Scrawler: handwritten notes with formulas from one reference line
Browse files- .gitattributes +5 -0
- .gitignore +5 -0
- LICENSE +202 -0
- LICENSE-WEIGHTS +5 -0
- NOTICE +6 -0
- README.md +125 -0
- ckpt_dl/b4styc/fp16/step003000_fp16.pt +3 -0
- ckpt_dl/glyph_cls2.pt +3 -0
- ckpt_dl/lstmal_v6isl3/step100000_fp16.pt +3 -0
- ckpt_dl/planner7/best.pt +3 -0
- ckpt_glyph_dn_u2s/fp16/step050000.pt +3 -0
- ckpt_semb3_all/best.pt +3 -0
- data/glyph_stroke_counts.json +1 -0
- data/ink_lines_u2/vocab.json +1 -0
- defaults.json +10 -0
- examples/demo_authors_hand/lecture_notes.pdf +3 -0
- examples/demo_authors_hand/page_01.png +3 -0
- examples/demo_authors_hand/page_05.png +3 -0
- examples/demo_released_weights/page_05.png +3 -0
- examples/demo_released_weights/pages.pdf +3 -0
- examples/physics_oscillator.txt +137 -0
- examples/ref_hwr200.json +5 -0
- examples/ref_hwr200.npy +3 -0
- requirements.txt +3 -0
- scripts/formula_line.py +227 -0
- scripts/formula_write.py +626 -0
- scripts/glyph_dn.py +60 -0
- scripts/make_ref.py +25 -0
- scripts/render_pages.py +80 -0
- scripts/style_classifier.py +269 -0
- scripts/train_glyph_cls.py +53 -0
- scripts/train_glyph_dn.py +318 -0
- scripts/train_ink.py +186 -0
- scripts/train_ink_al.py +119 -0
- scripts/train_ink_al2.py +231 -0
- scripts/train_ink_denoise.py +0 -0
- scripts/train_planner.py +108 -0
- scripts/write_pages.py +118 -0
- tools/mathjax/layout.mjs +68 -0
- tools/mathjax/package-lock.json +89 -0
- tools/mathjax/package.json +16 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
examples/demo_authors_hand/lecture_notes.pdf filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
examples/demo_authors_hand/page_01.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
examples/demo_authors_hand/page_05.png filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
examples/demo_released_weights/page_05.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
examples/demo_released_weights/pages.pdf filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
tools/mathjax/node_modules/
|
| 2 |
+
ckpt_denoise1/
|
| 3 |
+
samples_denoise1/
|
| 4 |
+
__pycache__/
|
| 5 |
+
out*/
|
LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
Apache License
|
| 3 |
+
Version 2.0, January 2004
|
| 4 |
+
http://www.apache.org/licenses/
|
| 5 |
+
|
| 6 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 7 |
+
|
| 8 |
+
1. Definitions.
|
| 9 |
+
|
| 10 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 11 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 12 |
+
|
| 13 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 14 |
+
the copyright owner that is granting the License.
|
| 15 |
+
|
| 16 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 17 |
+
other entities that control, are controlled by, or are under common
|
| 18 |
+
control with that entity. For the purposes of this definition,
|
| 19 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 20 |
+
direction or management of such entity, whether by contract or
|
| 21 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 22 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 23 |
+
|
| 24 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 25 |
+
exercising permissions granted by this License.
|
| 26 |
+
|
| 27 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 28 |
+
including but not limited to software source code, documentation
|
| 29 |
+
source, and configuration files.
|
| 30 |
+
|
| 31 |
+
"Object" form shall mean any form resulting from mechanical
|
| 32 |
+
transformation or translation of a Source form, including but
|
| 33 |
+
not limited to compiled object code, generated documentation,
|
| 34 |
+
and conversions to other media types.
|
| 35 |
+
|
| 36 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 37 |
+
Object form, made available under the License, as indicated by a
|
| 38 |
+
copyright notice that is included in or attached to the work
|
| 39 |
+
(an example is provided in the Appendix below).
|
| 40 |
+
|
| 41 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 42 |
+
form, that is based on (or derived from) the Work and for which the
|
| 43 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 44 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 45 |
+
of this License, Derivative Works shall not include works that remain
|
| 46 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 47 |
+
the Work and Derivative Works thereof.
|
| 48 |
+
|
| 49 |
+
"Contribution" shall mean any work of authorship, including
|
| 50 |
+
the original version of the Work and any modifications or additions
|
| 51 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 52 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 53 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 54 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 55 |
+
means any form of electronic, verbal, or written communication sent
|
| 56 |
+
to the Licensor or its representatives, including but not limited to
|
| 57 |
+
communication on electronic mailing lists, source code control systems,
|
| 58 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 59 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 60 |
+
excluding communication that is conspicuously marked or otherwise
|
| 61 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 62 |
+
|
| 63 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 64 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 65 |
+
subsequently incorporated within the Work.
|
| 66 |
+
|
| 67 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 68 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 69 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 70 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 71 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 72 |
+
Work and such Derivative Works in Source or Object form.
|
| 73 |
+
|
| 74 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 75 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 76 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 77 |
+
(except as stated in this section) patent license to make, have made,
|
| 78 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 79 |
+
where such license applies only to those patent claims licensable
|
| 80 |
+
by such Contributor that are necessarily infringed by their
|
| 81 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 82 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 83 |
+
institute patent litigation against any entity (including a
|
| 84 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 85 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 86 |
+
or contributory patent infringement, then any patent licenses
|
| 87 |
+
granted to You under this License for that Work shall terminate
|
| 88 |
+
as of the date such litigation is filed.
|
| 89 |
+
|
| 90 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 91 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 92 |
+
modifications, and in Source or Object form, provided that You
|
| 93 |
+
meet the following conditions:
|
| 94 |
+
|
| 95 |
+
(a) You must give any other recipients of the Work or
|
| 96 |
+
Derivative Works a copy of this License; and
|
| 97 |
+
|
| 98 |
+
(b) You must cause any modified files to carry prominent notices
|
| 99 |
+
stating that You changed the files; and
|
| 100 |
+
|
| 101 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 102 |
+
that You distribute, all copyright, patent, trademark, and
|
| 103 |
+
attribution notices from the Source form of the Work,
|
| 104 |
+
excluding those notices that do not pertain to any part of
|
| 105 |
+
the Derivative Works; and
|
| 106 |
+
|
| 107 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 108 |
+
distribution, then any Derivative Works that You distribute must
|
| 109 |
+
include a readable copy of the attribution notices contained
|
| 110 |
+
within such NOTICE file, excluding those notices that do not
|
| 111 |
+
pertain to any part of the Derivative Works, in at least one
|
| 112 |
+
of the following places: within a NOTICE text file distributed
|
| 113 |
+
as part of the Derivative Works; within the Source form or
|
| 114 |
+
documentation, if provided along with the Derivative Works; or,
|
| 115 |
+
within a display generated by the Derivative Works, if and
|
| 116 |
+
wherever such third-party notices normally appear. The contents
|
| 117 |
+
of the NOTICE file are for informational purposes only and
|
| 118 |
+
do not modify the License. You may add Your own attribution
|
| 119 |
+
notices within Derivative Works that You distribute, alongside
|
| 120 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 121 |
+
that such additional attribution notices cannot be construed
|
| 122 |
+
as modifying the License.
|
| 123 |
+
|
| 124 |
+
You may add Your own copyright statement to Your modifications and
|
| 125 |
+
may provide additional or different license terms and conditions
|
| 126 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 127 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 128 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 129 |
+
the conditions stated in this License.
|
| 130 |
+
|
| 131 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 132 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 133 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 134 |
+
this License, without any additional terms or conditions.
|
| 135 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 136 |
+
the terms of any separate license agreement you may have executed
|
| 137 |
+
with Licensor regarding such Contributions.
|
| 138 |
+
|
| 139 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 140 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 141 |
+
except as required for reasonable and customary use in describing the
|
| 142 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 143 |
+
|
| 144 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 145 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 146 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 147 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 148 |
+
implied, including, without limitation, any warranties or conditions
|
| 149 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 150 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 151 |
+
appropriateness of using or redistributing the Work and assume any
|
| 152 |
+
risks associated with Your exercise of permissions under this License.
|
| 153 |
+
|
| 154 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 155 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 156 |
+
unless required by applicable law (such as deliberate and grossly
|
| 157 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 158 |
+
liable to You for damages, including any direct, indirect, special,
|
| 159 |
+
incidental, or consequential damages of any character arising as a
|
| 160 |
+
result of this License or out of the use or inability to use the
|
| 161 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 162 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 163 |
+
other commercial damages or losses), even if such Contributor
|
| 164 |
+
has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 180 |
+
|
| 181 |
+
To apply the Apache License to your work, attach the following
|
| 182 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 183 |
+
replaced with your own identifying information. (Don't include
|
| 184 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 185 |
+
comment syntax for the file format. We also recommend that a
|
| 186 |
+
file or class name and description of purpose be included on the
|
| 187 |
+
same "printed page" as the copyright notice for easier
|
| 188 |
+
identification within third-party archives.
|
| 189 |
+
|
| 190 |
+
Copyright [yyyy] [name of copyright owner]
|
| 191 |
+
|
| 192 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 193 |
+
you may not use this file except in compliance with the License.
|
| 194 |
+
You may obtain a copy of the License at
|
| 195 |
+
|
| 196 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 197 |
+
|
| 198 |
+
Unless required by applicable law or agreed to in writing, software
|
| 199 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 200 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 201 |
+
See the License for the specific language governing permissions and
|
| 202 |
+
limitations under the License.
|
LICENSE-WEIGHTS
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
The model weights in ckpt_dl/, ckpt_glyph_dn_u2s/ and ckpt_semb3_all/ are licensed under the
|
| 2 |
+
Creative Commons Attribution-NonCommercial 4.0 International license (CC BY-NC 4.0):
|
| 3 |
+
https://creativecommons.org/licenses/by-nc/4.0/legalcode
|
| 4 |
+
|
| 5 |
+
The code in scripts/ and tools/ is licensed under Apache-2.0, see LICENSE.
|
NOTICE
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Scrawler-open
|
| 2 |
+
Copyright 2026 Lev Ossadtchi
|
| 3 |
+
|
| 4 |
+
MathJax (mathjax-full, installed by npm into tools/mathjax) is Copyright The MathJax Consortium, Apache-2.0.
|
| 5 |
+
examples/ref_hwr200.npy is one line of the HWR200 dataset (AntiplagiatCompany/HWR200, Apache-2.0), scan traced to pen strokes.
|
| 6 |
+
The style encoder was also trained on traced HWR200 lines.
|
README.md
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-4.0
|
| 3 |
+
language:
|
| 4 |
+
- ru
|
| 5 |
+
tags:
|
| 6 |
+
- handwriting-synthesis
|
| 7 |
+
- online-handwriting
|
| 8 |
+
- diffusion
|
| 9 |
+
- latex
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# Scrawler
|
| 13 |
+
|
| 14 |
+
Turns a text file with LaTeX formulas into handwritten notebook pages. The style comes from **one line** of the writer's handwriting, with no fine-tuning.
|
| 15 |
+
|
| 16 |
+

|
| 17 |
+
|
| 18 |
+
*The page above is in the author's hand, written by personal fine-tuned models that are **not** in this repository. For what these weights give out of the box, see `examples/demo_released_weights/`.*
|
| 19 |
+
|
| 20 |
+
The output is pen strokes, not pixels, so the PDF is sharp at any zoom. The models write Russian prose and mathematics.
|
| 21 |
+
|
| 22 |
+
## Quick start
|
| 23 |
+
|
| 24 |
+
Requires Python with `torch`, `numpy` and `pillow`, plus Node.js for the formula layout.
|
| 25 |
+
|
| 26 |
+
```bash
|
| 27 |
+
pip install -r requirements.txt
|
| 28 |
+
(cd tools/mathjax && npm ci)
|
| 29 |
+
python scripts/write_pages.py examples/physics_oscillator.txt out
|
| 30 |
+
```
|
| 31 |
+
|
| 32 |
+
This writes to `out/` (the folder must not exist yet):
|
| 33 |
+
|
| 34 |
+
- `pages.pdf`: vector, every stroke a smooth curve
|
| 35 |
+
- `page_NN.svg` and `page_NN.png` (300 dpi)
|
| 36 |
+
- `strokes.json`: the strokes, re-render with `scripts/render_pages.py`
|
| 37 |
+
- `report.json`: per-line timing and where each glyph came from
|
| 38 |
+
|
| 39 |
+
Tested only on Python 3.14, torch 2.13, Apple M5 Max (MPS), where 11 pages took about 7 minutes with another job running on the machine. The code selects CUDA or CPU when available, but this package was not run on them.
|
| 40 |
+
|
| 41 |
+
## Writing in your own hand
|
| 42 |
+
|
| 43 |
+
Export one line of your handwriting as pen strokes: 30-50 characters of ordinary prose, as a JSON list of strokes, each a list of `[x, y]` points with y pointing down, in writing order.
|
| 44 |
+
|
| 45 |
+
```bash
|
| 46 |
+
python scripts/make_ref.py my_line.json my_line.npy
|
| 47 |
+
REF=/absolute/path/to/my_line.npy python scripts/write_pages.py notes.txt out
|
| 48 |
+
```
|
| 49 |
+
|
| 50 |
+
The default reference is one traced line of the open HWR200 dataset.
|
| 51 |
+
|
| 52 |
+
## Input format
|
| 53 |
+
|
| 54 |
+
```text
|
| 55 |
+
# A heading
|
| 56 |
+
A line of text with an inline formula $\omega_0 = \sqrt{k/m}$ in it.
|
| 57 |
+
$x(t) = A\cos(\omega_0 t + \varphi)$
|
| 58 |
+
|
| 59 |
+
An empty line above makes a paragraph gap.
|
| 60 |
+
```
|
| 61 |
+
|
| 62 |
+
Long lines are wrapped at about 42 characters, never inside `$...$`. Formulas use the LaTeX that MathJax understands.
|
| 63 |
+
|
| 64 |
+
## How it works
|
| 65 |
+
|
| 66 |
+
| On the line | Written by |
|
| 67 |
+
|---|---|
|
| 68 |
+
| text only | **Text model**: a diffusion denoiser over pen points, with a duration planner for each character |
|
| 69 |
+
| formula only | **Formula module**: MathJax places every glyph, a diffusion **glyph model** draws each one, and a classifier picks the most recognisable variant |
|
| 70 |
+
| text + `$formula$` | The text model writes the line, then the formula spans are replaced by the formula module |
|
| 71 |
+
|
| 72 |
+
The style vector comes from a line-level writer encoder. Fraction bars, radicals, arrows and `|x|` are drawn as pen lines.
|
| 73 |
+
|
| 74 |
+
## Files
|
| 75 |
+
|
| 76 |
+
| Path | What | Size |
|
| 77 |
+
|---|---|---|
|
| 78 |
+
| `ckpt_dl/b4styc/fp16/step003000_fp16.pt` | text model | 44 MB |
|
| 79 |
+
| `ckpt_dl/lstmal_v6isl3/step100000_fp16.pt` | base network the text model depends on | 18 MB |
|
| 80 |
+
| `ckpt_dl/planner7/best.pt` | duration planner | 13 MB |
|
| 81 |
+
| `ckpt_semb3_all/best.pt` | writer style encoder | 7 MB |
|
| 82 |
+
| `ckpt_glyph_dn_u2s/fp16/step050000.pt` | glyph model | 14 MB |
|
| 83 |
+
| `ckpt_dl/glyph_cls2.pt` | glyph classifier | 1 MB |
|
| 84 |
+
|
| 85 |
+
`scripts/` also holds training code. It is research code as it was used, not a clean library, and its comments are partly in Russian. The training data is not included.
|
| 86 |
+
|
| 87 |
+
## Evaluation
|
| 88 |
+
|
| 89 |
+
240 lines of real notes from writers the generator never saw, each written from one reference line by the same writer. An LLM judge (Gemini 4 Argon) lists errors in each line; "clean" means no error of severity 2 or higher. Pairs are re-sampled for the 95% intervals.
|
| 90 |
+
|
| 91 |
+
| Run | Clean, % [95%] | Prose | Mixed | Formula |
|
| 92 |
+
|---|---|---|---|---|
|
| 93 |
+
| Real handwriting of the same lines | 45.0 [38.8, 51.2] | 47.0 | 35.7 | 51.4 |
|
| 94 |
+
| Text model alone | 47.1 [40.8, 53.3] | 63.0 | 41.4 | 30.0 |
|
| 95 |
+
| **This release** | **61.3 [55.0, 67.5]** | 63.0 | 54.3 | 65.7 |
|
| 96 |
+
| Same, with a library of real glyphs (not released) | 65.4 [59.2, 71.2] | 62.0 | 58.6 | 77.1 |
|
| 97 |
+
|
| 98 |
+
How to read this table:
|
| 99 |
+
|
| 100 |
+
- The score rewards legibility. Scoring above real notes means the output is tidier than hurried human writing, not that it looks more human.
|
| 101 |
+
- Prose is the same set of generated lines in every run but the first, so 62-63 shows judge noise of a few points.
|
| 102 |
+
- The private library variant scores 4.2 points higher, 95% interval [-9.6, +1.2] for the difference. That is within the noise, but a small loss from removing it is possible.
|
| 103 |
+
|
| 104 |
+
Style similarity by the writer encoder (cosine to the writer's real line, one reference line, 3 unseen writers): 0.59-0.68 generated, against 0.55-0.70 between real lines of the same writers.
|
| 105 |
+
|
| 106 |
+
## Known weaknesses
|
| 107 |
+
|
| 108 |
+
- The text model sometimes swaps, drops or adds a letter. There is no spell check.
|
| 109 |
+
- A one-letter formula inside a sentence can collide with the neighbouring word.
|
| 110 |
+
- About 2% of glyphs are the best of the variants the classifier rejected, and may be misshapen.
|
| 111 |
+
- Rare symbols without their own class are drawn as a similar one (for example `\simeq` as `\approx`).
|
| 112 |
+
- A heading of one short word may come out smaller than the other lines.
|
| 113 |
+
|
| 114 |
+
## Training data
|
| 115 |
+
|
| 116 |
+
- **Generator models:** tablet handwriting of 13 writers who agreed to this use, about 63 000 lines and 86 000 glyphs. Writer names are removed from the weights. The notes themselves are not published.
|
| 117 |
+
- **Style encoder:** the same kind of tablet data, plus traced lines of [HWR200](https://huggingface.co/datasets/AntiplagiatCompany/HWR200) (Apache-2.0).
|
| 118 |
+
|
| 119 |
+
## Responsible use
|
| 120 |
+
|
| 121 |
+
This is for notes, experiments and research. Do not use it to forge signatures, documents or exam work, or to imitate someone's handwriting without their consent.
|
| 122 |
+
|
| 123 |
+
## License
|
| 124 |
+
|
| 125 |
+
Code: Apache-2.0 (`LICENSE`). Weights: CC BY-NC 4.0 (`LICENSE-WEIGHTS`). See `NOTICE` for third-party components.
|
ckpt_dl/b4styc/fp16/step003000_fp16.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a72c85d0cf11ffe153421f1c8779b5f83a80ce1f412eda781d2e346925f5d3a
|
| 3 |
+
size 45379397
|
ckpt_dl/glyph_cls2.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:97b7cac6b4bee3c04a91791f4f5284e177a27437a03af4256289dda8880c317e
|
| 3 |
+
size 1020533
|
ckpt_dl/lstmal_v6isl3/step100000_fp16.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:eea3551c05cefe2558690550a990ae5f87f32ef7da0ebbe7a24b47ab0b298286
|
| 3 |
+
size 18912713
|
ckpt_dl/planner7/best.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:696c639a721b98f51a3b5e1d110a01090514e07505d608005b597ba731978646
|
| 3 |
+
size 13501883
|
ckpt_glyph_dn_u2s/fp16/step050000.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3ae974e2fa7f2408d293104f7adc08816226db9b4c4e464a78657f4f179bb650
|
| 3 |
+
size 13940100
|
ckpt_semb3_all/best.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5959c972791660071426a1b6ab12d0378494946e2638bf42df79723bc3b85126
|
| 3 |
+
size 7279759
|
data/glyph_stroke_counts.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"4": {"2": 31, "1": 353, "4": 1, "3": 2}, "BAR": {"1": 2158}, "=": {"2": 3214, "4": 4, "3": 9}, "u": {"1": 364, "2": 2}, "x": {"2": 2592, "1": 626, "3": 26}, "2": {"1": 3831, "2": 30, "4": 1, "3": 1}, "y": {"1": 866, "2": 34, "3": 4, "4": 1}, "+": {"2": 1883, "1": 10, "3": 57, "4": 5}, "5": {"2": 128, "1": 14, "5": 1, "3": 1}, "−": {"1": 1017, "2": 4}, "n": {"1": 1544, "2": 44}, "b": {"1": 276, "2": 10, "3": 1}, ">": {"1": 120, "2": 1}, "χ": {"1": 7, "2": 2}, "c": {"1": 373, "2": 1}, "a": {"1": 703, "2": 12, "3": 2}, "0": {"1": 1312, "2": 15, "3": 2}, "1": {"1": 1507, "2": 24}, "N": {"1": 153, "2": 5, "3": 1}, "*": {"3": 2}, "8": {"1": 96, "2": 1}, "6": {"1": 134, "2": 1}, "7": {"2": 45, "3": 1, "1": 2}, "±": {"3": 23}, "d": {"1": 1478, "2": 25}, "L": {"1": 432, "2": 18, "3": 1, "5": 1}, "λ": {"1": 333, "2": 41}, "(": {"1": 2371, "2": 15, "4": 1, "3": 2}, "z": {"2": 754, "3": 6, "1": 3}, "t": {"1": 9, "2": 629, "3": 6, "4": 1}, "D": {"1": 37, "2": 15, "3": 1}, ",": {"1": 855, "2": 37, "3": 2}, "VEC": {"1": 159}, "r": {"1": 421, "2": 17}, ")": {"1": 2553, "2": 21, "4": 3, "3": 2}, "P": {"2": 24, "1": 110, "3": 3, "4": 1}, "V": {"1": 182, "2": 19, "3": 3, "4": 1}, "{": {"1": 165, "2": 7}, "f": {"2": 434, "3": 13, "4": 3, "1": 10, "5": 2}, "}": {"1": 112, "2": 1}, "∈": {"2": 325, "3": 1, "1": 1}, "X": {"2": 153, "3": 16}, "m": {"1": 969, "2": 57, "3": 1}, "→": {"1": 168, "2": 7}, "∞": {"1": 96, "2": 1}, "<": {"1": 133, "2": 2}, "|": {"1": 137}, "ε": {"1": 93, "2": 2}, "α": {"1": 566, "2": 35, "3": 1}, "φ": {"1": 316, "2": 15, "3": 1}, "e": {"1": 797, "2": 11}, "p": {"1": 156, "2": 24, "3": 2}, "j": {"2": 142, "3": 15, "5": 1}, "ψ": {"2": 56, "3": 2, "1": 1}, "≠": {"3": 66, "4": 2}, "A": {"2": 916, "4": 5, "1": 9, "3": 102, "5": 3}, "s": {"1": 751, "2": 4}, "↑": {"2": 3}, "ω": {"1": 399, "2": 4, "3": 1}, "3": {"1": 656, "2": 8}, "l": {"1": 224, "2": 4, "3": 1}, "F": {"2": 89, "3": 40, "4": 3, "6": 1}, "ρ": {"1": 156, "2": 2, "3": 1}, "R": {"1": 211, "2": 27, "3": 6}, "S": {"1": 131, "2": 8}, "δ": {"1": 117, "2": 3, "3": 2}, "∂": {"1": 556, "2": 12, "3": 1}, "q": {"1": 118, "2": 10}, "Δ": {"2": 27, "1": 143, "3": 3, "4": 1}, "k": {"3": 25, "1": 105, "2": 430, "4": 6}, "∀": {"2": 142, "1": 2}, "∙": {"1": 1}, "√": {"2": 19, "4": 12, "6": 2, "3": 23, "5": 10, "1": 2, "10": 1}, "w": {"1": 71, "2": 1}, "π": {"3": 125, "2": 100, "4": 1, "1": 4}, "σ": {"1": 70, "2": 4}, "E": {"3": 146, "4": 7, "2": 7, "1": 4, "5": 3}, "μ": {"1": 33, "2": 2}, "v": {"1": 450, "2": 3}, "h": {"1": 96, "2": 7}, "g": {"1": 83, "3": 1, "2": 2}, "∑": {"1": 51, "5": 2, "4": 4, "2": 39, "3": 8}, "B": {"1": 243, "2": 54, "4": 1, "3": 3}, "×": {"2": 49, "1": 3, "4": 5}, "⊃": {"1": 2}, "′": {"1": 39}, "C": {"1": 199, "2": 10, "3": 1}, "T": {"2": 242, "1": 20, "3": 3}, "η": {"1": 16}, "Q": {"2": 127, "1": 33, "3": 1}, "∃": {"2": 53, "3": 3, "1": 3}, ":": {"2": 17, "1": 3}, "G": {"1": 55, "2": 3}, "ħ": {"2": 46, "1": 2}, "?": {"2": 30, "1": 1}, "≤": {"2": 59, "3": 1}, "≥": {"2": 15, "1": 4}, "∫": {"1": 91, "2": 14, "3": 2}, "θ": {"2": 161, "1": 17, "5": 1, "3": 2, "4": 1}, "K": {"2": 107, "1": 2, "3": 4}, "∗": {"3": 37, "4": 6, "2": 1}, "H": {"1": 35, "2": 6}, "≈": {"2": 24}, "∮": {"2": 8, "3": 4, "4": 1, "5": 1}, "∣": {"1": 10}, "⊂": {"1": 16, "3": 3}, "∅": {"1": 1, "2": 6}, "U": {"1": 98}, "M": {"2": 22, "1": 43, "3": 1}, "!": {"1": 8, "2": 1}, "⊤": {"2": 3}, "∋": {"1": 1, "2": 1, "3": 1}, "⟨": {"1": 27}, "⟩": {"1": 25}, "I": {"1": 48, "3": 5, "2": 3}, "τ": {"2": 21, "1": 3}, "Y": {"1": 7, "2": 30, "4": 3, "3": 2}, "Σ": {"2": 2, "1": 10}, "9": {"1": 57}, "⇒": {"1": 13, "2": 3}, "i": {"2": 107, "3": 1}, "∼": {"1": 34}, " ̃": {"1": 20}, "Γ": {"2": 7, "1": 6}, "Ω": {"1": 26, "2": 2}, "⋃": {"1": 1}, "Z": {"2": 16, "3": 1}, "[": {"1": 8, "2": 8}, "]": {"1": 8}, "⊥": {"1": 5, "2": 16}, "o": {"1": 9}, "∇": {"3": 2, "2": 4, "1": 2}, "Φ": {"1": 4, "2": 2, "3": 2}, " ̈": {"1": 1}, "∩": {"1": 12, "2": 3}, "̸": {"3": 1}, " ̇": {"1": 12}, "ξ": {"1": 37}, "⟶": {"1": 5}, "κ": {"1": 1, "2": 1}, "W": {"1": 15, "2": 1}, "γ": {"1": 24, "2": 3}, "β": {"1": 17, "2": 2}, "∘": {"2": 1}, "^": {"1": 14}, ".": {"2": 1, "1": 3}, ";": {"2": 9, "1": 6}, "⇔": {"1": 4}, "⋯": {"2": 2}, "∓": {"3": 2}, "⊗": {"3": 1}, "%": {"3": 1}, "ν": {"1": 7}, "J": {"2": 16, "1": 31}, "ζ": {"1": 17}, "⊕": {"3": 8}, "≃": {"2": 1}, "✓": {"1": 2}, "~": {"1": 8}, "Θ": {"2": 1}, "Λ": {"1": 2}, "Ξ": {"3": 2, "4": 1}, "–": {"1": 7}, "←": {"2": 2, "3": 1}, "⇓": {"2": 1}, "∉": {"3": 6}, "⟸": {"3": 1}, "⎧": {"1": 1}, "⎩": {"1": 1}, "∥": {"1": 10}, "↓": {"1": 2, "3": 1}, "⎛": {"2": 1}, "O": {"2": 1}, "⇉": {"2": 1}, "⟹": {"1": 1}, "∪": {"1": 1}, "⊖": {"3": 1}, "≡": {"3": 1}, "Ψ": {"2": 1}, "∫∫": {"1": 1}, "△": {"1": 1}}
|
data/ink_lines_u2/vocab.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
["<pad>", "<bos>", "<eos>", "{", "}", "^", "_", "\\frac", "1", "2", "x", "=", ")", "(", "i", "t", "a", "-", "0", "r", "m", "d", "3", "n", "\\sqrt", "4", "+", "5", "6", ",", "k", "9", "7", "\\\\", "\\begin", "\\end", "8", "f", "\\cdot", "\\partial", "c", "|", "s", "p", "y", "\\int", "&", "e", "A", "v", "/", ".", "\\hat", "b", "[", "]", "g", "z", "u", "l", "j", "T", "P", "\\tilde", "R", "X", "C", "\\overline", "E", "S", "N", "o", "\\pi", "V", "q", "B", "\\prime", "L", "F", "h", "M", "\\alpha", "\\mu", "I", "D", "w", "G", "H", "\\sum", "\\theta", "\\prod", "\\lambda", "\\sigma", "\\rho", "Q", "\\}", "\\{", "\\omega", "K", "\\infty", "\\beta", "\\in", "\\epsilon", "U", "Z", "\\phi", "\\gamma", "<", "*", "\\Delta", "\\underline", ":", "\\rightarrow", "W", "Y", "O", "\\nu", "\\le", "\\delta", "\\notin", "\\psi", "\\tau", "!", "J", "\\rangle", "\\vec", "\\times", ">", "\\varphi", "\\mathbb", "\\approx", "\\pm", "\\dot", "\\Omega", "\\eta", "\\ge", "\\langle", "\\xi", "\\Gamma", "\\zeta", "\\Phi", "\\nabla", "\\hbar", "\\equiv", "\\chi", "\\kappa", "\\circ", ";", "\\ne", "\\Sigma", "\\sim", "\\Psi", "\\forall", "\\Lambda", "\\rfloor", "\\lfloor", "\\Theta", "\\Pi", "\\mapsto", "\\aleph", "\\otimes", "\\neg", "\\wedge", "\\vdash", "\\cup", "\\dagger", "\\propto", "\\cap", "\\backslash", "\\subseteq", "\\vee", "\\subset", "\\oplus", "\\simeq", "\\Rightarrow", "\\vartheta", "\\perp", "\\upsilon", "\\emptyset", "\\ll", "\\bigcup", "\\Xi", "\\cong", "\\leftarrow", "\\#", "\\bigcap", "\\varpi", "\\|", "?", "\\oint", "\\%", "\\iint", "\\odot", "\\iota", "\\exists", "\\bigoplus", "\\top", "\\rceil", "\\lceil", "\\iff", "\\gg", "\\subsetneq", "\\Upsilon", "\\bullet", "\\mp", "\\varsigma", "\\models", "\\div", "\\ominus", "\\angle", "\\vdots", "\\longrightarrow", "\\Leftrightarrow", "\\triangle", "\\leftrightarrow", "\\hookrightarrow", "\\bigwedge", "\\Vdash", "\\_", "\\not", "\\rightleftharpoons", "\\supset", "\\ni", "\\triangleq", "\\supseteq", "\\bigvee", "\\bigcirc", "\\&", "\\triangleleft", "\\sqsubseteq", " ", "\"", "#", "$", "%", "'", "@", "\\", "~", "§", "«", "°", "·", "»", "½", "é", "ó", "Δ", "α", "β", "δ", "λ", "μ", "π", "ρ", "τ", "φ", "Ё", "А", "Б", "В", "Г", "Д", "Е", "Ж", "З", "И", "Й", "К", "Л", "М", "Н", "О", "П", "Р", "С", "Т", "У", "Ф", "Х", "Ц", "Ч", "Ш", "Ы", "Э", "Ю", "Я", "а", "б", "в", "г", "д", "е", "ж", "з", "и", "й", "к", "л", "м", "н", "о", "п", "р", "с", "т", "у", "ф", "х", "ц", "ч", "ш", "щ", "ъ", "ы", "ь", "э", "ю", "я", "ё", "і", "—", "“", "„", "•", "₀", "₁", "₂", "ₓ", "ₜ", "№", "←", "↑", "→", "↓", "↔", "↻", "⇒", "⇔", "∀", "∃", "∈", "√", "∞", "∨", "≈", "≠", "≤", "①", "②", "③", "④", "⑤", "⑥", "⑦", "⑧", "⑩", "⑭", "⑱", "○", "♥", "✓"]
|
defaults.json
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"B_CKPT": "ckpt_dl/b4styc/fp16/step003000_fp16.pt",
|
| 3 |
+
"GDN_CKPT": "ckpt_glyph_dn_u2s/fp16/step050000.pt",
|
| 4 |
+
"FW_LIB": "none",
|
| 5 |
+
"SEL_RETRY": "16",
|
| 6 |
+
"FW_NS": "data/glyph_stroke_counts.json",
|
| 7 |
+
"LINE_MARKS": "1",
|
| 8 |
+
"KEY": "ref",
|
| 9 |
+
"REF": "examples/ref_hwr200.npy"
|
| 10 |
+
}
|
examples/demo_authors_hand/lecture_notes.pdf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0f852fed683768424138a89b45bde5b963995960896017b00778e6c8adf204e1
|
| 3 |
+
size 1707520
|
examples/demo_authors_hand/page_01.png
ADDED
|
Git LFS Details
|
examples/demo_authors_hand/page_05.png
ADDED
|
Git LFS Details
|
examples/demo_released_weights/page_05.png
ADDED
|
Git LFS Details
|
examples/demo_released_weights/pages.pdf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a679392952e1e17d43a7c56afd19b1209682e731fee7a047fea9fa01331f362a
|
| 3 |
+
size 1924132
|
examples/physics_oscillator.txt
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Лекция 7. Гармонический осциллятор
|
| 2 |
+
Осциллятор - самая важная модель в физике: почти любая система вблизи устойчивого равновесия колеблется гармонически.
|
| 3 |
+
|
| 4 |
+
# 1. Пружинный маятник
|
| 5 |
+
Груз массы $m$ на пружине жёсткости $k$. Сила Гука:
|
| 6 |
+
$F = -kx$
|
| 7 |
+
Второй закон Ньютона даёт уравнение движения:
|
| 8 |
+
$m\ddot{x} = -kx$
|
| 9 |
+
Обозначим $\omega_0^2 = k/m$, тогда
|
| 10 |
+
$\ddot{x} + \omega_0^2 x = 0$
|
| 11 |
+
Общее решение - гармоническое колебание:
|
| 12 |
+
$x(t) = A\cos(\omega_0 t + \varphi)$
|
| 13 |
+
Здесь $A$ - амплитуда, $\varphi$ - начальная фаза. Они находятся из начальных условий $x(0)=x_0$ и $\dot{x}(0)=v_0$:
|
| 14 |
+
$A = \sqrt{x_0^2 + v_0^2/\omega_0^2}$
|
| 15 |
+
$\tan\varphi = -\frac{v_0}{\omega_0 x_0}$
|
| 16 |
+
Период и частота не зависят от амплитуды:
|
| 17 |
+
$T = \frac{2\pi}{\omega_0} = 2\pi\sqrt{\frac{m}{k}}$
|
| 18 |
+
Это свойство называется изохронностью.
|
| 19 |
+
|
| 20 |
+
# 2. Энергия колебаний
|
| 21 |
+
Кинетическая и потенциальная энергия:
|
| 22 |
+
$K = \frac{m\dot{x}^2}{2}, \quad U = \frac{kx^2}{2}$
|
| 23 |
+
Подставим решение $x(t)$:
|
| 24 |
+
$K = \frac{kA^2}{2}\sin^2(\omega_0 t + \varphi)$
|
| 25 |
+
$U = \frac{kA^2}{2}\cos^2(\omega_0 t + \varphi)$
|
| 26 |
+
Сумма постоянна:
|
| 27 |
+
$E = K + U = \frac{kA^2}{2} = \frac{m\omega_0^2 A^2}{2}$
|
| 28 |
+
Средние за период равны друг другу:
|
| 29 |
+
$\langle K \rangle = \langle U \rangle = \frac{E}{2}$
|
| 30 |
+
Энергия перекачивается из кинетической в потенциальную и обратно дважды за период.
|
| 31 |
+
|
| 32 |
+
# 3. Математический маятник
|
| 33 |
+
Точечная масса на нити длины $l$. Уравнение для угла:
|
| 34 |
+
$ml\ddot{\theta} = -mg\sin\theta$
|
| 35 |
+
При малых углах $\sin\theta \approx \theta$, и
|
| 36 |
+
$\ddot{\theta} + \frac{g}{l}\theta = 0$
|
| 37 |
+
$\omega_0 = \sqrt{g/l}, \quad T = 2\pi\sqrt{l/g}$
|
| 38 |
+
Для конечной амплитуды $\theta_0$ период растёт:
|
| 39 |
+
$T \approx 2\pi\sqrt{\frac{l}{g}}\left(1 + \frac{\theta_0^2}{16}\right)$
|
| 40 |
+
Изохронность нарушается - колебания становятся ангармоническими.
|
| 41 |
+
|
| 42 |
+
# 4. Фазовая плоскость
|
| 43 |
+
Состояние осциллятора задаётся парой $(x, p)$, где $p = m\dot{x}$.
|
| 44 |
+
Из закона сохранения энергии:
|
| 45 |
+
$\frac{p^2}{2m} + \frac{m\omega_0^2 x^2}{2} = E$
|
| 46 |
+
Фазовая траектория - эллипс с полуосями
|
| 47 |
+
$a = \sqrt{\frac{2E}{m\omega_0^2}}, \quad b = \sqrt{2mE}$
|
| 48 |
+
Площадь эллипса:
|
| 49 |
+
$S = \pi a b = \frac{2\pi E}{\omega_0}$
|
| 50 |
+
Отношение $E/\omega_0$ - адиабатический инвариант: оно сохраняется при медленном изменении параметров.
|
| 51 |
+
|
| 52 |
+
# 5. Затухающие колебания
|
| 53 |
+
Добавим силу вязкого трения $F = -b\dot{x}$:
|
| 54 |
+
$\ddot{x} + 2\gamma\dot{x} + \omega_0^2 x = 0, \quad \gamma = \frac{b}{2m}$
|
| 55 |
+
Ищем решение в виде $x = e^{\lambda t}$:
|
| 56 |
+
$\lambda^2 + 2\gamma\lambda + \omega_0^2 = 0$
|
| 57 |
+
$\lambda = -\gamma \pm \sqrt{\gamma^2 - \omega_0^2}$
|
| 58 |
+
При слабом затухании $\gamma < \omega_0$:
|
| 59 |
+
$x(t) = A e^{-\gamma t}\cos(\omega t + \varphi)$
|
| 60 |
+
$\omega = \sqrt{\omega_0^2 - \gamma^2}$
|
| 61 |
+
Амплитуда спадает экспоненциально, время жизни $\tau = 1/\gamma$.
|
| 62 |
+
Логарифмический декремент затухания:
|
| 63 |
+
$\delta = \ln\frac{x(t)}{x(t+T)} = \gamma T$
|
| 64 |
+
Добротность - число колебаний за время затухания, умноженное на $\pi$:
|
| 65 |
+
$Q = \frac{\omega_0}{2\gamma}$
|
| 66 |
+
При $\gamma = \omega_0$ - критическое затухание, система возвращается быстрее всего и без колебаний.
|
| 67 |
+
При $\gamma > \omega_0$ - апериодический режим.
|
| 68 |
+
|
| 69 |
+
# 6. Вынужденные колебания
|
| 70 |
+
Внешняя сила $F_0\cos\Omega t$:
|
| 71 |
+
$\ddot{x} + 2\gamma\dot{x} + \omega_0^2 x = \frac{F_0}{m}\cos\Omega t$
|
| 72 |
+
После затухания переходного процесса остаётся установившееся решение:
|
| 73 |
+
$x(t) = a(\Omega)\cos(\Omega t - \psi)$
|
| 74 |
+
Амплитуда и сдвиг фазы:
|
| 75 |
+
$a(\Omega) = \frac{F_0/m}{\sqrt{(\omega_0^2 - \Omega^2)^2 + 4\gamma^2\Omega^2}}$
|
| 76 |
+
$\tan\psi = \frac{2\gamma\Omega}{\omega_0^2 - \Omega^2}$
|
| 77 |
+
Резонанс - максимум амплитуды при
|
| 78 |
+
$\Omega_r = \sqrt{\omega_0^2 - 2\gamma^2}$
|
| 79 |
+
При малом затухании ширина резонансной кривой на половине мощности:
|
| 80 |
+
$\Delta\Omega \approx 2\gamma = \frac{\omega_0}{Q}$
|
| 81 |
+
Чем выше добротность, тем острее резонанс.
|
| 82 |
+
Средняя поглощаемая мощность:
|
| 83 |
+
$\langle P \rangle = \frac{F_0^2}{m}\frac{\gamma\Omega^2}{(\omega_0^2 - \Omega^2)^2 + 4\gamma^2\Omega^2}$
|
| 84 |
+
|
| 85 |
+
# 7. Комплексная запись
|
| 86 |
+
Удобно писать $x = \mathrm{Re}\, z$, где $z$ удовлетворяет
|
| 87 |
+
$\ddot{z} + 2\gamma\dot{z} + \omega_0^2 z = \frac{F_0}{m}e^{i\Omega t}$
|
| 88 |
+
Подстановка $z = z_0 e^{i\Omega t}$ сразу даёт
|
| 89 |
+
$z_0 = \frac{F_0/m}{\omega_0^2 - \Omega^2 + 2i\gamma\Omega}$
|
| 90 |
+
Модуль $z_0$ - амплитуда, аргумент - фаза. Так считать проще, чем через тригонометрию.
|
| 91 |
+
|
| 92 |
+
# 8. Связанные осцилляторы
|
| 93 |
+
Два одинаковых маятника, соединённые пружиной жёсткости $\kappa$:
|
| 94 |
+
$m\ddot{x}_1 = -kx_1 - \kappa(x_1 - x_2)$
|
| 95 |
+
$m\ddot{x}_2 = -kx_2 - \kappa(x_2 - x_1)$
|
| 96 |
+
Нормальные координаты:
|
| 97 |
+
$q_1 = x_1 + x_2, \quad q_2 = x_1 - x_2$
|
| 98 |
+
Они колеблются независимо с частотами
|
| 99 |
+
$\omega_1 = \sqrt{k/m}, \quad \omega_2 = \sqrt{(k + 2\kappa)/m}$
|
| 100 |
+
Если сначала качнуть только первый маятник, энергия переходит ко второму и обратно - биения с частотой
|
| 101 |
+
$\Omega_b = \omega_2 - \omega_1$
|
| 102 |
+
В общем случае $N$ степеней свободы: ищем $x = a e^{i\omega t}$ и решаем
|
| 103 |
+
$\det(K - \omega^2 M) = 0$
|
| 104 |
+
|
| 105 |
+
# 9. Квантовый осциллятор
|
| 106 |
+
Гамильтониан:
|
| 107 |
+
$\hat{H} = \frac{\hat{p}^2}{2m} + \frac{m\omega^2\hat{x}^2}{2}$
|
| 108 |
+
Уравнение Шрёдингера для стационарных состояний:
|
| 109 |
+
$-\frac{\hbar^2}{2m}\psi'' + \frac{m\omega^2 x^2}{2}\psi = E\psi$
|
| 110 |
+
Уровни энергии эквидистантны:
|
| 111 |
+
$E_n = \hbar\omega\left(n + \frac{1}{2}\right), \quad n = 0, 1, 2, \ldots$
|
| 112 |
+
Энергия нулевых колебаний $E_0 = \hbar\omega/2$ не равна нулю - это следствие соотношения неопределённостей:
|
| 113 |
+
$\Delta x \, \Delta p \geq \frac{\hbar}{2}$
|
| 114 |
+
Основное состояние - гауссиана:
|
| 115 |
+
$\psi_0(x) = \left(\frac{m\omega}{\pi\hbar}\right)^{1/4} e^{-m\omega x^2/2\hbar}$
|
| 116 |
+
|
| 117 |
+
# 10. Лестничные операторы
|
| 118 |
+
Введём
|
| 119 |
+
$\hat{a} = \sqrt{\frac{m\omega}{2\hbar}}\left(\hat{x} + \frac{i\hat{p}}{m\omega}\right)$
|
| 120 |
+
и сопряжённый к нему $\hat{a}^\dagger$. Коммутатор:
|
| 121 |
+
$[\hat{a}, \hat{a}^\dagger] = 1$
|
| 122 |
+
Гамильтониан принимает вид
|
| 123 |
+
$\hat{H} = \hbar\omega\left(\hat{a}^\dagger\hat{a} + \frac{1}{2}\right)$
|
| 124 |
+
Операторы повышают и понижают номер уровня:
|
| 125 |
+
$\hat{a}^\dagger|n\rangle = \sqrt{n+1}\,|n+1\rangle$
|
| 126 |
+
$\hat{a}|n\rangle = \sqrt{n}\,|n-1\rangle$
|
| 127 |
+
Все состояния получаются из основного:
|
| 128 |
+
$|n\rangle = \frac{(\hat{a}^\dagger)^n}{\sqrt{n!}}|0\rangle$
|
| 129 |
+
Среднее значение координаты в стационарном состоянии равно нулю, а дисперсия растёт с номером:
|
| 130 |
+
$\langle x^2 \rangle_n = \frac{\hbar}{m\omega}\left(n + \frac{1}{2}\right)$
|
| 131 |
+
|
| 132 |
+
# 11. Итоги
|
| 133 |
+
Гармонический осциллятор описывает колебания молекул, фононы в кристалле, моды электромагнитного поля в резонаторе.
|
| 134 |
+
Главное запомнить:
|
| 135 |
+
$\omega_0 = \sqrt{k/m}, \quad E = \frac{m\omega_0^2 A^2}{2}, \quad Q = \frac{\omega_0}{2\gamma}$
|
| 136 |
+
$E_n = \hbar\omega\left(n + \frac{1}{2}\right)$
|
| 137 |
+
Классический и квантовый осцилляторы имеют одну и ту же частоту, но квантовый не может покоиться.
|
examples/ref_hwr200.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"text": "незаменима, как средство развития",
|
| 3 |
+
"source": "HWR200 (AntiplagiatCompany/HWR200, Apache-2.0), scan traced to pen strokes",
|
| 4 |
+
"writer": "hwr200_w133"
|
| 5 |
+
}
|
examples/ref_hwr200.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ca7fb9773234c253c8e87230d607fb8d8e32a1046dd367cb2d4dde116fcb826d
|
| 3 |
+
size 13196
|
requirements.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch
|
| 2 |
+
numpy
|
| 3 |
+
pillow
|
scripts/formula_line.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""04.11 (И164, Лев: «обучить модель формул и собрать устойчивый модуль их написания … генерируем символы и расставляем по латех»):
|
| 3 |
+
a formula LINE in the hand of ONE reference line, in the line format of B (dx, dy, pen; line height 1, arc step STEP), so it can be
|
| 4 |
+
measured and judged exactly like B's lines (product/eval/run_eval.py FORMULA=1).
|
| 5 |
+
layout : MathJax boxes (formula_write.layout / compose_layout with its FIX40 fixes: script sizes, no-collision push, dots, donors)
|
| 6 |
+
glyphs : the glyph denoiser with style from a LINE (train_glyph_dn GSTYLE=semb, GDN_CKPT, default ckpt_glyph_dn_u2s/fp16/step050000.pt) —
|
| 7 |
+
the reference line's semb3_all embedding goes to glyph_dn.set_style; SEL_K variants per glyph, the glyph classifier picks
|
| 8 |
+
the most typical (formula_write SEL_PICK=best); FIX_LIBCLS off (every class the net knows is generated, none from w01's
|
| 9 |
+
library); classes the net does not know -> the real library (all writers) -> never silently dropped (counted in `missing`)
|
| 10 |
+
marks : fraction bars / radicals / arrows from real hand strokes (formula_write LINE_MARKS=2)
|
| 11 |
+
marks : slash, hat, tilde, overline, <-> (unknown to all of the above) -> pen lines over the box (own_marks)
|
| 12 |
+
SLANT=1 : shear the line to the reference line's slant
|
| 13 |
+
text : \\text{...} words (Cyrillic notes in formulas) are written by B itself with the same reference line (text_words replaced)
|
| 14 |
+
python scripts/formula_line.py <out.png> "<tex>" [...] (eyeball: reference = REF_ID line of u2 test, default the first line of WRITER)
|
| 15 |
+
As a module: setup(b_ckpt, planner) once, then formula_line(tex, ref_ink, rng) -> (seq [n,3] float32 or None, missing classes)."""
|
| 16 |
+
import os, sys, json, numpy as np
|
| 17 |
+
P = os.path.dirname(os.path.dirname(os.path.abspath(__file__))); sys.path.insert(0, os.path.join(P, 'scripts')); E = os.environ.get
|
| 18 |
+
os.environ.update(PROJ=P); os.environ.setdefault('INK_DS', 'ink_lines_u2')
|
| 19 |
+
for _k, _v in (('GEN', 'dn'), ('FIX40', '1'), ('LINE_MARKS', '2'), ('FIX_LIBCLS', ''), ('GDN_CFG', '1.2'), ('SEL_K', '4'), # SEL_K 4 (05.11, fgen_u2s_vbar_k4 vs _vbar: clean 65.4 vs 59.2, style 0.787 vs 0.779, 3x faster); was FIX40's 16
|
| 20 |
+
('GDN_CKPT', os.path.join(P, 'ckpt_glyph_dn_u2s', 'fp16', 'step050000.pt'))): os.environ.setdefault(_k, _v)
|
| 21 |
+
STEP = float(E('STEP', '0.05'))
|
| 22 |
+
import torch, train_ink_denoise as D, formula_write as FW, glyph_dn as GD
|
| 23 |
+
_B = {}
|
| 24 |
+
|
| 25 |
+
def setup(b_ckpt=None, planner=None):
|
| 26 |
+
if _B: return _B
|
| 27 |
+
import train_planner as TP
|
| 28 |
+
net, step = D.load_net(b_ckpt or os.path.join(P, 'ckpt_dl', 'b4styc', 'fp16', 'step003000_fp16.pt')); base = D.load_base()
|
| 29 |
+
_B.update(net=net, base=base, TP=TP, pnet=TP.load(planner or os.path.join(P, 'ckpt_dl', 'planner7', 'best.pt')), ref=None)
|
| 30 |
+
FW.text_words = _text_words_b; FW.compose_text = _compose_text
|
| 31 |
+
# 05.11: formula_write switches its generator on only in its own CLI (GEN='VAE' in __main__); imported, FW.GEN stayed None and EVERY glyph
|
| 32 |
+
# came from the real library (glyphs_clean3: 94% w01) — the 04.11 runs fdn_u2s50k / fdn_mix2 and pages_v1 were library glyphs, not the net.
|
| 33 |
+
# FL_LIB=1 reproduces that (library glyphs in the MathJax layout).
|
| 34 |
+
if E('FL_LIB', '0') != '1': FW.GEN = 'VAE'
|
| 35 |
+
return _B
|
| 36 |
+
|
| 37 |
+
def _text_words_b(words): # formula_write.compose_text hook: words -> {word: [variant strokes [[x,y,p],...]]}, written by B in the reference hand
|
| 38 |
+
B = _B; out = {}; words = [w for w in dict.fromkeys(words) if w.strip()]
|
| 39 |
+
if not words or B.get('ref') is None: return out
|
| 40 |
+
ref = B['ref']; toks = [list(w) for w in words]
|
| 41 |
+
sv = torch.cat([D.style_of(B['net'], B['base'], [ref])] * len(words), 0); se = torch.cat([D.semb_vec([ref])] * len(words), 0)
|
| 42 |
+
durs = B['TP'].predict(B['pnet'], toks, se); B['net']._tok_base = None
|
| 43 |
+
for w, q in zip(words, D.sample(B['net'], toks, durs, sv)):
|
| 44 |
+
q = np.asarray(q, np.float32); xy = np.cumsum(q[:, :2], 0) * 100.0; st, cur = [], []
|
| 45 |
+
for k in range(len(q)):
|
| 46 |
+
cur.append([float(xy[k, 0]), float(xy[k, 1]), 0.5])
|
| 47 |
+
if q[k, 2] > 0.5: st.append(cur); cur = []
|
| 48 |
+
if cur: st.append(cur)
|
| 49 |
+
out[w] = [st]
|
| 50 |
+
return out
|
| 51 |
+
|
| 52 |
+
def _compose_text(texts, s, rng):
|
| 53 |
+
"""formula_write.compose_text with the word FITTED BY X-HEIGHT: B's word x-height (15..85% of its y) = the height of the layout's
|
| 54 |
+
letter box (MathJax's Cyrillic box is about one x-height); the old 1.3 x box / interquartile rule made a subscript «отн» bigger than v"""
|
| 55 |
+
words, cur = [], []
|
| 56 |
+
for g in sorted(texts, key=lambda g: g['bbox'][0]):
|
| 57 |
+
if not g.get('ch', '').strip():
|
| 58 |
+
if cur: words.append(cur); cur = []
|
| 59 |
+
continue
|
| 60 |
+
if cur and g['bbox'][0] - cur[-1]['bbox'][2] > 50: words.append(cur); cur = []
|
| 61 |
+
cur.append(g)
|
| 62 |
+
if cur: words.append(cur)
|
| 63 |
+
W = FW.text_words([''.join(g['ch'] for g in w) for w in words]); out = []
|
| 64 |
+
for w in words:
|
| 65 |
+
var = W.get(''.join(g['ch'] for g in w))
|
| 66 |
+
if not var: continue
|
| 67 |
+
st = var[0]; A = np.array([p[:2] for t in st for p in t], np.float32)
|
| 68 |
+
bx0 = min(g['bbox'][0] for g in w) * s; bx1 = max(g['bbox'][2] for g in w) * s; yb = float(np.median([g['bbox'][3] for g in w])) * s
|
| 69 |
+
xhb = float(np.median([g['bbox'][3] - g['bbox'][1] for g in w])) * s; xhw = max(np.percentile(A[:, 1], 85) - np.percentile(A[:, 1], 15), 1e-6)
|
| 70 |
+
k = min(1.1 * (bx1 - bx0) / max(np.ptp(A[:, 0]), 1e-6), xhb / xhw); ay = np.percentile(A[:, 1], 85)
|
| 71 |
+
for t in st: out.append([[float((p[0] - A[:, 0].min()) * k + bx0), float((p[1] - ay) * k + yb), float(p[2] if len(p) > 2 else 0.5)] for p in t])
|
| 72 |
+
return out
|
| 73 |
+
|
| 74 |
+
def fix_limits(gl):
|
| 75 |
+
"""an integral's lower limit sits under its foot in the layout (MathJax tucks it in); a hand writes it to the right of the foot:
|
| 76 |
+
glyphs right after ∫/∮ whose box lies in the lower half of the sign and starts before its right edge are moved right of it"""
|
| 77 |
+
for i, g in enumerate(gl):
|
| 78 |
+
if g['c'] not in ('222B', '222E'): continue
|
| 79 |
+
x0, y0, x1, y1 = g['bbox']; my = (y0 + y1) / 2; grp = []
|
| 80 |
+
for h in gl[i + 1:i + 4]:
|
| 81 |
+
if h['bbox'][0] < x1 and h['bbox'][1] > my: grp.append(h)
|
| 82 |
+
if grp:
|
| 83 |
+
dx = x1 + 0.05 * (y1 - y0) - min(h['bbox'][0] for h in grp)
|
| 84 |
+
for h in grp: h['bbox'] = [h['bbox'][0] + dx, h['bbox'][1], h['bbox'][2] + dx, h['bbox'][3]]
|
| 85 |
+
return gl
|
| 86 |
+
|
| 87 |
+
MINE = {'/', '↔', '↦', 'ˆ', '^', '˜', '~', ' ̄', '¯', ' ̃', ' ̂', '‾', '\u0304', '\u0302', '\u0303', '\u0305'} # always pen marks (the net's '/' was rejected by the picker -> dropped; '↔' went to the hand marks -> library -> dropped)
|
| 88 |
+
SUBST = {'2A7E': '2265', '2A7D': '2264'} # ⩾ ⩽ -> ≥ ≤ (the net has no slanted variants)
|
| 89 |
+
def own_marks(gl, writer, rng):
|
| 90 |
+
"""glyphs neither the net, the library nor the hand marks can draw (slash, hat, tilde, overline, <->: they were silently dropped,
|
| 91 |
+
6 of 70 formulas of the measure) -> pen-line strokes over their layout box; returns (the other glyphs, strokes in page units)"""
|
| 92 |
+
S = GD._load(); lib, scale = FW.library(); s = scale.get(writer, scale.get('w01', 0.1)); xh = 450 * s; keep, out = [], []
|
| 93 |
+
for g in gl:
|
| 94 |
+
c = FW.cls_of(g['c']) if g['c'] not in ('BAR', 'TEXT') else g['c']
|
| 95 |
+
if g['c'] in SUBST: keep.append(dict(g, c=SUBST[g['c']])); continue
|
| 96 |
+
if c not in MINE: keep.append(g); continue
|
| 97 |
+
x0, y0, x1, y1 = [v * s for v in g['bbox']]; w = max(x1 - x0, 1e-6); h = max(y1 - y0, 1e-6); j = 0.015 * xh; ym = (y0 + y1) / 2
|
| 98 |
+
if c == '↔': out += FW.line_mark('↔', [x0, y0, x1, y1], rng, xh)
|
| 99 |
+
elif c == '↦': out += FW.line_mark('→', [x0, y0, x1, y1], rng, xh) + [FW._poly([[x0, ym - 0.4 * h], [x0, ym + 0.4 * h]], rng, j, 5)]
|
| 100 |
+
elif c == '/': out.append(FW._poly([[x0 + 0.15 * w, y1], [x1 - 0.15 * w, y0]], rng, j, 10))
|
| 101 |
+
elif c in ('ˆ', '^', ' ̂', '\u0302'): out.append(FW._poly([[x0 + 0.1 * w, y1], [(x0 + x1) / 2, y0], [x1 - 0.1 * w, y1]], rng, j, 5))
|
| 102 |
+
elif c in ('˜', '~', ' ̃', '\u0303'): out.append([[float(x0 + t * w), float(ym - 0.35 * h * np.sin(2 * np.pi * t) + rng.normal(0, j)), 0.5] for t in np.linspace(0, 1, 14)])
|
| 103 |
+
else: out.append(FW._poly([[x0, ym], [x1, ym]], rng, j, 10)) # overline / macron
|
| 104 |
+
FW.SEL_STATS['own_mark_' + c] += 1
|
| 105 |
+
return keep, out
|
| 106 |
+
|
| 107 |
+
def slant_of(q): # as scripts/w03_style_eval.feats: median dx/dy over near-vertical inked steps
|
| 108 |
+
q = np.asarray(q, np.float32); st = np.hypot(q[:, 0], q[:, 1]); v = (q[:, 2] < 0.5) & (np.abs(q[:, 1]) > 2 * np.abs(q[:, 0])) & (st > 0)
|
| 109 |
+
return float(np.median(q[v, 0] / q[v, 1])) if v.sum() > 5 else None
|
| 110 |
+
def shear(q, k): # x += k * (y - mean y) on the absolute points of a [n,3] delta line
|
| 111 |
+
a = np.cumsum(q[:, :2], 0); a[:, 0] += k * (a[:, 1] - a[:, 1].mean()); d = q.copy(); d[1:, :2] = a[1:] - a[:-1]; d[0, :2] = 0; return d
|
| 112 |
+
|
| 113 |
+
def to_line(strokes):
|
| 114 |
+
"""[[x,y,p],...] strokes (page units, y down) -> [n,3] dx,dy,pen (line height 1, arc step STEP) as scripts/ink_lines_prep.resample"""
|
| 115 |
+
S = [np.asarray(s, np.float32)[:, :2] for s in strokes if len(s)]
|
| 116 |
+
if not S: return None
|
| 117 |
+
A = np.concatenate(S); x0, y0 = A.min(0); h = max(float(A[:, 1].max() - y0), 1e-3); out = []
|
| 118 |
+
for p in S:
|
| 119 |
+
p = (p - [x0, y0]) / h
|
| 120 |
+
if len(p) == 1: out.append(np.array([[p[0, 0], p[0, 1], 1.0]], np.float32)); continue
|
| 121 |
+
cum = np.r_[0, np.cumsum(np.hypot(*np.diff(p, axis=0).T))]
|
| 122 |
+
s_ = np.r_[np.arange(0, cum[-1], STEP), cum[-1]] if cum[-1] > 1e-6 else np.array([0.0, cum[-1]])
|
| 123 |
+
r = np.stack([np.interp(s_, cum, p[:, 0]), np.interp(s_, cum, p[:, 1]), np.zeros(len(s_))], 1).astype(np.float32); r[-1, 2] = 1; out.append(r)
|
| 124 |
+
a = np.concatenate(out); d = a.copy(); d[1:, :2] = a[1:, :2] - a[:-1, :2]; d[0, :2] = 0; return d
|
| 125 |
+
|
| 126 |
+
def compose_em(tex, ref_ink, rng, key='ref'):
|
| 127 |
+
"""tex -> (strokes [[x,y,p],...] in EM units: MathJax baseline y = 0, y down, x-height of 'x' 0.442; or None, missing classes)"""
|
| 128 |
+
B = setup(); B['ref'] = np.asarray(ref_ink, np.float32); FW._TW.clear()
|
| 129 |
+
GD.set_style(key, D.semb_vec([B['ref']])[0].detach().cpu().numpy())
|
| 130 |
+
if E('GLYPH_B') == '1': import glyph_bdn; glyph_bdn.set_ref(key, B['ref']) # 05.11 И165: glyphs by B fine-tuned on glyphs
|
| 131 |
+
t = tex.strip().strip('$').strip() # $...$ and $$...$$
|
| 132 |
+
try:
|
| 133 |
+
o = FW.layout([t])[0]
|
| 134 |
+
if not o or not o.get('ok'): return None, ['LAYOUT']
|
| 135 |
+
fix_limits(o['glyphs']); o['glyphs'], extra = own_marks(o['glyphs'], key, rng); strokes, missing = FW.compose_layout(o, key, rng); strokes += extra
|
| 136 |
+
except Exception as e: return None, ['ERROR ' + repr(e)[:80]]
|
| 137 |
+
if not strokes: return None, missing or ['EMPTY']
|
| 138 |
+
s = FW.library()[1]; s = s.get(key, s.get('w01', 0.1)) * 1000.0 # compose_layout draws at layout units x s
|
| 139 |
+
return [[[p[0] / s, p[1] / s, p[2] if len(p) > 2 else 0.5] for p in st] for st in strokes], missing
|
| 140 |
+
|
| 141 |
+
def formula_line(tex, ref_ink, rng, key='ref'):
|
| 142 |
+
"""tex with or without the $...$; ref_ink [n,3] one line of the writer -> (seq [n,3] or None, missing classes)"""
|
| 143 |
+
strokes, missing = compose_em(tex, ref_ink, rng, key)
|
| 144 |
+
if not strokes: return None, missing
|
| 145 |
+
q = to_line(strokes)
|
| 146 |
+
if q is not None and E('SLANT', '0') == '1': # match the reference line's slant (the layout and the picker write upright)
|
| 147 |
+
sr, sq = slant_of(_B['ref']), slant_of(q)
|
| 148 |
+
if sr is not None and sq is not None: q = shear(q, float(np.clip(sr - sq, -0.5, 0.5)))
|
| 149 |
+
return q, missing
|
| 150 |
+
|
| 151 |
+
def spans(toks):
|
| 152 |
+
"""[(a, b)] token index ranges of $...$ (both dollars included)"""
|
| 153 |
+
out, a, k = [], None, 0
|
| 154 |
+
while k < len(toks):
|
| 155 |
+
if toks[k] == '$':
|
| 156 |
+
n = 2 if k + 1 < len(toks) and toks[k + 1] == '$' else 1 # $$ ... $$ = one delimiter
|
| 157 |
+
if a is None: a = k
|
| 158 |
+
else: out.append((a, k + n)); a = None
|
| 159 |
+
k += n; continue
|
| 160 |
+
k += 1
|
| 161 |
+
return out
|
| 162 |
+
|
| 163 |
+
def mixed_line(seq, toks, durs, ref_ink, rng, key='ref'):
|
| 164 |
+
"""MIXED line: B's line `seq` (written with the duration plan `durs` over `toks`) with every $...$ span replaced by the module's
|
| 165 |
+
formula: its points (by the plan) are cut out, the formula is scaled so its x-height (0.442 em) = the x-height of B's text on the line
|
| 166 |
+
(15..85% of y of the text points), put on B's text baseline (85% of y) one word-space after the text before it, and the text after it
|
| 167 |
+
is shifted to follow one word-space later. -> (new seq or None when the plan does not match the line, missing classes, n replaced)"""
|
| 168 |
+
q = np.asarray(seq, np.float32); ci = np.repeat(np.arange(len(toks)), [max(1, int(x)) for x in durs])
|
| 169 |
+
if len(ci) != len(q): return None, ['PLAN'], 0
|
| 170 |
+
A = np.cumsum(q[:, :2], 0); pen = q[:, 2].copy(); sp = spans(toks)
|
| 171 |
+
if not sp: return q, [], 0
|
| 172 |
+
infm = np.zeros(len(q), bool)
|
| 173 |
+
for a, b in sp: infm |= (ci >= a) & (ci < b)
|
| 174 |
+
txt = ~infm & np.array([not toks[c].isspace() for c in ci])
|
| 175 |
+
if txt.sum() < 10: return None, ['NOTEXT'], 0
|
| 176 |
+
y = A[txt, 1]; base = float(np.percentile(y, 85)); xh = max(float(np.percentile(y, 85) - np.percentile(y, 15)), 1e-3); gap = 0.6 * xh
|
| 177 |
+
k = xh / 0.442; pieces = []; shift = 0.0; miss = []; prev = 0; nrep = 0
|
| 178 |
+
for a, b in sp:
|
| 179 |
+
m = (ci >= a) & (ci < b); ix = np.where(m)[0]
|
| 180 |
+
before = np.arange(prev, ix[0]); pieces.append((A[before] + [shift, 0], pen[before]))
|
| 181 |
+
x_left = (A[before, 0].max() + shift) if len(before) else A[ix, 0].min() + shift
|
| 182 |
+
st, ms = compose_em(''.join(toks[a:b]).strip('$'), ref_ink, rng, key); miss += ms
|
| 183 |
+
if not st: # the module could not write it -> keep B's own points of the span
|
| 184 |
+
pieces.append((A[ix] + [shift, 0], pen[ix])); prev = ix[-1] + 1; continue
|
| 185 |
+
f = to_line_abs(st, k); fx0 = f[:, 0].min(); f[:, 0] += x_left + gap - fx0; f[:, 1] += base
|
| 186 |
+
if len(before): pieces[-1][1][-1] = 1.0 # lift the pen before the formula
|
| 187 |
+
pieces.append((f[:, :2], f[:, 2])); nrep += 1
|
| 188 |
+
nxt = np.arange(ix[-1] + 1, len(q)); x_right = A[nxt, 0].min() if len(nxt) else None
|
| 189 |
+
if x_right is not None: shift = f[:, 0].max() + gap - x_right
|
| 190 |
+
prev = ix[-1] + 1
|
| 191 |
+
rest = np.arange(prev, len(q)); pieces.append((A[rest] + [shift, 0], pen[rest]))
|
| 192 |
+
P_ = np.concatenate([p for p, _ in pieces if len(p)]); pn = np.concatenate([p for _, p in pieces if len(p)]); pn[-1] = 1.0
|
| 193 |
+
d = np.zeros((len(P_), 3), np.float32); d[1:, :2] = P_[1:] - P_[:-1]; d[0, :2] = 0; d[:, 2] = pn; return d, miss, nrep
|
| 194 |
+
|
| 195 |
+
def to_line_abs(strokes, k):
|
| 196 |
+
"""em strokes -> absolute points [n,3] (x, y, pen) scaled by k, resampled at STEP (line units), pen=1 on each stroke's last point"""
|
| 197 |
+
out = []
|
| 198 |
+
for st in strokes:
|
| 199 |
+
p = np.asarray(st, np.float32)[:, :2] * k
|
| 200 |
+
if len(p) == 1: out.append(np.array([[p[0, 0], p[0, 1], 1.0]], np.float32)); continue
|
| 201 |
+
cum = np.r_[0, np.cumsum(np.hypot(*np.diff(p, axis=0).T))]
|
| 202 |
+
s_ = np.r_[np.arange(0, cum[-1], STEP), cum[-1]] if cum[-1] > 1e-6 else np.array([0.0, cum[-1]])
|
| 203 |
+
r = np.stack([np.interp(s_, cum, p[:, 0]), np.interp(s_, cum, p[:, 1]), np.zeros(len(s_))], 1).astype(np.float32); r[-1, 2] = 1; out.append(r)
|
| 204 |
+
return np.concatenate(out)
|
| 205 |
+
|
| 206 |
+
if __name__ == '__main__':
|
| 207 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 208 |
+
out = sys.argv[1]
|
| 209 |
+
if os.path.exists(out): sys.exit(f'{out} exists')
|
| 210 |
+
z = np.load(os.path.join(P, 'data', 'ink_lines_u2', 'test_raw.npz'), allow_pickle=True); off = z['pt_off']; ink = lambda i: z['pts'][off[i]:off[i + 1]].astype(np.float32)
|
| 211 |
+
meta = [json.loads(l) for l in open(os.path.join(P, 'data', 'ink_lines_u2', 'meta_test.jsonl'))]
|
| 212 |
+
rid = int(E('REF_ID')) if E('REF_ID') else next(i for i, m in enumerate(meta) if m.get('writer_id') == E('WRITER', 'w13') and m.get('kind') == 'prose')
|
| 213 |
+
rng = np.random.default_rng(int(E('SEED', '0'))); rows = [('образец', ink(rid))]
|
| 214 |
+
for tex in sys.argv[2:]:
|
| 215 |
+
q, miss = formula_line(tex, ink(rid), rng); print(tex, 'missing', miss, flush=True)
|
| 216 |
+
if q is not None: rows.append((tex[:40], q))
|
| 217 |
+
font = ImageFont.truetype(os.path.join(P, 'DejaVuSans.ttf'), 14); S = 80; ims = []
|
| 218 |
+
for lab, q in rows:
|
| 219 |
+
a = np.cumsum(q[:, :2], 0); a -= a.min(0); im = Image.new('L', (int(a[:, 0].max() * S) + 20, int(a[:, 1].max() * S) + 20), 255); d = ImageDraw.Draw(im); prev = None
|
| 220 |
+
for (x, y), pen in zip(a, q[:, 2]):
|
| 221 |
+
p_ = (10 + x * S, 10 + y * S)
|
| 222 |
+
if prev is not None: d.line([prev, p_], fill=0, width=2)
|
| 223 |
+
prev = None if pen > 0.5 else p_
|
| 224 |
+
ims.append((lab, im))
|
| 225 |
+
W = max(im.size[0] for _, im in ims) + 20; sh = Image.new('L', (min(W, 2400), sum(im.size[1] + 20 for _, im in ims)), 255); dr = ImageDraw.Draw(sh); y = 0
|
| 226 |
+
for lab, im in ims: dr.text((6, y), lab, fill=120, font=font); sh.paste(im, (10, y + 16)); y += im.size[1] + 20
|
| 227 |
+
sh.save(out); print(out, sh.size)
|
scripts/formula_write.py
ADDED
|
@@ -0,0 +1,626 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""Write a LaTeX formula in a writer's hand from the library of their REAL glyphs (data/formula_glyphs/glyphs_clean.jsonl):
|
| 3 |
+
MathJax layout (tools/mathjax/layout.mjs) gives each glyph's box; every box gets a random real sample of that class (same writer
|
| 4 |
+
if available), scaled to the box height (BAR and wide glyphs: to the width), centred on the box, with human jitter: per-glyph
|
| 5 |
+
offset (JIT_XY of the writer's x-height), size (JIT_S), rotation (JIT_R degrees), and a slow random walk of the baseline (DRIFT).
|
| 6 |
+
Scale em->page units = median over the writer's glyphs of hand box height / layout box height.
|
| 7 |
+
As a module: compose(tex, writer, rng, exclude_docs=()) -> (strokes [[x,y,p],...], missing classes).
|
| 8 |
+
CLI (judge): N formulas from the valid notebooks of ink_dataset_v2 (their glyphs excluded from the library), each composed and
|
| 9 |
+
read by the external VLM as LaTeX; the same is done for the REAL handwritten crops of those formulas (control). Prints CER on
|
| 10 |
+
normalised LaTeX and writes figures/formula_write_<tag>.png. Env: PROJ, WRITER (w01), N (40), SEED (5), TAG, JIT_XY (0.06),
|
| 11 |
+
JIT_S (0.06), JIT_R (2.5), DRIFT (0.02), OPENROUTER_KEY.
|
| 12 |
+
21.10: SEL_K (1 = old behaviour) -- with GEN_CKPT, draw K samples per glyph from the network and keep the one the glyph
|
| 13 |
+
classifier (SEL_CLS, default ckpt_dl/glyph_cls1.pt from scripts/train_glyph_cls.py, trained without the valid notebooks) gives
|
| 14 |
+
the highest probability of its own class, among samples whose stroke count has share >= SEL_NMIN (0.4) among the
|
| 15 |
+
library's real glyphs of that class (the net draws x as ONE stroke, a half-x that the classifier still calls x); if no sample is recognised as its class (argmax != own for all K) -> a real glyph
|
| 16 |
+
of that class from the library. Classes the classifier does not know keep sample 0. LINE_MARKS=1: fraction bars, radicals and
|
| 17 |
+
arrows (BAR, sqrt, -> <- up down => <= <=>, <->, vector arrow) are drawn as pen lines over their layout box, not as glyphs.
|
| 18 |
+
LINE_MARKS=2 (hand marks): the same marks from the writer's REAL strokes: a real fraction bar of similar length, stretched to
|
| 19 |
+
box width x a real hand/layout length ratio (wobble, tilt, pen speed and pressure stay real); a radical = the hook of a real
|
| 20 |
+
radical (cut where it reaches the top) scaled to the box height + a real bar continuing it in the same stroke to the box end;
|
| 21 |
+
-> => <=> and the vector arrow = a real arrow whose shaft is stretched and head kept; rare arrows (up/down/left) = a real glyph
|
| 22 |
+
as in the library mode; if the writer has no samples -> the LINE_MARKS=1 line.
|
| 23 |
+
GEN=vae (default: the glyph1 net via GEN_CKPT): glyphs from the glyph VAE (scripts/glyph_vae.py, ckpt_glyph_vae/); with SEL_K the
|
| 24 |
+
K variants differ by z (a random real glyph of the writer, shrunk: GVAE_SHRINK, default 0.75) and MDN temperature (GVAE_TAUS,
|
| 25 |
+
cycled, default 0.2..0.4); with GEN=vae the pick defaults to SEL_PICK=any: a random variant among those with p(own class) >=
|
| 26 |
+
SEL_PMIN (0.8), else the best (SEL_PICK=best = always the most probable = least varied). Classes unknown to the VAE -> library.
|
| 27 |
+
XF (I90, 22.10; off by default) = per-glyph deformations on top of the layout: XF=w03 -> shear + parabolas (w03's list), XF=all ->
|
| 28 |
+
all below; each amplitude has its own variable (0 switches it off). Glyph coordinates centred and divided by L = max(w, h).
|
| 29 |
+
Measured (logs/glyph_transform_stats.py -> figures/glyph_transform_stats.json): top-up of w01's within-class spread over the
|
| 30 |
+
VAE's, sqrt(real^2 - vae^2), median of the 30 most frequent classes: shear_x 0.14, shear_y 0.19, parabola_v 1.47, parabola_h 0.37,
|
| 31 |
+
log aspect 0.15 -- but this spread is mostly DIFFERENT SHAPES of a class (allographs), not deformation: at 0.8 x these values the
|
| 32 |
+
classifier recognised 73% instead of 94% (parabola_v alone -12 pp). Defaults = the largest set that keeps recognition within ~1 pp
|
| 33 |
+
(400 held-out glyphs): XF_SHX 0.06 (x += s y), XF_SHY 0.05 (y += s x), XF_PV 0.10 (y += a x^2), XF_PH 0.08 (x += b y^2),
|
| 34 |
+
XF_ASP 0.04 (log aspect stretch), XF_ASC 0.04 (separate stretch of the parts above/below the middle band |y|>0.25), XF_PERSP 0.02
|
| 35 |
+
(mild projective, not measured), XF_TREM 0 (low-pass tremor; w01's strokes are smoother than the VAE's -> off), XF_SPEED 0.5
|
| 36 |
+
(uneven point density along a stroke: her step-length CV 0.64 vs VAE 0.11; shape unchanged), XF_PRES 0.27 (log pressure scale, her
|
| 37 |
+
per-class spread 0.34), XF_GAP 0.22 and XF_BASE 0.08 (AR(0.7)/AR(0.8) offsets along the formula in glyph heights: her residual of
|
| 38 |
+
hand vs layout centres 0.32 in x, 0.125 in y, minus JIT_XY, x0.7).
|
| 39 |
+
GEN=vae2 (M3, 22.10; THE DEFAULT since 22.10 ~11:00 -- GEN=vae = previous VAE mode, GEN=lib = library/glyph1 as before): GEN=vae with the
|
| 40 |
+
M3 defaults -- GVAE_CKPT ckpt_glyph_vae_iiC3L/last.pt (22.10 ~12:40, coordinator: best formula judge 35/30/0.07, diversity 0.93; GVAE_CKPT=ckpt_glyph_vae_iiC3/last.pt = the previous one) (VAE on glyphs_clean3 + classifier penalty,
|
| 41 |
+
free bits 0.75/dim, KL weight 0.1, decoder input dropout 0.5: z carries ~24 nats -> within-class diversity 0.93 of w01's), GVAE_SHRINK 1.0,
|
| 42 |
+
GVAE_TAUS 0.1-0.3,
|
| 43 |
+
SEL_CLS ckpt_dl/glyph_cls2.pt (clean classifier), SEL_K 8, library FW_LIB=glyphs_clean3.jsonl (fallback and hand marks); each can be
|
| 44 |
+
overridden by its variable. Shape donors (DONORS=1, vae2 only): O <- 0, κ <- k, Σ <- ∑ (the cleaned library has 1, 2 and 12 such
|
| 45 |
+
glyphs; the net cannot learn them).
|
| 46 |
+
FIX40=1 (22.10, w03: 40/40 by eye; DEFAULT for GEN=vae2, FIX40=0 = without) sets SEL_PICK=best SEL_K=16 XF_GAP=0.1, keeps operators (= + - < > etc.) unsheared under XF, draws
|
| 47 |
+
FIX_LIBCLS (λ,∩,d,μ: the VAE draws them badly) and [ ] from the real library, dot accents as a pen dot, and switches on three fixes, each also by its own flag (default off = old results):
|
| 48 |
+
FIX_SCRIPT -- sub/superscripts at w01's measured size (logs/script_scale_stats.py: script = layout box < 0.85 x the same char at base
|
| 49 |
+
level; her script size relative to MathJax's: digits 0.71, Latin letters 1.08 -> kept 1.0, other 0.92; SCRIPT_DIGIT/SCRIPT_OTHER);
|
| 50 |
+
FIX_TEXT -- \text{...} (Cyrillic) written by the LINE model (scripts/text_words.py, dpo1@1500, w01 refs) into the layout's text
|
| 51 |
+
boxes (tools/mathjax/layout.mjs LAYOUT_TEXT=1); FIX_DONOR -- << = two '<' glyphs; ->, <-, down/up arrows, vector arrow = a real bar
|
| 52 |
+
(shaft) + a real '>' (head), rotated as needed (w01's own => is '=' + '>'; her curled real arrows read as =>).
|
| 53 |
+
The 21.10 vae40_sel_hand run = GVAE_SHRINK=0.5 GVAE_TAUS=0.1,0.2,0.3,0.15,0.1,0.25,0.2,0.3 SEL_PICK=best."""
|
| 54 |
+
import os, sys, json, math, subprocess, collections, unicodedata, re, numpy as np
|
| 55 |
+
from PIL import Image, ImageDraw
|
| 56 |
+
PROJ=os.environ.get('PROJ',os.getcwd()); E=os.environ.get
|
| 57 |
+
if __name__=='__main__': os.environ.setdefault('GEN','vae2') # only when run as the CLI (importers of compose() keep their own env); M3 (22.10 ~11:00, coordinator): vae2 is the default; GEN=vae = the 21.10/22.10 VAE mode, GEN=lib (any other value) = library / glyph1 via GEN_CKPT as before
|
| 58 |
+
if E('GEN') in ('vae2','dn'): os.environ.setdefault('FIX40','1') # 22.10: the 40/40 fixes are the vae2 default (FIX40=0 = M3's vae2 as before)
|
| 59 |
+
if E('FIX40')=='1': # 22.10 40/40: the most typical of 16 variants (correct shape over variety), smaller gap jitter (0.22 made glyphs collide)
|
| 60 |
+
for _k,_v in (('SEL_PICK','best'),('SEL_K','16'),('XF_GAP','0.1')): os.environ.setdefault(_k,_v)
|
| 61 |
+
if E('GEN')=='dn': # И116 (22.10): GEN=dn = the glyph DENOISER (scripts/glyph_dn.py, ckpt GDN_CKPT, CFG GDN_CFG, DDIM GDN_STEPS) with the vae2 picker/library defaults
|
| 62 |
+
for _k,_v in (('GDN_CKPT',os.path.join(PROJ,'ckpt_dl','glyphdn','glyphdnB','fp16','step050000.pt')),('GDN_CFG','1.2'),('SEL_CLS',os.path.join(PROJ,'ckpt_dl','glyph_cls2.pt')),('SEL_K','8'),('FW_LIB','glyphs_clean3.jsonl')): os.environ.setdefault(_k,_v)
|
| 63 |
+
if E('GEN')=='vae2': # M3 (22.10): the M3 glyph VAE trained on the cleaned library + the clean classifier as the picker; other modes untouched
|
| 64 |
+
for _k,_v in (('GVAE_CKPT',os.path.join(PROJ,'ckpt_glyph_vae_iiC3L','last.pt')),('GVAE_SHRINK','1.0'),('GVAE_TAUS','0.1,0.2,0.3,0.15,0.25'),('SEL_CLS',os.path.join(PROJ,'ckpt_dl','glyph_cls2.pt')),('SEL_K','8'),('FW_LIB','glyphs_clean3.jsonl')): os.environ.setdefault(_k,_v)
|
| 65 |
+
FIX40=E('FIX40','0')=='1'; FIX_LIBCLS=set(E('FIX_LIBCLS','λ,∩,d,μ,=,M,θ,j,t').split(',')) if E('FIX_LIBCLS','λ,∩,d,μ,=,M,θ,j,t') else set(); XF_KEEP=set('=+−-<>≠±') # XF_KEEP: operators keep their shape under FIX40 (a sheared '=' reads as '>')
|
| 66 |
+
FIX_SCRIPT=E('FIX_SCRIPT','1' if FIX40 else '0')=='1'; FIX_TEXT=E('FIX_TEXT','1' if FIX40 else '0')=='1'; FIX_DONOR=E('FIX_DONOR','1' if FIX40 else '0')=='1' # 22.10 40/40 task, see FIX40 in the docstring
|
| 67 |
+
SEL_K=int(E('SEL_K','1')); SEL_NMIN=float(E('SEL_NMIN','0.4')); LINE_MARKS=int(E('LINE_MARKS','0')); SEL_STATS=collections.Counter()
|
| 68 |
+
JIT_XY=float(E('JIT_XY','0.06')); JIT_S=float(E('JIT_S','0.06')); JIT_R=float(E('JIT_R','2.5')); DRIFT=float(E('DRIFT','0.02'))
|
| 69 |
+
D=os.path.join(PROJ,'data','formula_glyphs')
|
| 70 |
+
def cls_of(c):
|
| 71 |
+
if c=='BAR': return 'BAR'
|
| 72 |
+
ch=chr(int(c,16))
|
| 73 |
+
if ch=='\u20d7': return 'VEC'
|
| 74 |
+
n=unicodedata.normalize('NFKC',ch); return n if n.strip() else ch
|
| 75 |
+
_LIB=None
|
| 76 |
+
def library():
|
| 77 |
+
global _LIB
|
| 78 |
+
if _LIB is None and E('FW_LIB')=='none': _LIB=(collections.defaultdict(list),{}) # 06.11 open release: no library of real people's strokes (LINE_MARKS=1, FW_NS)
|
| 79 |
+
if _LIB is None:
|
| 80 |
+
lib=collections.defaultdict(list); sc=collections.defaultdict(list)
|
| 81 |
+
for l in open(os.path.join(D,E('FW_LIB','glyphs_clean.jsonl'))):
|
| 82 |
+
g=json.loads(l); lib[(g['src'],g['cls'])].append(g); lib[('*',g['cls'])].append(g)
|
| 83 |
+
lh=g['lbox'][3]-g['lbox'][1]
|
| 84 |
+
if g['cls'] not in ('BAR',) and lh>200: sc[g['src']].append((g['box'][3]-g['box'][1])/lh)
|
| 85 |
+
_LIB=(lib,{k:float(np.median(v)) for k,v in sc.items()})
|
| 86 |
+
return _LIB
|
| 87 |
+
def layout(texs):
|
| 88 |
+
inp='\n'.join(json.dumps(dict(id=i,tex=t),ensure_ascii=False) for i,t in enumerate(texs))+'\n'
|
| 89 |
+
res=subprocess.run(['node','layout.mjs'],input=inp,capture_output=True,text=True,cwd=os.path.join(PROJ,'tools','mathjax'),env=dict(os.environ,LAYOUT_TEXT='1' if FIX_TEXT else '0'))
|
| 90 |
+
out={}
|
| 91 |
+
for l in res.stdout.splitlines():
|
| 92 |
+
try: o=json.loads(l); out[o['id']]=o
|
| 93 |
+
except ValueError: pass
|
| 94 |
+
return [out.get(i) for i in range(len(texs))]
|
| 95 |
+
def merge_radicals(gl):
|
| 96 |
+
out=[]; i=0
|
| 97 |
+
while i<len(gl):
|
| 98 |
+
x=gl[i]
|
| 99 |
+
if x['c']=='221A' and i+1<len(gl) and gl[i+1]['c']=='BAR' and gl[i+1].get('node') in ('msqrt','mroot'):
|
| 100 |
+
b=gl[i+1]['bbox']; x=dict(c='221A',sw=x['bbox'][2]-x['bbox'][0],bbox=[min(x['bbox'][0],b[0]),min(x['bbox'][1],b[1]),max(x['bbox'][2],b[2]),max(x['bbox'][3],b[3])]); i+=2 # sw: width of the sign itself (hand radicals, LINE_MARKS=2)
|
| 101 |
+
else: i+=1
|
| 102 |
+
out.append(x)
|
| 103 |
+
return out
|
| 104 |
+
GEN=None
|
| 105 |
+
def glyph_char(c):
|
| 106 |
+
if c=='BAR': return '\u2014'
|
| 107 |
+
if c=='VEC': return '\u20d7'
|
| 108 |
+
return c
|
| 109 |
+
def load_gen(ckpt):
|
| 110 |
+
"""glyph generator (train_ink_al2 trained on data/ink_glyphs): returns (module, model)"""
|
| 111 |
+
global GEN
|
| 112 |
+
os.environ.setdefault('INK_DS','ink_glyphs'); os.environ.setdefault('META_DS','ink_dataset_glyphs'); os.environ.setdefault('ALIGN','align_all.npz'); os.environ.setdefault('HW_RUN','glyph_gen')
|
| 113 |
+
os.environ.setdefault('MAXPTS','600'); os.environ.setdefault('MAXTOK','4'); os.environ.setdefault('PRES','1'); os.environ.setdefault('HID','384')
|
| 114 |
+
sys.path.insert(0,os.path.join(PROJ,'scripts')); import importlib; TI=importlib.import_module('train_ink_al2'); import torch
|
| 115 |
+
st=torch.load(ckpt,map_location=TI.dev); m=TI.Model().to(TI.dev); m.load_state_dict({k:v.float() for k,v in st.get('e',st.get('m',st)).items()}); m.eval(); GEN=(TI,m); return GEN
|
| 116 |
+
def gen_samples(chars,writer,rng,exclude_docs=()):
|
| 117 |
+
"""one generated glyph per char, style from 3 real glyphs of the writer (same page) -> list of strokes lists [[x,y,0,p],...]"""
|
| 118 |
+
TI,m=GEN; lib,_=library(); import torch
|
| 119 |
+
pool=[q for q in lib.get(('*','x'),[])+lib.get(('*','2'),[])+lib.get(('*','a'),[]) if q['src']==writer and q['doc'] not in exclude_docs]
|
| 120 |
+
ref=[]
|
| 121 |
+
for q in [pool[int(i)] for i in rng.choice(len(pool),3,replace=False)]:
|
| 122 |
+
pts=[np.asarray(st,np.float32) for st in q['strokes']]; allp=np.concatenate([p[:,:2] for p in pts]); h=max(np.ptp(allp[:,1])*2.0,1.0)
|
| 123 |
+
seq=[]; prev=allp[0]
|
| 124 |
+
for p in pts:
|
| 125 |
+
for j,pt in enumerate(p): seq.append([(pt[0]-prev[0])/h,(pt[1]-prev[1])/h,1.0 if j==len(p)-1 else 0.0]); prev=pt[:2]
|
| 126 |
+
ref.append(np.asarray(seq,np.float32))
|
| 127 |
+
sv=TI.style_from_seqs(m,ref); torch.manual_seed(int(rng.integers(1<<30)))
|
| 128 |
+
toks=[[glyph_char(c)] for c in chars]; seqs=TI.sample(m,toks,bias=float(os.environ.get('GBIAS','1.0')),sv=sv) if toks else []
|
| 129 |
+
out=[]
|
| 130 |
+
for q in seqs:
|
| 131 |
+
q=np.asarray(q,np.float32); xy=np.cumsum(q[:,:2],0); pr=q[:,3] if q.shape[1]>3 else np.full(len(q),0.5,np.float32)
|
| 132 |
+
st=[]; cur=[]
|
| 133 |
+
for k in range(len(q)):
|
| 134 |
+
cur.append([float(xy[k,0]),float(xy[k,1]),0.0,float(pr[k])])
|
| 135 |
+
if q[k,2]>0.5: st.append(cur); cur=[]
|
| 136 |
+
if cur: st.append(cur)
|
| 137 |
+
out.append(dict(strokes=st))
|
| 138 |
+
return out
|
| 139 |
+
LINE_SET={'BAR','√','→','←','↑','↓','⇒','⇐','⇔','↔','VEC'}
|
| 140 |
+
def _seg(a,b,n=8): return [[a[0]+(b[0]-a[0])*t,a[1]+(b[1]-a[1])*t] for t in np.linspace(0,1,n)]
|
| 141 |
+
def _poly(P,rng,w,n=6):
|
| 142 |
+
out=[]
|
| 143 |
+
for i in range(len(P)-1): q=_seg(P[i],P[i+1],n); out+=q if not out else q[1:]
|
| 144 |
+
return [[x+rng.normal(0,w),y+rng.normal(0,w),0.5] for x,y in out]
|
| 145 |
+
def line_mark(c,b,rng,xh):
|
| 146 |
+
"""pen-line strokes for a fraction bar / radical / arrow over layout box b (page units, y down)"""
|
| 147 |
+
x0,y0,x1,y1=b; w=x1-x0; h=max(y1-y0,1e-6); ym=(y0+y1)/2; xm=(x0+x1)/2; j=0.02*xh; tilt=rng.normal(0,0.01)*w
|
| 148 |
+
if c=='BAR': return [_poly([[x0-0.03*w,ym-tilt],[x1+0.03*w,ym+tilt]],rng,j,12)]
|
| 149 |
+
if c=='√':
|
| 150 |
+
tw=min(0.5*h,0.4*w); return [_poly([[x0,y0+0.6*h],[x0+0.18*tw,y0+0.52*h],[x0+0.55*tw,y1],[x0+tw,y0],[x1,y0+tilt]],rng,j,6)]
|
| 151 |
+
hl=min(0.35*w,max(0.9*h,0.25*xh)) if c not in ('↑','↓') else min(0.35*h,max(0.9*w,0.25*xh))
|
| 152 |
+
if c=='VEC': return [_poly([[x0,ym],[x1,ym]],rng,j,8),_poly([[x1-hl,ym-0.5*hl],[x1,ym],[x1-hl,ym+0.5*hl]],rng,j,4)]
|
| 153 |
+
if c in ('↑','↓'):
|
| 154 |
+
tip,sg=(y1,-1) if c=='↓' else (y0,1); return [_poly([[xm,y0],[xm,y1]],rng,j,10),_poly([[xm-0.5*hl,tip+sg*hl],[xm,tip],[xm+0.5*hl,tip+sg*hl]],rng,j,4)]
|
| 155 |
+
dbl=c in ('⇒','⇐','⇔'); d=0.22*h if dbl else 0.0; st=[]
|
| 156 |
+
for yy in ([ym-d,ym+d] if dbl else [ym]): st.append(_poly([[x0+(0.15*hl if c in '⇐⇔' else 0),yy],[x1-(0.15*hl if c in '⇒⇔' else 0),yy]],rng,j,10))
|
| 157 |
+
if c in ('→','⇒','⇔','↔'): st.append(_poly([[x1-hl,ym-0.55*hl-d],[x1,ym],[x1-hl,ym+0.55*hl+d]],rng,j,4))
|
| 158 |
+
if c in ('←','⇐','⇔','↔'): st.append(_poly([[x0+hl,ym-0.55*hl-d],[x0,ym],[x0+hl,ym+0.55*hl+d]],rng,j,4))
|
| 159 |
+
return st
|
| 160 |
+
CLS=None
|
| 161 |
+
def classifier():
|
| 162 |
+
global CLS
|
| 163 |
+
if CLS is None:
|
| 164 |
+
sys.path.insert(0,os.path.join(PROJ,'scripts')); import train_glyph_cls as GC
|
| 165 |
+
net,classes=GC.load(E('SEL_CLS',os.path.join(PROJ,'ckpt_dl','glyph_cls1.pt'))); CLS=(GC,net,{c:i for i,c in enumerate(classes)})
|
| 166 |
+
return CLS
|
| 167 |
+
_NS=None
|
| 168 |
+
def nstroke_p(c,n):
|
| 169 |
+
"""share of the library's real glyphs of class c with exactly n strokes (structure check: the net draws x as one stroke = half an x)"""
|
| 170 |
+
global _NS
|
| 171 |
+
if _NS is None and E('FW_NS'): # 06.11: the same statistics precomputed (per class: stroke count -> glyphs), no strokes
|
| 172 |
+
_NS={c:(collections.Counter({int(k):v for k,v in d.items()}),sum(d.values())) for c,d in json.load(open(E('FW_NS'))).items()}
|
| 173 |
+
if _NS is None:
|
| 174 |
+
lib,_=library(); _NS={}
|
| 175 |
+
for (src,cc),L in lib.items():
|
| 176 |
+
if src=='*': cnt=collections.Counter(len(g['strokes']) for g in L); _NS[cc]=(cnt,len(L))
|
| 177 |
+
cnt,tot=_NS.get(c,(None,0))
|
| 178 |
+
return cnt[n]/tot if tot>=20 else 1.0
|
| 179 |
+
def pick_sample(cands,c,relh,rng=None,force=False):
|
| 180 |
+
"""index of the best of K generated samples for class c, or None (no sample recognised as c -> library)"""
|
| 181 |
+
GC,net,ci=classifier(); import torch
|
| 182 |
+
if c not in ci: SEL_STATS['unscored']+=1; return 0
|
| 183 |
+
ok=[k for k,q in enumerate(cands) if q['strokes'] and sum(len(t) for t in q['strokes'])>=2 and nstroke_p(c,len(q['strokes']))>=SEL_NMIN]
|
| 184 |
+
if FIX40 and c in ('0','O','o'): ok=[k for k in ok if len(cands[k]['strokes'])==1] # no slashed zeros (Ø)
|
| 185 |
+
if not ok and force: ok=[k for k,q in enumerate(cands) if q['strokes'] and sum(len(t) for t in q['strokes'])>=2] # force (06.11): the best of what there is, never None while anything was drawn
|
| 186 |
+
if not ok: SEL_STATS['fallback_nstrokes']+=1; return None
|
| 187 |
+
X=np.stack([GC.render(cands[k]['strokes']) for k in ok]); A=np.stack([GC.feats(cands[k]['strokes'],relh) for k in ok])
|
| 188 |
+
with torch.no_grad(): P=torch.softmax(net(torch.from_numpy(X)[:,None].to(GC.dev),torch.from_numpy(A).to(GC.dev)),-1).cpu().numpy()
|
| 189 |
+
own=P[:,ci[c]]; rec=P.argmax(1)==ci[c]
|
| 190 |
+
if FIX40: # 22.10: a look-alike twin (b/ℓ, t/τ, 0/θ) must lose clearly: rank by the margin over the best other class, need >= SEL_MARGIN
|
| 191 |
+
oth=P.copy(); oth[:,ci[c]]=-1; own=own-oth.max(1); rec=rec&(own>=float(E('SEL_MARGIN','0.25')))
|
| 192 |
+
if not rec.any() and force: SEL_STATS['forced']+=1; return ok[int(np.argmax(own))]
|
| 193 |
+
if not rec.any(): SEL_STATS['fallback']+=1; return None
|
| 194 |
+
SEL_STATS['selected']+=1; SEL_STATS['first_ok']+=int(rec[0])
|
| 195 |
+
if E('SEL_PICK','any' if GEN=='VAE' else 'best')=='any': # I88: a random variant among the confidently recognised ones (argmax p_own always picks the most typical shape)
|
| 196 |
+
top=np.zeros(len(own),bool); top[np.argsort(-own)[:int(E('SEL_TOP','2'))]]=True # only among the SEL_TOP most probable: the classifier gives alpha-shaped lambdas p 0.80-0.97 (lambda labels hold alphas)
|
| 197 |
+
r=np.flatnonzero(rec&(own>=float(E('SEL_PMIN','0.8')))&top)
|
| 198 |
+
if len(r): return ok[int(r[(rng or np.random.default_rng()).integers(len(r))])]
|
| 199 |
+
return ok[int(np.argmax(own))]
|
| 200 |
+
_HM=None
|
| 201 |
+
def _hm_pools(writer,exclude_docs):
|
| 202 |
+
"""real strokes for hand marks: bars (one stroke, with hand/layout length ratios), radical hooks, arrows"""
|
| 203 |
+
global _HM
|
| 204 |
+
key=(writer,tuple(sorted(exclude_docs)))
|
| 205 |
+
if _HM is not None and _HM[0]==key: return _HM[1]
|
| 206 |
+
lib,scale=library(); sc=scale.get(writer,scale.get('w01',0.1)); P={}
|
| 207 |
+
def pool(c): return [g for g in lib.get((writer,c),[]) if g['doc'] not in exclude_docs] or [g for g in lib.get(('*',c),[]) if g['doc'] not in exclude_docs]
|
| 208 |
+
bars=[]
|
| 209 |
+
for g in pool('BAR'):
|
| 210 |
+
if len(g['strokes'])!=1 or len(g['strokes'][0])<4: continue
|
| 211 |
+
q=np.asarray(g['strokes'][0],np.float32); L=float(np.ptp(q[:,0]))
|
| 212 |
+
if L<1e-3: continue
|
| 213 |
+
bars.append((q,L,L/max((g['lbox'][2]-g['lbox'][0])*sc,1e-6)))
|
| 214 |
+
P['BAR']=bars; P['BAR_L']=np.array([b[1] for b in bars]); P['BAR_OV']=np.array([b[2] for b in bars])
|
| 215 |
+
hooks=[]
|
| 216 |
+
for g in pool('√'):
|
| 217 |
+
hh=np.ptp(np.concatenate([np.asarray(t,np.float32)[:,1] for t in g['strokes']]))
|
| 218 |
+
for t in sorted(g['strokes'],key=lambda t:min(p[0] for p in t)):
|
| 219 |
+
q=np.asarray(t,np.float32)
|
| 220 |
+
if len(q)<5 or np.ptp(q[:,1])<0.6*hh: continue
|
| 221 |
+
lo=int(np.argmax(q[:,1])); top=q[:,1].min()+0.12*np.ptp(q[:,1]); j=next((k for k in range(lo,len(q)) if q[k,1]<=top),None)
|
| 222 |
+
if j is None or j-lo<2: break
|
| 223 |
+
hk=q[:j+1]; w=np.ptp(hk[:,0]); h=np.ptp(hk[:,1])
|
| 224 |
+
if 0.25*h<=w<=1.2*h and hk[0,0]<=q[lo,0]<=hk[-1,0]+0.1*h and hk[0,1]<q[lo,1]-0.2*h: hooks.append(hk) # left start above the bottom, down, then up to the top-right
|
| 225 |
+
break
|
| 226 |
+
P['√']=hooks
|
| 227 |
+
for c in ('=','>','<'): P['G'+c]=[[np.asarray(t,np.float32) for t in g['strokes']] for g in pool(c)] # w01 writes => as '=' + '>' (real column)
|
| 228 |
+
def _chev_ok(q,right): # a clean chevron: one point of the extreme x in the middle third of the height, both ends on the other side
|
| 229 |
+
A=np.concatenate(q)[:,:2]; w=np.ptp(A[:,0]); h=np.ptp(A[:,1])
|
| 230 |
+
if w<1e-6 or not 0.6<=h/w<=3.0: return False
|
| 231 |
+
k=int(np.argmax(A[:,0])) if right else int(np.argmin(A[:,0])); yk=(A[k,1]-A[:,1].min())/max(h,1e-6)
|
| 232 |
+
ends=[t[0,0] for t in q]+[t[-1,0] for t in q]; xr=[(e-A[:,0].min())/w for e in ends]
|
| 233 |
+
return 0.25<=yk<=0.75 and (max(xr)<0.6 if right else min(xr)>0.4)
|
| 234 |
+
if FIX_DONOR: P['G>']=[q for q in P['G>'] if _chev_ok(q,True)] or P['G>']; P['G<']=[q for q in P['G<'] if _chev_ok(q,False)] or P['G<']
|
| 235 |
+
for c in ('→','⇒','⇔','VEC'):
|
| 236 |
+
A=[]
|
| 237 |
+
for g in pool(c):
|
| 238 |
+
q=[np.asarray(t,np.float32) for t in g['strokes']]; allp=np.concatenate(q); W=float(np.ptp(allp[:,0]))
|
| 239 |
+
if W<1e-3 or np.ptp(allp[:,1])<1e-3: continue
|
| 240 |
+
r=W/max((g['lbox'][2]-g['lbox'][0])*sc,1e-6)
|
| 241 |
+
if W/np.ptp(allp[:,1])<(2.0 if c=="→" else 1.4) or not 0.6<=r<=1.6: continue # whole, straight arrows only (no lone heads / merged neighbours)
|
| 242 |
+
A.append((q,r))
|
| 243 |
+
P[c]=A
|
| 244 |
+
_HM=(key,P); return P
|
| 245 |
+
def _real_bar(P,Lt,rng):
|
| 246 |
+
L=P['BAR_L']; ok=np.where((L>=0.75*Lt)&(L<=1.33*Lt))[0]
|
| 247 |
+
if len(ok)==0: ok=np.argsort(np.abs(np.log(L/Lt)))[:20]
|
| 248 |
+
q=P['BAR'][int(ok[rng.integers(len(ok))])][0]; k=Lt/max(float(np.ptp(q[:,0])),1e-6); ky=min(max(k,0.6),1.6)
|
| 249 |
+
x=(q[:,0]-q[:,0].min())*k; y=(q[:,1]-q[:,1].mean())*ky; pr=q[:,3] if q.shape[1]>3 else np.full(len(q),0.5,np.float32)
|
| 250 |
+
return np.stack([x,y,pr],1)
|
| 251 |
+
VBAR=E('VBAR','1')=='1' # 05.11: | and the norm sign from real bars (VBAR=0: the old way, the glyph net / library)
|
| 252 |
+
def vbar_mark(c,b,rng,xh,P):
|
| 253 |
+
"""| (one) or the norm / parallel sign (two) as REAL bars of the hand (fraction bars, _real_bar) stood upright, drawn top-down, the top
|
| 254 |
+
leaning right by 0..6 deg; length 0.9..1.0 of the layout box height; the two bars of a norm 0.35..0.45 x-height apart"""
|
| 255 |
+
x0,y0,x1,y1=b; bh=max(y1-y0,1e-6); cx=(x0+x1)/2; out=[]
|
| 256 |
+
n=1 if c in ('|','∣') else 2; gap=rng.uniform(0.35,0.45)*xh; tl=np.tan(np.radians(rng.uniform(0,6)))
|
| 257 |
+
for k in range(n):
|
| 258 |
+
Lt=bh*rng.uniform(0.9,1.0); q=_real_bar(P,Lt,rng); dx=(k-(n-1)/2)*gap+rng.normal(0,0.02*xh)
|
| 259 |
+
Y=q[:,0]+y0+(bh-Lt)/2+rng.normal(0,0.02*xh); X=cx+dx+q[:,1]+(Lt-q[:,0])*tl-Lt*tl/2
|
| 260 |
+
out.append(np.stack([X,Y,q[:,2]],1))
|
| 261 |
+
return out
|
| 262 |
+
def hand_mark(c,b,rng,xh,writer,exclude_docs,sw=None):
|
| 263 |
+
"""LINE_MARKS=2: real-stroke mark over layout box b (page units, y down); 'LIB' = use a real library glyph; None = pen line"""
|
| 264 |
+
P=_hm_pools(writer,exclude_docs); x0,y0,x1,y1=b; bw=x1-x0; bh=max(y1-y0,1e-6); j=0.03*xh
|
| 265 |
+
if c=='BAR' and len(P['BAR'])>=20:
|
| 266 |
+
ov=float(P['BAR_OV'][rng.integers(len(P['BAR_OV']))]); ov=min(max(ov,0.85),1.1) # real hand/layout length ratio, clipped so the bar still covers the fraction
|
| 267 |
+
q=_real_bar(P,bw*ov,rng); cx=(x0+x1)/2+rng.normal(0,j); q[:,0]+=cx-q[:,0].max()/2; q[:,1]+=(y0+y1)/2+rng.normal(0,j)
|
| 268 |
+
return [q.tolist()]
|
| 269 |
+
if c=='√' and len(P['√'])>=5 and len(P['BAR'])>=20:
|
| 270 |
+
hk=P['√'][int(rng.integers(len(P['√'])))]; k=bh/max(float(np.ptp(hk[:,1])),1e-6)*(1+rng.normal(0,0.04)); kx=k
|
| 271 |
+
if sw: kx=min(max(sw*(1+rng.normal(0,0.06))/max(float(np.ptp(hk[:,0])),1e-6),0.35*k),1.2*k) # hook as wide as the layout's sign, not stretched with a tall radicand
|
| 272 |
+
hx=(hk[:,0]-hk[:,0].min())*kx+x0; hy=(hk[:,1]-hk[:,1].min())*k+y0; hp=hk[:,3] if hk.shape[1]>3 else np.full(len(hk),0.5,np.float32)
|
| 273 |
+
ex,ey=hx[-1],hy[-1]; Lt=max(x1-ex,0.2*xh)*(1+abs(rng.normal(0,0.05)))
|
| 274 |
+
q=_real_bar(P,Lt,rng); bx=q[:,0]-q[0,0]+ex; by=q[:,1]-q[0,1]+ey
|
| 275 |
+
return [np.stack([np.concatenate([hx,bx[1:]]),np.concatenate([hy,by[1:]]),np.concatenate([hp,q[1:,2]])],1).tolist()]
|
| 276 |
+
if c in ('⇒','⇔') and len(P['G='])>=20 and len(P['G>'])>=20 and (c=='⇒' or len(P['G<'])>=20):
|
| 277 |
+
def place(q,tb):
|
| 278 |
+
allp=np.concatenate(q)[:,:2]; qx0,qy0=allp.min(0); qx1,qy1=allp.max(0); tw=tb[2]-tb[0]; th=tb[3]-tb[1]
|
| 279 |
+
ky=th/max(qy1-qy0,1e-6)*(1+rng.normal(0,0.04)); kx=min(max(tw/max(qx1-qx0,1e-6),0.6*ky),1.8*ky); cx=(tb[0]+tb[2])/2+rng.normal(0,j); cy=(tb[1]+tb[3])/2+rng.normal(0,j)
|
| 280 |
+
return [np.stack([(t[:,0]-(qx0+qx1)/2)*kx+cx,(t[:,1]-(qy0+qy1)/2)*ky+cy,t[:,3] if t.shape[1]>3 else np.full(len(t),0.5,np.float32)],1).tolist() for t in q]
|
| 281 |
+
pick=lambda k: P[k][int(rng.integers(len(P[k])))]; ym=(y0+y1)/2
|
| 282 |
+
if c=='⇒': return place(pick('G='),[x0,ym-0.3*bh,x0+0.68*bw,ym+0.3*bh])+place(pick('G>'),[x0+0.55*bw,y0,x1,y1])
|
| 283 |
+
return place(pick('G<'),[x0,y0,x0+0.4*bw,y1])+place(pick('G='),[x0+0.3*bw,ym-0.3*bh,x0+0.7*bw,ym+0.3*bh])+place(pick('G>'),[x0+0.6*bw,y0,x1,y1])
|
| 284 |
+
if c in ('→','VEC') and len(P[c])>=5:
|
| 285 |
+
q,ovs=P[c][int(rng.integers(len(P[c])))]; allp=np.concatenate(q)[:,:2]; qx0,qy0=allp.min(0); qx1,qy1=allp.max(0); qh=max(qy1-qy0,1e-6)
|
| 286 |
+
ky=bh/qh*(1+rng.normal(0,0.04)) if c!='VEC' else min(bh/qh,bw/max(qx1-qx0,1e-6)); ov=min(max(float([a[1] for a in P[c]][int(rng.integers(len(P[c])))]),0.85),1.15)
|
| 287 |
+
hw=1.1*qh; W0=(qx1-qx0)*ky; Wt=bw*ov; heads=('R','L') if c=='⇔' else ('R',)
|
| 288 |
+
S0=W0-hw*ky*len(heads); St=Wt-hw*ky*len(heads); f=St/S0 if S0>0.2*W0 and St>0.2*Wt else Wt/max(W0,1e-6)
|
| 289 |
+
def mx(x):
|
| 290 |
+
u=(x-qx0)*ky
|
| 291 |
+
if f==Wt/max(W0,1e-6) and not (S0>0.2*W0 and St>0.2*Wt): return u*f
|
| 292 |
+
a=hw*ky if 'L' in heads else 0.0; e=W0-hw*ky
|
| 293 |
+
return np.where(u<a,u,np.where(u<=e,a+(u-a)*f,a+(e-a)*f+(u-e)))
|
| 294 |
+
cx=(x0+x1)/2+rng.normal(0,j); cy=(y0+y1)/2+rng.normal(0,j); out=[]
|
| 295 |
+
for t in q:
|
| 296 |
+
X=mx(t[:,0]); Y=(t[:,1]-(qy0+qy1)/2)*ky+cy; pr=t[:,3] if t.shape[1]>3 else np.full(len(t),0.5,np.float32); out.append(np.stack([X,Y,pr],1))
|
| 297 |
+
xs=np.concatenate([o[:,0] for o in out]); sh=cx-(xs.min()+xs.max())/2
|
| 298 |
+
for o in out: o[:,0]+=sh
|
| 299 |
+
return [o.tolist() for o in out]
|
| 300 |
+
if c in ('↑','↓','←','⇐','↔','⇒','⇔'): return 'LIB'
|
| 301 |
+
return None
|
| 302 |
+
DONOR={'O':'0','κ':'k','Σ':'∑'} if (E('GEN') in ('vae2','dn') and E('DONORS','1')=='1') else {} # M3 (22.10): shape donors for classes the library has (almost) no glyphs of: O 1, κ 2, Σ 12
|
| 303 |
+
if E('FW_LIB')=='none': DONOR=dict(DONOR,**{'≃':'≈','∙':'⋅','*':'∗','⋯':'…','⇓':'↓','⟸':'←','⇉':'→','⎧':'{','⎩':'{','⎛':'('}) # 06.11 no library: its rare classes the net has no row for
|
| 304 |
+
if FIX_DONOR: DONOR=dict(DONOR,**{'∩':'u','Θ':'θ'}) # 22.10: w01's library has 15 odd ∩; drawn as her u turned upside down
|
| 305 |
+
def cls_net(c): return DONOR.get(c,c)
|
| 306 |
+
def vae_samples(chars,writer,rng,k=1):
|
| 307 |
+
"""chars repeated k times (variant-major, as for glyph1); variant j uses temperature GVAE_TAUS[j % len]"""
|
| 308 |
+
if E('GEN')=='dn': import glyph_dn as GV # И116: the glyph denoiser, same API (tau/shrink ignored)
|
| 309 |
+
else: import glyph_vae as GV
|
| 310 |
+
n=len(chars)//max(k,1); taus=[float(t) for t in E('GVAE_TAUS','0.2,0.3,0.4,0.25,0.2,0.35,0.3,0.4').split(',')]; shrink=float(E('GVAE_SHRINK','0.75')); out=[] # I88 defaults (22.10); the 21.10 run used GVAE_SHRINK=0.5 GVAE_TAUS=0.1,0.2,0.3,0.15,0.1,0.25,0.2,0.3 SEL_PICK=best
|
| 311 |
+
for j in range(max(k,1)):
|
| 312 |
+
r=GV.sample_batch(chars[j*n:(j+1)*n],[writer]*n,rng,tau=taus[j%len(taus)],shrink=shrink)
|
| 313 |
+
out+=[q if isinstance(q,dict) else dict(strokes=[]) for q in r]
|
| 314 |
+
return out
|
| 315 |
+
XF=E('XF','all') # 22.10 00:5x: w03 I90 — transforms on by default (XF= to switch off, XF=w03 for shear+parabolas only)
|
| 316 |
+
def _xa(name,dflt):
|
| 317 |
+
on={'w03':('SHX','SHY','PV','PH'),'all':None}.get(XF,())
|
| 318 |
+
if on is not None and name not in on: return 0.0
|
| 319 |
+
return float(E('XF_'+name,str(dflt)))
|
| 320 |
+
def xf_glyph(pts,mx,my,L,rng):
|
| 321 |
+
"""I90 per-glyph deformation (in glyph units: centred on (mx,my), divided by L); returns new point arrays (same columns)"""
|
| 322 |
+
shx=rng.normal(0,_xa('SHX',0.06)); shy=rng.normal(0,_xa('SHY',0.05)); a=rng.normal(0,_xa('PV',0.10)); bb=rng.normal(0,_xa('PH',0.08))
|
| 323 |
+
asp=rng.normal(0,_xa('ASP',0.04)); eu=rng.normal(0,_xa('ASC',0.04)); ed=rng.normal(0,_xa('ASC',0.04)); px,py=rng.normal(0,_xa('PERSP',0.02),2)
|
| 324 |
+
trem=_xa('TREM',0.0); spd=min(_xa('SPEED',0.5),0.9); prs=math.exp(rng.normal(0,_xa('PRES',0.27)))
|
| 325 |
+
allp=np.concatenate([(p[:,:2]-[mx,my])/L for p in pts]); mx2=float(np.mean(allp[:,0]**2)); my2=float(np.mean(allp[:,1]**2)); out=[]
|
| 326 |
+
for p in pts:
|
| 327 |
+
q=p.copy(); x=(p[:,0]-mx)/L; y=(p[:,1]-my)/L
|
| 328 |
+
if spd>0 and len(p)>3: # uneven pen speed: re-place the points along the same path (shape unchanged, density not)
|
| 329 |
+
d=np.r_[0,np.cumsum(np.linalg.norm(np.diff(np.stack([x,y],1),axis=0),axis=1))]
|
| 330 |
+
if d[-1]>0:
|
| 331 |
+
u=np.linspace(0,1,len(p)); ph=rng.uniform(0,6.283); w=u+spd*np.sin(2*np.pi*u+ph)/(2*np.pi)-spd*np.sin(ph)/(2*np.pi)*(1-u)-spd*np.sin(2*np.pi+ph)/(2*np.pi)*u
|
| 332 |
+
w=np.clip(np.maximum.accumulate(w),0,1)*d[-1]; x=np.interp(w,d,x); y=np.interp(w,d,y)
|
| 333 |
+
if q.shape[1]>3: q[:,3]=np.interp(w,d,p[:,3])
|
| 334 |
+
x=x+shx*y; y=y+shy*x; y=y+a*(x**2-mx2); x=x+bb*(y**2-my2); x=x*math.exp(asp/2); y=y*math.exp(-asp/2)
|
| 335 |
+
y=np.where(y<-0.25,-0.25+(y+0.25)*(1+eu),np.where(y>0.25,0.25+(y-0.25)*(1+ed),y))
|
| 336 |
+
den=1+px*x+py*y; x=x/den; y=y/den
|
| 337 |
+
if trem>0 and len(p)>3:
|
| 338 |
+
k=np.exp(-0.5*(np.arange(-6,7)/3.0)**2); k/=k.sum(); n=rng.normal(0,1,(len(p)+12,2)); n=np.stack([np.convolve(n[:,i],k,'valid') for i in (0,1)],1)[:len(p)]
|
| 339 |
+
n/=max(n.std(),1e-6); x=x+trem*n[:,0]; y=y+trem*n[:,1]
|
| 340 |
+
q[:,0]=x*L+mx; q[:,1]=y*L+my
|
| 341 |
+
if q.shape[1]>3: q[:,3]=np.clip(q[:,3]*prs,0,1)
|
| 342 |
+
out.append(q)
|
| 343 |
+
return out
|
| 344 |
+
_BASEH={}
|
| 345 |
+
def base_heights(codes):
|
| 346 |
+
"""layout box height of each char at base level (cache) -> script detection"""
|
| 347 |
+
new=[c for c in set(codes) if c not in _BASEH and c not in ('BAR','TEXT')]
|
| 348 |
+
if new:
|
| 349 |
+
for c,o in zip(new,layout([chr(int(c,16)) for c in new])):
|
| 350 |
+
_BASEH[c]=(o['glyphs'][0]['bbox'][3]-o['glyphs'][0]['bbox'][1]) if o and o.get('ok') and o['glyphs'] else None
|
| 351 |
+
return _BASEH
|
| 352 |
+
def script_factor(g):
|
| 353 |
+
if not FIX_SCRIPT or g['c'] in ('BAR','TEXT'): return 1.0
|
| 354 |
+
bh0=_BASEH.get(g['c'])
|
| 355 |
+
if not bh0 or (g['bbox'][3]-g['bbox'][1])>=0.85*bh0: return 1.0
|
| 356 |
+
ch=unicodedata.normalize('NFKC',chr(int(g['c'],16)))
|
| 357 |
+
if ch.isdigit(): return float(E('SCRIPT_DIGIT','0.71'))
|
| 358 |
+
if ch.isascii() and ch.isalpha(): return 1.0
|
| 359 |
+
return float(E('SCRIPT_OTHER','0.92'))
|
| 360 |
+
_TW={}
|
| 361 |
+
def text_words(words):
|
| 362 |
+
need=[w for w in dict.fromkeys(words) if w not in _TW]
|
| 363 |
+
if need:
|
| 364 |
+
r=subprocess.run([sys.executable,os.path.join(PROJ,'scripts','text_words.py')],input=json.dumps(need,ensure_ascii=False),capture_output=True,text=True,env=dict(os.environ,PROJ=PROJ))
|
| 365 |
+
try: _TW.update(json.loads(r.stdout.strip().splitlines()[-1]))
|
| 366 |
+
except Exception: print('text_words failed',r.stderr[-300:],flush=True)
|
| 367 |
+
return _TW
|
| 368 |
+
def compose_text(texts,s,rng):
|
| 369 |
+
"""TEXT boxes (one per char) -> words -> line-model ink fitted to each word box (width; x-height from the box)"""
|
| 370 |
+
words=[]; cur=[]
|
| 371 |
+
for g in sorted(texts,key=lambda g:g['bbox'][0]):
|
| 372 |
+
if not g.get('ch','').strip():
|
| 373 |
+
if cur: words.append(cur); cur=[]
|
| 374 |
+
continue
|
| 375 |
+
if cur and g['bbox'][0]-cur[-1]['bbox'][2]>50: words.append(cur); cur=[]
|
| 376 |
+
cur.append(g)
|
| 377 |
+
if cur: words.append(cur)
|
| 378 |
+
W=text_words([''.join(g['ch'] for g in w) for w in words]); out=[]
|
| 379 |
+
for w in words:
|
| 380 |
+
var=W.get(''.join(g['ch'] for g in w)) or []
|
| 381 |
+
if not var: continue
|
| 382 |
+
def asp(v):
|
| 383 |
+
B=np.array([p[:2] for t in v for p in t],np.float32); return np.ptp(B[:,0])/max(np.percentile(B[:,1],75)-np.percentile(B[:,1],25),1e-6)
|
| 384 |
+
a=np.array([asp(v) for v in var]); order=np.argsort(np.abs(np.log(a/np.median(a)))) # a typical variant: some samples are garbage (cut failed, scribbles)
|
| 385 |
+
st=var[int(order[int(rng.integers(min(2,len(order))))])]; A=np.array([p[:2] for t in st for p in t],np.float32)
|
| 386 |
+
bx0=min(g['bbox'][0] for g in w)*s; bx1=max(g['bbox'][2] for g in w)*s; yb=w[0]['bbox'][3]*s; xhb=(w[0]['bbox'][3]-w[0]['bbox'][1])*s
|
| 387 |
+
core=max(np.percentile(A[:,1],75)-np.percentile(A[:,1],25),1e-6); k=min((bx1-bx0)/max(np.ptp(A[:,0]),1e-6),1.3*xhb/core)
|
| 388 |
+
ax=(A[:,0].min()+A[:,0].max())/2; ay=np.percentile(A[:,1],75)
|
| 389 |
+
for t in st: out.append([[float((p[0]-ax)*k+(bx0+bx1)/2),float((p[1]-ay)*k+yb-0.1*xhb),float(p[2] if len(p)>2 else 0.5)] for p in t])
|
| 390 |
+
return out
|
| 391 |
+
def arrow_build(c,b,rng,P):
|
| 392 |
+
"""FIX_DONOR arrows: real bar shaft + real '>' head; c in -> <- down up VEC; b = box (page units)"""
|
| 393 |
+
x0,y0,x1,y1=b; bw=x1-x0; bh=max(y1-y0,1e-6); vert=c in ('↓','↑'); L=(bh if vert else bw)*min(max(1+rng.normal(0,0.05),0.9),1.1); T=bw if vert else bh
|
| 394 |
+
hh=min(max(0.75*T,0.12*L),0.45*L) if c!='VEC' else min(0.9*T,0.35*L)
|
| 395 |
+
q=_real_bar(P,L,rng); sx=q[:,0]-q[:,0].max(); sy=q[:,1]-q[:,1].mean()
|
| 396 |
+
g=P['G>'][int(rng.integers(len(P['G>'])))]; A=np.concatenate(g)[:,:2]; ax1=A[:,0].max(); ay=(A[:,1].min()+A[:,1].max())/2; kh=hh/max(np.ptp(A[:,1]),1e-6)
|
| 397 |
+
parts=[np.stack([sx,sy,q[:,2]],1)]+[np.stack([(t[:,0]-ax1)*kh,(t[:,1]-ay)*kh,t[:,3] if t.shape[1]>3 else np.full(len(t),0.5,np.float32)],1) for t in g]
|
| 398 |
+
cx=(x0+x1)/2; cy=(y0+y1)/2; out=[]
|
| 399 |
+
for t in parts:
|
| 400 |
+
X,Y=t[:,0]+L/2,t[:,1]
|
| 401 |
+
if c=='←': X=-X
|
| 402 |
+
if c=='↓': X,Y=-Y,X
|
| 403 |
+
if c=='↑': X,Y=Y,-X
|
| 404 |
+
out.append(np.stack([X+cx,Y+cy,t[:,2]],1).tolist())
|
| 405 |
+
return out
|
| 406 |
+
def _push(gs,placed,xh):
|
| 407 |
+
"""FIX40 no-collision: shift strokes gs (list of arrays [x,y,...]) right past any earlier glyph box on the same line it overlaps"""
|
| 408 |
+
if not gs or not placed: return
|
| 409 |
+
A=np.concatenate([np.asarray(q)[:,:2] for q in gs]); bx0,by0,bx1,by1=A[:,0].min(),A[:,1].min(),A[:,0].max(),A[:,1].max(); sh=0.0
|
| 410 |
+
for px0,py0,px1,py1 in placed:
|
| 411 |
+
ov=min(by1,py1)-max(by0,py0)
|
| 412 |
+
mh=min(by1-by0,py1-py0)
|
| 413 |
+
same_line=abs((py0+py1)/2-(by0+by1)/2)<0.3*max(by1-by0,py1-py0) # a subscript next to its base is not a collision
|
| 414 |
+
if (py1-py0)>1.8*(by1-by0) and py0<=by0 and py1>=by1: continue # an encloser (radical, tall bracket) is not a neighbour
|
| 415 |
+
if same_line and mh>0.12*xh and ov>0.5*mh and (px0+px1)/2<(bx0+bx1)/2 and px1+0.35*xh>bx0: sh=max(sh,px1+0.35*xh-bx0) # thin things (bars) never push
|
| 416 |
+
if 0<sh<2.0*xh:
|
| 417 |
+
for q in gs: q[:,0]+=sh
|
| 418 |
+
SEL_STATS['pushed']+=1
|
| 419 |
+
def compose_layout(o,writer,rng,exclude_docs=()):
|
| 420 |
+
lib,scale=library(); s=scale.get(writer,scale.get('w01',0.1)); xh=450*s # ~x-height in page units
|
| 421 |
+
strokes=[]; missing=[]; gl=merge_radicals(o['glyphs']); texts=[g for g in gl if g['c']=='TEXT']; gl=[g for g in gl if g['c']!='TEXT']
|
| 422 |
+
if FIX_DONOR: # << = two '<'
|
| 423 |
+
g2=[]
|
| 424 |
+
for g in gl:
|
| 425 |
+
if g['c']=='226A': b=g['bbox']; m=(b[0]+b[2])/2; w=b[2]-b[0]; g2+=[dict(g,c='3C',bbox=[b[0],b[1],m+0.08*w,b[3]]),dict(g,c='3C',bbox=[m-0.08*w,b[1],b[2],b[3]])]
|
| 426 |
+
else: g2.append(g)
|
| 427 |
+
gl=g2
|
| 428 |
+
if FIX_SCRIPT: base_heights([g['c'] for g in gl])
|
| 429 |
+
if texts: strokes+=compose_text(texts,s,rng)
|
| 430 |
+
if not gl: return strokes,missing
|
| 431 |
+
x_min=min(g['bbox'][0] for g in gl)
|
| 432 |
+
gen=None; n=len(gl); fy0=min(g['bbox'][1] for g in gl); fH=max(max(g['bbox'][3] for g in gl)-fy0,1e-6)
|
| 433 |
+
if GEN=='VAE': gen=vae_samples([cls_net(cls_of(g['c'])) for g in gl]*SEL_K,writer,rng,SEL_K)
|
| 434 |
+
elif GEN is not None: gen=gen_samples([cls_of(g['c']) for g in gl]*SEL_K,writer,rng,exclude_docs)
|
| 435 |
+
xr=np.random.default_rng(int(rng.integers(1<<31))) if XF else None; gap=0.0; base=0.0 # XF draws from its own stream: XF off = old results bit for bit
|
| 436 |
+
placed=[]; last_n=0 # text words (added above) count as placed too
|
| 437 |
+
def _bb(sts):
|
| 438 |
+
A=np.array([p[:2] for t in sts for p in t],np.float32); return (float(A[:,0].min()),float(A[:,1].min()),float(A[:,0].max()),float(A[:,1].max()))
|
| 439 |
+
for gi,g in enumerate(gl):
|
| 440 |
+
if FIX40 and len(strokes)>last_n: placed.append(_bb(strokes[last_n:])); last_n=len(strokes)
|
| 441 |
+
rot180=cls_of(g['c'])=='∩' and FIX_DONOR
|
| 442 |
+
c=cls_of(g['c'])
|
| 443 |
+
if c in DONOR: SEL_STATS['donor_'+c]+=1; c=DONOR[c] # drawn as the donor shape (O <- 0, kappa <- k, Sigma <- sum), sized by the layout box
|
| 444 |
+
force_lib=False
|
| 445 |
+
if FIX_DONOR and c in ('⋅','·','…','...','⋯','.',':'): # centred dot / ellipsis / period / colon: pen dots (the net draws a ring)
|
| 446 |
+
b=[v*s for v in g['bbox']]; r=0.05*xh
|
| 447 |
+
if c==':': pts_d=[((b[0]+b[2])/2,b[1]+0.15*(b[3]-b[1])),((b[0]+b[2])/2,b[3]-0.15*(b[3]-b[1]))]
|
| 448 |
+
else:
|
| 449 |
+
nd=1 if c in ('⋅','·','.') else 3; cyd=(b[1]+b[3])/2 if c!='.' else b[3]-r
|
| 450 |
+
pts_d=[(b[0]+(k+0.5)*(b[2]-b[0])/nd,cyd) for k in range(nd)]
|
| 451 |
+
for cxd,cyd in pts_d: strokes.append([[float(cxd-r),float(cyd),0.5],[float(cxd),float(cyd+r*0.6),0.5],[float(cxd+r),float(cyd),0.5]])
|
| 452 |
+
SEL_STATS['dot_mark']+=1; continue
|
| 453 |
+
if FIX_DONOR and c in (' \u0307','\u02d9','\u0307'): # dot accent: a small pen dot (the net draws it as a ring)
|
| 454 |
+
b=[v*s for v in g['bbox']]; cxd=(b[0]+b[2])/2; cyd=(b[1]+b[3])/2; r=0.05*xh
|
| 455 |
+
strokes.append([[float(cxd-r),float(cyd),0.5],[float(cxd),float(cyd+r*0.6),0.5],[float(cxd+r),float(cyd),0.5]]); SEL_STATS['dot_mark']+=1; continue
|
| 456 |
+
if FIX_DONOR and (c in ('[',']') or c in FIX_LIBCLS): force_lib_d=True # real library glyphs for classes the VAE draws badly (by eye, 22.10)
|
| 457 |
+
else: force_lib_d=False
|
| 458 |
+
if FIX40 and c=='BAR' and g.get('node')=='menclose' and LINE_MARKS==2: # \boxed{}: a frame of four real bars (w01 boxes results)
|
| 459 |
+
P_=_hm_pools(writer,exclude_docs); b=[v*s for v in g['bbox']]; bw_=b[2]-b[0]; bh_=b[3]-b[1]; box=[]
|
| 460 |
+
for (xa,ya,L_,vert) in ((b[0],b[1],bw_,False),(b[0],b[3],bw_,False),(b[0],b[1],bh_,True),(b[2],b[1],bh_,True)):
|
| 461 |
+
q_=_real_bar(P_,L_*(1+abs(rng.normal(0,0.03))),rng)
|
| 462 |
+
X_,Y_=(q_[:,0]+xa-0.02*L_,q_[:,1]+ya) if not vert else (q_[:,1]+xa,q_[:,0]+ya-0.02*L_)
|
| 463 |
+
box.append(np.stack([X_,Y_,q_[:,2]],1).tolist())
|
| 464 |
+
strokes+=box; SEL_STATS['boxed']+=1; continue
|
| 465 |
+
if LINE_MARKS==2 and FIX_DONOR and c in ('→','←','↓','↑','VEC') and len(_hm_pools(writer,exclude_docs).get('G>',[]))>=20:
|
| 466 |
+
am=[np.asarray(t,np.float64) for t in arrow_build(c,[v*s for v in g['bbox']],rng,_hm_pools(writer,exclude_docs))]; _push(am,placed,xh); strokes+=[t.tolist() for t in am]; SEL_STATS['arrow_built']+=1; continue
|
| 467 |
+
if LINE_MARKS==2 and VBAR and c in ('|','∥','‖','∣') and len(_hm_pools(writer,exclude_docs)['BAR'])>=20: # 05.11: |x| and the norm ||x|| — the glyph nets draw | as / ( ) ? 7
|
| 468 |
+
vb=[np.asarray(t,np.float64) for t in vbar_mark(c,[v*s for v in g['bbox']],rng,xh,_hm_pools(writer,exclude_docs))]; _push(vb,placed,xh); strokes+=[t.tolist() for t in vb]; SEL_STATS['vbar']+=1; continue
|
| 469 |
+
if LINE_MARKS==1 and VBAR and c in ('|','∥','‖','∣'): # 06.11 no library: |x| and the norm as pen lines (the glyph nets draw | as / ( ) ? 7)
|
| 470 |
+
b_=[v*s for v in g['bbox']]; xm_=(b_[0]+b_[2])/2; dx_=[0.0] if c in ('|','∣') else [-0.22*(b_[2]-b_[0]),0.22*(b_[2]-b_[0])]
|
| 471 |
+
vb=[np.asarray(_poly([[xm_+d_,b_[1]],[xm_+d_,b_[3]]],rng,0.015*xh,10),np.float64) for d_ in dx_]; _push(vb,placed,xh); strokes+=[t.tolist() for t in vb]; SEL_STATS['vbar_pen']+=1; continue
|
| 472 |
+
if LINE_MARKS==2 and c in LINE_SET:
|
| 473 |
+
hm=hand_mark(c,[v*s for v in g['bbox']],rng,xh,writer,exclude_docs,sw=g['sw']*s if 'sw' in g else None)
|
| 474 |
+
if hm=='LIB': force_lib=True; SEL_STATS['hand_mark_lib']+=1
|
| 475 |
+
elif hm is not None:
|
| 476 |
+
if FIX40 and c not in ('BAR','√'): hm=[np.asarray(t,np.float64) for t in hm]; _push(hm,placed,xh); hm=[t.tolist() for t in hm] # a radical encloses its radicand: never pushed
|
| 477 |
+
strokes+=hm; SEL_STATS['hand_mark']+=1; continue
|
| 478 |
+
if LINE_MARKS and c in LINE_SET and not force_lib:
|
| 479 |
+
strokes+=line_mark(c,[v*s for v in g['bbox']],rng,xh); SEL_STATS['line_mark']+=1; continue
|
| 480 |
+
q=None
|
| 481 |
+
if gen is not None and not force_lib and not force_lib_d:
|
| 482 |
+
if SEL_K>1:
|
| 483 |
+
k=pick_sample([gen[k*n+gi] for k in range(SEL_K)],c,(g['bbox'][3]-g['bbox'][1])/fH,rng)
|
| 484 |
+
q=gen[k*n+gi] if k is not None else None
|
| 485 |
+
if q is not None and not q['strokes']: q=None # empty variant (class unknown to the generator) -> library
|
| 486 |
+
else:
|
| 487 |
+
q=gen[gi]
|
| 488 |
+
if not q['strokes'] or sum(len(t) for t in q['strokes'])<2: missing.append(c); continue
|
| 489 |
+
if q is None:
|
| 490 |
+
pool=[q for q in lib.get((writer,c),[]) if q['doc'] not in exclude_docs] or [q for q in lib.get(('*',c),[]) if q['doc'] not in exclude_docs]
|
| 491 |
+
if not pool and gen is not None and SEL_K>1 and gen[gi]['strokes'] and int(E('SEL_RETRY','0')): # 06.11 no library (fgen_nolib_k4: -10 p.p., the first rejected variant was used): SEL_RETRY more variants, the first recognised one, else the best
|
| 492 |
+
ex_=vae_samples([cls_net(cls_of(g['c']))]*int(E('SEL_RETRY')),writer,rng,1); rh_=(g['bbox'][3]-g['bbox'][1])/fH; kk=pick_sample(ex_,c,rh_,rng)
|
| 493 |
+
if kk is not None: q=ex_[kk]; SEL_STATS['retry_ok']+=1
|
| 494 |
+
else: al_=ex_+[gen[k_*n+gi] for k_ in range(SEL_K)]; kk=pick_sample(al_,c,rh_,rng,force=True); q=al_[kk if kk is not None else 0]; SEL_STATS['retry_forced']+=1
|
| 495 |
+
elif not pool and gen is not None and SEL_K>1 and gen[gi]['strokes']: q=gen[gi]; SEL_STATS['net_unpicked']+=1 # 04.11: the picker rejected every variant and there is no library glyph -> the net's variant, not a dropped glyph
|
| 496 |
+
elif not pool: missing.append(c); continue
|
| 497 |
+
if gen is not None and SEL_K>1: pool=[q for q in pool if nstroke_p(c,len(q['strokes']))>=SEL_NMIN] or pool
|
| 498 |
+
if FIX40 and c in ('0','O','o'): pool=[q for q in pool if len(q['strokes'])==1] or pool # no slashed zeros (Ø) from the library
|
| 499 |
+
if FIX40 and c=='=': # a real '=' with two near-parallel bars (some library '=' converge and read as '>')
|
| 500 |
+
def _par(q):
|
| 501 |
+
if len(q['strokes'])!=2: return False
|
| 502 |
+
a=[math.degrees(math.atan2(t[-1][1]-t[0][1],abs(t[-1][0]-t[0][0])+1e-6)) for t in q['strokes']]
|
| 503 |
+
L=[abs(t[-1][0]-t[0][0]) for t in q['strokes']]
|
| 504 |
+
return abs(a[0]-a[1])<6 and min(L)>0.6*max(L)
|
| 505 |
+
pool=[q for q in pool if _par(q)] or pool # selection fallback: library glyph with a typical stroke count (no half-x)
|
| 506 |
+
if FIX40 and len(pool)>1: # the classifier picks the most typical of up to 16 real glyphs (the library still holds odd ones: lambda like d)
|
| 507 |
+
cands=[pool[int(i)] for i in rng.choice(len(pool),min(16,len(pool)),replace=False)]; kk=pick_sample(cands,c,(g['bbox'][3]-g['bbox'][1])/fH,rng)
|
| 508 |
+
q=cands[kk if kk is not None else 0]; SEL_STATS['lib_picked']+=1
|
| 509 |
+
elif pool: q=pool[int(rng.integers(len(pool)))] # (empty pool: q is the net's variant, set above)
|
| 510 |
+
b=[v*s for v in g['bbox']]; bw=b[2]-b[0]; bh=b[3]-b[1]
|
| 511 |
+
pts=[np.asarray(st,np.float32) for st in q['strokes']]; allp=np.concatenate([p[:,:2] for p in pts]); x0,y0=allp.min(0); x1,y1=allp.max(0); qw=max(x1-x0,1e-3); qh=max(y1-y0,1e-3)
|
| 512 |
+
if c=='BAR' or bw>2.5*bh: k=bw/qw; kx=k; ky=min(k,bh/qh*1.5) if c!='BAR' else k
|
| 513 |
+
elif bh>2.2*bw and c in '()[]{}|∫∮∑∏': ky=bh/qh; kx=min(ky,1.6*bw/qw)
|
| 514 |
+
else: ky=bh/qh; kx=ky
|
| 515 |
+
f=1.0+rng.normal(0,JIT_S); kx*=f; ky*=f; th=math.radians(rng.normal(0,JIT_R)); ct,sn=math.cos(th),math.sin(th)
|
| 516 |
+
drift=DRIFT*xh*math.sin((b[0]-x_min*s)/(6*xh)+rng.uniform(0,6.28))
|
| 517 |
+
cx=(b[0]+b[2])/2+rng.normal(0,JIT_XY*xh); cy=(b[1]+b[3])/2+rng.normal(0,JIT_XY*xh)+drift
|
| 518 |
+
if FIX40 and c not in ('BAR','(',')','[',']','{','}','|') and qw*kx>1.3*bw and bw>0: f2=1.3*bw/(qw*kx); kx*=f2; ky*=f2; SEL_STATS['narrowed']+=1 # a glyph much wider than its layout box crowds its neighbours
|
| 519 |
+
if FIX_DONOR and cls_of(g['c'])=='Θ': kx*=1.35 # capital Theta from her lowercase theta: wider, as a capital
|
| 520 |
+
fs=script_factor(g)
|
| 521 |
+
if fs!=1.0: kx*=fs; ky*=fs; cx=b[0]+(cx-b[0])*fs; SEL_STATS['script_scaled']+=1 # keep the left edge, shrink to her script size
|
| 522 |
+
mx=(x0+x1)/2; my=(y0+y1)/2
|
| 523 |
+
if XF:
|
| 524 |
+
if not (FIX40 and (c in XF_KEEP or c=='BAR')): pts=xf_glyph(pts,mx,my,max(qw,qh),xr)
|
| 525 |
+
else: xf_glyph(pts,mx,my,max(qw,qh),xr) # draw the same random numbers (stream stays aligned), keep the shape
|
| 526 |
+
gh=bh if c!='BAR' else xh
|
| 527 |
+
gap=0.7*gap+math.sqrt(1-0.49)*xr.normal(0,_xa('GAP',0.22)); base=0.8*base+0.6*xr.normal(0,_xa('BASE',0.08))
|
| 528 |
+
upright=FIX40 and int(g['c'],16)<128 and chr(int(g['c'],16)).isalpha() # letters of cos/sin/det/res: no gap jitter inside the word
|
| 529 |
+
cx+=(0 if upright else gap)*min(gh,1.5*xh); cy+=base*min(gh,1.5*xh)
|
| 530 |
+
if rot180: pts=[np.concatenate([2*mx-p[:,:1],2*my-p[:,1:2],p[:,2:]],1) for p in pts]
|
| 531 |
+
gs=[]
|
| 532 |
+
for p in pts:
|
| 533 |
+
u=(p[:,0]-mx)*kx; v=(p[:,1]-my)*ky; X=cx+u*ct-v*sn; Y=cy+u*sn+v*ct
|
| 534 |
+
pr=p[:,3] if p.shape[1]>3 else np.full(len(p),0.5,np.float32)
|
| 535 |
+
gs.append(np.stack([X,Y,pr],1))
|
| 536 |
+
if FIX40: _push(gs,placed,xh)
|
| 537 |
+
strokes+=[q.tolist() for q in gs]
|
| 538 |
+
return strokes,missing
|
| 539 |
+
def compose(tex,writer='w01',rng=None,exclude_docs=()):
|
| 540 |
+
rng=rng or np.random.default_rng(); o=layout([tex])[0]
|
| 541 |
+
if not o or not o.get('ok'): return None,['LAYOUT']
|
| 542 |
+
return compose_layout(o,writer,rng,exclude_docs)
|
| 543 |
+
def render(strokes,H=96,pen=2.4):
|
| 544 |
+
if not strokes or not any(len(s) for s in strokes): return Image.new('L',(32,H),255)
|
| 545 |
+
pts=np.array([p[:2] for s in strokes for p in s]); x0,y0=pts.min(0); x1,y1=pts.max(0); sc=(H-16)/max(y1-y0,1e-6)
|
| 546 |
+
W=int((x1-x0)*sc)+16; im=Image.new('L',(max(W,32),H),255); d=ImageDraw.Draw(im)
|
| 547 |
+
for s in strokes:
|
| 548 |
+
q=[((p[0]-x0)*sc+8,(p[1]-y0)*sc+8) for p in s]
|
| 549 |
+
if len(q)>1: d.line(q,fill=0,width=max(1,int(pen)),joint='curve')
|
| 550 |
+
elif q: d.point(q[0],fill=0)
|
| 551 |
+
return im
|
| 552 |
+
_NSYN=[('\\dfrac','\\frac'),('\\tfrac','\\frac'),('\\to','\\rightarrow'),('\\neq','\\ne'),('\\leq','\\le'),('\\geq','\\ge'),('\\varepsilon','\\epsilon'),
|
| 553 |
+
('\\varphi','\\phi'),('\\vartheta','\\theta'),('\\ldots','\\dots'),('\\cdots','\\dots'),('\\varrho','\\rho'),('\\langle','<'),('\\rangle','>'),
|
| 554 |
+
('\\overrightarrow','\\vec'),('\\neq','\\ne')] # 22.10: handwritten <T> and \\langle T\\rangle look the same; so do \\vec and \\overrightarrow
|
| 555 |
+
def norm_tex2(t):
|
| 556 |
+
"""22.10 (40/40 task): EQUIVALENT notations only -- spacing, \\left/\\right, font wrappers incl. \\mathcal (handwriting has no font),
|
| 557 |
+
synonyms (\\to = \\rightarrow, \\neq = \\ne, \\varphi = \\phi ...), prime forms (' = ^{\\prime}), cases = \\{ array{l} \\.,
|
| 558 |
+
pmatrix = ( array ), array column specs; then braces and whitespace dropped. FW_NORM=1 -> the old norm_tex."""
|
| 559 |
+
t=re.sub(r'\\(left|right)(?![A-Za-z])','',t.strip().strip('$')).replace('\\displaystyle','').replace('\\limits','') # not the \\right of \\rightarrow
|
| 560 |
+
t=re.sub(r'(?<!\\)\\(qquad|quad|[,;!: ])','',t).replace('~','') # spacing commands (not the row break \\\\)
|
| 561 |
+
for a,b in _NSYN: t=re.sub(re.escape(a)+r'(?![A-Za-z])',b.replace('\\','\\\\'),t)
|
| 562 |
+
t=re.sub(r'\s+','',t) # synonyms first: after the spaces go, '\\to x' becomes '\\tox' and no longer matches
|
| 563 |
+
for _ in range(3): t=re.sub(r'\\(mathrm|text|mathbf|mathit|operatorname|mathcal|textrm|textit)\{([^{}]*)\}',r'\2',t)
|
| 564 |
+
t=re.sub(r'\\(mathcal|mathrm|mathit)([A-Za-z])',r'\2',t)
|
| 565 |
+
t=re.sub(r"\^\{?\\prime\}?","'",t).replace('\\prime',"'")
|
| 566 |
+
t=re.sub(r'[{}]','',t)
|
| 567 |
+
t=re.sub(r'\\beginarray[lcr|]+',r'\\beginarray',t)
|
| 568 |
+
for a,b in (('\\\\beginarray','\\begincases'),('\\endarray.','\\endcases'),('(\\beginarray','\\beginpmatrix'),('\\endarray)','\\endpmatrix'),('[\\beginarray','\\beginbmatrix'),('\\endarray]','\\endbmatrix')): t=t.replace(a,b)
|
| 569 |
+
return t
|
| 570 |
+
def norm_tex(t):
|
| 571 |
+
if E('FW_NORM','2')!='1': return norm_tex2(t)
|
| 572 |
+
t=t.strip().strip('$').replace('\\left','').replace('\\right','').replace('\\displaystyle','').replace('\\limits','')
|
| 573 |
+
for a in ('\\,','\\;','\\!','\\:','\\quad','\\qquad','~'): t=t.replace(a,'')
|
| 574 |
+
t=re.sub(r'\\(mathrm|text|mathbf|mathit|operatorname)\{([^{}]*)\}',r'\2',t); t=re.sub(r'[{}\s]','',t); return t
|
| 575 |
+
def cer(a,b):
|
| 576 |
+
a=norm_tex(a); b=norm_tex(b)
|
| 577 |
+
if not a: return 0.0 if not b else 1.0
|
| 578 |
+
d=list(range(len(b)+1))
|
| 579 |
+
for i in range(1,len(a)+1):
|
| 580 |
+
p=d[:]; d[0]=i
|
| 581 |
+
for j in range(1,len(b)+1): d[j]=min(p[j]+1,d[j-1]+1,p[j-1]+(a[i-1]!=b[j-1]))
|
| 582 |
+
return d[-1]/len(a)
|
| 583 |
+
if __name__=='__main__':
|
| 584 |
+
import io, base64, urllib.request, time
|
| 585 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 586 |
+
WRITER=E('WRITER','w01'); N=int(E('N','40')); SEED=int(E('SEED','5')); TAG=E('TAG','lib'); KEY=E('OPENROUTER_KEY'); MODEL=E('MODEL','qwen/qwen3-vl-235b-a22b-instruct')
|
| 587 |
+
print('cfg',dict(XF=E('XF'),GVAE_SHRINK=E('GVAE_SHRINK'),GVAE_TAUS=E('GVAE_TAUS'),SEL_PICK=E('SEL_PICK'),SEL_PMIN=E('SEL_PMIN'),GEN=E('GEN','glyph1'),TAG=E('TAG'),N=E('N'),SEED=E('SEED'),GEN_CKPT=E('GEN_CKPT'),SEL_K=SEL_K,SEL_NMIN=SEL_NMIN,LINE_MARKS=int(LINE_MARKS),SEL_CLS=E('SEL_CLS','ckpt_dl/glyph_cls1.pt')),flush=True)
|
| 588 |
+
valid_docs={json.loads(l)['doc'] for l in open(os.path.join(PROJ,'data','ink_dataset_v2','valid.jsonl'))}
|
| 589 |
+
src={'w01':'data/sdocx_in/w01/lines_v3/all_full.jsonl'}.get(WRITER)
|
| 590 |
+
cand=[]
|
| 591 |
+
for l in open(os.path.join(PROJ,src)):
|
| 592 |
+
r=json.loads(l); lab=r['label']
|
| 593 |
+
if r.get('ok') and r['doc'] in valid_docs and lab.startswith('$') and lab.endswith('$') and lab.count('$')==2 and 6<=len(lab)<=70 and r.get('rows',1)<=1.4: cand.append(r)
|
| 594 |
+
rng=np.random.default_rng(SEED); pick=[cand[i] for i in rng.choice(len(cand),min(N,len(cand)),replace=False)]
|
| 595 |
+
if E('GEN') in ('vae','vae2','dn'): GEN='VAE'; print('glyphs from the glyph VAE',E('GEN'),E('GDN_CKPT') if E('GEN')=='dn' else E('GVAE_CKPT','ckpt_glyph_vae/last.pt'),'picker',E('SEL_CLS','ckpt_dl/glyph_cls1.pt'),'library',E('FW_LIB','glyphs_clean.jsonl'),flush=True)
|
| 596 |
+
elif E('GEN_CKPT'): load_gen(E('GEN_CKPT')); print('glyphs from the generator',E('GEN_CKPT'),flush=True)
|
| 597 |
+
lays=layout([r['label'][1:-1] for r in pick]); gen=[]; miss=collections.Counter()
|
| 598 |
+
for r,o in zip(pick,lays):
|
| 599 |
+
if not o or not o.get('ok'): gen.append(None); continue
|
| 600 |
+
st,m=compose_layout(o,WRITER,rng,exclude_docs=valid_docs); miss.update(m); gen.append(st)
|
| 601 |
+
print('composed',sum(g is not None for g in gen),'/',len(pick),'missing classes',dict(miss.most_common(10)),flush=True)
|
| 602 |
+
print('SEL_K',SEL_K,'LINE_MARKS',int(LINE_MARKS),'glyph sourcing',dict(SEL_STATS),flush=True)
|
| 603 |
+
P=("This is a handwritten mathematical formula. Transcribe it to LaTeX exactly as written (no $ signs, no explanations). "
|
| 604 |
+
"Output only the LaTeX.")
|
| 605 |
+
def ask(im):
|
| 606 |
+
buf=io.BytesIO(); im.convert('RGB').save(buf,'PNG'); d64=base64.b64encode(buf.getvalue()).decode()
|
| 607 |
+
body=json.dumps({"model":MODEL,"temperature":0,"max_tokens":300,"messages":[{"role":"user","content":[{"type":"text","text":P},{"type":"image_url","image_url":{"url":"data:image/png;base64,"+d64}}]}]}).encode()
|
| 608 |
+
req=urllib.request.Request("https://openrouter.ai/api/v1/chat/completions",data=body,headers={"Authorization":"Bearer "+KEY,"Content-Type":"application/json"})
|
| 609 |
+
for a in range(3):
|
| 610 |
+
try: return json.load(urllib.request.urlopen(req,timeout=120))["choices"][0]["message"]["content"].strip().strip('`').replace('latex\n','')
|
| 611 |
+
except Exception: time.sleep(3)
|
| 612 |
+
return ''
|
| 613 |
+
real_ims=[render(r['strokes']) for r in pick]; gen_ims=[render(g) if g else Image.new('L',(32,32),255) for g in gen]
|
| 614 |
+
with ThreadPoolExecutor(8) as ex: rr=list(ex.map(ask,real_ims)); gr=list(ex.map(ask,gen_ims))
|
| 615 |
+
labs=[r['label'][1:-1] for r in pick]; cr=[cer(l,x) for l,x in zip(labs,rr)]; cg=[cer(l,x) if g else 1.0 for l,x,g in zip(labs,gr,gen)]
|
| 616 |
+
for name,c in (('REAL (control)',cr),('NET composed' if (E('GEN_CKPT') or E('GEN') in ('vae','vae2','dn')) else 'LIBRARY composed',cg)):
|
| 617 |
+
print(f'=== {name}: <=0.2 {sum(x<=0.2 for x in c)}/{len(c)} <=0.05 {sum(x<=0.05 for x in c)}/{len(c)} mean {np.mean(c):.2f} ===',flush=True)
|
| 618 |
+
rows=[]
|
| 619 |
+
for i in range(min(16,len(pick))):
|
| 620 |
+
a=real_ims[i]; b=gen_ims[i]; w=max(a.size[0],b.size[0]); t=Image.new('L',(w*2+30,max(a.size[1],b.size[1])),255); t.paste(a,(0,0)); t.paste(b,(w+30,0)); rows.append(t)
|
| 621 |
+
W=min(1800,max(t.size[0] for t in rows)); sh=Image.new('L',(W,sum(t.size[1]+10 for t in rows)),255); y=0
|
| 622 |
+
for t in rows:
|
| 623 |
+
if t.size[0]>W: t=t.resize((W,int(t.size[1]*W/t.size[0])))
|
| 624 |
+
sh.paste(t,(0,y)); y+=t.size[1]+10
|
| 625 |
+
sh.save(os.path.join(PROJ,'figures',f'formula_write_{TAG}.png'))
|
| 626 |
+
json.dump(dict(labels=labs,real_reads=rr,gen_reads=gr,cer_real=cr,cer_gen=cg),open(os.path.join(PROJ,'figures',f'formula_write_{TAG}.json'),'w'),ensure_ascii=False)
|
scripts/glyph_dn.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""Glyph denoiser sampler (И116, 22.10): draws ONE formula glyph of a class in a writer's style with the diffusion denoiser of
|
| 3 |
+
scripts/train_glyph_dn.py. Same API and output format as scripts/glyph_vae.py (formula_write GEN=vae2 / glyph_m3_eval GEN=vae):
|
| 4 |
+
sample(cls, writer, rng, n=1, tau=None, shrink=None) -> list of n dicts {'strokes': [[[x,y,0,p],...], ...]} ([] if the class is
|
| 5 |
+
unknown to the net -> the caller falls back to the library)
|
| 6 |
+
sample_batch(classes, writers, rng, tau=None, shrink=None) -> one per (class, writer) pair.
|
| 7 |
+
_load() -> dict with c2i / cmean (class membership) as glyph_vae._load.
|
| 8 |
+
tau / shrink are accepted and ignored (the VAE's MDN temperature / z shrink have no counterpart; variants differ by the noise
|
| 9 |
+
seed, drawn from rng). Points: x, y in norm_h units centred on the glyph box, y down, pressure p in 0..1.
|
| 10 |
+
Env: PROJ, GDN_CKPT (ckpt_dl/glyphdn/glyphdnB/fp16/step050000.pt), GDN_CFG (1.5; formula_write GEN=dn sets 1.2: classifier-free guidance over the class), GDN_STEPS (10 DDIM
|
| 11 |
+
steps), GDN_DEV, GDN_LEN (dist = draw the point count from the (writer, class) train distribution; or an int multiplier x100 of the
|
| 12 |
+
median, e.g. 100), GDN_WRITER_NULL (1: an unknown writer -> the null «average» writer; 0 -> w01), GDN_NOISE (1.0: scale of the starting noise, >1 = more diversity), GDN_ETA (0: DDIM noise, 1 = fully stochastic steps -> more within-class diversity), GDN_PEN_THR (0: pen-up logit threshold; the net was trained with pos_weight 3, so 0 = calibrated p > 0.25, 1.1 = p > 0.5).
|
| 13 |
+
04.11 (И164): a GSTYLE=semb checkpoint takes the style from set_style(key, vec) (vec = semb3_all embedding of the reference line(s));
|
| 14 |
+
the `writers` given to sample_batch are those keys (unknown key -> the null «average» writer)."""
|
| 15 |
+
import os, sys, numpy as np, torch
|
| 16 |
+
PROJ=os.environ.get('PROJ',os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); E=os.environ.get
|
| 17 |
+
sys.path.insert(0,os.path.join(PROJ,'scripts'))
|
| 18 |
+
_S=None; STYLES={}; import collections; N=collections.Counter() # 06.11 N: glyphs drawn by the net ('net') / classes it does not know ('unknown' -> library)
|
| 19 |
+
def set_style(key,vec): # 04.11 И164: GSTYLE=semb checkpoints — key -> semb3_all embedding of the reference line(s) [256]
|
| 20 |
+
STYLES[key]=np.asarray(vec,np.float32).reshape(-1)
|
| 21 |
+
def _load():
|
| 22 |
+
global _S
|
| 23 |
+
if _S is not None: return _S
|
| 24 |
+
if E('GDN_DEV'): os.environ['DEV']=E('GDN_DEV')
|
| 25 |
+
import train_glyph_dn as T
|
| 26 |
+
ck=E('GDN_CKPT',os.path.join(PROJ,'ckpt_dl','glyphdn','glyphdnB','fp16','step050000.pt')); net,classes,writers,lens,step=T.load_net(ck) # 22.10 22:15: run B (w01-only ink_glyphs3, 50k) — the better of A/B on the M3 instruments
|
| 27 |
+
c2i={c:i for i,c in enumerate(classes)}; w2i={w:i for i,w in enumerate(writers)}
|
| 28 |
+
_S=dict(T=T,net=net,classes=classes,c2i=c2i,writers=writers,w2i=w2i,lens=lens,step=step,cmean={c:True for c in classes},dev=T.dev)
|
| 29 |
+
print(f'glyph denoiser {ck} step {step} classes {len(classes)} writers {len(writers)} cfg {E("GDN_CFG","1.5")} steps {E("GDN_STEPS","10")}',flush=True)
|
| 30 |
+
return _S
|
| 31 |
+
def label_of(cls):
|
| 32 |
+
if cls=='BAR': return '—'
|
| 33 |
+
if cls=='VEC': return '⃗'
|
| 34 |
+
return cls
|
| 35 |
+
def sample_batch(classes,writers,rng,tau=None,shrink=None,maxlen=200):
|
| 36 |
+
if E('GLYPH_B')=='1': # 05.11 И165: B fine-tuned on glyphs (scripts/glyph_bdn.py); classes it has no LaTeX for come back to _sample_batch
|
| 37 |
+
import glyph_bdn; return glyph_bdn.sample_batch(classes,writers,rng,tau,shrink,maxlen)
|
| 38 |
+
return _sample_batch(classes,writers,rng,tau,shrink,maxlen)
|
| 39 |
+
@torch.no_grad()
|
| 40 |
+
def _sample_batch(classes,writers,rng,tau=None,shrink=None,maxlen=200):
|
| 41 |
+
S=_load(); T=S['T']; labs=[label_of(c) for c in classes]; ok=[i for i,l in enumerate(labs) if l in S['c2i']]; res=[[] for _ in labs]
|
| 42 |
+
N['net']+=len(ok); N['unknown']+=len(labs)-len(ok)
|
| 43 |
+
if not ok: return res
|
| 44 |
+
nw=len(S['writers']); wnull=E('GDN_WRITER_NULL','1')=='1'
|
| 45 |
+
C=[S['c2i'][labs[i]] for i in ok]; W=[S['w2i'].get(writers[i],nw if wnull else S['w2i'].get('w01',nw)) for i in ok]
|
| 46 |
+
L=[T.draw_len(S['lens'],w,c,rng) for w,c in zip(W,C)]
|
| 47 |
+
if T.GSTYLE: # style rows [emb, null flag] from set_style; the length plan is then class-wide (W ids above are 'unknown')
|
| 48 |
+
SD=T.SDIM; W=np.zeros((len(ok),SD+1),np.float32)
|
| 49 |
+
for j,i in enumerate(ok):
|
| 50 |
+
if writers[i] in STYLES: W[j,:SD]=STYLES[writers[i]]
|
| 51 |
+
else: W[j,-1]=1
|
| 52 |
+
if E('GDN_LEN','dist')!='dist':
|
| 53 |
+
f=float(E('GDN_LEN'))/100.0; L=[max(4,int(round(float(np.median(S['lens'].get((-1,c),[30])))*f))) for c in C]
|
| 54 |
+
g=torch.Generator(device='cpu').manual_seed(int(rng.integers(1<<31)))
|
| 55 |
+
out=T.sample(S['net'],C,W,L,cfg=float(E('GDN_CFG','1.5')),steps=int(E('GDN_STEPS','10')),gen=g,pen_thr=float(E('GDN_PEN_THR','0')),eta=float(E('GDN_ETA','0')),noise=float(E('GDN_NOISE','1.0')))
|
| 56 |
+
for j,i in enumerate(ok): res[i]=dict(strokes=T.to_strokes(out[j]))
|
| 57 |
+
return res
|
| 58 |
+
def sample(cls,writer,rng,n=1,tau=None,shrink=None):
|
| 59 |
+
r=sample_batch([cls]*n,[writer]*n,rng,tau,shrink)
|
| 60 |
+
return [] if not r or r[0]==[] else r
|
scripts/make_ref.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""Your own reference line for write_pages.py: pen strokes of ONE handwritten line -> the [n,3] array the models take as the style
|
| 3 |
+
(dx, dy, pen_up; line height = 1, points every 0.05 of it along the pen path, as in training).
|
| 4 |
+
input : JSON, a list of strokes, each a list of points [x, y] in any units, y pointing down, in writing order
|
| 5 |
+
python scripts/make_ref.py my_line.json my_line.npy
|
| 6 |
+
then REF=/path/to/my_line.npy python scripts/write_pages.py notes.txt out
|
| 7 |
+
Write a line of ordinary prose of 30-50 characters: the style is taken from this one line only."""
|
| 8 |
+
import sys, json, numpy as np
|
| 9 |
+
STEP = 0.05
|
| 10 |
+
def to_ref(strokes):
|
| 11 |
+
S = [np.asarray(s, np.float64)[:, :2] for s in strokes if len(s)]
|
| 12 |
+
if not S: raise SystemExit('no strokes in the input')
|
| 13 |
+
A = np.concatenate(S); o = A.min(0); h = float(A[:, 1].max() - o[1])
|
| 14 |
+
if h <= 0: raise SystemExit('the line has zero height')
|
| 15 |
+
out = []
|
| 16 |
+
for p in S:
|
| 17 |
+
p = (p - o) / h
|
| 18 |
+
if len(p) == 1: out.append(np.array([[p[0, 0], p[0, 1], 1.0]])); continue
|
| 19 |
+
cum = np.r_[0, np.cumsum(np.hypot(*np.diff(p, axis=0).T))]
|
| 20 |
+
u = np.r_[np.arange(0, cum[-1], STEP), cum[-1]] if cum[-1] > 1e-6 else np.array([0.0, cum[-1]])
|
| 21 |
+
r = np.stack([np.interp(u, cum, p[:, 0]), np.interp(u, cum, p[:, 1]), np.zeros(len(u))], 1); r[-1, 2] = 1; out.append(r)
|
| 22 |
+
a = np.concatenate(out); d = a.copy(); d[1:, :2] = a[1:, :2] - a[:-1, :2]; d[0, :2] = 0
|
| 23 |
+
return d.astype(np.float32)
|
| 24 |
+
if __name__ == '__main__':
|
| 25 |
+
q = to_ref(json.load(open(sys.argv[1]))); np.save(sys.argv[2], q); print(f'{sys.argv[2]}: {len(q)} points, {int(q[:, 2].sum())} strokes')
|
scripts/render_pages.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""06.11 (Лев на увеличенный кусок страницы: «А нахрена такое уродское качество?»): the pages were a 150 dpi raster drawn with
|
| 3 |
+
PIL lines without antialiasing, and the PDF re-packed that raster as JPEG (staircase edges + grey JPEG dirt around the strokes).
|
| 4 |
+
Now the page is kept as STROKES and drawn three ways from them:
|
| 5 |
+
pages.pdf VECTOR: every stroke a smooth curve (Catmull-Rom through the model's points -> cubic Bezier), round caps and joins,
|
| 6 |
+
A4; sharp at any zoom, a few hundred KB (Flate-compressed streams). Written by hand, no extra libraries.
|
| 7 |
+
page_NN.svg the same curves (for the browser / an editor)
|
| 8 |
+
page_NN.png 300 dpi, antialiased (drawn at 4x over the curve sampled 8 points per segment, then downscaled with LANCZOS)
|
| 9 |
+
Strokes are lightly smoothed first ([1 2 1]/4 once on strokes of >= 5 points; ends kept): the models' points carry a small jitter
|
| 10 |
+
that the old 2-px raster hid and a vector line shows.
|
| 11 |
+
Input (strokes.json, written by write_pages.py): {"W": px, "H": px, "dpi": 150, "ruling": [y px, ...], "pages": [[stroke, ...], ...]},
|
| 12 |
+
stroke = [[x, y], ...] in page px (y down).
|
| 13 |
+
python scripts/render_pages.py <strokes.json> <out_dir> [PEN=1.1 (pt) PNG_DPI=300 SMOOTH=1] (re-render without the models)"""
|
| 14 |
+
import os, sys, json, zlib, numpy as np
|
| 15 |
+
from PIL import Image, ImageDraw
|
| 16 |
+
E = os.environ.get
|
| 17 |
+
|
| 18 |
+
def smooth(s):
|
| 19 |
+
s = np.asarray(s, np.float64)
|
| 20 |
+
if E('SMOOTH', '1') != '1' or len(s) < 5: return s
|
| 21 |
+
t = s.copy(); t[1:-1] = 0.25 * s[:-2] + 0.5 * s[1:-1] + 0.25 * s[2:]; return t
|
| 22 |
+
def bezier(s):
|
| 23 |
+
"""points [n,2] -> list of cubic segments (p0, c1, c2, p1), Catmull-Rom through all points"""
|
| 24 |
+
n = len(s)
|
| 25 |
+
if n < 2: return []
|
| 26 |
+
P = np.vstack([s[:1], s, s[-1:]]); out = []
|
| 27 |
+
for i in range(1, n):
|
| 28 |
+
p0, p1 = P[i], P[i + 1]; out.append((p0, p0 + (P[i + 1] - P[i - 1]) / 6, p1 - (P[i + 2] - P[i]) / 6, p1))
|
| 29 |
+
return out
|
| 30 |
+
def sample(seg, k=8):
|
| 31 |
+
p0, c1, c2, p1 = seg; t = np.linspace(0, 1, k + 1)[1:, None]
|
| 32 |
+
return (1 - t) ** 3 * p0 + 3 * (1 - t) ** 2 * t * c1 + 3 * (1 - t) * t ** 2 * c2 + t ** 3 * p1
|
| 33 |
+
|
| 34 |
+
def pdf(pages, ruling, W, H, dpi, path, pen):
|
| 35 |
+
k = 72.0 / dpi; PW, PH = W * k, H * k; f = lambda x, y: f'{x * k:.2f} {PH - y * k:.2f}'
|
| 36 |
+
objs = {1: b'<< /Type /Catalog /Pages 2 0 R >>'}; kids = []
|
| 37 |
+
for i, strokes in enumerate(pages):
|
| 38 |
+
c = ['0.86 G 0.5 w'] + [f'{f(0, y)} m {f(W, y)} l S' for y in ruling]
|
| 39 |
+
c.append(f'0.08 G {pen:.2f} w 1 J 1 j')
|
| 40 |
+
for st in strokes:
|
| 41 |
+
s = smooth(st)
|
| 42 |
+
if len(s) == 1: c.append(f'{f(*s[0])} m {f(*s[0])} l S'); continue
|
| 43 |
+
c.append(f'{f(*s[0])} m ' + ' '.join(f'{f(*a)} {f(*b)} {f(*d)} c' for _, a, b, d in bezier(s)) + ' S')
|
| 44 |
+
data = zlib.compress('\n'.join(c).encode()); po, co = 3 + 2 * i, 4 + 2 * i; kids.append(po)
|
| 45 |
+
objs[po] = f'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {PW:.2f} {PH:.2f}] /Contents {co} 0 R >>'.encode()
|
| 46 |
+
objs[co] = f'<< /Length {len(data)} /Filter /FlateDecode >>\nstream\n'.encode() + data + b'\nendstream'
|
| 47 |
+
objs[2] = f'<< /Type /Pages /Kids [{" ".join(f"{p} 0 R" for p in kids)}] /Count {len(kids)} >>'.encode()
|
| 48 |
+
b = bytearray(b'%PDF-1.4\n'); off = {}
|
| 49 |
+
for n in sorted(objs): off[n] = len(b); b += f'{n} 0 obj\n'.encode() + objs[n] + b'\nendobj\n'
|
| 50 |
+
x = len(b); b += f'xref\n0 {len(objs) + 1}\n0000000000 65535 f \n'.encode() + b''.join(f'{off[n]:010d} 00000 n \n'.encode() for n in sorted(objs))
|
| 51 |
+
b += f'trailer\n<< /Size {len(objs) + 1} /Root 1 0 R >>\nstartxref\n{x}\n%%EOF\n'.encode(); open(path, 'wb').write(b)
|
| 52 |
+
|
| 53 |
+
def svg(strokes, ruling, W, H, dpi, path, pen):
|
| 54 |
+
w = pen * dpi / 72.0; g = lambda p: f'{p[0]:.2f},{p[1]:.2f}'; L = []
|
| 55 |
+
for st in strokes:
|
| 56 |
+
s = smooth(st)
|
| 57 |
+
L.append(f'M{g(s[0])} L{g(s[0])}' if len(s) == 1 else f'M{g(s[0])} ' + ' '.join(f'C{g(a)} {g(b)} {g(d)}' for _, a, b, d in bezier(s)))
|
| 58 |
+
open(path, 'w').write(f'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {W} {H}" width="210mm" height="297mm"><rect width="100%" height="100%" fill="#fff"/>'
|
| 59 |
+
+ ''.join(f'<line x1="0" y1="{y:.1f}" x2="{W}" y2="{y:.1f}" stroke="#dcdcdc" stroke-width="1"/>' for y in ruling)
|
| 60 |
+
+ f'<path d="{" ".join(L)}" fill="none" stroke="#141414" stroke-width="{w:.2f}" stroke-linecap="round" stroke-linejoin="round"/></svg>')
|
| 61 |
+
|
| 62 |
+
def png(strokes, ruling, W, H, dpi, path, pen, out_dpi):
|
| 63 |
+
SS = 4; k = out_dpi / dpi * SS; im = Image.new('L', (int(W * k), int(H * k)), 255); d = ImageDraw.Draw(im); w = max(1, int(round(pen / 72.0 * out_dpi * SS)))
|
| 64 |
+
for y in ruling: d.line([(0, y * k), (W * k, y * k)], fill=222, width=SS)
|
| 65 |
+
for st in strokes:
|
| 66 |
+
s = smooth(st); pts = np.vstack([s[:1]] + [sample(seg) for seg in bezier(s)]) * k if len(s) > 1 else s * k
|
| 67 |
+
if len(pts) > 1: d.line([tuple(p) for p in pts], fill=20, width=w, joint='curve')
|
| 68 |
+
for p in (pts[0], pts[-1]): d.ellipse([p[0] - w / 2, p[1] - w / 2, p[0] + w / 2, p[1] + w / 2], fill=20)
|
| 69 |
+
im.resize((int(W * out_dpi / dpi), int(H * out_dpi / dpi)), Image.LANCZOS).save(path, dpi=(out_dpi, out_dpi))
|
| 70 |
+
|
| 71 |
+
def render(doc, out):
|
| 72 |
+
W, H, dpi, R = doc['W'], doc['H'], doc.get('dpi', 150), doc['ruling']; pen = float(E('PEN', '1.1')); od = int(E('PNG_DPI', '300'))
|
| 73 |
+
pdf(doc['pages'], R, W, H, dpi, os.path.join(out, 'pages.pdf'), pen)
|
| 74 |
+
for i, st in enumerate(doc['pages']):
|
| 75 |
+
svg(st, R, W, H, dpi, os.path.join(out, f'page_{i + 1:02d}.svg'), pen); png(st, R, W, H, dpi, os.path.join(out, f'page_{i + 1:02d}.png'), pen, od)
|
| 76 |
+
|
| 77 |
+
if __name__ == '__main__':
|
| 78 |
+
src, out = sys.argv[1], sys.argv[2]
|
| 79 |
+
if os.path.exists(out): sys.exit(f'{out} exists')
|
| 80 |
+
os.makedirs(out); render(json.load(open(src)), out); print('rendered ->', out)
|
scripts/style_classifier.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""Writer classifier on REAL lines (data/unified export data/ink_lines_u1) + the numeric STYLE metric for generators (22.10, w03:
|
| 3 |
+
«ты это как увидел? померил?» — instead of eyeballing figures/denoise_style_sensitivity*.png). DiffInk's «Style» = accuracy of a
|
| 4 |
+
writer classifier on generated lines; here: share of generated lines of writer W (style from K=3 reference lines of W) that the
|
| 5 |
+
classifier assigns to W.
|
| 6 |
+
|
| 7 |
+
Input of the classifier = the generator's representation of a line: (dx, dy, pen) in line heights, resampled to the dataset's arc
|
| 8 |
+
step 0.05 inside every stroke (so point density can not be used as a cue; real lines are already at that step, generated ones are
|
| 9 |
+
not exactly), then per point [dx*SCALE, dy*SCALE, pen, y - median(y)] (SCALE = 50 as in train_ink_al2.Inks / the style encoder).
|
| 10 |
+
Net: Conv1d stem (x4 downsample) -> BiGRU 2x192 -> masked mean+max pooling -> 15 writers (~1.7M params). Training: balanced
|
| 11 |
+
sampling over writers, random 512-point crops, the trainer's affine jitter (AUG_S 0.1, AUG_SH 0.15), label smoothing 0.1.
|
| 12 |
+
Splits: the unified train/dev/test by notebooks for the 12 writers with train data; the 3 held-out writers (w17, w06,
|
| 13 |
+
w13 — only in test by splits.json) are split here BY NOTEBOOK 70/30 (seed 0) into cls-train / cls-test, so the test is always
|
| 14 |
+
on unseen notebooks for all 15 writers (STATS in the eval json say which notebooks).
|
| 15 |
+
|
| 16 |
+
MODES (PY=~/.venvs/whisper/bin/python; MPS)
|
| 17 |
+
MODE=train [STEPS=8000 BS=64 LR=1e-3 OUT=ckpt_stylecls] -> OUT/best.pt (best dev), OUT/last.pt, OUT/train.log
|
| 18 |
+
MODE=eval [CKPT=ckpt_stylecls/best.pt] -> accuracy on test (15 cl.), per writer, prose-only, the 5 sheet writers; OUT/eval.json
|
| 19 |
+
MODE=metric DN_CKPT=<denoise ckpt> [TAG=denoise1 CFG=1.5 SSTEPS=20 SCFGS=1,1.5,2,3 N=60 SEED=3 BASE=<lstm fp16>]
|
| 20 |
+
-> out/style_metric/<TAG>/: seqs_<writer>_<cond>.npz (cached, generation skipped if present), metric.json, metric.md,
|
| 21 |
+
figures/style_metric_<TAG>.png (table + confusion of denoiser SCFG=1), figures/style_metric_<TAG>_sheet.png (eyeball)
|
| 22 |
+
Conditions: real (dev+test prose lines of the writer, unseen notebooks — the ceiling), lstm (BASE = lstmal_v6isl3@100k, bias 1.0,
|
| 23 |
+
the same refs), dn_s<SCFG> (denoiser, CFG, SSTEPS). Texts = the standard judge's 60 v2-valid prose texts (seed 3), the same for
|
| 24 |
+
every writer and condition; refs = exactly those of scripts/denoise_style_sheet.py (w01: v2 valid 0,16,17; others: 3 prose lines
|
| 25 |
+
of the largest v6 page); durations from the duration head of BASE; the same torch seed per batch of 16 in every condition."""
|
| 26 |
+
import os, sys, json, math, time, collections, numpy as np
|
| 27 |
+
PROJ=os.path.dirname(os.path.dirname(os.path.abspath(__file__))); sys.path.insert(0,os.path.join(PROJ,'scripts'))
|
| 28 |
+
E=os.environ.get; MODE=E('MODE','train')
|
| 29 |
+
import torch, torch.nn as nn, torch.nn.functional as F
|
| 30 |
+
dev='cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu')
|
| 31 |
+
SCALE=50.0; STEP=0.05; MAXPTS=1200; CROP=int(E('CROP','512'))
|
| 32 |
+
WRITERS=['w01','w03','w05','w06','w07','w08','w09','w10','w11','w12','w13','w14','w15','w16','w17']
|
| 33 |
+
NAMES={'w01':'w01','w03':'Лев','w05':'w05','w06':'w06','w07':'w07','w08':'w08','w09':'w09','w10':'w10','w11':'w11','w12':'w12','w13':'w13','w14':'w14','w15':'w15','w16':'w16','w17':'w17'}
|
| 34 |
+
WID={w:i for i,w in enumerate(WRITERS)}; HOLDOUT=['w17','w06','w13']; SHEET5=['w01','w03','w09','w12','w11']
|
| 35 |
+
U1=os.path.join(PROJ,'data','ink_lines_u1'); OUT=os.path.join(PROJ,E('OUT','ckpt_stylecls'))
|
| 36 |
+
|
| 37 |
+
# ---------------- representation
|
| 38 |
+
def resample(seq,step=STEP):
|
| 39 |
+
"""seq [n,3] (dx,dy,pen: pen-up AFTER this point) raw units -> same format, every stroke resampled to arc step `step`."""
|
| 40 |
+
seq=np.asarray(seq,np.float32)
|
| 41 |
+
if len(seq)==0: return np.zeros((1,3),np.float32)
|
| 42 |
+
P=np.cumsum(seq[:,:2],0); pen=seq[:,2]>0.5; ends=np.flatnonzero(pen); starts=np.concatenate([[0],ends[:-1]+1]) if len(ends) else np.array([0])
|
| 43 |
+
if len(ends)==0 or ends[-1]!=len(seq)-1: ends=np.append(ends,len(seq)-1); starts=np.concatenate([[0],ends[:-1]+1])
|
| 44 |
+
outP=[];outpen=[]
|
| 45 |
+
for a,b in zip(starts,ends):
|
| 46 |
+
if b<a: continue
|
| 47 |
+
s=P[a:b+1]
|
| 48 |
+
if len(s)>1:
|
| 49 |
+
L=np.concatenate([[0],np.cumsum(np.hypot(*np.diff(s,axis=0).T))]); tot=L[-1]
|
| 50 |
+
if tot>step*0.5:
|
| 51 |
+
t=np.arange(0,tot,step)
|
| 52 |
+
if tot-t[-1]>step*0.25: t=np.append(t,tot)
|
| 53 |
+
s=np.stack([np.interp(t,L,s[:,0]),np.interp(t,L,s[:,1])],1)
|
| 54 |
+
else: s=s[[0]]
|
| 55 |
+
outP.append(s); p=np.zeros(len(s),np.float32); p[-1]=1; outpen.append(p)
|
| 56 |
+
P2=np.concatenate(outP); pen2=np.concatenate(outpen); d=np.zeros((len(P2),3),np.float32); d[1:,:2]=np.diff(P2,axis=0); d[:,2]=pen2; return d
|
| 57 |
+
def feats(seq):
|
| 58 |
+
"""resampled (dx,dy,pen) -> [n,4] features: dx*SCALE, dy*SCALE, pen, y-median(y)"""
|
| 59 |
+
seq=np.asarray(seq,np.float32)[:MAXPTS]; y=np.cumsum(seq[:,1]); return np.stack([seq[:,0]*SCALE,seq[:,1]*SCALE,seq[:,2],y-np.median(y)],1).astype(np.float32)
|
| 60 |
+
|
| 61 |
+
# ---------------- data
|
| 62 |
+
def load_split(sp):
|
| 63 |
+
z=np.load(os.path.join(U1,f'{sp}_raw.npz'),allow_pickle=True); meta=[json.loads(l) for l in open(os.path.join(U1,f'meta_{sp}.jsonl'))]
|
| 64 |
+
assert len(meta)==len(z['labels']); return z['pts'],z['pt_off'],meta
|
| 65 |
+
def cls_splits():
|
| 66 |
+
"""-> dict split -> list of (pts_array, off_start, off_end, writer, kind, doc, src_split). Held-out writers' notebooks: 70/30 by notebook (seed 0)."""
|
| 67 |
+
out={'train':[],'dev':[],'test':[]}; rng=np.random.default_rng(0); hold_docs={}
|
| 68 |
+
for sp in ('train','dev','test'):
|
| 69 |
+
pts,off,meta=load_split(sp)
|
| 70 |
+
for i,r in enumerate(meta):
|
| 71 |
+
w=r['writer_id']; rec=(pts,int(off[i]),int(off[i+1]),w,r['kind'],r['doc'],sp)
|
| 72 |
+
if w in HOLDOUT:
|
| 73 |
+
hold_docs.setdefault(w,{}).setdefault(r['doc'],[]).append(rec)
|
| 74 |
+
else: out[sp].append(rec)
|
| 75 |
+
assign={}
|
| 76 |
+
for w,docs in hold_docs.items():
|
| 77 |
+
names=sorted(docs); rng.shuffle(names); n=len(names); ntest=max(1,int(round(0.3*n)))
|
| 78 |
+
for k,d in enumerate(names):
|
| 79 |
+
s='test' if k<ntest else 'train'; assign[(w,d)]=s; out[s]+=docs[d]
|
| 80 |
+
return out,assign
|
| 81 |
+
class Lines:
|
| 82 |
+
def __init__(s,recs,cache_tag=None):
|
| 83 |
+
s.recs=recs; s.y=np.array([WID[r[3]] for r in recs]); s.kind=[r[4] for r in recs]; s.n=len(recs)
|
| 84 |
+
s.cache=[None]*s.n
|
| 85 |
+
def seq(s,i):
|
| 86 |
+
pts,a,b=s.recs[i][:3]; return pts[a:b]
|
| 87 |
+
def feat(s,i):
|
| 88 |
+
if s.cache[i] is None: s.cache[i]=feats(resample(s.seq(i)))
|
| 89 |
+
return s.cache[i]
|
| 90 |
+
def by_writer(s):
|
| 91 |
+
d=collections.defaultdict(list)
|
| 92 |
+
for i,w in enumerate(s.y): d[int(w)].append(i)
|
| 93 |
+
return d
|
| 94 |
+
AUG_S=float(E('AUG_S','0.1')); AUG_SH=float(E('AUG_SH','0.15'))
|
| 95 |
+
def make_batch(fs,train,rng=None):
|
| 96 |
+
"""fs: list of [n,4] features -> X [B,T,4] padded, lengths. train: random crop CROP + affine jitter (on dx,dy; y recomputed)."""
|
| 97 |
+
xs=[]
|
| 98 |
+
for f in fs:
|
| 99 |
+
f=f.copy()
|
| 100 |
+
if train:
|
| 101 |
+
if len(f)>CROP: a=rng.integers(0,len(f)-CROP+1); f=f[a:a+CROP]
|
| 102 |
+
sx=1+(rng.random()*2-1)*AUG_S; sy=1+(rng.random()*2-1)*AUG_S; sh=(rng.random()*2-1)*AUG_SH
|
| 103 |
+
dx=f[:,0]*sx+f[:,1]*sh; dy=f[:,1]*sy; y=np.cumsum(dy)/SCALE; f=np.stack([dx,dy,f[:,2],y-np.median(y)],1)
|
| 104 |
+
xs.append(f)
|
| 105 |
+
L=np.array([len(x) for x in xs]); T=L.max(); X=np.zeros((len(xs),T,4),np.float32)
|
| 106 |
+
for b,x in enumerate(xs): X[b,:len(x)]=x
|
| 107 |
+
return torch.from_numpy(X).to(dev),torch.from_numpy(L)
|
| 108 |
+
|
| 109 |
+
# ---------------- model
|
| 110 |
+
class Net(nn.Module):
|
| 111 |
+
def __init__(s,nc=len(WRITERS),d=192):
|
| 112 |
+
super().__init__()
|
| 113 |
+
s.stem=nn.Sequential(nn.Conv1d(4,128,5,padding=2),nn.GELU(),nn.Conv1d(128,192,5,stride=2,padding=2),nn.GELU(),nn.Conv1d(192,256,5,stride=2,padding=2),nn.GELU())
|
| 114 |
+
s.gru=nn.GRU(256,d,num_layers=2,batch_first=True,bidirectional=True,dropout=0.1)
|
| 115 |
+
s.head=nn.Sequential(nn.Linear(4*d,256),nn.GELU(),nn.Dropout(0.2),nn.Linear(256,nc))
|
| 116 |
+
def forward(s,X,L):
|
| 117 |
+
h=s.stem(X.transpose(1,2)).transpose(1,2); L2=torch.clamp((L+3)//4,min=1) # x4 downsample
|
| 118 |
+
p=nn.utils.rnn.pack_padded_sequence(h,L2.cpu(),batch_first=True,enforce_sorted=False); o,_=s.gru(p); o,_=nn.utils.rnn.pad_packed_sequence(o,batch_first=True)
|
| 119 |
+
m=(torch.arange(o.size(1),device=o.device)[None]<L2.to(o.device)[:,None]).float()[...,None]
|
| 120 |
+
mean=(o*m).sum(1)/m.sum(1); mx=(o+(m-1)*1e4).max(1).values; return s.head(torch.cat([mean,mx],-1))
|
| 121 |
+
def load_cls(path=None):
|
| 122 |
+
path=path or E('CKPT',os.path.join(OUT,'best.pt')); st=torch.load(path,map_location=dev); net=Net().to(dev); net.load_state_dict(st['m']); net.eval(); return net,st
|
| 123 |
+
@torch.no_grad()
|
| 124 |
+
def predict(net,fs,bs=64):
|
| 125 |
+
"""fs: list of [n,4] features -> logits [N,15] (numpy). Sorted by length for small padding."""
|
| 126 |
+
net.eval(); order=np.argsort([len(f) for f in fs]); out=np.zeros((len(fs),len(WRITERS)),np.float32)
|
| 127 |
+
for i in range(0,len(fs),bs):
|
| 128 |
+
ids=order[i:i+bs]; X,L=make_batch([fs[j] for j in ids],False); out[ids]=net(X,L).float().cpu().numpy()
|
| 129 |
+
return out
|
| 130 |
+
|
| 131 |
+
# ---------------- train / eval
|
| 132 |
+
def train():
|
| 133 |
+
os.makedirs(OUT,exist_ok=True); log=open(os.path.join(OUT,'train.log'),'a'); P=lambda *a: (print(*a,flush=True),log.write(' '.join(str(x) for x in a)+'\n'),log.flush())
|
| 134 |
+
sp,assign=cls_splits(); tr=Lines(sp['train']); dv=Lines(sp['dev']); P(f'train {tr.n} dev {dv.n} test {len(sp["test"])} dev={dev} holdout notebooks: {sum(v=="train" for v in assign.values())} train / {sum(v=="test" for v in assign.values())} test')
|
| 135 |
+
t0=time.time(); [tr.feat(i) for i in range(tr.n)]; [dv.feat(i) for i in range(dv.n)]; P(f'features (resample+feats) {time.time()-t0:.0f}s')
|
| 136 |
+
bw=tr.by_writer(); ws=sorted(bw); P('train per writer',{WRITERS[w]:len(bw[w]) for w in ws})
|
| 137 |
+
STEPS=int(E('STEPS','8000')); BS=int(E('BS','64')); LR=float(E('LR','1e-3')); WARM=300
|
| 138 |
+
net=Net().to(dev); P(f'params {sum(p.numel() for p in net.parameters())/1e6:.2f}M'); opt=torch.optim.AdamW(net.parameters(),LR,weight_decay=0.01); rng=np.random.default_rng(int(E('SEED','0')))
|
| 139 |
+
best=0; step=0; t0=time.time(); lsum=0; asum=0
|
| 140 |
+
dvf=[dv.feat(i) for i in range(dv.n)]
|
| 141 |
+
while step<STEPS:
|
| 142 |
+
lr=LR*min(1,(step+1)/WARM)*0.5*(1+math.cos(math.pi*step/STEPS))
|
| 143 |
+
for g in opt.param_groups: g['lr']=lr
|
| 144 |
+
wsel=rng.choice(ws,BS); ids=[bw[w][rng.integers(len(bw[w]))] for w in wsel] # balanced over writers
|
| 145 |
+
X,L=make_batch([tr.feat(i) for i in ids],True,rng); y=torch.from_numpy(tr.y[ids]).to(dev)
|
| 146 |
+
net.train(); o=net(X,L); loss=F.cross_entropy(o,y,label_smoothing=0.1); opt.zero_grad(); loss.backward(); nn.utils.clip_grad_norm_(net.parameters(),5.0); opt.step(); step+=1
|
| 147 |
+
lsum+=loss.item(); asum+=(o.argmax(-1)==y).float().mean().item()
|
| 148 |
+
if step%100==0: P(f'step {step}/{STEPS} loss {lsum/100:.3f} acc {asum/100:.3f} lr {lr:.2e} {step/(time.time()-t0):.1f}it/s eta {(STEPS-step)/(step/(time.time()-t0))/60:.0f}min'); lsum=asum=0
|
| 149 |
+
if step%500==0 or step==STEPS:
|
| 150 |
+
pr=predict(net,dvf).argmax(-1); acc=(pr==dv.y).mean(); bal=np.mean([(pr[dv.y==w]==w).mean() for w in np.unique(dv.y)])
|
| 151 |
+
P(f' dev acc {acc:.4f} balanced {bal:.4f}'); torch.save(dict(m=net.state_dict(),step=step,dev_acc=float(acc),dev_bal=float(bal)),os.path.join(OUT,'last.pt'))
|
| 152 |
+
if bal>best: best=bal; torch.save(dict(m=net.state_dict(),step=step,dev_acc=float(acc),dev_bal=float(bal)),os.path.join(OUT,'best.pt')); P(' best')
|
| 153 |
+
P('CLS DONE best dev balanced',best)
|
| 154 |
+
def evaluate():
|
| 155 |
+
sp,assign=cls_splits(); te=Lines(sp['test']); net,st=load_cls(); print('ckpt step',st.get('step'),'dev acc',st.get('dev_acc'),'dev bal',st.get('dev_bal'))
|
| 156 |
+
fs=[te.feat(i) for i in range(te.n)]; pr=predict(net,fs).argmax(-1); y=te.y; kind=np.array(te.kind)
|
| 157 |
+
res=dict(step=st.get('step'),n_test=int(te.n),acc=float((pr==y).mean()),balanced=float(np.mean([(pr[y==w]==w).mean() for w in np.unique(y)])),
|
| 158 |
+
acc_prose=float((pr==y)[kind=='prose'].mean()),n_prose=int((kind=='prose').sum()),per_writer={},sheet5={},holdout_notebooks={f'{w}:{d}':s for (w,d),s in assign.items()})
|
| 159 |
+
for w in WRITERS:
|
| 160 |
+
m=y==WID[w]; mp=m&(kind=='prose')
|
| 161 |
+
if m.sum(): res['per_writer'][w]=dict(n=int(m.sum()),acc=float((pr[m]==WID[w]).mean()),n_prose=int(mp.sum()),acc_prose=float((pr[mp]==WID[w]).mean()) if mp.sum() else None)
|
| 162 |
+
m5=np.isin(y,[WID[w] for w in SHEET5]); res['sheet5']=dict(n=int(m5.sum()),acc=float((pr[m5]==y[m5]).mean()),acc_prose=float((pr==y)[m5&(kind=='prose')].mean()),
|
| 163 |
+
acc_within5=float(np.mean([pr[i] in [WID[w] for w in SHEET5] and pr[i]==y[i] for i in np.flatnonzero(m5)])))
|
| 164 |
+
cm=np.zeros((len(WRITERS),len(WRITERS)),int)
|
| 165 |
+
for a,b in zip(y,pr): cm[a,b]+=1
|
| 166 |
+
res['confusion']=cm.tolist(); json.dump(res,open(os.path.join(OUT,'eval.json'),'w'),ensure_ascii=False,indent=1)
|
| 167 |
+
print(f"TEST acc {res['acc']:.4f} balanced {res['balanced']:.4f} prose {res['acc_prose']:.4f} (n {res['n_prose']}) | sheet5 acc {res['sheet5']['acc']:.4f} prose {res['sheet5']['acc_prose']:.4f}")
|
| 168 |
+
for w,r in res['per_writer'].items(): print(f" {w:10s} n {r['n']:5d} acc {r['acc']:.3f} prose n {r['n_prose']:4d} acc {r['acc_prose'] if r['acc_prose'] is None else round(r['acc_prose'],3)}")
|
| 169 |
+
|
| 170 |
+
# ---------------- metric on generated lines
|
| 171 |
+
def v6_lines(): # copied from scripts/denoise_style_sheet.py (refs identical to the sheet)
|
| 172 |
+
out={}
|
| 173 |
+
for sp in ('valid','train'):
|
| 174 |
+
z=np.load(os.path.join(PROJ,'data','ink_lines_v6',f'{sp}_raw.npz'),allow_pickle=True); labels=[str(l) for l in z['labels']]; off=z['pt_off']; k=0
|
| 175 |
+
for l in open(os.path.join(PROJ,'data','ink_dataset_v6_sorted',f'meta_{sp}.jsonl')):
|
| 176 |
+
r=json.loads(l)
|
| 177 |
+
if k<len(labels) and r['label']==labels[k]:
|
| 178 |
+
if '$' not in labels[k] and 20<=len(labels[k])<=60 and 150<=off[k+1]-off[k]<=1200: out.setdefault(r['src'],{}).setdefault((sp,r['doc'],r['page']),[]).append(z['pts'][off[k]:off[k+1]].astype(np.float32))
|
| 179 |
+
k+=1
|
| 180 |
+
return out
|
| 181 |
+
def refs_for(src,L):
|
| 182 |
+
if src=='w01':
|
| 183 |
+
z=np.load(os.path.join(PROJ,'data','ink_lines_v2','valid_raw.npz'),allow_pickle=True); off=z['pt_off']; return [z['pts'][off[i]:off[i+1]].astype(np.float32) for i in (0,16,17)],('v2valid',0,16,17)
|
| 184 |
+
pages=sorted(L[src].items(),key=lambda kv:-len(kv[1])); return pages[0][1][:3],pages[0][0]
|
| 185 |
+
def judge_texts(T2,N,SEED): # exactly the standard judge's pool/choice (train_ink_denoise.judge, KIND=text)
|
| 186 |
+
va=T2.Inks('valid'); rng=np.random.RandomState(SEED); pool=[i for i in va.idx if 4<=len(str(va.labels[i]))<=90 and '$' not in str(va.labels[i])]; ids=rng.choice(pool,min(N,len(pool)),replace=False)
|
| 187 |
+
return [str(va.labels[i]) for i in ids]
|
| 188 |
+
def metric():
|
| 189 |
+
os.environ['MODE']='judge'; os.environ.setdefault('PROJ',PROJ) # train_ink_denoise reads MODE before importing the trainer (v2 valid / align_all for the judge's texts)
|
| 190 |
+
import train_ink_denoise as DN; T2=DN.T2
|
| 191 |
+
TAG=E('TAG','denoise1'); CFG=float(E('CFG','1.5')); SST=int(E('SSTEPS','20')); SCFGS=[float(x) for x in E('SCFGS','1,1.5,2,3').split(',')]; N=int(E('N','60')); SEED=int(E('SEED','3'))
|
| 192 |
+
od=os.path.join(PROJ,'out','style_metric',TAG); os.makedirs(od,exist_ok=True); fig=os.path.join(PROJ,'figures')
|
| 193 |
+
texts=judge_texts(T2,N,SEED); toks=[list(t) for t in texts]; json.dump(texts,open(os.path.join(od,'texts.json'),'w'),ensure_ascii=False)
|
| 194 |
+
base=DN.load_base(); L=v6_lines(); refs={}; refsrc={}
|
| 195 |
+
for w in SHEET5: refs[w],refsrc[w]=refs_for(w,L)
|
| 196 |
+
svs={w:T2.style_from_seqs(base,refs[w]) for w in SHEET5}; durs=DN.head_durs(base,toks)
|
| 197 |
+
conds=['lstm']+[f'dn_s{s:g}' for s in SCFGS]; net=None
|
| 198 |
+
def gen(w,cond):
|
| 199 |
+
f=os.path.join(od,f'seqs_{w}_{cond}.npz')
|
| 200 |
+
if os.path.exists(f): z=np.load(f,allow_pickle=True); return list(z['seqs'])
|
| 201 |
+
nonlocal net
|
| 202 |
+
if cond!='lstm' and net is None: net,step=DN.load_net(E('DN_CKPT')); print('denoiser step',step,flush=True)
|
| 203 |
+
seqs=[]; t0=time.time()
|
| 204 |
+
for i in range(0,N,16):
|
| 205 |
+
torch.manual_seed(1000+i)
|
| 206 |
+
if cond=='lstm': seqs+=T2._sample(base,toks[i:i+16],bias=1.0,sv=svs[w])
|
| 207 |
+
else: os.environ['SCFG']=cond[4:]; seqs+=DN.sample(net,toks[i:i+16],durs[i:i+16],svs[w],cfg=CFG,steps=SST)
|
| 208 |
+
np.savez(f,seqs=np.array([np.asarray(q,np.float32) for q in seqs],dtype=object),labels=np.array(texts,dtype=object)); print(f' {w} {cond}: {len(seqs)} lines {time.time()-t0:.0f}s',flush=True); return seqs
|
| 209 |
+
# real ceiling: dev+test prose lines of the writer (unseen notebooks), up to N, seed-fixed
|
| 210 |
+
real={}; rng=np.random.default_rng(SEED)
|
| 211 |
+
for sp in ('dev','test'):
|
| 212 |
+
pts,off,meta=load_split(sp)
|
| 213 |
+
for i,r in enumerate(meta):
|
| 214 |
+
if r['writer_id'] in SHEET5 and r['kind']=='prose' and 4<=len(r['label'])<=90: real.setdefault(r['writer_id'],[]).append(pts[off[i]:off[i+1]].astype(np.float32))
|
| 215 |
+
for w in SHEET5: ids=rng.permutation(len(real[w]))[:N]; real[w]=[real[w][i] for i in ids]
|
| 216 |
+
allseqs={}
|
| 217 |
+
for w in SHEET5:
|
| 218 |
+
allseqs[(w,'real')]=real[w]
|
| 219 |
+
for c in conds: allseqs[(w,c)]=gen(w,c)
|
| 220 |
+
cls,st=load_cls(); res=dict(tag=TAG,cfg=CFG,ssteps=SST,n_texts=N,seed=SEED,cls_step=st.get('step'),dn_ckpt=E('DN_CKPT'),refs={w:str(refsrc[w]) for w in SHEET5},acc={},conf={},n={})
|
| 221 |
+
for (w,c),seqs in allseqs.items():
|
| 222 |
+
fs=[feats(resample(q)) for q in seqs]; pr=predict(cls,fs).argmax(-1); res['acc'].setdefault(c,{})[w]=float((pr==WID[w]).mean()); res['n'].setdefault(c,{})[w]=len(seqs)
|
| 223 |
+
res['conf'].setdefault(c,{})[w]=np.bincount(pr,minlength=len(WRITERS)).tolist()
|
| 224 |
+
json.dump(res,open(os.path.join(od,'metric.json'),'w'),ensure_ascii=False,indent=1)
|
| 225 |
+
# markdown table
|
| 226 |
+
allc=['real']+conds; lines=[f'### Метрика стиля ({TAG}, CFG {CFG}, {SST} шагов, {N} текстов seed {SEED}, классификатор шаг {st.get("step")})','',
|
| 227 |
+
'| условие | '+' | '.join(NAMES[w] for w in SHEET5)+' | среднее |','|---|'+'---|'*(len(SHEET5)+1)]
|
| 228 |
+
for c in allc: a=[res['acc'][c][w] for w in SHEET5]; lines.append(f'| {c} | '+' | '.join(f'{x*100:.0f}%' for x in a)+f' | {np.mean(a)*100:.0f}% |')
|
| 229 |
+
lines+=['','Матрицы «заданный писец × узнанный» (строки = заданный, столбцы = узнанный; число строк из %d):'%N]
|
| 230 |
+
for c in allc:
|
| 231 |
+
lines+=['',f'{c}:','| задан \\ узнан | '+' | '.join(NAMES[w] for w in WRITERS)+' |','|---|'+'---|'*len(WRITERS)]
|
| 232 |
+
for w in SHEET5: lines.append(f'| {NAMES[w]} | '+' | '.join(('**%d**' if WRITERS[j]==w else '%d')%v for j,v in enumerate(res['conf'][c][w]))+' |')
|
| 233 |
+
open(os.path.join(od,'metric.md'),'w').write('\n'.join(lines)+'\n'); print('\n'.join(lines[:4+len(allc)]),flush=True)
|
| 234 |
+
plot(res,allc,os.path.join(fig,f'style_metric_{TAG}.png')); sheet(allseqs,allc,texts,os.path.join(fig,f'style_metric_{TAG}_sheet.png'),DN.render)
|
| 235 |
+
print('METRIC DONE',od,flush=True)
|
| 236 |
+
def plot(res,allc,path):
|
| 237 |
+
import matplotlib; matplotlib.use('Agg'); import matplotlib.pyplot as plt
|
| 238 |
+
fig,ax=plt.subplots(1,2,figsize=(15,5.2),gridspec_kw=dict(width_ratios=[1,1.4]))
|
| 239 |
+
A=np.array([[res['acc'][c][w] for w in SHEET5] for c in allc])*100; im=ax[0].imshow(A,cmap='Greens',vmin=0,vmax=100)
|
| 240 |
+
ax[0].set_xticks(range(len(SHEET5))); ax[0].set_xticklabels([NAMES[w] for w in SHEET5]); ax[0].set_yticks(range(len(allc))); ax[0].set_yticklabels(allc)
|
| 241 |
+
for i in range(len(allc)):
|
| 242 |
+
for j in range(len(SHEET5)): ax[0].text(j,i,f'{A[i,j]:.0f}',ha='center',va='center',color='black' if A[i,j]<60 else 'white',fontsize=11)
|
| 243 |
+
ax[0].set_title('узнан заданный писец, % строк'); ax[0].set_xlabel('заданный писец (опоры K=3)')
|
| 244 |
+
c0=[c for c in allc if c.startswith('dn_s1') and c!='dn_s1.5'][0] if any(c=='dn_s1' for c in allc) else allc[-1]
|
| 245 |
+
C=np.array([res['conf'][c0][w] for w in SHEET5],float); C=C/C.sum(1,keepdims=True)*100; im2=ax[1].imshow(C,cmap='Blues',vmin=0,vmax=100)
|
| 246 |
+
ax[1].set_xticks(range(len(WRITERS))); ax[1].set_xticklabels([NAMES[w] for w in WRITERS],rotation=60,ha='right'); ax[1].set_yticks(range(len(SHEET5))); ax[1].set_yticklabels([NAMES[w] for w in SHEET5])
|
| 247 |
+
for i in range(len(SHEET5)):
|
| 248 |
+
for j in range(len(WRITERS)):
|
| 249 |
+
if C[i,j]>=3: ax[1].text(j,i,f'{C[i,j]:.0f}',ha='center',va='center',color='black' if C[i,j]<60 else 'white',fontsize=9)
|
| 250 |
+
ax[1].set_title(f'{c0}: заданный (строки) × узнанный (столбцы), %'); ax[1].set_xlabel('узнанный писец')
|
| 251 |
+
fig.suptitle(f"Метрика стиля: классификатор писца (15 кл.) на строках {res['tag']} — {time.strftime('%Y-%m-%d %H:%M')}",fontsize=11); fig.tight_layout(); fig.savefig(path,dpi=130); print('fig',path,flush=True)
|
| 252 |
+
def sheet(allseqs,allc,texts,path,render,nt=3):
|
| 253 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 254 |
+
os.environ.setdefault('RENDER_S','40'); f=ImageFont.truetype(os.path.join(PROJ,'DejaVuSans.ttf'),14); CW=420
|
| 255 |
+
def fit(im):
|
| 256 |
+
im=im.convert('L'); r=min(1.0,(CW-8)/im.width); return im.resize((max(1,int(im.width*r)),max(1,int(im.height*r))),Image.LANCZOS)
|
| 257 |
+
rows=[]
|
| 258 |
+
for ti in range(min(nt,len(texts))):
|
| 259 |
+
for c in allc: rows.append((f'{c}: {texts[ti][:50]}',[fit(render(allseqs[(w,c)][ti])) for w in SHEET5]))
|
| 260 |
+
H=sum(max(im.height for im in ims)+24 for _,ims in rows)+30; sh=Image.new('L',(CW*len(SHEET5)+10,H),255); d=ImageDraw.Draw(sh)
|
| 261 |
+
for k,w in enumerate(SHEET5): d.text((10+k*CW,4),NAMES[w],fill=0,font=f)
|
| 262 |
+
y=30
|
| 263 |
+
for cap,ims in rows:
|
| 264 |
+
d.text((10,y),cap,fill=0,font=f); y+=20; hh=max(im.height for im in ims)
|
| 265 |
+
for k,im in enumerate(ims): sh.paste(im,(10+k*CW,y))
|
| 266 |
+
d.line([(0,y+hh+2),(sh.width,y+hh+2)],fill=200); y+=hh+4
|
| 267 |
+
sh.save(path); print('sheet',path,flush=True)
|
| 268 |
+
if __name__=='__main__':
|
| 269 |
+
{'train':train,'eval':evaluate,'metric':metric}[MODE]()
|
scripts/train_glyph_cls.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""Glyph classifier for SELECTING generated formula glyphs (formula_write.py SEL_K). Same CNN/features as formula_glyph_clean.py
|
| 3 |
+
(32x32 render + relative height box_h/fh + log aspect), trained ONCE on data/formula_glyphs/glyphs_clean.jsonl WITHOUT the valid
|
| 4 |
+
notebooks of ink_dataset_v2 (the formula judge's 40 formulas come from them), classes with >= MINC samples. Accuracy is reported on
|
| 5 |
+
the held-out valid-notebook glyphs. Output: ckpt_dl/glyph_cls1.pt {state, classes}. Env: PROJ, MINC (30), EPOCHS (8)."""
|
| 6 |
+
import os, sys, json, collections, numpy as np, torch, torch.nn as nn, torch.nn.functional as F
|
| 7 |
+
from PIL import Image, ImageDraw
|
| 8 |
+
PROJ=os.environ.get('PROJ',os.getcwd()); E=os.environ.get; MINC=int(E('MINC','30')); EPOCHS=int(E('EPOCHS','8'))
|
| 9 |
+
dev='cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu')
|
| 10 |
+
def render(strokes,S=32):
|
| 11 |
+
pts=np.array([p[:2] for s in strokes for p in s],np.float32); x0,y0=pts.min(0); x1,y1=pts.max(0); sc=(S-4)/max(x1-x0,y1-y0,1e-6)
|
| 12 |
+
im=Image.new('L',(S,S),0); d=ImageDraw.Draw(im); ox=(S-(x1-x0)*sc)/2; oy=(S-(y1-y0)*sc)/2
|
| 13 |
+
for s in strokes:
|
| 14 |
+
q=[((p[0]-x0)*sc+ox,(p[1]-y0)*sc+oy) for p in s]
|
| 15 |
+
if len(q)>1: d.line(q,fill=255,width=2)
|
| 16 |
+
elif q: d.point(q[0],fill=255)
|
| 17 |
+
return np.asarray(im,np.float32)/255.0
|
| 18 |
+
def feats(strokes,relh):
|
| 19 |
+
pts=np.array([p[:2] for s in strokes for p in s],np.float32); w=np.ptp(pts[:,0]); h=np.ptp(pts[:,1])
|
| 20 |
+
return np.array([relh,np.log((w+1e-3)/(h+1e-3))],np.float32)
|
| 21 |
+
class Net(nn.Module):
|
| 22 |
+
def __init__(s,n):
|
| 23 |
+
super().__init__(); s.c=nn.Sequential(nn.Conv2d(1,32,3,padding=1),nn.ReLU(),nn.MaxPool2d(2),nn.Conv2d(32,64,3,padding=1),nn.ReLU(),nn.MaxPool2d(2),nn.Conv2d(64,128,3,padding=1),nn.ReLU(),nn.AdaptiveAvgPool2d(2))
|
| 24 |
+
s.f=nn.Sequential(nn.Linear(128*4+2,256),nn.ReLU(),nn.Dropout(0.3),nn.Linear(256,n))
|
| 25 |
+
def forward(s,x,a): return s.f(torch.cat([s.c(x).flatten(1),a],1))
|
| 26 |
+
def load(path):
|
| 27 |
+
st=torch.load(path,map_location=dev); net=Net(len(st['classes'])).to(dev); net.load_state_dict(st['state']); net.eval(); return net,st['classes']
|
| 28 |
+
if __name__=='__main__':
|
| 29 |
+
valid_docs={json.loads(l)['doc'] for l in open(os.path.join(PROJ,'data','ink_dataset_v2','valid.jsonl'))} if os.path.exists(os.path.join(PROJ,'data','ink_dataset_v2','valid.jsonl')) else set(json.load(open(os.path.join(PROJ,'data','m3','valid_docs.json'))))
|
| 30 |
+
LIB=E('LIB','') # M3 22.10: LIB=data/m3/clean2.npz (glyph_m3.pack of glyphs_clean2.jsonl), OUT=ckpt_dl/glyph_cls2.pt, EPOCHS 12, MINC 20
|
| 31 |
+
if LIB.endswith('.npz'):
|
| 32 |
+
sys.path.insert(0,os.path.dirname(os.path.abspath(__file__))); import glyph_m3 as GM
|
| 33 |
+
G=[dict(cls=g['cls'],doc=g['doc'],strokes=g['strokes'],relh=g['relh']) for g in GM.load(os.path.join(PROJ,LIB))]
|
| 34 |
+
else:
|
| 35 |
+
G=[json.loads(l) for l in open(os.path.join(PROJ,'data','formula_glyphs','glyphs_clean.jsonl'))]
|
| 36 |
+
for g in G: g['relh']=(g['box'][3]-g['box'][1])/max(g['fh'],1e-6)
|
| 37 |
+
tr=[g for g in G if g['doc'] not in valid_docs]; te=[g for g in G if g['doc'] in valid_docs]
|
| 38 |
+
cnt=collections.Counter(g['cls'] for g in tr); classes=sorted(c for c,k in cnt.items() if k>=MINC); ci={c:i for i,c in enumerate(classes)}
|
| 39 |
+
tr=[g for g in tr if g['cls'] in ci]; te=[g for g in te if g['cls'] in ci]; print('train',len(tr),'held-out (valid notebooks)',len(te),'classes',len(classes),flush=True)
|
| 40 |
+
def XY(L): return (np.stack([render(g['strokes']) for g in L]),np.stack([feats(g['strokes'],g['relh']) for g in L]),np.array([ci[g['cls']] for g in L]))
|
| 41 |
+
Xa,Aa,Ya=XY(tr); Xt,At,Yt=XY(te); rng=np.random.default_rng(0); torch.manual_seed(0); net=Net(len(classes)).to(dev); opt=torch.optim.AdamW(net.parameters(),2e-3,weight_decay=1e-4)
|
| 42 |
+
for ep in range(EPOCHS):
|
| 43 |
+
perm=rng.permutation(len(Ya)); net.train()
|
| 44 |
+
for b in range(0,len(perm),256):
|
| 45 |
+
j=perm[b:b+256]; loss=F.cross_entropy(net(torch.from_numpy(Xa[j])[:,None].to(dev),torch.from_numpy(Aa[j]).to(dev)),torch.from_numpy(Ya[j]).to(dev),label_smoothing=0.05); opt.zero_grad(); loss.backward(); opt.step()
|
| 46 |
+
net.eval()
|
| 47 |
+
with torch.no_grad(): pr=torch.cat([net(torch.from_numpy(Xt[b:b+512])[:,None].to(dev),torch.from_numpy(At[b:b+512]).to(dev)).argmax(1).cpu() for b in range(0,len(Yt),512)]).numpy()
|
| 48 |
+
print(f'epoch {ep+1}: held-out acc {(pr==Yt).mean():.3f}',flush=True)
|
| 49 |
+
OUT=E('OUT','ckpt_dl/glyph_cls1.pt'); os.makedirs(os.path.dirname(os.path.join(PROJ,OUT)),exist_ok=True); torch.save(dict(state=net.state_dict(),classes=classes),os.path.join(PROJ,OUT)); print('saved',OUT)
|
| 50 |
+
conf=collections.Counter((classes[a],classes[b]) for a,b in zip(Yt,pr) if a!=b); per={c:[0,0] for c in classes}
|
| 51 |
+
for a,b in zip(Yt,pr): per[classes[a]][0]+=1; per[classes[a]][1]+=int(a==b)
|
| 52 |
+
json.dump(dict(acc=float((pr==Yt).mean()),n=int(len(Yt)),per={c:dict(n=v[0],acc=v[1]/v[0]) for c,v in per.items() if v[0]},confusions=[[a,b,k] for (a,b),k in conf.most_common()]),open(os.path.join(PROJ,OUT.replace('.pt','_confusion.json')),'w'),ensure_ascii=False,indent=1)
|
| 53 |
+
print('top confusions (true -> predicted, count):',conf.most_common(25))
|
scripts/train_glyph_dn.py
ADDED
|
@@ -0,0 +1,318 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""И116 (22.10, w03: «перенести рецепт денойзера на формулы — генерировать только символы»): a diffusion DENOISER of ONE formula
|
| 3 |
+
glyph, the recipe of scripts/train_ink_denoise.py (И100) transplanted from lines to glyphs.
|
| 4 |
+
x0 per point : [ (x - cx)/XS , (y - cy)/YS , pen ±1 , 2*pressure-1 ] — ABSOLUTE coordinates in the glyph's box (cx, cy = box
|
| 5 |
+
centre, norm_h units of the VAE data = 2 x median glyph height of its formula), pen = «pen up after this point»
|
| 6 |
+
as an analog bit, pressure as an analog channel (XS = YS = 0.15: measured residual sd on 3 000 train glyphs)
|
| 7 |
+
condition : class (embedding + FiLM; NULL row for CFG, dropped with p=PDROP per glyph), writer (embedding + FiLM, 17 writers
|
| 8 |
+
of v7 + a null row dropped with p=PWDROP = «average writer», also the fallback for an unknown writer), per-point
|
| 9 |
+
phase (pos/n, sin, cos), log length, position index; noise level t -> FiLM in every block; self-conditioning
|
| 10 |
+
on the previous x0 estimate.
|
| 11 |
+
Style: a learned writer vector instead of the frozen v6isl3 line encoder — that encoder reads prose LINES of
|
| 12 |
+
~1 200 points and followed style weakly even there (DIARY 22.10 20:29); a glyph is 27 points, outside its domain.
|
| 13 |
+
denoiser : ConvNeXt-1D (dilated depthwise k=7) -> BiGRU -> ConvNeXt-1D, FiLM by (t, class, writer); predicts x0 (xy, pressure)
|
| 14 |
+
+ a pen logit. D=256, pre 4 / post 3, GRU 2x256 -> ~6M params (glyphs are short: median 27 points, cap MAXLEN 128).
|
| 15 |
+
diffusion : cosine schedule, T=100, DDIM (eta 0) with GDN_STEPS (10) at sampling, CFG over the class (GDN_CFG 1.5).
|
| 16 |
+
loss : L1 positions (z units) + L1 x20 on point-to-point increments (raw units) + BCE pen (pos_weight 3) + L1 pressure.
|
| 17 |
+
plan (length) : at sampling the point count is drawn from the empirical length distribution of the (writer, class) train glyphs
|
| 18 |
+
(class over all writers when the writer has < LEN_MIN of it) — saved in the checkpoint (lens).
|
| 19 |
+
Data: train_glyph_vae.load_split on INK_DS / META_DS (default ink_glyphs_v7 / ink_dataset_glyphs_v7 = clean3 of all 17 writers,
|
| 20 |
+
110 102 train glyphs, valid = w01's held-out notebooks). Length-bucketed batches, affine augmentation as the VAE (AUG_S, AUG_SH).
|
| 21 |
+
GSTYLE=semb (04.11, И164: one reference LINE -> glyphs of that hand, also for writers never seen): the writer condition is the
|
| 22 |
+
frozen semb3_all embedding (data/<SEMB_DS>/semb_<SEMB_TAG>_train.npy, the style vector B takes) of a random line of the same writer
|
| 23 |
+
from ANOTHER doc (another page if the writer has one doc), drawn afresh every batch; dropped with p=PWDROP to a null flag (the
|
| 24 |
+
«average writer»). src in the data becomes "writer<TAB>doc" (wr() strips it). sample() then takes W as float rows [sdim+1].
|
| 25 |
+
MODES: MODE=train (06.11: INIT = warm-start checkpoint, INK_DS2 / META_DS2 / REP2 = a second set mixed in; env PROJ, RUN (glyphdn1), STEPS (50000), BS (256), LR (3e-4), DN_D (256), DN_GRU (256), DN_PRE (4), DN_POST (3),
|
| 26 |
+
PDROP (0.15), PWDROP (0.1), MAXLEN (128), CK_EVERY (2000), DEV) -> ckpt_glyph_dn_<RUN>/{last.pt, fp16/stepNNNNNN.pt, classes.json,
|
| 27 |
+
writers.json, lens.npz}, samples_glyph_dn_<RUN>/stepNNNNNN.png (12 classes x 6 real | 6 generated, w01);
|
| 28 |
+
MODE=sheet (CKPT, OUT, CLASSES, WRITER, N) -> an eyeball sheet."""
|
| 29 |
+
import os, sys, json, math, time, copy, collections, numpy as np, torch, torch.nn as nn, torch.nn.functional as F
|
| 30 |
+
PROJ=os.environ.get('PROJ',os.path.dirname(os.path.dirname(os.path.abspath(__file__)))); E=os.environ.get
|
| 31 |
+
sys.path.insert(0,os.path.join(PROJ,'scripts'))
|
| 32 |
+
os.environ.setdefault('INK_DS','ink_glyphs_v7'); os.environ.setdefault('META_DS','ink_dataset_glyphs_v7')
|
| 33 |
+
dev=E('DEV') or ('cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu'))
|
| 34 |
+
RUN=E('RUN','glyphdn1'); CKPT=os.path.join(PROJ,'ckpt_glyph_dn_'+RUN); SAMP=os.path.join(PROJ,'samples_glyph_dn_'+RUN)
|
| 35 |
+
STEPS=int(E('STEPS','50000')); BS=int(E('BS','256')); LR=float(E('LR','3e-4')); WD=float(E('WD','0.01')); WARM=int(E('WARM','500'))
|
| 36 |
+
TD=int(E('TD','100')); PDROP=float(E('PDROP','0.15')); PWDROP=float(E('PWDROP','0.1')); SELFCOND=E('SELFCOND','1')=='1'
|
| 37 |
+
GSTYLE=E('GSTYLE',''); SEMB_DS=E('SEMB_DS','ink_lines_u2'); SEMB_TAG=E('SEMB_TAG','3all'); SDIM=256
|
| 38 |
+
wr=lambda src: src.split('\t')[0]
|
| 39 |
+
DN_D=int(E('DN_D','256')); DN_GRU=int(E('DN_GRU','256')); DN_PRE=int(E('DN_PRE','4')); DN_POST=int(E('DN_POST','3')); G_DIM=192
|
| 40 |
+
XS=float(E('XS','0.15')); YS=float(E('YS','0.15')); MAXLEN=int(E('MAXLEN','128'))
|
| 41 |
+
DELW=float(E('DELW','1.0')); PENW=float(E('PENW','3.0')); POSW=float(E('POSW','1.0')); PRW=float(E('PRW','0.5'))
|
| 42 |
+
AMP=E('AMP','1')=='1' and dev=='cuda'; CK_EVERY=int(E('CK_EVERY','2000')); LOG_EVERY=int(E('LOG_EVERY','100')); LEN_MIN=int(E('LEN_MIN','5'))
|
| 43 |
+
if dev=='cuda': torch.backends.cuda.matmul.allow_tf32=True; torch.backends.cudnn.allow_tf32=True
|
| 44 |
+
|
| 45 |
+
# ---------------- diffusion schedule (cosine, discrete T=TD) — as train_ink_denoise.alpha_bar
|
| 46 |
+
def alpha_bar(Tn=TD,s=0.008):
|
| 47 |
+
t=np.arange(1,Tn+1)/Tn; f=np.cos((t+s)/(1+s)*math.pi/2)**2/math.cos(s/(1+s)*math.pi/2)**2; return np.clip(f,1e-4,0.9999).astype(np.float32)
|
| 48 |
+
AB=torch.tensor(alpha_bar(),device=dev)
|
| 49 |
+
|
| 50 |
+
# ---------------- data
|
| 51 |
+
def load_data(split):
|
| 52 |
+
import train_glyph_vae as TV
|
| 53 |
+
TV.MAXLEN=10**6 # we cap ourselves
|
| 54 |
+
out=[]
|
| 55 |
+
if GSTYLE: # = TV.load_split, plus the doc of each record (load_split drops 1-point records, so its index is not the meta row)
|
| 56 |
+
z=np.load(os.path.join(TV.DS,split+'_raw.npz'),allow_pickle=True); off=z['pt_off']; pts=z['pts']; pres=z['pres']; labs=list(z['labels'])
|
| 57 |
+
meta=[json.loads(l) for l in open(os.path.join(TV.META,'meta_'+split+'.jsonl'))]; assert len(meta)==len(labs)
|
| 58 |
+
for i in range(len(labs)):
|
| 59 |
+
assert meta[i]['label']==labs[i]; d=np.concatenate([pts[off[i]:off[i+1]],pres[off[i]:off[i+1],None]],1).astype(np.float32)
|
| 60 |
+
if 2<=len(d)<=MAXLEN: out.append((d,labs[i],meta[i]['src']+'\t'+meta[i].get('doc','')))
|
| 61 |
+
return out
|
| 62 |
+
for d,lab,src in TV.load_split(split):
|
| 63 |
+
if len(d)>MAXLEN or len(d)<2: continue
|
| 64 |
+
out.append((d,lab,src))
|
| 65 |
+
return out
|
| 66 |
+
def load_ds(split,ds,meta): # 06.11: load_data on another INK_DS / META_DS
|
| 67 |
+
import train_glyph_vae as TV; o=(TV.DS,TV.META); TV.DS=os.path.join(PROJ,'data',ds); TV.META=os.path.join(PROJ,'data',meta)
|
| 68 |
+
try: return load_data(split)
|
| 69 |
+
finally: TV.DS,TV.META=o
|
| 70 |
+
def to_x0(d):
|
| 71 |
+
"""d [n,4] dx,dy,pen_up,pressure (norm_h units) -> x0 [n,4] (zx, zy, pen±1, 2p-1) and raw positions P [n,2]"""
|
| 72 |
+
P=np.cumsum(d[:,:2],0); c=(P.min(0)+P.max(0))/2; Q=P-c
|
| 73 |
+
return np.stack([Q[:,0]/XS,Q[:,1]/YS,d[:,2]*2-1,np.clip(d[:,3],0,1)*2-1],1).astype(np.float32),P.astype(np.float32)
|
| 74 |
+
_BANK=None
|
| 75 |
+
def style_bank():
|
| 76 |
+
"""GSTYLE=semb: writer -> (doc array, embedding rows) of its u2 train lines"""
|
| 77 |
+
global _BANK
|
| 78 |
+
if _BANK is None:
|
| 79 |
+
Em=np.load(os.path.join(PROJ,'data',SEMB_DS,f'semb_{SEMB_TAG}_train.npy')).astype(np.float32); M=[json.loads(l) for l in open(os.path.join(PROJ,'data',SEMB_DS,'meta_train.jsonl'))]
|
| 80 |
+
by=collections.defaultdict(list)
|
| 81 |
+
for i,m in enumerate(M): by[m.get('writer_id')].append(i)
|
| 82 |
+
_BANK={w:(np.array([M[i]['doc'] for i in ix]),Em[ix]) for w,ix in by.items()}
|
| 83 |
+
return _BANK
|
| 84 |
+
def style_rows(srcs,rng):
|
| 85 |
+
"""GSTYLE=semb: per item a [SDIM+1] row = embedding of a random line of its writer from another doc + null flag 0 (unknown writer -> null)"""
|
| 86 |
+
bk=style_bank(); S=np.zeros((len(srcs),SDIM+1),np.float32)
|
| 87 |
+
for b,src in enumerate(srcs):
|
| 88 |
+
w,_,doc=src.partition('\t')
|
| 89 |
+
if w not in bk: S[b,-1]=1; continue
|
| 90 |
+
dd,em=bk[w]; ok=np.where(dd!=doc)[0]; ok=ok if len(ok) else np.arange(len(dd)); S[b,:SDIM]=em[ok[rng.integers(len(ok))]]
|
| 91 |
+
return S
|
| 92 |
+
def make_batch(items,c2i,w2i,aug=False,rng=None):
|
| 93 |
+
B=len(items); Tn=max(len(d) for d,_,_ in items); Z=np.zeros((B,Tn,4),np.float32); Mk=np.zeros((B,Tn),np.float32); C=np.zeros(B,np.int64); W=np.zeros(B,np.int64)
|
| 94 |
+
if GSTYLE: W=style_rows([src for _,_,src in items],rng if rng is not None else np.random.default_rng(1))
|
| 95 |
+
for b,(d,lab,src) in enumerate(items):
|
| 96 |
+
q=d.copy()
|
| 97 |
+
if aug:
|
| 98 |
+
sx,sy=1+rng.uniform(-1,1,2)*float(E('AUG_S','0.08')); sh=rng.uniform(-1,1)*float(E('AUG_SH','0.1'))
|
| 99 |
+
dx=q[:,0]*sx+q[:,1]*sh; dy=q[:,1]*sy; q[:,0]=dx; q[:,1]=dy
|
| 100 |
+
z,_=to_x0(q); n=len(z); Z[b,:n]=z; Mk[b,:n]=1; C[b]=c2i[lab]
|
| 101 |
+
if not GSTYLE: W[b]=w2i.get(src,len(w2i))
|
| 102 |
+
t=lambda a: torch.from_numpy(a).to(dev)
|
| 103 |
+
return t(Z),t(Mk),t(C),t(W)
|
| 104 |
+
def buckets(data,bs,rng):
|
| 105 |
+
idx=np.argsort([len(d) for d,_,_ in data]+rng.uniform(0,3,len(data))); bl=[idx[i:i+bs] for i in range(0,len(idx),bs)]; rng.shuffle(bl); return bl
|
| 106 |
+
|
| 107 |
+
# ---------------- denoiser
|
| 108 |
+
def tembed(t,dim=128):
|
| 109 |
+
h=dim//2; fr=torch.exp(-math.log(10000)*torch.arange(h,device=t.device)/h); a=t.float()[:,None]*fr[None]*(1000.0/TD); return torch.cat([a.sin(),a.cos()],-1)
|
| 110 |
+
class FiLM(nn.Module):
|
| 111 |
+
def __init__(s,G,D): super().__init__(); s.l=nn.Linear(G,2*D); nn.init.zeros_(s.l.weight); nn.init.zeros_(s.l.bias)
|
| 112 |
+
def forward(s,x,g): a,b=s.l(g)[:,None].chunk(2,-1); return x*(1+a)+b
|
| 113 |
+
class CNX(nn.Module):
|
| 114 |
+
def __init__(s,D,G,dil):
|
| 115 |
+
super().__init__(); s.dw=nn.Conv1d(D,D,7,padding=3*dil,dilation=dil,groups=D); s.ln=nn.LayerNorm(D); s.film=FiLM(G,D); s.f1=nn.Linear(D,4*D); s.f2=nn.Linear(4*D,D); s.gam=nn.Parameter(torch.full((D,),0.1))
|
| 116 |
+
def forward(s,x,m,g):
|
| 117 |
+
h=s.dw((x*m).transpose(1,2)).transpose(1,2); h=s.film(s.ln(h),g); h=s.f2(F.gelu(s.f1(h))); return (x+s.gam*h)*m
|
| 118 |
+
class Denoiser(nn.Module):
|
| 119 |
+
def __init__(s,ncls,nwr):
|
| 120 |
+
super().__init__(); D=DN_D; G=G_DIM; s.ncls=ncls; s.nwr=nwr
|
| 121 |
+
s.ec=nn.Embedding(ncls+1,D); s.gc=nn.Embedding(ncls+1,G)
|
| 122 |
+
if GSTYLE: s.ew=nn.Linear(SDIM+1,D); s.gw=nn.Linear(SDIM+1,G) # W = [embedding, null flag]
|
| 123 |
+
else: s.ew=nn.Embedding(nwr+1,D); s.gw=nn.Embedding(nwr+1,G)
|
| 124 |
+
s.inp=nn.Linear(4+4+5,D); s.tm=nn.Sequential(nn.Linear(128,G),nn.SiLU(),nn.Linear(G,G))
|
| 125 |
+
dil=[1,2,4,1,2,4,8]
|
| 126 |
+
s.pre=nn.ModuleList([CNX(D,G,dil[k%len(dil)]) for k in range(DN_PRE)]); s.gln=nn.LayerNorm(D); s.gfilm=FiLM(G,D)
|
| 127 |
+
s.gru=nn.GRU(D,DN_GRU,num_layers=2,batch_first=True,bidirectional=True); s.gout=nn.Linear(2*DN_GRU,D)
|
| 128 |
+
s.post=nn.ModuleList([CNX(D,G,dil[k%len(dil)]) for k in range(DN_POST)]); s.oln=nn.LayerNorm(D); s.head=nn.Linear(D,4)
|
| 129 |
+
def forward(s,zt,t,C,W,Mk,sc):
|
| 130 |
+
"""zt [B,T,4] noisy x0; t [B]; C [B] class (ncls = NULL); W [B] writer (nwr = null); Mk [B,T]; sc [B,T,4] self-cond (zeros = none)
|
| 131 |
+
-> x0 estimate [B,T,3] (zx, zy, pressure) and pen logit [B,T]"""
|
| 132 |
+
m=Mk[...,None]; Bn,Tn=Mk.shape; L=Mk.sum(1).clamp(min=1)
|
| 133 |
+
i=torch.arange(Tn,device=zt.device,dtype=torch.float32)[None].expand(Bn,-1); pc=(i/L[:,None])[...,None]
|
| 134 |
+
feats=torch.cat([zt,sc,pc,torch.sin(2*math.pi*pc),torch.cos(2*math.pi*pc),torch.log(L)[:,None,None].expand(-1,Tn,-1)/4.0,i[...,None]/100.0],-1)
|
| 135 |
+
x=(s.inp(feats)+s.ec(C)[:,None]+s.ew(W)[:,None])*m
|
| 136 |
+
g=F.silu(s.tm(tembed(t))+s.gc(C)+s.gw(W))
|
| 137 |
+
for b in s.pre: x=b(x,m,g)
|
| 138 |
+
h=s.gfilm(s.gln(x),g); Ll=L.long()
|
| 139 |
+
with torch.autocast(dev if dev!='mps' else 'cpu',enabled=False):
|
| 140 |
+
p=nn.utils.rnn.pack_padded_sequence(h.float(),Ll.cpu(),batch_first=True,enforce_sorted=False); o,_=s.gru(p); o,_=nn.utils.rnn.pad_packed_sequence(o,batch_first=True,total_length=Tn)
|
| 141 |
+
x=(x+s.gout(o.to(x.dtype)))*m
|
| 142 |
+
for b in s.post: x=b(x,m,g)
|
| 143 |
+
o=s.head(s.oln(x)).float(); return o[...,:3]*m,o[...,3]
|
| 144 |
+
|
| 145 |
+
# ---------------- sampling (DDIM, x0-prediction, CFG over the class)
|
| 146 |
+
@torch.no_grad()
|
| 147 |
+
def sample(net,C,W,lens,cfg=1.5,steps=10,seed=None,gen=None,pen_thr=0.0,eta=0.0,noise=1.0):
|
| 148 |
+
"""C, W: lists of class / writer indices; lens: point counts -> list of [n,4] (x, y, pen_up, pressure) in norm_h units, centred on the box."""
|
| 149 |
+
net.eval(); Bn=len(C); Tn=int(max(lens)); Mk=torch.zeros(Bn,Tn,device=dev)
|
| 150 |
+
for b,n in enumerate(lens): Mk[b,:int(n)]=1
|
| 151 |
+
m=Mk[...,None]; C=torch.tensor(C,device=dev,dtype=torch.long); W=torch.tensor(np.asarray(W),device=dev,dtype=torch.float32 if GSTYLE else torch.long) # GSTYLE: rows [SDIM+1]
|
| 152 |
+
if gen is None and seed is not None: gen=torch.Generator(device='cpu').manual_seed(int(seed))
|
| 153 |
+
z=(torch.randn(Bn,Tn,4,generator=gen).to(dev) if gen is not None else torch.randn(Bn,Tn,4,device=dev))*m*noise # noise>1: a hotter start (diversity lever)
|
| 154 |
+
sc=torch.zeros_like(z); ts=np.linspace(TD-1,0,steps).round().astype(int)
|
| 155 |
+
if cfg!=1.0: C2=torch.cat([C,torch.full_like(C,net.ncls)]); W2=torch.cat([W,W]); Mk2=torch.cat([Mk,Mk])
|
| 156 |
+
for k,t in enumerate(ts):
|
| 157 |
+
tt=torch.full((Bn,),int(t),device=dev,dtype=torch.long)
|
| 158 |
+
with torch.autocast('cuda',dtype=torch.bfloat16,enabled=AMP):
|
| 159 |
+
if cfg!=1.0:
|
| 160 |
+
xy,pl=net(torch.cat([z,z]),torch.cat([tt,tt]),C2,W2,Mk2,torch.cat([sc,sc]) if SELFCOND else torch.zeros(2*Bn,Tn,4,device=dev))
|
| 161 |
+
xy=xy[Bn:]+cfg*(xy[:Bn]-xy[Bn:]); pl=pl[Bn:]+cfg*(pl[:Bn]-pl[Bn:])
|
| 162 |
+
else: xy,pl=net(z,tt,C,W,Mk,sc if SELFCOND else torch.zeros_like(z))
|
| 163 |
+
x0=torch.cat([xy[...,:2].float(),torch.tanh(pl.float()/2)[...,None],xy[...,2:3].float()],-1)*m; sc=x0
|
| 164 |
+
if k==len(ts)-1: break
|
| 165 |
+
a=AB[t]; a2=AB[ts[k+1]]; eps=(z-a.sqrt()*x0)/(1-a).sqrt()
|
| 166 |
+
if eta>0: # stochastic DDIM (Song et al. eta): fresh noise at every step -> within-class diversity (eta 0 = deterministic, the x0 of step 1 almost decides the glyph)
|
| 167 |
+
sig=eta*((1-a2)/(1-a)).sqrt()*(1-a/a2).sqrt(); nz=torch.randn(z.shape,generator=gen).to(dev) if gen is not None else torch.randn_like(z)
|
| 168 |
+
z=(a2.sqrt()*x0+(1-a2-sig**2).clamp(min=0).sqrt()*eps+sig*nz)*m
|
| 169 |
+
else: z=(a2.sqrt()*x0+(1-a2).sqrt()*eps)*m
|
| 170 |
+
X=x0.cpu().numpy(); pen=(pl.float()>pen_thr).cpu().numpy(); out=[] # pen_thr: logit threshold; trained with pos_weight PENW -> 0 = calibrated p > 1/(1+PENW), log(PENW) = p > 0.5
|
| 171 |
+
for b in range(Bn):
|
| 172 |
+
n=int(lens[b]); q=np.zeros((n,4),np.float32); q[:,0]=X[b,:n,0]*XS; q[:,1]=X[b,:n,1]*YS; q[:,2]=pen[b,:n]; q[-1,2]=1; q[:,3]=np.clip((X[b,:n,3]+1)/2,0,1); out.append(q)
|
| 173 |
+
net.train(); return out
|
| 174 |
+
def to_strokes(q):
|
| 175 |
+
"""[n,4] x,y,pen_up,pressure -> list of strokes [[x,y,0,p],...] (the glyph_vae.sample_batch format)"""
|
| 176 |
+
st=[];cur=[]
|
| 177 |
+
for k in range(len(q)):
|
| 178 |
+
cur.append([float(q[k,0]),float(q[k,1]),0.0,float(q[k,3])])
|
| 179 |
+
if q[k,2]>0.5 or k==len(q)-1: st.append(cur); cur=[]
|
| 180 |
+
return [s for s in st if s]
|
| 181 |
+
|
| 182 |
+
# ---------------- length plan
|
| 183 |
+
def len_table(tr,c2i,w2i):
|
| 184 |
+
"""lens[(w,c)] = sorted array of train point counts; lens[(-1,c)] = over all writers"""
|
| 185 |
+
T=collections.defaultdict(list)
|
| 186 |
+
for d,lab,src in tr: T[(w2i.get(wr(src),len(w2i)),c2i[lab])].append(len(d)); T[(-1,c2i[lab])].append(len(d))
|
| 187 |
+
return {k:np.array(sorted(v),np.int32) for k,v in T.items()}
|
| 188 |
+
def draw_len(lens,w,c,rng):
|
| 189 |
+
a=lens.get((w,c)); a=a if a is not None and len(a)>=LEN_MIN else lens.get((-1,c))
|
| 190 |
+
if a is None or len(a)==0: return 30
|
| 191 |
+
return int(a[int(rng.integers(len(a)))])
|
| 192 |
+
|
| 193 |
+
# ---------------- eyeball sheet: real | generated, w01
|
| 194 |
+
def cell(st,S=56):
|
| 195 |
+
from PIL import Image, ImageDraw
|
| 196 |
+
im=Image.new('L',(S,S),255)
|
| 197 |
+
if not st or not any(len(s) for s in st): return im
|
| 198 |
+
d=ImageDraw.Draw(im); pts=np.array([p[:2] for s in st for p in s]); x0,y0=pts.min(0); x1,y1=pts.max(0); sc=(S-10)/max(x1-x0,y1-y0,1e-6); ox=(S-(x1-x0)*sc)/2; oy=(S-(y1-y0)*sc)/2
|
| 199 |
+
for s in st:
|
| 200 |
+
q=[((p[0]-x0)*sc+ox,(p[1]-y0)*sc+oy) for p in s]
|
| 201 |
+
if len(q)>1: d.line(q,fill=0,width=2)
|
| 202 |
+
elif q: d.ellipse([q[0][0]-1.5,q[0][1]-1.5,q[0][0]+1.5,q[0][1]+1.5],fill=0)
|
| 203 |
+
return im
|
| 204 |
+
def real_strokes(d):
|
| 205 |
+
P=np.cumsum(d[:,:2],0); q=np.concatenate([P,d[:,2:3],d[:,3:4]],1); return to_strokes(q)
|
| 206 |
+
def sheet(net,data,c2i,w2i,lens,path,classes=None,writer='w01',nreal=6,ngen=6,seed=0,cfg=1.5,steps=10,eta=0.0):
|
| 207 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 208 |
+
classes=classes or [c for c in ('x','2','=','λ','α','∂','ψ','i','±','θ','∫','Q') if c in c2i]; rng=np.random.default_rng(seed); w=w2i.get(writer,len(w2i))
|
| 209 |
+
byc=collections.defaultdict(list)
|
| 210 |
+
for d,lab,src in data:
|
| 211 |
+
if wr(src)==writer and lab in classes: byc[lab].append(d)
|
| 212 |
+
S=56; gap=16; f=ImageFont.truetype(os.path.join(PROJ,'DejaVuSans.ttf'),18); fs=ImageFont.truetype(os.path.join(PROJ,'DejaVuSans.ttf'),12)
|
| 213 |
+
im=Image.new('L',(40+(nreal+ngen)*S+gap,28+len(classes)*(S+4)),255); dr=ImageDraw.Draw(im); dr.text((40,6),f'real ({writer})',fill=0,font=fs); dr.text((40+nreal*S+gap,6),f'glyph denoiser cfg {cfg} steps {steps} eta {eta}',fill=0,font=fs)
|
| 214 |
+
C=[c2i[c] for c in classes for _ in range(ngen)]; W=[w]*len(C); L=[draw_len(lens,w,ci,rng) for ci in C]
|
| 215 |
+
if GSTYLE: W=style_rows([writer+'\t']*len(C),rng) # a random line of the writer per glyph
|
| 216 |
+
gl=sample(net,C,W,L,cfg=cfg,steps=steps,seed=seed,eta=eta)
|
| 217 |
+
for r,c in enumerate(classes):
|
| 218 |
+
y=28+r*(S+4); dr.text((8,y+16),c,fill=0,font=f); R=byc.get(c,[])
|
| 219 |
+
for j,i in enumerate(rng.choice(len(R),min(nreal,len(R)),replace=False) if R else []): im.paste(cell(real_strokes(R[i])),(40+j*S,y))
|
| 220 |
+
for j in range(ngen): im.paste(cell(to_strokes(gl[r*ngen+j])),(40+nreal*S+gap+j*S,y))
|
| 221 |
+
im.save(path); return path
|
| 222 |
+
|
| 223 |
+
def save_meta(classes,writers,lens):
|
| 224 |
+
os.makedirs(os.path.join(CKPT,'fp16'),exist_ok=True); json.dump(classes,open(os.path.join(CKPT,'classes.json'),'w'),ensure_ascii=False); json.dump(writers,open(os.path.join(CKPT,'writers.json'),'w'),ensure_ascii=False)
|
| 225 |
+
np.savez(os.path.join(CKPT,'lens.npz'),keys=np.array([f'{w}|{c}' for (w,c) in lens]),**{f'l{k}':v for k,v in enumerate(lens.values())})
|
| 226 |
+
def cfg_dict(): return dict(DN_D=DN_D,DN_GRU=DN_GRU,DN_PRE=DN_PRE,DN_POST=DN_POST,G_DIM=G_DIM,XS=XS,YS=YS,TD=TD,MAXLEN=MAXLEN,SELFCOND=SELFCOND,GSTYLE=GSTYLE,SEMB_DS=SEMB_DS,SEMB_TAG=SEMB_TAG)
|
| 227 |
+
|
| 228 |
+
# ---------------- training
|
| 229 |
+
def train():
|
| 230 |
+
tr=load_data('train'); va=load_data('valid')
|
| 231 |
+
if E('INK_DS2'): # 06.11 (Лев: «Дообучи на моей руке модель для формул»): a second glyph set mixed in x REP2 (its valid replaces the valid)
|
| 232 |
+
tr2=load_ds('train',E('INK_DS2'),E('META_DS2')); va=load_ds('valid',E('INK_DS2'),E('META_DS2')); tr=tr+tr2*int(E('REP2','20'))
|
| 233 |
+
print(f'mix: {len(tr2)} x {E("REP2","20")} glyphs of {E("INK_DS2")}, valid = its {len(va)}',flush=True)
|
| 234 |
+
ini=torch.load(E('INIT'),map_location='cpu') if E('INIT') else None # 06.11: warm start from a checkpoint (its classes / writers kept)
|
| 235 |
+
if ini: classes=list(ini['classes']); writers=list(ini['writers']); tr=[x for x in tr if x[1] in set(classes)]
|
| 236 |
+
else: classes=sorted(set(l for _,l,_ in tr)); writers=sorted(set(wr(s) for _,_,s in tr))
|
| 237 |
+
c2i={c:i for i,c in enumerate(classes)}; w2i={w:i for i,w in enumerate(writers)}; va=[x for x in va if x[1] in c2i]; lens=len_table(tr,c2i,w2i)
|
| 238 |
+
os.makedirs(SAMP,exist_ok=True); save_meta(classes,writers,lens)
|
| 239 |
+
net=Denoiser(len(classes),len(writers)).to(dev)
|
| 240 |
+
if ini: net.load_state_dict({k:v.float() for k,v in ini['e'].items()}); print('init from',E('INIT'),'step',ini.get('step'),flush=True)
|
| 241 |
+
ema=copy.deepcopy(net).eval()
|
| 242 |
+
for p in ema.parameters(): p.requires_grad_(False)
|
| 243 |
+
opt=torch.optim.AdamW(net.parameters(),LR,weight_decay=WD,betas=(0.9,0.99)); step=0; last=os.path.join(CKPT,'last.pt')
|
| 244 |
+
if os.path.exists(last):
|
| 245 |
+
st=torch.load(last,map_location=dev); net.load_state_dict(st['m']); ema.load_state_dict(st['e']); opt.load_state_dict(st['o']); step=st['step']; print('resumed',step,flush=True)
|
| 246 |
+
npar=sum(p.numel() for p in net.parameters())
|
| 247 |
+
print(f'glyph denoiser [{RUN}] dev={dev} train {len(tr)} valid {len(va)} classes {len(classes)} writers {len(writers)} D {DN_D} GRU {DN_GRU} pre/post {DN_PRE}/{DN_POST} params {npar/1e6:.2f}M steps {STEPS} bs {BS} lr {LR} T {TD} pdrop {PDROP} pwdrop {PWDROP} amp {AMP}',flush=True)
|
| 248 |
+
SEED=int(E('SEED','0')); torch.manual_seed(SEED); rng=np.random.default_rng(SEED+step); bl=[]; t0=time.time(); s0=step; acc={}; nbad=0
|
| 249 |
+
meta=dict(classes=classes,writers=writers,cfg=cfg_dict(),lens={f'{w}|{c}':v.tolist() for (w,c),v in lens.items()})
|
| 250 |
+
while step<STEPS:
|
| 251 |
+
lr=LR*min(1.0,(step+1)/WARM)*(0.5*(1+math.cos(math.pi*min(1.0,step/STEPS))))
|
| 252 |
+
for g in opt.param_groups: g['lr']=lr
|
| 253 |
+
if not bl: bl=buckets(tr,BS,rng)
|
| 254 |
+
z0,Mk,C,W=make_batch([tr[i] for i in bl.pop()],c2i,w2i,aug=True,rng=rng); Bn,Tn=Mk.shape; m=Mk[...,None]
|
| 255 |
+
C=torch.where(torch.rand(Bn,device=dev)<PDROP,torch.full_like(C,len(classes)),C); dw=torch.rand(Bn,device=dev)<PWDROP
|
| 256 |
+
if GSTYLE: W=torch.where(dw[:,None],F.one_hot(torch.full_like(C,SDIM),SDIM+1).float(),W)
|
| 257 |
+
else: W=torch.where(dw,torch.full_like(W,len(writers)),W)
|
| 258 |
+
t=torch.randint(0,TD,(Bn,),device=dev); a=AB[t][:,None,None]; eps=torch.randn_like(z0); zt=(a.sqrt()*z0+(1-a).sqrt()*eps)*m; sc=torch.zeros_like(zt)
|
| 259 |
+
with torch.autocast('cuda',dtype=torch.bfloat16,enabled=AMP):
|
| 260 |
+
if SELFCOND and np.random.rand()<0.5:
|
| 261 |
+
with torch.no_grad(): xy0,pl0=net(zt,t,C,W,Mk,sc); sc=torch.cat([xy0[...,:2].float(),torch.tanh(pl0.float()/2)[...,None],xy0[...,2:3].float()],-1).detach()*m
|
| 262 |
+
xy,pl=net(zt,t,C,W,Mk,sc)
|
| 263 |
+
xy=xy.float(); pl=pl.float(); nm=Mk.sum()
|
| 264 |
+
l_pos=((xy[...,:2]-z0[...,:2]).abs().sum(-1)*Mk).sum()/nm
|
| 265 |
+
dm=Mk[:,1:]*Mk[:,:-1]; sc_=torch.tensor([XS,YS],device=dev); dP=(xy[:,1:,:2]-xy[:,:-1,:2])*sc_*20; dT=(z0[:,1:,:2]-z0[:,:-1,:2])*sc_*20
|
| 266 |
+
l_del=((dP-dT).abs().sum(-1)*dm).sum()/dm.sum().clamp(min=1)
|
| 267 |
+
l_pen=(F.binary_cross_entropy_with_logits(pl,(z0[...,2]>0).float(),reduction='none',pos_weight=torch.tensor(PENW,device=dev))*Mk).sum()/nm
|
| 268 |
+
l_pr=((xy[...,2]-z0[...,3]).abs()*Mk).sum()/nm
|
| 269 |
+
loss=POSW*l_pos+DELW*l_del+l_pen+PRW*l_pr
|
| 270 |
+
if not torch.isfinite(loss):
|
| 271 |
+
nbad+=1; print(f' non-finite loss at step {step} (#{nbad})',flush=True)
|
| 272 |
+
if nbad>=50: print('GLYPHDN ABORT: persistent NaN',flush=True); raise SystemExit(3)
|
| 273 |
+
continue
|
| 274 |
+
opt.zero_grad(set_to_none=True); loss.backward(); gn=torch.nn.utils.clip_grad_norm_(net.parameters(),1.0)
|
| 275 |
+
if not torch.isfinite(gn): opt.zero_grad(set_to_none=True); continue
|
| 276 |
+
opt.step(); step+=1; dec=min(0.999,(1+step)/(10+step))
|
| 277 |
+
with torch.no_grad():
|
| 278 |
+
for pe,pm in zip(ema.parameters(),net.parameters()): pe.mul_(dec).add_(pm.detach(),alpha=1-dec)
|
| 279 |
+
for k_,v_ in (('pos',l_pos),('del',l_del),('pen',l_pen),('pr',l_pr)): acc[k_]=acc.get(k_,0)+float(v_.detach())
|
| 280 |
+
if step%LOG_EVERY==0:
|
| 281 |
+
sp=(step-s0)/(time.time()-t0); msg=' '.join(f'{k_} {v_/LOG_EVERY:.4f}' for k_,v_ in acc.items()); acc={}
|
| 282 |
+
print(f'step {step}/{STEPS} {msg} gn {float(gn):.2f} lr {lr:.2e} {sp:.2f}it/s eta {(STEPS-step)/max(sp,1e-6)/60:.0f}min',flush=True)
|
| 283 |
+
open(os.path.join(CKPT,'loss.csv'),'a').write(f'{step},{msg}\n')
|
| 284 |
+
if step%CK_EVERY==0 or step==STEPS:
|
| 285 |
+
vl=valid_loss(ema,va,c2i,w2i); print(f' valid step {step}: pos {vl[0]:.4f} del {vl[1]:.4f} pen {vl[2]:.4f}',flush=True)
|
| 286 |
+
tmp=last+'.tmp'; torch.save(dict(m=net.state_dict(),e=ema.state_dict(),o=opt.state_dict(),step=step,**meta),tmp); os.replace(tmp,last)
|
| 287 |
+
torch.save(dict(e={k:v.half() for k,v in ema.state_dict().items()},step=step,**meta),os.path.join(CKPT,'fp16',f'step{step:06d}.pt'))
|
| 288 |
+
try: sheet(ema,va+tr[:20000],c2i,w2i,lens,os.path.join(SAMP,f'step{step:06d}.png')); ema.eval()
|
| 289 |
+
except Exception as e: print('sheet failed',repr(e),flush=True)
|
| 290 |
+
print(' ckpt+sheet',step,flush=True)
|
| 291 |
+
print('GLYPHDN DONE',flush=True)
|
| 292 |
+
@torch.no_grad()
|
| 293 |
+
def valid_loss(net,va,c2i,w2i,n=1024):
|
| 294 |
+
net.eval(); rng=np.random.default_rng(1); ids=rng.choice(len(va),min(n,len(va)),replace=False); out=[]
|
| 295 |
+
for b in range(0,len(ids),256):
|
| 296 |
+
z0,Mk,C,W=make_batch([va[i] for i in ids[b:b+256]],c2i,w2i); Bn=Mk.size(0); m=Mk[...,None]
|
| 297 |
+
t=torch.full((Bn,),TD//4,device=dev,dtype=torch.long); a=AB[t][:,None,None]; zt=(a.sqrt()*z0+(1-a).sqrt()*torch.randn_like(z0))*m
|
| 298 |
+
with torch.autocast('cuda',dtype=torch.bfloat16,enabled=AMP): xy,pl=net(zt,t,C,W,Mk,torch.zeros_like(zt))
|
| 299 |
+
xy=xy.float(); pl=pl.float(); nm=Mk.sum(); sc_=torch.tensor([XS,YS],device=dev); dm=Mk[:,1:]*Mk[:,:-1]
|
| 300 |
+
out.append((float(((xy[...,:2]-z0[...,:2]).abs().sum(-1)*Mk).sum()/nm),float((((xy[:,1:,:2]-xy[:,:-1,:2])*sc_*20-(z0[:,1:,:2]-z0[:,:-1,:2])*sc_*20).abs().sum(-1)*dm).sum()/dm.sum()),float((F.binary_cross_entropy_with_logits(pl,(z0[...,2]>0).float(),reduction='none')*Mk).sum()/nm)))
|
| 301 |
+
net.train(); return np.mean(out,0)
|
| 302 |
+
|
| 303 |
+
def load_net(path):
|
| 304 |
+
st=torch.load(path,map_location=dev); cfg=st['cfg']
|
| 305 |
+
for k in ('DN_D','DN_GRU','DN_PRE','DN_POST','G_DIM','XS','YS','TD','MAXLEN','SELFCOND'): globals()[k]=cfg[k]
|
| 306 |
+
for k in ('GSTYLE','SEMB_DS','SEMB_TAG'): globals()[k]=cfg.get(k,'') # old checkpoints: writer ids
|
| 307 |
+
globals()['AB']=torch.tensor(alpha_bar(),device=dev)
|
| 308 |
+
net=Denoiser(len(st['classes']),len(st['writers'])).to(dev); net.load_state_dict({k:v.float() for k,v in st['e'].items()}); net.eval()
|
| 309 |
+
lens={(int(k.split('|')[0]),int(k.split('|')[1])):np.array(v,np.int32) for k,v in st['lens'].items()}
|
| 310 |
+
return net,st['classes'],st['writers'],lens,st.get('step')
|
| 311 |
+
|
| 312 |
+
if __name__=='__main__':
|
| 313 |
+
MODE=E('MODE','train')
|
| 314 |
+
if MODE=='train': train()
|
| 315 |
+
elif MODE=='sheet':
|
| 316 |
+
net,classes,writers,lens,step=load_net(E('CKPT')); c2i={c:i for i,c in enumerate(classes)}; w2i={w:i for i,w in enumerate(writers)}; data=load_data('valid')+load_data('train')
|
| 317 |
+
cl=E('CLASSES','').split(',') if E('CLASSES') else None
|
| 318 |
+
print('sheet',sheet(net,data,c2i,w2i,lens,E('OUT',os.path.join(PROJ,'figures',f'glyph_dn_{RUN}_sheet.png')),classes=cl,writer=E('WRITER','w01'),nreal=int(E('N','6')),ngen=int(E('N','6')),seed=int(E('SEED','0')),cfg=float(E('GDN_CFG','1.5')),steps=int(E('GDN_STEPS','10')),eta=float(E('GDN_ETA','0'))),flush=True)
|
scripts/train_ink.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""B2: online handwriting synthesis for formulas (Graves 2013 style, modernized).
|
| 3 |
+
Input: LaTeX tokens (from mw_ink_prep). Output: pen trajectory (dx,dy,pen_up) autoregressively.
|
| 4 |
+
Model: token encoder (embedding + 2-layer transformer, bidirectional) -> decoder = 3 stacked LSTMs; a Gaussian
|
| 5 |
+
MIXTURE ATTENTION WINDOW (K components, monotonic kappa) after LSTM-1 reads the token sequence; output head =
|
| 6 |
+
MDN (M bivariate gaussians over (dx,dy)) + Bernoulli pen_up + Bernoulli EOS. Teacher forcing, NLL loss.
|
| 7 |
+
Sampling: ancestral with probability bias; renders strokes to PNG with a round pen for eyeballing.
|
| 8 |
+
Data: data/mw_ink/{train,valid}_raw.npz + vocab.json. Env: PROJ, STEPS, BS, LR, HID, K, M, MAXPTS, HW_RUN, INIT_FROM."""
|
| 9 |
+
import os, sys, json, math, time, glob, copy, numpy as np, torch, torch.nn as nn, torch.nn.functional as F
|
| 10 |
+
PROJ=os.environ['PROJ']; D=os.path.join(PROJ,'data',os.environ.get('INK_DS','mw_ink')); RUN=os.environ.get('HW_RUN','ink1'); CKPT=os.path.join(PROJ,'ckpt_'+RUN); SAMP=os.path.join(PROJ,'samples_'+RUN)
|
| 11 |
+
os.makedirs(os.path.join(CKPT,'fp16'),exist_ok=True); os.makedirs(SAMP,exist_ok=True)
|
| 12 |
+
dev='cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu')
|
| 13 |
+
STEPS=int(os.environ.get('STEPS','60000')); BS=int(os.environ.get('BS','64')); LR=float(os.environ.get('LR','5e-4'))
|
| 14 |
+
WFB=int(os.environ.get('WFB','0')) # 17.10: 1 = exact Graves: LSTM-1 input is [x_t, w_{t-1}] (sequential loop); 0 = old parallel approximation where LSTM-1 never sees the text (cursive text alignment failed: lines1/lines2)
|
| 15 |
+
HID=int(os.environ.get('HID','400')); K=int(os.environ.get('K','10')); M=int(os.environ.get('M','20')); MAXPTS=int(os.environ.get('MAXPTS','600')); MAXTOK=int(os.environ.get('MAXTOK','48')); SCALE=float(os.environ.get('SCALE','50.0')); STOP_MARGIN=float(os.environ.get('STOP_MARGIN','1.5')); KSTEP=float(os.environ.get('KSTEP','0.1')); PENW=float(os.environ.get('PENW','5.0')); LAMK=float(os.environ.get('LAMK','0.1')); PTS_PER_TOK=float(os.environ.get('PTS_PER_TOK','60'))
|
| 16 |
+
itos=json.load(open(os.path.join(D,'vocab.json'))); stoi={s:i for i,s in enumerate(itos)}; V=len(itos)
|
| 17 |
+
PRESW=float(os.environ.get('PRESW','1.0')); PRES=os.environ.get('PRES','0')=='1' # 20.10: pen pressure as an extra target/output (npz 'pres' from ink_lines_prep)
|
| 18 |
+
class Inks:
|
| 19 |
+
def __init__(s,split):
|
| 20 |
+
z=np.load(os.path.join(D,f'{split}_raw.npz'),allow_pickle=True); s.pts=z['pts']; s.off=z['pt_off']; s.toks=[json.loads(t) for t in z['toks']]; s.labels=list(z['labels']); s.pres=z['pres'] if (PRES and 'pres' in z) else None
|
| 21 |
+
s.idx=[i for i in range(len(s.toks)) if (s.off[i+1]-s.off[i])<=MAXPTS and len(s.toks[i])<=MAXTOK]
|
| 22 |
+
print(f"{split}: {len(s.idx)}/{len(s.toks)} usable",flush=True)
|
| 23 |
+
def __len__(s): return len(s.idx)
|
| 24 |
+
def get(s,i):
|
| 25 |
+
j=s.idx[i]; d=s.pts[s.off[j]:s.off[j+1]]; t=[1]+[stoi.get(x,0) for x in s.toks[j]]+[2]; return d,t
|
| 26 |
+
def batch(s,ids):
|
| 27 |
+
ds,ts=zip(*[s.get(i) for i in ids]); Tn=max(len(d) for d in ds)+1; Un=max(len(t) for t in ts)
|
| 28 |
+
X=np.zeros((len(ids),Tn,3),np.float32); Y=np.zeros((len(ids),Tn,5 if PRES else 4),np.float32); Mk=np.zeros((len(ids),Tn),np.float32); Tk=np.zeros((len(ids),Un),np.int64); Tm=np.zeros((len(ids),Un),np.float32)
|
| 29 |
+
for b,(d,t) in enumerate(zip(ds,ts)):
|
| 30 |
+
n=len(d); X[b,1:n+1]=d; Y[b,:n,:3]=d; Y[b,n-1,3]=1; Mk[b,:n]=1 # input = shifted (start token = zeros), target = next point; EOS at last point
|
| 31 |
+
if PRES and s.pres is not None:
|
| 32 |
+
j=s.idx[ids[b]]; pv=s.pres[s.off[j]:s.off[j+1]]
|
| 33 |
+
# 20.10 (w03: data without pressure must not break the model): a line whose pressure is constant has NO real
|
| 34 |
+
# pressure (GoodNotes/Freenotes: 1.0; old sources: 0.5) -> target -1 = masked out of the pressure loss
|
| 35 |
+
Y[b,:n,4]=pv if float(np.ptp(pv[:n]))>1e-3 else -1.0
|
| 36 |
+
Tk[b,:len(t)]=t; Tm[b,:len(t)]=1
|
| 37 |
+
X[...,:2]*=SCALE; Y[...,:2]*=SCALE # deltas ~0.08 -> ~4: sane MDN gradients (overfit test 14.10)
|
| 38 |
+
return [torch.from_numpy(a).to(dev) for a in (X,Y,Mk,Tk,Tm)]
|
| 39 |
+
class Model(nn.Module):
|
| 40 |
+
def __init__(s):
|
| 41 |
+
super().__init__(); s.emb=nn.Embedding(V,256); s.tpos=nn.Parameter(torch.randn(MAXTOK+2,256)*0.02)
|
| 42 |
+
s.enc=nn.TransformerEncoder(nn.TransformerEncoderLayer(256,4,512,batch_first=True,norm_first=True),2)
|
| 43 |
+
if WFB: s.l1c=nn.LSTMCell(3+256,HID) # sees the previous window (Graves eq. 46-52)
|
| 44 |
+
else: s.l1=nn.LSTM(3,HID,batch_first=True)
|
| 45 |
+
s.win=nn.Linear(HID,3*K)
|
| 46 |
+
s.l2=nn.LSTM(3+256+HID,HID,batch_first=True); s.l3=nn.LSTM(3+256+HID,HID,batch_first=True)
|
| 47 |
+
s.out=nn.Linear(3*HID,6*M+(3 if PRES else 2))
|
| 48 |
+
def encode(s,tk,tm):
|
| 49 |
+
h=s.emb(tk)+s.tpos[:tk.size(1)][None]; return s.enc(h,src_key_padding_mask=(tm<0.5))
|
| 50 |
+
def window(s,h1,ctx,tm,kappa_prev):
|
| 51 |
+
"""h1 [B,T,HID] -> per-step window vectors [B,T,256] with monotone kappa (sequential over T)."""
|
| 52 |
+
B,T,_=h1.shape; U=ctx.size(1); u=torch.arange(U,device=h1.device).float()[None,None,:]
|
| 53 |
+
p=s.win(h1).view(B,T,3,K); alpha=p[:,:,0].exp(); beta=p[:,:,1].exp(); dk=p[:,:,2].exp() # [B,T,K]
|
| 54 |
+
Umax=tm.sum(1)[:,None,None]
|
| 55 |
+
kappa=torch.minimum(kappa_prev[:,None,:]+torch.cumsum(dk*KSTEP,1), Umax) # [B,T,K]; window cannot run past the text
|
| 56 |
+
phi=(alpha[...,None]*torch.exp(-beta[...,None]*(kappa[...,None]-u[:,:,None,:])**2)).sum(2) # [B,T,U]
|
| 57 |
+
phi=phi*tm[:,None,:]; w=phi@ctx # [B,T,256]
|
| 58 |
+
return w,kappa[:,-1],phi
|
| 59 |
+
def forward(s,X,tk,tm,state=None):
|
| 60 |
+
ctx=s.encode(tk,tm); B=X.size(0)
|
| 61 |
+
if state is None: st1=st2=st3=None; kap=torch.zeros(B,K,device=X.device); w0=torch.zeros(B,1,256,device=X.device)
|
| 62 |
+
else: st1,st2,st3,kap,w0=state
|
| 63 |
+
if WFB: # exact Graves: step t of LSTM-1 reads [x_t, w_{t-1}]; window from h1_t; sequential over T
|
| 64 |
+
T=X.size(1); U=ctx.size(1); u=torch.arange(U,device=X.device).float()[None,:]; Umax=tm.sum(1)[:,None]
|
| 65 |
+
if st1 is None: st1=(torch.zeros(B,HID,device=X.device),torch.zeros(B,HID,device=X.device))
|
| 66 |
+
h,c=st1; w=w0[:,0]; hs=[]; ws=[]; phis=[]
|
| 67 |
+
for t in range(T):
|
| 68 |
+
h,c=s.l1c(torch.cat([X[:,t],w],-1),(h,c))
|
| 69 |
+
p=s.win(h).view(B,3,K).clamp(-8,8); alpha=p[:,0].exp(); beta=p[:,1].exp(); dk=p[:,2].exp() # clamp: wfb1 17.10 got inf*0=NaN grads at step 0 (win from a non-WFB ckpt on a fresh LSTMCell)
|
| 70 |
+
kap=torch.minimum(kap+dk*KSTEP,Umax)
|
| 71 |
+
phi=(alpha[:,:,None]*torch.exp(-beta[:,:,None]*(kap[:,:,None]-u[:,None,:])**2)).sum(1)*tm # [B,U]
|
| 72 |
+
w=torch.bmm(phi[:,None,:],ctx)[:,0]; hs.append(h); ws.append(w); phis.append(phi)
|
| 73 |
+
h1=torch.stack(hs,1); w=torch.stack(ws,1); phi=torch.stack(phis,1); st1=(h,c)
|
| 74 |
+
else:
|
| 75 |
+
# old approximation: LSTM-1 sees strokes only; window recomputed from h1 in parallel
|
| 76 |
+
h1,st1=s.l1(X,st1)
|
| 77 |
+
w,kap,phi=s.window(h1,ctx,tm,kap)
|
| 78 |
+
h2,st2=s.l2(torch.cat([X,w,h1],-1),st2); h3,st3=s.l3(torch.cat([X,w,h2],-1),st3)
|
| 79 |
+
o=s.out(torch.cat([h1,h2,h3],-1)); return o,(st1,st2,st3,kap,w[:,-1:]),phi
|
| 80 |
+
def mdn_nll(o,Y,Mk):
|
| 81 |
+
pi=F.log_softmax(o[...,:M],-1); mu=o[...,M:3*M].view(*o.shape[:2],M,2); ls=o[...,3*M:5*M].view(*o.shape[:2],M,2).clamp(-7,5); rho=torch.tanh(o[...,5*M:6*M]).clamp(-0.98,0.98) # |rho|->1 made log1p(-rho^2) -> -inf -> NaN (ink1 run)
|
| 82 |
+
pen=o[...,6*M]; eos=o[...,6*M+1]; x=Y[...,None,:2]; z=(x-mu)/ls.exp(); zx,zy=z[...,0],z[...,1]
|
| 83 |
+
Z=zx**2+zy**2-2*rho*zx*zy; lp=-Z/(2*(1-rho**2))-ls.sum(-1)-math.log(2*math.pi)-0.5*torch.log1p(-rho**2)
|
| 84 |
+
nll=-(torch.logsumexp(pi+lp,-1))+F.binary_cross_entropy_with_logits(pen,Y[...,2],reduction='none',pos_weight=torch.tensor(PENW,device=pen.device))+F.binary_cross_entropy_with_logits(eos,Y[...,3],reduction='none',pos_weight=torch.tensor(PENW,device=pen.device))
|
| 85 |
+
if PRES and Y.size(-1)>4: nll=nll+PRESW*F.binary_cross_entropy_with_logits(o[...,6*M+2],Y[...,4].clamp(0,1),reduction='none')*(Y[...,4]>=0).float()
|
| 86 |
+
return (nll*Mk).sum()/Mk.sum()
|
| 87 |
+
@torch.no_grad()
|
| 88 |
+
def sample(model,toks_list,bias=1.0,maxlen=700):
|
| 89 |
+
model.eval(); B=len(toks_list); Un=max(len(t) for t in toks_list)+2
|
| 90 |
+
Tk=torch.zeros(B,Un,dtype=torch.long,device=dev); Tm=torch.zeros(B,Un,device=dev)
|
| 91 |
+
for b,t in enumerate(toks_list): ids=[1]+[stoi.get(x,0) for x in t]+[2]; Tk[b,:len(ids)]=torch.tensor(ids); Tm[b,:len(ids)]=1
|
| 92 |
+
x=torch.zeros(B,1,3,device=dev); state=None; out=[]; done=torch.zeros(B,dtype=torch.bool,device=dev); stop_idx=torch.zeros(B,dtype=torch.long,device=dev)
|
| 93 |
+
for step in range(maxlen):
|
| 94 |
+
o,state,phi=model(x,Tk,Tm,state); o=o[:,-1]
|
| 95 |
+
pi=F.softmax(o[:,:M]*(1+bias),-1); mu=o[:,M:3*M].view(B,M,2); ls=(o[:,3*M:5*M].view(B,M,2).clamp(-7,5)-bias); rho=torch.tanh(o[:,5*M:6*M])
|
| 96 |
+
k=torch.multinomial(pi,1)[:,0]; m=mu[torch.arange(B),k]; sd=ls[torch.arange(B),k].exp(); r=rho[torch.arange(B),k]
|
| 97 |
+
e1=torch.randn(B,device=dev); e2=torch.randn(B,device=dev); dx=m[:,0]+sd[:,0]*e1; dy=m[:,1]+sd[:,1]*(r*e1+torch.sqrt(1-r**2)*e2)
|
| 98 |
+
pen=(torch.sigmoid(o[:,6*M])>0.5).float(); eos=torch.sigmoid(o[:,6*M+1])>0.5
|
| 99 |
+
U_=Tm.sum(1); kap=state[3].mean(1); eos=eos|((kap>=U_-STOP_MARGIN)&(pen>0.5)) # Graves-style stop: window past the last token and pen up
|
| 100 |
+
eos=eos|(torch.full_like(U_,float(step+1))>=(U_-2).clamp(min=1)*PTS_PER_TOK) # runaway guard: hard cap on points per token
|
| 101 |
+
x=torch.stack([dx,dy,pen],-1)[:,None]; out.append(x[:,0].clone())
|
| 102 |
+
newly=eos&~done; stop_idx[newly]=step+1; done=done|eos
|
| 103 |
+
if done.all(): break
|
| 104 |
+
model.train(); seq=torch.stack(out,1).cpu().numpy(); seq[:,:,:2]/=SCALE; si=stop_idx.cpu().numpy()
|
| 105 |
+
return [seq[b,:int(si[b]) if si[b]>0 else seq.shape[1]] for b in range(B)] # list of per-sample sequences truncated at EOS/stop
|
| 106 |
+
def render(seq,S=None,pen_w=None,aa=None):
|
| 107 |
+
"""Stroke render. Defaults from env RENDER_S (px per unit, 48), RENDER_PEN (2), RENDER_AA (supersampling, 1):
|
| 108 |
+
w03 16–17.10 asked for HIGH-RES stroke pictures (they look great) -> for anything shown to him use RENDER_S=160
|
| 109 |
+
RENDER_PEN=6 RENDER_AA=2; judges keep the small defaults (CRNN input size)."""
|
| 110 |
+
from PIL import Image, ImageDraw
|
| 111 |
+
S=S or int(os.environ.get('RENDER_S','48')); pen_w=pen_w or int(os.environ.get('RENDER_PEN','2')); aa=aa or int(os.environ.get('RENDER_AA','1'))
|
| 112 |
+
Sa=S*aa; pw=pen_w*aa; m=10*aa
|
| 113 |
+
xy=np.cumsum(seq[:,:2],0); pen=seq[:,2]; x0,y0=xy[:,0].min(),xy[:,1].min(); W=int((xy[:,0].max()-x0)*Sa)+2*m; H=int((xy[:,1].max()-y0)*Sa)+2*m
|
| 114 |
+
im=Image.new('L',(max(W,40*aa),max(H,40*aa)),255); dr=ImageDraw.Draw(im); prev=None
|
| 115 |
+
DOTS=os.environ.get('RENDER_DOTS','0')=='1'
|
| 116 |
+
for k,((x,y),p) in enumerate(zip(xy,pen)):
|
| 117 |
+
q=(m+(x-x0)*Sa,m+(y-y0)*Sa)
|
| 118 |
+
if prev is not None:
|
| 119 |
+
dr.line([prev,q],fill=0,width=pw)
|
| 120 |
+
if pw>2: dr.ellipse([q[0]-pw/2,q[1]-pw/2,q[0]+pw/2,q[1]+pw/2],fill=0) # round joints at high res
|
| 121 |
+
# RENDER_DOTS=1 (21.10): a one-point stroke (a tapped dot, comma or the dot of «!») is drawn as a pen-width dot; off by default so the
|
| 122 |
+
# readability judge stays comparable with earlier runs (PDF/sdocx output already pads such strokes, write_page.py)
|
| 123 |
+
if DOTS and p>0.5 and prev is None and (k==0 or pen[k-1]>0.5): dr.ellipse([q[0]-pw/2,q[1]-pw/2,q[0]+pw/2,q[1]+pw/2],fill=0)
|
| 124 |
+
prev=None if p>0.5 else q
|
| 125 |
+
if aa>1: im=im.resize((im.size[0]//aa,im.size[1]//aa),Image.LANCZOS)
|
| 126 |
+
return im
|
| 127 |
+
def demo(model,va,path,n=8):
|
| 128 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 129 |
+
ids=list(range(0,len(va),max(1,len(va)//n)))[:n]; toks=[va.get(i)[1] for i in ids]; tl=[[itos[t] for t in tk[1:-1]] for tk in toks]
|
| 130 |
+
seqs=sample(model,tl,bias=1.0); ims=[]
|
| 131 |
+
f=ImageFont.truetype(os.path.join(PROJ,'DejaVuSans.ttf'),12)
|
| 132 |
+
for i,sq in zip(ids,seqs):
|
| 133 |
+
real=render(va.get(i)[0]); gen=render(sq)
|
| 134 |
+
c=Image.new('L',(max(real.width,gen.width,300),real.height+gen.height+18),255); c.paste(real,(0,0)); c.paste(gen,(0,real.height)); ImageDraw.Draw(c).text((2,real.height+gen.height+2),''.join(va.labels[va.idx[i]])[:70],fill=0,font=f); ims.append(c)
|
| 135 |
+
W=max(c.width for c in ims); sheet=Image.new('L',(W,sum(c.height for c in ims)+8*len(ims)),255); y=0
|
| 136 |
+
for c in ims: sheet.paste(c,(0,y)); y+=c.height+8
|
| 137 |
+
sheet.save(path)
|
| 138 |
+
def main():
|
| 139 |
+
tr=Inks('train'); va=Inks('valid'); model=Model().to(dev); ema=copy.deepcopy(model).eval()
|
| 140 |
+
for p in ema.parameters(): p.requires_grad_(False)
|
| 141 |
+
opt=torch.optim.Adam(model.parameters(),LR); step=0; ck=sorted(glob.glob(os.path.join(CKPT,'step*.pt')))
|
| 142 |
+
if ck: st=torch.load(ck[-1],map_location=dev); model.load_state_dict(st['m']); ema.load_state_dict(st['e']); opt.load_state_dict(st['o']); step=st['step']; print('resumed',step,flush=True)
|
| 143 |
+
elif os.environ.get('INIT_FROM'):
|
| 144 |
+
st=torch.load(os.environ['INIT_FROM'],map_location=dev); src=st.get('m',st.get('e',st)); src={k:v.float() for k,v in src.items()}
|
| 145 |
+
if WFB and not any(k.startswith('l1c.') for k in src): src={k:v for k,v in src.items() if not k.startswith('win.')} # window head trained on the OLD LSTM-1 + fresh LSTMCell = huge alpha -> exploding recurrent grads (wfb1 NaN, 18.10)
|
| 146 |
+
tgt=model.state_dict(); ok={k:v for k,v in src.items() if k in tgt and tgt[k].shape==v.shape}; skip=[k for k in src if k not in ok]
|
| 147 |
+
for k in list(skip): # vocab grew (joint set): copy the overlapping embedding rows, keep new rows random
|
| 148 |
+
if k in tgt and src[k].dim()==2 and tgt[k].dim()==2 and src[k].shape[1]==tgt[k].shape[1]:
|
| 149 |
+
n=min(src[k].shape[0],tgt[k].shape[0]); w=tgt[k].clone(); w[:n]=src[k][:n]; ok[k]=w; skip.remove(k); print(f' partial copy {k}: {n} rows',flush=True)
|
| 150 |
+
model.load_state_dict(ok,strict=False); ema.load_state_dict({k:v.clone() for k,v in ok.items()},strict=False)
|
| 151 |
+
print(f'warm-start from {os.path.basename(os.environ["INIT_FROM"])}: loaded {len(ok)} tensors, skipped {len(skip)} (shape/vocab mismatch): {skip[:6]}',flush=True)
|
| 152 |
+
print(f"ink[{RUN}] dev={dev} V={V} HID={HID} K={K} M={M} BS={BS} params {sum(p.numel() for p in model.parameters())/1e6:.1f}M target {STEPS}",flush=True)
|
| 153 |
+
rng=np.random.default_rng(0); t0=time.time()
|
| 154 |
+
while step<STEPS:
|
| 155 |
+
lr=LR*min(1.0,(step+1)/500)*(0.5*(1+math.cos(math.pi*min(1.0,step/STEPS))))
|
| 156 |
+
for g in opt.param_groups: g['lr']=lr
|
| 157 |
+
X,Y,Mk,Tk,Tm=tr.batch(rng.integers(0,len(tr),BS))
|
| 158 |
+
o,(_,_,_,kap_end,_),phi=model(X,Tk,Tm); loss=mdn_nll(o,Y,Mk)
|
| 159 |
+
U_=Tm.sum(1); loss=loss+LAMK*((kap_end.mean(1)/U_.clamp(min=1)-1.0)**2).mean() # alignment: window should END at the last token
|
| 160 |
+
if not torch.isfinite(loss):
|
| 161 |
+
nbad=globals().get('_nbad',0)+1; globals()['_nbad']=nbad; print(f' non-finite loss at step {step} (#{nbad}) -> skipped',flush=True)
|
| 162 |
+
if nbad>=50: print('INK ABORT: persistent NaN',flush=True); raise SystemExit(3)
|
| 163 |
+
continue
|
| 164 |
+
opt.zero_grad(); loss.backward(); gn=torch.nn.utils.clip_grad_norm_(model.parameters(),5.0)
|
| 165 |
+
if not torch.isfinite(gn):
|
| 166 |
+
nbad=globals().get('_nbad',0)+1; globals()['_nbad']=nbad; print(f' non-finite grad at step {step} (#{nbad}) -> skipped',flush=True); opt.zero_grad()
|
| 167 |
+
if nbad>=50: print('INK ABORT: persistent NaN grad',flush=True); raise SystemExit(3) # wfb1 17.10 looped forever on NaN grads at step 0
|
| 168 |
+
continue
|
| 169 |
+
opt.step(); step+=1
|
| 170 |
+
with torch.no_grad():
|
| 171 |
+
for pe,pm in zip(ema.parameters(),model.parameters()): pe.mul_(0.999).add_(pm,alpha=0.001)
|
| 172 |
+
if step%100==0:
|
| 173 |
+
sp=step/(time.time()-t0)
|
| 174 |
+
with torch.no_grad():
|
| 175 |
+
U=Tm.sum(1); kr=(kap_end.mean(1)/U.clamp(min=1)).mean().item() # ~1.0 when the window ends at the last token
|
| 176 |
+
pen_acc=(((o[...,6*M]>0)==(Y[...,2]>0.5)).float()*Mk).sum()/Mk.sum(); pu=((Y[...,2]>0.5).float()*Mk).sum()/Mk.sum()
|
| 177 |
+
print(f"step {step}/{STEPS} nll {loss.item():.3f} kappa_end/U {kr:.2f} pen_acc {pen_acc.item():.3f} (penup {pu.item():.3f}) lr {lr:.2e} {sp:.2f}it/s eta {(STEPS-step)/sp/60:.0f}min",flush=True)
|
| 178 |
+
open(os.path.join(CKPT,'loss.csv'),'a').write(f"{step},{loss.item():.4f}\n")
|
| 179 |
+
if step%2000==0 or step==STEPS:
|
| 180 |
+
torch.save(dict(m=model.state_dict(),e=ema.state_dict(),o=opt.state_dict(),step=step),os.path.join(CKPT,f'step{step:06d}.pt'))
|
| 181 |
+
torch.save(dict(e={k:v.half() for k,v in ema.state_dict().items()},step=step,itos=itos),os.path.join(CKPT,'fp16',f'step{step:06d}_fp16.pt'))
|
| 182 |
+
try: demo(ema,va,os.path.join(SAMP,f'step{step:06d}.png'))
|
| 183 |
+
except Exception as e: print('demo failed',e,flush=True)
|
| 184 |
+
print(' ckpt+demo',step,flush=True)
|
| 185 |
+
print("INK DONE",flush=True)
|
| 186 |
+
if __name__=='__main__': main()
|
scripts/train_ink_al.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""ALIGNED LSTM trajectory model: train_ink's 3-layer LSTM + MDN (which fits stroke dynamics well, nll -3) with the
|
| 3 |
+
Graves attention window REPLACED by the CTC forced alignment (scripts/ink_ctc_align.py): every input point carries the
|
| 4 |
+
embedding of its character and phase (FastSpeech-style), so nothing has to be aligned in free running. A duration head
|
| 5 |
+
(log1p points per char) on the text encoder provides the plan at sampling. cuDNN LSTMs -> fast (parallel over T).
|
| 6 |
+
Env: as train_ink (PROJ, INK_DS, HW_RUN, STEPS, BS, LR, HID, M, MAXPTS, MAXTOK, SCALE, PENW) + ALIGN (align_text.npz),
|
| 7 |
+
NPH (8), DURW (0.1), DUR_SCALE (1.0). MODEL_PY=train_ink_al for judge/demo."""
|
| 8 |
+
import os, sys, json, math, time, glob, copy, numpy as np, torch, torch.nn as nn, torch.nn.functional as F
|
| 9 |
+
sys.path.insert(0,os.path.dirname(os.path.abspath(__file__)))
|
| 10 |
+
os.environ.setdefault('HW_RUN','lstmal1'); os.environ.setdefault('INK_DS','ink_lines5s'); os.environ.setdefault('MAXPTS','1200'); os.environ.setdefault('MAXTOK','200')
|
| 11 |
+
import train_ink as B
|
| 12 |
+
from train_ink import * # noqa: Inks(base), mdn_nll, render, itos, stoi, V, HID, M, SCALE, PENW, PTS_PER_TOK, dev, D, PROJ, RUN, CKPT, SAMP, STEPS, BS, LR, MAXPTS, MAXTOK
|
| 13 |
+
E=os.environ.get; ALIGN=os.path.join(D,E('ALIGN','align_text.npz')); NPH=int(E('NPH','8')); DURW=float(E('DURW','0.1')); DUR_SCALE=float(E('DUR_SCALE','1.0'))
|
| 14 |
+
class Inks(B.Inks):
|
| 15 |
+
def __init__(s,split):
|
| 16 |
+
super().__init__(split); z=np.load(ALIGN,allow_pickle=True); s.al={int(i):a for i,sp,a in zip(z['ids'],z['split'],z['char_idx']) if sp==split}
|
| 17 |
+
s.idx=[i for i in s.idx if i in s.al]; print(f"{split}: {len(s.idx)} aligned lines",flush=True)
|
| 18 |
+
def batch(s,ids):
|
| 19 |
+
X,Y,Mk,Tk,Tm=super().batch(ids); Bn,Tn=Mk.shape; C=np.zeros((Bn,Tn),np.int64); Ph=np.zeros((Bn,Tn),np.int64); Cn=np.zeros((Bn,Tn),np.int64); Du=np.zeros((Bn,Tk.size(1)),np.float32); Dm=np.zeros((Bn,Tk.size(1)),np.float32)
|
| 20 |
+
for b,i in enumerate(ids):
|
| 21 |
+
j=s.idx[i]; ci=s.al[j].astype(np.int64); n=len(ci); toks=s.toks[j]; tid=np.array([stoi.get(t,0) for t in toks]+[0])
|
| 22 |
+
cnt=np.bincount(ci,minlength=len(toks)); start=np.concatenate([[0],np.cumsum(cnt)[:-1]])
|
| 23 |
+
pos=np.arange(n)-start[ci]; ph=np.minimum(NPH-1,(pos*NPH)//np.maximum(cnt[ci],1))
|
| 24 |
+
C[b,:n]=tid[ci]; Cn[b,:n]=tid[np.minimum(ci+1,len(toks))]; Ph[b,:n]=ph; Du[b,1:1+len(toks)]=np.log1p(cnt); Dm[b,1:1+len(toks)]=1 # next char vectorised (the per-point Python loop took minutes per batch, 18.10)
|
| 25 |
+
return X,Y,Mk,Tk,Tm,torch.from_numpy(C).to(dev),torch.from_numpy(Ph).to(dev),torch.from_numpy(Cn).to(dev),torch.from_numpy(Du).to(dev),torch.from_numpy(Dm).to(dev)
|
| 26 |
+
class Model(nn.Module):
|
| 27 |
+
def __init__(s):
|
| 28 |
+
super().__init__(); s.emb=nn.Embedding(V,256); s.tpos=nn.Parameter(torch.randn(MAXTOK+2,256)*0.02)
|
| 29 |
+
s.enc=nn.TransformerEncoder(nn.TransformerEncoderLayer(256,4,512,batch_first=True,norm_first=True),2,enable_nested_tensor=False)
|
| 30 |
+
s.dur=nn.Sequential(nn.LayerNorm(256),nn.Linear(256,256),nn.GELU(),nn.Linear(256,1))
|
| 31 |
+
s.ec=nn.Embedding(V,256); s.eph=nn.Embedding(NPH,256); s.enext=nn.Embedding(V,256) # current char, phase, NEXT char (connections)
|
| 32 |
+
s.l1=nn.LSTM(3+256,HID,batch_first=True); s.l2=nn.LSTM(3+256+HID,HID,batch_first=True); s.l3=nn.LSTM(3+256+HID,HID,batch_first=True); s.out=nn.Linear(3*HID,6*M+(3 if PRES else 2))
|
| 33 |
+
def encode(s,tk,tm): h=s.emb(tk)+s.tpos[:tk.size(1)][None]; return s.enc(h,src_key_padding_mask=(tm<0.5))
|
| 34 |
+
def cond(s,C,Ph,Cn): return s.ec(C)+s.eph(Ph)+s.enext(Cn)
|
| 35 |
+
def forward(s,X,C,Ph,Cn,state=None):
|
| 36 |
+
w=s.cond(C,Ph,Cn); st1,st2,st3=state if state is not None else (None,None,None)
|
| 37 |
+
h1,st1=s.l1(torch.cat([X,w],-1),st1); h2,st2=s.l2(torch.cat([X,w,h1],-1),st2); h3,st3=s.l3(torch.cat([X,w,h2],-1),st3)
|
| 38 |
+
return s.out(torch.cat([h1,h2,h3],-1)),(st1,st2,st3)
|
| 39 |
+
def next_char(C):
|
| 40 |
+
"""per-point id of the NEXT character (0 at the end)."""
|
| 41 |
+
Cn=torch.zeros_like(C); Bn,Tn=C.shape
|
| 42 |
+
for b in range(Bn):
|
| 43 |
+
c=C[b].tolist(); nxt=0
|
| 44 |
+
for t in range(Tn-1,-1,-1):
|
| 45 |
+
if t<Tn-1 and c[t+1]!=c[t]: nxt=c[t+1]
|
| 46 |
+
Cn[b,t]=nxt
|
| 47 |
+
return Cn
|
| 48 |
+
def expand(tok_ids,durs):
|
| 49 |
+
C=[];Ph=[];Cn=[]
|
| 50 |
+
for k,(t,d) in enumerate(zip(tok_ids,durs)):
|
| 51 |
+
d=max(1,int(d)); nx=tok_ids[k+1] if k+1<len(tok_ids) else 0
|
| 52 |
+
for q in range(d): C.append(t); Ph.append(min(NPH-1,(q*NPH)//d)); Cn.append(nx)
|
| 53 |
+
return C,Ph,Cn
|
| 54 |
+
@torch.no_grad()
|
| 55 |
+
def sample(model,toks_list,bias=1.0,maxlen=None):
|
| 56 |
+
model.eval(); Bn=len(toks_list); Un=max(len(t) for t in toks_list)+2
|
| 57 |
+
Tk=torch.zeros(Bn,Un,dtype=torch.long,device=dev); Tm=torch.zeros(Bn,Un,device=dev)
|
| 58 |
+
for b,t in enumerate(toks_list): ids=[1]+[stoi.get(x,0) for x in t]+[2]; Tk[b,:len(ids)]=torch.tensor(ids); Tm[b,:len(ids)]=1
|
| 59 |
+
ctx=model.encode(Tk,Tm); durs=(torch.expm1(model.dur(ctx)[...,0]).clamp(min=0)*DUR_SCALE).round().long().cpu().numpy()
|
| 60 |
+
plans=[expand([stoi.get(x,0) for x in t],durs[b,1:1+len(t)]) for b,t in enumerate(toks_list)]; Tn=min(maxlen or MAXPTS,max(len(p[0]) for p in plans)); L=[min(Tn,len(p[0])) for p in plans]
|
| 61 |
+
C=torch.zeros(Bn,Tn,dtype=torch.long,device=dev); Ph=torch.zeros_like(C); Cn=torch.zeros_like(C)
|
| 62 |
+
for b,(c,p,nx) in enumerate(plans): n=L[b]; C[b,:n]=torch.tensor(c[:n]); Ph[b,:n]=torch.tensor(p[:n]); Cn[b,:n]=torch.tensor(nx[:n])
|
| 63 |
+
x=torch.zeros(Bn,1,3,device=dev); state=None; out=[]
|
| 64 |
+
for step in range(Tn):
|
| 65 |
+
o,state=model(x,C[:,step:step+1],Ph[:,step:step+1],Cn[:,step:step+1],state); o=o[:,-1]
|
| 66 |
+
pi=F.softmax(o[:,:M]*(1+bias),-1); mu=o[:,M:3*M].view(Bn,M,2); ls=(o[:,3*M:5*M].view(Bn,M,2).clamp(-7,5)-bias); rho=torch.tanh(o[:,5*M:6*M])
|
| 67 |
+
k=torch.multinomial(pi,1)[:,0]; m=mu[torch.arange(Bn),k]; sd=ls[torch.arange(Bn),k].exp(); r=rho[torch.arange(Bn),k]
|
| 68 |
+
e1=torch.randn(Bn,device=dev); e2=torch.randn(Bn,device=dev); dx=m[:,0]+sd[:,0]*e1; dy=m[:,1]+sd[:,1]*(r*e1+torch.sqrt(1-r**2)*e2)
|
| 69 |
+
pen=(torch.sigmoid(o[:,6*M])>0.5).float(); x=torch.stack([dx,dy,pen],-1)[:,None]; out.append(x[:,0].clone())
|
| 70 |
+
model.train(); seq=torch.stack(out,1).cpu().numpy(); seq[:,:,:2]/=SCALE
|
| 71 |
+
res=[]
|
| 72 |
+
for b in range(Bn): q=seq[b,:L[b]].copy(); q[-1,2]=1; res.append(q)
|
| 73 |
+
return res
|
| 74 |
+
def demo(model,va,path,n=8):
|
| 75 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 76 |
+
ids=list(range(0,len(va),max(1,len(va)//n)))[:n]; toks=[va.get(i)[1] for i in ids]; tl=[[itos[t] for t in tk[1:-1]] for tk in toks]
|
| 77 |
+
seqs=sample(model,tl,bias=1.0); ims=[]; f=ImageFont.truetype(os.path.join(PROJ,'DejaVuSans.ttf'),12)
|
| 78 |
+
for i,sq in zip(ids,seqs):
|
| 79 |
+
real=render(va.get(i)[0]); gen=render(sq)
|
| 80 |
+
c=Image.new('L',(max(real.width,gen.width,300),real.height+gen.height+18),255); c.paste(real,(0,0)); c.paste(gen,(0,real.height)); ImageDraw.Draw(c).text((2,real.height+gen.height+2),''.join(va.labels[va.idx[i]])[:70],fill=0,font=f); ims.append(c)
|
| 81 |
+
W=max(c.width for c in ims); sheet=Image.new('L',(W,sum(c.height for c in ims)+8*len(ims)),255); y=0
|
| 82 |
+
for c in ims: sheet.paste(c,(0,y)); y+=c.height+8
|
| 83 |
+
sheet.save(path)
|
| 84 |
+
def main():
|
| 85 |
+
tr=Inks('train'); va=Inks('valid'); model=Model().to(dev); ema=copy.deepcopy(model).eval()
|
| 86 |
+
for p in ema.parameters(): p.requires_grad_(False)
|
| 87 |
+
opt=torch.optim.Adam(model.parameters(),LR); step=0; ck=sorted(glob.glob(os.path.join(CKPT,'step*.pt')))
|
| 88 |
+
if ck: st=torch.load(ck[-1],map_location=dev); model.load_state_dict(st['m']); ema.load_state_dict(st['e']); opt.load_state_dict(st['o']); step=st['step']; print('resumed',step,flush=True)
|
| 89 |
+
elif os.environ.get('INIT_FROM') and os.path.exists(os.environ['INIT_FROM']):
|
| 90 |
+
st=torch.load(os.environ['INIT_FROM'],map_location=dev); src={k:v.float() for k,v in st.get('e',st).items()}; tgt=model.state_dict(); ok={k:v for k,v in src.items() if k in tgt and tgt[k].shape==v.shape}; model.load_state_dict(ok,strict=False); ema.load_state_dict({k:v.clone() for k,v in ok.items()},strict=False); print('warm-start',len(ok),'of',len(tgt),flush=True)
|
| 91 |
+
print(f"lstmal[{RUN}] dev={dev} V={V} HID={HID} M={M} NPH={NPH} BS={BS} params {sum(p.numel() for p in model.parameters())/1e6:.1f}M target {STEPS}",flush=True)
|
| 92 |
+
rng=np.random.default_rng(0); t0=time.time(); nbad=0
|
| 93 |
+
while step<STEPS:
|
| 94 |
+
lr=LR*min(1.0,(step+1)/500)*(0.5*(1+math.cos(math.pi*min(1.0,step/STEPS))))
|
| 95 |
+
for g in opt.param_groups: g['lr']=lr
|
| 96 |
+
X,Y,Mk,Tk,Tm,C,Ph,Cn,Du,Dm=tr.batch(rng.integers(0,len(tr),BS))
|
| 97 |
+
o,_=model(X,C,Ph,Cn); ld=model.dur(model.encode(Tk,Tm))[...,0]; loss_tok=mdn_nll(o,Y,Mk); loss_dur=((ld-Du)**2*Dm).sum()/Dm.sum(); loss=loss_tok+DURW*loss_dur
|
| 98 |
+
if not torch.isfinite(loss):
|
| 99 |
+
nbad+=1; print(f' non-finite loss at step {step} (#{nbad})',flush=True)
|
| 100 |
+
if nbad>=50: print('INK ABORT: persistent NaN',flush=True); raise SystemExit(3)
|
| 101 |
+
continue
|
| 102 |
+
opt.zero_grad(); loss.backward(); gn=torch.nn.utils.clip_grad_norm_(model.parameters(),5.0)
|
| 103 |
+
if not torch.isfinite(gn): opt.zero_grad(); continue
|
| 104 |
+
opt.step(); step+=1
|
| 105 |
+
with torch.no_grad():
|
| 106 |
+
for pe,pm in zip(ema.parameters(),model.parameters()): pe.mul_(0.999).add_(pm,alpha=0.001)
|
| 107 |
+
if step%100==0:
|
| 108 |
+
sp=step/(time.time()-t0)
|
| 109 |
+
with torch.no_grad(): pen_acc=(((o[...,6*M]>0)==(Y[...,2]>0.5)).float()*Mk).sum()/Mk.sum()
|
| 110 |
+
print(f"step {step}/{STEPS} nll {loss_tok.item():.3f} dur {loss_dur.item():.3f} pen_acc {pen_acc.item():.3f} lr {lr:.2e} {sp:.2f}it/s eta {(STEPS-step)/sp/60:.0f}min",flush=True)
|
| 111 |
+
open(os.path.join(CKPT,'loss.csv'),'a').write(f"{step},{loss_tok.item():.4f},{loss_dur.item():.4f}\n")
|
| 112 |
+
if step%2000==0 or step==STEPS:
|
| 113 |
+
torch.save(dict(m=model.state_dict(),e=ema.state_dict(),o=opt.state_dict(),step=step),os.path.join(CKPT,f'step{step:06d}.pt'))
|
| 114 |
+
torch.save(dict(e={k:v.half() for k,v in ema.state_dict().items()},step=step,itos=itos),os.path.join(CKPT,'fp16',f'step{step:06d}_fp16.pt'))
|
| 115 |
+
try: demo(ema,va,os.path.join(SAMP,f'step{step:06d}.png'))
|
| 116 |
+
except Exception as e: print('demo failed',repr(e),flush=True)
|
| 117 |
+
print(' ckpt+demo',step,flush=True)
|
| 118 |
+
print("INK DONE",flush=True)
|
| 119 |
+
if __name__=='__main__': main()
|
scripts/train_ink_al2.py
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""train_ink_al + STYLE conditioning (w03 18.10: a page must come out in ONE hand, and his hand must be set by a few
|
| 3 |
+
of his lines): a style encoder (BiGRU over the points of K reference lines from the same page, mean-pooled -> 128-d)
|
| 4 |
+
feeds (a) the per-point conditioning and (b) the initial LSTM states (also fixes garbled line starts: the decoder no
|
| 5 |
+
longer starts from an all-zero state). Reference lines are OTHER lines of the same page (fallback: same doc / itself);
|
| 6 |
+
at sampling, pass reference strokes (real lines of the target writer) via sample(..., ref=[seq,...]).
|
| 7 |
+
Env: as train_ink_al + K (3), SDIM (128), ALIGN (align_all.npz by default), PSTYLE_DROP (0.1: style dropped -> model
|
| 8 |
+
also works without a reference). MODEL_PY=train_ink_al2."""
|
| 9 |
+
import os, sys, json, math, time, glob, copy, numpy as np, torch, torch.nn as nn, torch.nn.functional as F
|
| 10 |
+
sys.path.insert(0,os.path.dirname(os.path.abspath(__file__)))
|
| 11 |
+
os.environ.setdefault('HW_RUN','lstmal2'); os.environ.setdefault('INK_DS','ink_lines5s'); os.environ.setdefault('MAXPTS','1200'); os.environ.setdefault('MAXTOK','200'); os.environ.setdefault('ALIGN','align_all.npz')
|
| 12 |
+
import train_ink_al as A
|
| 13 |
+
from train_ink_al import * # noqa
|
| 14 |
+
K=int(os.environ.get('K','3')); SDIM=int(os.environ.get('SDIM','128')); PSTYLE_DROP=float(os.environ.get('PSTYLE_DROP','0.1'))
|
| 15 |
+
AUG=int(os.environ.get('AUG','0')); AUG_S=float(os.environ.get('AUG_S','0.1')); AUG_SH=float(os.environ.get('AUG_SH','0.15')); AUG_N=float(os.environ.get('AUG_N','0.0')) # 19.10 (Лев: «аугментируешь?»): affine jitter per line, train only
|
| 16 |
+
class Inks(A.Inks):
|
| 17 |
+
def __init__(s,split):
|
| 18 |
+
super().__init__(split)
|
| 19 |
+
# page key per line from the ink_dataset jsonl order is not stored in the npz -> use doc/page from the dataset jsonl
|
| 20 |
+
meta=os.path.join(PROJ,'data',os.environ.get('META_DS','ink_dataset_all_sorted'),'meta_'+split+'.jsonl'); s.page=[]; k=0
|
| 21 |
+
for l in open(meta): # ink_lines_prep drops a few over-long lines: walk the jsonl in order and match labels to the npz order
|
| 22 |
+
r=json.loads(l)
|
| 23 |
+
if k<len(s.labels) and r['label']==s.labels[k]: s.page.append((r['src'],r['doc'],r['page'])); k+=1
|
| 24 |
+
assert len(s.page)==len(s.labels), (len(s.page),len(s.labels))
|
| 25 |
+
s.bypage={}
|
| 26 |
+
for i in s.idx: s.bypage.setdefault(s.page[i],[]).append(i)
|
| 27 |
+
def refs(s,j,rng):
|
| 28 |
+
cands=[q for q in s.bypage[s.page[j]] if q!=j] or [j]; out=[]
|
| 29 |
+
for _ in range(K):
|
| 30 |
+
q=cands[rng.integers(len(cands))]; d=s.pts[s.off[q]:s.off[q+1]][:MAXPTS].copy(); d[:,:2]*=SCALE; out.append(d)
|
| 31 |
+
return out
|
| 32 |
+
def batch(s,ids,rng=None,aug=None):
|
| 33 |
+
rng=rng or np.random.default_rng(); res=super().batch(ids); Rs=[s.refs(s.idx[i],rng) for i in ids]; L=max(len(d) for rs in Rs for d in rs)
|
| 34 |
+
R=np.zeros((len(ids),K,L,3),np.float32); RL=np.zeros((len(ids),K),np.int64)
|
| 35 |
+
for b,rs in enumerate(Rs):
|
| 36 |
+
for k,d in enumerate(rs): R[b,k,:len(d)]=d; RL[b,k]=len(d)
|
| 37 |
+
R=torch.from_numpy(R).to(dev); RL=torch.from_numpy(RL).to(dev)
|
| 38 |
+
if aug is None: aug=AUG
|
| 39 |
+
if aug: # one affine jitter per line, applied to the deltas of the line (inputs X and targets Y) AND its style refs: scale ±AUG_S, shear ±AUG_SH, point noise AUG_N (scaled units)
|
| 40 |
+
X,Y=res[0],res[1]; Bn=X.size(0); sx=1+(torch.rand(Bn,device=dev)*2-1)*AUG_S; sy=1+(torch.rand(Bn,device=dev)*2-1)*AUG_S; sh=(torch.rand(Bn,device=dev)*2-1)*AUG_SH
|
| 41 |
+
def tf(d,sx,sy,sh): # d [..., 3]: dx' = sx*dx + sh*dy ; dy' = sy*dy
|
| 42 |
+
dx=d[...,0]*sx+d[...,1]*sh; dy=d[...,1]*sy; return torch.stack([dx,dy,d[...,2]],-1)
|
| 43 |
+
X=tf(X,sx[:,None],sy[:,None],sh[:,None]); Y=torch.cat([tf(Y[...,:3],sx[:,None],sy[:,None],sh[:,None]),Y[...,3:]],-1)
|
| 44 |
+
if AUG_N>0: nz=torch.randn_like(X[...,:2])*AUG_N*(X[...,:2].abs()>0); X=torch.cat([X[...,:2]+nz,X[...,2:]],-1)
|
| 45 |
+
R=tf(R,sx[:,None,None],sy[:,None,None],sh[:,None,None]); res=(X,Y,*res[2:])
|
| 46 |
+
return (*res,R,RL)
|
| 47 |
+
class Model(A.Model):
|
| 48 |
+
def __init__(s):
|
| 49 |
+
super().__init__(); s.sgru=nn.GRU(3,SDIM,batch_first=True,bidirectional=True); s.sproj=nn.Linear(2*SDIM,256); s.h0=nn.Linear(256,6*HID); s.null_style=nn.Parameter(torch.zeros(256))
|
| 50 |
+
def style(s,R,RL):
|
| 51 |
+
Bn,Kn,L,_=R.shape; x=R.flatten(0,1); p=nn.utils.rnn.pack_padded_sequence(x,RL.flatten().clamp(min=1).cpu(),batch_first=True,enforce_sorted=False); _,h=s.sgru(p) # h [2,B*K,SDIM]
|
| 52 |
+
v=s.sproj(torch.cat([h[0],h[1]],-1)).view(Bn,Kn,256).mean(1); return v # [B,256]
|
| 53 |
+
def forward(s,X,C,Ph,Cn,state=None,sv=None):
|
| 54 |
+
w=s.cond(C,Ph,Cn)+(sv[:,None,:] if sv is not None else 0)
|
| 55 |
+
if state is None and sv is not None:
|
| 56 |
+
h=s.h0(sv).view(X.size(0),6,HID).permute(1,0,2).contiguous(); state=((h[0:1],h[1:2]),(h[2:3],h[3:4]),(h[4:5],h[5:6]))
|
| 57 |
+
st1,st2,st3=state if state is not None else (None,None,None)
|
| 58 |
+
h1,st1=s.l1(torch.cat([X,w],-1),st1); h2,st2=s.l2(torch.cat([X,w,h1],-1),st2); h3,st3=s.l3(torch.cat([X,w,h2],-1),st3)
|
| 59 |
+
return s.out(torch.cat([h1,h2,h3],-1)),(st1,st2,st3)
|
| 60 |
+
@torch.no_grad()
|
| 61 |
+
def style_from_seqs(model,seqs):
|
| 62 |
+
"""seqs: list of [n,3] (dx,dy,pen) in model units (as returned by sample / stored in npz) -> style vector [1,256]."""
|
| 63 |
+
L=max(len(q) for q in seqs); R=np.zeros((1,len(seqs),L,3),np.float32); RL=np.zeros((1,len(seqs)),np.int64)
|
| 64 |
+
for k,q in enumerate(seqs): d=np.asarray(q,np.float32)[:MAXPTS].copy(); d[:,:2]*=SCALE; R[0,k,:len(d)]=d; RL[0,k]=len(d)
|
| 65 |
+
return model.style(torch.from_numpy(R).to(dev),torch.from_numpy(RL).to(dev))
|
| 66 |
+
_DUR_STATS=None
|
| 67 |
+
def dur_floor(toks_list,durs):
|
| 68 |
+
"""DUR_MIN=1 (И78, 21.10; default off): clamp each char's planned points to at least the DUR_MIN_Q quantile (0.25) of the REAL
|
| 69 |
+
per-letter point counts (data/<INK_DS>/dur_stats.json from scripts/dur_stats.py, or DUR_STATS=<path>), scaled by the line's
|
| 70 |
+
planned mean / real mean so the hand's overall speed is kept. durs [B,Un] (index 1..len(t) = chars) -> new array."""
|
| 71 |
+
global _DUR_STATS
|
| 72 |
+
if _DUR_STATS is None: _DUR_STATS=json.load(open(os.environ.get('DUR_STATS') or os.path.join(A.D,'dur_stats.json')))['chars']
|
| 73 |
+
q=f"{float(os.environ.get('DUR_MIN_Q','0.25')):.2f}"; durs=durs.copy()
|
| 74 |
+
for b,t in enumerate(toks_list):
|
| 75 |
+
ks=[k for k,c in enumerate(t) if c in _DUR_STATS]
|
| 76 |
+
if not ks: continue
|
| 77 |
+
r=max(1.0,float(np.mean([durs[b,1+k] for k in ks])))/np.mean([_DUR_STATS[t[k]]['mean'] for k in ks])
|
| 78 |
+
for k,c in enumerate(t):
|
| 79 |
+
if c in _DUR_STATS: durs[b,1+k]=max(int(durs[b,1+k]),int(round(_DUR_STATS[c]['q'][q]*r)))
|
| 80 |
+
return durs
|
| 81 |
+
@torch.no_grad()
|
| 82 |
+
def sample(model,toks_list,bias=1.0,maxlen=None,ref=None,sv=None,durs_list=None,prime=None,return_plan=False):
|
| 83 |
+
"""prime: (X [1,n,3] model inputs (scaled deltas, as Inks.batch gives), C, Ph, Cn [1,n]) of a REAL aligned line of the
|
| 84 |
+
writer -> the decoder is run over it teacher-forced (Graves' priming) before drawing the target text (fixes line starts)."""
|
| 85 |
+
if prime is not None: return _sample_primed(model,toks_list,bias,maxlen,ref,sv,durs_list,prime)
|
| 86 |
+
return _sample(model,toks_list,bias,maxlen,ref,sv,durs_list,return_plan)
|
| 87 |
+
@torch.no_grad()
|
| 88 |
+
def _sample_primed(model,toks_list,bias,maxlen,ref,sv,durs_list,prime):
|
| 89 |
+
model.eval(); Bn=len(toks_list); Xp,Cp,Pp,Np=[t.to(dev).expand(Bn,*t.shape[1:]).contiguous() for t in prime]
|
| 90 |
+
if sv is None and ref is not None: sv=style_from_seqs(model,ref)
|
| 91 |
+
if sv is None: sv=model.null_style[None]
|
| 92 |
+
sv=sv.expand(Bn,-1); Un=max(len(t) for t in toks_list)+2
|
| 93 |
+
Tk=torch.zeros(Bn,Un,dtype=torch.long,device=dev); Tm=torch.zeros(Bn,Un,device=dev)
|
| 94 |
+
for b,t in enumerate(toks_list): ids=[1]+[stoi.get(x,0) for x in t]+[2]; Tk[b,:len(ids)]=torch.tensor(ids); Tm[b,:len(ids)]=1
|
| 95 |
+
ctx=model.encode(Tk,Tm); durs=(torch.expm1(model.dur(ctx)[...,0]).clamp(min=0)*DUR_SCALE).round().long().cpu().numpy()
|
| 96 |
+
if os.environ.get('DUR_MIN')=='1': durs=dur_floor(toks_list,durs)
|
| 97 |
+
plans=[expand([stoi.get(x,0) for x in t],(durs_list[b] if durs_list is not None else durs[b,1:1+len(t)])) for b,t in enumerate(toks_list)]; Tn=min(maxlen or MAXPTS,max(len(p[0]) for p in plans)); L=[min(Tn,len(p[0])) for p in plans]
|
| 98 |
+
C=torch.zeros(Bn,Tn,dtype=torch.long,device=dev); Ph=torch.zeros_like(C); Cn=torch.zeros_like(C)
|
| 99 |
+
for b,(c,p,nx) in enumerate(plans): n=L[b]; C[b,:n]=torch.tensor(c[:n]); Ph[b,:n]=torch.tensor(p[:n]); Cn[b,:n]=torch.tensor(nx[:n])
|
| 100 |
+
_,state=model(Xp,Cp,Pp,Np,None,sv) # teacher-forced pass over the primer line
|
| 101 |
+
x=Xp[:,-1:].clone(); x[:,:,2]=1.0; out=[] # continue from the primer's last point with pen up
|
| 102 |
+
for step in range(Tn):
|
| 103 |
+
o,state=model(x,C[:,step:step+1],Ph[:,step:step+1],Cn[:,step:step+1],state,sv); o=o[:,-1]
|
| 104 |
+
pi=F.softmax(o[:,:M]*(1+bias),-1); mu=o[:,M:3*M].view(Bn,M,2); ls=(o[:,3*M:5*M].view(Bn,M,2).clamp(-7,5)-bias); rho=torch.tanh(o[:,5*M:6*M])
|
| 105 |
+
k=torch.multinomial(pi,1)[:,0]; m=mu[torch.arange(Bn),k]; sd=ls[torch.arange(Bn),k].exp(); r=rho[torch.arange(Bn),k]
|
| 106 |
+
e1=torch.randn(Bn,device=dev); e2=torch.randn(Bn,device=dev); dx=m[:,0]+sd[:,0]*e1; dy=m[:,1]+sd[:,1]*(r*e1+torch.sqrt(1-r**2)*e2)
|
| 107 |
+
pen=(torch.sigmoid(o[:,6*M])>0.5).float(); x=torch.stack([dx,dy,pen],-1)[:,None]; out.append(x[:,0].clone())
|
| 108 |
+
model.train(); seq=torch.stack(out,1).cpu().numpy(); seq[:,:,:2]/=SCALE; res=[]
|
| 109 |
+
for b in range(Bn): q=seq[b,:L[b]].copy(); q[0,:2]=0; q[-1,2]=1; res.append(q) # first delta = the jump from the primer: zero it
|
| 110 |
+
return res
|
| 111 |
+
PEN_SAMPLE=os.environ.get('PEN_SAMPLE','0')=='1'; PEN_THR=float(os.environ.get('PEN_THR','0.5'))
|
| 112 |
+
@torch.no_grad()
|
| 113 |
+
def _sample(model,toks_list,bias=1.0,maxlen=None,ref=None,sv=None,durs_list=None,return_plan=False):
|
| 114 |
+
"""ref: list of reference stroke sequences of the target writer (model units) -> one style for all lines; sv: precomputed style [1,256];
|
| 115 |
+
durs_list: optional per-line integer durations (points per char) to use instead of the duration head (oracle diagnostic)."""
|
| 116 |
+
model.eval(); Bn=len(toks_list); Un=max(len(t) for t in toks_list)+2
|
| 117 |
+
if sv is None and ref is None and os.environ.get('REF_DS'): # judge/demo without explicit refs: style from REF_N lines of data/<REF_DS>/<REF_SPLIT>_raw.npz
|
| 118 |
+
z=np.load(os.path.join(PROJ,'data',os.environ['REF_DS'],os.environ.get('REF_SPLIT','valid')+'_raw.npz'),allow_pickle=True); off=z['pt_off']
|
| 119 |
+
idx=[int(x) for x in os.environ['REF_IDX'].split(',')] if os.environ.get('REF_IDX') else list(range(min(int(os.environ.get('REF_N','3')),len(off)-1)))
|
| 120 |
+
ref=[z['pts'][off[i]:off[i+1]].astype(np.float32) for i in idx]
|
| 121 |
+
if sv is None and ref is not None: sv=style_from_seqs(model,ref)
|
| 122 |
+
if sv is None: sv=model.null_style[None]
|
| 123 |
+
sv=sv.expand(Bn,-1)
|
| 124 |
+
Tk=torch.zeros(Bn,Un,dtype=torch.long,device=dev); Tm=torch.zeros(Bn,Un,device=dev)
|
| 125 |
+
for b,t in enumerate(toks_list): ids=[1]+[stoi.get(x,0) for x in t]+[2]; Tk[b,:len(ids)]=torch.tensor(ids); Tm[b,:len(ids)]=1
|
| 126 |
+
ctx=model.encode(Tk,Tm); durs=(torch.expm1(model.dur(ctx)[...,0]).clamp(min=0)*DUR_SCALE).round().long().cpu().numpy()
|
| 127 |
+
if os.environ.get('DUR_MIN')=='1': durs=dur_floor(toks_list,durs)
|
| 128 |
+
plans=[expand([stoi.get(x,0) for x in t],(durs_list[b] if durs_list is not None else durs[b,1:1+len(t)])) for b,t in enumerate(toks_list)]; Tn=min(maxlen or MAXPTS,max(len(p[0]) for p in plans)); L=[min(Tn,len(p[0])) for p in plans]
|
| 129 |
+
C=torch.zeros(Bn,Tn,dtype=torch.long,device=dev); Ph=torch.zeros_like(C); Cn=torch.zeros_like(C)
|
| 130 |
+
for b,(c,p,nx) in enumerate(plans): n=L[b]; C[b,:n]=torch.tensor(c[:n]); Ph[b,:n]=torch.tensor(p[:n]); Cn[b,:n]=torch.tensor(nx[:n])
|
| 131 |
+
# 22.10 И101 (all off by default = old behaviour): sampling knobs SAMP_PI_BIAS / SAMP_SD_BIAS (separate bias of the mixture weights and of
|
| 132 |
+
# the sigmas; default = bias), SAMP_GREEDY=1 (argmax component), SAMP_TOPP (nucleus over components, 1 = off), SAMP_SDMUL (sigma x, 0 = component mean);
|
| 133 |
+
# NEG_GUIDE=A|B|R «negative letter»: a 2nd stream of the same LSTM over the same drawn points with the plan's look-alikes swapped in C and Cn
|
| 134 |
+
# (NEG_PAIRS, default ы:п,я:л,ю:н,м:л); on those spans (C or Cn differs) with strength NEG_W: A = log pi <- (1+w)log pi_c - w log pi_n,
|
| 135 |
+
# B = A + mu <- mu_c + w(mu_c - mu_n) of the chosen component, R = draw NEG_S points from the positive mixture and resample by (p_c/p_n)^w
|
| 136 |
+
# (target ~ p_c^(1+w)/p_n^w); the pen logit is mixed the same way unless NEG_PEN=0.
|
| 137 |
+
E_=os.environ.get; NG=E_('NEG_GUIDE',''); GW=float(E_('NEG_W','1.0')); NGPEN=E_('NEG_PEN','1')=='1'; NS=int(E_('NEG_S','32'))
|
| 138 |
+
pib=float(E_('SAMP_PI_BIAS',bias)); sdb=float(E_('SAMP_SD_BIAS',bias)); GREEDY=E_('SAMP_GREEDY')=='1'; TOPP=float(E_('SAMP_TOPP','1')); SDMUL=float(E_('SAMP_SDMUL','1'))
|
| 139 |
+
if NG:
|
| 140 |
+
sub=torch.arange(V,device=dev)
|
| 141 |
+
for a_,b_ in (q.split(':') for q in E_('NEG_PAIRS','ы:п,я:л,ю:н,м:л').split(',')):
|
| 142 |
+
if a_ in stoi and b_ in stoi: sub[stoi[a_]]=stoi[b_]
|
| 143 |
+
Cg=sub[C]; Cng=sub[Cn]; SPN=E_('NEG_SPAN','any'); GM=((Cg!=C) if SPN=='c' else (Cng!=Cn) if SPN=='cn' else ((Cg!=C)|(Cng!=Cn))).float()*GW # NEG_SPAN: where to guide (c = the letter's own plan span, cn = previous letter's tail)
|
| 144 |
+
C2=torch.cat([C,Cg]); Ph2=torch.cat([Ph,Ph]); Cn2=torch.cat([Cn,Cng]); sv2=torch.cat([sv,sv])
|
| 145 |
+
ar=torch.arange(Bn,device=dev)
|
| 146 |
+
def mix(o_): # -> log pi (bias, top-p applied), mu, log sd, rho
|
| 147 |
+
lp=F.log_softmax(o_[:,:M],-1)*(1+pib); mu_=o_[:,M:3*M].view(Bn,M,2); ls_=(o_[:,3*M:5*M].view(Bn,M,2).clamp(-7,5)-sdb); return lp,mu_,ls_,torch.tanh(o_[:,5*M:6*M])
|
| 148 |
+
def mdn_logp(lp,mu_,ls_,rho_,xy): # xy [Bn,S,2] -> log density [Bn,S] under the mixture
|
| 149 |
+
d=(xy[:,:,None,:]-mu_[:,None])/ls_.exp()[:,None]; r_=rho_[:,None]; om=(1-r_**2).clamp(min=1e-6)
|
| 150 |
+
lg=-(d[...,0]**2+d[...,1]**2-2*r_*d[...,0]*d[...,1])/(2*om)-0.5*torch.log(om)-ls_.sum(-1)[:,None]-math.log(2*math.pi)
|
| 151 |
+
return torch.logsumexp(F.log_softmax(lp,-1)[:,None]+lg,-1)
|
| 152 |
+
x=torch.zeros(Bn,1,3,device=dev); state=None; out=[]
|
| 153 |
+
for step in range(Tn):
|
| 154 |
+
if NG:
|
| 155 |
+
o2,state=model(torch.cat([x,x]),C2[:,step:step+1],Ph2[:,step:step+1],Cn2[:,step:step+1],state,sv2); o=o2[:Bn,-1]; on=o2[Bn:,-1]; g=GM[:,step]
|
| 156 |
+
else: o,state=model(x,C[:,step:step+1],Ph[:,step:step+1],Cn[:,step:step+1],state,sv); o=o[:,-1]
|
| 157 |
+
lp,mu,ls,rho=mix(o)
|
| 158 |
+
if NG: lpn,mun,lsn,rhon=mix(on)
|
| 159 |
+
if NG in ('A','B'): lp=(1+g[:,None])*lp-g[:,None]*lpn
|
| 160 |
+
pi=F.softmax(lp,-1)
|
| 161 |
+
if TOPP<1:
|
| 162 |
+
ps,ix=pi.sort(-1,descending=True); keep=(ps.cumsum(-1)-ps)<TOPP; pi=torch.zeros_like(pi).scatter(-1,ix,ps*keep); pi=pi/pi.sum(-1,keepdim=True)
|
| 163 |
+
if NG=='R':
|
| 164 |
+
ks=pi.argmax(-1,keepdim=True).expand(Bn,NS) if GREEDY else torch.multinomial(pi,NS,replacement=True); m=mu[ar[:,None],ks]; sd=ls[ar[:,None],ks].exp()*SDMUL; r=rho[ar[:,None],ks]
|
| 165 |
+
e1=torch.randn(Bn,NS,device=dev); e2=torch.randn(Bn,NS,device=dev); cand=torch.stack([m[...,0]+sd[...,0]*e1,m[...,1]+sd[...,1]*(r*e1+torch.sqrt(1-r**2)*e2)],-1)
|
| 166 |
+
wgt=g[:,None]*(mdn_logp(torch.log(pi+1e-12),mu,ls,rho,cand)-mdn_logp(lpn,mun,lsn,rhon,cand)); j=torch.multinomial(F.softmax(wgt,-1),1)[:,0]; dx,dy=cand[ar,j,0],cand[ar,j,1]
|
| 167 |
+
else:
|
| 168 |
+
k=pi.argmax(-1) if GREEDY else torch.multinomial(pi,1)[:,0]; m=mu[ar,k]; sd=ls[ar,k].exp()*SDMUL; r=rho[ar,k]
|
| 169 |
+
if NG=='B': m=m+g[:,None]*(m-mun[ar,k])
|
| 170 |
+
e1=torch.randn(Bn,device=dev); e2=torch.randn(Bn,device=dev); dx=m[:,0]+sd[:,0]*e1; dy=m[:,1]+sd[:,1]*(r*e1+torch.sqrt(1-r**2)*e2)
|
| 171 |
+
zp=o[:,6*M]
|
| 172 |
+
if NG and NGPEN: zp=zp+g*(zp-on[:,6*M])
|
| 173 |
+
pp=torch.sigmoid(zp); pen=(torch.bernoulli(pp) if PEN_SAMPLE else (pp>PEN_THR).float()); x=torch.stack([dx,dy,pen],-1)[:,None]; out.append(torch.cat([x[:,0],torch.sigmoid(o[:,6*M+2:6*M+3])],-1).clone() if PRES else x[:,0].clone()) # 21.10: PEN_SAMPLE=1 / PEN_THR (default 0.5 = old) for the glyph generator
|
| 174 |
+
model.train(); seq=torch.stack(out,1).cpu().numpy(); seq[:,:,:2]/=SCALE; res=[]
|
| 175 |
+
for b in range(Bn): q=seq[b,:L[b]].copy(); q[-1,2]=1; res.append(q)
|
| 176 |
+
# return_plan: also the per-point conditioning (char, phase, next char) of each line, so another model can be run
|
| 177 |
+
# teacher-forced over the sampled trajectory (write_page PRES_MODEL: pressure from a PRES model on grpo3's strokes)
|
| 178 |
+
if return_plan: return res,[(c[:L[b]],p[:L[b]],nx[:L[b]]) for b,(c,p,nx) in enumerate(plans)]
|
| 179 |
+
return res
|
| 180 |
+
def demo(model,va,path,n=8):
|
| 181 |
+
"""rows: real line | generated with the style of OTHER lines of the same page."""
|
| 182 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 183 |
+
rng=np.random.default_rng(1); ids=list(range(0,len(va),max(1,len(va)//n)))[:n]; f=ImageFont.truetype(os.path.join(PROJ,'DejaVuSans.ttf'),12); ims=[]
|
| 184 |
+
for i in ids:
|
| 185 |
+
d,t=va.get(i); refs=[q.copy() for q in va.refs(va.idx[i],rng)]
|
| 186 |
+
for q in refs: q[:,:2]/=SCALE
|
| 187 |
+
sq=sample(model,[[itos[x] for x in t[1:-1]]],bias=1.0,ref=refs)[0]; real=render(d); gen=render(sq)
|
| 188 |
+
c=Image.new('L',(max(real.width,gen.width,300),real.height+gen.height+18),255); c.paste(real,(0,0)); c.paste(gen,(0,real.height)); ImageDraw.Draw(c).text((2,real.height+gen.height+2),''.join(va.labels[va.idx[i]])[:70],fill=0,font=f); ims.append(c)
|
| 189 |
+
W=max(c.width for c in ims); sheet=Image.new('L',(W,sum(c.height for c in ims)+8*len(ims)),255); y=0
|
| 190 |
+
for c in ims: sheet.paste(c,(0,y)); y+=c.height+8
|
| 191 |
+
sheet.save(path)
|
| 192 |
+
def main():
|
| 193 |
+
tr=Inks('train'); va=Inks('valid'); model=Model().to(dev); ema=copy.deepcopy(model).eval()
|
| 194 |
+
for p in ema.parameters(): p.requires_grad_(False)
|
| 195 |
+
opt=torch.optim.Adam(model.parameters(),LR); step=0; ck=sorted(glob.glob(os.path.join(CKPT,'step*.pt')))
|
| 196 |
+
if ck: st=torch.load(ck[-1],map_location=dev); model.load_state_dict(st['m']); ema.load_state_dict(st['e']); opt.load_state_dict(st['o']); step=st['step']; print('resumed',step,flush=True)
|
| 197 |
+
elif os.environ.get('INIT_FROM') and os.path.exists(os.environ['INIT_FROM']):
|
| 198 |
+
st=torch.load(os.environ['INIT_FROM'],map_location=dev); src={k:v.float() for k,v in st.get('e',st).items()}; tgt=model.state_dict(); ok={k:v for k,v in src.items() if k in tgt and tgt[k].shape==v.shape}
|
| 199 |
+
for k_ in ('out.weight','out.bias'):
|
| 200 |
+
if k_ in src and k_ in tgt and tgt[k_].shape!=src[k_].shape and tgt[k_].shape[1:]==src[k_].shape[1:]: t_=tgt[k_].clone(); n_=min(t_.shape[0],src[k_].shape[0]); t_[:n_]=src[k_][:n_]; ok[k_]=t_
|
| 201 |
+
model.load_state_dict(ok,strict=False); ema.load_state_dict({k:v.clone() for k,v in ok.items()},strict=False); print('warm-start',len(ok),'of',len(tgt),flush=True)
|
| 202 |
+
print(f"lstmal2[{RUN}] dev={dev} V={V} HID={HID} K={K} SDIM={SDIM} BS={BS} params {sum(p.numel() for p in model.parameters())/1e6:.1f}M target {STEPS} align {ALIGN}",flush=True)
|
| 203 |
+
rng=np.random.default_rng(0); t0=time.time(); nbad=0
|
| 204 |
+
while step<STEPS:
|
| 205 |
+
lr=LR*min(1.0,(step+1)/500)*(0.5*(1+math.cos(math.pi*min(1.0,step/STEPS))))
|
| 206 |
+
for g in opt.param_groups: g['lr']=lr
|
| 207 |
+
X,Y,Mk,Tk,Tm,C,Ph,Cn,Du,Dm,R,RL=tr.batch(rng.integers(0,len(tr),BS),rng)
|
| 208 |
+
sv=model.style(R,RL); drop=(torch.rand(X.size(0),device=dev)<PSTYLE_DROP)[:,None]; sv=torch.where(drop,model.null_style[None].expand_as(sv),sv)
|
| 209 |
+
o,_=model(X,C,Ph,Cn,None,sv); ld=model.dur(model.encode(Tk,Tm))[...,0]; loss_tok=mdn_nll(o,Y,Mk); loss_dur=((ld-Du)**2*Dm).sum()/Dm.sum(); loss=loss_tok+DURW*loss_dur
|
| 210 |
+
if not torch.isfinite(loss):
|
| 211 |
+
nbad+=1; print(f' non-finite loss at step {step} (#{nbad})',flush=True)
|
| 212 |
+
if nbad>=50: print('INK ABORT: persistent NaN',flush=True); raise SystemExit(3)
|
| 213 |
+
continue
|
| 214 |
+
opt.zero_grad(); loss.backward(); gn=torch.nn.utils.clip_grad_norm_(model.parameters(),5.0)
|
| 215 |
+
if not torch.isfinite(gn): opt.zero_grad(); continue
|
| 216 |
+
opt.step(); step+=1
|
| 217 |
+
with torch.no_grad():
|
| 218 |
+
for pe,pm in zip(ema.parameters(),model.parameters()): pe.mul_(0.999).add_(pm,alpha=0.001)
|
| 219 |
+
if step%100==0:
|
| 220 |
+
sp=step/(time.time()-t0)
|
| 221 |
+
with torch.no_grad(): pen_acc=(((o[...,6*M]>0)==(Y[...,2]>0.5)).float()*Mk).sum()/Mk.sum()
|
| 222 |
+
print(f"step {step}/{STEPS} nll {loss_tok.item():.3f} dur {loss_dur.item():.3f} pen_acc {pen_acc.item():.3f} lr {lr:.2e} {sp:.2f}it/s eta {(STEPS-step)/sp/60:.0f}min",flush=True)
|
| 223 |
+
open(os.path.join(CKPT,'loss.csv'),'a').write(f"{step},{loss_tok.item():.4f},{loss_dur.item():.4f}\n")
|
| 224 |
+
if step%2000==0 or step==STEPS:
|
| 225 |
+
torch.save(dict(m=model.state_dict(),e=ema.state_dict(),o=opt.state_dict(),step=step),os.path.join(CKPT,f'step{step:06d}.pt'))
|
| 226 |
+
torch.save(dict(e={k:v.half() for k,v in ema.state_dict().items()},step=step,itos=itos),os.path.join(CKPT,'fp16',f'step{step:06d}_fp16.pt'))
|
| 227 |
+
try: demo(ema,va,os.path.join(SAMP,f'step{step:06d}.png'))
|
| 228 |
+
except Exception as e: print('demo failed',repr(e),flush=True)
|
| 229 |
+
print(' ckpt+demo',step,flush=True)
|
| 230 |
+
print("INK DONE",flush=True)
|
| 231 |
+
if __name__=='__main__': main()
|
scripts/train_ink_denoise.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
scripts/train_planner.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""И160 (01.11, Лев: «Давай тогда модель длительностей переобучим, а кодировщик оставим»): планировщик длительностей для B — сколько
|
| 3 |
+
точек пера на каждую букву строки. Заменяет голову длительностей lstmal_v6isl3 (22.10, старый набор v6, почти одна w01, видит только
|
| 4 |
+
текст: ритм у всех одинаковый).
|
| 5 |
+
данные : data/ink_lines_u2 train, разметка align_islands3c_clean.npz (та же, на которой учились A и B): длительность буквы = число её точек
|
| 6 |
+
вход : текст (тот же словарь) + вектор образца semb3_all (data/ink_lines_u2/semb_3all_train.npy) ДРУГОЙ строки того же писца, по
|
| 7 |
+
возможности с другой страницы — как у B; с вероятностью PSTYLE_DROP образец заменяется «никаким» (обучаемый вектор)
|
| 8 |
+
модель : эмбеддинг букв + позиция -> 4 слоя Transformer-энкодера d 256 (вектор образца прибавлен к каждой букве) -> log1p(длительность)
|
| 9 |
+
лосс : L1 по log1p(длительности) (медиана, устойчиво к выбросам разметки)
|
| 10 |
+
отложено: PAGE_HOLD (3%) страниц каждого писца целиком — по ним мера и выбор шага; там же мерится старая голова
|
| 11 |
+
Мера (до прогона): ошибка на букву = медиана |log(pred/true)| (старая голова на u2 train ×1.25–1.50), длина строки = pred/true,
|
| 12 |
+
разброс 10–90% (старая голова ±25%) и её медиана по писцам.
|
| 13 |
+
python scripts/train_planner.py <out_dir> [STEPS=20000 BS=64 LR=1e-3 MAXMIN=40] -> <out_dir>/best.pt, log.txt (перезаписывать отказывается)"""
|
| 14 |
+
import os, sys, json, math, time, copy, numpy as np
|
| 15 |
+
P = os.path.dirname(os.path.dirname(os.path.abspath(__file__))); sys.path.insert(0, os.path.join(P, 'scripts'))
|
| 16 |
+
import torch, torch.nn as nn, torch.nn.functional as F
|
| 17 |
+
E = os.environ.get; OUT = sys.argv[1] if __name__ == "__main__" else None
|
| 18 |
+
dev = 'cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu')
|
| 19 |
+
DS = os.path.join(P, 'data', E('INK_DS', 'ink_lines_u2')); itos = json.load(open(os.path.join(DS, 'vocab.json'))); stoi = {s: i for i, s in enumerate(itos)}
|
| 20 |
+
MAXTOK = 200; PSTYLE_DROP = float(E('PSTYLE_DROP', '0.2'))
|
| 21 |
+
|
| 22 |
+
class Planner(nn.Module):
|
| 23 |
+
def __init__(s, V, d=256, L=4):
|
| 24 |
+
super().__init__(); s.emb = nn.Embedding(V, d); s.pos = nn.Embedding(MAXTOK + 2, d); s.sty = nn.Linear(256, d); s.null = nn.Parameter(torch.zeros(256))
|
| 25 |
+
s.enc = nn.TransformerEncoder(nn.TransformerEncoderLayer(d, 4, 4 * d, 0.1, batch_first=True, norm_first=True), L); s.out = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, 1))
|
| 26 |
+
def forward(s, Tk, Tm, Se): # Tk [B,U] ids with <bos>/<eos>; Se [B,256] style (rows = null where no reference) -> log1p durations [B,U]
|
| 27 |
+
h = s.emb(Tk) + s.pos(torch.arange(Tk.shape[1], device=Tk.device))[None] + s.sty(Se)[:, None]
|
| 28 |
+
return s.out(s.enc(h, src_key_padding_mask=Tm < 0.5))[..., 0]
|
| 29 |
+
|
| 30 |
+
def batch_of(toks_list, se, d=None):
|
| 31 |
+
U = min(MAXTOK + 2, -(-(max(len(t) for t in toks_list) + 2) // 16) * 16) # pad to a multiple of 16: few distinct shapes for MPS (planner1 slowed 20x after step 1000)
|
| 32 |
+
Tk = torch.zeros(len(toks_list), U, dtype=torch.long); Tm = torch.zeros(len(toks_list), U)
|
| 33 |
+
for b, t in enumerate(toks_list): ids = [1] + [stoi.get(x, 0) for x in t] + [2]; Tk[b, :len(ids)] = torch.tensor(ids); Tm[b, :len(ids)] = 1
|
| 34 |
+
d = d or dev; return Tk.to(d), Tm.to(d), se.to(d)
|
| 35 |
+
|
| 36 |
+
def load(path):
|
| 37 |
+
st = torch.load(path, map_location=dev); n = Planner(len(st['itos'])).to(dev); n.load_state_dict(st['m']); n.eval(); return n
|
| 38 |
+
|
| 39 |
+
@torch.no_grad()
|
| 40 |
+
def predict(net, toks_list, se):
|
| 41 |
+
"""toks_list: list of char lists; se [B,256] semb3_all of the reference (None -> no reference) -> per-char point counts (lists of int)"""
|
| 42 |
+
d_ = next(net.parameters()).device; se = net.null[None].expand(len(toks_list), -1) if se is None else se.to(d_).float()
|
| 43 |
+
Tk, Tm, Se = batch_of(toks_list, se, d_); d = torch.expm1(net(Tk, Tm, Se)).clamp(min=1).round().long().cpu().numpy()
|
| 44 |
+
return [list(d[b, 1:1 + len(t)]) for b, t in enumerate(toks_list)]
|
| 45 |
+
|
| 46 |
+
if __name__ == '__main__':
|
| 47 |
+
if os.path.exists(OUT): sys.exit(f'{OUT} exists')
|
| 48 |
+
os.makedirs(OUT); LOG = open(os.path.join(OUT, 'log.txt'), 'w')
|
| 49 |
+
def log(*a): s_ = ' '.join(str(x) for x in a); print(s_, flush=True); LOG.write(s_ + '\n'); LOG.flush()
|
| 50 |
+
z = np.load(os.path.join(DS, 'train_raw.npz'), allow_pickle=True); TK = [json.loads(t) for t in z['toks']]
|
| 51 |
+
meta = [json.loads(l) for l in open(os.path.join(DS, 'meta_train.jsonl'))]; assert len(meta) == len(TK)
|
| 52 |
+
SE = np.load(os.path.join(DS, 'semb_3all_train.npy')).astype(np.float32)
|
| 53 |
+
al = np.load(os.path.join(DS, 'align_islands3c_clean.npz'), allow_pickle=True)
|
| 54 |
+
DU = {int(i): np.bincount(c.astype(np.int64), minlength=len(TK[int(i)])) for i, s, c in zip(al['ids'], al['split'], al['char_idx']) if s == 'train'}
|
| 55 |
+
ok = [i for i in DU if 1 <= len(TK[i]) <= MAXTOK and len(DU[i]) == len(TK[i])]
|
| 56 |
+
rng = np.random.default_rng(0); pages = {}
|
| 57 |
+
for i in ok: pages.setdefault(meta[i]['writer_id'], {}).setdefault((meta[i]['doc'], meta[i]['page']), []).append(i)
|
| 58 |
+
hold = set()
|
| 59 |
+
for w, pg in pages.items():
|
| 60 |
+
ks = sorted(pg); rng.shuffle(ks); n = max(1, int(round(float(E('PAGE_HOLD', '0.03')) * len(ks))))
|
| 61 |
+
for k in ks[:n]: hold.update(pg[k])
|
| 62 |
+
tr = [i for i in ok if i not in hold]; va = sorted(hold)
|
| 63 |
+
byw = {}; byp = {}
|
| 64 |
+
for i in tr: w = meta[i]['writer_id']; byw.setdefault(w, []).append(i); byp[i] = (meta[i]['doc'], meta[i]['page'])
|
| 65 |
+
def ref_of(i, r): # another TRAIN line of the same writer, preferably from another page (held-out lines take their reference from train too)
|
| 66 |
+
c = byw[meta[i]['writer_id']]; q = i
|
| 67 |
+
for _ in range(6):
|
| 68 |
+
q = c[r.integers(len(c))]
|
| 69 |
+
if q != i and byp.get(q) != (meta[i]['doc'], meta[i]['page']): break
|
| 70 |
+
return q
|
| 71 |
+
log(f'planner: dev {dev}, {len(ok)} aligned lines, train {len(tr)}, held-out {len(va)} lines ({sum(len(v) for v in pages.values())} pages, {PSTYLE_DROP} style drop)')
|
| 72 |
+
log('held-out per writer: ' + ', '.join(f'{w} {sum(1 for i in va if meta[i]["writer_id"] == w)}' for w in sorted(pages)))
|
| 73 |
+
vr = np.random.default_rng(1); vref = {i: ref_of(i, vr) for i in va}
|
| 74 |
+
def evaluate(fn): # fn(toks_list, idx_list) -> durations; returns letter error (x), length ratio stats, per-writer length medians
|
| 75 |
+
lr, tot, W = [], [], {}
|
| 76 |
+
for k in range(0, len(va), 128):
|
| 77 |
+
ids = va[k:k + 128]; pd = fn([TK[i] for i in ids], ids)
|
| 78 |
+
for i, p in zip(ids, pd):
|
| 79 |
+
p = np.maximum(np.array(p, float), 1); t = np.maximum(DU[i].astype(float), 1); lr.append(np.abs(np.log(p / t))); r = p.sum() / t.sum(); tot.append(r); W.setdefault(meta[i]['writer_id'], []).append(r)
|
| 80 |
+
lr = np.concatenate(lr); tot = np.array(tot)
|
| 81 |
+
return dict(letter=float(np.exp(np.median(lr))), len_med=float(np.median(tot)), len10=float(np.percentile(tot, 10)), len90=float(np.percentile(tot, 90)),
|
| 82 |
+
len_abs=float(np.median(np.abs(np.log(tot)))), per_w={w: round(float(np.median(v)), 3) for w, v in W.items()})
|
| 83 |
+
fmt = lambda r: f"letter x{r['letter']:.3f} | length med {r['len_med']:.3f} 10-90% {r['len10']:.2f}-{r['len90']:.2f} |log| {r['len_abs']:.3f} | " + ' '.join(f'{w} {v:.2f}' for w, v in sorted(r['per_w'].items()))
|
| 84 |
+
if E('OLD_HEAD', '1') == '1': # the old head on the same held-out lines (text only)
|
| 85 |
+
os.environ.update(PROJ=P, INK_DS='ink_lines_u2', META_DS='ink_lines_u2', HW_RUN='planner_cmp'); import train_ink_denoise as D; base = D.load_base()
|
| 86 |
+
log('OLD HEAD ' + fmt(evaluate(lambda tl, ids: D.head_durs(base, tl)))); del base
|
| 87 |
+
net = Planner(len(itos)).to(dev); STEPS = int(E('STEPS', '20000')); BS = int(E('BS', '64')); LR = float(E('LR', '1e-3')); WARM = 500
|
| 88 |
+
opt = torch.optim.AdamW(net.parameters(), LR, weight_decay=0.01); best = 9.0; t0 = time.time(); r = np.random.default_rng(2); hist = []
|
| 89 |
+
log(f'params {sum(p.numel() for p in net.parameters()) / 1e6:.2f}M, steps {STEPS}, BS {BS}, LR {LR}')
|
| 90 |
+
for step in range(1, STEPS + 1):
|
| 91 |
+
lr = LR * min(1.0, step / WARM) * (0.05 + 0.95 * 0.5 * (1 + math.cos(math.pi * step / STEPS)))
|
| 92 |
+
for g in opt.param_groups: g['lr'] = lr
|
| 93 |
+
ids = r.choice(tr, BS); se = torch.from_numpy(SE[[ref_of(i, r) for i in ids]])
|
| 94 |
+
Tk, Tm, Se = batch_of([TK[i] for i in ids], se); drop = (torch.rand(BS, device=dev) < PSTYLE_DROP)[:, None]; Se = torch.where(drop, net.null[None].expand_as(Se), Se)
|
| 95 |
+
Y = np.zeros(Tm.shape, np.float32); M = np.zeros(Tm.shape, np.float32) # built on CPU, one copy: per-row slice writes into an MPS tensor compile a graph per (row length, U) and stalled planner1-3 for minutes
|
| 96 |
+
for b, i in enumerate(ids): Y[b, 1:1 + len(DU[i])] = np.log1p(DU[i].astype(np.float32)); M[b, 1:1 + len(DU[i])] = 1
|
| 97 |
+
Y = torch.from_numpy(Y).to(dev); M = torch.from_numpy(M).to(dev)
|
| 98 |
+
net.train(); loss = ((net(Tk, Tm, Se) - Y).abs() * M).sum() / M.sum(); opt.zero_grad(); loss.backward(); nn.utils.clip_grad_norm_(net.parameters(), 1.0); opt.step(); hist.append(loss.item())
|
| 99 |
+
stop = (time.time() - t0) / 60 > float(E('MAXMIN', '40'))
|
| 100 |
+
if step % 100 == 0: log(f' {step}/{STEPS} loss {np.mean(hist[-100:]):.4f} {time.time() - t0:.0f}s')
|
| 101 |
+
if step % 1000 == 0 or step == STEPS or stop:
|
| 102 |
+
cn = copy.deepcopy(net).cpu().eval() # held-out check on a CPU copy: planner1-4 stalled for minutes on MPS right after the eval
|
| 103 |
+
rs = evaluate(lambda tl, ids_: predict(cn, tl, torch.from_numpy(SE[[vref[i] for i in ids_]])))
|
| 104 |
+
rn = evaluate(lambda tl, ids_: predict(cn, tl, None)); del cn
|
| 105 |
+
log(f'step {step} loss {np.mean(hist[-1000:]):.4f} {time.time() - t0:.0f}s\n with ref ' + fmt(rs) + '\n no ref ' + fmt(rn))
|
| 106 |
+
if rs['letter'] < best: best = rs['letter']; torch.save(dict(m=net.state_dict(), itos=itos, step=step, held=rs, held_noref=rn), os.path.join(OUT, 'best.pt'))
|
| 107 |
+
if stop: log(f'MAXMIN reached at step {step}'); break
|
| 108 |
+
log(f'PLANNER DONE best letter x{best:.3f} -> {OUT}/best.pt')
|
scripts/write_pages.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# -*- coding: utf-8 -*-
|
| 2 |
+
"""06.11 (Лев: «подготовь в отдельной папке полный продукт для генерации страниц с формулами и текстом моей рукой»): a text file ->
|
| 3 |
+
notebook pages in the hand of ONE reference line.
|
| 4 |
+
input : UTF-8 text, one line of the notes per line; $...$ = a formula (LaTeX); an empty line = a paragraph gap;
|
| 5 |
+
a line starting with '# ' = a heading (written the same way, with extra space above). Long lines are wrapped at WRAP
|
| 6 |
+
characters (never inside $...$).
|
| 7 |
+
prose (no $) -> B (b4styc@3000 + planner7, CFG 1.5, 20 steps), style from the reference line
|
| 8 |
+
formula (whole $..$) -> formula_line.compose_em: MathJax layout + glyph denoiser (GDN_CKPT; default the general u2s),
|
| 9 |
+
scaled so its x-height = the page's x-height
|
| 10 |
+
mixed (text + $..$) -> B's line with every $...$ replaced by the module (formula_line.mixed_line)
|
| 11 |
+
the module fails -> B writes the line itself (counted in report.json)
|
| 12 |
+
Page: A4 proportions, lines on a baseline grid (pitch >= 3 x-heights, more when a line is taller), x-height from B's text.
|
| 13 |
+
python scripts/write_pages.py <in.txt> <out_dir> [REF=examples/ref_hwr200.npy WRAP=42 LINES_PER_PAGE=99 SEED=0 KEY=w03]
|
| 14 |
+
-> <out_dir>/pages.pdf (VECTOR), page_NN.svg, page_NN.png (300 dpi, antialiased), strokes.json (re-render with render_pages.py),
|
| 15 |
+
report.json (per line: kind, seconds, fallbacks, glyph sources). Refuses to overwrite."""
|
| 16 |
+
import os, sys, json, re, time, collections, numpy as np
|
| 17 |
+
P = os.path.dirname(os.path.dirname(os.path.abspath(__file__))); sys.path.insert(0, os.path.join(P, 'scripts')); E = os.environ.get
|
| 18 |
+
IN, OUT = sys.argv[1], sys.argv[2]
|
| 19 |
+
if os.path.exists(OUT): sys.exit(f'{OUT} exists')
|
| 20 |
+
if os.path.exists(os.path.join(P, 'defaults.json')): # a packaged release keeps its default models and paths here (the env still wins)
|
| 21 |
+
for _k, _v in json.load(open(os.path.join(P, 'defaults.json'))).items(): os.environ.setdefault(_k, _v if os.path.isabs(_v) or not _k.endswith(('CKPT', 'FW_NS')) else os.path.join(P, _v))
|
| 22 |
+
os.environ.setdefault('CFG', '1.5'); os.environ.setdefault('SSTEPS', '20')
|
| 23 |
+
os.environ.setdefault('GDN_CKPT', os.path.join(P, 'ckpt_glyph_dn_u2s', 'fp16', 'step050000.pt'))
|
| 24 |
+
import torch, formula_line as FL, formula_write as FW, glyph_dn as GD, train_ink_denoise as D
|
| 25 |
+
|
| 26 |
+
# ---------------- text -> lines
|
| 27 |
+
WRAP = int(E('WRAP', '42')); KEY = E('KEY', 'ref')
|
| 28 |
+
def wrap(s):
|
| 29 |
+
parts = re.split(r'(\$[^$]*\$)', s); out, cur = [], ''
|
| 30 |
+
for p in parts:
|
| 31 |
+
for w in ([p] if p.startswith('$') else re.split(r'(?<= )', p)):
|
| 32 |
+
if not w: continue
|
| 33 |
+
if len(cur) + len(w) > WRAP and cur.strip(): out.append(cur.rstrip()); cur = w.lstrip()
|
| 34 |
+
else: cur += w
|
| 35 |
+
if cur.strip(): out.append(cur.rstrip())
|
| 36 |
+
return out
|
| 37 |
+
items = [] # (text or None for a gap, heading?)
|
| 38 |
+
for raw in open(IN, encoding='utf-8').read().splitlines():
|
| 39 |
+
s = raw.strip()
|
| 40 |
+
if not s: items.append((None, False)); continue
|
| 41 |
+
head = s.startswith('# '); s = s[2:].strip() if head else s
|
| 42 |
+
if head: items.append((None, False))
|
| 43 |
+
for l in wrap(s):
|
| 44 |
+
m = re.fullmatch(r'\s*\$([^$]+)\$\s*([.,:;!?]*)\s*', l) # a wrapped tail «$...$:» -> the formula with its punctuation inside
|
| 45 |
+
items.append(('$' + m.group(1) + m.group(2) + '$' if m else l, head)) # (as text the module refused it: no letters for the x-height)
|
| 46 |
+
def kind(lab):
|
| 47 |
+
if '$' not in lab: return 'prose'
|
| 48 |
+
s = lab.strip(); return 'formula' if s.startswith('$') and s.endswith('$') and s.strip('$').count('$') == 0 else 'mixed'
|
| 49 |
+
unk = sorted({c for t, _ in items if t for c in t if c not in D.stoi})
|
| 50 |
+
if unk: print('characters B does not know (dropped from prose):', ''.join(unk), flush=True)
|
| 51 |
+
|
| 52 |
+
# ---------------- models
|
| 53 |
+
B = FL.setup(os.path.join(P, E('B_CKPT', 'ckpt_dl/b4styc/fp16/step003000_fp16.pt'))); net, base, TP, pnet = B['net'], B['base'], B['TP'], B['pnet']
|
| 54 |
+
ref = np.load(os.path.join(P, E('REF', 'examples/ref_hwr200.npy'))).astype(np.float32)
|
| 55 |
+
print(f'reference line {E("REF", "examples/ref_hwr200.npy")} ({len(ref)} points), text model {E("B_CKPT", "ckpt_dl/b4styc/fp16/step003000_fp16.pt")}, glyph model {E("GDN_CKPT")}', flush=True)
|
| 56 |
+
def absl(q): a = np.cumsum(np.asarray(q, np.float32)[:, :2], 0); return a, np.asarray(q, np.float32)[:, 2]
|
| 57 |
+
def xheight(q):
|
| 58 |
+
"""B normalises every line by its own height, so the x-height is measured per line: the SHORTEST y-interval holding 60% of the
|
| 59 |
+
inked points (the x-band) x 1.29 (= the old 15..85% spread on long lines: 8 B lines of >= 25 chars, 06.11). The old 15..85%
|
| 60 |
+
spread blew up on short words with ascenders / descenders («Проба» came out half size). Baseline = the bottom of that band."""
|
| 61 |
+
a, pen = absl(q); y = np.sort(a[pen < 0.5, 1] if (pen < 0.5).sum() > 10 else a[:, 1]); n = len(y); k = max(1, int(0.6 * n))
|
| 62 |
+
if n - k < 1: return max(float(y[-1] - y[0]), 1e-3), float(y[-1])
|
| 63 |
+
w = y[k:] - y[:n - k]; j = int(np.argmin(w)); return max(1.29 * float(w[j]), 1e-3), float(y[j + k])
|
| 64 |
+
rng = np.random.default_rng(int(E('SEED', '0'))); torch.manual_seed(int(E('SEED', '0')))
|
| 65 |
+
lines = [(k, t) for k, (t, _) in enumerate(items) if t]; T0 = time.time(); rep = []
|
| 66 |
+
toks = {k: [c for c in t if c in D.stoi] for k, t in lines}; ks = {k: kind(t) for k, t in lines}
|
| 67 |
+
bi = [k for k, _ in lines if ks[k] != 'formula']; seqB = {}; durs = {}
|
| 68 |
+
if bi:
|
| 69 |
+
sv1 = D.style_of(net, base, [ref]); se1 = D.semb_vec([ref])
|
| 70 |
+
for i0 in range(0, len(bi), 16):
|
| 71 |
+
kk = bi[i0:i0 + 16]; tl = [toks[k] for k in kk]; dd = TP.predict(pnet, tl, torch.cat([se1] * len(kk), 0)); net._tok_base = None
|
| 72 |
+
for k, d, q in zip(kk, dd, D.sample(net, tl, dd, torch.cat([sv1] * len(kk), 0))): seqB[k] = np.asarray(q, np.float32); durs[k] = d
|
| 73 |
+
print(f' B: {min(i0 + 16, len(bi))}/{len(bi)} lines', flush=True)
|
| 74 |
+
xs = [xheight(seqB[k]) for k in bi if ks[k] == 'prose'] or [xheight(seqB[k]) for k in bi]; t_b = time.time() - T0
|
| 75 |
+
out = {}; FW.SEL_STATS.clear(); GD.N.clear() if hasattr(GD, 'N') else None
|
| 76 |
+
for n_, (k, t) in enumerate(lines):
|
| 77 |
+
t0 = time.time(); r = dict(line=k, kind=ks[k], text=t)
|
| 78 |
+
if ks[k] == 'formula':
|
| 79 |
+
st, ms = FL.compose_em(t, ref, rng, key=KEY); r['missing'] = ms
|
| 80 |
+
if st: a = FL.to_line_abs(st, 1 / 0.442); out[k] = (a[:, :2], a[:, 2])
|
| 81 |
+
elif ks[k] == 'mixed':
|
| 82 |
+
q2, ms, nr = FL.mixed_line(seqB[k], toks[k], durs[k], ref, rng, key=KEY); r['missing'] = ms
|
| 83 |
+
if q2 is not None:
|
| 84 |
+
xh, bl = xheight(seqB[k]); a, pen = absl(q2); out[k] = ((a - [0, bl]) / xh, pen)
|
| 85 |
+
else:
|
| 86 |
+
xh, bl = xheight(seqB[k]); a, pen = absl(seqB[k]); out[k] = ((a - [0, bl]) / xh, pen)
|
| 87 |
+
if k not in out: # the module failed -> B writes the whole line
|
| 88 |
+
r['fallback_B'] = True; tq = toks[k]; d_ = TP.predict(pnet, [tq], D.semb_vec([ref])); net._tok_base = None
|
| 89 |
+
q = np.asarray(D.sample(net, [tq], d_, D.style_of(net, base, [ref]))[0], np.float32); xh, bl = xheight(q); a, pen = absl(q); out[k] = ((a - [0, bl]) / xh, pen)
|
| 90 |
+
r['sec'] = round(time.time() - t0, 2); rep.append(r)
|
| 91 |
+
if ks[k] != 'prose': print(f' [{n_ + 1}/{len(lines)}] {ks[k]} {r["sec"]:.1f} s{" FALLBACK" if r.get("fallback_B") else ""}: {t[:60]}', flush=True)
|
| 92 |
+
|
| 93 |
+
# ---------------- pages
|
| 94 |
+
XH = float(E('XH', '15')); W, H = 1240, 1754; ML, MT = 90, 110; LPP = int(E('LINES_PER_PAGE', '99')); PITCH = 3.2 * XH
|
| 95 |
+
def on_grid(v): return MT + np.ceil((v - MT) / PITCH - 1e-6) * PITCH # the next ruling line at or below v
|
| 96 |
+
RULING = [float(v) for v in np.arange(MT, H - 60, PITCH)] # faint ruling; baselines sit on it
|
| 97 |
+
pages = [[]]; y = MT - PITCH; prev_bot = float(MT - 2 * XH); nline = 0 # 06.11: pages are kept as strokes in page px -> render_pages (vector PDF, SVG, 300 dpi PNG)
|
| 98 |
+
for k, (t, head) in enumerate(items):
|
| 99 |
+
if t is None:
|
| 100 |
+
y += PITCH; prev_bot += PITCH; continue # a paragraph gap = one empty ruling line
|
| 101 |
+
a, pen = out[k]; top, bot = a[:, 1].min(), a[:, 1].max(); s = XH; wid = (a[:, 0].max() - a[:, 0].min()) * s
|
| 102 |
+
if wid > W - ML - 50: s *= (W - ML - 50) / wid
|
| 103 |
+
base_y = on_grid(max(y + PITCH, prev_bot + 0.6 * XH - top * s))
|
| 104 |
+
if base_y + bot * s > H - 60 or nline >= LPP:
|
| 105 |
+
pages.append([]); y = MT - PITCH; prev_bot = float(MT - 2 * XH); nline = 0; base_y = on_grid(max(y + PITCH, prev_bot + 0.6 * XH - top * s))
|
| 106 |
+
x0 = a[:, 0].min(); ind = 0 if ks.get(k) != 'formula' else 40; cur = []
|
| 107 |
+
for (x, yy), p in zip(a, pen):
|
| 108 |
+
cur.append([round(float(ML + ind + (x - x0) * s), 2), round(float(base_y + yy * s), 2)])
|
| 109 |
+
if p > 0.5: pages[-1].append(cur); cur = []
|
| 110 |
+
if cur: pages[-1].append(cur)
|
| 111 |
+
y = base_y; prev_bot = base_y + bot * s; nline += 1
|
| 112 |
+
os.makedirs(OUT); doc = dict(W=W, H=H, dpi=150, ruling=RULING, pages=pages)
|
| 113 |
+
json.dump(doc, open(os.path.join(OUT, 'strokes.json'), 'w')); import render_pages; render_pages.render(doc, OUT)
|
| 114 |
+
src = dict(FW.SEL_STATS); src.update({f'glyph_net_{k}': v for k, v in getattr(GD, 'N', {}).items()})
|
| 115 |
+
json.dump(dict(input=os.path.abspath(IN), pages=len(pages), lines=len(lines), kinds=dict(collections.Counter(ks.values())), sec_total=round(time.time() - T0, 1),
|
| 116 |
+
sec_B=round(t_b, 1), fallbacks=sum(1 for r in rep if r.get('fallback_B')), glyph_sources=src, glyph_model=E('GDN_CKPT'), text_model=E('B_CKPT', 'ckpt_dl/b4styc/fp16/step003000_fp16.pt'), reference=E('REF', 'examples/ref_hwr200.npy'),
|
| 117 |
+
per_line=rep), open(os.path.join(OUT, 'report.json'), 'w'), ensure_ascii=False, indent=1)
|
| 118 |
+
print(f'PAGES DONE {len(pages)} pages, {len(lines)} lines in {time.time() - T0:.0f} s -> {OUT} | glyph sources {src}', flush=True)
|
tools/mathjax/layout.mjs
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// LaTeX math -> glyph layout with MathJax 3 (SVG output, no font cache).
|
| 2 |
+
// stdin: JSON lines {"id":..., "tex":"..."}; stdout: JSON lines {"id", "ok", "glyphs":[{"c":<codepoint hex or "BAR">,
|
| 3 |
+
// "bbox":[x0,y0,x1,y1], "node":<mml node>}], "w", "h", "err"}. Units: 1/1000 em, y grows DOWN (page convention).
|
| 4 |
+
// Glyph bbox = extent of the outline's absolute coordinates (MathJax glyph paths use absolute M/L/Q/T commands) under the
|
| 5 |
+
// accumulated translate/scale chain; <rect> (fraction bars, radical overbars, table rules) are emitted as c="BAR".
|
| 6 |
+
// Used for (1) free labelling of handwritten formula glyphs (align them to the layout of the line's LaTeX label) and
|
| 7 |
+
// (2) placing generated glyphs when writing formulas. Usage: node layout.mjs < in.jsonl > out.jsonl
|
| 8 |
+
import {mathjax} from 'mathjax-full/js/mathjax.js';
|
| 9 |
+
import {TeX} from 'mathjax-full/js/input/tex.js';
|
| 10 |
+
import {SVG} from 'mathjax-full/js/output/svg.js';
|
| 11 |
+
import {liteAdaptor} from 'mathjax-full/js/adaptors/liteAdaptor.js';
|
| 12 |
+
import {RegisterHTMLHandler} from 'mathjax-full/js/handlers/html.js';
|
| 13 |
+
import {AllPackages} from 'mathjax-full/js/input/tex/AllPackages.js';
|
| 14 |
+
import readline from 'node:readline';
|
| 15 |
+
const adaptor=liteAdaptor(); RegisterHTMLHandler(adaptor);
|
| 16 |
+
const tex=new TeX({packages:AllPackages, formatError:(jax,err)=>{throw err;}});
|
| 17 |
+
const doc=mathjax.document('',{InputJax:tex,OutputJax:new SVG({fontCache:'none'})});
|
| 18 |
+
function parseT(s){ // -> affine [a,b,c,d,e,f] (x'=a x + c y + e, y'=b x + d y + f)
|
| 19 |
+
let m=[1,0,0,1,0,0]; if(!s) return m;
|
| 20 |
+
const re=/(translate|scale)\(([^)]*)\)/g; let r;
|
| 21 |
+
while((r=re.exec(s))){ const v=r[2].split(/[ ,]+/).filter(x=>x!=='').map(Number); let t;
|
| 22 |
+
if(r[1]==='translate') t=[1,0,0,1,v[0]||0,v[1]||0]; else t=[v[0],0,0,(v.length>1?v[1]:v[0]),0,0];
|
| 23 |
+
m=mul(m,t); }
|
| 24 |
+
return m; }
|
| 25 |
+
function mul(A,B){ return [A[0]*B[0]+A[2]*B[1], A[1]*B[0]+A[3]*B[1], A[0]*B[2]+A[2]*B[3], A[1]*B[2]+A[3]*B[3], A[0]*B[4]+A[2]*B[5]+A[4], A[1]*B[4]+A[3]*B[5]+A[5]]; }
|
| 26 |
+
function app(M,x,y){ return [M[0]*x+M[2]*y+M[4], M[1]*x+M[3]*y+M[5]]; }
|
| 27 |
+
function boxOf(M,pts){ let x0=1e9,y0=1e9,x1=-1e9,y1=-1e9; for(const [x,y] of pts){ const [u,v]=app(M,x,y); x0=Math.min(x0,u); x1=Math.max(x1,u); y0=Math.min(y0,v); y1=Math.max(y1,v);} return [x0,y0,x1,y1]; }
|
| 28 |
+
function pathPoints(d){ // SVG path -> list of points (end points and control points), handles M L T Q C S H V Z, abs and rel
|
| 29 |
+
const tk=d.match(/[A-Za-z]|-?\d*\.?\d+(?:e-?\d+)?/g)||[]; const pts=[]; let cx=0,cy=0,sx=0,sy=0,cmd='M',i=0;
|
| 30 |
+
const n={M:2,L:2,T:2,Q:4,C:6,S:4,H:1,V:1,Z:0,A:7};
|
| 31 |
+
while(i<tk.length){ if(/[A-Za-z]/.test(tk[i])){ cmd=tk[i]; i++; if(cmd==='Z'||cmd==='z'){ cx=sx; cy=sy; continue; } }
|
| 32 |
+
const U=cmd.toUpperCase(), rel=cmd!==U, k=n[U]; if(!k){ i++; continue; }
|
| 33 |
+
const v=tk.slice(i,i+k).map(Number); if(v.length<k) break; i+=k;
|
| 34 |
+
if(U==='H'){ cx=rel?cx+v[0]:v[0]; pts.push([cx,cy]); continue; }
|
| 35 |
+
if(U==='V'){ cy=rel?cy+v[0]:v[0]; pts.push([cx,cy]); continue; }
|
| 36 |
+
if(U==='A'){ cx=rel?cx+v[5]:v[5]; cy=rel?cy+v[6]:v[6]; pts.push([cx,cy]); continue; }
|
| 37 |
+
for(let j=0;j<k;j+=2){ const x=rel?cx+v[j]:v[j], y=rel?cy+v[j+1]:v[j+1]; pts.push([x,y]); if(j===k-2){ cx=x; cy=y; } }
|
| 38 |
+
if(U==='M'){ sx=cx; sy=cy; cmd=rel?'l':'L'; } }
|
| 39 |
+
return pts; }
|
| 40 |
+
function walk(node,M,out,mml){
|
| 41 |
+
if(!adaptor.kind || typeof node!=='object') return;
|
| 42 |
+
const kind=adaptor.kind(node); if(kind==='#text'||kind==='#comment') return;
|
| 43 |
+
const t=adaptor.getAttribute(node,'transform'); const M2=t?mul(M,parseT(t)):M;
|
| 44 |
+
const mm=adaptor.getAttribute(node,'data-mml-node')||mml;
|
| 45 |
+
if(kind==='path'){ const c=adaptor.getAttribute(node,'data-c'); const d=adaptor.getAttribute(node,'d')||'';
|
| 46 |
+
const pts=pathPoints(d);
|
| 47 |
+
if(c&&pts.length) out.push({c, bbox:boxOf(M2,pts), node:mm}); return; }
|
| 48 |
+
if(kind==='text' && process.env.LAYOUT_TEXT==='1'){ // 22.10: unknown-font text (Cyrillic in \\text{}) -> one TEXT box per char:
|
| 49 |
+
// advance 600 (MathJax's estimate, see translate(600k) of the siblings), x-height 450 above the baseline, in the accumulated frame
|
| 50 |
+
const txt=(adaptor.textContent?adaptor.textContent(node):'')||''; const sc=Math.hypot(M2[0],M2[1])||1; const [bx,by]=app(M2,0,0);
|
| 51 |
+
out.push({c:'TEXT', ch:txt, bbox:[bx,by-450*sc,bx+600*sc,by], node:mm}); return; }
|
| 52 |
+
if(kind==='rect'){ const x=+adaptor.getAttribute(node,'x')||0, y=+adaptor.getAttribute(node,'y')||0, w=+adaptor.getAttribute(node,'width')||0, h=+adaptor.getAttribute(node,'height')||0;
|
| 53 |
+
out.push({c:'BAR', bbox:boxOf(M2,[[x,y],[x+w,y+h]]), node:mm}); return; }
|
| 54 |
+
for(const ch of adaptor.childNodes(node)) walk(ch,M2,out,mm);
|
| 55 |
+
}
|
| 56 |
+
function layout(s){
|
| 57 |
+
const node=doc.convert(s,{display:true}); const svg=adaptor.firstChild(node);
|
| 58 |
+
const out=[]; walk(svg,[1,0,0,1,0,0],out,null);
|
| 59 |
+
// the top <g> flips y (scale(1,-1)), so y already grows down in the accumulated frame
|
| 60 |
+
for(const g of out){ const b=g.bbox; g.bbox=[+b[0].toFixed(1),+Math.min(b[1],b[3]).toFixed(1),+b[2].toFixed(1),+Math.max(b[1],b[3]).toFixed(1)]; }
|
| 61 |
+
const err=adaptor.getAttribute(svg,'data-mjx-error')||null; let x0=1e9,y0=1e9,x1=-1e9,y1=-1e9;
|
| 62 |
+
for(const g of out){ x0=Math.min(x0,g.bbox[0]); y0=Math.min(y0,g.bbox[1]); x1=Math.max(x1,g.bbox[2]); y1=Math.max(y1,g.bbox[3]); }
|
| 63 |
+
return {glyphs:out, w:x1-x0, h:y1-y0, err};
|
| 64 |
+
}
|
| 65 |
+
const rl=readline.createInterface({input:process.stdin});
|
| 66 |
+
for await (const line of rl){ if(!line.trim()) continue; let o; try{o=JSON.parse(line);}catch(e){continue;}
|
| 67 |
+
try{ const r=layout(o.tex); console.log(JSON.stringify({id:o.id, ok:!r.err && r.glyphs.length>0, ...r})); }
|
| 68 |
+
catch(e){ console.log(JSON.stringify({id:o.id, ok:false, err:String(e.message||e).slice(0,200)})); } }
|
tools/mathjax/package-lock.json
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "mathjax",
|
| 3 |
+
"version": "1.0.0",
|
| 4 |
+
"lockfileVersion": 3,
|
| 5 |
+
"requires": true,
|
| 6 |
+
"packages": {
|
| 7 |
+
"": {
|
| 8 |
+
"name": "mathjax",
|
| 9 |
+
"version": "1.0.0",
|
| 10 |
+
"license": "ISC",
|
| 11 |
+
"dependencies": {
|
| 12 |
+
"mathjax-full": "^3.2.2"
|
| 13 |
+
}
|
| 14 |
+
},
|
| 15 |
+
"node_modules/@xmldom/xmldom": {
|
| 16 |
+
"version": "0.9.10",
|
| 17 |
+
"resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.9.10.tgz",
|
| 18 |
+
"integrity": "sha512-A9gOqLdi6cV4ibazAjcQufGj0B1y/vDqYrcuP6d/6x8P27gRS8643Dj9o1dEKtB6O7fwxb2FgBmJS2mX7gpvdw==",
|
| 19 |
+
"deprecated": "this version has critical issues, please update to the latest version",
|
| 20 |
+
"license": "MIT",
|
| 21 |
+
"engines": {
|
| 22 |
+
"node": ">=14.6"
|
| 23 |
+
}
|
| 24 |
+
},
|
| 25 |
+
"node_modules/commander": {
|
| 26 |
+
"version": "13.1.0",
|
| 27 |
+
"resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz",
|
| 28 |
+
"integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==",
|
| 29 |
+
"license": "MIT",
|
| 30 |
+
"engines": {
|
| 31 |
+
"node": ">=18"
|
| 32 |
+
}
|
| 33 |
+
},
|
| 34 |
+
"node_modules/esm": {
|
| 35 |
+
"version": "3.2.25",
|
| 36 |
+
"resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz",
|
| 37 |
+
"integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==",
|
| 38 |
+
"license": "MIT",
|
| 39 |
+
"engines": {
|
| 40 |
+
"node": ">=6"
|
| 41 |
+
}
|
| 42 |
+
},
|
| 43 |
+
"node_modules/mathjax-full": {
|
| 44 |
+
"version": "3.2.2",
|
| 45 |
+
"resolved": "https://registry.npmjs.org/mathjax-full/-/mathjax-full-3.2.2.tgz",
|
| 46 |
+
"integrity": "sha512-+LfG9Fik+OuI8SLwsiR02IVdjcnRCy5MufYLi0C3TdMT56L/pjB0alMVGgoWJF8pN9Rc7FESycZB9BMNWIid5w==",
|
| 47 |
+
"deprecated": "Version 4 replaces this package with the scoped package @mathjax/src",
|
| 48 |
+
"license": "Apache-2.0",
|
| 49 |
+
"dependencies": {
|
| 50 |
+
"esm": "^3.2.25",
|
| 51 |
+
"mhchemparser": "^4.1.0",
|
| 52 |
+
"mj-context-menu": "^0.6.1",
|
| 53 |
+
"speech-rule-engine": "^4.0.6"
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
"node_modules/mhchemparser": {
|
| 57 |
+
"version": "4.2.1",
|
| 58 |
+
"resolved": "https://registry.npmjs.org/mhchemparser/-/mhchemparser-4.2.1.tgz",
|
| 59 |
+
"integrity": "sha512-kYmyrCirqJf3zZ9t/0wGgRZ4/ZJw//VwaRVGA75C4nhE60vtnIzhl9J9ndkX/h6hxSN7pjg/cE0VxbnNM+bnDQ==",
|
| 60 |
+
"license": "Apache-2.0"
|
| 61 |
+
},
|
| 62 |
+
"node_modules/mj-context-menu": {
|
| 63 |
+
"version": "0.6.1",
|
| 64 |
+
"resolved": "https://registry.npmjs.org/mj-context-menu/-/mj-context-menu-0.6.1.tgz",
|
| 65 |
+
"integrity": "sha512-7NO5s6n10TIV96d4g2uDpG7ZDpIhMh0QNfGdJw/W47JswFcosz457wqz/b5sAKvl12sxINGFCn80NZHKwxQEXA==",
|
| 66 |
+
"license": "Apache-2.0"
|
| 67 |
+
},
|
| 68 |
+
"node_modules/speech-rule-engine": {
|
| 69 |
+
"version": "4.1.4",
|
| 70 |
+
"resolved": "https://registry.npmjs.org/speech-rule-engine/-/speech-rule-engine-4.1.4.tgz",
|
| 71 |
+
"integrity": "sha512-i/VCLG1fvRc95pMHRqG4aQNscv+9aIsqA2oI7ZQS51sTdUcDHYX6cpT8/tqZ+enjs1tKVwbRBWgxut9SWn+f9g==",
|
| 72 |
+
"license": "Apache-2.0",
|
| 73 |
+
"dependencies": {
|
| 74 |
+
"@xmldom/xmldom": "0.9.10",
|
| 75 |
+
"commander": "13.1.0",
|
| 76 |
+
"wicked-good-xpath": "1.3.0"
|
| 77 |
+
},
|
| 78 |
+
"bin": {
|
| 79 |
+
"sre": "bin/sre"
|
| 80 |
+
}
|
| 81 |
+
},
|
| 82 |
+
"node_modules/wicked-good-xpath": {
|
| 83 |
+
"version": "1.3.0",
|
| 84 |
+
"resolved": "https://registry.npmjs.org/wicked-good-xpath/-/wicked-good-xpath-1.3.0.tgz",
|
| 85 |
+
"integrity": "sha512-Gd9+TUn5nXdwj/hFsPVx5cuHHiF5Bwuc30jZ4+ronF1qHK5O7HD0sgmXWSEgwKquT3ClLoKPVbO6qGwVwLzvAw==",
|
| 86 |
+
"license": "MIT"
|
| 87 |
+
}
|
| 88 |
+
}
|
| 89 |
+
}
|
tools/mathjax/package.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "mathjax",
|
| 3 |
+
"version": "1.0.0",
|
| 4 |
+
"description": "",
|
| 5 |
+
"main": "index.js",
|
| 6 |
+
"scripts": {
|
| 7 |
+
"test": "echo \"Error: no test specified\" && exit 1"
|
| 8 |
+
},
|
| 9 |
+
"keywords": [],
|
| 10 |
+
"author": "",
|
| 11 |
+
"license": "ISC",
|
| 12 |
+
"type": "commonjs",
|
| 13 |
+
"dependencies": {
|
| 14 |
+
"mathjax-full": "^3.2.2"
|
| 15 |
+
}
|
| 16 |
+
}
|