CORTEG Stanford pooled model, converted from the official release
Browse files- LICENSE +50 -0
- README.md +93 -0
- config.json +26 -0
- convert_corteg_checkpoint.py +370 -0
- model.safetensors +3 -0
- pytorch_model.bin +3 -0
- report.json +18 -0
LICENSE
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
The CORTEG adapter (checkpoints/corteg_stanford_pooled.pt of
|
| 2 |
+
https://github.com/LiuyinYang1101/CORTEG at caafb78fbebdd9448814f96d7e61083a3eb62663):
|
| 3 |
+
|
| 4 |
+
MIT License
|
| 5 |
+
|
| 6 |
+
Copyright (c) 2026 Liuyin Yang
|
| 7 |
+
|
| 8 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 9 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 10 |
+
in the Software without restriction, including without limitation the rights
|
| 11 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 12 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 13 |
+
furnished to do so, subject to the following conditions:
|
| 14 |
+
|
| 15 |
+
The above copyright notice and this permission notice shall be included in all
|
| 16 |
+
copies or substantial portions of the Software.
|
| 17 |
+
|
| 18 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 19 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 20 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 21 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 22 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 23 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 24 |
+
SOFTWARE.
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
The ST-EEGFormer small backbone (release ST-EEGFormer-small of
|
| 28 |
+
https://github.com/LiuyinYang1101/STEEGFormer):
|
| 29 |
+
|
| 30 |
+
MIT License
|
| 31 |
+
|
| 32 |
+
Copyright (c) 2025 LIUYIN YANG
|
| 33 |
+
|
| 34 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 35 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 36 |
+
in the Software without restriction, including without limitation the rights
|
| 37 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 38 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 39 |
+
furnished to do so, subject to the following conditions:
|
| 40 |
+
|
| 41 |
+
The above copyright notice and this permission notice shall be included in all
|
| 42 |
+
copies or substantial portions of the Software.
|
| 43 |
+
|
| 44 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 45 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 46 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 47 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 48 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 49 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 50 |
+
SOFTWARE.
|
README.md
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: braindecode
|
| 3 |
+
license: mit
|
| 4 |
+
base_model: braindecode/STEEGFormer-small
|
| 5 |
+
tags:
|
| 6 |
+
- braindecode
|
| 7 |
+
- CORTEG
|
| 8 |
+
- ieeg
|
| 9 |
+
- ecog
|
| 10 |
+
- finger-flexion
|
| 11 |
+
- pytorch_model_hub_mixin
|
| 12 |
+
---
|
| 13 |
+
|
| 14 |
+
# CORTEG, Stanford finger flexion (pooled)
|
| 15 |
+
|
| 16 |
+
The released CORTEG model of [Yang et al. (2026)](https://arxiv.org/abs/2605.10337),
|
| 17 |
+
ST-EEGFormer small adapted to ECoG and trained on the finger-flexion recordings
|
| 18 |
+
of the 9 Stanford patients pooled (Table 1, "CORTEG (pooled)", Pearson r 0.554),
|
| 19 |
+
25.5M parameters, converted to
|
| 20 |
+
[`braindecode.models.CORTEG`](https://braindecode.org/stable/generated/braindecode.models.CORTEG.html).
|
| 21 |
+
It decodes the flexion of the five fingers (thumb, index, middle, ring, pinky) from 1 s of ECoG.
|
| 22 |
+
|
| 23 |
+
Sources:
|
| 24 |
+
|
| 25 |
+
- the adapter, `checkpoints/corteg_stanford_pooled.pt` of [LiuyinYang1101/CORTEG](https://github.com/LiuyinYang1101/CORTEG/tree/caafb78fbebdd9448814f96d7e61083a3eb62663),
|
| 26 |
+
sha256 `77d63a214e0e0f2c790e90878a0ef6b525f6e24e1453fcd94e118c63ff7e8396`: the spatial adapter, LoRA factors (r=4, alpha=16) on
|
| 27 |
+
blocks 4-7, the high-gamma patch embedding, those blocks' LayerNorms and the
|
| 28 |
+
readout, 297,236 parameters;
|
| 29 |
+
- the frozen backbone it was trained on, the ST-EEGFormer small release
|
| 30 |
+
([`checkpoint-300.pth`](https://github.com/LiuyinYang1101/STEEGFormer/releases/download/ST-EEGFormer-small/checkpoint-300.pth), sha256 `98d2848abdaea7a143d399c07f67f8782e07dd66e7c309ddb058c7158a8a8b02`), here
|
| 31 |
+
through its conversion [braindecode/STEEGFormer-small](https://huggingface.co/braindecode/STEEGFormer-small)
|
| 32 |
+
at revision `633c9c0d`.
|
| 33 |
+
|
| 34 |
+
Conversion folds the LoRA factors into the fused qkv, proj, fc1 and fc2 weights
|
| 35 |
+
(scale alpha/r = 4), so the model is a plain `CORTEG`;
|
| 36 |
+
`convert_corteg_checkpoint.py` in this repository reproduces it. Outputs match
|
| 37 |
+
the authors' `load_corteg` on the two release files to 1.5e-06
|
| 38 |
+
in float32 on CPU, for montages of 8 and 46
|
| 39 |
+
contacts.
|
| 40 |
+
|
| 41 |
+
## Usage
|
| 42 |
+
|
| 43 |
+
The model reads two streams of the same 1 s window, as the paper's Stanford
|
| 44 |
+
pipeline (`data/stanford_preprocessing/` of the official repository): after a
|
| 45 |
+
notch at 60, 120 and 180 Hz and a common average reference, `x` is the signal
|
| 46 |
+
resampled to 128 Hz and `x_hi` its 70-200 Hz Hilbert envelope at 200 Hz, each
|
| 47 |
+
z-scored per channel with the statistics of the training windows. Electrode
|
| 48 |
+
positions are in metres, in `chs_info` or `pos`.
|
| 49 |
+
|
| 50 |
+
```python
|
| 51 |
+
import torch
|
| 52 |
+
from braindecode.models import CORTEG
|
| 53 |
+
|
| 54 |
+
model = CORTEG.from_pretrained(
|
| 55 |
+
"braindecode/corteg-stanford-pooled", chs_info=raw.info["chs"]
|
| 56 |
+
).eval()
|
| 57 |
+
flexion = model(x, x_hi) # x: (batch, n_chans, 128), x_hi: (batch, n_chans, 200)
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
The output, `(batch, 5)`, is the flexion of each finger, z-scored per finger
|
| 61 |
+
with the training statistics of the patient, at the time 40 ms before the
|
| 62 |
+
window's end. The saved geometry is 8 channels; the model
|
| 63 |
+
reads any montage, so pass your own `chs_info` (or `n_chans` and `pos` at each
|
| 64 |
+
call) when loading.
|
| 65 |
+
|
| 66 |
+
## Limitations
|
| 67 |
+
|
| 68 |
+
The readout is at its random initialization, as released: the paper run built
|
| 69 |
+
it after the optimizer, so it was never trained, and the adapter learned around
|
| 70 |
+
it (see the release manifest, `checkpoints/corteg_stanford_pooled.json`).
|
| 71 |
+
The model was trained on the training split of the 9 Stanford patients (the
|
| 72 |
+
first 400 s of each recording, two thirds of one under 600 s); other patients
|
| 73 |
+
or tasks need fine-tuning, as in the paper's leave-one-subject-out protocol. The check above covers float32 CPU outputs, not
|
| 74 |
+
decoding accuracy, GPU kernels or mixed precision; the authors report that this
|
| 75 |
+
checkpoint reproduces the cohort mean of 0.554 (manifest, `paper.verified`).
|
| 76 |
+
|
| 77 |
+
## Citation
|
| 78 |
+
|
| 79 |
+
```bibtex
|
| 80 |
+
@inproceedings{yang2026corteg,
|
| 81 |
+
title = {{CORTEG}: Foundation Models Enable Cross-Modality Representation Transfer
|
| 82 |
+
from Scalp to Intracranial Brain Recordings},
|
| 83 |
+
author = {Yang, Liuyin and Sun, Qiang and Van Dyck, Bob and Calvo Merino, Eva and
|
| 84 |
+
Van Hulle, Marc M.},
|
| 85 |
+
booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
|
| 86 |
+
year = {2026}
|
| 87 |
+
}
|
| 88 |
+
```
|
| 89 |
+
|
| 90 |
+
## License
|
| 91 |
+
|
| 92 |
+
The adapter and the ST-EEGFormer backbone are both released by Liuyin Yang
|
| 93 |
+
under the MIT licence (`LICENSE`). Braindecode's code is BSD-3-Clause.
|
config.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_outputs": 5,
|
| 3 |
+
"n_chans": 8,
|
| 4 |
+
"chs_info": null,
|
| 5 |
+
"n_times": 128,
|
| 6 |
+
"input_window_seconds": null,
|
| 7 |
+
"sfreq": 128.0,
|
| 8 |
+
"patch_size": 16,
|
| 9 |
+
"hi_patch_size": 25,
|
| 10 |
+
"embed_dim": 512,
|
| 11 |
+
"depth": 8,
|
| 12 |
+
"num_heads": 8,
|
| 13 |
+
"mlp_ratio": 4,
|
| 14 |
+
"drop_prob": 0.0,
|
| 15 |
+
"drop_path": 0.1,
|
| 16 |
+
"activation": "torch.nn.modules.activation.GELU",
|
| 17 |
+
"n_chans_pos": 145,
|
| 18 |
+
"merge_strategy": "average",
|
| 19 |
+
"n_merge_blocks": 4,
|
| 20 |
+
"knn_k": 8,
|
| 21 |
+
"adapter_hidden": 128,
|
| 22 |
+
"n_fourier_freqs": 32,
|
| 23 |
+
"gate_bottleneck": 16,
|
| 24 |
+
"on_non_divisible": "pad",
|
| 25 |
+
"braindecode_version": "2.1.0dev0"
|
| 26 |
+
}
|
convert_corteg_checkpoint.py
ADDED
|
@@ -0,0 +1,370 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Authors: Julien Gadonneix <juliengado.2001@gmail.com>
|
| 2 |
+
# License: BSD (3-clause)
|
| 3 |
+
"""Convert, check and publish the released CORTEG Stanford model on CPU.
|
| 4 |
+
|
| 5 |
+
Run from an editable Braindecode checkout containing CORTEG, with the ``hub``
|
| 6 |
+
extra and ``timm`` and ``einops`` (for the reference) installed::
|
| 7 |
+
|
| 8 |
+
git clone https://github.com/LiuyinYang1101/CORTEG && git -C CORTEG checkout caafb78
|
| 9 |
+
curl -L -o checkpoint-300.pth \\
|
| 10 |
+
https://github.com/LiuyinYang1101/STEEGFormer/releases/download/ST-EEGFormer-small/checkpoint-300.pth
|
| 11 |
+
python convert_corteg_checkpoint.py --reference-dir CORTEG \\
|
| 12 |
+
--backbone checkpoint-300.pth --output-dir /tmp/corteg-converted
|
| 13 |
+
|
| 14 |
+
``--reference-dir`` is a clone of the official repository at ``REVISION``, which
|
| 15 |
+
ships the adapter (``ADAPTER``); ``--backbone`` is the ST-EEGFormer small
|
| 16 |
+
release it was trained on. The SHA-256 values below must match. The released
|
| 17 |
+
adapter holds the trainable parameters only: the spatial adapter, LoRA factors
|
| 18 |
+
on blocks 4-7, the high-gamma patch embedding, those blocks' LayerNorms and the
|
| 19 |
+
readout. The port takes the frozen backbone from its Braindecode conversion
|
| 20 |
+
(``BACKBONE_REPO`` at ``BACKBONE_REVISION``) and folds the LoRA factors into the
|
| 21 |
+
fused qkv, proj, fc1 and fc2 weights, so the published model is a plain
|
| 22 |
+
``CORTEG``. The authors' ``load_corteg`` builds the reference from the same two
|
| 23 |
+
release files; both run on the same inputs and their outputs must agree within
|
| 24 |
+
``TOLERANCE``. Download this script from the Braindecode CORTEG model
|
| 25 |
+
repository on Hugging Face; it is not bundled in the Braindecode source tree.
|
| 26 |
+
|
| 27 |
+
Add ``--push-to braindecode`` to upload the converted model, this script, the
|
| 28 |
+
MIT licences of the adapter and backbone, a report and a model card in one
|
| 29 |
+
commit.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
import argparse
|
| 33 |
+
import hashlib
|
| 34 |
+
import json
|
| 35 |
+
import shutil
|
| 36 |
+
import sys
|
| 37 |
+
import tempfile
|
| 38 |
+
from pathlib import Path
|
| 39 |
+
|
| 40 |
+
import numpy as np
|
| 41 |
+
import torch
|
| 42 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 43 |
+
from safetensors.torch import load_file
|
| 44 |
+
|
| 45 |
+
from braindecode.models import CORTEG
|
| 46 |
+
|
| 47 |
+
REPOSITORY = "https://github.com/LiuyinYang1101/CORTEG"
|
| 48 |
+
REVISION = "caafb78fbebdd9448814f96d7e61083a3eb62663"
|
| 49 |
+
ADAPTER = "checkpoints/corteg_stanford_pooled.pt"
|
| 50 |
+
ADAPTER_SHA256 = "77d63a214e0e0f2c790e90878a0ef6b525f6e24e1453fcd94e118c63ff7e8396"
|
| 51 |
+
ADAPTER_PARAMS = 297_236 # the release manifest's trainable_params
|
| 52 |
+
BACKBONE_URL = (
|
| 53 |
+
"https://github.com/LiuyinYang1101/STEEGFormer/releases/download/"
|
| 54 |
+
"ST-EEGFormer-small/checkpoint-300.pth"
|
| 55 |
+
)
|
| 56 |
+
BACKBONE_SHA256 = "98d2848abdaea7a143d399c07f67f8782e07dd66e7c309ddb058c7158a8a8b02"
|
| 57 |
+
BACKBONE_REPO = "braindecode/STEEGFormer-small"
|
| 58 |
+
BACKBONE_REVISION = "633c9c0d76c6e26459e1e0d1337c91858578f13d"
|
| 59 |
+
HUB_REPO = "corteg-stanford-pooled"
|
| 60 |
+
# The released run (manifest architecture_args): 1 s windows of 128 low-frequency
|
| 61 |
+
# and 200 high-gamma samples, five fingers, the "average" fusion before the last
|
| 62 |
+
# four blocks.
|
| 63 |
+
CONFIG = dict(
|
| 64 |
+
n_outputs=5,
|
| 65 |
+
n_times=128,
|
| 66 |
+
sfreq=128.0,
|
| 67 |
+
merge_strategy="average",
|
| 68 |
+
n_merge_blocks=4,
|
| 69 |
+
hi_patch_size=25,
|
| 70 |
+
knn_k=8,
|
| 71 |
+
drop_path=0.1,
|
| 72 |
+
)
|
| 73 |
+
LORA_SCALE = 16 / 4 # alpha / r
|
| 74 |
+
# Reference layer of a block -> its Braindecode modules; the fused qkv splits in three.
|
| 75 |
+
LAYERS = {
|
| 76 |
+
"norm1": ["0.fn.0"],
|
| 77 |
+
"norm2": ["1.fn.0"],
|
| 78 |
+
"attn.qkv": ["0.fn.1.queries", "0.fn.1.keys", "0.fn.1.values"],
|
| 79 |
+
"attn.proj": ["0.fn.1.projection"],
|
| 80 |
+
"mlp.fc1": ["1.fn.1.0"],
|
| 81 |
+
"mlp.fc2": ["1.fn.1.3"],
|
| 82 |
+
}
|
| 83 |
+
TOLERANCE = 1e-4
|
| 84 |
+
FINGERS = ["thumb", "index", "middle", "ring", "pinky"]
|
| 85 |
+
MIT = """MIT License
|
| 86 |
+
|
| 87 |
+
Copyright (c) {year} {holder}
|
| 88 |
+
|
| 89 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 90 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 91 |
+
in the Software without restriction, including without limitation the rights
|
| 92 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 93 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 94 |
+
furnished to do so, subject to the following conditions:
|
| 95 |
+
|
| 96 |
+
The above copyright notice and this permission notice shall be included in all
|
| 97 |
+
copies or substantial portions of the Software.
|
| 98 |
+
|
| 99 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 100 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 101 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 102 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 103 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 104 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 105 |
+
SOFTWARE.
|
| 106 |
+
"""
|
| 107 |
+
LICENSE = (
|
| 108 |
+
"The CORTEG adapter (checkpoints/corteg_stanford_pooled.pt of\n"
|
| 109 |
+
f"{REPOSITORY} at {REVISION}):\n\n"
|
| 110 |
+
+ MIT.format(year=2026, holder="Liuyin Yang")
|
| 111 |
+
+ "\n\nThe ST-EEGFormer small backbone (release ST-EEGFormer-small of\n"
|
| 112 |
+
"https://github.com/LiuyinYang1101/STEEGFormer):\n\n"
|
| 113 |
+
+ MIT.format(year=2025, holder="LIUYIN YANG")
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def sha256(path):
|
| 118 |
+
digest = hashlib.sha256()
|
| 119 |
+
with open(path, "rb") as f:
|
| 120 |
+
for block in iter(lambda: f.read(1 << 20), b""):
|
| 121 |
+
digest.update(block)
|
| 122 |
+
return digest.hexdigest()
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def montages():
|
| 126 |
+
"""Electrode positions in metres: the Braindecode test montage and 46 random contacts."""
|
| 127 |
+
rng = np.random.default_rng(0)
|
| 128 |
+
random = rng.uniform([-0.07, -0.10, -0.04], [0.07, 0.07, 0.08], size=(46, 3))
|
| 129 |
+
return [
|
| 130 |
+
torch.tensor(
|
| 131 |
+
[[-0.04 + 0.004 * i, -0.02 + 0.003 * i, 0.06 - 0.002 * i] for i in range(8)]
|
| 132 |
+
),
|
| 133 |
+
torch.tensor(random, dtype=torch.float32),
|
| 134 |
+
]
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def convert(adapter, model):
|
| 138 |
+
"""Load the backbone and the adapter into ``model``, the LoRA factors folded in."""
|
| 139 |
+
path = hf_hub_download(BACKBONE_REPO, "model.safetensors", revision=BACKBONE_REVISION)
|
| 140 |
+
backbone = {k: v for k, v in load_file(path).items() if not k.startswith("final_layer.")}
|
| 141 |
+
missing, unexpected = model.load_state_dict(backbone, strict=False)
|
| 142 |
+
if unexpected or {k.split(".")[0] for k in missing} != {"channel_adapter", "final_layer"}:
|
| 143 |
+
raise SystemExit(f"backbone does not fit CORTEG: {missing=}, {unexpected=}")
|
| 144 |
+
state = model.state_dict()
|
| 145 |
+
filled = set()
|
| 146 |
+
for key, value in adapter.items():
|
| 147 |
+
key = key.removeprefix("backbone.").replace("head.head.", "final_layer.")
|
| 148 |
+
if not key.startswith("blocks."):
|
| 149 |
+
if key not in state or state[key].shape != value.shape:
|
| 150 |
+
raise SystemExit(f"adapter tensor {key} has no home in CORTEG")
|
| 151 |
+
state[key] = value
|
| 152 |
+
filled.add(key)
|
| 153 |
+
continue
|
| 154 |
+
_, block, rest = key.split(".", 2)
|
| 155 |
+
layer, _, param = rest.rpartition(".")
|
| 156 |
+
if layer.endswith(".B"): # folded with its A factor
|
| 157 |
+
continue
|
| 158 |
+
if layer.endswith(".A"):
|
| 159 |
+
layer = layer.removesuffix(".A")
|
| 160 |
+
lora_b = adapter[f"backbone.blocks.{block}.{layer}.B.weight"]
|
| 161 |
+
delta = (LORA_SCALE * lora_b @ value).chunk(len(LAYERS[layer]))
|
| 162 |
+
for name, part in zip(LAYERS[layer], delta):
|
| 163 |
+
state[f"encoder.{block}.{name}.weight"] += part
|
| 164 |
+
else:
|
| 165 |
+
target = f"encoder.{block}.{LAYERS[layer][0]}.{param}"
|
| 166 |
+
state[target] = value
|
| 167 |
+
filled.add(target)
|
| 168 |
+
unfilled = {k for k in missing if k not in filled}
|
| 169 |
+
if unfilled:
|
| 170 |
+
raise SystemExit(f"the adapter leaves {sorted(unfilled)} at their initialization")
|
| 171 |
+
model.load_state_dict(state, strict=True)
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def reference_outputs(reference_dir, backbone, x, x_hi, pos):
|
| 175 |
+
"""Outputs of the authors' ``load_corteg`` on the two release files."""
|
| 176 |
+
sys.path.insert(0, str(reference_dir))
|
| 177 |
+
from load_corteg import load_corteg, predict
|
| 178 |
+
|
| 179 |
+
config = json.loads((reference_dir / "configs/steegformer_small.json").read_text())
|
| 180 |
+
config["pretrained"]["path"] = str(backbone.resolve())
|
| 181 |
+
with tempfile.TemporaryDirectory() as tmp:
|
| 182 |
+
config_path = Path(tmp) / "steegformer_small.json"
|
| 183 |
+
config_path.write_text(json.dumps(config))
|
| 184 |
+
reference = load_corteg(
|
| 185 |
+
C_in=x.shape[1],
|
| 186 |
+
T_in=x.shape[2],
|
| 187 |
+
ecog_xyz_mm=1000 * pos.numpy(),
|
| 188 |
+
d_out=CONFIG["n_outputs"],
|
| 189 |
+
backbone_config=str(config_path),
|
| 190 |
+
)
|
| 191 |
+
return torch.from_numpy(predict(reference, x.numpy(), x_hi.numpy()))
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def model_card(report):
|
| 195 |
+
return f"""---
|
| 196 |
+
library_name: braindecode
|
| 197 |
+
license: mit
|
| 198 |
+
base_model: {BACKBONE_REPO}
|
| 199 |
+
tags:
|
| 200 |
+
- braindecode
|
| 201 |
+
- CORTEG
|
| 202 |
+
- ieeg
|
| 203 |
+
- ecog
|
| 204 |
+
- finger-flexion
|
| 205 |
+
- pytorch_model_hub_mixin
|
| 206 |
+
---
|
| 207 |
+
|
| 208 |
+
# CORTEG, Stanford finger flexion (pooled)
|
| 209 |
+
|
| 210 |
+
The released CORTEG model of [Yang et al. (2026)](https://arxiv.org/abs/2605.10337),
|
| 211 |
+
ST-EEGFormer small adapted to ECoG and trained on the finger-flexion recordings
|
| 212 |
+
of the 9 Stanford patients pooled (Table 1, "CORTEG (pooled)", Pearson r 0.554),
|
| 213 |
+
{report["n_params"] / 1e6:.1f}M parameters, converted to
|
| 214 |
+
[`braindecode.models.CORTEG`](https://braindecode.org/stable/generated/braindecode.models.CORTEG.html).
|
| 215 |
+
It decodes the flexion of the five fingers ({", ".join(FINGERS)}) from 1 s of ECoG.
|
| 216 |
+
|
| 217 |
+
Sources:
|
| 218 |
+
|
| 219 |
+
- the adapter, `{ADAPTER}` of [LiuyinYang1101/CORTEG]({REPOSITORY}/tree/{REVISION}),
|
| 220 |
+
sha256 `{ADAPTER_SHA256}`: the spatial adapter, LoRA factors (r=4, alpha=16) on
|
| 221 |
+
blocks 4-7, the high-gamma patch embedding, those blocks' LayerNorms and the
|
| 222 |
+
readout, {ADAPTER_PARAMS:,} parameters;
|
| 223 |
+
- the frozen backbone it was trained on, the ST-EEGFormer small release
|
| 224 |
+
([`checkpoint-300.pth`]({BACKBONE_URL}), sha256 `{BACKBONE_SHA256}`), here
|
| 225 |
+
through its conversion [{BACKBONE_REPO}](https://huggingface.co/{BACKBONE_REPO})
|
| 226 |
+
at revision `{BACKBONE_REVISION[:8]}`.
|
| 227 |
+
|
| 228 |
+
Conversion folds the LoRA factors into the fused qkv, proj, fc1 and fc2 weights
|
| 229 |
+
(scale alpha/r = 4), so the model is a plain `CORTEG`;
|
| 230 |
+
`convert_corteg_checkpoint.py` in this repository reproduces it. Outputs match
|
| 231 |
+
the authors' `load_corteg` on the two release files to {report["max_abs_diff"]:.1e}
|
| 232 |
+
in float32 on CPU, for montages of {" and ".join(str(n) for n in report["n_chans_checked"])}
|
| 233 |
+
contacts.
|
| 234 |
+
|
| 235 |
+
## Usage
|
| 236 |
+
|
| 237 |
+
The model reads two streams of the same 1 s window, as the paper's Stanford
|
| 238 |
+
pipeline (`data/stanford_preprocessing/` of the official repository): after a
|
| 239 |
+
notch at 60, 120 and 180 Hz and a common average reference, `x` is the signal
|
| 240 |
+
resampled to 128 Hz and `x_hi` its 70-200 Hz Hilbert envelope at 200 Hz, each
|
| 241 |
+
z-scored per channel with the statistics of the training windows. Electrode
|
| 242 |
+
positions are in metres, in `chs_info` or `pos`.
|
| 243 |
+
|
| 244 |
+
```python
|
| 245 |
+
import torch
|
| 246 |
+
from braindecode.models import CORTEG
|
| 247 |
+
|
| 248 |
+
model = CORTEG.from_pretrained(
|
| 249 |
+
"braindecode/{HUB_REPO}", chs_info=raw.info["chs"]
|
| 250 |
+
).eval()
|
| 251 |
+
flexion = model(x, x_hi) # x: (batch, n_chans, 128), x_hi: (batch, n_chans, 200)
|
| 252 |
+
```
|
| 253 |
+
|
| 254 |
+
The output, `(batch, 5)`, is the flexion of each finger, z-scored per finger
|
| 255 |
+
with the training statistics of the patient, at the time 40 ms before the
|
| 256 |
+
window's end. The saved geometry is {report["n_chans"]} channels; the model
|
| 257 |
+
reads any montage, so pass your own `chs_info` (or `n_chans` and `pos` at each
|
| 258 |
+
call) when loading.
|
| 259 |
+
|
| 260 |
+
## Limitations
|
| 261 |
+
|
| 262 |
+
The readout is at its random initialization, as released: the paper run built
|
| 263 |
+
it after the optimizer, so it was never trained, and the adapter learned around
|
| 264 |
+
it (see the release manifest, `checkpoints/corteg_stanford_pooled.json`).
|
| 265 |
+
The model was trained on the training split of the 9 Stanford patients (the
|
| 266 |
+
first 400 s of each recording, two thirds of one under 600 s); other patients
|
| 267 |
+
or tasks need fine-tuning, as in the paper's leave-one-subject-out protocol. The check above covers float32 CPU outputs, not
|
| 268 |
+
decoding accuracy, GPU kernels or mixed precision; the authors report that this
|
| 269 |
+
checkpoint reproduces the cohort mean of 0.554 (manifest, `paper.verified`).
|
| 270 |
+
|
| 271 |
+
## Citation
|
| 272 |
+
|
| 273 |
+
```bibtex
|
| 274 |
+
@inproceedings{{yang2026corteg,
|
| 275 |
+
title = {{{{CORTEG}}: Foundation Models Enable Cross-Modality Representation Transfer
|
| 276 |
+
from Scalp to Intracranial Brain Recordings}},
|
| 277 |
+
author = {{Yang, Liuyin and Sun, Qiang and Van Dyck, Bob and Calvo Merino, Eva and
|
| 278 |
+
Van Hulle, Marc M.}},
|
| 279 |
+
booktitle = {{Advances in Neural Information Processing Systems (NeurIPS)}},
|
| 280 |
+
year = {{2026}}
|
| 281 |
+
}}
|
| 282 |
+
```
|
| 283 |
+
|
| 284 |
+
## License
|
| 285 |
+
|
| 286 |
+
The adapter and the ST-EEGFormer backbone are both released by Liuyin Yang
|
| 287 |
+
under the MIT licence (`LICENSE`). Braindecode's code is BSD-3-Clause.
|
| 288 |
+
"""
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def main():
|
| 292 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 293 |
+
parser.add_argument("--reference-dir", type=Path, required=True)
|
| 294 |
+
parser.add_argument("--backbone", type=Path, required=True)
|
| 295 |
+
parser.add_argument("--output-dir", type=Path, required=True)
|
| 296 |
+
parser.add_argument("--push-to", help="Hub namespace to publish the model to")
|
| 297 |
+
args = parser.parse_args()
|
| 298 |
+
torch.set_num_threads(1)
|
| 299 |
+
adapter_path = args.reference_dir / ADAPTER
|
| 300 |
+
for path, expected in ((adapter_path, ADAPTER_SHA256), (args.backbone, BACKBONE_SHA256)):
|
| 301 |
+
if sha256(path) != expected:
|
| 302 |
+
raise SystemExit(f"{path} is not the released file (sha256 mismatch)")
|
| 303 |
+
adapter = torch.load(adapter_path, map_location="cpu", weights_only=True)
|
| 304 |
+
n_adapter = sum(t.numel() for t in adapter.values())
|
| 305 |
+
if n_adapter != ADAPTER_PARAMS:
|
| 306 |
+
raise SystemExit(f"adapter holds {n_adapter} parameters, not {ADAPTER_PARAMS}")
|
| 307 |
+
|
| 308 |
+
max_abs_diff, n_chans_checked = 0.0, []
|
| 309 |
+
for pos in montages():
|
| 310 |
+
n_chans = len(pos)
|
| 311 |
+
model = CORTEG(n_chans=n_chans, **CONFIG).eval()
|
| 312 |
+
convert(adapter, model)
|
| 313 |
+
g = torch.Generator().manual_seed(0)
|
| 314 |
+
x = torch.randn(4, n_chans, 128, generator=g)
|
| 315 |
+
x_hi = torch.randn(4, n_chans, 200, generator=g)
|
| 316 |
+
with torch.no_grad():
|
| 317 |
+
out = model(x, x_hi, pos=pos)
|
| 318 |
+
expected = reference_outputs(args.reference_dir, args.backbone, x, x_hi, pos)
|
| 319 |
+
max_abs_diff = max(max_abs_diff, (out - expected).abs().max().item())
|
| 320 |
+
n_chans_checked.append(n_chans)
|
| 321 |
+
if max_abs_diff > TOLERANCE:
|
| 322 |
+
raise SystemExit(f"outputs differ by {max_abs_diff:.2e} > {TOLERANCE}")
|
| 323 |
+
|
| 324 |
+
# Users pass their own montage when loading; the weights do not depend on it.
|
| 325 |
+
n_chans = n_chans_checked[0]
|
| 326 |
+
model = CORTEG(n_chans=n_chans, **CONFIG).eval()
|
| 327 |
+
convert(adapter, model)
|
| 328 |
+
report = {
|
| 329 |
+
"source": f"{REPOSITORY}@{REVISION}:{ADAPTER}",
|
| 330 |
+
"sha256": ADAPTER_SHA256,
|
| 331 |
+
"backbone": f"{BACKBONE_URL} via {BACKBONE_REPO}@{BACKBONE_REVISION}",
|
| 332 |
+
"backbone_sha256": BACKBONE_SHA256,
|
| 333 |
+
"adapter_tensors": len(adapter),
|
| 334 |
+
"adapter_params": n_adapter,
|
| 335 |
+
"lora_scale": LORA_SCALE,
|
| 336 |
+
"n_params": sum(t.numel() for t in model.state_dict().values()),
|
| 337 |
+
"max_abs_diff": max_abs_diff,
|
| 338 |
+
"n_chans_checked": n_chans_checked,
|
| 339 |
+
"n_chans": n_chans,
|
| 340 |
+
"n_times": CONFIG["n_times"],
|
| 341 |
+
"torch_version": torch.__version__,
|
| 342 |
+
}
|
| 343 |
+
print(json.dumps(report, indent=2))
|
| 344 |
+
destination = args.output_dir / HUB_REPO
|
| 345 |
+
destination.mkdir(parents=True, exist_ok=True)
|
| 346 |
+
model.save_pretrained(destination)
|
| 347 |
+
# The Hub rejects NaN and Infinity, which Python's json writes by default.
|
| 348 |
+
json.loads(
|
| 349 |
+
(destination / "config.json").read_text(),
|
| 350 |
+
parse_constant=lambda c: (_ for _ in ()).throw(ValueError(f"{c} in config.json")),
|
| 351 |
+
)
|
| 352 |
+
shutil.copyfile(Path(__file__).resolve(), destination / Path(__file__).name)
|
| 353 |
+
(destination / "LICENSE").write_text(LICENSE)
|
| 354 |
+
(destination / "report.json").write_text(json.dumps(report, indent=2))
|
| 355 |
+
# Written last: save_pretrained leaves a placeholder card behind.
|
| 356 |
+
(destination / "README.md").write_text(model_card(report))
|
| 357 |
+
if args.push_to:
|
| 358 |
+
repo_id = f"{args.push_to}/{HUB_REPO}"
|
| 359 |
+
api = HfApi()
|
| 360 |
+
api.create_repo(repo_id, exist_ok=True)
|
| 361 |
+
api.upload_folder(
|
| 362 |
+
folder_path=str(destination),
|
| 363 |
+
repo_id=repo_id,
|
| 364 |
+
commit_message="CORTEG Stanford pooled model, converted from the official release",
|
| 365 |
+
)
|
| 366 |
+
print(f"published https://huggingface.co/{repo_id}")
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
if __name__ == "__main__":
|
| 370 |
+
main()
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:01211bbda61d0691386c7544449d039d7a5c34537c2ddec543800eb5234af5cb
|
| 3 |
+
size 101860424
|
pytorch_model.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d8372cf7c96a6e87f32932e9303fc169cdfdf911b8d31cb1dc4f0c6826729b32
|
| 3 |
+
size 101901883
|
report.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"source": "https://github.com/LiuyinYang1101/CORTEG@caafb78fbebdd9448814f96d7e61083a3eb62663:checkpoints/corteg_stanford_pooled.pt",
|
| 3 |
+
"sha256": "77d63a214e0e0f2c790e90878a0ef6b525f6e24e1453fcd94e118c63ff7e8396",
|
| 4 |
+
"backbone": "https://github.com/LiuyinYang1101/STEEGFormer/releases/download/ST-EEGFormer-small/checkpoint-300.pth via braindecode/STEEGFormer-small@633c9c0d76c6e26459e1e0d1337c91858578f13d",
|
| 5 |
+
"backbone_sha256": "98d2848abdaea7a143d399c07f67f8782e07dd66e7c309ddb058c7158a8a8b02",
|
| 6 |
+
"adapter_tensors": 64,
|
| 7 |
+
"adapter_params": 297236,
|
| 8 |
+
"lora_scale": 4.0,
|
| 9 |
+
"n_params": 25461524,
|
| 10 |
+
"max_abs_diff": 1.4603137969970703e-06,
|
| 11 |
+
"n_chans_checked": [
|
| 12 |
+
8,
|
| 13 |
+
46
|
| 14 |
+
],
|
| 15 |
+
"n_chans": 8,
|
| 16 |
+
"n_times": 128,
|
| 17 |
+
"torch_version": "2.15.0+cu132"
|
| 18 |
+
}
|