NeuroRVQ neurorvq-tokenizer-eeg-pretrained: converted from ntinosbarmpas/NeuroRVQ@d944b87
Browse files- README.md +68 -0
- config.json +102 -0
- convert_neurorvq_checkpoint.py +114 -0
- model.safetensors +3 -0
- pytorch_model.bin +3 -0
README.md
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-4.0
|
| 3 |
+
library_name: braindecode
|
| 4 |
+
tags:
|
| 5 |
+
- braindecode
|
| 6 |
+
- pytorch
|
| 7 |
+
- safetensors
|
| 8 |
+
- eeg
|
| 9 |
+
- neurorvq
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# neurorvq-tokenizer-eeg-pretrained
|
| 13 |
+
|
| 14 |
+
Weights of the NeuroRVQ-EEG tokenizer (four-scale encoder, residual vector quantizers, decoder) for
|
| 15 |
+
[`braindecode.models.NeuroRVQTokenizer`](https://braindecode.org/stable/generated/braindecode.models.NeuroRVQTokenizer.html)
|
| 16 |
+
(`modality="eeg"`), converted from the authors' release. All weights are pretrained; `model(x)` returns the per-window z-scored target and reconstruction, `model.tokenize(x)` the codes.
|
| 17 |
+
|
| 18 |
+
```python
|
| 19 |
+
from braindecode.models import NeuroRVQTokenizer
|
| 20 |
+
model = NeuroRVQTokenizer.from_pretrained("braindecode/neurorvq-tokenizer-eeg-pretrained", chs_info=raw.info["chs"])
|
| 21 |
+
```
|
| 22 |
+
|
| 23 |
+
Channel names must come from the model's EEG channel list (see the class docstring). The
|
| 24 |
+
channels and window in `config.json` (19 EEG channels (10-20), 800 samples at 200 Hz) are only a default; pass `chs_info` (or
|
| 25 |
+
`channel_names` and `n_chans`) and `n_times`. Input is expected at that sampling rate,
|
| 26 |
+
preprocessed as in the authors' examples; the model does not preprocess.
|
| 27 |
+
|
| 28 |
+
## Source and conversion
|
| 29 |
+
|
| 30 |
+
- Source: [ntinosbarmpas/NeuroRVQ](https://huggingface.co/ntinosbarmpas/NeuroRVQ) at revision
|
| 31 |
+
`d944b87f44ae0ba2923b2f10d0518f23f6803b76`, file `pretrained_models/tokenizers/NeuroRVQ_EEG_tokenizer_v1.pt` (sha256 `d3e850e160b1a558c529466b1c0b8cebe8888a5e02824979886dd226187b2725`), CC BY-NC 4.0.
|
| 32 |
+
Code: [KonstantinosBarmpas/NeuroRVQ](https://github.com/KonstantinosBarmpas/NeuroRVQ).
|
| 33 |
+
- `convert_neurorvq_checkpoint.py` (in this repository) renames the Transformer
|
| 34 |
+
feed-forward `mlp.fc1`/`mlp.fc2` keys to braindecode's `mlp.0`/`mlp.2`, drops the
|
| 35 |
+
pretraining-only tensors (`mask_token`, `norm_pre`, `head_pre_*`) and writes
|
| 36 |
+
`config.json`, `model.safetensors` and `pytorch_model.bin` with `save_pretrained`.
|
| 37 |
+
- The converted model's outputs equal the authors' module loading the original file
|
| 38 |
+
(max-abs difference 0.0 on random inputs).
|
| 39 |
+
- Requires a braindecode version newer than 1.8.1.
|
| 40 |
+
|
| 41 |
+
## Citation
|
| 42 |
+
|
| 43 |
+
```bibtex
|
| 44 |
+
@misc{neurorvq,
|
| 45 |
+
title = {NeuroRVQ: Multi-Scale Biosignal Tokenization for Generative Foundation Models},
|
| 46 |
+
author = {Konstantinos Barmpas and Na Lee and Dimitrios Chalatsis and William Raftery and
|
| 47 |
+
Yannis Panagakis and Dimitrios A. Adamos and Nikolaos Laskaris and
|
| 48 |
+
Alexandros Koliousis and Dario Farina and Stefanos Zafeiriou},
|
| 49 |
+
year = {2026},
|
| 50 |
+
eprint = {2510.13068},
|
| 51 |
+
archivePrefix = {arXiv},
|
| 52 |
+
primaryClass = {cs.LG},
|
| 53 |
+
url = {https://arxiv.org/abs/2510.13068},
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
@article{aristimunha2025braindecode,
|
| 57 |
+
title = {Braindecode: a deep learning library for raw electrophysiological data},
|
| 58 |
+
author = {Aristimunha, Bruno and others},
|
| 59 |
+
journal = {Zenodo},
|
| 60 |
+
year = {2025},
|
| 61 |
+
doi = {10.5281/zenodo.17699192},
|
| 62 |
+
}
|
| 63 |
+
```
|
| 64 |
+
|
| 65 |
+
## License
|
| 66 |
+
|
| 67 |
+
CC BY-NC 4.0 (Attribution-NonCommercial), as the original NeuroRVQ release: non-commercial
|
| 68 |
+
use only, with attribution to Barmpas et al. and the source repository above.
|
config.json
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_outputs": null,
|
| 3 |
+
"n_chans": null,
|
| 4 |
+
"chs_info": [
|
| 5 |
+
{
|
| 6 |
+
"ch_name": "Fp1",
|
| 7 |
+
"kind": "eeg"
|
| 8 |
+
},
|
| 9 |
+
{
|
| 10 |
+
"ch_name": "Fp2",
|
| 11 |
+
"kind": "eeg"
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"ch_name": "F7",
|
| 15 |
+
"kind": "eeg"
|
| 16 |
+
},
|
| 17 |
+
{
|
| 18 |
+
"ch_name": "F3",
|
| 19 |
+
"kind": "eeg"
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"ch_name": "Fz",
|
| 23 |
+
"kind": "eeg"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"ch_name": "F4",
|
| 27 |
+
"kind": "eeg"
|
| 28 |
+
},
|
| 29 |
+
{
|
| 30 |
+
"ch_name": "F8",
|
| 31 |
+
"kind": "eeg"
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"ch_name": "T7",
|
| 35 |
+
"kind": "eeg"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"ch_name": "C3",
|
| 39 |
+
"kind": "eeg"
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"ch_name": "Cz",
|
| 43 |
+
"kind": "eeg"
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"ch_name": "C4",
|
| 47 |
+
"kind": "eeg"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"ch_name": "T8",
|
| 51 |
+
"kind": "eeg"
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"ch_name": "P7",
|
| 55 |
+
"kind": "eeg"
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"ch_name": "P3",
|
| 59 |
+
"kind": "eeg"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"ch_name": "Pz",
|
| 63 |
+
"kind": "eeg"
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"ch_name": "P4",
|
| 67 |
+
"kind": "eeg"
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"ch_name": "P8",
|
| 71 |
+
"kind": "eeg"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"ch_name": "O1",
|
| 75 |
+
"kind": "eeg"
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"ch_name": "O2",
|
| 79 |
+
"kind": "eeg"
|
| 80 |
+
}
|
| 81 |
+
],
|
| 82 |
+
"n_times": 800,
|
| 83 |
+
"input_window_seconds": null,
|
| 84 |
+
"sfreq": 200,
|
| 85 |
+
"channel_names": null,
|
| 86 |
+
"modality": "eeg",
|
| 87 |
+
"patch_size": null,
|
| 88 |
+
"max_patches": null,
|
| 89 |
+
"out_chans": 8,
|
| 90 |
+
"encoder_depth": 12,
|
| 91 |
+
"decoder_depth": 3,
|
| 92 |
+
"num_heads": 10,
|
| 93 |
+
"n_code": 8192,
|
| 94 |
+
"code_dim": 128,
|
| 95 |
+
"num_quantizers": null,
|
| 96 |
+
"drop_prob": 0.0,
|
| 97 |
+
"attn_drop_rate": 0.0,
|
| 98 |
+
"drop_path_rate": 0.0,
|
| 99 |
+
"init_values": 0.0,
|
| 100 |
+
"activation": "torch.nn.modules.activation.GELU",
|
| 101 |
+
"braindecode_version": "1.8.1"
|
| 102 |
+
}
|
convert_neurorvq_checkpoint.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Convert a released NeuroRVQ checkpoint to a braindecode-native repository.
|
| 2 |
+
|
| 3 |
+
Source: https://huggingface.co/ntinosbarmpas/NeuroRVQ at revision
|
| 4 |
+
d944b87f44ae0ba2923b2f10d0518f23f6803b76 (CC BY-NC 4.0), one file per
|
| 5 |
+
repository (table below, with its sha256).
|
| 6 |
+
|
| 7 |
+
Usage::
|
| 8 |
+
|
| 9 |
+
python convert_neurorvq_checkpoint.py REPO OUT_DIR [SOURCE_DIR]
|
| 10 |
+
|
| 11 |
+
writes ``OUT_DIR/REPO`` with ``config.json``, ``model.safetensors`` and
|
| 12 |
+
``pytorch_model.bin`` (``save_pretrained``). ``SOURCE_DIR`` is a local copy of
|
| 13 |
+
the source repository; without it the file is downloaded.
|
| 14 |
+
|
| 15 |
+
Key changes: the Transformer feed-forward ``mlp.fc1``/``mlp.fc2`` become
|
| 16 |
+
braindecode's ``MLP`` children ``mlp.0``/``mlp.2``; the foundation models'
|
| 17 |
+
pretraining-only tensors (``mask_token``, ``norm_pre``, ``head_pre_*``) are
|
| 18 |
+
dropped; every other tensor is kept as released. The foundation models carry
|
| 19 |
+
no classification head (``fc_norm``/``final_layer`` are left out of the weight
|
| 20 |
+
files, so any ``n_chans``/``n_times``/``n_outputs`` loads). The channels and
|
| 21 |
+
window in ``config.json`` are only a default; pass ``chs_info`` (or
|
| 22 |
+
``channel_names`` and ``n_chans``) and ``n_times`` to ``from_pretrained``.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
import hashlib
|
| 26 |
+
import sys
|
| 27 |
+
from pathlib import Path
|
| 28 |
+
|
| 29 |
+
import torch
|
| 30 |
+
|
| 31 |
+
REPO, REVISION = "ntinosbarmpas/NeuroRVQ", "d944b87f44ae0ba2923b2f10d0518f23f6803b76"
|
| 32 |
+
_FM, _TOK = "pretrained_models/foundation_models", "pretrained_models/tokenizers"
|
| 33 |
+
CHECKPOINTS = {
|
| 34 |
+
"neurorvq-eeg-pretrained": (f"{_FM}/NeuroRVQ_EEG_foundation_model_v1.pt", "d14ef4eae2d4594d5f1f709226a3bb5f362afd84b68050d5da0fa0586aea886a"),
|
| 35 |
+
"neurorvq-ecg-pretrained": (f"{_FM}/NeuroRVQ_ECG_foundation_model_v1.pt", "d32f00bdc80e7dff508da289d85fec243aeaf6541acd89d608bc321ba156e1a7"),
|
| 36 |
+
"neurorvq-emg-pretrained": (f"{_FM}/NeuroRVQ_EMG_foundation_model_v1.pt", "a0bb153cea45f61179c7ffe8d5aa0d41c7a7f800129cb23974caf653c7266907"),
|
| 37 |
+
"neurorvq-tokenizer-eeg-pretrained": (f"{_TOK}/NeuroRVQ_EEG_tokenizer_v1.pt", "d3e850e160b1a558c529466b1c0b8cebe8888a5e02824979886dd226187b2725"),
|
| 38 |
+
"neurorvq-tokenizer-ecg-pretrained": (f"{_TOK}/NeuroRVQ_ECG_tokenizer_v1.pt", "b1524ed02fd912c86d3ba13a50375312578bc1ef7afe39cddbcac2dfd97680ea"),
|
| 39 |
+
"neurorvq-tokenizer-emg-pretrained": (f"{_TOK}/NeuroRVQ_EMG_tokenizer_v1.pt", "0d255bcc9f1c75ccc374cba06eab476f15bf5fb2a87115d6dc8d2dc0adadce49"),
|
| 40 |
+
"neurorvq-tokenizer-ppg-pretrained": (f"{_TOK}/NeuroRVQ_PPG_tokenizer_v1.pt", "8221b752e645d0819d02d81f036f90d434fe2bc34e708c78a263c480e1009e79"),
|
| 41 |
+
} # fmt: skip
|
| 42 |
+
TEN_TWENTY = "Fp1 Fp2 F7 F3 Fz F4 F8 T7 C3 Cz C4 T8 P7 P3 Pz P4 P8 O1 O2".split()
|
| 43 |
+
# Default window: the authors' create_patches length, max_patches // n_chans patches.
|
| 44 |
+
GEOMETRY = {
|
| 45 |
+
"eeg": dict(names=TEN_TWENTY, kind="eeg", n_times=800, sfreq=200),
|
| 46 |
+
"ecg": dict(names="i ii iii avr avl avf v1 v2 v3 v4 v5 v6".split(), kind="ecg", n_times=2000, sfreq=200),
|
| 47 |
+
"emg": dict(names=[f"c{i}" for i in range(1, 17)], kind="emg", n_times=3200, sfreq=1000),
|
| 48 |
+
"ppg": dict(names=["ppg_c1"], kind="misc", n_times=960, sfreq=100),
|
| 49 |
+
} # fmt: skip
|
| 50 |
+
HEAD = ("fc_norm.", "final_layer.")
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def build(repo):
|
| 54 |
+
from braindecode.models import NeuroRVQ, NeuroRVQTokenizer
|
| 55 |
+
|
| 56 |
+
modality = repo.split("-")[-2]
|
| 57 |
+
g = GEOMETRY[modality]
|
| 58 |
+
kw = dict(
|
| 59 |
+
chs_info=[{"ch_name": n, "kind": g["kind"]} for n in g["names"]],
|
| 60 |
+
n_times=g["n_times"],
|
| 61 |
+
sfreq=g["sfreq"],
|
| 62 |
+
modality=modality,
|
| 63 |
+
)
|
| 64 |
+
if "tokenizer" in repo:
|
| 65 |
+
return NeuroRVQTokenizer(**kw)
|
| 66 |
+
return NeuroRVQ(n_outputs=2, **kw)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def released_state(path):
|
| 70 |
+
state = torch.load(path, map_location="cpu", weights_only=True)
|
| 71 |
+
state = {
|
| 72 |
+
k.replace(".mlp.fc1.", ".mlp.0.").replace(".mlp.fc2.", ".mlp.2."): v
|
| 73 |
+
for k, v in state.items()
|
| 74 |
+
}
|
| 75 |
+
dropped = sorted(
|
| 76 |
+
k
|
| 77 |
+
for k in state
|
| 78 |
+
if k == "mask_token" or k.startswith(("norm_pre.", "head_pre_"))
|
| 79 |
+
)
|
| 80 |
+
return {k: v for k, v in state.items() if k not in dropped}, dropped
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def convert(repo, out, source_dir=None):
|
| 84 |
+
filename, sha = CHECKPOINTS[repo]
|
| 85 |
+
if source_dir is None:
|
| 86 |
+
from huggingface_hub import hf_hub_download
|
| 87 |
+
|
| 88 |
+
source = hf_hub_download(REPO, filename, revision=REVISION)
|
| 89 |
+
else:
|
| 90 |
+
source = Path(source_dir) / filename
|
| 91 |
+
assert hashlib.sha256(Path(source).read_bytes()).hexdigest() == sha
|
| 92 |
+
state, dropped = released_state(source)
|
| 93 |
+
print(f"{repo}: {len(state)} tensors kept, dropped {dropped}")
|
| 94 |
+
model = build(repo)
|
| 95 |
+
if "tokenizer" in repo:
|
| 96 |
+
model.load_state_dict(state, strict=True)
|
| 97 |
+
else:
|
| 98 |
+
bad = model.load_state_dict(state, strict=False)
|
| 99 |
+
assert not bad.unexpected_keys, bad.unexpected_keys
|
| 100 |
+
assert all(k.startswith(HEAD) for k in bad.missing_keys), bad.missing_keys
|
| 101 |
+
out = Path(out) / repo
|
| 102 |
+
model.save_pretrained(out)
|
| 103 |
+
if "tokenizer" not in repo: # no head in the weight files
|
| 104 |
+
from safetensors.torch import save_file
|
| 105 |
+
|
| 106 |
+
encoder = {k: v for k, v in model.state_dict().items() if not k.startswith(HEAD)}
|
| 107 |
+
assert encoder.keys() == state.keys()
|
| 108 |
+
save_file(encoder, out / "model.safetensors", metadata={"format": "pt"})
|
| 109 |
+
torch.save(encoder, out / "pytorch_model.bin")
|
| 110 |
+
return model
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
if __name__ == "__main__":
|
| 114 |
+
convert(sys.argv[1], sys.argv[2], sys.argv[3] if len(sys.argv) > 3 else None)
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ef0f97571fd9cccc436879139503bf33088caf3635e4890f2efb30685f43d418
|
| 3 |
+
size 304005768
|
pytorch_model.bin
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7205f7086f7be38203e8cbe592f83c3d8ee77515a2bc270e7b02111b22174e96
|
| 3 |
+
size 304124871
|