First commit

.gitattributes

@@ -1,35 +1,6 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+Data/celer/data_v2.0/sent_fix.tsv filter=lfs diff=lfs merge=lfs -text
+Data/celer/data_v2.0/sent_ia.tsv filter=lfs diff=lfs merge=lfs -text
+Data/beijing-sentence-corpus/BSC.EMD/BSC.EMD.txt filter=lfs diff=lfs merge=lfs -text
+Data/beijing-sentence-corpus/BSC.Word.Info.v2.xlsx filter=lfs diff=lfs merge=lfs -text
+results/BSC/Eyettention_chinese.pth filter=lfs diff=lfs merge=lfs -text
+results/CELER/Eyettention_english.pth filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,7 @@
+# Python bytecode/cache files
+__pycache__/
+*.py[cod]
+*$py.class
+.cache/
+results/
+/Data/

Data/beijing-sentence-corpus/BSC.EMD/BSC.EMD.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35b388e9355bec51b2a0ae0e872cbe03abaf5325ee9be8bf89d37a2eefd0e55d
+size 25362710

Data/beijing-sentence-corpus/BSC.Word.Info.v2.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c96e829a3de8203739893eef6b54e6ebddf055976919d9b29b2797053d81876
+size 208488

Data/beijing-sentence-corpus/README.md

@@ -0,0 +1,7 @@
+### not part of the original data repository, written by david
+
+for more info see: https://link.springer.com/article/10.3758/s13428-021-01730-2
+    lena, paul, shuwen and david have the paper.
+
+data from: https://osf.io/vr3k8/
+    downloadable via osfclient

Eyettention_examples_chinese.ipynb

@@ -0,0 +1,761 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "nuclear-dream",
+   "metadata": {
+    "id": "nuclear-dream"
+   },
+   "source": [
+    "# Eyettention"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "g-FLEOyvYJUG",
+   "metadata": {
+    "id": "g-FLEOyvYJUG"
+   },
+   "source": [
+    "**Configuration**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "neutral-montreal",
+   "metadata": {
+    "id": "neutral-montreal"
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "#configuration\n",
+    "#Chinese\n",
+    "cf = {\"model_pretrained\": \"bert-base-chinese\",\n",
+    "      \"atten_type\": 'local-g',\n",
+    "      #batch computation for training (BSC dataset)\n",
+    "      \"max_sn_len\": 27,\n",
+    "      \"max_sp_len\": 40\n",
+    "     }"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "uhLxInukW82r",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "uhLxInukW82r",
+    "outputId": "76c33a63-4d13-4496-976d-126fe4ebb1b0"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: transformers==4.27.0 in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (4.27.0)\n",
+      "Requirement already satisfied: packaging>=20.0 in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from transformers==4.27.0) (20.9)\n",
+      "Requirement already satisfied: numpy>=1.17 in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from transformers==4.27.0) (1.20.3)\n",
+      "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from transformers==4.27.0) (0.15.1)\n",
+      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from transformers==4.27.0) (0.12.1)\n",
+      "Requirement already satisfied: requests in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from transformers==4.27.0) (2.25.1)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from transformers==4.27.0) (2021.4.4)\n",
+      "Requirement already satisfied: tqdm>=4.27 in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from transformers==4.27.0) (4.65.0)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from transformers==4.27.0) (5.4.1)\n",
+      "Requirement already satisfied: filelock in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from transformers==4.27.0) (3.0.12)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from huggingface-hub<1.0,>=0.11.0->transformers==4.27.0) (4.5.0)\n",
+      "Requirement already satisfied: fsspec in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from huggingface-hub<1.0,>=0.11.0->transformers==4.27.0) (2023.6.0)\n",
+      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from packaging>=20.0->transformers==4.27.0) (2.4.7)\n",
+      "Requirement already satisfied: colorama in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from tqdm>=4.27->transformers==4.27.0) (0.4.4)\n",
+      "Requirement already satisfied: chardet<5,>=3.0.2 in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from requests->transformers==4.27.0) (3.0.4)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from requests->transformers==4.27.0) (2020.12.5)\n",
+      "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from requests->transformers==4.27.0) (2.10)\n",
+      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from requests->transformers==4.27.0) (1.26.5)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install --upgrade transformers==4.27.0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "8-8qDf4EXljd",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "8-8qDf4EXljd",
+    "outputId": "97cffda3-ffde-4231-c80b-0bfbc9f98a68"
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "fatal: destination path 'Eyettention' already exists and is not an empty directory.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# You can also just download the file model.py, which is only used in this tutorial\n",
+    "!git clone https://github.com/aeye-lab/Eyettention"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "jbqmeRpRY5Sg",
+   "metadata": {
+    "id": "jbqmeRpRY5Sg"
+   },
+   "source": [
+    "**Load the model and download pre-trained weights**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "stylish-member",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "stylish-member",
+    "outputId": "2307c733-ee9e-4ffb-92ef-f9d973f53fd6"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "keeping Bert with pre-trained weights\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
+      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+     ]
+    }
+   ],
+   "source": [
+    "from Eyettention import model\n",
+    "import torch\n",
+    "from torch.utils import model_zoo\n",
+    "\n",
+    "#DEVICE = 'cuda'\n",
+    "DEVICE = 'cpu'\n",
+    "\n",
+    "pretrained=True\n",
+    "\n",
+    "#load model\n",
+    "dnn = model.Eyettention(cf)\n",
+    "dnn.eval()\n",
+    "\n",
+    "if pretrained:  \n",
+    "    #for Chinese\n",
+    "    dnn.load_state_dict(model_zoo.load_url('https://github.com/aeye-lab/Eyettention/releases/download/v1.1/Eyettention_chinese.pth', map_location=torch.device('cpu')))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f0DvCfMrZMYg",
+   "metadata": {
+    "id": "f0DvCfMrZMYg"
+   },
+   "source": [
+    "**Tokenization**\n",
+    "<br>\n",
+    "**Introduction**: https://huggingface.co/learn/nlp-course/chapter2/4?fw=pt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "severe-findings",
+   "metadata": {
+    "id": "severe-findings"
+   },
+   "outputs": [],
+   "source": [
+    "from transformers import BertTokenizerFast\n",
+    "#initialize tokenizer\n",
+    "tokenizer = BertTokenizerFast.from_pretrained(cf['model_pretrained'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "funded-validity",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "funded-validity",
+    "outputId": "fbc6c028-3049-42ab-8187-6034e787c59d"
+   },
+   "outputs": [],
+   "source": [
+    "sentence = '中国选手在男子滑雪比赛中有望蝉联冠军'\n",
+    "#add special tokens\n",
+    "#text = ('[CLS]' + ' ' + sentence + ' ' + '[SEP]').split()\n",
+    "#print(text)\n",
+    "tokens = tokenizer(sentence, add_special_tokens=True, max_length=cf['max_sn_len'], padding='max_length')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "vulnerable-figure",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "vulnerable-figure",
+    "outputId": "9dcba3a3-b8b8-4d77-bed9-3b6059f9cea8"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['[CLS]', '中', '国', '选', '手', '在', '男', '子', '滑', '雪', '比', '赛', '中', '有', '望', '蝉', '联', '冠', '军', '[SEP]']\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tokenizer.tokenize(sentence, add_special_tokens=True))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "inappropriate-marketplace",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "inappropriate-marketplace",
+    "outputId": "6ca3b171-cd18-4a82-8770-871fd6fad840"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'input_ids': [101, 704, 1744, 6848, 2797, 1762, 4511, 2094, 3998, 7434, 3683, 6612, 704, 3300, 3307, 6068, 5468, 1094, 1092, 102, 0, 0, 0, 0, 0, 0, 0], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0]}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tokens)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "intelligent-express",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "intelligent-express",
+    "outputId": "3a185313-5e28-450b-cd8c-4b885bac07f0"
+   },
+   "outputs": [],
+   "source": [
+    "#we do not need word index in Chinese language since each character is encoded independently. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "Zz6pcnFualMb",
+   "metadata": {
+    "id": "Zz6pcnFualMb"
+   },
+   "source": [
+    "**Prepare features**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "specified-stomach",
+   "metadata": {
+    "id": "specified-stomach"
+   },
+   "outputs": [],
+   "source": [
+    "def compute_BSC_word_length(sentence, lac):\n",
+    "    word_string = lac.run(sentence)\n",
+    "    #print(word_string)\n",
+    "    word_len = [len(i) for i in word_string]\n",
+    "    wl_list = []\n",
+    "    for wl in word_len:\n",
+    "        wl_list.extend([wl]*wl)\n",
+    "    arr = np.asarray(wl_list, dtype=np.float32)\n",
+    "    #length of a punctuation is 0, plus an epsilon to avoid division output inf\n",
+    "    arr[arr==0] = 1/(0+0.5)\n",
+    "    arr[arr!=0] = 1/(arr[arr!=0])\n",
+    "    return arr\n",
+    "\n",
+    "def pad_seq(seqs, max_len, dtype=np.int32, fill_value=np.nan):\n",
+    "    padded = np.full((len(seqs), max_len), fill_value=fill_value, dtype=dtype)\n",
+    "    for i, seq in enumerate(seqs):\n",
+    "        padded[i, 1:(len(seq)+1)] = seq\n",
+    "    return padded"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "59e9c3cf",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: LAC in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (2.1.2)\n"
+     ]
+    }
+   ],
+   "source": [
+    "#segment sentene into word (not character) to calculate word length features\n",
+    "!pip install LAC"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "ce1a958a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: paddlepaddle==2.5.2 in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (2.5.2)\n",
+      "Requirement already satisfied: astor in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from paddlepaddle==2.5.2) (0.8.1)\n",
+      "Requirement already satisfied: numpy>=1.13 in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from paddlepaddle==2.5.2) (1.20.3)\n",
+      "Requirement already satisfied: protobuf<=3.20.2,>=3.1.0 in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from paddlepaddle==2.5.2) (3.14.0)\n",
+      "Requirement already satisfied: opt-einsum==3.3.0 in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from paddlepaddle==2.5.2) (3.3.0)\n",
+      "Requirement already satisfied: Pillow in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from paddlepaddle==2.5.2) (10.1.0)\n",
+      "Requirement already satisfied: httpx in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from paddlepaddle==2.5.2) (0.26.0)\n",
+      "Requirement already satisfied: decorator in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from paddlepaddle==2.5.2) (4.4.2)\n",
+      "Requirement already satisfied: six>=1.9 in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from protobuf<=3.20.2,>=3.1.0->paddlepaddle==2.5.2) (1.16.0)\n",
+      "Requirement already satisfied: sniffio in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from httpx->paddlepaddle==2.5.2) (1.2.0)\n",
+      "Requirement already satisfied: certifi in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from httpx->paddlepaddle==2.5.2) (2020.12.5)\n",
+      "Requirement already satisfied: idna in c:\\users\\shuwe\\appdata\\roaming\\python\\python38\\site-packages (from httpx->paddlepaddle==2.5.2) (2.10)\n",
+      "Requirement already satisfied: httpcore==1.* in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from httpx->paddlepaddle==2.5.2) (1.0.2)\n",
+      "Requirement already satisfied: anyio in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from httpx->paddlepaddle==2.5.2) (2.2.0)\n",
+      "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\shuwe\\anaconda3\\envs\\wenwen\\lib\\site-packages (from httpcore==1.*->httpx->paddlepaddle==2.5.2) (0.14.0)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install paddlepaddle==2.5.2\n",
+    "from LAC import LAC\n",
+    "lac = LAC(mode=\"seg\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "proprietary-efficiency",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "proprietary-efficiency",
+    "outputId": "a2b21281-ff87-4757-d3ff-88b9702436de"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[[nan 0.5 0.5 0.5 0.5 1.  0.5 0.5 0.5 0.5 0.5 0.5 1.  0.5 0.5 0.5 0.5 0.5\n",
+      "  0.5 nan nan nan nan nan nan nan nan]]\n"
+     ]
+    }
+   ],
+   "source": [
+    "#compute the word length feature\n",
+    "text_word_len = [compute_BSC_word_length(txt, lac) for txt in [sentence]]\n",
+    "text_word_len = pad_seq(text_word_len, cf['max_sn_len'], fill_value=np.nan, dtype=np.float32)\n",
+    "print(text_word_len)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "severe-industry",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "severe-industry",
+    "outputId": "93474602-f510-4841-c612-dca9de230a9b"
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "'wget' is not recognized as an internal or external command,\n",
+      "operable program or batch file.\n"
+     ]
+    }
+   ],
+   "source": [
+    "#Download precomputed sn_word_len mean and std (from CELER dataset) for feature normalization\n",
+    "import pickle\n",
+    "!wget https://github.com/aeye-lab/Eyettention/releases/download/v1.1/feature_norm_BSC.pickle\n",
+    "path = 'feature_norm_BSC.pickle'\n",
+    "file_to_read = open(path, \"rb\")\n",
+    "loaded_dictionary = pickle.load(file_to_read)\n",
+    "sn_word_len_mean = loaded_dictionary['sn_word_len_mean'].numpy()\n",
+    "sn_word_len_std = loaded_dictionary['sn_word_len_std'].numpy()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "dried-biodiversity",
+   "metadata": {
+    "id": "dried-biodiversity"
+   },
+   "outputs": [],
+   "source": [
+    "#normalize word length feature\n",
+    "text_word_len = (text_word_len - sn_word_len_mean)/sn_word_len_std\n",
+    "text_word_len = np.nan_to_num(text_word_len)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "according-decimal",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "according-decimal",
+    "outputId": "e88d7af1-1d6a-4d65-86cc-f6bb50dd3c2a"
+   },
+   "outputs": [],
+   "source": [
+    "tokens['input_ids'] = torch.tensor([tokens['input_ids']]).to(DEVICE)\n",
+    "tokens['attention_mask'] = torch.tensor([tokens['attention_mask']]).to(DEVICE)\n",
+    "word_len = torch.tensor([text_word_len.squeeze()]).to(DEVICE)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "Nrm-Rq4Ya2TX",
+   "metadata": {
+    "id": "Nrm-Rq4Ya2TX"
+   },
+   "source": [
+    "**Generate synthetic scanpath**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "novel-monaco",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 74
+    },
+    "id": "novel-monaco",
+    "outputId": "8169a529-a161-4e48-d324-0c4399d8ff70"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LabelEncoder()"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from sklearn.preprocessing import LabelEncoder\n",
+    "#prepare the saccade range classes for model predictions, cf[\"max_sn_len\"]-1 -> stop class\n",
+    "le = LabelEncoder()\n",
+    "le.fit(np.append(np.arange(-cf[\"max_sn_len\"]+3, cf[\"max_sn_len\"]-1), cf[\"max_sn_len\"]-1))\n",
+    "#le.classes_"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "mathematical-dimension",
+   "metadata": {
+    "id": "mathematical-dimension"
+   },
+   "outputs": [],
+   "source": [
+    "syn_scanpath, density_pred = dnn.scanpath_generation(sn_emd=tokens['input_ids'],\n",
+    "                                                     sn_mask=tokens['attention_mask'],\n",
+    "                                                     word_ids_sn=None,\n",
+    "                                                     sn_word_len = word_len,\n",
+    "                                                     le=le,\n",
+    "                                                     max_pred_len=50)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "bacterial-cardiff",
+   "metadata": {
+    "id": "bacterial-cardiff"
+   },
+   "outputs": [],
+   "source": [
+    "def post_process_scanpath(syn_scanpath, sn_len):\n",
+    "    syn_scanpath = syn_scanpath.detach().to('cpu').numpy()\n",
+    "    max_sp_len = syn_scanpath.shape[1]\n",
+    "\n",
+    "    #Find the number \"sn_len+1\" -> the stop class\n",
+    "    stop_indx = []\n",
+    "    for i in range(syn_scanpath.shape[0]):\n",
+    "        stop = np.where(syn_scanpath[i,:]==(sn_len[i]+1))[0]\n",
+    "        if len(stop)==0:#no stop class can be find -> exceeds the maximum length of the generated scanpath\n",
+    "            stop_indx.append(max_sp_len-1)\n",
+    "        else:\n",
+    "            stop_indx.append(stop[0])\n",
+    "\n",
+    "    #Truncating data after the stop class\n",
+    "    syn_scanpath_cut = [syn_scanpath[i][1:stop_indx[i]] for i in range(syn_scanpath.shape[0])]\n",
+    "    return syn_scanpath_cut"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "southern-cycling",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "southern-cycling",
+    "outputId": "32c519fa-5251-4869-fc78-242bae3cc5f3"
+   },
+   "outputs": [],
+   "source": [
+    "sn_len = (torch.sum(tokens['attention_mask'], axis=1) - 2).detach().to('cpu').numpy()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "563c5aa6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[ 1  2  4  6  9 12 16]\n"
+     ]
+    }
+   ],
+   "source": [
+    "syn_scanpath = post_process_scanpath(syn_scanpath, sn_len)\n",
+    "print(syn_scanpath[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "imposed-passing",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "imposed-passing",
+    "outputId": "2400559b-3d51-461b-8df2-19f1ccc621da"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['中', '国', '手', '男', '雪', '中', '联']\n"
+     ]
+    }
+   ],
+   "source": [
+    "fixated_character = [sentence[idx-1] for idx in syn_scanpath[0]] #python index start from 0, our model prediction index start from 1\n",
+    "print(fixated_character)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "Q9ojBr5rba-R",
+   "metadata": {
+    "id": "Q9ojBr5rba-R"
+   },
+   "source": [
+    "**Prediction visualization**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "dental-knock",
+   "metadata": {
+    "id": "dental-knock"
+   },
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "plt.rcParams[\"font.sans-serif\"]=['SimHei']\n",
+    "def visualize_distribution(syn_scanpath, density_pred, timestep, text):\n",
+    "    #visualize density map\n",
+    "    plt.figure(figsize = (25,5))\n",
+    "    sac_range = np.append(np.arange(-cf[\"max_sn_len\"]+3, cf[\"max_sn_len\"]-1), cf[\"max_sn_len\"]-1)\n",
+    "\n",
+    "    #current gaze position\n",
+    "    cur_loc = syn_scanpath[timestep]\n",
+    "    #convert the saccade range classes to the target word indexes\n",
+    "    target_loc = sac_range + cur_loc\n",
+    "\n",
+    "    prob_valid = density_pred[np.where((0 <= target_loc) & (target_loc <= (len(text)-2)))]#exclude SEP token\n",
+    "    prob_valid = np.append(prob_valid, density_pred[-1])\n",
+    "\n",
+    "    plt.subplot(2, 1, 2)\n",
+    "    sns.heatmap(prob_valid[np.newaxis, :], annot=np.array(text)[np.newaxis,:], fmt=\"\", linewidths=2, cmap=\"YlGnBu\", xticklabels=False, yticklabels=False, cbar=True, annot_kws={\"fontsize\":15}, cbar_kws = dict(use_gridspec=False,location=\"top\"))\n",
+    "    y = 3\n",
+    "    for past_ts in range(timestep):\n",
+    "        dy = 0\n",
+    "        dx = syn_scanpath[past_ts+1] - syn_scanpath[past_ts]\n",
+    "        #if refixation, display on next line\n",
+    "        if dx <= 0:\n",
+    "            dy = -0.5\n",
+    "            plt.arrow(syn_scanpath[past_ts]+0.5, y, dx, dy, color='b', ec ='blue', head_width=0.2)\n",
+    "            y = y + dy\n",
+    "        else:\n",
+    "            plt.arrow(syn_scanpath[past_ts]+0.5, y, dx, dy, color='b', ec ='blue', head_width=0.2)\n",
+    "\n",
+    "    plt.arrow(syn_scanpath[timestep]+0.5, y, (syn_scanpath[timestep+1]-syn_scanpath[timestep]), 0, color='r', ec ='red', head_width=0.2)\n",
+    "    plt.xlim([0, len(text)])\n",
+    "    plt.ylim([0, 3.5])\n",
+    "    ax = plt.gca()\n",
+    "    ax.axes.get_xaxis().set_visible(False)\n",
+    "    ax.axes.get_yaxis().set_visible(False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "cross-commercial",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 130
+    },
+    "id": "cross-commercial",
+    "outputId": "66a6e06d-7473-412f-9dba-10b9db130b9a"
+   },
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAABYEAAACYCAYAAACyLiZBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAssUlEQVR4nO3dd5gT5f7+8XuSbLbRqxSRjjTxq4DtJ+CCIoqIiEhRj4oiFlQUKSoWQERFOIIoCigHRTw0kSpSVBQERakKohwROPRlKVvT5vdH2LDZAnvOgczu5P26rr1IZp4kn5kMyeSeZ54xTNMUAAAAAAAAAMCeHFYXAAAAAAAAAAA4fwiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAGAbhmGUMwzjesMwKlhdCwAAAFBUEAIDAABbMwxjimEYawzDeP4MbSobhvFtJOs61862nIZhlDYMY4lhGMsMw/jMMAx3pGv8XxViGatIWiSppaSvDMOoGNECz5HCbLOn2lU2DGNDpOo61wrxfroMw9htGMbXp/6aRrrG/9V/8F6+YxjGLZGqCwAARB9CYAAAYFuGYXSR5DRN82pJVQ3DqJdPm7KS/iEpMdL1nSuFWU5JvSSNMU3zekkHJN0YyRr/V4VcxsaS+pum+YqkpZIui2SN50IhlzPbaEnxkans3Crkcl4iaYZpmm1O/W2JbJX/m8K+l4ZhXCvpAtM0F0S0QAAAEFUIgQEAgJ21kTTz1O2Vkv5fPm38ku6UdCJCNZ0PbXSW5TRN8x3TNJedultR0qHIlHbOtNHZl3G5aZprDcNopWBv4O8jV94500Zn32ZlGEaSpDQFA/3iqI3OvpxXSrrNMIzvDMOYbhiGK1LFnSNtdJZlNAwjRtIkSbsMw7g1cqUBAIBoQwgMAADsLFHSv0/dPiGpcu4GpmmeME3zeESrOvfOupzZDMO4SlJZ0zTXRqKwc6hQy2gYhqFgqO9VMOAvbs66nKeG8nhB0uAI1nWuFeb9/FFSa9M0/5+kY5Juikxp50xhlvEeSb9Kel1SS8Mw+kWoNgAAEGUIgQEAgJ2l6vTp8iVk332fQi2nYRjlJI2XdH+E6jqXCrWMZtCjktZI6hih2s6lwiznYEkTTNM8FqmizoPCLOdm0zT3n7q9XdKZhsYoigqzjP8n6X3TNA9I+ljSdRGqDQAARBm7/hACAACQpJ90+hTsZpJ2WVfKeXXW5TzVe3SmpCGmaf4VudLOmcIs4yDDMO45dbeMgr1Hi5vCbLPtJD1qGMbXki41DGNyZEo7pwqznB8ZhtHMMAynpNskbYpQbedKYZbxD0m1T91uLqk4/t8EAADFgGGaptU1AAAAnBeGYZSS9K2kFZI6SOou6Q7TNJ/Pp+3Xpmm2iWyF50ZhltMwjIcljdTpIO1d0zT/Gela/1uFXMayCgbdsZK2SnrULGY7u//JNnuqfbHcbgv5fjaR9IkkQ9J80zSfs6LW/1Yhl7GkpA8UHCoiRlJX0zT/nc/TAQAA/E8IgQEAgK2dCgavl7Tq1CnXthQNyxkNyyixnHYSDcsIAACKB0JgAAAAAAAAALAxxgQGAAAAAAAAABsjBAYAAFHJMIw+VtcQCdGwnNGwjBLLaTfRspwAAKBoIAQGAADRKloCmGhYzmhYRonltJtoWU4AAFAEEAIDAAAAAAAAgI1xYTgAAJDLDlOSTJmSApKk4P5CcJ8h9/Tg/ex5gdO3zYKnhz1Hzseb4dNNZe+nmDLNHM8R9phArnbBW6dbS6YZvG2axqlW0kdTvlCv+288NV2h6WGPN3M+hxG6Hcg5Pdfjw+blenwg3+nB1wpkv64pBU7V6TdzTFcB00/d9p+aF8j1mOUzlui67jeFTzezn+P0Y/zK+XxGWLuczxuqoYCacy6LaRq56s+/XcHTg4/PXm8FtfvXggW66OZOhXgdI+w1Q++bmf1nnr4fyPH+5Gp3uo0ZNr3gdtm3zbDpYTUEctdj5nmuE98tUImrbgl7ISN0O9cKzbEhG2aO6bnmKce8fJ8ru9BTy2DkXtCcb07O58pTj/JZoadf38huY5o68a+VKlUzKdcKyed1cryGkd0mn+c+0+sbOZch58rOsW6CnzE5PvPMQOhNNRU49dmUu10gx3uYe3pAUkDpKRsVX+aSXI83c7U7/fic95W7nnxeJ2P3DEMAAACnEAIDAIBcoiMEzi8rCsuNcjxOKr4hcL7TTXuFwIV/neIdAgeyN2qbh8B5arNpCBx+nxAYAACcXwwHAQAAAAAAAAA2RggMAAAAAAAAADZGCAwAAAAAAAAANkYIDAAAAAAAAAA2RggMAAAAAAAAADZGCAwAAAAAAAAANkYIDAAAAAAAAAA2RggMAAAAAAAAADZGCAwAAAAAAAAANkYIDAAAAAAAAAA2RggMAAAAAAAAADZGCAwAAAAAAAAANkYIDAAAAAAAAAA2RggMAAAAAAAAADZGCAwAAAAAAAAANkYIDAAAAAAAAAA2RggMAAAAAAAAADZmmKZpdQ0AAAAAAAAAgPOEnsAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY66zzDcjUgUAAAAAAAAA4H9hFDSDnsAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMCwnb17pb/+kvx+qytBURcISJs3SydOWF0JigU+XAAAAAAAxZRhmuaZ5p9xJlAUVaggpaVJPl/wds2aUsOGUqNGUt26Uu3awb8SJayuFFabPFl65JHg7dhYqVo1qV49qUkTqX794HZSp45Utark4JAZ+HABAAAAABRtRoEzCIFhN4mJUnp63ukxMVJ8vGSaUkaGlJAgVa8ezG6aNg2Gf3XqBDOcKlUko8D/NrCL8eOlZ56RsrLCpxtGcPtwuSSvV/J4pEqVpFq1pMaNg7lf9rZSu3Zwu0IU4MMFAAAAAFC0EQLb0aZN0r33BgMqnPbrr//d4wwj+GeawT+XS3K7gz1ES5UK5j+wl32/puhTo4eqm3v+o8dlbytScEgJhyOYA8a6pdi4YCdRBzmf/fw3Hy45jyh4PMFexJUrBwPhxo2lrl2lpKRzXysAAAAAIBoVmEa4IlkFzq3du6WNG62uwj6yw99sPl/wLz1dSkmxri6cP7WUotbm14pT1tkb52Qq/BBZQFLWqb+Tkg6fqwpR7JlmcAiJhITgUaVAQNq3L9jFPCtLatnS6goBAAAAAFGAnsCwnYLO2JYkpzOYxRiGlJkZvF+1avCs7SZNpAYNTo8DW716sPMe7Kug4SCyxccHczufL7i9lCsnXXSRdPHFwU6cdeuePsu/dOnI1g4L8OECAAAAACja6AmM6OFwSCVLnh6es0wZqUaNYAbTpMnp6zfVqRMM9RC9sod0KF06mNtJwSFb69QJbisXX3x6W6lRIzjkA6IYHy4AAAAAgGKKnsCwndWrpRMngllMzZrBMX2B/KSmSgsXBreTOnWCY/lyzS4UiA8XAAAAAEDRxoXhAAAAAAAAAMDGCgyBHZGsAgAAAAAAAAAQWYTAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjhMAAAAAAAAAAYGOEwAAAAAAAAABgY4TAAAAAAAAAAGBjrrM32XH+qyhW6p/6l/USjvWSV3CdeAI/WVxH0eJ2XC5JyvSvtbiSoiXOeaUkKb5GD4srKVoyds+QJA1Yt9LiSoqO0VckSZKy/D9aXEnREutsIUky9ZvFlRQthhqcusX3czj2W/JineSP9ZI/1kv+WC95sU7yx3rJH+slf6yXvFgn+at/xrn0BAYAAAAAAAAAGyMEBgAAAAAAAAAbIwQGAAAAAAAAABsjBAYAAAAAAAAAGyMEBgCEub51MzWoW9XqMoBiwzRNq0sokjIzs+T1+s7YxjRNZWZmRagiAHbk8XitLqHI2Lx5h77+mgvHns3hwymh23/8sVuBQMDCagAgclxWF4D/3vr1v6hXr8Fas+YjlS9fxupyLDF27EeaNm2+ypYtdda2KSkn1KvXzRow4N7zX1gRsO/fh1W+QmnFxroVCAT07oS56tT5Wm3c8LueH/Kuft78kZzO6DwO5PP5lZGRpZMn0nX8eJqSjxzT/v1HtWvXfm3ZtFNt212uv91/k9VlWmbcK/frk7nfaviY2VaXgiLI4/HK4XDI5XJaXUqR8PNPv2rkyMkaM/YZ1ahRRRs2bFevnoOVmBgf1i4zM0v9Hu+pPn26WlRp5A0b9p62bNkhh+P0d83OnXtUtWpFxcfHSQqGwFlZHs2ZM1YlSiRYVWpE/PTTr+rZc5C2bJkrtzsmNH3evJV6440PtXr1RxZWVzQdPXpciYnxio11W11KkcC+f/6GDZuotLQMjR070OpSLPfbb7s0Y8ZitWnTQpKUlpahyy7rpqpVK8owDEnS/v1H9Pnn41S//kVWlhoRHo9XTZt20aZNsxUXFytJSk4+pg4dHtaCBeNVqVI53XXXEI0Y8ZjatbvK4mrPv0AgIL8/oJiY0zGQaZryen3asuV3HT+eqqSklpKkyZPnqmbNqmrX7kqryo2o11//UFOmzC10++HDH1O3bu3PY0VFw4ABb2rBgq8L3b5jx1Z6881nzls9+N8RAhdj2T/Anc7o/SHudseobdsrNXr002dtO3jw2Kj5EeHz+fW3u4fp4Ue6qEvX6/TL1n/pH1MXqUevG+R2uxQXFxuVAfDdPYbpl61/yu8PyOVyKjExTqmpGbqo5gWqW7eaypUvrf/X6hJVrFTW6lIjxu12yWEYyvL4ZJqm6tepqpo1KunjOavytHU6HYp1xyg9Izp67S3v/5yyTp6Uw3XqqzJgypeZKVdcnOQI/pAyfX453DG68Z3RFlYaWa8Mm6ovlqwtMATOyvLK4/HquqTL9dbb/SNcXeRd3LC26tarodu7PKVFiycoNjZG1atX0pfL3g9rN/KVSWHBXzQYOfLxPNOSknpr5Mgn1Lx5YwsqslZsrFuGYeTZDtzumLAf5DitV6/B6tu3m2699TqrSykS2PeXUlPT5fF4w76DVq/eoNtvv14nTqSGtQ0EguFWuXKlbL3OUlPT1a/fq4qNjVFaWoa2bftTffsOl9fr1ciRT0iSFi6cEDo4ecUVPaPm+yj7szXnb8Dy5cuoT5/btWvXPu3Y8Zdq1KgSFQGwJP388zb17v2CsrK8qlSpnDwer1JT01WyZKLGjHlGw4dP1DXXXKrYWLfmzVuhXr06Wl1yxLjdMWrf/mqNGzfkrG3vvntI1HSGiIlx6bbbkjRq1Ol9+rfe+lgdO7ZWnToXhrUdPHisDCP6Mobihj3OYsx1KpiIlg+g/GQHmaZpKiMjS7GxMWE7edk9jLJ3AHL2RrIzl8up+3p31KefLFOXrtdp8aLvZchQjzueV2amRxkZWWrb+lFlZnlUtWoFzZr7qtUlR8Q77w+Q1+tTYmJcaGewQ7un1fvBjurY6RqLq7PGM4/equf75+2Z+Ou3b+Xb3ufzq2Ttu853WUWCw+VS455dVTOplSQp/XCyVjz9vK55YYBKVa8mSdq7ep22zfzMyjIj7uURD+rlEQ/mO+/rr37W0GffV7NL6+qRx7pEuDJrJCTEadSoJ7Vi+VpVqlROycnHtHfvIbVo3iPUZuVXkyVJhlVFWmDEiPe0du3mPNMPHTqqQYPGKj4+Ns+82bPHhHpq2ZHT6ZDDkXcriIlxKSYmPJCZMmWuWrZsqqZN60WqPEuNHTtNn3yyWJUrlw+b7nQ6NGnSbE2adPrMlJMn01WpUjnNmvVmpMu0HPv+0vvvz9a0afND68Lr9Soz06OpUz/X1Kmfh7UNBALKyvLoyy/fV7VqlawoNyJcLqdatbpc8+d/pfr1a+qFF1opEAjI6/WFfvu0bn1fqH1qarrtt6HDh1N0550DQvfbtn1AkrR48Tvq02eY9u49oE8//UInTqRJCh6gvOWWNurf/25L6o2Uyy5rqEWL3lHnzk9o1aqpWrdui1577QPNnTtWklSmTEktWfKdrriiqQ4eTFbnztFzAO6yyxqqXr0aZ2yTkZGpI0eO6ZFHuqtq1YoRqsxaMTFOORyn91EOHkzWxImzVKfOhXlC4Isvri2fzx/pEiNix46/NHz4RG3d+ocqVSqvfv16qGPH1ho8eKw++2xlnva//bZA69Zt0T33PBuaVqVKBd1ySxs9/ngvSw/+EwIXEz//vE09egxUyZKJoWl+f3DsojZt7g9NM01TqanpWrjwbdWrZ/9TfLJ/TB08mKykpN6KjY2VYUjp6ZkqUSIhFAJPnz5KkkKnQUWDzre1liFDaWkZWjj/W737/kBddvnFWrH8Rw199j2t+GaC1SVGXFysW3Fx7jN+6Pr9Afl8vqjpNf7a+HkaNe4z+f0BJSbEasuqsRo2epamfvpVnrZOp0MxNv/RkJOjkMtq2Lh3UWGlp2dq3NiZ+mzuN3pmUC917ZZkdUkRkZmZpdmzl6l79w5qm+N0yZw9gZOSHojKMYMPH05Rr14d1aNHh7DpSUm99dpr/fP0BG7Q4Jao6ZmWW85dk8zMLI0aNUUzZizR3XffEjUhsNPpVFLSFRo+/DEdPJisQYPG6uOPX5XD4ZDH41WXLk/q/fdfVIUKZbVo0SrNnLnU6pLPO/b98/fUU/foqafuCd2/776hatSojp555l7rirJYXFys7ruvszZs2KYrrmiqLl3ahebt339YCQlxWr/+09C0K67oaUWZEeVyOXXiRJrWr/9UDRrcopUrp6h58+6KiXEpJeWEJkx4TiVLJuqee57VypVTNH78Jzp+PPXsT1zMnalD1OHDKRo48H41bFhbc+cuV/v21ygtLUOZmZ5CDb1Y3LVqdbmWLl2tZs1uV0xMjEzTlNPp0A8/zAi12bRphwYOHKNVq6ZaV2iEDRv2WOigksvl1NSp81SrVjXdfHOwk4zP51dKyglt3/6nuna93pZDe2VlefTggy+pa9frNXr0AK1c+YMGDhyjhg1rS5KSkq7Qa689WeDjV62aqpgYl3bs+EuDBweHP3vooTsiVH1ehMDFRPYPo2+++TB0Ks+2bf9S585PhE07cOCIWre+L2oCrGwXXFBBv/4aPPq/Y8df6tt3mFaunBLWZsaMxVaUFnGmacrn8yshIU49et2gzEyPXh7RRzExLv2y9V/au+eQ/P6Aftn6L0kKta1X/8KzPHPxN+ylDzV/3nd5pj83+H09N/j0qdtOp0M/b/kwkqVZJvtobc0alfTyM3dKpvTJ3G8lSe1aXaK/3dlGdz86TlLwx2f2D9Bo8csns7Vt5rzgnUAwyFs9bHTYcBDuUiUsqq5oSE1NV9fOzyouzq0ZM4epdp1qVpcUMXv2HNSHH8zTwoWr9MYbT+mRh0coK8uj/fuP6JaOj0mSDh86qh7dByk5+ZhiYlyaPXuZXnixr1q0aGJx9edX9mfLa69N0U8/bZMkTZw4VFLwh2a3bsEeWjVrVtXrrz8lSfJ6o+MA3M6deyQF11HOg5KrVv2kV16ZJNMM6L33XgiN6RkNunRpp717D+qWWx6TaZr666/96tSpX2j+77/v1n33DZVpmurf/x717n2bhdVGBvv+ZzdnznKtWbNRmzfv0D//+UWe+XXrXqhPP33Dgsqs4fcHdOBAspYuXa3fftulqlUryel0KCMjS82bdw+1S01Nt7DKyCho2Dun05nvGRlSdHUWys8NN/RRenpm2LRZs77UTTddGzXjbbdte6V+/PHT0OdvRkamfv55mz79dIlcLpeOHj2mkyfTNWTIW7rwwgv0yCN3WlxxZPz22y517vxE2LSLL+6Up92KFZNtGQLv3LlHBw4cUY8eHVShQln16NFB69dv1e7d+yUFe0uXOsPvwRIlEpSYGK8rr7xEd9zRXitWrCMExtkV9GVVkGj8EsvIyMxzOqXP55fH41VCQpxFVUXevn1HdHP7/nI6nSpTpoT+OfsVTf/oCzmdTu3de0hZWV41bFRTPboN1VVXN1EgYKrpJXVUr779v8Sef/Fvev7Fv8ntjgn9H+nQ7mk9+niX0HAQPp9fnqzou8r0/GmDVa92FT3x3AfyeHySpNKlEtSscU1rC7NYYYaD2D5nvpUlWs7n82vfviNauGS0alx0gdXlRFS9ejX0+fxxGjNmmhIS4vT777u1fMWkMz4mGBTb/zNmxIh+io11KyvLo6wsjySpdOkSoX/HjRss6fR4jd9882FU9AT2+wPq2vWp0AHbiy+upb5979CePQf0/PPj9dBDd6hbt/ZRN0Zw9eqVVb16Zd1xR3sdPnxU06YtUKdObULz33xzmtq1u0put0uXXtpAVarY/zRc9v3PbOPG7Xr55XfVt2+3fE/hnzVrqWbMyBsM21lCQpy+/36jDh8+qrp1a6hq1YoaMuQtPfXU3erT5w6lpWUoOfmYbr31iai8NkhOvXu/KIfD0NGjx9Wq1b1KTU3X7bdfb3VZEXPyZJqaN+8uv9+vWrWqS5ImTXpJiYnxWrDgG+3YsUsDB94vj8cbFd/N0ukLH+dc3thYtxIS4tSwYW05nQ7t3XtQGzZsV8OGtaLq4pz16l2kFSsm68EHX9J117XUwIH3aenS1WrQoJZq1LhAmZnBfb3s/Ty7ueCCCoqPj9Vbb03XoEH3q0SJhNDF75YuXf0fPVdMjFM+n+98lFlo0bWHWYz9pzt20bYjKEmdOj2uI0dSZBiG0tMz1bx5d/l8PiUmxkfVFberVauojVs/1o8//KrnBk9UhYplNGXq85KkN177WJLU8672url9f703+ewD39tJYXrJuFxO24+Tllt8nFsJ8W5t2PKnZs5fo4rlS8nj9Sk2NrgTVLpUghyGIbfbpcPJJxQIRMep7QF/4ca0MqOkd3T2qWA5D6JIOS5UlOv/jcfj1ckT6SpRMt7WPdRKlEjQCy/0lcfj1Q3tr1aZMiXVqVPeC6JJ0shX+qlVq8tVpkzJCFcZedmnjvbrN1IHDyZLkl588WHVqlVNXq9PDzzwYqjtlCnDdMEFFSypM9KcToc2bJgVNm358u9VuXJ5LVv2ftT82M7N7/fL6XSqTp3qKlEi2MM199AGtWpVU4kSCYqNdcvn89v+u5p9/4Jt2vSbHnpo2FmDcrtvIzn5fH4NH/6Y3O6Y0Cn/ixd/q6uvbqbevYNj9GdledSv36vq0qWt7QOsQMBUZmaWnnsueDbbc8+NU2ZmlrxenwKBgKZMeTmf4SBOWlx15JQsmaj16z8NjQksKTRM06pVP6lChTKqX9/+w8vkNGbMNM2cuVRpaRlKTIxXRkaW3G6XbrutrZo3b6wbbrhamzfv0PLla3XPPXl7wdrZr7/u1FNPvaH27a/RI4/cqbS0DL366mQ9+GBXVayYJNM05XA45PP55Xbb7wBTuXKlNX78s3r22be0ePG3uvPOG9WnT9fQ/vzKlT+EnW1xww1X53tx5L17D+rzz79SUtIVEas9P4TAxUQgEAwZcg7qn31ads5p2eMORuP4g8tOjb+YPRzEZ5+9pffem6W+fa3ram81V0z4zu/uvw7o6msusaga6wQCAaUcPSl3bIycucbCMk1THq9P6Wnhpz9lebxKSIi1dXglBXsaTZvwuKpVKa+nXpiqPndfr2ef6KLMLK9cLofiYt3avnqcHA5D8XFuNWnVX7v/fcTqsiMivkJ57Vr2tXYt+1qSlHbwsBzuGP08IXyomYSK5fN5tP38e+9h3dZpsNxuV75hQ9fOQ/Ida+6tt/ureYuGkSjRMh99tFAHDxzRuHGD5fF4te/fh7Rp8+ywz48bru8jSRoQReNWZmV59O67QxUTc3qbyR4Go3Xr5laWVuS43TFRGwBLUqdO/eTzBeRyOXT8eKoCgYDefPMfofkOh0Pjx3+i+PhYeTxeXXlls3x/YNkJ+/75mzt3uYYPf0/9+vXU6tUb9cEHn2n69EV52nm9XjVoUMuCCq2xYsVaPf30aMXExIT18j15Mk1ffvl96DPYNE3t3LlHzZo1COttbzcej1dxcbF6+eVHNXv2Mr388qOqVq2yTNNUZqZHPXsODl1Hpnnz7srK8ui229paXbZlvF6f/H6/YmPdOnkyTfHxwbNos6+vY+eLtmYbPLi3nnzyLjVr1lXz54/XffcN1YgR/dSiRWN16dJf+/Yd0mWXNbK6TEusXbtZe/Yc0OTJczR58pzQ9GHDJmrYsImh+xMmPKt27a6yosTz7tprL9OKFZO1cOE3evfdmVqy5Ft9/HHwulNXXdVML774cKht7rPQW7e+T35/8Az1Dh2utXwYEULgYiJ7bL3Cjgvm8dj/VNOcstdPziP+pUuXUKlSifrll522H3uxMJKTj+uHdb/quRfuCxvXddef+7Vv3xFdfU1TC6s7v44fT1O7Nk/k6cEoSZmZHr06/CO9PnJ62HSv16fXRj+sdjfYd0zG8mVLatqEfmrcoIYOHDqmjEyvXn97nl5/e54k6babWurFAd10adKAMz+RDQV8Pl054DEZOULN74aPVokqlXXpA/a+cnRBLqxRWes35h0rOz0tU1e2eECz572qatXsf4p2fubOWab7T/W0yg7CO9z4cNgVvw4dTA7bnqLBG29M1Zo1G+VyOQvspZiSckLx8bFauvS9CFeHomTRoneUnHxMc+Ys05w5y/Xssw/qppuuDc2fM2e5Jk6cqb59u6l9+6tDAYWdse+fV0rKCa1YsU6jRw9Q27ZXaPXqjbr//tsKHA5i1qxlFlRpjfbtr1H79tfkmd6gwS1atGhC1Jxtka106RIaP35I6LdhIBBQenqG3O6YUMehvXsPhnoCR5vcw0EsXPiNhg59OzSMk2maWrDgm3wvjmZnhw+nyOFwqHLl0x08HA6HnnzyLv3xxx4LK7NWr14364YbrlblyuVCw2926PCwHnroDnXunKRAIJDnOgd2cvBgslJT01WnzoXq0qWd2re/Rt27P6N33glecDM+PlbVq1cu8PGffvq6EhLiVLFiuSKxjqyvAIVSt24NzZ49RvHxZz4KV758Gc2ePUbVqlWKUGXWyu708MEHn+mttz5WbKxbDodDfn9AzZt3Dx297Nu3m7WFWszn8+uloZPU4eardMEF5bV/3xEFAqY2b/pDX3/1sz6ft0orvn7b6jLPm7JlS2rD1qn5zss9JnA0SUyMldfrV9vbX9L0d584+wOiyNaPZuqvr77NMz3l953as2pNnumNundRnZuiZyw5nPbjj1t16NBR3Xhj+GfIki/ezbcncDQZPLi3fD5f2OnJuX388ULNnLk0wpUVDTt37tGiRavUqFFtq0uxVErKCfXtO0y7du1TixZNdPx4qkaPnqrRo6fmandSH3+8UCNHTlKNGlX02mv9VaeOfS9qy75/XmXLltKECc+FTaMn8GnZY5qeaRgMn88f6vFpZ7Gxbl11VbPQ/ZgYl2bN+lIdO7ZWzZpV8yx/IBCQx+OVy+WKimFEcg8HcdttbUM9oTt2fFSHD6do1aqptt9Oclu7dpMuuaRenqCuTZsWqlmzmr78ck2eaxDZ3ZEjKXK7Y1SxYlkFAmboOg/Baxv4Qvel4BlgJ06kqmzZUgXu9xVHS5Z8p+nTF4UOICUmxuvyyxspOflY6IDsmVSpUrFQ7SKFELiYSEyMV9Om9c7aLibGVah2dpHdS6JPn67q06frGdsOHjxW/kKO8WkHfn9wZ8br8enN0dO1Z/dBjXztEUlSxUpl1aRpbfXq/oIMw9Dj/e1/UTjktXvvEXX+22tWl1EkNbm7m5r+rXuo56YvM1NLH3lG1zz/tMrUrhnW9ou+Tym+QnQMCYG8xo+boVtvTcrzgyG/nsDRZt26zRo2bKLcbneBY3empJwIjR9sd9mn62/f/qcmTZqjpUtX6+abW+UbAq9f/4v++c8v9OKLD9vySts5lS1bSiNHPqELL7xA27f/qQ0btmnlyinas+eAhgz5u/7+90GqUKGsWrW6V4MG3a/Gjetq06bfbB0AS+z7FxY9gU/r2XOQduz4K2wIHkm6+eZHQ/e9Xq/KlSutr776wKoyIyYzMyvUe/Pll9+Vz+fXuHHTtWHDdsXFueX3B3TkSIqSknorEAgoK8urUaOejIrhigKBgNas2aivvvpRu3fv1913D9FHH72q33//S4cOHdXNN7fSP/7xufr0iZ5hFQOBgGbM+EIdO7bKd/6wYe9qy5bf9dhjPbVx43Z5PF61bGnfM2mlYNbSps39cjodeULdjIwsvfLKpNCY0tKpYRY9Xn333TSVK1c60uWeN9dee5lGj56qSZPmqFOnNtq5c4+WLPlOgwbdrx9+2CKv168TJ1LDHpOQUHRC39wIgYsxrzd4VcFoGQMsP//plRWzQ2O7W/v9Vg1/aYquuLKJ7ur5oi79v/q6s8f12v3XAZUpW1KxsW5NmPiMfF6/0tMzdfToCSUnH1f58vb5sC4sfyA6Lup1Nk6nQy6XQzExTnm9Bf8/Odt8u3C4wr8ed61YpZjEBJWuWSNsut/rlTc9QwkVykWyPBQRK1f+oB9/3KrhIx4NTcv+Ts7ZEzgQCKht0gNnvYiR3bRs2VQLF04446lv0dQTeMOG7fL7A7r99v7q2LG15s8fr9q1q+vrr3/UkSPHtHfvQZUqlSjTDA5/sHbt5jzjytlVdqCbs0fR2LEfKT4+TomJ8dq1a5/q1ashtztGCQlxYT38ogn7/uEC+ezDBQIB/fzzNm3atCOqejF6PF7NmvVmnqF3ChoOwuPx5gmL7eTXX3dq0KCxuvzyRho58gk1alRbjRvX0fHjaXrnneAFs6NxOIidO/eoS5cnVb58GS1Z8p0SE+NVtWpFffjhCAUCAQ0b9p7uvbezune/UZ07P6Hq1S8IG5rHzt555586dChZ3bt3kCS53S5t3LhdLVs2kWEYevvt50JnZrzyyvvatu1PTZ8+ysqSzzuXy6mtWz/Ld9711/fRww93U5cu7SJcVeTVqXOhxo8for///WO9/fYnqlixnB566A516dJOP/ywRStXrlOLFj3CHjNx4tAiGwQTAhdjJ0+mSVJUjAFWEL8/oBUr1iopqfdZ26aknFDPnjdHoCpreT0+Tf1wkZ58uoeuv6GlMjM9+mrFeq1b+4sWL1qjgweOKjUtXT6vP7TzXKpUor5Y/pbFlUfWZ3NW6fs1W3XoYIpKlU60uhzLxce5Va1KeSVvnyqvz69A4PQPzMPbguPAGoYU43Kp/lWP6eDh41aVGlGe1DTtXLxMO5cs12UP359nTNfU/QclSfEVo2usvZwOHz4mSWEXookWSUkt9fn8cbrooqqhaTlDLCkYSNza6XHVr19TDRtG12n/48d/oiVLvivwYoKSdOxYapE6Re58uu66Flq9eoMGDLhXtWtXD01v1qyBqlatoLZtHwhNS0yM1/Dhj9nqdMrCaNasgd5442k9/fQbWr/+F82cOVqbN/+ufv1G6ppr/s/2vaLPhn3/cLk/b6Xg+J3Dh7+nY8dO6tlnH8jnUfaTleXRpZfeoZgYV54LTCYkxKljx8fCpvn9fmVmerRs2ftnHMeyOGvUqI4WLAgf6i73d7DX64uaDkLZ6tS5UPPmjVOtWtUkSWvWbNS6dZvlcBgaMeJ9paWl64EHusjtjtHYsQPVp8/LWrRolQYOvC9sX8duDh5M1vTpizRmzDOhg689etykiRNnatKkOWFtDSM4nMbQoQ9ZUWqRcfjwUWVm5v0Mtqvrrmup665rmWf6qFH9NWpU/wIf99tvC85nWf8V4yxHkk1pR6RqKSbqn/qX9RLOmvWSleVRIBAoohcICa4TT+Ani+soWtyOyyVJmf61ltbx1YqfNWfW12p5ZSPddc8Nlv/QjnNeKUmKr9HjLC2jS8bu4IUoBqxbGbHXDPj92jFvsZK371DKH3+qXP06atDlFpVvUPd0XclHtWP+EiVv/11Ol0utX3k+YvWNviJJkpTl/zFir5kfvz+gO7o8q3/t/LeqVK2gRV+8aen/o1hn8CKOpn6zrIaiyFCDU7fYbwnH/lxe1q0Tv9+vxx8fpR9+2CKn06lu3drr3ntvDZ1OevTocY0d+5E+//wrffnlexG+yBXbSv6K7nrxen0WXnyn6K4X67BO8mf9elm8+FuNH/+Jxo8foqFD39bYsQPDPl//+GO3pk2br5deeiSC+3jWrJfU1PQifqDR+u2l6GGd5K++JBV4mgch8H+MDS1/rJe8CIHzU1RC4KKGEDh/VoTAknRw4xal7jugKi0uU0LF/Mf73T5ngdwlE1XtqpaKLVkiYrUVlRBYkjZt/F0+n1+Nm9RWXJy1p90SAuePELgg7LfkZe06+eOP3QoETNWte2GBYcPevQct6LnItpI/1kv+WC95sU7yx3rJH+slf6yXvFgn+TtzCMxwEAAA5FL50qaqfOmZL/Zw8e23RKiaoqvZpdF7MSIA51bdujXO2saup64DAABEQnQNNAYAAAAAAAAAUYYQGAAAAAAAAABsjBAYAAAAAAAAAGyMEBgAAAAAAAAAbMwwTfNM8884EwAAAAAAAABQJBgFzXD9tw8EAAAAAAAAABR9DAcBAAAAAAAAADZGCAwAAAAAAAAANkYIDAAAAAAAAAA2RggMAAAAAAAAADZGCAwAAAAAAAAANkYIDAAAAAAAAAA29v8B5e+bGE8lNXMAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 1800x360 with 2 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sn_char = [c for c in sentence]\n",
+    "sn_char = [''] + sn_char + ['SEP']\n",
+    "sample_index=0\n",
+    "timestep = 2\n",
+    "visualize_distribution(syn_scanpath=syn_scanpath[sample_index],\n",
+    "                       density_pred=density_pred[timestep][sample_index].detach().numpy(),\n",
+    "                       timestep=timestep,\n",
+    "                       text=sn_char)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "orange-persian",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 130
+    },
+    "id": "orange-persian",
+    "outputId": "df44ad8a-8a17-4f20-9c1d-3dea8be063a7"
+   },
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAABYEAAACYCAYAAACyLiZBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAttUlEQVR4nO3de5yMdf/H8fe1OzN7cj7mGK1DkpR09HNIopAkJMpNStI5herugNKZpKTQkVSSRKSiUipy51ROpZxSiGXtcU7X74/Znd3ZnbXbYefaveb1fDw8zFzzndnPde3M7DXv+R4M0zQFAAAAAAAAALCnGKsLAAAAAAAAAACUHkJgAAAAAAAAALAxQmAAAAAAAAAAsDFCYAAAAAAAAACwMUJgAAAAAAAAALAxQmAAAAAAAAAAsDFCYAAAAEQVwzCqGYZxkWEYNayuBQAAAIgEQmAAABA1DMOYZRjG14Zh/Pc4bWobhvFlJOsqLcXtr2EYlQ3DWGoYxieGYSwwDMMV6Rr/bSXY5zqSPpR0tqTPDMOoGdECS0FJntc57WobhrEuUnWVlhL8jh2GYew2DOPznH+tIl3jv+0v/I6nGYZxaaTqAgAA5QchMAAAiAqGYfSRFGua5vmS6hqG0TRMm6qSXpOUFOn6/m0l2V9JgyRNMk3zIkl/SLo4kjX+20q4zy0l3WGa5iOSlklqE8ka/20l3OdcT0lKiExlpaOE+3uapLmmaXbK+bcpslX+u0r6OzYMo72kE0zTXBTRAgEAQLlACAwAAKJFJ0nv5FxeIen/wrTxSbpSUmqEaipNnVTM/pqmOc00zU9yrtaUdCAypZWaTip+nz81TfNbwzA6KNAb+JvIlVcqOqn457UMw+gsKV2BsL8866Ti9/dcSZcbhvGVYRhzDMNwRKq4UtJJxeyzYRhOSTMk7TQM47LIlQYAAMoLQmAAABAtkiT9lnM5VVLtgg1M00w1TfNoRKsqPcXuby7DMM6TVNU0zW8jUVgpKtE+G4ZhKBD2exQI/suzYvc5Z5qPBySNjWBdpaUkv+PvJHU0TfP/JB2R1D0ypZWakuzzYEmbJT0h6WzDMG6JUG0AAKCcIAQGAADRIk15Q+EryP7nQSXaX8MwqkmaKunaCNVVmkq0z2bATZK+ltQzQrWVlpLs81hJz5umeSRSRZWikuzvRtM0f8+5vFXS8abIKA9Kss9nSHrJNM0/JM2WdEGEagMAAOWE3T/8AAAA5Pqf8oZRt5a007pSIqLY/c3pIfqOpHtM09wVudJKTUn2eYxhGINzrlZRoKdoeVaS53UXSTcZhvG5pNMNw5gZmdJKRUn29w3DMFobhhEr6XJJGyJUW2kpyT7/LOmknMttJdnh9QwAAP5FhmmaVtcAAABQ6gzDqCTpS0nLJV0iaYCkfqZp/jdM289N0+wU2Qr/XSXZX8MwbpQ0UXkh2Qumab4d6Vr/LSXc56oKBN9xkn6QdJNZjk+I/8rzOqd9uX5ul/B3fKqkNyUZkj4wTfM+K2r9t5RwnytKelmBqSKckvqapvlbmIcDAABRihAYAABEjZwA8CJJK3OGTdtatO2vxD5Hwz5H2/5K0bnPAADg30UIDAAAAAAAAAA2xpzAAAAAAAAAAGBjhMAAACDqGYYx3OoaIi3a9jna9leKvn2Otv2VonOfAQDA30MIDAAAIEVjkBJt+xxt+ytF3z5H2/5K0bnPAADgbyAEBgAAAAAAAAAbY2E4AABQwHZTkkyZkvySpMD5QuCcoeD2wPXc2/x5l82it4c8Rv77m6HbTeWep5gyzXyPEXIff4F2gUt5rSXTDFw2TSOnleQ3Fbw8Z9ZHGnjtxTlt8t3fzP8YRvCyP//2fPcxC95W4P7+sNsDPyu3Hr8p+XPq9Jn5tquI7TmXfTm3+QvcJ+x2U/rs7SX6v349gtd9yv94effxFfg5wRqKqDn/vpimUaD+8O2K3h64f+5x+yc/Z++SD1T3kstCfmbw92bm/jPzrvvz/X4KtMtrY4ZsL7pd7mUzZHtIDf6C9ZiFHsuf+6TO94OM/E/kAjuXtuZDVWjbQ4aZb3v+H5r7JM+5Lexj5Raasw9GwR3N/8vJ/1iF6lGYA5r3843cNuFqy/+CCTloeT/DME0d3f2FKtfrUOixj/fzjfz7EHKw8+4feI/J955n+oO/VFP+nPemgu38+X6HBbf7pULXQ9/LCm8PvZ57f3fqVjkqNA37czJ3zzUEAACQgxAYAAAUEH0hsBnmst1DYF++nxMNIXD+x8t9LLuHwLkHIFpC4CJf2DYOgYP1hPk5hMAAACA/poMAAAAAAAAAABsjBAYAAAAAAAAAGyMEBgAAAAAAAAAbIwQGAAAAAAAAABsjBAYAAAAAAAAAGyMEBgAAAAAAAAAbIwQGAAAAAAAAABsjBAYAAAAAAAAAGyMEBgAAAAAAAAAbIwQGAAAAAAAAABsjBAYAAAAAAAAAGyMEBgAAAAAAAAAbIwQGAAAAAAAAABsjBAYAAAAAAAAAGyMEBgAAAAAAAAAbIwQGAAAAAAAAABsjBAYAAAAAAAAAGzNM07S6BgAAAAAAAABAKaEnMAAAAAAAAADYGCEwAAAAAAAAANgYITAAAAAAAAAA2BghMAAAAAAAAADYGCEwAAAAAAAAANiYo5jbzYhUAQAAAAAAAAD4J4yibqAnMAAAAAAAAADYGCEwAAAAAAAAANgYITAAAAAAAAAA2BghMAAAAAAAAADYGCEwAAAAAAAAANgYITAAAAAAAAAA2BghMAAAAAAAAADYGCEwAAAAAAAAANgYITAAAAAAAAAA2BghMGxn715p1y7J57O6EpR1fr+0caOUmmp1JSgPeG8B/qGjR6WtW6XsbKsrAQAAAKKOYZrm8W4/7o1AWVSjhpSeLnm9gcuNGkktWkinnCI1aSKddFLgX4UKVlcKq82cKY0cGbgcFyfVqyc1bSqdeqrUrFngeZKcLNWtK8XwlVnU470F+Id695Y++ijwIqpUSWrYUGreXGrVKvBmm5wceBFVry4ZhtXVAgAAAOVRkSfShMCwnaQkKSOj8HanU0pIkExTysyUEhOl+vUD4U2rVoHwL/fzZ506fP6MBlOnSnffXbhTmmEEnh8Oh+TxSG63VKuW1Lix1LJlIPjLfa6cdFLgeQX7470F+Ie6dJGWLy+8PSYm8AIzDCkrK3C9Tp3Ai6hlS+nkk/O+lWvYMPDmDAAAACAcQmA72rBBGjIkEFAhz+bNf+9+hhH4Z5qBfw6H5HIFeohWqhT4fAp72bw573f+V+Q+V6TAlBIxMYEg0OWS4uMDvUQJ+uznzM2va4we/8v3izEk5T7PTCm2wHtLBd5bEC3+7h/ohITAi8brDYTE1atLJ54Y+EaufXvpuuv+3ToBAACA8qvINIKuFOXY7t3S+vVWV2EfueFvLq838C8jQ0pJsa4ulK6/GgDn3if//fz+QG/i7Gzp2DHp4MF/rz6UHYO0RS31N0KsnPA3yJvzL0MS7y1A8TIzA2+0cXGBfwcOBIZpZGcHAmEAAAAAxaInMGynqCHbkhQbGxiqnTviNDY2MN9rkyaBeWCbN88bcVq/PiNO7a6o6SByFex8Vq1aoPPZyScHRig3aZI3zL9y5cjWjsjjvQX4h4qaDiJXUlLgxeF2B0Le2rUDL5yC8/A0bhwYdgEAAACgIHoCI3rExEgVK+bNz1mlSt7aM6eemreAU3JyINRD9MqdsqFy5UBwJwWmoUxODjxXCk5D6XRaVyusx3sL8A/Fxga+XXM4Ai+i+HipQYOiJ9BmRU4AAADgX0NPYNjOqlVSamrgM2SjRoGRo0A4aWnS4sWB50lyMnP54vh4bwH+oR07AgsaJCcHevNWqmR1RQAAAIDdsDAcAAAAAAAAANhYkSEw4+wAAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxgiBAQAAAAAAAMDGCIEBAAAAAAAAwMYIgQEAAAAAAADAxhzFN9le+lWUK81y/ue4hOK4FMYxCS9wXExts7iOssVQc0lShvdLiyspWxId7SVJbv9aiyspO1wxbSVJ+zIWWVxJ2VI38VJJUkLDqyyupGzJ3D1XkvTmjo8srqRsGZh8sSTJ419ncSVlhzPmDEmSqS0WV1K2GGqRc4nzuVCc54bHcSmMYxIexyU8jkt4HJfCOCbhNTvurfQEBgAAAAAAAAAbIwQGAAAAAAAAABsjBAYAAAAAAAAAGyMEBgAAAAAAAAAbIwQGAIRY9eUP+mXHPqvLAFDOebLd8nl9x21jmqY82e4IVQTAjtxuj9UllBkbN27X559/Z3UZZd7BgynByz//vFt+v9/CagAgchxWF4C/b+3aHzVo0Fh9/fUbql69itXlWGLy5Df0+usfqGrVSsW2TUlJ1aBBPXTXXUNKv7Ay4LffDqhGjSqKi3PJ7/fruefm6vLLL9T332/R2LHP6Icf3lNsbKzVZVrC6/UpMzNLqanpOnr0mP48eET7fj+oX3/9TRs2bNNFF52nYcMut7pMy0ycMFs9Lj1XI2/pbXUpKIPcbq9iYwzFOqLz/aOg89o205MPDtY1Nz+rX3cd0Dltmmr5/Id0LC0zpF1CvEsPT35XT037wKJKI2/JC+9q37bdMmLz+hwc3P2HqtSqKmd8XGCDacqT7dHwZ+9SXGK8RZVGxvffb9PgQQ9q3YbZcrryTsE/WLhSTz81R198+aKF1ZVNKYdTlZgUr7g4l9WllAmc+4c3fvx0padnavLk0VaXYrlt23Zq7twl6tTpLElSenqm2rTpr7p1a8owDEnS77//qYULn1WzZidaWWpEuN0etWrVRxs2vKv4nL87hw4d0SWX3KhFi6aqVq1quvrqe/TwwzerS5fzLK629Pn9fvl8fjmdeX+DTNOUx+PVpk0/6ejRNHXufLYkaebM99SoUV116XKuVeVG1BNPvKJZs94rcfsJE25W//7dSrGisuGuu57WokWfl7h9z54d9PTTd5daPfjnCIHLMUfOB/BoDfIkyeVy6sILz9VTT40qtu3YsZOj5kOE1+vToEFjdNNNV6lfv6764Yef9cor7+vqq3vK5XIqPj4uKp83V155t37Y9JN8Pr8cjlglVUhU2rF0NW5cT02aNFT1GlXUscOZqlWrmtWlRozb7ZHfb8rlcigmJka//vK79v32py697PxCbb1enzxurxIS4yyoNPK6dr5VKSnHgifKfr9f6elZSkqKV0xMINTyen2Ki3Pqy2+iJ7x5ZuJ7+uzj9YqNDT+YyO32yuP2ql2nlnp48tAIVxd5G37cpc3b9+rrxRN1xoV3KSvbo517DujUDneEtHvywcHKzo6u3mqX3T6w0LZnhozTZXcMVMOWyRZUZK24OKcMwwgJgCXJ6XSEfCBHnkGD7tWIEX3V67JOVpdSJnDuL6WlZcjt9gSPhSStWrVOV1xxkVJT00La+v2BcKtatUq2PmZpaRm65ZZHFRfnVHp6prZs+VUjRkyQx+PRxIm3SZIWL35eSUkJkqRzzhkol8tpZckRk/vemv8zYPXqVTR8+BXauXOftm/fpYYN60RFACxJ33+/RcOGPaDsbI9q1aomt9ujtLQMVayYpEmT7taECdPVrt3piotz6f33l2vQoJ5WlxwxLpdT3bqdr2efvafYttdcc0/Ie5CdOZ0OXX55Zz32WN557ZQps9WzZ0clJzcIaTt27GQZBpMNlHWccZZjDocj5//oeAMKJzeEME1TmZnZiotzhpzkmaap7Gx38AQgN7ixO4cjVtddd4XefPND9evXVYsXfyHJUN++dyozM1uZmVlq3/4/ys52q27dWnr//SlWlxwRM2c+JI/Hq6SkhODJYOfO12n48L7qddkFFldnjZdnLNGL0xYV2t7rknvDto+NjdHajS+Vdlllgsvl1N1jBqn/gC6SpN9+O6iLu9yuN958SE2bBU56Fn3wlZ55+i0ry4y40Q/11+iH+oe97esvftTjD76tlqedqCEj7N87QpIyMrM1fNR09bzoTP1x4Ihq1aisRg1q6fdNM4Ntmp9/iyTJtKpICyydPl+/btheaPuxw0e14Ok5csYVDiCuf2aUnDb+sjY2JkYxMUah7eFC4FdeXqSzzj5Fp54aHWH55MmzNffNpapdu3rI9tjYGM2Y8Z5mzMjrnXXsWLpq1aqmd+Y9GekyLce5v/TSS+/q9dc/CB4Lj8ejrCy3Xn11oV59dWFIW7/fr+xstz7++CXVq1fLinIjwuGIVYcOZ+qDDz5Ts2aN9MADHeT3++XxeIOffTp2zPtSNi0tw/bPoYMHU3TllXcFr1944XWSpCVLpmn48PHau/cPvfXWR0pNTZckde48TJde2kl33HGNJfVGSps2LfThh9PUu/dtWrnyVa1evUmPP/6y3ntvsiSpSpWKWrr0K51zTivt339IvXtHz+ejNm1aqGnThsdtk5mZpT//PKKRIweobt2aEarMWk5nrGJi8s7Z9u8/pOnT5yk5uUGhEPjkk0+St5hpwMqr7dt3acKE6frhh59Vq1Z13XLLVerZs6PGjp2sBQtWFGq/bdsirV69SYMH532mrlOnhi69tJNuvXWQpV/+EwKXE99/v0VXXTVaFSsmBbf5fIG5izp1uja4zTRNpaVlaPHi59S0qf2H+OR+mNq//5A6dx6muLg4GYaUkZGlChUSgyHwnDmPSVJwGFQ06NOniwzDUFpahhYu/EwzZjyotm1b6tNPv9HYsVP05ZevWV1ixMXFuRQfH3fcN12fz5fTu9O+QUR+w4b30HU39FRsbIwyM7J1Wff7dOMtl6lP3w6F2nq9Ptv+YQ+npH+c7f5BqiQyM7M149mlWrpwjUaO6qVLr4iOoYPxcU4NGXCBZsz+VIs/+V9we/6ewFtXPRtVf3typR1O1dk926ttj/8L2f7MkHG6fNSgQj2Bx3W/TY4o7Q2b//mRleXWk4+/obff+kSDrrk4akLg2NhYXdD5bE2YMFL79x/S2DFT9MbsRxQTEyO326Mr+ozSiy/drxo1qujDD7/UvHc+sbrkUse5f3h33jlYd945OHh96ND7dcopybr77iHWFWWx+Pg4DR3aW+vWbdE557RSnz5dgrf9/vtBJSbGa+3avC+szzmn8CgNu3E4YpWamq61a99S8+aXasWKWWrbdoCcTodSUlL1/PP3qWLFJA0efK9WrJilqVPf1NGjacU/cDl3vA5RBw+maPToa9WixUl6771P1a1bO6WnZyory12iqRfLuw4dztSyZavUuvUVcjqdMk1TsbExWrNmbrDNhg3bNXr0JK1c+ap1hUbY+PE3B79Ucjhi9eqr76tx43rq0SPwWdHr9SklJVVbt/6qvn0vUoUKiRZX/O/Lznbr+usfUt++F+mpp+7SihVrNHr0JLVocZIkqXPnc/T447cXef+VK1+V0+nQ9u27NHbsZFWokKgbbugXoeoLi86z7XIod8jOF1+8EhzKs2XLL+rd+7aQbX/88ac6dhwaNQFWrhNOqKHNmwPf/m/fvksjRozXihWzQtrMnbvEitIizjRNeb0+JSbGa9CgHsrKytYjj9wql8upTZt+0u7df8jn82vTpp8kBULPxMSEqJgX7IEHntf7Yb6pGz16skaPnhy8Hhsbox83vx/ByqyTG3Tu3XNQz01ZIBlSz16BIXFff/WDFi74So8/PUJS4IQ62gLPJx+fo2cmvS1JwUVDrhn4UMh0EFWrVrSsvrIgPS1Lw/o/rfh4p6bPvk0nnlTb6pIipvGJtXXb9T105WXtNPT25/XurLsUH+dU/TrVtfaTJyRJdWpX1WcLxqlm9cryeL0aMuAC3Xbfy1q1ZqvF1Zcuvy/whdHHM9/X7h9/kSRd9dBwSdKxw6maecckSVL1erV0+V1XS5J8Xp8cLvuP2Pnll98kBd4/8n/Z9NWX6/XYxNfk95t6/oXR6tipjVUlRlyfPp21d+8BXXrpbZJpateu33VZr9uDt//0024NHfqgZJq6446rde2wy6wrNkI49y/e/Pmf6uuv12vjxu16++2PCt3epEkDvfVW9PQY9/n8+uOPQ1q2bJW2bdupunVrBb7kz8xW27YDgu3S0jIsrDIyipqyKjY2NuyIDCm6OguF07XrcGVkZIVsmzfvY3Xv3j5q5tu+8MJz9d13bwXffzMzs/T991v01ltL5XA4dPjwER07lqF77pmiBg1O0MiRV1pccWRs27ZTvXvfFrLt5JN7FWq3fPlMW4bAO3bs0R9//KmrrrpENWpU1VVXXaK1a3/Q7t2/Swr0lq5UqUKR969QIVFJSQk699zT1K9fNy1fvpoQGMUr6o9VUaLxj1hmZpacztDhpV6vT263R4k2X2wmv99+O6CuXYcrNjZWVapU1HvvPaPXX18khyNGe/bsV3a2Wy1bnqR+/UapXbvT5febOu20pmrWbHDxD17OjRs3UuPGjZTL5Qy+Rjp3vk633zYoOB2E1+tTdhSuVH/TDc9o9679uue/g4InPmlpmdq6ZY/FlVmrJNNBPDdlnpUlWs7r9Wn/7ymavXCs6jWsYXU5EbVl+16d1W2MJowZoPT0LLVs3iA49UNR3p11t+LDTIVgN5fedpUcLoe8bq+87sBcyAk5HwwSKiSq/32Bnoy5Cwze8fo4xUZBT2Cfz68r+90rmZLH69XJJzfS9cN7a++eA3rw/pd03fWXqW//C6NujuD69Wurfv3a6t/vIh04mKI3Xl+sS3t1DN4+6ek3dFGXc+RyOdX69GaqU8f+w3A59z++9eu3aty4FzRiRP+wQ/jnzVumuXMLB8N2lpgYr2++Wa+DBw+rSZOGqlu3pu65Z4ruvPMaDR/eT+npmTp06Iguu+y2IkPSaDFs2IOKiTF0+PBRdegwRGlpGbriiousLitijh1LV9u2A+Tz+dS4cX1J0owZDykpKUGLFn2h7dt3avToa+V2e6Jm/mi326OYmJiQ/Y2LcykxMV4tWpyk2NgY7d27X+vWbVWLFo2janHOpk1P1PLlM3X99Q/pggvO1ujRQ7Vs2So1b95YDRueoKwst7Kz3apcueggtDw74YQaSkiI05QpczRmzLWqUCExuPjdsmWr/tJjOZ2x8nq9pVFmiUXXGWY59ldP7KLtRFCSevW6VX/+mSLDMJSRkaW2bQfI6w3M/7pq1RtWlxcx9evX1ubNC7V69SaNHTtZNWtW1euvPyJJeuyxQO/oq6/uqa5dh2vWrPFWlhpxJeklE+jtmhCBasqOzMxsZWW5dXKLhrq4+9k6fChVTqcjuIjVsdSMnGFAPlWtVjFqPjiUdOoLry86psjw+/3yenxyuhwhf2OCCxU5Qp8XbrdXaccyVaFCvFw2Dj3T0rN0xwOvyumM1YIlq5VyJF3fLXs8bNsb7n5RH3++XoeP2H/IaWKlwBD2dx5+WamHjkiSeozspxr1a8nn9Wn2/S8E217z8EhVqlHFgiojLzY2Rt/9L3Q6puWffqdatatq6bIpUfNhuyCfz6fY2FidlFxfSTlfFhScm7Fx43pKqpCo+Lg4eb0+249M4dy/aBs2bNMNN4wvNii3+3MkP6/XpwkTbpbL5QyOVlqy5Eudf35rDRvWR1JgSPMttzyqPn0utH2A5febysrK1n33PStJuu++Z5WVlS2Pxyu/369Zs8aFmQ7imMVVR07Fiklau/at4JzAktS2bUtJ0sqV/1ONGlWiYpRofpMmva533lmm9PRMJSUlKDMzWy6XQ5dffqHatm2prl3P18aN2/Xpp99q8ODCvWDtbPPmHbrzzifVrVs7jRx5pdLTM/XoozN1/fV9VbNmZ5mmqZiYGHm9PrlsOKKrWrXKmjr1Xt177xQtWfKlrrzyYg0f3ldVqgRGg65YsSZktEXXrudr4sRbCz3O3r37tXDhZ+rc+ZyI1R4OIXA5kTsMOf+k/rnzguXfZppmyP/R5JNPAotV5U4HsWDBFL344jyNGGFdV3ur5S6akWvnzn1q3z56hpfm8vv9Onz4qOLiXIXnwjJNud0epadnhmx2Z3uUmBRv++GVPp9fY+96SQf2p2jMvVfpnbc+00svLFZcnFNen0/ubI+6dx0j028qO9ujhUsnqm7d6sU/sA3UqVtdc+d8orlzAnNP7tr1h+LjXRo96rmQdvXr23exmfx+/+2whvZ9Si5X+FOHYf2fDhtCPDx5qE5va+95TW8c0k316lTTwBHPyOVy6MQGNVWl6eDgFymS9MPKwJQz/310blEPYztet0cDHrxesY7Y4HPjxFZNJElNzzrFytLKHJfLGbUBsCRd1uv2YLB79Gia/H6/Jj2d9wV+TEyMpk6dq4SEeLndHp17bis9MvH4ve7LO879w3vvvU81YcKLuuWWgVq1ar1efnmB5sz5sFA7j8ej5s0bW1ChNZYv/1ajRj0lp9MZ8mX9sWPp+vjjb4LvwaZpaseOPWrdurl69epkUbWlz+32KD4+TuPG3aR33/1E48bdpHr1ass0TWVluTVw4NjgOjJt2w5QdrZbl19+odVlW8bj8crnC6yJcuxYuhISAqNoc9fXiY+Ps7jC0jd27DDdfvvVat26rz74YKqGDr1fDz98i846q6X69LlD+/YdUJs20Xnu8u23G7Vnzx+aOXO+Zs6cH9w+fvx0jR8/PXj9+efvVZcu51lRYqlr376Nli+fqcWLv9ALL7yjpUu/1OzZgXWnzjuvtR588MZg24Kj0Dt2HCqfLzBC/ZJL2ls+jQghcDmR2yOtpPOCud2eIh/LjnKPT/5v/CtXrqBKlZL04487dNZZp1pVWplx6NARrV69SQ89dGNID8dff/1N+/YdULt2Z1hYXek6eiRNHdoPlcvlKBQCZ2Zma/z4FzVx4syQ7W63R09Pulvdup0fyVIjKiXlmMbe9ZJ+/mmvqlevpLg4l667oaeuu6GnJOmTZWs1ber7WrD4YYsrjTyP26vpM8aEPF8GDxqnRo3qaPwjwy2szDr1GtTQx6sfK7Q9MyNb3dvdp1nvjNIJdatZUJn1BvfvpGdeXCQpL6TZ+NnTyp/J1D2hajDUiRafvPyBflm3LTDlQxG9FDOOpskZ59ItM/8b4epQliz+cKoOHTqi+fOXa/785Zp87zBd0j1vUcH585frxenzdMOIvurW7XwlJNg/kODcv7CUlFQtX75aTz11ly688BytWrVe1157eZHTQcybZ/8FBHN169ZO3bq1K7S9efNL9eGHz+uEE6JruqbKlSto6tR7gp8N/X6/MjIy5XI5gx2H9u7dH+wJHG0KTgexePEXuv/+5xQX51J2tlumaWrRoi/CLo5mZwcPpigmJka1a+d1eImJidHtt1+tn3+O3inyBg3qoa5dz1ft2tWC029ecsmNuuGGfurdu3NgtGCBdQ7sZP/+Q0pLy1BycgP16dNF3bq104ABd2vatMCCmwkJcapfv+g1Ud566wklJsarZs1qZeIYWV8BSqRJk4Z6991JxZ70Vq9eRe++O0n16kVHz7TcD9gvv7xAU6bMDvb09Pn8att2QPDbyxEj+ltbqMW8Xp/uu2+qevbsoBNOqKF9+w7I7ze1YcM2rVixRgsWLLf1KqdVq1XS5i3vh72t4JzA0SQzI1sOR6xefmOsRt8xvfg7RJFHJ76meW8XXkRw3ffbteC9LwptH3X3QA25tkckSkMZ0+7sk1WndlXN/3B1yPbTLhgVtidwNOl2fe/AYm9Oh4wiViRfs2ilvv/omwhXVjbs2PGbli75Wi1aNLK6FEulpKRqxIhHtGvnPp11VkulHk3Tk0+9piefCp0240jKMc2ZvUSPTpylhg3r6PHHb9NJyfUtqrr0ce5fWNWqlfT88/eFbKMncJ7cOU2PNw2G1+sL9vi0s7g4l847r3XwutPp0Lx5H6tnz45q1Khuof33+/1yuz1yOBxRMY1IwekgLr/8wmBP6J49b9LBgylaufJV2z9PCvr22w067bSmhYK6Tp3OUqNG9fTxx18XWoPI7v78M0Uul1M1a1aV328G184JLEbvDVlLJzvbrdTUNFWtWqnw6NtybOnSrzRnzofBL5CSkhJ05pmn6NChI8EvZI+nTp2aJWoXKYTA5URSUoJatWpabDun01GidnaR20ti+PC+Gj6873Hbjh07Wb4ombtTyjuZcbs9euKJV7Rnz+968sk7JUm1alVXq1ZN1b//XTIMQ3feWbgHBeyvbr0aev7F260uo0y6597/6L77hwaHVGakZ+n/zrtBr895QKe2Cp3a4PxzrlfdetHVwwZ57r+zr96cv7LQHNLhegJHm183/KQl096Vw1V0CJxxNE2JNl1IpKDc4frbtu3SyzM/0LJl36p793ZhQ+D/rd2iee8s138fuNaWK23nV7VqJU2ceLMaNDhBW7f+qnXrtmrFihnau2e/xt7zrJ555m7VqFFFHTsM0+gxQ9SyZbI2bthu6wBY4ty/pOgJnGfgwDHavn2XnM7Quft79LgpeN3j8ahatcr67LOXrSozYrKysoO9N8eNe0Fer0/PPjtH69ZtVXy8Sz6fX3/+maLOnYfJ7/crO9ujxx67XR07trW48tLn9/v19dfr9dln32n37t91zTX36I03HtVPP+3SgQOH1aNHB7322kINHx490yr6/X7NnfuRevbsEPb28eNf0KZNP+nmmwdq/fqtcrs9OvvsVhGuMrK8Xp86dbpWsbExYUfTPvLIjOCc0lLgPMft9uirr15XtWqVI11uqWnfvo2eeupVzZgxX716ddKOHXu0dOlXGjPmWq1Zs0kej0+pqaHrfSQmlp3QtyBC4HLM4wmsKhgtc4CF81dXVizpQk/l3TffbNCDDz6v8847TQMGjFabNidr4MAe2rlzn6pWraT4eJdefPEBeb0+ZWRk6dChIzp06IjtF4kIx++LruHZRfH7/fL5fPK4vXI4Y4tcYKa42+3CWWDe27ff+lSVK1fQKS1DexW53R4dS81Q3bqEwNGoe5c2an9uC40cMyO4Lfe1kb8nsGEY2vb1s/L7o+vvdaPTmmrk9HsC00EUIZp6Aq9ft10+n19X9r1X3Xu004KFT6px47r64vPvdejPo/rttwOqWDFJpmlqwYIvtGb1j4XmlbOr5OQGkhTSe37S5NlKTIhTUlKCdu7cpyZNG8jlcioxMV7nnneaVaVainP/UOGm2PH7/fr++y3asGF7VPVidLs9mjev8Nz8RU0H4XZ7CoXFdrJ58w6NGTNZZ555iiZOvE2nnHKSWrZM1tGj6Zo2LTD9UDROB7Fjxx716XO7qlevoqVLv1JSUoLq1q2pV155WH6/X+PHv6ghQ3prwICL1bv3bapf/wR1797e6rIjYtq0t3XgwCENGHCJJMnlcmj9+q06++xTZRiGnnvuvuDIjEceeUlbtvyqOXMKT5NmJw5HrH74YUHY2y66aLhuvLG/+vTpEuGqIi85uYGmTr1HzzwzW88996Zq1qymG27opz59umjNmk1asWK1zjrrqpD7TJ9+f5kNggmBy7Fjx9IlKSrmACuKz+fX8uXfqnPnYcW2TUlJ1cCB9h+u7XZ7NGvWexo16j/q1q2dsrKytXz5an3zzQYtWvSF9u//U2lpGTmr4wY+RFSqVEErVsws5pHtZf67n2jVqvXav/+QKkVJL7Tjyc5ya//+FJ1/1k1yOEJD3vPb3iRJMmXK6/Fp6adPqEZN+3y7ezxHj6TplZcX67VXl+jxJ28q9C34r7/skyTVi5LF4cI5dDBVkkIWookWSz79Xmd1HaNfdu0PbkuIDw0dDMPQdx8/rh+27tHGzbsiXaKlPp+9RD9+uV4Op0MqImfIPJahuIToCDo7XXCmvvlmk+4YNVCNG9cNbm/duqnq1Kmhbl3yVpJOSkrQg+Out9VwypJo3bqZnnjyDo0a9bT+t3az3n7ncW3cuF233vK42rU7XRUqlM0PVJHCuX+o/MOQc8XExGjChBd15Mgx3XvvdRZUFXnZ2W6dfno/OZ2OQgtMJibGq2fPm0O2+Xw+ZWW59cknLx13Hsvy7JRTkrVoUehCvi1anBRy3ePxRk0HoVzJyQ30/vvPqnHjepKkr79er9WrNyomxtDDD7+k9PQMXXddH7lcTk2ePFrDh4/Thx+u1OjRQ3XiiXWLefTya//+Q5oz50NNmnR38MvXq67qrunT39GMGfND2hpGYDqN+++/wYpSy4yDBw8rK6vwe7BdXXDB2brggrMLbX/ssTv02GN3FHm/bdsWlWZZf4tRzDfJprQ9UrWUE81y/ue4hLLmuGRnu+X3+4MrmJYtPFfCCxwXU9ssrWL5p9/qnXc+1rnnnab//KeX5R+0DTWXJGV4v7S0jrIm0RHofeD2r43Yz/R6fZo+bYG+W7NZGzf+rDPOaKabb+2nNmc2D7b54/dDenH6+1r73Ra5XE7Nf//RiNXnigkMU9yXYe1Jhc/n1/UDJmnXL/tVu05Vzf5grKWvo7qJl0qSEhpeVUzL6JK5O7CYy5s7PrK4krJlYPLFkiSPf53FlZQdzpjAArGmtkT8Z/t8Pt126xNas+YHxcbGql//izRkSK/gcNKUw6maNPkNfbDwCy37eFpEF7ky1CLnEudzocruea7H47Vw8Z2ye1yswzEJz/rjsmTJl5o69U1NnXqP7r//OU2ePDrk/fXnn3fr9dc/0EMPjYzgOZ41xyUtLaOMT79k/fOl7OGYhNdMKrL7BSHw38ATLTyOS2Eck/DKRghc1hACh2dFCCxJKz9fp19+2aeLup2tevVqhm3z3LPvqmrVCures52qVq0YsdrKSggsST9u2Cmfz6/mpzRQXLy1C2UQAodHCBweIXBhVobAkvTzz3vk9/vVpEmDIsOGvXv3R7znIiFwUTjPDY/jUhjHJDyOS3gcl/A4LoVxTMI7fgjMdBAAABTQodMZ6tDpjOO2ufnW4y9GGQ1atm5kdQkAbKJJkwbFtrHr0HUAAIBIiK6JxgAAAAAAAAAgyhACAwAAAAAAAICNEQIDAAAAAAAAgI0RAgMAAAAAAACAjRmmaR7v9uPeCAAAAAAAAAAoE4yibnD83TsCAAAAAAAAAMo+poMAAAAAAAAAABsjBAYAAAAAAAAAGyMEBgAAAAAAAAAbIwQGAAAAAAAAABsjBAYAAAAAAAAAGyMEBgAAAAAAAAAb+38j5POkb5QpGgAAAABJRU5ErkJggg==",
+      "text/plain": [
+       "<Figure size 1800x360 with 2 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sample_index=0\n",
+    "timestep = 3\n",
+    "visualize_distribution(syn_scanpath=syn_scanpath[sample_index],\n",
+    "                       density_pred=density_pred[timestep][sample_index].detach().numpy(),\n",
+    "                       timestep=timestep,\n",
+    "                       text=sn_char)"
+   ]
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 AEye
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,132 @@
----
-license: mit
----
+# Eyettention: An Attention-based Dual-Sequence Model for Predicting Human Scanpaths during Reading
+
+In this paper, we develop Eyettention, the first dual-sequence model that simultaneously processes the sequence of words and the chronological sequence of fixations. The alignment of the two sequences is achieved by a cross-sequence attention mechanism. We show that Eyettention outperforms state-of-the-art models in predicting scanpaths. We provide an extensive within- and across-data set evaluation on different languages. An ablation study and qualitative analysis support an in-depth understanding of the model's behavior.
+
+The overview above describes the [original authors' work](https://arxiv.org/abs/2304.10784). This project uses the [original Eyettention implementation](https://github.com/aeye-lab/Eyettention), reorganized as the `Eyettention` package, with raw-text inference, scanpath-prefix replay, an endpoint handler, and a Gradio interface.
+
+## Setup
+
+For this checkout, run setup and inference commands from the project root (the directory containing `Eyettention/`). Install dependencies:
+
+```bash
+python -m pip install -r Eyettention/requirements.txt
+```
+
+The requirements retain historical PyTorch and Transformers pins that may need adjustment for your Python version and platform. The local preprocessing code also requires `LAC` and a compatible PaddlePaddle installation; BSC Excel loading requires `openpyxl`. Install `gradio` if using the web interface. Use a separate environment from ScanDL2 because their dependency versions differ.
+
+## Dataset
+
+For the CELER dataset, follow the instructions at [the CELER repository](https://github.com/berzak/celer). In order to run experiments, place the downloaded dataset in `Eyettention/Data/celer/`. The local loaders expect `data_v2.0/sent_fix.tsv`, `data_v2.0/sent_ia.tsv`, and `metadata.tsv` beneath that directory.
+
+The Chinese [Beijing Sentence Corpus (BSC)](https://osf.io/vr3k8/) belongs in `Eyettention/Data/beijing-sentence-corpus/`, with `BSC.Word.Info.v2.xlsx` and `BSC.EMD/BSC.EMD.txt`. See [utils.py](utils.py) for the dataset loaders.
+
+Raw-text inference uses the trained checkpoints and normalization files already stored under `results/` and `Data/`; it does not require the full training corpora.
+
+## Run Experiments
+
+The experiment scripts use paths relative to `Eyettention/`. Starting at the project root:
+
+```bash
+cd Eyettention
+mkdir -p results/BSC results/CELER
+```
+
+Keep the project root on `PYTHONPATH` when running the package modules.
+
+### For Chinese BSC dataset
+
+```bash
+PYTHONPATH=.. python -m Eyettention.main_BSC --test_mode text --gpu 0
+PYTHONPATH=.. python -m Eyettention.main_BSC --test_mode subject --gpu 0
+PYTHONPATH=.. python -m Eyettention.main_BSC_NRS_setting --gpu 0
+PYTHONPATH=.. python -m Eyettention.main_BSC_reader_identifier --gpu 0
+```
+
+### For English CELER dataset
+
+```bash
+PYTHONPATH=.. python -m Eyettention.main_celer --test_mode text --gpu 0
+PYTHONPATH=.. python -m Eyettention.main_celer --test_mode subject --gpu 0
+PYTHONPATH=.. python -m Eyettention.main_celer_NRS_setting --gpu 0
+PYTHONPATH=.. python -m Eyettention.main_celer_reader_identifier --gpu 0
+```
+
+`text` evaluates new sentences; `subject` evaluates new readers. Select an available GPU with `--gpu`. Several historical experiment scripts call CUDA directly, so use a CUDA-enabled environment for these commands. Review output paths before running; `--save_data_folder` selects the results directory.
+
+## Raw-text inference
+
+Run the following from the project root:
+
+```python
+from Eyettention import EyettentionRawTextInference
+
+model = EyettentionRawTextInference(
+    checkpoint_path="Eyettention/results/CELER/Eyettention_english.pth",
+    dataset="celer",
+    device="cpu",
+)
+scanpath, density = model.generate_from_english_text(
+    "The quick brown fox jumps.",
+    max_pred_len=20,
+    # Optional: replay observed fixation positions before sampling.
+    # previous_scanpath=[0, 1, 2],
+)
+print(scanpath[0].tolist())
+```
+
+For Chinese, use `dataset="BSC"`, checkpoint `Eyettention/results/BSC/Eyettention_chinese.pth`, and `generate_from_chinese_text(...)`. Dataset names are case-sensitive. Use `device="cuda"` for GPU inference. BERT assets must be downloadable from Hugging Face or cached locally.
+
+Keep `Data/feature_norm_celer.pickle` and `Data/feature_norm_BSC.pickle` alongside the package. The output contains fixation indices and per-step probability distributions, not fixation durations. Index `0` denotes CLS, positions `1..N` refer to words or Chinese characters, and `N+1` denotes SEP; stop at the first SEP when interpreting fixations. Use short sentences within the checkpoint's configured input limits. Prefix replay extends the generation loop, so `max_pred_len` is not a strict total-length cap when a prefix is provided.
+
+## Endpoint handler
+
+The added [handler.py](handler.py) wraps inference in a dictionary-based interface:
+
+```python
+from Eyettention.handler import EndpointHandler
+
+handler = EndpointHandler()
+result = handler({
+    "inputs": "The quick brown fox jumps.",
+    "parameters": {
+        "dataset": "celer",
+        "device": "cpu",
+        "max_pred_len": 20,
+    },
+})
+```
+
+It returns `scanpath` and `density_steps`. Optional parameters include `checkpoint_path` and `previous_scanpath`. The handler defaults to BSC on CPU and does not itself start an HTTP server.
+
+## Gradio interface
+
+The added [app.py](app.py) provides a web interface. From the project root, point it to the local checkpoints and launch:
+
+```bash
+export EYETTENTION_BSC_CHECKPOINT="Eyettention/results/BSC/Eyettention_chinese.pth"
+export EYETTENTION_CELER_CHECKPOINT="Eyettention/results/CELER/Eyettention_english.pth"
+python -m Eyettention.app
+```
+
+Open `http://localhost:7860`, choose the dataset, enter text, and click **Run**. The interface also supports an observed scanpath prefix. It selects CUDA when available, otherwise CPU. Set `PORT` to change the default port; the app binds to `0.0.0.0`.
+
+## Cite our work
+
+If you use this code for your research, please consider citing the original authors' paper:
+
+```bibtex
+@article{deng2023eyettention,
+  title={Eyettention: {A}n Attention-based Dual-Sequence Model for Predicting Human Scanpaths during Reading},
+  author={Deng, Shuwen and Reich, David R and Prasse, Paul and Haller, Patrick and Scheffer, Tobias and J{\"a}ger, Lena A},
+  journal={Proceedings of the {ACM} on Human-Computer Interaction},
+  volume={7},
+  number={ETRA},
+  pages={1--24},
+  year={2023},
+  publisher={ACM New York, NY, USA}
+}
+```
+
+## License
+
+The original code is provided under the [MIT License](LICENSE.txt), copyright © 2023 AEye. Include the copyright and permission notice when redistributing copies or substantial portions of the software. See the license file for the full terms and warranty disclaimer. Datasets and dependencies retain their own licenses.

__init__.py

@@ -0,0 +1,29 @@
+# API implementation
+
+from Eyettention.model import Eyettention, Eyettention_readerID
+from Eyettention.raw_text_inference import EyettentionRawTextInference
+
+from Eyettention.utils import (
+    BSCdataset,
+    calculate_mean_std,
+    celerdataset,
+    eval_log_llh,
+    gradient_clipping,
+    load_corpus,
+    load_label,
+    prepare_scanpath,
+)
+
+__all__ = [
+    "Eyettention",
+    "Eyettention_readerID",
+    "EyettentionRawTextInference",
+    "BSCdataset",
+    "calculate_mean_std",
+    "celerdataset",
+    "eval_log_llh",
+    "gradient_clipping",
+    "load_corpus",
+    "load_label",
+    "prepare_scanpath",
+]

app.py

@@ -0,0 +1,353 @@
+import os
+import sys
+import json
+from typing import Any, Dict, List, Optional, Tuple
+
+import gradio as gr
+import torch
+import numpy as np
+
+
+from Eyettention.raw_text_inference import EyettentionRawTextInference
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+BSC_CHECKPOINT = os.environ.get(
+    "EYETTENTION_BSC_CHECKPOINT",
+    os.path.join(os.path.dirname(__file__), "checkpoints", "bsc.pt"),
+)
+CELER_CHECKPOINT = os.environ.get(
+    "EYETTENTION_CELER_CHECKPOINT",
+    os.path.join(os.path.dirname(__file__), "checkpoints", "celer.pt"),
+)
+
+_MODELS: Dict[str, EyettentionRawTextInference] = {}
+
+
+def get_model(dataset: str) -> EyettentionRawTextInference:
+    """Load and cache an Eyettention model for the given dataset."""
+    if dataset not in _MODELS:
+        if dataset == "BSC":
+            ckpt = BSC_CHECKPOINT
+        elif dataset == "celer":
+            ckpt = CELER_CHECKPOINT
+        else:
+            raise ValueError(f"Unsupported dataset: {dataset}")
+
+        if not os.path.exists(ckpt):
+            raise FileNotFoundError(
+                f"Checkpoint not found at {ckpt!r}. "
+                f"Set EYETTENTION_{dataset.upper()}_CHECKPOINT to the correct path."
+            )
+
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        _MODELS[dataset] = EyettentionRawTextInference(
+            checkpoint_path=ckpt,
+            dataset=dataset,
+            device=device,
+        )
+    return _MODELS[dataset]
+
+
+def words_for_text(text: str, dataset: str) -> List[str]:
+    """
+    Return the list of words that the model's fixation indices refer to.
+
+    - For BSC (Chinese): the model tokenizes per character, so we return the
+      list of characters in the input (excluding whitespace).
+    - For CELER (English): the model tokenizes with a BERT wordpiece tokenizer
+      on whitespace-split words, so we return the whitespace-split words.
+    """
+    if dataset == "BSC":
+
+        return [ch for ch in text if not ch.isspace()]
+    else:
+        return text.split()
+
+
+# ---------------------------------------------------------------------------
+# Decoding helpers
+# ---------------------------------------------------------------------------
+def decode_scanpath(
+    scanpath_indices: torch.Tensor,
+    words: List[str],
+    dataset: str,
+) -> List[Dict[str, Any]]:
+    """
+    Convert a single predicted scanpath (indices into the word/char sequence)
+    into a list of dicts describing each fixation.
+
+    Convention in Eyettention's `scanpath_generation`:
+      * index 0 corresponds to the CLS token (sentence start),
+      * indices 1..len(words) correspond to words/characters,
+      * index len(words)+1 corresponds to SEP (sentence end).
+
+    We drop the leading CLS and any trailing SEP here.
+    """
+    if isinstance(scanpath_indices, torch.Tensor):
+        scanpath_indices = scanpath_indices.detach().cpu().tolist()
+
+    fixations: List[Dict[str, Any]] = []
+    n_words = len(words)
+
+    for step, idx in enumerate(scanpath_indices):
+        idx = int(idx)
+
+        if step == 0 and idx == 0:
+            continue
+
+        if idx == 0:
+            word = "<CLS>"
+        elif idx == n_words + 1:
+            word = "<SEP>"
+        elif 1 <= idx <= n_words:
+            word = words[idx - 1]
+        else:
+            word = f"<OOR:{idx}>"
+
+        fixations.append(
+            {
+                "step": len(fixations) + 1,
+                "word": word,
+                "word_index": idx,
+            }
+        )
+
+    return fixations
+
+
+def fixations_to_markdown(
+    fixations: List[Dict[str, Any]],
+    original_text: str,
+    dataset: str,
+) -> str:
+    """Render a scanpath as a Markdown table."""
+    lines = [
+        f"**Input ({dataset}):** {original_text}",
+        "",
+        "| Step | Fixated unit | Index |",
+        "|------|--------------|-------|",
+    ]
+    for f in fixations:
+        lines.append(f"| {f['step']} | {f['word']} | {f['word_index']} |")
+    return "\n".join(lines)
+
+
+def predict(
+    text: str,
+    dataset: str,
+    max_pred_len: int,
+    use_previous_scanpath: bool,
+    previous_scanpath: str,
+    progress=gr.Progress(track_tqdm=True),
+) -> Tuple[str, str, str]:
+    """
+    Run Eyettention on the input text and return:
+      - a Markdown table of the predicted scanpath,
+      - a space-separated string of the predicted word/char sequence,
+      - the raw JSON of the scanpath indices and fixated units.
+    """
+    if text is None or text.strip() == "":
+        raise gr.Error("Please provide some input text.")
+
+    if dataset == "celer" and not any(c.isalpha() for c in text):
+        raise gr.Error("CELER is the English model — please provide English text.")
+    if dataset == "BSC" and not any("\u4e00" <= c <= "\u9fff" for c in text):
+
+        pass
+
+    if max_pred_len <= 0:
+        raise gr.Error("max_pred_len must be a positive integer.")
+
+    progress(0.05, desc="Loading Eyettention model...")
+    model = get_model(dataset)
+
+    prev: Optional[List[int]] = None
+    if use_previous_scanpath and previous_scanpath.strip():
+        try:
+            prev = [int(x) for x in previous_scanpath.replace(",", " ").split()]
+        except ValueError:
+            raise gr.Error(
+                "previous_scanpath must be a whitespace/comma-separated list of integers."
+            )
+
+    progress(0.25, desc="Running autoregressive scanpath generation...")
+    with torch.no_grad():
+        if dataset == "BSC":
+            scanpath, _density = model.generate_from_chinese_text(
+                text=text,
+                max_pred_len=max_pred_len,
+                previous_scanpath=prev,
+            )
+        else:
+            scanpath, _density = model.generate_from_english_text(
+                text=text,
+                max_pred_len=max_pred_len,
+                previous_scanpath=prev,
+            )
+
+    scanpath = scanpath[0]
+
+    progress(0.9, desc="Formatting output...")
+    words = words_for_text(text, dataset)
+    fixations = decode_scanpath(scanpath, words, dataset)
+
+    markdown = fixations_to_markdown(fixations, text, dataset)
+    word_seq = " ".join(f["word"] for f in fixations)
+    json_out = json.dumps(
+        {
+            "dataset": dataset,
+            "input": text,
+            "scanpath_indices": [int(i) for i in scanpath.detach().cpu().tolist()],
+            "fixated_units": [f["word"] for f in fixations],
+        },
+        ensure_ascii=False,
+        indent=2,
+    )
+
+    return markdown, word_seq, json_out
+
+
+DESCRIPTION = """
+# Eyettention — Scanpath Prediction
+
+**Eyettention** predicts human-like **eye-movement scanpaths** from raw text.
+Given a sentence, it autoregressively generates a sequence of fixation
+locations (word or character indices) that approximate where a reader would
+look, in order.
+
+Two checkpoints are supported:
+
+| Dataset | Language | Tokenization |
+|---------|----------|--------------|
+| **BSC** | Chinese | Character-level |
+| **CELER** | English | Word-level (BERT wordpieces pooled) |
+
+### How to use
+1. Choose the **dataset / language**.
+2. Paste your text.
+3. Adjust `max_pred_len` if you want shorter or longer scanpaths.
+4. Click **Run**.
+
+### Optional: replay an observed prefix
+Enable **Use previous scanpath** and provide a whitespace- or comma-separated
+list of integer fixation indices. Those fixations will be replayed before the
+model starts sampling new ones (useful for conditional generation / prefix
+completion).
+"""
+
+EXAMPLES = [
+    [
+        "He said BankEast's offer appears to be \"attractive to the bank's shareholders.\"",
+        "celer",
+        20,
+        False,
+        "",
+    ],
+    ["中国选手在男子滑雪比赛中有望蝉联冠军", "BSC", 20, False, ""],
+]
+
+
+def build_demo() -> gr.Blocks:
+    with gr.Blocks(
+        title="Eyettention — Scanpath Prediction",
+        theme=gr.themes.Soft(),
+    ) as demo:
+        gr.Markdown(DESCRIPTION)
+
+        with gr.Row():
+            with gr.Column(scale=3):
+                text_in = gr.Textbox(
+                    label="Input text",
+                    placeholder="Enter a sentence...",
+                    lines=5,
+                )
+                dataset_in = gr.Radio(
+                    choices=["celer", "BSC"],
+                    value="celer",
+                    label="Dataset / language",
+                    info="celer = English, BSC = Chinese",
+                )
+                max_len_in = gr.Slider(
+                    minimum=1,
+                    maximum=120,
+                    value=30,
+                    step=1,
+                    label="max_pred_len",
+                    info="Maximum number of fixations to generate.",
+                )
+                with gr.Accordion("Advanced: replay an observed prefix", open=False):
+                    use_prev_in = gr.Checkbox(
+                        value=False,
+                        label="Use previous scanpath",
+                    )
+                    prev_in = gr.Textbox(
+                        label="Previous scanpath (integer indices)",
+                        placeholder="e.g. 0 1 3 5 4",
+                        lines=2,
+                    )
+                with gr.Row():
+                    run_btn = gr.Button("Run", variant="primary")
+                    clear_btn = gr.Button("Clear")
+
+            with gr.Column(scale=4):
+                table_out = gr.Markdown(
+                    label="Predicted scanpath",
+                    value="_Results will appear here._",
+                )
+                seq_out = gr.Textbox(
+                    label="Fixated units (sequence)",
+                    lines=3,
+                    interactive=False,
+                )
+                json_out = gr.Code(
+                    label="Raw JSON output",
+                    language="json",
+                    value="{}",
+                )
+
+        gr.Examples(
+            examples=EXAMPLES,
+            inputs=[text_in, dataset_in, max_len_in, use_prev_in, prev_in],
+        )
+
+        def _run(text, dataset, max_pred_len, use_prev, prev):
+            md, seq, js = predict(
+                text=text,
+                dataset=dataset,
+                max_pred_len=int(max_pred_len),
+                use_previous_scanpath=use_prev,
+                previous_scanpath=prev,
+            )
+            return md, seq, js
+
+        run_btn.click(
+            fn=_run,
+            inputs=[text_in, dataset_in, max_len_in, use_prev_in, prev_in],
+            outputs=[table_out, seq_out, json_out],
+        )
+        clear_btn.click(
+            fn=lambda: ("", "celer", 30, False, "", "_Results will appear here._", "", "{}"),
+            inputs=None,
+            outputs=[
+                text_in,
+                dataset_in,
+                max_len_in,
+                use_prev_in,
+                prev_in,
+                table_out,
+                seq_out,
+                json_out,
+            ],
+        )
+
+    return demo
+
+
+if __name__ == "__main__":
+    demo = build_demo()
+    demo.queue(max_size=16).launch(
+        server_name="0.0.0.0",
+        server_port=int(os.environ.get("PORT", 7860)),
+        show_error=True,
+    )

handler.py

@@ -0,0 +1,83 @@
+from pathlib import Path
+import torch
+
+from Eyettention import Eyettention
+from Eyettention import EyettentionRawTextInference
+
+
+class EndpointHandler:
+    def __init__(self, path: str = ""):
+        self.path = Path(path) if path else Path(__file__).resolve().parent
+        self.inference = None
+        self.dataset = None
+        self.checkpoint_path = None
+        self.device = None
+
+    def __call__(self, data):
+        inputs = data.get("inputs", data)
+        parameters = data.get("parameters", {})
+        dataset = parameters.get("dataset", "BSC")  # default = BSC
+        max_pred_len = parameters.get("max_pred_len", 60)
+        previous_scanpath = parameters.get("previous_scanpath")
+
+        inference = self._get_inference(dataset, parameters)
+
+        if isinstance(inputs, str):
+            if dataset == "BSC":
+                scanpath, density = inference.generate_from_chinese_text(
+                    text=inputs, max_pred_len=max_pred_len, previous_scanpath=previous_scanpath
+                )
+
+            elif dataset == "celer":
+                scanpath, density = inference.generate_from_english_text(
+                    text=inputs, max_pred_len=max_pred_len, previous_scanpath=previous_scanpath
+                )
+            else:
+                raise ValueError(f"Unsupported dataset: {dataset}")
+
+        elif isinstance(inputs, dict):
+            with torch.no_grad():
+                scanpath, density = inference.model.scanpath_generation(
+                    sn_emd=inputs["sn_input_ids"],
+                    sn_mask=inputs["sn_mask"],
+                    word_ids_sn=inputs.get("word_ids_sn"),
+                    sn_word_len=inputs["sn_word_len"],
+                    le=inference.label_encoder,
+                    max_pred_len=max_pred_len or inference.cf["max_pred_len"],
+                    previous_scanpath=previous_scanpath,
+                )
+        else:
+            raise ValueError("'inputs' must be raw text or model-ready tensors.")
+
+        return {
+            "scanpath": scanpath.tolist(),
+            "density_steps": len(density),
+        }
+
+    def _get_inference(self, dataset, parameters):
+        device = parameters.get("device", "cpu")
+        checkpoint_path = parameters.get("checkpoint_path") or self._default_checkpoint_path(
+            dataset
+        )
+
+        if (
+            self.inference is None
+            or self.dataset != dataset
+            or self.checkpoint_path != checkpoint_path
+            or self.device != device
+        ):
+            self.inference = EyettentionRawTextInference(
+                checkpoint_path=str(checkpoint_path), dataset=dataset, device=device
+            )
+            self.dataset = dataset
+            self.checkpoint_path = checkpoint_path
+            self.device = device
+
+        return self.inference
+
+    def _default_checkpoint_path(self, dataset):
+        if dataset == "BSC":
+            return self.path / "results" / "BSC" / "Eyettention_chinese.pth"
+        if dataset == "celer":
+            return self.path / "results" / "CELER" / "Eyettention_english.pth"
+        raise ValueError(f"Unsupported dataset: {dataset}")

main_BSC.py

@@ -0,0 +1,406 @@
+import numpy as np
+import pandas as pd
+import os
+from sklearn.model_selection import StratifiedKFold, KFold
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+from torch.optim import Adam, RMSprop
+from transformers import BertTokenizer
+from sklearn.preprocessing import LabelEncoder, OneHotEncoder
+from torch.nn.functional import cross_entropy, softmax
+from collections import deque
+import pickle
+import json
+import matplotlib.pyplot as plt
+import argparse
+
+from Eyettention.utils import *
+from Eyettention.model import Eyettention
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="run Eyettention on BSC dataset")
+    parser.add_argument(
+        "--test_mode",
+        help="New Sentence Split: text, New Reader Split: subject",
+        type=str,
+        default="text",
+    )
+    parser.add_argument(
+        "--atten_type", help="attention type: global, local, local-g", type=str, default="local-g"
+    )
+    parser.add_argument(
+        "--save_data_folder",
+        help="folder path for saving results",
+        type=str,
+        default="./results/BSC/",
+    )
+    parser.add_argument(
+        "--scanpath_gen_flag", help="whether to generate scanpath", type=int, default=1
+    )
+    parser.add_argument(
+        "--max_pred_len",
+        help="if scanpath_gen_flag is True, you can determine the longest scanpath that you want to generate, which should depend on the sentence length",
+        type=int,
+        default=60,
+    )
+    parser.add_argument("--gpu", help="gpu index", type=int, default=7)
+    args = parser.parse_args()
+    gpu = args.gpu
+    os.makedirs(args.save_data_folder, exist_ok=True)
+
+    torch.set_default_tensor_type("torch.FloatTensor")
+    availbl = torch.cuda.is_available()
+    print(torch.cuda.is_available())
+    if availbl:
+        device = f"cuda:{gpu}"
+    else:
+        device = "cpu"
+    if availbl:
+        torch.cuda.set_device(gpu)
+
+    cf = build_bsc_config(
+        atten_type=args.atten_type,
+        max_pred_len=args.max_pred_len,
+    )
+
+    # Encode the label into interger categories, setting the exclusive category 'cf["max_sn_len"]-1' as the end sign
+    le = build_label_encoder(cf)
+    # le.classes_
+
+    # load corpus
+    word_info_df, pos_info_df, eyemovement_df = load_corpus(cf["dataset"])
+    # Make list with sentence index
+    sn_list = np.unique(eyemovement_df.sn.values).tolist()
+    # Make list with reader index
+    reader_list = np.unique(eyemovement_df.id.values).tolist()
+
+    # Split training&test sets by text or reader, depending on configuration
+    if args.test_mode == "text":
+        print("Start evaluating on new sentences.")
+        split_list = sn_list
+    elif args.test_mode == "subject":
+        print("Start evaluating on new readers.")
+        split_list = reader_list
+
+    n_folds = cf["n_folds"]
+    kf = KFold(n_splits=n_folds, shuffle=True, random_state=0)
+    fold_indx = 0
+    # for scanpath generation
+    sp_dnn_list = []
+    sp_human_list = []
+    for train_idx, test_idx in kf.split(split_list):
+        loss_dict = {"val_loss": [], "train_loss": [], "test_ll": []}
+        list_train = [split_list[i] for i in train_idx]
+        list_test = [split_list[i] for i in test_idx]
+
+        # create train validation split for training the models:
+        kf_val = KFold(n_splits=n_folds, shuffle=True, random_state=0)
+        for train_index, val_index in kf_val.split(list_train):
+            # we only evaluate a single fold
+            break
+        list_train_net = [list_train[i] for i in train_index]
+        list_val_net = [list_train[i] for i in val_index]
+
+        if args.test_mode == "text":
+            sn_list_train = list_train_net
+            sn_list_val = list_val_net
+            sn_list_test = list_test
+            reader_list_train, reader_list_val, reader_list_test = (
+                reader_list,
+                reader_list,
+                reader_list,
+            )
+
+        elif args.test_mode == "subject":
+            reader_list_train = list_train_net
+            reader_list_val = list_val_net
+            reader_list_test = list_test
+            sn_list_train, sn_list_val, sn_list_test = sn_list, sn_list, sn_list
+
+        # initialize tokenizer
+        tokenizer = BertTokenizer.from_pretrained(cf["model_pretrained"])
+        # Preparing batch data
+        dataset_train = BSCdataset(
+            word_info_df, eyemovement_df, cf, reader_list_train, sn_list_train, tokenizer
+        )
+        train_dataloaderr = DataLoader(
+            dataset_train, batch_size=cf["batch_size"], shuffle=True, drop_last=True
+        )
+
+        dataset_val = BSCdataset(
+            word_info_df, eyemovement_df, cf, reader_list_val, sn_list_val, tokenizer
+        )
+        val_dataloaderr = DataLoader(
+            dataset_val, batch_size=cf["batch_size"], shuffle=False, drop_last=True
+        )
+
+        dataset_test = BSCdataset(
+            word_info_df, eyemovement_df, cf, reader_list_test, sn_list_test, tokenizer
+        )
+        test_dataloaderr = DataLoader(
+            dataset_test, batch_size=cf["batch_size"], shuffle=False, drop_last=False
+        )
+
+        # z-score normalization for gaze features
+        fix_dur_mean, fix_dur_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_fix_dur", padding_value=0, scale=1000
+        )
+        landing_pos_mean, landing_pos_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_landing_pos", padding_value=0
+        )
+        sn_word_len_mean, sn_word_len_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sn_word_len"
+        )
+
+        # load model
+        dnn = Eyettention(cf)
+
+        # training
+        episode = 0
+        optimizer = Adam(dnn.parameters(), lr=cf["lr"])
+        dnn.train()
+        dnn.to(device)
+        av_score = deque(maxlen=100)
+        old_score = 1e10
+        save_ep_couter = 0
+        print("Start training")
+        for episode_i in range(episode, cf["n_epochs"] + 1):
+            dnn.train()
+            print("episode:", episode_i)
+            counter = 0
+            for batchh in train_dataloaderr:
+                counter += 1
+                batchh.keys()
+                sn_input_ids = batchh["sn_input_ids"].to(device)
+                sn_attention_mask = batchh["sn_attention_mask"].to(device)
+                sp_input_ids = batchh["sp_input_ids"].to(device)
+                sp_attention_mask = batchh["sp_attention_mask"].to(device)
+                sp_pos = batchh["sp_pos"].to(device)
+                sp_landing_pos = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur = (batchh["sp_fix_dur"] / 1000).to(device)
+                sn_word_len = batchh["sn_word_len"].to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur, 0)
+                sp_fix_dur = (sp_fix_dur - fix_dur_mean) / fix_dur_std * mask
+                sp_fix_dur = torch.nan_to_num(sp_fix_dur)
+                sp_landing_pos = (sp_landing_pos - landing_pos_mean) / landing_pos_std * mask
+                sp_landing_pos = torch.nan_to_num(sp_landing_pos)
+                sn_word_len = (sn_word_len - sn_word_len_mean) / sn_word_len_std
+                sn_word_len = torch.nan_to_num(sn_word_len)
+
+                # zero old gradients
+                optimizer.zero_grad()
+                # predict output with DNN
+                dnn_out, atten_weights = dnn(
+                    sn_emd=sn_input_ids,
+                    sn_mask=sn_attention_mask,
+                    sp_emd=sp_input_ids,
+                    sp_pos=sp_pos,
+                    word_ids_sn=None,
+                    word_ids_sp=None,
+                    sp_fix_dur=sp_fix_dur,
+                    sp_landing_pos=sp_landing_pos,
+                    sn_word_len=sn_word_len,
+                )  # [batch, step, dec_o_dim]
+
+                dnn_out = dnn_out.permute(0, 2, 1)  # [batch, dec_o_dim, step]
+
+                # prepare label and mask
+                pad_mask, label = load_label(sp_pos, cf, le, device)
+                loss = nn.CrossEntropyLoss(reduction="none")
+                batch_error = torch.mean(torch.masked_select(loss(dnn_out, label), ~pad_mask))
+
+                # backpropagate loss
+                batch_error.backward()
+                # clip gradients
+                gradient_clipping(dnn, cf["max_grad_norm"])
+
+                # learn
+                optimizer.step()
+                av_score.append(batch_error.to("cpu").detach().numpy())
+                print("counter:", counter)
+                print(
+                    "\rSample {}\tAverage Error: {:.10f} ".format(counter, np.mean(av_score)),
+                    end=" ",
+                )
+            loss_dict["train_loss"].append(np.mean(av_score))
+
+            val_loss = []
+            dnn.eval()
+            for batchh in val_dataloaderr:
+                with torch.no_grad():
+                    sn_input_ids_val = batchh["sn_input_ids"].to(device)
+                    sn_attention_mask_val = batchh["sn_attention_mask"].to(device)
+                    sp_input_ids_val = batchh["sp_input_ids"].to(device)
+                    sp_attention_mask_val = batchh["sp_attention_mask"].to(device)
+                    sp_pos_val = batchh["sp_pos"].to(device)
+                    sp_landing_pos_val = batchh["sp_landing_pos"].to(device)
+                    sp_fix_dur_val = (batchh["sp_fix_dur"] / 1000).to(device)
+                    sn_word_len_val = batchh["sn_word_len"].to(device)
+
+                    # normalize gaze features
+                    mask = ~torch.eq(sp_fix_dur_val, 0)
+                    sp_fix_dur_val = (sp_fix_dur_val - fix_dur_mean) / fix_dur_std * mask
+                    sp_landing_pos_val = (
+                        (sp_landing_pos_val - landing_pos_mean) / landing_pos_std * mask
+                    )
+                    sp_fix_dur_val = torch.nan_to_num(sp_fix_dur_val)
+                    sp_landing_pos_val = torch.nan_to_num(sp_landing_pos_val)
+                    sn_word_len_val = (sn_word_len_val - sn_word_len_mean) / sn_word_len_std
+                    sn_word_len_val = torch.nan_to_num(sn_word_len_val)
+
+                    dnn_out_val, atten_weights_val = dnn(
+                        sn_emd=sn_input_ids_val,
+                        sn_mask=sn_attention_mask_val,
+                        sp_emd=sp_input_ids_val,
+                        sp_pos=sp_pos_val,
+                        word_ids_sn=None,
+                        word_ids_sp=None,
+                        sp_fix_dur=sp_fix_dur_val,
+                        sp_landing_pos=sp_landing_pos_val,
+                        sn_word_len=sn_word_len_val,
+                    )  # [batch, step, dec_o_dim]
+                    dnn_out_val = dnn_out_val.permute(0, 2, 1)  # [batch, dec_o_dim, step
+
+                    # prepare label and mask
+                    loss = nn.CrossEntropyLoss(reduction="none")
+                    pad_mask_val, label_val = load_label(sp_pos_val, cf, le, device)
+                    batch_error_val = torch.mean(
+                        torch.masked_select(loss(dnn_out_val, label_val), ~pad_mask_val)
+                    )
+                    val_loss.append(batch_error_val.detach().to("cpu").numpy())
+            print("\nvalidation loss is {} \n".format(np.mean(val_loss)))
+            loss_dict["val_loss"].append(np.mean(val_loss))
+
+            if np.mean(val_loss) < old_score:
+                # save model if val loss is smallest
+                torch.save(
+                    dnn.state_dict(),
+                    "{}/CELoss_BSC_{}_eyettention_{}_newloss_fold{}.pth".format(
+                        args.save_data_folder, args.test_mode, args.atten_type, fold_indx
+                    ),
+                )
+                old_score = np.mean(val_loss)
+                print("\nsaved model state dict\n")
+                save_ep_couter = episode_i
+            else:
+                # early stopping
+                if episode_i - save_ep_couter >= cf["earlystop_patience"]:
+                    break
+
+        # evaluation
+        dnn.eval()
+        res_llh = []
+        dnn.load_state_dict(
+            torch.load(
+                os.path.join(
+                    args.save_data_folder,
+                    f"CELoss_BSC_{args.test_mode}_eyettention_{args.atten_type}_newloss_fold{fold_indx}.pth",
+                ),
+                map_location="cpu",
+            )
+        )
+        dnn.to(device)
+        batch_indx = 0
+        for batchh in test_dataloaderr:
+            with torch.no_grad():
+                sn_input_ids_test = batchh["sn_input_ids"].to(device)
+                sn_attention_mask_test = batchh["sn_attention_mask"].to(device)
+                sp_input_ids_test = batchh["sp_input_ids"].to(device)
+                sp_attention_mask_test = batchh["sp_attention_mask"].to(device)
+                sp_pos_test = batchh["sp_pos"].to(device)  # 28: '<Sep>', 29: '<'Pad'>'
+                sp_landing_pos_test = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur_test = (batchh["sp_fix_dur"] / 1000).to(device)
+                sn_word_len_test = batchh["sn_word_len"].to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur_test, 0)
+                sp_fix_dur_test = (sp_fix_dur_test - fix_dur_mean) / fix_dur_std * mask
+                sp_landing_pos_test = (
+                    (sp_landing_pos_test - landing_pos_mean) / landing_pos_std * mask
+                )
+                sp_fix_dur_test = torch.nan_to_num(sp_fix_dur_test)
+                sp_landing_pos_test = torch.nan_to_num(sp_landing_pos_test)
+                sn_word_len_test = (sn_word_len_test - sn_word_len_mean) / sn_word_len_std
+                sn_word_len_test = torch.nan_to_num(sn_word_len_test)
+
+                dnn_out_test, atten_weights_test = dnn(
+                    sn_emd=sn_input_ids_test,
+                    sn_mask=sn_attention_mask_test,
+                    sp_emd=sp_input_ids_test,
+                    sp_pos=sp_pos_test,
+                    word_ids_sn=None,
+                    word_ids_sp=None,
+                    sp_fix_dur=sp_fix_dur_test,
+                    sp_landing_pos=sp_landing_pos_test,
+                    sn_word_len=sn_word_len_test,
+                )  # [batch, step, dec_o_dim]
+
+                # We do not use nn.CrossEntropyLoss here to calculate the likelihood because it combines nn.LogSoftmax and nn.NLL,
+                # while nn.LogSoftmax returns a log value based on e, we want 2 instead
+                # m = nn.LogSoftmax(dim=2) -- base e, we want base 2
+                m = nn.Softmax(dim=2)
+                dnn_out_test = m(dnn_out_test).detach().to("cpu").numpy()
+
+                # prepare label and mask
+                pad_mask_test, label_test = load_label(sp_pos_test, cf, le, "cpu")
+
+                # compute log likelihood for the batch samples
+                res_batch = eval_log_llh(dnn_out_test, label_test, pad_mask_test)
+                res_llh.append(np.array(res_batch))
+
+                if bool(args.scanpath_gen_flag) == True:
+                    sn_len = (
+                        (torch.sum(sn_attention_mask_test, axis=1) - 2).detach().to("cpu").numpy()
+                    )
+                    # compute the scan path generated from the model when the first CLS token is given
+                    sp_dnn = dnn.scanpath_generation(
+                        sn_emd=sn_input_ids_test,
+                        sn_mask=sn_attention_mask_test,
+                        word_ids_sn=None,
+                        sn_word_len=sn_word_len_test,
+                        le=le,
+                        max_pred_len=cf["max_pred_len"],
+                    )
+
+                    sp_dnn, sp_human = prepare_scanpath(
+                        sp_dnn.detach().to("cpu").numpy(), sn_len, sp_pos_test, cf
+                    )
+                    sp_dnn_list.extend(sp_dnn)
+                    sp_human_list.extend(sp_human)
+
+                batch_indx += 1
+
+        res_llh = np.concatenate(res_llh).ravel()
+        loss_dict["test_ll"].append(res_llh)
+        loss_dict["fix_dur_mean"] = fix_dur_mean
+        loss_dict["fix_dur_std"] = fix_dur_std
+        loss_dict["landing_pos_mean"] = landing_pos_mean
+        loss_dict["landing_pos_std"] = landing_pos_std
+        loss_dict["sn_word_len_mean"] = sn_word_len_mean
+        loss_dict["sn_word_len_std"] = sn_word_len_std
+
+        print("\nTest likelihood is {} \n".format(np.mean(res_llh)))
+        # save results
+        with open(
+            "{}/res_BSC_{}_eyettention_{}_Fold{}.pickle".format(
+                args.save_data_folder, args.test_mode, args.atten_type, fold_indx
+            ),
+            "wb",
+        ) as handle:
+            pickle.dump(loss_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
+        fold_indx += 1
+
+    if bool(args.scanpath_gen_flag) == True:
+        # save results
+        dic = {"sp_dnn": sp_dnn_list, "sp_human": sp_human_list}
+        with open(
+            os.path.join(
+                args.save_data_folder,
+                f"BSC_scanpath_generation_eyettention_{args.test_mode}_{args.atten_type}.pickle",
+            ),
+            "wb",
+        ) as handle:
+            pickle.dump(dic, handle, protocol=pickle.HIGHEST_PROTOCOL)

main_BSC_NRS_setting.py

@@ -0,0 +1,390 @@
+import numpy as np
+import pandas as pd
+import os
+from sklearn.model_selection import StratifiedKFold, KFold
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+from torch.optim import Adam, RMSprop
+from transformers import BertTokenizer
+from sklearn.preprocessing import LabelEncoder, OneHotEncoder
+from torch.nn.functional import cross_entropy, softmax
+from collections import deque
+import pickle
+import json
+import random
+import matplotlib.pyplot as plt
+import argparse
+
+from Eyettention.utils import *
+from Eyettention.model import Eyettention
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="run seq2seq model evaluation")
+    parser.add_argument(
+        "--test_mode", help="test mode: New Reader / New Sentence Split", type=str, default="NRS"
+    )
+    parser.add_argument("--atten_type", help="attention type", type=str, default="local-g")
+    parser.add_argument(
+        "--save_data_folder",
+        help="folder path for saving results",
+        type=str,
+        default="./results/BSC/",
+    )
+    parser.add_argument(
+        "--scanpath_gen_flag", help="whether to generate scanpath", type=int, default=1
+    )
+    parser.add_argument(
+        "--max_pred_len",
+        help="if scanpath_gen_flag is True, you can determine the longest scanpath that you want to generate, which should depend on the sentence length",
+        type=int,
+        default=60,
+    )
+    parser.add_argument("--sampl_time", help="Sampling times", type=int, default=5)
+    parser.add_argument("--gpu", help="gpu index", type=int, default=6)
+    args = parser.parse_args()
+    gpu = args.gpu
+
+    torch.set_default_tensor_type("torch.FloatTensor")
+    availbl = torch.cuda.is_available()
+    print(torch.cuda.is_available())
+    if availbl:
+        device = f"cuda:{gpu}"
+    else:
+        device = "cpu"
+    torch.cuda.set_device(gpu)
+
+    cf = {
+        "model_pretrained": "bert-base-chinese",
+        "lr": 1e-3,
+        "max_grad_norm": 10,
+        "n_epochs": 1000,
+        "dataset": "BSC",
+        "atten_type": args.atten_type,
+        "batch_size": 256,
+        "max_sn_len": 27,  # include start token and end token
+        "max_sp_len": 40,  # include start token and end token
+        "norm_type": "z-score",
+        "earlystop_patience": 20,
+        "max_pred_len": args.max_pred_len,
+    }
+
+    # Encode the label into interger categories, setting the exclusive category 'cf["max_sn_len"]-1' as the end sign
+    le = LabelEncoder()
+    le.fit(np.append(np.arange(-cf["max_sn_len"] + 3, cf["max_sn_len"] - 1), cf["max_sn_len"] - 1))
+    # le.classes_
+
+    # load corpus
+    word_info_df, pos_info_df, eyemovement_df = load_corpus(cf["dataset"])
+    # Make list with sentence index
+    sn_list = np.unique(eyemovement_df.sn.values).tolist()
+    # Make list with reader index
+    reader_list = np.unique(eyemovement_df.id.values).tolist()
+
+    random.seed(0)
+    fold_indx = 0
+    # for scanpath generation
+    sp_dnn_list = []
+    sp_human_list = []
+    for i in range(args.sampl_time):
+        print("time:", i)
+        loss_dict = {"val_loss": [], "train_loss": [], "test_ll": []}
+        readers_test = random.sample(reader_list, int(np.ceil(len(reader_list) * 0.3)))
+        reader_list_val = random.sample(readers_test, int(np.ceil(len(readers_test) * 0.3)))
+        reader_list_test = set(readers_test) - set(reader_list_val)
+        reader_list_test = list(reader_list_test)
+        reader_list_train = set(reader_list) - set(reader_list_test) - set(reader_list_val)
+        reader_list_train = list(reader_list_train)
+
+        sn_test = random.sample(sn_list, int(np.ceil(len(sn_list) * 0.3)))
+        sn_list_val = random.sample(sn_test, int(np.ceil(len(sn_test) * 0.3)))
+        sn_list_test = set(sn_test) - set(sn_list_val)
+        sn_list_test = list(sn_list_test)
+        sn_list_train = set(sn_list) - set(sn_list_test) - set(sn_list_val)
+        sn_list_train = list(sn_list_train)
+
+        # initialize tokenizer
+        tokenizer = BertTokenizer.from_pretrained(cf["model_pretrained"])
+        # Preparing batch data
+        dataset_train = BSCdataset(
+            word_info_df, eyemovement_df, cf, reader_list_train, sn_list_train, tokenizer
+        )
+        train_dataloaderr = DataLoader(
+            dataset_train, batch_size=cf["batch_size"], shuffle=True, drop_last=True
+        )
+
+        dataset_val = BSCdataset(
+            word_info_df, eyemovement_df, cf, reader_list_val, sn_list_val, tokenizer
+        )
+        val_dataloaderr = DataLoader(
+            dataset_val, batch_size=cf["batch_size"], shuffle=False, drop_last=False
+        )
+
+        dataset_test = BSCdataset(
+            word_info_df, eyemovement_df, cf, reader_list_test, sn_list_test, tokenizer
+        )
+        test_dataloaderr = DataLoader(
+            dataset_test, batch_size=cf["batch_size"], shuffle=False, drop_last=False
+        )
+
+        # z-score normalization for gaze features
+        fix_dur_mean, fix_dur_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_fix_dur", padding_value=0, scale=1000
+        )
+        landing_pos_mean, landing_pos_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_landing_pos", padding_value=0
+        )
+        sn_word_len_mean, sn_word_len_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sn_word_len"
+        )
+
+        # load model
+        dnn = Eyettention(cf)
+
+        # training
+        episode = 0
+        optimizer = Adam(dnn.parameters(), lr=cf["lr"])
+        dnn.train()
+        dnn.to(device)
+        av_score = deque(maxlen=100)
+        old_score = 1e10
+        save_ep_couter = 0
+        print("Start training")
+        for episode_i in range(episode, cf["n_epochs"] + 1):
+            dnn.train()
+            print("episode:", episode_i)
+            counter = 0
+            for batchh in train_dataloaderr:
+                counter += 1
+                batchh.keys()
+                sn_input_ids = batchh["sn_input_ids"].to(device)
+                sn_attention_mask = batchh["sn_attention_mask"].to(device)
+                sp_input_ids = batchh["sp_input_ids"].to(device)
+                sp_attention_mask = batchh["sp_attention_mask"].to(device)
+                sp_pos = batchh["sp_pos"].to(device)
+                sp_landing_pos = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur = (batchh["sp_fix_dur"] / 1000).to(device)
+                sn_word_len = batchh["sn_word_len"].to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur, 0)
+                sp_fix_dur = (sp_fix_dur - fix_dur_mean) / fix_dur_std * mask
+                sp_landing_pos = (sp_landing_pos - landing_pos_mean) / landing_pos_std * mask
+                sp_fix_dur = torch.nan_to_num(sp_fix_dur)
+                sp_landing_pos = torch.nan_to_num(sp_landing_pos)
+                sn_word_len = (sn_word_len - sn_word_len_mean) / sn_word_len_std
+                sn_word_len = torch.nan_to_num(sn_word_len)
+
+                # zero old gradients
+                optimizer.zero_grad()
+                # predict output with DNN
+                dnn_out, atten_weights = dnn(
+                    sn_emd=sn_input_ids,
+                    sn_mask=sn_attention_mask,
+                    sp_emd=sp_input_ids,
+                    sp_pos=sp_pos,
+                    word_ids_sn=None,
+                    word_ids_sp=None,
+                    sp_fix_dur=sp_fix_dur,
+                    sp_landing_pos=sp_landing_pos,
+                    sn_word_len=sn_word_len,
+                )  # [batch, step, dec_o_dim]
+
+                dnn_out = dnn_out.permute(0, 2, 1)  # [batch, dec_o_dim, step]
+
+                # prepare label and mask
+                pad_mask, label = load_label(sp_pos, cf, le, device)
+                loss = nn.CrossEntropyLoss(reduction="none")
+                batch_error = torch.mean(torch.masked_select(loss(dnn_out, label), ~pad_mask))
+
+                # backpropagate loss
+                batch_error.backward()
+                # clip gradients
+                gradient_clipping(dnn, cf["max_grad_norm"])
+
+                # learn
+                optimizer.step()
+                av_score.append(batch_error.to("cpu").detach().numpy())
+                print("counter:", counter)
+                print(
+                    "\rSample {}\tAverage Error: {:.10f} ".format(counter, np.mean(av_score)),
+                    end=" ",
+                )
+            loss_dict["train_loss"].append(np.mean(av_score))
+
+            val_loss = []
+            dnn.eval()
+            for batchh in val_dataloaderr:
+                with torch.no_grad():
+                    sn_input_ids_val = batchh["sn_input_ids"].to(device)
+                    sn_attention_mask_val = batchh["sn_attention_mask"].to(device)
+                    sp_input_ids_val = batchh["sp_input_ids"].to(device)
+                    sp_attention_mask_val = batchh["sp_attention_mask"].to(device)
+                    sp_pos_val = batchh["sp_pos"].to(device)
+                    sp_landing_pos_val = batchh["sp_landing_pos"].to(device)
+                    sp_fix_dur_val = (batchh["sp_fix_dur"] / 1000).to(device)
+                    sn_word_len_val = batchh["sn_word_len"].to(device)
+
+                    # normalize gaze features
+                    mask = ~torch.eq(sp_fix_dur_val, 0)
+                    sp_fix_dur_val = (sp_fix_dur_val - fix_dur_mean) / fix_dur_std * mask
+                    sp_landing_pos_val = (
+                        (sp_landing_pos_val - landing_pos_mean) / landing_pos_std * mask
+                    )
+                    sp_fix_dur_val = torch.nan_to_num(sp_fix_dur_val)
+                    sp_landing_pos_val = torch.nan_to_num(sp_landing_pos_val)
+                    sn_word_len_val = (sn_word_len_val - sn_word_len_mean) / sn_word_len_std
+                    sn_word_len_val = torch.nan_to_num(sn_word_len_val)
+
+                    dnn_out_val, atten_weights_val = dnn(
+                        sn_emd=sn_input_ids_val,
+                        sn_mask=sn_attention_mask_val,
+                        sp_emd=sp_input_ids_val,
+                        sp_pos=sp_pos_val,
+                        word_ids_sn=None,
+                        word_ids_sp=None,
+                        sp_fix_dur=sp_fix_dur_val,
+                        sp_landing_pos=sp_landing_pos_val,
+                        sn_word_len=sn_word_len_val,
+                    )  # [batch, step, dec_o_dim]
+                    dnn_out_val = dnn_out_val.permute(0, 2, 1)  # [batch, dec_o_dim, step]
+
+                    # prepare label and mask
+                    pad_mask_val, label_val = load_label(sp_pos_val, cf, le, device)
+                    batch_error_val = torch.mean(
+                        torch.masked_select(loss(dnn_out_val, label_val), ~pad_mask_val)
+                    )
+                    val_loss.append(batch_error_val.detach().to("cpu").numpy())
+            print("\nvalidation loss is {} \n".format(np.mean(val_loss)))
+            loss_dict["val_loss"].append(np.mean(val_loss))
+
+            if np.mean(val_loss) < old_score:
+                # save model if val loss is smallest
+                torch.save(
+                    dnn.state_dict(),
+                    "{}/CELoss_BSC_NRS_eyettention_{}_newloss_fold{}.pth".format(
+                        args.save_data_folder, args.atten_type, fold_indx
+                    ),
+                )
+                old_score = np.mean(val_loss)
+                print("\nsaved model state dict\n")
+                save_ep_couter = episode_i
+            else:
+                # early stopping
+                if episode_i - save_ep_couter >= cf["earlystop_patience"]:
+                    break
+
+        # evaluation
+        dnn.eval()
+        res_llh = []
+        dnn.load_state_dict(
+            torch.load(
+                os.path.join(
+                    args.save_data_folder,
+                    f"CELoss_BSC_NRS_eyettention_{args.atten_type}_newloss_fold{fold_indx}.pth",
+                ),
+                map_location="cpu",
+            )
+        )
+        dnn.to(device)
+        batch_indx = 0
+        for batchh in test_dataloaderr:
+            with torch.no_grad():
+                sn_input_ids_test = batchh["sn_input_ids"].to(device)
+                sn_attention_mask_test = batchh["sn_attention_mask"].to(device)
+                sp_input_ids_test = batchh["sp_input_ids"].to(device)
+                sp_attention_mask_test = batchh["sp_attention_mask"].to(device)
+                sp_pos_test = batchh["sp_pos"].to(device)  # 28: '<Sep>', 29: '<'Pad'>'
+                sp_landing_pos_test = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur_test = (batchh["sp_fix_dur"] / 1000).to(device)
+                sn_word_len_test = batchh["sn_word_len"].to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur_test, 0)
+                sp_fix_dur_test = (sp_fix_dur_test - fix_dur_mean) / fix_dur_std * mask
+                sp_landing_pos_test = (
+                    (sp_landing_pos_test - landing_pos_mean) / landing_pos_std * mask
+                )
+                sp_fix_dur_test = torch.nan_to_num(sp_fix_dur_test)
+                sp_landing_pos_test = torch.nan_to_num(sp_landing_pos_test)
+                sn_word_len_test = (sn_word_len_test - sn_word_len_mean) / sn_word_len_std
+                sn_word_len_test = torch.nan_to_num(sn_word_len_test)
+
+                dnn_out_test, atten_weights_test = dnn(
+                    sn_emd=sn_input_ids_test,
+                    sn_mask=sn_attention_mask_test,
+                    sp_emd=sp_input_ids_test,
+                    sp_pos=sp_pos_test,
+                    word_ids_sn=None,
+                    word_ids_sp=None,
+                    sp_fix_dur=sp_fix_dur_test,
+                    sp_landing_pos=sp_landing_pos_test,
+                    sn_word_len=sn_word_len_test,
+                )  # [batch, step, dec_o_dim]
+
+                # We do not use nn.CrossEntropyLoss here to calculate the likelihood because it combines nn.LogSoftmax and nn.NLL,
+                # while nn.LogSoftmax returns a log value based on e, we want 2 instead
+                # m = nn.LogSoftmax(dim=2) -- base e, we want base 2
+                m = nn.Softmax(dim=2)
+                dnn_out_test = m(dnn_out_test).detach().to("cpu").numpy()
+
+                # prepare label and mask
+                pad_mask_test, label_test = load_label(sp_pos_test, cf, le, "cpu")
+                pred = dnn_out_test.argmax(axis=2)
+                # compute log likelihood for the batch samples
+                res_batch = eval_log_llh(dnn_out_test, label_test, pad_mask_test)
+                res_llh.append(np.array(res_batch))
+
+                if bool(args.scanpath_gen_flag) == True:
+                    sn_len = (
+                        (torch.sum(sn_attention_mask_test, axis=1) - 2).detach().to("cpu").numpy()
+                    )
+                    # compute the scan path generated from the model when the first CLS token is given
+                    sp_dnn = dnn.scanpath_generation(
+                        sn_emd=sn_input_ids_test,
+                        sn_mask=sn_attention_mask_test,
+                        word_ids_sn=None,
+                        sn_word_len=sn_word_len_test,
+                        le=le,
+                        max_pred_len=cf["max_pred_len"],
+                    )
+
+                    sp_dnn, sp_human = prepare_scanpath(
+                        sp_dnn.detach().to("cpu").numpy(), sn_len, sp_pos_test, cf
+                    )
+                    sp_dnn_list.extend(sp_dnn)
+                    sp_human_list.extend(sp_human)
+
+                batch_indx += 1
+
+        res_llh = np.concatenate(res_llh).ravel()
+        loss_dict["test_ll"].append(res_llh)
+        loss_dict["fix_dur_mean"] = fix_dur_mean
+        loss_dict["fix_dur_std"] = fix_dur_std
+        loss_dict["landing_pos_mean"] = landing_pos_mean
+        loss_dict["landing_pos_std"] = landing_pos_std
+        loss_dict["sn_word_len_mean"] = sn_word_len_mean
+        loss_dict["sn_word_len_std"] = sn_word_len_std
+
+        print("\nTest likelihood is {} \n".format(np.mean(res_llh)))
+        # save results
+        with open(
+            "{}/res_BSC_NRS_eyettention_{}_Fold{}.pickle".format(
+                args.save_data_folder, args.atten_type, fold_indx
+            ),
+            "wb",
+        ) as handle:
+            pickle.dump(loss_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
+        fold_indx += 1
+
+    if bool(args.scanpath_gen_flag) == True:
+        # save results
+        dic = {"sp_dnn": sp_dnn_list, "sp_human": sp_human_list}
+        with open(
+            os.path.join(
+                args.save_data_folder,
+                f"BSC_scanpath_generation_eyettention_NRS_{args.atten_type}.pickle",
+            ),
+            "wb",
+        ) as handle:
+            pickle.dump(dic, handle, protocol=pickle.HIGHEST_PROTOCOL)

main_BSC_reader_identifier.py

@@ -0,0 +1,358 @@
+import numpy as np
+import pandas as pd
+import os
+from sklearn.model_selection import StratifiedKFold, KFold
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+from torch.optim import Adam, RMSprop
+from transformers import BertTokenizer
+from sklearn.preprocessing import LabelEncoder, OneHotEncoder
+from torch.nn.functional import cross_entropy, softmax
+from collections import deque
+import pickle
+import json
+import matplotlib.pyplot as plt
+import argparse
+
+from Eyettention.utils import *
+from Eyettention.model import Eyettention_readerID
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="run uniform baseline")
+    parser.add_argument("--test_mode", help="test mode: text", type=str, default="text")
+    parser.add_argument("--atten_type", help="attention type", type=str, default="local-g")
+    parser.add_argument(
+        "--save_data_folder",
+        help="folder path for saving results",
+        type=str,
+        default="./results/BSC/",
+    )
+    parser.add_argument("--gpu", help="gpu index", type=int, default=3)
+    parser.add_argument("--emb_size", help="readerID embedding size", type=int, default=16)
+    args = parser.parse_args()
+    gpu = args.gpu
+
+    torch.set_default_tensor_type("torch.FloatTensor")
+    availbl = torch.cuda.is_available()
+    print(torch.cuda.is_available())
+    if availbl:
+        device = f"cuda:{gpu}"
+    else:
+        device = "cpu"
+    torch.cuda.set_device(gpu)
+
+    cf = {
+        "model_pretrained": "bert-base-chinese",
+        "lr": 1e-3,
+        "max_grad_norm": 10,
+        "n_epochs": 1000,
+        "n_folds": 5,
+        "dataset": "BSC",
+        "atten_type": args.atten_type,
+        "subid_emb_size": args.emb_size,
+        "batch_size": 256,
+        "max_sn_len": 27,  # include start token and end token
+        "max_sp_len": 40,  # include start token and end token
+        "norm_type": "z-score",
+        "earlystop_patience": 20,
+    }
+
+    # Encode the label into interger categories, setting the exclusive category 'cf["max_sn_len"]-1' as the end sign
+    le = LabelEncoder()
+    le.fit(np.append(np.arange(-cf["max_sn_len"] + 3, cf["max_sn_len"] - 1), cf["max_sn_len"] - 1))
+    # le.classes_
+
+    # load corpus
+    word_info_df, pos_info_df, eyemovement_df = load_corpus(cf["dataset"])
+    # Make list with sentence index
+    sn_list = np.unique(eyemovement_df.sn.values).tolist()
+    # Make list with reader index
+    reader_list = np.unique(eyemovement_df.id.values).tolist()
+
+    # Split training&test sets by text
+    print("Start evaluating on new sentences.")
+    split_list = sn_list
+
+    n_folds = cf["n_folds"]
+    kf = KFold(n_splits=n_folds, shuffle=True, random_state=0)
+    fold_indx = 0
+    for train_idx, test_idx in kf.split(split_list):
+        loss_dict = {"val_loss": [], "train_loss": [], "test_ll": []}
+        list_train = [split_list[i] for i in train_idx]
+        list_test = [split_list[i] for i in test_idx]
+
+        # create train validation split for training the models:
+        kf_val = KFold(n_splits=n_folds, shuffle=True, random_state=0)
+        for train_index, val_index in kf_val.split(list_train):
+            # we only evaluate a single fold
+            break
+        list_train_net = [list_train[i] for i in train_index]
+        list_val_net = [list_train[i] for i in val_index]
+
+        sn_list_train = list_train_net
+        sn_list_val = list_val_net
+        sn_list_test = list_test
+        reader_list_train, reader_list_val, reader_list_test = reader_list, reader_list, reader_list
+
+        # initialize tokenizer
+        tokenizer = BertTokenizer.from_pretrained(cf["model_pretrained"])
+        # Preparing batch data
+        dataset_train = BSCdataset(
+            word_info_df, eyemovement_df, cf, reader_list_train, sn_list_train, tokenizer
+        )
+        train_dataloaderr = DataLoader(
+            dataset_train, batch_size=cf["batch_size"], shuffle=True, drop_last=True
+        )
+
+        dataset_val = BSCdataset(
+            word_info_df, eyemovement_df, cf, reader_list_val, sn_list_val, tokenizer
+        )
+        val_dataloaderr = DataLoader(
+            dataset_val, batch_size=cf["batch_size"], shuffle=False, drop_last=True
+        )
+
+        dataset_test = BSCdataset(
+            word_info_df, eyemovement_df, cf, reader_list_test, sn_list_test, tokenizer
+        )
+        test_dataloaderr = DataLoader(
+            dataset_test, batch_size=cf["batch_size"], shuffle=False, drop_last=False
+        )
+
+        # z-score normalization for gaze features
+        fix_dur_mean, fix_dur_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_fix_dur", padding_value=0, scale=1000
+        )
+        landing_pos_mean, landing_pos_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_landing_pos", padding_value=0
+        )
+        sn_word_len_mean, sn_word_len_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sn_word_len"
+        )
+
+        # load model here
+        dnn = Eyettention_readerID(cf)
+
+        # training
+        episode = 0
+        optimizer = Adam(dnn.parameters(), lr=cf["lr"])
+        dnn.train()
+        dnn.to(device)
+        av_score = deque(maxlen=100)
+        old_score = 1e10
+        save_ep_couter = 0
+        print("Start training")
+        for episode_i in range(episode, cf["n_epochs"] + 1):
+            dnn.train()
+            print("episode:", episode_i)
+            counter = 0
+            for batchh in train_dataloaderr:
+                counter += 1
+                batchh.keys()
+                sn_input_ids = batchh["sn_input_ids"].to(device)
+                sn_attention_mask = batchh["sn_attention_mask"].to(device)
+                sp_input_ids = batchh["sp_input_ids"].to(device)
+                sp_attention_mask = batchh["sp_attention_mask"].to(device)
+                sp_pos = batchh["sp_pos"].to(device)
+                sp_landing_pos = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur = (batchh["sp_fix_dur"] / 1000).to(device)
+                sn_word_len = batchh["sn_word_len"].to(device)
+                sub_id = batchh["sub_id"].to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur, 0)
+                sp_fix_dur = (sp_fix_dur - fix_dur_mean) / fix_dur_std * mask
+                sp_landing_pos = (sp_landing_pos - landing_pos_mean) / landing_pos_std * mask
+                sp_fix_dur = torch.nan_to_num(sp_fix_dur)
+                sp_landing_pos = torch.nan_to_num(sp_landing_pos)
+                sn_word_len = (sn_word_len - sn_word_len_mean) / sn_word_len_std
+                sn_word_len = torch.nan_to_num(sn_word_len)
+
+                # zero old gradients
+                optimizer.zero_grad()
+                # predict output with DNN
+                dnn_out, atten_weights = dnn(
+                    sn_emd=sn_input_ids,
+                    sn_mask=sn_attention_mask,
+                    sp_emd=sp_input_ids,
+                    sp_pos=sp_pos,
+                    word_ids_sn=None,
+                    word_ids_sp=None,
+                    sp_fix_dur=sp_fix_dur,
+                    sp_landing_pos=sp_landing_pos,
+                    sn_word_len=sn_word_len,
+                    sub_id=sub_id,
+                )
+
+                dnn_out = dnn_out.permute(0, 2, 1)  # [batch, dec_o_dim, step]
+
+                # prepare label and mask
+                pad_mask, label = load_label(sp_pos, cf, le, device)
+                loss = nn.CrossEntropyLoss(reduction="none")
+                batch_error = torch.mean(torch.masked_select(loss(dnn_out, label), ~pad_mask))
+
+                # backpropagate loss
+                batch_error.backward()
+                # clip gradients
+                gradient_clipping(dnn, cf["max_grad_norm"])
+
+                # learn
+                optimizer.step()
+                av_score.append(batch_error.to("cpu").detach().numpy())
+                print("counter:", counter)
+                print(
+                    "\rSample {}\tAverage Error: {:.10f} ".format(counter, np.mean(av_score)),
+                    end=" ",
+                )
+            loss_dict["train_loss"].append(np.mean(av_score))
+
+            val_loss = []
+            dnn.eval()
+            for batchh in val_dataloaderr:
+                with torch.no_grad():
+                    sn_input_ids_val = batchh["sn_input_ids"].to(device)
+                    sn_attention_mask_val = batchh["sn_attention_mask"].to(device)
+                    sp_input_ids_val = batchh["sp_input_ids"].to(device)
+                    sp_attention_mask_val = batchh["sp_attention_mask"].to(device)
+                    sp_pos_val = batchh["sp_pos"].to(device)
+                    sp_landing_pos_val = batchh["sp_landing_pos"].to(device)
+                    sp_fix_dur_val = (batchh["sp_fix_dur"] / 1000).to(device)
+                    sn_word_len_val = batchh["sn_word_len"].to(device)
+                    sub_id_val = batchh["sub_id"].to(device)
+
+                    # normalize gaze features
+                    mask = ~torch.eq(sp_fix_dur_val, 0)
+                    sp_fix_dur_val = (sp_fix_dur_val - fix_dur_mean) / fix_dur_std * mask
+                    sp_landing_pos_val = (
+                        (sp_landing_pos_val - landing_pos_mean) / landing_pos_std * mask
+                    )
+                    sp_fix_dur_val = torch.nan_to_num(sp_fix_dur_val)
+                    sp_landing_pos_val = torch.nan_to_num(sp_landing_pos_val)
+                    sn_word_len_val = (sn_word_len_val - sn_word_len_mean) / sn_word_len_std
+                    sn_word_len_val = torch.nan_to_num(sn_word_len_val)
+
+                    dnn_out_val, atten_weights_val = dnn(
+                        sn_emd=sn_input_ids_val,
+                        sn_mask=sn_attention_mask_val,
+                        sp_emd=sp_input_ids_val,
+                        sp_pos=sp_pos_val,
+                        word_ids_sn=None,
+                        word_ids_sp=None,
+                        sp_fix_dur=sp_fix_dur_val,
+                        sp_landing_pos=sp_landing_pos_val,
+                        sn_word_len=sn_word_len_val,
+                        sub_id=sub_id_val,
+                    )
+
+                    dnn_out_val = dnn_out_val.permute(0, 2, 1)  # [batch, dec_o_dim, step
+
+                    # prepare label and mask
+                    loss = nn.CrossEntropyLoss(reduction="none")
+                    pad_mask_val, label_val = load_label(sp_pos_val, cf, le, device)
+                    batch_error_val = torch.mean(
+                        torch.masked_select(loss(dnn_out_val, label_val), ~pad_mask_val)
+                    )
+                    val_loss.append(batch_error_val.detach().to("cpu").numpy())
+            print("\nvalidation loss is {} \n".format(np.mean(val_loss)))
+            loss_dict["val_loss"].append(np.mean(val_loss))
+
+            if np.mean(val_loss) < old_score:
+                # save model if val loss is smallest
+                torch.save(
+                    dnn.state_dict(),
+                    "{}/CELoss_BSC_text_eyettention_readerID_{}_emb{}_newloss_fold{}.pth".format(
+                        args.save_data_folder, args.atten_type, args.emb_size, fold_indx
+                    ),
+                )
+                old_score = np.mean(val_loss)
+                print("\nsaved model state dict\n")
+                save_ep_couter = episode_i
+            else:
+                # early stopping
+                if episode_i - save_ep_couter >= cf["earlystop_patience"]:
+                    break
+
+        # evaluation
+        dnn.eval()
+        res_llh = []
+        dnn.load_state_dict(
+            torch.load(
+                os.path.join(
+                    args.save_data_folder,
+                    f"CELoss_BSC_text_eyettention_readerID_{args.atten_type}_emb{args.emb_size}_newloss_fold{fold_indx}.pth",
+                ),
+                map_location="cpu",
+            )
+        )
+        dnn.to(device)
+        batch_indx = 0
+        for batchh in test_dataloaderr:
+            with torch.no_grad():
+                sn_input_ids_test = batchh["sn_input_ids"].to(device)
+                sn_attention_mask_test = batchh["sn_attention_mask"].to(device)
+                sp_input_ids_test = batchh["sp_input_ids"].to(device)
+                sp_attention_mask_test = batchh["sp_attention_mask"].to(device)
+                sp_pos_test = batchh["sp_pos"].to(device)  # 28: '<Sep>', 29: '<'Pad'>'
+                sp_landing_pos_test = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur_test = (batchh["sp_fix_dur"] / 1000).to(device)
+                sn_word_len_test = batchh["sn_word_len"].to(device)
+                sub_id_test = batchh["sub_id"].to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur_test, 0)
+                sp_fix_dur_test = (sp_fix_dur_test - fix_dur_mean) / fix_dur_std * mask
+                sp_landing_pos_test = (
+                    (sp_landing_pos_test - landing_pos_mean) / landing_pos_std * mask
+                )
+                sp_fix_dur_test = torch.nan_to_num(sp_fix_dur_test)
+                sp_landing_pos_test = torch.nan_to_num(sp_landing_pos_test)
+                sn_word_len_test = (sn_word_len_test - sn_word_len_mean) / sn_word_len_std
+                sn_word_len_test = torch.nan_to_num(sn_word_len_test)
+
+                dnn_out_test, atten_weights_test = dnn(
+                    sn_emd=sn_input_ids_test,
+                    sn_mask=sn_attention_mask_test,
+                    sp_emd=sp_input_ids_test,
+                    sp_pos=sp_pos_test,
+                    word_ids_sn=None,
+                    word_ids_sp=None,
+                    sp_fix_dur=sp_fix_dur_test,
+                    sp_landing_pos=sp_landing_pos_test,
+                    sn_word_len=sn_word_len_test,
+                    sub_id=sub_id_test,
+                )
+
+                # We do not use nn.CrossEntropyLoss here to calculate the likelihood because it combines nn.LogSoftmax and nn.NLL,
+                # while nn.LogSoftmax returns a log value based on e, we want 2 instead
+                # m = nn.LogSoftmax(dim=2) -- base e, we want base 2
+                m = nn.Softmax(dim=2)
+                dnn_out_test = m(dnn_out_test).detach().to("cpu").numpy()
+
+                # prepare label and mask
+                pad_mask_test, label_test = load_label(sp_pos_test, cf, le, "cpu")
+                pred = dnn_out_test.argmax(axis=2)
+                # compute log likelihood for the batch samples
+                res_batch = eval_log_llh(dnn_out_test, label_test, pad_mask_test)
+                res_llh.append(np.array(res_batch))
+
+                batch_indx += 1
+
+        res_llh = np.concatenate(res_llh).ravel()
+        loss_dict["test_ll"].append(res_llh)
+        loss_dict["fix_dur_mean"] = fix_dur_mean
+        loss_dict["fix_dur_std"] = fix_dur_std
+        loss_dict["landing_pos_mean"] = landing_pos_mean
+        loss_dict["landing_pos_std"] = landing_pos_std
+        loss_dict["sn_word_len_mean"] = sn_word_len_mean
+        loss_dict["sn_word_len_std"] = sn_word_len_std
+        print("\nTest likelihood is {} \n".format(np.mean(res_llh)))
+        # save results
+        with open(
+            "{}/res_BSC_NRS_eyettention_readerID_{}_emb{}_Fold{}.pickle".format(
+                args.save_data_folder, args.atten_type, args.emb_size, fold_indx
+            ),
+            "wb",
+        ) as handle:
+            pickle.dump(loss_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
+
+        fold_indx += 1

main_celer.py

@@ -0,0 +1,439 @@
+import numpy as np
+import pandas as pd
+import os
+from sklearn.model_selection import StratifiedKFold, KFold
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+from torch.optim import Adam, RMSprop
+from transformers import BertTokenizerFast
+from sklearn.preprocessing import LabelEncoder, OneHotEncoder
+from torch.nn.functional import cross_entropy, softmax
+from collections import deque
+import pickle
+import json
+import matplotlib.pyplot as plt
+import argparse
+
+from Eyettention.utils import *
+from Eyettention.model import Eyettention
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="run Eyettention on CELER dataset")
+    parser.add_argument(
+        "--test_mode",
+        help="New Sentence Split: text, New Reader Split: subject",
+        type=str,
+        default="text",
+    )
+    parser.add_argument(
+        "--atten_type", help="attention type: global, local, local-g", type=str, default="local-g"
+    )
+    parser.add_argument(
+        "--save_data_folder",
+        help="folder path for saving results",
+        type=str,
+        default="./results/CELER/",
+    )
+    parser.add_argument(
+        "--scanpath_gen_flag", help="whether to generate scanpath", type=int, default=1
+    )
+    parser.add_argument(
+        "--max_pred_len",
+        help="if scanpath_gen_flag is True, you can determine the longest scanpath that you want to generate, which should depend on the sentence length",
+        type=int,
+        default=60,
+    )
+    parser.add_argument("--gpu", help="gpu index", type=int, default=6)
+    args = parser.parse_args()
+    gpu = args.gpu
+
+    # use FastTokenizer lead to warning -> The current process just got forked
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+    torch.set_default_tensor_type("torch.FloatTensor")
+    availbl = torch.cuda.is_available()
+    print(torch.cuda.is_available())
+    if availbl:
+        device = f"cuda:{gpu}"
+    else:
+        device = "cpu"
+    print(device)
+    torch.cuda.set_device(gpu)
+
+    cf = {
+        "model_pretrained": "bert-base-cased",
+        "lr": 1e-3,
+        "max_grad_norm": 10,
+        "n_epochs": 1000,
+        "n_folds": 5,
+        "dataset": "celer",
+        "atten_type": args.atten_type,
+        "batch_size": 256,
+        "max_sn_len": 24,  # max number of words in a sentence, include start token and end token,
+        "max_sn_token": 35,  # maximum number of tokens a sentence includes. include start token and end token,
+        "max_sp_len": 52,  # max number of words in a scanpath, include start token and end token
+        "max_sp_token": 395,  # maximum number of tokens a scanpath includes. include start token and end token
+        "norm_type": "z-score",
+        "earlystop_patience": 20,
+        "max_pred_len": args.max_pred_len,
+    }
+
+    # Encode the label into interger categories, setting the exclusive category 'cf["max_sn_len"]-1' as the end sign
+    le = LabelEncoder()
+    le.fit(np.append(np.arange(-cf["max_sn_len"] + 3, cf["max_sn_len"] - 1), cf["max_sn_len"] - 1))
+    # le.classes_
+
+    # load corpus
+    word_info_df, _, eyemovement_df = load_corpus(cf["dataset"])
+
+    # only use native speaker
+    # Make list with reader index
+    reader_list = celer_load_native_speaker()
+
+    # Make list with sentence index
+    sn_list = np.unique(
+        word_info_df[word_info_df["list"].isin(reader_list)].sentenceid.values
+    ).tolist()
+
+    # Split training&test sets by text or reader, depending on configuration
+    if args.test_mode == "text":
+        print("Start evaluating on new sentences.")
+        split_list = sn_list
+    elif args.test_mode == "subject":
+        print("Start evaluating on new readers.")
+        split_list = reader_list
+
+    n_folds = cf["n_folds"]
+    kf = KFold(n_splits=n_folds, shuffle=True, random_state=0)
+    fold_indx = 0
+    # for scanpath generation
+    sp_dnn_list = []
+    sp_human_list = []
+    for train_idx, test_idx in kf.split(split_list):
+        loss_dict = {"val_loss": [], "train_loss": [], "test_ll": [], "test_AUC": []}
+        list_train = [split_list[i] for i in train_idx]
+        list_test = [split_list[i] for i in test_idx]
+
+        # create train validation split for training the models:
+        kf_val = KFold(n_splits=n_folds, shuffle=True, random_state=0)
+        for train_index, val_index in kf_val.split(list_train):
+            # we only evaluate a single fold
+            break
+        list_train_net = [list_train[i] for i in train_index]
+        list_val_net = [list_train[i] for i in val_index]
+
+        if args.test_mode == "text":
+            sn_list_train = list_train_net
+            sn_list_val = list_val_net
+            sn_list_test = list_test
+            reader_list_train, reader_list_val, reader_list_test = (
+                reader_list,
+                reader_list,
+                reader_list,
+            )
+
+        elif args.test_mode == "subject":
+            reader_list_train = list_train_net
+            reader_list_val = list_val_net
+            reader_list_test = list_test
+            sn_list_train, sn_list_val, sn_list_test = sn_list, sn_list, sn_list
+
+        # initialize tokenizer
+        tokenizer = BertTokenizerFast.from_pretrained(cf["model_pretrained"])
+        # Preparing batch data
+        dataset_train = celerdataset(
+            word_info_df, eyemovement_df, cf, reader_list_train, sn_list_train, tokenizer
+        )
+        train_dataloaderr = DataLoader(
+            dataset_train, batch_size=cf["batch_size"], shuffle=True, drop_last=True
+        )
+
+        dataset_val = celerdataset(
+            word_info_df, eyemovement_df, cf, reader_list_val, sn_list_val, tokenizer
+        )
+        val_dataloaderr = DataLoader(
+            dataset_val, batch_size=cf["batch_size"], shuffle=False, drop_last=True
+        )
+
+        dataset_test = celerdataset(
+            word_info_df, eyemovement_df, cf, reader_list_test, sn_list_test, tokenizer
+        )
+        test_dataloaderr = DataLoader(
+            dataset_test, batch_size=cf["batch_size"], shuffle=False, drop_last=False
+        )
+
+        # z-score normalization for gaze features
+        fix_dur_mean, fix_dur_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_fix_dur", padding_value=0, scale=1000
+        )
+        landing_pos_mean, landing_pos_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_landing_pos", padding_value=0
+        )
+        sn_word_len_mean, sn_word_len_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sn_word_len"
+        )
+
+        # load model
+        dnn = Eyettention(cf)
+
+        # training
+        episode = 0
+        optimizer = Adam(dnn.parameters(), lr=cf["lr"])
+        dnn.train()
+        dnn.to(device)
+        av_score = deque(maxlen=100)
+        old_score = 1e10
+        save_ep_couter = 0
+        print("Start training")
+        for episode_i in range(episode, cf["n_epochs"] + 1):
+            dnn.train()
+            print("episode:", episode_i)
+            counter = 0
+            for batchh in train_dataloaderr:
+                counter += 1
+                batchh.keys()
+                sn_input_ids = batchh["sn_input_ids"].to(device)
+                sn_attention_mask = batchh["sn_attention_mask"].to(device)
+                word_ids_sn = batchh["word_ids_sn"].to(device)
+                sn_word_len = batchh["sn_word_len"].to(device)
+
+                sp_input_ids = batchh["sp_input_ids"].to(device)
+                sp_attention_mask = batchh["sp_attention_mask"].to(device)
+                word_ids_sp = batchh["word_ids_sp"].to(device)
+
+                sp_pos = batchh["sp_pos"].to(device)
+                sp_landing_pos = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur = (batchh["sp_fix_dur"] / 1000).to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur, 0)
+                sp_fix_dur = (sp_fix_dur - fix_dur_mean) / fix_dur_std * mask
+                sp_landing_pos = (sp_landing_pos - landing_pos_mean) / landing_pos_std * mask
+                sp_fix_dur = torch.nan_to_num(sp_fix_dur)
+                sp_landing_pos = torch.nan_to_num(sp_landing_pos)
+                sn_word_len = (sn_word_len - sn_word_len_mean) / sn_word_len_std
+                sn_word_len = torch.nan_to_num(sn_word_len)
+
+                # zero old gradients
+                optimizer.zero_grad()
+                # predict output with DNN
+                dnn_out, atten_weights = dnn(
+                    sn_emd=sn_input_ids,
+                    sn_mask=sn_attention_mask,
+                    sp_emd=sp_input_ids,
+                    sp_pos=sp_pos,
+                    word_ids_sn=word_ids_sn,
+                    word_ids_sp=word_ids_sp,
+                    sp_fix_dur=sp_fix_dur,
+                    sp_landing_pos=sp_landing_pos,
+                    sn_word_len=sn_word_len,
+                )
+
+                dnn_out = dnn_out.permute(0, 2, 1)  # [batch, dec_o_dim, step]
+
+                # prepare label and mask
+                pad_mask, label = load_label(sp_pos, cf, le, device)
+                loss = nn.CrossEntropyLoss(reduction="none")
+                batch_error = torch.mean(torch.masked_select(loss(dnn_out, label), ~pad_mask))
+
+                # backpropagate loss
+                batch_error.backward()
+                # clip gradients
+                gradient_clipping(dnn, cf["max_grad_norm"])
+
+                # learn
+                optimizer.step()
+                av_score.append(batch_error.to("cpu").detach().numpy())
+                print("counter:", counter)
+                print(
+                    "\rSample {}\tAverage Error: {:.10f} ".format(counter, np.mean(av_score)),
+                    end=" ",
+                )
+            loss_dict["train_loss"].append(np.mean(av_score))
+
+            val_loss = []
+            dnn.eval()
+            for batchh in val_dataloaderr:
+                with torch.no_grad():
+                    sn_input_ids_val = batchh["sn_input_ids"].to(device)
+                    sn_attention_mask_val = batchh["sn_attention_mask"].to(device)
+                    word_ids_sn_val = batchh["word_ids_sn"].to(device)
+                    sn_word_len_val = batchh["sn_word_len"].to(device)
+
+                    sp_input_ids_val = batchh["sp_input_ids"].to(device)
+                    sp_attention_mask_val = batchh["sp_attention_mask"].to(device)
+                    word_ids_sp_val = batchh["word_ids_sp"].to(device)
+
+                    sp_pos_val = batchh["sp_pos"].to(device)
+                    sp_landing_pos_val = batchh["sp_landing_pos"].to(device)
+                    sp_fix_dur_val = (batchh["sp_fix_dur"] / 1000).to(device)
+
+                    # normalize gaze features
+                    mask = ~torch.eq(sp_fix_dur_val, 0)
+                    sp_fix_dur_val = (sp_fix_dur_val - fix_dur_mean) / fix_dur_std * mask
+                    sp_landing_pos_val = (
+                        (sp_landing_pos_val - landing_pos_mean) / landing_pos_std * mask
+                    )
+                    sp_fix_dur_val = torch.nan_to_num(sp_fix_dur_val)
+                    sp_landing_pos_val = torch.nan_to_num(sp_landing_pos_val)
+                    sn_word_len_val = (sn_word_len_val - sn_word_len_mean) / sn_word_len_std
+                    sn_word_len_val = torch.nan_to_num(sn_word_len_val)
+
+                    dnn_out_val, atten_weights_val = dnn(
+                        sn_emd=sn_input_ids_val,
+                        sn_mask=sn_attention_mask_val,
+                        sp_emd=sp_input_ids_val,
+                        sp_pos=sp_pos_val,
+                        word_ids_sn=word_ids_sn_val,
+                        word_ids_sp=word_ids_sp_val,
+                        sp_fix_dur=sp_fix_dur_val,
+                        sp_landing_pos=sp_landing_pos_val,
+                        sn_word_len=sn_word_len_val,
+                    )
+                    dnn_out_val = dnn_out_val.permute(0, 2, 1)  # [batch, dec_o_dim, step
+
+                    # prepare label and mask
+                    pad_mask_val, label_val = load_label(sp_pos_val, cf, le, device)
+                    batch_error_val = torch.mean(
+                        torch.masked_select(loss(dnn_out_val, label_val), ~pad_mask_val)
+                    )
+                    val_loss.append(batch_error_val.detach().to("cpu").numpy())
+            print("\nvalidation loss is {} \n".format(np.mean(val_loss)))
+            loss_dict["val_loss"].append(np.mean(val_loss))
+
+            if np.mean(val_loss) < old_score:
+                # save model if val loss is smallest
+                torch.save(
+                    dnn.state_dict(),
+                    "{}/CELoss_CELER_{}_eyettention_{}_newloss_fold{}.pth".format(
+                        args.save_data_folder, args.test_mode, args.atten_type, fold_indx
+                    ),
+                )
+                old_score = np.mean(val_loss)
+                print("\nsaved model state dict\n")
+                save_ep_couter = episode_i
+            else:
+                # early stopping
+                if episode_i - save_ep_couter >= cf["earlystop_patience"]:
+                    break
+
+        # evaluation
+        dnn.eval()
+        res_llh = []
+        dnn.load_state_dict(
+            torch.load(
+                os.path.join(
+                    args.save_data_folder,
+                    f"CELoss_CELER_{args.test_mode}_eyettention_{args.atten_type}_newloss_fold{fold_indx}.pth",
+                ),
+                map_location="cpu",
+            )
+        )
+        dnn.to(device)
+        batch_indx = 0
+        for batchh in test_dataloaderr:
+            with torch.no_grad():
+                sn_input_ids_test = batchh["sn_input_ids"].to(device)
+                sn_attention_mask_test = batchh["sn_attention_mask"].to(device)
+                word_ids_sn_test = batchh["word_ids_sn"].to(device)
+                sn_word_len_test = batchh["sn_word_len"].to(device)
+
+                sp_input_ids_test = batchh["sp_input_ids"].to(device)
+                sp_attention_mask_test = batchh["sp_attention_mask"].to(device)
+                word_ids_sp_test = batchh["word_ids_sp"].to(device)
+
+                sp_pos_test = batchh["sp_pos"].to(device)
+                sp_landing_pos_test = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur_test = (batchh["sp_fix_dur"] / 1000).to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur_test, 0)
+                sp_fix_dur_test = (sp_fix_dur_test - fix_dur_mean) / fix_dur_std * mask
+                sp_landing_pos_test = (
+                    (sp_landing_pos_test - landing_pos_mean) / landing_pos_std * mask
+                )
+                sp_fix_dur_test = torch.nan_to_num(sp_fix_dur_test)
+                sp_landing_pos_test = torch.nan_to_num(sp_landing_pos_test)
+                sn_word_len_test = (sn_word_len_test - sn_word_len_mean) / sn_word_len_std
+                sn_word_len_test = torch.nan_to_num(sn_word_len_test)
+
+                dnn_out_test, atten_weights_test = dnn(
+                    sn_emd=sn_input_ids_test,
+                    sn_mask=sn_attention_mask_test,
+                    sp_emd=sp_input_ids_test,
+                    sp_pos=sp_pos_test,
+                    word_ids_sn=word_ids_sn_test,
+                    word_ids_sp=word_ids_sp_test,
+                    sp_fix_dur=sp_fix_dur_test,
+                    sp_landing_pos=sp_landing_pos_test,
+                    sn_word_len=sn_word_len_test,
+                )
+
+                # We do not use nn.CrossEntropyLoss here to calculate the likelihood because it combines nn.LogSoftmax and nn.NLL,
+                # while nn.LogSoftmax returns a log value based on e, we want 2 instead
+                # m = nn.LogSoftmax(dim=2) -- base e, we want base 2
+                m = nn.Softmax(dim=2)
+                dnn_out_test = m(dnn_out_test).detach().to("cpu").numpy()
+
+                # prepare label and mask
+                pad_mask_test, label_test = load_label(sp_pos_test, cf, le, "cpu")
+                pred = dnn_out_test.argmax(axis=2)
+                # compute log likelihood for the batch samples
+                res_batch = eval_log_llh(dnn_out_test, label_test, pad_mask_test)
+                res_llh.append(np.array(res_batch))
+
+                if bool(args.scanpath_gen_flag) == True:
+                    sn_len = (
+                        (torch.max(torch.nan_to_num(word_ids_sn_test), dim=1)[0] + 1 - 2)
+                        .detach()
+                        .to("cpu")
+                        .numpy()
+                    )
+                    # compute the scan path generated from the model when the first few fixed points are given
+                    sp_dnn = dnn.scanpath_generation(
+                        sn_emd=sn_input_ids_test,
+                        sn_mask=sn_attention_mask_test,
+                        word_ids_sn=word_ids_sn_test,
+                        sn_word_len=sn_word_len_test,
+                        le=le,
+                        max_pred_len=cf["max_pred_len"],
+                    )
+
+                    sp_dnn, sp_human = prepare_scanpath(
+                        sp_dnn.detach().to("cpu").numpy(), sn_len, sp_pos_test, cf
+                    )
+                    sp_dnn_list.extend(sp_dnn)
+                    sp_human_list.extend(sp_human)
+
+                batch_indx += 1
+
+        res_llh = np.concatenate(res_llh).ravel()
+        loss_dict["test_ll"].append(res_llh)
+        loss_dict["fix_dur_mean"] = fix_dur_mean
+        loss_dict["fix_dur_std"] = fix_dur_std
+        loss_dict["landing_pos_mean"] = landing_pos_mean
+        loss_dict["landing_pos_std"] = landing_pos_std
+        loss_dict["sn_word_len_mean"] = sn_word_len_mean
+        loss_dict["sn_word_len_std"] = sn_word_len_std
+        print("\nTest likelihood is {} \n".format(np.mean(res_llh)))
+        # save results
+        with open(
+            "{}/res_CELER_{}_eyettention_{}_Fold{}.pickle".format(
+                args.save_data_folder, args.test_mode, args.atten_type, fold_indx
+            ),
+            "wb",
+        ) as handle:
+            pickle.dump(loss_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
+        fold_indx += 1
+
+    if bool(args.scanpath_gen_flag) == True:
+        # save results
+        dic = {"sp_dnn": sp_dnn_list, "sp_human": sp_human_list}
+        with open(
+            os.path.join(
+                args.save_data_folder,
+                f"CELER_scanpath_generation_eyettention_{args.test_mode}_{args.atten_type}.pickle",
+            ),
+            "wb",
+        ) as handle:
+            pickle.dump(dic, handle, protocol=pickle.HIGHEST_PROTOCOL)

main_celer_NRS_setting.py

@@ -0,0 +1,419 @@
+import numpy as np
+import pandas as pd
+import os
+from sklearn.model_selection import StratifiedKFold, KFold
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+from torch.optim import Adam, RMSprop
+from transformers import BertTokenizerFast
+from sklearn.preprocessing import LabelEncoder, OneHotEncoder
+from torch.nn.functional import cross_entropy, softmax
+from collections import deque
+import pickle
+import json
+import random
+import matplotlib.pyplot as plt
+import argparse
+
+from Eyettention.utils import *
+from Eyettention.model import Eyettention
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="run uniform baseline")
+    parser.add_argument(
+        "--test_mode", help="test mode: New Reader / New Sentence Split", type=str, default="NRS"
+    )
+    parser.add_argument("--atten_type", help="attention type", type=str, default="local-g")
+    parser.add_argument(
+        "--save_data_folder",
+        help="folder path for saving results",
+        type=str,
+        default="./results/CELER/",
+    )
+    parser.add_argument(
+        "--scanpath_gen_flag", help="whether to generate scanpath", type=int, default=1
+    )
+    parser.add_argument(
+        "--max_pred_len",
+        help="if scanpath_gen_flag is True, you can determine the longest scanpath that you want to generate, which should depend on the sentence length",
+        type=int,
+        default=60,
+    )
+    parser.add_argument("--sampl_time", help="Sampling times", type=int, default=5)
+    parser.add_argument("--gpu", help="gpu index", type=int, default=6)
+    args = parser.parse_args()
+    gpu = args.gpu
+
+    # use FastTokenizer lead to warning -> The current process just got forked
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+    torch.set_default_tensor_type("torch.FloatTensor")
+    availbl = torch.cuda.is_available()
+    print(torch.cuda.is_available())
+    if availbl:
+        device = f"cuda:{gpu}"
+    else:
+        device = "cpu"
+    print(device)
+    torch.cuda.set_device(gpu)
+
+    cf = {
+        "model_pretrained": "bert-base-cased",
+        "lr": 1e-3,
+        "max_grad_norm": 10,
+        "n_epochs": 1000,
+        "dataset": "celer",
+        "atten_type": args.atten_type,
+        "batch_size": 256,
+        "max_sn_len": 24,  # max number of words in a sentence, include start token and end token,
+        "max_sn_token": 35,  # maximum number of tokens a sentence includes. include start token and end token,
+        "max_sp_len": 52,  # max number of words in a scanpath, include start token and end token
+        "max_sp_token": 395,  # maximum number of tokens a scanpath includes. include start token and end token
+        "norm_type": "z-score",
+        "earlystop_patience": 20,
+        "max_pred_len": args.max_pred_len,
+    }
+
+    # Encode the label into interger categories, setting the exclusive category 'cf["max_sn_len"]-1' as the end sign
+    le = LabelEncoder()
+    le.fit(np.append(np.arange(-cf["max_sn_len"] + 3, cf["max_sn_len"] - 1), cf["max_sn_len"] - 1))
+    # le.classes_
+
+    # load corpus
+    word_info_df, _, eyemovement_df = load_corpus(cf["dataset"])
+
+    # only use native speaker
+    # Make list with reader index
+    reader_list = celer_load_native_speaker()
+    # Make list with sentence index
+    # sn_list = np.unique(eyemovement_df.sentenceid.values).tolist()
+    sn_list = np.unique(
+        word_info_df[word_info_df["list"].isin(reader_list)].sentenceid.values
+    ).tolist()
+
+    random.seed(0)
+    fold_indx = 0
+    # for scanpath generation
+    sp_dnn_list = []
+    sp_human_list = []
+    for i in range(args.sampl_time):
+        print("time:", i)
+        loss_dict = {"val_loss": [], "train_loss": [], "test_ll": []}
+        readers_test = random.sample(reader_list, int(np.ceil(len(reader_list) * 0.3)))
+        reader_list_val = random.sample(readers_test, int(np.ceil(len(readers_test) * 0.3)))
+        reader_list_test = set(readers_test) - set(reader_list_val)
+        reader_list_test = list(reader_list_test)
+        reader_list_train = set(reader_list) - set(reader_list_test) - set(reader_list_val)
+        reader_list_train = list(reader_list_train)
+        # Make the order of the test samples unchanged on each re-run
+        reader_list_test.sort()
+
+        sn_test = random.sample(sn_list, int(np.ceil(len(sn_list) * 0.3)))
+        sn_list_val = random.sample(sn_test, int(np.ceil(len(sn_test) * 0.3)))
+        sn_list_test = set(sn_test) - set(sn_list_val)
+        sn_list_test = list(sn_list_test)
+        sn_list_train = set(sn_list) - set(sn_list_test) - set(sn_list_val)
+        sn_list_train = list(sn_list_train)
+        # Make the order of the test samples unchanged on each re-run
+        sn_list_test.sort()
+
+        # initialize tokenizer
+        tokenizer = BertTokenizerFast.from_pretrained(cf["model_pretrained"])
+        # Preparing batch data
+        dataset_train = celerdataset(
+            word_info_df, eyemovement_df, cf, reader_list_train, sn_list_train, tokenizer
+        )
+        train_dataloaderr = DataLoader(
+            dataset_train, batch_size=cf["batch_size"], shuffle=True, drop_last=True
+        )
+
+        dataset_val = celerdataset(
+            word_info_df, eyemovement_df, cf, reader_list_val, sn_list_val, tokenizer
+        )
+        val_dataloaderr = DataLoader(
+            dataset_val, batch_size=cf["batch_size"], shuffle=False, drop_last=False
+        )
+
+        dataset_test = celerdataset(
+            word_info_df, eyemovement_df, cf, reader_list_test, sn_list_test, tokenizer
+        )
+        test_dataloaderr = DataLoader(
+            dataset_test, batch_size=cf["batch_size"], shuffle=False, drop_last=False
+        )
+
+        # z-score normalization for gaze features
+        fix_dur_mean, fix_dur_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_fix_dur", padding_value=0, scale=1000
+        )
+        landing_pos_mean, landing_pos_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_landing_pos", padding_value=0
+        )
+        sn_word_len_mean, sn_word_len_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sn_word_len"
+        )
+
+        # load model
+        dnn = Eyettention(cf)
+
+        # training
+        episode = 0
+        optimizer = Adam(dnn.parameters(), lr=cf["lr"])
+        dnn.train()
+        dnn.to(device)
+        av_score = deque(maxlen=100)
+        old_score = 1e10
+        save_ep_couter = 0
+        print("Start training")
+        for episode_i in range(episode, cf["n_epochs"] + 1):
+            dnn.train()
+            print("episode:", episode_i)
+            counter = 0
+            for batchh in train_dataloaderr:
+                counter += 1
+                batchh.keys()
+                sn_input_ids = batchh["sn_input_ids"].to(device)
+                sn_attention_mask = batchh["sn_attention_mask"].to(device)
+                word_ids_sn = batchh["word_ids_sn"].to(device)
+                sn_word_len = batchh["sn_word_len"].to(device)
+
+                sp_input_ids = batchh["sp_input_ids"].to(device)
+                sp_attention_mask = batchh["sp_attention_mask"].to(device)
+                word_ids_sp = batchh["word_ids_sp"].to(device)
+
+                sp_pos = batchh["sp_pos"].to(device)
+                sp_landing_pos = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur = (batchh["sp_fix_dur"] / 1000).to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur, 0)
+                sp_fix_dur = (sp_fix_dur - fix_dur_mean) / fix_dur_std * mask
+                sp_landing_pos = (sp_landing_pos - landing_pos_mean) / landing_pos_std * mask
+                sp_fix_dur = torch.nan_to_num(sp_fix_dur)
+                sp_landing_pos = torch.nan_to_num(sp_landing_pos)
+                sn_word_len = (sn_word_len - sn_word_len_mean) / sn_word_len_std
+                sn_word_len = torch.nan_to_num(sn_word_len)
+
+                # zero old gradients
+                optimizer.zero_grad()
+                # predict output with DNN
+                dnn_out, atten_weights = dnn(
+                    sn_emd=sn_input_ids,
+                    sn_mask=sn_attention_mask,
+                    sp_emd=sp_input_ids,
+                    sp_pos=sp_pos,
+                    word_ids_sn=word_ids_sn,
+                    word_ids_sp=word_ids_sp,
+                    sp_fix_dur=sp_fix_dur,
+                    sp_landing_pos=sp_landing_pos,
+                    sn_word_len=sn_word_len,
+                )
+
+                dnn_out = dnn_out.permute(0, 2, 1)  # [batch, dec_o_dim, step]
+
+                # prepare label and mask
+                pad_mask, label = load_label(sp_pos, cf, le, device)
+                loss = nn.CrossEntropyLoss(reduction="none")
+                batch_error = torch.mean(torch.masked_select(loss(dnn_out, label), ~pad_mask))
+
+                # backpropagate loss
+                batch_error.backward()
+                # clip gradients
+                gradient_clipping(dnn, cf["max_grad_norm"])
+
+                # learn
+                optimizer.step()
+                av_score.append(batch_error.to("cpu").detach().numpy())
+                print("counter:", counter)
+                print(
+                    "\rSample {}\tAverage Error: {:.10f} ".format(counter, np.mean(av_score)),
+                    end=" ",
+                )
+            loss_dict["train_loss"].append(np.mean(av_score))
+
+            val_loss = []
+            dnn.eval()
+            for batchh in val_dataloaderr:
+                with torch.no_grad():
+                    sn_input_ids_val = batchh["sn_input_ids"].to(device)
+                    sn_attention_mask_val = batchh["sn_attention_mask"].to(device)
+                    word_ids_sn_val = batchh["word_ids_sn"].to(device)
+                    sn_word_len_val = batchh["sn_word_len"].to(device)
+
+                    sp_input_ids_val = batchh["sp_input_ids"].to(device)
+                    sp_attention_mask_val = batchh["sp_attention_mask"].to(device)
+                    word_ids_sp_val = batchh["word_ids_sp"].to(device)
+
+                    sp_pos_val = batchh["sp_pos"].to(device)
+                    sp_landing_pos_val = batchh["sp_landing_pos"].to(device)
+                    sp_fix_dur_val = (batchh["sp_fix_dur"] / 1000).to(device)
+
+                    # normalize gaze features
+                    mask = ~torch.eq(sp_fix_dur_val, 0)
+                    sp_fix_dur_val = (sp_fix_dur_val - fix_dur_mean) / fix_dur_std * mask
+                    sp_landing_pos_val = (
+                        (sp_landing_pos_val - landing_pos_mean) / landing_pos_std * mask
+                    )
+                    sp_fix_dur_val = torch.nan_to_num(sp_fix_dur_val)
+                    sp_landing_pos_val = torch.nan_to_num(sp_landing_pos_val)
+                    sn_word_len_val = (sn_word_len_val - sn_word_len_mean) / sn_word_len_std
+                    sn_word_len_val = torch.nan_to_num(sn_word_len_val)
+
+                    dnn_out_val, atten_weights_val = dnn(
+                        sn_emd=sn_input_ids_val,
+                        sn_mask=sn_attention_mask_val,
+                        sp_emd=sp_input_ids_val,
+                        sp_pos=sp_pos_val,
+                        word_ids_sn=word_ids_sn_val,
+                        word_ids_sp=word_ids_sp_val,
+                        sp_fix_dur=sp_fix_dur_val,
+                        sp_landing_pos=sp_landing_pos_val,
+                        sn_word_len=sn_word_len_val,
+                    )
+                    dnn_out_val = dnn_out_val.permute(0, 2, 1)  # [batch, dec_o_dim, step
+
+                    # prepare label and mask
+                    pad_mask_val, label_val = load_label(sp_pos_val, cf, le, device)
+                    batch_error_val = torch.mean(
+                        torch.masked_select(loss(dnn_out_val, label_val), ~pad_mask_val)
+                    )
+                    val_loss.append(batch_error_val.detach().to("cpu").numpy())
+            print("\nvalidation loss is {} \n".format(np.mean(val_loss)))
+            loss_dict["val_loss"].append(np.mean(val_loss))
+
+            if np.mean(val_loss) < old_score:
+                # save model if val loss is smallest
+                torch.save(
+                    dnn.state_dict(),
+                    "{}/CELoss_CELER_NRS_eyettention_{}_newloss_fold{}.pth".format(
+                        args.save_data_folder, args.atten_type, fold_indx
+                    ),
+                )
+                old_score = np.mean(val_loss)
+                print("\nsaved model state dict\n")
+                save_ep_couter = episode_i
+            else:
+                # early stopping
+                if episode_i - save_ep_couter >= cf["earlystop_patience"]:
+                    break
+
+        # evaluation
+        dnn.eval()
+        res_llh = []
+        dnn.load_state_dict(
+            torch.load(
+                os.path.join(
+                    args.save_data_folder,
+                    f"CELoss_CELER_NRS_eyettention_{args.atten_type}_newloss_fold{fold_indx}.pth",
+                ),
+                map_location="cpu",
+            )
+        )
+        dnn.to(device)
+        batch_indx = 0
+        for batchh in test_dataloaderr:
+            with torch.no_grad():
+                sn_input_ids_test = batchh["sn_input_ids"].to(device)
+                sn_attention_mask_test = batchh["sn_attention_mask"].to(device)
+                word_ids_sn_test = batchh["word_ids_sn"].to(device)
+                sn_word_len_test = batchh["sn_word_len"].to(device)
+
+                sp_input_ids_test = batchh["sp_input_ids"].to(device)
+                sp_attention_mask_test = batchh["sp_attention_mask"].to(device)
+                word_ids_sp_test = batchh["word_ids_sp"].to(device)
+
+                sp_pos_test = batchh["sp_pos"].to(device)
+                sp_landing_pos_test = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur_test = (batchh["sp_fix_dur"] / 1000).to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur_test, 0)
+                sp_fix_dur_test = (sp_fix_dur_test - fix_dur_mean) / fix_dur_std * mask
+                sp_landing_pos_test = (
+                    (sp_landing_pos_test - landing_pos_mean) / landing_pos_std * mask
+                )
+                sp_fix_dur_test = torch.nan_to_num(sp_fix_dur_test)
+                sp_landing_pos_test = torch.nan_to_num(sp_landing_pos_test)
+                sn_word_len_test = (sn_word_len_test - sn_word_len_mean) / sn_word_len_std
+                sn_word_len_test = torch.nan_to_num(sn_word_len_test)
+
+                dnn_out_test, atten_weights_test = dnn(
+                    sn_emd=sn_input_ids_test,
+                    sn_mask=sn_attention_mask_test,
+                    sp_emd=sp_input_ids_test,
+                    sp_pos=sp_pos_test,
+                    word_ids_sn=word_ids_sn_test,
+                    word_ids_sp=word_ids_sp_test,
+                    sp_fix_dur=sp_fix_dur_test,
+                    sp_landing_pos=sp_landing_pos_test,
+                    sn_word_len=sn_word_len_test,
+                )
+
+                # We do not use nn.CrossEntropyLoss here to calculate the likelihood because it combines nn.LogSoftmax and nn.NLL,
+                # while nn.LogSoftmax returns a log value based on e, we want 2 instead
+                # m = nn.LogSoftmax(dim=2) -- base e, we want base 2
+                m = nn.Softmax(dim=2)
+                dnn_out_test = m(dnn_out_test).detach().to("cpu").numpy()
+
+                # prepare label and mask
+                pad_mask_test, label_test = load_label(sp_pos_test, cf, le, "cpu")
+                pred = dnn_out_test.argmax(axis=2)
+                # compute log likelihood for the batch samples
+                res_batch = eval_log_llh(dnn_out_test, label_test, pad_mask_test)
+                res_llh.append(np.array(res_batch))
+
+                # compute the scan path generated from the model when the first few fixed points are given
+                if bool(args.scanpath_gen_flag) == True:
+                    sn_len = (
+                        (torch.max(torch.nan_to_num(word_ids_sn_test), dim=1)[0] + 1 - 2)
+                        .detach()
+                        .to("cpu")
+                        .numpy()
+                    )
+                    # compute the scan path generated from the model when the first few fixed points are given
+                    sp_dnn = dnn.scanpath_generation(
+                        sn_emd=sn_input_ids_test,
+                        sn_mask=sn_attention_mask_test,
+                        word_ids_sn=word_ids_sn_test,
+                        sn_word_len=sn_word_len_test,
+                        le=le,
+                        max_pred_len=cf["max_pred_len"],
+                    )
+
+                    sp_dnn, sp_human = prepare_scanpath(
+                        sp_dnn.detach().to("cpu").numpy(), sn_len, sp_pos_test, cf
+                    )
+                    sp_dnn_list.extend(sp_dnn)
+                    sp_human_list.extend(sp_human)
+
+                batch_indx += 1
+
+        res_llh = np.concatenate(res_llh).ravel()
+        loss_dict["test_ll"].append(res_llh)
+        loss_dict["fix_dur_mean"] = fix_dur_mean
+        loss_dict["fix_dur_std"] = fix_dur_std
+        loss_dict["landing_pos_mean"] = landing_pos_mean
+        loss_dict["landing_pos_std"] = landing_pos_std
+        loss_dict["sn_word_len_mean"] = sn_word_len_mean
+        loss_dict["sn_word_len_std"] = sn_word_len_std
+        print("\nTest likelihood is {} \n".format(np.mean(res_llh)))
+        # save results
+        with open(
+            "{}/res_CELER_NRS_eyettention_{}_Fold{}.pickle".format(
+                args.save_data_folder, args.atten_type, fold_indx
+            ),
+            "wb",
+        ) as handle:
+            pickle.dump(loss_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
+        fold_indx += 1
+
+    if bool(args.scanpath_gen_flag) == True:
+        # save results
+        dic = {"sp_dnn": sp_dnn_list, "sp_human": sp_human_list}
+        with open(
+            os.path.join(
+                args.save_data_folder,
+                f"CELER_scanpath_generation_eyettention_NRS_{args.atten_type}.pickle",
+            ),
+            "wb",
+        ) as handle:
+            pickle.dump(dic, handle, protocol=pickle.HIGHEST_PROTOCOL)

main_celer_reader_identifier.py

@@ -0,0 +1,374 @@
+import numpy as np
+import pandas as pd
+import os
+from sklearn.model_selection import StratifiedKFold, KFold
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader
+from torch.optim import Adam, RMSprop
+from transformers import BertTokenizerFast
+from sklearn.preprocessing import LabelEncoder, OneHotEncoder
+from torch.nn.functional import cross_entropy, softmax
+from collections import deque
+import pickle
+import json
+import matplotlib.pyplot as plt
+import argparse
+
+from Eyettention.utils import *
+from Eyettention.model import Eyettention_readerID
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="run uniform baseline")
+    parser.add_argument("--test_mode", help="test mode: text", type=str, default="text")
+    parser.add_argument("--atten_type", help="attention type", type=str, default="local-g")
+    parser.add_argument(
+        "--save_data_folder",
+        help="folder path for saving results",
+        type=str,
+        default="./results/CELER/",
+    )
+    parser.add_argument("--gpu", help="gpu index", type=int, default=6)
+    parser.add_argument("--emb_size", help="readerID embedding size", type=int, default=32)
+    args = parser.parse_args()
+    gpu = args.gpu
+
+    # use FastTokenizer lead to warning -> The current process just got forked
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+    torch.set_default_tensor_type("torch.FloatTensor")
+    availbl = torch.cuda.is_available()
+    print(torch.cuda.is_available())
+    if availbl:
+        device = f"cuda:{gpu}"
+    else:
+        device = "cpu"
+    print(device)
+    torch.cuda.set_device(gpu)
+
+    cf = {
+        "model_pretrained": "bert-base-cased",
+        "lr": 1e-3,
+        "max_grad_norm": 10,
+        "n_epochs": 1000,
+        "n_folds": 5,
+        "dataset": "celer",
+        "atten_type": args.atten_type,
+        "subid_emb_size": args.emb_size,
+        "batch_size": 256,
+        "max_sn_len": 24,  # max number of words in a sentence, include start token and end token,
+        "max_sn_token": 35,  # maximum number of tokens a sentence includes. include start token and end token,
+        "max_sp_len": 52,  # max number of words in a scanpath, include start token and end token
+        "max_sp_token": 395,  # maximum number of tokens a scanpath includes. include start token and end token
+        "norm_type": "z-score",
+        "earlystop_patience": 20,
+    }
+
+    # Encode the label into interger categories, setting the exclusive category 'cf["max_sn_len"]-1' as the end sign
+    le = LabelEncoder()
+    le.fit(np.append(np.arange(-cf["max_sn_len"] + 3, cf["max_sn_len"] - 1), cf["max_sn_len"] - 1))
+    # le.classes_
+
+    # load corpus
+    word_info_df, _, eyemovement_df = load_corpus(cf["dataset"])
+
+    # only use native speaker
+    # Make list with reader index
+    reader_list = celer_load_native_speaker()
+    # Make list with sentence index
+    sn_list = np.unique(
+        word_info_df[word_info_df["list"].isin(reader_list)].sentenceid.values
+    ).tolist()
+
+    # Split training&test sets by text
+    print("Start evaluating on new sentences.")
+    split_list = sn_list
+
+    n_folds = cf["n_folds"]
+    kf = KFold(n_splits=n_folds, shuffle=True, random_state=0)
+    fold_indx = 0
+    for train_idx, test_idx in kf.split(split_list):
+        loss_dict = {"val_loss": [], "train_loss": [], "test_ll": [], "test_AUC": []}
+        list_train = [split_list[i] for i in train_idx]
+        list_test = [split_list[i] for i in test_idx]
+        # create train validation split for training the models:
+        kf_val = KFold(n_splits=n_folds, shuffle=True, random_state=0)
+        for train_index, val_index in kf_val.split(list_train):
+            # we only evaluate a single fold
+            break
+        list_train_net = [list_train[i] for i in train_index]
+        list_val_net = [list_train[i] for i in val_index]
+
+        sn_list_train = list_train_net
+        sn_list_val = list_val_net
+        sn_list_test = list_test
+        reader_list_train, reader_list_val, reader_list_test = reader_list, reader_list, reader_list
+
+        # initialize tokenizer
+        tokenizer = BertTokenizerFast.from_pretrained(cf["model_pretrained"])
+        # Preparing batch data
+        dataset_train = celerdataset(
+            word_info_df, eyemovement_df, cf, reader_list_train, sn_list_train, tokenizer
+        )
+        train_dataloaderr = DataLoader(
+            dataset_train, batch_size=cf["batch_size"], shuffle=True, drop_last=True
+        )
+        dataset_val = celerdataset(
+            word_info_df, eyemovement_df, cf, reader_list_val, sn_list_val, tokenizer
+        )
+        val_dataloaderr = DataLoader(
+            dataset_val, batch_size=cf["batch_size"], shuffle=False, drop_last=True
+        )
+        dataset_test = celerdataset(
+            word_info_df, eyemovement_df, cf, reader_list_test, sn_list_test, tokenizer
+        )
+        test_dataloaderr = DataLoader(
+            dataset_test, batch_size=cf["batch_size"], shuffle=False, drop_last=False
+        )
+
+        # z-score normalization for gaze features
+        fix_dur_mean, fix_dur_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_fix_dur", padding_value=0, scale=1000
+        )
+        landing_pos_mean, landing_pos_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sp_landing_pos", padding_value=0
+        )
+        sn_word_len_mean, sn_word_len_std = calculate_mean_std(
+            dataloader=train_dataloaderr, feat_key="sn_word_len"
+        )
+
+        # load model
+        dnn = Eyettention_readerID(cf)
+
+        # training
+        episode = 0
+        optimizer = Adam(dnn.parameters(), lr=cf["lr"])
+        dnn.train()
+        dnn.to(device)
+        av_score = deque(maxlen=100)
+        old_score = 1e10
+        save_ep_couter = 0
+        print("Start training")
+        for episode_i in range(episode, cf["n_epochs"] + 1):
+            dnn.train()
+            print("episode:", episode_i)
+            counter = 0
+            for batchh in train_dataloaderr:
+                counter += 1
+                batchh.keys()
+                sn_input_ids = batchh["sn_input_ids"].to(device)
+                sn_attention_mask = batchh["sn_attention_mask"].to(device)
+                word_ids_sn = batchh["word_ids_sn"].to(device)
+                sn_word_len = batchh["sn_word_len"].to(device)
+
+                sp_input_ids = batchh["sp_input_ids"].to(device)
+                sp_attention_mask = batchh["sp_attention_mask"].to(device)
+                word_ids_sp = batchh["word_ids_sp"].to(device)
+
+                sp_pos = batchh["sp_pos"].to(device)
+                sp_landing_pos = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur = (batchh["sp_fix_dur"] / 1000).to(device)
+                sub_id = batchh["sub_id"].to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur, 0)
+                sp_fix_dur = (sp_fix_dur - fix_dur_mean) / fix_dur_std * mask
+                sp_landing_pos = (sp_landing_pos - landing_pos_mean) / landing_pos_std * mask
+                sp_fix_dur = torch.nan_to_num(sp_fix_dur)
+                sp_landing_pos = torch.nan_to_num(sp_landing_pos)
+                sn_word_len = (sn_word_len - sn_word_len_mean) / sn_word_len_std
+                sn_word_len = torch.nan_to_num(sn_word_len)
+
+                # zero old gradients
+                optimizer.zero_grad()
+                # predict output with DNN
+                dnn_out, atten_weights = dnn(
+                    sn_emd=sn_input_ids,
+                    sn_mask=sn_attention_mask,
+                    sp_emd=sp_input_ids,
+                    sp_pos=sp_pos,
+                    word_ids_sn=word_ids_sn,
+                    word_ids_sp=word_ids_sp,
+                    sp_fix_dur=sp_fix_dur,
+                    sp_landing_pos=sp_landing_pos,
+                    sn_word_len=sn_word_len,
+                    sub_id=sub_id,
+                )
+
+                dnn_out = dnn_out.permute(0, 2, 1)  # [batch, dec_o_dim, step]
+
+                # prepare label and mask
+                pad_mask, label = load_label(sp_pos, cf, le, device)
+                loss = nn.CrossEntropyLoss(reduction="none")
+                batch_error = torch.mean(torch.masked_select(loss(dnn_out, label), ~pad_mask))
+
+                # backpropagate loss
+                batch_error.backward()
+                # clip gradients
+                gradient_clipping(dnn, cf["max_grad_norm"])
+
+                # learn
+                optimizer.step()
+                av_score.append(batch_error.to("cpu").detach().numpy())
+                print("counter:", counter)
+                print(
+                    "\rSample {}\tAverage Error: {:.10f} ".format(counter, np.mean(av_score)),
+                    end=" ",
+                )
+            loss_dict["train_loss"].append(np.mean(av_score))
+
+            val_loss = []
+            dnn.eval()
+            for batchh in val_dataloaderr:
+                with torch.no_grad():
+                    sn_input_ids_val = batchh["sn_input_ids"].to(device)
+                    sn_attention_mask_val = batchh["sn_attention_mask"].to(device)
+                    word_ids_sn_val = batchh["word_ids_sn"].to(device)
+                    sn_word_len_val = batchh["sn_word_len"].to(device)
+
+                    sp_input_ids_val = batchh["sp_input_ids"].to(device)
+                    sp_attention_mask_val = batchh["sp_attention_mask"].to(device)
+                    word_ids_sp_val = batchh["word_ids_sp"].to(device)
+
+                    sp_pos_val = batchh["sp_pos"].to(device)
+                    sp_landing_pos_val = batchh["sp_landing_pos"].to(device)
+                    sp_fix_dur_val = (batchh["sp_fix_dur"] / 1000).to(device)
+                    sub_id_val = batchh["sub_id"].to(device)
+
+                    # normalize gaze features
+                    mask = ~torch.eq(sp_fix_dur_val, 0)
+                    sp_fix_dur_val = (sp_fix_dur_val - fix_dur_mean) / fix_dur_std * mask
+                    sp_landing_pos_val = (
+                        (sp_landing_pos_val - landing_pos_mean) / landing_pos_std * mask
+                    )
+                    sp_fix_dur_val = torch.nan_to_num(sp_fix_dur_val)
+                    sp_landing_pos_val = torch.nan_to_num(sp_landing_pos_val)
+                    sn_word_len_val = (sn_word_len_val - sn_word_len_mean) / sn_word_len_std
+                    sn_word_len_val = torch.nan_to_num(sn_word_len_val)
+
+                    dnn_out_val, atten_weights_val = dnn(
+                        sn_emd=sn_input_ids_val,
+                        sn_mask=sn_attention_mask_val,
+                        sp_emd=sp_input_ids_val,
+                        sp_pos=sp_pos_val,
+                        word_ids_sn=word_ids_sn_val,
+                        word_ids_sp=word_ids_sp_val,
+                        sp_fix_dur=sp_fix_dur_val,
+                        sp_landing_pos=sp_landing_pos_val,
+                        sn_word_len=sn_word_len_val,
+                        sub_id=sub_id_val,
+                    )
+                    dnn_out_val = dnn_out_val.permute(0, 2, 1)  # [batch, dec_o_dim, step
+
+                    # prepare label and mask
+                    pad_mask_val, label_val = load_label(sp_pos_val, cf, le, device)
+                    batch_error_val = torch.mean(
+                        torch.masked_select(loss(dnn_out_val, label_val), ~pad_mask_val)
+                    )
+                    val_loss.append(batch_error_val.detach().to("cpu").numpy())
+            print("\nvalidation loss is {} \n".format(np.mean(val_loss)))
+            loss_dict["val_loss"].append(np.mean(val_loss))
+
+            if np.mean(val_loss) < old_score:
+                # save model if val loss is smallest
+                torch.save(
+                    dnn.state_dict(),
+                    "{}/CELoss_CELER_text_eyettention_readerID_{}_emb{}_newloss_fold{}.pth".format(
+                        args.save_data_folder, args.atten_type, args.emb_size, fold_indx
+                    ),
+                )
+                old_score = np.mean(val_loss)
+                print("\nsaved model state dict\n")
+                save_ep_couter = episode_i
+            else:
+                # early stopping
+                if episode_i - save_ep_couter >= cf["earlystop_patience"]:
+                    break
+
+        # evaluation
+        dnn.eval()
+        res_llh = []
+        dnn.load_state_dict(
+            torch.load(
+                os.path.join(
+                    args.save_data_folder,
+                    f"CELoss_CELER_text_eyettention_readerID_{args.atten_type}_emb{args.emb_size}_newloss_fold{fold_indx}.pth",
+                ),
+                map_location="cpu",
+            )
+        )
+        dnn.to(device)
+        batch_indx = 0
+        for batchh in test_dataloaderr:
+            with torch.no_grad():
+                sn_input_ids_test = batchh["sn_input_ids"].to(device)
+                sn_attention_mask_test = batchh["sn_attention_mask"].to(device)
+                word_ids_sn_test = batchh["word_ids_sn"].to(device)
+                sn_word_len_test = batchh["sn_word_len"].to(device)
+
+                sp_input_ids_test = batchh["sp_input_ids"].to(device)
+                sp_attention_mask_test = batchh["sp_attention_mask"].to(device)
+                word_ids_sp_test = batchh["word_ids_sp"].to(device)
+
+                sp_pos_test = batchh["sp_pos"].to(device)
+                sp_landing_pos_test = batchh["sp_landing_pos"].to(device)
+                sp_fix_dur_test = (batchh["sp_fix_dur"] / 1000).to(device)
+                sub_id_test = batchh["sub_id"].to(device)
+
+                # normalize gaze features
+                mask = ~torch.eq(sp_fix_dur_test, 0)
+                sp_fix_dur_test = (sp_fix_dur_test - fix_dur_mean) / fix_dur_std * mask
+                sp_landing_pos_test = (
+                    (sp_landing_pos_test - landing_pos_mean) / landing_pos_std * mask
+                )
+                sp_fix_dur_test = torch.nan_to_num(sp_fix_dur_test)
+                sp_landing_pos_test = torch.nan_to_num(sp_landing_pos_test)
+                sn_word_len_test = (sn_word_len_test - sn_word_len_mean) / sn_word_len_std
+                sn_word_len_test = torch.nan_to_num(sn_word_len_test)
+
+                dnn_out_test, atten_weights_test = dnn(
+                    sn_emd=sn_input_ids_test,
+                    sn_mask=sn_attention_mask_test,
+                    sp_emd=sp_input_ids_test,
+                    sp_pos=sp_pos_test,
+                    word_ids_sn=word_ids_sn_test,
+                    word_ids_sp=word_ids_sp_test,
+                    sp_fix_dur=sp_fix_dur_test,
+                    sp_landing_pos=sp_landing_pos_test,
+                    sn_word_len=sn_word_len_test,
+                    sub_id=sub_id_test,
+                )
+
+                # We do not use nn.CrossEntropyLoss here to calculate the likelihood because it combines nn.LogSoftmax and nn.NLL,
+                # while nn.LogSoftmax returns a log value based on e, we want 2 instead
+                # m = nn.LogSoftmax(dim=2) -- base e, we want base 2
+                m = nn.Softmax(dim=2)
+                dnn_out_test = m(dnn_out_test).detach().to("cpu").numpy()
+
+                # prepare label and mask
+                pad_mask_test, label_test = load_label(sp_pos_test, cf, le, "cpu")
+                pred = dnn_out_test.argmax(axis=2)
+                # compute log likelihood for the batch samples
+                res_batch = eval_log_llh(dnn_out_test, label_test, pad_mask_test)
+                res_llh.append(np.array(res_batch))
+
+                batch_indx += 1
+
+        res_llh = np.concatenate(res_llh).ravel()
+        loss_dict["test_ll"].append(res_llh)
+        loss_dict["fix_dur_mean"] = fix_dur_mean
+        loss_dict["fix_dur_std"] = fix_dur_std
+        loss_dict["landing_pos_mean"] = landing_pos_mean
+        loss_dict["landing_pos_std"] = landing_pos_std
+        loss_dict["sn_word_len_mean"] = sn_word_len_mean
+        loss_dict["sn_word_len_std"] = sn_word_len_std
+        print("\nTest likelihood is {} \n".format(np.mean(res_llh)))
+        # save results
+        with open(
+            "{}/res_CELER_text_eyettention_readerID_{}_emb{}_Fold{}.pickle".format(
+                args.save_data_folder, args.atten_type, args.emb_size, fold_indx
+            ),
+            "wb",
+        ) as handle:
+            pickle.dump(loss_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
+
+        fold_indx += 1

model.py

@@ -0,0 +1,702 @@
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+import torch.nn.functional as F
+import numpy as np
+from torch.nn.functional import cross_entropy, softmax
+from transformers import BertModel, BertConfig
+
+
+class Eyettention(nn.Module):
+    def __init__(self, cf):
+        super(Eyettention, self).__init__()
+        self.cf = cf
+        self.window_width = 1  # D
+        self.atten_type = cf["atten_type"]
+        self.hidden_size = 128
+
+        # Word-Sequence Encoder
+        encoder_config = BertConfig.from_pretrained(self.cf["model_pretrained"])
+        encoder_config.output_hidden_states = True
+        # initiate Bert with pre-trained weights
+        print("keeping Bert with pre-trained weights")
+        self.encoder = BertModel.from_pretrained(self.cf["model_pretrained"], config=encoder_config)
+        self.encoder.eval()
+        # freeze the parameters in Bert model
+        for param in self.encoder.parameters():
+            param.requires_grad = False
+
+        self.embedding_dropout = nn.Dropout(0.4)
+        self.encoder_lstm = nn.LSTM(
+            input_size=768,  # BERT embedding size
+            hidden_size=int(self.hidden_size / 2),
+            num_layers=8,
+            batch_first=True,
+            bidirectional=True,
+            dropout=0.2,
+        )
+
+        # Fixation-Sequence Encoder
+        self.position_embeddings = nn.Embedding(
+            encoder_config.max_position_embeddings, encoder_config.hidden_size
+        )
+        self.LayerNorm = nn.LayerNorm(encoder_config.hidden_size, eps=encoder_config.layer_norm_eps)
+
+        # The scanpath is generated in an autoregressive manner, the output of the previous timestep is fed to the input of the next time step.
+        # So we use decoder cells and loop over all timesteps.
+        # initialize eight decoder cells
+        self.decoder_cell1 = nn.LSTMCell(
+            768 + 2, self.hidden_size
+        )  # first layer input size = #BERT embedding size + two fixation attributes:landing position and fixiation duration
+        self.decoder_cell2 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell3 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell4 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell5 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell6 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell7 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell8 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.dropout_LSTM = nn.Dropout(0.2)
+
+        # Cross-Attention
+        self.attn = nn.Linear(
+            self.hidden_size, self.hidden_size + 1
+        )  # +1 acount for the word length feature
+
+        # Decoder
+        # initialize five dense layers
+        self.dropout_dense = nn.Dropout(0.2)
+        self.decoder_dense = nn.Sequential(
+            self.dropout_dense,
+            nn.Linear(self.hidden_size * 2 + 1, 512),
+            nn.ReLU(),
+            self.dropout_dense,
+            nn.Linear(512, 256),
+            nn.ReLU(),
+            self.dropout_dense,
+            nn.Linear(256, 256),
+            nn.ReLU(),
+            self.dropout_dense,
+            nn.Linear(256, 256),
+            nn.ReLU(),
+            nn.Linear(256, self.cf["max_sn_len"] * 2 - 3),  # number of output classes
+        )
+
+        # for scanpath generation
+        self.softmax = nn.Softmax(dim=1)
+
+    def pool_subword_to_word(self, subword_emb, word_ids_sn, target, pool_method="sum"):
+        # batching computing
+        # Pool bert token (subword) to word level
+        if target == "sn":
+            max_len = self.cf["max_sn_len"]  # CLS and SEP included
+        elif target == "sp":
+            max_len = self.cf["max_sp_len"] - 1  # do not account the 'SEP' token
+
+        merged_word_emb = torch.empty(subword_emb.shape[0], 0, 768).to(subword_emb.device)
+        for word_idx in range(max_len):
+            word_mask = (word_ids_sn == word_idx).unsqueeze(2).repeat(1, 1, 768)
+            # pooling method -> sum
+            if pool_method == "sum":
+                pooled_word_emb = torch.sum(subword_emb * word_mask, 1).unsqueeze(
+                    1
+                )  # [batch, 1, 768]
+            elif pool_method == "mean":
+                pooled_word_emb = torch.mean(subword_emb * word_mask, 1).unsqueeze(
+                    1
+                )  # [batch, 1, 768]
+            merged_word_emb = torch.cat([merged_word_emb, pooled_word_emb], dim=1)
+
+        mask_word = torch.sum(merged_word_emb, 2).bool()
+        return merged_word_emb, mask_word
+
+    def encode(self, sn_emd, sn_mask, word_ids_sn, sn_word_len):
+        # Word-Sequence Encoder
+        outputs = self.encoder(input_ids=sn_emd, attention_mask=sn_mask)
+        hidden_rep_orig, pooled_rep = outputs[0], outputs[1]
+        if word_ids_sn != None:
+            # Pool bert subword to word level for english corpus
+            merged_word_emb, sn_mask_word = self.pool_subword_to_word(
+                hidden_rep_orig, word_ids_sn, target="sn", pool_method="sum"
+            )
+        else:  # no pooling for Chinese corpus
+            merged_word_emb, sn_mask_word = hidden_rep_orig, None
+
+        hidden_rep = self.embedding_dropout(merged_word_emb)
+        x, (hn, hc) = self.encoder_lstm(hidden_rep, None)
+
+        # concatenate with the word length feature
+        x = torch.cat((x, sn_word_len[:, :, None]), dim=2)
+        return x, sn_mask_word
+
+    def cross_attention(self, ht, hs, sn_mask, cur_word_index):
+        # General Attention:
+        # score(ht,hs) = (ht^T)(Wa)hs
+        # hs is the output from word-Sequence Encoder
+        # ht is the previous hidden state from Fixation-Sequence Encoder
+        # self.attn(o): [batch, step, units]
+        attn_prod = torch.matmul(
+            self.attn(ht.unsqueeze(1)), hs.permute(0, 2, 1)
+        )  # [batch, 1, step]
+        if self.atten_type == "global":  # global attention
+            attn_prod += (~sn_mask).unsqueeze(1) * -1e9
+            att_weight = softmax(attn_prod, dim=2)  # [batch, 1, step]
+
+        else:  # local attention
+            # current fixated word index
+            aligned_position = cur_word_index
+            # Get window borders
+            left = torch.where(
+                aligned_position - self.window_width >= 0, aligned_position - self.window_width, 0
+            )
+            right = torch.where(
+                aligned_position + self.window_width <= self.cf["max_sn_len"] - 1,
+                aligned_position + self.window_width,
+                self.cf["max_sn_len"] - 1,
+            )
+
+            # exclude padding tokens
+            # only consider words in the window
+            sen_seq = (
+                torch.arange(self.cf["max_sn_len"])[None, :]
+                .expand(sn_mask.shape[0], self.cf["max_sn_len"])
+                .to(sn_mask.device)
+            )
+            outside_win_mask = (sen_seq < left.unsqueeze(1)) + (sen_seq > right.unsqueeze(1))
+
+            attn_prod += (~sn_mask + outside_win_mask).unsqueeze(1) * -1e9
+            att_weight = softmax(attn_prod, dim=2)  # [batch, 1, step]
+
+            if self.atten_type == "local-g":  # local attention with Gaussian Kernel
+                gauss = lambda s: torch.exp(
+                    -torch.square(s - aligned_position.unsqueeze(1))
+                    / (2 * torch.square(torch.tensor(self.window_width / 2)))
+                )
+                gauss_factor = gauss(sen_seq)
+                att_weight = att_weight * gauss_factor.unsqueeze(1)
+
+        return att_weight
+
+    def decode(self, sp_emd, sn_mask, sp_pos, enc_out, sp_fix_dur, sp_landing_pos, word_ids_sp):
+        # Fixation-Sequence Encoder + Decoder
+        # Initialize hidden state and cell state with zeros,
+        hn = torch.zeros(8, sp_emd.shape[0], self.hidden_size).to(sp_emd.device)
+        hc = torch.zeros(8, sp_emd.shape[0], self.hidden_size).to(sp_emd.device)
+        hx, cx = hn[0, :, :], hc[0, :, :]
+        hx2, cx2 = hn[1, :, :], hc[1, :, :]
+        hx3, cx3 = hn[2, :, :], hc[2, :, :]
+        hx4, cx4 = hn[3, :, :], hc[3, :, :]
+        hx5, cx5 = hn[4, :, :], hc[4, :, :]
+        hx6, cx6 = hn[5, :, :], hc[5, :, :]
+        hx7, cx7 = hn[6, :, :], hc[6, :, :]
+        hx8, cx8 = hn[7, :, :], hc[7, :, :]
+
+        dec_emb_in = self.encoder.embeddings.word_embeddings(sp_emd[:, :-1])
+        if word_ids_sp is not None:
+            # Pool bert subword to word level for English corpus
+            sp_merged_word_emd, sp_mask_word = self.pool_subword_to_word(
+                dec_emb_in, word_ids_sp[:, :-1], target="sp", pool_method="sum"
+            )
+        else:  # no pooling for Chinese corpus
+            sp_merged_word_emd, sp_mask_word = dec_emb_in, None
+
+        # add positional embeddings
+        position_embeddings = self.position_embeddings(sp_pos[:, :-1])
+        dec_emb_in = sp_merged_word_emd + position_embeddings
+        dec_emb_in = self.LayerNorm(dec_emb_in)
+        dec_emb_in = dec_emb_in.permute(1, 0, 2)  # [step, n, emb_dim]
+        dec_emb_in = self.embedding_dropout(dec_emb_in)
+
+        # concatenate two additional gaze features
+        if sp_landing_pos is not None:
+            dec_emb_in = torch.cat((dec_emb_in, sp_landing_pos.permute(1, 0)[:-1, :, None]), dim=2)
+
+        if sp_fix_dur is not None:
+            dec_emb_in = torch.cat((dec_emb_in, sp_fix_dur.permute(1, 0)[:-1, :, None]), dim=2)
+
+        # Predict output for each time step in turn
+        output = []
+        # save attention scores for visualization
+        atten_weights_batch = torch.empty(sp_emd.shape[0], 0, self.cf["max_sn_len"]).to(
+            sp_emd.device
+        )
+        for i in range(dec_emb_in.shape[0]):
+            hx, cx = self.decoder_cell1(dec_emb_in[i], (hx, cx))  # [batch, units]
+            hx2, cx2 = self.decoder_cell2(self.dropout_LSTM(hx), (hx2, cx2))
+            hx3, cx3 = self.decoder_cell3(self.dropout_LSTM(hx2), (hx3, cx3))
+            hx4, cx4 = self.decoder_cell4(self.dropout_LSTM(hx3), (hx4, cx4))
+            hx5, cx5 = self.decoder_cell5(self.dropout_LSTM(hx4), (hx5, cx5))
+            hx6, cx6 = self.decoder_cell6(self.dropout_LSTM(hx5), (hx6, cx6))
+            hx7, cx7 = self.decoder_cell7(self.dropout_LSTM(hx6), (hx7, cx7))
+            hx8, cx8 = self.decoder_cell8(self.dropout_LSTM(hx7), (hx8, cx8))
+
+            att_weight = self.cross_attention(
+                ht=hx8, hs=enc_out, sn_mask=sn_mask, cur_word_index=sp_pos[:, i]
+            )
+            atten_weights_batch = torch.cat([atten_weights_batch, att_weight], dim=1)
+
+            context = torch.matmul(att_weight, enc_out)  # [batch, 1, units]
+
+            # Decoder
+            hc = torch.cat([context.squeeze(1), hx8], dim=1)  # [batch, units *2]
+            result = self.decoder_dense(hc)  # [batch, dec_o_dim]
+            output.append(result)
+
+        output = torch.stack(output, dim=0)  # [step, batch, dec_o_dim]
+        # output = F.softmax(output, dim=2) # cross entropy in pytorch includes softmax
+        return output.permute(1, 0, 2), atten_weights_batch  # [batch, step, dec_o_dim]
+
+    def forward(
+        self,
+        sn_emd,
+        sn_mask,
+        sp_emd,
+        sp_pos,
+        word_ids_sn,
+        word_ids_sp,
+        sp_fix_dur,
+        sp_landing_pos,
+        sn_word_len,
+    ):
+        x, sn_mask_word = self.encode(
+            sn_emd, sn_mask, word_ids_sn, sn_word_len
+        )  # [batch, step, units], [batch, units]
+
+        if sn_mask_word is None:  # for Chinese dataset without token pooling
+            sn_mask = torch.Tensor.bool(sn_mask)
+            pred, atten_weights = self.decode(
+                sp_emd, sn_mask, sp_pos, x, sp_fix_dur, sp_landing_pos, word_ids_sp
+            )  # [batch, step, dec_o_dim]
+
+        else:  # for English dataset with token pooling
+            pred, atten_weights = self.decode(
+                sp_emd, sn_mask_word, sp_pos, x, sp_fix_dur, sp_landing_pos, word_ids_sp
+            )  # [batch, step, dec_o_dim]
+
+        return pred, atten_weights
+
+    def scanpath_generation(
+        self, sn_emd, sn_mask, word_ids_sn, sn_word_len, le, max_pred_len=60, previous_scanpath=None
+    ):
+
+        if max_pred_len <= 0:
+            raise ValueError("max_pred_len must be positive.")
+
+        prev_scanpath_len = 0
+
+        if previous_scanpath is not None:
+            if (
+                len(previous_scanpath) == 0 or previous_scanpath[0] != 0
+            ):  # prepend with 0 if no 0 at the start of previous scanpaths
+                previous_scanpath = [0] + previous_scanpath
+
+            prev_scanpath_len = len(previous_scanpath)
+
+            previous_scanpath = torch.as_tensor(
+                previous_scanpath, dtype=torch.long, device=sn_emd.device
+            )
+            if previous_scanpath.ndim == 1:
+                previous_scanpath = previous_scanpath.unsqueeze(0)
+            # if input includes CLS: [0, 1, 2, ...]
+            previous_scanpath = previous_scanpath[:, 1:]
+
+        # compute the scan path generated from the model when the first CLS taken is given
+        enc_out, sn_mask_word = self.encode(sn_emd, sn_mask, word_ids_sn, sn_word_len)
+        if sn_mask_word is None:
+            sn_mask = torch.Tensor.bool(sn_mask)
+        else:
+            sn_mask = sn_mask_word
+        sn_len = torch.sum(sn_mask, axis=1) - 2
+
+        # decode
+        # Initialize hidden state and cell state with zeros,
+        hn = torch.zeros(8, sn_emd.shape[0], self.hidden_size).to(sn_emd.device)
+        hc = torch.zeros(8, sn_emd.shape[0], self.hidden_size).to(sn_emd.device)
+        hx, cx = hn[0, :, :], hc[0, :, :]
+        hx2, cx2 = hn[1, :, :], hc[1, :, :]
+        hx3, cx3 = hn[2, :, :], hc[2, :, :]
+        hx4, cx4 = hn[3, :, :], hc[3, :, :]
+        hx5, cx5 = hn[4, :, :], hc[4, :, :]
+        hx6, cx6 = hn[5, :, :], hc[5, :, :]
+        hx7, cx7 = hn[6, :, :], hc[6, :, :]
+        hx8, cx8 = hn[7, :, :], hc[7, :, :]
+
+        # use CLS token (101) as start token
+        dec_in_start = (torch.ones(sn_mask.shape[0]) * 101).long().to(sn_mask.device)
+        dec_emb_in = self.encoder.embeddings.word_embeddings(dec_in_start)  # [batch, emb_dim]
+        # dec_in_start = sp_emd[:, 0]
+        # dec_emb_in = self.encoder.embeddings.word_embeddings(dec_in_start) # [batch, emb_dim]
+
+        # add positional embeddings
+        start_pos = torch.zeros(sn_mask.shape[0]).to(sn_mask.device)
+        position_embeddings = self.position_embeddings(start_pos.long())
+        dec_emb_in = dec_emb_in + position_embeddings
+        dec_emb_in = self.LayerNorm(dec_emb_in)
+
+        # concatenate two additional gaze features, which are set to zeros for CLS token
+        dec_in = torch.cat(
+            (dec_emb_in, torch.zeros(dec_emb_in.shape[0], 2).to(sn_emd.device)), dim=1
+        )
+
+        # generate fixation one by one in an autoregressive way
+        output = []
+        density_prediction = []
+        pred_counter = 0
+        # output.append(sp_pos[:, pred_counter])
+        output.append(start_pos.long())
+        for p in range(prev_scanpath_len + max_pred_len - 1):
+            hx, cx = self.decoder_cell1(dec_in, (hx, cx))  # [batch, units]
+            hx2, cx2 = self.decoder_cell2(self.dropout_LSTM(hx), (hx2, cx2))
+            hx3, cx3 = self.decoder_cell3(self.dropout_LSTM(hx2), (hx3, cx3))
+            hx4, cx4 = self.decoder_cell4(self.dropout_LSTM(hx3), (hx4, cx4))
+            hx5, cx5 = self.decoder_cell5(self.dropout_LSTM(hx4), (hx5, cx5))
+            hx6, cx6 = self.decoder_cell6(self.dropout_LSTM(hx5), (hx6, cx6))
+            hx7, cx7 = self.decoder_cell7(self.dropout_LSTM(hx6), (hx7, cx7))
+            hx8, cx8 = self.decoder_cell8(self.dropout_LSTM(hx7), (hx8, cx8))
+
+            att_weight = self.cross_attention(
+                ht=hx8, hs=enc_out, sn_mask=sn_mask, cur_word_index=output[-1]
+            )
+
+            context = torch.matmul(att_weight, enc_out)  # [batch, 1, units]
+            hc = torch.cat([context.squeeze(1), hx8], dim=1)  # [batch, units *2]
+
+            result = self.decoder_dense(hc)  # [batch, dec_o_dim]
+            result = self.softmax(result)  # [batch, dec_o_dim]
+            density_prediction.append(result)
+
+            # we can either take argmax or sampling from the output distribution,
+            # we do sampling in the paper
+            # pred_indx = result.argmax(dim=1)
+            # sampling next fixation location according to the distribution
+            if previous_scanpath is not None and p < previous_scanpath.shape[1]:
+                # replay observed fixation
+                pred_pos = previous_scanpath[:, p].clone()
+            else:
+                pred_indx = torch.multinomial(result, 1)
+                pred_class = [le.classes_[pred_indx[i]] for i in torch.arange(result.shape[0])]
+                pred_class = torch.from_numpy(np.array(pred_class)).to(sn_emd.device)
+                # predict fixation word index = last fixation word index + predicted saccade range
+                pred_pos = output[-1] + pred_class
+
+            # larger than sentence max length -- set to sentence length+1, i.e. token <'SEP'>
+            # prepare the input to the next timstep
+            input_ids = []
+            for i in range(pred_pos.shape[0]):
+                if pred_pos[i] > sn_len[i]:
+                    pred_pos[i] = sn_len[i] + 1
+                elif pred_pos[i] < 1:
+                    pred_pos[i] = 1
+
+                if word_ids_sn is not None:
+                    input_ids.append(sn_emd[i, word_ids_sn[i, :] == pred_pos[i]])
+                else:
+                    input_ids.append(sn_emd[i, pred_pos[i]])
+            output.append(pred_pos)
+
+            # prepare next timestamp input token
+            pred_counter += 1
+            if word_ids_sn is not None:
+                # merge tokens
+                dec_emb_in = torch.empty(0, 768).to(sn_emd.device)
+                for id in input_ids:
+                    dec_emb_in = torch.cat(
+                        [
+                            dec_emb_in,
+                            torch.sum(self.encoder.embeddings.word_embeddings(id), axis=0)[None, :],
+                        ],
+                        dim=0,
+                    )
+
+            else:
+                input_ids = torch.stack(input_ids)
+                dec_emb_in = self.encoder.embeddings.word_embeddings(input_ids)  # [batch, emb_dim]
+            # add positional embeddings
+            position_embeddings = self.position_embeddings(output[-1])
+            dec_emb_in = dec_emb_in + position_embeddings
+            dec_emb_in = self.LayerNorm(dec_emb_in)
+            # concatenate two additional gaze features
+            dec_in = torch.cat(
+                (dec_emb_in, torch.zeros(dec_emb_in.shape[0], 2).to(sn_emd.device)), dim=1
+            )
+
+        output = torch.stack(output, dim=0)  # [step, batch]
+        return output.permute(1, 0), density_prediction  # [batch, step]
+
+
+class Eyettention_readerID(nn.Module):
+    def __init__(self, cf):
+        super(Eyettention_readerID, self).__init__()
+        self.cf = cf
+        self.window_width = 1  # D
+        self.atten_type = cf["atten_type"]
+        self.hidden_size = 128
+        self.sub_emb_size = cf["subid_emb_size"]
+
+        # Word-Sequence Encoder
+        encoder_config = BertConfig.from_pretrained(self.cf["model_pretrained"])
+        encoder_config.output_hidden_states = True
+        # initiate Bert with pre-trained weights
+        print("keeping Bert with pre-trained weights")
+        self.encoder = BertModel.from_pretrained(self.cf["model_pretrained"], config=encoder_config)
+        self.encoder.eval()
+        # freeze the parameters in Bert model
+        for param in self.encoder.parameters():
+            param.requires_grad = False
+
+        self.embedding_dropout = nn.Dropout(0.4)
+        self.encoder_lstm = nn.LSTM(
+            input_size=768,  # BERT embedding size
+            hidden_size=int(self.hidden_size / 2),
+            num_layers=8,
+            batch_first=True,
+            bidirectional=True,
+            dropout=0.2,
+        )
+
+        # Fixation-Sequence Encoder
+        self.position_embeddings = nn.Embedding(
+            encoder_config.max_position_embeddings, encoder_config.hidden_size
+        )
+        self.LayerNorm = nn.LayerNorm(encoder_config.hidden_size, eps=encoder_config.layer_norm_eps)
+        # for reader-specific setting
+        self.sub_embeddings = nn.Embedding(400, self.sub_emb_size)
+
+        # The scanpath is generated in an autoregressive manner, the output of the previous timestep is fed to the input of the next time step.
+        # So we use decoder cells and loop over all timesteps.
+        # initialize eight decoder cells
+        self.decoder_cell1 = nn.LSTMCell(
+            768 + 2 + self.sub_emb_size, self.hidden_size
+        )  # first layer input size = #BERT embedding size + two fixation attributes:landing position and fixiation duration
+        self.decoder_cell2 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell3 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell4 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell5 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell6 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell7 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.decoder_cell8 = nn.LSTMCell(self.hidden_size, self.hidden_size)
+        self.dropout_LSTM = nn.Dropout(0.2)
+
+        # Cross-Attention
+        self.attn = nn.Linear(
+            self.hidden_size, self.hidden_size + 1
+        )  # +1 acoount for the word length feature
+
+        # Decoder
+        # initialize five dense layers
+        self.dropout_dense = nn.Dropout(0.2)
+        self.decoder_dense = nn.Sequential(
+            self.dropout_dense,
+            nn.Linear(self.hidden_size * 2 + 1, 512),
+            nn.ReLU(),
+            self.dropout_dense,
+            nn.Linear(512, 256),
+            nn.ReLU(),
+            self.dropout_dense,
+            nn.Linear(256, 256),
+            nn.ReLU(),
+            self.dropout_dense,
+            nn.Linear(256, 256),
+            nn.ReLU(),
+            nn.Linear(256, self.cf["max_sn_len"] * 2 - 3),  # number of output classes
+        )
+
+        # for scanpath generation
+        self.softmax = nn.Softmax(dim=1)
+
+    def pool_subword_to_word(self, subword_emb, word_ids_sn, target, pool_method="sum"):
+        # batching computing
+        # Pool bert token (subword) to word level
+        if target == "sn":
+            max_len = self.cf["max_sn_len"]  # CLS and SEP included
+        elif target == "sp":
+            max_len = self.cf["max_sp_len"] - 1  # do not account the 'SEP' token
+
+        merged_word_emb = torch.empty(subword_emb.shape[0], 0, 768).to(subword_emb.device)
+        for word_idx in range(max_len):
+            word_mask = (word_ids_sn == word_idx).unsqueeze(2).repeat(1, 1, 768)
+            # pooling method -> sum
+            if pool_method == "sum":
+                pooled_word_emb = torch.sum(subword_emb * word_mask, 1).unsqueeze(
+                    1
+                )  # [batch, 1, 768]
+            elif pool_method == "mean":
+                pooled_word_emb = torch.mean(subword_emb * word_mask, 1).unsqueeze(
+                    1
+                )  # [batch, 1, 768]
+            merged_word_emb = torch.cat([merged_word_emb, pooled_word_emb], dim=1)
+
+        mask_word = torch.sum(merged_word_emb, 2).bool()
+        return merged_word_emb, mask_word
+
+    def encode(self, sn_emd, sn_mask, word_ids_sn, sn_word_len):
+        # Word-Sequence Encoder
+        outputs = self.encoder(input_ids=sn_emd, attention_mask=sn_mask)
+        hidden_rep_orig, pooled_rep = outputs[0], outputs[1]
+        if word_ids_sn != None:
+            # Pool bert subword to word level for english corpus
+            merged_word_emb, sn_mask_word = self.pool_subword_to_word(
+                hidden_rep_orig, word_ids_sn, target="sn", pool_method="sum"
+            )
+        else:  # no pooling for Chinese corpus
+            merged_word_emb, sn_mask_word = hidden_rep_orig, None
+
+        hidden_rep = self.embedding_dropout(merged_word_emb)
+        x, (hn, hc) = self.encoder_lstm(hidden_rep, None)
+
+        # concatenate with the word length feature
+        x = torch.cat((x, sn_word_len[:, :, None]), dim=2)
+        return x, sn_mask_word
+
+    def cross_attention(self, ht, hs, sn_mask, cur_word_index):
+        # General Attention:
+        # score(ht,hs) = (ht^T)(Wa)hs
+        # hs is the output from word-Sequence Encoder
+        # ht is the previous hidden state from Fixation-Sequence Encoder
+        # self.attn(o): [batch, step, units]
+        attn_prod = torch.matmul(
+            self.attn(ht.unsqueeze(1)), hs.permute(0, 2, 1)
+        )  # [batch, 1, step]
+        if self.atten_type == "global":  # global attention
+            attn_prod += (~sn_mask).unsqueeze(1) * -1e9
+            att_weight = softmax(attn_prod, dim=2)  # [batch, 1, step]
+
+        else:  # local attention
+            # current fixated word index
+            aligned_position = cur_word_index
+            # Get window borders
+            left = torch.where(
+                aligned_position - self.window_width >= 0, aligned_position - self.window_width, 0
+            )
+            right = torch.where(
+                aligned_position + self.window_width <= self.cf["max_sn_len"] - 1,
+                aligned_position + self.window_width,
+                self.cf["max_sn_len"] - 1,
+            )
+
+            # exclude padding tokens
+            # only consider words in the window
+            sen_seq = (
+                torch.arange(self.cf["max_sn_len"])[None, :]
+                .expand(sn_mask.shape[0], self.cf["max_sn_len"])
+                .to(sn_mask.device)
+            )
+            outside_win_mask = (sen_seq < left.unsqueeze(1)) + (sen_seq > right.unsqueeze(1))
+
+            attn_prod += (~sn_mask + outside_win_mask).unsqueeze(1) * -1e9
+            att_weight = softmax(attn_prod, dim=2)  # [batch, 1, step]
+
+            if self.atten_type == "local-g":  # local attention with Gaussian Kernel
+                gauss = lambda s: torch.exp(
+                    -torch.square(s - aligned_position.unsqueeze(1))
+                    / (2 * torch.square(torch.tensor(self.window_width / 2)))
+                )
+                gauss_factor = gauss(sen_seq)
+                att_weight = att_weight * gauss_factor.unsqueeze(1)
+
+        return att_weight
+
+    def decode(
+        self, sp_emd, sn_mask, sp_pos, enc_out, sp_fix_dur, sp_landing_pos, word_ids_sp, sub_id
+    ):
+        # Fixation-Sequence Encoder + Decoder
+        # Initialize hidden state and cell state with zeros
+        hn = torch.zeros(8, sp_emd.shape[0], self.hidden_size).to(sp_emd.device)
+        hc = torch.zeros(8, sp_emd.shape[0], self.hidden_size).to(sp_emd.device)
+        hx, cx = hn[0, :, :], hc[0, :, :]
+        hx2, cx2 = hn[1, :, :], hc[1, :, :]
+        hx3, cx3 = hn[2, :, :], hc[2, :, :]
+        hx4, cx4 = hn[3, :, :], hc[3, :, :]
+        hx5, cx5 = hn[4, :, :], hc[4, :, :]
+        hx6, cx6 = hn[5, :, :], hc[5, :, :]
+        hx7, cx7 = hn[6, :, :], hc[6, :, :]
+        hx8, cx8 = hn[7, :, :], hc[7, :, :]
+
+        dec_emb_in = self.encoder.embeddings.word_embeddings(sp_emd[:, :-1])
+        if word_ids_sp is not None:
+            # Pool bert subword to word level for English corpus
+            sp_merged_word_emd, sp_mask_word = self.pool_subword_to_word(
+                dec_emb_in, word_ids_sp[:, :-1], target="sp", pool_method="sum"
+            )
+        else:  # no pooling for Chinese
+            sp_merged_word_emd, sp_mask_word = dec_emb_in, None
+
+        # add positional embeddings and layer normalization
+        position_embeddings = self.position_embeddings(sp_pos[:, :-1])
+        dec_emb_in = sp_merged_word_emd + position_embeddings
+        dec_emb_in = self.LayerNorm(dec_emb_in)
+        dec_emb_in = dec_emb_in.permute(1, 0, 2)  # [step, n, emb_dim]
+        dec_emb_in = self.embedding_dropout(dec_emb_in)
+
+        # concatenate two additional gaze features
+        if sp_landing_pos is not None:
+            dec_emb_in = torch.cat((dec_emb_in, sp_landing_pos.permute(1, 0)[:-1, :, None]), dim=2)
+
+        if sp_fix_dur is not None:
+            dec_emb_in = torch.cat((dec_emb_in, sp_fix_dur.permute(1, 0)[:-1, :, None]), dim=2)
+
+        # concatenate subject id for Eyettention_reader setting
+        if sub_id is not None:
+            dec_emb_in = torch.cat(
+                (dec_emb_in, self.sub_embeddings(sub_id).repeat(dec_emb_in.shape[0], 1, 1)), dim=2
+            )
+
+        # Predict output for each time step in turn
+        output = []
+        # save attention scores for visualization
+        atten_weights_batch = torch.empty(sp_emd.shape[0], 0, self.cf["max_sn_len"]).to(
+            sp_emd.device
+        )
+        for i in range(dec_emb_in.shape[0]):
+            hx, cx = self.decoder_cell1(dec_emb_in[i], (hx, cx))  # [batch, units]
+            hx2, cx2 = self.decoder_cell2(self.dropout_LSTM(hx), (hx2, cx2))
+            hx3, cx3 = self.decoder_cell3(self.dropout_LSTM(hx2), (hx3, cx3))
+            hx4, cx4 = self.decoder_cell4(self.dropout_LSTM(hx3), (hx4, cx4))
+            hx5, cx5 = self.decoder_cell5(self.dropout_LSTM(hx4), (hx5, cx5))
+            hx6, cx6 = self.decoder_cell6(self.dropout_LSTM(hx5), (hx6, cx6))
+            hx7, cx7 = self.decoder_cell7(self.dropout_LSTM(hx6), (hx7, cx7))
+            hx8, cx8 = self.decoder_cell8(self.dropout_LSTM(hx7), (hx8, cx8))
+
+            att_weight = self.cross_attention(
+                ht=hx8, hs=enc_out, sn_mask=sn_mask, cur_word_index=sp_pos[:, i]
+            )
+            atten_weights_batch = torch.cat([atten_weights_batch, att_weight], dim=1)
+
+            context = torch.matmul(att_weight, enc_out)  # [batch, 1, units]
+
+            # Decoder
+            hc = torch.cat([context.squeeze(1), hx8], dim=1)  # [batch, units *2]
+            result = self.decoder_dense(hc)  # [batch, dec_o_dim]
+            output.append(result)
+
+        output = torch.stack(output, dim=0)  # [step, batch, dec_o_dim]
+        # output = F.softmax(output, dim=2) # cross entropy in pytorch includes softmax
+        return output.permute(1, 0, 2), atten_weights_batch  # [batch, step, dec_o_dim]
+
+    def forward(
+        self,
+        sn_emd,
+        sn_mask,
+        sp_emd,
+        sp_pos,
+        word_ids_sn,
+        word_ids_sp,
+        sp_fix_dur,
+        sp_landing_pos,
+        sn_word_len,
+        sub_id,
+    ):
+        x, sn_mask_word = self.encode(
+            sn_emd, sn_mask, word_ids_sn, sn_word_len
+        )  # [batch, step, units], [batch, units]
+
+        if sn_mask_word is None:  # for Chinese dataset without token pooling
+            sn_mask = torch.Tensor.bool(sn_mask)
+            pred, atten_weights = self.decode(
+                sp_emd, sn_mask, sp_pos, x, sp_fix_dur, sp_landing_pos, word_ids_sp, sub_id
+            )  # [batch, step, dec_o_dim]
+
+        else:  # for English dataset with token pooling
+            pred, atten_weights = self.decode(
+                sp_emd, sn_mask_word, sp_pos, x, sp_fix_dur, sp_landing_pos, word_ids_sp, sub_id
+            )  # [batch, step, dec_o_dim]
+        return pred, atten_weights

raw_text_inference.py

@@ -0,0 +1,98 @@
+import torch
+from transformers import BertTokenizer, BertTokenizerFast
+
+from Eyettention.model import Eyettention
+from Eyettention.utils import (
+    build_bsc_config,
+    build_celer_config,
+    build_label_encoder,
+    text_to_bsc_inputs,
+    text_to_celer_inputs,
+)
+
+
+class EyettentionRawTextInference:
+    def __init__(self, checkpoint_path, dataset="BSC", cf=None, device="cpu"):
+        if cf is not None:
+            self.cf = cf
+        elif dataset == "BSC":
+            self.cf = build_bsc_config()
+        elif dataset == "celer":
+            self.cf = build_celer_config()
+        else:
+            raise ValueError(f"Unsupported dataset: {dataset}")
+
+        self.device = device
+        if self.cf["dataset"] == "celer":
+            self.tokenizer = BertTokenizerFast.from_pretrained(self.cf["model_pretrained"])
+        else:
+            self.tokenizer = BertTokenizer.from_pretrained(self.cf["model_pretrained"])
+        self.label_encoder = build_label_encoder(self.cf)
+
+        self.model = Eyettention(self.cf)
+        state_dict = torch.load(checkpoint_path, map_location=device)
+        if (
+            "encoder.embeddings.position_ids" not in self.model.state_dict()
+        ):  # Based on transformers version
+            state_dict.pop("encoder.embeddings.position_ids", None)
+        self.model.load_state_dict(state_dict)
+        self.model.to(device)
+        self.model.eval()
+
+    def generate_from_chinese_text(self, text, max_pred_len=None, previous_scanpath=None):
+        """Generate from raw Chinese text."""
+        if self.cf["dataset"] != "BSC":
+            raise ValueError("generate_from_chinese_text requires a BSC config.")
+        if max_pred_len is not None and max_pred_len <= 0:
+            raise ValueError("max_pred_len must be positive.")
+        if not isinstance(text, str):
+            raise TypeError("text must be a string.")
+        if not text.strip():
+            raise ValueError("text must not be empty.")
+
+        sn_input_ids, sn_mask, sn_word_len = text_to_bsc_inputs(
+            sn_str=text,
+            tokenizer=self.tokenizer,
+            cf=self.cf,
+            device=self.device,
+        )
+
+        with torch.no_grad():
+            return self.model.scanpath_generation(
+                sn_emd=sn_input_ids,
+                sn_mask=sn_mask,
+                word_ids_sn=None,
+                sn_word_len=sn_word_len,
+                le=self.label_encoder,
+                max_pred_len=max_pred_len or self.cf["max_pred_len"],
+                previous_scanpath=previous_scanpath,
+            )
+
+    def generate_from_english_text(self, text, max_pred_len=None, previous_scanpath=None):
+        """Generate from raw English text."""
+        if self.cf["dataset"] != "celer":
+            raise ValueError("generate_from_english_text requires a CELER config.")
+        if max_pred_len is not None and max_pred_len <= 0:
+            raise ValueError("max_pred_len must be positive.")
+        if not isinstance(text, str):
+            raise TypeError("text must be a string.")
+        if not text.strip():
+            raise ValueError("text must not be empty.")
+
+        sn_input_ids, sn_mask, word_ids_sn, sn_word_len = text_to_celer_inputs(
+            sn_str=text,
+            tokenizer=self.tokenizer,
+            cf=self.cf,
+            device=self.device,
+        )
+
+        with torch.no_grad():
+            return self.model.scanpath_generation(
+                sn_emd=sn_input_ids,
+                sn_mask=sn_mask,
+                word_ids_sn=word_ids_sn,
+                sn_word_len=sn_word_len,
+                le=self.label_encoder,
+                max_pred_len=max_pred_len or self.cf["max_pred_len"],
+                previous_scanpath=previous_scanpath,
+            )

requirements.txt

@@ -0,0 +1,10 @@
+matplotlib==3.8.4
+numpy==1.23.5
+pandas==1.5.3
+scikit_learn==1.6.1
+seaborn==0.12.2
+torch==2.4.1
+tqdm==4.66.4
+transformers==4.27.0
+paddlepaddle==2.4.2
+--find-links https://www.paddlepaddle.org.cn/whl/windows/mkl/avx/stable.html

results/BSC/Eyettention_chinese.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f5b992f0fa720f6f2d9c9d932b2062e2e9eeea4f5d3fd421dcd1bd68f14b059
+size 422488401

results/CELER/Eyettention_english.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d51b0dc3cedbb93519054d0c43a0c768b434a74b6774311458810e28293b4e50
+size 446653649

tests/test_imports.py

@@ -0,0 +1,83 @@
+import ast
+import sys
+import unittest
+from pathlib import Path
+
+EYETTENTION_ROOT = Path(__file__).resolve().parents[1]
+PROJECT_ROOT = EYETTENTION_ROOT.parent
+EXCLUDED_DIRS = {"__pycache__", ".git", "tests"}
+EXPECTED_IMPORT_FAILURES = {
+    ("Eyettention/app.py", "import gradio as gr"),
+}
+
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+
+class EyettentionImportTests(unittest.TestCase):
+    def test_import_lines(self):
+        failures = []
+
+        for file_path in _python_files(EYETTENTION_ROOT):
+            source = file_path.read_text()
+            tree = ast.parse(source, filename=str(file_path))
+
+            for node in ast.walk(tree):
+                if not isinstance(node, (ast.Import, ast.ImportFrom)):
+                    continue
+                if isinstance(node, ast.ImportFrom) and node.module == "__future__":
+                    continue
+
+                import_line = ast.get_source_segment(source, node)
+                try:
+                    exec(
+                        compile(import_line, str(file_path), "exec"),
+                        _import_globals(file_path),
+                    )
+                except Exception as exc:
+                    relative_path = str(file_path.relative_to(PROJECT_ROOT))
+                    if _is_expected_failure(relative_path, import_line):
+                        continue
+                    failures.append(
+                        f"{relative_path}:{node.lineno}\n"
+                        f"{import_line}\n"
+                        f"{type(exc).__name__}: {exc}"
+                    )
+
+        if failures:
+            self.fail("Failed import line(s):\n\n" + "\n\n".join(failures))
+
+
+def _python_files(root):
+    for file_path in root.rglob("*.py"):
+        if any(part in EXCLUDED_DIRS for part in file_path.parts):
+            continue
+        yield file_path
+
+
+def _import_globals(file_path):
+    module_path = file_path.relative_to(PROJECT_ROOT).with_suffix("")
+    module_parts = module_path.parts
+
+    if module_parts[-1] == "__init__":
+        module_name = ".".join(module_parts[:-1])
+        package = module_name
+    else:
+        module_name = ".".join(module_parts)
+        package = ".".join(module_parts[:-1])
+
+    return {
+        "__name__": module_name,
+        "__package__": package,
+    }
+
+
+def _is_expected_failure(relative_path, import_line):
+    return any(
+        relative_path == expected_path and import_line.startswith(expected_import)
+        for expected_path, expected_import in EXPECTED_IMPORT_FAILURES
+    )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_raw_text_inference.py

@@ -0,0 +1,198 @@
+import functools
+import sys
+import unittest
+import warnings
+from pathlib import Path
+
+import numpy as np
+import torch
+
+warnings.simplefilter("ignore")
+
+EYETTENTION_ROOT = Path(__file__).resolve().parents[1]
+PROJECT_ROOT = EYETTENTION_ROOT.parent
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+BSC_CHECKPOINT = EYETTENTION_ROOT / "results" / "BSC" / "Eyettention_chinese.pth"
+CELER_CHECKPOINT = EYETTENTION_ROOT / "results" / "CELER" / "Eyettention_english.pth"
+
+
+def _import_utils():
+    try:
+        from Eyettention.utils import (
+            build_bsc_config,
+            build_celer_config,
+            build_label_encoder,
+            text_to_bsc_inputs,
+            text_to_celer_inputs,
+        )
+    except ImportError as exc:
+        raise unittest.SkipTest(f"Eyettention utils dependencies are unavailable: {exc}") from exc
+
+    return (
+        build_bsc_config,
+        build_celer_config,
+        build_label_encoder,
+        text_to_bsc_inputs,
+        text_to_celer_inputs,
+    )
+
+
+def _load_tokenizer(tokenizer_cls, model_name):
+    try:
+        return tokenizer_cls.from_pretrained(model_name)
+    except OSError as exc:
+        raise unittest.SkipTest(
+            f"{model_name} is not available in the local Hugging Face cache"
+        ) from exc
+
+
+@functools.lru_cache(maxsize=None)
+def _raw_text_runner(dataset, checkpoint_path):
+    from Eyettention.raw_text_inference import EyettentionRawTextInference
+
+    if not Path(checkpoint_path).exists():
+        raise unittest.SkipTest(f"Missing checkpoint: {checkpoint_path}")
+
+    try:
+        return EyettentionRawTextInference(
+            checkpoint_path=str(checkpoint_path),
+            dataset=dataset,
+            device="cpu",
+        )
+    except OSError as exc:
+        raise unittest.SkipTest(f"Pretrained model for {dataset} is not available locally") from exc
+    except ImportError as exc:
+        raise unittest.SkipTest(f"Raw text dependency for {dataset} is unavailable: {exc}") from exc
+
+
+class ConfigAndInputTests(unittest.TestCase):
+
+    def test_text_to_bsc_inputs_returns_model_ready_tensors(self):
+        from transformers import BertTokenizer
+
+        build_bsc_config, _, _, text_to_bsc_inputs, _ = _import_utils()
+        cf = build_bsc_config(max_pred_len=5)
+        tokenizer = _load_tokenizer(BertTokenizer, cf["model_pretrained"])
+
+        sn_input_ids, sn_mask, sn_word_len = text_to_bsc_inputs(
+            "中国选手在男子滑雪比赛中有望蝉联冠军",
+            tokenizer,
+            cf,
+            device="cpu",
+        )
+
+        self.assertEqual(tuple(sn_input_ids.shape), (1, cf["max_sn_len"]))
+        self.assertEqual(tuple(sn_mask.shape), (1, cf["max_sn_len"]))
+        self.assertEqual(tuple(sn_word_len.shape), (1, cf["max_sn_len"]))
+        self.assertTrue(torch.is_floating_point(sn_mask))
+        self.assertTrue(torch.isfinite(sn_word_len).all())
+
+    def test_text_to_celer_inputs_returns_model_ready_tensors(self):
+        from transformers import BertTokenizerFast
+
+        _, build_celer_config, _, _, text_to_celer_inputs = _import_utils()
+        cf = build_celer_config(max_pred_len=5)
+        tokenizer = _load_tokenizer(BertTokenizerFast, cf["model_pretrained"])
+
+        sn_input_ids, sn_mask, word_ids_sn, sn_word_len = text_to_celer_inputs(
+            "He said BankEast's offer appears to be \"attractive to the bank's shareholders.\"",
+            tokenizer,
+            cf,
+            device="cpu",
+        )
+
+        self.assertEqual(tuple(sn_input_ids.shape), (1, cf["max_sn_token"]))
+        self.assertEqual(tuple(sn_mask.shape), (1, cf["max_sn_token"]))
+        self.assertEqual(tuple(word_ids_sn.shape), (1, cf["max_sn_token"]))
+        self.assertEqual(tuple(sn_word_len.shape), (1, cf["max_sn_len"]))
+        self.assertTrue(torch.is_floating_point(sn_mask))
+        self.assertTrue(torch.isfinite(sn_word_len).all())
+        self.assertGreaterEqual(np.nanmax(word_ids_sn.numpy()), 1)
+
+
+class RawTextInferenceSmokeTests(unittest.TestCase):
+    def test_chinese_raw_text_generation_smoke(self):
+        runner = _raw_text_runner("BSC", BSC_CHECKPOINT)
+        torch.manual_seed(0)
+
+        scanpath, density = runner.generate_from_chinese_text(
+            "中国选手在男子滑雪比赛中有望蝉联冠军",
+            max_pred_len=5,
+        )
+
+        self.assertEqual(tuple(scanpath.shape), (1, 5))
+        self.assertEqual(len(density), 4)
+        self.assertEqual(scanpath[0, 0].item(), 0)
+
+    def test_english_raw_text_generation_smoke(self):
+        runner = _raw_text_runner("celer", CELER_CHECKPOINT)
+        torch.manual_seed(0)
+
+        scanpath, density = runner.generate_from_english_text(
+            "The quick brown fox jumps.",
+            max_pred_len=5,
+        )
+
+        self.assertEqual(tuple(scanpath.shape), (1, 5))
+        self.assertEqual(len(density), 4)
+        self.assertEqual(scanpath[0, 0].item(), 0)
+
+    def test_previous_scanpath_is_replayed_before_sampling(self):
+        runner = _raw_text_runner("BSC", BSC_CHECKPOINT)
+        torch.manual_seed(0)
+
+        scanpath, _ = runner.generate_from_chinese_text(
+            "中国选手在男子滑雪比赛中有望蝉联冠军",
+            max_pred_len=3,
+            previous_scanpath=[0, 1, 2],
+        )
+
+        self.assertEqual(scanpath[0, :3].tolist(), [0, 1, 2])
+        self.assertEqual(scanpath.shape[1], 6)
+
+    def test_dataset_specific_methods_guard_against_wrong_config(self):
+        from Eyettention.raw_text_inference import EyettentionRawTextInference
+
+        bsc_runner = object.__new__(EyettentionRawTextInference)
+        bsc_runner.cf = {"dataset": "celer"}
+        with self.assertRaises(ValueError):
+            bsc_runner.generate_from_chinese_text("中国选手在男子滑雪比赛中有望蝉联冠军")
+
+        celer_runner = object.__new__(EyettentionRawTextInference)
+        celer_runner.cf = {"dataset": "BSC"}
+        with self.assertRaises(ValueError):
+            celer_runner.generate_from_english_text(
+                "He said BankEast's offer appears to be \"attractive to the bank's shareholders.\""
+            )
+
+    def test_text_is_none(self):
+        from Eyettention.raw_text_inference import EyettentionRawTextInference
+
+        bsc_runner = object.__new__(EyettentionRawTextInference)
+        bsc_runner.cf = {"dataset": "celer"}
+        with self.assertRaises(ValueError):
+            bsc_runner.generate_from_chinese_text(None)
+
+        celer_runner = object.__new__(EyettentionRawTextInference)
+        celer_runner.cf = {"dataset": "BSC"}
+        with self.assertRaises(ValueError):
+            celer_runner.generate_from_english_text(None)
+
+    def test_text_is_empty(self):
+        from Eyettention.raw_text_inference import EyettentionRawTextInference
+
+        bsc_runner = object.__new__(EyettentionRawTextInference)
+        bsc_runner.cf = {"dataset": "celer"}
+        with self.assertRaises(ValueError):
+            bsc_runner.generate_from_chinese_text(" ")
+
+        celer_runner = object.__new__(EyettentionRawTextInference)
+        celer_runner.cf = {"dataset": "BSC"}
+        with self.assertRaises(ValueError):
+            celer_runner.generate_from_english_text(" ")
+
+
+if __name__ == "__main__":
+    unittest.main()

utils.py

@@ -0,0 +1,716 @@
+# coding=utf-8
+import numpy as np
+import pandas as pd
+import os
+import seaborn as sns
+from typing import Tuple
+from torch.utils.data import Dataset, DataLoader
+from transformers import BertTokenizerFast, BertTokenizer, BertForTokenClassification
+import torch
+import torch.nn.functional as F
+import seaborn as sns
+import matplotlib.pyplot as plt
+from tqdm import tqdm
+import json
+from collections import Counter
+import torch.nn as nn
+from sklearn.preprocessing import LabelEncoder
+import pickle
+from LAC import LAC
+
+
+def build_bsc_config(
+    atten_type="local-g",
+    max_pred_len=60,
+    batch_size=256,
+    n_epochs=1000,
+    n_folds=5,
+):
+    return {
+        "model_pretrained": "bert-base-chinese",
+        "lr": 1e-3,
+        "max_grad_norm": 10,
+        "n_epochs": n_epochs,
+        "n_folds": n_folds,
+        "dataset": "BSC",
+        "atten_type": atten_type,
+        "batch_size": batch_size,
+        "max_sn_len": 27,
+        "max_sp_len": 40,
+        "norm_type": "z-score",
+        "earlystop_patience": 20,
+        "max_pred_len": max_pred_len,
+    }
+
+
+def build_celer_config(
+    atten_type="local-g",
+    max_pred_len=60,
+    batch_size=256,
+    n_epochs=1000,
+    n_folds=5,
+):
+    return {
+        "model_pretrained": "bert-base-cased",
+        "lr": 1e-3,
+        "max_grad_norm": 10,
+        "n_epochs": n_epochs,
+        "n_folds": n_folds,
+        "dataset": "celer",
+        "atten_type": atten_type,
+        "batch_size": batch_size,
+        "max_sn_len": 24,
+        "max_sn_token": 35,
+        "max_sp_len": 52,
+        "max_sp_token": 395,
+        "norm_type": "z-score",
+        "earlystop_patience": 20,
+        "max_pred_len": max_pred_len,
+    }
+
+
+def build_label_encoder(cf):
+    le = LabelEncoder()
+    le.fit(np.append(np.arange(-cf["max_sn_len"] + 3, cf["max_sn_len"] - 1), cf["max_sn_len"] - 1))
+    return le
+
+
+# Raw text to tokenised inputs for both BSC and CELER
+def text_to_bsc_inputs(sn_str, tokenizer, cf, device="cpu"):
+    lac = LAC(mode="seg")
+    tokens = tokenizer.encode_plus(
+        sn_str,
+        add_special_tokens=True,
+        max_length=cf["max_sn_len"],
+        padding="max_length",
+    )
+
+    sn_input_ids = torch.tensor([tokens["input_ids"]], device=device)
+    sn_mask = torch.tensor([tokens["attention_mask"]], dtype=torch.float32, device=device)
+
+    sn_word_len = compute_BSC_word_length_for_input(sn_str, lac)
+    sn_word_len = pad_seq_with_nan([sn_word_len], max_len=cf["max_sn_len"], dtype=np.float32)
+
+    # Load precomputed BSC statistics for feature normalization.
+    path = os.path.join(os.path.dirname(__file__), "Data", "feature_norm_BSC.pickle")
+    with open(path, "rb") as file_to_read:
+        loaded_dictionary = pickle.load(file_to_read)
+    sn_word_len_mean = loaded_dictionary["sn_word_len_mean"].numpy()
+    sn_word_len_std = loaded_dictionary["sn_word_len_std"].numpy()
+
+    sn_word_len = (sn_word_len - sn_word_len_mean) / sn_word_len_std
+    sn_word_len = np.nan_to_num(sn_word_len)
+    sn_word_len = torch.tensor(sn_word_len, dtype=torch.float32, device=device)
+
+    return sn_input_ids, sn_mask, sn_word_len
+
+
+def text_to_celer_inputs(sn_str, tokenizer, cf, device="cpu"):
+    text = sn_str.split()
+
+    tokens = tokenizer(
+        [tokenizer.cls_token, *text, tokenizer.sep_token],
+        add_special_tokens=False,
+        max_length=cf["max_sn_token"],
+        padding="max_length",
+        is_split_into_words=True,
+    )
+
+    sn_input_ids = torch.tensor([tokens["input_ids"]], device=device)
+    sn_mask = torch.tensor([tokens["attention_mask"]], dtype=torch.float32, device=device)
+
+    word_ids_sn = tokens.word_ids()
+    word_ids_sn = [val if val is not None else np.nan for val in word_ids_sn]
+    word_ids_sn = torch.tensor([word_ids_sn], dtype=torch.float32, device=device)
+
+    word_lengths = np.asarray([len(t) for t in text], dtype=np.float32)
+    sn_word_len = compute_word_length_celer(word_lengths)
+    sn_word_len = pad_seq_with_nan([sn_word_len], max_len=cf["max_sn_len"], dtype=np.float32)
+
+    # Load precomputed CELER statistics for feature normalization.
+    path = os.path.join(os.path.dirname(__file__), "Data", "feature_norm_celer.pickle")
+    with open(path, "rb") as file_to_read:
+        loaded_dictionary = pickle.load(file_to_read)
+    sn_word_len_mean = float(loaded_dictionary["sn_word_len_mean"])
+    sn_word_len_std = float(loaded_dictionary["sn_word_len_std"])
+
+    sn_word_len = (sn_word_len - sn_word_len_mean) / sn_word_len_std
+    sn_word_len = np.nan_to_num(sn_word_len)
+    sn_word_len = torch.tensor(sn_word_len, dtype=torch.float32, device=device)
+
+    return sn_input_ids, sn_mask, word_ids_sn, sn_word_len
+
+
+def load_bsc() -> Tuple[pd.DataFrame, ...]:
+    """
+    :return: word info dataframe, part-of-speech info, eye movements
+    """
+    bsc_path = "./Data/beijing-sentence-corpus/"
+    info_path = os.path.join(bsc_path, "BSC.Word.Info.v2.xlsx")
+    bsc_emd_path = os.path.join(bsc_path, "BSC.EMD/BSC.EMD.txt")
+    word_info_df = pd.read_excel(info_path, "word")
+    pos_info_df = pd.read_excel(info_path, header=None)
+    eyemovement_df = pd.read_csv(bsc_emd_path, delimiter="\t")
+    return word_info_df, pos_info_df, eyemovement_df
+
+
+def load_corpus(corpus, task=None):
+    if corpus == "BSC":
+        # load word data, POS data, EM data
+        word_info_df, pos_info_df, eyemovement_df = load_bsc()
+        return word_info_df, pos_info_df, eyemovement_df
+    elif corpus == "celer":
+        eyemovement_df = pd.read_csv("./Data/celer/data_v2.0/sent_fix.tsv", delimiter="\t")
+        eyemovement_df["CURRENT_FIX_INTEREST_AREA_LABEL"] = (
+            eyemovement_df.CURRENT_FIX_INTEREST_AREA_LABEL.replace("\t(.*)", "", regex=True)
+        )
+        word_info_df = pd.read_csv("./Data/celer/data_v2.0/sent_ia.tsv", delimiter="\t")
+        word_info_df["IA_LABEL"] = word_info_df.IA_LABEL.replace("\t(.*)", "", regex=True)
+        return word_info_df, None, eyemovement_df
+
+
+def compute_BSC_word_length(sn_df):
+    word_len = sn_df.LEN.values
+    wl_list = []
+    for wl in word_len:
+        wl_list.extend([wl] * wl)
+    arr = np.asarray(wl_list, dtype=np.float32)
+    # length of a punctuation is 0, plus an epsilon to avoid division output inf
+    arr[arr == 0] = 1 / (0 + 0.5)
+    arr[arr != 0] = 1 / (arr[arr != 0])
+    return arr
+
+
+def compute_BSC_word_length_for_input(sentence, lac):
+    word_string = lac.run(sentence)
+    # print(word_string)
+    word_len = [len(i) for i in word_string]
+    wl_list = []
+    for wl in word_len:
+        wl_list.extend([wl] * wl)
+    arr = np.asarray(wl_list, dtype=np.float32)
+    # length of a punctuation is 0, plus an epsilon to avoid division output inf
+    arr[arr == 0] = 1 / (0 + 0.5)
+    arr[arr != 0] = 1 / (arr[arr != 0])
+    return arr
+
+
+def pad_seq(seqs, max_len, pad_value, dtype=np.compat.long):
+    padded = np.full((len(seqs), max_len), fill_value=pad_value, dtype=dtype)
+    for i, seq in enumerate(seqs):
+        padded[i, 0] = 0
+        padded[i, 1 : (len(seq) + 1)] = seq
+        if pad_value != 0:
+            padded[i, len(seq) + 1] = pad_value - 1
+
+    return padded
+
+
+def pad_seq_with_nan(seqs, max_len, dtype=np.compat.long):
+    padded = np.full((len(seqs), max_len), fill_value=np.nan, dtype=dtype)
+    for i, seq in enumerate(seqs):
+        padded[i, 1 : (len(seq) + 1)] = seq
+    return padded
+
+
+def _process_BSC_corpus(sn_list, reader_list, word_info_df, eyemovement_df, tokenizer, cf):
+    """
+    SN_token_embedding   <CLS>, bla, bla, <SEP>
+    SP_token_embedding   <CLS>, bla, bla, <SEP>
+    SP_ordinal_pos 0, bla, bla, max_sp_len
+    SP_fix_dur     0, bla, bla, 0
+    SN_len         original sentence length without start and end tokens
+    """
+    SN_input_ids, SN_attention_mask, SN_WORD_len = [], [], []
+    SP_input_ids, SP_attention_mask = [], []
+    SP_ordinal_pos, SP_landing_pos, SP_fix_dur = [], [], []
+    sub_id_list = []
+    for sn_id in sn_list:
+        # print('sub_id:', sub_id)
+        # process sentence sequence
+        sn_df = eyemovement_df[eyemovement_df.sn == sn_id]
+        sn = word_info_df[word_info_df.SN == sn_id]
+        sn_str = "".join(sn.WORD.values)
+        sn_word_len = compute_BSC_word_length(sn)
+
+        # tokenization and padding
+        tokenizer.padding_side = "right"
+        tokens = tokenizer.encode_plus(
+            sn_str,
+            add_special_tokens=True,
+            truncation=True,
+            max_length=cf["max_sn_len"],
+            padding="max_length",
+            return_attention_mask=True,
+        )
+        encoded_sn = tokens["input_ids"]
+        mask_sn = tokens["attention_mask"]
+
+        # process fixation sequence
+        for sub_id in reader_list:
+            sub_df = sn_df[sn_df.id == sub_id]
+            if len(sub_df) == 0:
+                # no scanpath data found for the subject
+                continue
+
+            # last fixation go back to the first character with fl = 0 -- seems to be outlier point? remove
+            if sub_df.iloc[-1].wn == 1 and sub_df.iloc[-1].fl == 0:
+                sub_df = sub_df.iloc[:-1]
+
+            sp_word_pos, sp_fix_loc, sp_fix_dur = (
+                sub_df.wn.values,
+                sub_df.fl.values,
+                sub_df.dur.values,
+            )
+            sp_landing_pos_char = np.modf(sp_fix_loc)[0]
+            SP_landing_pos.append(sp_landing_pos_char)
+
+            # Convert word-based ordinal positions to token(character)-based ordinal positions
+            # When the fixated word index is less than 0, set it to 0
+            sp_fix_loc = np.where(sp_fix_loc < 0, 0, sp_fix_loc)
+            sp_ordinal_pos = [
+                np.sum(sn[sn.NW < value].LEN) + np.ceil(sp_fix_loc[count] + 1e-10)
+                for count, value in enumerate(sp_word_pos)
+            ]
+            SP_ordinal_pos.append(sp_ordinal_pos)
+            SP_fix_dur.append(sp_fix_dur)
+
+            # tokenization and padding for scanpath, i.e. fixated word sequence
+            sp_token = [sn_str[int(i - 1)] for i in sp_ordinal_pos]
+            sp_token_str = "[CLS]" + "".join(sp_token) + "[SEP]"
+            sp_tokens = tokenizer.encode_plus(
+                sp_token_str,
+                add_special_tokens=False,
+                truncation=True,
+                max_length=cf["max_sp_len"],
+                padding="max_length",
+                return_attention_mask=True,
+            )
+            encoded_sp = sp_tokens["input_ids"]
+            mask_sp = sp_tokens["attention_mask"]
+            SP_input_ids.append(encoded_sp)
+            SP_attention_mask.append(mask_sp)
+
+            # sentence information
+            SN_input_ids.append(encoded_sn)
+            SN_attention_mask.append(mask_sn)
+            SN_WORD_len.append(sn_word_len)
+            sub_id_list.append(sub_id)
+
+    # padding for batch computation
+    SP_ordinal_pos = pad_seq(SP_ordinal_pos, max_len=(cf["max_sp_len"]), pad_value=cf["max_sn_len"])
+    SP_fix_dur = pad_seq(SP_fix_dur, max_len=(cf["max_sp_len"]), pad_value=0)
+    SP_landing_pos = pad_seq(SP_landing_pos, cf["max_sp_len"], pad_value=0, dtype=np.float32)
+    SN_WORD_len = pad_seq_with_nan(SN_WORD_len, cf["max_sn_len"], dtype=np.float32)
+
+    # assign type
+    SN_input_ids = np.asarray(SN_input_ids, dtype=np.int64)
+    SN_attention_mask = np.asarray(SN_attention_mask, dtype=np.float32)
+    SP_input_ids = np.asarray(SP_input_ids, dtype=np.int64)
+    SP_attention_mask = np.asarray(SP_attention_mask, dtype=np.float32)
+    sub_id_list = np.asarray(sub_id_list, dtype=np.int64)
+
+    data = {
+        "SN_input_ids": SN_input_ids,
+        "SN_attention_mask": SN_attention_mask,
+        "SN_WORD_len": SN_WORD_len,
+        "SP_input_ids": SP_input_ids,
+        "SP_attention_mask": SP_attention_mask,
+        "SP_ordinal_pos": np.array(SP_ordinal_pos),
+        "SP_landing_pos": np.array(SP_landing_pos),
+        "SP_fix_dur": np.array(SP_fix_dur),
+        "sub_id": sub_id_list,
+    }
+
+    return data
+
+
+class BSCdataset(Dataset):
+    """Return BSC dataset."""
+
+    def __init__(self, word_info_df, eyemovement_df, cf, reader_list, sn_list, tokenizer):
+        self.data = _process_BSC_corpus(
+            sn_list, reader_list, word_info_df, eyemovement_df, tokenizer, cf
+        )
+
+    def __len__(self):
+        return len(self.data["SN_input_ids"])
+
+    def __getitem__(self, idx):
+        sample = {}
+        sample["sn_input_ids"] = self.data["SN_input_ids"][idx, :]
+        sample["sn_attention_mask"] = self.data["SN_attention_mask"][idx, :]
+        sample["sn_word_len"] = self.data["SN_WORD_len"][idx, :]
+
+        sample["sp_input_ids"] = self.data["SP_input_ids"][idx, :]
+        sample["sp_attention_mask"] = self.data["SP_attention_mask"][idx, :]
+
+        sample["sp_pos"] = self.data["SP_ordinal_pos"][idx, :]
+        sample["sp_fix_dur"] = self.data["SP_fix_dur"][idx, :]
+        sample["sp_landing_pos"] = self.data["SP_landing_pos"][idx, :]
+
+        sample["sub_id"] = self.data["sub_id"][idx]
+
+        return sample
+
+
+def calculate_mean_std(dataloader, feat_key, padding_value=0, scale=1):
+    # calculate mean
+    total_sum = 0
+    total_num = 0
+    for batchh in dataloader:
+        batchh.keys()
+        feat = batchh[feat_key] / scale
+        feat = torch.nan_to_num(feat)
+        total_num += len(feat.view(-1).nonzero())
+        total_sum += feat.sum()
+    feat_mean = total_sum / total_num
+    # calculate std
+    sum_of_squared_error = 0
+    for batchh in dataloader:
+        batchh.keys()
+        feat = batchh[feat_key] / scale
+        feat = torch.nan_to_num(feat)
+        mask = ~torch.eq(feat, padding_value)
+        sum_of_squared_error += (((feat - feat_mean).pow(2)) * mask).sum()
+    feat_std = torch.sqrt(sum_of_squared_error / total_num)
+    return feat_mean, feat_std
+
+
+def load_label(sp_pos, cf, labelencoder, device):
+    # prepare label and mask
+    pad_mask = torch.eq(sp_pos[:, 1:], cf["max_sn_len"])
+    end_mask = torch.eq(sp_pos[:, 1:], cf["max_sn_len"] - 1)
+    mask = pad_mask + end_mask
+    sac_amp = sp_pos[:, 1:] - sp_pos[:, :-1]
+    label = sp_pos[:, 1:] * mask + sac_amp * ~mask
+    label = (
+        torch.where(label > cf["max_sn_len"] - 1, cf["max_sn_len"] - 1, label)
+        .to("cpu")
+        .detach()
+        .numpy()
+    )
+    label = labelencoder.transform(label.reshape(-1)).reshape(label.shape[0], label.shape[1])
+    label = torch.from_numpy(label).long().to(device)
+    pad_mask = pad_mask.to(device)
+    return pad_mask, label
+
+
+def likelihood(pred, label, mask):
+    if torch.is_tensor(label):
+        label = label.detach().to("cpu").numpy()
+    if torch.is_tensor(mask):
+        mask = mask.detach().to("cpu").numpy()
+    # test
+    # res = F.nll_loss(torch.tensor(pred), torch.tensor(label))
+    label = one_hot_encode(label, pred.shape[1])
+    res = np.sum(np.multiply(pred, label), axis=1)
+    res = np.sum(res * ~mask) / np.sum(~mask)
+    return res
+
+
+def eval_log_llh(dnn_out, label, pad_mask):
+    res = []
+    dnn_out = np.log2(dnn_out + 1e-10)
+    # For each scanpath calculate the likelihood and then find the average
+    for sp_indx in range(dnn_out.shape[0]):
+        out = likelihood(dnn_out[sp_indx, :, :], label[sp_indx, :], pad_mask[sp_indx, :])
+        res.append(out)
+
+    return res
+
+
+def prepare_scanpath(sp_dnn, sn_len, sp_human, cf):
+    max_sp_len = sp_dnn.shape[1]
+    sp_human = sp_human.detach().to("cpu").numpy()
+
+    # stop_indx = [np.where(sp_dnn[i,:]==(sn_len[i]+1))[0][0] for i in range(sp_dnn.shape[0])]
+    # Find the number "sn_len+1" -> the end point
+    stop_indx = []
+    for i in range(sp_dnn.shape[0]):
+        stop = np.where(sp_dnn[i, :] == (sn_len[i] + 1))[0]
+        if (
+            len(stop) == 0
+        ):  # no end point can be find -> exceeds the maximum length of the generated scanpath
+            stop_indx.append(max_sp_len - 1)
+        else:
+            stop_indx.append(stop[0])
+
+    # Truncating data after the end point
+    sp_dnn_cut = [sp_dnn[i][: stop_indx[i] + 1] for i in range(sp_dnn.shape[0])]
+    # replace the last teminal number to cf["max_sn_len"]-1, keep the same as the human scanpath label
+    for i in range(len(sp_dnn_cut)):
+        sp_dnn_cut[i][-1] = cf["max_sn_len"] - 1
+
+    # process the human scanpath data, truncating data after the end point
+    stop_indx = [
+        np.where(sp_human[i, :] == cf["max_sn_len"] - 1)[0][0] for i in range(sp_human.shape[0])
+    ]
+    sp_human_cut = [sp_human[i][: stop_indx[i] + 1] for i in range(sp_human.shape[0])]
+    return sp_dnn_cut, sp_human_cut
+
+
+def celer_load_native_speaker():
+    sub_metadata_path = "./Data/celer/metadata.tsv"
+    sub_infor = pd.read_csv(sub_metadata_path, delimiter="\t")
+    native_sub_list = sub_infor[sub_infor.L1 == "English"].List.values
+    return native_sub_list.tolist()
+
+
+def compute_word_length_celer(arr):
+    # length of a punctuation is 0, plus an epsilon to avoid division output inf
+    arr = arr.astype("float64")
+    arr[arr == 0] = 1 / (0 + 0.5)
+    arr[arr != 0] = 1 / (arr[arr != 0])
+    return arr
+
+
+def _process_celer(sn_list, reader_list, word_info_df, eyemovement_df, tokenizer, cf):
+    """
+    SN_token_embedding   <CLS>, bla, bla, <SEP>
+    SP_token_embedding       <CLS>, bla, bla, <SEP>
+    SP_ordinal_pos 0, bla, bla, max_sp_len
+    SP_fix_dur     0, bla, bla, 0
+    """
+    SN_input_ids, SN_attention_mask, SN_WORD_len, WORD_ids_sn = [], [], [], []
+    SP_input_ids, SP_attention_mask, WORD_ids_sp = [], [], []
+    SP_ordinal_pos, SP_landing_pos, SP_fix_dur = [], [], []
+    sub_id_list = []
+    for sn_id in tqdm(sn_list):
+        # process sentence sequence
+        sn_df = eyemovement_df[eyemovement_df.sentenceid == sn_id]
+        # notice: Each sentence is recorded multiple times in file |word_info_df|.
+        sn = word_info_df[word_info_df.sentenceid == sn_id]
+        sn = sn[sn["list"] == sn.list.values.tolist()[0]]
+        # compute word length for each word
+        sn_word_len = compute_word_length_celer(sn.WORD_LEN.values)
+
+        sn_str = sn.sentence.iloc[-1]
+        # nessacery sanity check, when split sentence to words, the length of sentence should match the sentence length recorded in celer dataset
+        if sn_id in [
+            "1987/w7_019/w7_019.295-3",
+            "1987/w7_036/w7_036.147-43",
+            "1987/w7_091/w7_091.360-6",
+        ]:
+            # extra inverted commas at the end of the sentence
+            sn_str = sn_str[:-3] + sn_str[-1:]
+        if sn_id == "1987/w7_085/w7_085.200-18":
+            sn_str = sn_str[:43] + sn_str[44:]
+        sn_len = len(sn_str.split())
+
+        # tokenization and padding
+        tokenizer.padding_side = "right"
+        sn_str = "[CLS]" + " " + sn_str + " " + "[SEP]"
+        # pre-tokenized input
+        tokens = tokenizer.encode_plus(
+            sn_str.split(),
+            add_special_tokens=False,
+            truncation=False,
+            max_length=cf["max_sn_token"],
+            padding="max_length",
+            return_attention_mask=True,
+            is_split_into_words=True,
+        )
+        encoded_sn = tokens["input_ids"]
+        mask_sn = tokens["attention_mask"]
+        # use offset mapping to determine if two tokens are in the same word.
+        # index start from 0, CLS -> 0 and SEP -> last index
+        word_ids_sn = tokens.word_ids()
+        word_ids_sn = [val if val is not None else np.nan for val in word_ids_sn]
+
+        # process fixation sequence
+        for sub_id in reader_list:
+            sub_df = sn_df[sn_df.list == sub_id]
+            # remove fixations on non-words
+            sub_df = sub_df.loc[sub_df.CURRENT_FIX_INTEREST_AREA_LABEL != "."]
+            if len(sub_df) == 0:
+                # no scanpath data found for the subject
+                continue
+
+            # prepare decoder input and output
+            sp_word_pos, sp_fix_loc, sp_fix_dur = (
+                sub_df.CURRENT_FIX_INTEREST_AREA_ID.values,
+                sub_df.CURRENT_FIX_NEAREST_INTEREST_AREA_DISTANCE.values,
+                sub_df.CURRENT_FIX_DURATION.values,
+            )
+
+            # dataset is noisy -> sanity check
+            # 1) check if recorded fixation duration are within reasonable limits
+            # Less than 50ms attempt to merge with neighbouring fixation if fixate is on the same word, otherwise delete
+            outlier_indx = np.where(sp_fix_dur < 50)[0]
+            if outlier_indx.size > 0:
+                for out_idx in range(len(outlier_indx)):
+                    outlier_i = outlier_indx[out_idx]
+                    merge_flag = False
+
+                    # outliers are commonly found in the fixation of the last record and the first record, and are removed directly
+                    if outlier_i == len(sp_fix_dur) - 1 or outlier_i == 0:
+                        merge_flag = True
+
+                    else:
+                        if outlier_i - 1 >= 0 and merge_flag == False:
+                            # try to merge with the left fixation
+                            if (
+                                sub_df.iloc[outlier_i].CURRENT_FIX_INTEREST_AREA_LABEL
+                                == sub_df.iloc[outlier_i - 1].CURRENT_FIX_INTEREST_AREA_LABEL
+                            ):
+                                sp_fix_dur[outlier_i - 1] = (
+                                    sp_fix_dur[outlier_i - 1] + sp_fix_dur[outlier_i]
+                                )
+                                merge_flag = True
+
+                        if outlier_i + 1 < len(sp_fix_dur) and merge_flag == False:
+                            # try to merge with the right fixation
+                            if (
+                                sub_df.iloc[outlier_i].CURRENT_FIX_INTEREST_AREA_LABEL
+                                == sub_df.iloc[outlier_i + 1].CURRENT_FIX_INTEREST_AREA_LABEL
+                            ):
+                                sp_fix_dur[outlier_i + 1] = (
+                                    sp_fix_dur[outlier_i + 1] + sp_fix_dur[outlier_i]
+                                )
+                                merge_flag = True
+
+                    sp_word_pos = np.delete(sp_word_pos, outlier_i)
+                    sp_fix_loc = np.delete(sp_fix_loc, outlier_i)
+                    sp_fix_dur = np.delete(sp_fix_dur, outlier_i)
+                    sub_df.drop(sub_df.index[outlier_i], axis=0, inplace=True)
+                    outlier_indx = outlier_indx - 1
+
+            # 2) scanpath too long, remove outliers, speed up the inference
+            if len(sp_word_pos) > 50:  # 72/10684
+                continue
+            # 3)scanpath too short for a normal length sentence
+            if len(sp_word_pos) <= 1 and sn_len > 10:
+                continue
+
+            # 4) check landing position feature
+            # assign missing value to 'nan'
+            sp_fix_loc = np.where(sp_fix_loc == ".", np.nan, sp_fix_loc)
+            # convert string of number of float type
+            sp_fix_loc = [float(i) for i in sp_fix_loc]
+
+            # Outliers in calculated landing positions due to lack of valid AOI data, assign to 'nan'
+            if np.nanmax(sp_fix_loc) > 35:
+                missing_idx = np.where(np.array(sp_fix_loc) > 5)[0]
+                for miss in missing_idx:
+                    if sub_df.iloc[miss].CURRENT_FIX_INTEREST_AREA_LEFT in [
+                        "NONE",
+                        "BEFORE",
+                        "AFTER",
+                        "BOTH",
+                    ]:
+                        sp_fix_loc[miss] = np.nan
+                    else:
+                        print(
+                            "Landing position calculation error. Unknown cause, needs to be checked"
+                        )
+
+            sp_ordinal_pos = sp_word_pos.astype(int)
+            SP_ordinal_pos.append(sp_ordinal_pos)
+            SP_fix_dur.append(sp_fix_dur)
+            SP_landing_pos.append(sp_fix_loc)
+
+            sp_token = [sn_str.split()[int(i)] for i in sp_ordinal_pos]
+            sp_token_str = "[CLS]" + " " + " ".join(sp_token) + " " + "[SEP]"
+
+            # tokenization and padding for scanpath, i.e. fixated word sequence
+            sp_tokens = tokenizer.encode_plus(
+                sp_token_str.split(),
+                add_special_tokens=False,
+                truncation=False,
+                max_length=cf["max_sp_token"],
+                padding="max_length",
+                return_attention_mask=True,
+                is_split_into_words=True,
+            )
+            encoded_sp = sp_tokens["input_ids"]
+            mask_sp = sp_tokens["attention_mask"]
+            # index start from 0, CLS -> 0 and SEP -> last index
+            word_ids_sp = sp_tokens.word_ids()
+            word_ids_sp = [val if val is not None else np.nan for val in word_ids_sp]
+            SP_input_ids.append(encoded_sp)
+            SP_attention_mask.append(mask_sp)
+            WORD_ids_sp.append(word_ids_sp)
+
+            # sentence information
+            SN_input_ids.append(encoded_sn)
+            SN_attention_mask.append(mask_sn)
+            SN_WORD_len.append(sn_word_len)
+            WORD_ids_sn.append(word_ids_sn)
+            sub_id_list.append(int(sub_id))
+
+    # padding for batch computation
+    SP_ordinal_pos = pad_seq(SP_ordinal_pos, max_len=(cf["max_sp_len"]), pad_value=cf["max_sn_len"])
+    SP_fix_dur = pad_seq(SP_fix_dur, max_len=(cf["max_sp_len"]), pad_value=0)
+    SP_landing_pos = pad_seq(SP_landing_pos, cf["max_sp_len"], pad_value=0, dtype=np.float32)
+    SN_WORD_len = pad_seq_with_nan(SN_WORD_len, cf["max_sn_len"], dtype=np.float32)
+
+    # assign type
+    SN_input_ids = np.asarray(SN_input_ids, dtype=np.int64)
+    SN_attention_mask = np.asarray(SN_attention_mask, dtype=np.float32)
+    SP_input_ids = np.asarray(SP_input_ids, dtype=np.int64)
+    SP_attention_mask = np.asarray(SP_attention_mask, dtype=np.float32)
+    sub_id_list = np.asarray(sub_id_list, dtype=np.int64)
+    WORD_ids_sn = np.asarray(WORD_ids_sn)
+    WORD_ids_sp = np.asarray(WORD_ids_sp)
+
+    data = {
+        "SN_input_ids": SN_input_ids,
+        "SN_attention_mask": SN_attention_mask,
+        "SN_WORD_len": SN_WORD_len,
+        "WORD_ids_sn": WORD_ids_sn,
+        "SP_input_ids": SP_input_ids,
+        "SP_attention_mask": SP_attention_mask,
+        "WORD_ids_sp": WORD_ids_sp,
+        "SP_ordinal_pos": np.array(SP_ordinal_pos),
+        "SP_landing_pos": np.array(SP_landing_pos),
+        "SP_fix_dur": np.array(SP_fix_dur),
+        "sub_id": sub_id_list,
+    }
+
+    return data
+
+
+class celerdataset(Dataset):
+    """Return celer dataset."""
+
+    def __init__(self, word_info_df, eyemovement_df, cf, reader_list, sn_list, tokenizer):
+
+        self.data = _process_celer(
+            sn_list, reader_list, word_info_df, eyemovement_df, tokenizer, cf
+        )
+
+    def __len__(self):
+        return len(self.data["SN_input_ids"])
+
+    def __getitem__(self, idx):
+        sample = {}
+        sample["sn_input_ids"] = self.data["SN_input_ids"][idx, :]
+        sample["sn_attention_mask"] = self.data["SN_attention_mask"][idx, :]
+        sample["sn_word_len"] = self.data["SN_WORD_len"][idx, :]
+        sample["word_ids_sn"] = self.data["WORD_ids_sn"][idx, :]
+
+        sample["sp_input_ids"] = self.data["SP_input_ids"][idx, :]
+        sample["sp_attention_mask"] = self.data["SP_attention_mask"][idx, :]
+        sample["word_ids_sp"] = self.data["WORD_ids_sp"][idx, :]
+
+        sample["sp_pos"] = self.data["SP_ordinal_pos"][idx, :]
+        sample["sp_fix_dur"] = self.data["SP_fix_dur"][idx, :]
+        sample["sp_landing_pos"] = self.data["SP_landing_pos"][idx, :]
+
+        sample["sub_id"] = self.data["sub_id"][idx]
+
+        return sample
+
+
+def one_hot_encode(arr, dim):
+    # one hot encode
+    onehot_encoded = np.zeros((arr.shape[0], dim))
+    for idx, value in enumerate(arr):
+        onehot_encoded[idx, value] = 1
+
+    return onehot_encoded
+
+
+def gradient_clipping(dnn_model, clip=10):
+    torch.nn.utils.clip_grad_norm_(dnn_model.parameters(), clip)