diff --git a/fixation_correction_test.ipynb b/fixation_correction_test.ipynb new file mode 100644 index 00000000..63c269af --- /dev/null +++ b/fixation_correction_test.ipynb @@ -0,0 +1,384 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Fixation correction\n", + "\n", + "The code tests algorithms from `preprocessing.events.fixation_correction.py`\n" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "### Preparation - the same as in the `preprocessing.ipynb`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "# from preprocessing.data_collection.multipleye_data_collection import prepare_language_folder\n", + "from preprocessing.data_collection.multipleye_data_collection import (\n", + " MultipleyeDataCollection,\n", + ")\n", + "from pathlib import Path\n", + "\n", + "import preprocessing\n", + "\n", + "# the config will be loaded into general constants module, so we can access all settings at the same place\n", + "from preprocessing import constants\n", + "from preprocessing.scripts.prepare_language_folder import prepare_language_folder\n", + "\n", + "\n", + "import polars as pl" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "# get the data collection name from the config and create the path to the data folder\n", + "this_repo = Path().resolve()\n", + "data_collection_name = constants.DATA_COLLECTION_NAME\n", + "data_folder_path = this_repo / \"data\" / data_collection_name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# run the preparation function to prepare the language folder structure\n", + "prepare_language_folder(data_collection_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "multipleye = MultipleyeDataCollection.create_from_data_folder(\n", + " data_folder_path,\n", + " include_pilots=constants.INCLUDE_PILOTS,\n", + " excluded_sessions=constants.EXCLUDE_SESSIONS,\n", + " included_sessions=constants.INCLUDE_SESSIONS,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "multipleye.convert_edf_to_asc()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "multipleye.prepare_session_level_information()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "# print an overview on the data collection and the sessions\n", + "multipleye" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### Choosing the first session to work with" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "# pick only one session as an example to work with in the next steps\n", + "sessions = [s for s in multipleye]\n", + "sess = sessions[0]\n", + "idf = sess.session_identifier" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "# get the path to the .asc file for the session\n", + "asc = sess.asc_path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "# load gaze data\n", + "gaze = preprocessing.load_gaze_data(\n", + " asc_file=asc,\n", + " lab_config=sess.lab_config,\n", + " session_idf=idf,\n", + " trial_cols=constants.TRIAL_COLS,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "preprocessing.preprocess_gaze(gaze)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "preprocessing.detect_fixations(\n", + " gaze,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "sess.stimuli" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "gaze" + ] + }, + { + "cell_type": "markdown", + "id": "17", + "metadata": {}, + "source": [ + "## And now it begins with the fixation correction!" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "### Add \"fixation_correction_{algorithm}\" events to all stimuli and pages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "\n", + "import preprocessing.events.fixation_correction as fc\n", + "\n", + "importlib.reload(fc)\n", + "\n", + "algorithm = \"chain\"\n", + "\n", + "### Necessary to remove previous corrections if you want to add new ones,\n", + "# otherwise you will get an error because the function is designed\n", + "# to not allow multiple corrections with the same algorithm in the events\n", + "# dataframe to avoid confusion and mistakes.\n", + "fc.remove_previous_fixation_corrections(gaze.events, algorithm)\n", + "\n", + "### Add the corrected fixations to the events dataframe\n", + "fc.add_corrected_fixations(sess, gaze.events, algorithm, verbose=1)" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "## Visualisation for testing purposes - requires opencv-python and matplotlib!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "#!pip install opencv-python\n", + "#!pip install matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "import cv2\n", + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "def compare_events(stimulus, page, events, event1, event2):\n", + " stimulus_name = f\"{stimulus.name}_{stimulus.id}\"\n", + " page_name = f\"page_{page.number}\"\n", + "\n", + " fix1 = events.frame.filter(\n", + " (pl.col(\"page\") == f\"{page_name}\")\n", + " & (pl.col(\"stimulus\") == f\"{stimulus_name}\")\n", + " & (pl.col(\"name\") == f\"{event1}\")\n", + " )\n", + "\n", + " fix2 = events.frame.filter(\n", + " (pl.col(\"page\") == f\"{page_name}\")\n", + " & (pl.col(\"stimulus\") == f\"{stimulus_name}\")\n", + " & (pl.col(\"name\") == f\"{event2}\")\n", + " )\n", + "\n", + " if len(fix1) != len(fix2):\n", + " raise ValueError(\n", + " f\"Number of events in {event1} and {event2} do not match, cannot compare them. Number of events in {event1}: {len(fix1)}, number of events in {event2}: {len(fix2)}\"\n", + " )\n", + "\n", + " image = cv2.imread(stimulus.pages[page.number - 1].aoi_image_path)\n", + " org_coords = []\n", + " previous_coords = [-1, -1]\n", + " for i, p in enumerate(fix1.iter_rows(named=True)):\n", + " x = int(p[\"location\"][0])\n", + " y = int(p[\"location\"][1])\n", + " cv2.circle(image, (x, y), radius=5, color=(255, 0, 0), thickness=-1)\n", + " if previous_coords != [-1, -1]:\n", + " cv2.line(\n", + " image,\n", + " (x, y),\n", + " (previous_coords[0], previous_coords[1]),\n", + " (255, 0, 0),\n", + " thickness=1,\n", + " )\n", + " previous_coords = [x, y]\n", + " org_coords.append((x, y))\n", + "\n", + " previous_coords = [-1, -1]\n", + " for i, p in enumerate(fix2.iter_rows(named=True)):\n", + " x = int(p[\"location\"][0])\n", + " y = int(p[\"location\"][1])\n", + " cv2.circle(image, (x, y), radius=5, color=(0, 0, 255), thickness=-1)\n", + " if previous_coords != [-1, -1]:\n", + " cv2.line(\n", + " image,\n", + " (x, y),\n", + " (previous_coords[0], previous_coords[1]),\n", + " (0, 0, 255),\n", + " thickness=1,\n", + " )\n", + " previous_coords = [x, y]\n", + " cv2.line(\n", + " image, (x, y), (org_coords[i][0], org_coords[i][1]), (0, 0, 0), thickness=1\n", + " )\n", + "\n", + " show_image(\n", + " image,\n", + " title=f\"{stimulus.name}_{stimulus.id} page: {page_name} 'fixation' vs. 'fixation_corrected_{algorithm}'\",\n", + " )\n", + "\n", + "\n", + "def show_image(image, title=\"image\"):\n", + " # print(image.shape)\n", + " plt.figure(figsize=(10, 10))\n", + " plt.imshow(image)\n", + " plt.title(title)\n", + " plt.axis(\"off\")\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "algorithm = \"chain\"\n", + "for stimulus in sess.stimuli:\n", + " for page in stimulus.pages:\n", + " compare_events(\n", + " stimulus, page, gaze.events, \"fixation\", f\"fixation_corrected_{algorithm}\"\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pymov2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/preprocessing/api.py b/preprocessing/api.py index 59464ba4..c04bc5e9 100644 --- a/preprocessing/api.py +++ b/preprocessing/api.py @@ -4,6 +4,7 @@ from .signals.preprocess import preprocess_gaze from .events.properties import compute_event_properties from .events.detect import detect_fixations, detect_saccades +from .events.fixation_correction import correct_fixations from .mapping.aoi import map_fixations_to_aois from .io.save import ( save_raw_data, @@ -50,4 +51,5 @@ "parse_answers_from_logfile", "collect_session_answers", "run_preflight_check", + "correct_fixations", ] diff --git a/preprocessing/config.py b/preprocessing/config.py index ef013e69..81065c7f 100644 --- a/preprocessing/config.py +++ b/preprocessing/config.py @@ -436,6 +436,8 @@ def _init_defaults(self) -> None: self.RUN_SANITY_CHECKS = True #: Whether to process psychometric tests. self.RUN_PSYCHOMETRIC_TESTS = True + #: Whether to correct fixations for line drift. + self.RUN_FIXATION_CORRECTION = True #: Column name for the trial identifier. self.TRIAL_COL = "trial" diff --git a/preprocessing/data_collection/multipleye_data_collection.py b/preprocessing/data_collection/multipleye_data_collection.py index 3536f493..9443a602 100644 --- a/preprocessing/data_collection/multipleye_data_collection.py +++ b/preprocessing/data_collection/multipleye_data_collection.py @@ -104,6 +104,7 @@ def __init__( self.include_pilots = kwargs.get("include_pilots", False) self.reports_dir = kwargs.get("output_dir", "") self.pilot_folder = kwargs.get("pilot_folder", "") + self.enlarge_aois = kwargs.get("enlarge_aois", True) for short_name, long_name in settings.EYETRACKER_NAMES.items(): if eye_tracker in long_name: @@ -400,6 +401,7 @@ def create_from_data_folder( include_pilots: bool = False, excluded_sessions: list[str] | None = None, included_sessions: list[str] | None = None, + enlarge_aois: bool = True, ) -> "MultipleyeDataCollection": """ :param data_dir: str path to the data folder @@ -502,6 +504,7 @@ def create_from_data_folder( ps_tests_path=ps_tests_path, included_sessions=included_sessions, excluded_sessions=excluded_sessions, + enlarge_aois=enlarge_aois, ) def create_sanity_check_report( @@ -782,6 +785,7 @@ def _load_session_stimuli( stimulus_name, stimulus_order_version, trial_id[0], + enlarge_aois=self.enlarge_aois, ) stimuli.append(stimulus) diff --git a/preprocessing/data_collection/stimulus.py b/preprocessing/data_collection/stimulus.py index ac2a4f92..b0f1c2d0 100644 --- a/preprocessing/data_collection/stimulus.py +++ b/preprocessing/data_collection/stimulus.py @@ -12,7 +12,7 @@ from ..config import settings from ..utils.data_path_utils import _ci_resolve -from ..mapping.aoi import enlarge_aois +from ..mapping.aoi import enlarge_aois as _enlarge_aois from ..utils.logging import get_logger logger = get_logger() @@ -108,6 +108,7 @@ def load( stimulus_name: str, question_version: int, trial: str, + enlarge_aois: bool = True, ) -> "Stimulus": # assert stimulus_name in NAMES, f"{stimulus_name!r} is not a valid stimulus name" stimulus_df_path = _ci_resolve( @@ -166,7 +167,8 @@ def load( ) aois_df = pl.read_csv(aoi_path) - aois_df = enlarge_aois(aois_df) + if enlarge_aois: + aois_df = _enlarge_aois(aois_df) text_stimulus = TextStimulus( aois_df, diff --git a/preprocessing/events/drift_algorithms.py b/preprocessing/events/drift_algorithms.py new file mode 100644 index 00000000..e144c3fe --- /dev/null +++ b/preprocessing/events/drift_algorithms.py @@ -0,0 +1,512 @@ +""" +Algorithms for drift correction in gaze data recorded during reading. + +Algorithms described here: +Carr, J. W., Pescuma, V. N., Furlan, M., Ktori, M., & Crepaldi, D. (2022). +Algorithms for the automated correction of vertical drift in eye-tracking data. +Behavior Research Methods, 54(1), 287-310. +https://doi.org/10.3758/s13428-021-01554-0 + +Code taken from publication: +Al Madi, N. (2025). +Identifying Eye Movement Patterns for An Adaptive Approach to Correcting Eye Tracking Data +in Reading Tasks. Proceedings of the ACM on Human-Computer Interaction, 9(3), 1-16. + +Available here: +https://osf.io/khrqp/overview + +Parameters: + +fixation_XY - list of original fixation locations +line_Y - list of vertical coordinates of lines of text (AOIs) + +""" + +import numpy as np +from scipy.optimize import minimize +from scipy.stats import norm +from sklearn.cluster import KMeans + +###################################################################### +# ATTACH +###################################################################### + + +def attach(fixation_XY, line_Y): + n = len(fixation_XY) + for fixation_i in range(n): + line_i = np.argmin(abs(line_Y - fixation_XY[fixation_i, 1])) + fixation_XY[fixation_i, 1] = line_Y[line_i] + return fixation_XY + + +###################################################################### +# CHAIN +# +# https://github.com/sascha2schroeder/popEye/ +###################################################################### + + +def chain(fixation_XY, line_Y, x_thresh=192, y_thresh=32): + n = len(fixation_XY) + dist_X = abs(np.diff(fixation_XY[:, 0])) + dist_Y = abs(np.diff(fixation_XY[:, 1])) + end_chain_indices = list( + np.where(np.logical_or(dist_X > x_thresh, dist_Y > y_thresh))[0] + 1 + ) + end_chain_indices.append(n) + start_of_chain = 0 + for end_of_chain in end_chain_indices: + mean_y = np.mean(fixation_XY[start_of_chain:end_of_chain, 1]) + line_i = np.argmin(abs(line_Y - mean_y)) + fixation_XY[start_of_chain:end_of_chain, 1] = line_Y[line_i] + start_of_chain = end_of_chain + return fixation_XY + + +###################################################################### +# CLUSTER +# +# https://github.com/sascha2schroeder/popEye/ +###################################################################### + + +def cluster(fixation_XY, line_Y): + m = len(line_Y) + fixation_Y = fixation_XY[:, 1].reshape(-1, 1) + clusters = KMeans(m, n_init=100, max_iter=300).fit_predict(fixation_Y) + centers = [fixation_Y[clusters == i].mean() for i in range(m)] + ordered_cluster_indices = np.argsort(centers) + for fixation_i, cluster_i in enumerate(clusters): + line_i = np.where(ordered_cluster_indices == cluster_i)[0][0] + fixation_XY[fixation_i, 1] = line_Y[line_i] + return fixation_XY + + +###################################################################### +# COMPARE +# +# Lima Sanches, C., Kise, K., & Augereau, O. (2015). Eye gaze and text +# line matching for reading analysis. In Adjunct proceedings of the +# 2015 ACM International Joint Conference on Pervasive and +# Ubiquitous Computing and proceedings of the 2015 ACM International +# Symposium on Wearable Computers (pp. 1227–1233). Association for +# Computing Machinery. +# +# https://doi.org/10.1145/2800835.2807936 +###################################################################### + + +def compare(fixation_XY, word_XY, x_thresh=512, n_nearest_lines=3): + line_Y = np.unique(word_XY[:, 1]) + n = len(fixation_XY) + diff_X = np.diff(fixation_XY[:, 0]) + end_line_indices = list(np.where(diff_X < -x_thresh)[0] + 1) + end_line_indices.append(n) + start_of_line = 0 + for end_of_line in end_line_indices: + gaze_line = fixation_XY[start_of_line:end_of_line] + mean_y = np.mean(gaze_line[:, 1]) + lines_ordered_by_proximity = np.argsort(abs(line_Y - mean_y)) + nearest_line_I = lines_ordered_by_proximity[:n_nearest_lines] + line_costs = np.zeros(n_nearest_lines) + for candidate_i in range(n_nearest_lines): + candidate_line_i = nearest_line_I[candidate_i] + text_line = word_XY[word_XY[:, 1] == line_Y[candidate_line_i]] + dtw_cost, _ = dynamic_time_warping(gaze_line[:, 0:1], text_line[:, 0:1]) + line_costs[candidate_i] = dtw_cost + line_i = nearest_line_I[np.argmin(line_costs)] + fixation_XY[start_of_line:end_of_line, 1] = line_Y[line_i] + start_of_line = end_of_line + return fixation_XY + + +###################################################################### +# MERGE +# +# Špakov, O., Istance, H., Hyrskykari, A., Siirtola, H., & Räihä, +# K.-J. (2019). Improving the performance of eye trackers with +# limited spatial accuracy and low sampling rates for reading +# analysis by heuristic fixation-to-word mapping. Behavior Research +# Methods, 51(6), 2661–2687. +# +# https://doi.org/10.3758/s13428-018-1120-x +# https://github.com/uta-gasp/sgwm +###################################################################### + +phases = [ + {"min_i": 3, "min_j": 3, "no_constraints": False}, # Phase 1 + {"min_i": 1, "min_j": 3, "no_constraints": False}, # Phase 2 + {"min_i": 1, "min_j": 1, "no_constraints": False}, # Phase 3 + {"min_i": 1, "min_j": 1, "no_constraints": True}, +] # Phase 4 + + +def merge(fixation_XY, line_Y, y_thresh=32, g_thresh=0.1, e_thresh=20): + n = len(fixation_XY) + m = len(line_Y) + diff_X = np.diff(fixation_XY[:, 0]) + dist_Y = abs(np.diff(fixation_XY[:, 1])) + sequence_boundaries = list( + np.where(np.logical_or(diff_X < 0, dist_Y > y_thresh))[0] + 1 + ) + sequence_starts = [0] + sequence_boundaries + sequence_ends = sequence_boundaries + [n] + sequences = [ + list(range(start, end)) for start, end in zip(sequence_starts, sequence_ends) + ] + for phase in phases: + while len(sequences) > m: + best_merger = None + best_error = np.inf + for i in range(len(sequences) - 1): + if len(sequences[i]) < phase["min_i"]: + continue # first sequence too short, skip to next i + for j in range(i + 1, len(sequences)): + if len(sequences[j]) < phase["min_j"]: + continue # second sequence too short, skip to next j + candidate_XY = fixation_XY[sequences[i] + sequences[j]] + gradient, intercept = np.polyfit( + candidate_XY[:, 0], candidate_XY[:, 1], 1 + ) + residuals = candidate_XY[:, 1] - ( + gradient * candidate_XY[:, 0] + intercept + ) + error = np.sqrt(sum(residuals**2) / len(candidate_XY)) + if phase["no_constraints"] or ( + abs(gradient) < g_thresh and error < e_thresh + ): + if error < best_error: + best_merger = (i, j) + best_error = error + if best_merger is None: + break # no possible mergers, break while and move to next phase + merge_i, merge_j = best_merger + merged_sequence = sequences[merge_i] + sequences[merge_j] + sequences.append(merged_sequence) + del sequences[merge_j], sequences[merge_i] + mean_Y = [fixation_XY[sequence, 1].mean() for sequence in sequences] + ordered_sequence_indices = np.argsort(mean_Y) + for line_i, sequence_i in enumerate(ordered_sequence_indices): + fixation_XY[sequences[sequence_i], 1] = line_Y[line_i] + return fixation_XY + + +###################################################################### +# REGRESS +# +# Cohen, A. L. (2013). Software for the automatic correction of +# recorded eye fixation locations in reading experiments. Behavior +# Research Methods, 45(3), 679–683. +# +# https://doi.org/10.3758/s13428-012-0280-3 +# https://blogs.umass.edu/rdcl/resources/ +###################################################################### + + +def regress( + fixation_XY, line_Y, k_bounds=(-0.1, 0.1), o_bounds=(-50, 50), s_bounds=(1, 20) +): + n = len(fixation_XY) + m = len(line_Y) + + def fit_lines(params, return_line_assignments=False): + k = k_bounds[0] + (k_bounds[1] - k_bounds[0]) * norm.cdf(params[0]) + o = o_bounds[0] + (o_bounds[1] - o_bounds[0]) * norm.cdf(params[1]) + s = s_bounds[0] + (s_bounds[1] - s_bounds[0]) * norm.cdf(params[2]) + predicted_Y_from_slope = fixation_XY[:, 0] * k + line_Y_plus_offset = line_Y + o + density = np.zeros((n, m)) + for line_i in range(m): + fit_Y = predicted_Y_from_slope + line_Y_plus_offset[line_i] + density[:, line_i] = norm.logpdf(fixation_XY[:, 1], fit_Y, s) + if return_line_assignments: + return density.argmax(axis=1) + return -sum(density.max(axis=1)) + + best_fit = minimize(fit_lines, [0, 0, 0]) + line_assignments = fit_lines(best_fit.x, True) + for fixation_i, line_i in enumerate(line_assignments): + fixation_XY[fixation_i, 1] = line_Y[line_i] + return fixation_XY + + +###################################################################### +# SEGMENT +# +# Abdulin, E. R., & Komogortsev, O. V. (2015). Person verification via +# eye movement-driven text reading model, In 2015 IEEE 7th +# International Conference on Biometrics Theory, Applications and +# Systems. IEEE. +# +# https://doi.org/10.1109/BTAS.2015.7358786 +###################################################################### + + +def segment(fixation_XY, line_Y): + n = len(fixation_XY) + m = len(line_Y) + diff_X = np.diff(fixation_XY[:, 0]) + saccades_ordered_by_length = np.argsort(diff_X) + line_change_indices = saccades_ordered_by_length[: m - 1] + current_line_i = 0 + for fixation_i in range(n): + fixation_XY[fixation_i, 1] = line_Y[current_line_i] + if fixation_i in line_change_indices: + current_line_i += 1 + return fixation_XY + + +###################################################################### +# SPLIT +# +# Carr, J. W., Pescuma, V. N., Furlan, M., Ktori, M., & Crepaldi, D. +# (2021). Algorithms for the automated correction of vertical drift +# in eye-tracking data. Behavior Research Methods. +# +# https://doi.org/10.3758/s13428-021-01554-0 +# https://github.com/jwcarr/drift +###################################################################### + + +def split(fixation_XY, line_Y): + n = len(fixation_XY) + diff_X = np.diff(fixation_XY[:, 0]) + clusters = KMeans(2, n_init=10, max_iter=300).fit_predict(diff_X.reshape(-1, 1)) + centers = [diff_X[clusters == 0].mean(), diff_X[clusters == 1].mean()] + sweep_marker = np.argmin(centers) + end_line_indices = list(np.where(clusters == sweep_marker)[0] + 1) + end_line_indices.append(n) + start_of_line = 0 + for end_of_line in end_line_indices: + mean_y = np.mean(fixation_XY[start_of_line:end_of_line, 1]) + line_i = np.argmin(abs(line_Y - mean_y)) + fixation_XY[start_of_line:end_of_line, 1] = line_Y[line_i] + start_of_line = end_of_line + return fixation_XY + + +###################################################################### +# STRETCH +# +# Lohmeier, S. (2015). Experimental evaluation and modelling of the +# comprehension of indirect anaphors in a programming language +# (Master’s thesis). Technische Universität Berlin. +# +# http://www.monochromata.de/master_thesis/ma1.3.pdf +###################################################################### + + +def stretch(fixation_XY, line_Y, scale_bounds=(0.9, 1.1), offset_bounds=(-50, 50)): + n = len(fixation_XY) + fixation_Y = fixation_XY[:, 1] + + def fit_lines(params, return_correction=False): + candidate_Y = fixation_Y * params[0] + params[1] + corrected_Y = np.zeros(n) + for fixation_i in range(n): + line_i = np.argmin(abs(line_Y - candidate_Y[fixation_i])) + corrected_Y[fixation_i] = line_Y[line_i] + if return_correction: + return corrected_Y + return sum(abs(candidate_Y - corrected_Y)) + + best_fit = minimize(fit_lines, [1, 0], bounds=[scale_bounds, offset_bounds]) + fixation_XY[:, 1] = fit_lines(best_fit.x, return_correction=True) + return fixation_XY + + +###################################################################### +# WARP +# +# Carr, J. W., Pescuma, V. N., Furlan, M., Ktori, M., & Crepaldi, D. +# (2021). Algorithms for the automated correction of vertical drift +# in eye-tracking data. Behavior Research Methods. +# +# https://doi.org/10.3758/s13428-021-01554-0 +# https://github.com/jwcarr/drift +###################################################################### + + +def warp(fixation_XY, word_XY): + _, dtw_path = dynamic_time_warping(fixation_XY, word_XY) + for fixation_i, words_mapped_to_fixation_i in enumerate(dtw_path): + candidate_Y = word_XY[words_mapped_to_fixation_i, 1] + fixation_XY[fixation_i, 1] = mode(candidate_Y) + return fixation_XY + + +def mode(values): + values = list(values) + return max(set(values), key=values.count) + + +# def time_warp(fixation_XY, word_XY): + +# durations = np.delete(fixation_XY, 0, 1) +# durations = np.delete(durations, 0, 1) +# fixation_XY = np.delete(fixation_XY, 2, 1) + +# word_durations = np.delete(word_XY, 0, 1) +# word_durations = np.delete(word_durations, 0, 1) +# word_XY = np.delete(word_XY, 2, 1) + +# _, dtw_path = dynamic_time_warping(durations, word_durations) + +# for fixation_i, words_mapped_to_fixation_i in enumerate(dtw_path): +# candidate_Y = word_XY[words_mapped_to_fixation_i, 1] +# fixation_XY[fixation_i, 1] = mode(candidate_Y) +# return fixation_XY + + +###################################################################### +# Dynamic Time Warping adapted from https://github.com/talcs/simpledtw +# This is used by the COMPARE and WARP algorithms +###################################################################### + + +def dynamic_time_warping(sequence1, sequence2): + n1 = len(sequence1) + n2 = len(sequence2) + dtw_cost = np.zeros((n1 + 1, n2 + 1)) + dtw_cost[0, :] = np.inf + dtw_cost[:, 0] = np.inf + dtw_cost[0, 0] = 0 + for i in range(n1): + for j in range(n2): + this_cost = np.sqrt(sum((sequence1[i] - sequence2[j]) ** 2)) + dtw_cost[i + 1, j + 1] = this_cost + min( + dtw_cost[i, j + 1], dtw_cost[i + 1, j], dtw_cost[i, j] + ) + dtw_cost = dtw_cost[1:, 1:] + dtw_path = [[] for _ in range(n1)] + while i > 0 or j > 0: + dtw_path[i].append(j) + possible_moves = [np.inf, np.inf, np.inf] + if i > 0 and j > 0: + possible_moves[0] = dtw_cost[i - 1, j - 1] + if i > 0: + possible_moves[1] = dtw_cost[i - 1, j] + if j > 0: + possible_moves[2] = dtw_cost[i, j - 1] + best_move = np.argmin(possible_moves) + if best_move == 0: + i -= 1 + j -= 1 + elif best_move == 1: + i -= 1 + else: + j -= 1 + dtw_path[0].append(0) + return dtw_cost[-1, -1], dtw_path + + +###################################################################### +# SLICE +# +# Glandorf, D., & Schroeder, S. (2021). Slice: An algorithm to assign +# fixations in multi-line texts. Procedia Computer Science, 192, +# 2971–2979. +# +# https://doi.org/10.1016/j.procs.2021.09.069 +###################################################################### + + +def slice(fixation_XY, line_Y, x_thresh=192, y_thresh=32, w_thresh=32, n_thresh=90): + n = len(fixation_XY) + line_height = np.mean(np.diff(line_Y)) + proto_lines, phantom_proto_lines = {}, {} + # 1. Segment runs + dist_X = abs(np.diff(fixation_XY[:, 0])) + dist_Y = abs(np.diff(fixation_XY[:, 1])) + end_run_indices = list( + np.where(np.logical_or(dist_X > x_thresh, dist_Y > y_thresh))[0] + 1 + ) + run_starts = [0] + end_run_indices + run_ends = end_run_indices + [n] + runs = [list(range(start, end)) for start, end in zip(run_starts, run_ends)] + # 2. Determine starting run + longest_run_i = np.argmax( + [fixation_XY[run[-1], 0] - fixation_XY[run[0], 0] for run in runs] + ) + proto_lines[0] = runs.pop(longest_run_i) + # 3. Group runs into proto lines + while runs: + merger_on_this_iteration = False + for proto_line_i, direction in [(min(proto_lines), -1), (max(proto_lines), 1)]: + # Create new proto line above or below (depending on direction) + proto_lines[proto_line_i + direction] = [] + # Get current proto line XY coordinates (if proto line is empty, get phanton coordinates) + if proto_lines[proto_line_i]: + proto_line_XY = fixation_XY[proto_lines[proto_line_i]] + else: + proto_line_XY = phantom_proto_lines[proto_line_i] + # Compute differences between current proto line and all runs + run_differences = np.zeros(len(runs)) + for run_i, run in enumerate(runs): + y_diffs = [ + y - proto_line_XY[np.argmin(abs(proto_line_XY[:, 0] - x)), 1] + for x, y in fixation_XY[run] + ] + run_differences[run_i] = np.mean(y_diffs) + # Find runs that can be merged into this proto line + merge_into_current = list(np.where(abs(run_differences) < w_thresh)[0]) + # Find runs that can be merged into the adjacent proto line + merge_into_adjacent = list( + np.where( + np.logical_and( + run_differences * direction >= w_thresh, + run_differences * direction < n_thresh, + ) + )[0] + ) + # Perform mergers + for index in merge_into_current: + proto_lines[proto_line_i].extend(runs[index]) + for index in merge_into_adjacent: + proto_lines[proto_line_i + direction].extend(runs[index]) + # If no, mergers to the adjacent, create phantom line for the adjacent + if not merge_into_adjacent: + average_x, average_y = np.mean(proto_line_XY, axis=0) + adjacent_y = average_y + line_height * direction + phantom_proto_lines[proto_line_i + direction] = np.array( + [[average_x, adjacent_y]] + ) + # Remove all runs that were merged on this iteration + for index in sorted(merge_into_current + merge_into_adjacent, reverse=True): + del runs[index] + merger_on_this_iteration = True + # If no mergers were made, break the while loop + if not merger_on_this_iteration: + break + # 4. Assign any leftover runs to the closest proto lines + for run in runs: + best_pl_distance = np.inf + best_pl_assignemnt = None + for proto_line_i in proto_lines: + if proto_lines[proto_line_i]: + proto_line_XY = fixation_XY[proto_lines[proto_line_i]] + else: + proto_line_XY = phantom_proto_lines[proto_line_i] + y_diffs = [ + y - proto_line_XY[np.argmin(abs(proto_line_XY[:, 0] - x)), 1] + for x, y in fixation_XY[run] + ] + pl_distance = abs(np.mean(y_diffs)) + if pl_distance < best_pl_distance: + best_pl_distance = pl_distance + best_pl_assignemnt = proto_line_i + proto_lines[best_pl_assignemnt].extend(run) + # 5. Prune proto lines + while len(proto_lines) > len(line_Y): + top, bot = min(proto_lines), max(proto_lines) + if len(proto_lines[top]) < len(proto_lines[bot]): + proto_lines[top + 1].extend(proto_lines[top]) + del proto_lines[top] + else: + proto_lines[bot - 1].extend(proto_lines[bot]) + del proto_lines[bot] + # 6. Map proto lines to text lines + for line_i, proto_line_i in enumerate(sorted(proto_lines)): + fixation_XY[proto_lines[proto_line_i], 1] = line_Y[line_i] + return fixation_XY diff --git a/preprocessing/events/fixation_correction.py b/preprocessing/events/fixation_correction.py new file mode 100644 index 00000000..0023829a --- /dev/null +++ b/preprocessing/events/fixation_correction.py @@ -0,0 +1,223 @@ +import preprocessing.events.drift_algorithms as da +import numpy as np +import polars as pl +import pymovements as pm + +from ..config import settings + + +def _filter_gaze_events( + events: pl.DataFrame, + stimulus_name: str, + page_name: str, + event_type: str = "fixation", +) -> pl.DataFrame: + """Filters the gaze events for a specific stimulus, page and event type (e.g. fixation, saccade, etc.) + + Parameters: + events (pl.DataFrame): The gaze events to filter + stimulus_name (str): The name of the stimulus to filter for + page_name (str): The name of the page to filter for + event_type (str): The type of event to filter for (default is "fixation") + + Returns: + pl.DataFrame: The filtered gaze events + """ + return events.frame.filter( + (pl.col("page") == f"{page_name}") + & (pl.col("stimulus") == f"{stimulus_name}") + & (pl.col("name") == f"{event_type}") + ) + + +def _filter_aois(stimulus, page_name) -> pl.DataFrame: + """Filters the AOIs for a specific stimulus and page name + + Parameters: + stimulus: The stimulus to filter the AOIs for + page_name: The name of the page to filter for + + Returns: + pl.DataFrame: The filtered AOIs + """ + + return stimulus.text_stimulus.aois.filter((pl.col("page") == f"{page_name}")) + + +def _get_lines_of_text_form_aois(aois: pl.DataFrame) -> list: + """Calculates the line positions of the text based on the AOIs. This is necessary for some of the drift correction algorithms to calculate the line positions. + + Parameters: + aois: The AOIs to calculate the line positions from + + Returns: + list: The line positions of the text + """ + + return ( + aois.filter(pl.col("line_idx").is_not_null()) + .group_by("line_idx") + .agg( + (pl.col("top_left_y") + pl.col("height") / 2) + .mean() + .alias("line_center") + ) + .sort("line_idx")["line_center"] + .to_list() + ) + + +def create_corrected_fixations_locations( + events: pl.DataFrame, aois: pl.DataFrame, algorithm: str = "chain" +): + """Corrects the fixations based on the specified algorithm and the AOIs. + The AOIs are necessary to calculate the line positions. + + Parameters: + events: The gaze events to correct + aois: The AOIs to use for the correction + algorithm: The algorithm to use for the correction (default is "chain") + algorithms available: attach, cluster, compare, merge, regress, segment, split, stretch, warp, slice + some have additional parameters, e.g. attach has a parameter "threshold" which defines the + maximum distance to the line for a fixation to be attached to it, default is 50 pixels + + Returns: + list: The corrected fixation locations + """ + + fixationXY = events.filter(pl.col("name") == "fixation").select(pl.col("location")) + fixationXY = fixationXY["location"].to_list() + lines = _get_lines_of_text_form_aois(aois) + func = getattr(da, algorithm) + return func(np.array(fixationXY), lines) + + +def _add_corrected_fixations_to_events( + original_events: pl.DataFrame, + corrected_fixations: list, + algorithm: str, + events_destination: pl.DataFrame, +) -> pl.DataFrame: + """Adds the corrected fixations to the gaze events dataframe + with the appropriate event type name (e.g. fixation_corrected_attach) and returns the updated dataframe. + + Parameters: + original_events: The original gaze events dataframe with fixations that where corrected + (used to get the other information of the events, e.g. start and end time, page, stimulus, etc.) + corrected_fixations: The list of corrected fixation locations + algorithm: The algorithm that was used to correct the fixations + events_destination: The destination dataframe to which the corrected fixations should be added + + Returns: + pl.DataFrame: The updated gaze events dataframe with the corrected fixations added + """ + + for i, event in enumerate(original_events.iter_rows(named=True)): + new_event = event.copy() + new_event["location"] = corrected_fixations[i].tolist() + new_event["name"] = f"fixation_corrected_{algorithm}" + events_destination = events_destination.vstack(pl.DataFrame([new_event])) + return events_destination + + +def add_corrected_fixations_for_page( + stimulus, page, gaze_events: pl.DataFrame, algorithm: str = "chain" +) -> pl.DataFrame: + """Corrects the fixations based on the specified algorithm and the AOIs and adds them to the events dataframe.""" + + stimulus_name = f"{stimulus.name}_{stimulus.id}" + page_name = f"page_{page.number}" + events = _filter_gaze_events(gaze_events, stimulus_name, page_name) + + if ( + events.select( + pl.col("name").str.contains(f"fixation_corrected_{algorithm}").sum() + ).item() + > 0 + ): + raise ValueError( + f"Fixation corrections with algorithm {algorithm} already exist for {stimulus_name}/{page_name} in the gaze events dataframe, cannot add corrected fixations again without removing the previous ones first. Remove them with the function remove_previous_fixation_corrections(gaze_events, algorithm)" + ) + + aois = _filter_aois(stimulus, page_name) + corrected_fixations = create_corrected_fixations_locations(events, aois, algorithm) + gaze_events.frame = _add_corrected_fixations_to_events( + events, corrected_fixations, algorithm, gaze_events.frame + ) + + +def add_corrected_fixations( + sess, gaze_events: pl.DataFrame, algorithm: str = "chain", verbose=0 +) -> pl.DataFrame: + """Adds the corrected fixations for all stimuli and pages in the session to the gaze events dataframe.""" + + if ( + gaze_events.frame.select( + pl.col("name").str.contains(f"fixation_corrected_{algorithm}").sum() + ).item() + > 0 + ): + raise ValueError( + f"Fixation corrections with algorithm {algorithm} already exist in the gaze events dataframe, cannot add corrected fixations again without removing the previous ones first. Remove them with the function remove_previous_fixation_corrections(gaze_events, algorithm)" + ) + + for stimulus in sess.stimuli: + if verbose == 1: + print(stimulus.name) + for page in stimulus.pages: + if verbose == 1: + print(page.number, end=",") + add_corrected_fixations_for_page(stimulus, page, gaze_events, algorithm) + if verbose == 1: + print() + + +def remove_previous_fixation_corrections( + gaze_events, algorithm: str = "chain" +) -> pl.DataFrame: + gaze_events.frame = gaze_events.frame.filter( + ~pl.col("name").str.contains(f"fixation_corrected_{algorithm}") + ) + return gaze_events + + +def correct_fixations( + gaze: pm.Gaze, + sess, + algorithm: str = "chain", +) -> None: + """Correct fixation locations for vertical line drift. + + Runs the specified drift-correction algorithm (default ``"chain"``) on the + detected fixations in ``gaze.events`` and replaces the original + ``fixation`` events with the corrected ones. This ensures that downstream + AOI mapping and reading measures operate on the corrected locations. + + Parameters + ---------- + gaze : pm.Gaze + Gaze object with detected fixations in ``gaze.events``. + sess : Session + Session object providing ``stimuli`` and page AOI information. + algorithm : str, optional + Drift-correction algorithm to use. Default is ``"chain"``. + """ + if gaze.events is None or gaze.events.frame is None or gaze.events.frame.is_empty(): + return + + remove_previous_fixation_corrections(gaze.events, algorithm) + add_corrected_fixations(sess, gaze.events, algorithm, verbose=1) + + corrected_name = f"fixation_corrected_{algorithm}" + corrected = ( + gaze.events.frame.filter(pl.col("name") == corrected_name) + .with_columns(pl.lit(settings.FIXATION).alias("name")) + ) + + if corrected.is_empty(): + return + + other_events = gaze.events.frame.filter( + (pl.col("name") != settings.FIXATION) & (pl.col("name") != corrected_name) + ) + gaze.events.frame = pl.concat([other_events, corrected], how="vertical") diff --git a/preprocessing/scripts/run_preprocessing.py b/preprocessing/scripts/run_preprocessing.py index 34495adb..cc6c51de 100644 --- a/preprocessing/scripts/run_preprocessing.py +++ b/preprocessing/scripts/run_preprocessing.py @@ -40,12 +40,15 @@ def run_preprocessing(config_path: str | None = None): f"And check if you have filled in the correct data collection name in the settings." ) + enlarge_aois = not settings.RUN_FIXATION_CORRECTION + if settings.EXPERIMENT_TYPE == "MultiplEYE": data_collection = preprocessing.data_collection.MultipleyeDataCollection.create_from_data_folder( data_folder_path, include_pilots=settings.INCLUDE_PILOTS, excluded_sessions=settings.EXCLUDE_SESSIONS, included_sessions=settings.INCLUDE_SESSIONS, + enlarge_aois=enlarge_aois, ) elif settings.EXPERIMENT_TYPE == "MeRID": @@ -55,6 +58,7 @@ def run_preprocessing(config_path: str | None = None): include_pilots=settings.INCLUDE_PILOTS, excluded_sessions=settings.EXCLUDE_SESSIONS, included_sessions=settings.INCLUDE_SESSIONS, + enlarge_aois=enlarge_aois, ) ) @@ -180,6 +184,11 @@ def run_preprocessing(config_path: str | None = None): if settings.RUN_FIXATION_DETECTION: preprocessing.detect_fixations(gaze) + + if settings.RUN_FIXATION_CORRECTION: + pbar.set_description(f"Correcting fixations {sess.sid}:") + preprocessing.correct_fixations(gaze, sess) + if settings.RUN_SACCADE_DETECTION: preprocessing.detect_saccades(gaze) diff --git a/templates_and_notes/multipleye_settings_preprocessing.template.yaml b/templates_and_notes/multipleye_settings_preprocessing.template.yaml index 399c7df5..1927ac99 100644 --- a/templates_and_notes/multipleye_settings_preprocessing.template.yaml +++ b/templates_and_notes/multipleye_settings_preprocessing.template.yaml @@ -43,6 +43,9 @@ plot_aoi_overlay: true # Whether to copy AOI images overlay to the output directory. copy_aoi_images_overlay: false +# Whether to run fixation correction. +run_fixation_correction: true + # ----------------------- Internal Usage ----------------------- # --- LOGGING --- diff --git a/tests/MultiplEYE_toy_X_x_1_1/eye-tracking-sessions/001_TOY_X_1_ET1/001toyx1.asc b/tests/MultiplEYE_toy_X_x_1_1/eye-tracking-sessions/001_TOY_X_1_ET1/001toyx1.asc index f98ea8f7..1dadedea 100644 --- a/tests/MultiplEYE_toy_X_x_1_1/eye-tracking-sessions/001_TOY_X_1_ET1/001toyx1.asc +++ b/tests/MultiplEYE_toy_X_x_1_1/eye-tracking-sessions/001_TOY_X_1_ET1/001toyx1.asc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ec52b4c59d633a34db12f2953a673c8f8d34f5e40789b3abf00d248343091586 -size 132353014 +oid sha256:8f1bcb3359db962513c49796fea4c7a2046d6848037814dbc5e7d76ced9a2cb4 +size 132352226