Main analysis

The primary analysis for the thesis, where we train a classifier for the code vs prose task.

---

Table of Contents

• Setup
  • Imports
  • Configuration
• Loading
  • Loading EEG data
  • Loading markers
• Preprocessing
  • Filter short trials
  • Filter no answer trials
  • Bandpass filtering
  • Constructing epochs
  • Epochs to windows
  • Constructing our X and y
• Training
  • Learning curves

NOTE: This TOC is manually built and may not be up to date.

---

Setup

# Imports
import re
import logging
from pathlib import Path
from datetime import datetime, timezone, timedelta
from typing import Dict, List, Set
from pprint import pprint
from collections import Counter, defaultdict

import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.cm import get_cmap
import mne
from sklearn.base import clone

from eegclassify import main, load, clean, features, preprocess
from eegclassify.plot import TimelineFigure

%matplotlib inline
plt.rcParams['figure.dpi'] = 300
plt.rc('text', usetex=True)  # render text with TeX

Configuration

This cell contains all the configuration options available for the analysis.

# Configuration
use_bandpass_filter = True
classify_breaks = False
single_subject = None    # set to subject number, or False
balance_dataset = True
min_task_duration = 5
standardize = False
sliding_windows = True

# Constants
sfreq = 256  # sampling frequency of the Muse S

# Color maps
cmap = get_cmap('RdYlGn')
cmap_2cls = get_cmap('tab10')
cmap_discrete = get_cmap('tab20')

Misc setup

# Set up logging
logging.basicConfig(format="%(levelname)s: %(msg)s", level=logging.INFO, force=True)
logger = logging.getLogger(__name__)
mne.set_log_level('WARNING')

# Set this to True to run on testing data
simulate_test = False
if simulate_test:
    import os
    os.environ['PYTEST_CURRENT_TEST'] = "true"

%%javascript
document.title='erb-thesis/Main - Jupyter'  // Set the document title to be able to track time spent working on the notebook with ActivityWatch

Loading

Loading EEG

First we need to load the EEG data used during the experiments.

data_dir = Path('../data').resolve()
  
files = sorted([
    *list((data_dir / "tasks/visual-codeprose/").glob("*/*/recording_*.csv")),
])
#pprint(files)

eeg = load.load_eeg(files)
eeg = eeg.set_index('timestamp').sort_index()
print(eeg.shape)

INFO: Loading EEG recordings... (from 32 files)
INFO: Concatenating...
INFO: Concatenated!

(3317436, 4)

Lets have a look at the loaded data:

def plot_eeg(X, offset, window, span=None):
    plt.figure(figsize=(20, 3))
    data = X[offset : offset + window, :]
    plt.plot(data)
    plt.xlim(0, window);
    if span is None:
        span = max(np.std(data, axis=1))
    plt.ylim(-span, span);

plot_eeg(eeg.to_numpy(), offset = 1000, window = 3 * sfreq)
plot_eeg(eeg.to_numpy(), offset = 20000, window = 3 * sfreq)

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 6000x900 with 1 Axes>

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 6000x900 with 1 Axes>

We see that some channels are bad some of the time, we will deal with that later.

Loading markers

Now we need to load the markers produced during each trial of the experiment, so we can annotate the EEG data.

marker_files = [
    *list((data_dir / 'tasks/visual-codeprose/').glob('*/*/*_behOutput_*.csv')),
]

def _build_breaks(df):
    starts = df['t_answered'].iloc[:-1].shift()
    starts_utc = df['t_answered_utc'].iloc[:-1].shift()
    stops = df['t_presented'].iloc[1:]
    stops_utc = df['t_presented_utc'].iloc[1:]
    
    breaks = pd.DataFrame({
        "t_presented": starts, 
        "t_answered": stops, 
        "t_presented_utc": starts_utc, 
        "t_answered_utc": stops_utc, 
        "type": "relax", 
        "duration": stops - starts, 
        'subject': df['subject'],
        'image_path': 'none',
        'response': 'up',  # as placeholder
    })
    return breaks

dfs = []
for file in marker_files:
    df = pd.read_csv(file, index_col=0)
    df['duration'] = df['t_answered'] - df['t_presented']
    
    match = re.search('subject(\d+)', str(file))
    assert match
    df['subject'] = int(match.group(1))
    
    match = re.search('session(\d+)', str(file))
    assert match
    df['session'] = int(match.group(1))
    
    if classify_breaks:
        breaks = _build_breaks(df)
        df = df.append(breaks)
    dfs.append(df)
df_markers = pd.concat(dfs).sort_values(by=['t_presented_utc'])

Lets take a look at some of the marker rows:

df_markers['img'] = df_markers['image_path'].apply((lambda c: c.split("/")[-1]))
df_markers.drop(columns=['image_path'])

	type	response	t_presented	t_presented_utc	t_answered	t_answered_utc	duration	subject	session	img
0	code	up	0.048869	1.616768e+09	41.687711	1.616768e+09	41.638842	1	0	final-9-1.png
1	code	up	46.757057	1.616768e+09	107.743816	1.616768e+09	60.986759	1	0	final-14-1.png
2	code	up	112.792900	1.616768e+09	173.725393	1.616768e+09	60.932493	1	0	final-2-1.png
3	code	down	178.779508	1.616768e+09	220.707851	1.616768e+09	41.928343	1	0	final-10-2.png
4	code	down	225.770106	1.616768e+09	331.949693	1.616768e+09	106.179587	1	0	final-7-1.png
...	...	...	...	...	...	...	...	...	...	...
43	prose	space	1849.720940	1.623680e+09	1849.872283	1.623680e+09	0.151344	10	1	bugs_19.PNG
44	code	space	1850.054168	1.623680e+09	1850.193749	1.623680e+09	0.139581	10	1	final-3-1.png
45	prose	space	1850.370756	1.623680e+09	1850.527017	1.623680e+09	0.156261	10	1	bugs_18.PNG
46	code	space	1850.737257	1.623680e+09	1850.874336	1.623680e+09	0.137079	10	1	final-15-1.png
47	prose	space	1851.070524	1.623680e+09	1851.215915	1.623680e+09	0.145390	10	1	bugs_15.PNG

478 rows × 10 columns

Inspect the markers

fig, axs = plt.subplots(1, len(set(df_markers['subject'])), figsize=(16, 4), sharey=True)
for subject, ax in zip(set(df_markers['subject']), axs):
    #plt.figure()
    ax.set_title(f"Subject {subject}")
    _df = df_markers[df_markers['subject'] == subject]
    _df = _df[_df["response"] != "space"]
    _df['duration'].hist(ax=ax)

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2879                 artists.remove(spine)
   2880 
-> 2881         self._update_title_position(renderer)
   2882 
   2883         if not self.axison or inframe:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in _update_title_position(self, renderer)
   2820                 _log.debug('top of axes not in the figure, so title not moved')
   2821                 return
-> 2822             if title.get_window_extent(renderer).ymin < top:
   2823                 _, y = self.transAxes.inverted().transform((0, top))
   2824                 title.set_position((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 4800x1200 with 10 Axes>

Preprocessing

Now we need to preprocess the data a bit, gathering the EEG data for each trial in the experiment.

• [ ] Better cleaning/rejection of bad epochs/windows/samples

Select subjects

if single_subject is not None:
    logger.info(f"Selecting subject {single_subject}")
    n_prev = len(df_markers)
    new_markers = df_markers[df_markers['subject'] == single_subject]
    if not new_markers.empty:
        df_markers = new_markers
    print(f"old size: {n_prev}, new size: {len(df_markers)}")
else:
    logger.info("Using all subjects")

INFO: Using all subjects

Filter no answer trials

We filter away rows where space was clicked (didn't answer/skipped/unsure?)

n_prev = len(df_markers)
df_markers = df_markers[df_markers['response'].isin(['up', 'down'])]
print(f"Filtered away {n_prev - len(df_markers)} epochs due skipped by subject")

Filtered away 186 epochs due skipped by subject

Filter short trials

We filter away rows where the subject didn't spend at least min_task_duration seconds with the task.

n_prev = len(df_markers)
df_markers = df_markers[df_markers['duration'] > min_task_duration]
print(f"Filtered away {n_prev - len(df_markers)} epochs due to short duration")

Filtered away 0 epochs due to short duration

Bandpass filtering

• [x] FIXME: Why is the signal shifted after filtering? Should this be accounted for somehow?

# TODO: Should this maybe be done per epoch/window to avoid discontinuities? (although discontinuities should be rare...)

# Bandpass-filter the signal
plot_offset = 10000
filter_shift = sfreq - 50

if use_bandpass_filter:
    logger.info("Bandpass-filtering the signal")
    plot_eeg(eeg.to_numpy(), plot_offset, 10 * sfreq)
    
    eeg_clean = clean.filter(eeg, sfreq)
    
    for ch_idx, col in enumerate(eeg.columns):
        eeg[col][:-filter_shift] = eeg_clean[filter_shift:, ch_idx]
        
    # plot the new result
    plot_eeg(eeg.to_numpy(), plot_offset, 10 * sfreq)
else:
    logger.info("Skipping bandpass filtering")

INFO: Bandpass-filtering the signal

Setting up band-pass filter from 3 - 40 Hz

FIR filter parameters
---------------------
Designing a one-pass, zero-phase, non-causal bandpass filter:
- Windowed time-domain design (firwin) method
- Hamming window with 0.0194 passband ripple and 53 dB stopband attenuation
- Lower passband edge: 3.00
- Lower transition bandwidth: 2.00 Hz (-6 dB cutoff frequency: 2.00 Hz)
- Upper passband edge: 40.00 Hz
- Upper transition bandwidth: 10.00 Hz (-6 dB cutoff frequency: 45.00 Hz)
- Filter length: 423 samples (1.652 sec)

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 6000x900 with 1 Axes>

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 6000x900 with 1 Axes>

Exponential moving standardize

# TODO: This should be done per epoch/window to avoid discontinuities
# NOTE: This is needed for the neural nets to behave

if standardize:
    from braindecode.datautil import exponential_moving_standardize
    output = exponential_moving_standardize(eeg.to_numpy().T)
    
    for ch_idx, col in enumerate(eeg.columns):
        eeg[col] = output.T[:, ch_idx]
else:
    logger.info("Skipping exponential standardize")

INFO: Skipping exponential standardize

plot_eeg(eeg.to_numpy(), 10000, 10 * sfreq)

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 6000x900 with 1 Axes>

Constructing epochs

Now we match up the EEG data with the markers to create our epochs.

epochs = []

skip_first_s = 1  # skip first 1 second
skip_last_s = 1   # skip last 1 second

for _, row in df_markers.iterrows():
    start = datetime.fromtimestamp(row['t_presented_utc'], timezone.utc) + timedelta(seconds=skip_first_s)
    stop = datetime.fromtimestamp(row['t_answered_utc'], timezone.utc) - timedelta(seconds=skip_last_s)
    epoch = eeg.truncate(start, stop)
    
    # Check that sample count aligns with epoch duration
    data_duration = row['duration'] - skip_first_s - skip_last_s
    expected_samples = round(data_duration * sfreq)
    actual_samples = len(epoch)
    diff = expected_samples - actual_samples
    if abs(diff) > 5:
        logger.warning(f"Expected {expected_samples} samples, found {actual_samples}")
        
    epochs.append((epoch, row['type'], row['subject'], row['img'], row["session"], start))
print(f"epochs: {len(epochs)}")

WARNING: Expected 15101 samples, found 14141
WARNING: Expected 14243 samples, found 13139
WARNING: Expected 6170 samples, found 5066
WARNING: Expected 41657 samples, found 40856
WARNING: Expected 10233 samples, found 9130
WARNING: Expected 10802 samples, found 9710
WARNING: Expected 8679 samples, found 7588
WARNING: Expected 3885 samples, found 2926
WARNING: Expected 16917 samples, found 15814
WARNING: Expected 20673 samples, found 19571
WARNING: Expected 19187 samples, found 18096
WARNING: Expected 13757 samples, found 12666
WARNING: Expected 27185 samples, found 0
WARNING: Expected 24049 samples, found 0
WARNING: Expected 17681 samples, found 0
WARNING: Expected 3278 samples, found 0
WARNING: Expected 1740 samples, found 0
WARNING: Expected 5939 samples, found 0
WARNING: Expected 4528 samples, found 0
WARNING: Expected 7316 samples, found 0
WARNING: Expected 7621 samples, found 0
WARNING: Expected 3750 samples, found 0
WARNING: Expected 5016 samples, found 0
WARNING: Expected 1737 samples, found 0
WARNING: Expected 1370 samples, found 0
WARNING: Expected 44213 samples, found 0
WARNING: Expected 20452 samples, found 0
WARNING: Expected 11174 samples, found 763
WARNING: Expected 10231 samples, found 0
WARNING: Expected 11115 samples, found 0
WARNING: Expected 9825 samples, found 0
WARNING: Expected 11232 samples, found 0
WARNING: Expected 20366 samples, found 0
WARNING: Expected 12162 samples, found 0
WARNING: Expected 15459 samples, found 0
WARNING: Expected 15540 samples, found 0
WARNING: Expected 7302 samples, found 0
WARNING: Expected 16345 samples, found 0
WARNING: Expected 19058 samples, found 0
WARNING: Expected 21405 samples, found 0
WARNING: Expected 6790 samples, found 0
WARNING: Expected 7104 samples, found 0
WARNING: Expected 13695 samples, found 0
WARNING: Expected 7750 samples, found 0
WARNING: Expected 4341 samples, found 0
WARNING: Expected 4213 samples, found 0
WARNING: Expected 5823 samples, found 0
WARNING: Expected 4807 samples, found 0
WARNING: Expected 9769 samples, found 0
WARNING: Expected 7677 samples, found 0
WARNING: Expected 7395 samples, found 0
WARNING: Expected 5852 samples, found 0
WARNING: Expected 9744 samples, found 0
WARNING: Expected 7911 samples, found 0
WARNING: Expected 22241 samples, found 0
WARNING: Expected 3723 samples, found 0
WARNING: Expected 6942 samples, found 0
WARNING: Expected 7225 samples, found 7232
WARNING: Expected 22445 samples, found 22460
WARNING: Expected 26541 samples, found 5534
WARNING: Expected 5057 samples, found 0
WARNING: Expected 13191 samples, found 0
WARNING: Expected 23824 samples, found 0
WARNING: Expected 10259 samples, found 0
WARNING: Expected 18668 samples, found 0
WARNING: Expected 16063 samples, found 0
WARNING: Expected 19616 samples, found 0
WARNING: Expected 26956 samples, found 0
WARNING: Expected 20593 samples, found 0
WARNING: Expected 8452 samples, found 0
WARNING: Expected 8338 samples, found 0
WARNING: Expected 10701 samples, found 0
WARNING: Expected 4595 samples, found 0
WARNING: Expected 15045 samples, found 0
WARNING: Expected 14929 samples, found 0
WARNING: Expected 8860 samples, found 0
WARNING: Expected 7389 samples, found 0
WARNING: Expected 6632 samples, found 0
WARNING: Expected 12985 samples, found 0
WARNING: Expected 6154 samples, found 0
WARNING: Expected 57496 samples, found 0

epochs: 292

Inspecting epochs

pd.DataFrame([np.std(epoch) for epoch, *_ in epochs])

	TP9	AF7	AF8	TP10
0	20.020678	6.778347	6.917218	17.755212
1	17.734952	6.419294	6.785364	15.456685
2	17.756778	7.088328	7.111105	17.374138
3	12.668252	6.692826	6.884935	13.280354
4	16.936969	8.073649	9.080483	13.390705
...	...	...	...	...
287	NaN	NaN	NaN	NaN
288	NaN	NaN	NaN	NaN
289	NaN	NaN	NaN	NaN
290	NaN	NaN	NaN	NaN
291	NaN	NaN	NaN	NaN

292 rows × 4 columns

Epochs to windows

Now we split up the epochs into windows of a fixed size.

• [ ] TODO: Use a sliding window approach, similar to the one in braindecode (needs care with CV)

# Split epochs into windows
WINDOW_SIZE = int(min_task_duration * sfreq)
WINDOW_STEP = 256

if sliding_windows:
    print("Using sliding window to generate windows")
else:
    print("Not using sliding window to generate windows")
    WINDOW_STEP = WINDOW_SIZE

windows = []
for epoch, type, subject, img, session, timestamp in epochs:
    for i in range(0, len(epoch), WINDOW_STEP):
        #print(i, i+WINDOW_SIZE)
        window = epoch.iloc[i:i+WINDOW_SIZE]
        if len(window) == WINDOW_SIZE:
            windows.append((window, type, subject, img, session, timestamp))
        else:
            logger.debug(f'epoch too small ({len(window)}), skipping')    

print(f"windows: {len(windows)}")

Using sliding window to generate windows
windows: 7788

Filter away windows with bad signal

filter_bad_signal = True
if filter_bad_signal:
    for window, *_ in windows:
        # print(np.std(window))
        pass

Constructing our X and y

Now to actually construct matrices that we can feed into the classifier.

X, y, subjs, imgs, sessions, ts = zip(*windows)
X = np.array([x.to_numpy().T for x in X])
print(X.shape)

(7788, 4, 1280)

# Lets take a look at the signal in a few of the windows to make sure it looks ok
fig = plt.figure(figsize=(25, 10))
axs = fig.subplots(4, 1, sharex=True, sharey=True)
for i, ax in enumerate(axs):
    if standardize:
        plt.ylim(-1, 1)
    else:
        plt.ylim(-40, 40)
    plt.xlim(0, WINDOW_SIZE)
    ax.plot(X[i, :, :].T);

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 7500x3000 with 4 Axes>

y = np.array(y)
print(y.shape)
print(Counter(y))

(7788,)
Counter({'code': 5465, 'prose': 2323})

subjs = np.array(subjs)
sessions = np.array(sessions)
imgs = np.array(imgs)

assert len(subjs) == len(sessions) == len(imgs)

print(Counter(subjs))
print(Counter(sessions))
print(Counter(zip(subjs, sessions)))
print(Counter(imgs))

Counter({0: 2237, 5: 1023, 6: 1001, 1: 819, 3: 730, 4: 703, 8: 518, 7: 458, 10: 227, 9: 72})
Counter({0: 4299, 1: 3489})
Counter({(0, 1): 1213, (0, 0): 1024, (5, 0): 1023, (6, 1): 1001, (1, 0): 819, (3, 0): 730, (4, 0): 703, (8, 1): 518, (7, 1): 458, (10, 1): 227, (9, 1): 72})
Counter({'final-7-1.png': 477, 'final-11-1.png': 465, 'final-3-1.png': 445, 'final-5-2.png': 408, 'final-12-2.png': 403, 'final-5-3.png': 372, 'final-15-1.png': 324, 'final-4-1.png': 315, 'final-2-2.png': 278, 'final-1-1.png': 277, 'final-5-1.png': 273, 'final-8-1.png': 250, 'final-2-1.png': 216, 'final-10-1.png': 211, 'final-14-1.png': 205, 'final-9-1.png': 184, 'bugs_20.PNG': 178, 'bugs_11.PNG': 173, 'bugs_10.PNG': 166, 'final-13-1.png': 153, 'bugs_2.PNG': 151, 'bugs_12.PNG': 149, 'bugs_19.PNG': 143, 'final-10-2.png': 137, 'bugs_5.PNG': 132, 'bugs_4.PNG': 110, 'bugs_18.PNG': 105, 'bugs_1.PNG': 82, 'bugs_16.PNG': 76, 'bugs_15.PNG': 76, 'final-12-1.png': 72, 'bugs_8.PNG': 68, 'sat_6.PNG': 65, 'bugs_6.PNG': 63, 'bugs_9.PNG': 63, 'bugs_3.PNG': 54, 'bugs_13.PNG': 53, 'sat_8.PNG': 52, 'bugs_17.PNG': 50, 'bugs_14.PNG': 50, 'sat_9.PNG': 49, 'sat_5.PNG': 42, 'sat_10.PNG': 36, 'sat_2.PNG': 34, 'sat_4.PNG': 33, 'sat_1.PNG': 31, 'sat_7.PNG': 31, 'sat_3.PNG': 8})

Balance the dataset

from numpy.random import shuffle

if balance_dataset:
    assert len(set(y)) == 2, 'only supports two classes'
    
    c_y: dict = Counter(y)
    c_smallest, min_count = min(c_y.items(), key=lambda t: t[1])
    
    print(f"Smallest class: {c_smallest} with {min_count} samples")
    
    large_class_samples = np.argwhere(y != c_smallest).flatten()
    other_class_samples = np.argwhere(y == c_smallest).flatten()
    
    # Randomly undersample the largest class
    shuffle(large_class_samples)
    subsample = large_class_samples[:min_count]
    idx = np.array(sorted(np.concatenate([subsample, other_class_samples])))
    print(idx)
    X, y, subjs, imgs, sessions = X[idx], y[idx], subjs[idx], imgs[idx], sessions[idx]
    print(f"Total samples: {len(y)}")
    
    # Doesn't work as fit_resample expects X to be a 2D matrix
    #from imblearn.under_sampling import RandomUnderSampler
    #rus = RandomUnderSampler(random_state=0)
    #X_resampled, y_resampled = rus.fit_resample(X, y)
    #print(sorted(Counter(y_resampled).items()))

Smallest class: prose with 2323 samples
[   1    2    6 ... 7784 7786 7787]
Total samples: 4646

plt.plot(subjs)
plt.plot(sessions)

[<matplotlib.lines.Line2D at 0x7f9f48383a90>]

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 1800x1200 with 1 Axes>

Inspect the dataset

img_to_id: Dict[str, int] = {img: idx for idx, img in enumerate(sorted(set(imgs)))}
    
tl = TimelineFigure(title="Classification", figsize=(16, 4))
tl.add_chunked(y, lambda v: cmap_discrete({"code": 0, "prose": 1}[v]), "label")
tl.add_chunked(subjs, cmap_discrete, "subject")
tl.add_chunked(sessions, cmap_discrete, "session")
tl.add_chunked(imgs, lambda v: cmap_discrete(img_to_id[v] % 10), "image")

thres = 10
bad = 30
quality = [1 - (np.clip(np.std(xx), thres, bad) - thres) / (bad - thres) for xx in X]
tl.add_chunked(quality, cmap, "quality (stddev)")
tl.add_chunked(map(np.std, X), lambda v: 'g' if v < 20 else 'r', "quality (thres)")

tl.plot()

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2879                 artists.remove(spine)
   2880 
-> 2881         self._update_title_position(renderer)
   2882 
   2883         if not self.axison or inframe:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in _update_title_position(self, renderer)
   2820                 _log.debug('top of axes not in the figure, so title not moved')
   2821                 return
-> 2822             if title.get_window_extent(renderer).ymin < top:
   2823                 _, y = self.transAxes.inverted().transform((0, top))
   2824                 title.set_position((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 4800x1200 with 1 Axes>

Training

Here we train our model using pyRiemann.

• [ ] Much of this code is from eegclassify/main.py, code should probably be reused better

First we set up the different classifiers we want to train:

import sklearn
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import LeaveOneGroupOut, learning_curve
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA

from pyriemann.estimation import Covariances, ERPCovariances, XdawnCovariances
from pyriemann.spatialfilters import CSP
from pyriemann.tangentspace import TangentSpace


# Fixes non-convergence for binary classification
dual = set(y) == 2

clfs: Dict[str, Pipeline] = {
    # These four are from https://neurotechx.github.io/eeg-notebooks/auto_examples/visual_ssvep/02r__ssvep_decoding.html
    "Cov + CSP + TS": make_pipeline(
        Covariances(),
        CSP(4, log=False),
        TangentSpace(),
        LogisticRegression(dual=dual),
    ),
    
    #"Cov + TS": make_pipeline(
    #    Covariances(), TangentSpace(), LogisticRegression(dual=dual)
    #),
    
    # This started failing with the error: AttributeError: 'str' object has no attribute 'decode'
    # Don't know why
    #"Xdawn + TS": make_pipeline(
    #    XdawnCovariances(2),
    #    TangentSpace(metric='riemann'),
    #    LogisticRegression()
    #),
    
    # Performs meh
    #"CSP + RegLDA": make_pipeline(
    #    Covariances(), CSP(4), LDA(shrinkage="auto", solver="eigen")
    #),
    
    # Performs badly
    # "Cov + MDM": make_pipeline(Covariances(), MDM()),
}

And then we train each classifier and plot their respective confusion matrices:

from sklearn.metrics import plot_confusion_matrix
from eegclassify.util import unison_shuffled_copies
from eegclassify.main import _performance

for name, clf in clfs.items():
    logger.info(f"===== Training with {name} =====")
    clf = clone(clf)

    # Shuffled split
    X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
        X, y, test_size=0.3, shuffle=False
    )

    clf.fit(X_train, y_train)
    logger.info(f"Test score: {clf.score(X_test, y_test)}")

    y_pred = clf.predict(X_test)
    perf = _performance(y_test, y_pred)
    logger.info(perf)
    
    disp = plot_confusion_matrix(clf, X_test, y_test,
                                 display_labels=['code', 'prose'],
                                 cmap=plt.cm.Blues,
                                 normalize='true')
    plt.show()

INFO: ===== Training with Cov + CSP + TS =====
INFO: Test score: 0.6255380200860832
INFO: {'precision': 0.6259613326087314, 'recall': 0.6276791974092746, 'fbeta': 0.6244216654795454, 'support': array([778, 616]), 'bac': 0.6276791974092746, 'confusion_matrix': array([[474, 304],
       [218, 398]])}
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 1800x1200 with 2 Axes>

for name, clf in clfs.items():
    logger.info(f"===== Training with {name} =====")
    clf = clone(clf)
    
    ret = sklearn.model_selection.cross_validate(clf, X, y, cv=5, n_jobs=-1, return_estimator=True)
    logger.info(f"CV score (shuffled):   {ret['test_score']}")
    
    clf_trained = ret["estimator"][0]
    
    plot_confusion_matrix(clf_trained, X, y,
                          display_labels=['code', 'prose'],
                          cmap=plt.cm.Blues,
                          normalize='true')
    plt.show()

INFO: ===== Training with Cov + CSP + TS =====
INFO: CV score (shuffled):   [0.73870968 0.62648009 0.6846071  0.61679225 0.63832078]
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 1800x1200 with 2 Axes>

# LORO/LOGO split, where we leave out one subject for each fold
logo = LeaveOneGroupOut()
groups = subjs
#groups = subjs * 1000 + sessions

# LORO
for name, clf in clfs.items():
    clf = clone(clf)
    logger.info(f"===== Training with {name} =====")
    
    res = sklearn.model_selection.cross_validate(clf, X, y, cv=logo, groups=groups, n_jobs=-1, return_estimator=True)
    score = res["test_score"]
    logger.info(f"CV score (LORO):   {np.mean(score) :.3f} (mean), {np.std(score) :.3f} (std)")
    logger.info(f"                   {score}")

    fig, axs = plt.subplots(1, len(set(groups)), figsize=(20, 4))
        
    for idx, (train, test) in enumerate(logo.split(X, y, groups)):  # type: ignore
        ax = axs[idx]
        clf = clone(clf)
        clf.fit(X[train], y[train])
    
        plot_confusion_matrix(clf, X[test], y[test],
                              display_labels=['code', 'prose'],
                              cmap=plt.cm.Blues,
                              normalize='true', ax=ax)
    plt.show()

INFO: ===== Training with Cov + CSP + TS =====
INFO: CV score (LORO):   0.610 (mean), 0.081 (std)
INFO:                    [0.62472406 0.81541582 0.63934426 0.61832061 0.63318113 0.60264901
 0.58119658 0.52664577 0.52380952 0.53389831]
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 6000x1200 with 20 Axes>

# Grid search for best params
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import FeatureUnion

pipeline_csp = make_pipeline(
    Covariances(),
    CSP(log=False),
    TangentSpace()
)

# TODO: add frequency based features
pipeline = make_pipeline(
    FeatureUnion([("csp", pipeline_csp),]),
    LogisticRegression(dual=dual),
)

#print(pipeline.get_params().keys())
param_grid = dict(featureunion__csp__csp__nfilter=[3, 4, 6])

grid_search = GridSearchCV(pipeline, param_grid=param_grid, cv=logo.split(X, y, groups=groups), n_jobs=-1, verbose=10)
grid_search.fit(X, y)

Fitting 10 folds for each of 3 candidates, totalling 30 fits

GridSearchCV(cv=<generator object BaseCrossValidator.split at 0x7f9eed8ce820>,
             estimator=Pipeline(steps=[('featureunion',
                                        FeatureUnion(transformer_list=[('csp',
                                                                        Pipeline(steps=[('covariances',
                                                                                         Covariances()),
                                                                                        ('csp',
                                                                                         CSP(log=False)),
                                                                                        ('tangentspace',
                                                                                         TangentSpace())]))])),
                                       ('logisticregression',
                                        LogisticRegression())]),
             n_jobs=-1,
             param_grid={'featureunion__csp__csp__nfilter': [3, 4, 6]},
             verbose=10)

best = grid_search.best_estimator_
#print(pipeline.get_params().keys())
# Why doesn't the best estimator use one of the values in param_grid?
print(best.get_params()['featureunion__csp__csp__nfilter'])

4

Learning curves

Now to check the learning curves and see if the train and validation scores converge.

Note: Performance is currently terrible as there isn't enough data for the model to learn to generalize across subjects (easily seen by changing to shuffled CV).

A great example of how to plot learning curves is available here: https://scikit-learn.org/stable/auto_examples/model_selection/plot_learning_curve.html

from eegclassify.util import powspace

# TODO: Add one participant to the training set each run instead of randomly sampled by trainsizes
# TODO: Evaluate 

for name, clf in clfs.items():
    logger.info(f"===== Training with {name} =====")
    clf = clone(clf)
    
    # We create shuffled versions of the dataset to ensure that all stimuli of the same type aren't in sequence
    # (as is the case for subject 1 which didn't have shuffled stimuli)
    x_l, y_l, groups_l = unison_shuffled_copies(X, y, groups)
    #x_l, y_l, groups_l = X, y, groups
    
    # We build a train_sizes to train across several sample sizes
    groups_c: dict = Counter(groups_l)
    largest_group, largest_group_count = max(groups_c.items(), key=lambda g: g[1])
    largest_loro_train = len(y) - largest_group_count
    smallest_train = 10
    
    train_sizes = np.floor(powspace(smallest_train, largest_loro_train, power=4, num=20)) / largest_loro_train
    # print(train_sizes)
    
    # Compute the learning curve
    train_sizes, train_scores, valid_scores = learning_curve(
        clf, x_l, y_l, groups=groups_l, 
        train_sizes=train_sizes, 
        cv=logo, n_jobs=-1
    )
    
    m = np.mean(train_scores, axis=1)
    std = np.std(train_scores, axis=1)
    plt.plot(train_sizes, m, label="training score", marker='.')
    plt.fill_between(train_sizes, m-std, m+std, alpha=0.2)
    
    m = np.mean(valid_scores, axis=1)
    std = np.std(valid_scores, axis=1)
    plt.plot(train_sizes, m, label="validation score", marker='.')
    plt.fill_between(train_sizes, m-std, m+std, alpha=0.2)
    
    plt.axhline(0.5, color='grey', linestyle='--', linewidth=0.8)
    plt.ylim(0.4, 1)
    plt.xlim(train_sizes[0], train_sizes[-1])
        
    plt.legend()
    plt.show()

INFO: ===== Training with Cov + CSP + TS =====

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 1800x1200 with 1 Axes>

LORO validation

# Use a specific subject/session for validation
print(set(subjs))

def make_split(subj_val, sess_val=None, excl_subjs=set()):
    # Exclude bad subjects
    incl_subjs = np.isin(subjs, excl_subjs) ^ True

    train_idx = ((subjs != subj_val) if subj_val is not None else True) & \
                ((sessions != sess_val) if sess_val is not None else True)
    train = np.argwhere(incl_subjs & train_idx).flatten()
    test = np.argwhere(incl_subjs & ~train_idx).flatten()
    assert len(train)
    assert len(test)
    return train, test
    
    
subj_val = 6
excl_subjs = {3, 4, 8, 9, 10}
train, test = make_split(subj_val=subj_val, excl_subjs=excl_subjs)

print(Counter(subjs))

{0, 1, 3, 4, 5, 6, 7, 8, 9, 10}
Counter({0: 1359, 5: 657, 6: 604, 1: 493, 3: 427, 4: 393, 8: 319, 7: 234, 10: 118, 9: 42})

# Fit the first classifier on the training set
clf = clone(list(clfs.values())[0])
clf.fit(X[train], y[train])

predicted = clf.predict(X)
predicted_proba = clf.predict_proba(X)

import eegclassify.plot
import importlib
importlib.reload(eegclassify.plot)

plt.rcParams["font.family"] = "serif"

def whichsplit(idx):
    if idx in train:
        return "train"
    elif idx in test:
        return "test"
    else:
        return None

cmap_class = lambda s: get_cmap('tab10')({"code": 0, "prose": 1}[s])
cmap_split = lambda s: {"train": '#4c92fc', "test": '#ffd000'}[s] if s else 'w'
img_to_id = {img: idx for idx, img in enumerate(sorted(set(imgs)))}

tl = eegclassify.plot.TimelineFigure(title="", figsize=(16, 4))
tl.add_chunked(imgs, lambda v: cmap_discrete(img_to_id[v] % 10), "Image")
tl.add_chunked(y, cmap_class, "Label")
tl.add_chunked(predicted, cmap_class, "Predicted")
#tl.add_chunked(predicted_proba[:, 1], cmap, "predicted(prob)")
tl.add_chunked(zip(y, predicted), lambda v: 'g' if v[0] == v[1] else '#dddddd', "Correct?")
tl.add_chunked(subjs, cmap_discrete, "Subject", show_label=True)
tl.add_chunked(map(whichsplit, range(len(imgs))), cmap_split, "Split")

thres = 5
bad = 40
quality = [1 - (np.clip(np.std(xx), thres, bad) - thres) / (bad - thres) for xx in X]

#tl.add_chunked(quality, cmap, "quality (stddev)")
tl.add_chunked(map(np.std, X), lambda v: 'g' if v < (thres + bad) / 2 else '#dddddd', "Quality")  # thres

tl.plot()
plt.rcParams["font.family"] = "sans"

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 4800x1200 with 1 Axes>

from sklearn.metrics import plot_roc_curve

ax = plt.gca()
ax.plot([0, 1], [0, 1], '--', label="Chance")
for clf in clfs.values():
    print(clf)
    clf = clone(clf)
    clf.fit(X[train], y[train])
    plot_roc_curve(clf, X[test], y[test], ax=ax)
plt.xlim(0, 1)
plt.ylim(0, 1);

Pipeline(steps=[('covariances', Covariances()), ('csp', CSP(log=False)),
                ('tangentspace', TangentSpace()),
                ('logisticregression', LogisticRegression())])

/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function `plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: RocCurveDisplay.from_predictions or RocCurveDisplay.from_estimator.
  warnings.warn(msg, category=FutureWarning)
INFO: font.family must be one of (%s) when text.usetex is True. serif will be used by default.
INFO: font.family must be one of (%s) when text.usetex is True. serif will be used by default.

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    251         try:
--> 252             report = subprocess.check_output(
    253                 command, cwd=cwd if cwd is not None else self.texcache,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in check_output(timeout, *popenargs, **kwargs)
    414 
--> 415     return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
    416                **kwargs).stdout

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in run(input, capture_output, timeout, check, *popenargs, **kwargs)
    492 
--> 493     with Popen(*popenargs, **kwargs) as process:
    494         try:

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors, text)
    857 
--> 858             self._execute_child(args, executable, preexec_fn, close_fds,
    859                                 pass_fds, cwd, env,

/opt/hostedtoolcache/Python/3.8.12/x64/lib/python3.8/subprocess.py in _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
   1703                         err_msg = os.strerror(errno_num)
-> 1704                     raise child_exception_type(errno_num, err_msg, err_filename)
   1705                 raise child_exception_type(err_msg)

FileNotFoundError: [Errno 2] No such file or directory: 'latex'

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    339                 pass
    340             else:
--> 341                 return printer(obj)
    342             # Finally look for special method names
    343             method = get_real_method(obj, self.print_method)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in <lambda>(fig)
    251 
    252     if 'png' in formats:
--> 253         png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png', **kwargs))
    254     if 'retina' in formats or 'png2x' in formats:
    255         png_formatter.for_type(Figure, lambda fig: retina_figure(fig, **kwargs))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/IPython/core/pylabtools.py in print_figure(fig, fmt, bbox_inches, **kwargs)
    135         FigureCanvasBase(fig)
    136 
--> 137     fig.canvas.print_figure(bytes_io, **kw)
    138     data = bytes_io.getvalue()
    139     if fmt == 'svg':

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2228                        else suppress())
   2229                 with ctx:
-> 2230                     self.figure.draw(renderer)
   2231 
   2232             if bbox_inches:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     72     @wraps(draw)
     73     def draw_wrapper(artist, renderer, *args, **kwargs):
---> 74         result = draw(artist, renderer, *args, **kwargs)
     75         if renderer._rasterizing:
     76             renderer.stop_rasterizing()

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/figure.py in draw(self, renderer)
   2788 
   2789             self.patch.draw(renderer)
-> 2790             mimage._draw_list_compositing_images(
   2791                 renderer, self, artists, self.suppressComposite)
   2792 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/_api/deprecation.py in wrapper(*inner_args, **inner_kwargs)
    429                          else deprecation_addendum,
    430                 **kwargs)
--> 431         return func(*inner_args, **inner_kwargs)
    432 
    433     return wrapper

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axes/_base.py in draw(self, renderer, inframe)
   2919             renderer.stop_rasterizing()
   2920 
-> 2921         mimage._draw_list_compositing_images(renderer, self, artists)
   2922 
   2923         renderer.close_group('axes')

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/image.py in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    130     if not_composite or not has_images:
    131         for a in artists:
--> 132             a.draw(renderer)
    133     else:
    134         # Composite any adjacent images together

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/artist.py in draw_wrapper(artist, renderer, *args, **kwargs)
     49                 renderer.start_filter()
     50 
---> 51             return draw(artist, renderer, *args, **kwargs)
     52         finally:
     53             if artist.get_agg_filter() is not None:

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in draw(self, renderer, *args, **kwargs)
   1140 
   1141         ticks_to_draw = self._update_ticks()
-> 1142         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,
   1143                                                                 renderer)
   1144 

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in _get_tick_bboxes(self, ticks, renderer)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/axis.py in <listcomp>(.0)
   1066     def _get_tick_bboxes(self, ticks, renderer):
   1067         """Return lists of bboxes for ticks' label1's and label2's."""
-> 1068         return ([tick.label1.get_window_extent(renderer)
   1069                  for tick in ticks if tick.label1.get_visible()],
   1070                 [tick.label2.get_window_extent(renderer)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in get_window_extent(self, renderer, dpi)
    901 
    902         with cbook._setattr_cm(self.figure, dpi=dpi):
--> 903             bbox, info, descent = self._get_layout(self._renderer)
    904             x, y = self.get_unitless_position()
    905             x, y = self.get_transform().transform((x, y))

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/text.py in _get_layout(self, renderer)
    304 
    305         # Full vertical extent of font, including ascenders and descenders:
--> 306         _, lp_h, lp_d = renderer.get_text_width_height_descent(
    307             "lp", self._fontproperties,
    308             ismath="TeX" if self.get_usetex() else False)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in get_text_width_height_descent(self, s, prop, ismath)
    227             texmanager = self.get_texmanager()
    228             fontsize = prop.get_size_in_points()
--> 229             w, h, d = texmanager.get_text_width_height_descent(
    230                 s, fontsize, renderer=self)
    231             return w, h, d

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in get_text_width_height_descent(self, tex, fontsize, renderer)
    397         else:
    398             # use dviread.
--> 399             dvifile = self.make_dvi(tex, fontsize)
    400             with dviread.Dvi(dvifile, 72 * dpi_fraction) as dvi:
    401                 page, = dvi

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in make_dvi(self, tex, fontsize)
    289             # and thus replace() works atomically.
    290             with TemporaryDirectory(dir=Path(dvifile).parent) as tmpdir:
--> 291                 self._run_checked_subprocess(
    292                     ["latex", "-interaction=nonstopmode", "--halt-on-error",
    293                      texfile], tex, cwd=tmpdir)

~/work/thesis/thesis/.venv/lib/python3.8/site-packages/matplotlib/texmanager.py in _run_checked_subprocess(self, command, tex, cwd)
    254                 stderr=subprocess.STDOUT)
    255         except FileNotFoundError as exc:
--> 256             raise RuntimeError(
    257                 'Failed to process string with tex because {} could not be '
    258                 'found'.format(command[0])) from exc

RuntimeError: Failed to process string with tex because latex could not be found

<Figure size 1800x1200 with 1 Axes>

# Aggregate the window classification into epoch classification by taking the mean of window predictions
# TODO: Train a classifier on the probability outputs
logger.info(f"Using {clf}")

# Window-level performance
logger.info("=== Window performance ===")
perf = _performance(y[test], clf.predict(X[test]))
logger.info(perf)

# Epoch-aggregated performance
ys_epoch, votes = eegclassify.util.aggregate_windows_to_epochs(clf, X, y, subjs, imgs, test)
perf = _performance(ys_epoch, votes)
logger.info("=== Epoch performance ===")
logger.info(perf)

INFO: Using Pipeline(steps=[('covariances', Covariances()), ('csp', CSP(log=False)),
                ('tangentspace', TangentSpace()),
                ('logisticregression', LogisticRegression())])
INFO: === Window performance ===
INFO: {'precision': 0.701845274192151, 'recall': 0.6286114883140683, 'fbeta': 0.5734613890752445, 'support': array([327, 277]), 'bac': 0.6286114883140683, 'confusion_matrix': array([[103, 224],
       [ 16, 261]])}
INFO: === Epoch performance ===
INFO: {'precision': 0.8035714285714286, 'recall': 0.6333333333333333, 'fbeta': 0.5883040935672514, 'support': array([15, 17]), 'bac': 0.6333333333333333, 'confusion_matrix': array([[ 4, 11],
       [ 0, 17]])}

Compare to bandpower features

Now we're interested in if our classifiers outperform basic bandpower features with common classifers such as SVM and random forest.

This code is largely based on the training for device activity in eegclassify.main._train_features.

from eegclassify.main import df_to_vectors
importlib.reload(eegclassify.features)

print(X.shape, y.shape)
raw_data = [X[i, :, :].T for i in range(X.shape[0])]
assert len(raw_data) == len(y)
assert raw_data[0].shape[1] == 4

df_feat = pd.DataFrame({"class": pd.Series(y, dtype="category"), "raw_data": raw_data})
df_feat = features.compute_features(df_feat, ratios=True)
X_feat, y_feat = df_to_vectors(df_feat)

INFO: Computing features...

(4646, 4, 1280) (4646,)

/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/outdated/utils.py:14: OutdatedPackageWarning: The package yasa is out of date. Your version is 0.5.1, the latest is 0.6.1.
Set the environment variable OUTDATED_IGNORE=1 to disable these warnings.
  return warn(
INFO: Done computing features!
INFO: Classes: {0: 'code', 1: 'prose'}

print(df_feat.iloc[0]["bandpower"])

[['TP9', 0.22772545684310566, 0.3080763221440419, 0.09296107698163092, 0.05618869534545469, 0.20049993432137617, 0.11454851436439037, 0.739185196896216, 2.449686086232673, 4.052869628721985, 1.1357881867337194, 1.9880262795786947, 1.3528409445953817, 3.314035638861173, 5.482887976842184, 1.5365407634011208, 2.689483349945684, 0.40821556917845, 0.3017469058792734, 1.6544444822947981, 0.46364642111366483, 0.8115432792599332, 0.2467387534287245, 0.18238563403513858, 0.6044324912087407, 0.28024296135374926, 0.4905231260067921, 0.8804458539719305, 0.6508125419246984, 2.156815958156282, 3.5683322612970287, 1.7503494954423038, 0.5030114592911323, 0.3718186245771686, 1.2322201730410796, 2.0386398662601555, 0.5713144732545573], ['AF7', 0.38866623519926774, 0.2448498438503438, 0.11051636254151122, 0.06116849691698342, 0.1345879819265577, 0.060211079565336015, 1.5873656649616932, 3.516820733701584, 6.354026251891677, 2.8878227434256063, 6.455061726264511, 0.6299745686033421, 2.2155076245689447, 4.0028749469297695, 1.8192548869924663, 4.066524726311431, 0.2843477321482528, 0.4513638269218608, 1.8067529547358612, 0.821145847939217, 1.8354821627402997, 0.15738052698512017, 0.249820444869757, 0.5534791003821523, 0.4544870651999374, 1.0159010162009883, 0.34628164151577234, 0.5496755881487106, 1.2178104565828873, 2.20028264073936, 2.23526937065651, 0.1549171862960157, 0.24591022243876945, 0.5448159727725389, 0.9843478685940772, 0.4473733739331358], ['AF8', 0.2740763940110105, 0.16681654881134797, 0.10940099047234292, 0.1021488443088578, 0.22312996620920442, 0.12442725618723621, 1.642980843111448, 2.505246002140157, 2.6831081238894066, 1.2283262471076577, 2.2027038320171957, 0.6086498233942994, 1.5248175367618813, 1.6330732857530972, 0.7476205533726584, 1.3406752983472126, 0.3991623972838315, 0.6558161720277763, 1.070995870903419, 0.4903016494421446, 0.8792365420942659, 0.3727020879614833, 0.6123423907083549, 0.9337104158547953, 0.45779975699491693, 0.8209523173535692, 0.8141159584879848, 1.337576923867074, 2.039560750280526, 2.184361142007123, 1.7932563414678364, 0.45398749730426463, 0.7458927610830169, 1.1373503626431243, 1.2180975421612925, 0.5576447587975453], ['TP10', 0.3067785285135468, 0.35664209354685517, 0.12440402332401249, 0.03495806988019976, 0.12724467951550733, 0.04997260521987851, 0.8601859793458246, 2.4659855872549774, 8.77561403031882, 2.410934034190087, 6.138934065248893, 1.1625392926777354, 2.8668051403608907, 10.201996127619653, 2.802805546800023, 7.136752066009702, 0.40551737413565114, 0.34882035961262225, 3.558663958002867, 0.9776756387590363, 2.489444422131629, 0.11395214016308208, 0.09802003328473344, 0.28100433527901947, 0.2747310929879737, 0.6995446750551612, 0.414777005848662, 0.35678536498606017, 1.022834118347574, 3.63992291197915, 2.5462886906862905, 0.16289472885215892, 0.14011976186796687, 0.4016960535892314, 1.4295012679799857, 0.39272844577983584]]

from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC
from sklearn.linear_model import LogisticRegression

importlib.reload(eegclassify.util)

clfs_feat = {
    "SVM-2000": make_pipeline(StandardScaler(), LinearSVC(max_iter=5000)),
    "LR": make_pipeline(StandardScaler(), LogisticRegression(max_iter=5000)),
    "randomforest-20": make_pipeline(RandomForestClassifier(n_estimators=20)),
    #"randomforest-50": make_pipeline(RandomForestClassifier(n_estimators=50)),
    "randomforest-100": make_pipeline(RandomForestClassifier(n_estimators=100)),
    
    # Old classifiers:
    
    # Performs basically the same as when using StandardScaler
    #"SVM-noscale": make_pipeline(LinearSVC(max_iter=10000)),
    
    # 2000 iterations is more than enough
    #"SVM-20000": make_pipeline(StandardScaler(), LinearSVC(max_iter=20000)),
}

scores_window: Dict[str, List[float]] = defaultdict(list)
scores_epoch: Dict[str, List[float]] = defaultdict(list)

all_subjs: Set[int] = set(subjs)
excl_subjs = {3, 4, 8, 9, 10}
incl_subjs: Set[int] = all_subjs - excl_subjs
    
for subj in incl_subjs:
    logger.info(f"*** Running split with subj {subj} as validation set ***")
    train, test = make_split(subj_val=subj, excl_subjs=excl_subjs)
    
    for clf_name, clf in clfs_feat.items():
        logger.info(f"=== Training with {clf_name} ===")
        clf.fit(X_feat[train], y_feat[train])
        
        y_feat_pred = clf.predict(X_feat[test])
        perf = _performance(y_feat[test], y_feat_pred)
        logger.info(f"Window performance: {perf['bac']}")
        scores_window[clf_name] += [perf['bac']]
        
        ys_epoch, votes = eegclassify.util.aggregate_windows_to_epochs(clf, X_feat, y_feat, subjs, imgs, test)
        perf = _performance(ys_epoch, votes)
        logger.info(f"Epoch performance: {perf['bac']}")
        scores_epoch[clf_name] += [perf['bac']]
            
np.set_printoptions(precision=3)
for clf_name in clfs_feat.keys():
    print(f"=== Results for {clf_name} ===")
    print(f'Median & mean BAC (window): {np.median(scores_window[clf_name])} & {np.mean(scores_window[clf_name])} ({np.array(scores_window[clf_name])})')
    print(f'Median & mean BAC (epoch):  {np.median(scores_epoch[clf_name])} & {np.mean(scores_epoch[clf_name])} ({np.array(scores_epoch[clf_name])})')

INFO: *** Running split with subj 0 as validation set ***
INFO: === Training with SVM-2000 ===
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/svm/_base.py:1199: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.
  warnings.warn(
INFO: Window performance: 0.5214818828546958
WARNING: Classifier does not support predict_proba, using majority vote
INFO: Epoch performance: 0.6017369727047146
INFO: === Training with LR ===
INFO: Window performance: 0.517485710789883
INFO: Epoch performance: 0.521505376344086
INFO: === Training with randomforest-20 ===
INFO: Window performance: 0.5487865095873171
INFO: Epoch performance: 0.5607940446650124
INFO: === Training with randomforest-100 ===
INFO: Window performance: 0.5523172991207985
INFO: Epoch performance: 0.5095119933829612
INFO: *** Running split with subj 1 as validation set ***
INFO: === Training with SVM-2000 ===
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/svm/_base.py:1199: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.
  warnings.warn(
INFO: Window performance: 0.7121803100622099
WARNING: Classifier does not support predict_proba, using majority vote
INFO: Epoch performance: 0.8136363636363637
INFO: === Training with LR ===
INFO: Window performance: 0.7121885388894376
INFO: Epoch performance: 0.9090909090909092
INFO: === Training with randomforest-20 ===
INFO: Window performance: 0.6633751357756492
INFO: Epoch performance: 0.8090909090909091
INFO: === Training with randomforest-100 ===
INFO: Window performance: 0.7180886080115861
INFO: Epoch performance: 0.8136363636363637
INFO: *** Running split with subj 5 as validation set ***
INFO: === Training with SVM-2000 ===
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/svm/_base.py:1199: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.
  warnings.warn(
INFO: Window performance: 0.6238215230138351
WARNING: Classifier does not support predict_proba, using majority vote
INFO: Epoch performance: 0.75
INFO: === Training with LR ===
INFO: Window performance: 0.6293977311922738
INFO: Epoch performance: 0.625
INFO: === Training with randomforest-20 ===
INFO: Window performance: 0.5997585559345419
INFO: Epoch performance: 0.5833333333333334
INFO: === Training with randomforest-100 ===
INFO: Window performance: 0.6210956961637221
INFO: Epoch performance: 0.6666666666666666
INFO: *** Running split with subj 6 as validation set ***
INFO: === Training with SVM-2000 ===
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/svm/_base.py:1199: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.
  warnings.warn(
INFO: Window performance: 0.7263438545358196
WARNING: Classifier does not support predict_proba, using majority vote
INFO: Epoch performance: 0.803921568627451
INFO: === Training with LR ===
INFO: Window performance: 0.7312070126629793
INFO: Epoch performance: 0.803921568627451
INFO: === Training with randomforest-20 ===
INFO: Window performance: 0.7515538921825147
INFO: Epoch performance: 0.9
INFO: === Training with randomforest-100 ===
INFO: Window performance: 0.7380463462833549
INFO: Epoch performance: 0.8372549019607842
INFO: *** Running split with subj 7 as validation set ***
INFO: === Training with SVM-2000 ===
/home/runner/work/thesis/thesis/.venv/lib/python3.8/site-packages/sklearn/svm/_base.py:1199: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.
  warnings.warn(
INFO: Window performance: 0.729067657478614
WARNING: Classifier does not support predict_proba, using majority vote
INFO: Epoch performance: 0.8333333333333333
INFO: === Training with LR ===
INFO: Window performance: 0.7321351421411907
INFO: Epoch performance: 0.9
INFO: === Training with randomforest-20 ===
INFO: Window performance: 0.6553616175581094
INFO: Epoch performance: 0.8333333333333333
INFO: === Training with randomforest-100 ===
INFO: Window performance: 0.6721679771882831
INFO: Epoch performance: 0.75

=== Results for SVM-2000 ===
Median & mean BAC (window): 0.7121803100622099 & 0.6625790455890349 ([0.521 0.712 0.624 0.726 0.729])
Median & mean BAC (epoch):  0.803921568627451 & 0.7605256476603725 ([0.602 0.814 0.75  0.804 0.833])
=== Results for LR ===
Median & mean BAC (window): 0.7121885388894376 & 0.6644828271351528 ([0.517 0.712 0.629 0.731 0.732])
Median & mean BAC (epoch):  0.803921568627451 & 0.7519035708124892 ([0.522 0.909 0.625 0.804 0.9  ])
=== Results for randomforest-20 ===
Median & mean BAC (window): 0.6553616175581094 & 0.6437671422076265 ([0.549 0.663 0.6   0.752 0.655])
Median & mean BAC (epoch):  0.8090909090909091 & 0.7373103240845176 ([0.561 0.809 0.583 0.9   0.833])
=== Results for randomforest-100 ===
Median & mean BAC (window): 0.6721679771882831 & 0.660343185353549 ([0.552 0.718 0.621 0.738 0.672])
Median & mean BAC (epoch):  0.75 & 0.7154139851293552 ([0.51  0.814 0.667 0.837 0.75 ])

# Using a permutation test to calculate the p-value
from sklearn.model_selection import permutation_test_score
from time import time

def permutation_test():
    # This takes a long time to run! (especially with high n_permutations)
    clf = list(clfs.values())[0]
    print(clf)
    _start = time()
    score, perm_scores, pvalue = permutation_test_score(
        clf, X, y, scoring="balanced_accuracy", cv=logo, groups=groups, n_permutations=10
    )
    print(f"Took {(time() - _start)/60} minutes")
    print(score, pvalue)
    
# permutation_test()