From b16827ca47d1d6953c1eddc76127e60af1f28f57 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 11:37:37 +0200 Subject: [PATCH 01/34] rm travis CIs --- .travis.yml | 90 ----------------------------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cbee697..0000000 --- a/.travis.yml +++ /dev/null @@ -1,90 +0,0 @@ -language: python - -sudo: false - -env: -# - PYTHON=2.7 TEST_LOCATION=src -- PYTHON=3.5 TEST_LOCATION=src - -before_install: -- openssl aes-256-cbc -K $encrypted_73bd2a498087_key -iv $encrypted_73bd2a498087_iv - -in .s3cfg.enc -out $HOME/.s3cfg -d -- python -c "import os.path as op; assert op.exists(op.join(op.expanduser('~'), '.s3cfg'))" -- wget -q http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh -- chmod +x miniconda.sh -- "./miniconda.sh -b -p /home/travis/miniconda" -- export PATH=/home/travis/miniconda/bin:$PATH -- conda update --yes --quiet conda -- export DISPLAY=:99.0 -- "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile - --background --exec /usr/bin/Xvfb -- :99 -screen 0 1400x900x24 -ac +extension GLX - +render -noreset" -- conda create -n get-hcp-env --yes pip python=2.7 -- source activate get-hcp-env -- conda install scipy --yes --quiet -- pip install -q s3cmd mne -- python setup.py develop -- export MNE_HCP_N_RUNS=3 -- mkdir $HOME/mne-hcp-data -- s3cmd get s3://mne-hcp-testing-data/mne-hcp-testing.tar.gz $HOME/mne-hcp-data/ -- (cd $HOME/mne-hcp-data/ && tar -xvzf mne-hcp-testing.tar.gz) -- find $HOME/mne-hcp-data/mne-hcp-testing/105923 -name "*" > output.txt -- cat output.txt | wc -l -- head output.txt -# get fsaverage -- pushd ~ -- wget --quiet http://faculty.washington.edu/larsoner/fsaverage_min.zip -- mkdir subjects -- cd subjects -- unzip ../fsaverage_min.zip -- cd .. -- export SUBJECTS_DIR="${PWD}/subjects" -- popd - -install: -- conda create -n testenv --yes pip python=$PYTHON -- source activate testenv -- ENSURE_PACKAGES="numpy$NUMPY scipy$SCIPY matplotlib$MPL libpng$LIBPNG scikit-learn$SKLEARN" -- conda install --yes --quiet $ENSURE_PACKAGES nose coverage -- if [ "${PYTHON}" == "3.5" ]; then - conda install --yes --quiet $ENSURE_PACKAGES ipython; - else - conda install --yes --quiet $ENSURE_PACKAGES ipython==1.1.0; - pip install nitime faulthandler; - if [ "${NUMPY}" != "=1.8" ]; then - conda install --yes --quiet $ENSURE_PACKAGES mayavi$MAYAVI; - pip install pysurfer; - fi; - fi; -- pip install -q codecov nose-timer mne nibabel -- NP_VERSION=`python -c 'import numpy; print(numpy.__version__)'` -- if [ -n "$NUMPY" ] && [ "${NUMPY:(-3)}" != "${NP_VERSION::3}" ]; then echo "Incorrect - numpy version $NP_VERSION"; exit 1; fi; -- SP_VERSION=`python -c 'import scipy; print(scipy.__version__)'` -- if [ -n "$SCIPY" ] && [ "${SCIPY:(-4)}" != "${SP_VERSION::4}" ]; then echo "Incorrect - scipy version $SP_VERSION"; exit 1; fi; -- MPL_VERSION=`python -c 'import matplotlib; print(matplotlib.__version__)'` -- if [ -n "$MPL" ] && [ "${MPL:(-3)}" != "${MPL_VERSION::3}" ]; then echo "Incorrect - matplotlib version $MPL_VERSION"; exit 1; fi; -- export MNE_LOGGING_LEVEL=warning -- python setup.py build -- python setup.py install -- SRC_DIR=$(pwd) -- cd ~ -- HCP_DIR=$(python -c 'import hcp;print(hcp.__path__[0])') -- if [ "${TEST_LOCATION}" == "install" ]; then - ln -s ${SRC_DIR}/setup.cfg ${HCP_DIR}/../setup.cfg; - ln -s ${SRC_DIR}/.coveragerc ${HCP_DIR}/../.coveragerc; cd ${MNE_DIR}/../; - COVERAGE=; - else cd ${SRC_DIR}; - COVERAGE=--with-coverage; fi; -- export MNE_HCP_N_RUNS=1 - -script: -- nosetests --with-timer --timer-top-n 30 --verbosity=2 $COVERAGE -- if [ "${DEPS}" == "nodata" ]; then make flake; fi; -- if [ "${DEPS}" == "nodata" ]; then make codespell-error; fi; - -after_success: -- if [ "${TEST_LOCATION}" == "src" ]; then echo "Running codecov"; cd ${SRC_DIR}; - codecov; fi; From 382d41c2df1fcb4d7c07df010bb7fa146da3fad4 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 12:02:44 +0200 Subject: [PATCH 02/34] clean-up imports and files --- hcp/__init__.py | 29 +- hcp/anatomy.py | 167 ++-- hcp/io/__init__.py | 13 +- hcp/io/file_mapping/__init__.py | 5 +- hcp/io/file_mapping/file_mapping.py | 413 +++++----- hcp/io/file_mapping/s3.py | 73 +- .../file_mapping/tests/test_file_mapping.py | 100 ++- hcp/io/read.py | 730 +++++++----------- hcp/io/tests/__init__.py | 1 - hcp/io/tests/test_io.py | 197 ++--- hcp/preprocessing.py | 138 ++-- hcp/viz.py | 50 +- 12 files changed, 911 insertions(+), 1005 deletions(-) diff --git a/hcp/__init__.py b/hcp/__init__.py index bd6323f..d00640e 100644 --- a/hcp/__init__.py +++ b/hcp/__init__.py @@ -1,20 +1,17 @@ # Author: Denis A. Engemann # License: BSD (3-clause) -from . import viz -from . import preprocessing -from . import anatomy -from .anatomy import make_mne_anatomy, compute_forward_stack -from . import tests +from . import anatomy, preprocessing, tests, viz +from .anatomy import compute_forward_stack, make_mne_anatomy +from .io import ( + file_mapping, + read_annot, + read_epochs, + read_evokeds, + read_ica, + read_info, + read_raw, + read_trial_info, +) -from .io import read_raw -from .io import read_epochs -from .io import read_evokeds -from .io import read_info -from .io import read_annot -from .io import read_ica -from .io import read_trial_info - -from .io import file_mapping - -__version__ = '0.1.dev12' +__version__ = "0.1.dev12" diff --git a/hcp/anatomy.py b/hcp/anatomy.py index fb26fc7..327f3d0 100644 --- a/hcp/anatomy.py +++ b/hcp/anatomy.py @@ -4,21 +4,25 @@ import shutil import sys -import numpy as np -from scipy import linalg - import mne -from mne.io.pick import _pick_data_channels, pick_info -from mne import write_trans, write_surface +import numpy as np +from mne import write_surface, write_trans +from mne._fiff.pick import _pick_data_channels, pick_info from mne.transforms import Transform, apply_trans from mne.utils import logger +from scipy import linalg from .io.file_mapping import get_file_paths -from .io.read import _read_trans_hcp, _get_head_model, read_info +from .io.read import _get_head_model, _read_trans_hcp, read_info -def make_mne_anatomy(subject, subjects_dir, recordings_path=None, - hcp_path=op.curdir, outputs=('label', 'mri', 'surf')): +def make_mne_anatomy( + subject, + subjects_dir, + recordings_path=None, + hcp_path=op.curdir, + outputs=("label", "mri", "surf"), +): """Extract relevant anatomy and create MNE friendly directory layout The function will create the following outputs by default: @@ -62,66 +66,64 @@ def make_mne_anatomy(subject, subjects_dir, recordings_path=None, for output in outputs: if not op.exists(op.join(this_subjects_dir, output)): os.makedirs(op.join(this_subjects_dir, output)) - if output == 'mri': - for suboutput in ['orig', 'transforms']: - if not op.exists( - op.join(this_subjects_dir, output, suboutput)): + if output == "mri": + for suboutput in ["orig", "transforms"]: + if not op.exists(op.join(this_subjects_dir, output, suboutput)): os.makedirs(op.join(this_subjects_dir, output, suboutput)) files = get_file_paths( - subject=subject, data_type='freesurfer', output=output, - hcp_path=hcp_path) + subject=subject, data_type="freesurfer", output=output, hcp_path=hcp_path + ) for source in files: match = [match for match in re.finditer(subject, source)][-1] - split_path = source[:match.span()[1] + 1] + split_path = source[: match.span()[1] + 1] target = op.join(this_subjects_dir, source.split(split_path)[-1]) - if (not op.isfile(target) and not op.islink(target) and - op.exists(source)): # don't link if it's not there. - if sys.platform != 'win32': + if ( + not op.isfile(target) and not op.islink(target) and op.exists(source) + ): # don't link if it's not there. + if sys.platform != "win32": os.symlink(source, target) else: shutil.copyfile(source, target) - logger.info('reading extended structural processing ...') + logger.info("reading extended structural processing ...") # Step 1 ################################################################# # transform head models to expected coordinate system # make hcp trans transforms_fname = get_file_paths( - subject=subject, data_type='meg_anatomy', output='transforms', - hcp_path=hcp_path) - transforms_fname = [k for k in transforms_fname if - k.endswith('transform.txt')][0] + subject=subject, data_type="meg_anatomy", output="transforms", hcp_path=hcp_path + ) + transforms_fname = [k for k in transforms_fname if k.endswith("transform.txt")][0] hcp_trans = _read_trans_hcp(fname=transforms_fname, convert_to_meter=False) # get RAS freesurfer trans c_ras_trans_fname = get_file_paths( - subject=subject, data_type='freesurfer', output='mri', - hcp_path=hcp_path) - c_ras_trans_fname = [k for k in c_ras_trans_fname if - k.endswith('c_ras.mat')][0] - logger.info('reading RAS freesurfer transform') + subject=subject, data_type="freesurfer", output="mri", hcp_path=hcp_path + ) + c_ras_trans_fname = [k for k in c_ras_trans_fname if k.endswith("c_ras.mat")][0] + logger.info("reading RAS freesurfer transform") # ceci n'est pas un .mat file ... with open(op.join(subjects_dir, c_ras_trans_fname)) as fid: - ras_trans = np.array([ - r.split() for r in fid.read().split('\n') if r], - dtype=np.float64) + ras_trans = np.array( + [r.split() for r in fid.read().split("\n") if r], dtype=np.float64 + ) - logger.info('Combining RAS transform and coregistration') + logger.info("Combining RAS transform and coregistration") ras_trans_m = linalg.inv(ras_trans) # and the inversion - logger.info('extracting head model') + logger.info("extracting head model") head_model_fname = get_file_paths( - subject=subject, data_type='meg_anatomy', output='head_model', - hcp_path=hcp_path)[0] + subject=subject, data_type="meg_anatomy", output="head_model", hcp_path=hcp_path + )[0] pnts, faces = _get_head_model(head_model_fname=head_model_fname) - logger.info('coregistring head model to MNE-HCP coordinates') - pnts = apply_trans(ras_trans_m.dot(hcp_trans['bti2spm']), pnts) + logger.info("coregistring head model to MNE-HCP coordinates") + pnts = apply_trans(ras_trans_m.dot(hcp_trans["bti2spm"]), pnts) - tri_fname = op.join(this_subjects_dir, 'bem', 'inner_skull.surf') + tri_fname = op.join(this_subjects_dir, "bem", "inner_skull.surf") if not op.exists(op.dirname(tri_fname)): os.makedirs(op.dirname(tri_fname)) write_surface(tri_fname, pnts, faces) @@ -129,24 +131,31 @@ def make_mne_anatomy(subject, subjects_dir, recordings_path=None, # Step 2 ################################################################# # write corresponding device to MRI transform - logger.info('extracting coregistration') + logger.info("extracting coregistration") # now convert to everything meter too here ras_trans_m[:3, 3] *= 1e-3 - bti2spm = hcp_trans['bti2spm'] + bti2spm = hcp_trans["bti2spm"] bti2spm[:3, 3] *= 1e-3 head_mri_t = Transform( # we're lying here for a good purpose - 'head', 'mri', np.dot(ras_trans_m, bti2spm)) # it should be 'ctf_head' - write_trans(op.join(this_recordings_path, - '%s-head_mri-trans.fif' % subject), head_mri_t) + "head", "mri", np.dot(ras_trans_m, bti2spm) + ) # it should be 'ctf_head' + write_trans( + op.join(this_recordings_path, "%s-head_mri-trans.fif" % subject), head_mri_t + ) @mne.utils.verbose -def compute_forward_stack(subjects_dir, - subject, - recordings_path, - info_from=(('data_type', 'rest'), ('run_index', 0)), - fwd_params=None, src_params=None, - hcp_path=op.curdir, n_jobs=1, verbose=None): +def compute_forward_stack( + subjects_dir, + subject, + recordings_path, + info_from=(("data_type", "rest"), ("run_index", 0)), + fwd_params=None, + src_params=None, + hcp_path=op.curdir, + n_jobs=1, + verbose=None, +): """ Convenience function for conducting standard MNE analyses. @@ -205,38 +214,44 @@ def compute_forward_stack(subjects_dir, info_from = dict(info_from) head_mri_t = mne.read_trans( - op.join(recordings_path, subject, '{}-head_mri-trans.fif'.format( - subject))) - - src_defaults = dict(subject='fsaverage', spacing='oct6', n_jobs=n_jobs, - surface='white', subjects_dir=subjects_dir, add_dist=True) - if 'fname' in mne.fixes._get_args(mne.setup_source_space): + op.join(recordings_path, subject, "{}-head_mri-trans.fif".format(subject)) + ) + + src_defaults = dict( + subject="fsaverage", + spacing="oct6", + n_jobs=n_jobs, + surface="white", + subjects_dir=subjects_dir, + add_dist=True, + ) + if "fname" in mne.fixes._get_args(mne.setup_source_space): # needed for mne-0.14 and below src_defaults.update(dict(fname=None)) else: # remove 'fname' argument (if necessary) when using mne-0.15+ - if 'fname' in src_params: - del src_params['fname'] + if "fname" in src_params: + del src_params["fname"] src_params = _update_dict_defaults(src_params, src_defaults) add_source_space_distances = False - if src_params['add_dist']: # we want the distances on the morphed space - src_params['add_dist'] = False + if src_params["add_dist"]: # we want the distances on the morphed space + src_params["add_dist"] = False add_source_space_distances = True src_fsaverage = mne.setup_source_space(**src_params) src_subject = mne.morph_source_spaces( - src_fsaverage, subject, subjects_dir=subjects_dir) + src_fsaverage, subject, subjects_dir=subjects_dir + ) if add_source_space_distances: # and here we compute them post hoc. - src_subject = mne.add_source_space_distances( - src_subject, n_jobs=n_jobs) + src_subject = mne.add_source_space_distances(src_subject, n_jobs=n_jobs) - bems = mne.make_bem_model(subject, conductivity=(0.3,), - subjects_dir=subjects_dir, - ico=None) # ico = None for morphed SP. + bems = mne.make_bem_model( + subject, conductivity=(0.3,), subjects_dir=subjects_dir, ico=None + ) # ico = None for morphed SP. bem_sol = mne.make_bem_solution(bems) - bem_sol['surfs'][0]['coord_frame'] = 5 + bem_sol["surfs"][0]["coord_frame"] = 5 info = read_info(subject=subject, hcp_path=hcp_path, **info_from) picks = _pick_data_channels(info, with_ref_meg=False) @@ -244,17 +259,21 @@ def compute_forward_stack(subjects_dir, # here we assume that as a result of our MNE-HCP processing # all other transforms in info are identity - for trans in ['dev_head_t', 'ctf_head_t']: + for trans in ["dev_head_t", "ctf_head_t"]: # 'dev_ctf_t' is not identity - assert np.sum(info[trans]['trans'] - np.eye(4)) == 0 + assert np.sum(info[trans]["trans"] - np.eye(4)) == 0 fwd = mne.make_forward_solution( - info, trans=head_mri_t, bem=bem_sol, src=src_subject, - n_jobs=n_jobs) - - return dict(fwd=fwd, src_subject=src_subject, - src_fsaverage=src_fsaverage, - bem_sol=bem_sol, info=info) + info, trans=head_mri_t, bem=bem_sol, src=src_subject, n_jobs=n_jobs + ) + + return dict( + fwd=fwd, + src_subject=src_subject, + src_fsaverage=src_fsaverage, + bem_sol=bem_sol, + info=info, + ) def _update_dict_defaults(values, defaults): diff --git a/hcp/io/__init__.py b/hcp/io/__init__.py index ebb3d2b..83a7230 100644 --- a/hcp/io/__init__.py +++ b/hcp/io/__init__.py @@ -1,5 +1,10 @@ -from .read import ( - read_ica, read_raw, read_info, read_annot, read_epochs, - read_evokeds, read_trial_info) - from . import file_mapping +from .read import ( + read_annot, + read_epochs, + read_evokeds, + read_ica, + read_info, + read_raw, + read_trial_info, +) diff --git a/hcp/io/file_mapping/__init__.py b/hcp/io/file_mapping/__init__.py index 9bf2e12..acf4c97 100644 --- a/hcp/io/file_mapping/__init__.py +++ b/hcp/io/file_mapping/__init__.py @@ -1,3 +1,2 @@ -from . file_mapping import get_file_paths -from . s3 import get_s3_keys_meg -from . s3 import get_s3_keys_anatomy +from .file_mapping import get_file_paths +from .s3 import get_s3_keys_anatomy, get_s3_keys_meg diff --git a/hcp/io/file_mapping/file_mapping.py b/hcp/io/file_mapping/file_mapping.py index ac65622..7301396 100644 --- a/hcp/io/file_mapping/file_mapping.py +++ b/hcp/io/file_mapping/file_mapping.py @@ -21,184 +21,188 @@ """ unprocessed = { - 'path': '{subject}/unprocessed/MEG/{run}-{kind}/4D', - 'patterns': ['c,rfDC', 'config'], + "path": "{subject}/unprocessed/MEG/{run}-{kind}/4D", + "patterns": ["c,rfDC", "config"], } preprocessed = { - 'meg': { - 'path': '{subject}/MEG/{kind}/{pipeline}/', - 'patterns': { - ('epochs', 'rmeg'): [ - '{subject}_MEG_{run}-{kind}_{context}preproc.mat'], - ('epochs', 'tmeg'): [ - '{subject}_MEG_{run}-{kind}_tmegpreproc_{onset}.mat', + "meg": { + "path": "{subject}/MEG/{kind}/{pipeline}/", + "patterns": { + ("epochs", "rmeg"): ["{subject}_MEG_{run}-{kind}_{context}preproc.mat"], + ("epochs", "tmeg"): [ + "{subject}_MEG_{run}-{kind}_tmegpreproc_{onset}.mat", ], - 'bads': [ - '{subject}_MEG_{run}-{kind}_baddata_badchannels.txt', - '{subject}_MEG_{run}-{kind}_baddata_badsegments.txt', - '{subject}_MEG_{run}-{kind}_baddata_manual_badchannels.txt', - '{subject}_MEG_{run}-{kind}_baddata_manual_badsegments.txt' + "bads": [ + "{subject}_MEG_{run}-{kind}_baddata_badchannels.txt", + "{subject}_MEG_{run}-{kind}_baddata_badsegments.txt", + "{subject}_MEG_{run}-{kind}_baddata_manual_badchannels.txt", + "{subject}_MEG_{run}-{kind}_baddata_manual_badsegments.txt", ], - 'ica': [ - '{subject}_MEG_{run}-{kind}_icaclass_vs.mat', - '{subject}_MEG_{run}-{kind}_icaclass_vs.txt', - '{subject}_MEG_{run}-{kind}_icaclass.mat', - '{subject}_MEG_{run}-{kind}_icaclass.txt' + "ica": [ + "{subject}_MEG_{run}-{kind}_icaclass_vs.mat", + "{subject}_MEG_{run}-{kind}_icaclass_vs.txt", + "{subject}_MEG_{run}-{kind}_icaclass.mat", + "{subject}_MEG_{run}-{kind}_icaclass.txt", ], - 'psd': ['{subject}_MEG_{run}-{kind}_powavg.mat'], - 'evoked': [ - (r'{subject}_MEG_{kind}_eravg_[{condition}]_{diff_modes}_' - '[{sensor_mode}].mat') - ], - 'tfr': [ - (r'{subject}_MEG_{kind}_tfavg_[{condition}]_{diff_modes}_' - '[{sensor_mode}].mat') + "psd": ["{subject}_MEG_{run}-{kind}_powavg.mat"], + "evoked": [ + ( + r"{subject}_MEG_{kind}_eravg_[{condition}]_{diff_modes}_" + "[{sensor_mode}].mat" + ) ], - 'trial_info': [ - '{subject}_MEG_{run}-{kind}_tmegpreproc_trialinfo.mat' - ] - } + "tfr": [ + ( + r"{subject}_MEG_{kind}_tfavg_[{condition}]_{diff_modes}_" + "[{sensor_mode}].mat" + ) + ], + "trial_info": ["{subject}_MEG_{run}-{kind}_tmegpreproc_trialinfo.mat"], + }, }, - 'meg_anatomy': { - 'path': '{subject}/MEG/anatomy', - 'patterns': { - 'transforms': [ - '{subject}_MEG_anatomy_transform.txt', + "meg_anatomy": { + "path": "{subject}/MEG/anatomy", + "patterns": { + "transforms": [ + "{subject}_MEG_anatomy_transform.txt", ], - 'head_model': [ - '{subject}_MEG_anatomy_headmodel.mat' + "head_model": ["{subject}_MEG_anatomy_headmodel.mat"], + "source_model": [ + "{subject}_MEG_anatomy_sourcemodel_2d.mat", + "{subject}_MEG_anatomy_sourcemodel_3d4mm.mat", + "{subject}_MEG_anatomy_sourcemodel_3d6mm.mat", + "{subject}_MEG_anatomy_sourcemodel_3d8mm.mat", ], - 'source_model': [ - '{subject}_MEG_anatomy_sourcemodel_2d.mat', - '{subject}_MEG_anatomy_sourcemodel_3d4mm.mat', - '{subject}_MEG_anatomy_sourcemodel_3d6mm.mat', - '{subject}_MEG_anatomy_sourcemodel_3d8mm.mat' + "freesurfer": [ + "{subject}.L.inflated.4k_fs_LR.surf.gii", + "{subject}.R.inflated.4k_fs_LR.surf.gii", + "{subject}.L.midthickness.4k_fs_LR.surf.gii", ], - 'freesurfer': [ - '{subject}.L.inflated.4k_fs_LR.surf.gii', - '{subject}.R.inflated.4k_fs_LR.surf.gii', - '{subject}.L.midthickness.4k_fs_LR.surf.gii'] - } + }, + }, + "freesurfer": { + "path": "{subject}/T1w/{subject}", + "patterns": {"label": [], "surf": [], "mri": [], "stats": [], "touch": []}, }, - 'freesurfer': { - 'path': '{subject}/T1w/{subject}', - 'patterns': { - 'label': [], - 'surf': [], - 'mri': [], - 'stats': [], - 'touch': [] - } - } } evoked_map = { - 'modes': {'planar': 'MODE-planar', 'mag': 'MODE-mag'}, + "modes": {"planar": "MODE-planar", "mag": "MODE-mag"}, # always BT-diff - 'task_motor': ( - 'LM-TEMG-LF', - 'LM-TEMG-LH', - 'LM-TEMG-RF', - 'LM-TEMG-RH', - 'LM-TFLA-LF', - 'LM-TFLA-LH', - 'LM-TFLA-RF', - 'LM-TFLA-RH'), + "task_motor": ( + "LM-TEMG-LF", + "LM-TEMG-LH", + "LM-TEMG-RF", + "LM-TEMG-RH", + "LM-TFLA-LF", + "LM-TFLA-LH", + "LM-TFLA-RF", + "LM-TFLA-RH", + ), # if versus in name then OP-diff + BT-diff, else BT-diff - 'task_working_memory': ( - 'LM-TIM-0B', - 'LM-TIM-0B-versus-2B', - 'LM-TIM-2B', - 'LM-TIM-face', - 'LM-TIM-face-versus-tool', - 'LM-TIM-tool', - 'LM-TRESP-0B', - 'LM-TRESP-0B-versus-2B', - 'LM-TRESP-2B', - 'LM-TRESP-face', - 'LM-TRESP-face-versus-tool', - 'LM-TRESP-tool'), + "task_working_memory": ( + "LM-TIM-0B", + "LM-TIM-0B-versus-2B", + "LM-TIM-2B", + "LM-TIM-face", + "LM-TIM-face-versus-tool", + "LM-TIM-tool", + "LM-TRESP-0B", + "LM-TRESP-0B-versus-2B", + "LM-TRESP-2B", + "LM-TRESP-face", + "LM-TRESP-face-versus-tool", + "LM-TRESP-tool", + ), # if versus in name then OP-diff + BT-diff, else BT-diff - 'task_story_math': ( - 'LM-TEV-mathnumopt', - 'LM-TEV-mathnumoptcor-versus-mathnumoptwro', - 'LM-TEV-mathnumque', - 'LM-TEV-mathnumque-versus-mathoper', - 'LM-TEV-mathnumquelate-versus-mathnumqueearly', - 'LM-TEV-mathoper', - 'LM-TEV-mathsentnon', - 'LM-TEV-storoptcor-versus-storoptwro', - 'LM-TEV-storsentnon', - 'LM-TEV-storsentnon-versus-mathsentnon', - 'LM-TRESP-all') + "task_story_math": ( + "LM-TEV-mathnumopt", + "LM-TEV-mathnumoptcor-versus-mathnumoptwro", + "LM-TEV-mathnumque", + "LM-TEV-mathnumque-versus-mathoper", + "LM-TEV-mathnumquelate-versus-mathnumqueearly", + "LM-TEV-mathoper", + "LM-TEV-mathsentnon", + "LM-TEV-storoptcor-versus-storoptwro", + "LM-TEV-storsentnon", + "LM-TEV-storsentnon-versus-mathsentnon", + "LM-TRESP-all", + ), } -freesurfer_files = op.join(op.dirname(__file__), 'data', '%s.txt') -for kind, patterns in preprocessed['freesurfer']['patterns'].items(): +freesurfer_files = op.join(op.dirname(__file__), "data", "%s.txt") +for kind, patterns in preprocessed["freesurfer"]["patterns"].items(): with open(freesurfer_files % kind) as fid: - patterns.extend([k.rstrip('\n') for k in fid.readlines()]) + patterns.extend([k.rstrip("\n") for k in fid.readlines()]) pipeline_map = { - 'ica': 'icaclass', - 'bads': 'baddata', - 'psd': 'powavg', - 'evoked': 'eravg', - 'tfr': 'tfavg', - 'trial_info': 'tmegpreproc' + "ica": "icaclass", + "bads": "baddata", + "psd": "powavg", + "evoked": "eravg", + "tfr": "tfavg", + "trial_info": "tmegpreproc", } kind_map = { - 'task_motor': 'Motort', - 'task_working_memory': 'Wrkmem', - 'task_story_math': 'StoryM', - 'rest': 'Restin', - 'noise_empty_room': 'Rnoise', - 'noise_subject': 'Pnoise', - 'meg_anatomy': 'anatomy', - 'freesurfer': 'freesurfer' + "task_motor": "Motort", + "task_working_memory": "Wrkmem", + "task_story_math": "StoryM", + "rest": "Restin", + "noise_empty_room": "Rnoise", + "noise_subject": "Pnoise", + "meg_anatomy": "anatomy", + "freesurfer": "freesurfer", } run_map = { - 'noise_empty_room': ['1'], - 'noise_subject': ['2'], - 'rest': ['3', '4', '5'], - 'task_working_memory': ['6', '7'], - 'task_story_math': ['8', '9'], - 'task_motor': ['10', '11'], - 'meg_anatomy': [], - 'freesurfer': [] + "noise_empty_room": ["1"], + "noise_subject": ["2"], + "rest": ["3", "4", "5"], + "task_working_memory": ["6", "7"], + "task_story_math": ["8", "9"], + "task_motor": ["10", "11"], + "meg_anatomy": [], + "freesurfer": [], } def _map_onset(onset, data_type, output): """Helper to resolve stim and resp according to context""" out = onset - if data_type == 'task_working_memory': - out = {'stim': 'TIM', 'resp': 'TRESP'}[onset] - elif data_type == 'task_motor': - out = {'stim': 'TFLA', 'resp': 'TEMG'}[onset] - elif data_type == 'task_story_math' and output == 'evoked': - out = {'stim': 'TEV', 'resp': 'TRESP'}[onset] - elif data_type == 'task_story_math' and output == 'epochs': - out = {'stim': 'TEV', 'resp': 'TRESP', 'sentence': 'BSENT', - 'block': 'BUN'}[onset] + if data_type == "task_working_memory": + out = {"stim": "TIM", "resp": "TRESP"}[onset] + elif data_type == "task_motor": + out = {"stim": "TFLA", "resp": "TEMG"}[onset] + elif data_type == "task_story_math" and output == "evoked": + out = {"stim": "TEV", "resp": "TRESP"}[onset] + elif data_type == "task_story_math" and output == "epochs": + out = {"stim": "TEV", "resp": "TRESP", "sentence": "BSENT", "block": "BUN"}[ + onset + ] return out def _map_diff_mode(condition, data_type): """Helper to resolve diff mode according to context""" - diff_mode = '[BT-diff]' - if 'versus' in condition: - diff_mode = '[OP-diff]_[BT-diff]' + diff_mode = "[BT-diff]" + if "versus" in condition: + diff_mode = "[OP-diff]_[BT-diff]" return diff_mode -def get_file_paths(subject, data_type, output, run_index=0, - onset='stim', - sensor_mode='mag', hcp_path='.'): +def get_file_paths( + subject, + data_type, + output, + run_index=0, + onset="stim", + sensor_mode="mag", + hcp_path=".", +): """This is the MNE-HCP file path synthesizer An easy conceptual mapper from questions to file paths @@ -245,108 +249,127 @@ def get_file_paths(subject, data_type, output, run_index=0, The file names. """ if data_type not in kind_map: - raise ValueError('I never heard of `%s` -- are you sure this is a' - ' valid HCP type? I currenlty support:\n%s' % ( - data_type, ' \n'.join( - [k for k in kind_map if '_' in k]))) - context = ('rmeg' if 'rest' in data_type else 'tmeg') - sensor_mode = evoked_map['modes'][sensor_mode] + raise ValueError( + "I never heard of `%s` -- are you sure this is a" + " valid HCP type? I currenlty support:\n%s" + % (data_type, " \n".join([k for k in kind_map if "_" in k])) + ) + context = "rmeg" if "rest" in data_type else "tmeg" + sensor_mode = evoked_map["modes"][sensor_mode] my_onset = _map_onset(onset, data_type, output) - if data_type not in ('meg_anatomy', 'freesurfer'): + if data_type not in ("meg_anatomy", "freesurfer"): my_runs = run_map[data_type] if run_index >= len(my_runs): - raise ValueError('For `data_type=%s` we have %d runs. ' - 'You asked for run index %d.' % ( - data_type, len(my_runs), run_index)) + raise ValueError( + "For `data_type=%s` we have %d runs. " + "You asked for run index %d." % (data_type, len(my_runs), run_index) + ) run_label = my_runs[run_index] else: run_label = None - if (data_type in ('noise_subject', - 'noise_empty_room') and output in - ('epochs', 'evoked', 'ica', 'annot')): - raise ValueError('You requested preprocessed data of type "%s" ' - 'and output "%s". HCP does not provide these data' % - (data_type, output)) - if (data_type in ('rest', 'noise_subject', 'noise_empty_room') and - output in ('trial_info', 'evoked')): - raise ValueError('%s not defined for %s' % (output, data_type)) + if data_type in ("noise_subject", "noise_empty_room") and output in ( + "epochs", + "evoked", + "ica", + "annot", + ): + raise ValueError( + 'You requested preprocessed data of type "%s" ' + 'and output "%s". HCP does not provide these data' % (data_type, output) + ) + if data_type in ("rest", "noise_subject", "noise_empty_room") and output in ( + "trial_info", + "evoked", + ): + raise ValueError("%s not defined for %s" % (output, data_type)) files = list() pipeline = pipeline_map.get(output, output) - processing = 'preprocessed' - if output == 'raw': - processing = 'unprocessed' + processing = "preprocessed" + if output == "raw": + processing = "unprocessed" - if processing == 'preprocessed': - file_map = preprocessed[(data_type if data_type in ( - 'meg_anatomy', 'freesurfer') else 'meg')] - path = file_map['path'].format( + if processing == "preprocessed": + file_map = preprocessed[ + (data_type if data_type in ("meg_anatomy", "freesurfer") else "meg") + ] + path = file_map["path"].format( subject=subject, - pipeline=(context + 'preproc' if output == 'epochs' - else pipeline), - kind=kind_map[data_type]) + pipeline=(context + "preproc" if output == "epochs" else pipeline), + kind=kind_map[data_type], + ) - if output == 'epochs': + if output == "epochs": pattern_key = (output, context) else: pattern_key = output - my_pattern = file_map['patterns'].get(pattern_key, None) + my_pattern = file_map["patterns"].get(pattern_key, None) if my_pattern is None: - raise ValueError('What is output "%s"? I don\'t know about this.' % - output) + raise ValueError('What is output "%s"? I don\'t know about this.' % output) - if output in ('bads', 'ica'): + if output in ("bads", "ica"): files.extend( - [op.join(path, - p.format(subject=subject, run=run_label, - kind=kind_map[data_type])) - for p in my_pattern]) + [ + op.join( + path, + p.format( + subject=subject, run=run_label, kind=kind_map[data_type] + ), + ) + for p in my_pattern + ] + ) - elif output == 'epochs': + elif output == "epochs": my_pattern = my_pattern[0] formats = dict( - subject=subject, run=run_label, kind=kind_map[data_type], - context=context) - if context != 'rest': + subject=subject, + run=run_label, + kind=kind_map[data_type], + context=context, + ) + if context != "rest": formats.update(onset=my_onset) this_file = my_pattern.format(**formats) files.append(op.join(path, this_file)) - elif output == 'evoked': + elif output == "evoked": # XXX add evoked template checks for condition in evoked_map[data_type]: if my_onset not in condition: continue this_file = my_pattern[0].format( - subject=subject, kind=kind_map[data_type], + subject=subject, + kind=kind_map[data_type], condition=condition, diff_modes=_map_diff_mode(condition, data_type), - sensor_mode=sensor_mode) + sensor_mode=sensor_mode, + ) files.append(op.join(path, this_file)) - elif output == 'trial_info': + elif output == "trial_info": this_file = my_pattern[0].format( - subject=subject, run=run_label, kind=kind_map[data_type]) + subject=subject, run=run_label, kind=kind_map[data_type] + ) files.append(op.join(path, this_file)) - elif data_type == 'meg_anatomy': - path = file_map['path'].format(subject=subject) - files.extend([op.join(path, pa.format(subject=subject)) - for pa in my_pattern]) - elif data_type == 'freesurfer': - path = file_map['path'].format(subject=subject) + elif data_type == "meg_anatomy": + path = file_map["path"].format(subject=subject) + files.extend( + [op.join(path, pa.format(subject=subject)) for pa in my_pattern] + ) + elif data_type == "freesurfer": + path = file_map["path"].format(subject=subject) for pa in my_pattern: - files.append( - op.join(path, output, pa.format(subject=subject))) + files.append(op.join(path, output, pa.format(subject=subject))) else: raise ValueError('I never heard of `output` "%s".' % output) - elif processing == 'unprocessed': - path = unprocessed['path'].format( - subject=subject, kind=kind_map[data_type], pipeline=pipeline, - run=run_label) - files.extend([op.join(path, p) for p in unprocessed['patterns']]) + elif processing == "unprocessed": + path = unprocessed["path"].format( + subject=subject, kind=kind_map[data_type], pipeline=pipeline, run=run_label + ) + files.extend([op.join(path, p) for p in unprocessed["patterns"]]) else: - raise ValueError('`processing` %s should be "unprocessed"' - ' or "preprocessed"') + raise ValueError('`processing` %s should be "unprocessed"' ' or "preprocessed"') return [op.join(hcp_path, pa) for pa in files] diff --git a/hcp/io/file_mapping/s3.py b/hcp/io/file_mapping/s3.py index 5ae9fa0..4aa4661 100644 --- a/hcp/io/file_mapping/s3.py +++ b/hcp/io/file_mapping/s3.py @@ -2,10 +2,11 @@ def get_s3_keys_anatomy( - subject, - freesurfer_outputs=('label', 'mri', 'surf'), - meg_anatomy_outputs=('head_model', 'transforms'), - hcp_path_bucket='HCP_900'): + subject, + freesurfer_outputs=("label", "mri", "surf"), + meg_anatomy_outputs=("head_model", "transforms"), + hcp_path_bucket="HCP_900", +): """Helper to prepare AWS downloads for anatomy data A helper function useful for working with Amazon EC2 and S3. @@ -36,20 +37,33 @@ def get_s3_keys_anatomy( aws_keys = list() for output in freesurfer_outputs: aws_keys.extend( - get_file_paths(subject=subject, data_type='freesurfer', - output=output, - hcp_path=hcp_path_bucket)) + get_file_paths( + subject=subject, + data_type="freesurfer", + output=output, + hcp_path=hcp_path_bucket, + ) + ) for output in meg_anatomy_outputs: aws_keys.extend( - get_file_paths(subject=subject, data_type='meg_anatomy', - output=output, - hcp_path=hcp_path_bucket)) + get_file_paths( + subject=subject, + data_type="meg_anatomy", + output=output, + hcp_path=hcp_path_bucket, + ) + ) return aws_keys def get_s3_keys_meg( - subject, data_types, outputs=('raw', 'bads', 'ica'), - run_inds=0, hcp_path_bucket='HCP_900', onsets='stim'): + subject, + data_types, + outputs=("raw", "bads", "ica"), + run_inds=0, + hcp_path_bucket="HCP_900", + onsets="stim", +): """Helper to prepare AWS downloads for MEG data A helper function useful for working with Amazon EC2 and S3. @@ -109,30 +123,39 @@ def get_s3_keys_meg( data_types = [data_types] if not all(isinstance(rr, int) for rr in run_inds): - raise ValueError('Rund indices must be integers. I found: ' + - ', '.join(['%s' % type(rr) for rr in run_inds - if not isinstance(rr, int)])) + raise ValueError( + "Rund indices must be integers. I found: " + + ", ".join(["%s" % type(rr) for rr in run_inds if not isinstance(rr, int)]) + ) elif max(run_inds) > 2: - raise ValueError('For HCP MEG data there is no task with more' - 'than three runs. Among your requests there ' - 'is run index %i. Did you forget about ' - 'zero-based indexing?' % max(run_inds)) + raise ValueError( + "For HCP MEG data there is no task with more" + "than three runs. Among your requests there " + "is run index %i. Did you forget about " + "zero-based indexing?" % max(run_inds) + ) elif min(run_inds) < 0: - raise ValueError('Run indices must be positive') + raise ValueError("Run indices must be positive") for data_type in data_types: for output in outputs: - if 'noise' in data_type and output != 'raw': + if "noise" in data_type and output != "raw": continue # we only have raw for noise data - elif data_type == 'rest' and output in ('evoked', 'trial_info'): + elif data_type == "rest" and output in ("evoked", "trial_info"): continue # there is no such thing as evoked resting state data for run_index in run_inds: if run_index + 1 > len(run_map[data_type]): continue # ignore irrelevant run indices for onset in onsets: aws_keys.extend( - fun(subject=subject, data_type=data_type, - output=output, run_index=run_index, onset=onset, - hcp_path=hcp_path_bucket)) + fun( + subject=subject, + data_type=data_type, + output=output, + run_index=run_index, + onset=onset, + hcp_path=hcp_path_bucket, + ) + ) return aws_keys diff --git a/hcp/io/file_mapping/tests/test_file_mapping.py b/hcp/io/file_mapping/tests/test_file_mapping.py index f37101c..ae3e462 100644 --- a/hcp/io/file_mapping/tests/test_file_mapping.py +++ b/hcp/io/file_mapping/tests/test_file_mapping.py @@ -1,20 +1,32 @@ +from nose.tools import assert_equal, assert_raises + +import hcp.tests.config as tconf from hcp.io.file_mapping import get_file_paths from hcp.io.file_mapping.file_mapping import run_map -import hcp.tests.config as tconf -from nose.tools import assert_raises, assert_equal def test_basic_file_mapping(): """Test construction of file paths and names""" - assert_raises(ValueError, get_file_paths, - subject=tconf.subject, data_type='sushi', - output='raw', run_index=0, hcp_path=tconf.hcp_path) + assert_raises( + ValueError, + get_file_paths, + subject=tconf.subject, + data_type="sushi", + output="raw", + run_index=0, + hcp_path=tconf.hcp_path, + ) - assert_raises(ValueError, get_file_paths, - subject=tconf.subject, data_type='rest', - output='kimchi', run_index=0, - hcp_path=tconf.hcp_path) + assert_raises( + ValueError, + get_file_paths, + subject=tconf.subject, + data_type="rest", + output="kimchi", + run_index=0, + hcp_path=tconf.hcp_path, + ) for run_index in range(3): for output in tconf.hcp_outputs: @@ -22,35 +34,53 @@ def test_basic_file_mapping(): # check too many runs if run_index >= len(run_map[data_type]): assert_raises( - ValueError, get_file_paths, - subject=tconf.subject, data_type=data_type, - output=output, run_index=run_index, - hcp_path=tconf.hcp_path) + ValueError, + get_file_paths, + subject=tconf.subject, + data_type=data_type, + output=output, + run_index=run_index, + hcp_path=tconf.hcp_path, + ) # check no event related outputs - elif (data_type in ('rest', 'noise_subject', - 'noise_empty_room') and - output in ('trial_info', 'evoked')): + elif data_type in ( + "rest", + "noise_subject", + "noise_empty_room", + ) and output in ("trial_info", "evoked"): assert_raises( - ValueError, get_file_paths, - subject=tconf.subject, data_type=data_type, - output=output, run_index=run_index, - hcp_path=tconf.hcp_path) + ValueError, + get_file_paths, + subject=tconf.subject, + data_type=data_type, + output=output, + run_index=run_index, + hcp_path=tconf.hcp_path, + ) # check no preprocessing - elif (data_type in ('noise_subject', - 'noise_empty_room') and output in - ('epochs', 'evoked', 'ica', 'annot')): + elif data_type in ("noise_subject", "noise_empty_room") and output in ( + "epochs", + "evoked", + "ica", + "annot", + ): assert_raises( - ValueError, get_file_paths, - subject=tconf.subject, data_type=data_type, - output=output, run_index=run_index, - hcp_path=tconf.hcp_path) + ValueError, + get_file_paths, + subject=tconf.subject, + data_type=data_type, + output=output, + run_index=run_index, + hcp_path=tconf.hcp_path, + ) else: file_names = get_file_paths( - subject=tconf.subject, data_type=data_type, - output=output, run_index=run_index, - hcp_path=tconf.hcp_path) - if output == 'raw': - assert_equal( - sum('config' in fn for fn in file_names), 1) - assert_equal( - sum('c,rfDC' in fn for fn in file_names), 1) + subject=tconf.subject, + data_type=data_type, + output=output, + run_index=run_index, + hcp_path=tconf.hcp_path, + ) + if output == "raw": + assert_equal(sum("config" in fn for fn in file_names), 1) + assert_equal(sum("c,rfDC" in fn for fn in file_names), 1) diff --git a/hcp/io/read.py b/hcp/io/read.py index 702caef..c0c4e1e 100644 --- a/hcp/io/read.py +++ b/hcp/io/read.py @@ -7,350 +7,92 @@ import numpy as np import scipy.io as scio -from scipy import linalg - -from mne import (EpochsArray, EvokedArray, pick_info, - rename_channels) +from mne import EpochsArray, EvokedArray, pick_info, rename_channels +from mne._fiff.tag import _loc_to_coil_trans from mne.io.bti.bti import _get_bti_info, read_raw_bti -from mne.io import _loc_to_coil_trans from mne.utils import logger +from scipy import linalg from .file_mapping import get_file_paths +# fmt: off _data_labels = [ - 'TRIGGER', - 'RESPONSE', - 'MLzA', - 'MLyA', - 'MLzaA', - 'MLyaA', - 'MLxA', - 'A22', - 'MLxaA', - 'A2', - 'MRzA', - 'MRxA', - 'MRzaA', - 'MRxaA', - 'MRyA', - 'MCzA', - 'MRyaA', - 'MCzaA', - 'MCyA', - 'GzxA', - 'MCyaA', - 'A104', - 'SA1', - 'A241', - 'MCxA', - 'A138', - 'MCxaA', - 'A214', - 'SA2', - 'SA3', - 'A71', - 'A26', - 'A93', - 'A39', - 'A125', - 'A20', - 'A65', - 'A9', - 'A8', - 'A95', - 'A114', - 'A175', - 'A16', - 'A228', - 'A35', - 'A191', - 'A37', - 'A170', - 'A207', - 'A112', - 'A224', - 'A82', - 'A238', - 'A202', - 'A220', - 'A28', - 'A239', - 'A13', - 'A165', - 'A204', - 'A233', - 'A98', - 'A25', - 'A70', - 'A72', - 'A11', - 'A47', - 'A160', - 'A64', - 'A3', - 'A177', - 'A63', - 'A155', - 'A10', - 'A127', - 'A67', - 'A115', - 'A247', - 'A174', - 'A194', - 'A5', - 'A242', - 'A176', - 'A78', - 'A168', - 'A31', - 'A223', - 'A245', - 'A219', - 'A12', - 'A186', - 'A105', - 'A222', - 'A76', - 'A50', - 'A188', - 'A231', - 'A45', - 'A180', - 'A99', - 'A234', - 'A215', - 'A235', - 'A181', - 'A38', - 'A230', - 'A91', - 'A212', - 'A24', - 'A66', - 'A42', - 'A96', - 'A57', - 'A86', - 'A56', - 'A116', - 'A151', - 'A141', - 'A120', - 'A189', - 'A80', - 'A210', - 'A143', - 'A113', - 'A27', - 'A137', - 'A135', - 'A167', - 'A75', - 'A240', - 'A206', - 'A107', - 'A130', - 'A100', - 'A43', - 'A200', - 'A102', - 'A132', - 'A183', - 'A199', - 'A122', - 'A19', - 'A62', - 'A21', - 'A229', - 'A84', - 'A213', - 'A55', - 'A32', - 'A85', - 'A146', - 'A58', - 'A60', - 'GyyA', - 'A88', - 'A79', - 'GzyA', - 'GxxA', - 'A169', - 'A54', - 'GyxA', - 'A203', - 'A145', - 'A103', - 'A163', - 'A139', - 'A49', - 'A166', - 'A156', - 'A128', - 'A68', - 'A159', - 'A236', - 'A161', - 'A121', - 'A4', - 'A61', - 'A6', - 'A126', - 'A14', - 'A94', - 'A15', - 'A193', - 'A150', - 'A227', - 'A59', - 'A36', - 'A225', - 'A195', - 'A30', - 'A109', - 'A172', - 'A108', - 'A81', - 'A171', - 'A218', - 'A173', - 'A201', - 'A74', - 'A29', - 'A164', - 'A205', - 'A232', - 'A69', - 'A157', - 'A97', - 'A217', - 'A101', - 'A124', - 'A40', - 'A123', - 'A153', - 'A178', - 'A1', - 'A179', - 'A33', - 'A147', - 'A117', - 'A148', - 'A87', - 'A89', - 'A243', - 'A119', - 'A52', - 'A142', - 'A211', - 'A190', - 'A53', - 'A192', - 'A73', - 'A226', - 'A136', - 'A184', - 'A51', - 'A237', - 'A77', - 'A129', - 'A131', - 'A198', - 'A197', - 'A182', - 'A46', - 'A92', - 'A41', - 'A90', - 'A7', - 'A23', - 'A83', - 'A154', - 'A34', - 'A17', - 'A18', - 'A248', - 'A149', - 'A118', - 'A208', - 'A152', - 'A140', - 'A144', - 'A209', - 'A110', - 'A111', - 'A244', - 'A185', - 'A246', - 'A162', - 'A106', - 'A187', - 'A48', - 'A221', - 'A196', - 'A133', - 'A158', - 'A44', - 'A134', - 'A216', - 'UACurrent', - 'ECG+', - 'VEOG+', - 'HEOG+', - 'EMG_LF', - 'EMG_LH', - 'ECG-', - 'VEOG-', - 'HEOG-', - 'EMG_RF', - 'EMG_RH' + "TRIGGER", "RESPONSE", + "MLzA", "MLyA", "MLzaA", "MLyaA", "MLxA", "A22", "MLxaA", "A2", "MRzA", "MRxA", + "MRzaA", "MRxaA", "MRyA", "MCzA", "MRyaA", "MCzaA", "MCyA", "GzxA", "MCyaA", "A104", + "SA1", "A241", "MCxA", "A138", "MCxaA", "A214", "SA2", "SA3", "A71", "A26", "A93", + "A39", "A125", "A20", "A65", "A9", "A8", "A95", "A114", "A175", "A16", "A228", + "A35", "A191", "A37", "A170", "A207", "A112", "A224", "A82", "A238", "A202", "A220", + "A28", "A239", "A13", "A165", "A204", "A233", "A98", "A25", "A70", "A72", "A11", + "A47", "A160", "A64", "A3", "A177", "A63", "A155", "A10", "A127", "A67", "A115", + "A247", "A174", "A194", "A5", "A242", "A176", "A78", "A168", "A31", "A223", "A245", + "A219", "A12", "A186", "A105", "A222", "A76", "A50", "A188", "A231", "A45", "A180", + "A99", "A234", "A215", "A235", "A181", "A38", "A230", "A91", "A212", "A24", "A66", + "A42", "A96", "A57", "A86", "A56", "A116", "A151", "A141", "A120", "A189", "A80", + "A210", "A143", "A113", "A27", "A137", "A135", "A167", "A75", "A240", "A206", + "A107", "A130", "A100", "A43", "A200", "A102", "A132", "A183", "A199", "A122", + "A19", "A62", "A21", "A229", "A84", "A213", "A55", "A32", "A85", "A146", "A58", + "A60", "GyyA", "A88", "A79", "GzyA", "GxxA", "A169", "A54", "GyxA", "A203", "A145", + "A103", "A163", "A139", "A49", "A166", "A156", "A128", "A68", "A159", "A236", + "A161", "A121", "A4", "A61", "A6", "A126", "A14", "A94", "A15", "A193", "A150", + "A227", "A59", "A36", "A225", "A195", "A30", "A109", "A172", "A108", "A81", "A171", + "A218", "A173", "A201", "A74", "A29", "A164", "A205", "A232", "A69", "A157", "A97", + "A217", "A101", "A124", "A40", "A123", "A153", "A178", "A1", "A179", "A33", "A147", + "A117", "A148", "A87", "A89", "A243", "A119", "A52", "A142", "A211", "A190", "A53", + "A192", "A73", "A226", "A136", "A184", "A51", "A237", "A77", "A129", "A131", "A198", + "A197", "A182", "A46", "A92", "A41", "A90", "A7", "A23", "A83", "A154", "A34", + "A17", "A18", "A248", "A149", "A118", "A208", "A152", "A140", "A144", "A209", + "A110", "A111", "A244", "A185", "A246", "A162", "A106", "A187", "A48", "A221", + "A196", "A133", "A158", "A44", "A134", "A216", + "UACurrent", "ECG+", "VEOG+", "HEOG+", "EMG_LF", "EMG_LH", "ECG-", "VEOG-", "HEOG-", + "EMG_RF", "EMG_RH", ] +# fmt: on _label_mapping = [ - ('E1', 'ECG+'), - ('E3', 'VEOG+'), - ('E5', 'HEOG+'), - ('E63', 'EMG_LF'), - ('E31', 'EMG_LH'), - ('E2', 'ECG-'), - ('E4', 'VEOG-'), - ('E6', 'HEOG-'), - ('E64', 'EMG_RF'), - ('E32', 'EMG_RH') + ("E1", "ECG+"), + ("E3", "VEOG+"), + ("E5", "HEOG+"), + ("E63", "EMG_LF"), + ("E31", "EMG_LH"), + ("E2", "ECG-"), + ("E4", "VEOG-"), + ("E6", "HEOG-"), + ("E64", "EMG_RF"), + ("E32", "EMG_RH"), ] _time_lock_mapping = dict( - TRESP='resp', - TEMG='resp', - TIM='stim', - TEV='stim', - TFLA='stim', - BSENT='stim', - BU='stim' + TRESP="resp", + TEMG="resp", + TIM="stim", + TEV="stim", + TFLA="stim", + BSENT="stim", + BU="stim", ) def _parse_trans(string): """helper to parse transforms""" - return np.array(string.replace('\n', '') - .strip('[] ') - .split(' '), dtype=float).reshape(4, 4) + return np.array( + string.replace("\n", "").strip("[] ").split(" "), dtype=float + ).reshape(4, 4) def _parse_hcp_trans(fid, transforms, convert_to_meter): - """"another helper""" + """ "another helper""" contents = fid.read() - for trans in contents.split(';'): - if 'filename' in trans or trans == '\n': + for trans in contents.split(";"): + if "filename" in trans or trans == "\n": continue - key, trans = trans.split(' = ') - key = key.lstrip('\ntransform.') + key, trans = trans.split(" = ") + key = key.lstrip("\ntransform.") transforms[key] = _parse_trans(trans) if convert_to_meter: transforms[key][:3, 3] *= 1e-3 # mm to m if not transforms: - raise RuntimeError('Could not parse the transforms.') + raise RuntimeError("Could not parse the transforms.") def _read_trans_hcp(fname, convert_to_meter): @@ -368,13 +110,13 @@ def _read_landmarks_hcp(fname): out = dict() with open(fname) as fid: for line in fid: - kind, data = line.split(' = ') - kind = kind.split('.')[1] - if kind == 'coordsys': - out['coord_frame'] = data.split(';')[0].replace("'", "") + kind, data = line.split(" = ") + kind = kind.split(".")[1] + if kind == "coordsys": + out["coord_frame"] = data.split(";")[0].replace("'", "") else: data = data.split() - for c in ('[', '];'): + for c in ("[", "];"): if c in data: data.remove(c) out[kind] = np.array(data, dtype=int) * 1e-3 # mm to m @@ -384,8 +126,8 @@ def _read_landmarks_hcp(fname): def _get_head_model(head_model_fname): """helper to parse head model from matfile""" head_mat = scio.loadmat(head_model_fname, squeeze_me=False) - pnts = head_mat['headmodel']['bnd'][0][0][0][0][0] - faces = head_mat['headmodel']['bnd'][0][0][0][0][1] + pnts = head_mat["headmodel"]["bnd"][0][0][0][0][0] + faces = head_mat["headmodel"]["bnd"][0][0][0][0][1] faces -= 1 # correct for Matlab's 1-based index return pnts, faces @@ -393,59 +135,54 @@ def _get_head_model(head_model_fname): def _read_bti_info(raw_fid, config): """helper to only access bti info from pdf file""" info, bti_info = _get_bti_info( - pdf_fname=raw_fid, config_fname=config, head_shape_fname=None, - rotation_x=0.0, translation=(0.0, 0.02, 0.11), - ecg_ch='E31', eog_ch=('E63', 'E64'), + pdf_fname=raw_fid, + config_fname=config, + head_shape_fname=None, + rotation_x=0.0, + translation=(0.0, 0.02, 0.11), + ecg_ch="E31", + eog_ch=("E63", "E64"), convert=False, # no conversion to neuromag coordinates rename_channels=False, # keep native channel names - sort_by_ch_name=False) # do not change native order + sort_by_ch_name=False, + ) # do not change native order return info -def _read_raw_bti(raw_fid, config_fid, convert, verbose=None): - """Convert and raw file from HCP input""" - raw = read_raw_bti( # no convrt + no rename for HCP compatibility - raw_fid, config_fid, convert=convert, head_shape_fname=None, - sort_by_ch_name=False, rename_channels=False, preload=False, - verbose=verbose) - - return raw - - def _check_raw_config_runs(raws, configs): - """XXX this goes to tests later, currently not used """ + """XXX this goes to tests later, currently not used""" for raw, config in zip(raws, configs): assert op.split(raw)[0] == op.split(config)[0] - run_str = set([configs[0].split('/')[-3]]) + run_str = set([configs[0].split("/")[-3]]) for config in configs[1:]: - assert set(configs[0].split('/')) - set(config.split('/')) == run_str + assert set(configs[0].split("/")) - set(config.split("/")) == run_str def _check_infos_trans(infos): """XXX this goes to tests later, currently not used""" - chan_max_idx = np.argmax([c['nchan'] for c in infos]) - chan_template = infos[chan_max_idx]['ch_names'] - channels = [c['ch_names'] for c in infos] + chan_max_idx = np.argmax([c["nchan"] for c in infos]) + chan_template = infos[chan_max_idx]["ch_names"] + channels = [c["ch_names"] for c in infos] common_channels = set(chan_template).intersection(*channels) - - common_chs = [[c['chs'][c['ch_names'].index(ch)] for ch in common_channels] - for c in infos] - dev_ctf_trans = [i['dev_ctf_t']['trans'] for i in infos] - cns = [[c['ch_name'] for c in cc] for cc in common_chs] + common_chs = [ + [c["chs"][c["ch_names"].index(ch)] for ch in common_channels] for c in infos + ] + dev_ctf_trans = [i["dev_ctf_t"]["trans"] for i in infos] + cns = [[c["ch_name"] for c in cc] for cc in common_chs] for cn1, cn2 in itt.combinations(cns, 2): assert cn1 == cn2 # BTI stores data in head coords, as a consequence the coordinates # change across run, we apply the ctf->ctf_head transform here # to check that all transforms are correct. - cts = [np.array([linalg.inv(_loc_to_coil_trans(c['loc'])).dot(t) - for c in cc]) - for t, cc in zip(dev_ctf_trans, common_chs)] + cts = [ + np.array([linalg.inv(_loc_to_coil_trans(c["loc"])).dot(t) for c in cc]) + for t, cc in zip(dev_ctf_trans, common_chs) + ] for ct1, ct2 in itt.combinations(cts, 2): np.testing.assert_array_almost_equal(ct1, ct2, 12) -def read_raw(subject, data_type, run_index=0, hcp_path=op.curdir, - verbose=None): +def read_raw(subject, data_type, run_index=0, hcp_path=op.curdir, verbose=None): """Read HCP raw data Parameters @@ -475,11 +212,22 @@ def read_raw(subject, data_type, run_index=0, hcp_path=op.curdir, The MNE raw object. """ pdf, config = get_file_paths( - subject=subject, data_type=data_type, output='raw', - run_index=run_index, hcp_path=hcp_path) - - raw = _read_raw_bti(pdf, config, convert=False, verbose=verbose) - return raw + subject=subject, + data_type=data_type, + output="raw", + run_index=run_index, + hcp_path=hcp_path, + ) + return read_raw_bti( # no convrt + no rename for HCP compatibility + pdf, + config, + convert=False, + head_shape_fname=None, + sort_by_ch_name=False, + rename_channels=False, + preload=False, + verbose=verbose, + ) def read_info(subject, data_type, run_index=0, hcp_path=op.curdir): @@ -513,23 +261,30 @@ def read_info(subject, data_type, run_index=0, hcp_path=op.curdir): HCP MEG does not deliver only 3 of the 5 task packages from MRI HCP. """ raw, config = get_file_paths( - subject=subject, data_type=data_type, output='raw', - run_index=run_index, hcp_path=hcp_path) + subject=subject, + data_type=data_type, + output="raw", + run_index=run_index, + hcp_path=hcp_path, + ) if not op.exists(raw): raw = None - meg_info = _read_bti_info(raw, config) - if raw is None: - logger.info('Did not find Raw data. Guessing EMG, ECG and EOG ' - 'channels') + logger.info("Did not find Raw data. Guessing EMG, ECG and EOG " "channels") rename_channels(meg_info, dict(_label_mapping)) return meg_info -def read_epochs(subject, data_type, onset='stim', run_index=0, - hcp_path=op.curdir, return_fixations_motor=False): +def read_epochs( + subject, + data_type, + onset="stim", + run_index=0, + hcp_path=op.curdir, + return_fixations_motor=False, +): """Read HCP processed data Parameters @@ -561,39 +316,45 @@ def read_epochs(subject, data_type, onset='stim', run_index=0, The MNE epochs. Note, these are pseudo-epochs in the case of onset == 'rest'. """ - info = read_info(subject=subject, data_type=data_type, - run_index=run_index, hcp_path=hcp_path) + info = read_info( + subject=subject, data_type=data_type, run_index=run_index, hcp_path=hcp_path + ) epochs_mat_fname = get_file_paths( - subject=subject, data_type=data_type, output='epochs', + subject=subject, + data_type=data_type, + output="epochs", onset=onset, - run_index=run_index, hcp_path=hcp_path)[0] + run_index=run_index, + hcp_path=hcp_path, + )[0] - if data_type != 'task_motor': + if data_type != "task_motor": return_fixations_motor = None - epochs = _read_epochs(epochs_mat_fname=epochs_mat_fname, info=info, - return_fixations_motor=return_fixations_motor) - if data_type == 'task_motor': - epochs.set_channel_types( - {ch: 'emg' for ch in epochs.ch_names if 'EMG' in ch}) + epochs = _read_epochs( + epochs_mat_fname=epochs_mat_fname, + info=info, + return_fixations_motor=return_fixations_motor, + ) + if data_type == "task_motor": + epochs.set_channel_types({ch: "emg" for ch in epochs.ch_names if "EMG" in ch}) return epochs def _read_epochs(epochs_mat_fname, info, return_fixations_motor): """read the epochs from matfile""" - data = scio.loadmat(epochs_mat_fname, - squeeze_me=True)['data'] - ch_names = [ch for ch in data['label'].tolist()] - info['sfreq'] = data['fsample'].tolist() - times = data['time'].tolist()[0] + data = scio.loadmat(epochs_mat_fname, squeeze_me=True)["data"] + ch_names = [ch for ch in data["label"].tolist()] + info["sfreq"] = data["fsample"].tolist() + times = data["time"].tolist()[0] # deal with different event lengths if return_fixations_motor is not None: - fixation_mask = data['trialinfo'].tolist()[:, 1] == 6 + fixation_mask = data["trialinfo"].tolist()[:, 1] == 6 if return_fixations_motor is False: fixation_mask = ~fixation_mask - data = np.array(data['trial'].tolist()[fixation_mask].tolist()) + data = np.array(data["trial"].tolist()[fixation_mask].tolist()) else: - data = np.array(data['trial'].tolist().tolist()) + data = np.array(data["trial"].tolist().tolist()) # warning: data are not chronologically ordered but # match the trial info @@ -604,8 +365,7 @@ def _read_epochs(epochs_mat_fname, info, return_fixations_motor): # as from the data['trialinfo'] arbitrary events can be constructed. # and it is task specific. this_info = _hcp_pick_info(info, ch_names) - epochs = EpochsArray(data=data, info=this_info, events=events, - tmin=times.min()) + epochs = EpochsArray(data=data, info=this_info, events=events, tmin=times.min()) # XXX hack for now due to issue with EpochsArray constructor # cf https://github.com/mne-tools/mne-hcp/issues/9 epochs.times = times @@ -614,9 +374,7 @@ def _read_epochs(epochs_mat_fname, info, return_fixations_motor): def _hcp_pick_info(info, ch_names): """helper to subset info""" - return pick_info( - info, [info['ch_names'].index(ch) for ch in ch_names], - copy=True) + return pick_info(info, [info["ch_names"].index(ch) for ch in ch_names], copy=True) def read_trial_info(subject, data_type, run_index=0, hcp_path=op.curdir): @@ -645,9 +403,12 @@ def read_trial_info(subject, data_type, run_index=0, hcp_path=op.curdir): """ trial_info_mat_fname = get_file_paths( - subject=subject, data_type=data_type, - output='trial_info', run_index=run_index, - hcp_path=hcp_path)[0] + subject=subject, + data_type=data_type, + output="trial_info", + run_index=run_index, + hcp_path=hcp_path, + )[0] trl_info = _read_trial_info(trial_info_mat_fname=trial_info_mat_fname) return trl_info @@ -656,14 +417,15 @@ def read_trial_info(subject, data_type, run_index=0, hcp_path=op.curdir): def _read_trial_info(trial_info_mat_fname): """helper to read trial info""" # XXX FIXME index -1 - data = scio.loadmat(trial_info_mat_fname, squeeze_me=True)['trlInfo'] + data = scio.loadmat(trial_info_mat_fname, squeeze_me=True)["trlInfo"] out = dict() - for idx, lock_name in enumerate(data['lockNames'].tolist()): + for idx, lock_name in enumerate(data["lockNames"].tolist()): key = _time_lock_mapping[lock_name] out[key] = dict( - comments=data['trlColDescr'].tolist()[idx], - codes=data['lockTrl'].tolist().tolist()[idx]) + comments=data["trlColDescr"].tolist()[idx], + codes=data["lockTrl"].tolist().tolist()[idx], + ) return out @@ -680,22 +442,25 @@ def _check_sorting_runs(candidates, id_char): def _parse_annotations_segments(segment_strings): """Read bad segments defintions from text file""" - for char in '}]': # multi line array definitions - segment_strings = segment_strings.replace( - char + ';', 'splitme' - ) - split = segment_strings.split('splitme') + for char in "}]": # multi line array definitions + segment_strings = segment_strings.replace(char + ";", "splitme") + split = segment_strings.split("splitme") out = dict() for entry in split: - if len(entry) == 1 or entry == '\n': + if len(entry) == 1 or entry == "\n": continue - key, rest = entry.split(' = ') + key, rest = entry.split(" = ") val = np.array( - [k for k in [''.join([c for c in e if c.isdigit()]) - for e in rest.split()] if k.isdigit()], dtype=int) + [ + k + for k in ["".join([c for c in e if c.isdigit()]) for e in rest.split()] + if k.isdigit() + ], + dtype=int, + ) # reshape and map to Python index val = val.reshape(-1, 2) - 1 - out[key.split('.')[1]] = val + out[key.split(".")[1]] = val return out @@ -725,28 +490,33 @@ def read_annot(subject, data_type, run_index=0, hcp_path=op.curdir): The annotations. """ bads_files = get_file_paths( - subject=subject, data_type=data_type, - output='bads', run_index=run_index, - hcp_path=hcp_path) - segments_fname = [k for k in bads_files if - k.endswith('baddata_badsegments.txt')][0] - bads_fname = [k for k in bads_files if - k.endswith('baddata_badchannels.txt')][0] + subject=subject, + data_type=data_type, + output="bads", + run_index=run_index, + hcp_path=hcp_path, + ) + segments_fname = [k for k in bads_files if k.endswith("baddata_badsegments.txt")][0] + bads_fname = [k for k in bads_files if k.endswith("baddata_badchannels.txt")][0] ica_files = get_file_paths( - subject=subject, data_type=data_type, - output='ica', run_index=run_index, - hcp_path=hcp_path) - ica_fname = [k for k in ica_files if k.endswith('icaclass_vs.txt')][0] + subject=subject, + data_type=data_type, + output="ica", + run_index=run_index, + hcp_path=hcp_path, + ) + ica_fname = [k for k in ica_files if k.endswith("icaclass_vs.txt")][0] out = dict() iter_fun = [ - ('channels', _parse_annotations_bad_channels, bads_fname), - ('segments', _parse_annotations_segments, segments_fname), - ('ica', _parse_annotations_ica, ica_fname)] + ("channels", _parse_annotations_bad_channels, bads_fname), + ("segments", _parse_annotations_segments, segments_fname), + ("ica", _parse_annotations_ica, ica_fname), + ] for subtype, fun, fname in iter_fun: - with open(fname, 'r') as fid: + with open(fname, "r") as fid: out[subtype] = fun(fid.read()) return out @@ -777,72 +547,76 @@ def read_ica(subject, data_type, run_index=0, hcp_path=op.curdir): mat : numpy structured array The ICA mat struct. """ - ica_files = get_file_paths( - subject=subject, data_type=data_type, - output='ica', run_index=run_index, - hcp_path=hcp_path) - ica_fname_mat = [k for k in ica_files if k.endswith('icaclass.mat')][0] - - mat = scio.loadmat(ica_fname_mat, squeeze_me=True)['comp_class'] + subject=subject, + data_type=data_type, + output="ica", + run_index=run_index, + hcp_path=hcp_path, + ) + ica_fname_mat = [k for k in ica_files if k.endswith("icaclass.mat")][0] + + mat = scio.loadmat(ica_fname_mat, squeeze_me=True)["comp_class"] return mat def _parse_annotations_bad_channels(bads_strings): """Read bad channel definitions from text file""" - for char in '}]': - bads_strings = bads_strings.replace( - char + ';', 'splitme' - ) - split = bads_strings.split('splitme') + for char in "}]": + bads_strings = bads_strings.replace(char + ";", "splitme") + split = bads_strings.split("splitme") out = dict() for entry in split: - if len(entry) == 1 or entry == '\n': + if len(entry) == 1 or entry == "\n": continue - key, rest = entry.split(' = ') + key, rest = entry.split(" = ") val = [ch for ch in rest.split("'") if ch.isalnum()] - out[key.split('.')[1]] = val + out[key.split(".")[1]] = val return out def _parse_annotations_ica(ica_strings): """Read bad channel definitions from text file""" # prepare splitting - for char in '}]': # multi line array definitions - ica_strings = ica_strings.replace( - char + ';', 'splitme' - ) + for char in "}]": # multi line array definitions + ica_strings = ica_strings.replace(char + ";", "splitme") # scalar variables match_inds = list() - for match in re.finditer(';', ica_strings): + for match in re.finditer(";", ica_strings): ii = match.start() if ica_strings[ii - 1].isalnum(): match_inds.append(ii) ica_strings = list(ica_strings) for ii in match_inds: - ica_strings[ii] = 'splitme' - ica_strings = ''.join(ica_strings) + ica_strings[ii] = "splitme" + ica_strings = "".join(ica_strings) - split = ica_strings.split('splitme') + split = ica_strings.split("splitme") out = dict() for entry in split: - if len(entry) == 1 or entry == '\n': + if len(entry) == 1 or entry == "\n": continue - key, rest = entry.split(' = ') - if '[' in rest: - sep = ' ' + key, rest = entry.split(" = ") + if "[" in rest: + sep = " " else: sep = "'" val = [ch for ch in rest.split(sep) if ch.isalnum()] if all(v.isdigit() for v in val): val = [int(v) - 1 for v in val] # map to Python index - out[key.split('.')[1]] = val + out[key.split(".")[1]] = val return out -def read_evokeds(subject, data_type, onset='stim', sensor_mode='mag', - hcp_path=op.curdir, kind='average'): +def read_evokeds( + subject, + data_type, + onset="stim", + sensor_mode="mag", + hcp_path=op.curdir, + kind="average", +): """Read HCP processed data Parameters @@ -871,42 +645,54 @@ def read_evokeds(subject, data_type, onset='stim', sensor_mode='mag', The MNE epochs. Note, these are pseudo-epochs in the case of onset == 'rest'. """ - info = read_info(subject=subject, data_type=data_type, - hcp_path=hcp_path, run_index=0) - + info = read_info( + subject=subject, data_type=data_type, hcp_path=hcp_path, run_index=0 + ) evoked_files = list() for fname in get_file_paths( - subject=subject, data_type=data_type, onset=onset, - output='evoked', sensor_mode=sensor_mode, hcp_path=hcp_path): + subject=subject, + data_type=data_type, + onset=onset, + output="evoked", + sensor_mode=sensor_mode, + hcp_path=hcp_path, + ): evoked_files.extend(_read_evoked(fname, sensor_mode, info, kind)) return evoked_files def _read_evoked(fname, sensor_mode, info, kind): """helper to read evokeds""" - data = scio.loadmat(fname, squeeze_me=True)['data'] - ch_names = [ch for ch in data['label'].tolist()] + data = scio.loadmat(fname, squeeze_me=True)["data"] + ch_names = [ch for ch in data["label"].tolist()] - times = data['time'].tolist() - sfreq = 1. / np.diff(times)[0] + times = data["time"].tolist() + sfreq = 1.0 / np.diff(times)[0] info = _hcp_pick_info(info, ch_names) - info['sfreq'] = sfreq + info["sfreq"] = sfreq out = list() - comment = ('_'.join(fname.split('/')[-1].split('_')[2:]) - .replace('.mat', '') - .replace('_eravg_', '_') - .replace('[', '') - .replace(']', '')) - nave = np.unique(data['dof'].tolist()) + comment = ( + "_".join(fname.split("/")[-1].split("_")[2:]) + .replace(".mat", "") + .replace("_eravg_", "_") + .replace("[", "") + .replace("]", "") + ) + nave = np.unique(data["dof"].tolist()) assert len(nave) == 1 nave = nave[0] - for key, this_kind in (('var', 'standard_error'), ('avg', 'average')): + for key, this_kind in (("var", "standard_error"), ("avg", "average")): if this_kind != kind: continue evoked = EvokedArray( - data=data[key].tolist(), info=info, tmin=min(times), - kind=this_kind, comment=comment, nave=nave) + data=data[key].tolist(), + info=info, + tmin=min(times), + kind=this_kind, + comment=comment, + nave=nave, + ) out.append(evoked) return out diff --git a/hcp/io/tests/__init__.py b/hcp/io/tests/__init__.py index 177e82a..e69de29 100644 --- a/hcp/io/tests/__init__.py +++ b/hcp/io/tests/__init__.py @@ -1 +0,0 @@ -from . import test_io diff --git a/hcp/io/tests/test_io.py b/hcp/io/tests/test_io.py index 2b1ec8c..f48543d 100644 --- a/hcp/io/tests/test_io.py +++ b/hcp/io/tests/test_io.py @@ -1,111 +1,120 @@ -import shutil import os import os.path as op +import shutil +import mne import numpy as np - +from mne.utils import _TempDir +from nose.tools import assert_equal, assert_raises, assert_true from numpy.testing import assert_array_equal -from nose.tools import assert_equal, assert_true, assert_raises - -import mne import hcp -from mne.utils import _TempDir -from hcp.tests import config as tconf from hcp.io.read import _hcp_pick_info +from hcp.tests import config as tconf -hcp_params = dict(hcp_path=tconf.hcp_path, - subject=tconf.test_subject) +hcp_params = dict(hcp_path=tconf.hcp_path, subject=tconf.test_subject) def test_read_annot(): """Test reading annotations.""" for run_index in tconf.run_inds: - annots = hcp.read_annot(data_type='rest', run_index=run_index, - **hcp_params) + annots = hcp.read_annot(data_type="rest", run_index=run_index, **hcp_params) # channels - assert_equal(list(sorted(annots['channels'])), - ['all', 'ica', 'manual', 'neigh_corr', - 'neigh_stdratio']) - for channels in annots['channels'].values(): + assert_equal( + list(sorted(annots["channels"])), + ["all", "ica", "manual", "neigh_corr", "neigh_stdratio"], + ) + for channels in annots["channels"].values(): for chan in channels: assert_true(chan in tconf.bti_chans) # segments - assert_equal(list(sorted(annots['ica'])), - ['bad', 'brain_ic', 'brain_ic_number', - 'brain_ic_vs', 'brain_ic_vs_number', - 'ecg_eog_ic', 'flag', 'good', - 'physio', 'total_ic_number']) - for components in annots['ica'].values(): + assert_equal( + list(sorted(annots["ica"])), + [ + "bad", + "brain_ic", + "brain_ic_number", + "brain_ic_vs", + "brain_ic_vs_number", + "ecg_eog_ic", + "flag", + "good", + "physio", + "total_ic_number", + ], + ) + for components in annots["ica"].values(): if len(components) > 0: assert_true(min(components) >= 0) assert_true(max(components) <= 248) def _basic_raw_checks(raw): - """Helper for testing raw files """ + """Helper for testing raw files""" picks = mne.pick_types(raw.info, meg=True, ref_meg=False) assert_equal(len(picks), 248) ch_names = [raw.ch_names[pp] for pp in picks] - assert_true(all(ch.startswith('A') for ch in ch_names)) + assert_true(all(ch.startswith("A") for ch in ch_names)) ch_sorted = list(sorted(ch_names)) assert_true(ch_sorted != ch_names) - assert_equal(np.round(raw.info['sfreq'], 4), - tconf.sfreq_raw) + assert_equal(np.round(raw.info["sfreq"], 4), tconf.sfreq_raw) def test_read_raw_rest(): """Test reading raw for resting state""" - for run_index in tconf.run_inds[:tconf.max_runs]: - raw = hcp.read_raw(data_type='rest', run_index=run_index, - **hcp_params) + for run_index in tconf.run_inds[: tconf.max_runs]: + raw = hcp.read_raw(data_type="rest", run_index=run_index, **hcp_params) _basic_raw_checks(raw=raw) def test_read_raw_task(): """Test reading raw for tasks""" - for run_index in tconf.run_inds[:tconf.max_runs]: + for run_index in tconf.run_inds[: tconf.max_runs]: for data_type in tconf.task_types: if run_index == 2: assert_raises( - ValueError, hcp.read_raw, - data_type=data_type, run_index=run_index, **hcp_params) + ValueError, + hcp.read_raw, + data_type=data_type, + run_index=run_index, + **hcp_params, + ) continue - raw = hcp.read_raw( - data_type=data_type, run_index=run_index, **hcp_params) + raw = hcp.read_raw(data_type=data_type, run_index=run_index, **hcp_params) _basic_raw_checks(raw=raw) def test_read_raw_noise(): """Test reading raw for empty room noise""" - for run_index in tconf.run_inds[:tconf.max_runs][:2]: + for run_index in tconf.run_inds[: tconf.max_runs][:2]: for data_type in tconf.noise_types: if run_index == 1: assert_raises( - ValueError, hcp.read_raw, - data_type=data_type, run_index=run_index, **hcp_params) + ValueError, + hcp.read_raw, + data_type=data_type, + run_index=run_index, + **hcp_params, + ) continue - raw = hcp.read_raw( - data_type=data_type, run_index=run_index, **hcp_params) + raw = hcp.read_raw(data_type=data_type, run_index=run_index, **hcp_params) _basic_raw_checks(raw=raw) def _epochs_basic_checks(epochs, annots, data_type): - n_good = 248 - len(annots['channels']['all']) - if data_type == 'task_motor': + n_good = 248 - len(annots["channels"]["all"]) + if data_type == "task_motor": n_good += 4 assert_equal(len(epochs.ch_names), n_good) - assert_equal( - round(epochs.info['sfreq'], 3), - round(tconf.sfreq_preproc, 3)) - assert_array_equal( - np.unique(epochs.events[:, 2]), - np.array([99], dtype=np.int)) + assert_equal(round(epochs.info["sfreq"], 3), round(tconf.sfreq_preproc, 3)) + assert_array_equal(np.unique(epochs.events[:, 2]), np.array([99], dtype=np.int)) assert_true( - _check_bounds(epochs.times, - tconf.epochs_bounds[data_type], - atol=1. / epochs.info['sfreq']) # decim tolerance + _check_bounds( + epochs.times, + tconf.epochs_bounds[data_type], + atol=1.0 / epochs.info["sfreq"], + ) # decim tolerance ) # XXX these seem not to be reliably set. checkout later. @@ -119,33 +128,34 @@ def _epochs_basic_checks(epochs, annots, data_type): def test_read_epochs_rest(): """Test reading epochs for resting state""" - for run_index in tconf.run_inds[:tconf.max_runs][:2]: - annots = hcp.read_annot( - data_type='rest', run_index=run_index, **hcp_params) + for run_index in tconf.run_inds[: tconf.max_runs][:2]: + annots = hcp.read_annot(data_type="rest", run_index=run_index, **hcp_params) - epochs = hcp.read_epochs( - data_type='rest', run_index=run_index, **hcp_params) + epochs = hcp.read_epochs(data_type="rest", run_index=run_index, **hcp_params) - _epochs_basic_checks(epochs, annots, data_type='rest') + _epochs_basic_checks(epochs, annots, data_type="rest") def test_read_epochs_task(): """Test reading epochs for task""" - for run_index in tconf.run_inds[:tconf.max_runs][:2]: + for run_index in tconf.run_inds[: tconf.max_runs][:2]: for data_type in tconf.task_types: annots = hcp.read_annot( - data_type=data_type, run_index=run_index, **hcp_params) + data_type=data_type, run_index=run_index, **hcp_params + ) epochs = hcp.read_epochs( - data_type=data_type, run_index=run_index, **hcp_params) + data_type=data_type, run_index=run_index, **hcp_params + ) _epochs_basic_checks(epochs, annots, data_type) def _check_bounds(array, bounds, atol=0.01): """helper for bounds checking""" - return (np.allclose(np.min(array), min(bounds), atol=atol) and - np.allclose(np.max(array), max(bounds), atol=atol)) + return np.allclose(np.min(array), min(bounds), atol=atol) and np.allclose( + np.max(array), max(bounds), atol=atol + ) def test_read_evoked(): @@ -154,74 +164,73 @@ def test_read_evoked(): all_annots = list() for run_index in tconf.run_inds[:2]: annots = hcp.read_annot( - data_type=data_type, run_index=run_index, **hcp_params) + data_type=data_type, run_index=run_index, **hcp_params + ) all_annots.append(annots) - evokeds = hcp.read_evokeds(data_type=data_type, - kind='average', **hcp_params) + evokeds = hcp.read_evokeds(data_type=data_type, kind="average", **hcp_params) - n_average = sum(ee.kind == 'average' for ee in evokeds) + n_average = sum(ee.kind == "average" for ee in evokeds) assert_equal(n_average, len(evokeds)) n_chans = 248 - if data_type == 'task_motor': + if data_type == "task_motor": n_chans += 4 - n_chans -= len(set(sum( - [an['channels']['all'] for an in all_annots], []))) + n_chans -= len(set(sum([an["channels"]["all"] for an in all_annots], []))) assert_equal(n_chans, len(evokeds[0].ch_names)) - assert_true( - _check_bounds(evokeds[0].times, - tconf.epochs_bounds[data_type]) - ) + assert_true(_check_bounds(evokeds[0].times, tconf.epochs_bounds[data_type])) def test_read_info(): """Test reading info.""" tempdir = _TempDir() for data_type in tconf.task_types: - for run_index in tconf.run_inds[:tconf.max_runs][:2]: + for run_index in tconf.run_inds[: tconf.max_runs][:2]: # with pdf file - info = hcp.read_info( - data_type=data_type, run_index=run_index, **hcp_params) + info = hcp.read_info(data_type=data_type, run_index=run_index, **hcp_params) assert_equal( - {k for k in info['ch_names'] if k.startswith('A')}, - tconf.bti_chans + {k for k in info["ch_names"] if k.startswith("A")}, tconf.bti_chans ) # without pdf file # in this case the hcp code guesses certain channel labels cp_paths = hcp.io.file_mapping.get_file_paths( - subject=tconf.test_subject, data_type=data_type, - run_index=run_index, output='raw', hcp_path='', + subject=tconf.test_subject, + data_type=data_type, + run_index=run_index, + output="raw", + hcp_path="", ) for pp in cp_paths: - if 'c,' in pp: # don't copy pdf + if "c," in pp: # don't copy pdf continue os.makedirs(op.join(tempdir, op.dirname(pp))) shutil.copy(op.join(tconf.hcp_path, pp), op.join(tempdir, pp)) info2 = hcp.read_info( - subject=tconf.test_subject, data_type=data_type, + subject=tconf.test_subject, + data_type=data_type, hcp_path=tempdir, - run_index=run_index) - assert_true(len(info['chs']) != len(info2['chs'])) - common_chs = [ch for ch in info2['ch_names'] if - ch in info['ch_names']] - assert_equal(len(common_chs), len(info['chs'])) + run_index=run_index, + ) + assert_true(len(info["chs"]) != len(info2["chs"])) + common_chs = [ch for ch in info2["ch_names"] if ch in info["ch_names"]] + assert_equal(len(common_chs), len(info["chs"])) info2 = _hcp_pick_info(info2, common_chs) - assert_equal(info['ch_names'], info2['ch_names']) - for ch1, ch2 in zip(info['chs'], info2['chs']): - assert_array_equal(ch1['loc'], ch2['loc']) + assert_equal(info["ch_names"], info2["ch_names"]) + for ch1, ch2 in zip(info["chs"], info2["chs"]): + assert_array_equal(ch1["loc"], ch2["loc"]) def test_read_trial_info(): """Test reading trial info basics.""" for data_type in tconf.task_types: - for run_index in tconf.run_inds[:tconf.max_runs][:2]: + for run_index in tconf.run_inds[: tconf.max_runs][:2]: trial_info = hcp.read_trial_info( - data_type=data_type, run_index=run_index, **hcp_params) - assert_true('stim' in trial_info) - assert_true('resp' in trial_info) + data_type=data_type, run_index=run_index, **hcp_params + ) + assert_true("stim" in trial_info) + assert_true("resp" in trial_info) assert_equal(2, len(trial_info)) for key, val in trial_info.items(): - assert_array_equal(np.ndim(val['comments']), 1) - assert_array_equal(np.ndim(val['codes']), 2) + assert_array_equal(np.ndim(val["comments"]), 1) + assert_array_equal(np.ndim(val["codes"]), 2) diff --git a/hcp/preprocessing.py b/hcp/preprocessing.py index 92b5376..ba2ba3a 100644 --- a/hcp/preprocessing.py +++ b/hcp/preprocessing.py @@ -1,18 +1,20 @@ # Author: Denis A. Engemann # License: BSD (3-clause) -import numpy as np import mne -from mne.io import set_bipolar_reference +import numpy as np +from mne import set_bipolar_reference from mne.io.bti.bti import ( - _convert_coil_trans, _coil_trans_to_loc, _get_bti_dev_t, - _loc_to_coil_trans) + _coil_trans_to_loc, + _convert_coil_trans, + _get_bti_dev_t, + _loc_to_coil_trans, +) from mne.transforms import Transform from mne.utils import logger from .io import read_info -from .io.read import _hcp_pick_info -from .io.read import _data_labels +from .io.read import _data_labels, _hcp_pick_info def set_eog_ecg_channels(raw): @@ -26,11 +28,11 @@ def set_eog_ecg_channels(raw): raw : instance of Raw the hcp raw data. """ - for kind in ['ECG', 'VEOG', 'HEOG']: + for kind in ["ECG", "VEOG", "HEOG"]: set_bipolar_reference( - raw, anode=kind + '-', cathode=kind + '+', ch_name=kind, - copy=False) - raw.set_channel_types({'ECG': 'ecg', 'VEOG': 'eog', 'HEOG': 'eog'}) + raw, anode=kind + "-", cathode=kind + "+", ch_name=kind, copy=False + ) + raw.set_channel_types({"ECG": "ecg", "VEOG": "eog", "HEOG": "eog"}) def apply_ica_hcp(raw, ica_mat, exclude): @@ -49,18 +51,17 @@ def apply_ica_hcp(raw, ica_mat, exclude): the components to be excluded. """ if not raw.preload: - raise RuntimeError('raw data must be loaded, use raw.load_data()') - ch_names = ica_mat['topolabel'].tolist().tolist() - picks = mne.pick_channels(raw.info['ch_names'], include=ch_names) + raise RuntimeError("raw data must be loaded, use raw.load_data()") + ch_names = ica_mat["topolabel"].tolist().tolist() + picks = mne.pick_channels(raw.info["ch_names"], include=ch_names) assert ch_names == [raw.ch_names[p] for p in picks] - unmixing_matrix = np.array(ica_mat['unmixing'].tolist()) + unmixing_matrix = np.array(ica_mat["unmixing"].tolist()) n_components, n_channels = unmixing_matrix.shape - mixing = np.array(ica_mat['topo'].tolist()) + mixing = np.array(ica_mat["topo"].tolist()) - proj_mat = (np.eye(n_channels) - np.dot( - mixing[:, exclude], unmixing_matrix[exclude])) + proj_mat = np.eye(n_channels) - np.dot(mixing[:, exclude], unmixing_matrix[exclude]) raw._data *= 1e15 raw._data[picks] = np.dot(proj_mat, raw._data[picks]) raw._data /= 1e15 @@ -91,18 +92,20 @@ def apply_ref_correction(raw, decim_fit=100): Defaults to 100. """ from sklearn.linear_model import LinearRegression - from sklearn.preprocessing import StandardScaler from sklearn.pipeline import Pipeline + from sklearn.preprocessing import StandardScaler + meg_picks = mne.pick_types(raw.info, ref_meg=False, meg=True) ref_picks = mne.pick_types(raw.info, ref_meg=True, meg=False) if len(ref_picks) == 0: - raise ValueError('Could not find meg ref channels.') + raise ValueError("Could not find meg ref channels.") - estimator = Pipeline([('scaler', StandardScaler()), ('estimator', LinearRegression())]) # ref MAG + GRAD + estimator = Pipeline( + [("scaler", StandardScaler()), ("estimator", LinearRegression())] + ) # ref MAG + GRAD Y_pred = estimator.fit( - raw[ref_picks][0][:, ::decim_fit].T, - raw[meg_picks][0][:, ::decim_fit].T).predict( - raw[ref_picks][0].T) + raw[ref_picks][0][:, ::decim_fit].T, raw[meg_picks][0][:, ::decim_fit].T + ).predict(raw[ref_picks][0].T) raw._data[meg_picks] -= Y_pred.T @@ -121,20 +124,19 @@ def map_ch_coords_to_mne(inst): inst : MNE data containers Raw, Epochs, Evoked. """ - bti_dev_t = Transform('ctf_meg', 'meg', _get_bti_dev_t()) - dev_ctf_t = inst.info['dev_ctf_t'] - for ch in inst.info['chs']: - loc = ch['loc'][:] + bti_dev_t = Transform("ctf_meg", "meg", _get_bti_dev_t()) + dev_ctf_t = inst.info["dev_ctf_t"] + for ch in inst.info["chs"]: + loc = ch["loc"][:] if loc is not None: - logger.debug('converting %s' % ch['ch_name']) + logger.debug("converting %s" % ch["ch_name"]) t = _loc_to_coil_trans(loc) t = _convert_coil_trans(t, dev_ctf_t, bti_dev_t) loc = _coil_trans_to_loc(t) - ch['loc'] = loc + ch["loc"] = loc -def interpolate_missing(inst, subject, data_type, hcp_path, - run_index=0, mode='fast'): +def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="fast"): """Interpolate all MEG channels that are missing .. warning:: @@ -172,54 +174,55 @@ def interpolate_missing(inst, subject, data_type, hcp_path, """ try: info = read_info( - subject=subject, data_type=data_type, hcp_path=hcp_path, - run_index=run_index if run_index is None else run_index) + subject=subject, + data_type=data_type, + hcp_path=hcp_path, + run_index=run_index if run_index is None else run_index, + ) except (ValueError, IOError): raise ValueError( - 'could not find config to complete info.' - 'reading only channel positions without ' - 'transforms.') + "could not find config to complete info." + "reading only channel positions without " + "transforms." + ) # full BTI MEG channels - bti_meg_channel_names = ['A%i' % ii for ii in range(1, 249, 1)] + bti_meg_channel_names = ["A%i" % ii for ii in range(1, 249, 1)] # figure out which channels are missing bti_meg_channel_missing_names = [ - ch for ch in bti_meg_channel_names if ch not in inst.ch_names] + ch for ch in bti_meg_channel_names if ch not in inst.ch_names + ] # get meg picks picks_meg = mne.pick_types(inst.info, meg=True, ref_meg=False) # some non-contiguous block in the middle so let's try to invert - picks_other = [ii for ii in range(len(inst.ch_names)) if ii not in - picks_meg] + picks_other = [ii for ii in range(len(inst.ch_names)) if ii not in picks_meg] other_chans = [inst.ch_names[po] for po in picks_other] # compute new n channels - n_channels = (len(picks_meg) + - len(bti_meg_channel_missing_names) + - len(other_chans)) + n_channels = len(picks_meg) + len(bti_meg_channel_missing_names) + len(other_chans) # restrict info to final channels # ! info read from config file is not sorted like inst.info # ! therefore picking order matters, but we don't know it. # ! so far we will rely on the consistent layout for raw files - final_names = [ch for ch in _data_labels if ch in bti_meg_channel_names or - ch in other_chans] + final_names = [ + ch for ch in _data_labels if ch in bti_meg_channel_names or ch in other_chans + ] info = _hcp_pick_info(info, final_names) - assert len(info['ch_names']) == n_channels - existing_channels_index = [ii for ii, ch in enumerate(info['ch_names']) if - ch in inst.ch_names] + assert len(info["ch_names"]) == n_channels + existing_channels_index = [ + ii for ii, ch in enumerate(info["ch_names"]) if ch in inst.ch_names + ] - info['sfreq'] = inst.info['sfreq'] + info["sfreq"] = inst.info["sfreq"] # compute shape of data to be added - is_raw = isinstance(inst, (mne.io.Raw, - mne.io.RawArray, - mne.io.bti.bti.RawBTi)) + is_raw = isinstance(inst, (mne.io.Raw, mne.io.RawArray, mne.io.bti.bti.RawBTi)) is_epochs = isinstance(inst, mne.BaseEpochs) is_evoked = isinstance(inst, (mne.Evoked, mne.EvokedArray)) if is_raw: - shape = (n_channels, - (inst.last_samp - inst.first_samp) + 1) + shape = (n_channels, (inst.last_samp - inst.first_samp) + 1) data = inst._data elif is_epochs: shape = (n_channels, len(inst.events), len(inst.times)) @@ -228,8 +231,7 @@ def interpolate_missing(inst, subject, data_type, hcp_path, shape = (n_channels, len(inst.times)) data = inst.data else: - raise ValueError('instance must be Raw, Epochs ' - 'or Evoked') + raise ValueError("instance must be Raw, Epochs " "or Evoked") out_data = np.empty(shape, dtype=data.dtype) out_data[existing_channels_index] = data @@ -238,18 +240,26 @@ def interpolate_missing(inst, subject, data_type, hcp_path, if inst.annotations is not None: out.annotations = inst.annotations elif is_epochs: - out = mne.EpochsArray(data=np.transpose(out_data, (1, 0, 2)), - info=info, events=inst.events, - tmin=inst.times.min(), event_id=inst.event_id) + out = mne.EpochsArray( + data=np.transpose(out_data, (1, 0, 2)), + info=info, + events=inst.events, + tmin=inst.times.min(), + event_id=inst.event_id, + ) elif is_evoked: out = mne.EvokedArray( - data=out_data, info=info, tmin=inst.times.min(), - comment=inst.comment, nave=inst.nave, kind=inst.kind) + data=out_data, + info=info, + tmin=inst.times.min(), + comment=inst.comment, + nave=inst.nave, + kind=inst.kind, + ) else: - raise ValueError('instance must be Raw, Epochs ' - 'or Evoked') + raise ValueError("instance must be Raw, Epochs " "or Evoked") # set "bad" channels and interpolate. - out.info['bads'] = bti_meg_channel_missing_names + out.info["bads"] = bti_meg_channel_missing_names out.interpolate_bads(mode=mode) return out diff --git a/hcp/viz.py b/hcp/viz.py index b2e75d3..a375f18 100644 --- a/hcp/viz.py +++ b/hcp/viz.py @@ -1,12 +1,11 @@ import os.path as op -import numpy as np - import mne -from mne.io.pick import _pick_data_channels, pick_info -from mne import read_trans, read_surface +import numpy as np +from mne import read_surface, read_trans +from mne._fiff.pick import _pick_data_channels, pick_info +from mne.channels.layout import _find_topomap_coords from mne.transforms import apply_trans -from mne.viz.topomap import _find_topomap_coords from .io import read_info @@ -27,14 +26,22 @@ def make_hcp_bti_layout(info): picks = list(range(248)) pos = _find_topomap_coords(info, picks=picks) return mne.channels.layout.Layout( - box=(-42.19, 43.52, -41.7, 28.71), pos=pos, - names=[info['ch_names'][idx] for idx in picks], ids=picks, - kind='magnesWH3600_hcp') - - -def plot_coregistration(subject, subjects_dir, hcp_path, recordings_path, - info_from=(('data_type', 'rest'), ('run_index', 0)), - view_init=(('azim', 0), ('elev', 0))): + box=(-42.19, 43.52, -41.7, 28.71), + pos=pos, + names=[info["ch_names"][idx] for idx in picks], + ids=picks, + kind="magnesWH3600_hcp", + ) + + +def plot_coregistration( + subject, + subjects_dir, + hcp_path, + recordings_path, + info_from=(("data_type", "rest"), ("run_index", 0)), + view_init=(("azim", 0), ("elev", 0)), +): """A diagnostic plot to show the HCP coregistration Parameters @@ -64,7 +71,7 @@ def plot_coregistration(subject, subjects_dir, hcp_path, recordings_path, The figure object. """ import matplotlib.pyplot as plt - from mpl_toolkits.mplot3d import Axes3D # noqa + from mpl_toolkits.mplot3d import Axes3D # noqa if isinstance(info_from, tuple): info_from = dict(info_from) @@ -72,23 +79,22 @@ def plot_coregistration(subject, subjects_dir, hcp_path, recordings_path, view_init = dict(view_init) head_mri_t = read_trans( - op.join(recordings_path, subject, - '{}-head_mri-trans.fif'.format(subject))) + op.join(recordings_path, subject, "{}-head_mri-trans.fif".format(subject)) + ) info = read_info(subject=subject, hcp_path=hcp_path, **info_from) info = pick_info(info, _pick_data_channels(info, with_ref_meg=False)) - sens_pnts = np.array([c['loc'][:3] for c in info['chs']]) + sens_pnts = np.array([c["loc"][:3] for c in info["chs"]]) sens_pnts = apply_trans(head_mri_t, sens_pnts) sens_pnts *= 1e3 # put in mm scale - pnts, tris = read_surface( - op.join(subjects_dir, subject, 'bem', 'inner_skull.surf')) + pnts, tris = read_surface(op.join(subjects_dir, subject, "bem", "inner_skull.surf")) fig = plt.figure() - ax = fig.add_subplot(111, projection='3d') - ax.scatter(*sens_pnts.T, color='purple', marker='o') - ax.scatter(*pnts.T, color='green', alpha=0.3) + ax = fig.add_subplot(111, projection="3d") + ax.scatter(*sens_pnts.T, color="purple", marker="o") + ax.scatter(*pnts.T, color="green", alpha=0.3) ax.view_init(**view_init) fig.tight_layout() return fig From 5e19498b518806d5ff024a8e375e547126738a25 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 12:47:42 +0200 Subject: [PATCH 03/34] clean-up tutorials --- examples/make_mne_anatomy.py | 10 +- examples/plot_bem.py | 7 +- examples/plot_evoked_data.py | 32 +++-- examples/plot_interpolate_missing_channels.py | 29 ++-- .../plot_compute_evoked_inverse_solution.py | 78 ++++++----- tutorials/plot_compute_forward.py | 76 +++++----- tutorials/plot_reference_correction.py | 84 +++++------ tutorials/plot_reproduce_erf.py | 132 ++++++++++-------- .../plot_temporal_searchlight_decoding.py | 57 ++++---- 9 files changed, 271 insertions(+), 234 deletions(-) diff --git a/examples/make_mne_anatomy.py b/examples/make_mne_anatomy.py index ed8c095..7b11063 100644 --- a/examples/make_mne_anatomy.py +++ b/examples/make_mne_anatomy.py @@ -23,14 +23,14 @@ # License: BSD 3 clause import os.path as op -import hcp -storage_dir = op.expanduser('~/mne-hcp-data') + +storage_dir = op.expanduser("~/mne-hcp-data") # This is where the data are after downloading from HCP -hcp_path = storage_dir + '/HCP' +hcp_path = storage_dir + "/HCP" # this is the new directory to be created -subjects_dir = storage_dir + '/hcp-subjects' +subjects_dir = storage_dir + "/hcp-subjects" # it will make the subfolders 'bem', 'mir', 'surf' and 'label' # and populate it with symbolic links if possible and write some files @@ -38,7 +38,7 @@ # this is where the coregistration matrix is written as # `105923-head_mri-transform.fif` -recordings_path = storage_dir + '/hcp-meg' +recordings_path = storage_dir + "/hcp-meg" # comment out to write files on your disk # hcp.make_mne_anatomy( diff --git a/examples/plot_bem.py b/examples/plot_bem.py index 9287f17..ae38030 100644 --- a/examples/plot_bem.py +++ b/examples/plot_bem.py @@ -14,11 +14,12 @@ import os.path as op from mne.viz import plot_bem + # from mne.bem import make_watershed_bem -storage_dir = op.expanduser('~/mne-hcp-data') -subject = '105923' -subjects_dir = storage_dir + '/hcp-subjects' +storage_dir = op.expanduser("~/mne-hcp-data") +subject = "105923" +subjects_dir = storage_dir + "/hcp-subjects" ############################################################################### # We assume that all directory structures are set up. diff --git a/examples/plot_evoked_data.py b/examples/plot_evoked_data.py index b43564d..e88b39a 100644 --- a/examples/plot_evoked_data.py +++ b/examples/plot_evoked_data.py @@ -12,29 +12,31 @@ # License: BSD 3 clause import os.path as op + import hcp from hcp import preprocessing as preproc -############################################################################## -# we assume our data is inside a designated folder under $HOME -storage_dir = op.expanduser('~/mne-hcp-data') -hcp_path = op.join(storage_dir, 'HCP') -recordings_path = op.join(storage_dir, 'hcp-meg') -subjects_dir = op.join(storage_dir, 'hcp-subjects') -subject = '105923' # our test subject -data_type = 'task_working_memory' +# %% +# We assume our data is inside a designated folder under $HOME. +storage_dir = op.expanduser("~/mne-hcp-data") +hcp_path = op.join(storage_dir, "HCP") +recordings_path = op.join(storage_dir, "hcp-meg") +subjects_dir = op.join(storage_dir, "hcp-subjects") +subject = "105923" # our test subject +data_type = "task_working_memory" run_index = 0 -############################################################################## +# %% # Let's get the evoked data. -hcp_evokeds = hcp.read_evokeds(onset='stim', subject=subject, - data_type=data_type, hcp_path=hcp_path) +hcp_evokeds = hcp.read_evokeds( + onset="stim", subject=subject, data_type=data_type, hcp_path=hcp_path +) for evoked in hcp_evokeds: - if not evoked.comment == 'Wrkmem_LM-TIM-face_BT-diff_MODE-mag': + if not evoked.comment == "Wrkmem_LM-TIM-face_BT-diff_MODE-mag": continue -############################################################################## +# %% # In order to plot topographic patterns we need to transform the sensor # positions to MNE coordinates one a copy of the data. # We're not using this these transformed data for any source analyses. @@ -44,6 +46,6 @@ preproc.map_ch_coords_to_mne(evoked_viz) evoked_viz.plot_joint() -############################################################################## -# for subsequent analyses we would use `evoked` not `evoked_viz`. +# %% +# For subsequent analyses we would use `evoked` not `evoked_viz`. # See also :ref:`tut_compute_inverse_erf` to see how the story continues. diff --git a/examples/plot_interpolate_missing_channels.py b/examples/plot_interpolate_missing_channels.py index 570d5d5..11bf6e1 100644 --- a/examples/plot_interpolate_missing_channels.py +++ b/examples/plot_interpolate_missing_channels.py @@ -5,6 +5,7 @@ We'll take a look at the coregistration here. """ + # Author: Denis A. Engemann # License: BSD 3 clause @@ -15,29 +16,31 @@ import hcp from hcp import preprocessing as preproc -mne.set_log_level('WARNING') +mne.set_log_level("WARNING") -############################################################################## -# we assume our data is inside a designated folder under $HOME -# and set the IO params +# %% +# We assume our data is inside a designated folder under $HOME +# and set the IO params. -storage_dir = op.expanduser('~') +storage_dir = op.expanduser("~") -hcp_params = dict(hcp_path=op.join(storage_dir, 'mne-hcp-data', 'HCP'), - subject='105923', - data_type='task_working_memory') +hcp_params = dict( + hcp_path=op.join(storage_dir, "mne-hcp-data", "HCP"), + subject="105923", + data_type="task_working_memory", +) -############################################################################## -# we take the some evoked and create an interpolated copy +# %% +# We take the some evoked and create an interpolated copy. evoked = hcp.read_evokeds(**hcp_params)[0] # The HCP pipelines don't interpolate missing channels -print('%i channels out of 248 expected' % len(evoked.ch_names)) +print(f"{len(evoked.ch_names)} channels out of 248 expected") evoked_interpolated = preproc.interpolate_missing(evoked, **hcp_params) -############################################################################## +# %% # Let's visualize what has changed! # we calculate the difference ... @@ -45,7 +48,7 @@ print(bads) # ... and mark the respective channels as bad ... -evoked_interpolated.info['bads'] += list(bads) +evoked_interpolated.info["bads"] += list(bads) # ... such that MNE is displaying the interpolated time series in red ... evoked_interpolated.plot(exclude=[]) diff --git a/tutorials/plot_compute_evoked_inverse_solution.py b/tutorials/plot_compute_evoked_inverse_solution.py index f0b4ae2..240504c 100644 --- a/tutorials/plot_compute_evoked_inverse_solution.py +++ b/tutorials/plot_compute_evoked_inverse_solution.py @@ -8,85 +8,93 @@ Here we'll use our knowledge from the other examples and tutorials to compute an inverse solution and apply it on event related fields. """ + # Author: Denis A. Engemann # License: BSD 3 clause import os.path as op + import mne + import hcp from hcp import preprocessing as preproc -############################################################################## -# we assume our data is inside a designated folder under $HOME -storage_dir = op.expanduser('~/mne-hcp-data') -hcp_path = op.join(storage_dir, 'HCP') -recordings_path = op.join(storage_dir, 'hcp-meg') -subjects_dir = op.join(storage_dir, 'hcp-subjects') -subject = '105923' # our test subject -data_type = 'task_working_memory' +# %% +# We assume our data is inside a designated folder under $HOME. +storage_dir = op.expanduser("~/mne-hcp-data") +hcp_path = op.join(storage_dir, "HCP") +recordings_path = op.join(storage_dir, "hcp-meg") +subjects_dir = op.join(storage_dir, "hcp-subjects") +subject = "105923" # our test subject +data_type = "task_working_memory" run_index = 0 -############################################################################## +# %% # We're reading the evoked data. # These are the same as in :ref:`tut_plot_evoked` -hcp_evokeds = hcp.read_evokeds(onset='stim', subject=subject, - data_type=data_type, hcp_path=hcp_path) +hcp_evokeds = hcp.read_evokeds( + onset="stim", subject=subject, data_type=data_type, hcp_path=hcp_path +) for evoked in hcp_evokeds: - if not evoked.comment == 'Wrkmem_LM-TIM-face_BT-diff_MODE-mag': + if not evoked.comment == "Wrkmem_LM-TIM-face_BT-diff_MODE-mag": continue -############################################################################## +# %% # We'll now use a convenience function to get our forward and source models # instead of computing them by hand. src_outputs = hcp.anatomy.compute_forward_stack( - subject=subject, subjects_dir=subjects_dir, - hcp_path=hcp_path, recordings_path=recordings_path, + subject=subject, + subjects_dir=subjects_dir, + hcp_path=hcp_path, + recordings_path=recordings_path, # speed up computations here. Setting `add_dist` to True may improve the # accuracy. src_params=dict(add_dist=False), - info_from=dict(data_type=data_type, run_index=run_index)) + info_from=dict(data_type=data_type, run_index=run_index), +) -fwd = src_outputs['fwd'] +fwd = src_outputs["fwd"] -############################################################################## +# %% # Now we can compute the noise covariance. For this purpose we will apply # the same filtering as was used for the computations of the ERF in the first # place. See also :ref:`tut_reproduce_erf`. -raw_noise = hcp.read_raw(subject=subject, hcp_path=hcp_path, - data_type='noise_empty_room') +raw_noise = hcp.read_raw( + subject=subject, hcp_path=hcp_path, data_type="noise_empty_room" +) raw_noise.load_data() # apply ref channel correction and drop ref channels preproc.apply_ref_correction(raw_noise) -# Note: MNE complains on Python 2.7 -raw_noise.filter(0.50, None, method='iir', - iir_params=dict(order=4, ftype='butter'), n_jobs=1) -raw_noise.filter(None, 60, method='iir', - iir_params=dict(order=4, ftype='butter'), n_jobs=1) +raw_noise.filter( + 0.50, None, method="iir", iir_params=dict(order=4, ftype="butter"), n_jobs=1 +) +raw_noise.filter( + None, 60, method="iir", iir_params=dict(order=4, ftype="butter"), n_jobs=1 +) -############################################################################## +# %% # Note that using the empty room noise covariance will inflate the SNR of the # evkoked and renders comparisons to `baseline` rather uninformative. -noise_cov = mne.compute_raw_covariance(raw_noise, method='empirical') +noise_cov = mne.compute_raw_covariance(raw_noise, method="empirical") -############################################################################## +# %% # Now we assemble the inverse operator, project the data and show the results # on the `fsaverage` surface, the freesurfer average brain. -inv_op = mne.minimum_norm.make_inverse_operator( - evoked.info, fwd, noise_cov=noise_cov) +inv_op = mne.minimum_norm.make_inverse_operator(evoked.info, fwd, noise_cov=noise_cov) stc = mne.minimum_norm.apply_inverse( # these data have a pretty high SNR and - evoked, inv_op, method='MNE', lambda2=1./9.**2) # 9 is a lovely number. + evoked, inv_op, method="MNE", lambda2=1.0 / 9.0**2 +) # 9 is a lovely number. -stc = stc.to_original_src( - src_outputs['src_fsaverage'], subjects_dir=subjects_dir) +stc = stc.to_original_src(src_outputs["src_fsaverage"], subjects_dir=subjects_dir) -brain = stc.plot(subject='fsaverage', subjects_dir=subjects_dir, hemi='both') +brain = stc.plot(subject="fsaverage", subjects_dir=subjects_dir, hemi="both") brain.set_time(145) # we take the peak seen in :ref:`tut_plot_evoked` and -brain.show_view('caudal') # admire wide spread visual activation. +brain.show_view("caudal") # admire wide spread visual activation. diff --git a/tutorials/plot_compute_forward.py b/tutorials/plot_compute_forward.py index 4eb0169..2877576 100644 --- a/tutorials/plot_compute_forward.py +++ b/tutorials/plot_compute_forward.py @@ -12,31 +12,34 @@ # License: BSD 3 clause import os.path as op + import mne + import hcp -############################################################################## -# we assume our data is inside a designated folder under $HOME -storage_dir = op.expanduser('~/mne-hcp-data') -hcp_path = op.join(storage_dir, 'HCP') -recordings_path = op.join(storage_dir, 'hcp-meg') -subjects_dir = op.join(storage_dir, 'hcp-subjects') -subject = '105923' # our test subject +# %% +# We assume our data is inside a designated folder under $HOME. + +storage_dir = op.expanduser("~/mne-hcp-data") +hcp_path = op.join(storage_dir, "HCP") +recordings_path = op.join(storage_dir, "hcp-meg") +subjects_dir = op.join(storage_dir, "hcp-subjects") +subject = "105923" # our test subject -############################################################################## -# and we assume to have the downloaded data, the MNE/freesurfer style +# %% +# And we assume to have the downloaded data, the MNE/freesurfer style # anatomy directory, and the MNE style MEG directory. # these can be obtained from :func:`make_mne_anatomy`. # See also :ref:`tut_make_anatomy`. -############################################################################## -# first we read the coregistration. +# %% +# First we read the coregistration. head_mri_t = mne.read_trans( - op.join(recordings_path, subject, '{}-head_mri-trans.fif'.format( - subject))) + op.join(recordings_path, subject, "{}-head_mri-trans.fif".format(subject)) +) -############################################################################## +# %% # Now we can setup our source model. # Note that spacing has to be set to 'all' since no common MNE resampling # scheme has been employed in the HCP pipelines. @@ -46,44 +49,49 @@ # the subject's native space. With `oct6` we have ~8000 dipole locations. src_fsaverage = mne.setup_source_space( - subject='fsaverage', subjects_dir=subjects_dir, add_dist=False, - spacing='oct6', overwrite=True) + subject="fsaverage", + subjects_dir=subjects_dir, + add_dist=False, + spacing="oct6", + overwrite=True, +) # now we morph it onto the subject. -src_subject = mne.morph_source_spaces( - src_fsaverage, subject, subjects_dir=subjects_dir) +src_subject = mne.morph_source_spaces(src_fsaverage, subject, subjects_dir=subjects_dir) -############################################################################## +# %% # For the same reason `ico` has to be set to `None` when computing the bem. # The headshape is not computed with MNE and has a none standard configuration. -bems = mne.make_bem_model(subject, conductivity=(0.3,), - subjects_dir=subjects_dir, - ico=None) # ico = None for morphed SP. +bems = mne.make_bem_model( + subject, conductivity=(0.3,), subjects_dir=subjects_dir, ico=None +) # ico = None for morphed SP. bem_sol = mne.make_bem_solution(bems) -bem_sol['surfs'][0]['coord_frame'] = 5 +bem_sol["surfs"][0]["coord_frame"] = 5 -############################################################################## +# %% # Now we can read the channels that we want to map to the cortical locations. # Then we can compute the forward solution. -info = hcp.read_info(subject=subject, hcp_path=hcp_path, data_type='rest', - run_index=0) +info = hcp.read_info(subject=subject, hcp_path=hcp_path, data_type="rest", run_index=0) picks = mne.pick_types(info, meg=True, ref_meg=False) info = mne.pick_info(info, picks) -fwd = mne.make_forward_solution(info, trans=head_mri_t, bem=bem_sol, - src=src_subject) +fwd = mne.make_forward_solution(info, trans=head_mri_t, bem=bem_sol, src=src_subject) mag_map = mne.sensitivity_map( - fwd, projs=None, ch_type='mag', mode='fixed', exclude=[], verbose=None) + fwd, projs=None, ch_type="mag", mode="fixed", exclude=[], verbose=None +) -############################################################################## -# we display sensitivity map on the original surface with little smoothing +# %% +# We display sensitivity map on the original surface with little smoothing # and admire the expected curvature-driven sensitivity pattern. mag_map = mag_map.to_original_src(src_fsaverage, subjects_dir=subjects_dir) -mag_map.plot(subject='fsaverage', subjects_dir=subjects_dir, - clim=dict(kind='percent', lims=[0, 50, 99]), - smoothing_steps=2) +mag_map.plot( + subject="fsaverage", + subjects_dir=subjects_dir, + clim=dict(kind="percent", lims=[0, 50, 99]), + smoothing_steps=2, +) diff --git a/tutorials/plot_reference_correction.py b/tutorials/plot_reference_correction.py index 4afa2ff..9c1be0b 100644 --- a/tutorials/plot_reference_correction.py +++ b/tutorials/plot_reference_correction.py @@ -11,48 +11,48 @@ # import os import os.path as op -import numpy as np import matplotlib.pyplot as plt +import mne +import numpy as np from scipy.signal import welch -import mne import hcp from hcp.preprocessing import apply_ref_correction -############################################################################### -# We first set parameters +# %% +# We first set parameters. -storage_dir = op.join(op.expanduser('~'), 'mne-hcp-data') -hcp_path = op.join(storage_dir, 'HCP') -subject = '105923' -data_type = 'rest' +storage_dir = op.join(op.expanduser("~"), "mne-hcp-data") +hcp_path = op.join(storage_dir, "HCP") +subject = "105923" +data_type = "rest" run_index = 0 -############################################################################### -# Then we define a spectral plotter for convenience +# %% +# Then we define a spectral plotter for convenience. def plot_psd(X, label, Fs, NFFT, color=None): - - freqs, psd = welch(X, fs=Fs, window='hanning', nperseg=NFFT, - noverlap=int(NFFT * 0.8)) + freqs, psd = welch( + X, fs=Fs, window="hanning", nperseg=NFFT, noverlap=int(NFFT * 0.8) + ) freqs = freqs[freqs > 0] psd = psd[freqs > 0] - plt.plot(np.log10(freqs), 10 * np.log10(psd.ravel()), label=label, - color=color) + plt.plot(np.log10(freqs), 10 * np.log10(psd.ravel()), label=label, color=color) -############################################################################### -# Now we read in the data +# %% +# Now we read in the data. # # Then we plot the power spectrum of the MEG and reference channels, # apply the reference correction and add the resulting cleaned MEG channels # to our comparison. -raw = hcp.read_raw(subject=subject, hcp_path=hcp_path, - run_index=run_index, data_type=data_type) +raw = hcp.read_raw( + subject=subject, hcp_path=hcp_path, run_index=run_index, data_type=data_type +) raw.load_data() # get meg and ref channels @@ -66,53 +66,53 @@ def plot_psd(X, label, Fs, NFFT, color=None): # add some plotting parameter decim_fit = 100 # we lean a purely spatial model, we don't need all samples decim_show = 10 # we can make plotting faster -n_fft = 2 ** 15 # let's use long windows to see low frequencies +n_fft = 2**15 # let's use long windows to see low frequencies # we put aside the time series for later plotting x_meg = raw[meg_picks][0][:, ::decim_show].mean(0) x_meg_ref = raw[ref_picks][0][:, ::decim_show].mean(0) -############################################################################### +# %% # Now we apply the ref correction (in place). apply_ref_correction(raw) -############################################################################### +# %% # That was the easiest part! Let's now plot everything. plt.figure(figsize=(9, 6)) -plot_psd(x_meg, Fs=raw.info['sfreq'], NFFT=n_fft, label='MEG', color='black') -plot_psd(x_meg_ref, Fs=raw.info['sfreq'], NFFT=n_fft, label='MEG-REF', - color='red') -plot_psd(raw[meg_picks][0][:, ::decim_show].mean(0), Fs=raw.info['sfreq'], - NFFT=n_fft, label='MEG-corrected', color='orange') +plot_psd(x_meg, Fs=raw.info["sfreq"], NFFT=n_fft, label="MEG", color="black") +plot_psd(x_meg_ref, Fs=raw.info["sfreq"], NFFT=n_fft, label="MEG-REF", color="red") +plot_psd( + raw[meg_picks][0][:, ::decim_show].mean(0), + Fs=raw.info["sfreq"], + NFFT=n_fft, + label="MEG-corrected", + color="orange", +) plt.legend() plt.xticks(np.log10([0.1, 1, 10, 100]), [0.1, 1, 10, 100]) plt.xlim(np.log10([0.1, 300])) -plt.xlabel('log10(frequency) [Hz]') -plt.ylabel('Power Spectral Density [dB]') +plt.xlabel("log10(frequency) [Hz]") +plt.ylabel("Power Spectral Density [dB]") plt.grid() plt.show() -############################################################################### -# We can see that the ref correction removes low frequencies which is expected - - -############################################################################### -# By comparing single channel time series we can also see the detrending effect +# %% +# We can see that the ref correction removes low frequencies which is expected. +# +# By comparing single channel time series we can also see the detrending effect. chan1c = raw[meg_picks[0]][0] ch_name = raw.ch_names[meg_picks[0]] plt.figure() -plt.plot(raw.times, chan1.ravel() * 1e15, label='%s before' % ch_name, - color='black') -plt.plot(raw.times, chan1c.ravel() * 1e15, label='%s after' % ch_name, - color='orange') +plt.plot(raw.times, chan1.ravel() * 1e15, label="%s before" % ch_name, color="black") +plt.plot(raw.times, chan1c.ravel() * 1e15, label="%s after" % ch_name, color="orange") plt.xlim(raw.times[[0, -1]]) -plt.legend(loc='upper left') -plt.ylabel('Magnetometer [fT]') -plt.xlabel('Time [seconds]') +plt.legend(loc="upper left") +plt.ylabel("Magnetometer [fT]") +plt.xlabel("Time [seconds]") plt.grid() plt.show() diff --git a/tutorials/plot_reproduce_erf.py b/tutorials/plot_reproduce_erf.py index ce95b22..4689ed5 100644 --- a/tutorials/plot_reproduce_erf.py +++ b/tutorials/plot_reproduce_erf.py @@ -16,24 +16,26 @@ import os.path as op -import numpy as np import matplotlib.pyplot as plt import mne +import numpy as np + import hcp import hcp.preprocessing as preproc -mne.set_log_level('WARNING') +mne.set_log_level("WARNING") # we assume our data is inside its designated folder under $HOME -storage_dir = op.expanduser('~') +storage_dir = op.expanduser("~") hcp_params = dict( - hcp_path=op.join(storage_dir, 'mne-hcp-data', 'HCP'), - subject='105923', - data_type='task_working_memory') + hcp_path=op.join(storage_dir, "mne-hcp-data", "HCP"), + subject="105923", + data_type="task_working_memory", +) -############################################################################## -# We first reprocess the data from scratch +# %% +# We first reprocess the data from scratch. # # That is, almost from scratch. We're relying on the ICA solutions and # data annotations. @@ -59,8 +61,8 @@ # trial_info is a dict # it contains a 'comments' vector that maps on the columns of 'codes' # 'codes is a matrix with its length corresponding to the number of trials -print(trial_info['stim']['comments'][:10]) # which column? -print(set(trial_info['stim']['codes'][:, 3])) # check values +print(trial_info["stim"]["comments"][:10]) # which column? +print(set(trial_info["stim"]["codes"][:, 3])) # check values # so according to this we need to use the column 7 (index 6) # for the time sample and column 4 (index 3) to get the image types @@ -69,9 +71,9 @@ all_events = list() for trial_info in trial_infos: events = np.c_[ - trial_info['stim']['codes'][:, 6] - 1, # time sample - np.zeros(len(trial_info['stim']['codes'])), - trial_info['stim']['codes'][:, 3] # event codes + trial_info["stim"]["codes"][:, 6] - 1, # time sample + np.zeros(len(trial_info["stim"]["codes"])), + trial_info["stim"]["codes"][:, 3], # event codes ].astype(int) events = events[np.argsort(events[:, 0])] # chronological order # for some reason in the HCP data the time events may not always be unique @@ -83,7 +85,6 @@ # now we can go ahead evokeds = list() for run_index, events in zip([0, 1], all_events): - raw = hcp.read_raw(run_index=run_index, **hcp_params) raw.load_data() # apply ref channel correction and drop ref channels @@ -91,35 +92,44 @@ annots = hcp.read_annot(run_index=run_index, **hcp_params) # construct MNE annotations - bad_seg = (annots['segments']['all']) / raw.info['sfreq'] + bad_seg = (annots["segments"]["all"]) / raw.info["sfreq"] annotations = mne.Annotations( - bad_seg[:, 0], (bad_seg[:, 1] - bad_seg[:, 0]), - description='bad') + bad_seg[:, 0], (bad_seg[:, 1] - bad_seg[:, 0]), description="bad" + ) raw.annotations = annotations - raw.info['bads'].extend(annots['channels']['all']) + raw.info["bads"].extend(annots["channels"]["all"]) raw.pick_types(meg=True, ref_meg=False) # Note: MNE complains on Python 2.7 - raw.filter(0.50, None, method='iir', - iir_params=dict(order=4, ftype='butter'), n_jobs=1) - raw.filter(None, 60, method='iir', - iir_params=dict(order=4, ftype='butter'), n_jobs=1) + raw.filter( + 0.50, None, method="iir", iir_params=dict(order=4, ftype="butter"), n_jobs=1 + ) + raw.filter( + None, 60, method="iir", iir_params=dict(order=4, ftype="butter"), n_jobs=1 + ) # read ICA and remove EOG ECG # note that the HCP ICA assumes that bad channels have already been removed ica_mat = hcp.read_ica(run_index=run_index, **hcp_params) # We will select the brain ICs only - exclude = annots['ica']['ecg_eog_ic'] + exclude = annots["ica"]["ecg_eog_ic"] preproc.apply_ica_hcp(raw, ica_mat=ica_mat, exclude=exclude) # now we can epoch events = np.sort(events, 0) - epochs = mne.Epochs(raw, events=events[events[:, 2] == 1], - event_id=event_id, tmin=tmin, tmax=tmax, - reject=None, baseline=baseline, decim=decim, - preload=True) + epochs = mne.Epochs( + raw, + events=events[events[:, 2] == 1], + event_id=event_id, + tmin=tmin, + tmax=tmax, + reject=None, + baseline=baseline, + decim=decim, + preload=True, + ) evoked = epochs.average() # now we need to add back out channels for comparison across runs. @@ -127,8 +137,8 @@ evokeds.append(evoked) del epochs, raw -############################################################################## -# Now we can compute the same ERF based on the preprocessed epochs +# %% +# Now we can compute the same ERF based on the preprocessed epochs. # # These are obtained from the 'tmegpreproc' pipeline. # Things are pythonized and simplified however, so @@ -136,7 +146,6 @@ evokeds_from_epochs_hcp = list() for run_index, events in zip([0, 1], all_events): - unique_subset = np.nonzero(np.r_[1, np.diff(events[:, 0])])[0] # use diff to find first unique events this_events = events[unique_subset] @@ -149,7 +158,7 @@ epochs_hcp.events[:, 2] = events[subset, 2] epochs_hcp.event_id = event_id - evoked = epochs_hcp['face'].average() + evoked = epochs_hcp["face"].average() del epochs_hcp # These epochs have different channels. @@ -163,8 +172,8 @@ evokeds_from_epochs_hcp.append(evoked) -############################################################################## -# Finally we can read the actual official ERF file +# %% +# Finally we can read the actual official ERF file. # # These are obtained from the 'eravg' pipelines. # We read the matlab file, MNE-HCP is doing some conversions, and then we @@ -172,34 +181,32 @@ # and we want the average, not the standard deviation. evoked_hcp = None -hcp_evokeds = hcp.read_evokeds(onset='stim', **hcp_params) +hcp_evokeds = hcp.read_evokeds(onset="stim", **hcp_params) for ev in hcp_evokeds: - if not ev.comment == 'Wrkmem_LM-TIM-face_BT-diff_MODE-mag': + if not ev.comment == "Wrkmem_LM-TIM-face_BT-diff_MODE-mag": continue # Once more we add and interpolate missing channels evoked_hcp = preproc.interpolate_missing(ev, **hcp_params) -############################################################################## -# Time to compare the outputs -# +# %% +# Time to compare the outputs. -evoked = mne.combine_evoked(evokeds, weights='equal') -evoked_from_epochs_hcp = mne.combine_evoked( - evokeds_from_epochs_hcp, weights='equal') +evoked = mne.combine_evoked(evokeds, weights="equal") +evoked_from_epochs_hcp = mne.combine_evoked(evokeds_from_epochs_hcp, weights="equal") fig1, axes = plt.subplots(3, 1, figsize=(12, 8)) evoked.plot(axes=axes[0], show=False) -axes[0].set_title('MNE-HCP') +axes[0].set_title("MNE-HCP") evoked_from_epochs_hcp.plot(axes=axes[1], show=False) -axes[1].set_title('HCP epochs') +axes[1].set_title("HCP epochs") evoked_hcp.plot(axes=axes[2], show=False) -axes[2].set_title('HCP evoked') +axes[2].set_title("HCP evoked") fig1.canvas.draw() plt.show() @@ -207,24 +214,33 @@ # now some correlations plt.figure() -r1 = np.corrcoef(evoked_from_epochs_hcp.data.ravel(), - evoked_hcp.data.ravel())[0][1] -plt.plot(evoked_from_epochs_hcp.data.ravel()[::10] * 1e15, - evoked_hcp.data.ravel()[::10] * 1e15, - linestyle='None', marker='o', alpha=0.1, - mec='orange', color='orange') +r1 = np.corrcoef(evoked_from_epochs_hcp.data.ravel(), evoked_hcp.data.ravel())[0][1] +plt.plot( + evoked_from_epochs_hcp.data.ravel()[::10] * 1e15, + evoked_hcp.data.ravel()[::10] * 1e15, + linestyle="None", + marker="o", + alpha=0.1, + mec="orange", + color="orange", +) plt.annotate("r=%0.3f" % r1, xy=(-300, 250)) -plt.ylabel('evoked from HCP epochs') -plt.xlabel('evoked from HCP evoked') +plt.ylabel("evoked from HCP epochs") +plt.xlabel("evoked from HCP evoked") plt.show() plt.figure() r1 = np.corrcoef(evoked.data.ravel(), evoked_hcp.data.ravel())[0][1] -plt.plot(evoked.data.ravel()[::10] * 1e15, - evoked_hcp.data.ravel()[::10] * 1e15, - linestyle='None', marker='o', alpha=0.1, - mec='orange', color='orange') +plt.plot( + evoked.data.ravel()[::10] * 1e15, + evoked_hcp.data.ravel()[::10] * 1e15, + linestyle="None", + marker="o", + alpha=0.1, + mec="orange", + color="orange", +) plt.annotate("r=%0.3f" % r1, xy=(-300, 250)) -plt.ylabel('evoked from scratch with MNE-HCP') -plt.xlabel('evoked from HCP evoked file') +plt.ylabel("evoked from scratch with MNE-HCP") +plt.xlabel("evoked from HCP evoked file") plt.show() diff --git a/tutorials/plot_temporal_searchlight_decoding.py b/tutorials/plot_temporal_searchlight_decoding.py index 17e8550..c5c05c2 100644 --- a/tutorials/plot_temporal_searchlight_decoding.py +++ b/tutorials/plot_temporal_searchlight_decoding.py @@ -15,46 +15,44 @@ import os.path as op -import numpy as np import mne +import numpy as np + import hcp from hcp import preprocessing as preproc -from sklearn.preprocessing import LabelBinarizer -from sklearn.metrics import roc_auc_score -from sklearn.cross_validation import StratifiedKFold -from mne.decoding import GeneralizationAcrossTime -mne.set_log_level('WARNING') +mne.set_log_level("WARNING") # we assume our data is inside its designated folder under $HOME -storage_dir = op.expanduser('~') +storage_dir = op.expanduser("~") hcp_params = dict( - hcp_path=op.join(storage_dir, 'mne-hcp-data', 'HCP'), - subject='105923', - data_type='task_working_memory') + hcp_path=op.join(storage_dir, "mne-hcp-data", "HCP"), + subject="105923", + data_type="task_working_memory", +) # these values are looked up from the HCP manual tmin, tmax = -1.5, 2.5 decim = 3 -############################################################################## +# %% # We know from studying either the manual or the trial info about the mapping # of events. event_id = dict(face=1, tool=2) -############################################################################## -# we first collect epochs across runs and essentially adopt the code +# %% +# We first collect epochs across runs and essentially adopt the code # shown in :ref:`tut_reproduce_erf`. epochs = list() for run_index in [0, 1]: - hcp_params['run_index'] = run_index + hcp_params["run_index"] = run_index trial_info = hcp.read_trial_info(**hcp_params) events = np.c_[ - trial_info['stim']['codes'][:, 6] - 1, # time sample - np.zeros(len(trial_info['stim']['codes'])), - trial_info['stim']['codes'][:, 3] # event codes + trial_info["stim"]["codes"][:, 6] - 1, # time sample + np.zeros(len(trial_info["stim"]["codes"])), + trial_info["stim"]["codes"][:, 3], # event codes ].astype(int) # for some reason in the HCP data the time events may not always be unique @@ -71,20 +69,21 @@ epochs = mne.concatenate_epochs(epochs) del epochs_hcp -############################################################################## -# Now we can proceed as shown in the MNE-Python decoding tutorials - -y = LabelBinarizer().fit_transform(epochs.events[:, 2]).ravel() - -cv = StratifiedKFold(y=y) # do a stratified cross-validation +# %% +# Now we can proceed as shown in the MNE-Python decoding tutorials, +# Incompatible with recent versions of MNE/scikit-learn which should use +# mne.decoding.GeneralizingEstimator -gat = GeneralizationAcrossTime(predict_mode='cross-validation', n_jobs=1, - cv=cv, scorer=roc_auc_score) +# y = LabelBinarizer().fit_transform(epochs.events[:, 2]).ravel() +# cv = StratifiedKFold(y=y) # do a stratified cross-validation +# gat = GeneralizationAcrossTime( +# predict_mode="cross-validation", n_jobs=1, cv=cv, scorer=roc_auc_score +# ) # fit and score -gat.fit(epochs, y=y) -gat.score(epochs) +# gat.fit(epochs, y=y) +# gat.score(epochs) ############################################################################## # Ploting the temporal connectome and the evolution of discriminability. -gat.plot() -gat.plot_diagonal() +# gat.plot() +# gat.plot_diagonal() From bec0d1e99f2b5aaa74a38303fa4bad5c33b6e35d Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 12:58:47 +0200 Subject: [PATCH 04/34] clean-up packaging --- .codespellignore | 0 .coveragerc | 6 - .gitignore | 170 +- .s3cfg.enc | Bin 1952 -> 0 bytes AUTHORS.rst | 6 - LICENSE.txt => LICENSE | 0 MANIFEST.in | 5 - README.rst | 2 +- codecov.yml | 33 +- dictionary.txt | 4499 ---------------------------------------- pyproject.toml | 157 ++ setup.cfg | 8 - setup.py | 69 - 13 files changed, 300 insertions(+), 4655 deletions(-) create mode 100644 .codespellignore delete mode 100644 .coveragerc delete mode 100644 .s3cfg.enc delete mode 100644 AUTHORS.rst rename LICENSE.txt => LICENSE (100%) delete mode 100644 MANIFEST.in delete mode 100644 dictionary.txt create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100755 setup.py diff --git a/.codespellignore b/.codespellignore new file mode 100644 index 0000000..e69de29 diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 6eed3f3..0000000 --- a/.coveragerc +++ /dev/null @@ -1,6 +0,0 @@ -[run] -branch = True -source = hcp -include = */hcp/* -omit = - */setup.py diff --git a/.gitignore b/.gitignore index 4af0651..f4c6b1c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,56 +1,126 @@ -*.pyc -*.pyo -*.sh -*.so -*.fif -*.tar.gz -*.log -*.stc -*~ -.#* -*.swp -*.lprof -*.npy -*.zip -*.fif.gz -*.nii.gz -*.tar.* -*.egg* -*.tmproj -*.png +# macOS .DS_Store -events.eve -foo-lh.label -foo.lout -bar.lout -foobar.lout -epochs_data.mat -memmap*.dat -tmp-*.w -tmtags -auto_examples -MNE-eegbci-data* -MNE-misc-data* -MNE-sample-data* -MNE-somato-data* -MNE-spm-face* -MNE-testing-data* -MNE-brainstorm-data* -MEGSIM* -build -coverage +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ dist/ -doc/_build/ -doc/generated/ -doc/auto_examples/ -doc/auto_tutorials/ -doc/modules/generated/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ .coverage -tags -doc/coverages -doc/samples -cover +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ +junit-results.xml + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +doc/_build/ +doc/generated/ +doc/api/generated/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# PyCharm +**/.idea/ -*.orig +# VSCode +.vscode diff --git a/.s3cfg.enc b/.s3cfg.enc deleted file mode 100644 index 1dbf89b8c447eeea54eb73b5aee2732fbf3bf80b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1952 zcmV;R2VeNy5g#Qw7)jF&2d@{!p#}@F8atSf*qG~$>$)oLjwr7kW8dH!ZNX#gos_fO z+!O(}crI}f$u)amM1^Y+G=k4ecr&Gk>-p5lRH4ph5Tge@bYutF)nycfcaLXGXf#^( zP^tzh9aSmVi5rmqPgto}q$4(g3rfI!AwQDHh>IhDD_IhFy8k{D_)4l2R2rOHM(g)m zv@YLr98v5pe+x(oOr{OW{?Ae$z6dV&b{r#u7Ehx+af?eA1T^1%-})6a0e~$LR0Um->5y3Dk{%Wvp&U)isA>WdVtFfY-)>~6tEWlmR(Ewtx-`x>#R3Uv-X6$ad71lMa%9Z_dT{1 zTOdJaI3|;b@)Lemr!C_#eh;0190G0w-EK;uXZ}pI;_8t#-?ujKT#x2*GyB<@2PLX= z;A?xH#5axr1Z}x5A4A$5``R|!IAU-)q6hTNfbWoKFR*vg2J>Adty%&-yS!=BtjMY> z7FMDBS@x+Ai{zUI*C;Ss=yocAu^5j2Om?!WT)Xbf7b%m7f6z#Aid-&xe&(<%x67XQ z5k$c58s@LetYKU!*a!+$(SM z)bP59K$Ywkup~J3sbd$&jEN$REdU=onYpl>YXlFE`&Yb&BzARZBqe^F=u3(4h`|?u z-#dsT1nuJOYxVN86#b||nEcl$w^Qwv{i?rR-c7Oy1qd*%uK#`*NP;~L+KvP!kq|dLj=WKudI0Z7$Wfwk z*-f0mGkbE=EN2x~5@qFTC0paY2WyBiP{9-xXJ?gKt|GdDd?G_UyGuEmpB$zh8PZ)^^i8?YSE3t;+ z2Ec@58BKCIL@27%K-vm;+07D|wE}9wR=&2T%)$dlP*IfwtMxF-#) z#cn{nD(TaR{&v+q|JA3p7GrhBQY=fz-p1EFCf7ZFFRe!+Wdx)`y#Kg0AHs%Qw8IJ+ zf3PKJ!8bVZdi*bf>f@YAs=B0R4q1$6+HGjhDZ5_8%#A9$4W|}RAqz&TKNfM>XBfqO zzAxH*lRdIc8r->R_;e^hn#!CO06=h9&0;D1g2v9y3XS#Fa@gE|NT zIwP1b{)M}>AN|J4K%8VrkF9KMXwRiUNdvvv8b++dC$CJ$s~?A5d98d;J&z?CdB2k` z4?5T?cG{IdaAb=@4k!=L$Dzq|Sv^9=LQD5y)|FS&PdO^&Jo5^;vPfb3hht-welT9d z+5Q6yMtkmrXC>b7kx$?kNP_q?4_t->gzXk%_p$p;FJKY3y}U^WY6?khL+3zI54B<$ zPDmFt^ECb+OaU0pr6~<<2*hny1CajNl__oa#G3%o6S6MvNqM=cc_Lp-IzbcLkdjDv zOhs_ws=09t(O>~pH%4@iGVDxsQS3=8s`lVLTE7$2V2XVzT0~TByc`It2&=>L(ikVj zP3~k*-i-1Oi-Ahc;_{k0GG@k$*Lz}DshesmH)o7$Ad4dG7e~V!rQ;r=ipOb;S-woo zcD#^Oa2Nz%eTalFnT55_PF&(aM_*BLzwJQ?PEpyq^gGP*9v&a(9aUj-eC}X@wx}GWk?abandoned -abbout->about -aberation->aberration -abilties->abilities -abilty->ability -abondon->abandon -abondoned->abandoned -abondoning->abandoning -abondons->abandons -aborigene->aborigine -abortificant->abortifacient -abotu->about -abouta->about a -aboutit->about it -aboutthe->about the -abreviate->abbreviate -abreviated->abbreviated -abreviation->abbreviation -abritrary->arbitrary -absail->abseil -absailing->abseiling -absance->absence -abscence->absence -absense->absence -absolutly->absolutely -absorbsion->absorption -absorbtion->absorption -absymal->abysmal -abudance->abundance -abundacies->abundances -abundancies->abundances -abundunt->abundant -abutts->abuts -acadamy->academy -acadmic->academic -accademic->academic -accademy->academy -acccused->accused -accelleration->acceleration -accension->accession, ascension, -acceptence->acceptance -acceptible->acceptable -acces->access -accesories->accessories -accessable->accessible -accidant->accident -accidentaly->accidentally -accidently->accidentally -acclimitization->acclimatization -accomadate->accommodate -accomadated->accommodated -accomadates->accommodates -accomadating->accommodating -accomadation->accommodation -accomadations->accommodations -accomdate->accommodate -accomodate->accommodate -accomodated->accommodated -accomodates->accommodates -accomodating->accommodating -accomodation->accommodation -accomodations->accommodations -accompagnied->accompanied -accompanyed->accompanied -accordeon->accordion -accordian->accordion -accoring->according -accoustic->acoustic -accquainted->acquainted -accrediation->accreditation -accredidation->accreditation -accross->across -accussed->accused -acedemic->academic -acess->access -acheive->achieve -acheived->achieved -acheivement->achievement -acheivements->achievements -acheives->achieves -acheiving->achieving -acheivment->achievement -acheivments->achievements -achievment->achievement -achievments->achievements -achive->achieve, archive, -achived->achieved, archived, -achivement->achievement -achivements->achievements -acident->accident -acknowldeged->acknowledged -acknowledgeing->acknowledging -ackward->awkward, backward, -acommodate->accommodate -acomplish->accomplish -acomplished->accomplished -acomplishment->accomplishment -acomplishments->accomplishments -acording->according -acordingly->accordingly -acquaintence->acquaintance -acquaintences->acquaintances -acquiantence->acquaintance -acquiantences->acquaintances -acquited->acquitted -actived->activated -activites->activities -activly->actively -actualy->actually -acuracy->accuracy -acused->accused -acustom->accustom -acustommed->accustomed -acutal->actual -adavanced->advanced -adbandon->abandon -addional->additional -addionally->additionally -additinally->additionally -additionaly->additionally -additon->addition -additonal->additional -additonally->additionally -addmission->admission -addopt->adopt -addopted->adopted -addoptive->adoptive -addres->address -addresable->addressable -addresed->addressed -addresing->addressing -addressess->addresses -addtion->addition -addtional->additional -adecuate->adequate -adequit->adequate -adhearing->adhering -adherance->adherence -admendment->amendment -admininistrative->administrative -adminstered->administered -adminstrate->administrate -adminstration->administration -adminstrative->administrative -adminstrator->administrator -admissability->admissibility -admissable->admissible -admited->admitted -admitedly->admittedly -adn->and -adolecent->adolescent -adquire->acquire -adquired->acquired -adquires->acquires -adquiring->acquiring -adres->address -adresable->addressable -adresing->addressing -adress->address -adressable->addressable -adressed->addressed -adressing->addressing -adventrous->adventurous -advertisment->advertisement -advertisments->advertisements -advesary->adversary -adviced->advised -aeriel->aerial -aeriels->aerials -afair->affair -afficianados->aficionados -afficionado->aficionado -afficionados->aficionados -affilate->affiliate -affilliate->affiliate -affort->afford, effort, -aforememtioned->aforementioned -againnst->against -agains->against -agaisnt->against -aganist->against -aggaravates->aggravates -aggreed->agreed -aggreement->agreement -aggregious->egregious -aggresive->aggressive -agian->again -agianst->against -agin->again -agina->again, angina, -aginst->against -agravate->aggravate -agre->agree -agred->agreed -agreeement->agreement -agreemnt->agreement -agregate->aggregate -agregates->aggregates -agregation->aggregation -agreing->agreeing -agression->aggression -agressive->aggressive -agressively->aggressively -agressor->aggressor -agricuture->agriculture -agrieved->aggrieved -ahev->have -ahppen->happen -ahve->have -aicraft->aircraft -aiport->airport -airbourne->airborne -aircaft->aircraft -aircrafts->aircraft -airporta->airports -airrcraft->aircraft -aisian->asian -albiet->albeit -alchohol->alcohol -alchoholic->alcoholic -alchol->alcohol -alcholic->alcoholic -alcohal->alcohol -alcoholical->alcoholic -aleady->already -aledge->allege -aledged->alleged -aledges->alleges -alege->allege -aleged->alleged -alegience->allegiance -algebraical->algebraic -algorhitms->algorithms -algoritm->algorithm -algoritms->algorithms -algorythm->algorithm -alientating->alienating -alledge->allege -alledged->alleged -alledgedly->allegedly -alledges->alleges -allegedely->allegedly -allegedy->allegedly -allegely->allegedly -allegence->allegiance -allegience->allegiance -allign->align -alligned->aligned -alliviate->alleviate -allopone->allophone -allopones->allophones -allready->already -allthough->although -alltime->all-time -alltogether->altogether -allways->always -almsot->almost -alochol->alcohol -alomst->almost -alot->a lot, allot, -alotted->allotted -alowed->allowed -alowing->allowing -alreayd->already -alse->also, else -alsot->also -alternitives->alternatives -altho->although -althought->although -altough->although -alusion->allusion, illusion, -alwasy->always -alwyas->always -amalgomated->amalgamated -amatuer->amateur -amature->armature, amateur, -amendmant->amendment -Amercia->America -amerliorate->ameliorate -amke->make -amking->making -ammend->amend -ammended->amended -ammendment->amendment -ammendments->amendments -ammount->amount -ammused->amused -amoung->among -amoungst->amongst -amung->among -amunition->ammunition -analagous->analogous -analitic->analytic -analogeous->analogous -anarchim->anarchism -anarchistm->anarchism -anbd->and -ancestory->ancestry -ancilliary->ancillary -andd->and -androgenous->androgynous -androgeny->androgyny -anihilation->annihilation -aniversary->anniversary -annoint->anoint -annointed->anointed -annointing->anointing -annoints->anoints -annother->another -annouced->announced -annualy->annually -annuled->annulled -anohter->another -anomolies->anomalies -anomolous->anomalous -anomoly->anomaly -anonimity->anonymity -anounced->announced -anouncement->announcement -ansalisation->nasalisation -ansalization->nasalization -ansestors->ancestors -antartic->antarctic -anthromorphization->anthropomorphization -anthropolgist->anthropologist -anthropolgy->anthropology -anual->annual -anulled->annulled -anwsered->answered -anyhwere->anywhere -anyother->any other -anytying->anything -aparent->apparent -aparment->apartment -apenines->apennines, Apennines, -aplication->application -aplied->applied -apolegetics->apologetics -apon->upon, apron, -apparant->apparent -apparantly->apparently -appart->apart -appartment->apartment -appartments->apartments -appealling->appealing, appalling, -appeareance->appearance -appearence->appearance -appearences->appearances -appeneded->appended -appenines->apennines, Apennines, -apperance->appearance -apperances->appearances -appereance->appearance -appereances->appearances -applicaiton->application -applicaitons->applications -appologies->apologies -appology->apology -apprearance->appearance -apprieciate->appreciate -approachs->approaches -appropiate->appropriate -appropiately->appropriately -appropraite->appropriate -appropraitely->appropriately -appropriatly->appropriately -appropriatness->appropriateness -appropropiate->appropriate -appropropiately->appropriately -approproximate->approximate -approxamately->approximately -approxiately->approximately -approximitely->approximately -aprehensive->apprehensive -apropriate->appropriate -apropriately->appropriately -aproval->approval -aproximate->approximate -aproximately->approximately -aquaduct->aqueduct -aquaintance->acquaintance -aquainted->acquainted -aquiantance->acquaintance -aquire->acquire -aquired->acquired -aquiring->acquiring -aquisition->acquisition -aquitted->acquitted -aranged->arranged -arangement->arrangement -arbitarily->arbitrarily -arbitary->arbitrary -archaelogical->archaeological -archaelogists->archaeologists -archaelogy->archaeology -archaoelogy->archeology, archaeology, -archaology->archeology, archaeology, -archeaologist->archeologist, archaeologist, -archeaologists->archeologists, archaeologists, -archetect->architect -archetects->architects -archetectural->architectural -archetecturally->architecturally -archetecture->architecture -archiac->archaic -archictect->architect -archimedian->archimedean -architecht->architect -architechturally->architecturally -architechture->architecture -architechtures->architectures -architectual->architectural -archtype->archetype -archtypes->archetypes -aready->already -areodynamics->aerodynamics -argubly->arguably -arguement->argument -arguements->arguments -arised->arose -arithmentic->arithmetic -arival->arrival -armamant->armament -armistace->armistice -arogant->arrogant -arogent->arrogant -aroud->around -arrangment->arrangement -arrangments->arrangements -arrengement->arrangement -arrengements->arrangements -arround->around -artcile->article -artical->article -artice->article -articel->article -artifical->artificial -artifically->artificially -artillary->artillery -arund->around -asetic->ascetic -asfar->as far -asign->assign -asigned->assigned -aslo->also -asociated->associated -asorbed->absorbed -asphyxation->asphyxiation -assasin->assassin -assasinate->assassinate -assasinated->assassinated -assasinates->assassinates -assasination->assassination -assasinations->assassinations -assasined->assassinated -assasins->assassins -assassintation->assassination -assemple->assemble -assertation->assertion -asside->aside -assisnate->assassinate -assit->assist -assitant->assistant -assocaited->associated -assocation->association -assoicate->associate -assoicated->associated -assoicates->associates -assosication->assassination -asssassans->assassins -asssertion->assertion -assualt->assault -assualted->assaulted -assymetric->asymmetric -assymetrical->asymmetrical -asteriod->asteroid -asthetic->aesthetic -asthetical->aesthetical -asthetically->aesthetically -asume->assume -aswell->as well -asychronously->asynchronously -atain->attain -atempting->attempting -atheistical->atheistic -athenean->athenian -atheneans->athenians -athiesm->atheism -athiest->atheist -atorney->attorney -atribute->attribute -atributed->attributed -atributes->attributes -attaindre->attainder, attained, -attemp->attempt -attemped->attempted -attemt->attempt -attemted->attempted -attemting->attempting -attemts->attempts -attendence->attendance -attendent->attendant -attendents->attendants -attened->attended -attension->attention -attitide->attitude -attributred->attributed -attrocities->atrocities -audeince->audience -auromated->automated -austrailia->Australia -austrailian->Australian -auther->author -authetication->authentication -authobiographic->autobiographic -authobiography->autobiography -authorative->authoritative -authorites->authorities -authorithy->authority -authoritiers->authorities -authoritive->authoritative -authrorities->authorities -autochtonous->autochthonous -autoctonous->autochthonous -autoincrememnt->autoincrement -automaticly->automatically -automibile->automobile -automonomous->autonomous -autor->author -autority->authority -auxilary->auxiliary -auxiliar->auxiliary -auxillaries->auxiliaries -auxillary->auxiliary -auxilliaries->auxiliaries -auxilliary->auxiliary -availabe->available -availablity->availability -availaible->available -availble->available -availiable->available -availible->available -avalable->available -avalance->avalanche -avaliable->available -avation->aviation -avengence->a vengeance -averageed->averaged -avilable->available -awared->awarded -awya->away -baceause->because -backgorund->background -backrounds->backgrounds -bakc->back -banannas->bananas -bandwith->bandwidth -bankrupcy->bankruptcy -banruptcy->bankruptcy -baout->about, bout, -basicaly->basically -basicly->basically -bcak->back -beachead->beachhead -beacuse->because -beastiality->bestiality -beatiful->beautiful -beaurocracy->bureaucracy -beaurocratic->bureaucratic -beautyfull->beautiful -becamae->became -becames->becomes, became, -becasue->because -becausee->because -beccause->because -becomeing->becoming -becomming->becoming -becouse->because -becuase->because -bedore->before -beeing->being -befoer->before -beggin->begin, begging, -begginer->beginner -begginers->beginners -beggining->beginning -begginings->beginnings -beggins->begins -begining->beginning -beginnig->beginning -behavour->behavior, behaviour, -beleagured->beleaguered -beleif->belief -beleive->believe -beleived->believed -beleives->believes -beleiving->believing -beligum->belgium -belive->believe -belived->believed, beloved, -belives->believes, beliefs, -belligerant->belligerent -bellweather->bellwether -bemusemnt->bemusement -beneficary->beneficiary -beng->being -benificial->beneficial -benifit->benefit -benifits->benefits -bergamont->bergamot -Bernouilli->Bernoulli -beseige->besiege -beseiged->besieged -beseiging->besieging -beteen->between -betwen->between -beween->between -bewteen->between -bianry->binary -bilateraly->bilaterally -billingualism->bilingualism -binominal->binomial -bizzare->bizarre -blaim->blame -blaimed->blamed -blessure->blessing -Blitzkreig->Blitzkrieg -boaut->bout, boat, about, -bodydbuilder->bodybuilder -bombardement->bombardment -bombarment->bombardment -bondary->boundary -Bonnano->Bonanno -boradcast->broadcast -borke->broke -boundry->boundary -bouyancy->buoyancy -bouyant->buoyant -boyant->buoyant -Brasillian->Brazilian -breakthough->breakthrough -breakthroughts->breakthroughs -breif->brief -breifly->briefly -brethen->brethren -bretheren->brethren -brigth->bright -briliant->brilliant -brillant->brilliant -brimestone->brimstone -Britian->Britain -Brittish->British -broacasted->broadcast -broadacasting->broadcasting -broady->broadly -Buddah->Buddha -Buddist->Buddhist -buisness->business -buisnessman->businessman -buoancy->buoyancy -buring->burying, burning, burin, during, -burried->buried -busines->business -busineses->business, businesses, -busness->business -bussiness->business -cacheing->caching -caculater->calculator -cacuses->caucuses -cahracters->characters -calaber->caliber -calander->calendar, calender, colander, -calculater->calculator -calculs->calculus -calender->calendar -calenders->calendars -caligraphy->calligraphy -caluclate->calculate -caluclated->calculated -caluculate->calculate -caluculated->calculated -calulate->calculate -calulated->calculated -calulater->calculator -Cambrige->Cambridge -camoflage->camouflage -campagin->campaign -campain->campaign -campaing->campaign -campains->campaigns -cancelation->cancellation -candadate->candidate -candiate->candidate -candidiate->candidate -cannister->canister -cannisters->canisters -cannnot->cannot -cannonical->canonical -cannotation->connotation -cannotations->connotations -cant'->can't -cant->can't -caost->coast -caperbility->capability -Capetown->Cape Town -capible->capable -captial->capital -captued->captured -capturd->captured -carachter->character -caracter->character -caracterized->characterized -carcas->carcass, Caracas, -carefull->careful -careing->caring -carismatic->charismatic -Carmalite->Carmelite -Carnagie->Carnegie -Carnagie-Mellon->Carnegie-Mellon -carnege->carnage, Carnegie, -carnige->carnage, Carnegie, -Carnigie->Carnegie -Carnigie-Mellon->Carnegie-Mellon -carniverous->carnivorous -carreer->career -carrers->careers -Carribbean->Caribbean -Carribean->Caribbean -cartdridge->cartridge -Carthagian->Carthaginian -carthographer->cartographer -cartilege->cartilage -cartilidge->cartilage -cartrige->cartridge -casette->cassette -casion->caisson -cassawory->cassowary -cassowarry->cassowary -casue->cause -casued->caused -casues->causes -casuing->causing -casulaties->casualties -casulaty->casualty -catagories->categories -catagorized->categorized -catagory->category -Cataline->Catiline, Catalina, -catapillar->caterpillar -catapillars->caterpillars -catapiller->caterpillar -catapillers->caterpillars -catepillar->caterpillar -catepillars->caterpillars -catergorize->categorize -catergorized->categorized -caterpilar->caterpillar -caterpilars->caterpillars -caterpiller->caterpillar -caterpillers->caterpillars -cathlic->catholic -catholocism->catholicism -catterpilar->caterpillar -catterpilars->caterpillars -catterpillar->caterpillar -catterpillars->caterpillars -cattleship->battleship -caugt->caught -cauhgt->caught -causalities->casualties -Ceasar->Caesar -Celcius->Celsius -cellpading->cellpadding -cementary->cemetery -cemetarey->cemetery -cemetaries->cemeteries -cemetary->cemetery -cencus->census -censur->censor, censure, -cententenial->centennial -centruies->centuries -centruy->century -centuties->centuries -centuty->century -ceratin->certain, keratin, -cerimonial->ceremonial -cerimonies->ceremonies -cerimonious->ceremonious -cerimony->ceremony -ceromony->ceremony -certainity->certainty -certian->certain -cervial->cervical, servile, serval, -chalenging->challenging -challange->challenge -challanged->challenged -challege->challenge -Champange->Champagne -changable->changeable -characer->character -charachter->character -charachters->characters -charactersistic->characteristic -charactor->character -charactors->characters -charasmatic->charismatic -charaterized->characterized -chariman->chairman -charistics->characteristics -chasr->chaser, chase, -cheif->chief -cheifs->chiefs -chek->check -chemcial->chemical -chemcially->chemically -chemestry->chemistry -chemicaly->chemically -childbird->childbirth -childen->children -choosed->chose, chosen, -choosen->chosen -chould->should, could, -chracter->character -chuch->church -churchs->churches -Cincinatti->Cincinnati -Cincinnatti->Cincinnati -circulaton->circulation -circumsicion->circumcision -circut->circuit -ciricuit->circuit -ciriculum->curriculum -civillian->civilian -claer->clear -claerer->clearer -claerly->clearly -claimes->claims -clas->class, disabled because of name clash in c++ -clasic->classic -clasical->classical -clasically->classically -cleareance->clearance -clera->clear, sclera, -clincial->clinical -clinicaly->clinically -cmo->com, disabled due to lots of false positives -cmoputer->computer -co-incided->coincided -Coca Cola->Coca-Cola -coctail->cocktail -codespel->codespell -coform->conform -cognizent->cognizant -coincedentally->coincidentally -colaborations->collaborations -colateral->collateral -colelctive->collective -collaberative->collaborative -collecton->collection -collegue->colleague -collegues->colleagues -collonade->colonnade -collonies->colonies -collony->colony -collosal->colossal -colonizators->colonizers -colum->column -comander->commander, commandeer, -comando->commando -comandos->commandos -comany->company -comapany->company -comback->comeback -combanations->combinations -combinatins->combinations -combinded->combined -combusion->combustion -comdemnation->condemnation -comemmorates->commemorates -comemoretion->commemoration -comision->commission -comisioned->commissioned -comisioner->commissioner -comisioning->commissioning -comisions->commissions -comission->commission -comissioned->commissioned -comissioner->commissioner -comissioning->commissioning -comissions->commissions -comited->committed -comiting->committing -comitted->committed -comittee->committee -comitting->committing -commandoes->commandos -commedic->comedic -commemerative->commemorative -commemmorate->commemorate -commemmorating->commemorating -commerical->commercial -commerically->commercially -commericial->commercial -commericially->commercially -commerorative->commemorative -comming->coming -comminication->communication -commision->commission -commisioned->commissioned -commisioner->commissioner -commisioning->commissioning -commisions->commissions -commited->committed -commitee->committee -commiting->committing -committe->committee -committment->commitment -committments->commitments -commmemorated->commemorated -commnad->command -commnads->commands -commongly->commonly -commonweath->commonwealth -commuications->communications -commuinications->communications -communciation->communication -communiation->communication -communites->communities -comonent->component -compability->compatibility -compair->compare -compairs->compares -comparision->comparison -comparisions->comparisons -comparitive->comparative -comparitively->comparatively -compatabilities->compatibilities -compatability->compatibility -compatable->compatible -compatablities->compatibilities -compatablity->compatibility -compatiable->compatible -compatibilty->compatibility -compatibily->compatibility -compatiblities->compatibilities -compatiblity->compatibility -compeitions->competitions -compensantion->compensation -competance->competence -competant->competent -competative->competitive -competion->competition, completion, -competitiion->competition -competive->competitive -competiveness->competitiveness -comphrehensive->comprehensive -compitent->competent -completedthe->completed the -completelyl->completely -completetion->completion -complier->compiler -componant->component -comprable->comparable -comprimise->compromise -compulsary->compulsory -compulsery->compulsory -computarized->computerized -concensus->consensus -concider->consider -concidered->considered -concidering->considering -conciders->considers -concieted->conceited -concieved->conceived -concious->conscious -conciously->consciously -conciousness->consciousness -condamned->condemned -condemmed->condemned -condidtion->condition -condidtions->conditions -conditionsof->conditions of -conected->connected -conection->connection -conectix->connectix -conesencus->consensus -confidental->confidential -confidentally->confidentially -confids->confides -configureable->configurable -confortable->comfortable -congradulations->congratulations -congresional->congressional -conived->connived -conjecutre->conjecture -conjuction->conjunction -Conneticut->Connecticut -conotations->connotations -conquerd->conquered -conquerer->conqueror -conquerers->conquerors -conqured->conquered -conscent->consent -consciouness->consciousness -consdider->consider -consdidered->considered -consdiered->considered -consectutive->consecutive -consenquently->consequently -consentrate->concentrate -consentrated->concentrated -consentrates->concentrates -consept->concept -consequentually->consequently -consequeseces->consequences -consern->concern -conserned->concerned -conserning->concerning -conservitive->conservative -consiciousness->consciousness -consicousness->consciousness -considerd->considered -consideres->considered, considers, -consious->conscious -consistant->consistent -consistantly->consistently -consituencies->constituencies -consituency->constituency -consituted->constituted -consitution->constitution -consitutional->constitutional -consolodate->consolidate -consolodated->consolidated -consonent->consonant -consonents->consonants -consorcium->consortium -conspiracys->conspiracies -conspiriator->conspirator -constaints->constraints -constanly->constantly -constarnation->consternation -constatn->constant -constinually->continually -constituant->constituent -constituants->constituents -constituion->constitution -constituional->constitutional -constructes->constructs -construtor->constructor -consttruction->construction -constuction->construction -consulant->consultant -consumate->consummate -consumated->consummated -contaiminate->contaminate -containes->contains -contamporaries->contemporaries -contamporary->contemporary -contempoary->contemporary -contemporaneus->contemporaneous -contempory->contemporary -contendor->contender -contian->contain -contians->contains -contibute->contribute -contibuted->contributed -contibutes->contributes -contigent->contingent -contigious->contiguous -contined->continued -continiously->continuously -continous->continuous -continously->continuously -continueing->continuing -contiuguous->contiguous -contravercial->controversial -contraversy->controversy -contributer->contributor -contributers->contributors -contritutions->contributions -controled->controlled -controler->controller -controling->controlling -controll->control -controlls->controls -controvercial->controversial -controvercy->controversy -controveries->controversies -controversal->controversial -controversey->controversy -controvertial->controversial -controvery->controversy -contruction->construction -contstruction->construction -conveinent->convenient -convenant->covenant -convential->conventional -convertables->convertibles -convertion->conversion -convery->convert -conveyer->conveyor -conviced->convinced -convienient->convenient -convinience->convenience -coordiantion->coordination -coorperation->cooperation, corporation, -coorperations->corporations -copmetitors->competitors -coputer->computer -copywrite->copyright -coridal->cordial -cornmitted->committed -corosion->corrosion -corparate->corporate -corperations->corporations -correcters->correctors -correponding->corresponding -correposding->corresponding -correspoinding->corresponding -correspondant->correspondent -correspondants->correspondents -corresponsing->corresponding -corridoors->corridors -corrispond->correspond -corrispondant->correspondent -corrispondants->correspondents -corrisponded->corresponded -corrisponding->corresponding -corrisponds->corresponds -costitution->constitution -cotrol->control -coucil->council -coudl->could, cloud, -councellor->councillor, counselor, councilor, -councellors->councillors, counselors, councilors, -counries->countries -countains->contains -countires->countries -countrie's->countries, countries', country's, -coururier->courier, couturier, -coverted->converted, covered, coveted, -cpoy->coy, copy, -creaeted->created -creche->crèche -creedence->credence -critereon->criterion -criterias->criteria -criticists->critics -critising->criticising, criticizing, -critisising->criticising -critisism->criticism -critisisms->criticisms -critisize->criticise, criticize, -critisized->criticised, criticized, -critisizes->criticises, criticizes, -critisizing->criticising, criticizing, -critized->criticized -critizing->criticizing -crockodiles->crocodiles -crowm->crown -crtical->critical -crticised->criticised -crucifiction->crucifixion -crusies->cruises -crystalisation->crystallisation -culiminating->culminating -cumulatative->cumulative -curch->church -curcuit->circuit -curcumstance->circumstance -curcumstances->circumstances -currenly->currently -curriculem->curriculum -cxan->cyan -cyclinder->cylinder -dacquiri->daiquiri -daed->dead -dael->deal, dial, dahl, -dalmation->dalmatian -damenor->demeanor -dammage->damage -Dardenelles->Dardanelles -daugher->daughter -deamon->daemon -deamons->daemons -debateable->debatable -decendant->descendant -decendants->descendants -decendent->descendant -decendents->descendants -decideable->decidable -decidely->decidedly -decieved->deceived -decison->decision -decomissioned->decommissioned -decomposit->decompose -decomposited->decomposed -decompositing->decomposing -decomposits->decomposes -decress->decrees -decribe->describe -decribed->described -decribes->describes -decribing->describing -decriptor->descriptor -dectect->detect -defendent->defendant -defendents->defendants -deffensively->defensively -deffine->define -deffined->defined -definance->defiance -definate->definite -definately->definitely -definatly->definitely -definetly->definitely -definining->defining -definit->definite -definitly->definitely -definiton->definition -defintion->definition -degrate->degrade -delagates->delegates -delapidated->dilapidated -delerious->delirious -delevopment->development -delevopp->develop -deliberatly->deliberately -delusionally->delusively -demenor->demeanor -demographical->demographic -demolision->demolition -demorcracy->democracy -demostration->demonstration -denegrating->denigrating -densly->densely -deparment->department -deparmental->departmental -deparments->departments -depdendencies->dependencies -depdendency->dependency -dependance->dependence -dependancies->dependencies -dependancy->dependency -dependant->dependent -deployement->deployment -deram->dram, dream, -derectory->directory -deriviated->derived -derivitive->derivative -derogitory->derogatory -derprecated->deprecated -descendands->descendants -descibed->described -descision->decision -descisions->decisions -descriibes->describes -descripters->descriptors -descripton->description -desctruction->destruction -descuss->discuss -desgined->designed -deside->decide -desigining->designing -desinations->destinations -desintegrated->disintegrated -desintegration->disintegration -desireable->desirable -desitned->destined -desktiop->desktop -desorder->disorder -desoriented->disoriented -desparate->desperate, disparate, -despict->depict -despiration->desperation -desription->description -dessicated->desiccated -dessigned->designed -destablized->destabilized -destory->destroy -destuction->destruction -detailled->detailed -detatched->detached -deteoriated->deteriorated -deteriate->deteriorate -deterioriating->deteriorating -determinining->determining -detremental->detrimental -devasted->devastated -develope->develop -developement->development -developped->developed -develpment->development -devels->delves -devestated->devastated -devestating->devastating -devide->divide -devided->divided -devistating->devastating -devolopement->development -diablical->diabolical -diamons->diamonds -diaster->disaster -dichtomy->dichotomy -diconnects->disconnects -dicover->discover -dicovered->discovered -dicovering->discovering -dicovers->discovers -dicovery->discovery -dictionarys->dictionaries -dictionnary->dictionary -dicussed->discussed -didnt'->didn't -didnt->didn't -diea->idea, die, -dieing->dying, dyeing, -dieties->deities -diety->deity -diferent->different -diferrent->different -differentiatiations->differentiations -differnt->different -difficulity->difficulty -diffrent->different -dificulties->difficulties -dificulty->difficulty -dimenions->dimensions -dimention->dimension -dimentional->dimensional -dimentions->dimensions -dimesnional->dimensional -diminuitive->diminutive -dimunitive->diminutive -diosese->diocese -diphtong->diphthong -diphtongs->diphthongs -diplomancy->diplomacy -dipthong->diphthong -dipthongs->diphthongs -directoty->directory -dirived->derived -disagreeed->disagreed -disapeared->disappeared -disapointing->disappointing -disappearred->disappeared -disaproval->disapproval -disasterous->disastrous -disatisfaction->dissatisfaction -disatisfied->dissatisfied -disatrous->disastrous -discontentment->discontent -discribe->describe -discribed->described -discribes->describes -discribing->describing -disctinction->distinction -disctinctive->distinctive -disemination->dissemination -disenchanged->disenchanted -disiplined->disciplined -disobediance->disobedience -disobediant->disobedient -disolved->dissolved -disover->discover -dispair->despair -disparingly->disparagingly -dispence->dispense -dispenced->dispensed -dispencing->dispensing -dispicable->despicable -dispite->despite -dispostion->disposition -disproportiate->disproportionate -disputandem->disputandum -disricts->districts -dissagreement->disagreement -dissapear->disappear -dissapearance->disappearance -dissapeared->disappeared -dissapearing->disappearing -dissapears->disappears -dissappear->disappear -dissappears->disappears -dissappointed->disappointed -dissarray->disarray -dissobediance->disobedience -dissobediant->disobedient -dissobedience->disobedience -dissobedient->disobedient -distiction->distinction -distingish->distinguish -distingished->distinguished -distingishes->distinguishes -distingishing->distinguishing -distingquished->distinguished -distrubution->distribution -distruction->destruction -distructive->destructive -ditributed->distributed -diversed->diverse, diverged, -divice->device -divinition->definition,divination, -divison->division -divisons->divisions -doccument->document -doccumented->documented -doccuments->documents -docrines->doctrines -doctines->doctrines -documenatry->documentary -doens->does -doesnt'->doesn't -doesnt->doesn't -doign->doing -dominaton->domination -dominent->dominant -dominiant->dominant -donig->doing -dont->don't -dosen't->doesn't -dosent'->doesn't -doub->doubt, daub, -doulbe->double -dowloads->downloads -dramtic->dramatic -draughtman->draughtsman -Dravadian->Dravidian -dreasm->dreams -driectly->directly -drnik->drink -druming->drumming -drummless->drumless -dstination->destination -dum->dumb -dupicate->duplicate -durig->during -durring->during -duting->during -dyas->dryas -dynamicaly->dynamically -eahc->each -ealier->earlier -earlies->earliest -earnt->earned -ecclectic->eclectic -eceonomy->economy -ecidious->deciduous -eclispe->eclipse -ecomonic->economic -ect->etc -eearly->early -efect->effect -efel->evil -effeciency->efficiency -effecient->efficient -effeciently->efficiently -efficency->efficiency -efficent->efficient -efficently->efficiently -efford->effort, afford, -effords->efforts, affords, -effulence->effluence -eigth->eighth, eight, -eiter->either -elction->election -electic->eclectic, electric, -electon->election, electron, -electrial->electrical -electricly->electrically -electricty->electricity -elemenet->element -elemenets->elements -elementay->elementary -eleminated->eliminated -eleminating->eliminating -eles->else -eletricity->electricity -elicided->elicited -eligable->eligible -elimentary->elementary -ellected->elected -elphant->elephant -emabaroged->embargoed -embarass->embarrass -embarassed->embarrassed -embarassing->embarrassing -embarassment->embarrassment -embargos->embargoes -embarras->embarrass -embarrased->embarrassed -embarrasing->embarrassing -embarrasment->embarrassment -embeded->embedded -embezelled->embezzled -emblamatic->emblematic -eminate->emanate -eminated->emanated -emision->emission -emited->emitted -emiting->emitting -emition->emission, emotion, -emmediately->immediately -emmigrated->emigrated, immigrated, -emminent->eminent, imminent, -emminently->eminently -emmisaries->emissaries -emmisarries->emissaries -emmisarry->emissary -emmisary->emissary -emmision->emission -emmisions->emissions -emmited->emitted -emmiting->emitting -emmitted->emitted -emmitting->emitting -emnity->enmity -emperical->empirical -emphaised->emphasised -emphsis->emphasis -emphysyma->emphysema -empirial->empirical, imperial, -emporer->emperor -emprisoned->imprisoned -enameld->enameled -enchancement->enhancement -encouraing->encouraging -encryptiion->encryption -encylopedia->encyclopedia -endevors->endeavors -endevour->endeavour -endianess->endianness -endig->ending -endolithes->endoliths -enduce->induce -ened->need -enflamed->inflamed -enforceing->enforcing -engagment->engagement -engeneer->engineer -engeneering->engineering -engieneer->engineer -engieneers->engineers -enlargment->enlargement -enlargments->enlargements -Enlish->English, enlist, -enourmous->enormous -enourmously->enormously -enque->enqueue -ensconsed->ensconced -entaglements->entanglements -enteratinment->entertainment -enthousiasm->enthusiasm -enthusiatic->enthusiastic -entitity->entity -entitiy->entity -entitlied->entitled -entrepeneur->entrepreneur -entrepeneurs->entrepreneurs -enviorment->environment -enviormental->environmental -enviormentally->environmentally -enviorments->environments -enviornment->environment -enviornmental->environmental -enviornmentalist->environmentalist -enviornmentally->environmentally -enviornments->environments -enviroment->environment -enviromental->environmental -enviromentalist->environmentalist -enviromentally->environmentally -enviroments->environments -envolutionary->evolutionary -envrionments->environments -enxt->next -epidsodes->episodes -epsiode->episode -equialent->equivalent -equilibium->equilibrium -equilibrum->equilibrium -equiped->equipped -equippment->equipment -equitorial->equatorial -equivelant->equivalent -equivelent->equivalent -equivilant->equivalent -equivilent->equivalent -equivlalent->equivalent -erally->orally, really, -eratic->erratic -eratically->erratically -eraticly->erratically -erested->arrested, erected, -erronous->erroneous -errupted->erupted -esential->essential -esitmated->estimated -esle->else -especialy->especially -essencial->essential -essense->essence -essentail->essential -essentialy->essentially -essentual->essential -essesital->essential -estabishes->establishes -establising->establishing -ethnocentricm->ethnocentrism -ethose->those, ethos, -Europian->European -Europians->Europeans -Eurpean->European -Eurpoean->European -evenhtually->eventually -eventally->eventually -eventhough->even though -eventially->eventually -eventualy->eventually -everthing->everything -everytime->every time -everyting->everything -eveyr->every -evidentally->evidently -exagerate->exaggerate -exagerated->exaggerated -exagerates->exaggerates -exagerating->exaggerating -exagerrate->exaggerate -exagerrated->exaggerated -exagerrates->exaggerates -exagerrating->exaggerating -examinated->examined -exampt->exempt -exapansion->expansion -excact->exact -excange->exchange -excecute->execute -excecuted->executed -excecutes->executes -excecuting->executing -excecution->execution -excedded->exceeded -excelent->excellent -excell->excel -excellance->excellence -excellant->excellent -excells->excels -excercise->exercise -excerciser->exerciser -exchanching->exchanging -excisted->existed -exculsivly->exclusively -execising->exercising -exection->execution -exectued->executed -exeedingly->exceedingly -exelent->excellent -exellent->excellent -exemple->example -exept->except -exeptional->exceptional -exerbate->exacerbate -exerbated->exacerbated -exerciese->exercises -exerpt->excerpt -exerpts->excerpts -exersize->exercise -exerternal->external -exhalted->exalted -exhibtion->exhibition -exibition->exhibition -exibitions->exhibitions -exicting->exciting -exinct->extinct -existance->existence -existant->existent -existince->existence -exliled->exiled -exludes->excludes -exmaple->example -exonorate->exonerate -exoskelaton->exoskeleton -expalin->explain -expatriot->expatriate -expeced->expected -expecially->especially -expeditonary->expeditionary -expeiments->experiments -expell->expel -expells->expels -experiance->experience -experianced->experienced -expiditions->expeditions -expierence->experience -explaination->explanation -explaning->explaining -explictly->explicitly -exploititive->exploitative -explotation->exploitation -expropiated->expropriated -expropiation->expropriation -exressed->expressed -extemely->extremely -extention->extension -extentions->extensions -exteral->external -extered->exerted -extermist->extremist -extint->extinct, extant, -extradiction->extradition -extraterrestial->extraterrestrial -extraterrestials->extraterrestrials -extravagent->extravagant -extrememly->extremely -extremeophile->extremophile -extremly->extremely -extrordinarily->extraordinarily -extrordinary->extraordinary -eyar->year, eyas, -eyars->years, eyas, -eyasr->years, eyas, -faciliate->facilitate -faciliated->facilitated -faciliates->facilitates -facilites->facilities -facillitate->facilitate -facinated->fascinated -facist->fascist -familes->families -familiies->families -familliar->familiar -famoust->famous -fanatism->fanaticism -Farenheit->Fahrenheit -fatc->fact -faught->fought -favoutrable->favourable -feasable->feasible -Febuary->February -Feburary->February -fedreally->federally -femminist->feminist -feromone->pheromone -fertily->fertility -fianite->finite -fianlly->finally -ficticious->fictitious -fictious->fictitious -fidn->find -fiel->feel, field, file, phial, -fiels->feels, fields, files, phials, -fiercly->fiercely -fightings->fighting -filiament->filament -fimilies->families -finacial->financial -finaly->finally -financialy->financially -firends->friends -firts->flirts, first, -fisionable->fissionable -flaged->flagged -flakyness->flakiness -flamable->flammable -flawess->flawless -fleed->fled, freed, -Flemmish->Flemish -florescent->fluorescent -flourescent->fluorescent -flourine->fluorine -flourishment->flourishing -fluorish->flourish -follwoing->following -folowing->following -fomed->formed -fomr->from, form, -fonetic->phonetic -fontrier->fontier -foootball->football -forbad->forbade -forbiden->forbidden -foreward->foreword, forward, -forfiet->forfeit -forhead->forehead -foriegn->foreign -Formalhaut->Fomalhaut -formallize->formalize -formallized->formalized -formaly->formally, formerly, -formated->formatted -formelly->formerly -formidible->formidable -formost->foremost -forsaw->foresaw -forseeable->foreseeable -fortelling->foretelling -forunner->forerunner -foucs->focus -foudn->found -fougth->fought -foundaries->foundries -foundary->foundry -Foundland->Newfoundland -fourties->forties -fourty->forty -fouth->fourth -foward->forward -fragement->fragment -Fransiscan->Franciscan -Fransiscans->Franciscans -freind->friend -freindly->friendly -frequentily->frequently -frome->from -fromed->formed -froniter->frontier -fucntion->function -fucntioning->functioning -fufill->fulfill -fufilled->fulfilled -fulfiled->fulfilled -fullfill->fulfill -fullfilled->fulfilled -funciton->function -functino->function -functionnality->functionality -fundametal->fundamental -fundametals->fundamentals -funguses->fungi -funtion->function -furuther->further -futher->further -futhermore->furthermore -futhroc->futhark, futhorc, -gae->game, Gael, gale, -galatic->galactic -Galations->Galatians -gallaxies->galaxies -galvinized->galvanized -Gameboy->Game Boy -ganerate->generate -ganes->games -ganster->gangster -garantee->guarantee -garanteed->guaranteed -garantees->guarantees -gardai->gardaí -garnison->garrison -gauarana->guaraná -gaurantee->guarantee -gauranteed->guaranteed -gaurantees->guarantees -gaurd->guard, gourd, -gaurentee->guarantee -gaurenteed->guaranteed -gaurentees->guarantees -geneological->genealogical -geneologies->genealogies -geneology->genealogy -generaly->generally -generatting->generating -genialia->genitalia -geographicial->geographical -geometrician->geometer -geometricians->geometers -gerat->great -Ghandi->Gandhi -glight->flight -gnawwed->gnawed -godess->goddess -godesses->goddesses -Godounov->Godunov -gogin->going, Gauguin, -goign->going -gonig->going -Gothenberg->Gothenburg -Gottleib->Gottlieb -gouvener->governor -govement->government -govenment->government -govenrment->government -goverance->governance -goverment->government -govermental->governmental -governer->governor -governmnet->government -govorment->government -govormental->governmental -govornment->government -gracefull->graceful -graet->great -grafitti->graffiti -gramatically->grammatically -grammaticaly->grammatically -grammer->grammar -grat->great -gratuitious->gratuitous -greatful->grateful -greatfully->gratefully -greif->grief -gridles->griddles -gropu->group -grwo->grow -Guaduloupe->Guadalupe, Guadeloupe, -Guadulupe->Guadalupe, Guadeloupe, -guage->gauge -guarentee->guarantee -guarenteed->guaranteed -guarentees->guarantees -guarrenteed->guaranteed -Guatamala->Guatemala -Guatamalan->Guatemalan -guerilla->guerrilla -guerillas->guerrillas -guerrila->guerrilla -guerrilas->guerrillas -gueswork->guesswork -guidence->guidance -Guilia->Giulia -Guilio->Giulio -Guiness->Guinness -Guiseppe->Giuseppe -gunanine->guanine -gurantee->guarantee -guranteed->guaranteed -gurantees->guarantees -guttaral->guttural -gutteral->guttural -habaeus->habeas -habeus->habeas -Habsbourg->Habsburg -haemorrage->haemorrhage -haev->have, heave, -halarious->hilarious -Hallowean->Hallowe'en, Halloween, -halp->help -hapen->happen -hapened->happened -hapening->happening -happend->happened -happended->happened -happenned->happened -harased->harassed -harases->harasses -harasment->harassment -harasments->harassments -harassement->harassment -harras->harass -harrased->harassed -harrases->harasses -harrasing->harassing -harrasment->harassment -harrasments->harassments -harrassed->harassed -harrasses->harassed -harrassing->harassing -harrassment->harassment -harrassments->harassments -hasnt'->hasn't -hasnt->hasn't -Hatian->Haitian -haviest->heaviest -headquarer->headquarter -headquater->headquarter -headquatered->headquartered -headquaters->headquarters -healthercare->healthcare -heared->heard -heathy->healthy -Heidelburg->Heidelberg -heigher->higher -heirachies->hierarchies -heirarchy->hierarchy -heiroglyphics->hieroglyphics -helment->helmet -helpfull->helpful -helpped->helped -hemmorhage->hemorrhage -herad->heard, Hera, -heridity->heredity -heroe->hero -heros->heroes -hertiage->heritage -hertzs->hertz -hesistant->hesitant -heterogenous->heterogeneous -hexidecimal->hexadecimal -hieght->height -hierachical->hierarchical -hierachies->hierarchies -hierachy->hierarchy -hierarcical->hierarchical -hierarcy->hierarchy -hieroglph->hieroglyph -hieroglphs->hieroglyphs -higer->higher -higest->highest -higway->highway -hillarious->hilarious -himselv->himself -hinderance->hindrance -hinderence->hindrance -hindrence->hindrance -hipopotamus->hippopotamus -hismelf->himself -histocompatability->histocompatibility -historicians->historians -hitsingles->hit singles -holf->hold -holliday->holiday -homestate->home state -homogeneize->homogenize -homogeneized->homogenized -honory->honorary -horrifing->horrifying -hosited->hoisted -hospitible->hospitable -hounour->honour -housr->hours, house, -howver->however -hsitorians->historians -hstory->history -hten->then, hen, the, -htere->there, here, -htey->they -htikn->think -hting->thing -htink->think -htis->this -humer->humor, humour, -humerous->humorous, humerus, -huminoid->humanoid -humoural->humoral -humurous->humorous -husban->husband -hvae->have -hvaing->having -hvea->have, heave, -hwihc->which -hwile->while -hwole->whole -hydogen->hydrogen -hydropile->hydrophile -hydropilic->hydrophilic -hydropobe->hydrophobe -hydropobic->hydrophobic -hygeine->hygiene -hypocracy->hypocrisy -hypocrasy->hypocrisy -hypocricy->hypocrisy -hypocrit->hypocrite -hypocrits->hypocrites -iconclastic->iconoclastic -idaeidae->idea -idaes->ideas -idealogies->ideologies -idealogy->ideology -identicial->identical -identifers->identifiers -ideosyncratic->idiosyncratic -idesa->ideas, ides, -idiosyncracy->idiosyncrasy -Ihaca->Ithaca -illegimacy->illegitimacy -illegitmate->illegitimate -illess->illness -illiegal->illegal -illution->illusion -ilness->illness -ilogical->illogical -imagenary->imaginary -imagin->imagine -imaginery->imaginary, imagery, -imanent->eminent, imminent, -imcoming->incoming -imcomplete->incomplete -imediately->immediately -imense->immense -imigrant->emigrant, immigrant, -imigrated->emigrated, immigrated, -imigration->emigration, immigration, -iminent->eminent, imminent, immanent, -immediatelly->immediately -immediatley->immediately -immediatly->immediately -immidately->immediately -immidiately->immediately -immitate->imitate -immitated->imitated -immitating->imitating -immitator->imitator -immunosupressant->immunosuppressant -impecabbly->impeccably -impedence->impedance -implamenting->implementing -impliment->implement -implimented->implemented -imploys->employs -importamt->important -impovements->improvements -impressario->impresario -imprioned->imprisoned -imprisonned->imprisoned -improvision->improvisation -improvments->improvements -inablility->inability -inaccesible->inaccessible -inaccessable->inaccessible -inadiquate->inadequate -inadquate->inadequate -inadvertant->inadvertent -inadvertantly->inadvertently -inagurated->inaugurated -inaguration->inauguration -inappropiate->inappropriate -inaugures->inaugurates -inbalance->imbalance -inbalanced->imbalanced -inbetween->between -incarcirated->incarcerated -incidentially->incidentally -incidently->incidentally -inclreased->increased -includ->include -includng->including -incomming->incoming -incompatabilities->incompatibilities -incompatability->incompatibility -incompatable->incompatible -incompatablities->incompatibilities -incompatablity->incompatibility -incompatiblities->incompatibilities -incompatiblity->incompatibility -incompetance->incompetence -incompetant->incompetent -incomptable->incompatible -incomptetent->incompetent -inconsistant->inconsistent -inconsitent->inconsistent -incoroporated->incorporated -incorperation->incorporation -incorportaed->incorporated -incorprates->incorporates -incorruptable->incorruptible -incramentally->incrementally -increadible->incredible -incredable->incredible -incremeantal->incremental -inctroduce->introduce -inctroduced->introduced -incuding->including -incunabla->incunabula -indefinately->indefinitely -indefineable->undefinable -indefinitly->indefinitely -indentical->identical -indentifier->identifier -indepedantly->independently -indepedence->independence -independance->independence -independant->independent -independantly->independently -independece->independence -independendet->independent -indespensable->indispensable -indespensible->indispensable -indicies->indices -indictement->indictment -indigineous->indigenous -indipendence->independence -indipendent->independent -indipendently->independently -indispensible->indispensable -indisputible->indisputable -indisputibly->indisputably -indite->indict -individualy->individually -indpendent->independent -indpendently->independently -indulgue->indulge -indutrial->industrial -indviduals->individuals -inefficienty->inefficiently -inevatible->inevitable -inevitible->inevitable -inevititably->inevitably -infalability->infallibility -infallable->infallible -infectuous->infectious -infered->inferred -infilitrate->infiltrate -infilitrated->infiltrated -infilitration->infiltration -infinit->infinite -inflamation->inflammation -influencial->influential -influented->influenced -infomation->information -informtion->information -infrantryman->infantryman -infrigement->infringement -ingenius->ingenious -ingration->integration -ingreediants->ingredients -inhabitans->inhabitants -inherantly->inherently -inheritage->heritage, inheritance, -inheritence->inheritance -inifinite->infinite -inital->initial -initalise->initialise -initalization->initialization -initalize->initialize -initalizer->initializer -initally->initially -initation->initiation -initiaitive->initiative -initializiation->initialization -initilize->initialize -inlcuding->including -inmigrant->immigrant -inmigrants->immigrants -innoculated->inoculated -inocence->innocence -inofficial->unofficial -inot->into -inpeach->impeach -inpolite->impolite -inprisonment->imprisonment -inproving->improving -insectiverous->insectivorous -insensative->insensitive -insensetive->insensitive -inseperable->inseparable -insistance->insistence -insitution->institution -insitutions->institutions -inspite->in spite, inspire, -instade->instead -instanciation->instantiation -instatance->instance -institue->institute -instuction->instruction -instuments->instruments -instutionalized->institutionalized -instutions->intuitions -insurence->insurance -intead->instead -intelectual->intellectual -inteligence->intelligence -inteligent->intelligent -intenational->international -intented->intended, indented, -intepretation->interpretation -intepretator->interpretor -interal->internal, integral, -interational->international -interbread->interbreed, interbred, -interchangable->interchangeable -interchangably->interchangeably -intercontinential->intercontinental -intercontinetal->intercontinental -intered->interred, interned, -interelated->interrelated -interferance->interference -interfereing->interfering -interger->integer -intergrated->integrated -intergration->integration -interm->interim -internation->international -interpet->interpret -interrim->interim -interrput->interrupt -interrugum->interregnum -intertaining->entertaining -interupt->interrupt -intervines->intervenes -intevene->intervene -intial->initial -intialization->initialization -intially->initially -intrduced->introduced -intrest->interest -introdued->introduced -intruction->instruction -intruduced->introduced -intrument->instrument -intrumental->instrumental -intruments->instruments -intrusted->entrusted -intutive->intuitive -intutively->intuitively -inudstry->industry -inumerable->enumerable, innumerable, -invaild->invalid -inventer->inventor -invertibrates->invertebrates -investingate->investigate -involvment->involvement -irelevent->irrelevant -iresistable->irresistible -iresistably->irresistibly -iresistible->irresistible -iresistibly->irresistibly -iritable->irritable -iritated->irritated -ironicly->ironically -irregardless->regardless -irrelevent->irrelevant -irreplacable->irreplaceable -irresistable->irresistible -irresistably->irresistibly -isnt'->isn't -isnt->isn't -Israelies->Israelis -issueing->issuing -itnernal->internal -itnroduced->introduced -iunior->junior -iwll->will -iwth->with -Janurary->January -Januray->January -Japanes->Japanese -jaques->jacques -jeapardy->jeopardy -jewllery->jewellery -Johanine->Johannine -jorunal->journal -Jospeh->Joseph -jouney->journey -journied->journeyed -journies->journeys -jstu->just -jsut->just -Juadaism->Judaism -Juadism->Judaism -judical->judicial -judisuary->judiciary -juducial->judicial -juristiction->jurisdiction -juristictions->jurisdictions -kenrel->kernel -kindergarden->kindergarten -klenex->kleenex -knifes->knives -knive->knife -knowlege->knowledge -knowlegeable->knowledgeable -knwo->know -knwos->knows -konw->know -konws->knows -kwno->know -labatory->lavatory, laboratory, -labled->labelled, labeled, -labratory->laboratory -laguage->language -laguages->languages -larg->large -largst->largest -larrry->larry -lastr->last -lattitude->latitude -launchs->launch, launches, -launhed->launched -lavae->larvae -layed->laid -lazyness->laziness -leage->league -leanr->lean, learn, leaner, -leathal->lethal -lefted->left -legitamate->legitimate -legitmate->legitimate -leibnitz->leibniz -lenght->length -leran->learn -lerans->learns -leutenant->lieutenant -levetate->levitate -levetated->levitated -levetates->levitates -levetating->levitating -levle->level -liasion->liaison -liason->liaison -liasons->liaisons -libary->library -libell->libel -libguistic->linguistic -libguistics->linguistics -libitarianisn->libertarianism -lible->libel, liable, -licenced->licensed -lieing->lying -liek->like -liekd->liked -liesure->leisure -lieuenant->lieutenant -lieved->lived -liftime->lifetime -lightyear->light year -lightyears->light years -likelyhood->likelihood -linnaena->linnaean -lippizaner->lipizzaner -liquify->liquefy -liscense->license, licence, -lisence->license, licence, -lisense->license, licence, -listners->listeners -litature->literature -literaly->literally -literture->literature -littel->little -litterally->literally -liuke->like -livley->lively -lmits->limits -loev->love -loger->logger, longer, -lonelyness->loneliness -longitudonal->longitudinal -longuer->longer -lonley->lonely -lonly->lonely, only, -loosing->losing -lotharingen->lothringen -lsat->last -lukid->likud -lveo->love -lvoe->love -Lybia->Libya -maching->machine, marching, matching, -mackeral->mackerel -magasine->magazine -magincian->magician -magisine->magazine -magizine->magazine -magnificient->magnificent -magolia->magnolia -mailny->mainly -maintainance->maintenance -maintainence->maintenance -maintance->maintenance -maintenence->maintenance -maintinaing->maintaining -maintioned->mentioned -majoroty->majority -maked->marked, made, -makse->makes -Malcom->Malcolm -maltesian->Maltese -mamal->mammal -mamalian->mammalian -managable->manageable, manageably, -managment->management -maneouvre->manoeuvre -maneouvred->manoeuvred -maneouvres->manoeuvres -maneouvring->manoeuvring -manisfestations->manifestations -mannor->manner -manoeuverability->maneuverability -manouver->maneuver, manoeuvre, -manouverability->maneuverability, manoeuvrability, manoeuverability, -manouverable->maneuverable, manoeuvrable, -manouvers->maneuvers, manoeuvres, -mantained->maintained -manuever->maneuver, manoeuvre, -manuevers->maneuvers, manoeuvres, -manufacturedd->manufactured -manufature->manufacture -manufatured->manufactured -manufaturing->manufacturing -manuver->maneuver -mapp->map -mariage->marriage -marjority->majority -markes->marks, marked, -marketting->marketing -marmelade->marmalade -marrage->marriage -marraige->marriage -marrtyred->martyred -marryied->married -Massachussets->Massachusetts -Massachussetts->Massachusetts -massmedia->mass media -masterbation->masturbation -mataphysical->metaphysical -materalists->materialist -mathamatics->mathematics -mathematican->mathematician -mathematicas->mathematics -matheticians->mathematicians -mathmatically->mathematically -mathmatician->mathematician -mathmaticians->mathematicians -mccarthyst->mccarthyist -mchanics->mechanics -meaninng->meaning -mear->wear, mere, mare, -mechandise->merchandise -medacine->medicine -medeival->medieval -medevial->medieval -mediciney->mediciny -medievel->medieval -mediterainnean->mediterranean -Mediteranean->Mediterranean -meerkrat->meerkat -melieux->milieux -membranaphone->membranophone -memeber->member -menally->mentally -meranda->veranda, Miranda, -mercentile->mercantile -messagin->messaging -messanger->messenger -messenging->messaging -metalic->metallic -metalurgic->metallurgic -metalurgical->metallurgical -metalurgy->metallurgy -metamorphysis->metamorphosis -metaphoricial->metaphorical -meterologist->meteorologist -meterology->meteorology -methaphor->metaphor -methaphors->metaphors -Michagan->Michigan -micoscopy->microscopy -midwifes->midwives -mileau->milieu -milennia->millennia -milennium->millennium -mileu->milieu -miliary->military -miligram->milligram -milion->million -miliraty->military -millenia->millennia -millenial->millennial -millenialism->millennialism -millenium->millennium -millepede->millipede -millioniare->millionaire -millitary->military -millon->million -miltary->military -minature->miniature -minerial->mineral -MingGW->MinGW -miniscule->minuscule -ministery->ministry -minsitry->ministry -minstries->ministries -minstry->ministry -minumum->minimum -mirrorred->mirrored -miscelaneous->miscellaneous -miscellanious->miscellaneous -miscellanous->miscellaneous -mischeivous->mischievous -mischevious->mischievous -mischievious->mischievous -misdameanor->misdemeanor -misdameanors->misdemeanors -misdemenor->misdemeanor -misdemenors->misdemeanors -misfourtunes->misfortunes -misile->missile -Misouri->Missouri -mispell->misspell -mispelled->misspelled -mispelling->misspelling -missen->mizzen -Missisipi->Mississippi -Missisippi->Mississippi -missle->missile -missonary->missionary -misterious->mysterious -mistery->mystery -misteryous->mysterious -mkae->make -mkaes->makes -mkaing->making -mkea->make -moderm->modem -modle->model -moduel->module -moduels->modules -moent->moment -moeny->money -mohammedans->muslims -moil->mohel -moil->soil -moleclues->molecules -momento->memento -monestaries->monasteries -monestary->monastery, monetary, -monickers->monikers -monolite->monolithic -Monserrat->Montserrat -montains->mountains -montanous->mountainous -Montnana->Montana -monts->months -montypic->monotypic -moreso->more, more so, -morgage->mortgage -Morisette->Morissette -Morrisette->Morissette -morroccan->moroccan -morrocco->morocco -morroco->morocco -mortage->mortgage -mosture->moisture -motiviated->motivated -mounth->month -movei->movie, disabled due to assembly code -movment->movement -mroe->more -mucuous->mucous -muder->murder -mudering->murdering -muhammadan->muslim -multicultralism->multiculturalism -multifuction->multifunction -multipled->multiplied -multiplers->multipliers -multy-thread->multithread -munbers->numbers -muncipalities->municipalities -muncipality->municipality -munnicipality->municipality -muscels->mussels, muscles, -muscial->musical -muscician->musician -muscicians->musicians -mutiliated->mutilated -myraid->myriad -mysef->myself -mysogynist->misogynist -mysogyny->misogyny -mysterous->mysterious -Mythraic->Mithraic -naieve->naive -Naploeon->Napoleon -Napolean->Napoleon -Napoleonian->Napoleonic -naturaly->naturally -naturely->naturally -naturual->natural -naturually->naturally -Nazereth->Nazareth -neccesarily->necessarily -neccesary->necessary -neccessarily->necessarily -neccessary->necessary -neccessities->necessities -necesarily->necessarily -necesary->necessary -necessiate->necessitate -negitive->negative -neglible->negligible -negligable->negligible -negligble->negligible -negociate->negotiate -negociation->negotiation -negociations->negotiations -negotation->negotiation -neice->niece, nice, -neigborhood->neighborhood -neigbour->neighbour, neighbor, -neigbourhood->neighbourhood -neigbouring->neighbouring, neighboring, -neigbours->neighbours, neighbors, -neolitic->neolithic -nescessary->necessary -nessasarily->necessarily -nessecary->necessary -nestin->nesting -neverthless->nevertheless -newletters->newsletters -Newyorker->New Yorker -nickle->nickel -nightfa;;->nightfall -nightime->nighttime -nineth->ninth -ninteenth->nineteenth -ninties->1990s -ninty->ninety -nkow->know -nkwo->know -nmae->name -noncombatents->noncombatants -nonexistant->nonexistent -nonsence->nonsense -nontheless->nonetheless -noone->no one -norhern->northern -northen->northern -northereastern->northeastern -notabley->notably -noteable->notable -noteably->notably -noteriety->notoriety -noth->north -nothern->northern -noticable->noticeable -noticably->noticeably -noticeing->noticing -noticible->noticeable -notifcation->notification -notwhithstanding->notwithstanding -noveau->nouveau -Novermber->November -nowdays->nowadays -nowe->now -nto->not, disable due to \n -nucular->nuclear -nuculear->nuclear -nuisanse->nuisance -Nullabour->Nullarbor -numberous->numerous -Nuremburg->Nuremberg -nusance->nuisance -nutritent->nutrient -nutritents->nutrients -nuturing->nurturing -obediance->obedience -obediant->obedient -obession->obsession -obssessed->obsessed -obstacal->obstacle -obstancles->obstacles -obstruced->obstructed -ocasion->occasion -ocasional->occasional -ocasionally->occasionally -ocasionaly->occasionally -ocasioned->occasioned -ocasions->occasions -ocassion->occasion -ocassional->occasional -ocassionally->occasionally -ocassionaly->occasionally -ocassioned->occasioned -ocassions->occasions -occaison->occasion -occassion->occasion -occassional->occasional -occassionally->occasionally -occassionaly->occasionally -occassioned->occasioned -occassions->occasions -occationally->occasionally -occour->occur -occurance->occurrence -occurances->occurrences -occure->occur -occured->occurred -occurence->occurrence -occurences->occurrences -occures->occurs -occuring->occurring -occurr->occur -occurrance->occurrence -occurrances->occurrences -octohedra->octahedra -octohedral->octahedral -octohedron->octahedron -ocuntries->countries -ocuntry->country -ocurr->occur -ocurrance->occurrence -ocurred->occurred -ocurrence->occurrence -offcers->officers -offcially->officially -offereings->offerings -offical->official -offically->officially -officals->officials -officaly->officially -officialy->officially -offred->offered -oftenly->often -oging->going, ogling, -olny->only -omision->omission -omited->omitted -omiting->omitting -omlette->omelette -ommision->omission -ommited->omitted -ommiting->omitting -ommitted->omitted -ommitting->omitting -omniverous->omnivorous -omniverously->omnivorously -omre->more -onot->note, not, -onyl->only -openess->openness -openin->opening -oponent->opponent -oportunity->opportunity -opose->oppose -oposite->opposite -oposition->opposition -oppenly->openly -oppinion->opinion -opponant->opponent -oppononent->opponent -oppositition->opposition -oppossed->opposed -opprotunity->opportunity -opression->oppression -opressive->oppressive -opthalmic->ophthalmic -opthalmologist->ophthalmologist -opthalmology->ophthalmology -opthamologist->ophthalmologist -optmizations->optimizations -optomism->optimism -orded->ordered -organim->organism -organistion->organisation -organiztion->organization -orgin->origin, organ, -orginal->original -orginal->orignal -orginally->originally -orginize->organise, organize, -oridinarily->ordinarily -origanaly->originally -originall->original, originally, -originaly->originally -originially->originally -originnally->originally -origional->original -orignally->originally -orignially->originally -otehr->other -oublisher->publisher -ouevre->oeuvre -oustanding->outstanding -oveerun->overrun -overlayed->overlaid -overriddden->overridden -overshaddowed->overshadowed -overthere->over there -overwelming->overwhelming -overwheliming->overwhelming -owrk->work -owudl->would -oxigen->oxygen -oximoron->oxymoron -p0enis->penis -paide->paid -paitience->patience -palce->place, palace, -paleolitic->paleolithic -paliamentarian->parliamentarian -Palistian->Palestinian -Palistinian->Palestinian -Palistinians->Palestinians -pallete->palette -pamflet->pamphlet -pamplet->pamphlet -pantomine->pantomime -Papanicalou->Papanicolaou -paralel->parallel -paralell->parallel -paralelly->parallelly -paralely->parallelly -parallell->parallel -parallely->parallelly -paramter->parameter -paramters->parameters -paranthesis->parenthesis -paraphenalia->paraphernalia -parellels->parallels -parisitic->parasitic -parituclar->particular -parliment->parliament -parrakeets->parakeets -parralel->parallel -parrallel->parallel -parrallell->parallel -parrallelly->parallelly -parrallely->parallelly -partialy->partially -particually->particularly -particualr->particular -particuarly->particularly -particularily->particularly -particulary->particularly -pary->party -pased->passed -pasengers->passengers -passerbys->passersby -pasttime->pastime -pastural->pastoral -pathes->paths -paticular->particular -pattented->patented -pavillion->pavilion -payed->paid -pblisher->publisher -pbulisher->publisher -peacefuland->peaceful and -peageant->pageant -peculure->peculiar -pedestrain->pedestrian -peformed->performed -peice->piece -Peloponnes->Peloponnesus -penatly->penalty -penerator->penetrator -penisula->peninsula -penisular->peninsular -penninsula->peninsula -penninsular->peninsular -pennisula->peninsula -Pennyslvania->Pennsylvania -pensinula->peninsula -pensle->pencil -peom->poem -peoms->poems -peopel->people -peotry->poetry -perade->parade -percepted->perceived -percieve->perceive -percieved->perceived -perenially->perennially -perfomance->performance -perfomers->performers -performence->performance -performes->performed, performs, -perfrom->perform -perfromance->performance -perfroms->performs -perhasp->perhaps -perheaps->perhaps -perhpas->perhaps -peripathetic->peripatetic -peristent->persistent -perjery->perjury -perjorative->pejorative -permanant->permanent -permenant->permanent -permenantly->permanently -permissable->permissible -perogative->prerogative -peronal->personal -perosnality->personality -perpertrated->perpetrated -perphas->perhaps -perpindicular->perpendicular -perseverence->perseverance -persistance->persistence -persistant->persistent -personel->personnel, personal, -personell->personnel -personnell->personnel -persuded->persuaded -persue->pursue -persued->pursued -persuing->pursuing -persuit->pursuit -persuits->pursuits -pertubation->perturbation -pertubations->perturbations -pessiary->pessary -petetion->petition -Pharoah->Pharaoh -phenomenom->phenomenon -phenomenonal->phenomenal -phenomenonly->phenomenally -phenomonenon->phenomenon -phenomonon->phenomenon -phenonmena->phenomena -Philipines->Philippines -philisopher->philosopher -philisophical->philosophical -philisophy->philosophy -Phillipine->Philippine -Phillipines->Philippines -Phillippines->Philippines -phillosophically->philosophically -philospher->philosopher -philosphies->philosophies -philosphy->philosophy -Phonecian->Phoenecian -phongraph->phonograph -phsyically->physically -phylosophical->philosophical -physcial->physical -physicaly->physically -piblisher->publisher -pich->pitch -pilgrimmage->pilgrimage -pilgrimmages->pilgrimages -pinapple->pineapple -pinnaple->pineapple -pinoneered->pioneered -plaftorm->platform -plaftorms->platforms -plagarism->plagiarism -planation->plantation -planed->planned -plantiff->plaintiff -plateu->plateau -platfrom->platform -plathome->platform -plausable->plausible -playright->playwright -playwrite->playwright -playwrites->playwrights -pleasent->pleasant -plebicite->plebiscite -plesant->pleasant -poenis->penis -poeoples->peoples -poety->poetry -poisin->poison -polical->political -polinator->pollinator -polinators->pollinators -politican->politician -politicans->politicians -poltical->political -polute->pollute -poluted->polluted -polutes->pollutes -poluting->polluting -polution->pollution -polyphonyic->polyphonic -polysaccaride->polysaccharide -polysaccharid->polysaccharide -pomegranite->pomegranate -pomotion->promotion -poportional->proportional -popoulation->population -popularaty->popularity -populare->popular -populer->popular -porshan->portion -porshon->portion -portait->portrait -portayed->portrayed -portraing->portraying -Portugese->Portuguese -portuguease->portuguese -portugues->Portuguese -posess->possess -posessed->possessed -posesses->possesses -posessing->possessing -posession->possession -posessions->possessions -posion->poison -positon->position -possable->possible -possably->possibly -posseses->possesses -possesing->possessing -possesion->possession -possessess->possesses -possibile->possible -possibilty->possibility -possiblility->possibility -possiblilty->possibility -possiblities->possibilities -possiblity->possibility -possition->position -Postdam->Potsdam -posthomous->posthumous -postion->position -postive->positive -potatos->potatoes -potrait->portrait -potrayed->portrayed -poulations->populations -poverful->powerful -poweful->powerful -powerfull->powerful -ppublisher->publisher -practial->practical -practially->practically -practicaly->practically -practicioner->practitioner -practicioners->practitioners -practicly->practically -practioner->practitioner -practioners->practitioners -prairy->prairie -prarie->prairie -praries->prairies -pratice->practice -preample->preamble -precedessor->predecessor -preceed->precede -preceeded->preceded -preceeding->preceding -preceeds->precedes -precentage->percentage -precice->precise -precisly->precisely -precurser->precursor -predecesors->predecessors -predicatble->predictable -predicitons->predictions -predomiantly->predominately -prefered->preferred -prefering->preferring -preferrable->preferable -preferrably->preferably -preform->perform -preformance->performance -preforms->performs -pregancies->pregnancies -preiod->period -preliferation->proliferation -premeire->premiere -premeired->premiered -premillenial->premillennial -preminence->preeminence -premission->permission -Premonasterians->Premonstratensians -preocupation->preoccupation -prepair->prepare -prepartion->preparation -prepatory->preparatory -preperation->preparation -preperations->preparations -prepresent->represent -preriod->period -presance->presence -presedential->presidential -presense->presence -presidenital->presidential -presidental->presidential -presist->persist -presitgious->prestigious -prespective->perspective -prestigeous->prestigious -prestigous->prestigious -presumabely->presumably -presumibly->presumably -pretection->protection -prevelant->prevalent -preverse->perverse -previvous->previous -pricipal->principal -priciple->principle -priestood->priesthood -primarly->primarily -primative->primitive -primatively->primitively -primatives->primitives -primordal->primordial -principaly->principality -principial->principal -principlaity->principality -principly->principally -prinicipal->principal -printting->printing -privalege->privilege -privaleges->privileges -priveledges->privileges -privelege->privilege -priveleged->privileged -priveleges->privileges -privelige->privilege -priveliged->privileged -priveliges->privileges -privelleges->privileges -privilage->privilege -priviledge->privilege -priviledges->privileges -privledge->privilege -privte->private -probabilaty->probability -probablistic->probabilistic -probablly->probably -probalibity->probability -probaly->probably -probelm->problem -proccess->process -proccessing->processing -procede->proceed, precede, -proceded->proceeded, preceded, -procedes->proceeds, precedes, -procedger->procedure -proceding->proceeding, preceding, -procedings->proceedings -proceedure->procedure -proces->process -processer->processor -proclaimation->proclamation -proclamed->proclaimed -proclaming->proclaiming -proclomation->proclamation -profesion->profusion, profession, -profesor->professor -professer->professor -proffesed->professed -proffesion->profession -proffesional->professional -proffesor->professor -profilic->prolific -progessed->progressed -progidy->prodigy -programable->programmable -progrom->pogrom, program, -progroms->pogroms, programs, -prohabition->prohibition -prologomena->prolegomena -prominance->prominence -prominant->prominent -prominantly->prominently -prominately->prominently, predominately, -promiscous->promiscuous -promiss->promise -promotted->promoted -pronomial->pronominal -pronouced->pronounced -pronounched->pronounced -pronounciation->pronunciation -proove->prove -prooved->proved -prophacy->prophecy -propietary->proprietary -propmted->prompted -propoganda->propaganda -propogate->propagate -propogates->propagates -propogation->propagation -propostion->proposition -propotions->proportions -propper->proper -propperly->properly -proprietory->proprietary -proseletyzing->proselytizing -protaganist->protagonist -protaganists->protagonists -protocal->protocol -protoganist->protagonist -protrayed->portrayed -protruberance->protuberance -protruberances->protuberances -prouncements->pronouncements -provacative->provocative -provded->provided -provicial->provincial -provinicial->provincial -provisiosn->provision -provisonal->provisional -proximty->proximity -pseudononymous->pseudonymous -pseudonyn->pseudonym -psuedo->pseudo -psycology->psychology -psyhic->psychic -pubilsher->publisher -pubisher->publisher -publiaher->publisher -publically->publicly -publicaly->publicly -publicher->publisher -publihser->publisher -publisehr->publisher -publiser->publisher -publisger->publisher -publisheed->published -publisherr->publisher -publishher->publisher -publishor->publisher -publishre->publisher -publissher->publisher -publlisher->publisher -publsiher->publisher -publusher->publisher -puchasing->purchasing -Pucini->Puccini -Puertorrican->Puerto Rican -Puertorricans->Puerto Ricans -pulisher->publisher -pumkin->pumpkin -puplisher->publisher -puritannical->puritanical -purposedly->purposely -purpotedly->purportedly -pursuade->persuade -pursuaded->persuaded -pursuades->persuades -pususading->persuading -puting->putting -pwoer->power -pyscic->psychic -qtuie->quite, quiet, -quantaty->quantity -quantitiy->quantity -quarantaine->quarantine -Queenland->Queensland -questonable->questionable -quicklyu->quickly -quinessential->quintessential -quitted->quit -quizes->quizzes -qutie->quite, quiet, -rabinnical->rabbinical -racaus->raucous -radiactive->radioactive -radify->ratify -raelly->really -rarified->rarefied -reaccurring->recurring -reacing->reaching -reacll->recall -readible->readable -readmition->readmission -realitvely->relatively -realsitic->realistic -realtions->relations -realy->really -realyl->really -reaons->reasons -reasearch->research -rebiulding->rebuilding -rebllions->rebellions -rebounce->rebound -reccomend->recommend -reccomendations->recommendations -reccomended->recommended -reccomending->recommending -reccommend->recommend -reccommended->recommended -reccommending->recommending -reccuring->recurring -receeded->receded -receeding->receding -receivedfrom->received from -recepient->recipient -recepients->recipients -receving->receiving -rechargable->rechargeable -reched->reached -recide->reside -recided->resided -recident->resident -recidents->residents -reciding->residing -reciepents->recipients -reciept->receipt -recieve->receive -recieved->received -reciever->receiver -recievers->receivers -recieves->receives -recieving->receiving -recipiant->recipient -recipiants->recipients -recived->received -recivership->receivership -recogise->recognise -recogize->recognize -recomend->recommend -recomended->recommended -recomending->recommending -recomends->recommends -recommedations->recommendations -recompence->recompense -reconaissance->reconnaissance -reconcilation->reconciliation -reconized->recognized -reconnaisance->reconnaissance -reconnaissence->reconnaissance -recontructed->reconstructed -recordproducer->record producer -recquired->required -recrational->recreational -recrod->record -recuiting->recruiting -recuring->recurring -recurrance->recurrence -rediculous->ridiculous -reedeming->redeeming -reenforced->reinforced -refect->reflect -refedendum->referendum -referal->referral -referece->reference -refereces->references -refered->referred -referemce->reference -referemces->references -referencs->references -referenece->reference -refereneced->referenced -refereneces->references -referiang->referring -refering->referring -refernce->reference -refernce->references -refernces->references -referrence->reference -referrences->references -referrs->refers -reffered->referred -refference->reference -reffering->referring -refrence->reference -refrences->references -refrers->refers -refridgeration->refrigeration -refridgerator->refrigerator -refromist->reformist -refusla->refusal -regardes->regards -registrs->registers -regluar->regular -reguarly->regularly -regulaion->regulation -regulaotrs->regulators -regularily->regularly -rehersal->rehearsal -reicarnation->reincarnation -reigining->reigning -reknown->renown -reknowned->renowned -rela->real, disabled due to lots of false positives -relaly->really -relatiopnship->relationship -relativly->relatively -relected->reelected -releive->relieve -releived->relieved -releiver->reliever -releses->releases -relevence->relevance -relevent->relevant -reliablity->reliability -relient->reliant -religeous->religious -religous->religious -religously->religiously -relinqushment->relinquishment -relitavely->relatively -relized->realised, realized, -relpacement->replacement -relys->relies -remaing->remaining -remeber->remember -rememberable->memorable -rememberance->remembrance -remembrence->remembrance -remenant->remnant -remenicent->reminiscent -reminent->remnant -reminescent->reminiscent -reminscent->reminiscent -reminsicent->reminiscent -rendevous->rendezvous -rendezous->rendezvous -renedered->rende -renewl->renewal -rennovate->renovate -rennovated->renovated -rennovating->renovating -rennovation->renovation -rentors->renters -reoccurrence->recurrence -reorganision->reorganisation -repatition->repetition, repartition, -repblic->republic -repblican->republican -repblicans->republicans -repblics->republics -repectively->respectively -repeition->repetition -repentence->repentance -repentent->repentant -repeteadly->repeatedly -repetion->repetition -repid->rapid -reponse->response -reponses->responses -reponsible->responsible -reportadly->reportedly -represantative->representative -representiative->representative -representive->representative -representives->representatives -represnted->represented -reproducabely->reproducibly -reproducable->reproducible -reprtoire->repertoire -repsectively->respectively -reptition->repetition -repubic->republic -repubican->republican -repubicans->republicans -repubics->republics -republi->republic -republian->republican -republians->republicans -republis->republics -repulic->republic -repulican->republican -repulicans->republicans -repulics->republics -reqest->request -requirment->requirement -requred->required -resaurant->restaurant -resembelance->resemblance -resembes->resembles -resemblence->resemblance -resevoir->reservoir -residental->residential -resignement->resignment -resistable->resistible -resistence->resistance -resistent->resistant -resopnse->response -respectivly->respectively -responce->response -responibilities->responsibilities -responisble->responsible -responnsibilty->responsibility -responsability->responsibility -responsibile->responsible -responsibilites->responsibilities -responsiblities->responsibilities -responsiblity->responsibility -ressemblance->resemblance -ressemble->resemble -ressembled->resembled -ressemblence->resemblance -ressembling->resembling -resssurecting->resurrecting -ressurect->resurrect -ressurected->resurrected -ressurection->resurrection -ressurrection->resurrection -restarant->restaurant -restarants->restaurants -restaraunt->restaurant -restaraunteur->restaurateur -restaraunteurs->restaurateurs -restaraunts->restaurants -restauranteurs->restaurateurs -restauration->restoration -restauraunt->restaurant -resteraunt->restaurant -resteraunts->restaurants -resticted->restricted -restraunt->restraint, restaurant, -resturant->restaurant -resturants->restaurants -resturaunt->restaurant -resturaunts->restaurants -resurecting->resurrecting -retalitated->retaliated -retalitation->retaliation -retreive->retrieve -retuns->returns -returnd->returned -revaluated->reevaluated -reveiw->review -reveral->reversal -reversable->reversible -revison->revision -revisons->revisions -revolutionar->revolutionary -rewitten->rewritten -rewriet->rewrite -rewuired->required -rference->reference -rferences->references -rhymme->rhyme -rhythem->rhythm -rhythim->rhythm -rhytmic->rhythmic -rigeur->rigueur, rigour, rigor, -rigourous->rigorous -rininging->ringing -rised->raised, rose, -Rockerfeller->Rockefeller -rococco->rococo -rocord->record -roomate->roommate -rougly->roughly -rucuperate->recuperate -rudimentatry->rudimentary -rulle->rule -runing->running -runnning->running -runnung->running -russina->Russian -Russion->Russian -rwite->write -rythem->rhythm -rythim->rhythm -rythm->rhythm -rythmic->rhythmic -rythyms->rhythms -sacrafice->sacrifice -sacreligious->sacrilegious -Sacremento->Sacramento -sacrifical->sacrificial -saftey->safety -safty->safety -salery->salary -sanctionning->sanctioning -sandwhich->sandwich -Sanhedrim->Sanhedrin -santioned->sanctioned -sargant->sergeant -sargeant->sergeant -sasy->says, sassy, -satelite->satellite -satelites->satellites -Saterday->Saturday -Saterdays->Saturdays -satisfactority->satisfactorily -satric->satiric -satrical->satirical -satrically->satirically -sattelite->satellite -sattelites->satellites -saught->sought -saveing->saving -saxaphone->saxophone -scaleable->scalable -scandanavia->Scandinavia -scaricity->scarcity -scavanged->scavenged -schedual->schedule -scholarhip->scholarship -scholarstic->scholastic, scholarly, -schould->should -scientfic->scientific -scientifc->scientific -scientis->scientist -scince->science -scinece->science -scirpt->script -scoll->scroll -screenwrighter->screenwriter -scriping->scripting -scrutinity->scrutiny -scuptures->sculptures -seach->search -seached->searched -seaches->searches -secceeded->seceded, succeeded, -seceed->succeed, secede, -seceeded->succeeded, seceded, -secratary->secretary -secretery->secretary -sedereal->sidereal -seeked->sought -segementation->segmentation -segmentaion->segmentation -seguoys->segues -seige->siege -seing->seeing -seinor->senior -seldomly->seldom -senarios->scenarios -sence->sense, since, -senstive->sensitive -sensure->censure -sepcial->special -seperate->separate -seperated->separated -seperately->separately -seperates->separates -seperating->separating -seperation->separation -seperatism->separatism -seperatist->separatist -sepina->subpoena -sepulchure->sepulchre, sepulcher, -sepulcre->sepulchre, sepulcher, -sergent->sergeant -setted->set -settelement->settlement -settlment->settlement -severeal->several -severley->severely -severly->severely -sevice->service -shadasloo->shadaloo -shaddow->shadow -shadoloo->shadaloo -shamen->shaman, shamans, -shashes->slashes -sheat->sheath, sheet, cheat, -sheild->shield -sherif->sheriff -shineing->shining -shiped->shipped -shiping->shipping -shopkeeepers->shopkeepers -shorly->shortly -shortwhile->short while -shoudl->should -shoudln->should, shouldn't, -shouldnt'->shouldn't -shouldnt->shouldn't -shreak->shriek -shrinked->shrunk -sicne->since -sideral->sidereal -sieze->seize, size, -siezed->seized, sized, -siezing->seizing, sizing, -siezure->seizure -siezures->seizures -siginificant->significant -signficant->significant -signficiant->significant -signfies->signifies -signifantly->significantly -significently->significantly -signifigant->significant -signifigantly->significantly -signitories->signatories -signitory->signatory -similarily->similarly -similiar->similar -similiarity->similarity -similiarly->similarly -simmilar->similar -simpley->simply -simplier->simpler -simultanous->simultaneous -simultanously->simultaneously -sincerley->sincerely -singsog->singsong -sinse->sines, since, -Sionist->Zionist -Sionists->Zionists -Sixtin->Sistine -Skagerak->Skagerrak -skateing->skating -slaugterhouses->slaughterhouses -slighly->slightly -slippy->slippery -slowy->slowly -smae->same -smealting->smelting -smoe->some -sneeks->sneaks -snese->sneeze -socalism->socialism -socities->societies -soem->some -sofware->software -sohw->show -soilders->soldiers -solatary->solitary -soley->solely -soliders->soldiers -soliliquy->soliloquy -soluable->soluble -somene->someone -somtimes->sometimes -somwhere->somewhere -sonething->something -sophicated->sophisticated -sophmore->sophomore -sorceror->sorcerer -sorrounding->surrounding -sotry->story -sotyr->satyr, story, -soudn->sound -soudns->sounds -sould->could, should, sold, -sountrack->soundtrack -sourth->south -sourthern->southern -southbrige->southbridge -souvenier->souvenir -souveniers->souvenirs -soveits->soviets -sovereignity->sovereignty -soverign->sovereign -soverignity->sovereignty -soverignty->sovereignty -spagetti->spaghetti -spainish->Spanish -speach->speech -specfic->specific -speciallized->specialised, specialized, -specif->specific, specify, -specifiying->specifying -speciman->specimen -spectauclar->spectacular -spectaulars->spectaculars -spects->aspects, expects, -spectum->spectrum -speices->species -spendour->splendour -spermatozoan->spermatozoon -spoace->space -sponser->sponsor -sponsered->sponsored -spontanous->spontaneous -sponzored->sponsored -spoonfulls->spoonfuls -sppeches->speeches -spreaded->spread -sprech->speech -spred->spread -spriritual->spiritual -spritual->spiritual -spurrious->spurious -sqaure->square -stablility->stability -stainlees->stainless -staion->station -standars->standards -stange->strange -startegic->strategic -startegies->strategies -startegy->strategy -stateman->statesman -statememts->statements -statment->statement -steriods->steroids -sterotypes->stereotypes -stilus->stylus -stingent->stringent -stiring->stirring -stirrs->stirs -stlye->style -stomache->stomach -stong->strong -stopry->story -storeis->stories -storise->stories -stornegst->strongest -stoyr->story -stpo->stop -stradegies->strategies -stradegy->strategy -strat->start, strata, -stratagically->strategically -streemlining->streamlining -stregth->strength -strenghen->strengthen -strenghened->strengthened -strenghening->strengthening -strenght->strength -strenghten->strengthen -strenghtened->strengthened -strenghtening->strengthening -strengtened->strengthened -strenous->strenuous -strictist->strictest -strikely->strikingly -strnad->strand -stroy->story, destroy, -structered->structured -structual->structural -stubborness->stubbornness -stucture->structure -stuctured->structured -studdy->study -studing->studying -stuggling->struggling -sturcture->structure -subcatagories->subcategories -subcatagory->subcategory -subconsiously->subconsciously -subjudgation->subjugation -submachne->submachine -subpecies->subspecies -subsidary->subsidiary -subsiduary->subsidiary -subsituting->substituting -subsquent->subsequent -subsquently->subsequently -substace->substance -substancial->substantial -substatial->substantial -substituded->substituted -substract->subtract -substracted->subtracted -substracting->subtracting -substraction->subtraction -substracts->subtracts -subsytems->subsystems -subtances->substances -subterranian->subterranean -suburburban->suburban -succceeded->succeeded -succcesses->successes -succedded->succeeded -succeded->succeeded -succeds->succeeds -succesful->successful -succesfully->successfully -succesfuly->successfully -succesion->succession -succesive->successive -successfull->successful -successfuly->successfully -successully->successfully -succsess->success -succsessfull->successful -suceed->succeed -suceeded->succeeded -suceeding->succeeding -suceeds->succeeds -sucesful->successful -sucesfully->successfully -sucesfuly->successfully -sucesion->succession -sucess->success -sucesses->successes -sucessful->successful -sucessfull->successful -sucessfully->successfully -sucessfuly->successfully -sucession->succession -sucessive->successive -sucessor->successor -sucessot->successor -sucide->suicide -sucidial->suicidal -sufferage->suffrage -sufferred->suffered -sufferring->suffering -sufficent->sufficient -sufficently->sufficiently -sumary->summary -sunglases->sunglasses -suop->soup -superceeded->superseded -superintendant->superintendent -suphisticated->sophisticated -suplimented->supplemented -supose->suppose -suposed->supposed -suposedly->supposedly -suposes->supposes -suposing->supposing -supplamented->supplemented -suppliementing->supplementing -suppoed->supposed -supposingly->supposedly -suppy->supply -supress->suppress -supressed->suppressed -supresses->suppresses -supressing->suppressing -suprise->surprise -suprised->surprised -suprising->surprising -suprisingly->surprisingly -suprize->surprise -suprized->surprised -suprizing->surprising -suprizingly->surprisingly -surfce->surface -surley->surly, surely, -suround->surround -surounded->surrounded -surounding->surrounding -suroundings->surroundings -surounds->surrounds -surplanted->supplanted -surpress->suppress -surpressed->suppressed -surprize->surprise -surprized->surprised -surprizing->surprising -surprizingly->surprisingly -surrended->surrounded, surrendered, -surrepetitious->surreptitious -surrepetitiously->surreptitiously -surreptious->surreptitious -surreptiously->surreptitiously -surronded->surrounded -surrouded->surrounded -surrouding->surrounding -surrundering->surrendering -surveilence->surveillance -surveill->surveil -surveyer->surveyor -surviver->survivor -survivers->survivors -survivied->survived -suseptable->susceptible -suseptible->susceptible -suspention->suspension -swaer->swear -swaers->swears -swepth->swept -swiming->swimming -syas->says -symetrical->symmetrical -symetrically->symmetrically -symetry->symmetry -symettric->symmetric -symmetral->symmetric -symmetricaly->symmetrically -synagouge->synagogue -syncronization->synchronization -synonomous->synonymous -synonymns->synonyms -synphony->symphony -synronous->synchronous -syphyllis->syphilis -sypmtoms->symptoms -syrap->syrup -sysmatically->systematically -sytem->system -sytems->systems -sytle->style -tabacco->tobacco -tahn->than -taht->that -talekd->talked -targetted->targeted -targetting->targeting -tast->taste -tath->that -tattooes->tattoos -taxanomic->taxonomic -taxanomy->taxonomy -teached->taught -techician->technician -techicians->technicians -techiniques->techniques -technitian->technician -technnology->technology -technolgy->technology -teh->the -tehy->they -telelevision->television -televsion->television -telphony->telephony -temerature->temperature -tempalte->template -tempaltes->templates -temparate->temperate -temperarily->temporarily -temperment->temperament -tempertaure->temperature -temperture->temperature -temprary->temporary -tenacle->tentacle -tenacles->tentacles -tendacy->tendency -tendancies->tendencies -tendancy->tendency -tennisplayer->tennis player -tepmorarily->temporarily -terrestial->terrestrial -terriories->territories -terriory->territory -territorist->terrorist -territoy->territory -terroist->terrorist -testiclular->testicular -testomony->testimony -tghe->the -thast->that, that's, -theather->theater -theese->these -theif->thief -theives->thieves -themselfs->themselves -themslves->themselves -ther->there, their, the, other, -therafter->thereafter -therby->thereby -theri->their -theshold->threshold -theyre->they're, -thgat->that -thge->the -thier->their -thign->thing -thigns->things -thigsn->things -thikn->think -thikning->thinking, thickening, -thikns->thinks -thiunk->think -thn->then -thna->than -thne->then -thnig->thing -thnigs->things -thoughout->throughout -threatend->threatened -threatning->threatening -threee->three -threshhold->threshold -thrid->third -throrough->thorough -throughly->thoroughly -throught->thought, through, throughout, -througout->throughout -thru->through -ths->the, this, -thsi->this -thsoe->those -thta->that -thyat->that -tiem->time, item, -tihkn->think -tihs->this -timne->time -tiome->time, tome, -tje->the -tjhe->the -tjpanishad->upanishad -tkae->take -tkaes->takes -tkaing->taking -tlaking->talking -tobbaco->tobacco -todays->today's, disable because of var names -todya->today -toghether->together -toke->took -tolerence->tolerance -Tolkein->Tolkien -tomatos->tomatoes -tommorow->tomorrow -tommorrow->tomorrow -tongiht->tonight -toriodal->toroidal -tormenters->tormentors -tornadoe->tornado -torpeados->torpedoes -torpedos->torpedoes -tothe->to the -toubles->troubles -tounge->tongue -tourch->torch, touch, -towords->towards -towrad->toward -tradionally->traditionally -traditionaly->traditionally -traditionnal->traditional -traditition->tradition -tradtionally->traditionally -trafficed->trafficked -trafficing->trafficking -trafic->traffic -trancendent->transcendent -trancending->transcending -tranfer->transfer -tranform->transform -tranformed->transformed -transcendance->transcendence -transcendant->transcendent -transcendentational->transcendental -transcripting->transcribing, transcription, -transending->transcending -transesxuals->transsexuals -transfered->transferred -transfering->transferring -transformaton->transformation -transistion->transition -translater->translator -translaters->translators -transmissable->transmissible -transmited->transmitted -transporation->transportation -transtion->transition -trasaction->transaction -trascation->transaction -trasnaction->transaction -tremelo->tremolo -tremelos->tremolos -triguered->triggered -triology->trilogy -troling->trolling -troup->troupe -troups->troupes, troops, -truely->truly -trustworthyness->trustworthiness -turnk->turnkey, trunk, -Tuscon->Tucson -tust->trust -twelth->twelfth -twon->town -twpo->two -tyhat->that -tyhe->they -typcial->typical -typicaly->typically -tyranies->tyrannies -tyrany->tyranny -tyrranies->tyrannies -tyrrany->tyranny -ubiquitious->ubiquitous -ublisher->publisher -uise->use -Ukranian->Ukrainian -ultimely->ultimately -unacompanied->unaccompanied -unahppy->unhappy -unanymous->unanimous -unathorised->unauthorised -unavailible->unavailable -unballance->unbalance -unbeknowst->unbeknownst -unbeleivable->unbelievable -uncertainity->uncertainty -unchallengable->unchallengeable -unchangable->unchangeable -uncompetive->uncompetitive -unconcious->unconscious -unconciousness->unconsciousness -unconfortability->discomfort -uncontitutional->unconstitutional -unconvential->unconventional -undecideable->undecidable -understoon->understood -undesireable->undesirable -undetecable->undetectable -undoubtely->undoubtedly -undreground->underground -uneccesary->unnecessary -unecessary->unnecessary -unequalities->inequalities -unflaged->unflagged -unforetunately->unfortunately -unforgetable->unforgettable -unforgiveable->unforgivable -unfortunatley->unfortunately -unfortunatly->unfortunately -unfourtunately->unfortunately -unihabited->uninhabited -unilateraly->unilaterally -unilatreal->unilateral -unilatreally->unilaterally -uninterruped->uninterrupted -uninterupted->uninterrupted -UnitesStates->UnitedStates -univeral->universal -univeristies->universities -univeristy->university -univerity->university -universtiy->university -univesities->universities -univesity->university -unkonwn->unknown -unkown->unknown -unlikey->unlikely -unmanouverable->unmaneuverable, unmanoeuvrable, -unmistakeably->unmistakably -unneccesarily->unnecessarily -unneccesary->unnecessary -unneccessarily->unnecessarily -unneccessary->unnecessary -unnecesarily->unnecessarily -unnecesary->unnecessary -unoffical->unofficial -unoperational->nonoperational -unoticeable->unnoticeable -unplease->displease -unplesant->unpleasant -unprecendented->unprecedented -unprecidented->unprecedented -unregnized->unrecognized -unrepentent->unrepentant -unrepetant->unrepentant -unrepetent->unrepentant -unsed->unused, used -unsubstanciated->unsubstantiated -unsuccesful->unsuccessful -unsuccesfully->unsuccessfully -unsuccessfull->unsuccessful -unsucesful->unsuccessful -unsucesfuly->unsuccessfully -unsucessful->unsuccessful -unsucessfull->unsuccessful -unsucessfully->unsuccessfully -unsuprised->unsurprised -unsuprising->unsurprising -unsuprisingly->unsurprisingly -unsuprized->unsurprised -unsuprizing->unsurprising -unsuprizingly->unsurprisingly -unsurprized->unsurprised -unsurprizing->unsurprising -unsurprizingly->unsurprisingly -untill->until -untranslateable->untranslatable -unuseable->unusable -unusuable->unusable -unviersity->university -unwarrented->unwarranted -unweildly->unwieldy -unwieldly->unwieldy -unwrritten->unwritten -upcomming->upcoming -upgradded->upgraded -upsteam->upstream -upto->up to -usally->usually -useage->usage -usefull->useful -usefuly->usefully -useing->using -usualy->usually -ususally->usually -vaccum->vacuum -vaccume->vacuum -vacinity->vicinity -vaguaries->vagaries -vaieties->varieties -vailidty->validity -valetta->valletta -valuble->valuable -valueable->valuable -varables->variables -varations->variations -varient->variant -variey->variety -varing->varying -varities->varieties -varity->variety -vasall->vassal -vasalls->vassals -vegatarian->vegetarian -vegitable->vegetable -vegitables->vegetables -vegtable->vegetable -vehicule->vehicle -vell->well -venemous->venomous -vengance->vengeance -vengence->vengeance -verfication->verification -verison->version -verisons->versions -vermillion->vermilion -versitilaty->versatility -versitlity->versatility -vetween->between -veyr->very -vigeur->vigueur, vigour, vigor, -vigilence->vigilance -vigourous->vigorous -vill->will -villian->villain -villification->vilification -villify->vilify -villin->villi, villain, villein, -vincinity->vicinity -violentce->violence -virtualy->virtually -virutal->virtual -virutally->virtually -visability->visibility -visable->visible -visably->visibly -visibile->visible -visting->visiting -vistors->visitors -vitories->victories -volcanoe->volcano -voleyball->volleyball -volontary->voluntary -volonteer->volunteer -volonteered->volunteered -volonteering->volunteering -volonteers->volunteers -volounteer->volunteer -volounteered->volunteered -volounteering->volunteering -volounteers->volunteers -volumne->volume -vreity->variety -vrey->very -vriety->variety -vulnerablility->vulnerability -vyer->very -vyre->very -waht->what -wanna->want to, disabled because one might want to allow informal spelling -warantee->warranty -wardobe->wardrobe -waring->warning -warrent->warrant -warrriors->warriors -wasnt'->wasn't -wasnt->wasn't -wass->was -watn->want -wayword->wayward -weaponary->weaponry -weas->was -wehn->when -weild->wield, wild, -weilded->wielded -wendsay->Wednesday -wensday->Wednesday -wereabouts->whereabouts -whant->want -whants->wants -whcih->which -whenver->whenever -wheras->whereas -wherease->whereas -whereever->wherever -whic->which -whihc->which -whith->with -whlch->which -whn->when -wholey->wholly -wholy->wholly, holy, -whta->what -whther->whether -wich->which -widesread->widespread -wief->wife -wierd->weird -wiew->view -wih->with -wiht->with -wille->will -willingless->willingness -willk->will -wirting->writing -withdrawl->withdrawal, withdraw, -witheld->withheld -withh->with -withing->within -withold->withhold -witht->with -witn->with -wiull->will -wnat->want -wnated->wanted -wnats->wants -wohle->whole -wokr->work -wokring->working -wonderfull->wonderful -wont->won't, wont, -wordlwide->worldwide -workststion->workstation -worls->world -worstened->worsened -woudl->would -wouldnt->wouldn't -wresters->wrestlers -wriet->write -writen->written -wroet->wrote -wrok->work -wroking->working -wtih->with -wupport->support -xenophoby->xenophobia -yaching->yachting -yaer->year -yaerly->yearly -yaers->years -yatch->yacht -yearm->year -yeasr->years -yeild->yield -yeilding->yielding -Yementite->Yemenite, Yemeni, -yera->year -yeras->years -yersa->years -yotube->youtube -youseff->yousef -youself->yourself -yrea->year -ytou->you -yuo->you -zeebra->zebra diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ffe8fe7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,157 @@ +[build-system] +build-backend = 'setuptools.build_meta' +requires = ['setuptools >= 64.0.0'] + +[project] +authors = [ + {email = 'denis.engemann@gmail.com', name = 'Denis A. Engemann'}, +] +classifiers = [ + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + 'Operating System :: MacOS', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: Unix', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.9', +] +dependencies = [ + 'matplotlib', + 'mne', + 'numpy>=1.21', + 'scikit-learn', + 'scipy', +] +description = 'MNE HCP project for accessing the human connectome MEG data in Python.' +keywords = [ + 'python', + 'mne', + 'hcp', +] +license = {file = 'LICENSE'} +maintainers = [ + {email = 'denis.engemann@gmail.com', name = 'Denis A. Engemann'}, + {email = 'mathieu.scheltienne@fcbg.ch', name = 'Mathieu Scheltienne'}, +] +name = 'mne-hcp' +readme = 'README.rst' +requires-python = '>=3.9' +version = '0.1.0' + +[project.optional-dependencies] +all = [ + 'mne-hcp[build]', + 'mne-hcp[style]', + 'mne-hcp[test]', +] +build = [ + 'build', + 'twine', +] +full = [ + 'template[all]', +] +style = [ + 'codespell[toml]>=2.2.4', + 'isort', + 'pydocstyle[toml]', + 'ruff>=0.1.8', + 'toml-sort', + 'yamllint', +] +test = [ + 'pytest-cov', + 'pytest-timeout', + 'pytest>=8.0', +] + +[project.urls] +documentation = 'https://mne.tools/mne-hcp/' +source = 'http://github.com/mne-tools/mne-hcp' +tracker = 'http://github.com/mne-tools/mne-hcp/issues' + +[tool.codespell] +check-filenames = true +check-hidden = true +ignore-words = '.codespellignore' +skip = 'build,.git,.mypy_cache,.pytest_cache' + +[tool.coverage.report] +exclude_lines = [ + 'if __name__ == .__main__.:', + 'if TYPE_CHECKING:', + 'pragma: no cover', +] +precision = 2 + +[tool.coverage.run] +branch = true +cover_pylib = false +omit = [ + '**/__init__.py', + '**/hcp/_version.py', + '**/tests/**', +] + +[tool.isort] +extend_skip_glob = [ + 'doc/*', + 'examples/*', + 'tutorials/*', +] +line_length = 88 +multi_line_output = 3 +profile = 'black' +py_version = 39 + +[tool.pydocstyle] +add_ignore = 'D100,D104,D107' +convention = 'numpy' +ignore-decorators = '(copy_doc|property|.*setter|.*getter|pyqtSlot|Slot)' +match = '^(?!__init__|test_).*\.py' +match-dir = '^hcp.*' + +[tool.pytest.ini_options] +addopts = '--durations 20 --junit-xml=junit-results.xml --verbose' +minversion = '8.0' + +[tool.ruff] +extend-exclude = [ + 'doc', +] +line-length = 88 +target-version = 'py39' + +[tool.ruff.format] +docstring-code-format = true +line-ending = "lf" + +[tool.ruff.lint] +ignore = [] +select = ['A', 'B', 'E', 'F', 'UP', 'W'] + +[tool.ruff.lint.per-file-ignores] +'*' = [ + 'B904', # 'Within an except clause, raise exceptions with raise ... from ...' + 'UP007', # 'Use `X | Y` for type annotations', requires python 3.10 +] +'*.pyi' = ['E501'] +'__init__.py' = ['F401'] + +[tool.setuptools] +include-package-data = false + +[tool.setuptools.package-data] +"hcp.io.file_mapping" = ["data/*.txt"] + +[tool.setuptools.packages.find] +exclude = ['hcp*tests'] +include = ['hcp*'] + +[tool.tomlsort] +all = true +ignore_case = true +trailing_comma_inline_array = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 8ac2f13..0000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[bdist_wheel] -# This flag says that the code is written to work on both Python 2 and Python -# 3. If at all possible, it is good practice to do this. If you cannot, you -# will need to generate wheels for each Python version that you support. -universal=1 - -[metadata] -description-file = README.rst diff --git a/setup.py b/setup.py deleted file mode 100755 index 7485498..0000000 --- a/setup.py +++ /dev/null @@ -1,69 +0,0 @@ -#! /usr/bin/env python -# - -# Copyright (C) 2015-2016 Denis Engemann -# - -import os -from os import path as op - -import setuptools # noqa; we are using a setuptools namespace -from numpy.distutils.core import setup - -# get the version (don't import mne here, so dependencies are not needed) -version = None -with open(os.path.join('hcp', '__init__.py'), 'r') as fid: - for line in (line.strip() for line in fid): - if line.startswith('__version__'): - version = line.split('=')[1].strip().strip('\'') - break -if version is None: - raise RuntimeError('Could not determine version') - - -descr = """MNE HCP project for accessing the human connectome MEG data in Python.""" # noqa - -DISTNAME = 'mne-hcp' -DESCRIPTION = descr -MAINTAINER = 'Denis A. Engemann' -MAINTAINER_EMAIL = 'denis.engemann@gmail.com' -URL = 'http://github.com/mne-tools/mne-hcp' -LICENSE = 'BSD (3-clause)' -DOWNLOAD_URL = 'http://github.com/mne-tools/mne-hcp' -VERSION = version - - -if __name__ == "__main__": - if os.path.exists('MANIFEST'): - os.remove('MANIFEST') - - setup(name=DISTNAME, - maintainer=MAINTAINER, - include_package_data=True, - maintainer_email=MAINTAINER_EMAIL, - description=DESCRIPTION, - license=LICENSE, - url=URL, - version=VERSION, - download_url=DOWNLOAD_URL, - long_description=open('README.rst').read(), - zip_safe=False, # the package can run out of an .egg file - classifiers=['Intended Audience :: Science/Research', - 'Intended Audience :: Developers', - 'License :: OSI Approved', - 'Programming Language :: Python', - 'Topic :: Software Development', - 'Topic :: Scientific/Engineering', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Operating System :: Unix', - 'Operating System :: MacOS'], - platforms='any', - packages=['hcp', - 'hcp.io', - 'hcp.io.tests', - 'hcp.io.file_mapping', - 'hcp.io.file_mapping.tests', - 'hcp.tests'], - package_data={'hcp': [ - op.join('io', 'file_mapping', 'data', '*txt')]}) From 3bfd2a3d63acf2d42c7514634b1384f0b33545df Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 13:52:16 +0200 Subject: [PATCH 05/34] bump version --- hcp/__init__.py | 2 -- hcp/_version.py | 5 +++++ pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 hcp/_version.py diff --git a/hcp/__init__.py b/hcp/__init__.py index d00640e..0866d85 100644 --- a/hcp/__init__.py +++ b/hcp/__init__.py @@ -13,5 +13,3 @@ read_raw, read_trial_info, ) - -__version__ = "0.1.dev12" diff --git a/hcp/_version.py b/hcp/_version.py new file mode 100644 index 0000000..77f24d8 --- /dev/null +++ b/hcp/_version.py @@ -0,0 +1,5 @@ +"""Version number.""" + +from importlib.metadata import version + +__version__ = version(__package__) diff --git a/pyproject.toml b/pyproject.toml index ffe8fe7..a71439f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ maintainers = [ name = 'mne-hcp' readme = 'README.rst' requires-python = '>=3.9' -version = '0.1.0' +version = '0.2.0.dev0' [project.optional-dependencies] all = [ From c89fed2e5f8099557f6d13d23c25c7431ab0e6cf Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 13:53:37 +0200 Subject: [PATCH 06/34] fix codecov --- codecov.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codecov.yml b/codecov.yml index 408f379..8b7d7dc 100644 --- a/codecov.yml +++ b/codecov.yml @@ -14,7 +14,7 @@ coverage: target: 95% if_no_uploads: error if_not_found: success - if_ci_failed: failure + if_ci_failed: error project: default: false library: @@ -22,4 +22,4 @@ coverage: target: 90% if_no_uploads: error if_not_found: success - if_ci_failed: failure + if_ci_failed: error From 993cd45977b76f90294684e944e27e727a8b6aa9 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 14:19:33 +0200 Subject: [PATCH 07/34] fix attribute setting --- hcp/io/read.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hcp/io/read.py b/hcp/io/read.py index c0c4e1e..b0778a2 100644 --- a/hcp/io/read.py +++ b/hcp/io/read.py @@ -368,7 +368,7 @@ def _read_epochs(epochs_mat_fname, info, return_fixations_motor): epochs = EpochsArray(data=data, info=this_info, events=events, tmin=times.min()) # XXX hack for now due to issue with EpochsArray constructor # cf https://github.com/mne-tools/mne-hcp/issues/9 - epochs.times = times + epochs._times = times return epochs @@ -516,7 +516,7 @@ def read_annot(subject, data_type, run_index=0, hcp_path=op.curdir): ] for subtype, fun, fname in iter_fun: - with open(fname, "r") as fid: + with open(fname) as fid: out[subtype] = fun(fid.read()) return out From c1e99212e4497eb97892d27a92442c2a5b67b7ae Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 14:26:05 +0200 Subject: [PATCH 08/34] add pre-commit --- .pre-commit-config.yaml | 44 +++++++++++++++++++++++++++++++++++++ .yamllint.yaml | 5 +++++ codecov.yml => codecov.yaml | 0 3 files changed, 49 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 .yamllint.yaml rename codecov.yml => codecov.yaml (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d3bd494 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,44 @@ +repos: + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + files: hcp + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.7 + hooks: + - id: ruff + name: ruff linter + args: [--fix] + files: hcp + - id: ruff-format + name: ruff formatter + files: hcp + + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + args: [--write-changes] + additional_dependencies: [tomli] + + - repo: https://github.com/pycqa/pydocstyle + rev: 6.3.0 + hooks: + - id: pydocstyle + files: hcp + additional_dependencies: [tomli] + + - repo: https://github.com/pappasam/toml-sort + rev: v0.23.1 + hooks: + - id: toml-sort-fix + files: pyproject.toml + + - repo: https://github.com/adrienverge/yamllint + rev: v1.35.1 + hooks: + - id: yamllint + args: [--strict, -c, .yamllint.yaml] + files: (.github/|.codecov.yaml|.pre-commit-config.yaml|.yamllint.yaml) diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..669c864 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,5 @@ +extends: default + +rules: + line-length: disable + document-start: disable diff --git a/codecov.yml b/codecov.yaml similarity index 100% rename from codecov.yml rename to codecov.yaml From 28e62b84e676f2c921b4aaa45691155d9e020a73 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 15:00:36 +0200 Subject: [PATCH 09/34] update doc build --- doc/Makefile | 159 +++-------------- doc/conf.py | 470 +++++++++++++++++-------------------------------- doc/make.bat | 55 ++++++ hcp/anatomy.py | 4 +- pyproject.toml | 13 ++ update_doc.sh | 10 -- 6 files changed, 256 insertions(+), 455 deletions(-) create mode 100644 doc/make.bat delete mode 100644 update_doc.sh diff --git a/doc/Makefile b/doc/Makefile index c1ea1fe..3165eea 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -1,152 +1,35 @@ -# Makefile for Sphinx documentation -# +# Minimal makefile for Sphinx documentation -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build +SPHINXOPTS ?= -nWT --keep-going +SPHINXBUILD ?= sphinx-build -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif +.PHONY: help Makefile clean html html-noplot linkcheck linkcheck-grep view -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +first_target: help -.PHONY: help help: @echo "Please use \`make ' where is one of" - @echo " html-noplot to make standalone HTML files, without plotting anything" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - @echo " coverage to run coverage check of the documentation (if enabled)" - @echo " install to make the html and push it online" + @echo " html to make standalone HTML files" + @echo " html-noplot to make standalone HTML files without plotting" + @echo " clean to clean HTML files" + @echo " linkcheck to check all external links for integrity" + @echo " linkcheck-grep to grep the linkcheck result" + @echo " view to view the built HTML" -.PHONY: clean - -clean: - rm -rf $(BUILDDIR)/* - rm -rf auto_examples/ - rm -rf generated/* - rm -rf auto_tutorials/* - rm -rf modules/* - -html-noplot: - $(SPHINXBUILD) -D plot_gallery=0 -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -.PHONY: html html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -.PHONY: dirhtml -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -.PHONY: singlehtml -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + $(SPHINXBUILD) . _build/html -b html $(SPHINXOPTS) -.PHONY: pickle -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -.PHONY: htmlhelp -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -.PHONY: qthelp -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/mne-hcp.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/mne-hcp.qhc" - -.PHONY: latex -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -.PHONY: latexpdf -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." +html-noplot: + $(SPHINXBUILD) . _build/html -b html $(SPHINXOPTS) -D plot_gallery=0 -.PHONY: changes -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." +clean: + rm -rf _build generated -.PHONY: linkcheck linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -.PHONY: doctest -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." + $(SPHINXBUILD) . _build/linkcheck -b linkcheck -D plot_gallery=0 -.PHONY: coverage -coverage: - $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage - @echo "Testing of coverage in the sources finished, look at the " \ - "results in $(BUILDDIR)/coverage/python.txt." +linkcheck-grep: + @! grep -h "^.*:.*: \[\(\(local\)\|\(broken\)\)\]" _build/linkcheck/output.txt -install: - rm -rf _build/doctrees _build/mne-hcp.github.io - # first clone the gh-pages branch because it may ask - # for password and we don't want to delay this long build in - # the middle of it - # --no-checkout just fetches the root folder without content - # --depth 1 is a speed optimization since we don't need the - # history prior to the last commit - git clone -b gh-pages --no-checkout --depth 1 https://github.com/mne-tools/mne-hcp.git _build/mne-hcp.github.io - touch _build/mne-hcp.github.io/.nojekyll - make html - cd _build/ && \ - cp -r html/* mne-hcp.github.io && \ - cd mne-hcp.github.io && \ - git add * && \ - git add .nojekyll && \ - git commit -a -m 'Make install' && \ - git push +view: + @python -c "import webbrowser; webbrowser.open_new_tab('file://$(PWD)/_build/html/index.html')" diff --git a/doc/conf.py b/doc/conf.py index c14062d..757d8ed 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,129 +1,85 @@ -# -*- coding: utf-8 -*- +# Configuration file for the Sphinx documentation builder. # -# autoreject documentation build configuration file, created by -# sphinx-quickstart on Mon May 23 16:22:52 2016. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys -import os -import sphinx_gallery -import sphinx_bootstrap_theme - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.mathjax', - 'sphinx_gallery.gen_gallery', -] - -# generate autosummary even if no references -autosummary_generate = True - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +from __future__ import annotations # c.f. PEP 563, PEP 649 -# The encoding of source files. -#source_encoding = 'utf-8-sig' +import inspect +import subprocess +import sys +from datetime import date +from importlib import import_module +from typing import TYPE_CHECKING -# The master toctree document. -master_doc = 'index' +from sphinx_gallery.sorting import FileNameSortKey -# General information about the project. -project = u'MNE-HCP' -copyright = u'2016, Denis A. Engemann' -author = u'Denis A. Engemann' +import hcp -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = u'0.1.dev12' -# The full version, including alpha/beta/rc tags. -release = u'0.1.dev12' +if TYPE_CHECKING: + from typing import Optional -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None +# -- project information --------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +project = "MNE-HCP" +author = "Denis A. Engemann" +copyright = f"{date.today().year}, {author}" +release = hcp.__version__ +package = hcp.__name__ +gh_url = "https://github.com/mscheltienne/template-python" -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] +# -- general configuration ------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None +# If your documentation needs a minimal Sphinx version, state it here. +needs_sphinx = "5.0" -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# The document name of the “root” document, that is, the document that contains the root +# toctree directive. +root_doc = "index" +source_suffix = ".rst" -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True +# Add any Sphinx extension module names here, as strings. They can be extensions coming +# with Sphinx (named "sphinx.ext.*") or your custom ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosectionlabel", + "sphinx.ext.autosummary", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.mathjax", + "numpydoc", + "sphinx_copybutton", + "sphinx_gallery.gen_gallery", +] -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +# Sphinx will warn about all references where the target cannot be found. +nitpicky = True +nitpick_ignore = [] # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False +modindex_common_prefix = [f"{package}."] +# The name of a reST role (builtin or Sphinx extension) to use as the default role, that +# is, for text marked up `like this`. This can be set to 'py:obj' to make `filter` a +# cross-reference to the Python function “filter”. +default_role = "py:obj" -# -- Options for HTML output ---------------------------------------------- +# list of warning types to suppress +suppress_warnings = ["config.cache"] -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'bootstrap' +# -- options for HTML output ----------------------------------------------------------- +html_css_files = [] +html_permalinks_icon = "🔗" +html_show_sphinx = False +html_static_path = ["_static"] +html_theme = "bootstrap" +html_title = project -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. html_theme_options = { 'navbar_sidebarrel': False, 'navbar_links': [ @@ -135,210 +91,114 @@ 'bootswatch_theme': "cosmo" } -# amelia -# cerulean -# cosmo -# cyborg -# journal -# readable -# simplex -# slate -# spacelab -# spruce -# superhero -# united - -# Add any paths that contain custom themes here, relative to this directory. -html_theme_path = sphinx_bootstrap_theme.get_html_theme_path() - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'mnehcpdoc' - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', +# -- autosummary ----------------------------------------------------------------------- +autosummary_generate = True -# Additional stuff for the LaTeX preamble. -#'preamble': '', +# -- autodoc --------------------------------------------------------------------------- +autodoc_typehints = "none" +autodoc_member_order = "groupwise" +autodoc_warningiserror = True +autoclass_content = "class" -# Latex figure (float) alignment -#'figure_align': 'htbp', +# -- intersphinx ----------------------------------------------------------------------- +intersphinx_mapping = { + "matplotlib": ("https://matplotlib.org/stable", None), + "mne": ("https://mne.tools/stable/", None), + "numpy": ("https://numpy.org/doc/stable", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), + "python": ("https://docs.python.org/3", None), + "scipy": ("https://docs.scipy.org/doc/scipy", None), + "sklearn": ("https://scikit-learn.org/stable/", None), +} +intersphinx_timeout = 5 + +# -- autosectionlabels ----------------------------------------------------------------- +autosectionlabel_prefix_document = True + +# -- numpydoc -------------------------------------------------------------------------- +numpydoc_class_members_toctree = False +numpydoc_attributes_as_param_list = False + +# x-ref +numpydoc_xref_param_type = True +numpydoc_xref_aliases = { + # Matplotlib + "Axes": "matplotlib.axes.Axes", + "Figure": "matplotlib.figure.Figure", + # MNE + "DigMontage": "mne.channels.DigMontage", + "Epochs": "mne.Epochs", + "Evoked": "mne.Evoked", + "Forward": "mne.Forward", + "Info": "mne.Info", + "Projection": "mne.Projection", + "Raw": "mne.io.Raw", + # Python + "bool": ":class:`python:bool`", + "Path": "pathlib.Path", + "TextIO": "io.TextIOBase", +} +numpydoc_xref_ignore = { + "instance", + "of", + "shape", } -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'mne-hcp.tex', u'mne-hcp Documentation', - u'Denis A. Engemann', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'mne-hcp', u'mn-hcp Documentation', - [author], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'mne-hcp', u'MNE-HCP Documentation', - author, 'mne-hcp', 'Reprocessing the Human Connectome Project MEG data.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False - -try: - from mayavi import mlab - find_mayavi_figures = True - # Do not pop up any mayavi windows while running the - # examples. These are very annoying since they steal the focus. - # mlab.options.offscreen = True XXX bug with mayavi -except Exception: - find_mayavi_figures = False - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = { - 'python': ('https://docs.python.org/', None), - 'mne': ('http://martinos.org/mne/stable/', None), +# validation +# https://numpydoc.readthedocs.io/en/latest/validation.html#validation-checks +error_ignores = { + "GL01", # docstring should start in the line immediately after the quotes + "EX01", # section 'Examples' not found + "ES01", # no extended summary found + "SA01", # section 'See Also' not found + "RT02", # The first line of the Returns section should contain only the type, unless multiple values are being returned # noqa: E501 +} +numpydoc_validate = True +numpydoc_validation_checks = {"all"} | set(error_ignores) +numpydoc_validation_exclude = { # regex to ignore during docstring check + r"\.__getitem__", + r"\.__contains__", + r"\.__hash__", + r"\.__mul__", + r"\.__sub__", + r"\.__add__", + r"\.__iter__", + r"\.__div__", + r"\.__neg__", } +# -- sphinx-gallery -------------------------------------------------------------------- +if sys.platform.startswith("win"): + try: + subprocess.check_call(["optipng", "--version"]) + compress_images = ("images", "thumbnails") + except Exception: + compress_images = () +else: + compress_images = ("images", "thumbnails") + sphinx_gallery_conf = { - 'examples_dirs': ['../examples', '../tutorials'], - 'gallery_dirs': ['auto_examples', 'auto_tutorials'], - 'find_mayavi_figures': find_mayavi_figures, - 'mod_example_dir': 'generated', - 'reference_url': { - 'mne': 'http://martinos.org/mne/stable', - 'numpy': 'http://docs.scipy.org/doc/numpy-1.9.1', - 'scipy': 'http://docs.scipy.org/doc/scipy-0.17.0/reference', - 'mayavi': 'http://docs.enthought.com/mayavi/mayavi'} + "backreferences_dir": "generated/backreferences", + "compress_images": compress_images, + "doc_module": (f"{package}",), + "examples_dirs": ["../tutorials"], + "exclude_implicit_doc": {}, # set + "filename_pattern": r"\d{2}_", + "gallery_dirs": ["generated/tutorials"], + "line_numbers": False, + "plot_gallery": "True", # str, to enable overwrite from CLI without warning + "reference_url": {f"{package}": None}, + "remove_config_comments": True, + "show_memory": True, + "within_subsection_order": FileNameSortKey, } + +# -- linkcheck ------------------------------------------------------------------------- +linkcheck_anchors = False # saves a bit of time +linkcheck_timeout = 15 # some can be quite slow +linkcheck_retries = 3 +linkcheck_ignore = [] # will be compiled to regex + +# -- sphinx_copybutton ----------------------------------------------------------------- +copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " +copybutton_prompt_is_regexp = True diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 0000000..dcd0a57 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,55 @@ +@echo off + +REM Minimal makefile for Sphinx documentation + +REM Set default options and commands +set SPHINXOPTS=-nWT --keep-going +set SPHINXBUILD=sphinx-build + +if "%1" == "html" goto html +if "%1" == "html-noplot" goto html-noplot +if "%1" == "clean" goto clean +if "%1" == "linkcheck" goto linkcheck +if "%1" == "linkcheck-grep" goto linkcheck-grep +if "%1" == "view" goto view + +REM Define targets +:help +echo Please use `make ^` where ^ is one of +echo html to make standalone HTML files +echo html-noplot to make standalone HTML files without plotting +echo clean to clean HTML files +echo linkcheck to check all external links for integrity +echo linkcheck-grep to grep the linkcheck result +echo view to view the built HTML +goto :eof + +:html +%SPHINXBUILD% . _build\html -b html %SPHINXOPTS% +goto :eof + +:html-noplot +%SPHINXBUILD% . _build\html -b html %SPHINXOPTS% -D plot_gallery=0 +goto :eof + +:clean +rmdir /s /q _build generated +goto :eof + +:linkcheck +%SPHINXBUILD% . _build\linkcheck -b linkcheck -D plot_gallery=0 +goto :eof + +:linkcheck-grep +findstr /C:"[broken]" _build\linkcheck\output.txt > nul +if %errorlevel% equ 0 ( + echo Lines with [broken]: + findstr /C:"[broken]" _build\linkcheck\output.txt +) else ( + echo No lines with [broken] found. +) +goto :eof + +:view +python -c "import webbrowser; webbrowser.open_new_tab(r'file:///%cd%\_build\html\index.html')" +goto :eof diff --git a/hcp/anatomy.py b/hcp/anatomy.py index 327f3d0..ea70f9a 100644 --- a/hcp/anatomy.py +++ b/hcp/anatomy.py @@ -199,7 +199,7 @@ def compute_forward_stack( ------- out : dict A dictionary with the following keys: - fwd : instance of mne.Forward + fwd : instance of Forward The forward solution. src_subject : instance of mne.SourceSpace The source model on the subject's surface @@ -214,7 +214,7 @@ def compute_forward_stack( info_from = dict(info_from) head_mri_t = mne.read_trans( - op.join(recordings_path, subject, "{}-head_mri-trans.fif".format(subject)) + op.join(recordings_path, subject, f"{subject}-head_mri-trans.fif") ) src_defaults = dict( diff --git a/pyproject.toml b/pyproject.toml index a71439f..438721a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ version = '0.2.0.dev0' [project.optional-dependencies] all = [ 'mne-hcp[build]', + 'mne-hcp[doc]', 'mne-hcp[style]', 'mne-hcp[test]', ] @@ -51,6 +52,18 @@ build = [ 'build', 'twine', ] +doc = [ + 'matplotlib', + 'memory-profiler', + 'numpydoc', + 'sphinx', + 'sphinx-bootstrap-theme', + 'sphinx-copybutton', + 'sphinx-design', + 'sphinx-gallery', + 'sphinx-issues', + 'sphinxcontrib-bibtex', +] full = [ 'template[all]', ] diff --git a/update_doc.sh b/update_doc.sh deleted file mode 100644 index 7e08719..0000000 --- a/update_doc.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/bash - -(git co gh-pages && - cd .. & - cp -R ./doc/_build/html/* . && - git add -f auto_examples/* && - git add -f auto_tutorials/* && - git add -f _images/* && - git add -f _sources/* && - git add -f _downloads/*) \ No newline at end of file From 5a8929ea2b3f720765da73c1e3cbf58412991e2e Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 15:11:31 +0200 Subject: [PATCH 10/34] fix more stuff --- doc/_static/.gitkeep | 0 doc/conf.py | 2 +- doc/python_reference.rst | 2 +- hcp/__init__.py | 3 ++- hcp/_version.py | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 doc/_static/.gitkeep diff --git a/doc/_static/.gitkeep b/doc/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/doc/conf.py b/doc/conf.py index 757d8ed..194c6b5 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -22,7 +22,7 @@ # -- project information --------------------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = "MNE-HCP" +project = "mne-hcp" author = "Denis A. Engemann" copyright = f"{date.today().year}, {author}" release = hcp.__version__ diff --git a/doc/python_reference.rst b/doc/python_reference.rst index b134a8f..51346da 100644 --- a/doc/python_reference.rst +++ b/doc/python_reference.rst @@ -33,7 +33,7 @@ below a module heading are found in the :py:mod:`hcp` namespace. Handling HCP files for downloading ================================== -.. currentmodule:: file_mapping +.. currentmodule:: hcp.io.file_mapping .. autosummary:: :toctree: generated/ diff --git a/hcp/__init__.py b/hcp/__init__.py index 0866d85..db048cb 100644 --- a/hcp/__init__.py +++ b/hcp/__init__.py @@ -1,7 +1,8 @@ # Author: Denis A. Engemann # License: BSD (3-clause) -from . import anatomy, preprocessing, tests, viz +from . import anatomy, preprocessing, viz +from ._version import __version__ from .anatomy import compute_forward_stack, make_mne_anatomy from .io import ( file_mapping, diff --git a/hcp/_version.py b/hcp/_version.py index 77f24d8..31bf0da 100644 --- a/hcp/_version.py +++ b/hcp/_version.py @@ -2,4 +2,4 @@ from importlib.metadata import version -__version__ = version(__package__) +__version__ = version("mne-hcp") From ce457b3ba1a566a61064edffefb719ee4d7ab9a4 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 15:22:58 +0200 Subject: [PATCH 11/34] more clean-up --- README.rst | 4 ++- doc/conf.py | 11 ------- doc/index.rst | 4 ++- doc/python_reference.rst | 2 -- doc/sg_execution_times.rst | 49 +++++++++++++++++++++++++++ hcp/anatomy.py | 14 ++++---- hcp/io/file_mapping/file_mapping.py | 51 +++++++++++++++-------------- hcp/preprocessing.py | 33 +++++++++++-------- tutorials/plot_compute_forward.py | 8 ++--- 9 files changed, 112 insertions(+), 64 deletions(-) create mode 100644 doc/sg_execution_times.rst diff --git a/README.rst b/README.rst index 8c6148e..1dfa3e9 100644 --- a/README.rst +++ b/README.rst @@ -289,7 +289,9 @@ Python conventions by subtracting 1. Contributions ------------- -- currently `@dengemann` is pushing frequently to master, if you plan to contribute, open issues and pull requests, or contact `@dengemann` directly. Discussions are welcomed. +- currently ``@dengemann`` is pushing frequently to master, if you plan to contribute, + open issues and pull requests, or contact ``@dengemann`` directly. Discussions are + welcomed. Acknowledgements ================ diff --git a/doc/conf.py b/doc/conf.py index 194c6b5..f7518ee 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -3,22 +3,15 @@ # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -from __future__ import annotations # c.f. PEP 563, PEP 649 -import inspect import subprocess import sys from datetime import date -from importlib import import_module -from typing import TYPE_CHECKING from sphinx_gallery.sorting import FileNameSortKey import hcp -if TYPE_CHECKING: - from typing import Optional - # -- project information --------------------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information @@ -44,7 +37,6 @@ # with Sphinx (named "sphinx.ext.*") or your custom ones. extensions = [ "sphinx.ext.autodoc", - "sphinx.ext.autosectionlabel", "sphinx.ext.autosummary", "sphinx.ext.doctest", "sphinx.ext.intersphinx", @@ -112,9 +104,6 @@ } intersphinx_timeout = 5 -# -- autosectionlabels ----------------------------------------------------------------- -autosectionlabel_prefix_document = True - # -- numpydoc -------------------------------------------------------------------------- numpydoc_class_members_toctree = False numpydoc_attributes_as_param_list = False diff --git a/doc/index.rst b/doc/index.rst index 46ed02b..ee22e70 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -289,7 +289,9 @@ Python conventions by subtracting 1. Contributions ------------- -- currently `@dengemann` is pushing frequently to master, if you plan to contribute, open issues and pull requests, or contact `@dengemann` directly. Discussions are welcomed. +- currently ``@dengemann`` is pushing frequently to master, if you plan to contribute, + open issues and pull requests, or contact ``@dengemann`` directly. Discussions are + welcomed. Acknowledgements ================ diff --git a/doc/python_reference.rst b/doc/python_reference.rst index 51346da..709e82c 100644 --- a/doc/python_reference.rst +++ b/doc/python_reference.rst @@ -66,8 +66,6 @@ Manipulating data and sensors Visualizing data ================ -:py:mod:`hcp.viz`: - .. currentmodule:: hcp.viz .. autosummary:: diff --git a/doc/sg_execution_times.rst b/doc/sg_execution_times.rst new file mode 100644 index 0000000..75c0b71 --- /dev/null +++ b/doc/sg_execution_times.rst @@ -0,0 +1,49 @@ + +:orphan: + +.. _sphx_glr_sg_execution_times: + + +Computation times +================= +**00:00.000** total execution time for 5 files **from all galleries**: + +.. container:: + + .. raw:: html + + + + + + + + .. list-table:: + :header-rows: 1 + :class: table table-striped sg-datatable + + * - Example + - Time + - Mem (MB) + * - :ref:`sphx_glr_generated_tutorials_plot_compute_evoked_inverse_solution.py` (``../tutorials/plot_compute_evoked_inverse_solution.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_generated_tutorials_plot_compute_forward.py` (``../tutorials/plot_compute_forward.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_generated_tutorials_plot_reference_correction.py` (``../tutorials/plot_reference_correction.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_generated_tutorials_plot_reproduce_erf.py` (``../tutorials/plot_reproduce_erf.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_generated_tutorials_plot_temporal_searchlight_decoding.py` (``../tutorials/plot_temporal_searchlight_decoding.py``) + - 00:00.000 + - 0.0 diff --git a/hcp/anatomy.py b/hcp/anatomy.py index ea70f9a..f963b27 100644 --- a/hcp/anatomy.py +++ b/hcp/anatomy.py @@ -23,7 +23,7 @@ def make_mne_anatomy( hcp_path=op.curdir, outputs=("label", "mri", "surf"), ): - """Extract relevant anatomy and create MNE friendly directory layout + """Extract relevant anatomy and create MNE friendly directory layout. The function will create the following outputs by default: @@ -33,7 +33,7 @@ def make_mne_anatomy( $subjects_dir/$subject/surf/* $recordings_path/$subject/$subject-head_mri-trans.fif - These can then be set as $SUBJECTS_DIR and as MEG directory, consistent + These can then be set as ``$SUBJECTS_DIR`` and as MEG directory, consistent with MNE examples. Parameters @@ -44,9 +44,9 @@ def make_mne_anatomy( The path corresponding to MNE/freesurfer SUBJECTS_DIR (to be created) hcp_path : str The path where the HCP files can be found. - outputs : {'label', 'mri', 'stats', 'surf', 'touch'} + outputs : ``'label'`` | ``'mri'`` | ``'stats'`` | ``'surf'`` | ``'touch'`` The outputs of the freesrufer pipeline shipped by HCP. Defaults to - ('mri', 'surf'), the minimum needed to extract MNE-friendly anatomy + (``'mri'``, ``'surf'``), the minimum needed to extract MNE-friendly anatomy files and data. """ if hcp_path == op.curdir: @@ -186,8 +186,10 @@ def compute_forward_stack( src_params : None | dict The src params. Defaults to: - dict(subject='fsaverage', fname=None, spacing='oct6', n_jobs=2, - surface='white', subjects_dir=subjects_dir, add_dist=True) + .. code-block:: python + + dict(subject='fsaverage', fname=None, spacing='oct6', n_jobs=2, + surface='white', subjects_dir=subjects_dir, add_dist=True) hcp_path : str The prefix of the path of the HCP data. n_jobs : int diff --git a/hcp/io/file_mapping/file_mapping.py b/hcp/io/file_mapping/file_mapping.py index 7301396..bc45758 100644 --- a/hcp/io/file_mapping/file_mapping.py +++ b/hcp/io/file_mapping/file_mapping.py @@ -203,9 +203,9 @@ def get_file_paths( sensor_mode="mag", hcp_path=".", ): - """This is the MNE-HCP file path synthesizer + """MNE-HCP file path synthesizer. - An easy conceptual mapper from questions to file paths + An easy conceptual mapper from questions to file paths. Parameters ---------- @@ -213,35 +213,37 @@ def get_file_paths( The subject, a 6 digit code. data_type : str The acquisition context of the data. The following ones are supported: - 'rest' - 'noise' - 'task_motor' - 'task_story_math' - 'task_working_memory' + + * ``'rest'`` + * ``'noise'`` + * ``'task_motor'`` + * ````'task_story_math'`` + * 'task_working_memory'`` output : str The kind of output. The following ones are supported: - 'raw', - 'epochs' - 'evoked' - 'ica' - 'annotations' - 'trial_info' - 'freesurfer' - 'meg_anatomy' - onset : {'stim', 'resp', 'sentence', 'block'} + + * ``'raw'`` + * ``'epochs'`` + * ``'evoked'`` + * ``'ica'`` + * ``'annotations'`` + * ``'trial_info'`` + * ``'freesurfer'`` + * ``'meg_anatomy'`` + onset : ``'stim`` | ``'resp'`` | ``'sentence'`` | ``'block'`` The event onset. Only considered for epochs and evoked outputs The mapping is generous, everything that is not a response is a stimulus, in the sense of internal or external events. sentence and block are specific to task_story_math. - sensor_mode : {'mag', 'planar'} - The sensor projection. Defaults to 'mag'. Only relevant for + sensor_mode : ``'mag'`` | ``'planar'`` + The sensor projection. Defaults to ``'mag'``. Only relevant for evoked output. run_index : int The run index. For the first run, use 0, for the second, use 1. Also see HCP documentation for the number of runs for a given data type. hcp_path : str - The HCP directory, defaults to op.curdir. + The HCP directory, defaults to ``op.curdir``. Returns ------- @@ -250,9 +252,8 @@ def get_file_paths( """ if data_type not in kind_map: raise ValueError( - "I never heard of `%s` -- are you sure this is a" - " valid HCP type? I currenlty support:\n%s" - % (data_type, " \n".join([k for k in kind_map if "_" in k])) + f"I never heard of `{data_type}` -- are you sure this is a valid HCP type? " + f"I currenlty support:\n{" \n".join([k for k in kind_map if "_" in k])}" ) context = "rmeg" if "rest" in data_type else "tmeg" sensor_mode = evoked_map["modes"][sensor_mode] @@ -274,14 +275,14 @@ def get_file_paths( "annot", ): raise ValueError( - 'You requested preprocessed data of type "%s" ' - 'and output "%s". HCP does not provide these data' % (data_type, output) + f'You requested preprocessed data of type "{data_type}" ' + f'and output "{output}". HCP does not provide these data' ) if data_type in ("rest", "noise_subject", "noise_empty_room") and output in ( "trial_info", "evoked", ): - raise ValueError("%s not defined for %s" % (output, data_type)) + raise ValueError(f"{output} not defined for {data_type}.") files = list() pipeline = pipeline_map.get(output, output) diff --git a/hcp/preprocessing.py b/hcp/preprocessing.py index ba2ba3a..27fd986 100644 --- a/hcp/preprocessing.py +++ b/hcp/preprocessing.py @@ -68,16 +68,18 @@ def apply_ica_hcp(raw, ica_mat, exclude): def apply_ref_correction(raw, decim_fit=100): - """Regress out MEG ref channels + """Regress out MEG ref channels. Computes linear models from MEG reference channels on each sensors, predicts the MEG data and subtracts and computes the residual by subtracting the predictions. .. note:: + Operates in place. .. note:: + Can be memory demanding. To alleviate this problem the model can be fit on decimated data. This is legitimate because the linear model does not have any representation of time, only the distributions @@ -88,8 +90,7 @@ def apply_ref_correction(raw, decim_fit=100): raw : instance of Raw The BTi/4D raw data. decim_fit : int - The decimation factor used for fitting the model. - Defaults to 100. + The decimation factor used for fitting the model. Defaults to 100. """ from sklearn.linear_model import LinearRegression from sklearn.pipeline import Pipeline @@ -110,12 +111,14 @@ def apply_ref_correction(raw, decim_fit=100): def map_ch_coords_to_mne(inst): - """Transform sensors to MNE coordinates + """Transform sensors to MNE coordinates. .. note:: - operates in place + + Operates in place. .. warning:: + For several reasons we do not use the MNE coordinates for the inverse modeling. This however won't always play nicely with visualization. @@ -137,9 +140,10 @@ def map_ch_coords_to_mne(inst): def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="fast"): - """Interpolate all MEG channels that are missing + """Interpolate all MEG channels that are missing. .. warning:: + This function may require some memory. Parameters @@ -150,12 +154,13 @@ def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="f The subject data_type : str The kind of data to read. The following options are supported: - 'rest' - 'task_motor' - 'task_story_math' - 'task_working_memory' - 'noise_empty_room' - 'noise_subject' + + * ``'rest'`` + * ``'task_motor'`` + * ``'task_story_math'`` + * ``'task_working_memory'`` + * ``'noise_empty_room'`` + * ``'noise_subject'`` run_index : int The run index. For the first run, use 0, for the second, use 1. Also see HCP documentation for the number of runs for a given data @@ -163,7 +168,7 @@ def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="f hcp_path : str The HCP directory, defaults to op.curdir. mode : str - Either `'accurate'` or `'fast'`, determines the quality of the + Either ``'accurate'`` or ``'fast'``, determines the quality of the Legendre polynomial expansion used for interpolation of MEG channels. @@ -179,7 +184,7 @@ def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="f hcp_path=hcp_path, run_index=run_index if run_index is None else run_index, ) - except (ValueError, IOError): + except (OSError, ValueError): raise ValueError( "could not find config to complete info." "reading only channel positions without " diff --git a/tutorials/plot_compute_forward.py b/tutorials/plot_compute_forward.py index 2877576..c64ba99 100644 --- a/tutorials/plot_compute_forward.py +++ b/tutorials/plot_compute_forward.py @@ -29,24 +29,24 @@ # %% # And we assume to have the downloaded data, the MNE/freesurfer style # anatomy directory, and the MNE style MEG directory. -# these can be obtained from :func:`make_mne_anatomy`. +# these can be obtained from :func:`~hcp.make_mne_anatomy`. # See also :ref:`tut_make_anatomy`. # %% # First we read the coregistration. head_mri_t = mne.read_trans( - op.join(recordings_path, subject, "{}-head_mri-trans.fif".format(subject)) + op.join(recordings_path, subject, f"{subject}-head_mri-trans.fif") ) # %% # Now we can setup our source model. -# Note that spacing has to be set to 'all' since no common MNE resampling +# Note that spacing has to be set to ``'all'`` since no common MNE resampling # scheme has been employed in the HCP pipelines. # Since this will take very long time to compute and at this point no other # decimation scheme is available inside MNE, we will compute the source # space on fsaverage, the freesurfer average brain, and morph it onto -# the subject's native space. With `oct6` we have ~8000 dipole locations. +# the subject's native space. With ``oct6`` we have ~8000 dipole locations. src_fsaverage = mne.setup_source_space( subject="fsaverage", From c4d5617198282d00fe7c08cbeee62c9f02a7de2b Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 15:27:07 +0200 Subject: [PATCH 12/34] more clean-up --- doc/conf.py | 1 + hcp/io/file_mapping/s3.py | 39 +++++++++++++++++++++------------------ hcp/preprocessing.py | 6 +++--- hcp/viz.py | 12 ++++++------ 4 files changed, 31 insertions(+), 27 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index f7518ee..a14b0a5 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -119,6 +119,7 @@ "Epochs": "mne.Epochs", "Evoked": "mne.Evoked", "Forward": "mne.Forward", + "Layout": "mne.channels.Layout", "Info": "mne.Info", "Projection": "mne.Projection", "Raw": "mne.io.Raw", diff --git a/hcp/io/file_mapping/s3.py b/hcp/io/file_mapping/s3.py index 4aa4661..a3c865e 100644 --- a/hcp/io/file_mapping/s3.py +++ b/hcp/io/file_mapping/s3.py @@ -7,7 +7,7 @@ def get_s3_keys_anatomy( meg_anatomy_outputs=("head_model", "transforms"), hcp_path_bucket="HCP_900", ): - """Helper to prepare AWS downloads for anatomy data + """Helper to prepare AWS downloads for anatomy data. A helper function useful for working with Amazon EC2 and S3. It compiles a list of related files. @@ -22,10 +22,10 @@ def get_s3_keys_anatomy( The subject, a 6 digit code. freesurfer_outputs : str | list | tuple The Freesurfer outputs to be downloaded. Defaults to - `('label', 'mri', 'surf')`. + ``('label', 'mri', 'surf')``. meg_anatomy_outputs : str | list | tuple The MEG anatomy contents to download. Defaults to - `('head_model', 'transforms')`. + ``('head_model', 'transforms')``. hcp_path_bucket : str The S3 bucket path. Will be prepended to each file path. @@ -64,12 +64,13 @@ def get_s3_keys_meg( hcp_path_bucket="HCP_900", onsets="stim", ): - """Helper to prepare AWS downloads for MEG data + """Helper to prepare AWS downloads for MEG data. A helper function useful for working with Amazon EC2 and S3. It compiles a list of related files. .. note:: + This function does not download anything. It only facilitates downloading by compiling the content. @@ -79,22 +80,24 @@ def get_s3_keys_meg( The subject, a 6 digit code. data_type : str | tuple of str | list of str The acquisition context of the data. The following ones are supported: - 'rest' - 'noise' - 'task_motor' - 'task_story_math' - 'task_working_memory' + + * ``'rest'`` + * ``'noise'`` + * ``'task_motor'`` + * ``'task_story_math'`` + * ``'task_working_memory'`` outputs : str | tuple of str | list of str The kind of output. The following ones are supported: - 'raw' - 'epochs' - 'evoked' - 'ica' - 'annotations' - 'trial_info' - 'freesurfer' - 'meg_anatomy' - onsets : {'stim', 'resp', 'sentence', 'block'} | list | tuple + + * ``'raw'`` + * ``'epochs'`` + * ``'evoked'`` + * ``'ica'`` + * ``'annotations'`` + * ``'trial_info'`` + * ``'freesurfer'`` + * ``'meg_anatomy'`` + onsets : ``'stim'`` | ``'resp'`` | ``'sentence'`` | ``'block'`` | list | tuple The event onsets. Only considered for epochs and evoked outputs The mapping is generous, everything that is not a response is a stimulus, in the sense of internal or external events. sentence and diff --git a/hcp/preprocessing.py b/hcp/preprocessing.py index 27fd986..4e43338 100644 --- a/hcp/preprocessing.py +++ b/hcp/preprocessing.py @@ -68,7 +68,7 @@ def apply_ica_hcp(raw, ica_mat, exclude): def apply_ref_correction(raw, decim_fit=100): - """Regress out MEG ref channels. + """Regresses out MEG ref channels. Computes linear models from MEG reference channels on each sensors, predicts the MEG data and subtracts @@ -161,12 +161,12 @@ def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="f * ``'task_working_memory'`` * ``'noise_empty_room'`` * ``'noise_subject'`` + hcp_path : str + The HCP directory, defaults to op.curdir. run_index : int The run index. For the first run, use 0, for the second, use 1. Also see HCP documentation for the number of runs for a given data type. - hcp_path : str - The HCP directory, defaults to op.curdir. mode : str Either ``'accurate'`` or ``'fast'``, determines the quality of the Legendre polynomial expansion used for interpolation of MEG diff --git a/hcp/viz.py b/hcp/viz.py index a375f18..8199b9e 100644 --- a/hcp/viz.py +++ b/hcp/viz.py @@ -11,16 +11,16 @@ def make_hcp_bti_layout(info): - """Get Layout of HCP Magnes3600WH data + """Get Layout of HCP Magnes3600WH data. Parameters ---------- - info : mne.io.meas_info.Info + info : Info The measurement info. Returns ------- - lout : mne.channels.Layout + lout : Layout The layout that can be used for plotting. """ picks = list(range(248)) @@ -42,7 +42,7 @@ def plot_coregistration( info_from=(("data_type", "rest"), ("run_index", 0)), view_init=(("azim", 0), ("elev", 0)), ): - """A diagnostic plot to show the HCP coregistration + """Plot diagnostic to show the HCP coregistration. Parameters ---------- @@ -67,7 +67,7 @@ def plot_coregistration( Returns ------- - fig : matplotlib.figure.Figure + fig : Figure The figure object. """ import matplotlib.pyplot as plt @@ -79,7 +79,7 @@ def plot_coregistration( view_init = dict(view_init) head_mri_t = read_trans( - op.join(recordings_path, subject, "{}-head_mri-trans.fif".format(subject)) + op.join(recordings_path, subject, f"{subject}-head_mri-trans.fif") ) info = read_info(subject=subject, hcp_path=hcp_path, **info_from) From cc65335cdce968c59138a07253e0966ee0bf7ce7 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 15:27:25 +0200 Subject: [PATCH 13/34] add back aws file --- .s3cfg.enc | Bin 0 -> 1952 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .s3cfg.enc diff --git a/.s3cfg.enc b/.s3cfg.enc new file mode 100644 index 0000000000000000000000000000000000000000..1dbf89b8c447eeea54eb73b5aee2732fbf3bf80b GIT binary patch literal 1952 zcmV;R2VeNy5g#Qw7)jF&2d@{!p#}@F8atSf*qG~$>$)oLjwr7kW8dH!ZNX#gos_fO z+!O(}crI}f$u)amM1^Y+G=k4ecr&Gk>-p5lRH4ph5Tge@bYutF)nycfcaLXGXf#^( zP^tzh9aSmVi5rmqPgto}q$4(g3rfI!AwQDHh>IhDD_IhFy8k{D_)4l2R2rOHM(g)m zv@YLr98v5pe+x(oOr{OW{?Ae$z6dV&b{r#u7Ehx+af?eA1T^1%-})6a0e~$LR0Um->5y3Dk{%Wvp&U)isA>WdVtFfY-)>~6tEWlmR(Ewtx-`x>#R3Uv-X6$ad71lMa%9Z_dT{1 zTOdJaI3|;b@)Lemr!C_#eh;0190G0w-EK;uXZ}pI;_8t#-?ujKT#x2*GyB<@2PLX= z;A?xH#5axr1Z}x5A4A$5``R|!IAU-)q6hTNfbWoKFR*vg2J>Adty%&-yS!=BtjMY> z7FMDBS@x+Ai{zUI*C;Ss=yocAu^5j2Om?!WT)Xbf7b%m7f6z#Aid-&xe&(<%x67XQ z5k$c58s@LetYKU!*a!+$(SM z)bP59K$Ywkup~J3sbd$&jEN$REdU=onYpl>YXlFE`&Yb&BzARZBqe^F=u3(4h`|?u z-#dsT1nuJOYxVN86#b||nEcl$w^Qwv{i?rR-c7Oy1qd*%uK#`*NP;~L+KvP!kq|dLj=WKudI0Z7$Wfwk z*-f0mGkbE=EN2x~5@qFTC0paY2WyBiP{9-xXJ?gKt|GdDd?G_UyGuEmpB$zh8PZ)^^i8?YSE3t;+ z2Ec@58BKCIL@27%K-vm;+07D|wE}9wR=&2T%)$dlP*IfwtMxF-#) z#cn{nD(TaR{&v+q|JA3p7GrhBQY=fz-p1EFCf7ZFFRe!+Wdx)`y#Kg0AHs%Qw8IJ+ zf3PKJ!8bVZdi*bf>f@YAs=B0R4q1$6+HGjhDZ5_8%#A9$4W|}RAqz&TKNfM>XBfqO zzAxH*lRdIc8r->R_;e^hn#!CO06=h9&0;D1g2v9y3XS#Fa@gE|NT zIwP1b{)M}>AN|J4K%8VrkF9KMXwRiUNdvvv8b++dC$CJ$s~?A5d98d;J&z?CdB2k` z4?5T?cG{IdaAb=@4k!=L$Dzq|Sv^9=LQD5y)|FS&PdO^&Jo5^;vPfb3hht-welT9d z+5Q6yMtkmrXC>b7kx$?kNP_q?4_t->gzXk%_p$p;FJKY3y}U^WY6?khL+3zI54B<$ zPDmFt^ECb+OaU0pr6~<<2*hny1CajNl__oa#G3%o6S6MvNqM=cc_Lp-IzbcLkdjDv zOhs_ws=09t(O>~pH%4@iGVDxsQS3=8s`lVLTE7$2V2XVzT0~TByc`It2&=>L(ikVj zP3~k*-i-1Oi-Ahc;_{k0GG@k$*Lz}DshesmH)o7$Ad4dG7e~V!rQ;r=ipOb;S-woo zcD#^Oa2Nz%eTalFnT55_PF&(aM_*BLzwJQ?PEpyq^gGP*9v&a(9aUj-eC}X@wx}GWk? Date: Mon, 22 Apr 2024 15:32:25 +0200 Subject: [PATCH 14/34] fix stuff --- doc/python_reference.rst | 2 ++ hcp/anatomy.py | 4 +++- hcp/io/file_mapping/s3.py | 12 ++++++------ hcp/preprocessing.py | 7 ++++--- hcp/viz.py | 6 +++--- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/doc/python_reference.rst b/doc/python_reference.rst index 709e82c..05c8876 100644 --- a/doc/python_reference.rst +++ b/doc/python_reference.rst @@ -1,3 +1,5 @@ +:orphan: + .. _api_reference: ==================== diff --git a/hcp/anatomy.py b/hcp/anatomy.py index f963b27..5ab4dc8 100644 --- a/hcp/anatomy.py +++ b/hcp/anatomy.py @@ -41,7 +41,9 @@ def make_mne_anatomy( subject : str The subject name. subjects_dir : str - The path corresponding to MNE/freesurfer SUBJECTS_DIR (to be created) + The path corresponding to MNE/freesurfer SUBJECTS_DIR (to be created). + recordings_path : str + The path to the recordings where the transformation files are written. hcp_path : str The path where the HCP files can be found. outputs : ``'label'`` | ``'mri'`` | ``'stats'`` | ``'surf'`` | ``'touch'`` diff --git a/hcp/io/file_mapping/s3.py b/hcp/io/file_mapping/s3.py index a3c865e..4912c97 100644 --- a/hcp/io/file_mapping/s3.py +++ b/hcp/io/file_mapping/s3.py @@ -78,7 +78,7 @@ def get_s3_keys_meg( ---------- subject : str The subject, a 6 digit code. - data_type : str | tuple of str | list of str + data_types : str | tuple of str | list of str The acquisition context of the data. The following ones are supported: * ``'rest'`` @@ -97,17 +97,17 @@ def get_s3_keys_meg( * ``'trial_info'`` * ``'freesurfer'`` * ``'meg_anatomy'`` - onsets : ``'stim'`` | ``'resp'`` | ``'sentence'`` | ``'block'`` | list | tuple - The event onsets. Only considered for epochs and evoked outputs - The mapping is generous, everything that is not a response is a - stimulus, in the sense of internal or external events. sentence and - block are specific to task_story_math. Can be a collection of those. run_inds : int | list of int | tuple of int The run index. For the first run, use 0, for the second, use 1. Also see HCP documentation for the number of runs for a given data type. hcp_path_bucket : str The S3 bucket path. Will be prepended to each file path. + onsets : ``'stim'`` | ``'resp'`` | ``'sentence'`` | ``'block'`` | list | tuple + The event onsets. Only considered for epochs and evoked outputs + The mapping is generous, everything that is not a response is a + stimulus, in the sense of internal or external events. sentence and + block are specific to task_story_math. Can be a collection of those. Returns ------- diff --git a/hcp/preprocessing.py b/hcp/preprocessing.py index 4e43338..ad9305f 100644 --- a/hcp/preprocessing.py +++ b/hcp/preprocessing.py @@ -39,16 +39,17 @@ def apply_ica_hcp(raw, ica_mat, exclude): """Apply the HCP ICA. .. note:: + Operates in place and data must be loaded. Parameters ---------- raw : instance of Raw - the hcp raw data. - ica_mat : numpy structured array + The hcp raw data. + ica_mat : array The hcp ICA solution exclude : array-like - the components to be excluded. + The components to be excluded. """ if not raw.preload: raise RuntimeError("raw data must be loaded, use raw.load_data()") diff --git a/hcp/viz.py b/hcp/viz.py index 8199b9e..4e1d038 100644 --- a/hcp/viz.py +++ b/hcp/viz.py @@ -47,9 +47,9 @@ def plot_coregistration( Parameters ---------- subject : str - The subject + The subject. subjects_dir : str - The path corresponding to MNE/freesurfer SUBJECTS_DIR (to be created) + The path corresponding to MNE/freesurfer SUBJECTS_DIR (to be created). hcp_path : str The path where the HCP files can be found. recordings_path : str @@ -63,7 +63,7 @@ def plot_coregistration( are smaller than 12 digits, hence negligible. view_init : tuple of tuples | dict The initival view, defaults to azimuth and elevation of 0, - a simple lateral view + a simple lateral view. Returns ------- From 5114b151c016645c5500e195f672c1343a5680d7 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 15:43:52 +0200 Subject: [PATCH 15/34] fix stuff --- doc/conf.py | 1 + hcp/anatomy.py | 27 +++---- hcp/io/file_mapping/file_mapping.py | 8 +- hcp/io/read.py | 120 ++++++++++++++-------------- hcp/preprocessing.py | 13 +-- 5 files changed, 85 insertions(+), 84 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index a14b0a5..844c1fa 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -123,6 +123,7 @@ "Info": "mne.Info", "Projection": "mne.Projection", "Raw": "mne.io.Raw", + "SourceSpaces": "mne.SourceSpaces", # Python "bool": ":class:`python:bool`", "Path": "pathlib.Path", diff --git a/hcp/anatomy.py b/hcp/anatomy.py index 5ab4dc8..302a9ab 100644 --- a/hcp/anatomy.py +++ b/hcp/anatomy.py @@ -152,17 +152,16 @@ def compute_forward_stack( subject, recordings_path, info_from=(("data_type", "rest"), ("run_index", 0)), - fwd_params=None, src_params=None, hcp_path=op.curdir, n_jobs=1, verbose=None, ): - """ - Convenience function for conducting standard MNE analyses. + """Convenience function for conducting standard MNE analyses. .. note:: - this function computes bem solutions, source spaces and forward models + + This function computes bem solutions, source spaces and forward models optimized for connectivity computation, i.e., the fsaverage space is morphed onto the subject's space. @@ -170,12 +169,10 @@ def compute_forward_stack( ---------- subject : str The subject name. - hcp_path : str - The directory containing the HCP data. - recordings_path : str - The path where MEG data and transformations are stored. subjects_dir : str The directory containing the extracted HCP subject data. + recordings_path : str + The path where MEG data and transformations are stored. info_from : tuple of tuples | dict The reader info concerning the data from which sensor positions should be read. @@ -183,8 +180,6 @@ def compute_forward_stack( coordinates for 4D systems, hence not available in that case. Note that differences between the sensor positions across runs are smaller than 12 digits, hence negligible. - fwd_params : None | dict - The forward parameters src_params : None | dict The src params. Defaults to: @@ -193,11 +188,11 @@ def compute_forward_stack( dict(subject='fsaverage', fname=None, spacing='oct6', n_jobs=2, surface='white', subjects_dir=subjects_dir, add_dist=True) hcp_path : str - The prefix of the path of the HCP data. + The directory containing the HCP data. n_jobs : int The number of jobs to use in parallel. - verbose : bool, str, int, or None - If not None, override default verbose level (see mne.verbose) + verbose : bool | str | int | None + If not None, override default verbose level (see mne.verbose). Returns ------- @@ -205,13 +200,13 @@ def compute_forward_stack( A dictionary with the following keys: fwd : instance of Forward The forward solution. - src_subject : instance of mne.SourceSpace + src_subject : instance of SourceSpaces The source model on the subject's surface - src_fsaverage : instance of mne.SourceSpace + src_fsaverage : instance of SourceSpaces The source model on fsaverage's surface bem_sol : dict The BEM. - info : instance of mne.io.meas_info.Info + info : instance of Info The actual measurement info used. """ if isinstance(info_from, tuple): diff --git a/hcp/io/file_mapping/file_mapping.py b/hcp/io/file_mapping/file_mapping.py index bc45758..869f255 100644 --- a/hcp/io/file_mapping/file_mapping.py +++ b/hcp/io/file_mapping/file_mapping.py @@ -230,6 +230,10 @@ def get_file_paths( * ``'trial_info'`` * ``'freesurfer'`` * ``'meg_anatomy'`` + run_index : int + The run index. For the first run, use 0, for the second, use 1. + Also see HCP documentation for the number of runs for a given data + type. onset : ``'stim`` | ``'resp'`` | ``'sentence'`` | ``'block'`` The event onset. Only considered for epochs and evoked outputs The mapping is generous, everything that is not a response is a @@ -238,10 +242,6 @@ def get_file_paths( sensor_mode : ``'mag'`` | ``'planar'`` The sensor projection. Defaults to ``'mag'``. Only relevant for evoked output. - run_index : int - The run index. For the first run, use 0, for the second, use 1. - Also see HCP documentation for the number of runs for a given data - type. hcp_path : str The HCP directory, defaults to ``op.curdir``. diff --git a/hcp/io/read.py b/hcp/io/read.py index b0778a2..108ad1c 100644 --- a/hcp/io/read.py +++ b/hcp/io/read.py @@ -285,36 +285,38 @@ def read_epochs( hcp_path=op.curdir, return_fixations_motor=False, ): - """Read HCP processed data + """Read HCP processed data. Parameters ---------- subject : str, file_map - The subject + The subject. data_type : str The kind of data to read. The following options are supported: - 'rest' - 'task_motor' - 'task_story_math' - 'task_working_memory' - onset : {'stim', 'resp', 'sentence', 'block'} + + * ``'rest'`` + * ``'task_motor'`` + * ``'task_story_math'`` + * ``'task_working_memory'`` + onset : ``'stim'`` | ``'resp'`` | ``'sentence'`` | ``'block'`` The event onset. The mapping is generous, everything that is not a response is a stimulus, in the sense of internal or external events. `sentence` and `block` are specific to task_story_math. run_index : int - The run index. For the first run, use 0, for the second, use 1. + The run index. For the first run, use ``0``, for the second, use ``1``. Also see HCP documentation for the number of runs for a given data type. hcp_path : str - The HCP directory, defaults to op.curdir. + The HCP directory, defaults to ``op.curdir``. return_fixations_motor : bool Weather to return fixations or regular trials. For motor data only. Defaults to False. + Returns ------- - epochs : instance of mne.Epochs + epochs : instance of Epochs The MNE epochs. Note, these are pseudo-epochs in the case of - onset == 'rest'. + ``onset == 'rest'``. """ info = read_info( subject=subject, data_type=data_type, run_index=run_index, hcp_path=hcp_path @@ -378,7 +380,7 @@ def _hcp_pick_info(info, ch_names): def read_trial_info(subject, data_type, run_index=0, hcp_path=op.curdir): - """Read information about trials + """Read information about trials. Parameters ---------- @@ -386,16 +388,17 @@ def read_trial_info(subject, data_type, run_index=0, hcp_path=op.curdir): The HCP subject. data_type : str The kind of data to read. The following options are supported: - 'rest' - 'task_motor' - 'task_story_math' - 'task_working_memory' + + * ``'rest'`` + * ``'task_motor'`` + * ``'task_story_math'`` + * ``'task_working_memory'`` run_index : int - The run index. For the first run, use 0, for the second, use 1. - Also see HCP documentation for the number of runs for a given data - type. + The run index. For the first run, use ``0``, for the second, use ``1``. + Also see HCP documentation for the number of runs for a given data type. hcp_path : str - The HCP directory, defaults to op.curdir. + The HCP directory, defaults to ``op.curdir``. + Returns ------- trial_info : dict @@ -441,7 +444,7 @@ def _check_sorting_runs(candidates, id_char): def _parse_annotations_segments(segment_strings): - """Read bad segments defintions from text file""" + """Read bad segments defintions from text file.""" for char in "}]": # multi line array definitions segment_strings = segment_strings.replace(char + ";", "splitme") split = segment_strings.split("splitme") @@ -470,19 +473,20 @@ def read_annot(subject, data_type, run_index=0, hcp_path=op.curdir): Parameters ---------- subject : str, file_map - The subject + The subject. data_type : str The kind of data to read. The following options are supported: - 'rest' - 'task_motor' - 'task_story_math' - 'task_working_memory' + + * ``'rest'`` + * ``'task_motor'`` + * ``'task_story_math'`` + * ``'task_working_memory'`` run_index : int - The run index. For the first run, use 0, for the second, use 1. + The run index. For the first run, use ``0``, for the second, use ``1``. Also see HCP documentation for the number of runs for a given data type. hcp_path : str - The HCP directory, defaults to op.curdir. + The HCP directory, defaults to ``op.curdir``. Returns ------- @@ -523,28 +527,28 @@ def read_annot(subject, data_type, run_index=0, hcp_path=op.curdir): def read_ica(subject, data_type, run_index=0, hcp_path=op.curdir): - """Read precomputed independent components from subject + """Read precomputed independent components from subject. Parameters ---------- subject : str, file_map - The subject + The subject. data_type : str The kind of data to read. The following options are supported: - 'rest' - 'task_motor' - 'task_story_math' - 'task_working_memory' + + * ``'rest'`` + * ``'task_motor'`` + * ``'task_story_math'`` + * ``'task_working_memory'`` run_index : int - The run index. For the first run, use 0, for the second, use 1. - Also see HCP documentation for the number of runs for a given data - type. + The run index. For the first run, use ``0``, for the second, use ``1``. + Also see HCP documentation for the number of runs for a given data type. hcp_path : str - The HCP directory, defaults to op.curdir. + The HCP directory, defaults to ``op.curdir``. Returns ------- - mat : numpy structured array + mat : array The ICA mat struct. """ ica_files = get_file_paths( @@ -555,9 +559,7 @@ def read_ica(subject, data_type, run_index=0, hcp_path=op.curdir): hcp_path=hcp_path, ) ica_fname_mat = [k for k in ica_files if k.endswith("icaclass.mat")][0] - - mat = scio.loadmat(ica_fname_mat, squeeze_me=True)["comp_class"] - return mat + return scio.loadmat(ica_fname_mat, squeeze_me=True)["comp_class"] def _parse_annotations_bad_channels(bads_strings): @@ -576,7 +578,7 @@ def _parse_annotations_bad_channels(bads_strings): def _parse_annotations_ica(ica_strings): - """Read bad channel definitions from text file""" + """Read bad channel definitions from text file.""" # prepare splitting for char in "}]": # multi line array definitions ica_strings = ica_strings.replace(char + ";", "splitme") @@ -617,33 +619,35 @@ def read_evokeds( hcp_path=op.curdir, kind="average", ): - """Read HCP processed data + """Read HCP processed data. Parameters ---------- subject : str, file_map - The subject + The subject. data_type : str The kind of data to read. The following options are supported: - 'rest' - 'task_motor' - 'task_story_math' - 'task_working_memory' - onset : {'stim', 'resp'} + + * ``'rest'`` + * ``'task_motor'`` + * ``'task_story_math'`` + * ``'task_working_memory'`` + onset : ``'stim'``, ``'resp'`` The event onset. The mapping is generous, everything that is not a response is a stimulus, in the sense of internal or external events. - sensor_mode : {'mag', 'planar'} - The sensor projection. Defaults to 'mag'. Only relevant for + sensor_mode : ``'mag'`` | ``'planar'`` + The sensor projection. Defaults to ``'mag'``. Only relevant for evoked output. hcp_path : str - The HCP directory, defaults to op.curdir. - kind : {'average', 'standard_error'} - The averaging mode. Defaults to 'average'. + The HCP directory, defaults to ``op.curdir``. + kind : ``'average'`` | ``'standard_error'`` + The averaging mode. Defaults to ``'average'``. + Returns ------- - epochs : instance of mne.Epochs + epochs : instance of Epochs The MNE epochs. Note, these are pseudo-epochs in the case of - onset == 'rest'. + ``onset == 'rest'``. """ info = read_info( subject=subject, data_type=data_type, hcp_path=hcp_path, run_index=0 @@ -662,7 +666,7 @@ def read_evokeds( def _read_evoked(fname, sensor_mode, info, kind): - """helper to read evokeds""" + """Helper to read evokeds.""" data = scio.loadmat(fname, squeeze_me=True)["data"] ch_names = [ch for ch in data["label"].tolist()] diff --git a/hcp/preprocessing.py b/hcp/preprocessing.py index ad9305f..15a3639 100644 --- a/hcp/preprocessing.py +++ b/hcp/preprocessing.py @@ -18,15 +18,16 @@ def set_eog_ecg_channels(raw): - """Set the HCP ECG and EOG channels + """Set the HCP ECG and EOG channels. .. note:: + Operates in place. Parameters ---------- raw : instance of Raw - the hcp raw data. + The hcp raw data. """ for kind in ["ECG", "VEOG", "HEOG"]: set_bipolar_reference( @@ -47,7 +48,7 @@ def apply_ica_hcp(raw, ica_mat, exclude): raw : instance of Raw The hcp raw data. ica_mat : array - The hcp ICA solution + The hcp ICA solution. exclude : array-like The components to be excluded. """ @@ -69,7 +70,7 @@ def apply_ica_hcp(raw, ica_mat, exclude): def apply_ref_correction(raw, decim_fit=100): - """Regresses out MEG ref channels. + """Regress out MEG ref channels. Computes linear models from MEG reference channels on each sensors, predicts the MEG data and subtracts @@ -152,7 +153,7 @@ def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="f inst : MNE data containers Raw, Epochs, Evoked. subject : str, file_map - The subject + The subject. data_type : str The kind of data to read. The following options are supported: @@ -175,7 +176,7 @@ def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="f Returns ------- - out : MNE data containers + out : MNE data containers Raw, Epochs, Evoked but with missing channels interpolated. """ try: From 70e4256ac80ac9fa87f5cf6e6652db1135078530 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 16:00:11 +0200 Subject: [PATCH 16/34] more fixes --- README.rst | 6 +- doc/conf.py | 2 + doc/index.rst | 187 +++++++++++++++++++-------------------- doc/python_reference.rst | 2 - hcp/io/read.py | 2 +- 5 files changed, 96 insertions(+), 103 deletions(-) diff --git a/README.rst b/README.rst index 1dfa3e9..b872075 100644 --- a/README.rst +++ b/README.rst @@ -289,9 +289,9 @@ Python conventions by subtracting 1. Contributions ------------- -- currently ``@dengemann`` is pushing frequently to master, if you plan to contribute, - open issues and pull requests, or contact ``@dengemann`` directly. Discussions are - welcomed. +Currently ``@dengemann`` is pushing frequently to master, if you plan to contribute, +open issues and pull requests, or contact ``@dengemann`` directly. Discussions are +welcomed. Acknowledgements ================ diff --git a/doc/conf.py b/doc/conf.py index 844c1fa..67892ab 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -133,6 +133,8 @@ "instance", "of", "shape", + "MNE", + "containers", } # validation diff --git a/doc/index.rst b/doc/index.rst index ee22e70..e776556 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,8 +1,5 @@ -|Travis|_ |Zenodo|_ |Codecov|_ - -.. |Travis| image:: https://api.travis-ci.org/mne-tools/mne-hcp.png?branch=master -.. _Travis: https://travis-ci.org/mne-tools/mne-hcp +|Zenodo|_ |Codecov|_ .. |Zenodo| image:: https://zenodo.org/badge/53261823.svg .. _Zenodo: https://zenodo.org/badge/latestdoi/53261823 @@ -14,37 +11,46 @@ MNE-HCP ======= -We provide Python tools for seamless integration of MEG data from the `Human Connectome Project `_ into the Python ecosystem. -In only a few lines of code, complex data retrieval requests can be readily executed on the resources from this neuroimaging reference dataset. Raw HCP data are translated into actionable MNE objects that we know and love. MNE-HCP abstracts away difficulties due to diverging coordinate systems, distributed information, and file format conventions. Providing a simple and consistent access to HCP MEG data will facilitate emergence of standardized data analysis practices. -By building on the `MNE software package `_, MNE-HCP naturally supplements a fast growing stack of Python data science toolkits. +We provide Python tools for seamless integration of MEG data from the +`Human Connectome Project `_ into the Python ecosystem. +In only a few lines of code, complex data retrieval requests can be readily executed on +the resources from this neuroimaging reference dataset. Raw HCP data are translated into +actionable MNE objects that we know and love. MNE-HCP abstracts away difficulties due to +diverging coordinate systems, distributed information, and file format conventions. +Providing a simple and consistent access to HCP MEG data will facilitate emergence of +standardized data analysis practices. By building on the +`MNE software package `_, MNE-HCP naturally supplements a fast +growing stack of Python data science toolkits. Fast interface to MEG data -------------------------- -Allow us to give you a flavor by a few example queries of MEG HCP data from subject 105923: +Allow us to give you a flavor by a few example queries of MEG HCP data from subject +105923: .. code-block:: python - # Get all entries from the MEG data header - info = hcp.read_info('105923', 'task_motor') + # Get all entries from the MEG data header + info = hcp.read_info('105923', 'task_motor') - # Get continuous MEG time series - raw = hcp.read_raw('105923', 'task_motor') + # Get continuous MEG time series + raw = hcp.read_raw('105923', 'task_motor') - # Get segmented MEG time series - epochs = hcp.read_epochs('105923', 'task_motor') + # Get segmented MEG time series + epochs = hcp.read_epochs('105923', 'task_motor') - # Get all MEG time series averaged across events - list_of_evoked = hcp.read_evokeds('105923', 'task_motor') + # Get all MEG time series averaged across events + list_of_evoked = hcp.read_evokeds('105923', 'task_motor') - # Get details on contamination and noise sources - annotations_dict = hcp.read_annot('105923', 'task_motor') + # Get details on contamination and noise sources + annotations_dict = hcp.read_annot('105923', 'task_motor') - # Get precomputed independent components that compose the signal time series - ica_mat = hcp.read_ica('105923', 'task_motor') + # Get precomputed independent components that compose the signal time series + ica_mat = hcp.read_ica('105923', 'task_motor') Scope and Disclaimer -------------------- + This code is under active research-driven development. The API is still changing, but is getting closer to a stable release. @@ -53,32 +59,19 @@ but is getting closer to a stable release. For now please consider the following caveats: - We only intend to support a subset of the files shipped with HCP. - - Specifically, for now it is not planned to support io and processing for any outputs of the HCP source space pipelines. - - This library breaks with some of MNE conventions in order to make the HCP outputs compatible with MNE. + - Specifically, for now it is not planned to support io and processing for any + outputs of the HCP source space pipelines. + - This library breaks with some of MNE conventions in order to make the HCP outputs + compatible with MNE. Installation ============ -We recommend the `Anaconda Python distribution `_, which comes with the necessary dependencies. Alternatively, to install ``mne-hcp``, you first need to install its dependencies:: - - $ pip install numpy matplotlib scipy scikit-learn mne joblib pandas - -Then clone the repository:: - - $ git clone http://github.com/mne-tools/mne-hcp +We recommend the `Anaconda Python distribution `_, +which comes with the necessary dependencies. To install ``mne-hcp``, you +can install from source with:: -and finally run `setup.py` to install the package:: - - $ cd mne-hcp/ - $ python setup.py install - -If you do not have admin privileges on the computer, use the ``--user`` flag -with `setup.py`. - -Alternatively, for a devoloper install based on symbolic links (which simplifies keeping up with code changes), do:: - - $ cd mne-hcp/ - $ python setup.py develop + $ pip install git+http://github.com/mne-tools/mne-hcp To check if everything worked fine, you can do:: @@ -86,23 +79,13 @@ To check if everything worked fine, you can do:: and it should not give any error messages. -Dependencies ------------- - -The following main and additional dependencies are required to use MNE-HCP: - - - MNE-Python master branch - - the MNE-Python dependencies, specifically - - scipy - - numpy - - matplotlib - - scikit-learn (optional) - Quickstart ========== -The following data layout is expected: a folder that contains the HCP data -as they are unpacked by a zip, subject wise. +The following data layout is expected: + +- a folder that contains the HCP data as they are unpacked by a zip, subject wise. + When data were downloaded via the Aspera connect client, the following command should produce the expected layout: @@ -113,13 +96,12 @@ command should produce the expected layout: $ unzip -o $fname; rm $fname; $ done -When files are downloaded using the `Amazon webservice tools `_, e.g. `s3rcmd`, -all should be fine. - -The code is organized by different modules. -The `io` module includes readers for sensor space data at different processing -stages and annotations for bad data. +When files are downloaded using the +`Amazon webservice tools `_, e.g. `s3rcmd`, all should be +fine. +The code is organized by different modules. The ``io`` module includes readers for +sensor space data at different processing stages and annotations for bad data. Types of Data ------------- @@ -146,8 +128,8 @@ table gives an overview: Functionality to make the HCP datasets compatible with MNE ---------------------------------------------------------- -MNE HCP comes with convenience functions such as `hcp.make_mne_anatomy`. This one will create an -MNE friendly anatomy directories and extracts the head model and +MNE HCP comes with convenience functions such as :func:`hcp.make_mne_anatomy`. This one +will create an MNE friendly anatomy directories and extracts the head model and coregistration MEG to MRI coregistration. (Yes, it maps to MRI, not to the helmet -- a peculiarity of the HCP data.) It can be used as follows: @@ -161,6 +143,7 @@ It can be used as follows: >>> '100307', subjects_dir=storage_dir + '/subjects', >>> hcp_path=storage_dir + '/HCP', >>> recordings_path=storage_dir + '/hcp-meg') + reading extended structural processing ... reading RAS freesurfer transform Combining RAS transform and coregistration @@ -186,6 +169,7 @@ Example usage: >>> files = hcp.file_mapping.et_file_paths( >>> subject='123455', data_type='task_motor', output='raw', >>> hcp_path='/media/storage/HCP') + ['/media/storage/HCP/123455/unprocessed/MEG/10-Motor/4D/c,rfDC', '/media/storage/HCP/123455/unprocessed/MEG/10-Motor/4D/config'] @@ -229,35 +213,41 @@ Reproducing HCP sensor space outputs A couple of steps are necessary to reproduce the original sensor space outputs. -1. Reference channels should be regressed out. Checkout :func:`hcp.preprocessing.apply_ref_correction`. +1. Reference channels should be regressed out. Checkout + :func:`hcp.preprocessing.apply_ref_correction`. 2. The trial info structure gives the correct latencies of the events The latencies in the trigger channel are shifted by around 18 ms. - For now we'd recommend using the events from the function :func:`hcp.read_trial_info`. + For now we'd recommend using the events from the function + :func:`hcp.read_trial_info`. 3. The default filters in MNE and FieldTrip are different. FieldTrip uses a 4th order butterworth filter. In MNE you might need - to adjust the `*_trans_bandwidth` parameter to avoid numerical errors. - In the HCP outputs, evoked responses were filtered between 0.5 and 30Hz prior + to adjust the ``*_trans_bandwidth`` parameter to avoid numerical errors. + In the HCP outputs, evoked responses were filtered between 0.5 and 30 Hz prior to baseline correction. -4. Annotations need to be loaded and registered. The HCP consortium ships annotations of bad segments and bad channels. - These have to be read and used. Check out `hcp.read_annot` and add bad - channel names to `raw.info['bads']` and create and set an `mne.Annotations` - object as attribute to `raw`, see below. +4. Annotations need to be loaded and registered. The HCP consortium ships annotations of + bad segments and bad channels. + These have to be read and used. Check out :func:`hcp.read_annot` and add bad + channel names to ``raw.info['bads']`` and create and set an :class:`mne.Annotations` + object as attribute to ``raw``, see below. - .. code-block:: python + .. code-block:: python - annots = hcp.read_annot(subject, data_type, hcp_path=hcp_path, - run_index=run_index) - bad_segments = annots['segments']['all'] / raw.info['sfreq'] - raw.annotations = mne.Annotations( - bad_segments[:, 0], (bad_segments[:, 1] - bad_segments[:, 0]), - description='bad') + annots = hcp.read_annot( + subject, data_type, hcp_path=hcp_path, run_index=run_index + ) + bad_segments = annots['segments']['all'] / raw.info['sfreq'] + raw.annotations = mne.Annotations( + bad_segments[:, 0], + (bad_segments[:, 1] - bad_segments[:, 0]), + description='bad', + ) 5. ICA components related to eye blinks and heart beats need to be removed from the data. Checkout the ICA slot in the output of - `hcp.read_annot` to get the HCP ICA components. + :func:`hcp.read_annot` to get the HCP ICA components. Convenience functions @@ -266,40 +256,43 @@ Convenience functions NNE-HCP includes convenience functions that help setting up directory and file layouts expected by MNE-Python. -:func:`hcp.make_mne_anatomy` will produce an MNE and Freesurfer compatible directory layout and will create the following outputs by default, mostly using sympbolic links: +:func:`hcp.make_mne_anatomy` will produce an MNE and Freesurfer compatible directory +layout and will create the following outputs by default, mostly using sympbolic links: .. code-block:: bash - $subjects_dir/$subject/bem/inner_skull.surf - $subjects_dir/$subject/label/* - $subjects_dir/$subject/mri/* - $subjects_dir/$subject/surf/* - $recordings_path/$subject/$subject-head_mri-trans.fif + $subjects_dir/$subject/bem/inner_skull.surf + $subjects_dir/$subject/label/* + $subjects_dir/$subject/mri/* + $subjects_dir/$subject/surf/* + $recordings_path/$subject/$subject-head_mri-trans.fif -These can then be set as $SUBJECTS_DIR and as MEG directory, consistent +These can then be set as ``$SUBJECTS_DIR`` and as MEG directory, consistent with MNE examples. -Here, `inner_skull.surf` and `$subject-head_mri-trans.fif` are written by the function such that they can be used by MNE. The latter is the coregistration matrix. +Here, ``inner_skull.surf`` and ``$subject-head_mri-trans.fif`` are written by the +function such that they can be used by MNE. The latter is the coregistration matrix. Python Indexing ^^^^^^^^^^^^^^^ MNE-HCP corrects on reading the indices it finds for data segments, events, or components. The indices it reads from the files will already be mapped to -Python conventions by subtracting 1. +Python conventions by subtracting ``1``. Contributions ------------- -- currently ``@dengemann`` is pushing frequently to master, if you plan to contribute, - open issues and pull requests, or contact ``@dengemann`` directly. Discussions are - welcomed. + +Currently ``@dengemann`` is pushing frequently to master, if you plan to contribute, +open issues and pull requests, or contact ``@dengemann`` directly. Discussions are +welcomed. Acknowledgements ================ -This project is supported by the Amazon Webservices Research grant issued to Denis A. Engemann and -by the ERC starting grant ERC StG 263584 issued to Virginie van Wassenhove. +This project is supported by the Amazon Webservices Research grant issued to +Denis A. Engemann and by the ERC starting grant ERC StG 263584 issued to +Virginie van Wassenhove. -I acknowledge support by Alex Gramfort, Mainak Jas, Jona Sassenhagen, Giorgos Michalareas, -Eric Larson, Danilo Bzdok, and Jan-Mathijs Schoffelen for discussions, -inputs and help with finding the best way to map -HCP data to the MNE world. +I acknowledge support by Alex Gramfort, Mainak Jas, Jona Sassenhagen, +Giorgos Michalareas, Eric Larson, Danilo Bzdok, and Jan-Mathijs Schoffelen for +discussions, inputs and help with finding the best way to map HCP data to the MNE world. diff --git a/doc/python_reference.rst b/doc/python_reference.rst index 05c8876..ce499e2 100644 --- a/doc/python_reference.rst +++ b/doc/python_reference.rst @@ -49,8 +49,6 @@ Handling HCP files for downloading Manipulating data and sensors ============================= -:py:mod:`hcp.preprocessing`: - .. currentmodule:: hcp.preprocessing .. autosummary:: diff --git a/hcp/io/read.py b/hcp/io/read.py index 108ad1c..17638f6 100644 --- a/hcp/io/read.py +++ b/hcp/io/read.py @@ -254,7 +254,7 @@ def read_info(subject, data_type, run_index=0, hcp_path=op.curdir): Returns ------- - info : instance of mne.io.meas_info.Info + info : instance of Info The MNE channel info object. .. note:: From b61111887164514cd2c5cf8781fa93fa654a237d Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 17:09:58 +0200 Subject: [PATCH 17/34] fix stuff --- doc/conf.py | 1 + doc/sg_execution_times.rst | 49 ----------------- examples/make_mne_anatomy.py | 10 ++-- examples/plot_bem.py | 1 + hcp/io/read.py | 52 +++++++++---------- .../plot_compute_evoked_inverse_solution.py | 8 +-- tutorials/plot_compute_forward.py | 11 ++-- tutorials/plot_reproduce_erf.py | 4 +- 8 files changed, 45 insertions(+), 91 deletions(-) delete mode 100644 doc/sg_execution_times.rst diff --git a/doc/conf.py b/doc/conf.py index 67892ab..2a2973c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -135,6 +135,7 @@ "shape", "MNE", "containers", + "file_map", } # validation diff --git a/doc/sg_execution_times.rst b/doc/sg_execution_times.rst deleted file mode 100644 index 75c0b71..0000000 --- a/doc/sg_execution_times.rst +++ /dev/null @@ -1,49 +0,0 @@ - -:orphan: - -.. _sphx_glr_sg_execution_times: - - -Computation times -================= -**00:00.000** total execution time for 5 files **from all galleries**: - -.. container:: - - .. raw:: html - - - - - - - - .. list-table:: - :header-rows: 1 - :class: table table-striped sg-datatable - - * - Example - - Time - - Mem (MB) - * - :ref:`sphx_glr_generated_tutorials_plot_compute_evoked_inverse_solution.py` (``../tutorials/plot_compute_evoked_inverse_solution.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_generated_tutorials_plot_compute_forward.py` (``../tutorials/plot_compute_forward.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_generated_tutorials_plot_reference_correction.py` (``../tutorials/plot_reference_correction.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_generated_tutorials_plot_reproduce_erf.py` (``../tutorials/plot_reproduce_erf.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_generated_tutorials_plot_temporal_searchlight_decoding.py` (``../tutorials/plot_temporal_searchlight_decoding.py``) - - 00:00.000 - - 0.0 diff --git a/examples/make_mne_anatomy.py b/examples/make_mne_anatomy.py index 7b11063..2a5f08e 100644 --- a/examples/make_mne_anatomy.py +++ b/examples/make_mne_anatomy.py @@ -11,7 +11,8 @@ files and information and creates a "subjects_dir" as known from MNE and freesurfer. -The most essential contents are +The most essential contents are: + - freesurfer surfaces - the MRI - the headmodel @@ -19,6 +20,7 @@ This function is your friend prior to running any source analysis. """ + # Author: Denis A. Engemann # License: BSD 3 clause @@ -32,12 +34,12 @@ # this is the new directory to be created subjects_dir = storage_dir + "/hcp-subjects" -# it will make the subfolders 'bem', 'mir', 'surf' and 'label' +# It will make the subfolders 'bem', 'mir', 'surf' and 'label' # and populate it with symbolic links if possible and write some files # where information has to be presented in a different format to satisfy MNE. -# this is where the coregistration matrix is written as -# `105923-head_mri-transform.fif` +# This is where the coregistration matrix is written as +# ``105923-head_mri-transform.fif``` recordings_path = storage_dir + "/hcp-meg" # comment out to write files on your disk diff --git a/examples/plot_bem.py b/examples/plot_bem.py index ae38030..f0a4210 100644 --- a/examples/plot_bem.py +++ b/examples/plot_bem.py @@ -7,6 +7,7 @@ MRT/anatomy files and the MEG data. Here, inspect the head model that is used to compute the BEM solution. """ + # Author: Jona Sassenhagen # Denis A. Engemann # License: BSD 3 clause diff --git a/hcp/io/read.py b/hcp/io/read.py index 17638f6..5a670da 100644 --- a/hcp/io/read.py +++ b/hcp/io/read.py @@ -183,32 +183,32 @@ def _check_infos_trans(infos): def read_raw(subject, data_type, run_index=0, hcp_path=op.curdir, verbose=None): - """Read HCP raw data + """Read HCP raw data. Parameters ---------- subject : str, file_map - The subject + The subject. data_type : str The kind of data to read. The following options are supported: - 'rest' - 'task_motor' - 'task_story_math' - 'task_working_memory' - 'noise_empty_room' - 'noise_subject' + + * ``'rest'`` + * ``'task_motor'`` + * ``'task_story_math'`` + * ``'task_working_memory'`` + * ``'noise_empty_room'`` + * ``'noise_subject'`` run_index : int - The run index. For the first run, use 0, for the second, use 1. - Also see HCP documentation for the number of runs for a given data - type. + The run index. For the first run, use ``0``, for the second, use ``1``. + Also see HCP documentation for the number of runs for a given data type. hcp_path : str - The HCP directory, defaults to op.curdir. + The HCP directory, defaults to ``op.curdir``. verbose : bool, str, int, or None If not None, override default verbose level (see mne.verbose). Returns ------- - raw : instance of mne.io.Raw + raw : instance of Raw The MNE raw object. """ pdf, config = get_file_paths( @@ -231,26 +231,25 @@ def read_raw(subject, data_type, run_index=0, hcp_path=op.curdir, verbose=None): def read_info(subject, data_type, run_index=0, hcp_path=op.curdir): - """Read info from unprocessed data + """Read info from unprocessed data. Parameters ---------- subject : str, file_map - The subject + The subject. data_type : str The kind of data to read. The following options are supported: - 'rest' - 'task_motor' - 'task_story_math' - 'task_working_memory' - 'noise_empty_room' - 'noise_subject' + * ``'rest'`` + * ``'task_motor'`` + * ``'task_story_math'`` + * ``'task_working_memory'`` + * ``'noise_empty_room'`` + * ``'noise_subject'`` run_index : int - The run index. For the first run, use 0, for the second, use 1. - Also see HCP documentation for the number of runs for a given data - type. + The run index. For the first run, use ``0``, for the second, use ``1``. + Also see HCP documentation for the number of runs for a given data type. hcp_path : str - The HCP directory, defaults to op.curdir. + The HCP directory, defaults to ``op.curdir``. Returns ------- @@ -258,6 +257,7 @@ def read_info(subject, data_type, run_index=0, hcp_path=op.curdir): The MNE channel info object. .. note:: + HCP MEG does not deliver only 3 of the 5 task packages from MRI HCP. """ raw, config = get_file_paths( @@ -375,7 +375,7 @@ def _read_epochs(epochs_mat_fname, info, return_fixations_motor): def _hcp_pick_info(info, ch_names): - """helper to subset info""" + """Helper to subset info.""" return pick_info(info, [info["ch_names"].index(ch) for ch in ch_names], copy=True) diff --git a/tutorials/plot_compute_evoked_inverse_solution.py b/tutorials/plot_compute_evoked_inverse_solution.py index 240504c..6f955ba 100644 --- a/tutorials/plot_compute_evoked_inverse_solution.py +++ b/tutorials/plot_compute_evoked_inverse_solution.py @@ -49,7 +49,7 @@ subjects_dir=subjects_dir, hcp_path=hcp_path, recordings_path=recordings_path, - # speed up computations here. Setting `add_dist` to True may improve the + # speed up computations here. Setting ``add_dist`` to True may improve the # accuracy. src_params=dict(add_dist=False), info_from=dict(data_type=data_type, run_index=run_index), @@ -79,13 +79,13 @@ # %% # Note that using the empty room noise covariance will inflate the SNR of the -# evkoked and renders comparisons to `baseline` rather uninformative. -noise_cov = mne.compute_raw_covariance(raw_noise, method="empirical") +# evkoked and renders comparisons to ``baseline`` rather uninformative. +noise_cov = mne.compute_raw_covariance(raw_noise, method="empirical") # %% # Now we assemble the inverse operator, project the data and show the results -# on the `fsaverage` surface, the freesurfer average brain. +# on the ``fsaverage`` surface, the freesurfer average brain. inv_op = mne.minimum_norm.make_inverse_operator(evoked.info, fwd, noise_cov=noise_cov) diff --git a/tutorials/plot_compute_forward.py b/tutorials/plot_compute_forward.py index c64ba99..e26b631 100644 --- a/tutorials/plot_compute_forward.py +++ b/tutorials/plot_compute_forward.py @@ -5,9 +5,10 @@ Compute forward model ===================== -Here we'll first compute a source space, then the bem model -and finally the forward solution. +Here we'll first compute a source space, then the bem model and finally the forward +solution. """ + # Author: Denis A. Engemann # License: BSD 3 clause @@ -18,7 +19,7 @@ import hcp # %% -# We assume our data is inside a designated folder under $HOME. +# We assume our data is inside a designated folder under ``$HOME``. storage_dir = op.expanduser("~/mne-hcp-data") hcp_path = op.join(storage_dir, "HCP") @@ -56,12 +57,12 @@ overwrite=True, ) -# now we morph it onto the subject. +# Now we morph it onto the subject. src_subject = mne.morph_source_spaces(src_fsaverage, subject, subjects_dir=subjects_dir) # %% -# For the same reason `ico` has to be set to `None` when computing the bem. +# For the same reason ``ico`` has to be set to ``None`` when computing the bem. # The headshape is not computed with MNE and has a none standard configuration. bems = mne.make_bem_model( diff --git a/tutorials/plot_reproduce_erf.py b/tutorials/plot_reproduce_erf.py index 4689ed5..8b4858d 100644 --- a/tutorials/plot_reproduce_erf.py +++ b/tutorials/plot_reproduce_erf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ .. _tut_reproduce_erf: @@ -11,6 +10,7 @@ the HCP dat from scratch, then read the preprocessed epochs, finally read the ERF files. Subsequently we will compare these outputs. """ + # Author: Denis A. Engemann # License: BSD 3 clause @@ -100,8 +100,6 @@ raw.annotations = annotations raw.info["bads"].extend(annots["channels"]["all"]) raw.pick_types(meg=True, ref_meg=False) - - # Note: MNE complains on Python 2.7 raw.filter( 0.50, None, method="iir", iir_params=dict(order=4, ftype="butter"), n_jobs=1 ) From a761eba8b1cc480f1e01fd2575a25b3682f500d8 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 17:25:45 +0200 Subject: [PATCH 18/34] fix more stuff --- doc/conf.py | 11 +++++++---- doc/index.rst | 2 +- doc/python_reference.rst | 2 +- hcp/anatomy.py | 6 +++--- hcp/io/file_mapping/file_mapping.py | 11 +++++------ hcp/io/read.py | 14 +++++++------- hcp/preprocessing.py | 8 ++++---- hcp/viz.py | 4 ++-- 8 files changed, 30 insertions(+), 28 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 2a2973c..2c57a7d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -71,7 +71,7 @@ html_static_path = ["_static"] html_theme = "bootstrap" html_title = project - +html_show_sourcelink = False html_theme_options = { 'navbar_sidebarrel': False, 'navbar_links': [ @@ -136,6 +136,8 @@ "MNE", "containers", "file_map", + "sentence", + "block", } # validation @@ -159,6 +161,7 @@ r"\.__iter__", r"\.__div__", r"\.__neg__", + r"\.apply_ref_correction" } # -- sphinx-gallery -------------------------------------------------------------------- @@ -175,10 +178,10 @@ "backreferences_dir": "generated/backreferences", "compress_images": compress_images, "doc_module": (f"{package}",), - "examples_dirs": ["../tutorials"], + "examples_dirs": ["../tutorials", "../examples"], "exclude_implicit_doc": {}, # set - "filename_pattern": r"\d{2}_", - "gallery_dirs": ["generated/tutorials"], + "filename_pattern": r"plot_", + "gallery_dirs": ["generated/tutorials", "generated/examples"], "line_numbers": False, "plot_gallery": "True", # str, to enable overwrite from CLI without warning "reference_url": {f"{package}": None}, diff --git a/doc/index.rst b/doc/index.rst index e776556..14ab03f 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -97,7 +97,7 @@ command should produce the expected layout: $ done When files are downloaded using the -`Amazon webservice tools `_, e.g. `s3rcmd`, all should be +`Amazon webservice tools `_, e.g. ``s3rcmd``, all should be fine. The code is organized by different modules. The ``io`` module includes readers for diff --git a/doc/python_reference.rst b/doc/python_reference.rst index ce499e2..8528bab 100644 --- a/doc/python_reference.rst +++ b/doc/python_reference.rst @@ -8,7 +8,7 @@ Python API Reference This is the functions reference of MNE-HCP. Functions are grouped thematically by analysis purpose. Functions that are not -below a module heading are found in the :py:mod:`hcp` namespace. +below a module heading are found in the ``hcp`` namespace. .. contents:: :local: diff --git a/hcp/anatomy.py b/hcp/anatomy.py index 302a9ab..aab9adc 100644 --- a/hcp/anatomy.py +++ b/hcp/anatomy.py @@ -167,13 +167,13 @@ def compute_forward_stack( Parameters ---------- - subject : str - The subject name. subjects_dir : str The directory containing the extracted HCP subject data. + subject : str + The subject name. recordings_path : str The path where MEG data and transformations are stored. - info_from : tuple of tuples | dict + info_from : tuple of tuple | dict The reader info concerning the data from which sensor positions should be read. Must not be empty room as sensor positions are in head diff --git a/hcp/io/file_mapping/file_mapping.py b/hcp/io/file_mapping/file_mapping.py index 869f255..8c24557 100644 --- a/hcp/io/file_mapping/file_mapping.py +++ b/hcp/io/file_mapping/file_mapping.py @@ -217,8 +217,8 @@ def get_file_paths( * ``'rest'`` * ``'noise'`` * ``'task_motor'`` - * ````'task_story_math'`` - * 'task_working_memory'`` + * ``'task_story_math'`` + * ``'task_working_memory'`` output : str The kind of output. The following ones are supported: @@ -231,10 +231,9 @@ def get_file_paths( * ``'freesurfer'`` * ``'meg_anatomy'`` run_index : int - The run index. For the first run, use 0, for the second, use 1. - Also see HCP documentation for the number of runs for a given data - type. - onset : ``'stim`` | ``'resp'`` | ``'sentence'`` | ``'block'`` + The run index. For the first run, use ``0``, for the second, use ``1``. + Also see HCP documentation for the number of runs for a given data type. + onset : ``'stim'`` | ``'resp'`` | ``'sentence'`` | ``'block'`` The event onset. Only considered for epochs and evoked outputs The mapping is generous, everything that is not a response is a stimulus, in the sense of internal or external events. sentence and diff --git a/hcp/io/read.py b/hcp/io/read.py index 5a670da..5faa62a 100644 --- a/hcp/io/read.py +++ b/hcp/io/read.py @@ -187,7 +187,7 @@ def read_raw(subject, data_type, run_index=0, hcp_path=op.curdir, verbose=None): Parameters ---------- - subject : str, file_map + subject : str | file_map The subject. data_type : str The kind of data to read. The following options are supported: @@ -203,7 +203,7 @@ def read_raw(subject, data_type, run_index=0, hcp_path=op.curdir, verbose=None): Also see HCP documentation for the number of runs for a given data type. hcp_path : str The HCP directory, defaults to ``op.curdir``. - verbose : bool, str, int, or None + verbose : bool | str | int | None If not None, override default verbose level (see mne.verbose). Returns @@ -235,7 +235,7 @@ def read_info(subject, data_type, run_index=0, hcp_path=op.curdir): Parameters ---------- - subject : str, file_map + subject : str | file_map The subject. data_type : str The kind of data to read. The following options are supported: @@ -289,7 +289,7 @@ def read_epochs( Parameters ---------- - subject : str, file_map + subject : str | file_map The subject. data_type : str The kind of data to read. The following options are supported: @@ -472,7 +472,7 @@ def read_annot(subject, data_type, run_index=0, hcp_path=op.curdir): Parameters ---------- - subject : str, file_map + subject : str | file_map The subject. data_type : str The kind of data to read. The following options are supported: @@ -531,7 +531,7 @@ def read_ica(subject, data_type, run_index=0, hcp_path=op.curdir): Parameters ---------- - subject : str, file_map + subject : str | file_map The subject. data_type : str The kind of data to read. The following options are supported: @@ -623,7 +623,7 @@ def read_evokeds( Parameters ---------- - subject : str, file_map + subject : str | file_map The subject. data_type : str The kind of data to read. The following options are supported: diff --git a/hcp/preprocessing.py b/hcp/preprocessing.py index 15a3639..04911bd 100644 --- a/hcp/preprocessing.py +++ b/hcp/preprocessing.py @@ -126,7 +126,7 @@ def map_ch_coords_to_mne(inst): Parameters ---------- - inst : MNE data containers + inst : Raw | Epochs | Evoked Raw, Epochs, Evoked. """ bti_dev_t = Transform("ctf_meg", "meg", _get_bti_dev_t()) @@ -150,9 +150,9 @@ def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="f Parameters ---------- - inst : MNE data containers + inst : Raw | Epochs | Evoked Raw, Epochs, Evoked. - subject : str, file_map + subject : str | file_map The subject. data_type : str The kind of data to read. The following options are supported: @@ -176,7 +176,7 @@ def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="f Returns ------- - out : MNE data containers + out : Raw | Epochs | Evoked Raw, Epochs, Evoked but with missing channels interpolated. """ try: diff --git a/hcp/viz.py b/hcp/viz.py index 4e1d038..2161f84 100644 --- a/hcp/viz.py +++ b/hcp/viz.py @@ -54,14 +54,14 @@ def plot_coregistration( The path where the HCP files can be found. recordings_path : str The path to converted data (including the head<->device transform). - info_from : tuple of tuples | dict + info_from : tuple of tuple | dict The reader info concerning the data from which sensor positions should be read. Must not be empty room as sensor positions are in head coordinates for 4D systems, hence not available in that case. Note that differences between the sensor positions across runs are smaller than 12 digits, hence negligible. - view_init : tuple of tuples | dict + view_init : tuple of tuple | dict The initival view, defaults to azimuth and elevation of 0, a simple lateral view. From 54ea404eaf32a3f6f30a3ebc47651835fca463c3 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 17:33:37 +0200 Subject: [PATCH 19/34] fix stuff --- doc/conf.py | 11 ++++++----- examples/README.rst | 4 ++++ examples/README.txt | 4 ---- tutorials/{README.txt => README.rst} | 0 4 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 examples/README.rst delete mode 100644 examples/README.txt rename tutorials/{README.txt => README.rst} (100%) diff --git a/doc/conf.py b/doc/conf.py index 2c57a7d..4b97b3a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -73,14 +73,15 @@ html_title = project html_show_sourcelink = False html_theme_options = { - 'navbar_sidebarrel': False, - 'navbar_links': [ - ("Examples", "auto_examples/index"), - ("Tutorials", "auto_tutorials/index"), + "navbar_pagenav": False, + "navbar_sidebarrel": False, + "navbar_links": [ + ("Examples", "generated/examples/index"), + ("Tutorials", "generated/tutorials/index"), ("API", "python_reference"), ("GitHub", "https://github.com/mne-tools/mne-hcp", True) ], - 'bootswatch_theme': "cosmo" + "bootswatch_theme": "cosmo" } # -- autosummary ----------------------------------------------------------------------- diff --git a/examples/README.rst b/examples/README.rst new file mode 100644 index 0000000..1a6c9a9 --- /dev/null +++ b/examples/README.rst @@ -0,0 +1,4 @@ +Examples +======== + +Short examples on how to do things with MNE-HCP. diff --git a/examples/README.txt b/examples/README.txt deleted file mode 100644 index 580c206..0000000 --- a/examples/README.txt +++ /dev/null @@ -1,4 +0,0 @@ -Examples -======== - -Short examples on how to do things with MNE hcp. diff --git a/tutorials/README.txt b/tutorials/README.rst similarity index 100% rename from tutorials/README.txt rename to tutorials/README.rst From 8cbb63de890993459a37f1fbf7acbdfb451af0ea Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 17:50:53 +0200 Subject: [PATCH 20/34] clean-up readme --- README.rst | 40 +++++++--------------------------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/README.rst b/README.rst index b872075..0cca0ca 100644 --- a/README.rst +++ b/README.rst @@ -53,32 +53,18 @@ but is getting closer to a stable release. For now please consider the following caveats: - We only intend to support a subset of the files shipped with HCP. - - Specifically, for now it is not planned to support io and processing for any outputs of the HCP source space pipelines. - - This library breaks with some of MNE conventions in order to make the HCP outputs compatible with MNE. + - Specifically, for now it is not planned to support io and processing for any + outputs of the HCP source space pipelines. + - This library breaks with some of MNE conventions in order to make the HCP outputs + compatible with MNE. Installation ============ -We recommend the `Anaconda Python distribution `_, which comes with the necessary dependencies. Alternatively, to install ``mne-hcp``, you first need to install its dependencies:: +We recommend the `Anaconda Python distribution `_, +which comes with the necessary dependencies. You can install ``mne-hcp``, from source:: - $ pip install numpy matplotlib scipy scikit-learn mne joblib pandas - -Then clone the repository:: - - $ git clone http://github.com/mne-tools/mne-hcp - -and finally run `setup.py` to install the package:: - - $ cd mne-hcp/ - $ python setup.py install - -If you do not have admin privileges on the computer, use the ``--user`` flag -with `setup.py`. - -Alternatively, for a devoloper install based on symbolic links (which simplifies keeping up with code changes), do:: - - $ cd mne-hcp/ - $ python setup.py develop + $ pip install git+http://github.com/mne-tools/mne-hcp To check if everything worked fine, you can do:: @@ -86,18 +72,6 @@ To check if everything worked fine, you can do:: and it should not give any error messages. -Dependencies ------------- - -The following main and additional dependencies are required to use MNE-HCP: - - - MNE-Python - - the MNE-Python dependencies, specifically - - scipy - - numpy - - matplotlib - - scikit-learn (optional) - Quickstart ========== From 3df7dd282b759ed5c577b7a0242b36f1acfbfe6b Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 17:54:28 +0200 Subject: [PATCH 21/34] more clean-up --- .codespellignore | 1 + hcp/io/file_mapping/file_mapping.py | 2 +- hcp/io/read.py | 6 +-- hcp/io/tests/test_io.py | 2 +- hcp/tests/config.py | 45 +++++++++------------ hcp/tests/test_anatomy.py | 56 ++++++++++++++----------- hcp/tests/test_preprocessing.py | 63 ++++++++++++++++------------- hcp/tests/test_viz.py | 13 +++--- 8 files changed, 98 insertions(+), 90 deletions(-) diff --git a/.codespellignore b/.codespellignore index e69de29..5f339bb 100644 --- a/.codespellignore +++ b/.codespellignore @@ -0,0 +1 @@ +BU diff --git a/hcp/io/file_mapping/file_mapping.py b/hcp/io/file_mapping/file_mapping.py index 8c24557..07c16d2 100644 --- a/hcp/io/file_mapping/file_mapping.py +++ b/hcp/io/file_mapping/file_mapping.py @@ -252,7 +252,7 @@ def get_file_paths( if data_type not in kind_map: raise ValueError( f"I never heard of `{data_type}` -- are you sure this is a valid HCP type? " - f"I currenlty support:\n{" \n".join([k for k in kind_map if "_" in k])}" + f"I currently support:\n{" \n".join([k for k in kind_map if "_" in k])}" ) context = "rmeg" if "rest" in data_type else "tmeg" sensor_mode = evoked_map["modes"][sensor_mode] diff --git a/hcp/io/read.py b/hcp/io/read.py index 5faa62a..a4c1310 100644 --- a/hcp/io/read.py +++ b/hcp/io/read.py @@ -87,7 +87,7 @@ def _parse_hcp_trans(fid, transforms, convert_to_meter): if "filename" in trans or trans == "\n": continue key, trans = trans.split(" = ") - key = key.lstrip("\ntransform.") + key = key.removeprefix("\ntransform.") transforms[key] = _parse_trans(trans) if convert_to_meter: transforms[key][:3, 3] *= 1e-3 # mm to m @@ -218,7 +218,7 @@ def read_raw(subject, data_type, run_index=0, hcp_path=op.curdir, verbose=None): run_index=run_index, hcp_path=hcp_path, ) - return read_raw_bti( # no convrt + no rename for HCP compatibility + return read_raw_bti( # no convert + no rename for HCP compatibility pdf, config, convert=False, @@ -444,7 +444,7 @@ def _check_sorting_runs(candidates, id_char): def _parse_annotations_segments(segment_strings): - """Read bad segments defintions from text file.""" + """Read bad segments definitions from text file.""" for char in "}]": # multi line array definitions segment_strings = segment_strings.replace(char + ";", "splitme") split = segment_strings.split("splitme") diff --git a/hcp/io/tests/test_io.py b/hcp/io/tests/test_io.py index f48543d..be5843a 100644 --- a/hcp/io/tests/test_io.py +++ b/hcp/io/tests/test_io.py @@ -231,6 +231,6 @@ def test_read_trial_info(): assert_true("stim" in trial_info) assert_true("resp" in trial_info) assert_equal(2, len(trial_info)) - for key, val in trial_info.items(): + for val in trial_info.values(): assert_array_equal(np.ndim(val["comments"]), 1) assert_array_equal(np.ndim(val["codes"]), 2) diff --git a/hcp/tests/config.py b/hcp/tests/config.py index 5433797..dbf1053 100644 --- a/hcp/tests/config.py +++ b/hcp/tests/config.py @@ -4,44 +4,37 @@ import os import os.path as op -hcp_prefix = 's3://hcp-openaccess/HCP_900' -subject = '105923' +hcp_prefix = "s3://hcp-openaccess/HCP_900" +subject = "105923" hcp_data_types = [ - 'rest', - 'task_working_memory', - 'task_story_math', - 'task_motor', - 'noise_empty_room' + "rest", + "task_working_memory", + "task_story_math", + "task_motor", + "noise_empty_room", ] -hcp_outputs = [ - 'raw', - 'epochs', - 'ica', - 'evoked', - 'trial_info', - 'bads' -] +hcp_outputs = ["raw", "epochs", "ica", "evoked", "trial_info", "bads"] -hcp_onsets = ['stim'] +hcp_onsets = ["stim"] # allow for downloading fewer data run_inds = [0, 1, 2] -max_runs = int(os.getenv('MNE_HCP_N_RUNS', 3)) +max_runs = int(os.getenv("MNE_HCP_N_RUNS", 3)) ############################################################################## # variable used in different tests -hcp_path = op.expanduser('~/mne-hcp-data/mne-hcp-testing') +hcp_path = op.expanduser("~/mne-hcp-data/mne-hcp-testing") -bti_chans = {'A' + str(i) for i in range(1, 249, 1)} +bti_chans = {"A" + str(i) for i in range(1, 249, 1)} -test_subject = '105923' -task_types = ['task_story_math', 'task_working_memory', 'task_motor'] -noise_types = ['noise_empty_room'] +test_subject = "105923" +task_types = ["task_story_math", "task_working_memory", "task_motor"] +noise_types = ["noise_empty_room"] test_decim = 100 sfreq_preproc = 508.63 / test_decim sfreq_raw = 2034.5101 @@ -49,10 +42,10 @@ highpass_preproc = 1.3 epochs_bounds = { - 'task_motor': (-1.2, 1.2), - 'task_working_memory': (-1.5, 2.5), - 'task_story_math': (-1.5, 4), - 'rest': (0, 2) + "task_motor": (-1.2, 1.2), + "task_working_memory": (-1.5, 2.5), + "task_story_math": (-1.5, 4), + "rest": (0, 2), } diff --git a/hcp/tests/test_anatomy.py b/hcp/tests/test_anatomy.py index 2b9ea38..6ef6fe8 100644 --- a/hcp/tests/test_anatomy.py +++ b/hcp/tests/test_anatomy.py @@ -1,47 +1,57 @@ import os.path as op import shutil -from nose.tools import assert_true import matplotlib - import mne +from nose.tools import assert_true + +from hcp import compute_forward_stack, make_mne_anatomy from hcp.tests import config as tconf -from hcp import make_mne_anatomy, compute_forward_stack -from hcp.viz import plot_coregistration from hcp.tests.config import expensive_test +from hcp.viz import plot_coregistration -matplotlib.use('Agg') +matplotlib.use("Agg") -hcp_params = dict(hcp_path=tconf.hcp_path, - subject=tconf.test_subject) +hcp_params = dict(hcp_path=tconf.hcp_path, subject=tconf.test_subject) @expensive_test def test_anatomy(): """Test anatomy functions (slow!).""" import matplotlib.pyplot as plt + # This is where the data are after downloading from HCP temp_dir = mne.utils._TempDir() - subjects_dir = op.join(temp_dir, 'hcp-subjects') - recordings_path = op.join(temp_dir, 'hcp-meg') - make_mne_anatomy(recordings_path=recordings_path, - subjects_dir=subjects_dir, **hcp_params) - assert_true(op.isfile(op.join(subjects_dir, hcp_params['subject'], 'bem', - 'inner_skull.surf'))) + subjects_dir = op.join(temp_dir, "hcp-subjects") + recordings_path = op.join(temp_dir, "hcp-meg") + make_mne_anatomy( + recordings_path=recordings_path, subjects_dir=subjects_dir, **hcp_params + ) + assert_true( + op.isfile( + op.join(subjects_dir, hcp_params["subject"], "bem", "inner_skull.surf") + ) + ) # Now we need fsaverage... - mne_subjects_dir = mne.get_config('SUBJECTS_DIR') + mne_subjects_dir = mne.get_config("SUBJECTS_DIR") assert_true(mne_subjects_dir is not None) - shutil.copytree(op.join(mne_subjects_dir, 'fsaverage'), - op.join(subjects_dir, 'fsaverage')) + shutil.copytree( + op.join(mne_subjects_dir, "fsaverage"), op.join(subjects_dir, "fsaverage") + ) compute_forward_stack( - subjects_dir=subjects_dir, recordings_path=recordings_path, - src_params=dict(add_dist=False, spacing='oct1'), - verbose=True, **hcp_params) + subjects_dir=subjects_dir, + recordings_path=recordings_path, + src_params=dict(add_dist=False, spacing="oct1"), + verbose=True, + **hcp_params, + ) # let's do our viz tests, too - plot_coregistration(subjects_dir=subjects_dir, - recordings_path=recordings_path, **hcp_params) - plt.close('all') + plot_coregistration( + subjects_dir=subjects_dir, recordings_path=recordings_path, **hcp_params + ) + plt.close("all") mne.viz.plot_bem(subject=tconf.test_subject, subjects_dir=subjects_dir) - plt.close('all') + plt.close("all") + mne.utils.run_tests_if_main() diff --git a/hcp/tests/test_preprocessing.py b/hcp/tests/test_preprocessing.py index a5c97a3..9524f38 100644 --- a/hcp/tests/test_preprocessing.py +++ b/hcp/tests/test_preprocessing.py @@ -1,23 +1,26 @@ import warnings -from nose.tools import assert_true, assert_raises +import mne import numpy as np +from nose.tools import assert_raises, assert_true from numpy.testing import assert_equal -import mne import hcp +from hcp.preprocessing import ( + apply_ica_hcp, + apply_ref_correction, + interpolate_missing, + map_ch_coords_to_mne, + set_eog_ecg_channels, +) from hcp.tests import config as tconf -from hcp.preprocessing import (apply_ref_correction, interpolate_missing, - map_ch_coords_to_mne, set_eog_ecg_channels, - apply_ica_hcp) -hcp_params = dict(hcp_path=tconf.hcp_path, - subject=tconf.test_subject) +hcp_params = dict(hcp_path=tconf.hcp_path, subject=tconf.test_subject) def test_set_eog_ecg_channels(): """Test setting of EOG and ECG channels.""" - raw = hcp.read_raw(data_type='rest', **hcp_params) + raw = hcp.read_raw(data_type="rest", **hcp_params) raw.crop(0, 1).load_data() assert_equal(len(mne.pick_types(raw.info, meg=False, eog=True)), 0) assert_equal(len(mne.pick_types(raw.info, meg=False, ecg=True)), 13) @@ -29,20 +32,23 @@ def test_set_eog_ecg_channels(): def test_apply_ica(): """Test ICA application.""" - raw = hcp.read_raw(data_type='rest', verbose='error', **hcp_params) - annots = hcp.read_annot(data_type='rest', **hcp_params) + raw = hcp.read_raw(data_type="rest", verbose="error", **hcp_params) + annots = hcp.read_annot(data_type="rest", **hcp_params) # construct MNE annotations - bad_seg = (annots['segments']['all']) / raw.info['sfreq'] + bad_seg = (annots["segments"]["all"]) / raw.info["sfreq"] annotations = mne.Annotations( - bad_seg[:, 0], (bad_seg[:, 1] - bad_seg[:, 0]), description='bad') + bad_seg[:, 0], (bad_seg[:, 1] - bad_seg[:, 0]), description="bad" + ) raw.annotations = annotations - raw.info['bads'].extend(annots['channels']['all']) - ica_mat = hcp.read_ica(data_type='rest', **hcp_params) - exclude = [ii for ii in range(annots['ica']['total_ic_number'][0]) - if ii not in annots['ica']['brain_ic_vs']] - assert_raises(RuntimeError, apply_ica_hcp, raw, ica_mat=ica_mat, - exclude=exclude) + raw.info["bads"].extend(annots["channels"]["all"]) + ica_mat = hcp.read_ica(data_type="rest", **hcp_params) + exclude = [ + ii + for ii in range(annots["ica"]["total_ic_number"][0]) + if ii not in annots["ica"]["brain_ic_vs"] + ] + assert_raises(RuntimeError, apply_ica_hcp, raw, ica_mat=ica_mat, exclude=exclude) # XXX right now this is just a smoke test, should really check some # values... with warnings.catch_warnings(record=True): @@ -52,7 +58,7 @@ def test_apply_ica(): def test_apply_ref_correction(): """Test reference correction.""" - raw = hcp.read_raw(data_type='rest', run_index=0, **hcp_params) + raw = hcp.read_raw(data_type="rest", run_index=0, **hcp_params) # raw.crop(0, 10).load_data() raw.load_data() # XXX terrible hack to have more samples. @@ -67,26 +73,24 @@ def test_apply_ref_correction(): def test_map_ch_coords_to_mne(): """Test mapping of channel coords to MNE.""" - data_type = 'task_working_memory' - hcp_evokeds = hcp.read_evokeds(onset='stim', data_type=data_type, - **hcp_params) + data_type = "task_working_memory" + hcp_evokeds = hcp.read_evokeds(onset="stim", data_type=data_type, **hcp_params) for evoked in hcp_evokeds: - if evoked.comment == 'Wrkmem_LM-TIM-face_BT-diff_MODE-mag': + if evoked.comment == "Wrkmem_LM-TIM-face_BT-diff_MODE-mag": break - old_coord = evoked.info['chs'][0]['loc'] + old_coord = evoked.info["chs"][0]["loc"] map_ch_coords_to_mne(evoked) - new_coord = evoked.info['chs'][0]['loc'] + new_coord = evoked.info["chs"][0]["loc"] assert_true((old_coord != new_coord).any()) def test_interpolate_missing(): """Test interpolation of missing channels.""" - data_type = 'task_working_memory' - raw = hcp.read_raw(data_type='task_working_memory', run_index=0, - **hcp_params) + data_type = "task_working_memory" + raw = hcp.read_raw(data_type="task_working_memory", run_index=0, **hcp_params) raw.load_data() n_chan = len(raw.ch_names) - raw.drop_channels(['A1']) + raw.drop_channels(["A1"]) assert_equal(len(raw.ch_names), n_chan - 1) raw = interpolate_missing(raw, data_type=data_type, **hcp_params) assert_equal(len(raw.ch_names), n_chan) @@ -96,4 +100,5 @@ def test_interpolate_missing(): evoked_int = interpolate_missing(evoked, data_type=data_type, **hcp_params) assert_equal(len(evoked_int.ch_names), 248) + mne.utils.run_tests_if_main() diff --git a/hcp/tests/test_viz.py b/hcp/tests/test_viz.py index 9897c0e..189d000 100644 --- a/hcp/tests/test_viz.py +++ b/hcp/tests/test_viz.py @@ -1,23 +1,22 @@ -from numpy.testing import assert_equal import matplotlib - import mne +from numpy.testing import assert_equal + import hcp from hcp.tests import config as tconf from hcp.viz import make_hcp_bti_layout -matplotlib.use('Agg') +matplotlib.use("Agg") -hcp_params = dict(hcp_path=tconf.hcp_path, - subject=tconf.test_subject) +hcp_params = dict(hcp_path=tconf.hcp_path, subject=tconf.test_subject) def test_make_layout(): """Test making a layout.""" - raw = hcp.read_raw(data_type='rest', **hcp_params).crop(0, 1).load_data() + raw = hcp.read_raw(data_type="rest", **hcp_params).crop(0, 1).load_data() raw.pick_types() lout = make_hcp_bti_layout(raw.info) - assert_equal(lout.names, raw.info['ch_names']) + assert_equal(lout.names, raw.info["ch_names"]) mne.utils.run_tests_if_main() From c0bfacf13bec89721806ceb915a500fda59d6101 Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 18:08:00 +0200 Subject: [PATCH 22/34] more fixes --- doc/conf.py | 1 - examples/make_mne_anatomy.py | 2 +- examples/plot_evoked_data.py | 2 +- hcp/anatomy.py | 4 +-- hcp/io/file_mapping/file_mapping.py | 4 +-- hcp/io/read.py | 26 +++++++++---------- hcp/tests/config.py | 4 +-- pyproject.toml | 8 +++--- .../plot_temporal_searchlight_decoding.py | 2 +- 9 files changed, 26 insertions(+), 27 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 4b97b3a..9681412 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -20,7 +20,6 @@ copyright = f"{date.today().year}, {author}" release = hcp.__version__ package = hcp.__name__ -gh_url = "https://github.com/mscheltienne/template-python" # -- general configuration ------------------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/examples/make_mne_anatomy.py b/examples/make_mne_anatomy.py index 2a5f08e..e3ef3c1 100644 --- a/examples/make_mne_anatomy.py +++ b/examples/make_mne_anatomy.py @@ -7,7 +7,7 @@ The HCP anatomy data needed for MNE based source analysis Live in different places and subfolders. -MNE-HCP has a convenience function that exctracts the required +MNE-HCP has a convenience function that extracts the required files and information and creates a "subjects_dir" as known from MNE and freesurfer. diff --git a/examples/plot_evoked_data.py b/examples/plot_evoked_data.py index e88b39a..33751a6 100644 --- a/examples/plot_evoked_data.py +++ b/examples/plot_evoked_data.py @@ -5,7 +5,7 @@ Visualize evoked data ===================== -Here we'll generate some attention for pecularities of visualizing the +Here we'll generate some attention for peculiarities of visualizing the HCP evoked outputs using MNE plotting functions. """ # Author: Denis A. Engemann diff --git a/hcp/anatomy.py b/hcp/anatomy.py index aab9adc..5bb4584 100644 --- a/hcp/anatomy.py +++ b/hcp/anatomy.py @@ -157,7 +157,7 @@ def compute_forward_stack( n_jobs=1, verbose=None, ): - """Convenience function for conducting standard MNE analyses. + """Conduct standard MNE analyses. .. note:: @@ -276,7 +276,7 @@ def compute_forward_stack( def _update_dict_defaults(values, defaults): - """Helper to handle dict updates""" + """Helper to handle dict updates.""" out = {k: v for k, v in defaults.items()} if isinstance(values, dict): out.update(values) diff --git a/hcp/io/file_mapping/file_mapping.py b/hcp/io/file_mapping/file_mapping.py index 07c16d2..da02262 100644 --- a/hcp/io/file_mapping/file_mapping.py +++ b/hcp/io/file_mapping/file_mapping.py @@ -171,7 +171,7 @@ def _map_onset(onset, data_type, output): - """Helper to resolve stim and resp according to context""" + """Helper to resolve stim and resp according to context.""" out = onset if data_type == "task_working_memory": out = {"stim": "TIM", "resp": "TRESP"}[onset] @@ -187,7 +187,7 @@ def _map_onset(onset, data_type, output): def _map_diff_mode(condition, data_type): - """Helper to resolve diff mode according to context""" + """Helper to resolve diff mode according to context.""" diff_mode = "[BT-diff]" if "versus" in condition: diff_mode = "[OP-diff]_[BT-diff]" diff --git a/hcp/io/read.py b/hcp/io/read.py index a4c1310..721292c 100644 --- a/hcp/io/read.py +++ b/hcp/io/read.py @@ -74,14 +74,14 @@ def _parse_trans(string): - """helper to parse transforms""" + """Helper to parse transforms.""" return np.array( string.replace("\n", "").strip("[] ").split(" "), dtype=float ).reshape(4, 4) def _parse_hcp_trans(fid, transforms, convert_to_meter): - """ "another helper""" + """Another helper.""" contents = fid.read() for trans in contents.split(";"): if "filename" in trans or trans == "\n": @@ -96,7 +96,8 @@ def _parse_hcp_trans(fid, transforms, convert_to_meter): def _read_trans_hcp(fname, convert_to_meter): - """Read + parse transforms + """Read and parse transforms. + subject_MEG_anatomy_transform.txt """ transforms = dict() @@ -106,7 +107,7 @@ def _read_trans_hcp(fname, convert_to_meter): def _read_landmarks_hcp(fname): - """XXX parse landmarks currently not used""" + """XXX parse landmarks currently not used.""" out = dict() with open(fname) as fid: for line in fid: @@ -124,7 +125,7 @@ def _read_landmarks_hcp(fname): def _get_head_model(head_model_fname): - """helper to parse head model from matfile""" + """Helper to parse head model from matfile.""" head_mat = scio.loadmat(head_model_fname, squeeze_me=False) pnts = head_mat["headmodel"]["bnd"][0][0][0][0][0] faces = head_mat["headmodel"]["bnd"][0][0][0][0][1] @@ -133,7 +134,7 @@ def _get_head_model(head_model_fname): def _read_bti_info(raw_fid, config): - """helper to only access bti info from pdf file""" + """Helper to only access bti info from pdf file.""" info, bti_info = _get_bti_info( pdf_fname=raw_fid, config_fname=config, @@ -150,7 +151,7 @@ def _read_bti_info(raw_fid, config): def _check_raw_config_runs(raws, configs): - """XXX this goes to tests later, currently not used""" + """XXX this goes to tests later, currently not used.""" for raw, config in zip(raws, configs): assert op.split(raw)[0] == op.split(config)[0] run_str = set([configs[0].split("/")[-3]]) @@ -159,7 +160,7 @@ def _check_raw_config_runs(raws, configs): def _check_infos_trans(infos): - """XXX this goes to tests later, currently not used""" + """XXX this goes to tests later, currently not used.""" chan_max_idx = np.argmax([c["nchan"] for c in infos]) chan_template = infos[chan_max_idx]["ch_names"] channels = [c["ch_names"] for c in infos] @@ -344,7 +345,7 @@ def read_epochs( def _read_epochs(epochs_mat_fname, info, return_fixations_motor): - """read the epochs from matfile""" + """Read the epochs from matfile.""" data = scio.loadmat(epochs_mat_fname, squeeze_me=True)["data"] ch_names = [ch for ch in data["label"].tolist()] info["sfreq"] = data["fsample"].tolist() @@ -404,7 +405,6 @@ def read_trial_info(subject, data_type, run_index=0, hcp_path=op.curdir): trial_info : dict The trial info including event labels, indices and times. """ - trial_info_mat_fname = get_file_paths( subject=subject, data_type=data_type, @@ -418,7 +418,7 @@ def read_trial_info(subject, data_type, run_index=0, hcp_path=op.curdir): def _read_trial_info(trial_info_mat_fname): - """helper to read trial info""" + """Helper to read trial info.""" # XXX FIXME index -1 data = scio.loadmat(trial_info_mat_fname, squeeze_me=True)["trlInfo"] out = dict() @@ -434,7 +434,7 @@ def _read_trial_info(trial_info_mat_fname): def _check_sorting_runs(candidates, id_char): - """helper to ensure correct run-parsing and mapping""" + """Helper to ensure correct run-parsing and mapping.""" run_idx = [f.find(id_char) for f in candidates] for config, idx in zip(candidates, run_idx): assert config[idx - 1].isdigit() @@ -563,7 +563,7 @@ def read_ica(subject, data_type, run_index=0, hcp_path=op.curdir): def _parse_annotations_bad_channels(bads_strings): - """Read bad channel definitions from text file""" + """Read bad channel definitions from text file.""" for char in "}]": bads_strings = bads_strings.replace(char + ";", "splitme") split = bads_strings.split("splitme") diff --git a/hcp/tests/config.py b/hcp/tests/config.py index dbf1053..99303d3 100644 --- a/hcp/tests/config.py +++ b/hcp/tests/config.py @@ -50,13 +50,13 @@ def nottest(f): - """Decorator to mark a function as not a test""" + """Decorator to mark a function as not a test.""" f.__test__ = False return f @nottest def expensive_test(f): - """Decorator for expensive testing""" + """Decorator for expensive testing.""" f.expensive_test = True return f diff --git a/pyproject.toml b/pyproject.toml index 438721a..6e97f4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,9 +27,9 @@ dependencies = [ ] description = 'MNE HCP project for accessing the human connectome MEG data in Python.' keywords = [ - 'python', - 'mne', 'hcp', + 'mne', + 'python', ] license = {file = 'LICENSE'} maintainers = [ @@ -65,7 +65,7 @@ doc = [ 'sphinxcontrib-bibtex', ] full = [ - 'template[all]', + 'mne-hcp[all]', ] style = [ 'codespell[toml]>=2.2.4', @@ -121,7 +121,7 @@ profile = 'black' py_version = 39 [tool.pydocstyle] -add_ignore = 'D100,D104,D107' +add_ignore = 'D100,D104,D107,D401' convention = 'numpy' ignore-decorators = '(copy_doc|property|.*setter|.*getter|pyqtSlot|Slot)' match = '^(?!__init__|test_).*\.py' diff --git a/tutorials/plot_temporal_searchlight_decoding.py b/tutorials/plot_temporal_searchlight_decoding.py index c5c05c2..58ac573 100644 --- a/tutorials/plot_temporal_searchlight_decoding.py +++ b/tutorials/plot_temporal_searchlight_decoding.py @@ -84,6 +84,6 @@ # gat.score(epochs) ############################################################################## -# Ploting the temporal connectome and the evolution of discriminability. +# Plotting the temporal connectome and the evolution of discriminability. # gat.plot() # gat.plot_diagonal() From 370c487be36624fab4bb120d9c0031048d22160f Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Mon, 22 Apr 2024 18:13:02 +0200 Subject: [PATCH 23/34] fix and rm codespell that is misbehaving --- .codespellignore | 1 - .pre-commit-config.yaml | 7 ------- hcp/io/file_mapping/file_mapping.py | 2 +- pyproject.toml | 7 ------- 4 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 .codespellignore diff --git a/.codespellignore b/.codespellignore deleted file mode 100644 index 5f339bb..0000000 --- a/.codespellignore +++ /dev/null @@ -1 +0,0 @@ -BU diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d3bd494..d8aa233 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,13 +16,6 @@ repos: name: ruff formatter files: hcp - - repo: https://github.com/codespell-project/codespell - rev: v2.2.6 - hooks: - - id: codespell - args: [--write-changes] - additional_dependencies: [tomli] - - repo: https://github.com/pycqa/pydocstyle rev: 6.3.0 hooks: diff --git a/hcp/io/file_mapping/file_mapping.py b/hcp/io/file_mapping/file_mapping.py index da02262..c9503b1 100644 --- a/hcp/io/file_mapping/file_mapping.py +++ b/hcp/io/file_mapping/file_mapping.py @@ -252,7 +252,7 @@ def get_file_paths( if data_type not in kind_map: raise ValueError( f"I never heard of `{data_type}` -- are you sure this is a valid HCP type? " - f"I currently support:\n{" \n".join([k for k in kind_map if "_" in k])}" + f"I currently support:\n{r', '.join([k for k in kind_map if '_' in k])}" ) context = "rmeg" if "rest" in data_type else "tmeg" sensor_mode = evoked_map["modes"][sensor_mode] diff --git a/pyproject.toml b/pyproject.toml index 6e97f4b..043933c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,6 @@ full = [ 'mne-hcp[all]', ] style = [ - 'codespell[toml]>=2.2.4', 'isort', 'pydocstyle[toml]', 'ruff>=0.1.8', @@ -86,12 +85,6 @@ documentation = 'https://mne.tools/mne-hcp/' source = 'http://github.com/mne-tools/mne-hcp' tracker = 'http://github.com/mne-tools/mne-hcp/issues' -[tool.codespell] -check-filenames = true -check-hidden = true -ignore-words = '.codespellignore' -skip = 'build,.git,.mypy_cache,.pytest_cache' - [tool.coverage.report] exclude_lines = [ 'if __name__ == .__main__.:', From 5199dbc90e4cc73ee289a498619cf80a9d26b7ba Mon Sep 17 00:00:00 2001 From: Mathieu Scheltienne Date: Mon, 22 Apr 2024 19:39:33 +0200 Subject: [PATCH 24/34] add back codespell --- .codespellignore | 1 + .pre-commit-config.yaml | 7 +++++++ pyproject.toml | 7 +++++++ 3 files changed, 15 insertions(+) create mode 100644 .codespellignore diff --git a/.codespellignore b/.codespellignore new file mode 100644 index 0000000..7e6edb8 --- /dev/null +++ b/.codespellignore @@ -0,0 +1 @@ +bu diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d8aa233..d3bd494 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,13 @@ repos: name: ruff formatter files: hcp + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + args: [--write-changes] + additional_dependencies: [tomli] + - repo: https://github.com/pycqa/pydocstyle rev: 6.3.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 043933c..0705ca5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ full = [ 'mne-hcp[all]', ] style = [ + 'codespell[toml]>=2.2.4', 'isort', 'pydocstyle[toml]', 'ruff>=0.1.8', @@ -85,6 +86,12 @@ documentation = 'https://mne.tools/mne-hcp/' source = 'http://github.com/mne-tools/mne-hcp' tracker = 'http://github.com/mne-tools/mne-hcp/issues' +[tool.codespell] +check-filenames = true +check-hidden = true +ignore-words = '.codespellignore' +skip = 'build,.git,.mypy_cache,.pytest_cache,.venv' + [tool.coverage.report] exclude_lines = [ 'if __name__ == .__main__.:', From d19f34605b0e5097bb2d7c3b3732c5f1521866b4 Mon Sep 17 00:00:00 2001 From: Mathieu Scheltienne Date: Mon, 22 Apr 2024 19:48:38 +0200 Subject: [PATCH 25/34] more fixes --- doc/conf.py | 2 -- examples/plot_evoked_data.py | 2 +- hcp/io/read.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 9681412..950b446 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -136,8 +136,6 @@ "MNE", "containers", "file_map", - "sentence", - "block", } # validation diff --git a/examples/plot_evoked_data.py b/examples/plot_evoked_data.py index 33751a6..690273c 100644 --- a/examples/plot_evoked_data.py +++ b/examples/plot_evoked_data.py @@ -47,5 +47,5 @@ evoked_viz.plot_joint() # %% -# For subsequent analyses we would use `evoked` not `evoked_viz`. +# For subsequent analyses we would use ``evoked`` not ``evoked_viz``. # See also :ref:`tut_compute_inverse_erf` to see how the story continues. diff --git a/hcp/io/read.py b/hcp/io/read.py index 721292c..e503cd5 100644 --- a/hcp/io/read.py +++ b/hcp/io/read.py @@ -302,7 +302,7 @@ def read_epochs( onset : ``'stim'`` | ``'resp'`` | ``'sentence'`` | ``'block'`` The event onset. The mapping is generous, everything that is not a response is a stimulus, in the sense of internal or external events. - `sentence` and `block` are specific to task_story_math. + ``sentence`` and ``block`` are specific to ``task_story_math``. run_index : int The run index. For the first run, use ``0``, for the second, use ``1``. Also see HCP documentation for the number of runs for a given data From 5005321ec7a064c439d5ad77687b3c309564023b Mon Sep 17 00:00:00 2001 From: Mathieu Scheltienne Date: Mon, 22 Apr 2024 19:50:41 +0200 Subject: [PATCH 26/34] fix backref --- doc/_templates/function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/_templates/function.rst b/doc/_templates/function.rst index bdde242..2ae53dc 100644 --- a/doc/_templates/function.rst +++ b/doc/_templates/function.rst @@ -5,7 +5,7 @@ .. autofunction:: {{ objname }} -.. include:: {{module}}.{{objname}}.examples +.. include:: backreferences/{{module}}.{{objname}}.examples .. raw:: html From 02a88b3c1f70dd429d813b685818c7a22d44b3fe Mon Sep 17 00:00:00 2001 From: mscheltienne Date: Tue, 23 Apr 2024 10:03:43 +0200 Subject: [PATCH 27/34] fix ruff --- tutorials/plot_temporal_searchlight_decoding.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tutorials/plot_temporal_searchlight_decoding.py b/tutorials/plot_temporal_searchlight_decoding.py index 58ac573..ea46dd6 100644 --- a/tutorials/plot_temporal_searchlight_decoding.py +++ b/tutorials/plot_temporal_searchlight_decoding.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ .. _tut_searchlight_decoding: From 3c49713711a20cc590d7a74f480e9635a16b1425 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 23 Jul 2024 16:32:37 -0400 Subject: [PATCH 28/34] FIX: Eradicate nose --- .gitignore | 1 - .../file_mapping/tests/test_file_mapping.py | 3 ++- hcp/io/tests/test_io.py | 23 ++++++++++--------- hcp/tests/test_anatomy.py | 8 ++----- hcp/tests/test_preprocessing.py | 9 +++----- hcp/tests/test_viz.py | 3 --- 6 files changed, 19 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index f4c6b1c..da4a42e 100644 --- a/.gitignore +++ b/.gitignore @@ -45,7 +45,6 @@ htmlcov/ .coverage .coverage.* .cache -nosetests.xml coverage.xml *.cover .hypothesis/ diff --git a/hcp/io/file_mapping/tests/test_file_mapping.py b/hcp/io/file_mapping/tests/test_file_mapping.py index ae3e462..effba60 100644 --- a/hcp/io/file_mapping/tests/test_file_mapping.py +++ b/hcp/io/file_mapping/tests/test_file_mapping.py @@ -1,4 +1,5 @@ -from nose.tools import assert_equal, assert_raises +from pytest import raises as assert_raises +from numpy.testing import assert_equal import hcp.tests.config as tconf from hcp.io.file_mapping import get_file_paths diff --git a/hcp/io/tests/test_io.py b/hcp/io/tests/test_io.py index be5843a..f7300ae 100644 --- a/hcp/io/tests/test_io.py +++ b/hcp/io/tests/test_io.py @@ -5,7 +5,8 @@ import mne import numpy as np from mne.utils import _TempDir -from nose.tools import assert_equal, assert_raises, assert_true +from numpy.testing import assert_equal +from pytest import raises as assert_raises from numpy.testing import assert_array_equal import hcp @@ -26,7 +27,7 @@ def test_read_annot(): ) for channels in annots["channels"].values(): for chan in channels: - assert_true(chan in tconf.bti_chans) + assert chan in tconf.bti_chans # segments assert_equal( @@ -46,8 +47,8 @@ def test_read_annot(): ) for components in annots["ica"].values(): if len(components) > 0: - assert_true(min(components) >= 0) - assert_true(max(components) <= 248) + assert min(components) >= 0 + assert max(components) <= 248 def _basic_raw_checks(raw): @@ -55,9 +56,9 @@ def _basic_raw_checks(raw): picks = mne.pick_types(raw.info, meg=True, ref_meg=False) assert_equal(len(picks), 248) ch_names = [raw.ch_names[pp] for pp in picks] - assert_true(all(ch.startswith("A") for ch in ch_names)) + assert all(ch.startswith("A") for ch in ch_names) ch_sorted = list(sorted(ch_names)) - assert_true(ch_sorted != ch_names) + assert ch_sorted != ch_names assert_equal(np.round(raw.info["sfreq"], 4), tconf.sfreq_raw) @@ -109,7 +110,7 @@ def _epochs_basic_checks(epochs, annots, data_type): assert_equal(len(epochs.ch_names), n_good) assert_equal(round(epochs.info["sfreq"], 3), round(tconf.sfreq_preproc, 3)) assert_array_equal(np.unique(epochs.events[:, 2]), np.array([99], dtype=np.int)) - assert_true( + assert ( _check_bounds( epochs.times, tconf.epochs_bounds[data_type], @@ -178,7 +179,7 @@ def test_read_evoked(): n_chans += 4 n_chans -= len(set(sum([an["channels"]["all"] for an in all_annots], []))) assert_equal(n_chans, len(evokeds[0].ch_names)) - assert_true(_check_bounds(evokeds[0].times, tconf.epochs_bounds[data_type])) + assert _check_bounds(evokeds[0].times, tconf.epochs_bounds[data_type]) def test_read_info(): @@ -212,7 +213,7 @@ def test_read_info(): hcp_path=tempdir, run_index=run_index, ) - assert_true(len(info["chs"]) != len(info2["chs"])) + assert len(info["chs"]) != len(info2["chs"]) common_chs = [ch for ch in info2["ch_names"] if ch in info["ch_names"]] assert_equal(len(common_chs), len(info["chs"])) info2 = _hcp_pick_info(info2, common_chs) @@ -228,8 +229,8 @@ def test_read_trial_info(): trial_info = hcp.read_trial_info( data_type=data_type, run_index=run_index, **hcp_params ) - assert_true("stim" in trial_info) - assert_true("resp" in trial_info) + assert "stim" in trial_info + assert "resp" in trial_info assert_equal(2, len(trial_info)) for val in trial_info.values(): assert_array_equal(np.ndim(val["comments"]), 1) diff --git a/hcp/tests/test_anatomy.py b/hcp/tests/test_anatomy.py index 6ef6fe8..00f602e 100644 --- a/hcp/tests/test_anatomy.py +++ b/hcp/tests/test_anatomy.py @@ -3,7 +3,6 @@ import matplotlib import mne -from nose.tools import assert_true from hcp import compute_forward_stack, make_mne_anatomy from hcp.tests import config as tconf @@ -27,14 +26,14 @@ def test_anatomy(): make_mne_anatomy( recordings_path=recordings_path, subjects_dir=subjects_dir, **hcp_params ) - assert_true( + assert ( op.isfile( op.join(subjects_dir, hcp_params["subject"], "bem", "inner_skull.surf") ) ) # Now we need fsaverage... mne_subjects_dir = mne.get_config("SUBJECTS_DIR") - assert_true(mne_subjects_dir is not None) + assert mne_subjects_dir is not None shutil.copytree( op.join(mne_subjects_dir, "fsaverage"), op.join(subjects_dir, "fsaverage") ) @@ -52,6 +51,3 @@ def test_anatomy(): plt.close("all") mne.viz.plot_bem(subject=tconf.test_subject, subjects_dir=subjects_dir) plt.close("all") - - -mne.utils.run_tests_if_main() diff --git a/hcp/tests/test_preprocessing.py b/hcp/tests/test_preprocessing.py index 9524f38..3af51be 100644 --- a/hcp/tests/test_preprocessing.py +++ b/hcp/tests/test_preprocessing.py @@ -2,7 +2,7 @@ import mne import numpy as np -from nose.tools import assert_raises, assert_true +from pytest import raises as assert_raises from numpy.testing import assert_equal import hcp @@ -68,7 +68,7 @@ def test_apply_ref_correction(): orig = raw[meg_picks[0]][0][0] apply_ref_correction(raw) proc = raw[meg_picks[0]][0][0] - assert_true(np.linalg.norm(orig) > np.linalg.norm(proc)) + assert np.linalg.norm(orig) > np.linalg.norm(proc) def test_map_ch_coords_to_mne(): @@ -81,7 +81,7 @@ def test_map_ch_coords_to_mne(): old_coord = evoked.info["chs"][0]["loc"] map_ch_coords_to_mne(evoked) new_coord = evoked.info["chs"][0]["loc"] - assert_true((old_coord != new_coord).any()) + assert (old_coord != new_coord).any() def test_interpolate_missing(): @@ -99,6 +99,3 @@ def test_interpolate_missing(): assert_equal(len(evoked.ch_names), 243) evoked_int = interpolate_missing(evoked, data_type=data_type, **hcp_params) assert_equal(len(evoked_int.ch_names), 248) - - -mne.utils.run_tests_if_main() diff --git a/hcp/tests/test_viz.py b/hcp/tests/test_viz.py index 189d000..44406ca 100644 --- a/hcp/tests/test_viz.py +++ b/hcp/tests/test_viz.py @@ -17,6 +17,3 @@ def test_make_layout(): raw.pick_types() lout = make_hcp_bti_layout(raw.info) assert_equal(lout.names, raw.info["ch_names"]) - - -mne.utils.run_tests_if_main() From 40de5b037b8932cc4512eec2f7d4f4ef85ce62ba Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 23 Jul 2024 16:37:09 -0400 Subject: [PATCH 29/34] TST: Ping Travis From f6450d279b529d05b0232391a5a9ed9be6671c24 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 23 Jul 2024 17:41:34 -0400 Subject: [PATCH 30/34] TST: Fix test --- hcp/tests/test_viz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hcp/tests/test_viz.py b/hcp/tests/test_viz.py index 44406ca..fe8000d 100644 --- a/hcp/tests/test_viz.py +++ b/hcp/tests/test_viz.py @@ -14,6 +14,6 @@ def test_make_layout(): """Test making a layout.""" raw = hcp.read_raw(data_type="rest", **hcp_params).crop(0, 1).load_data() - raw.pick_types() + raw.pick_types(meg=True) lout = make_hcp_bti_layout(raw.info) assert_equal(lout.names, raw.info["ch_names"]) From d47b4d89c62dec70e3fa43535d5718e22e80a1ab Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 23 Jul 2024 18:26:12 -0400 Subject: [PATCH 31/34] FIX: Fixes --- README.rst | 13 +++++++- hcp/anatomy.py | 9 ++---- hcp/io/read.py | 8 +++-- hcp/io/tests/test_io.py | 5 ++-- hcp/preprocessing.py | 7 +++-- hcp/tests/config.py | 2 +- hcp/tests/test_preprocessing.py | 5 +++- tools/get_testing_data.sh | 53 +++++++++++++++++++++++++++++++++ 8 files changed, 84 insertions(+), 18 deletions(-) create mode 100755 tools/get_testing_data.sh diff --git a/README.rst b/README.rst index 0cca0ca..3c8d114 100644 --- a/README.rst +++ b/README.rst @@ -88,7 +88,18 @@ command should produce the expected layout: $ done When files are downloaded using the `Amazon webservice tools `_, e.g. `s3rcmd`, -all should be fine. +all should be fine. For example, you can use this to get the data for subject 105923, +register with HCP, get your access and secret keys, and create a ``~/.s3cfg`` with: + +.. code-block:: ini + + [default] + access_key = + secret_key = + +Then commands like ``s3cmd sync s3://hcp-openaccess/HCP/105923/`` should list available +files, and you can use for example ``s3cmd sync`` to get specific folders or files. +For more ideas, see `the HCP docs `__. The code is organized by different modules. The `io` module includes readers for sensor space data at different processing diff --git a/hcp/anatomy.py b/hcp/anatomy.py index 5bb4584..515d1b4 100644 --- a/hcp/anatomy.py +++ b/hcp/anatomy.py @@ -224,13 +224,8 @@ def compute_forward_stack( subjects_dir=subjects_dir, add_dist=True, ) - if "fname" in mne.fixes._get_args(mne.setup_source_space): - # needed for mne-0.14 and below - src_defaults.update(dict(fname=None)) - else: - # remove 'fname' argument (if necessary) when using mne-0.15+ - if "fname" in src_params: - del src_params["fname"] + if "fname" in src_params: + del src_params["fname"] src_params = _update_dict_defaults(src_params, src_defaults) add_source_space_distances = False diff --git a/hcp/io/read.py b/hcp/io/read.py index e503cd5..d0312c5 100644 --- a/hcp/io/read.py +++ b/hcp/io/read.py @@ -348,7 +348,8 @@ def _read_epochs(epochs_mat_fname, info, return_fixations_motor): """Read the epochs from matfile.""" data = scio.loadmat(epochs_mat_fname, squeeze_me=True)["data"] ch_names = [ch for ch in data["label"].tolist()] - info["sfreq"] = data["fsample"].tolist() + with info._unlock(): + info["sfreq"] = data["fsample"].tolist() times = data["time"].tolist()[0] # deal with different event lengths if return_fixations_motor is not None: @@ -361,7 +362,7 @@ def _read_epochs(epochs_mat_fname, info, return_fixations_motor): # warning: data are not chronologically ordered but # match the trial info - events = np.zeros((len(data), 3), dtype=np.int) + events = np.zeros((len(data), 3), dtype=np.int64) events[:, 0] = np.arange(len(data)) events[:, 2] = 99 # all events # we leave it to the user to construct his events @@ -674,7 +675,8 @@ def _read_evoked(fname, sensor_mode, info, kind): sfreq = 1.0 / np.diff(times)[0] info = _hcp_pick_info(info, ch_names) - info["sfreq"] = sfreq + with info._unlock(): + info["sfreq"] = sfreq out = list() comment = ( diff --git a/hcp/io/tests/test_io.py b/hcp/io/tests/test_io.py index f7300ae..35c431f 100644 --- a/hcp/io/tests/test_io.py +++ b/hcp/io/tests/test_io.py @@ -4,6 +4,7 @@ import mne import numpy as np +from numpy.testing import assert_allclose from mne.utils import _TempDir from numpy.testing import assert_equal from pytest import raises as assert_raises @@ -108,8 +109,8 @@ def _epochs_basic_checks(epochs, annots, data_type): if data_type == "task_motor": n_good += 4 assert_equal(len(epochs.ch_names), n_good) - assert_equal(round(epochs.info["sfreq"], 3), round(tconf.sfreq_preproc, 3)) - assert_array_equal(np.unique(epochs.events[:, 2]), np.array([99], dtype=np.int)) + assert_allclose(epochs.info["sfreq"], tconf.sfreq_preproc, rtol=1e-4) + assert_array_equal(np.unique(epochs.events[:, 2]), np.array([99], dtype=np.int64)) assert ( _check_bounds( epochs.times, diff --git a/hcp/preprocessing.py b/hcp/preprocessing.py index 04911bd..72410e7 100644 --- a/hcp/preprocessing.py +++ b/hcp/preprocessing.py @@ -222,7 +222,8 @@ def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="f ii for ii, ch in enumerate(info["ch_names"]) if ch in inst.ch_names ] - info["sfreq"] = inst.info["sfreq"] + with info._unlock(): + info["sfreq"] = inst.info["sfreq"] # compute shape of data to be added is_raw = isinstance(inst, (mne.io.Raw, mne.io.RawArray, mne.io.bti.bti.RawBTi)) @@ -245,7 +246,7 @@ def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="f if is_raw: out = mne.io.RawArray(out_data, info) if inst.annotations is not None: - out.annotations = inst.annotations + out.set_annotations(inst.annotations) elif is_epochs: out = mne.EpochsArray( data=np.transpose(out_data, (1, 0, 2)), @@ -268,5 +269,5 @@ def interpolate_missing(inst, subject, data_type, hcp_path, run_index=0, mode="f # set "bad" channels and interpolate. out.info["bads"] = bti_meg_channel_missing_names - out.interpolate_bads(mode=mode) + out.interpolate_bads(mode=mode, origin=(0., 0., 0.04)) return out diff --git a/hcp/tests/config.py b/hcp/tests/config.py index 99303d3..f67714b 100644 --- a/hcp/tests/config.py +++ b/hcp/tests/config.py @@ -35,7 +35,7 @@ test_subject = "105923" task_types = ["task_story_math", "task_working_memory", "task_motor"] noise_types = ["noise_empty_room"] -test_decim = 100 +test_decim = int(os.getenv("MNE_HCP_TEST_DECIM", "100")) sfreq_preproc = 508.63 / test_decim sfreq_raw = 2034.5101 lowpass_preproc = 150 diff --git a/hcp/tests/test_preprocessing.py b/hcp/tests/test_preprocessing.py index 3af51be..2572550 100644 --- a/hcp/tests/test_preprocessing.py +++ b/hcp/tests/test_preprocessing.py @@ -1,5 +1,7 @@ import warnings +import pytest + import mne import numpy as np from pytest import raises as assert_raises @@ -18,6 +20,7 @@ hcp_params = dict(hcp_path=tconf.hcp_path, subject=tconf.test_subject) +@pytest.mark.xfail(reason="TODO: Pick counts are wrong, need to update logic!") def test_set_eog_ecg_channels(): """Test setting of EOG and ECG channels.""" raw = hcp.read_raw(data_type="rest", **hcp_params) @@ -40,7 +43,7 @@ def test_apply_ica(): bad_seg[:, 0], (bad_seg[:, 1] - bad_seg[:, 0]), description="bad" ) - raw.annotations = annotations + raw.set_annotations(annotations) raw.info["bads"].extend(annots["channels"]["all"]) ica_mat = hcp.read_ica(data_type="rest", **hcp_params) exclude = [ diff --git a/tools/get_testing_data.sh b/tools/get_testing_data.sh new file mode 100755 index 0000000..f8fab54 --- /dev/null +++ b/tools/get_testing_data.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# On CIs, smaller files are downloaded and used (e.g., that have been cropped and +# decimated), see: +# +# https://github.com/dengemann/make-mne-hcp-testing-data/blob/master/make_mne_hcp_testing_data.ipynb +# +# Since the raw data are not decimated by a factor of 100, you need to set +# MNE_HCP_TEST_DECIM=1 when running the tests if you use these data. + +set -exo pipefail + +LOCAL=~/mne-hcp-data/mne-hcp-testing/105923 +REMOTE=s3://hcp-openaccess/HCP/105923 +mkdir -p $LOCAL/unprocessed/MEG +cd $LOCAL/unprocessed/MEG +s3cmd sync $REMOTE/unprocessed/MEG/3-Restin ./ +s3cmd sync $REMOTE/unprocessed/MEG/4-Restin ./ +s3cmd sync $REMOTE/unprocessed/MEG/5-Restin ./ +s3cmd sync $REMOTE/unprocessed/MEG/8-StoryM ./ +s3cmd sync $REMOTE/unprocessed/MEG/6-Wrkmem ./ +s3cmd sync $REMOTE/unprocessed/MEG/7-Wrkmem ./ +s3cmd sync $REMOTE/unprocessed/MEG/9-StoryM ./ +s3cmd sync $REMOTE/unprocessed/MEG/10-Motort ./ +s3cmd sync $REMOTE/unprocessed/MEG/11-Motort ./ +s3cmd sync $REMOTE/unprocessed/MEG/1-Rnoise ./ +cd $LOCAL/MEG +mkdir -p anatomy StoryM Wrkmem Motort Restin +cd anatomy +s3cmd sync $REMOTE/MEG/anatomy/105923_MEG_anatomy_transform.txt ./ +s3cmd sync $REMOTE/MEG/anatomy/105923_MEG_anatomy_headmodel.mat ./ +cd ../Restin +s3cmd sync $REMOTE/MEG/Restin/baddata ./ +s3cmd sync $REMOTE/MEG/Restin/icaclass ./ +s3cmd sync $REMOTE/MEG/Restin/rmegpreproc ./ +cd ../StoryM +s3cmd sync $REMOTE/MEG/StoryM/baddata ./ +s3cmd sync $REMOTE/MEG/StoryM/icaclass ./ +s3cmd sync $REMOTE/MEG/StoryM/tmegpreproc ./ +s3cmd sync $REMOTE/MEG/StoryM/eravg ./ +cd ../Wrkmem +s3cmd sync $REMOTE/MEG/Wrkmem/baddata ./ +s3cmd sync $REMOTE/MEG/Wrkmem/icaclass ./ +s3cmd sync $REMOTE/MEG/Wrkmem/tmegpreproc ./ +s3cmd sync $REMOTE/MEG/Wrkmem/eravg ./ +cd ../Motort +s3cmd sync $REMOTE/MEG/Motort/baddata ./ +s3cmd sync $REMOTE/MEG/Motort/icaclass ./ +s3cmd sync $REMOTE/MEG/Motort/tmegpreproc ./ +s3cmd sync $REMOTE/MEG/Motort/eravg ./ +cd $LOCAL +mkdir -p T1w/105923/mri +s3cmd sync $REMOTE/T1w/105923/mri/c_ras.mat ./T1w/105923/mri/ \ No newline at end of file From ebe391586fc470ff6fa58fb9366e2dbd6f4b36ae Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 24 Jul 2024 10:45:13 -0400 Subject: [PATCH 32/34] FIX: Fix a little more --- hcp/anatomy.py | 11 +- hcp/conftest.py | 44 +++ .../file_mapping/tests/test_file_mapping.py | 131 +++---- hcp/io/read.py | 3 - hcp/io/tests/test_io.py | 334 +++++++++--------- hcp/tests/config.py | 62 ---- hcp/tests/test_anatomy.py | 51 ++- hcp/tests/test_preprocessing.py | 13 +- hcp/tests/test_viz.py | 9 +- tools/get_testing_data.sh | 4 +- 10 files changed, 324 insertions(+), 338 deletions(-) create mode 100644 hcp/conftest.py delete mode 100644 hcp/tests/config.py diff --git a/hcp/anatomy.py b/hcp/anatomy.py index 515d1b4..6b2ea6e 100644 --- a/hcp/anatomy.py +++ b/hcp/anatomy.py @@ -9,19 +9,22 @@ from mne import write_surface, write_trans from mne._fiff.pick import _pick_data_channels, pick_info from mne.transforms import Transform, apply_trans -from mne.utils import logger +from mne.utils import logger, verbose from scipy import linalg from .io.file_mapping import get_file_paths from .io.read import _get_head_model, _read_trans_hcp, read_info +@verbose def make_mne_anatomy( subject, subjects_dir, recordings_path=None, hcp_path=op.curdir, outputs=("label", "mri", "surf"), + *, + verbose=None, ): """Extract relevant anatomy and create MNE friendly directory layout. @@ -50,6 +53,7 @@ def make_mne_anatomy( The outputs of the freesrufer pipeline shipped by HCP. Defaults to (``'mri'``, ``'surf'``), the minimum needed to extract MNE-friendly anatomy files and data. + %(verbose)s """ if hcp_path == op.curdir: hcp_path = op.realpath(hcp_path) @@ -146,7 +150,7 @@ def make_mne_anatomy( ) -@mne.utils.verbose +@verbose def compute_forward_stack( subjects_dir, subject, @@ -155,6 +159,7 @@ def compute_forward_stack( src_params=None, hcp_path=op.curdir, n_jobs=1, + *, verbose=None, ): """Conduct standard MNE analyses. @@ -246,6 +251,8 @@ def compute_forward_stack( ) # ico = None for morphed SP. bem_sol = mne.make_bem_solution(bems) bem_sol["surfs"][0]["coord_frame"] = 5 + # make the forward code happy + bem_sol["surfs"][0]["tris"] = bem_sol["surfs"][0]["tris"].astype(np.int64) info = read_info(subject=subject, hcp_path=hcp_path, **info_from) picks = _pick_data_channels(info, with_ref_meg=False) diff --git a/hcp/conftest.py b/hcp/conftest.py new file mode 100644 index 0000000..a212af7 --- /dev/null +++ b/hcp/conftest.py @@ -0,0 +1,44 @@ +# Author: Denis A. Engemann +# License: BSD (3-clause) + +import pytest +from pathlib import Path + + +def pytest_configure(config): + """Configure pytest options.""" + # Markers + for marker in ( + "slow", + ): + config.addinivalue_line("markers", marker) + + # Treat warnings as errors, plus an allowlist + warning_lines = f"error::" + warning_lines += r""" + ignore:.*is non-interactive.*:UserWarning + """ + for warning_line in warning_lines.split("\n"): + warning_line = warning_line.strip() + if warning_line and not warning_line.startswith("#"): + config.addinivalue_line("filterwarnings", warning_line) + + +@pytest.fixture(autouse=True) +def mpl_close(): + """Close all matplotlib windows after each test.""" + import matplotlib + matplotlib.use("Agg", force=True) + + import matplotlib.pyplot as plt + yield + plt.close("all") + + +@pytest.fixture() +def hcp_params(): + """The MNE-HCP hcp params.""" + return dict( + hcp_path=Path("~/mne-hcp-data/mne-hcp-testing").expanduser().resolve(strict=True), + subject="105923", + ) diff --git a/hcp/io/file_mapping/tests/test_file_mapping.py b/hcp/io/file_mapping/tests/test_file_mapping.py index effba60..e37c5ec 100644 --- a/hcp/io/file_mapping/tests/test_file_mapping.py +++ b/hcp/io/file_mapping/tests/test_file_mapping.py @@ -1,87 +1,88 @@ -from pytest import raises as assert_raises +import pytest from numpy.testing import assert_equal -import hcp.tests.config as tconf from hcp.io.file_mapping import get_file_paths from hcp.io.file_mapping.file_mapping import run_map -def test_basic_file_mapping(): +@pytest.mark.parametrize("data_type", [ + "rest", + "task_working_memory", + "task_story_math", + "task_motor", + "noise_empty_room", +]) +@pytest.mark.parametrize("run_index", range(3)) +@pytest.mark.parametrize("output", [ + "raw", "epochs", "ica", "evoked", "trial_info", "bads", +]) +def test_basic_file_mapping(hcp_params, data_type, run_index, output): """Test construction of file paths and names""" - assert_raises( + pytest.raises( ValueError, get_file_paths, - subject=tconf.subject, data_type="sushi", output="raw", run_index=0, - hcp_path=tconf.hcp_path, + **hcp_params, ) - assert_raises( + pytest.raises( ValueError, get_file_paths, - subject=tconf.subject, data_type="rest", output="kimchi", run_index=0, - hcp_path=tconf.hcp_path, + **hcp_params, ) - for run_index in range(3): - for output in tconf.hcp_outputs: - for data_type in tconf.hcp_data_types: - # check too many runs - if run_index >= len(run_map[data_type]): - assert_raises( - ValueError, - get_file_paths, - subject=tconf.subject, - data_type=data_type, - output=output, - run_index=run_index, - hcp_path=tconf.hcp_path, - ) - # check no event related outputs - elif data_type in ( - "rest", - "noise_subject", - "noise_empty_room", - ) and output in ("trial_info", "evoked"): - assert_raises( - ValueError, - get_file_paths, - subject=tconf.subject, - data_type=data_type, - output=output, - run_index=run_index, - hcp_path=tconf.hcp_path, - ) - # check no preprocessing - elif data_type in ("noise_subject", "noise_empty_room") and output in ( - "epochs", - "evoked", - "ica", - "annot", - ): - assert_raises( - ValueError, - get_file_paths, - subject=tconf.subject, - data_type=data_type, - output=output, - run_index=run_index, - hcp_path=tconf.hcp_path, - ) - else: - file_names = get_file_paths( - subject=tconf.subject, - data_type=data_type, - output=output, - run_index=run_index, - hcp_path=tconf.hcp_path, - ) - if output == "raw": - assert_equal(sum("config" in fn for fn in file_names), 1) - assert_equal(sum("c,rfDC" in fn for fn in file_names), 1) + # check too many runs + if run_index >= len(run_map[data_type]): + pytest.raises( + ValueError, + get_file_paths, + data_type=data_type, + output=output, + run_index=run_index, + **hcp_params, + ) + # check no event related outputs + elif data_type in ( + "rest", + "noise_subject", + "noise_empty_room", + ) and output in ("trial_info", "evoked"): + pytest.raises( + ValueError, + get_file_paths, + data_type=data_type, + output=output, + run_index=run_index, + **hcp_params, + ) + # check no preprocessing + elif data_type in ("noise_subject", "noise_empty_room") and output in ( + "epochs", + "evoked", + "ica", + "annot", + ): + pytest.raises( + ValueError, + get_file_paths, + data_type=data_type, + output=output, + run_index=run_index, + **hcp_params, + ) + else: + file_names = get_file_paths( + data_type=data_type, + output=output, + run_index=run_index, + **hcp_params, + ) + if output == "raw": + assert_equal(sum("config" in fn for fn in file_names), 1) + assert_equal(sum("c,rfDC" in fn for fn in file_names), 1) diff --git a/hcp/io/read.py b/hcp/io/read.py index d0312c5..6f059e0 100644 --- a/hcp/io/read.py +++ b/hcp/io/read.py @@ -370,9 +370,6 @@ def _read_epochs(epochs_mat_fname, info, return_fixations_motor): # and it is task specific. this_info = _hcp_pick_info(info, ch_names) epochs = EpochsArray(data=data, info=this_info, events=events, tmin=times.min()) - # XXX hack for now due to issue with EpochsArray constructor - # cf https://github.com/mne-tools/mne-hcp/issues/9 - epochs._times = times return epochs diff --git a/hcp/io/tests/test_io.py b/hcp/io/tests/test_io.py index 35c431f..045349a 100644 --- a/hcp/io/tests/test_io.py +++ b/hcp/io/tests/test_io.py @@ -2,54 +2,73 @@ import os.path as op import shutil +import pytest + import mne import numpy as np from numpy.testing import assert_allclose -from mne.utils import _TempDir from numpy.testing import assert_equal from pytest import raises as assert_raises from numpy.testing import assert_array_equal import hcp from hcp.io.read import _hcp_pick_info -from hcp.tests import config as tconf -hcp_params = dict(hcp_path=tconf.hcp_path, subject=tconf.test_subject) +run_inds=[0, 1, 2] +max_runs = int(os.getenv("MNE_HCP_N_RUNS", 3)) +bti_chans = {"A" + str(i) for i in range(1, 249, 1)} +task_types = pytest.mark.parametrize("data_type", ["task_story_math", "task_working_memory", "task_motor"]) +run_indices = pytest.mark.parametrize("run_index", run_inds[: max_runs]) +run_indices_2 = pytest.mark.parametrize("run_index", run_inds[: max_runs][:2]) + +test_decim = int(os.getenv("MNE_HCP_TEST_DECIM", "100")) +sfreq_preproc = 508.63 / test_decim +sfreq_raw = 2034.5101 +#lowpass_preproc = 150 +#highpass_preproc = 1.3 + +epochs_bounds = { + "task_motor": (-1.2, 1.2), + "task_working_memory": (-1.5, 2.5), + "task_story_math": (-1.5, 4), + "rest": (0, 2), +} -def test_read_annot(): + +@run_indices +def test_read_annot(hcp_params, run_index): """Test reading annotations.""" - for run_index in tconf.run_inds: - annots = hcp.read_annot(data_type="rest", run_index=run_index, **hcp_params) - # channels - assert_equal( - list(sorted(annots["channels"])), - ["all", "ica", "manual", "neigh_corr", "neigh_stdratio"], - ) - for channels in annots["channels"].values(): - for chan in channels: - assert chan in tconf.bti_chans - - # segments - assert_equal( - list(sorted(annots["ica"])), - [ - "bad", - "brain_ic", - "brain_ic_number", - "brain_ic_vs", - "brain_ic_vs_number", - "ecg_eog_ic", - "flag", - "good", - "physio", - "total_ic_number", - ], - ) - for components in annots["ica"].values(): - if len(components) > 0: - assert min(components) >= 0 - assert max(components) <= 248 + annots = hcp.read_annot(data_type="rest", run_index=run_index, **hcp_params) + # channels + assert_equal( + list(sorted(annots["channels"])), + ["all", "ica", "manual", "neigh_corr", "neigh_stdratio"], + ) + for channels in annots["channels"].values(): + for chan in channels: + assert chan in bti_chans + + # segments + assert_equal( + list(sorted(annots["ica"])), + [ + "bad", + "brain_ic", + "brain_ic_number", + "brain_ic_vs", + "brain_ic_vs_number", + "ecg_eog_ic", + "flag", + "good", + "physio", + "total_ic_number", + ], + ) + for components in annots["ica"].values(): + if len(components) > 0: + assert min(components) >= 0 + assert max(components) <= 248 def _basic_raw_checks(raw): @@ -60,61 +79,61 @@ def _basic_raw_checks(raw): assert all(ch.startswith("A") for ch in ch_names) ch_sorted = list(sorted(ch_names)) assert ch_sorted != ch_names - assert_equal(np.round(raw.info["sfreq"], 4), tconf.sfreq_raw) + assert_equal(np.round(raw.info["sfreq"], 4), sfreq_raw) -def test_read_raw_rest(): +@run_indices +def test_read_raw_rest(hcp_params, run_index): """Test reading raw for resting state""" - for run_index in tconf.run_inds[: tconf.max_runs]: - raw = hcp.read_raw(data_type="rest", run_index=run_index, **hcp_params) - _basic_raw_checks(raw=raw) + raw = hcp.read_raw(data_type="rest", run_index=run_index, **hcp_params) + _basic_raw_checks(raw=raw) -def test_read_raw_task(): +@task_types +@run_indices +def test_read_raw_task(hcp_params, data_type, run_index): """Test reading raw for tasks""" - for run_index in tconf.run_inds[: tconf.max_runs]: - for data_type in tconf.task_types: - if run_index == 2: - assert_raises( - ValueError, - hcp.read_raw, - data_type=data_type, - run_index=run_index, - **hcp_params, - ) - continue - raw = hcp.read_raw(data_type=data_type, run_index=run_index, **hcp_params) - _basic_raw_checks(raw=raw) - - -def test_read_raw_noise(): + if run_index == 2: + assert_raises( + ValueError, + hcp.read_raw, + data_type=data_type, + run_index=run_index, + **hcp_params, + ) + return + raw = hcp.read_raw(data_type=data_type, run_index=run_index, **hcp_params) + _basic_raw_checks(raw=raw) + + +@pytest.mark.parametrize("data_type", ["noise_empty_room"]) +@run_indices_2 +def test_read_raw_noise(hcp_params, data_type, run_index): """Test reading raw for empty room noise""" - for run_index in tconf.run_inds[: tconf.max_runs][:2]: - for data_type in tconf.noise_types: - if run_index == 1: - assert_raises( - ValueError, - hcp.read_raw, - data_type=data_type, - run_index=run_index, - **hcp_params, - ) - continue - raw = hcp.read_raw(data_type=data_type, run_index=run_index, **hcp_params) - _basic_raw_checks(raw=raw) - - -def _epochs_basic_checks(epochs, annots, data_type): + if run_index == 1: + assert_raises( + ValueError, + hcp.read_raw, + data_type=data_type, + run_index=run_index, + **hcp_params, + ) + return + raw = hcp.read_raw(data_type=data_type, run_index=run_index, **hcp_params) + _basic_raw_checks(raw=raw) + + +def _epochs_basic_checks(epochs, annots, *, data_type): n_good = 248 - len(annots["channels"]["all"]) if data_type == "task_motor": n_good += 4 assert_equal(len(epochs.ch_names), n_good) - assert_allclose(epochs.info["sfreq"], tconf.sfreq_preproc, rtol=1e-4) + assert_allclose(epochs.info["sfreq"], sfreq_preproc, rtol=1e-4) assert_array_equal(np.unique(epochs.events[:, 2]), np.array([99], dtype=np.int64)) assert ( _check_bounds( epochs.times, - tconf.epochs_bounds[data_type], + epochs_bounds[data_type], atol=1.0 / epochs.info["sfreq"], ) # decim tolerance ) @@ -128,29 +147,29 @@ def _epochs_basic_checks(epochs, annots, data_type): # highpass_preproc) -def test_read_epochs_rest(): +@run_indices_2 +def test_read_epochs_rest(hcp_params, run_index): """Test reading epochs for resting state""" - for run_index in tconf.run_inds[: tconf.max_runs][:2]: - annots = hcp.read_annot(data_type="rest", run_index=run_index, **hcp_params) + annots = hcp.read_annot(data_type="rest", run_index=run_index, **hcp_params) - epochs = hcp.read_epochs(data_type="rest", run_index=run_index, **hcp_params) + epochs = hcp.read_epochs(data_type="rest", run_index=run_index, **hcp_params) - _epochs_basic_checks(epochs, annots, data_type="rest") + _epochs_basic_checks(epochs, annots, data_type="rest") -def test_read_epochs_task(): +@task_types +@run_indices_2 +def test_read_epochs_task(hcp_params, data_type, run_index): """Test reading epochs for task""" - for run_index in tconf.run_inds[: tconf.max_runs][:2]: - for data_type in tconf.task_types: - annots = hcp.read_annot( - data_type=data_type, run_index=run_index, **hcp_params - ) + annots = hcp.read_annot( + data_type=data_type, run_index=run_index, **hcp_params + ) - epochs = hcp.read_epochs( - data_type=data_type, run_index=run_index, **hcp_params - ) + epochs = hcp.read_epochs( + data_type=data_type, run_index=run_index, **hcp_params + ) - _epochs_basic_checks(epochs, annots, data_type) + _epochs_basic_checks(epochs, annots, data_type=data_type) def _check_bounds(array, bounds, atol=0.01): @@ -160,79 +179,78 @@ def _check_bounds(array, bounds, atol=0.01): ) -def test_read_evoked(): +@task_types +def test_read_evoked(hcp_params, data_type): """Test reading evokeds.""" - for data_type in tconf.task_types: - all_annots = list() - for run_index in tconf.run_inds[:2]: - annots = hcp.read_annot( - data_type=data_type, run_index=run_index, **hcp_params - ) - all_annots.append(annots) + all_annots = list() + for run_index in run_inds[:2]: + annots = hcp.read_annot( + data_type=data_type, run_index=run_index, **hcp_params + ) + all_annots.append(annots) - evokeds = hcp.read_evokeds(data_type=data_type, kind="average", **hcp_params) + evokeds = hcp.read_evokeds(data_type=data_type, kind="average", **hcp_params) - n_average = sum(ee.kind == "average" for ee in evokeds) - assert_equal(n_average, len(evokeds)) + n_average = sum(ee.kind == "average" for ee in evokeds) + assert_equal(n_average, len(evokeds)) - n_chans = 248 - if data_type == "task_motor": - n_chans += 4 - n_chans -= len(set(sum([an["channels"]["all"] for an in all_annots], []))) - assert_equal(n_chans, len(evokeds[0].ch_names)) - assert _check_bounds(evokeds[0].times, tconf.epochs_bounds[data_type]) + n_chans = 248 + if data_type == "task_motor": + n_chans += 4 + n_chans -= len(set(sum([an["channels"]["all"] for an in all_annots], []))) + assert_equal(n_chans, len(evokeds[0].ch_names)) + assert _check_bounds(evokeds[0].times, epochs_bounds[data_type]) -def test_read_info(): +@task_types +@run_indices_2 +def test_read_info(tmp_path, hcp_params, data_type, run_index): """Test reading info.""" - tempdir = _TempDir() - for data_type in tconf.task_types: - for run_index in tconf.run_inds[: tconf.max_runs][:2]: - # with pdf file - info = hcp.read_info(data_type=data_type, run_index=run_index, **hcp_params) - assert_equal( - {k for k in info["ch_names"] if k.startswith("A")}, tconf.bti_chans - ) - # without pdf file - # in this case the hcp code guesses certain channel labels - cp_paths = hcp.io.file_mapping.get_file_paths( - subject=tconf.test_subject, - data_type=data_type, - run_index=run_index, - output="raw", - hcp_path="", - ) - for pp in cp_paths: - if "c," in pp: # don't copy pdf - continue - os.makedirs(op.join(tempdir, op.dirname(pp))) - shutil.copy(op.join(tconf.hcp_path, pp), op.join(tempdir, pp)) - - info2 = hcp.read_info( - subject=tconf.test_subject, - data_type=data_type, - hcp_path=tempdir, - run_index=run_index, - ) - assert len(info["chs"]) != len(info2["chs"]) - common_chs = [ch for ch in info2["ch_names"] if ch in info["ch_names"]] - assert_equal(len(common_chs), len(info["chs"])) - info2 = _hcp_pick_info(info2, common_chs) - assert_equal(info["ch_names"], info2["ch_names"]) - for ch1, ch2 in zip(info["chs"], info2["chs"]): - assert_array_equal(ch1["loc"], ch2["loc"]) - - -def test_read_trial_info(): + # with pdf file + info = hcp.read_info(data_type=data_type, run_index=run_index, **hcp_params) + assert_equal( + {k for k in info["ch_names"] if k.startswith("A")}, bti_chans + ) + # without pdf file + # in this case the hcp code guesses certain channel labels + cp_paths = hcp.io.file_mapping.get_file_paths( + subject=hcp_params["subject"], + data_type=data_type, + run_index=run_index, + output="raw", + hcp_path="", + ) + for pp in cp_paths: + if "c," in pp: # don't copy pdf + continue + (tmp_path / op.dirname(pp)).mkdir(parents=True, exist_ok=True) + shutil.copy(hcp_params["hcp_path"] / pp, tmp_path / pp) + + info2 = hcp.read_info( + subject=hcp_params["subject"], + data_type=data_type, + hcp_path=tmp_path, + run_index=run_index, + ) + assert len(info["chs"]) != len(info2["chs"]) + common_chs = [ch for ch in info2["ch_names"] if ch in info["ch_names"]] + assert_equal(len(common_chs), len(info["chs"])) + info2 = _hcp_pick_info(info2, common_chs) + assert_equal(info["ch_names"], info2["ch_names"]) + for ch1, ch2 in zip(info["chs"], info2["chs"]): + assert_array_equal(ch1["loc"], ch2["loc"]) + + +@task_types +@run_indices_2 +def test_read_trial_info(hcp_params, data_type, run_index): """Test reading trial info basics.""" - for data_type in tconf.task_types: - for run_index in tconf.run_inds[: tconf.max_runs][:2]: - trial_info = hcp.read_trial_info( - data_type=data_type, run_index=run_index, **hcp_params - ) - assert "stim" in trial_info - assert "resp" in trial_info - assert_equal(2, len(trial_info)) - for val in trial_info.values(): - assert_array_equal(np.ndim(val["comments"]), 1) - assert_array_equal(np.ndim(val["codes"]), 2) + trial_info = hcp.read_trial_info( + data_type=data_type, run_index=run_index, **hcp_params + ) + assert "stim" in trial_info + assert "resp" in trial_info + assert_equal(2, len(trial_info)) + for val in trial_info.values(): + assert_array_equal(np.ndim(val["comments"]), 1) + assert_array_equal(np.ndim(val["codes"]), 2) diff --git a/hcp/tests/config.py b/hcp/tests/config.py deleted file mode 100644 index f67714b..0000000 --- a/hcp/tests/config.py +++ /dev/null @@ -1,62 +0,0 @@ -# Author: Denis A. Engemann -# License: BSD (3-clause) - -import os -import os.path as op - -hcp_prefix = "s3://hcp-openaccess/HCP_900" -subject = "105923" - - -hcp_data_types = [ - "rest", - "task_working_memory", - "task_story_math", - "task_motor", - "noise_empty_room", -] - - -hcp_outputs = ["raw", "epochs", "ica", "evoked", "trial_info", "bads"] - -hcp_onsets = ["stim"] - -# allow for downloading fewer data -run_inds = [0, 1, 2] -max_runs = int(os.getenv("MNE_HCP_N_RUNS", 3)) - -############################################################################## -# variable used in different tests - -hcp_path = op.expanduser("~/mne-hcp-data/mne-hcp-testing") - -bti_chans = {"A" + str(i) for i in range(1, 249, 1)} - -test_subject = "105923" -task_types = ["task_story_math", "task_working_memory", "task_motor"] -noise_types = ["noise_empty_room"] -test_decim = int(os.getenv("MNE_HCP_TEST_DECIM", "100")) -sfreq_preproc = 508.63 / test_decim -sfreq_raw = 2034.5101 -lowpass_preproc = 150 -highpass_preproc = 1.3 - -epochs_bounds = { - "task_motor": (-1.2, 1.2), - "task_working_memory": (-1.5, 2.5), - "task_story_math": (-1.5, 4), - "rest": (0, 2), -} - - -def nottest(f): - """Decorator to mark a function as not a test.""" - f.__test__ = False - return f - - -@nottest -def expensive_test(f): - """Decorator for expensive testing.""" - f.expensive_test = True - return f diff --git a/hcp/tests/test_anatomy.py b/hcp/tests/test_anatomy.py index 00f602e..e9dc29b 100644 --- a/hcp/tests/test_anatomy.py +++ b/hcp/tests/test_anatomy.py @@ -1,42 +1,33 @@ -import os.path as op import shutil -import matplotlib import mne +import pytest from hcp import compute_forward_stack, make_mne_anatomy -from hcp.tests import config as tconf -from hcp.tests.config import expensive_test from hcp.viz import plot_coregistration -matplotlib.use("Agg") -hcp_params = dict(hcp_path=tconf.hcp_path, subject=tconf.test_subject) - - -@expensive_test -def test_anatomy(): +@pytest.mark.slow +def test_anatomy(tmp_path, hcp_params): """Test anatomy functions (slow!).""" - import matplotlib.pyplot as plt - # This is where the data are after downloading from HCP - temp_dir = mne.utils._TempDir() - subjects_dir = op.join(temp_dir, "hcp-subjects") - recordings_path = op.join(temp_dir, "hcp-meg") + subjects_dir = tmp_path / "hcp-subjects" + recordings_path = tmp_path / "hcp-meg" make_mne_anatomy( - recordings_path=recordings_path, subjects_dir=subjects_dir, **hcp_params - ) - assert ( - op.isfile( - op.join(subjects_dir, hcp_params["subject"], "bem", "inner_skull.surf") - ) + recordings_path=recordings_path, + subjects_dir=subjects_dir, + verbose=True, + **hcp_params, ) + subject_dir = subjects_dir / hcp_params["subject"] + inner_skull = subject_dir / "bem" / "inner_skull.surf" + assert inner_skull.is_file() + white = subject_dir / "surf" / "lh.white" + assert white.is_file() + # Now we need fsaverage... - mne_subjects_dir = mne.get_config("SUBJECTS_DIR") - assert mne_subjects_dir is not None - shutil.copytree( - op.join(mne_subjects_dir, "fsaverage"), op.join(subjects_dir, "fsaverage") - ) + mne_subjects_dir = mne.utils.get_subjects_dir(raise_error=True) + shutil.copytree(mne_subjects_dir / "fsaverage", subjects_dir / "fsaverage") compute_forward_stack( subjects_dir=subjects_dir, recordings_path=recordings_path, @@ -46,8 +37,8 @@ def test_anatomy(): ) # let's do our viz tests, too plot_coregistration( - subjects_dir=subjects_dir, recordings_path=recordings_path, **hcp_params + subjects_dir=subjects_dir, + recordings_path=recordings_path, + **hcp_params, ) - plt.close("all") - mne.viz.plot_bem(subject=tconf.test_subject, subjects_dir=subjects_dir) - plt.close("all") + mne.viz.plot_bem(subject=hcp_params["subject"], subjects_dir=subjects_dir) diff --git a/hcp/tests/test_preprocessing.py b/hcp/tests/test_preprocessing.py index 2572550..ba15d1b 100644 --- a/hcp/tests/test_preprocessing.py +++ b/hcp/tests/test_preprocessing.py @@ -15,13 +15,10 @@ map_ch_coords_to_mne, set_eog_ecg_channels, ) -from hcp.tests import config as tconf - -hcp_params = dict(hcp_path=tconf.hcp_path, subject=tconf.test_subject) @pytest.mark.xfail(reason="TODO: Pick counts are wrong, need to update logic!") -def test_set_eog_ecg_channels(): +def test_set_eog_ecg_channels(hcp_params): """Test setting of EOG and ECG channels.""" raw = hcp.read_raw(data_type="rest", **hcp_params) raw.crop(0, 1).load_data() @@ -33,7 +30,7 @@ def test_set_eog_ecg_channels(): assert_equal(len(mne.pick_types(raw.info, meg=False, ecg=True)), 8) -def test_apply_ica(): +def test_apply_ica(hcp_params): """Test ICA application.""" raw = hcp.read_raw(data_type="rest", verbose="error", **hcp_params) annots = hcp.read_annot(data_type="rest", **hcp_params) @@ -59,7 +56,7 @@ def test_apply_ica(): apply_ica_hcp(raw, ica_mat=ica_mat, exclude=exclude) -def test_apply_ref_correction(): +def test_apply_ref_correction(hcp_params): """Test reference correction.""" raw = hcp.read_raw(data_type="rest", run_index=0, **hcp_params) # raw.crop(0, 10).load_data() @@ -74,7 +71,7 @@ def test_apply_ref_correction(): assert np.linalg.norm(orig) > np.linalg.norm(proc) -def test_map_ch_coords_to_mne(): +def test_map_ch_coords_to_mne(hcp_params): """Test mapping of channel coords to MNE.""" data_type = "task_working_memory" hcp_evokeds = hcp.read_evokeds(onset="stim", data_type=data_type, **hcp_params) @@ -87,7 +84,7 @@ def test_map_ch_coords_to_mne(): assert (old_coord != new_coord).any() -def test_interpolate_missing(): +def test_interpolate_missing(hcp_params): """Test interpolation of missing channels.""" data_type = "task_working_memory" raw = hcp.read_raw(data_type="task_working_memory", run_index=0, **hcp_params) diff --git a/hcp/tests/test_viz.py b/hcp/tests/test_viz.py index fe8000d..4ba27bf 100644 --- a/hcp/tests/test_viz.py +++ b/hcp/tests/test_viz.py @@ -1,17 +1,10 @@ -import matplotlib -import mne from numpy.testing import assert_equal import hcp -from hcp.tests import config as tconf from hcp.viz import make_hcp_bti_layout -matplotlib.use("Agg") -hcp_params = dict(hcp_path=tconf.hcp_path, subject=tconf.test_subject) - - -def test_make_layout(): +def test_make_layout(hcp_params): """Test making a layout.""" raw = hcp.read_raw(data_type="rest", **hcp_params).crop(0, 1).load_data() raw.pick_types(meg=True) diff --git a/tools/get_testing_data.sh b/tools/get_testing_data.sh index f8fab54..6bd0d27 100755 --- a/tools/get_testing_data.sh +++ b/tools/get_testing_data.sh @@ -49,5 +49,5 @@ s3cmd sync $REMOTE/MEG/Motort/icaclass ./ s3cmd sync $REMOTE/MEG/Motort/tmegpreproc ./ s3cmd sync $REMOTE/MEG/Motort/eravg ./ cd $LOCAL -mkdir -p T1w/105923/mri -s3cmd sync $REMOTE/T1w/105923/mri/c_ras.mat ./T1w/105923/mri/ \ No newline at end of file +mkdir -p T1w/ +s3cmd sync $REMOTE/T1w/105923 ./T1w/ From dd177f9b5fd31675f0517134e599de07c79bd57e Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 24 Jul 2024 10:54:00 -0400 Subject: [PATCH 33/34] FIX: set types, too --- hcp/io/file_mapping/file_mapping.py | 6 +++++- hcp/preprocessing.py | 4 +++- hcp/tests/test_preprocessing.py | 8 +++----- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/hcp/io/file_mapping/file_mapping.py b/hcp/io/file_mapping/file_mapping.py index c9503b1..cc0f9fd 100644 --- a/hcp/io/file_mapping/file_mapping.py +++ b/hcp/io/file_mapping/file_mapping.py @@ -1,4 +1,5 @@ import os.path as op +import mne """Notes @@ -249,6 +250,9 @@ def get_file_paths( out : list of str The file names. """ + hcp_path = mne.utils._check_fname( + hcp_path, overwrite="read", need_dir=True, must_exist=True, + ) if data_type not in kind_map: raise ValueError( f"I never heard of `{data_type}` -- are you sure this is a valid HCP type? " @@ -372,4 +376,4 @@ def get_file_paths( else: raise ValueError('`processing` %s should be "unprocessed"' ' or "preprocessed"') - return [op.join(hcp_path, pa) for pa in files] + return [str(hcp_path / pa) for pa in files] diff --git a/hcp/preprocessing.py b/hcp/preprocessing.py index 72410e7..654718f 100644 --- a/hcp/preprocessing.py +++ b/hcp/preprocessing.py @@ -30,8 +30,10 @@ def set_eog_ecg_channels(raw): The hcp raw data. """ for kind in ["ECG", "VEOG", "HEOG"]: + anode, cathode = f"{kind}-", f"{kind}+" + raw.set_channel_types({anode: "eeg", cathode: "eeg"}) set_bipolar_reference( - raw, anode=kind + "-", cathode=kind + "+", ch_name=kind, copy=False + raw, anode=anode, cathode=cathode, ch_name=kind, copy=False ) raw.set_channel_types({"ECG": "ecg", "VEOG": "eog", "HEOG": "eog"}) diff --git a/hcp/tests/test_preprocessing.py b/hcp/tests/test_preprocessing.py index ba15d1b..b3334c5 100644 --- a/hcp/tests/test_preprocessing.py +++ b/hcp/tests/test_preprocessing.py @@ -17,17 +17,15 @@ ) -@pytest.mark.xfail(reason="TODO: Pick counts are wrong, need to update logic!") def test_set_eog_ecg_channels(hcp_params): """Test setting of EOG and ECG channels.""" raw = hcp.read_raw(data_type="rest", **hcp_params) raw.crop(0, 1).load_data() - assert_equal(len(mne.pick_types(raw.info, meg=False, eog=True)), 0) - assert_equal(len(mne.pick_types(raw.info, meg=False, ecg=True)), 13) + assert_equal(len(mne.pick_types(raw.info, meg=False, eog=True)), 4) + assert_equal(len(mne.pick_types(raw.info, meg=False, ecg=True)), 2) set_eog_ecg_channels(raw) - # XXX Probably shouldn't still have 8 ECG channels! assert_equal(len(mne.pick_types(raw.info, meg=False, eog=True)), 2) - assert_equal(len(mne.pick_types(raw.info, meg=False, ecg=True)), 8) + assert_equal(len(mne.pick_types(raw.info, meg=False, ecg=True)), 1) def test_apply_ica(hcp_params): From 07c3a51e77d8442a273493c1ea0062a52c5f52d3 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 24 Jul 2024 11:29:19 -0400 Subject: [PATCH 34/34] FIX: One more --- hcp/conftest.py | 9 ++++-- hcp/io/file_mapping/file_mapping.py | 50 ++++++++++++++++------------- hcp/io/tests/test_io.py | 2 +- tools/get_testing_data.sh | 5 +-- 4 files changed, 38 insertions(+), 28 deletions(-) diff --git a/hcp/conftest.py b/hcp/conftest.py index a212af7..bf5b1d8 100644 --- a/hcp/conftest.py +++ b/hcp/conftest.py @@ -1,6 +1,7 @@ # Author: Denis A. Engemann # License: BSD (3-clause) +import mne import pytest from pathlib import Path @@ -38,7 +39,9 @@ def mpl_close(): @pytest.fixture() def hcp_params(): """The MNE-HCP hcp params.""" - return dict( - hcp_path=Path("~/mne-hcp-data/mne-hcp-testing").expanduser().resolve(strict=True), - subject="105923", + # Point to the folder that has 105923 in it + hcp_path = Path( + mne.get_config("MNE_HCP_TESTING_PATH", "~/mne-hcp-data/mne-hcp-testing") ) + hcp_path = hcp_path.expanduser().resolve(strict=True) + return dict(hcp_path=hcp_path, subject="105923") diff --git a/hcp/io/file_mapping/file_mapping.py b/hcp/io/file_mapping/file_mapping.py index cc0f9fd..a839db9 100644 --- a/hcp/io/file_mapping/file_mapping.py +++ b/hcp/io/file_mapping/file_mapping.py @@ -1,4 +1,4 @@ -import os.path as op +from pathlib import Path import mne """Notes @@ -134,7 +134,7 @@ } -freesurfer_files = op.join(op.dirname(__file__), "data", "%s.txt") +freesurfer_files = str(Path(__file__).parent / "data" / "%s.txt") for kind, patterns in preprocessed["freesurfer"]["patterns"].items(): with open(freesurfer_files % kind) as fid: patterns.extend([k.rstrip("\n") for k in fid.readlines()]) @@ -250,9 +250,12 @@ def get_file_paths( out : list of str The file names. """ - hcp_path = mne.utils._check_fname( - hcp_path, overwrite="read", need_dir=True, must_exist=True, - ) + if isinstance(hcp_path, str) and hcp_path == "": # don't prefix anything + hcp_path = Path() + else: + hcp_path = mne.utils._check_fname( + hcp_path, overwrite="read", need_dir=True, must_exist=True, + ) if data_type not in kind_map: raise ValueError( f"I never heard of `{data_type}` -- are you sure this is a valid HCP type? " @@ -297,11 +300,11 @@ def get_file_paths( file_map = preprocessed[ (data_type if data_type in ("meg_anatomy", "freesurfer") else "meg") ] - path = file_map["path"].format( + path = Path(file_map["path"].format( subject=subject, pipeline=(context + "preproc" if output == "epochs" else pipeline), kind=kind_map[data_type], - ) + )) if output == "epochs": pattern_key = (output, context) @@ -315,11 +318,9 @@ def get_file_paths( if output in ("bads", "ica"): files.extend( [ - op.join( - path, - p.format( - subject=subject, run=run_label, kind=kind_map[data_type] - ), + path + / p.format( + subject=subject, run=run_label, kind=kind_map[data_type] ) for p in my_pattern ] @@ -336,7 +337,7 @@ def get_file_paths( if context != "rest": formats.update(onset=my_onset) this_file = my_pattern.format(**formats) - files.append(op.join(path, this_file)) + files.append(path / this_file) elif output == "evoked": # XXX add evoked template checks @@ -350,29 +351,34 @@ def get_file_paths( diff_modes=_map_diff_mode(condition, data_type), sensor_mode=sensor_mode, ) - files.append(op.join(path, this_file)) + files.append(path / this_file) elif output == "trial_info": this_file = my_pattern[0].format( subject=subject, run=run_label, kind=kind_map[data_type] ) - files.append(op.join(path, this_file)) + files.append(path / this_file) elif data_type == "meg_anatomy": - path = file_map["path"].format(subject=subject) + path = Path(file_map["path"].format(subject=subject)) files.extend( - [op.join(path, pa.format(subject=subject)) for pa in my_pattern] + [path / pa.format(subject=subject) for pa in my_pattern] ) elif data_type == "freesurfer": - path = file_map["path"].format(subject=subject) + path = Path(file_map["path"].format(subject=subject)) for pa in my_pattern: - files.append(op.join(path, output, pa.format(subject=subject))) + files.append(path / output / pa.format(subject=subject)) else: raise ValueError('I never heard of `output` "%s".' % output) elif processing == "unprocessed": - path = unprocessed["path"].format( - subject=subject, kind=kind_map[data_type], pipeline=pipeline, run=run_label + path = Path( + unprocessed["path"].format( + subject=subject, + kind=kind_map[data_type], + pipeline=pipeline, + run=run_label, + ) ) - files.extend([op.join(path, p) for p in unprocessed["patterns"]]) + files.extend([path / p for p in unprocessed["patterns"]]) else: raise ValueError('`processing` %s should be "unprocessed"' ' or "preprocessed"') diff --git a/hcp/io/tests/test_io.py b/hcp/io/tests/test_io.py index 045349a..b6a1ace 100644 --- a/hcp/io/tests/test_io.py +++ b/hcp/io/tests/test_io.py @@ -16,7 +16,7 @@ run_inds=[0, 1, 2] -max_runs = int(os.getenv("MNE_HCP_N_RUNS", 3)) +max_runs = 3 # from the test dataset creation step bti_chans = {"A" + str(i) for i in range(1, 249, 1)} task_types = pytest.mark.parametrize("data_type", ["task_story_math", "task_working_memory", "task_motor"]) run_indices = pytest.mark.parametrize("run_index", run_inds[: max_runs]) diff --git a/tools/get_testing_data.sh b/tools/get_testing_data.sh index 6bd0d27..63da61a 100755 --- a/tools/get_testing_data.sh +++ b/tools/get_testing_data.sh @@ -6,11 +6,12 @@ # https://github.com/dengemann/make-mne-hcp-testing-data/blob/master/make_mne_hcp_testing_data.ipynb # # Since the raw data are not decimated by a factor of 100, you need to set -# MNE_HCP_TEST_DECIM=1 when running the tests if you use these data. +# MNE_HCP_TEST_DECIM=1 when running the tests if you use these data directly +# (e.g., if you also set MNE_HCP_TESTING_PATH=~/mne-hcp-data/HCP). set -exo pipefail -LOCAL=~/mne-hcp-data/mne-hcp-testing/105923 +LOCAL=~/mne-hcp-data/HCP/105923 REMOTE=s3://hcp-openaccess/HCP/105923 mkdir -p $LOCAL/unprocessed/MEG cd $LOCAL/unprocessed/MEG