error loading for volume based atlases Kastner2015 or thalamus atlas (func1pt8mm)

[iancharest]

nsd_access/nsd_access/nsda.py

Line 220 in ffa837a

mapp_df = pd.read_csv(os.path.join(self.nsddata_folder, 'freesurfer', 'fsaverage',

@tknapen we have a bug with the mgz.ctab files not being read by pd.read_csv at the line referenced above. This works well for the nsdgeneral, visualsulc, or HCP_MMP1 atlases, but fails for Kastner2015 and thalmus.

Here is the full error trace (atlas=Kastner2015, sub=subj01, data_format='func1pt8mm'):

---------------------------------------------------------------------------
ParserError                               Traceback (most recent call last)
<ipython-input-30-9b391304df40> in <module>
      2 mmp1, atlas_mapping = nsda.read_atlas_results(subject=sub,
      3                                               atlas=atlas,
----> 4                                               data_format='func1pt8mm')

~/env/lib/python3.6/site-packages/nsd_access/nsda.py in read_atlas_results(self, subject, atlas, data_format)
    219 
    220         mapp_df = pd.read_csv(os.path.join(self.nsddata_folder, 'freesurfer', 'fsaverage',
--> 221                                            'label', f'{atlas_name}.mgz.ctab'), delimiter=' ', header=None, index_col=0)
    222         atlas_mapping = mapp_df.to_dict()[1]
    223         # dict((y,x) for x,y in atlas_mapping.iteritems())

~/env/lib/python3.6/site-packages/pandas/io/parsers.py in parser_f(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)
    683         )
    684 
--> 685         return _read(filepath_or_buffer, kwds)
    686 
    687     parser_f.__name__ = name

~/env/lib/python3.6/site-packages/pandas/io/parsers.py in _read(filepath_or_buffer, kwds)
    461 
    462     try:
--> 463         data = parser.read(nrows)
    464     finally:
    465         parser.close()

~/env/lib/python3.6/site-packages/pandas/io/parsers.py in read(self, nrows)
   1152     def read(self, nrows=None):
   1153         nrows = _validate_integer("nrows", nrows)
-> 1154         ret = self._engine.read(nrows)
   1155 
   1156         # May alter columns / col_dict

~/env/lib/python3.6/site-packages/pandas/io/parsers.py in read(self, nrows)
   2057     def read(self, nrows=None):
   2058         try:
-> 2059             data = self._reader.read(nrows)
   2060         except StopIteration:
   2061             if self._first_chunk:

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.read()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._read_low_memory()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._read_rows()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._tokenize_rows()

pandas/_libs/parsers.pyx in pandas._libs.parsers.raise_parser_error()

ParserError: Error tokenizing data. C error: Expected 2 fields in line 2, saw 6

[iancharest]

We also get the same error when trying to load it from the fsaverage space.

[iancharest]

Trying to load the ctab files with nibabel also leads to a wall:

In [36]: import nibabel as nib                                                                                                                           

In [37]: from nibabel.freesurfer.io import read_annot                                                                                                    

In [38]: atlas_name                                                                                                                                      
Out[38]: 'Kastner2015'

In [39]: nsddata_folder                                                                                                                                  
Out[39]: '/media/charesti-start/data/NSD/nsddata'

In [40]: read_annot(os.path.join(nsddata_folder, 'freesurfer', 'fsaverage', 
    ...:                                            'label', f'{atlas_name}.mgz.ctab'))                                                                  
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-40-d23abcaf4e95> in <module>
      1 read_annot(os.path.join(nsddata_folder, 'freesurfer', 'fsaverage',
----> 2                                            'label', f'{atlas_name}.mgz.ctab'))

~/env/lib/python3.6/site-packages/nibabel/freesurfer/io.py in read_annot(filepath, orig_ids)
    368 
    369         # vertex ids + annotation values
--> 370         data = np.fromfile(fobj, dt, vnum * 2).reshape(vnum, 2)
    371         labels = data[:, 1]
    372 

ValueError: cannot reshape array of size 70 into shape (807425390,2)

In [41]: label, ctab, names = read_annot(os.path.join(nsddata_folder, 'freesurfer', 'fsaverage', 
    ...:                                            'label', f'{atlas_name}.mgz.ctab'))                                                                  
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-41-38283f589a40> in <module>
      1 label, ctab, names = read_annot(os.path.join(nsddata_folder, 'freesurfer', 'fsaverage',
----> 2                                            'label', f'{atlas_name}.mgz.ctab'))

~/env/lib/python3.6/site-packages/nibabel/freesurfer/io.py in read_annot(filepath, orig_ids)
    368 
    369         # vertex ids + annotation values
--> 370         data = np.fromfile(fobj, dt, vnum * 2).reshape(vnum, 2)
    371         labels = data[:, 1]
    372 

ValueError: cannot reshape array of size 70 into shape (807425390,2)