diff --git a/.gitignore b/.gitignore index bab0bcd..b4ce445 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ Dummy garbage .ipynb* .venvtest +.DS_STORE +.DS_Store # Created by https://www.gitignore.io/api/python # Edit at https://www.gitignore.io/?templates=python @@ -118,4 +120,9 @@ dmypy.json # Pyre type checker .pyre/ +# vs code +.vscode/ + + # End of https://www.gitignore.io/api/python +.vscode/settings.json diff --git a/README.md b/README.md index 72ec841..779468d 100755 --- a/README.md +++ b/README.md @@ -88,13 +88,20 @@ supply the scan information manually. ## Credits and notes **This tool is not an official product of the TVIPS company. Use at your own risk. -I am not responsbile for loss or corruption of data.** The tool derives from python scripts +We are not responsbile for loss or corruption of data.** The tool derives from python scripts originally developed by the company. We have significantly modified these scripts mainly to make the conversion process possible on a computer with regular sized RAM and support loss-less export to hdf5. The GUI is also our addition. ## Changelog +### 0.1.2 +* calculate and store direct beam positions in the HDF5 file +* calculate mean and maximum images +* added choice in how to bin the data +* more options on export image depth +* bugfixes + ### 0.1.1 * Added an option to crop the file along the scan directions * Added an option to limit the conversion between a set number of frames diff --git a/setup.py b/setup.py index a330ebf..8be0544 100644 --- a/setup.py +++ b/setup.py @@ -5,13 +5,13 @@ setup( name="tvipsconverter", - version="0.1.1", + version="0.1.2", description=( "GUI converter for movie data from TVIPS cameras" "into other formats like .blo, .tiff and .hspy." "Mainly for orientation mapping (PED) or 4D STEM experiments."), url='https://github.com/din14970/TVIPSconverter', - author='Niels Cautaerts', + author='Niels Cautaerts, Paddy Harrison', author_email='nielscautaerts@hotmail.com', license='GPL-3.0', long_description=readme, @@ -37,5 +37,6 @@ 'Pillow', 'PyQt5>=5.13.2', 'h5py>=2.10.0', + 'scikit-image>=0.17.2', ], ) diff --git a/tvipsconverter/__main__.py b/tvipsconverter/__main__.py new file mode 100644 index 0000000..5bd1eb6 --- /dev/null +++ b/tvipsconverter/__main__.py @@ -0,0 +1,3 @@ +from .widgets import main + +main() diff --git a/tvipsconverter/utils/blockfile.py b/tvipsconverter/utils/blockfile.py index c88d2de..fac49fa 100644 --- a/tvipsconverter/utils/blockfile.py +++ b/tvipsconverter/utils/blockfile.py @@ -22,6 +22,7 @@ import warnings import datetime import dateutil +from skimage import util from dateutil import tz, parser @@ -52,8 +53,7 @@ def sarray2dict(sarray, dictionary=None): if dictionary is None: dictionary = OrderedDict() for name in sarray.dtype.names: - dictionary[name] = sarray[name][0] if len(sarray[name]) == 1 \ - else sarray[name] + dictionary[name] = sarray[name][0] if len(sarray[name]) == 1 else sarray[name] return dictionary @@ -87,14 +87,11 @@ def dict2sarray(dictionary, sarray=None, dtype=None): return sarray -def ISO_format_to_serial_date(date, time, timezone='UTC'): +def ISO_format_to_serial_date(date, time, timezone="UTC"): """ Convert ISO format to a serial date. """ - if timezone is None or timezone == 'Coordinated Universal Time': - timezone = 'UTC' - dt = parser.parse( - '%sT%s' % - (date, time)).replace( - tzinfo=tz.gettz(timezone)) + if timezone is None or timezone == "Coordinated Universal Time": + timezone = "UTC" + dt = parser.parse("%sT%s" % (date, time)).replace(tzinfo=tz.gettz(timezone)) return datetime_to_serial_date(dt) @@ -123,8 +120,7 @@ def serial_date_to_ISO_format(serial): """ dt_utc = serial_date_to_datetime(serial) dt_local = dt_utc.astimezone(tz.tzlocal()) - return (dt_local.date().isoformat(), dt_local.time().isoformat(), - dt_local.tzname()) + return (dt_local.date().isoformat(), dt_local.time().isoformat(), dt_local.tzname()) _logger = logging.getLogger(__name__) @@ -132,11 +128,11 @@ def serial_date_to_ISO_format(serial): # Plugin characteristics # ---------------------- -format_name = 'Blockfile' -description = 'Read/write support for ASTAR blockfiles' +format_name = "Blockfile" +description = "Read/write support for ASTAR blockfiles" full_support = False # Recognised file extension -file_extensions = ['blo', 'BLO'] +file_extensions = ["blo", "BLO"] default_extension = 0 # Writing capabilities: @@ -145,66 +141,73 @@ def serial_date_to_ISO_format(serial): mapping = { - 'blockfile_header.Beam_energy': - ("Acquisition_instrument.TEM.beam_energy", lambda x: x * 1e-3), - 'blockfile_header.Camera_length': - ("Acquisition_instrument.TEM.camera_length", lambda x: x * 1e-4), - 'blockfile_header.Scan_rotation': - ("Acquisition_instrument.TEM.rotation", lambda x: x * 1e-2), + "blockfile_header.Beam_energy": ( + "Acquisition_instrument.TEM.beam_energy", + lambda x: x * 1e-3, + ), + "blockfile_header.Camera_length": ( + "Acquisition_instrument.TEM.camera_length", + lambda x: x * 1e-4, + ), + "blockfile_header.Scan_rotation": ( + "Acquisition_instrument.TEM.rotation", + lambda x: x * 1e-2, + ), } -def get_header_dtype_list(endianess='<'): +def get_header_dtype_list(endianess="<"): end = endianess - dtype_list = \ + dtype_list = ( [ - ('ID', (bytes, 6)), - ('MAGIC', end + 'u2'), - ('Data_offset_1', end + 'u4'), # Offset VBF - ('Data_offset_2', end + 'u4'), # Offset DPs - ('UNKNOWN1', end + 'u4'), # Flags for ASTAR software? - ('DP_SZ', end + 'u2'), # Pixel dim DPs - ('DP_rotation', end + 'u2'), # [degrees ( * 100 ?)] - ('NX', end + 'u2'), # Scan dim 1 - ('NY', end + 'u2'), # Scan dim 2 - ('Scan_rotation', end + 'u2'), # [100 * degrees] - ('SX', end + 'f8'), # Pixel size [nm] - ('SY', end + 'f8'), # Pixel size [nm] - ('Beam_energy', end + 'u4'), # [V] - ('SDP', end + 'u2'), # Pixel size [100 * ppcm] - ('Camera_length', end + 'u4'), # [10 * mm] - ('Acquisition_time', end + 'f8'), # [Serial date] - ] + [ - ('Centering_N%d' % i, 'f8') for i in range(8) - ] + [ - ('Distortion_N%02d' % i, 'f8') for i in range(14) + ("ID", (bytes, 6)), + ("MAGIC", end + "u2"), + ("Data_offset_1", end + "u4"), # Offset VBF + ("Data_offset_2", end + "u4"), # Offset DPs + ("UNKNOWN1", end + "u4"), # Flags for ASTAR software? + ("DP_SZ", end + "u2"), # Pixel dim DPs + ("DP_rotation", end + "u2"), # [degrees ( * 100 ?)] + ("NX", end + "u2"), # Scan dim 1 + ("NY", end + "u2"), # Scan dim 2 + ("Scan_rotation", end + "u2"), # [100 * degrees] + ("SX", end + "f8"), # Pixel size [nm] + ("SY", end + "f8"), # Pixel size [nm] + ("Beam_energy", end + "u4"), # [V] + ("SDP", end + "u2"), # Pixel size [100 * ppcm] + ("Camera_length", end + "u4"), # [10 * mm] + ("Acquisition_time", end + "f8"), # [Serial date] ] + + [("Centering_N%d" % i, "f8") for i in range(8)] + + [("Distortion_N%02d" % i, "f8") for i in range(14)] + ) return dtype_list -def get_default_header(endianess='<'): +def get_default_header(endianess="<"): """Returns a header pre-populated with default values. """ dt = np.dtype(get_header_dtype_list()) header = np.zeros((1,), dtype=dt) - header['ID'][0] = 'IMGBLO'.encode() - header['MAGIC'][0] = magics[0] - header['Data_offset_1'][0] = 0x1000 # Always this value observed - header['UNKNOWN1'][0] = 131141 # Very typical value (always?) - header['Acquisition_time'][0] = datetime_to_serial_date( - datetime.datetime.fromtimestamp(86400, dateutil.tz.tzutc())) + header["ID"][0] = "IMGBLO".encode() + header["MAGIC"][0] = magics[0] + header["Data_offset_1"][0] = 0x1000 # Always this value observed + header["UNKNOWN1"][0] = 131141 # Very typical value (always?) + header["Acquisition_time"][0] = datetime_to_serial_date( + datetime.datetime.fromtimestamp(86400, dateutil.tz.tzutc()) + ) return header -def get_header_from_signal(signal, endianess='<'): +def get_header_from_signal(signal, endianess="<"): header = get_default_header(endianess) - if 'blockfile_header' in signal.original_metadata: - header = dict2sarray(signal.original_metadata['blockfile_header'], - sarray=header) - note = signal.original_metadata['blockfile_header']['Note'] + if "blockfile_header" in signal.original_metadata: + header = dict2sarray( + signal.original_metadata["blockfile_header"], sarray=header + ) + note = signal.original_metadata["blockfile_header"]["Note"] else: - note = '' + note = "" if signal.axes_manager.navigation_dimension == 2: NX, NY = signal.axes_manager.navigation_shape SX = signal.axes_manager.navigation_axes[0].scale @@ -219,21 +222,23 @@ def get_header_from_signal(signal, endianess='<'): DP_SZ = signal.axes_manager.signal_shape if DP_SZ[0] != DP_SZ[1]: - raise ValueError('Blockfiles require signal shape to be square!') + raise ValueError("Blockfiles require signal shape to be square!") DP_SZ = DP_SZ[0] - SDP = 100. / signal.axes_manager.signal_axes[0].scale + SDP = 100.0 / signal.axes_manager.signal_axes[0].scale - offset2 = NX * NY + header['Data_offset_1'] + offset2 = NX * NY + header["Data_offset_1"] # Based on inspected files, the DPs are stored at 16-bit boundary... # Normally, you'd expect word alignment (32-bits) ¯\_(°_o)_/¯ offset2 += offset2 % 16 header_sofar = { - 'NX': NX, 'NY': NY, - 'DP_SZ': DP_SZ, - 'SX': SX, 'SY': SY, - 'SDP': SDP, - 'Data_offset_2': offset2, + "NX": NX, + "NY": NY, + "DP_SZ": DP_SZ, + "SX": SX, + "SY": SY, + "SDP": SDP, + "Data_offset_2": offset2, } header = dict2sarray(header_sofar, sarray=header) @@ -242,7 +247,7 @@ def get_header_from_signal(signal, endianess='<'): def get_header(data_shape, scan_scale, diff_scale, endianess="<", **kwargs): header = get_default_header(endianess) - note = '' + note = "" if len(data_shape) == 4: NY, NX = data_shape[:2][::-1] # first dimension seems to by y in np SX = scan_scale @@ -259,21 +264,23 @@ def get_header(data_shape, scan_scale, diff_scale, endianess="<", **kwargs): DP_SZ = data_shape[-2:] if DP_SZ[0] != DP_SZ[1]: - raise ValueError('Blockfiles require signal shape to be square!') + raise ValueError("Blockfiles require signal shape to be square!") DP_SZ = DP_SZ[0] - SDP = 100. / diff_scale + SDP = 100.0 / diff_scale - offset2 = NX * NY + header['Data_offset_1'] + offset2 = NX * NY + header["Data_offset_1"] # Based on inspected files, the DPs are stored at 16-bit boundary... # Normally, you'd expect word alignment (32-bits) ¯\_(°_o)_/¯ offset2 += offset2 % 16 header_sofar = { - 'NX': NX, 'NY': NY, - 'DP_SZ': DP_SZ, - 'SX': SX, 'SY': SY, - 'SDP': SDP, - 'Data_offset_2': offset2, + "NX": NX, + "NY": NY, + "DP_SZ": DP_SZ, + "SX": SX, + "SY": SY, + "SDP": SDP, + "Data_offset_2": offset2, } header_sofar.update(kwargs) @@ -282,164 +289,177 @@ def get_header(data_shape, scan_scale, diff_scale, endianess="<", **kwargs): return header, note -def file_reader(filename, endianess='<', mmap_mode=None, - lazy=False, **kwds): +def file_reader(filename, endianess="<", mmap_mode=None, lazy=False, **kwds): _logger.debug("Reading blockfile: %s" % filename) metadata = {} if mmap_mode is None: - mmap_mode = 'r' if lazy else 'c' + mmap_mode = "r" if lazy else "c" # Makes sure we open in right mode: - if '+' in mmap_mode or ('write' in mmap_mode and - 'copyonwrite' != mmap_mode): + if "+" in mmap_mode or ("write" in mmap_mode and "copyonwrite" != mmap_mode): if lazy: raise ValueError("Lazy loading does not support in-place writing") - f = open(filename, 'r+b') + f = open(filename, "r+b") else: - f = open(filename, 'rb') + f = open(filename, "rb") _logger.debug("File opened") # Get header header = np.fromfile(f, dtype=get_header_dtype_list(endianess), count=1) - if header['MAGIC'][0] not in magics: - warnings.warn("Blockfile has unrecognized header signature. " - "Will attempt to read, but correcteness not guaranteed!") + if header["MAGIC"][0] not in magics: + warnings.warn( + "Blockfile has unrecognized header signature. " + "Will attempt to read, but correcteness not guaranteed!" + ) header = sarray2dict(header) - note = f.read(header['Data_offset_1'] - f.tell()) + note = f.read(header["Data_offset_1"] - f.tell()) # It seems it uses "\x00" for padding, so we remove it try: - header['Note'] = note.decode("latin1").strip("\x00") + header["Note"] = note.decode("latin1").strip("\x00") except Exception: # Not sure about the encoding so, if it fails, we carry on _logger.warn( "Reading the Note metadata of this file failed. " "You can help improving " "HyperSpy by reporting the issue in " - "https://github.com/hyperspy/hyperspy") + "https://github.com/hyperspy/hyperspy" + ) _logger.debug("File header: " + str(header)) - NX, NY = int(header['NX']), int(header['NY']) - DP_SZ = int(header['DP_SZ']) - if header['SDP']: - SDP = 100. / header['SDP'] + NX, NY = int(header["NX"]), int(header["NY"]) + DP_SZ = int(header["DP_SZ"]) + if header["SDP"]: + SDP = 100.0 / header["SDP"] else: SDP = -1 - original_metadata = {'blockfile_header': header} + original_metadata = {"blockfile_header": header} # Get data: # A Virtual BF/DF is stored first - offset1 = header['Data_offset_1'] + offset1 = header["Data_offset_1"] f.seek(offset1) - data_pre = np.fromfile(f, count=NX*NY, dtype=endianess+'u1' - ).squeeze().reshape((NY, NX), order='C') + data_pre = ( + np.fromfile(f, count=NX * NY, dtype=endianess + "u1") + .squeeze() + .reshape((NY, NX), order="C") + ) # Then comes actual blockfile - offset2 = header['Data_offset_2'] + offset2 = header["Data_offset_2"] if not lazy: f.seek(offset2) - data = np.fromfile(f, dtype=endianess + 'u1') + data = np.fromfile(f, dtype=endianess + "u1") else: - data = np.memmap(f, mode=mmap_mode, offset=offset2, - dtype=endianess + 'u1') + data = np.memmap(f, mode=mmap_mode, offset=offset2, dtype=endianess + "u1") try: data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) except ValueError: warnings.warn( - 'Blockfile header dimensions larger than file size! ' - 'Will attempt to load by zero padding incomplete frames.') + "Blockfile header dimensions larger than file size! " + "Will attempt to load by zero padding incomplete frames." + ) # Data is stored DP by DP: pw = [(0, NX * NY * (DP_SZ * DP_SZ + 6) - data.size)] - data = np.pad(data, pw, mode='constant') + data = np.pad(data, pw, mode="constant") data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) # Every frame is preceeded by a 6 byte sequence (AA 55, and then a 4 byte # integer specifying frame number) data = data[:, :, 6:] - data = data.reshape((NY, NX, DP_SZ, DP_SZ), order='C').squeeze() - - units = ['nm', 'nm', 'cm', 'cm'] - names = ['y', 'x', 'dy', 'dx'] - scales = [header['SY'], header['SX'], SDP, SDP] - date, time, time_zone = serial_date_to_ISO_format( - header['Acquisition_time']) - metadata = {'General': {'original_filename': os.path.split(filename)[1], - 'date': date, - 'time': time, - 'time_zone': time_zone, - 'notes': header['Note']}, - "Signal": {'signal_type': "diffraction", - 'record_by': 'image', }, - } + data = data.reshape((NY, NX, DP_SZ, DP_SZ), order="C").squeeze() + + units = ["nm", "nm", "cm", "cm"] + names = ["y", "x", "dy", "dx"] + scales = [header["SY"], header["SX"], SDP, SDP] + date, time, time_zone = serial_date_to_ISO_format(header["Acquisition_time"]) + metadata = { + "General": { + "original_filename": os.path.split(filename)[1], + "date": date, + "time": time, + "time_zone": time_zone, + "notes": header["Note"], + }, + "Signal": {"signal_type": "diffraction", "record_by": "image",}, + } # Create the axis objects for each axis dim = data.ndim axes = [ { - 'size': data.shape[i], - 'index_in_array': i, - 'name': names[i], - 'scale': scales[i], - 'offset': 0.0, - 'units': units[i], } - for i in range(dim)] - - dictionary = {'data': data, - 'vbf': data_pre, - 'axes': axes, - 'metadata': metadata, - 'original_metadata': original_metadata, - 'mapping': mapping, } + "size": data.shape[i], + "index_in_array": i, + "name": names[i], + "scale": scales[i], + "offset": 0.0, + "units": units[i], + } + for i in range(dim) + ] + + dictionary = { + "data": data, + "vbf": data_pre, + "axes": axes, + "metadata": metadata, + "original_metadata": original_metadata, + "mapping": mapping, + } f.close() - return [dictionary, ] + return [ + dictionary, + ] def file_writer(filename, signal, **kwds): - endianess = kwds.pop('endianess', '<') + endianess = kwds.pop("endianess", "<") header, note = get_header_from_signal(signal, endianess=endianess) - with open(filename, 'wb') as f: + with open(filename, "wb") as f: # Write header header.tofile(f) # Write header note field: - if len(note) > int(header['Data_offset_1']) - f.tell(): - note = note[:int(header['Data_offset_1']) - f.tell() - len(note)] + if len(note) > int(header["Data_offset_1"]) - f.tell(): + note = note[: int(header["Data_offset_1"]) - f.tell() - len(note)] f.write(note.encode()) # Zero pad until next data block - zero_pad = int(header['Data_offset_1']) - f.tell() + zero_pad = int(header["Data_offset_1"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write virtual bright field - vbf = signal.mean( - signal.axes_manager.signal_axes[ - :2]).data.astype( - endianess + - 'u1') + vbf = signal.mean(signal.axes_manager.signal_axes[:2]).data.astype( + endianess + "u1" + ) vbf.tofile(f) # Zero pad until next data block - if f.tell() > int(header['Data_offset_2']): - raise ValueError("Signal navigation size does not match " - "data dimensions.") - zero_pad = int(header['Data_offset_2']) - f.tell() + if f.tell() > int(header["Data_offset_2"]): + raise ValueError( + "Signal navigation size does not match " "data dimensions." + ) + zero_pad = int(header["Data_offset_2"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write full data stack: # We need to pad each image with magic 'AA55', then a u32 serial - dp_head = np.zeros((1,), dtype=[('MAGIC', endianess + 'u2'), - ('ID', endianess + 'u4')]) - dp_head['MAGIC'] = 0x55AA + dp_head = np.zeros( + (1,), dtype=[("MAGIC", endianess + "u2"), ("ID", endianess + "u4")] + ) + dp_head["MAGIC"] = 0x55AA # Write by loop: for img in signal._iterate_signal(): dp_head.tofile(f) - img.astype(endianess + 'u1').tofile(f) - dp_head['ID'] += 1 + img.astype(endianess + "u1").tofile(f) + dp_head["ID"] += 1 class bloFileWriter(QThread): """Write a blo file from an HDF5 without loading all data in memory""" + increase_progress = pyqtSignal(int) finish = pyqtSignal() - def __init__(self, fh, path_blo, shape, indexes, - scan_scale=5, diff_scale=1.075): + def __init__( + self, fh, path_blo, shape, indexes, scan_scale=5, diff_scale=1.075, **options + ): QThread.__init__(self) self.fh = fh # open hdf5 file in read mode self.path_blo = path_blo @@ -453,6 +473,9 @@ def __init__(self, fh, path_blo, shape, indexes, self.vbf_im = imagefun.normalize_convert(self.vbf_im, dtype=np.uint8) logger.debug("Initialized bloFileWriter") + # added for 8-bit rescaling option + self.options = options + def run(self): self.convert_to_blo() self.finish.emit() @@ -461,12 +484,16 @@ def run(self): def convert_to_blo(self): endianess = "<" header, note = get_header( - self.shape, self.scan_scale, - self.diff_scale, endianess, - Camera_length=100.0*self.fh["ImageStream"].attrs['magtotal'], - Beam_energy=self.fh["ImageStream"].attrs['ht']*1000, - Distortion_N01=1.0, Distortion_N09=1.0, - Note="Reconstructed from TVIPS image stream") + self.shape, + self.scan_scale, + self.diff_scale, + endianess, + Camera_length=100.0 * self.fh["ImageStream"].attrs["magtotal"], + Beam_energy=self.fh["ImageStream"].attrs["ht"] * 1000, + Distortion_N01=1.0, + Distortion_N09=1.0, + Note="Reconstructed from TVIPS image stream", + ) logger.debug("Created header of blo file") with open(self.path_blo, "wb") as f: @@ -474,59 +501,63 @@ def convert_to_blo(self): header.tofile(f) logger.debug("Wrote header to file") # Write header note field: - if len(note) > int(header['Data_offset_1']) - f.tell(): - note = note[:int(header['Data_offset_1']) - - f.tell() - len(note)] + if len(note) > int(header["Data_offset_1"]) - f.tell(): + note = note[: int(header["Data_offset_1"]) - f.tell() - len(note)] f.write(note.encode()) # Zero pad until next data block - zero_pad = int(header['Data_offset_1']) - f.tell() + zero_pad = int(header["Data_offset_1"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write virtual bright field vbf = self.vbf_im.astype(endianess + "u1") vbf.tofile(f) # Zero pad until next data block - if f.tell() > int(header['Data_offset_2']): - raise ValueError("Signal navigation size does not match " - "data dimensions.") - zero_pad = int(header['Data_offset_2']) - f.tell() + if f.tell() > int(header["Data_offset_2"]): + raise ValueError( + "Signal navigation size does not match " "data dimensions." + ) + zero_pad = int(header["Data_offset_2"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write full data stack: # We need to pad each image with magic 'AA55', then a u32 serial - dp_head = np.zeros((1,), dtype=[('MAGIC', endianess + 'u2'), - ('ID', endianess + 'u4')]) - dp_head['MAGIC'] = 0x55AA + dp_head = np.zeros( + (1,), dtype=[("MAGIC", endianess + "u2"), ("ID", endianess + "u4")] + ) + dp_head["MAGIC"] = 0x55AA # Write by loop: logger.debug("Wrote header part of blo file") for j, indx in enumerate(self.indexes): dp_head.tofile(f) c = f"{indx}".zfill(6) img = self.fh["ImageStream"][f"Frame_{c}"][:] - img = imagefun.normalize_convert(img, dtype=np.uint8) - img.astype(endianess + 'u1').tofile(f) - dp_head['ID'] += 1 + + if "rescale" in self.options and self.options["rescale"]: + img = imagefun.normalize_convert(img, dtype=np.uint8) + else: + img = util.img_as_ubyte(img) + img.astype(endianess + "u1").tofile(f) + dp_head["ID"] += 1 self.update_gui_progress(j) logger.debug(f"Wrote frame Frame_{c} to blo file") def update_gui_progress(self, j): """If using the GUI update features with progress""" - value = int((j+1)/len(self.indexes)*100) + value = int((j + 1) / len(self.indexes) * 100) self.increase_progress.emit(value) def file_writer_array(filename, array, scan_scale, diff_scale, **kwds): endianess = "<" - header, note = get_header(array.shape, scan_scale, diff_scale, endianess, - **kwds) + header, note = get_header(array.shape, scan_scale, diff_scale, endianess, **kwds) - with open(filename, 'wb') as f: + with open(filename, "wb") as f: # Write header header.tofile(f) # Write header note field: - if len(note) > int(header['Data_offset_1']) - f.tell(): - note = note[:int(header['Data_offset_1']) - f.tell() - len(note)] + if len(note) > int(header["Data_offset_1"]) - f.tell(): + note = note[: int(header["Data_offset_1"]) - f.tell() - len(note)] f.write(note.encode()) # Zero pad until next data block - zero_pad = int(header['Data_offset_1']) - f.tell() + zero_pad = int(header["Data_offset_1"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write virtual bright field vbf = None @@ -535,8 +566,7 @@ def file_writer_array(filename, array, scan_scale, diff_scale, **kwds): amean = (2, 3) # do proper vbf xx, yy = np.meshgrid(range(array.shape[2]), range(array.shape[3])) - mask = np.hypot(xx - 0.5 * array.shape[2], - yy - 0.5 * array.shape[3]) < 5 + mask = np.hypot(xx - 0.5 * array.shape[2], yy - 0.5 * array.shape[3]) < 5 # TODO: make radius and offset configurable vbffloat = np.zeros((array.shape[:2])) @@ -550,23 +580,25 @@ def file_writer_array(filename, array, scan_scale, diff_scale, **kwds): elif len(array.shape) == 3: amean = (1, 2) - vbf = array.mean(axis=amean).astype(endianess + 'u1') + vbf = array.mean(axis=amean).astype(endianess + "u1") vbf.tofile(f) # Zero pad until next data block - if f.tell() > int(header['Data_offset_2']): - raise ValueError("Signal navigation size does not match " - "data dimensions.") - zero_pad = int(header['Data_offset_2']) - f.tell() + if f.tell() > int(header["Data_offset_2"]): + raise ValueError( + "Signal navigation size does not match " "data dimensions." + ) + zero_pad = int(header["Data_offset_2"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write full data stack: # We need to pad each image with magic 'AA55', then a u32 serial - dp_head = np.zeros((1,), dtype=[('MAGIC', endianess + 'u2'), - ('ID', endianess + 'u4')]) - dp_head['MAGIC'] = 0x55AA + dp_head = np.zeros( + (1,), dtype=[("MAGIC", endianess + "u2"), ("ID", endianess + "u4")] + ) + dp_head["MAGIC"] = 0x55AA # Write by loop: - for img in array.reshape(array.shape[0]*array.shape[1], - *array.shape[2:]): + for img in array.reshape(array.shape[0] * array.shape[1], *array.shape[2:]): dp_head.tofile(f) - img.astype(endianess + 'u1').tofile(f) - dp_head['ID'] += 1 + img.astype(endianess + "u1").tofile(f) + dp_head["ID"] += 1 + diff --git a/tvipsconverter/utils/imagefun.py b/tvipsconverter/utils/imagefun.py index 1a0105b..4a2621b 100644 --- a/tvipsconverter/utils/imagefun.py +++ b/tvipsconverter/utils/imagefun.py @@ -13,10 +13,16 @@ def _get_dtype_min_max(dtype): if dtype == np.float or dtype == np.float32 or dtype == np.float64: max = 1 # np.finfo(dtype).max min = 0 # np.finfo(dtype).min - elif (dtype == np.int8 or dtype == np.uint8 or - dtype == np.int16 or dtype == np.uint16 or - dtype == np.int32 or dtype == np.uint32 or - dtype == np.int64 or dtype == np.uint64): + elif ( + dtype == np.int8 + or dtype == np.uint8 + or dtype == np.int16 + or dtype == np.uint16 + or dtype == np.int32 + or dtype == np.uint32 + or dtype == np.int64 + or dtype == np.uint64 + ): max = np.iinfo(dtype).max min = np.iinfo(dtype).min else: @@ -119,8 +125,8 @@ def linscale(arr, min=None, max=None, nmin=0, nmax=1, dtype=np.float): else: workarr[workarr > max] = max - a = (nmax-nmin)/(max-min) - result = (workarr-min)*a+nmin + a = (nmax - nmin) / (max - min) + result = (workarr - min) * a + nmin return result.astype(dtype) @@ -142,10 +148,10 @@ def matlab_style_gauss2D(shape=(3, 3), sigma=0.5): 2D gaussian mask - should give the same result as MATLAB's fspecial('gaussian',[shape],[sigma]) """ - m, n = [(ss-1.)/2. for ss in shape] - y, x = np.ogrid[-m:m+1, -n:n+1] - h = np.exp(-(x*x + y*y) / (2.*sigma*sigma)) - h[h < np.finfo(h.dtype).eps*h.max()] = 0 + m, n = [(ss - 1.0) / 2.0 for ss in shape] + y, x = np.ogrid[-m : m + 1, -n : n + 1] + h = np.exp(-(x * x + y * y) / (2.0 * sigma * sigma)) + h[h < np.finfo(h.dtype).eps * h.max()] = 0 sumh = h.sum() if sumh != 0: h /= sumh @@ -168,14 +174,14 @@ def findoutliers(raw, percent=0.07): Uses returns min and max values of the array with the upper and lower percent pixel values suppressed. """ - cnts, edges = np.histogram(raw, bins=2**16) - stats = np.zeros((2, 2**16), dtype=np.int) + cnts, edges = np.histogram(raw, bins=2 ** 16) + stats = np.zeros((2, 2 ** 16), dtype=np.int) stats[0] = np.cumsum(cnts) # low stats[1] = np.cumsum(cnts[::-1]) # high thresh = stats > percent * raw.shape[0] * raw.shape[1] min = (np.where(thresh[0]))[0][0] - max = 2**16 - (np.where(thresh[1]))[0][0] - return edges[min], edges[max+1] + max = 2 ** 16 - (np.where(thresh[1]))[0][0] + return edges[min], edges[max + 1] def suppressoutliers(raw, percent=0.07): @@ -191,12 +197,33 @@ def bin2(a, factor): imag = Image.fromarray(a) # binned = Image.resize(imag, (a.shape[0]//factor, a.shape[1]//factor), # resample=Image.BILINEAR) - binned = np.array(imag.resize((a.shape[0]//factor, a.shape[1]//factor), - resample=Image.NEAREST)) - print(binned.dtype) + binned = np.array( + imag.resize( + (a.shape[0] // factor, a.shape[1] // factor), resample=Image.NEAREST + ) + ) return binned +def bin_box(a, factor): + """ + + Use box averaging to bin the images. + + """ + assert all( + not i % factor for i in a.shape + ), "array shape is not factorisable by factor." + # should work ndim + slices = tuple( + tuple((slice(j, None, factor)) for i in range(a.ndim)) for j in range(factor) + ) + + # stack th offset slices and take mean down stack axis to finish binning + cube = np.stack([a[s] for s in slices], axis=0) + return cube.mean(axis=0) + + def getElectronWavelength(ht): # ht in Volts, length unit in meters h = 6.6e-34 @@ -204,5 +231,5 @@ def getElectronWavelength(ht): charge = 1.6e-19 c = 3e8 wavelength = h / math.sqrt(2 * m * charge * ht) - relativistic_correction = 1 / math.sqrt(1 + ht * charge/(2 * m * c * c)) + relativistic_correction = 1 / math.sqrt(1 + ht * charge / (2 * m * c * c)) return wavelength * relativistic_correction diff --git a/tvipsconverter/utils/recorder.py b/tvipsconverter/utils/recorder.py index b9b5975..6d82b2d 100644 --- a/tvipsconverter/utils/recorder.py +++ b/tvipsconverter/utils/recorder.py @@ -1,5 +1,6 @@ import numpy as np import os.path +from scipy.ndimage import gaussian_filter from tifffile import FileHandle import math import h5py @@ -8,70 +9,73 @@ from PyQt5.QtCore import QThread, pyqtSignal import logging -from .imagefun import (normalize_convert, bin2, gausfilter, - medfilter) +from .imagefun import normalize_convert, bin2, bin_box, gausfilter, medfilter # Initialize the Logger logger = logging.getLogger(__name__) TVIPS_RECORDER_GENERAL_HEADER = [ - ('size', 'u4'), # unused - likely the size of generalheader in bytes - ('version', 'u4'), # 1 or 2 - ('dimx', 'u4'), # dp image size width - ('dimy', 'u4'), # dp image size height - ('bitsperpixel', 'u4'), # 8 or 16 - ('offsetx', 'u4'), # generally 0 - ('offsety', 'u4'), - ('binx', 'u4'), # camera binning - ('biny', 'u4'), - ('pixelsize', 'u4'), # nm, physical pixel size - ('ht', 'u4'), # high tension, voltage - ('magtotal', 'u4'), # magnification/camera length? - ('frameheaderbytes', 'u4'), # number of bytes per frame header - ('dummy', 'S204'), # just writes out TVIPS TVIPS TVIPS - ] + ("size", "u4"), # unused - likely the size of generalheader in bytes + ("version", "u4"), # 1 or 2 + ("dimx", "u4"), # dp image size width + ("dimy", "u4"), # dp image size height + ("bitsperpixel", "u4"), # 8 or 16 + ("offsetx", "u4"), # generally 0 + ("offsety", "u4"), + ("binx", "u4"), # camera binning + ("biny", "u4"), + ("pixelsize", "u4"), # nm, physical pixel size + ("ht", "u4"), # high tension, voltage + ("magtotal", "u4"), # magnification/camera length? + ("frameheaderbytes", "u4"), # number of bytes per frame header + ("dummy", "S204"), # just writes out TVIPS TVIPS TVIPS +] TVIPS_RECORDER_FRAME_HEADER = [ - ('num', 'u4'), # seems to cycle also - ('timestamp', 'u4'), # seconds since 1.1.1970 - ('ms', 'u4'), # additional milliseconds to the timestamp - ('LUTidx', 'u4'), # always the same value - ('fcurrent', 'f4'), # 0 for all frames - ('mag', 'u4'), # same for all frames - ('mode', 'u4'), # 1 -> image 2 -> diff - ('stagex', 'f4'), - ('stagey', 'f4'), - ('stagez', 'f4'), - ('stagea', 'f4'), - ('stageb', 'f4'), - ('rotidx', 'u4'), - ('temperature', 'f4'), # cycles between 0.0 and 9.0 with step 1.0 - ('objective', 'f4'), # kind of randomly between 0.0 and 1.0 + ("num", "u4"), # seems to cycle also + ("timestamp", "u4"), # seconds since 1.1.1970 + ("ms", "u4"), # additional milliseconds to the timestamp + ("LUTidx", "u4"), # always the same value + ("fcurrent", "f4"), # 0 for all frames + ("mag", "u4"), # same for all frames + ("mode", "u4"), # 1 -> image 2 -> diff + ("stagex", "f4"), + ("stagey", "f4"), + ("stagez", "f4"), + ("stagea", "f4"), + ("stageb", "f4"), + ("rotidx", "u4"), + ("temperature", "f4"), # cycles between 0.0 and 9.0 with step 1.0 + ("objective", "f4"), # kind of randomly between 0.0 and 1.0 # for header version 2, some more data might be present - ] +] FILTER_DEFAULTS = { - "useint": False, "whichint": 65536, - "usebin": False, "whichbin": 1, "usegaus": False, - "gausks": 8, "gaussig": 4, "usemed": False, - "medks": 4, "usels": False, "lsmin": 10, - "lsmax": 1000, "usecoffset": False - } - - -VBF_DEFAULTS = { - "calcvbf": True, "vbfrad": 10, "vbfxoffset": 0, - "vbfyoffset": 0 - } - - -def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, - binning=1): + "useint": False, + "whichint": 65536, + "usebin": False, + "whichbin": 1, + "usegaus": False, + "gausks": 8, + "gaussig": 4, + "usemed": False, + "medks": 4, + "usels": False, + "lsmin": 10, + "lsmax": 1000, + "usecoffset": False, +} + + +VBF_DEFAULTS = {"calcvbf": True, "vbfrad": 10, "vbfxoffset": 0, "vbfyoffset": 0} + + +def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, binning=1): """Do some kind of intensity correction, unsure reason""" pixperchannel = int(128 / binning) # binning has to be an integer - if (128.0/binning != pixperchannel): + if 128.0 / binning != pixperchannel: logger.error("Can't figure out column offset dimension") return image numcol = int(image.shape[0] / 128 * binning) @@ -81,8 +85,7 @@ def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, for j in range(numcol): channel = imtemp[:, j, :] # pdb.set_trace() - mask = np.bitwise_and(channel < thresholdmax, - channel >= thresholdmin) + mask = np.bitwise_and(channel < thresholdmax, channel >= thresholdmin) value = np.mean(channel[mask]) offsets.append(value) # apply offset correction to images @@ -93,10 +96,23 @@ def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, return subtracted.reshape(image.shape) -def filter_image(imag, useint, whichint, usebin, - whichbin, usegaus, gausks, - gaussig, usemed, medks, - usels, lsmin, lsmax, usecoffset): +def filter_image( + imag, + useint, + whichint, + usebin, + whichbin, + usegaus, + gausks, + gaussig, + usemed, + medks, + usels, + lsmin, + lsmax, + usecoffset, + bintype, +): """ Filter an image and return the filtered image """ @@ -105,7 +121,17 @@ def filter_image(imag, useint, whichint, usebin, imag = np.where(imag > whichint, 0, imag) # binning by some factor if usebin: - imag = bin2(imag, whichbin) + if bintype: # if True use interpolate, else box + imag = bin2(imag, whichbin) + else: + # test binning factor will work + if all(not i % whichbin for i in imag.shape): + imag = bin_box(imag, whichbin) + else: + logger.warning( + "array shape is not factorisable by factor, using decimation instead." + ) + imag = bin2(imag, whichbin) # median filter if usemed: imag = medfilter(imag, medks) @@ -127,13 +153,31 @@ def getOriginalPreviewImage(path, improc, vbfsettings, frame=0): return firstframe +def write_scan_parameters_hdf5(path, **parameters): + """ + Write parameters as attrs to a hdf5 file. + + Parameters are written under h5['Scan'].attrs + """ + with h5py.File(path, "r+") as f: + for key, val in parameters.items(): + f["Scan"].attrs[key] = val + + class Recorder(QThread): increase_progress = pyqtSignal(int) finish = pyqtSignal() - def __init__(self, path, improc=None, vbfsettings=None, - outputpath=None, imrange=(None, None)): + def __init__( + self, + path, + improc=None, + vbfsettings=None, + outputpath=None, + imrange=(None, None), + **options, + ): QThread.__init__(self) logger.debug("Initializing recorder object") # filename @@ -186,6 +230,10 @@ def __init__(self, path, improc=None, vbfsettings=None, if self.outputpath is not None: self.outputpath = str(Path(self.outputpath)) + # other options, added 2.11.2020 + self.options = options + # print(options, self.options) + def run(self): self.convert_HDF5() self.finish.emit() @@ -211,13 +259,26 @@ def convert_HDF5(self): # This is important! also initializes the headers! firstframe = self.read_frame(0) pff = filter_image(firstframe, **self.improc) + + # initialise maximum_image if checkBox checked + # print(f"average image: {'calcmax' in self.options} {self.options['calcmax']}") + if "calcmax" in self.options and self.options["calcmax"]: + # print("creating maxiumum image") + self.maximum_image = np.zeros_like(pff) + + # print(f"average image: {'calcave' in self.options} {self.options['calcave']}") + # initialise average_image if checkBox checked + if "calcave" in self.options and self.options["calcave"]: + # print("creating average_image") + # as datatype is typcially 8- or 16-bit then 32-bit image should be fine + self.average_image = np.zeros_like(pff, dtype=np.float32) + self.scangroup = self.stream.create_group("Scan") # do we need a virtual bright field calculated? if self.vbfproc["calcvbf"]: # make a VBF mask. For this we need a frame # ZOB center offset - zoboffset = [self.vbfproc["vbfxoffset"], - self.vbfproc["vbfyoffset"]] + zoboffset = [self.vbfproc["vbfxoffset"], self.vbfproc["vbfyoffset"]] radius = self.vbfproc["vbfrad"] # generate mask self.mask = self._virtual_bf_mask(pff, zoboffset, radius) @@ -245,11 +306,13 @@ def valid_first_tvips_file(filename): if match is not None: num, ext = match.groups() if ext != "tvips": - raise ValueError(f"Invalid tvips file: extension {ext}, must " - f"be tvips") + raise ValueError( + f"Invalid tvips file: extension {ext}, must " f"be tvips" + ) if int(num) != 0: - raise ValueError("Can only read video sequences starting with " - "part 000") + raise ValueError( + "Can only read video sequences starting with " "part 000" + ) return True else: raise ValueError("Could not recognize as a valid tvips file") @@ -261,10 +324,8 @@ def read_frame(self, frame=0): fh = FileHandle(file=f) fh.seek(bite_start - self.ranges[toopen][0]) frame = np.fromfile( - fh, - count=self.general.dimx*self.general.dimy, - dtype=self.dtype - ) + fh, count=self.general.dimx * self.general.dimy, dtype=self.dtype + ) frame.shape = (self.general.dimx, self.general.dimy) return frame @@ -288,10 +349,14 @@ def _get_byte_filename(self, b): def _get_frame_byte_position(self, frame): """Get the byte where a frame starts""" - frame_byte_size = (self.general.dimx * self.general.dimy * - self.general.bitsperpixel//8) - bite_start = (self.generalheadersize + self.general.frameheaderbytes + - (frame_byte_size + self.general.frameheaderbytes)*frame) + frame_byte_size = ( + self.general.dimx * self.general.dimy * self.general.bitsperpixel // 8 + ) + bite_start = ( + self.generalheadersize + + self.general.frameheaderbytes + + (frame_byte_size + self.general.frameheaderbytes) * frame + ) return bite_start def _get_frame_byte_position_in_file(self, frame): @@ -303,21 +368,24 @@ def _get_frame_byte_position_in_file(self, frame): def _get_byte_frame_position(self, byte_start): """Get the frame index closest corresponding to a byte""" - frame_byte_size = (self.general.dimx * self.general.dimy * - self.general.bitsperpixel//8) - frame = ((byte_start - self.generalheadersize - - self.general.frameheaderbytes) // - (frame_byte_size + self.general.frameheaderbytes)) + frame_byte_size = ( + self.general.dimx * self.general.dimy * self.general.bitsperpixel // 8 + ) + frame = ( + byte_start - self.generalheadersize - self.general.frameheaderbytes + ) // (frame_byte_size + self.general.frameheaderbytes) return frame def _get_files_size_dictionary(self): """ Get a dictionary of files (keys) and total size (values) """ + def get_filesize(fn): with open(fn, "rb") as f: fh = FileHandle(file=f) return fn, fh.size + sizes = self._scan_over_all_files(get_filesize) return dict(sizes) @@ -329,7 +397,7 @@ def _get_files_ranges(self): ranges = {} starts = 0 for i in sizes: - ends = starts+sizes[i] + ends = starts + sizes[i] ranges[i] = (starts, ends) starts = ends return ranges @@ -354,8 +422,7 @@ def _frames_exceeded(self): """Have we read the number of frames?""" if self.finalim is not None: if self.current_frame >= self.finalim: - logger.debug(f"We are at frame {self.current_frame}. " - f"Quitting.") + logger.debug(f"We are at frame {self.current_frame}. " f"Quitting.") return True return False @@ -365,11 +432,10 @@ def _scan_over_all_files(self, func, *args, **kwargs): results = [] part = int(self.filename[-9:-6]) if part != 0: - raise ValueError("Can only read video sequences starting with " - "part 000") + raise ValueError("Can only read video sequences starting with " "part 000") try: while True: - fn = self.filename[:-9]+"{:03d}.tvips".format(part) + fn = self.filename[:-9] + "{:03d}.tvips".format(part) if not os.path.exists(fn): logger.debug(f"There is no file {fn}; breaking loop") break @@ -404,13 +470,15 @@ def _readGeneral(self, fh): self.inc = self.general.frameheaderbytes self.frame_header = TVIPS_RECORDER_FRAME_HEADER else: - raise NotImplementedError(f"Version {self.general.version} not " - f"yet supported.") + raise NotImplementedError( + f"Version {self.general.version} not " f"yet supported." + ) self.dt = np.dtype(self.frame_header) # make sure the record consumes less bytes than reported in the main # header - assert self.inc >= self.dt.itemsize, ("The record consumes more bytes " - "than stated in the main header") + assert self.inc >= self.dt.itemsize, ( + "The record consumes more bytes " "than stated in the main header" + ) def _readIndividualFile(self, fn): """ @@ -429,7 +497,8 @@ def _readIndividualFile(self, fn): if self.startim > self.current_frame: self.current_frame += 1 current_byte = self._get_frame_byte_position_in_file( - self.current_frame) + self.current_frame + ) fh.seek(current_byte - self.general.frameheaderbytes) continue if self.finalim < self.current_frame: @@ -442,16 +511,16 @@ def _readIndividualFile(self, fn): def _readFrame(self, fh, record=None): # read frame header header = fh.read_record(self.frame_header) - logger.debug(f"{self.current_frame}: Starting frame read " - f"(pos: {fh.tell()}). rot: {header['rotidx']}") + logger.debug( + f"{self.current_frame}: Starting frame read " + f"(pos: {fh.tell()}). rot: {header['rotidx']}" + ) skip = self.inc - self.dt.itemsize fh.seek(skip, 1) # read frame frame = np.fromfile( - fh, - count=self.general.dimx*self.general.dimy, - dtype=self.dtype - ) + fh, count=self.general.dimx * self.general.dimy, dtype=self.dtype + ) frame.shape = (self.general.dimx, self.general.dimy) # do calculations on the frame frame = filter_image(frame, **self.improc) @@ -461,16 +530,53 @@ def _readFrame(self, fh, record=None): for i in self.frame_header: ds.attrs[i[0]] = header[i[0]] # store the rotation index for finding start and stop later - self.rotidxs.append(header['rotidx']) + self.rotidxs.append(header["rotidx"]) # immediately calculate and store the VBF intensity if required if self.vbfproc["calcvbf"]: vbf_int = frame[self.mask].mean() self.vbfs.append(vbf_int) + # calculate images as specified in the options + if "calcmax" in self.options and self.options["calcmax"]: + # maximum_image should already be initialised in self.convert_HDF5 + self.maximum_image = np.stack((self.maximum_image, frame), axis=0).max( + axis=0 + ) + + if "calcave" in self.options and self.options["calcave"]: + # average_image should already be initialised in self.convert_HDF5 + self.average_image = np.stack( + ( + self.average_image, + # frame is scaled as a function of 1/total_frames and then summed (average) + (1 / (self.finalim - self.startim)) + * frame.astype(self.average_image.dtype), + ), + axis=0, + ).sum(axis=0) + + if ( + "refine_center" in self.options and self.options["refine_center"][0] + ): # make sure is checked + side, sigma = self.options["refine_center"][1:] + center = np.array(frame.shape) // 2 + + crop = frame[ + center[0] - side // 2 : center[0] + side // 2, + center[1] - side // 2 : center[1] + side // 2, + ] + # blur crop and find maximum -> use as center location + blurred = gaussian_filter(crop, sigma, mode="nearest") + # add crop offset (center - side//2) to get actual location on frame + ds.attrs["Center location"] = np.unravel_index( + blurred.argmax(), crop.shape + ) + (center - side // 2) + def _update_gui_progess(self): """If using the GUI update features with progress""" - value = int((self.current_frame - self.startim) / - (self.finalim - self.startim)*100) + value = int( + (self.current_frame - self.startim) / (self.finalim - self.startim) * 100 + ) self.increase_progress.emit(value) def _find_start_and_stop(self): @@ -478,40 +584,33 @@ def _find_start_and_stop(self): previous = self.rotidxs[0] for j, i in enumerate(self.rotidxs): if i > previous: - self.start = j-1 + self.start = j - 1 logger.info(f"Found start at frame {j-1}") - self.scangroup.attrs[ - "start_frame"] = self.start + self.scangroup.attrs["start_frame"] = self.start break previous = i else: self.start = None - self.scangroup.attrs[ - "start_frame"] = "None" + self.scangroup.attrs["start_frame"] = "None" # loop over it backwards to find the end # infact the index goes back to 1 for j, i in reversed(list(enumerate(self.rotidxs))): if i > 1: self.end = j logger.info(f"Found final at frame {j}") - self.scangroup.attrs[ - "end_frame"] = self.end - self.scangroup.attrs[ - "final_rotinx"] = i + self.scangroup.attrs["end_frame"] = self.end + self.scangroup.attrs["final_rotinx"] = i self.final_rotinx = i break else: self.end = None - self.scangroup.attrs[ - "end_frame"] = "None" + self.scangroup.attrs["end_frame"] = "None" self.final_rotinx = None - self.scangroup.attrs[ - "final_rotinx"] = "None" + self.scangroup.attrs["final_rotinx"] = "None" # add a couple more attributes for good measure self.scangroup.attrs["total_stream_frames"] = len(self.rotidxs) if self.end is not None and self.start is not None: - self.scangroup.attrs[ - "ims_between_start_end"] = self.end-self.start + self.scangroup.attrs["ims_between_start_end"] = self.end - self.start def _save_preliminary_scan_info(self): # save rotation indexes and vbf intensities @@ -519,11 +618,19 @@ def _save_preliminary_scan_info(self): if self.vbfproc["calcvbf"]: self.scangroup.create_dataset("vbf_intensities", data=self.vbfs) + # save options + if "calcmax" in self.options and self.options["calcmax"]: + self.scangroup.create_dataset("maximum_image", data=self.maximum_image) + if "calcave" in self.options and self.options["calcave"]: + self.scangroup.create_dataset("average_image", data=self.average_image) + @staticmethod def _virtual_bf_mask(arr, centeroffsetpx=(0, 0), radiuspx=10): """Create virtual bright field mask""" - xx, yy = np.meshgrid(np.arange(arr.shape[0], dtype=np.float), - np.arange(arr.shape[1], dtype=np.float)) + xx, yy = np.meshgrid( + np.arange(arr.shape[0], dtype=np.float), + np.arange(arr.shape[1], dtype=np.float), + ) xx -= 0.5 * arr.shape[0] + centeroffsetpx[0] yy -= 0.5 * arr.shape[1] + centeroffsetpx[1] mask = np.hypot(xx, yy) < radiuspx @@ -531,27 +638,33 @@ def _virtual_bf_mask(arr, centeroffsetpx=(0, 0), radiuspx=10): def determine_recorder_image_dimension(self, opts): # scan dimensions - if (opts.dimension is not None): - self.xdim, self.ydim = list(map(int, opts.dimension.split('x'))) + if opts.dimension is not None: + self.xdim, self.ydim = list(map(int, opts.dimension.split("x"))) else: dim = math.sqrt(self.final_rotinx) if not dim == int(dim): - raise ValueError("Can't determine correct image dimensions, " - "please supply values manually (--dimension)") + raise ValueError( + "Can't determine correct image dimensions, " + "please supply values manually (--dimension)" + ) self.xdim, self.ydim = dim, dim - logger.debug("Image dimensions: {}x{}".format(self.xdim, - self.ydim)) + logger.debug("Image dimensions: {}x{}".format(self.xdim, self.ydim)) class hdf5Intermediate(h5py.File): """This class represents the intermediate hdf5 file handle""" + def __init__(self, filepath, mode="r"): super().__init__(filepath, mode) - (self.total_frames, - self.start_frame, - self.end_frame, - self.final_rotator, - dim, self.imdimx, self.imdimy) = self.get_scan_info() + ( + self.total_frames, + self.start_frame, + self.end_frame, + self.final_rotator, + dim, + self.imdimx, + self.imdimy, + ) = self.get_scan_info() self.sdimx = dim self.sdimy = dim @@ -583,11 +696,21 @@ def get_scan_info(self): logger.debug(f"final rotator index not found, error: {e}") finrot = None try: - dim = round(np.sqrt(finrot), 6) - if not dim == int(dim): - raise Exception + if ( + "scan_dim_x" in self["Scan"].attrs + and "scan_dim_y" in self["Scan"].attrs + ): + dim = ( + self["Scan"].attrs["scan_dim_x"], + self["Scan"].attrs["scan_dim_y"], + ) + else: - dim = int(dim) + dim = round(np.sqrt(finrot), 6) + if not dim == int(dim): + raise Exception + else: + dim = int(dim) except Exception: logger.debug("Could not calculate scan dimensions") dim = None @@ -598,19 +721,33 @@ def get_scan_info(self): imdimy = None return (tot, start, end, finrot, dim, imdimx, imdimy) - def get_vbf_image(self, sdimx=None, sdimy=None, start_frame=None, - end_frame=None, hyst=0, snakescan=True): + def get_vbf_image( + self, + sdimx=None, + sdimy=None, + start_frame=None, + end_frame=None, + hyst=0, + snakescan=True, + ): # try to get the rotator data try: vbfs = self["Scan"]["vbf_intensities"][:] except Exception: - raise Exception("No VBF information found in dataset, please " - "calculate from TVIPS file.") + raise Exception( + "No VBF information found in dataset, please " + "calculate from TVIPS file." + ) logger.debug("Succesfully imported vbf intensities") logger.debug("Now calculating scan indexes") scan_indexes = self.calculate_scan_export_indexes( - sdimx=sdimx, sdimy=sdimy, start_frame=start_frame, - end_frame=end_frame, hyst=hyst, snakescan=snakescan) + sdimx=sdimx, + sdimy=sdimy, + start_frame=start_frame, + end_frame=end_frame, + hyst=hyst, + snakescan=snakescan, + ) logger.debug("Calculated scan indexes") if sdimx is None: sdimx = self.sdimx @@ -620,11 +757,25 @@ def get_vbf_image(self, sdimx=None, sdimy=None, start_frame=None, logger.debug("Applied calculated indexes and retrieved image") return img - def get_blo_export_data(self, sdimx=None, sdimy=None, start_frame=None, - end_frame=None, hyst=0, snakescan=True, crop=None): + def get_blo_export_data( + self, + sdimx=None, + sdimy=None, + start_frame=None, + end_frame=None, + hyst=0, + snakescan=True, + crop=None, + ): scan_indexes = self.calculate_scan_export_indexes( - sdimx=sdimx, sdimy=sdimy, start_frame=start_frame, - end_frame=end_frame, hyst=hyst, snakescan=snakescan, crop=crop) + sdimx=sdimx, + sdimy=sdimy, + start_frame=start_frame, + end_frame=end_frame, + hyst=hyst, + snakescan=snakescan, + crop=crop, + ) logger.debug("Calculated scan indexes") if sdimx is None: sdimx = self.sdimx @@ -637,20 +788,29 @@ def get_blo_export_data(self, sdimx=None, sdimy=None, start_frame=None, if crop is not None: xmin, xmax, ymin, ymax = crop sdimx = xmax - xmin - sdimy = ymax-ymin + sdimy = ymax - ymin shape = (sdimx, sdimy, *imshap) return shape, scan_indexes - def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, - start_frame=None, end_frame=None, - hyst=0, snakescan=True, crop=None): + def calculate_scan_export_indexes( + self, + sdimx=None, + sdimy=None, + start_frame=None, + end_frame=None, + hyst=0, + snakescan=True, + crop=None, + ): """Calculate the indexes of the list of frames to consider for the scan or VBF""" try: rots = self["Scan"]["rotation_indexes"][:] except Exception: - raise Exception("No VBF information found in dataset, please " - "calculate from TVIPS file.") + raise Exception( + "No VBF information found in dataset, please " + "calculate from TVIPS file." + ) logger.debug("Succesfully read rotator indexes") # set the scan info if sdimx is None: @@ -663,17 +823,17 @@ def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, # if a start frame is given, it's easy, we ignore rots if start_frame is not None: if end_frame is None: - end_frame = start_frame + sdimx*sdimy - 1 + end_frame = start_frame + sdimx * sdimy - 1 if end_frame >= self.total_frames: raise Exception("Final frame is out of bounds") if end_frame <= start_frame: - raise Exception("Final frame index must be larger than " - "first frame index") - if end_frame+1-start_frame != sdimx*sdimy: - raise Exception("Number of custom frames does not match " - "scan dimension") + raise Exception( + "Final frame index must be larger than first frame index" + ) + if end_frame + 1 - start_frame != sdimx * sdimy: + raise Exception("Number of custom frames does not match scan dimension") # just create an index array - sel = np.arange(start_frame, end_frame+1) + sel = np.arange(start_frame, end_frame + 1) sel = sel.reshape(sdimy, sdimx) # reverse correct even scan lines if snakescan: @@ -683,45 +843,56 @@ def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, # check for crop if crop is not None: - logger.info('Cropping to: {}'.format(crop)) + logger.info("Cropping to: {}".format(crop)) if all(i is not None for i in crop): xmin, xmax, ymin, ymax = crop - if all(i >= 0 for i in (xmin, ymin)) and xmax < sdimx and ymax < sdimy: - sel = sel[ymin: ymax, xmin: xmax] + if ( + all(i >= 0 for i in (xmin, ymin)) + and xmax < sdimx + and ymax < sdimy + ): + sel = sel[ymin:ymax, xmin:xmax] # +1 to include final frame else: - logger.warning('Aborting crop due to incorrect given dimensions: {}'.format(crop)) + logger.warning( + "Aborting crop due to incorrect given dimensions: {}".format( + crop + ) + ) return sel.ravel() # if frames or not given, we must use our best guess and match # rotators else: try: - rots = rots[self.start_frame:self.end_frame+1] + rots = rots[self.start_frame : self.end_frame + 1] except Exception: - raise Exception("No valid first or last scan frames detected, " - "must provide manual input") + raise Exception( + "No valid first or last scan frames detected, " + "must provide manual input" + ) # check whether sdimx*sdimy matches the final rotator index if not isinstance(self.final_rotator, int): - raise Exception("No final rotator index found, " - "can't align scan") - if sdimx*sdimy != self.final_rotator: - raise Exception("Scan dim x * scan dim y should match " - "the final rotator index if no custom " - "frames are specified") - indxs = np.zeros(sdimy*sdimx, dtype=int) + raise Exception("No final rotator index found, " "can't align scan") + if sdimx * sdimy != self.final_rotator: + raise Exception( + "Scan dim x * scan dim y should match " + "the final rotator index if no custom " + "frames are specified" + ) + indxs = np.zeros(sdimy * sdimx, dtype=int) prevw = 1 for j, _ in enumerate(indxs): # find where the argument is j - w = np.argwhere(rots == j+1) + w = np.argwhere(rots == j + 1) if w.size > 0: w = w[0, 0] prevw = w else: # move up if the rot index stays the same, otherwise copy - if prevw+1 < len(rots): - if rots[prevw+1] == rots[prevw]: - prevw = prevw+1 + if prevw + 1 < len(rots): + if rots[prevw + 1] == rots[prevw]: + prevw = prevw + 1 w = prevw indxs[j] = w # just an array of indexes @@ -732,4 +903,4 @@ def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, # hysteresis correction on even scan lines img[::2] = np.roll(img[::2], hyst, axis=1) # add the start index - return img.ravel()+self.start_frame + return img.ravel() + self.start_frame diff --git a/tvipsconverter/widget_2.ui b/tvipsconverter/widget.ui similarity index 87% rename from tvipsconverter/widget_2.ui rename to tvipsconverter/widget.ui index be0e70e..f76e1a7 100644 --- a/tvipsconverter/widget_2.ui +++ b/tvipsconverter/widget.ui @@ -7,7 +7,7 @@ 0 0 1171 - 755 + 1003 @@ -32,38 +32,38 @@ Progress - - + + 0 0 - - - 974 - 0 - + + Status: Idle - - 0 + + true - - + + 0 0 - - Status: Idle + + + 974 + 0 + - - true + + 0 @@ -82,7 +82,7 @@ <html><head/><body><p><br/></p></body></html> - 0 + 1 @@ -156,8 +156,8 @@ 0 0 - 1100 - 643 + 1115 + 780 @@ -286,6 +286,41 @@ + + + + Binning Type + + + + + + Use box averaging where the box size is the binning factor. + + + Box Average + + + true + + + buttonGroup + + + + + + + Decimation + + + buttonGroup + + + + + + @@ -685,6 +720,101 @@ + + + + Other calculations + + + + + + <html><head/><body><p>The maximum diffraction pattern represents the maximum intensity at each diffraction pattern pixel for all scanning positions. It is therefore instructive to the diffraction intensities recorded.</p></body></html> + + + Calculate maximum diffraction pattern + + + true + + + + + + + <html><head/><body><p>Sigma should be set at least as large as the direct beam disk radius to accurately find the center point.</p></body></html> + + + 5 + + + + + + + <html><head/><body><p>Sigma should be set at least as large as the direct beam disk radius to accurately find the center point.</p></body></html> + + + Sigma + + + + + + + <html><head/><body><p>The direct beam location will be refined and stored as metadata for each frame. This is done by blurring (sigma) a small central region (of diameter box size) and finding the peak position. Therefore box size should be large enough to encompass the direct beam entirely for all scnning pixels and sigma should be at least the disk radius.</p></body></html> + + + Refine direct beam position + + + true + + + + + + + <html><head/><body><p>Diameter of the bo in which the direct beam spot will be found for all scanning positions.</p></body></html> + + + Box size + + + + + + + <html><head/><body><p>Diameter of the bo in which the direct beam spot will be found for all scanning positions.</p></body></html> + + + 50 + + + + + + + Qt::Horizontal + + + + + + + <html><head/><body><p>The average diffraction pattern represents the average intensity at each diffraction pattern pixel for all scanning positions. It is therefore instructive to the average diffraction over the whole scanning area.</p></body></html> + + + Calculate average diffraction pattern + + + true + + + + + + @@ -1026,6 +1156,9 @@ + + Qt::ScrollBarAlwaysOn + true @@ -1034,8 +1167,8 @@ 0 0 - 1054 - 610 + 1069 + 628 @@ -1125,10 +1258,10 @@ Scaling - - + + - Scale of DP + nm^-1 / pixel @@ -1139,30 +1272,6 @@ - - - - nm / pixel - - - - - - - nm^-1 / pixel - - - - - - - 4 - - - 1.000000000000000 - - - @@ -1189,6 +1298,30 @@ + + + + nm / pixel + + + + + + + Scale of DP + + + + + + + 4 + + + 1.000000000000000 + + + @@ -1207,40 +1340,24 @@ Scan modification - - - - - 0 - 0 - + + + + <html><head/><body><p>In EM Konos, the scanning follows a winding scan pattern (check on), whereas in EM Scan it uses normal flyback scan (check off)</p></body></html> - pixels + Winding scan? - - - - - + true - - - 0 - 0 - - - - <html><head/><body><p>Number of scan points in the Y direction</p></body></html> - - - Scan dim. Y + + false - - + + true @@ -1257,13 +1374,32 @@ - -100000000 + 1 + + + 10000 + + + 1 + + + 100 + + + + + + + true + + + QAbstractSpinBox::NoButtons 100000000 - 5 + 0 @@ -1274,22 +1410,30 @@ - - - - true + + + + <html><head/><body><p>Frame in the stream where the scan began</p></body></html> - - - 0 - 0 - + + Scan start frame - - <html><head/><body><p>Number of scan points in the x direction</p></body></html> + + + + + + Use custom scan dimensions + + + false + + + + - Scan dim. X + Use custom scan frames @@ -1324,50 +1468,22 @@ - - - + + + true - - QAbstractSpinBox::NoButtons + + + 0 + 0 + - - 100000000 - - - 0 - - - - - - - true - - - - 0 - 0 - - - - - 93 - 0 - - - - 1 - - - 10000 - - - 1 + + <html><head/><body><p>Number of scan points in the Y direction</p></body></html> - - 100 + + Scan dim. Y @@ -1384,23 +1500,6 @@ - - - - <html><head/><body><p>Frame in the stream where the scan began</p></body></html> - - - Scan start frame - - - - - - - Use custom scan frames - - - @@ -1414,19 +1513,50 @@ - - - - <html><head/><body><p>In EM Konos, the scanning follows a winding scan pattern (check on), whereas in EM Scan it uses normal flyback scan (check off)</p></body></html> + + + + true - - Winding scan? + + + 0 + 0 + - + + + 93 + 0 + + + + -100000000 + + + 100000000 + + + 5 + + + + + + true - - false + + + 0 + 0 + + + + <html><head/><body><p>Number of scan points in the x direction</p></body></html> + + + Scan dim. X @@ -1440,13 +1570,16 @@ - - - - Use custom scan dimensions + + + + + 0 + 0 + - - false + + pixels @@ -1455,14 +1588,65 @@ + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + Crop - - + + + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + - ymin + xmin + + + + + + + + 93 + 0 + + + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + + + 1000 + + + 100 + + + + + + + + 93 + 0 + + + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + + + 1000 + + + + + + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + + + Show Cropped Region @@ -1474,6 +1658,9 @@ 0 + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + 1000 @@ -1482,51 +1669,66 @@ - - + + + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + - xmin + ymin - - + + 93 0 + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + + + 0 + 1000 - 100 + 0 - - + + + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + - Show Cropped Region + xmax + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + ymax - - + + - xmax + Apply crop? - + Qt::Horizontal @@ -1539,35 +1741,35 @@ - - - - - 93 - 0 - - - - 1000 + + + + + + + 8-bit conversion + + + + + + Qt::Horizontal - - - - - + - 93 - 0 + 40 + 20 - - 0 - - - 1000 + + + + + + <html><head/><body><p>If checked then the intensities in each frame will be rescaled between 0 and 255.</p></body></html> - - 0 + + Rescale intensities across 8-bit range @@ -1643,19 +1845,6 @@ Display settings - - - - 255 - - - 10 - - - Qt::Horizontal - - - @@ -1666,6 +1855,13 @@ + + + + Maximum + + + @@ -1673,10 +1869,16 @@ - - - - Maximum + + + + 255 + + + 10 + + + Qt::Horizontal @@ -1989,4 +2191,7 @@ + + + diff --git a/tvipsconverter/widgets.py b/tvipsconverter/widgets.py index 1d070b9..c8a74b4 100644 --- a/tvipsconverter/widgets.py +++ b/tvipsconverter/widgets.py @@ -1,17 +1,21 @@ from PyQt5 import uic -from PyQt5.QtWidgets import (QApplication, QFileDialog, QGraphicsScene) +from PyQt5.QtWidgets import QApplication, QFileDialog, QGraphicsScene from PyQt5.QtCore import QThread, pyqtSignal +from PyQt5.QtGui import QIcon, QPixmap import sys import matplotlib.pyplot as plt from matplotlib.patches import Circle -from matplotlib.backends.backend_qt5agg import (FigureCanvasQTAgg as - FigureCanvas) +from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from pathlib import Path import logging from time import sleep import numpy as np import os +# hotfix 3.9 MacOS Big Sur bug +if sys.platform == "darwin": + os.environ["QT_MAC_WANTS_LAYER"] = "1" + from .utils import recorder as rec from .utils import blockfile as blf from .utils import tiffexport as tfe @@ -22,14 +26,16 @@ logger = logging.getLogger(__name__) # import the UI interface -rawgui, Window = uic.loadUiType(str(Path(__file__).parent.absolute()) + - "/widget_2.ui") +rawgui, Window = uic.loadUiType( + str(Path(__file__).parent.absolute()) + os.sep + "widget.ui" +) class External(QThread): """ Runs a counter thread. """ + countChanged = pyqtSignal(int) finish = pyqtSignal() @@ -48,6 +54,7 @@ def run(self): class ConnectedWidget(rawgui): """Class connecting the gui elements to the back-end functionality""" + def __init__(self, window): super().__init__() self.window = window @@ -110,6 +117,10 @@ def connectUI(self): self.pushButton_11.clicked.connect(self.export_tiffs) # show cropped region self.pushButton_13.clicked.connect(self.show_cropped_region) + # write scan settings (from VBF preview) to hdf5 + # self.pushButton_write_scan_parameters.clicked.connect( + # self.write_scan_parameters_hdf5 + # ) def export_preview(self): try: @@ -141,9 +152,9 @@ def update_levels_vbf(self): vmax = self.horizontalSlider_2.value() mn = self.vbf_data.min() mx = self.vbf_data.max() - unit = (mx-mn)/100 - climmin = mn+vmin*unit - climmax = mn+(vmax+1)*unit + unit = (mx - mn) / 100 + climmin = mn + vmin * unit + climmax = mn + (vmax + 1) * unit try: self.vbf_im.set_clim(climmin, climmax) canvas = FigureCanvas(self.fig_vbf) @@ -151,9 +162,7 @@ def update_levels_vbf(self): scene = QGraphicsScene() scene.addWidget(canvas) self.graphicsView_3.setScene(scene) - self.graphicsView_3.fitInView( - scene.sceneRect(), - ) + self.graphicsView_3.fitInView(scene.sceneRect(),) self.repaint_widget(self.graphicsView_3) except Exception as e: logger.debug(f"Error: {e}") @@ -165,21 +174,21 @@ def update_final_frame(self): # update x, y crop box values self.spinBox_22.setValue(0) self.spinBox_20.setValue(0) - self.spinBox_23.setValue(self.spinBox.value() - 1) - self.spinBox_21.setValue(self.spinBox_2.value() - 1) + self.spinBox_23.setValue(self.spinBox.value()) + self.spinBox_21.setValue(self.spinBox_2.value()) if self.checkBox_2.checkState(): # we use self defined size start = self.spinBox_15.value() - frms = self.spinBox.value()*self.spinBox_2.value() - self.spinBox_16.setValue(start+frms-1) + frms = self.spinBox.value() * self.spinBox_2.value() + self.spinBox_16.setValue(start + frms - 1) else: # we use auto-size start = self.spinBox_15.value() frms = self.lineEdit_11.text() try: dim = np.sqrt(int(frms)) - self.spinBox_16.setValue(start+dim**2-1) + self.spinBox_16.setValue(start + dim ** 2 - 1) except Exception: self.spinBox_16.setValue(0) @@ -206,14 +215,12 @@ def open_tvips_file(self): self.statusedit.setText(str(e)) def openFileBrowser(self, fs): - path, okpres = QFileDialog.getOpenFileName(caption="Select file", - filter=fs) + path, okpres = QFileDialog.getOpenFileName(caption="Select file", filter=fs) if okpres: return str(Path(path)) def saveFileBrowser(self, fs): - path, okpres = QFileDialog.getSaveFileName(caption="Select file", - filter=fs) + path, okpres = QFileDialog.getSaveFileName(caption="Select file", filter=fs) if okpres: return str(Path(path)) @@ -237,13 +244,14 @@ def read_modsettings(self): "usels": self.checkBox_4.checkState(), "lsmin": self.spinBox_7.value(), "lsmax": self.spinBox_8.value(), - "usecoffset": self.checkBox.checkState() + "usecoffset": self.checkBox.checkState(), + "bintype": self.radioButton_decimation.isChecked(), # if True then use decimation otherwise box averaging } vbfsettings = { "calcvbf": self.checkBox_10.checkState(), "vbfrad": self.spinBox_12.value(), "vbfxoffset": self.spinBox_13.value(), - "vbfyoffset": self.spinBox_14.value() + "vbfyoffset": self.spinBox_14.value(), } return path, improc, vbfsettings @@ -263,9 +271,8 @@ def updatePreview(self): # (self.path_preview != path): self.update_line(self.statusedit, "Extracting frame...") self.original_preview = rec.getOriginalPreviewImage( - path, improc=improc, - vbfsettings=vbfsettings, - frame=framenum) + path, improc=improc, vbfsettings=vbfsettings, frame=framenum + ) # update the path self.path_preview = path ois = self.original_preview.shape @@ -273,36 +280,42 @@ def updatePreview(self): nis = filterframe.shape # check if the VBF aperture fits in the frame if vbfsettings["calcvbf"]: - midx = nis[1]//2 - midy = nis[0]//2 + midx = nis[1] // 2 + midy = nis[0] // 2 xx = vbfsettings["vbfxoffset"] yy = vbfsettings["vbfyoffset"] rr = vbfsettings["vbfrad"] - if (midx+xx-rr < 0 or - midx+xx+rr > nis[1] or - midy+yy-rr < 0 or - midy+yy-rr > nis[0]): - raise Exception("Virtual bright field aperture out " - "of bounds") + if ( + midx + xx - rr < 0 + or midx + xx + rr > nis[1] + or midy + yy - rr < 0 + or midy + yy - rr > nis[0] + ): + raise Exception("Virtual bright field aperture out " "of bounds") # plot the image and the circle over it if self.fig_prev is not None: plt.close(self.fig_prev) - self.fig_prev = plt.figure(frameon=False, - figsize=(filterframe.shape[1]/100, - filterframe.shape[0]/100)) + self.fig_prev = plt.figure( + frameon=False, + figsize=(filterframe.shape[1] / 100, filterframe.shape[0] / 100), + ) canvas = FigureCanvas(self.fig_prev) - ax = plt.Axes(self.fig_prev, [0., 0., 1., 1.]) + ax = plt.Axes(self.fig_prev, [0.0, 0.0, 1.0, 1.0]) ax.set_axis_off() self.fig_prev.add_axes(ax) ax.imshow(filterframe, cmap="Greys_r") if vbfsettings["calcvbf"]: xoff = vbfsettings["vbfxoffset"] yoff = vbfsettings["vbfyoffset"] - circ = Circle((filterframe.shape[1]//2+xoff, - filterframe.shape[0]//2+yoff), - vbfsettings["vbfrad"], - color="red", - alpha=0.5) + circ = Circle( + ( + filterframe.shape[1] // 2 + xoff, + filterframe.shape[0] // 2 + yoff, + ), + vbfsettings["vbfrad"], + color="red", + alpha=0.5, + ) ax.add_patch(circ) canvas.draw() scene = QGraphicsScene() @@ -311,8 +324,10 @@ def updatePreview(self): self.graphicsView.fitInView(scene.sceneRect()) self.repaint_widget(self.graphicsView) self.update_line(self.statusedit, "Succesfully created preview.") - self.update_line(self.lineEdit_8, f"Original: {ois[0]}x{ois[1]}. " - f"New: {nis[0]}x{nis[1]}.") + self.update_line( + self.lineEdit_8, + f"Original: {ois[0]}x{ois[1]}. " f"New: {nis[0]}x{nis[1]}.", + ) except Exception as e: self.update_line(self.statusedit, f"Error: {e}") # empty the preview @@ -336,8 +351,7 @@ def get_hdf5_path(self): # open a savefile browser try: # read the gui info - (self.inpath, self.improc, - self.vbfsettings) = self.read_modsettings() + (self.inpath, self.improc, self.vbfsettings) = self.read_modsettings() if not self.inpath: raise Exception("A TVIPS file must be selected!") self.oupath = self.saveFileBrowser("HDF5 (*.hdf5)") @@ -396,8 +410,7 @@ def image_range(self): def write_to_hdf5(self): try: - (self.inpath, self.improc, - self.vbfsettings) = self.read_modsettings() + (self.inpath, self.improc, self.vbfsettings) = self.read_modsettings() if not self.inpath: raise Exception("A TVIPS file must be selected!") self.oupath = self.lineEdit_2.text() @@ -410,11 +423,20 @@ def write_to_hdf5(self): improc = self.improc vbfsettings = self.vbfsettings start_frame, end_frame = self.image_range - self.get_thread = rec.Recorder(path, - improc=improc, - vbfsettings=vbfsettings, - outputpath=opath, - imrange=(start_frame, end_frame)) + self.get_thread = rec.Recorder( + path, + improc=improc, + vbfsettings=vbfsettings, + outputpath=opath, + imrange=(start_frame, end_frame), + calcmax=self.checkBox_maxiumum_image.isChecked(), # options kwarg + calcave=self.checkBox_average_image.isChecked(), # options kwarg + refine_center=( + self.checkBox_refine_center.isChecked(), + self.spinBox_refine_center_diameter.value(), + self.spinBox_refine_center_sigma.value(), + ), + ) self.get_thread.increase_progress.connect(self.increase_progbar) self.get_thread.finish.connect(self.done_hdf5export) self.get_thread.start() @@ -425,8 +447,7 @@ def write_to_hdf5(self): def done_hdf5export(self): self.window.setEnabled(True) # also update lines in the second pannel - self.update_line(self.statusedit, - "Succesfully exported to HDF5") + self.update_line(self.statusedit, "Succesfully exported to HDF5") # don't auto update, the gui may be before the file exists # self.update_line(self.lineEdit_4, self.lineEdit_2.text()) @@ -441,6 +462,9 @@ def auto_read_hdf5(self): self.update_line(self.lineEdit_3, "?") if star is not None: self.update_line(self.lineEdit_5, str(star)) + logger.info(f"start: {star}") + self.spinBox_15.setValue(star) + self.checkBox_11.setChecked(True) else: self.update_line(self.lineEdit_5, "?") if en is not None: @@ -452,12 +476,18 @@ def auto_read_hdf5(self): else: self.update_line(self.lineEdit_11, "?") if dim is not None: - self.update_line(self.lineEdit_12, - f"{str(int(dim))}x{str(int(dim))}") + if isinstance(dim, (list, tuple)): + self.update_line(self.lineEdit_12, str(dim)) + self.spinBox.setValue(dim[0]) + self.spinBox_2.setValue(dim[1]) + self.checkBox_2.setChecked(True) + else: + self.update_line( + self.lineEdit_12, f"{str(int(dim))}x{str(int(dim))}" + ) else: self.update_line(self.lineEdit_12, "?") - self.update_line(self.lineEdit_13, - f"{str(imdimx)}x{str(imdimy)}") + self.update_line(self.lineEdit_13, f"{str(imdimx)}x{str(imdimy)}") self.update_final_frame() f.close() except Exception as e: @@ -503,29 +533,36 @@ def update_vbf(self): else: snakescan = False # calculate the image - logger.debug(f"We try to create a VBF image with data: " - f"S.F. {start_frame}, E.F. {end_frame}, " - f"Dims: x {sdimx} y {sdimy}," - f"hyst: {hyst}") - self.vbf_data = f.get_vbf_image(sdimx, sdimy, start_frame, - end_frame, hyst, snakescan) + logger.debug( + f"We try to create a VBF image with data: " + f"S.F. {start_frame}, E.F. {end_frame}, " + f"Dims: x {sdimx} y {sdimy}," + f"hyst: {hyst}" + ) + self.vbf_data = f.get_vbf_image( + sdimx, sdimy, start_frame, end_frame, hyst, snakescan + ) logger.debug("Succesfully created the VBF array") # save the settings for later storage - self.vbf_sets = {"start_frame": start_frame, - "end_frame": end_frame, - "scan_dim_x": sdimx, - "scan_dim_y": sdimy, - "hysteresis": hyst, - "winding_scan": snakescan} + self.vbf_sets = { + "start_frame": start_frame, + "end_frame": end_frame, + "scan_dim_x": sdimx, + "scan_dim_y": sdimy, + "hysteresis": hyst, + "winding_scan": snakescan, + } + # plot the image and store it for further use. First close prior # image if self.fig_vbf is not None: plt.close(self.fig_vbf) - self.fig_vbf = plt.figure(frameon=False, - figsize=(self.vbf_data.shape[1]/100, - self.vbf_data.shape[0]/100)) + self.fig_vbf = plt.figure( + frameon=False, + figsize=(self.vbf_data.shape[1] / 100, self.vbf_data.shape[0] / 100), + ) canvas = FigureCanvas(self.fig_vbf) - ax = plt.Axes(self.fig_vbf, [0., 0., 1., 1.]) + ax = plt.Axes(self.fig_vbf, [0.0, 0.0, 1.0, 1.0]) ax.set_axis_off() self.fig_vbf.add_axes(ax) self.vbf_im = ax.imshow(self.vbf_data, cmap="plasma") @@ -540,6 +577,9 @@ def update_vbf(self): yshap, xshap = self.vbf_data.shape self.update_line(self.lineEdit_10, f"Size: {xshap}x{yshap}.") f.close() + + # add settings to hdf5 file, do this after file close + rec.write_scan_parameters_hdf5(path_hdf5, **self.vbf_sets) except Exception as e: self.update_line(self.statusedit, f"Error: {e}") @@ -584,31 +624,49 @@ def write_to_file(self): dp_scale = self.doubleSpinBox_3.value() # calculate the image filetyp = self.comboBox.currentText() - logger.debug(f"We try to create a {filetyp} file with data: " - f"S.F. {start_frame}, E.F. {end_frame}, " - f"Dims: x {sdimx} y {sdimy}," - f"hyst: {hyst}, snakescan: {snakescan}") + logger.debug( + f"We try to create a {filetyp} file with data: " + f"S.F. {start_frame}, E.F. {end_frame}, " + f"Dims: x {sdimx} y {sdimy}," + f"hyst: {hyst}, snakescan: {snakescan}" + ) logger.debug("Calculating shape and indexes") - # xmin, xmax, ymin, ymax - crop = (self.spinBox_22.value(), - self.spinBox_23.value(), - self.spinBox_20.value(), - self.spinBox_21.value(), + # check crop desired + if self.checkBox_apply_crop.isChecked(): + # check crop valid + if self.check_crop_limits() is None: + self.update_line( + self.statusedit, + f"File not exported, please check cropping limits.", ) - shape, indexes = f.get_blo_export_data(sdimx, sdimy, - start_frame, - end_frame, hyst, - snakescan, crop=crop) + return + # get crop + crop = self.get_crop_limits() + else: + crop = None + + shape, indexes = f.get_blo_export_data( + sdimx, sdimy, start_frame, end_frame, hyst, snakescan, crop=crop + ) logger.debug(f"Shape: {shape}") logger.debug(f"Starting to write {filetyp} file") self.update_line(self.statusedit, f"Writing {filetyp} file...") if filetyp == ".blo": self.get_thread = blf.bloFileWriter( - f, path_blo, shape, indexes, scan_scale, dp_scale) + f, + path_blo, + shape, + indexes, + scan_scale, + dp_scale, + # if rescale button is checked do rescale, otherwise no rescaling selected + rescale=self.checkBox_rescale.isChecked(), + ) elif filetyp == ".hspy": self.get_thread = hspf.hspyFileWriter( - f, path_blo, shape, indexes, scan_scale, dp_scale) + f, path_blo, shape, indexes, scan_scale, dp_scale + ) else: raise NotImplementedError("Unrecognized file type") self.get_thread.increase_progress.connect(self.increase_progbar) @@ -621,8 +679,7 @@ def write_to_file(self): def done_bloexport(self): self.window.setEnabled(True) # also update lines in the second pannel - self.update_line(self.statusedit, - "Succesfully exported to file") + self.update_line(self.statusedit, "Succesfully exported to file") def export_tiffs(self): path_hdf5 = self.lineEdit_4.text() @@ -648,7 +705,7 @@ def export_tiffs(self): raise Exception("Unexpected dtype") if tot_frames <= last_frame: raise Exception("Frames are out of range") - frames = np.arange(first_frame, last_frame+1) + frames = np.arange(first_frame, last_frame + 1) self.update_line(self.statusedit, "Exporting to tiff files...") self.get_thread = tfe.TiffFileWriter(f, frames, dtype, pre, fin) self.get_thread.increase_progress.connect(self.increase_progbar) @@ -661,8 +718,7 @@ def export_tiffs(self): def done_tiffexport(self): self.window.setEnabled(True) # also update lines in the second pannel - self.update_line(self.statusedit, - "Succesfully exported Tiff files") + self.update_line(self.statusedit, "Succesfully exported Tiff files") def increase_progbar(self, value): self.progressBar.setValue(value) @@ -671,49 +727,109 @@ def hardRepaint(self): self.window.hide() self.window.show() - def show_cropped_region(self): - # check for validity of cropped region + def remove_cropped_region(self): + # try to remove if it is there + label = "crop_rect" + try: + # see if rectangle already plotted and just update + index = [i.get_label() for i in self.fig_vbf.axes[0].patches].index(label) + rect = self.fig_vbf.axes[0].patches[index] + rect.remove() + self.fig_vbf.canvas.draw() + except ValueError: + logger.info("No crop rectange drawn.") + + def get_crop_limits(self): xmin = self.spinBox_22.value() xmax = self.spinBox_23.value() ymin = self.spinBox_20.value() ymax = self.spinBox_21.value() + + return xmin, xmax, ymin, ymax + + def check_crop_limits(self): + xmin, xmax, ymin, ymax = self.get_crop_limits() + + # check scan boundaries, returns None if any test fails + if ( + xmin < 0 + or ymin < 0 + or not xmax <= self.spinBox.value() + or not ymax <= self.spinBox_2.value() + ): + self.update_line( + self.statusedit, "Crop limits should be within scan bounds." + ) + self.remove_cropped_region() + return + + # check cropping dimension > 0 if not xmax - xmin > 0: - logger.warning('Not valid cropping dimensions: x.') + self.update_line( + self.statusedit, f"Crop dimension incorrect. Crop x: {xmax - xmin}.", + ) + self.remove_cropped_region() return if not ymax - ymin > 0: - logger.warning('Not valid cropping dimensions: x.') + self.update_line( + self.statusedit, f"Crop dimension incorrect. Crop y: {ymax - ymin}.", + ) + self.remove_cropped_region() + return + + # return non-None if passes checks + return True + + def show_cropped_region(self): + if self.check_crop_limits() is None: return + # check for validity of cropped region + xmin, xmax, ymin, ymax = self.get_crop_limits() + if self.fig_vbf is None: - logger.warning('No VBF figure plotted yet.') - # no VBF figure plotted yet + self.update_line( + self.statusedit, "No VBF figure plotted yet.", + ) return - - label = 'crop_rect' + label = "crop_rect" try: # see if rectangle already plotted and just update - index = [i.get_label() for i in - self.fig_vbf.axes[0].patches].index(label) + index = [i.get_label() for i in self.fig_vbf.axes[0].patches].index(label) rect = self.fig_vbf.axes[0].patches[index] - logger.info('Updating rectangle.') + logger.info("Updating rectangle.") rect.set_height(ymax - ymin) rect.set_width(xmax - xmin) rect.set_x(xmin) rect.set_y(ymin) except ValueError: - logger.info('Plotting rectangle.') - rect = plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin, - fill=False, ec='k', ls='dashed', label=label) + logger.info("Plotting rectangle.") + rect = plt.Rectangle( + (xmin, ymin), + xmax - xmin, + ymax - ymin, + fill=False, + ec="k", + ls="dashed", + label=label, + ) self.fig_vbf.axes[0].add_patch(rect) self.fig_vbf.canvas.draw() + self.update_line( + self.statusedit, f"Crop dimensions (x, y): ({xmax - xmin}, {ymax - ymin}).", + ) + def main(): app = QApplication([]) window = Window() _ = ConnectedWidget(window) window.setWindowTitle("TVIPS converter") + # window.setWindowIcon( + # QIcon(QPixmap(os.path.join(os.path.dirname(__file__), "Icon 128.png"))) + # ) window.show() app.exec_()