From e62088f0c1e4ea21bf81cdfb65c7a14ac3d78b11 Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Tue, 3 Nov 2020 11:45:05 +0100 Subject: [PATCH 01/21] add create DP ave and max images when convert_HDF5 --- tvipsconverter/__main__.py | 3 + tvipsconverter/utils/imagefun.py | 44 +-- tvipsconverter/utils/recorder.py | 456 ++++++++++++++++++++----------- tvipsconverter/widget_2.ui | 39 ++- tvipsconverter/widgets.py | 223 ++++++++------- 5 files changed, 471 insertions(+), 294 deletions(-) create mode 100644 tvipsconverter/__main__.py diff --git a/tvipsconverter/__main__.py b/tvipsconverter/__main__.py new file mode 100644 index 0000000..5bd1eb6 --- /dev/null +++ b/tvipsconverter/__main__.py @@ -0,0 +1,3 @@ +from .widgets import main + +main() diff --git a/tvipsconverter/utils/imagefun.py b/tvipsconverter/utils/imagefun.py index 1a0105b..6443ad1 100644 --- a/tvipsconverter/utils/imagefun.py +++ b/tvipsconverter/utils/imagefun.py @@ -13,10 +13,16 @@ def _get_dtype_min_max(dtype): if dtype == np.float or dtype == np.float32 or dtype == np.float64: max = 1 # np.finfo(dtype).max min = 0 # np.finfo(dtype).min - elif (dtype == np.int8 or dtype == np.uint8 or - dtype == np.int16 or dtype == np.uint16 or - dtype == np.int32 or dtype == np.uint32 or - dtype == np.int64 or dtype == np.uint64): + elif ( + dtype == np.int8 + or dtype == np.uint8 + or dtype == np.int16 + or dtype == np.uint16 + or dtype == np.int32 + or dtype == np.uint32 + or dtype == np.int64 + or dtype == np.uint64 + ): max = np.iinfo(dtype).max min = np.iinfo(dtype).min else: @@ -119,8 +125,8 @@ def linscale(arr, min=None, max=None, nmin=0, nmax=1, dtype=np.float): else: workarr[workarr > max] = max - a = (nmax-nmin)/(max-min) - result = (workarr-min)*a+nmin + a = (nmax - nmin) / (max - min) + result = (workarr - min) * a + nmin return result.astype(dtype) @@ -142,10 +148,10 @@ def matlab_style_gauss2D(shape=(3, 3), sigma=0.5): 2D gaussian mask - should give the same result as MATLAB's fspecial('gaussian',[shape],[sigma]) """ - m, n = [(ss-1.)/2. for ss in shape] - y, x = np.ogrid[-m:m+1, -n:n+1] - h = np.exp(-(x*x + y*y) / (2.*sigma*sigma)) - h[h < np.finfo(h.dtype).eps*h.max()] = 0 + m, n = [(ss - 1.0) / 2.0 for ss in shape] + y, x = np.ogrid[-m : m + 1, -n : n + 1] + h = np.exp(-(x * x + y * y) / (2.0 * sigma * sigma)) + h[h < np.finfo(h.dtype).eps * h.max()] = 0 sumh = h.sum() if sumh != 0: h /= sumh @@ -168,14 +174,14 @@ def findoutliers(raw, percent=0.07): Uses returns min and max values of the array with the upper and lower percent pixel values suppressed. """ - cnts, edges = np.histogram(raw, bins=2**16) - stats = np.zeros((2, 2**16), dtype=np.int) + cnts, edges = np.histogram(raw, bins=2 ** 16) + stats = np.zeros((2, 2 ** 16), dtype=np.int) stats[0] = np.cumsum(cnts) # low stats[1] = np.cumsum(cnts[::-1]) # high thresh = stats > percent * raw.shape[0] * raw.shape[1] min = (np.where(thresh[0]))[0][0] - max = 2**16 - (np.where(thresh[1]))[0][0] - return edges[min], edges[max+1] + max = 2 ** 16 - (np.where(thresh[1]))[0][0] + return edges[min], edges[max + 1] def suppressoutliers(raw, percent=0.07): @@ -191,9 +197,11 @@ def bin2(a, factor): imag = Image.fromarray(a) # binned = Image.resize(imag, (a.shape[0]//factor, a.shape[1]//factor), # resample=Image.BILINEAR) - binned = np.array(imag.resize((a.shape[0]//factor, a.shape[1]//factor), - resample=Image.NEAREST)) - print(binned.dtype) + binned = np.array( + imag.resize( + (a.shape[0] // factor, a.shape[1] // factor), resample=Image.NEAREST + ) + ) return binned @@ -204,5 +212,5 @@ def getElectronWavelength(ht): charge = 1.6e-19 c = 3e8 wavelength = h / math.sqrt(2 * m * charge * ht) - relativistic_correction = 1 / math.sqrt(1 + ht * charge/(2 * m * c * c)) + relativistic_correction = 1 / math.sqrt(1 + ht * charge / (2 * m * c * c)) return wavelength * relativistic_correction diff --git a/tvipsconverter/utils/recorder.py b/tvipsconverter/utils/recorder.py index b9b5975..a989fb4 100644 --- a/tvipsconverter/utils/recorder.py +++ b/tvipsconverter/utils/recorder.py @@ -8,70 +8,73 @@ from PyQt5.QtCore import QThread, pyqtSignal import logging -from .imagefun import (normalize_convert, bin2, gausfilter, - medfilter) +from .imagefun import normalize_convert, bin2, gausfilter, medfilter # Initialize the Logger logger = logging.getLogger(__name__) TVIPS_RECORDER_GENERAL_HEADER = [ - ('size', 'u4'), # unused - likely the size of generalheader in bytes - ('version', 'u4'), # 1 or 2 - ('dimx', 'u4'), # dp image size width - ('dimy', 'u4'), # dp image size height - ('bitsperpixel', 'u4'), # 8 or 16 - ('offsetx', 'u4'), # generally 0 - ('offsety', 'u4'), - ('binx', 'u4'), # camera binning - ('biny', 'u4'), - ('pixelsize', 'u4'), # nm, physical pixel size - ('ht', 'u4'), # high tension, voltage - ('magtotal', 'u4'), # magnification/camera length? - ('frameheaderbytes', 'u4'), # number of bytes per frame header - ('dummy', 'S204'), # just writes out TVIPS TVIPS TVIPS - ] + ("size", "u4"), # unused - likely the size of generalheader in bytes + ("version", "u4"), # 1 or 2 + ("dimx", "u4"), # dp image size width + ("dimy", "u4"), # dp image size height + ("bitsperpixel", "u4"), # 8 or 16 + ("offsetx", "u4"), # generally 0 + ("offsety", "u4"), + ("binx", "u4"), # camera binning + ("biny", "u4"), + ("pixelsize", "u4"), # nm, physical pixel size + ("ht", "u4"), # high tension, voltage + ("magtotal", "u4"), # magnification/camera length? + ("frameheaderbytes", "u4"), # number of bytes per frame header + ("dummy", "S204"), # just writes out TVIPS TVIPS TVIPS +] TVIPS_RECORDER_FRAME_HEADER = [ - ('num', 'u4'), # seems to cycle also - ('timestamp', 'u4'), # seconds since 1.1.1970 - ('ms', 'u4'), # additional milliseconds to the timestamp - ('LUTidx', 'u4'), # always the same value - ('fcurrent', 'f4'), # 0 for all frames - ('mag', 'u4'), # same for all frames - ('mode', 'u4'), # 1 -> image 2 -> diff - ('stagex', 'f4'), - ('stagey', 'f4'), - ('stagez', 'f4'), - ('stagea', 'f4'), - ('stageb', 'f4'), - ('rotidx', 'u4'), - ('temperature', 'f4'), # cycles between 0.0 and 9.0 with step 1.0 - ('objective', 'f4'), # kind of randomly between 0.0 and 1.0 + ("num", "u4"), # seems to cycle also + ("timestamp", "u4"), # seconds since 1.1.1970 + ("ms", "u4"), # additional milliseconds to the timestamp + ("LUTidx", "u4"), # always the same value + ("fcurrent", "f4"), # 0 for all frames + ("mag", "u4"), # same for all frames + ("mode", "u4"), # 1 -> image 2 -> diff + ("stagex", "f4"), + ("stagey", "f4"), + ("stagez", "f4"), + ("stagea", "f4"), + ("stageb", "f4"), + ("rotidx", "u4"), + ("temperature", "f4"), # cycles between 0.0 and 9.0 with step 1.0 + ("objective", "f4"), # kind of randomly between 0.0 and 1.0 # for header version 2, some more data might be present - ] +] FILTER_DEFAULTS = { - "useint": False, "whichint": 65536, - "usebin": False, "whichbin": 1, "usegaus": False, - "gausks": 8, "gaussig": 4, "usemed": False, - "medks": 4, "usels": False, "lsmin": 10, - "lsmax": 1000, "usecoffset": False - } - - -VBF_DEFAULTS = { - "calcvbf": True, "vbfrad": 10, "vbfxoffset": 0, - "vbfyoffset": 0 - } - - -def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, - binning=1): + "useint": False, + "whichint": 65536, + "usebin": False, + "whichbin": 1, + "usegaus": False, + "gausks": 8, + "gaussig": 4, + "usemed": False, + "medks": 4, + "usels": False, + "lsmin": 10, + "lsmax": 1000, + "usecoffset": False, +} + + +VBF_DEFAULTS = {"calcvbf": True, "vbfrad": 10, "vbfxoffset": 0, "vbfyoffset": 0} + + +def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, binning=1): """Do some kind of intensity correction, unsure reason""" pixperchannel = int(128 / binning) # binning has to be an integer - if (128.0/binning != pixperchannel): + if 128.0 / binning != pixperchannel: logger.error("Can't figure out column offset dimension") return image numcol = int(image.shape[0] / 128 * binning) @@ -81,8 +84,7 @@ def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, for j in range(numcol): channel = imtemp[:, j, :] # pdb.set_trace() - mask = np.bitwise_and(channel < thresholdmax, - channel >= thresholdmin) + mask = np.bitwise_and(channel < thresholdmax, channel >= thresholdmin) value = np.mean(channel[mask]) offsets.append(value) # apply offset correction to images @@ -93,10 +95,22 @@ def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, return subtracted.reshape(image.shape) -def filter_image(imag, useint, whichint, usebin, - whichbin, usegaus, gausks, - gaussig, usemed, medks, - usels, lsmin, lsmax, usecoffset): +def filter_image( + imag, + useint, + whichint, + usebin, + whichbin, + usegaus, + gausks, + gaussig, + usemed, + medks, + usels, + lsmin, + lsmax, + usecoffset, +): """ Filter an image and return the filtered image """ @@ -132,8 +146,15 @@ class Recorder(QThread): increase_progress = pyqtSignal(int) finish = pyqtSignal() - def __init__(self, path, improc=None, vbfsettings=None, - outputpath=None, imrange=(None, None)): + def __init__( + self, + path, + improc=None, + vbfsettings=None, + outputpath=None, + imrange=(None, None), + **options, + ): QThread.__init__(self) logger.debug("Initializing recorder object") # filename @@ -186,6 +207,10 @@ def __init__(self, path, improc=None, vbfsettings=None, if self.outputpath is not None: self.outputpath = str(Path(self.outputpath)) + # other options, added 2.11.2020 + self.options = options + # print(options, self.options) + def run(self): self.convert_HDF5() self.finish.emit() @@ -211,13 +236,26 @@ def convert_HDF5(self): # This is important! also initializes the headers! firstframe = self.read_frame(0) pff = filter_image(firstframe, **self.improc) + + # initialise maximum_image if checkBox checked + # print(f"average image: {'calcmax' in self.options} {self.options['calcmax']}") + if "calcmax" in self.options and self.options["calcmax"]: + # print("creating maxiumum image") + self.maximum_image = np.zeros_like(pff) + + # print(f"average image: {'calcave' in self.options} {self.options['calcave']}") + # initialise average_image if checkBox checked + if "calcave" in self.options and self.options["calcave"]: + # print("creating average_image") + # as datatype is typcially 8- or 16-bit then 32-bit image should be fine + self.average_image = np.zeros_like(pff, dtype=np.float32) + self.scangroup = self.stream.create_group("Scan") # do we need a virtual bright field calculated? if self.vbfproc["calcvbf"]: # make a VBF mask. For this we need a frame # ZOB center offset - zoboffset = [self.vbfproc["vbfxoffset"], - self.vbfproc["vbfyoffset"]] + zoboffset = [self.vbfproc["vbfxoffset"], self.vbfproc["vbfyoffset"]] radius = self.vbfproc["vbfrad"] # generate mask self.mask = self._virtual_bf_mask(pff, zoboffset, radius) @@ -245,11 +283,13 @@ def valid_first_tvips_file(filename): if match is not None: num, ext = match.groups() if ext != "tvips": - raise ValueError(f"Invalid tvips file: extension {ext}, must " - f"be tvips") + raise ValueError( + f"Invalid tvips file: extension {ext}, must " f"be tvips" + ) if int(num) != 0: - raise ValueError("Can only read video sequences starting with " - "part 000") + raise ValueError( + "Can only read video sequences starting with " "part 000" + ) return True else: raise ValueError("Could not recognize as a valid tvips file") @@ -261,10 +301,8 @@ def read_frame(self, frame=0): fh = FileHandle(file=f) fh.seek(bite_start - self.ranges[toopen][0]) frame = np.fromfile( - fh, - count=self.general.dimx*self.general.dimy, - dtype=self.dtype - ) + fh, count=self.general.dimx * self.general.dimy, dtype=self.dtype + ) frame.shape = (self.general.dimx, self.general.dimy) return frame @@ -288,10 +326,14 @@ def _get_byte_filename(self, b): def _get_frame_byte_position(self, frame): """Get the byte where a frame starts""" - frame_byte_size = (self.general.dimx * self.general.dimy * - self.general.bitsperpixel//8) - bite_start = (self.generalheadersize + self.general.frameheaderbytes + - (frame_byte_size + self.general.frameheaderbytes)*frame) + frame_byte_size = ( + self.general.dimx * self.general.dimy * self.general.bitsperpixel // 8 + ) + bite_start = ( + self.generalheadersize + + self.general.frameheaderbytes + + (frame_byte_size + self.general.frameheaderbytes) * frame + ) return bite_start def _get_frame_byte_position_in_file(self, frame): @@ -303,21 +345,24 @@ def _get_frame_byte_position_in_file(self, frame): def _get_byte_frame_position(self, byte_start): """Get the frame index closest corresponding to a byte""" - frame_byte_size = (self.general.dimx * self.general.dimy * - self.general.bitsperpixel//8) - frame = ((byte_start - self.generalheadersize - - self.general.frameheaderbytes) // - (frame_byte_size + self.general.frameheaderbytes)) + frame_byte_size = ( + self.general.dimx * self.general.dimy * self.general.bitsperpixel // 8 + ) + frame = ( + byte_start - self.generalheadersize - self.general.frameheaderbytes + ) // (frame_byte_size + self.general.frameheaderbytes) return frame def _get_files_size_dictionary(self): """ Get a dictionary of files (keys) and total size (values) """ + def get_filesize(fn): with open(fn, "rb") as f: fh = FileHandle(file=f) return fn, fh.size + sizes = self._scan_over_all_files(get_filesize) return dict(sizes) @@ -329,7 +374,7 @@ def _get_files_ranges(self): ranges = {} starts = 0 for i in sizes: - ends = starts+sizes[i] + ends = starts + sizes[i] ranges[i] = (starts, ends) starts = ends return ranges @@ -354,8 +399,7 @@ def _frames_exceeded(self): """Have we read the number of frames?""" if self.finalim is not None: if self.current_frame >= self.finalim: - logger.debug(f"We are at frame {self.current_frame}. " - f"Quitting.") + logger.debug(f"We are at frame {self.current_frame}. " f"Quitting.") return True return False @@ -365,11 +409,10 @@ def _scan_over_all_files(self, func, *args, **kwargs): results = [] part = int(self.filename[-9:-6]) if part != 0: - raise ValueError("Can only read video sequences starting with " - "part 000") + raise ValueError("Can only read video sequences starting with " "part 000") try: while True: - fn = self.filename[:-9]+"{:03d}.tvips".format(part) + fn = self.filename[:-9] + "{:03d}.tvips".format(part) if not os.path.exists(fn): logger.debug(f"There is no file {fn}; breaking loop") break @@ -404,13 +447,15 @@ def _readGeneral(self, fh): self.inc = self.general.frameheaderbytes self.frame_header = TVIPS_RECORDER_FRAME_HEADER else: - raise NotImplementedError(f"Version {self.general.version} not " - f"yet supported.") + raise NotImplementedError( + f"Version {self.general.version} not " f"yet supported." + ) self.dt = np.dtype(self.frame_header) # make sure the record consumes less bytes than reported in the main # header - assert self.inc >= self.dt.itemsize, ("The record consumes more bytes " - "than stated in the main header") + assert self.inc >= self.dt.itemsize, ( + "The record consumes more bytes " "than stated in the main header" + ) def _readIndividualFile(self, fn): """ @@ -429,7 +474,8 @@ def _readIndividualFile(self, fn): if self.startim > self.current_frame: self.current_frame += 1 current_byte = self._get_frame_byte_position_in_file( - self.current_frame) + self.current_frame + ) fh.seek(current_byte - self.general.frameheaderbytes) continue if self.finalim < self.current_frame: @@ -442,16 +488,16 @@ def _readIndividualFile(self, fn): def _readFrame(self, fh, record=None): # read frame header header = fh.read_record(self.frame_header) - logger.debug(f"{self.current_frame}: Starting frame read " - f"(pos: {fh.tell()}). rot: {header['rotidx']}") + logger.debug( + f"{self.current_frame}: Starting frame read " + f"(pos: {fh.tell()}). rot: {header['rotidx']}" + ) skip = self.inc - self.dt.itemsize fh.seek(skip, 1) # read frame frame = np.fromfile( - fh, - count=self.general.dimx*self.general.dimy, - dtype=self.dtype - ) + fh, count=self.general.dimx * self.general.dimy, dtype=self.dtype + ) frame.shape = (self.general.dimx, self.general.dimy) # do calculations on the frame frame = filter_image(frame, **self.improc) @@ -461,16 +507,35 @@ def _readFrame(self, fh, record=None): for i in self.frame_header: ds.attrs[i[0]] = header[i[0]] # store the rotation index for finding start and stop later - self.rotidxs.append(header['rotidx']) + self.rotidxs.append(header["rotidx"]) # immediately calculate and store the VBF intensity if required if self.vbfproc["calcvbf"]: vbf_int = frame[self.mask].mean() self.vbfs.append(vbf_int) + if "calcmax" in self.options and self.options["calcmax"]: + # maximum_image should already be initialised in self.convert_HDF5 + self.maximum_image = np.stack((self.maximum_image, frame), axis=0).max( + axis=0 + ) + + if "calcave" in self.options and self.options["calcave"]: + # average_image should already be initialised in self.convert_HDF5 + self.average_image = np.stack( + ( + self.average_image, + # frame is scaled as a function of 1/total_frames and then summed (average) + (1 / (self.finalim - self.startim)) + * frame.astype(self.average_image.dtype), + ), + axis=0, + ).sum(axis=0) + def _update_gui_progess(self): """If using the GUI update features with progress""" - value = int((self.current_frame - self.startim) / - (self.finalim - self.startim)*100) + value = int( + (self.current_frame - self.startim) / (self.finalim - self.startim) * 100 + ) self.increase_progress.emit(value) def _find_start_and_stop(self): @@ -478,40 +543,33 @@ def _find_start_and_stop(self): previous = self.rotidxs[0] for j, i in enumerate(self.rotidxs): if i > previous: - self.start = j-1 + self.start = j - 1 logger.info(f"Found start at frame {j-1}") - self.scangroup.attrs[ - "start_frame"] = self.start + self.scangroup.attrs["start_frame"] = self.start break previous = i else: self.start = None - self.scangroup.attrs[ - "start_frame"] = "None" + self.scangroup.attrs["start_frame"] = "None" # loop over it backwards to find the end # infact the index goes back to 1 for j, i in reversed(list(enumerate(self.rotidxs))): if i > 1: self.end = j logger.info(f"Found final at frame {j}") - self.scangroup.attrs[ - "end_frame"] = self.end - self.scangroup.attrs[ - "final_rotinx"] = i + self.scangroup.attrs["end_frame"] = self.end + self.scangroup.attrs["final_rotinx"] = i self.final_rotinx = i break else: self.end = None - self.scangroup.attrs[ - "end_frame"] = "None" + self.scangroup.attrs["end_frame"] = "None" self.final_rotinx = None - self.scangroup.attrs[ - "final_rotinx"] = "None" + self.scangroup.attrs["final_rotinx"] = "None" # add a couple more attributes for good measure self.scangroup.attrs["total_stream_frames"] = len(self.rotidxs) if self.end is not None and self.start is not None: - self.scangroup.attrs[ - "ims_between_start_end"] = self.end-self.start + self.scangroup.attrs["ims_between_start_end"] = self.end - self.start def _save_preliminary_scan_info(self): # save rotation indexes and vbf intensities @@ -519,11 +577,19 @@ def _save_preliminary_scan_info(self): if self.vbfproc["calcvbf"]: self.scangroup.create_dataset("vbf_intensities", data=self.vbfs) + # save options + if "calcmax" in self.options and self.options["calcmax"]: + self.scangroup.create_dataset("maximum_image", data=self.maximum_image) + if "calcave" in self.options and self.options["calcave"]: + self.scangroup.create_dataset("average_image", data=self.average_image) + @staticmethod def _virtual_bf_mask(arr, centeroffsetpx=(0, 0), radiuspx=10): """Create virtual bright field mask""" - xx, yy = np.meshgrid(np.arange(arr.shape[0], dtype=np.float), - np.arange(arr.shape[1], dtype=np.float)) + xx, yy = np.meshgrid( + np.arange(arr.shape[0], dtype=np.float), + np.arange(arr.shape[1], dtype=np.float), + ) xx -= 0.5 * arr.shape[0] + centeroffsetpx[0] yy -= 0.5 * arr.shape[1] + centeroffsetpx[1] mask = np.hypot(xx, yy) < radiuspx @@ -531,27 +597,33 @@ def _virtual_bf_mask(arr, centeroffsetpx=(0, 0), radiuspx=10): def determine_recorder_image_dimension(self, opts): # scan dimensions - if (opts.dimension is not None): - self.xdim, self.ydim = list(map(int, opts.dimension.split('x'))) + if opts.dimension is not None: + self.xdim, self.ydim = list(map(int, opts.dimension.split("x"))) else: dim = math.sqrt(self.final_rotinx) if not dim == int(dim): - raise ValueError("Can't determine correct image dimensions, " - "please supply values manually (--dimension)") + raise ValueError( + "Can't determine correct image dimensions, " + "please supply values manually (--dimension)" + ) self.xdim, self.ydim = dim, dim - logger.debug("Image dimensions: {}x{}".format(self.xdim, - self.ydim)) + logger.debug("Image dimensions: {}x{}".format(self.xdim, self.ydim)) class hdf5Intermediate(h5py.File): """This class represents the intermediate hdf5 file handle""" + def __init__(self, filepath, mode="r"): super().__init__(filepath, mode) - (self.total_frames, - self.start_frame, - self.end_frame, - self.final_rotator, - dim, self.imdimx, self.imdimy) = self.get_scan_info() + ( + self.total_frames, + self.start_frame, + self.end_frame, + self.final_rotator, + dim, + self.imdimx, + self.imdimy, + ) = self.get_scan_info() self.sdimx = dim self.sdimy = dim @@ -598,19 +670,33 @@ def get_scan_info(self): imdimy = None return (tot, start, end, finrot, dim, imdimx, imdimy) - def get_vbf_image(self, sdimx=None, sdimy=None, start_frame=None, - end_frame=None, hyst=0, snakescan=True): + def get_vbf_image( + self, + sdimx=None, + sdimy=None, + start_frame=None, + end_frame=None, + hyst=0, + snakescan=True, + ): # try to get the rotator data try: vbfs = self["Scan"]["vbf_intensities"][:] except Exception: - raise Exception("No VBF information found in dataset, please " - "calculate from TVIPS file.") + raise Exception( + "No VBF information found in dataset, please " + "calculate from TVIPS file." + ) logger.debug("Succesfully imported vbf intensities") logger.debug("Now calculating scan indexes") scan_indexes = self.calculate_scan_export_indexes( - sdimx=sdimx, sdimy=sdimy, start_frame=start_frame, - end_frame=end_frame, hyst=hyst, snakescan=snakescan) + sdimx=sdimx, + sdimy=sdimy, + start_frame=start_frame, + end_frame=end_frame, + hyst=hyst, + snakescan=snakescan, + ) logger.debug("Calculated scan indexes") if sdimx is None: sdimx = self.sdimx @@ -620,11 +706,25 @@ def get_vbf_image(self, sdimx=None, sdimy=None, start_frame=None, logger.debug("Applied calculated indexes and retrieved image") return img - def get_blo_export_data(self, sdimx=None, sdimy=None, start_frame=None, - end_frame=None, hyst=0, snakescan=True, crop=None): + def get_blo_export_data( + self, + sdimx=None, + sdimy=None, + start_frame=None, + end_frame=None, + hyst=0, + snakescan=True, + crop=None, + ): scan_indexes = self.calculate_scan_export_indexes( - sdimx=sdimx, sdimy=sdimy, start_frame=start_frame, - end_frame=end_frame, hyst=hyst, snakescan=snakescan, crop=crop) + sdimx=sdimx, + sdimy=sdimy, + start_frame=start_frame, + end_frame=end_frame, + hyst=hyst, + snakescan=snakescan, + crop=crop, + ) logger.debug("Calculated scan indexes") if sdimx is None: sdimx = self.sdimx @@ -637,20 +737,29 @@ def get_blo_export_data(self, sdimx=None, sdimy=None, start_frame=None, if crop is not None: xmin, xmax, ymin, ymax = crop sdimx = xmax - xmin - sdimy = ymax-ymin + sdimy = ymax - ymin shape = (sdimx, sdimy, *imshap) return shape, scan_indexes - def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, - start_frame=None, end_frame=None, - hyst=0, snakescan=True, crop=None): + def calculate_scan_export_indexes( + self, + sdimx=None, + sdimy=None, + start_frame=None, + end_frame=None, + hyst=0, + snakescan=True, + crop=None, + ): """Calculate the indexes of the list of frames to consider for the scan or VBF""" try: rots = self["Scan"]["rotation_indexes"][:] except Exception: - raise Exception("No VBF information found in dataset, please " - "calculate from TVIPS file.") + raise Exception( + "No VBF information found in dataset, please " + "calculate from TVIPS file." + ) logger.debug("Succesfully read rotator indexes") # set the scan info if sdimx is None: @@ -663,17 +772,19 @@ def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, # if a start frame is given, it's easy, we ignore rots if start_frame is not None: if end_frame is None: - end_frame = start_frame + sdimx*sdimy - 1 + end_frame = start_frame + sdimx * sdimy - 1 if end_frame >= self.total_frames: raise Exception("Final frame is out of bounds") if end_frame <= start_frame: - raise Exception("Final frame index must be larger than " - "first frame index") - if end_frame+1-start_frame != sdimx*sdimy: - raise Exception("Number of custom frames does not match " - "scan dimension") + raise Exception( + "Final frame index must be larger than " "first frame index" + ) + if end_frame + 1 - start_frame != sdimx * sdimy: + raise Exception( + "Number of custom frames does not match " "scan dimension" + ) # just create an index array - sel = np.arange(start_frame, end_frame+1) + sel = np.arange(start_frame, end_frame + 1) sel = sel.reshape(sdimy, sdimx) # reverse correct even scan lines if snakescan: @@ -683,45 +794,56 @@ def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, # check for crop if crop is not None: - logger.info('Cropping to: {}'.format(crop)) + logger.info("Cropping to: {}".format(crop)) if all(i is not None for i in crop): xmin, xmax, ymin, ymax = crop - if all(i >= 0 for i in (xmin, ymin)) and xmax < sdimx and ymax < sdimy: - sel = sel[ymin: ymax, xmin: xmax] + if ( + all(i >= 0 for i in (xmin, ymin)) + and xmax < sdimx + and ymax < sdimy + ): + sel = sel[ymin:ymax, xmin:xmax] else: - logger.warning('Aborting crop due to incorrect given dimensions: {}'.format(crop)) + logger.warning( + "Aborting crop due to incorrect given dimensions: {}".format( + crop + ) + ) return sel.ravel() # if frames or not given, we must use our best guess and match # rotators else: try: - rots = rots[self.start_frame:self.end_frame+1] + rots = rots[self.start_frame : self.end_frame + 1] except Exception: - raise Exception("No valid first or last scan frames detected, " - "must provide manual input") + raise Exception( + "No valid first or last scan frames detected, " + "must provide manual input" + ) # check whether sdimx*sdimy matches the final rotator index if not isinstance(self.final_rotator, int): - raise Exception("No final rotator index found, " - "can't align scan") - if sdimx*sdimy != self.final_rotator: - raise Exception("Scan dim x * scan dim y should match " - "the final rotator index if no custom " - "frames are specified") - indxs = np.zeros(sdimy*sdimx, dtype=int) + raise Exception("No final rotator index found, " "can't align scan") + if sdimx * sdimy != self.final_rotator: + raise Exception( + "Scan dim x * scan dim y should match " + "the final rotator index if no custom " + "frames are specified" + ) + indxs = np.zeros(sdimy * sdimx, dtype=int) prevw = 1 for j, _ in enumerate(indxs): # find where the argument is j - w = np.argwhere(rots == j+1) + w = np.argwhere(rots == j + 1) if w.size > 0: w = w[0, 0] prevw = w else: # move up if the rot index stays the same, otherwise copy - if prevw+1 < len(rots): - if rots[prevw+1] == rots[prevw]: - prevw = prevw+1 + if prevw + 1 < len(rots): + if rots[prevw + 1] == rots[prevw]: + prevw = prevw + 1 w = prevw indxs[j] = w # just an array of indexes @@ -732,4 +854,4 @@ def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, # hysteresis correction on even scan lines img[::2] = np.roll(img[::2], hyst, axis=1) # add the start index - return img.ravel()+self.start_frame + return img.ravel() + self.start_frame diff --git a/tvipsconverter/widget_2.ui b/tvipsconverter/widget_2.ui index 671f58d..363c976 100644 --- a/tvipsconverter/widget_2.ui +++ b/tvipsconverter/widget_2.ui @@ -82,7 +82,7 @@ <html><head/><body><p><br/></p></body></html> - 1 + 0 @@ -156,8 +156,8 @@ 0 0 - 1127 - 650 + 1115 + 705 @@ -685,6 +685,35 @@ + + + + Other calculations + + + + + + Calculate maximum diffraction pattern + + + true + + + + + + + Calculate average diffraction pattern + + + true + + + + + + @@ -1034,8 +1063,8 @@ 0 0 - 1086 - 565 + 1069 + 610 diff --git a/tvipsconverter/widgets.py b/tvipsconverter/widgets.py index 1d070b9..ec49adc 100644 --- a/tvipsconverter/widgets.py +++ b/tvipsconverter/widgets.py @@ -1,11 +1,10 @@ from PyQt5 import uic -from PyQt5.QtWidgets import (QApplication, QFileDialog, QGraphicsScene) +from PyQt5.QtWidgets import QApplication, QFileDialog, QGraphicsScene from PyQt5.QtCore import QThread, pyqtSignal import sys import matplotlib.pyplot as plt from matplotlib.patches import Circle -from matplotlib.backends.backend_qt5agg import (FigureCanvasQTAgg as - FigureCanvas) +from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from pathlib import Path import logging from time import sleep @@ -22,14 +21,14 @@ logger = logging.getLogger(__name__) # import the UI interface -rawgui, Window = uic.loadUiType(str(Path(__file__).parent.absolute()) + - "/widget_2.ui") +rawgui, Window = uic.loadUiType(str(Path(__file__).parent.absolute()) + "/widget_2.ui") class External(QThread): """ Runs a counter thread. """ + countChanged = pyqtSignal(int) finish = pyqtSignal() @@ -48,6 +47,7 @@ def run(self): class ConnectedWidget(rawgui): """Class connecting the gui elements to the back-end functionality""" + def __init__(self, window): super().__init__() self.window = window @@ -141,9 +141,9 @@ def update_levels_vbf(self): vmax = self.horizontalSlider_2.value() mn = self.vbf_data.min() mx = self.vbf_data.max() - unit = (mx-mn)/100 - climmin = mn+vmin*unit - climmax = mn+(vmax+1)*unit + unit = (mx - mn) / 100 + climmin = mn + vmin * unit + climmax = mn + (vmax + 1) * unit try: self.vbf_im.set_clim(climmin, climmax) canvas = FigureCanvas(self.fig_vbf) @@ -151,9 +151,7 @@ def update_levels_vbf(self): scene = QGraphicsScene() scene.addWidget(canvas) self.graphicsView_3.setScene(scene) - self.graphicsView_3.fitInView( - scene.sceneRect(), - ) + self.graphicsView_3.fitInView(scene.sceneRect(),) self.repaint_widget(self.graphicsView_3) except Exception as e: logger.debug(f"Error: {e}") @@ -171,15 +169,15 @@ def update_final_frame(self): if self.checkBox_2.checkState(): # we use self defined size start = self.spinBox_15.value() - frms = self.spinBox.value()*self.spinBox_2.value() - self.spinBox_16.setValue(start+frms-1) + frms = self.spinBox.value() * self.spinBox_2.value() + self.spinBox_16.setValue(start + frms - 1) else: # we use auto-size start = self.spinBox_15.value() frms = self.lineEdit_11.text() try: dim = np.sqrt(int(frms)) - self.spinBox_16.setValue(start+dim**2-1) + self.spinBox_16.setValue(start + dim ** 2 - 1) except Exception: self.spinBox_16.setValue(0) @@ -206,14 +204,12 @@ def open_tvips_file(self): self.statusedit.setText(str(e)) def openFileBrowser(self, fs): - path, okpres = QFileDialog.getOpenFileName(caption="Select file", - filter=fs) + path, okpres = QFileDialog.getOpenFileName(caption="Select file", filter=fs) if okpres: return str(Path(path)) def saveFileBrowser(self, fs): - path, okpres = QFileDialog.getSaveFileName(caption="Select file", - filter=fs) + path, okpres = QFileDialog.getSaveFileName(caption="Select file", filter=fs) if okpres: return str(Path(path)) @@ -237,13 +233,13 @@ def read_modsettings(self): "usels": self.checkBox_4.checkState(), "lsmin": self.spinBox_7.value(), "lsmax": self.spinBox_8.value(), - "usecoffset": self.checkBox.checkState() + "usecoffset": self.checkBox.checkState(), } vbfsettings = { "calcvbf": self.checkBox_10.checkState(), "vbfrad": self.spinBox_12.value(), "vbfxoffset": self.spinBox_13.value(), - "vbfyoffset": self.spinBox_14.value() + "vbfyoffset": self.spinBox_14.value(), } return path, improc, vbfsettings @@ -263,9 +259,8 @@ def updatePreview(self): # (self.path_preview != path): self.update_line(self.statusedit, "Extracting frame...") self.original_preview = rec.getOriginalPreviewImage( - path, improc=improc, - vbfsettings=vbfsettings, - frame=framenum) + path, improc=improc, vbfsettings=vbfsettings, frame=framenum + ) # update the path self.path_preview = path ois = self.original_preview.shape @@ -273,36 +268,42 @@ def updatePreview(self): nis = filterframe.shape # check if the VBF aperture fits in the frame if vbfsettings["calcvbf"]: - midx = nis[1]//2 - midy = nis[0]//2 + midx = nis[1] // 2 + midy = nis[0] // 2 xx = vbfsettings["vbfxoffset"] yy = vbfsettings["vbfyoffset"] rr = vbfsettings["vbfrad"] - if (midx+xx-rr < 0 or - midx+xx+rr > nis[1] or - midy+yy-rr < 0 or - midy+yy-rr > nis[0]): - raise Exception("Virtual bright field aperture out " - "of bounds") + if ( + midx + xx - rr < 0 + or midx + xx + rr > nis[1] + or midy + yy - rr < 0 + or midy + yy - rr > nis[0] + ): + raise Exception("Virtual bright field aperture out " "of bounds") # plot the image and the circle over it if self.fig_prev is not None: plt.close(self.fig_prev) - self.fig_prev = plt.figure(frameon=False, - figsize=(filterframe.shape[1]/100, - filterframe.shape[0]/100)) + self.fig_prev = plt.figure( + frameon=False, + figsize=(filterframe.shape[1] / 100, filterframe.shape[0] / 100), + ) canvas = FigureCanvas(self.fig_prev) - ax = plt.Axes(self.fig_prev, [0., 0., 1., 1.]) + ax = plt.Axes(self.fig_prev, [0.0, 0.0, 1.0, 1.0]) ax.set_axis_off() self.fig_prev.add_axes(ax) ax.imshow(filterframe, cmap="Greys_r") if vbfsettings["calcvbf"]: xoff = vbfsettings["vbfxoffset"] yoff = vbfsettings["vbfyoffset"] - circ = Circle((filterframe.shape[1]//2+xoff, - filterframe.shape[0]//2+yoff), - vbfsettings["vbfrad"], - color="red", - alpha=0.5) + circ = Circle( + ( + filterframe.shape[1] // 2 + xoff, + filterframe.shape[0] // 2 + yoff, + ), + vbfsettings["vbfrad"], + color="red", + alpha=0.5, + ) ax.add_patch(circ) canvas.draw() scene = QGraphicsScene() @@ -311,8 +312,10 @@ def updatePreview(self): self.graphicsView.fitInView(scene.sceneRect()) self.repaint_widget(self.graphicsView) self.update_line(self.statusedit, "Succesfully created preview.") - self.update_line(self.lineEdit_8, f"Original: {ois[0]}x{ois[1]}. " - f"New: {nis[0]}x{nis[1]}.") + self.update_line( + self.lineEdit_8, + f"Original: {ois[0]}x{ois[1]}. " f"New: {nis[0]}x{nis[1]}.", + ) except Exception as e: self.update_line(self.statusedit, f"Error: {e}") # empty the preview @@ -336,8 +339,7 @@ def get_hdf5_path(self): # open a savefile browser try: # read the gui info - (self.inpath, self.improc, - self.vbfsettings) = self.read_modsettings() + (self.inpath, self.improc, self.vbfsettings) = self.read_modsettings() if not self.inpath: raise Exception("A TVIPS file must be selected!") self.oupath = self.saveFileBrowser("HDF5 (*.hdf5)") @@ -396,8 +398,7 @@ def image_range(self): def write_to_hdf5(self): try: - (self.inpath, self.improc, - self.vbfsettings) = self.read_modsettings() + (self.inpath, self.improc, self.vbfsettings) = self.read_modsettings() if not self.inpath: raise Exception("A TVIPS file must be selected!") self.oupath = self.lineEdit_2.text() @@ -410,11 +411,15 @@ def write_to_hdf5(self): improc = self.improc vbfsettings = self.vbfsettings start_frame, end_frame = self.image_range - self.get_thread = rec.Recorder(path, - improc=improc, - vbfsettings=vbfsettings, - outputpath=opath, - imrange=(start_frame, end_frame)) + self.get_thread = rec.Recorder( + path, + improc=improc, + vbfsettings=vbfsettings, + outputpath=opath, + imrange=(start_frame, end_frame), + calcmax=self.checkBox_maxiumum_image.isChecked(), # options kwarg + calcave=self.checkBox_average_image.isChecked(), # options kwarg + ) self.get_thread.increase_progress.connect(self.increase_progbar) self.get_thread.finish.connect(self.done_hdf5export) self.get_thread.start() @@ -425,8 +430,7 @@ def write_to_hdf5(self): def done_hdf5export(self): self.window.setEnabled(True) # also update lines in the second pannel - self.update_line(self.statusedit, - "Succesfully exported to HDF5") + self.update_line(self.statusedit, "Succesfully exported to HDF5") # don't auto update, the gui may be before the file exists # self.update_line(self.lineEdit_4, self.lineEdit_2.text()) @@ -452,12 +456,10 @@ def auto_read_hdf5(self): else: self.update_line(self.lineEdit_11, "?") if dim is not None: - self.update_line(self.lineEdit_12, - f"{str(int(dim))}x{str(int(dim))}") + self.update_line(self.lineEdit_12, f"{str(int(dim))}x{str(int(dim))}") else: self.update_line(self.lineEdit_12, "?") - self.update_line(self.lineEdit_13, - f"{str(imdimx)}x{str(imdimy)}") + self.update_line(self.lineEdit_13, f"{str(imdimx)}x{str(imdimy)}") self.update_final_frame() f.close() except Exception as e: @@ -503,29 +505,35 @@ def update_vbf(self): else: snakescan = False # calculate the image - logger.debug(f"We try to create a VBF image with data: " - f"S.F. {start_frame}, E.F. {end_frame}, " - f"Dims: x {sdimx} y {sdimy}," - f"hyst: {hyst}") - self.vbf_data = f.get_vbf_image(sdimx, sdimy, start_frame, - end_frame, hyst, snakescan) + logger.debug( + f"We try to create a VBF image with data: " + f"S.F. {start_frame}, E.F. {end_frame}, " + f"Dims: x {sdimx} y {sdimy}," + f"hyst: {hyst}" + ) + self.vbf_data = f.get_vbf_image( + sdimx, sdimy, start_frame, end_frame, hyst, snakescan + ) logger.debug("Succesfully created the VBF array") # save the settings for later storage - self.vbf_sets = {"start_frame": start_frame, - "end_frame": end_frame, - "scan_dim_x": sdimx, - "scan_dim_y": sdimy, - "hysteresis": hyst, - "winding_scan": snakescan} + self.vbf_sets = { + "start_frame": start_frame, + "end_frame": end_frame, + "scan_dim_x": sdimx, + "scan_dim_y": sdimy, + "hysteresis": hyst, + "winding_scan": snakescan, + } # plot the image and store it for further use. First close prior # image if self.fig_vbf is not None: plt.close(self.fig_vbf) - self.fig_vbf = plt.figure(frameon=False, - figsize=(self.vbf_data.shape[1]/100, - self.vbf_data.shape[0]/100)) + self.fig_vbf = plt.figure( + frameon=False, + figsize=(self.vbf_data.shape[1] / 100, self.vbf_data.shape[0] / 100), + ) canvas = FigureCanvas(self.fig_vbf) - ax = plt.Axes(self.fig_vbf, [0., 0., 1., 1.]) + ax = plt.Axes(self.fig_vbf, [0.0, 0.0, 1.0, 1.0]) ax.set_axis_off() self.fig_vbf.add_axes(ax) self.vbf_im = ax.imshow(self.vbf_data, cmap="plasma") @@ -584,31 +592,35 @@ def write_to_file(self): dp_scale = self.doubleSpinBox_3.value() # calculate the image filetyp = self.comboBox.currentText() - logger.debug(f"We try to create a {filetyp} file with data: " - f"S.F. {start_frame}, E.F. {end_frame}, " - f"Dims: x {sdimx} y {sdimy}," - f"hyst: {hyst}, snakescan: {snakescan}") + logger.debug( + f"We try to create a {filetyp} file with data: " + f"S.F. {start_frame}, E.F. {end_frame}, " + f"Dims: x {sdimx} y {sdimy}," + f"hyst: {hyst}, snakescan: {snakescan}" + ) logger.debug("Calculating shape and indexes") # xmin, xmax, ymin, ymax - crop = (self.spinBox_22.value(), - self.spinBox_23.value(), - self.spinBox_20.value(), - self.spinBox_21.value(), - ) - shape, indexes = f.get_blo_export_data(sdimx, sdimy, - start_frame, - end_frame, hyst, - snakescan, crop=crop) + crop = ( + self.spinBox_22.value(), + self.spinBox_23.value(), + self.spinBox_20.value(), + self.spinBox_21.value(), + ) + shape, indexes = f.get_blo_export_data( + sdimx, sdimy, start_frame, end_frame, hyst, snakescan, crop=crop + ) logger.debug(f"Shape: {shape}") logger.debug(f"Starting to write {filetyp} file") self.update_line(self.statusedit, f"Writing {filetyp} file...") if filetyp == ".blo": self.get_thread = blf.bloFileWriter( - f, path_blo, shape, indexes, scan_scale, dp_scale) + f, path_blo, shape, indexes, scan_scale, dp_scale + ) elif filetyp == ".hspy": self.get_thread = hspf.hspyFileWriter( - f, path_blo, shape, indexes, scan_scale, dp_scale) + f, path_blo, shape, indexes, scan_scale, dp_scale + ) else: raise NotImplementedError("Unrecognized file type") self.get_thread.increase_progress.connect(self.increase_progbar) @@ -621,8 +633,7 @@ def write_to_file(self): def done_bloexport(self): self.window.setEnabled(True) # also update lines in the second pannel - self.update_line(self.statusedit, - "Succesfully exported to file") + self.update_line(self.statusedit, "Succesfully exported to file") def export_tiffs(self): path_hdf5 = self.lineEdit_4.text() @@ -648,7 +659,7 @@ def export_tiffs(self): raise Exception("Unexpected dtype") if tot_frames <= last_frame: raise Exception("Frames are out of range") - frames = np.arange(first_frame, last_frame+1) + frames = np.arange(first_frame, last_frame + 1) self.update_line(self.statusedit, "Exporting to tiff files...") self.get_thread = tfe.TiffFileWriter(f, frames, dtype, pre, fin) self.get_thread.increase_progress.connect(self.increase_progbar) @@ -661,8 +672,7 @@ def export_tiffs(self): def done_tiffexport(self): self.window.setEnabled(True) # also update lines in the second pannel - self.update_line(self.statusedit, - "Succesfully exported Tiff files") + self.update_line(self.statusedit, "Succesfully exported Tiff files") def increase_progbar(self, value): self.progressBar.setValue(value) @@ -678,33 +688,38 @@ def show_cropped_region(self): ymin = self.spinBox_20.value() ymax = self.spinBox_21.value() if not xmax - xmin > 0: - logger.warning('Not valid cropping dimensions: x.') + logger.warning("Not valid cropping dimensions: x.") return if not ymax - ymin > 0: - logger.warning('Not valid cropping dimensions: x.') + logger.warning("Not valid cropping dimensions: x.") return if self.fig_vbf is None: - logger.warning('No VBF figure plotted yet.') + logger.warning("No VBF figure plotted yet.") # no VBF figure plotted yet return - - label = 'crop_rect' + label = "crop_rect" try: # see if rectangle already plotted and just update - index = [i.get_label() for i in - self.fig_vbf.axes[0].patches].index(label) + index = [i.get_label() for i in self.fig_vbf.axes[0].patches].index(label) rect = self.fig_vbf.axes[0].patches[index] - logger.info('Updating rectangle.') + logger.info("Updating rectangle.") rect.set_height(ymax - ymin) rect.set_width(xmax - xmin) rect.set_x(xmin) rect.set_y(ymin) except ValueError: - logger.info('Plotting rectangle.') - rect = plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin, - fill=False, ec='k', ls='dashed', label=label) + logger.info("Plotting rectangle.") + rect = plt.Rectangle( + (xmin, ymin), + xmax - xmin, + ymax - ymin, + fill=False, + ec="k", + ls="dashed", + label=label, + ) self.fig_vbf.axes[0].add_patch(rect) self.fig_vbf.canvas.draw() From ccf28803c8233852db0d89d928156f1913d4e2fa Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Tue, 3 Nov 2020 11:49:30 +0100 Subject: [PATCH 02/21] add comment- creating options image in _readFrame --- tvipsconverter/utils/recorder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tvipsconverter/utils/recorder.py b/tvipsconverter/utils/recorder.py index a989fb4..5a03b95 100644 --- a/tvipsconverter/utils/recorder.py +++ b/tvipsconverter/utils/recorder.py @@ -513,6 +513,7 @@ def _readFrame(self, fh, record=None): vbf_int = frame[self.mask].mean() self.vbfs.append(vbf_int) + # calculate images as specified in the options if "calcmax" in self.options and self.options["calcmax"]: # maximum_image should already be initialised in self.convert_HDF5 self.maximum_image = np.stack((self.maximum_image, frame), axis=0).max( From 2d486d4aa44c8e9aaa1362b0e4dd67807ac1058a Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Tue, 3 Nov 2020 20:32:39 +0100 Subject: [PATCH 03/21] added direct 16- to 8-bit conversion (skimage) --- tvipsconverter/utils/blockfile.py | 416 +++++++++++++++------------- tvipsconverter/widget_2.ui | 438 ++++++++++++++++-------------- tvipsconverter/widgets.py | 9 +- 3 files changed, 467 insertions(+), 396 deletions(-) diff --git a/tvipsconverter/utils/blockfile.py b/tvipsconverter/utils/blockfile.py index c88d2de..fac49fa 100644 --- a/tvipsconverter/utils/blockfile.py +++ b/tvipsconverter/utils/blockfile.py @@ -22,6 +22,7 @@ import warnings import datetime import dateutil +from skimage import util from dateutil import tz, parser @@ -52,8 +53,7 @@ def sarray2dict(sarray, dictionary=None): if dictionary is None: dictionary = OrderedDict() for name in sarray.dtype.names: - dictionary[name] = sarray[name][0] if len(sarray[name]) == 1 \ - else sarray[name] + dictionary[name] = sarray[name][0] if len(sarray[name]) == 1 else sarray[name] return dictionary @@ -87,14 +87,11 @@ def dict2sarray(dictionary, sarray=None, dtype=None): return sarray -def ISO_format_to_serial_date(date, time, timezone='UTC'): +def ISO_format_to_serial_date(date, time, timezone="UTC"): """ Convert ISO format to a serial date. """ - if timezone is None or timezone == 'Coordinated Universal Time': - timezone = 'UTC' - dt = parser.parse( - '%sT%s' % - (date, time)).replace( - tzinfo=tz.gettz(timezone)) + if timezone is None or timezone == "Coordinated Universal Time": + timezone = "UTC" + dt = parser.parse("%sT%s" % (date, time)).replace(tzinfo=tz.gettz(timezone)) return datetime_to_serial_date(dt) @@ -123,8 +120,7 @@ def serial_date_to_ISO_format(serial): """ dt_utc = serial_date_to_datetime(serial) dt_local = dt_utc.astimezone(tz.tzlocal()) - return (dt_local.date().isoformat(), dt_local.time().isoformat(), - dt_local.tzname()) + return (dt_local.date().isoformat(), dt_local.time().isoformat(), dt_local.tzname()) _logger = logging.getLogger(__name__) @@ -132,11 +128,11 @@ def serial_date_to_ISO_format(serial): # Plugin characteristics # ---------------------- -format_name = 'Blockfile' -description = 'Read/write support for ASTAR blockfiles' +format_name = "Blockfile" +description = "Read/write support for ASTAR blockfiles" full_support = False # Recognised file extension -file_extensions = ['blo', 'BLO'] +file_extensions = ["blo", "BLO"] default_extension = 0 # Writing capabilities: @@ -145,66 +141,73 @@ def serial_date_to_ISO_format(serial): mapping = { - 'blockfile_header.Beam_energy': - ("Acquisition_instrument.TEM.beam_energy", lambda x: x * 1e-3), - 'blockfile_header.Camera_length': - ("Acquisition_instrument.TEM.camera_length", lambda x: x * 1e-4), - 'blockfile_header.Scan_rotation': - ("Acquisition_instrument.TEM.rotation", lambda x: x * 1e-2), + "blockfile_header.Beam_energy": ( + "Acquisition_instrument.TEM.beam_energy", + lambda x: x * 1e-3, + ), + "blockfile_header.Camera_length": ( + "Acquisition_instrument.TEM.camera_length", + lambda x: x * 1e-4, + ), + "blockfile_header.Scan_rotation": ( + "Acquisition_instrument.TEM.rotation", + lambda x: x * 1e-2, + ), } -def get_header_dtype_list(endianess='<'): +def get_header_dtype_list(endianess="<"): end = endianess - dtype_list = \ + dtype_list = ( [ - ('ID', (bytes, 6)), - ('MAGIC', end + 'u2'), - ('Data_offset_1', end + 'u4'), # Offset VBF - ('Data_offset_2', end + 'u4'), # Offset DPs - ('UNKNOWN1', end + 'u4'), # Flags for ASTAR software? - ('DP_SZ', end + 'u2'), # Pixel dim DPs - ('DP_rotation', end + 'u2'), # [degrees ( * 100 ?)] - ('NX', end + 'u2'), # Scan dim 1 - ('NY', end + 'u2'), # Scan dim 2 - ('Scan_rotation', end + 'u2'), # [100 * degrees] - ('SX', end + 'f8'), # Pixel size [nm] - ('SY', end + 'f8'), # Pixel size [nm] - ('Beam_energy', end + 'u4'), # [V] - ('SDP', end + 'u2'), # Pixel size [100 * ppcm] - ('Camera_length', end + 'u4'), # [10 * mm] - ('Acquisition_time', end + 'f8'), # [Serial date] - ] + [ - ('Centering_N%d' % i, 'f8') for i in range(8) - ] + [ - ('Distortion_N%02d' % i, 'f8') for i in range(14) + ("ID", (bytes, 6)), + ("MAGIC", end + "u2"), + ("Data_offset_1", end + "u4"), # Offset VBF + ("Data_offset_2", end + "u4"), # Offset DPs + ("UNKNOWN1", end + "u4"), # Flags for ASTAR software? + ("DP_SZ", end + "u2"), # Pixel dim DPs + ("DP_rotation", end + "u2"), # [degrees ( * 100 ?)] + ("NX", end + "u2"), # Scan dim 1 + ("NY", end + "u2"), # Scan dim 2 + ("Scan_rotation", end + "u2"), # [100 * degrees] + ("SX", end + "f8"), # Pixel size [nm] + ("SY", end + "f8"), # Pixel size [nm] + ("Beam_energy", end + "u4"), # [V] + ("SDP", end + "u2"), # Pixel size [100 * ppcm] + ("Camera_length", end + "u4"), # [10 * mm] + ("Acquisition_time", end + "f8"), # [Serial date] ] + + [("Centering_N%d" % i, "f8") for i in range(8)] + + [("Distortion_N%02d" % i, "f8") for i in range(14)] + ) return dtype_list -def get_default_header(endianess='<'): +def get_default_header(endianess="<"): """Returns a header pre-populated with default values. """ dt = np.dtype(get_header_dtype_list()) header = np.zeros((1,), dtype=dt) - header['ID'][0] = 'IMGBLO'.encode() - header['MAGIC'][0] = magics[0] - header['Data_offset_1'][0] = 0x1000 # Always this value observed - header['UNKNOWN1'][0] = 131141 # Very typical value (always?) - header['Acquisition_time'][0] = datetime_to_serial_date( - datetime.datetime.fromtimestamp(86400, dateutil.tz.tzutc())) + header["ID"][0] = "IMGBLO".encode() + header["MAGIC"][0] = magics[0] + header["Data_offset_1"][0] = 0x1000 # Always this value observed + header["UNKNOWN1"][0] = 131141 # Very typical value (always?) + header["Acquisition_time"][0] = datetime_to_serial_date( + datetime.datetime.fromtimestamp(86400, dateutil.tz.tzutc()) + ) return header -def get_header_from_signal(signal, endianess='<'): +def get_header_from_signal(signal, endianess="<"): header = get_default_header(endianess) - if 'blockfile_header' in signal.original_metadata: - header = dict2sarray(signal.original_metadata['blockfile_header'], - sarray=header) - note = signal.original_metadata['blockfile_header']['Note'] + if "blockfile_header" in signal.original_metadata: + header = dict2sarray( + signal.original_metadata["blockfile_header"], sarray=header + ) + note = signal.original_metadata["blockfile_header"]["Note"] else: - note = '' + note = "" if signal.axes_manager.navigation_dimension == 2: NX, NY = signal.axes_manager.navigation_shape SX = signal.axes_manager.navigation_axes[0].scale @@ -219,21 +222,23 @@ def get_header_from_signal(signal, endianess='<'): DP_SZ = signal.axes_manager.signal_shape if DP_SZ[0] != DP_SZ[1]: - raise ValueError('Blockfiles require signal shape to be square!') + raise ValueError("Blockfiles require signal shape to be square!") DP_SZ = DP_SZ[0] - SDP = 100. / signal.axes_manager.signal_axes[0].scale + SDP = 100.0 / signal.axes_manager.signal_axes[0].scale - offset2 = NX * NY + header['Data_offset_1'] + offset2 = NX * NY + header["Data_offset_1"] # Based on inspected files, the DPs are stored at 16-bit boundary... # Normally, you'd expect word alignment (32-bits) ¯\_(°_o)_/¯ offset2 += offset2 % 16 header_sofar = { - 'NX': NX, 'NY': NY, - 'DP_SZ': DP_SZ, - 'SX': SX, 'SY': SY, - 'SDP': SDP, - 'Data_offset_2': offset2, + "NX": NX, + "NY": NY, + "DP_SZ": DP_SZ, + "SX": SX, + "SY": SY, + "SDP": SDP, + "Data_offset_2": offset2, } header = dict2sarray(header_sofar, sarray=header) @@ -242,7 +247,7 @@ def get_header_from_signal(signal, endianess='<'): def get_header(data_shape, scan_scale, diff_scale, endianess="<", **kwargs): header = get_default_header(endianess) - note = '' + note = "" if len(data_shape) == 4: NY, NX = data_shape[:2][::-1] # first dimension seems to by y in np SX = scan_scale @@ -259,21 +264,23 @@ def get_header(data_shape, scan_scale, diff_scale, endianess="<", **kwargs): DP_SZ = data_shape[-2:] if DP_SZ[0] != DP_SZ[1]: - raise ValueError('Blockfiles require signal shape to be square!') + raise ValueError("Blockfiles require signal shape to be square!") DP_SZ = DP_SZ[0] - SDP = 100. / diff_scale + SDP = 100.0 / diff_scale - offset2 = NX * NY + header['Data_offset_1'] + offset2 = NX * NY + header["Data_offset_1"] # Based on inspected files, the DPs are stored at 16-bit boundary... # Normally, you'd expect word alignment (32-bits) ¯\_(°_o)_/¯ offset2 += offset2 % 16 header_sofar = { - 'NX': NX, 'NY': NY, - 'DP_SZ': DP_SZ, - 'SX': SX, 'SY': SY, - 'SDP': SDP, - 'Data_offset_2': offset2, + "NX": NX, + "NY": NY, + "DP_SZ": DP_SZ, + "SX": SX, + "SY": SY, + "SDP": SDP, + "Data_offset_2": offset2, } header_sofar.update(kwargs) @@ -282,164 +289,177 @@ def get_header(data_shape, scan_scale, diff_scale, endianess="<", **kwargs): return header, note -def file_reader(filename, endianess='<', mmap_mode=None, - lazy=False, **kwds): +def file_reader(filename, endianess="<", mmap_mode=None, lazy=False, **kwds): _logger.debug("Reading blockfile: %s" % filename) metadata = {} if mmap_mode is None: - mmap_mode = 'r' if lazy else 'c' + mmap_mode = "r" if lazy else "c" # Makes sure we open in right mode: - if '+' in mmap_mode or ('write' in mmap_mode and - 'copyonwrite' != mmap_mode): + if "+" in mmap_mode or ("write" in mmap_mode and "copyonwrite" != mmap_mode): if lazy: raise ValueError("Lazy loading does not support in-place writing") - f = open(filename, 'r+b') + f = open(filename, "r+b") else: - f = open(filename, 'rb') + f = open(filename, "rb") _logger.debug("File opened") # Get header header = np.fromfile(f, dtype=get_header_dtype_list(endianess), count=1) - if header['MAGIC'][0] not in magics: - warnings.warn("Blockfile has unrecognized header signature. " - "Will attempt to read, but correcteness not guaranteed!") + if header["MAGIC"][0] not in magics: + warnings.warn( + "Blockfile has unrecognized header signature. " + "Will attempt to read, but correcteness not guaranteed!" + ) header = sarray2dict(header) - note = f.read(header['Data_offset_1'] - f.tell()) + note = f.read(header["Data_offset_1"] - f.tell()) # It seems it uses "\x00" for padding, so we remove it try: - header['Note'] = note.decode("latin1").strip("\x00") + header["Note"] = note.decode("latin1").strip("\x00") except Exception: # Not sure about the encoding so, if it fails, we carry on _logger.warn( "Reading the Note metadata of this file failed. " "You can help improving " "HyperSpy by reporting the issue in " - "https://github.com/hyperspy/hyperspy") + "https://github.com/hyperspy/hyperspy" + ) _logger.debug("File header: " + str(header)) - NX, NY = int(header['NX']), int(header['NY']) - DP_SZ = int(header['DP_SZ']) - if header['SDP']: - SDP = 100. / header['SDP'] + NX, NY = int(header["NX"]), int(header["NY"]) + DP_SZ = int(header["DP_SZ"]) + if header["SDP"]: + SDP = 100.0 / header["SDP"] else: SDP = -1 - original_metadata = {'blockfile_header': header} + original_metadata = {"blockfile_header": header} # Get data: # A Virtual BF/DF is stored first - offset1 = header['Data_offset_1'] + offset1 = header["Data_offset_1"] f.seek(offset1) - data_pre = np.fromfile(f, count=NX*NY, dtype=endianess+'u1' - ).squeeze().reshape((NY, NX), order='C') + data_pre = ( + np.fromfile(f, count=NX * NY, dtype=endianess + "u1") + .squeeze() + .reshape((NY, NX), order="C") + ) # Then comes actual blockfile - offset2 = header['Data_offset_2'] + offset2 = header["Data_offset_2"] if not lazy: f.seek(offset2) - data = np.fromfile(f, dtype=endianess + 'u1') + data = np.fromfile(f, dtype=endianess + "u1") else: - data = np.memmap(f, mode=mmap_mode, offset=offset2, - dtype=endianess + 'u1') + data = np.memmap(f, mode=mmap_mode, offset=offset2, dtype=endianess + "u1") try: data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) except ValueError: warnings.warn( - 'Blockfile header dimensions larger than file size! ' - 'Will attempt to load by zero padding incomplete frames.') + "Blockfile header dimensions larger than file size! " + "Will attempt to load by zero padding incomplete frames." + ) # Data is stored DP by DP: pw = [(0, NX * NY * (DP_SZ * DP_SZ + 6) - data.size)] - data = np.pad(data, pw, mode='constant') + data = np.pad(data, pw, mode="constant") data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) # Every frame is preceeded by a 6 byte sequence (AA 55, and then a 4 byte # integer specifying frame number) data = data[:, :, 6:] - data = data.reshape((NY, NX, DP_SZ, DP_SZ), order='C').squeeze() - - units = ['nm', 'nm', 'cm', 'cm'] - names = ['y', 'x', 'dy', 'dx'] - scales = [header['SY'], header['SX'], SDP, SDP] - date, time, time_zone = serial_date_to_ISO_format( - header['Acquisition_time']) - metadata = {'General': {'original_filename': os.path.split(filename)[1], - 'date': date, - 'time': time, - 'time_zone': time_zone, - 'notes': header['Note']}, - "Signal": {'signal_type': "diffraction", - 'record_by': 'image', }, - } + data = data.reshape((NY, NX, DP_SZ, DP_SZ), order="C").squeeze() + + units = ["nm", "nm", "cm", "cm"] + names = ["y", "x", "dy", "dx"] + scales = [header["SY"], header["SX"], SDP, SDP] + date, time, time_zone = serial_date_to_ISO_format(header["Acquisition_time"]) + metadata = { + "General": { + "original_filename": os.path.split(filename)[1], + "date": date, + "time": time, + "time_zone": time_zone, + "notes": header["Note"], + }, + "Signal": {"signal_type": "diffraction", "record_by": "image",}, + } # Create the axis objects for each axis dim = data.ndim axes = [ { - 'size': data.shape[i], - 'index_in_array': i, - 'name': names[i], - 'scale': scales[i], - 'offset': 0.0, - 'units': units[i], } - for i in range(dim)] - - dictionary = {'data': data, - 'vbf': data_pre, - 'axes': axes, - 'metadata': metadata, - 'original_metadata': original_metadata, - 'mapping': mapping, } + "size": data.shape[i], + "index_in_array": i, + "name": names[i], + "scale": scales[i], + "offset": 0.0, + "units": units[i], + } + for i in range(dim) + ] + + dictionary = { + "data": data, + "vbf": data_pre, + "axes": axes, + "metadata": metadata, + "original_metadata": original_metadata, + "mapping": mapping, + } f.close() - return [dictionary, ] + return [ + dictionary, + ] def file_writer(filename, signal, **kwds): - endianess = kwds.pop('endianess', '<') + endianess = kwds.pop("endianess", "<") header, note = get_header_from_signal(signal, endianess=endianess) - with open(filename, 'wb') as f: + with open(filename, "wb") as f: # Write header header.tofile(f) # Write header note field: - if len(note) > int(header['Data_offset_1']) - f.tell(): - note = note[:int(header['Data_offset_1']) - f.tell() - len(note)] + if len(note) > int(header["Data_offset_1"]) - f.tell(): + note = note[: int(header["Data_offset_1"]) - f.tell() - len(note)] f.write(note.encode()) # Zero pad until next data block - zero_pad = int(header['Data_offset_1']) - f.tell() + zero_pad = int(header["Data_offset_1"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write virtual bright field - vbf = signal.mean( - signal.axes_manager.signal_axes[ - :2]).data.astype( - endianess + - 'u1') + vbf = signal.mean(signal.axes_manager.signal_axes[:2]).data.astype( + endianess + "u1" + ) vbf.tofile(f) # Zero pad until next data block - if f.tell() > int(header['Data_offset_2']): - raise ValueError("Signal navigation size does not match " - "data dimensions.") - zero_pad = int(header['Data_offset_2']) - f.tell() + if f.tell() > int(header["Data_offset_2"]): + raise ValueError( + "Signal navigation size does not match " "data dimensions." + ) + zero_pad = int(header["Data_offset_2"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write full data stack: # We need to pad each image with magic 'AA55', then a u32 serial - dp_head = np.zeros((1,), dtype=[('MAGIC', endianess + 'u2'), - ('ID', endianess + 'u4')]) - dp_head['MAGIC'] = 0x55AA + dp_head = np.zeros( + (1,), dtype=[("MAGIC", endianess + "u2"), ("ID", endianess + "u4")] + ) + dp_head["MAGIC"] = 0x55AA # Write by loop: for img in signal._iterate_signal(): dp_head.tofile(f) - img.astype(endianess + 'u1').tofile(f) - dp_head['ID'] += 1 + img.astype(endianess + "u1").tofile(f) + dp_head["ID"] += 1 class bloFileWriter(QThread): """Write a blo file from an HDF5 without loading all data in memory""" + increase_progress = pyqtSignal(int) finish = pyqtSignal() - def __init__(self, fh, path_blo, shape, indexes, - scan_scale=5, diff_scale=1.075): + def __init__( + self, fh, path_blo, shape, indexes, scan_scale=5, diff_scale=1.075, **options + ): QThread.__init__(self) self.fh = fh # open hdf5 file in read mode self.path_blo = path_blo @@ -453,6 +473,9 @@ def __init__(self, fh, path_blo, shape, indexes, self.vbf_im = imagefun.normalize_convert(self.vbf_im, dtype=np.uint8) logger.debug("Initialized bloFileWriter") + # added for 8-bit rescaling option + self.options = options + def run(self): self.convert_to_blo() self.finish.emit() @@ -461,12 +484,16 @@ def run(self): def convert_to_blo(self): endianess = "<" header, note = get_header( - self.shape, self.scan_scale, - self.diff_scale, endianess, - Camera_length=100.0*self.fh["ImageStream"].attrs['magtotal'], - Beam_energy=self.fh["ImageStream"].attrs['ht']*1000, - Distortion_N01=1.0, Distortion_N09=1.0, - Note="Reconstructed from TVIPS image stream") + self.shape, + self.scan_scale, + self.diff_scale, + endianess, + Camera_length=100.0 * self.fh["ImageStream"].attrs["magtotal"], + Beam_energy=self.fh["ImageStream"].attrs["ht"] * 1000, + Distortion_N01=1.0, + Distortion_N09=1.0, + Note="Reconstructed from TVIPS image stream", + ) logger.debug("Created header of blo file") with open(self.path_blo, "wb") as f: @@ -474,59 +501,63 @@ def convert_to_blo(self): header.tofile(f) logger.debug("Wrote header to file") # Write header note field: - if len(note) > int(header['Data_offset_1']) - f.tell(): - note = note[:int(header['Data_offset_1']) - - f.tell() - len(note)] + if len(note) > int(header["Data_offset_1"]) - f.tell(): + note = note[: int(header["Data_offset_1"]) - f.tell() - len(note)] f.write(note.encode()) # Zero pad until next data block - zero_pad = int(header['Data_offset_1']) - f.tell() + zero_pad = int(header["Data_offset_1"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write virtual bright field vbf = self.vbf_im.astype(endianess + "u1") vbf.tofile(f) # Zero pad until next data block - if f.tell() > int(header['Data_offset_2']): - raise ValueError("Signal navigation size does not match " - "data dimensions.") - zero_pad = int(header['Data_offset_2']) - f.tell() + if f.tell() > int(header["Data_offset_2"]): + raise ValueError( + "Signal navigation size does not match " "data dimensions." + ) + zero_pad = int(header["Data_offset_2"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write full data stack: # We need to pad each image with magic 'AA55', then a u32 serial - dp_head = np.zeros((1,), dtype=[('MAGIC', endianess + 'u2'), - ('ID', endianess + 'u4')]) - dp_head['MAGIC'] = 0x55AA + dp_head = np.zeros( + (1,), dtype=[("MAGIC", endianess + "u2"), ("ID", endianess + "u4")] + ) + dp_head["MAGIC"] = 0x55AA # Write by loop: logger.debug("Wrote header part of blo file") for j, indx in enumerate(self.indexes): dp_head.tofile(f) c = f"{indx}".zfill(6) img = self.fh["ImageStream"][f"Frame_{c}"][:] - img = imagefun.normalize_convert(img, dtype=np.uint8) - img.astype(endianess + 'u1').tofile(f) - dp_head['ID'] += 1 + + if "rescale" in self.options and self.options["rescale"]: + img = imagefun.normalize_convert(img, dtype=np.uint8) + else: + img = util.img_as_ubyte(img) + img.astype(endianess + "u1").tofile(f) + dp_head["ID"] += 1 self.update_gui_progress(j) logger.debug(f"Wrote frame Frame_{c} to blo file") def update_gui_progress(self, j): """If using the GUI update features with progress""" - value = int((j+1)/len(self.indexes)*100) + value = int((j + 1) / len(self.indexes) * 100) self.increase_progress.emit(value) def file_writer_array(filename, array, scan_scale, diff_scale, **kwds): endianess = "<" - header, note = get_header(array.shape, scan_scale, diff_scale, endianess, - **kwds) + header, note = get_header(array.shape, scan_scale, diff_scale, endianess, **kwds) - with open(filename, 'wb') as f: + with open(filename, "wb") as f: # Write header header.tofile(f) # Write header note field: - if len(note) > int(header['Data_offset_1']) - f.tell(): - note = note[:int(header['Data_offset_1']) - f.tell() - len(note)] + if len(note) > int(header["Data_offset_1"]) - f.tell(): + note = note[: int(header["Data_offset_1"]) - f.tell() - len(note)] f.write(note.encode()) # Zero pad until next data block - zero_pad = int(header['Data_offset_1']) - f.tell() + zero_pad = int(header["Data_offset_1"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write virtual bright field vbf = None @@ -535,8 +566,7 @@ def file_writer_array(filename, array, scan_scale, diff_scale, **kwds): amean = (2, 3) # do proper vbf xx, yy = np.meshgrid(range(array.shape[2]), range(array.shape[3])) - mask = np.hypot(xx - 0.5 * array.shape[2], - yy - 0.5 * array.shape[3]) < 5 + mask = np.hypot(xx - 0.5 * array.shape[2], yy - 0.5 * array.shape[3]) < 5 # TODO: make radius and offset configurable vbffloat = np.zeros((array.shape[:2])) @@ -550,23 +580,25 @@ def file_writer_array(filename, array, scan_scale, diff_scale, **kwds): elif len(array.shape) == 3: amean = (1, 2) - vbf = array.mean(axis=amean).astype(endianess + 'u1') + vbf = array.mean(axis=amean).astype(endianess + "u1") vbf.tofile(f) # Zero pad until next data block - if f.tell() > int(header['Data_offset_2']): - raise ValueError("Signal navigation size does not match " - "data dimensions.") - zero_pad = int(header['Data_offset_2']) - f.tell() + if f.tell() > int(header["Data_offset_2"]): + raise ValueError( + "Signal navigation size does not match " "data dimensions." + ) + zero_pad = int(header["Data_offset_2"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write full data stack: # We need to pad each image with magic 'AA55', then a u32 serial - dp_head = np.zeros((1,), dtype=[('MAGIC', endianess + 'u2'), - ('ID', endianess + 'u4')]) - dp_head['MAGIC'] = 0x55AA + dp_head = np.zeros( + (1,), dtype=[("MAGIC", endianess + "u2"), ("ID", endianess + "u4")] + ) + dp_head["MAGIC"] = 0x55AA # Write by loop: - for img in array.reshape(array.shape[0]*array.shape[1], - *array.shape[2:]): + for img in array.reshape(array.shape[0] * array.shape[1], *array.shape[2:]): dp_head.tofile(f) - img.astype(endianess + 'u1').tofile(f) - dp_head['ID'] += 1 + img.astype(endianess + "u1").tofile(f) + dp_head["ID"] += 1 + diff --git a/tvipsconverter/widget_2.ui b/tvipsconverter/widget_2.ui index 363c976..d9d1da2 100644 --- a/tvipsconverter/widget_2.ui +++ b/tvipsconverter/widget_2.ui @@ -7,7 +7,7 @@ 0 0 1171 - 898 + 990 @@ -32,38 +32,38 @@ Progress - - + + 0 0 - - - 974 - 0 - + + Status: Idle - - 0 + + true - - + + 0 0 - - Status: Idle + + + 974 + 0 + - - true + + 0 @@ -82,7 +82,7 @@ <html><head/><body><p><br/></p></body></html> - 0 + 1 @@ -157,7 +157,7 @@ 0 0 1115 - 705 + 707 @@ -1055,6 +1055,9 @@ + + Qt::ScrollBarAlwaysOn + true @@ -1064,7 +1067,7 @@ 0 0 1069 - 610 + 628 @@ -1154,10 +1157,10 @@ Scaling - - + + - Scale of DP + nm^-1 / pixel @@ -1168,30 +1171,6 @@ - - - - nm / pixel - - - - - - - nm^-1 / pixel - - - - - - - 4 - - - 1.000000000000000 - - - @@ -1218,6 +1197,30 @@ + + + + nm / pixel + + + + + + + Scale of DP + + + + + + + 4 + + + 1.000000000000000 + + + @@ -1236,40 +1239,24 @@ Scan modification - - - - - 0 - 0 - + + + + <html><head/><body><p>In EM Konos, the scanning follows a winding scan pattern (check on), whereas in EM Scan it uses normal flyback scan (check off)</p></body></html> - pixels + Winding scan? - - - - - + true - - - 0 - 0 - - - - <html><head/><body><p>Number of scan points in the Y direction</p></body></html> - - - Scan dim. Y + + false - - + + true @@ -1286,13 +1273,32 @@ - -100000000 + 1 + + + 10000 + + + 1 + + + 100 + + + + + + + true + + + QAbstractSpinBox::NoButtons 100000000 - 5 + 0 @@ -1303,22 +1309,30 @@ - - - - true + + + + <html><head/><body><p>Frame in the stream where the scan began</p></body></html> - - - 0 - 0 - + + Scan start frame - - <html><head/><body><p>Number of scan points in the x direction</p></body></html> + + + + + + Use custom scan dimensions + + + false + + + + - Scan dim. X + Use custom scan frames @@ -1353,50 +1367,22 @@ - - - - true - - - QAbstractSpinBox::NoButtons - - - 100000000 - - - 0 - - - - - + + true - + 0 0 - - - 93 - 0 - - - - 1 - - - 10000 - - - 1 + + <html><head/><body><p>Number of scan points in the Y direction</p></body></html> - - 100 + + Scan dim. Y @@ -1413,23 +1399,6 @@ - - - - <html><head/><body><p>Frame in the stream where the scan began</p></body></html> - - - Scan start frame - - - - - - - Use custom scan frames - - - @@ -1443,19 +1412,50 @@ - - - - <html><head/><body><p>In EM Konos, the scanning follows a winding scan pattern (check on), whereas in EM Scan it uses normal flyback scan (check off)</p></body></html> + + + + true - - Winding scan? + + + 0 + 0 + - + + + 93 + 0 + + + + -100000000 + + + 100000000 + + + 5 + + + + + + true - - false + + + 0 + 0 + + + + <html><head/><body><p>Number of scan points in the x direction</p></body></html> + + + Scan dim. X @@ -1469,13 +1469,16 @@ - - - - Use custom scan dimensions + + + + + 0 + 0 + - - false + + pixels @@ -1488,6 +1491,20 @@ Crop + + + + Show Cropped Region + + + + + + + xmin + + + @@ -1495,26 +1512,22 @@ - - + + 93 0 + + 0 + 1000 - 100 - - - - - - - xmin + 0 @@ -1534,20 +1547,6 @@ - - - - Show Cropped Region - - - - - - - ymax - - - @@ -1568,8 +1567,15 @@ - - + + + + ymax + + + + + 93 @@ -1579,24 +1585,50 @@ 1000 + + 100 + - - + + 93 0 - - 0 - 1000 - - 0 + + + + + + + + + 8-bit conversion + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + Rescale intensities across 8-bit range @@ -1672,19 +1704,6 @@ Display settings - - - - 255 - - - 10 - - - Qt::Horizontal - - - @@ -1695,6 +1714,13 @@ + + + + Maximum + + + @@ -1702,10 +1728,16 @@ - - - - Maximum + + + + 255 + + + 10 + + + Qt::Horizontal diff --git a/tvipsconverter/widgets.py b/tvipsconverter/widgets.py index ec49adc..4cbb29d 100644 --- a/tvipsconverter/widgets.py +++ b/tvipsconverter/widgets.py @@ -615,7 +615,14 @@ def write_to_file(self): self.update_line(self.statusedit, f"Writing {filetyp} file...") if filetyp == ".blo": self.get_thread = blf.bloFileWriter( - f, path_blo, shape, indexes, scan_scale, dp_scale + f, + path_blo, + shape, + indexes, + scan_scale, + dp_scale, + # if rescale button is checked do rescale, otherwise no rescaling selected + rescale=self.checkBox_rescale.isChecked(), ) elif filetyp == ".hspy": self.get_thread = hspf.hspyFileWriter( From 5927ce52283d0940d261d885a307ab8dadabab8b Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Wed, 4 Nov 2020 09:31:33 +0100 Subject: [PATCH 04/21] add .DS_STORE --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bab0bcd..acd31f7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ Dummy garbage .ipynb* .venvtest +.DS_STORE # Created by https://www.gitignore.io/api/python # Edit at https://www.gitignore.io/?templates=python From df1867cc7a64e16759caea4e4b8ed1a1e235ad25 Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Wed, 18 Nov 2020 09:34:43 +0100 Subject: [PATCH 05/21] Python 3.9 Big Sur bug fix App would run but not show window on MacOS Big Sur without this (temporary?) fix --- .DS_Store | Bin 0 -> 6148 bytes tvipsconverter/widgets.py | 4 ++++ 2 files changed, 4 insertions(+) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..025144fdfefec6ad5e5ba8e8df7f7a06b52f3f59 GIT binary patch literal 6148 zcmeHK-HOvd82zSO-Nd5yVxfAM8^Mbz?XpEgglxAJ3Myh2y-?Xrn%E6wGo?wl?AA(e zdjg-Cn}`L9*g)8jJ!)fxk@wv3A$#A@yj7y0ox?b(rt<r)58Q7Dto4>c$RFb3aa_ zktD}q1ecf3<22-xHXo;9CgXUj!?tb9-m6}pPB)v4y0dksHLE+*W@EEecUoI_XEWQX z-DuwL?H?YWOyA7jzQ+h*3_H2Bs~Ug62Ncd!?8qCX2~P*`6ESLMsf!tnXn@}>@$F!4 zc|q@wh_Lw8p=#`$70@$nRn9rC31<{A3jF&Di2cDq66k82E0kLYGI<36meDK?WpS4f z97m(8ajpSm!4w_sj`njk&J`*;F@5=9`pHb+P?&r=;ycovn5)p_MggP1 zvI1+WS{3Ji_t*FTWs<2G1&jj!l>)5X@j7h`NuRANgA-@1gS3w%O!& Date: Tue, 3 Nov 2020 11:45:05 +0100 Subject: [PATCH 06/21] add create DP ave and max images when convert_HDF5 --- tvipsconverter/__main__.py | 3 + tvipsconverter/utils/imagefun.py | 44 +-- tvipsconverter/utils/recorder.py | 456 ++++++++++++++++++++----------- tvipsconverter/widget_2.ui | 29 ++ tvipsconverter/widgets.py | 223 ++++++++------- 5 files changed, 466 insertions(+), 289 deletions(-) create mode 100644 tvipsconverter/__main__.py diff --git a/tvipsconverter/__main__.py b/tvipsconverter/__main__.py new file mode 100644 index 0000000..5bd1eb6 --- /dev/null +++ b/tvipsconverter/__main__.py @@ -0,0 +1,3 @@ +from .widgets import main + +main() diff --git a/tvipsconverter/utils/imagefun.py b/tvipsconverter/utils/imagefun.py index 1a0105b..6443ad1 100644 --- a/tvipsconverter/utils/imagefun.py +++ b/tvipsconverter/utils/imagefun.py @@ -13,10 +13,16 @@ def _get_dtype_min_max(dtype): if dtype == np.float or dtype == np.float32 or dtype == np.float64: max = 1 # np.finfo(dtype).max min = 0 # np.finfo(dtype).min - elif (dtype == np.int8 or dtype == np.uint8 or - dtype == np.int16 or dtype == np.uint16 or - dtype == np.int32 or dtype == np.uint32 or - dtype == np.int64 or dtype == np.uint64): + elif ( + dtype == np.int8 + or dtype == np.uint8 + or dtype == np.int16 + or dtype == np.uint16 + or dtype == np.int32 + or dtype == np.uint32 + or dtype == np.int64 + or dtype == np.uint64 + ): max = np.iinfo(dtype).max min = np.iinfo(dtype).min else: @@ -119,8 +125,8 @@ def linscale(arr, min=None, max=None, nmin=0, nmax=1, dtype=np.float): else: workarr[workarr > max] = max - a = (nmax-nmin)/(max-min) - result = (workarr-min)*a+nmin + a = (nmax - nmin) / (max - min) + result = (workarr - min) * a + nmin return result.astype(dtype) @@ -142,10 +148,10 @@ def matlab_style_gauss2D(shape=(3, 3), sigma=0.5): 2D gaussian mask - should give the same result as MATLAB's fspecial('gaussian',[shape],[sigma]) """ - m, n = [(ss-1.)/2. for ss in shape] - y, x = np.ogrid[-m:m+1, -n:n+1] - h = np.exp(-(x*x + y*y) / (2.*sigma*sigma)) - h[h < np.finfo(h.dtype).eps*h.max()] = 0 + m, n = [(ss - 1.0) / 2.0 for ss in shape] + y, x = np.ogrid[-m : m + 1, -n : n + 1] + h = np.exp(-(x * x + y * y) / (2.0 * sigma * sigma)) + h[h < np.finfo(h.dtype).eps * h.max()] = 0 sumh = h.sum() if sumh != 0: h /= sumh @@ -168,14 +174,14 @@ def findoutliers(raw, percent=0.07): Uses returns min and max values of the array with the upper and lower percent pixel values suppressed. """ - cnts, edges = np.histogram(raw, bins=2**16) - stats = np.zeros((2, 2**16), dtype=np.int) + cnts, edges = np.histogram(raw, bins=2 ** 16) + stats = np.zeros((2, 2 ** 16), dtype=np.int) stats[0] = np.cumsum(cnts) # low stats[1] = np.cumsum(cnts[::-1]) # high thresh = stats > percent * raw.shape[0] * raw.shape[1] min = (np.where(thresh[0]))[0][0] - max = 2**16 - (np.where(thresh[1]))[0][0] - return edges[min], edges[max+1] + max = 2 ** 16 - (np.where(thresh[1]))[0][0] + return edges[min], edges[max + 1] def suppressoutliers(raw, percent=0.07): @@ -191,9 +197,11 @@ def bin2(a, factor): imag = Image.fromarray(a) # binned = Image.resize(imag, (a.shape[0]//factor, a.shape[1]//factor), # resample=Image.BILINEAR) - binned = np.array(imag.resize((a.shape[0]//factor, a.shape[1]//factor), - resample=Image.NEAREST)) - print(binned.dtype) + binned = np.array( + imag.resize( + (a.shape[0] // factor, a.shape[1] // factor), resample=Image.NEAREST + ) + ) return binned @@ -204,5 +212,5 @@ def getElectronWavelength(ht): charge = 1.6e-19 c = 3e8 wavelength = h / math.sqrt(2 * m * charge * ht) - relativistic_correction = 1 / math.sqrt(1 + ht * charge/(2 * m * c * c)) + relativistic_correction = 1 / math.sqrt(1 + ht * charge / (2 * m * c * c)) return wavelength * relativistic_correction diff --git a/tvipsconverter/utils/recorder.py b/tvipsconverter/utils/recorder.py index b9b5975..a989fb4 100644 --- a/tvipsconverter/utils/recorder.py +++ b/tvipsconverter/utils/recorder.py @@ -8,70 +8,73 @@ from PyQt5.QtCore import QThread, pyqtSignal import logging -from .imagefun import (normalize_convert, bin2, gausfilter, - medfilter) +from .imagefun import normalize_convert, bin2, gausfilter, medfilter # Initialize the Logger logger = logging.getLogger(__name__) TVIPS_RECORDER_GENERAL_HEADER = [ - ('size', 'u4'), # unused - likely the size of generalheader in bytes - ('version', 'u4'), # 1 or 2 - ('dimx', 'u4'), # dp image size width - ('dimy', 'u4'), # dp image size height - ('bitsperpixel', 'u4'), # 8 or 16 - ('offsetx', 'u4'), # generally 0 - ('offsety', 'u4'), - ('binx', 'u4'), # camera binning - ('biny', 'u4'), - ('pixelsize', 'u4'), # nm, physical pixel size - ('ht', 'u4'), # high tension, voltage - ('magtotal', 'u4'), # magnification/camera length? - ('frameheaderbytes', 'u4'), # number of bytes per frame header - ('dummy', 'S204'), # just writes out TVIPS TVIPS TVIPS - ] + ("size", "u4"), # unused - likely the size of generalheader in bytes + ("version", "u4"), # 1 or 2 + ("dimx", "u4"), # dp image size width + ("dimy", "u4"), # dp image size height + ("bitsperpixel", "u4"), # 8 or 16 + ("offsetx", "u4"), # generally 0 + ("offsety", "u4"), + ("binx", "u4"), # camera binning + ("biny", "u4"), + ("pixelsize", "u4"), # nm, physical pixel size + ("ht", "u4"), # high tension, voltage + ("magtotal", "u4"), # magnification/camera length? + ("frameheaderbytes", "u4"), # number of bytes per frame header + ("dummy", "S204"), # just writes out TVIPS TVIPS TVIPS +] TVIPS_RECORDER_FRAME_HEADER = [ - ('num', 'u4'), # seems to cycle also - ('timestamp', 'u4'), # seconds since 1.1.1970 - ('ms', 'u4'), # additional milliseconds to the timestamp - ('LUTidx', 'u4'), # always the same value - ('fcurrent', 'f4'), # 0 for all frames - ('mag', 'u4'), # same for all frames - ('mode', 'u4'), # 1 -> image 2 -> diff - ('stagex', 'f4'), - ('stagey', 'f4'), - ('stagez', 'f4'), - ('stagea', 'f4'), - ('stageb', 'f4'), - ('rotidx', 'u4'), - ('temperature', 'f4'), # cycles between 0.0 and 9.0 with step 1.0 - ('objective', 'f4'), # kind of randomly between 0.0 and 1.0 + ("num", "u4"), # seems to cycle also + ("timestamp", "u4"), # seconds since 1.1.1970 + ("ms", "u4"), # additional milliseconds to the timestamp + ("LUTidx", "u4"), # always the same value + ("fcurrent", "f4"), # 0 for all frames + ("mag", "u4"), # same for all frames + ("mode", "u4"), # 1 -> image 2 -> diff + ("stagex", "f4"), + ("stagey", "f4"), + ("stagez", "f4"), + ("stagea", "f4"), + ("stageb", "f4"), + ("rotidx", "u4"), + ("temperature", "f4"), # cycles between 0.0 and 9.0 with step 1.0 + ("objective", "f4"), # kind of randomly between 0.0 and 1.0 # for header version 2, some more data might be present - ] +] FILTER_DEFAULTS = { - "useint": False, "whichint": 65536, - "usebin": False, "whichbin": 1, "usegaus": False, - "gausks": 8, "gaussig": 4, "usemed": False, - "medks": 4, "usels": False, "lsmin": 10, - "lsmax": 1000, "usecoffset": False - } - - -VBF_DEFAULTS = { - "calcvbf": True, "vbfrad": 10, "vbfxoffset": 0, - "vbfyoffset": 0 - } - - -def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, - binning=1): + "useint": False, + "whichint": 65536, + "usebin": False, + "whichbin": 1, + "usegaus": False, + "gausks": 8, + "gaussig": 4, + "usemed": False, + "medks": 4, + "usels": False, + "lsmin": 10, + "lsmax": 1000, + "usecoffset": False, +} + + +VBF_DEFAULTS = {"calcvbf": True, "vbfrad": 10, "vbfxoffset": 0, "vbfyoffset": 0} + + +def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, binning=1): """Do some kind of intensity correction, unsure reason""" pixperchannel = int(128 / binning) # binning has to be an integer - if (128.0/binning != pixperchannel): + if 128.0 / binning != pixperchannel: logger.error("Can't figure out column offset dimension") return image numcol = int(image.shape[0] / 128 * binning) @@ -81,8 +84,7 @@ def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, for j in range(numcol): channel = imtemp[:, j, :] # pdb.set_trace() - mask = np.bitwise_and(channel < thresholdmax, - channel >= thresholdmin) + mask = np.bitwise_and(channel < thresholdmax, channel >= thresholdmin) value = np.mean(channel[mask]) offsets.append(value) # apply offset correction to images @@ -93,10 +95,22 @@ def _correct_column_offsets(image, thresholdmin=0, thresholdmax=30, return subtracted.reshape(image.shape) -def filter_image(imag, useint, whichint, usebin, - whichbin, usegaus, gausks, - gaussig, usemed, medks, - usels, lsmin, lsmax, usecoffset): +def filter_image( + imag, + useint, + whichint, + usebin, + whichbin, + usegaus, + gausks, + gaussig, + usemed, + medks, + usels, + lsmin, + lsmax, + usecoffset, +): """ Filter an image and return the filtered image """ @@ -132,8 +146,15 @@ class Recorder(QThread): increase_progress = pyqtSignal(int) finish = pyqtSignal() - def __init__(self, path, improc=None, vbfsettings=None, - outputpath=None, imrange=(None, None)): + def __init__( + self, + path, + improc=None, + vbfsettings=None, + outputpath=None, + imrange=(None, None), + **options, + ): QThread.__init__(self) logger.debug("Initializing recorder object") # filename @@ -186,6 +207,10 @@ def __init__(self, path, improc=None, vbfsettings=None, if self.outputpath is not None: self.outputpath = str(Path(self.outputpath)) + # other options, added 2.11.2020 + self.options = options + # print(options, self.options) + def run(self): self.convert_HDF5() self.finish.emit() @@ -211,13 +236,26 @@ def convert_HDF5(self): # This is important! also initializes the headers! firstframe = self.read_frame(0) pff = filter_image(firstframe, **self.improc) + + # initialise maximum_image if checkBox checked + # print(f"average image: {'calcmax' in self.options} {self.options['calcmax']}") + if "calcmax" in self.options and self.options["calcmax"]: + # print("creating maxiumum image") + self.maximum_image = np.zeros_like(pff) + + # print(f"average image: {'calcave' in self.options} {self.options['calcave']}") + # initialise average_image if checkBox checked + if "calcave" in self.options and self.options["calcave"]: + # print("creating average_image") + # as datatype is typcially 8- or 16-bit then 32-bit image should be fine + self.average_image = np.zeros_like(pff, dtype=np.float32) + self.scangroup = self.stream.create_group("Scan") # do we need a virtual bright field calculated? if self.vbfproc["calcvbf"]: # make a VBF mask. For this we need a frame # ZOB center offset - zoboffset = [self.vbfproc["vbfxoffset"], - self.vbfproc["vbfyoffset"]] + zoboffset = [self.vbfproc["vbfxoffset"], self.vbfproc["vbfyoffset"]] radius = self.vbfproc["vbfrad"] # generate mask self.mask = self._virtual_bf_mask(pff, zoboffset, radius) @@ -245,11 +283,13 @@ def valid_first_tvips_file(filename): if match is not None: num, ext = match.groups() if ext != "tvips": - raise ValueError(f"Invalid tvips file: extension {ext}, must " - f"be tvips") + raise ValueError( + f"Invalid tvips file: extension {ext}, must " f"be tvips" + ) if int(num) != 0: - raise ValueError("Can only read video sequences starting with " - "part 000") + raise ValueError( + "Can only read video sequences starting with " "part 000" + ) return True else: raise ValueError("Could not recognize as a valid tvips file") @@ -261,10 +301,8 @@ def read_frame(self, frame=0): fh = FileHandle(file=f) fh.seek(bite_start - self.ranges[toopen][0]) frame = np.fromfile( - fh, - count=self.general.dimx*self.general.dimy, - dtype=self.dtype - ) + fh, count=self.general.dimx * self.general.dimy, dtype=self.dtype + ) frame.shape = (self.general.dimx, self.general.dimy) return frame @@ -288,10 +326,14 @@ def _get_byte_filename(self, b): def _get_frame_byte_position(self, frame): """Get the byte where a frame starts""" - frame_byte_size = (self.general.dimx * self.general.dimy * - self.general.bitsperpixel//8) - bite_start = (self.generalheadersize + self.general.frameheaderbytes + - (frame_byte_size + self.general.frameheaderbytes)*frame) + frame_byte_size = ( + self.general.dimx * self.general.dimy * self.general.bitsperpixel // 8 + ) + bite_start = ( + self.generalheadersize + + self.general.frameheaderbytes + + (frame_byte_size + self.general.frameheaderbytes) * frame + ) return bite_start def _get_frame_byte_position_in_file(self, frame): @@ -303,21 +345,24 @@ def _get_frame_byte_position_in_file(self, frame): def _get_byte_frame_position(self, byte_start): """Get the frame index closest corresponding to a byte""" - frame_byte_size = (self.general.dimx * self.general.dimy * - self.general.bitsperpixel//8) - frame = ((byte_start - self.generalheadersize - - self.general.frameheaderbytes) // - (frame_byte_size + self.general.frameheaderbytes)) + frame_byte_size = ( + self.general.dimx * self.general.dimy * self.general.bitsperpixel // 8 + ) + frame = ( + byte_start - self.generalheadersize - self.general.frameheaderbytes + ) // (frame_byte_size + self.general.frameheaderbytes) return frame def _get_files_size_dictionary(self): """ Get a dictionary of files (keys) and total size (values) """ + def get_filesize(fn): with open(fn, "rb") as f: fh = FileHandle(file=f) return fn, fh.size + sizes = self._scan_over_all_files(get_filesize) return dict(sizes) @@ -329,7 +374,7 @@ def _get_files_ranges(self): ranges = {} starts = 0 for i in sizes: - ends = starts+sizes[i] + ends = starts + sizes[i] ranges[i] = (starts, ends) starts = ends return ranges @@ -354,8 +399,7 @@ def _frames_exceeded(self): """Have we read the number of frames?""" if self.finalim is not None: if self.current_frame >= self.finalim: - logger.debug(f"We are at frame {self.current_frame}. " - f"Quitting.") + logger.debug(f"We are at frame {self.current_frame}. " f"Quitting.") return True return False @@ -365,11 +409,10 @@ def _scan_over_all_files(self, func, *args, **kwargs): results = [] part = int(self.filename[-9:-6]) if part != 0: - raise ValueError("Can only read video sequences starting with " - "part 000") + raise ValueError("Can only read video sequences starting with " "part 000") try: while True: - fn = self.filename[:-9]+"{:03d}.tvips".format(part) + fn = self.filename[:-9] + "{:03d}.tvips".format(part) if not os.path.exists(fn): logger.debug(f"There is no file {fn}; breaking loop") break @@ -404,13 +447,15 @@ def _readGeneral(self, fh): self.inc = self.general.frameheaderbytes self.frame_header = TVIPS_RECORDER_FRAME_HEADER else: - raise NotImplementedError(f"Version {self.general.version} not " - f"yet supported.") + raise NotImplementedError( + f"Version {self.general.version} not " f"yet supported." + ) self.dt = np.dtype(self.frame_header) # make sure the record consumes less bytes than reported in the main # header - assert self.inc >= self.dt.itemsize, ("The record consumes more bytes " - "than stated in the main header") + assert self.inc >= self.dt.itemsize, ( + "The record consumes more bytes " "than stated in the main header" + ) def _readIndividualFile(self, fn): """ @@ -429,7 +474,8 @@ def _readIndividualFile(self, fn): if self.startim > self.current_frame: self.current_frame += 1 current_byte = self._get_frame_byte_position_in_file( - self.current_frame) + self.current_frame + ) fh.seek(current_byte - self.general.frameheaderbytes) continue if self.finalim < self.current_frame: @@ -442,16 +488,16 @@ def _readIndividualFile(self, fn): def _readFrame(self, fh, record=None): # read frame header header = fh.read_record(self.frame_header) - logger.debug(f"{self.current_frame}: Starting frame read " - f"(pos: {fh.tell()}). rot: {header['rotidx']}") + logger.debug( + f"{self.current_frame}: Starting frame read " + f"(pos: {fh.tell()}). rot: {header['rotidx']}" + ) skip = self.inc - self.dt.itemsize fh.seek(skip, 1) # read frame frame = np.fromfile( - fh, - count=self.general.dimx*self.general.dimy, - dtype=self.dtype - ) + fh, count=self.general.dimx * self.general.dimy, dtype=self.dtype + ) frame.shape = (self.general.dimx, self.general.dimy) # do calculations on the frame frame = filter_image(frame, **self.improc) @@ -461,16 +507,35 @@ def _readFrame(self, fh, record=None): for i in self.frame_header: ds.attrs[i[0]] = header[i[0]] # store the rotation index for finding start and stop later - self.rotidxs.append(header['rotidx']) + self.rotidxs.append(header["rotidx"]) # immediately calculate and store the VBF intensity if required if self.vbfproc["calcvbf"]: vbf_int = frame[self.mask].mean() self.vbfs.append(vbf_int) + if "calcmax" in self.options and self.options["calcmax"]: + # maximum_image should already be initialised in self.convert_HDF5 + self.maximum_image = np.stack((self.maximum_image, frame), axis=0).max( + axis=0 + ) + + if "calcave" in self.options and self.options["calcave"]: + # average_image should already be initialised in self.convert_HDF5 + self.average_image = np.stack( + ( + self.average_image, + # frame is scaled as a function of 1/total_frames and then summed (average) + (1 / (self.finalim - self.startim)) + * frame.astype(self.average_image.dtype), + ), + axis=0, + ).sum(axis=0) + def _update_gui_progess(self): """If using the GUI update features with progress""" - value = int((self.current_frame - self.startim) / - (self.finalim - self.startim)*100) + value = int( + (self.current_frame - self.startim) / (self.finalim - self.startim) * 100 + ) self.increase_progress.emit(value) def _find_start_and_stop(self): @@ -478,40 +543,33 @@ def _find_start_and_stop(self): previous = self.rotidxs[0] for j, i in enumerate(self.rotidxs): if i > previous: - self.start = j-1 + self.start = j - 1 logger.info(f"Found start at frame {j-1}") - self.scangroup.attrs[ - "start_frame"] = self.start + self.scangroup.attrs["start_frame"] = self.start break previous = i else: self.start = None - self.scangroup.attrs[ - "start_frame"] = "None" + self.scangroup.attrs["start_frame"] = "None" # loop over it backwards to find the end # infact the index goes back to 1 for j, i in reversed(list(enumerate(self.rotidxs))): if i > 1: self.end = j logger.info(f"Found final at frame {j}") - self.scangroup.attrs[ - "end_frame"] = self.end - self.scangroup.attrs[ - "final_rotinx"] = i + self.scangroup.attrs["end_frame"] = self.end + self.scangroup.attrs["final_rotinx"] = i self.final_rotinx = i break else: self.end = None - self.scangroup.attrs[ - "end_frame"] = "None" + self.scangroup.attrs["end_frame"] = "None" self.final_rotinx = None - self.scangroup.attrs[ - "final_rotinx"] = "None" + self.scangroup.attrs["final_rotinx"] = "None" # add a couple more attributes for good measure self.scangroup.attrs["total_stream_frames"] = len(self.rotidxs) if self.end is not None and self.start is not None: - self.scangroup.attrs[ - "ims_between_start_end"] = self.end-self.start + self.scangroup.attrs["ims_between_start_end"] = self.end - self.start def _save_preliminary_scan_info(self): # save rotation indexes and vbf intensities @@ -519,11 +577,19 @@ def _save_preliminary_scan_info(self): if self.vbfproc["calcvbf"]: self.scangroup.create_dataset("vbf_intensities", data=self.vbfs) + # save options + if "calcmax" in self.options and self.options["calcmax"]: + self.scangroup.create_dataset("maximum_image", data=self.maximum_image) + if "calcave" in self.options and self.options["calcave"]: + self.scangroup.create_dataset("average_image", data=self.average_image) + @staticmethod def _virtual_bf_mask(arr, centeroffsetpx=(0, 0), radiuspx=10): """Create virtual bright field mask""" - xx, yy = np.meshgrid(np.arange(arr.shape[0], dtype=np.float), - np.arange(arr.shape[1], dtype=np.float)) + xx, yy = np.meshgrid( + np.arange(arr.shape[0], dtype=np.float), + np.arange(arr.shape[1], dtype=np.float), + ) xx -= 0.5 * arr.shape[0] + centeroffsetpx[0] yy -= 0.5 * arr.shape[1] + centeroffsetpx[1] mask = np.hypot(xx, yy) < radiuspx @@ -531,27 +597,33 @@ def _virtual_bf_mask(arr, centeroffsetpx=(0, 0), radiuspx=10): def determine_recorder_image_dimension(self, opts): # scan dimensions - if (opts.dimension is not None): - self.xdim, self.ydim = list(map(int, opts.dimension.split('x'))) + if opts.dimension is not None: + self.xdim, self.ydim = list(map(int, opts.dimension.split("x"))) else: dim = math.sqrt(self.final_rotinx) if not dim == int(dim): - raise ValueError("Can't determine correct image dimensions, " - "please supply values manually (--dimension)") + raise ValueError( + "Can't determine correct image dimensions, " + "please supply values manually (--dimension)" + ) self.xdim, self.ydim = dim, dim - logger.debug("Image dimensions: {}x{}".format(self.xdim, - self.ydim)) + logger.debug("Image dimensions: {}x{}".format(self.xdim, self.ydim)) class hdf5Intermediate(h5py.File): """This class represents the intermediate hdf5 file handle""" + def __init__(self, filepath, mode="r"): super().__init__(filepath, mode) - (self.total_frames, - self.start_frame, - self.end_frame, - self.final_rotator, - dim, self.imdimx, self.imdimy) = self.get_scan_info() + ( + self.total_frames, + self.start_frame, + self.end_frame, + self.final_rotator, + dim, + self.imdimx, + self.imdimy, + ) = self.get_scan_info() self.sdimx = dim self.sdimy = dim @@ -598,19 +670,33 @@ def get_scan_info(self): imdimy = None return (tot, start, end, finrot, dim, imdimx, imdimy) - def get_vbf_image(self, sdimx=None, sdimy=None, start_frame=None, - end_frame=None, hyst=0, snakescan=True): + def get_vbf_image( + self, + sdimx=None, + sdimy=None, + start_frame=None, + end_frame=None, + hyst=0, + snakescan=True, + ): # try to get the rotator data try: vbfs = self["Scan"]["vbf_intensities"][:] except Exception: - raise Exception("No VBF information found in dataset, please " - "calculate from TVIPS file.") + raise Exception( + "No VBF information found in dataset, please " + "calculate from TVIPS file." + ) logger.debug("Succesfully imported vbf intensities") logger.debug("Now calculating scan indexes") scan_indexes = self.calculate_scan_export_indexes( - sdimx=sdimx, sdimy=sdimy, start_frame=start_frame, - end_frame=end_frame, hyst=hyst, snakescan=snakescan) + sdimx=sdimx, + sdimy=sdimy, + start_frame=start_frame, + end_frame=end_frame, + hyst=hyst, + snakescan=snakescan, + ) logger.debug("Calculated scan indexes") if sdimx is None: sdimx = self.sdimx @@ -620,11 +706,25 @@ def get_vbf_image(self, sdimx=None, sdimy=None, start_frame=None, logger.debug("Applied calculated indexes and retrieved image") return img - def get_blo_export_data(self, sdimx=None, sdimy=None, start_frame=None, - end_frame=None, hyst=0, snakescan=True, crop=None): + def get_blo_export_data( + self, + sdimx=None, + sdimy=None, + start_frame=None, + end_frame=None, + hyst=0, + snakescan=True, + crop=None, + ): scan_indexes = self.calculate_scan_export_indexes( - sdimx=sdimx, sdimy=sdimy, start_frame=start_frame, - end_frame=end_frame, hyst=hyst, snakescan=snakescan, crop=crop) + sdimx=sdimx, + sdimy=sdimy, + start_frame=start_frame, + end_frame=end_frame, + hyst=hyst, + snakescan=snakescan, + crop=crop, + ) logger.debug("Calculated scan indexes") if sdimx is None: sdimx = self.sdimx @@ -637,20 +737,29 @@ def get_blo_export_data(self, sdimx=None, sdimy=None, start_frame=None, if crop is not None: xmin, xmax, ymin, ymax = crop sdimx = xmax - xmin - sdimy = ymax-ymin + sdimy = ymax - ymin shape = (sdimx, sdimy, *imshap) return shape, scan_indexes - def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, - start_frame=None, end_frame=None, - hyst=0, snakescan=True, crop=None): + def calculate_scan_export_indexes( + self, + sdimx=None, + sdimy=None, + start_frame=None, + end_frame=None, + hyst=0, + snakescan=True, + crop=None, + ): """Calculate the indexes of the list of frames to consider for the scan or VBF""" try: rots = self["Scan"]["rotation_indexes"][:] except Exception: - raise Exception("No VBF information found in dataset, please " - "calculate from TVIPS file.") + raise Exception( + "No VBF information found in dataset, please " + "calculate from TVIPS file." + ) logger.debug("Succesfully read rotator indexes") # set the scan info if sdimx is None: @@ -663,17 +772,19 @@ def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, # if a start frame is given, it's easy, we ignore rots if start_frame is not None: if end_frame is None: - end_frame = start_frame + sdimx*sdimy - 1 + end_frame = start_frame + sdimx * sdimy - 1 if end_frame >= self.total_frames: raise Exception("Final frame is out of bounds") if end_frame <= start_frame: - raise Exception("Final frame index must be larger than " - "first frame index") - if end_frame+1-start_frame != sdimx*sdimy: - raise Exception("Number of custom frames does not match " - "scan dimension") + raise Exception( + "Final frame index must be larger than " "first frame index" + ) + if end_frame + 1 - start_frame != sdimx * sdimy: + raise Exception( + "Number of custom frames does not match " "scan dimension" + ) # just create an index array - sel = np.arange(start_frame, end_frame+1) + sel = np.arange(start_frame, end_frame + 1) sel = sel.reshape(sdimy, sdimx) # reverse correct even scan lines if snakescan: @@ -683,45 +794,56 @@ def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, # check for crop if crop is not None: - logger.info('Cropping to: {}'.format(crop)) + logger.info("Cropping to: {}".format(crop)) if all(i is not None for i in crop): xmin, xmax, ymin, ymax = crop - if all(i >= 0 for i in (xmin, ymin)) and xmax < sdimx and ymax < sdimy: - sel = sel[ymin: ymax, xmin: xmax] + if ( + all(i >= 0 for i in (xmin, ymin)) + and xmax < sdimx + and ymax < sdimy + ): + sel = sel[ymin:ymax, xmin:xmax] else: - logger.warning('Aborting crop due to incorrect given dimensions: {}'.format(crop)) + logger.warning( + "Aborting crop due to incorrect given dimensions: {}".format( + crop + ) + ) return sel.ravel() # if frames or not given, we must use our best guess and match # rotators else: try: - rots = rots[self.start_frame:self.end_frame+1] + rots = rots[self.start_frame : self.end_frame + 1] except Exception: - raise Exception("No valid first or last scan frames detected, " - "must provide manual input") + raise Exception( + "No valid first or last scan frames detected, " + "must provide manual input" + ) # check whether sdimx*sdimy matches the final rotator index if not isinstance(self.final_rotator, int): - raise Exception("No final rotator index found, " - "can't align scan") - if sdimx*sdimy != self.final_rotator: - raise Exception("Scan dim x * scan dim y should match " - "the final rotator index if no custom " - "frames are specified") - indxs = np.zeros(sdimy*sdimx, dtype=int) + raise Exception("No final rotator index found, " "can't align scan") + if sdimx * sdimy != self.final_rotator: + raise Exception( + "Scan dim x * scan dim y should match " + "the final rotator index if no custom " + "frames are specified" + ) + indxs = np.zeros(sdimy * sdimx, dtype=int) prevw = 1 for j, _ in enumerate(indxs): # find where the argument is j - w = np.argwhere(rots == j+1) + w = np.argwhere(rots == j + 1) if w.size > 0: w = w[0, 0] prevw = w else: # move up if the rot index stays the same, otherwise copy - if prevw+1 < len(rots): - if rots[prevw+1] == rots[prevw]: - prevw = prevw+1 + if prevw + 1 < len(rots): + if rots[prevw + 1] == rots[prevw]: + prevw = prevw + 1 w = prevw indxs[j] = w # just an array of indexes @@ -732,4 +854,4 @@ def calculate_scan_export_indexes(self, sdimx=None, sdimy=None, # hysteresis correction on even scan lines img[::2] = np.roll(img[::2], hyst, axis=1) # add the start index - return img.ravel()+self.start_frame + return img.ravel() + self.start_frame diff --git a/tvipsconverter/widget_2.ui b/tvipsconverter/widget_2.ui index be0e70e..72219cc 100644 --- a/tvipsconverter/widget_2.ui +++ b/tvipsconverter/widget_2.ui @@ -685,6 +685,35 @@ + + + + Other calculations + + + + + + Calculate maximum diffraction pattern + + + true + + + + + + + Calculate average diffraction pattern + + + true + + + + + + diff --git a/tvipsconverter/widgets.py b/tvipsconverter/widgets.py index 1d070b9..ec49adc 100644 --- a/tvipsconverter/widgets.py +++ b/tvipsconverter/widgets.py @@ -1,11 +1,10 @@ from PyQt5 import uic -from PyQt5.QtWidgets import (QApplication, QFileDialog, QGraphicsScene) +from PyQt5.QtWidgets import QApplication, QFileDialog, QGraphicsScene from PyQt5.QtCore import QThread, pyqtSignal import sys import matplotlib.pyplot as plt from matplotlib.patches import Circle -from matplotlib.backends.backend_qt5agg import (FigureCanvasQTAgg as - FigureCanvas) +from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from pathlib import Path import logging from time import sleep @@ -22,14 +21,14 @@ logger = logging.getLogger(__name__) # import the UI interface -rawgui, Window = uic.loadUiType(str(Path(__file__).parent.absolute()) + - "/widget_2.ui") +rawgui, Window = uic.loadUiType(str(Path(__file__).parent.absolute()) + "/widget_2.ui") class External(QThread): """ Runs a counter thread. """ + countChanged = pyqtSignal(int) finish = pyqtSignal() @@ -48,6 +47,7 @@ def run(self): class ConnectedWidget(rawgui): """Class connecting the gui elements to the back-end functionality""" + def __init__(self, window): super().__init__() self.window = window @@ -141,9 +141,9 @@ def update_levels_vbf(self): vmax = self.horizontalSlider_2.value() mn = self.vbf_data.min() mx = self.vbf_data.max() - unit = (mx-mn)/100 - climmin = mn+vmin*unit - climmax = mn+(vmax+1)*unit + unit = (mx - mn) / 100 + climmin = mn + vmin * unit + climmax = mn + (vmax + 1) * unit try: self.vbf_im.set_clim(climmin, climmax) canvas = FigureCanvas(self.fig_vbf) @@ -151,9 +151,7 @@ def update_levels_vbf(self): scene = QGraphicsScene() scene.addWidget(canvas) self.graphicsView_3.setScene(scene) - self.graphicsView_3.fitInView( - scene.sceneRect(), - ) + self.graphicsView_3.fitInView(scene.sceneRect(),) self.repaint_widget(self.graphicsView_3) except Exception as e: logger.debug(f"Error: {e}") @@ -171,15 +169,15 @@ def update_final_frame(self): if self.checkBox_2.checkState(): # we use self defined size start = self.spinBox_15.value() - frms = self.spinBox.value()*self.spinBox_2.value() - self.spinBox_16.setValue(start+frms-1) + frms = self.spinBox.value() * self.spinBox_2.value() + self.spinBox_16.setValue(start + frms - 1) else: # we use auto-size start = self.spinBox_15.value() frms = self.lineEdit_11.text() try: dim = np.sqrt(int(frms)) - self.spinBox_16.setValue(start+dim**2-1) + self.spinBox_16.setValue(start + dim ** 2 - 1) except Exception: self.spinBox_16.setValue(0) @@ -206,14 +204,12 @@ def open_tvips_file(self): self.statusedit.setText(str(e)) def openFileBrowser(self, fs): - path, okpres = QFileDialog.getOpenFileName(caption="Select file", - filter=fs) + path, okpres = QFileDialog.getOpenFileName(caption="Select file", filter=fs) if okpres: return str(Path(path)) def saveFileBrowser(self, fs): - path, okpres = QFileDialog.getSaveFileName(caption="Select file", - filter=fs) + path, okpres = QFileDialog.getSaveFileName(caption="Select file", filter=fs) if okpres: return str(Path(path)) @@ -237,13 +233,13 @@ def read_modsettings(self): "usels": self.checkBox_4.checkState(), "lsmin": self.spinBox_7.value(), "lsmax": self.spinBox_8.value(), - "usecoffset": self.checkBox.checkState() + "usecoffset": self.checkBox.checkState(), } vbfsettings = { "calcvbf": self.checkBox_10.checkState(), "vbfrad": self.spinBox_12.value(), "vbfxoffset": self.spinBox_13.value(), - "vbfyoffset": self.spinBox_14.value() + "vbfyoffset": self.spinBox_14.value(), } return path, improc, vbfsettings @@ -263,9 +259,8 @@ def updatePreview(self): # (self.path_preview != path): self.update_line(self.statusedit, "Extracting frame...") self.original_preview = rec.getOriginalPreviewImage( - path, improc=improc, - vbfsettings=vbfsettings, - frame=framenum) + path, improc=improc, vbfsettings=vbfsettings, frame=framenum + ) # update the path self.path_preview = path ois = self.original_preview.shape @@ -273,36 +268,42 @@ def updatePreview(self): nis = filterframe.shape # check if the VBF aperture fits in the frame if vbfsettings["calcvbf"]: - midx = nis[1]//2 - midy = nis[0]//2 + midx = nis[1] // 2 + midy = nis[0] // 2 xx = vbfsettings["vbfxoffset"] yy = vbfsettings["vbfyoffset"] rr = vbfsettings["vbfrad"] - if (midx+xx-rr < 0 or - midx+xx+rr > nis[1] or - midy+yy-rr < 0 or - midy+yy-rr > nis[0]): - raise Exception("Virtual bright field aperture out " - "of bounds") + if ( + midx + xx - rr < 0 + or midx + xx + rr > nis[1] + or midy + yy - rr < 0 + or midy + yy - rr > nis[0] + ): + raise Exception("Virtual bright field aperture out " "of bounds") # plot the image and the circle over it if self.fig_prev is not None: plt.close(self.fig_prev) - self.fig_prev = plt.figure(frameon=False, - figsize=(filterframe.shape[1]/100, - filterframe.shape[0]/100)) + self.fig_prev = plt.figure( + frameon=False, + figsize=(filterframe.shape[1] / 100, filterframe.shape[0] / 100), + ) canvas = FigureCanvas(self.fig_prev) - ax = plt.Axes(self.fig_prev, [0., 0., 1., 1.]) + ax = plt.Axes(self.fig_prev, [0.0, 0.0, 1.0, 1.0]) ax.set_axis_off() self.fig_prev.add_axes(ax) ax.imshow(filterframe, cmap="Greys_r") if vbfsettings["calcvbf"]: xoff = vbfsettings["vbfxoffset"] yoff = vbfsettings["vbfyoffset"] - circ = Circle((filterframe.shape[1]//2+xoff, - filterframe.shape[0]//2+yoff), - vbfsettings["vbfrad"], - color="red", - alpha=0.5) + circ = Circle( + ( + filterframe.shape[1] // 2 + xoff, + filterframe.shape[0] // 2 + yoff, + ), + vbfsettings["vbfrad"], + color="red", + alpha=0.5, + ) ax.add_patch(circ) canvas.draw() scene = QGraphicsScene() @@ -311,8 +312,10 @@ def updatePreview(self): self.graphicsView.fitInView(scene.sceneRect()) self.repaint_widget(self.graphicsView) self.update_line(self.statusedit, "Succesfully created preview.") - self.update_line(self.lineEdit_8, f"Original: {ois[0]}x{ois[1]}. " - f"New: {nis[0]}x{nis[1]}.") + self.update_line( + self.lineEdit_8, + f"Original: {ois[0]}x{ois[1]}. " f"New: {nis[0]}x{nis[1]}.", + ) except Exception as e: self.update_line(self.statusedit, f"Error: {e}") # empty the preview @@ -336,8 +339,7 @@ def get_hdf5_path(self): # open a savefile browser try: # read the gui info - (self.inpath, self.improc, - self.vbfsettings) = self.read_modsettings() + (self.inpath, self.improc, self.vbfsettings) = self.read_modsettings() if not self.inpath: raise Exception("A TVIPS file must be selected!") self.oupath = self.saveFileBrowser("HDF5 (*.hdf5)") @@ -396,8 +398,7 @@ def image_range(self): def write_to_hdf5(self): try: - (self.inpath, self.improc, - self.vbfsettings) = self.read_modsettings() + (self.inpath, self.improc, self.vbfsettings) = self.read_modsettings() if not self.inpath: raise Exception("A TVIPS file must be selected!") self.oupath = self.lineEdit_2.text() @@ -410,11 +411,15 @@ def write_to_hdf5(self): improc = self.improc vbfsettings = self.vbfsettings start_frame, end_frame = self.image_range - self.get_thread = rec.Recorder(path, - improc=improc, - vbfsettings=vbfsettings, - outputpath=opath, - imrange=(start_frame, end_frame)) + self.get_thread = rec.Recorder( + path, + improc=improc, + vbfsettings=vbfsettings, + outputpath=opath, + imrange=(start_frame, end_frame), + calcmax=self.checkBox_maxiumum_image.isChecked(), # options kwarg + calcave=self.checkBox_average_image.isChecked(), # options kwarg + ) self.get_thread.increase_progress.connect(self.increase_progbar) self.get_thread.finish.connect(self.done_hdf5export) self.get_thread.start() @@ -425,8 +430,7 @@ def write_to_hdf5(self): def done_hdf5export(self): self.window.setEnabled(True) # also update lines in the second pannel - self.update_line(self.statusedit, - "Succesfully exported to HDF5") + self.update_line(self.statusedit, "Succesfully exported to HDF5") # don't auto update, the gui may be before the file exists # self.update_line(self.lineEdit_4, self.lineEdit_2.text()) @@ -452,12 +456,10 @@ def auto_read_hdf5(self): else: self.update_line(self.lineEdit_11, "?") if dim is not None: - self.update_line(self.lineEdit_12, - f"{str(int(dim))}x{str(int(dim))}") + self.update_line(self.lineEdit_12, f"{str(int(dim))}x{str(int(dim))}") else: self.update_line(self.lineEdit_12, "?") - self.update_line(self.lineEdit_13, - f"{str(imdimx)}x{str(imdimy)}") + self.update_line(self.lineEdit_13, f"{str(imdimx)}x{str(imdimy)}") self.update_final_frame() f.close() except Exception as e: @@ -503,29 +505,35 @@ def update_vbf(self): else: snakescan = False # calculate the image - logger.debug(f"We try to create a VBF image with data: " - f"S.F. {start_frame}, E.F. {end_frame}, " - f"Dims: x {sdimx} y {sdimy}," - f"hyst: {hyst}") - self.vbf_data = f.get_vbf_image(sdimx, sdimy, start_frame, - end_frame, hyst, snakescan) + logger.debug( + f"We try to create a VBF image with data: " + f"S.F. {start_frame}, E.F. {end_frame}, " + f"Dims: x {sdimx} y {sdimy}," + f"hyst: {hyst}" + ) + self.vbf_data = f.get_vbf_image( + sdimx, sdimy, start_frame, end_frame, hyst, snakescan + ) logger.debug("Succesfully created the VBF array") # save the settings for later storage - self.vbf_sets = {"start_frame": start_frame, - "end_frame": end_frame, - "scan_dim_x": sdimx, - "scan_dim_y": sdimy, - "hysteresis": hyst, - "winding_scan": snakescan} + self.vbf_sets = { + "start_frame": start_frame, + "end_frame": end_frame, + "scan_dim_x": sdimx, + "scan_dim_y": sdimy, + "hysteresis": hyst, + "winding_scan": snakescan, + } # plot the image and store it for further use. First close prior # image if self.fig_vbf is not None: plt.close(self.fig_vbf) - self.fig_vbf = plt.figure(frameon=False, - figsize=(self.vbf_data.shape[1]/100, - self.vbf_data.shape[0]/100)) + self.fig_vbf = plt.figure( + frameon=False, + figsize=(self.vbf_data.shape[1] / 100, self.vbf_data.shape[0] / 100), + ) canvas = FigureCanvas(self.fig_vbf) - ax = plt.Axes(self.fig_vbf, [0., 0., 1., 1.]) + ax = plt.Axes(self.fig_vbf, [0.0, 0.0, 1.0, 1.0]) ax.set_axis_off() self.fig_vbf.add_axes(ax) self.vbf_im = ax.imshow(self.vbf_data, cmap="plasma") @@ -584,31 +592,35 @@ def write_to_file(self): dp_scale = self.doubleSpinBox_3.value() # calculate the image filetyp = self.comboBox.currentText() - logger.debug(f"We try to create a {filetyp} file with data: " - f"S.F. {start_frame}, E.F. {end_frame}, " - f"Dims: x {sdimx} y {sdimy}," - f"hyst: {hyst}, snakescan: {snakescan}") + logger.debug( + f"We try to create a {filetyp} file with data: " + f"S.F. {start_frame}, E.F. {end_frame}, " + f"Dims: x {sdimx} y {sdimy}," + f"hyst: {hyst}, snakescan: {snakescan}" + ) logger.debug("Calculating shape and indexes") # xmin, xmax, ymin, ymax - crop = (self.spinBox_22.value(), - self.spinBox_23.value(), - self.spinBox_20.value(), - self.spinBox_21.value(), - ) - shape, indexes = f.get_blo_export_data(sdimx, sdimy, - start_frame, - end_frame, hyst, - snakescan, crop=crop) + crop = ( + self.spinBox_22.value(), + self.spinBox_23.value(), + self.spinBox_20.value(), + self.spinBox_21.value(), + ) + shape, indexes = f.get_blo_export_data( + sdimx, sdimy, start_frame, end_frame, hyst, snakescan, crop=crop + ) logger.debug(f"Shape: {shape}") logger.debug(f"Starting to write {filetyp} file") self.update_line(self.statusedit, f"Writing {filetyp} file...") if filetyp == ".blo": self.get_thread = blf.bloFileWriter( - f, path_blo, shape, indexes, scan_scale, dp_scale) + f, path_blo, shape, indexes, scan_scale, dp_scale + ) elif filetyp == ".hspy": self.get_thread = hspf.hspyFileWriter( - f, path_blo, shape, indexes, scan_scale, dp_scale) + f, path_blo, shape, indexes, scan_scale, dp_scale + ) else: raise NotImplementedError("Unrecognized file type") self.get_thread.increase_progress.connect(self.increase_progbar) @@ -621,8 +633,7 @@ def write_to_file(self): def done_bloexport(self): self.window.setEnabled(True) # also update lines in the second pannel - self.update_line(self.statusedit, - "Succesfully exported to file") + self.update_line(self.statusedit, "Succesfully exported to file") def export_tiffs(self): path_hdf5 = self.lineEdit_4.text() @@ -648,7 +659,7 @@ def export_tiffs(self): raise Exception("Unexpected dtype") if tot_frames <= last_frame: raise Exception("Frames are out of range") - frames = np.arange(first_frame, last_frame+1) + frames = np.arange(first_frame, last_frame + 1) self.update_line(self.statusedit, "Exporting to tiff files...") self.get_thread = tfe.TiffFileWriter(f, frames, dtype, pre, fin) self.get_thread.increase_progress.connect(self.increase_progbar) @@ -661,8 +672,7 @@ def export_tiffs(self): def done_tiffexport(self): self.window.setEnabled(True) # also update lines in the second pannel - self.update_line(self.statusedit, - "Succesfully exported Tiff files") + self.update_line(self.statusedit, "Succesfully exported Tiff files") def increase_progbar(self, value): self.progressBar.setValue(value) @@ -678,33 +688,38 @@ def show_cropped_region(self): ymin = self.spinBox_20.value() ymax = self.spinBox_21.value() if not xmax - xmin > 0: - logger.warning('Not valid cropping dimensions: x.') + logger.warning("Not valid cropping dimensions: x.") return if not ymax - ymin > 0: - logger.warning('Not valid cropping dimensions: x.') + logger.warning("Not valid cropping dimensions: x.") return if self.fig_vbf is None: - logger.warning('No VBF figure plotted yet.') + logger.warning("No VBF figure plotted yet.") # no VBF figure plotted yet return - - label = 'crop_rect' + label = "crop_rect" try: # see if rectangle already plotted and just update - index = [i.get_label() for i in - self.fig_vbf.axes[0].patches].index(label) + index = [i.get_label() for i in self.fig_vbf.axes[0].patches].index(label) rect = self.fig_vbf.axes[0].patches[index] - logger.info('Updating rectangle.') + logger.info("Updating rectangle.") rect.set_height(ymax - ymin) rect.set_width(xmax - xmin) rect.set_x(xmin) rect.set_y(ymin) except ValueError: - logger.info('Plotting rectangle.') - rect = plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin, - fill=False, ec='k', ls='dashed', label=label) + logger.info("Plotting rectangle.") + rect = plt.Rectangle( + (xmin, ymin), + xmax - xmin, + ymax - ymin, + fill=False, + ec="k", + ls="dashed", + label=label, + ) self.fig_vbf.axes[0].add_patch(rect) self.fig_vbf.canvas.draw() From 6d384bcf0ce7137d309cf85935c56a649f99c442 Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Tue, 3 Nov 2020 11:49:30 +0100 Subject: [PATCH 07/21] add comment- creating options image in _readFrame --- tvipsconverter/utils/recorder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tvipsconverter/utils/recorder.py b/tvipsconverter/utils/recorder.py index a989fb4..5a03b95 100644 --- a/tvipsconverter/utils/recorder.py +++ b/tvipsconverter/utils/recorder.py @@ -513,6 +513,7 @@ def _readFrame(self, fh, record=None): vbf_int = frame[self.mask].mean() self.vbfs.append(vbf_int) + # calculate images as specified in the options if "calcmax" in self.options and self.options["calcmax"]: # maximum_image should already be initialised in self.convert_HDF5 self.maximum_image = np.stack((self.maximum_image, frame), axis=0).max( From 9d0306f046c8aca0a1506914759a029e9ad84ffe Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Tue, 3 Nov 2020 20:32:39 +0100 Subject: [PATCH 08/21] added direct 16- to 8-bit conversion (skimage) --- tvipsconverter/utils/blockfile.py | 416 +++++++++++++++------------- tvipsconverter/widget_2.ui | 442 ++++++++++++++++-------------- tvipsconverter/widgets.py | 9 +- 3 files changed, 469 insertions(+), 398 deletions(-) diff --git a/tvipsconverter/utils/blockfile.py b/tvipsconverter/utils/blockfile.py index c88d2de..fac49fa 100644 --- a/tvipsconverter/utils/blockfile.py +++ b/tvipsconverter/utils/blockfile.py @@ -22,6 +22,7 @@ import warnings import datetime import dateutil +from skimage import util from dateutil import tz, parser @@ -52,8 +53,7 @@ def sarray2dict(sarray, dictionary=None): if dictionary is None: dictionary = OrderedDict() for name in sarray.dtype.names: - dictionary[name] = sarray[name][0] if len(sarray[name]) == 1 \ - else sarray[name] + dictionary[name] = sarray[name][0] if len(sarray[name]) == 1 else sarray[name] return dictionary @@ -87,14 +87,11 @@ def dict2sarray(dictionary, sarray=None, dtype=None): return sarray -def ISO_format_to_serial_date(date, time, timezone='UTC'): +def ISO_format_to_serial_date(date, time, timezone="UTC"): """ Convert ISO format to a serial date. """ - if timezone is None or timezone == 'Coordinated Universal Time': - timezone = 'UTC' - dt = parser.parse( - '%sT%s' % - (date, time)).replace( - tzinfo=tz.gettz(timezone)) + if timezone is None or timezone == "Coordinated Universal Time": + timezone = "UTC" + dt = parser.parse("%sT%s" % (date, time)).replace(tzinfo=tz.gettz(timezone)) return datetime_to_serial_date(dt) @@ -123,8 +120,7 @@ def serial_date_to_ISO_format(serial): """ dt_utc = serial_date_to_datetime(serial) dt_local = dt_utc.astimezone(tz.tzlocal()) - return (dt_local.date().isoformat(), dt_local.time().isoformat(), - dt_local.tzname()) + return (dt_local.date().isoformat(), dt_local.time().isoformat(), dt_local.tzname()) _logger = logging.getLogger(__name__) @@ -132,11 +128,11 @@ def serial_date_to_ISO_format(serial): # Plugin characteristics # ---------------------- -format_name = 'Blockfile' -description = 'Read/write support for ASTAR blockfiles' +format_name = "Blockfile" +description = "Read/write support for ASTAR blockfiles" full_support = False # Recognised file extension -file_extensions = ['blo', 'BLO'] +file_extensions = ["blo", "BLO"] default_extension = 0 # Writing capabilities: @@ -145,66 +141,73 @@ def serial_date_to_ISO_format(serial): mapping = { - 'blockfile_header.Beam_energy': - ("Acquisition_instrument.TEM.beam_energy", lambda x: x * 1e-3), - 'blockfile_header.Camera_length': - ("Acquisition_instrument.TEM.camera_length", lambda x: x * 1e-4), - 'blockfile_header.Scan_rotation': - ("Acquisition_instrument.TEM.rotation", lambda x: x * 1e-2), + "blockfile_header.Beam_energy": ( + "Acquisition_instrument.TEM.beam_energy", + lambda x: x * 1e-3, + ), + "blockfile_header.Camera_length": ( + "Acquisition_instrument.TEM.camera_length", + lambda x: x * 1e-4, + ), + "blockfile_header.Scan_rotation": ( + "Acquisition_instrument.TEM.rotation", + lambda x: x * 1e-2, + ), } -def get_header_dtype_list(endianess='<'): +def get_header_dtype_list(endianess="<"): end = endianess - dtype_list = \ + dtype_list = ( [ - ('ID', (bytes, 6)), - ('MAGIC', end + 'u2'), - ('Data_offset_1', end + 'u4'), # Offset VBF - ('Data_offset_2', end + 'u4'), # Offset DPs - ('UNKNOWN1', end + 'u4'), # Flags for ASTAR software? - ('DP_SZ', end + 'u2'), # Pixel dim DPs - ('DP_rotation', end + 'u2'), # [degrees ( * 100 ?)] - ('NX', end + 'u2'), # Scan dim 1 - ('NY', end + 'u2'), # Scan dim 2 - ('Scan_rotation', end + 'u2'), # [100 * degrees] - ('SX', end + 'f8'), # Pixel size [nm] - ('SY', end + 'f8'), # Pixel size [nm] - ('Beam_energy', end + 'u4'), # [V] - ('SDP', end + 'u2'), # Pixel size [100 * ppcm] - ('Camera_length', end + 'u4'), # [10 * mm] - ('Acquisition_time', end + 'f8'), # [Serial date] - ] + [ - ('Centering_N%d' % i, 'f8') for i in range(8) - ] + [ - ('Distortion_N%02d' % i, 'f8') for i in range(14) + ("ID", (bytes, 6)), + ("MAGIC", end + "u2"), + ("Data_offset_1", end + "u4"), # Offset VBF + ("Data_offset_2", end + "u4"), # Offset DPs + ("UNKNOWN1", end + "u4"), # Flags for ASTAR software? + ("DP_SZ", end + "u2"), # Pixel dim DPs + ("DP_rotation", end + "u2"), # [degrees ( * 100 ?)] + ("NX", end + "u2"), # Scan dim 1 + ("NY", end + "u2"), # Scan dim 2 + ("Scan_rotation", end + "u2"), # [100 * degrees] + ("SX", end + "f8"), # Pixel size [nm] + ("SY", end + "f8"), # Pixel size [nm] + ("Beam_energy", end + "u4"), # [V] + ("SDP", end + "u2"), # Pixel size [100 * ppcm] + ("Camera_length", end + "u4"), # [10 * mm] + ("Acquisition_time", end + "f8"), # [Serial date] ] + + [("Centering_N%d" % i, "f8") for i in range(8)] + + [("Distortion_N%02d" % i, "f8") for i in range(14)] + ) return dtype_list -def get_default_header(endianess='<'): +def get_default_header(endianess="<"): """Returns a header pre-populated with default values. """ dt = np.dtype(get_header_dtype_list()) header = np.zeros((1,), dtype=dt) - header['ID'][0] = 'IMGBLO'.encode() - header['MAGIC'][0] = magics[0] - header['Data_offset_1'][0] = 0x1000 # Always this value observed - header['UNKNOWN1'][0] = 131141 # Very typical value (always?) - header['Acquisition_time'][0] = datetime_to_serial_date( - datetime.datetime.fromtimestamp(86400, dateutil.tz.tzutc())) + header["ID"][0] = "IMGBLO".encode() + header["MAGIC"][0] = magics[0] + header["Data_offset_1"][0] = 0x1000 # Always this value observed + header["UNKNOWN1"][0] = 131141 # Very typical value (always?) + header["Acquisition_time"][0] = datetime_to_serial_date( + datetime.datetime.fromtimestamp(86400, dateutil.tz.tzutc()) + ) return header -def get_header_from_signal(signal, endianess='<'): +def get_header_from_signal(signal, endianess="<"): header = get_default_header(endianess) - if 'blockfile_header' in signal.original_metadata: - header = dict2sarray(signal.original_metadata['blockfile_header'], - sarray=header) - note = signal.original_metadata['blockfile_header']['Note'] + if "blockfile_header" in signal.original_metadata: + header = dict2sarray( + signal.original_metadata["blockfile_header"], sarray=header + ) + note = signal.original_metadata["blockfile_header"]["Note"] else: - note = '' + note = "" if signal.axes_manager.navigation_dimension == 2: NX, NY = signal.axes_manager.navigation_shape SX = signal.axes_manager.navigation_axes[0].scale @@ -219,21 +222,23 @@ def get_header_from_signal(signal, endianess='<'): DP_SZ = signal.axes_manager.signal_shape if DP_SZ[0] != DP_SZ[1]: - raise ValueError('Blockfiles require signal shape to be square!') + raise ValueError("Blockfiles require signal shape to be square!") DP_SZ = DP_SZ[0] - SDP = 100. / signal.axes_manager.signal_axes[0].scale + SDP = 100.0 / signal.axes_manager.signal_axes[0].scale - offset2 = NX * NY + header['Data_offset_1'] + offset2 = NX * NY + header["Data_offset_1"] # Based on inspected files, the DPs are stored at 16-bit boundary... # Normally, you'd expect word alignment (32-bits) ¯\_(°_o)_/¯ offset2 += offset2 % 16 header_sofar = { - 'NX': NX, 'NY': NY, - 'DP_SZ': DP_SZ, - 'SX': SX, 'SY': SY, - 'SDP': SDP, - 'Data_offset_2': offset2, + "NX": NX, + "NY": NY, + "DP_SZ": DP_SZ, + "SX": SX, + "SY": SY, + "SDP": SDP, + "Data_offset_2": offset2, } header = dict2sarray(header_sofar, sarray=header) @@ -242,7 +247,7 @@ def get_header_from_signal(signal, endianess='<'): def get_header(data_shape, scan_scale, diff_scale, endianess="<", **kwargs): header = get_default_header(endianess) - note = '' + note = "" if len(data_shape) == 4: NY, NX = data_shape[:2][::-1] # first dimension seems to by y in np SX = scan_scale @@ -259,21 +264,23 @@ def get_header(data_shape, scan_scale, diff_scale, endianess="<", **kwargs): DP_SZ = data_shape[-2:] if DP_SZ[0] != DP_SZ[1]: - raise ValueError('Blockfiles require signal shape to be square!') + raise ValueError("Blockfiles require signal shape to be square!") DP_SZ = DP_SZ[0] - SDP = 100. / diff_scale + SDP = 100.0 / diff_scale - offset2 = NX * NY + header['Data_offset_1'] + offset2 = NX * NY + header["Data_offset_1"] # Based on inspected files, the DPs are stored at 16-bit boundary... # Normally, you'd expect word alignment (32-bits) ¯\_(°_o)_/¯ offset2 += offset2 % 16 header_sofar = { - 'NX': NX, 'NY': NY, - 'DP_SZ': DP_SZ, - 'SX': SX, 'SY': SY, - 'SDP': SDP, - 'Data_offset_2': offset2, + "NX": NX, + "NY": NY, + "DP_SZ": DP_SZ, + "SX": SX, + "SY": SY, + "SDP": SDP, + "Data_offset_2": offset2, } header_sofar.update(kwargs) @@ -282,164 +289,177 @@ def get_header(data_shape, scan_scale, diff_scale, endianess="<", **kwargs): return header, note -def file_reader(filename, endianess='<', mmap_mode=None, - lazy=False, **kwds): +def file_reader(filename, endianess="<", mmap_mode=None, lazy=False, **kwds): _logger.debug("Reading blockfile: %s" % filename) metadata = {} if mmap_mode is None: - mmap_mode = 'r' if lazy else 'c' + mmap_mode = "r" if lazy else "c" # Makes sure we open in right mode: - if '+' in mmap_mode or ('write' in mmap_mode and - 'copyonwrite' != mmap_mode): + if "+" in mmap_mode or ("write" in mmap_mode and "copyonwrite" != mmap_mode): if lazy: raise ValueError("Lazy loading does not support in-place writing") - f = open(filename, 'r+b') + f = open(filename, "r+b") else: - f = open(filename, 'rb') + f = open(filename, "rb") _logger.debug("File opened") # Get header header = np.fromfile(f, dtype=get_header_dtype_list(endianess), count=1) - if header['MAGIC'][0] not in magics: - warnings.warn("Blockfile has unrecognized header signature. " - "Will attempt to read, but correcteness not guaranteed!") + if header["MAGIC"][0] not in magics: + warnings.warn( + "Blockfile has unrecognized header signature. " + "Will attempt to read, but correcteness not guaranteed!" + ) header = sarray2dict(header) - note = f.read(header['Data_offset_1'] - f.tell()) + note = f.read(header["Data_offset_1"] - f.tell()) # It seems it uses "\x00" for padding, so we remove it try: - header['Note'] = note.decode("latin1").strip("\x00") + header["Note"] = note.decode("latin1").strip("\x00") except Exception: # Not sure about the encoding so, if it fails, we carry on _logger.warn( "Reading the Note metadata of this file failed. " "You can help improving " "HyperSpy by reporting the issue in " - "https://github.com/hyperspy/hyperspy") + "https://github.com/hyperspy/hyperspy" + ) _logger.debug("File header: " + str(header)) - NX, NY = int(header['NX']), int(header['NY']) - DP_SZ = int(header['DP_SZ']) - if header['SDP']: - SDP = 100. / header['SDP'] + NX, NY = int(header["NX"]), int(header["NY"]) + DP_SZ = int(header["DP_SZ"]) + if header["SDP"]: + SDP = 100.0 / header["SDP"] else: SDP = -1 - original_metadata = {'blockfile_header': header} + original_metadata = {"blockfile_header": header} # Get data: # A Virtual BF/DF is stored first - offset1 = header['Data_offset_1'] + offset1 = header["Data_offset_1"] f.seek(offset1) - data_pre = np.fromfile(f, count=NX*NY, dtype=endianess+'u1' - ).squeeze().reshape((NY, NX), order='C') + data_pre = ( + np.fromfile(f, count=NX * NY, dtype=endianess + "u1") + .squeeze() + .reshape((NY, NX), order="C") + ) # Then comes actual blockfile - offset2 = header['Data_offset_2'] + offset2 = header["Data_offset_2"] if not lazy: f.seek(offset2) - data = np.fromfile(f, dtype=endianess + 'u1') + data = np.fromfile(f, dtype=endianess + "u1") else: - data = np.memmap(f, mode=mmap_mode, offset=offset2, - dtype=endianess + 'u1') + data = np.memmap(f, mode=mmap_mode, offset=offset2, dtype=endianess + "u1") try: data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) except ValueError: warnings.warn( - 'Blockfile header dimensions larger than file size! ' - 'Will attempt to load by zero padding incomplete frames.') + "Blockfile header dimensions larger than file size! " + "Will attempt to load by zero padding incomplete frames." + ) # Data is stored DP by DP: pw = [(0, NX * NY * (DP_SZ * DP_SZ + 6) - data.size)] - data = np.pad(data, pw, mode='constant') + data = np.pad(data, pw, mode="constant") data = data.reshape((NY, NX, DP_SZ * DP_SZ + 6)) # Every frame is preceeded by a 6 byte sequence (AA 55, and then a 4 byte # integer specifying frame number) data = data[:, :, 6:] - data = data.reshape((NY, NX, DP_SZ, DP_SZ), order='C').squeeze() - - units = ['nm', 'nm', 'cm', 'cm'] - names = ['y', 'x', 'dy', 'dx'] - scales = [header['SY'], header['SX'], SDP, SDP] - date, time, time_zone = serial_date_to_ISO_format( - header['Acquisition_time']) - metadata = {'General': {'original_filename': os.path.split(filename)[1], - 'date': date, - 'time': time, - 'time_zone': time_zone, - 'notes': header['Note']}, - "Signal": {'signal_type': "diffraction", - 'record_by': 'image', }, - } + data = data.reshape((NY, NX, DP_SZ, DP_SZ), order="C").squeeze() + + units = ["nm", "nm", "cm", "cm"] + names = ["y", "x", "dy", "dx"] + scales = [header["SY"], header["SX"], SDP, SDP] + date, time, time_zone = serial_date_to_ISO_format(header["Acquisition_time"]) + metadata = { + "General": { + "original_filename": os.path.split(filename)[1], + "date": date, + "time": time, + "time_zone": time_zone, + "notes": header["Note"], + }, + "Signal": {"signal_type": "diffraction", "record_by": "image",}, + } # Create the axis objects for each axis dim = data.ndim axes = [ { - 'size': data.shape[i], - 'index_in_array': i, - 'name': names[i], - 'scale': scales[i], - 'offset': 0.0, - 'units': units[i], } - for i in range(dim)] - - dictionary = {'data': data, - 'vbf': data_pre, - 'axes': axes, - 'metadata': metadata, - 'original_metadata': original_metadata, - 'mapping': mapping, } + "size": data.shape[i], + "index_in_array": i, + "name": names[i], + "scale": scales[i], + "offset": 0.0, + "units": units[i], + } + for i in range(dim) + ] + + dictionary = { + "data": data, + "vbf": data_pre, + "axes": axes, + "metadata": metadata, + "original_metadata": original_metadata, + "mapping": mapping, + } f.close() - return [dictionary, ] + return [ + dictionary, + ] def file_writer(filename, signal, **kwds): - endianess = kwds.pop('endianess', '<') + endianess = kwds.pop("endianess", "<") header, note = get_header_from_signal(signal, endianess=endianess) - with open(filename, 'wb') as f: + with open(filename, "wb") as f: # Write header header.tofile(f) # Write header note field: - if len(note) > int(header['Data_offset_1']) - f.tell(): - note = note[:int(header['Data_offset_1']) - f.tell() - len(note)] + if len(note) > int(header["Data_offset_1"]) - f.tell(): + note = note[: int(header["Data_offset_1"]) - f.tell() - len(note)] f.write(note.encode()) # Zero pad until next data block - zero_pad = int(header['Data_offset_1']) - f.tell() + zero_pad = int(header["Data_offset_1"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write virtual bright field - vbf = signal.mean( - signal.axes_manager.signal_axes[ - :2]).data.astype( - endianess + - 'u1') + vbf = signal.mean(signal.axes_manager.signal_axes[:2]).data.astype( + endianess + "u1" + ) vbf.tofile(f) # Zero pad until next data block - if f.tell() > int(header['Data_offset_2']): - raise ValueError("Signal navigation size does not match " - "data dimensions.") - zero_pad = int(header['Data_offset_2']) - f.tell() + if f.tell() > int(header["Data_offset_2"]): + raise ValueError( + "Signal navigation size does not match " "data dimensions." + ) + zero_pad = int(header["Data_offset_2"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write full data stack: # We need to pad each image with magic 'AA55', then a u32 serial - dp_head = np.zeros((1,), dtype=[('MAGIC', endianess + 'u2'), - ('ID', endianess + 'u4')]) - dp_head['MAGIC'] = 0x55AA + dp_head = np.zeros( + (1,), dtype=[("MAGIC", endianess + "u2"), ("ID", endianess + "u4")] + ) + dp_head["MAGIC"] = 0x55AA # Write by loop: for img in signal._iterate_signal(): dp_head.tofile(f) - img.astype(endianess + 'u1').tofile(f) - dp_head['ID'] += 1 + img.astype(endianess + "u1").tofile(f) + dp_head["ID"] += 1 class bloFileWriter(QThread): """Write a blo file from an HDF5 without loading all data in memory""" + increase_progress = pyqtSignal(int) finish = pyqtSignal() - def __init__(self, fh, path_blo, shape, indexes, - scan_scale=5, diff_scale=1.075): + def __init__( + self, fh, path_blo, shape, indexes, scan_scale=5, diff_scale=1.075, **options + ): QThread.__init__(self) self.fh = fh # open hdf5 file in read mode self.path_blo = path_blo @@ -453,6 +473,9 @@ def __init__(self, fh, path_blo, shape, indexes, self.vbf_im = imagefun.normalize_convert(self.vbf_im, dtype=np.uint8) logger.debug("Initialized bloFileWriter") + # added for 8-bit rescaling option + self.options = options + def run(self): self.convert_to_blo() self.finish.emit() @@ -461,12 +484,16 @@ def run(self): def convert_to_blo(self): endianess = "<" header, note = get_header( - self.shape, self.scan_scale, - self.diff_scale, endianess, - Camera_length=100.0*self.fh["ImageStream"].attrs['magtotal'], - Beam_energy=self.fh["ImageStream"].attrs['ht']*1000, - Distortion_N01=1.0, Distortion_N09=1.0, - Note="Reconstructed from TVIPS image stream") + self.shape, + self.scan_scale, + self.diff_scale, + endianess, + Camera_length=100.0 * self.fh["ImageStream"].attrs["magtotal"], + Beam_energy=self.fh["ImageStream"].attrs["ht"] * 1000, + Distortion_N01=1.0, + Distortion_N09=1.0, + Note="Reconstructed from TVIPS image stream", + ) logger.debug("Created header of blo file") with open(self.path_blo, "wb") as f: @@ -474,59 +501,63 @@ def convert_to_blo(self): header.tofile(f) logger.debug("Wrote header to file") # Write header note field: - if len(note) > int(header['Data_offset_1']) - f.tell(): - note = note[:int(header['Data_offset_1']) - - f.tell() - len(note)] + if len(note) > int(header["Data_offset_1"]) - f.tell(): + note = note[: int(header["Data_offset_1"]) - f.tell() - len(note)] f.write(note.encode()) # Zero pad until next data block - zero_pad = int(header['Data_offset_1']) - f.tell() + zero_pad = int(header["Data_offset_1"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write virtual bright field vbf = self.vbf_im.astype(endianess + "u1") vbf.tofile(f) # Zero pad until next data block - if f.tell() > int(header['Data_offset_2']): - raise ValueError("Signal navigation size does not match " - "data dimensions.") - zero_pad = int(header['Data_offset_2']) - f.tell() + if f.tell() > int(header["Data_offset_2"]): + raise ValueError( + "Signal navigation size does not match " "data dimensions." + ) + zero_pad = int(header["Data_offset_2"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write full data stack: # We need to pad each image with magic 'AA55', then a u32 serial - dp_head = np.zeros((1,), dtype=[('MAGIC', endianess + 'u2'), - ('ID', endianess + 'u4')]) - dp_head['MAGIC'] = 0x55AA + dp_head = np.zeros( + (1,), dtype=[("MAGIC", endianess + "u2"), ("ID", endianess + "u4")] + ) + dp_head["MAGIC"] = 0x55AA # Write by loop: logger.debug("Wrote header part of blo file") for j, indx in enumerate(self.indexes): dp_head.tofile(f) c = f"{indx}".zfill(6) img = self.fh["ImageStream"][f"Frame_{c}"][:] - img = imagefun.normalize_convert(img, dtype=np.uint8) - img.astype(endianess + 'u1').tofile(f) - dp_head['ID'] += 1 + + if "rescale" in self.options and self.options["rescale"]: + img = imagefun.normalize_convert(img, dtype=np.uint8) + else: + img = util.img_as_ubyte(img) + img.astype(endianess + "u1").tofile(f) + dp_head["ID"] += 1 self.update_gui_progress(j) logger.debug(f"Wrote frame Frame_{c} to blo file") def update_gui_progress(self, j): """If using the GUI update features with progress""" - value = int((j+1)/len(self.indexes)*100) + value = int((j + 1) / len(self.indexes) * 100) self.increase_progress.emit(value) def file_writer_array(filename, array, scan_scale, diff_scale, **kwds): endianess = "<" - header, note = get_header(array.shape, scan_scale, diff_scale, endianess, - **kwds) + header, note = get_header(array.shape, scan_scale, diff_scale, endianess, **kwds) - with open(filename, 'wb') as f: + with open(filename, "wb") as f: # Write header header.tofile(f) # Write header note field: - if len(note) > int(header['Data_offset_1']) - f.tell(): - note = note[:int(header['Data_offset_1']) - f.tell() - len(note)] + if len(note) > int(header["Data_offset_1"]) - f.tell(): + note = note[: int(header["Data_offset_1"]) - f.tell() - len(note)] f.write(note.encode()) # Zero pad until next data block - zero_pad = int(header['Data_offset_1']) - f.tell() + zero_pad = int(header["Data_offset_1"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write virtual bright field vbf = None @@ -535,8 +566,7 @@ def file_writer_array(filename, array, scan_scale, diff_scale, **kwds): amean = (2, 3) # do proper vbf xx, yy = np.meshgrid(range(array.shape[2]), range(array.shape[3])) - mask = np.hypot(xx - 0.5 * array.shape[2], - yy - 0.5 * array.shape[3]) < 5 + mask = np.hypot(xx - 0.5 * array.shape[2], yy - 0.5 * array.shape[3]) < 5 # TODO: make radius and offset configurable vbffloat = np.zeros((array.shape[:2])) @@ -550,23 +580,25 @@ def file_writer_array(filename, array, scan_scale, diff_scale, **kwds): elif len(array.shape) == 3: amean = (1, 2) - vbf = array.mean(axis=amean).astype(endianess + 'u1') + vbf = array.mean(axis=amean).astype(endianess + "u1") vbf.tofile(f) # Zero pad until next data block - if f.tell() > int(header['Data_offset_2']): - raise ValueError("Signal navigation size does not match " - "data dimensions.") - zero_pad = int(header['Data_offset_2']) - f.tell() + if f.tell() > int(header["Data_offset_2"]): + raise ValueError( + "Signal navigation size does not match " "data dimensions." + ) + zero_pad = int(header["Data_offset_2"]) - f.tell() np.zeros((zero_pad,), np.byte).tofile(f) # Write full data stack: # We need to pad each image with magic 'AA55', then a u32 serial - dp_head = np.zeros((1,), dtype=[('MAGIC', endianess + 'u2'), - ('ID', endianess + 'u4')]) - dp_head['MAGIC'] = 0x55AA + dp_head = np.zeros( + (1,), dtype=[("MAGIC", endianess + "u2"), ("ID", endianess + "u4")] + ) + dp_head["MAGIC"] = 0x55AA # Write by loop: - for img in array.reshape(array.shape[0]*array.shape[1], - *array.shape[2:]): + for img in array.reshape(array.shape[0] * array.shape[1], *array.shape[2:]): dp_head.tofile(f) - img.astype(endianess + 'u1').tofile(f) - dp_head['ID'] += 1 + img.astype(endianess + "u1").tofile(f) + dp_head["ID"] += 1 + diff --git a/tvipsconverter/widget_2.ui b/tvipsconverter/widget_2.ui index 72219cc..d9d1da2 100644 --- a/tvipsconverter/widget_2.ui +++ b/tvipsconverter/widget_2.ui @@ -7,7 +7,7 @@ 0 0 1171 - 755 + 990 @@ -32,38 +32,38 @@ Progress - - + + 0 0 - - - 974 - 0 - + + Status: Idle - - 0 + + true - - + + 0 0 - - Status: Idle + + + 974 + 0 + - - true + + 0 @@ -82,7 +82,7 @@ <html><head/><body><p><br/></p></body></html> - 0 + 1 @@ -156,8 +156,8 @@ 0 0 - 1100 - 643 + 1115 + 707 @@ -1055,6 +1055,9 @@ + + Qt::ScrollBarAlwaysOn + true @@ -1063,8 +1066,8 @@ 0 0 - 1054 - 610 + 1069 + 628 @@ -1154,10 +1157,10 @@ Scaling - - + + - Scale of DP + nm^-1 / pixel @@ -1168,30 +1171,6 @@ - - - - nm / pixel - - - - - - - nm^-1 / pixel - - - - - - - 4 - - - 1.000000000000000 - - - @@ -1218,6 +1197,30 @@ + + + + nm / pixel + + + + + + + Scale of DP + + + + + + + 4 + + + 1.000000000000000 + + + @@ -1236,40 +1239,24 @@ Scan modification - - - - - 0 - 0 - + + + + <html><head/><body><p>In EM Konos, the scanning follows a winding scan pattern (check on), whereas in EM Scan it uses normal flyback scan (check off)</p></body></html> - pixels + Winding scan? - - - - - + true - - - 0 - 0 - - - - <html><head/><body><p>Number of scan points in the Y direction</p></body></html> - - - Scan dim. Y + + false - - + + true @@ -1286,13 +1273,32 @@ - -100000000 + 1 + + + 10000 + + + 1 + + + 100 + + + + + + + true + + + QAbstractSpinBox::NoButtons 100000000 - 5 + 0 @@ -1303,22 +1309,30 @@ - - - - true + + + + <html><head/><body><p>Frame in the stream where the scan began</p></body></html> - - - 0 - 0 - + + Scan start frame - - <html><head/><body><p>Number of scan points in the x direction</p></body></html> + + + + + + Use custom scan dimensions + + + false + + + + - Scan dim. X + Use custom scan frames @@ -1353,50 +1367,22 @@ - - - - true - - - QAbstractSpinBox::NoButtons - - - 100000000 - - - 0 - - - - - + + true - + 0 0 - - - 93 - 0 - - - - 1 - - - 10000 - - - 1 + + <html><head/><body><p>Number of scan points in the Y direction</p></body></html> - - 100 + + Scan dim. Y @@ -1413,23 +1399,6 @@ - - - - <html><head/><body><p>Frame in the stream where the scan began</p></body></html> - - - Scan start frame - - - - - - - Use custom scan frames - - - @@ -1443,19 +1412,50 @@ - - - - <html><head/><body><p>In EM Konos, the scanning follows a winding scan pattern (check on), whereas in EM Scan it uses normal flyback scan (check off)</p></body></html> + + + + true - - Winding scan? + + + 0 + 0 + - + + + 93 + 0 + + + + -100000000 + + + 100000000 + + + 5 + + + + + + true - - false + + + 0 + 0 + + + + <html><head/><body><p>Number of scan points in the x direction</p></body></html> + + + Scan dim. X @@ -1469,13 +1469,16 @@ - - - - Use custom scan dimensions + + + + + 0 + 0 + - - false + + pixels @@ -1488,6 +1491,20 @@ Crop + + + + Show Cropped Region + + + + + + + xmin + + + @@ -1495,26 +1512,22 @@ - - + + 93 0 + + 0 + 1000 - 100 - - - - - - - xmin + 0 @@ -1534,20 +1547,6 @@ - - - - Show Cropped Region - - - - - - - ymax - - - @@ -1568,8 +1567,15 @@ - - + + + + ymax + + + + + 93 @@ -1579,24 +1585,50 @@ 1000 + + 100 + - - + + 93 0 - - 0 - 1000 - - 0 + + + + + + + + + 8-bit conversion + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + Rescale intensities across 8-bit range @@ -1672,19 +1704,6 @@ Display settings - - - - 255 - - - 10 - - - Qt::Horizontal - - - @@ -1695,6 +1714,13 @@ + + + + Maximum + + + @@ -1702,10 +1728,16 @@ - - - - Maximum + + + + 255 + + + 10 + + + Qt::Horizontal diff --git a/tvipsconverter/widgets.py b/tvipsconverter/widgets.py index ec49adc..4cbb29d 100644 --- a/tvipsconverter/widgets.py +++ b/tvipsconverter/widgets.py @@ -615,7 +615,14 @@ def write_to_file(self): self.update_line(self.statusedit, f"Writing {filetyp} file...") if filetyp == ".blo": self.get_thread = blf.bloFileWriter( - f, path_blo, shape, indexes, scan_scale, dp_scale + f, + path_blo, + shape, + indexes, + scan_scale, + dp_scale, + # if rescale button is checked do rescale, otherwise no rescaling selected + rescale=self.checkBox_rescale.isChecked(), ) elif filetyp == ".hspy": self.get_thread = hspf.hspyFileWriter( From d2fc9d103eeab366985df2817dd12a14235f321a Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Wed, 4 Nov 2020 09:31:33 +0100 Subject: [PATCH 09/21] add .DS_STORE --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bab0bcd..acd31f7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ Dummy garbage .ipynb* .venvtest +.DS_STORE # Created by https://www.gitignore.io/api/python # Edit at https://www.gitignore.io/?templates=python From ac707f6dc08d05d85d33e31f42386d8fc7c3e661 Mon Sep 17 00:00:00 2001 From: Niels Cautaerts Date: Wed, 18 Nov 2020 10:28:17 +0100 Subject: [PATCH 10/21] DS_Store ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index acd31f7..5e61abd 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ garbage .ipynb* .venvtest .DS_STORE +.DS_Store # Created by https://www.gitignore.io/api/python # Edit at https://www.gitignore.io/?templates=python From d7182005d2b92b0055402a42422784bcfb6eb249 Mon Sep 17 00:00:00 2001 From: Niels Cautaerts Date: Wed, 18 Nov 2020 10:31:31 +0100 Subject: [PATCH 11/21] removed ds store --- .DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 025144fdfefec6ad5e5ba8e8df7f7a06b52f3f59..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK-HOvd82zSO-Nd5yVxfAM8^Mbz?XpEgglxAJ3Myh2y-?Xrn%E6wGo?wl?AA(e zdjg-Cn}`L9*g)8jJ!)fxk@wv3A$#A@yj7y0ox?b(rt<r)58Q7Dto4>c$RFb3aa_ zktD}q1ecf3<22-xHXo;9CgXUj!?tb9-m6}pPB)v4y0dksHLE+*W@EEecUoI_XEWQX z-DuwL?H?YWOyA7jzQ+h*3_H2Bs~Ug62Ncd!?8qCX2~P*`6ESLMsf!tnXn@}>@$F!4 zc|q@wh_Lw8p=#`$70@$nRn9rC31<{A3jF&Di2cDq66k82E0kLYGI<36meDK?WpS4f z97m(8ajpSm!4w_sj`njk&J`*;F@5=9`pHb+P?&r=;ycovn5)p_MggP1 zvI1+WS{3Ji_t*FTWs<2G1&jj!l>)5X@j7h`NuRANgA-@1gS3w%O!& Date: Wed, 18 Nov 2020 10:46:28 +0100 Subject: [PATCH 12/21] added author, added scikit-image dependency --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a330ebf..75c8851 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ "into other formats like .blo, .tiff and .hspy." "Mainly for orientation mapping (PED) or 4D STEM experiments."), url='https://github.com/din14970/TVIPSconverter', - author='Niels Cautaerts', + author='Niels Cautaerts, Paddy Harrison', author_email='nielscautaerts@hotmail.com', license='GPL-3.0', long_description=readme, @@ -37,5 +37,6 @@ 'Pillow', 'PyQt5>=5.13.2', 'h5py>=2.10.0', + 'scikit-image>=0.17.2', ], ) From bb57eef8c107ac9ffb8b619596226efff4d80853 Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Wed, 18 Nov 2020 12:45:27 +0100 Subject: [PATCH 13/21] add box averaging ability --- .gitignore | 6 ++++ tvipsconverter/utils/imagefun.py | 19 ++++++++++ tvipsconverter/utils/recorder.py | 15 ++++++-- tvipsconverter/{widget_2.ui => widget.ui} | 44 +++++++++++++++++++++-- tvipsconverter/widgets.py | 13 +++++-- 5 files changed, 89 insertions(+), 8 deletions(-) rename tvipsconverter/{widget_2.ui => widget.ui} (97%) diff --git a/.gitignore b/.gitignore index acd31f7..ec698de 100644 --- a/.gitignore +++ b/.gitignore @@ -119,4 +119,10 @@ dmypy.json # Pyre type checker .pyre/ +# vs code +.vscode/ + +# Mac +.DS_Store + # End of https://www.gitignore.io/api/python diff --git a/tvipsconverter/utils/imagefun.py b/tvipsconverter/utils/imagefun.py index 6443ad1..4a2621b 100644 --- a/tvipsconverter/utils/imagefun.py +++ b/tvipsconverter/utils/imagefun.py @@ -205,6 +205,25 @@ def bin2(a, factor): return binned +def bin_box(a, factor): + """ + + Use box averaging to bin the images. + + """ + assert all( + not i % factor for i in a.shape + ), "array shape is not factorisable by factor." + # should work ndim + slices = tuple( + tuple((slice(j, None, factor)) for i in range(a.ndim)) for j in range(factor) + ) + + # stack th offset slices and take mean down stack axis to finish binning + cube = np.stack([a[s] for s in slices], axis=0) + return cube.mean(axis=0) + + def getElectronWavelength(ht): # ht in Volts, length unit in meters h = 6.6e-34 diff --git a/tvipsconverter/utils/recorder.py b/tvipsconverter/utils/recorder.py index 5a03b95..38fe3fd 100644 --- a/tvipsconverter/utils/recorder.py +++ b/tvipsconverter/utils/recorder.py @@ -8,7 +8,7 @@ from PyQt5.QtCore import QThread, pyqtSignal import logging -from .imagefun import normalize_convert, bin2, gausfilter, medfilter +from .imagefun import normalize_convert, bin2, bin_box, gausfilter, medfilter # Initialize the Logger logger = logging.getLogger(__name__) @@ -110,6 +110,7 @@ def filter_image( lsmin, lsmax, usecoffset, + bintype, ): """ Filter an image and return the filtered image @@ -119,7 +120,17 @@ def filter_image( imag = np.where(imag > whichint, 0, imag) # binning by some factor if usebin: - imag = bin2(imag, whichbin) + if bintype: # if True use interpolate, else box + imag = bin2(imag, whichbin) + else: + # test binning factor will work + if all(not i % whichbin for i in imag.shape): + imag = bin_box(imag, whichbin) + else: + logger.warning( + "array shape is not factorisable by factor, using decimation instead." + ) + imag = bin2(imag, whichbin) # median filter if usemed: imag = medfilter(imag, medks) diff --git a/tvipsconverter/widget_2.ui b/tvipsconverter/widget.ui similarity index 97% rename from tvipsconverter/widget_2.ui rename to tvipsconverter/widget.ui index d9d1da2..6c7267a 100644 --- a/tvipsconverter/widget_2.ui +++ b/tvipsconverter/widget.ui @@ -7,7 +7,7 @@ 0 0 1171 - 990 + 1003 @@ -82,7 +82,7 @@ <html><head/><body><p><br/></p></body></html> - 1 + 0 @@ -157,7 +157,7 @@ 0 0 1115 - 707 + 734 @@ -286,6 +286,41 @@ + + + + Binning Type + + + + + + Use box averaging where the box size is the binning factor. + + + Box Average + + + true + + + buttonGroup + + + + + + + Decimation + + + buttonGroup + + + + + + @@ -2050,4 +2085,7 @@ + + + diff --git a/tvipsconverter/widgets.py b/tvipsconverter/widgets.py index 7e1c29b..d041caa 100644 --- a/tvipsconverter/widgets.py +++ b/tvipsconverter/widgets.py @@ -1,6 +1,7 @@ from PyQt5 import uic from PyQt5.QtWidgets import QApplication, QFileDialog, QGraphicsScene from PyQt5.QtCore import QThread, pyqtSignal +from PyQt5.QtGui import QIcon, QPixmap import sys import matplotlib.pyplot as plt from matplotlib.patches import Circle @@ -12,8 +13,8 @@ import os # hotfix 3.9 MacOS Big Sur bug -if sys.platform == 'darwin': - os.environ['QT_MAC_WANTS_LAYER'] = '1' +if sys.platform == "darwin": + os.environ["QT_MAC_WANTS_LAYER"] = "1" from .utils import recorder as rec from .utils import blockfile as blf @@ -25,7 +26,9 @@ logger = logging.getLogger(__name__) # import the UI interface -rawgui, Window = uic.loadUiType(str(Path(__file__).parent.absolute()) + "/widget_2.ui") +rawgui, Window = uic.loadUiType( + str(Path(__file__).parent.absolute()) + os.sep + "widget.ui" +) class External(QThread): @@ -238,6 +241,7 @@ def read_modsettings(self): "lsmin": self.spinBox_7.value(), "lsmax": self.spinBox_8.value(), "usecoffset": self.checkBox.checkState(), + "bintype": self.radioButton_decimation.isChecked(), # if True then use decimation otherwise box averaging } vbfsettings = { "calcvbf": self.checkBox_10.checkState(), @@ -740,6 +744,9 @@ def main(): window = Window() _ = ConnectedWidget(window) window.setWindowTitle("TVIPS converter") + # window.setWindowIcon( + # QIcon(QPixmap(os.path.join(os.path.dirname(__file__), "Icon 128.png"))) + # ) window.show() app.exec_() From fbddd306c9790971994fab08c6016a014ee1a58e Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Wed, 18 Nov 2020 12:48:03 +0100 Subject: [PATCH 14/21] Update .gitignore --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index ec698de..69aab02 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ Dummy garbage .ipynb* .venvtest -.DS_STORE +.DS_Store # Created by https://www.gitignore.io/api/python # Edit at https://www.gitignore.io/?templates=python @@ -122,7 +122,5 @@ dmypy.json # vs code .vscode/ -# Mac -.DS_Store # End of https://www.gitignore.io/api/python From 3b50bf81a522e2a969d89d0fcdcdce9462f686bd Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Wed, 18 Nov 2020 14:23:36 +0100 Subject: [PATCH 15/21] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5e61abd..2cdccd9 100644 --- a/.gitignore +++ b/.gitignore @@ -121,3 +121,4 @@ dmypy.json .pyre/ # End of https://www.gitignore.io/api/python +.vscode/settings.json From 93992f226e0ca7db4db053fe12e0ae49c27e9b52 Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Thu, 19 Nov 2020 14:53:36 +0100 Subject: [PATCH 16/21] added refine_center fnality on export hdf --- tvipsconverter/utils/recorder.py | 18 ++++++ tvipsconverter/widget.ui | 105 ++++++++++++++++++++++++++++++- tvipsconverter/widgets.py | 5 ++ 3 files changed, 125 insertions(+), 3 deletions(-) diff --git a/tvipsconverter/utils/recorder.py b/tvipsconverter/utils/recorder.py index 38fe3fd..c15da8d 100644 --- a/tvipsconverter/utils/recorder.py +++ b/tvipsconverter/utils/recorder.py @@ -1,5 +1,6 @@ import numpy as np import os.path +from scipy.ndimage import gaussian_filter from tifffile import FileHandle import math import h5py @@ -543,6 +544,23 @@ def _readFrame(self, fh, record=None): axis=0, ).sum(axis=0) + if ( + "refine_center" in self.options and self.options["refine_center"][0] + ): # make sure is checked + side, sigma = self.options["refine_center"][1:] + center = np.array(frame.shape) // 2 + + crop = frame[ + center[0] - side // 2 : center[0] + side // 2, + center[1] - side // 2 : center[1] + side // 2, + ] + # blur crop and find maximum -> use as center location + blurred = gaussian_filter(crop, sigma) + # add crop offset (center - side//2) to get actual location on frame + ds.attrs["Center location"] = np.unravel_index( + blurred.argmax(), crop.shape + ) + (center - side // 2) + def _update_gui_progess(self): """If using the GUI update features with progress""" value = int( diff --git a/tvipsconverter/widget.ui b/tvipsconverter/widget.ui index 6c7267a..afaa794 100644 --- a/tvipsconverter/widget.ui +++ b/tvipsconverter/widget.ui @@ -157,7 +157,7 @@ 0 0 1115 - 734 + 780 @@ -728,6 +728,9 @@ + + <html><head/><body><p>The maximum diffraction pattern represents the maximum intensity at each diffraction pattern pixel for all scanning positions. It is therefore instructive to the diffraction intensities recorded.</p></body></html> + Calculate maximum diffraction pattern @@ -736,8 +739,71 @@ - + + + + <html><head/><body><p>Sigma should be set at least as large as the direct beam disk radius to accurately find the center point.</p></body></html> + + + 5 + + + + + + + <html><head/><body><p>Sigma should be set at least as large as the direct beam disk radius to accurately find the center point.</p></body></html> + + + Sigma + + + + + + + <html><head/><body><p>The direct beam location will be refined and stored as metadata for each frame. This is done by blurring (sigma) a small central region (of diameter box size) and finding the peak position. Therefore box size should be large enough to encompass the direct beam entirely for all scnning pixels and sigma should be at least the disk radius.</p></body></html> + + + Refine direct beam position + + + true + + + + + + + <html><head/><body><p>Diameter of the bo in which the direct beam spot will be found for all scanning positions.</p></body></html> + + + Box size + + + + + + + <html><head/><body><p>Diameter of the bo in which the direct beam spot will be found for all scanning positions.</p></body></html> + + + 30 + + + + + + + Qt::Horizontal + + + + + + <html><head/><body><p>The average diffraction pattern represents the average intensity at each diffraction pattern pixel for all scanning positions. It is therefore instructive to the average diffraction over the whole scanning area.</p></body></html> + Calculate average diffraction pattern @@ -1081,7 +1147,7 @@ - 0 + 1 @@ -1522,12 +1588,18 @@ + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + Crop + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + Show Cropped Region @@ -1535,6 +1607,9 @@ + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + xmin @@ -1542,6 +1617,9 @@ + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + ymin @@ -1555,6 +1633,9 @@ 0 + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + 0 @@ -1574,6 +1655,9 @@ 0 + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + 1000 @@ -1584,6 +1668,9 @@ + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + xmax @@ -1604,6 +1691,9 @@ + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + ymax @@ -1617,6 +1707,9 @@ 0 + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + 1000 @@ -1633,6 +1726,9 @@ 0 + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + 1000 @@ -1662,6 +1758,9 @@ + + <html><head/><body><p>If checked then the intensities in each frame will be rescaled between 0 and 255.</p></body></html> + Rescale intensities across 8-bit range diff --git a/tvipsconverter/widgets.py b/tvipsconverter/widgets.py index d041caa..9264763 100644 --- a/tvipsconverter/widgets.py +++ b/tvipsconverter/widgets.py @@ -427,6 +427,11 @@ def write_to_hdf5(self): imrange=(start_frame, end_frame), calcmax=self.checkBox_maxiumum_image.isChecked(), # options kwarg calcave=self.checkBox_average_image.isChecked(), # options kwarg + refine_center=( + self.checkBox_refine_center.isChecked(), + self.spinBox_refine_center_diameter.value(), + self.spinBox_refine_center_sigma.value(), + ), ) self.get_thread.increase_progress.connect(self.increase_progbar) self.get_thread.finish.connect(self.done_hdf5export) From c43a3b01ca54918e6083d0fe60fc4eadef0aa4c9 Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Thu, 19 Nov 2020 15:59:32 +0100 Subject: [PATCH 17/21] update default center box size and gaussian mode --- tvipsconverter/utils/recorder.py | 2 +- tvipsconverter/widget.ui | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tvipsconverter/utils/recorder.py b/tvipsconverter/utils/recorder.py index c15da8d..8c5d69d 100644 --- a/tvipsconverter/utils/recorder.py +++ b/tvipsconverter/utils/recorder.py @@ -555,7 +555,7 @@ def _readFrame(self, fh, record=None): center[1] - side // 2 : center[1] + side // 2, ] # blur crop and find maximum -> use as center location - blurred = gaussian_filter(crop, sigma) + blurred = gaussian_filter(crop, sigma, mode="nearest") # add crop offset (center - side//2) to get actual location on frame ds.attrs["Center location"] = np.unravel_index( blurred.argmax(), crop.shape diff --git a/tvipsconverter/widget.ui b/tvipsconverter/widget.ui index afaa794..05414d3 100644 --- a/tvipsconverter/widget.ui +++ b/tvipsconverter/widget.ui @@ -788,7 +788,7 @@ <html><head/><body><p>Diameter of the bo in which the direct beam spot will be found for all scanning positions.</p></body></html> - 30 + 50 @@ -1167,7 +1167,7 @@ 0 0 - 1069 + 925 628 From 05a0ab454973d7bd65e4970fec36fb7dabe99970 Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Thu, 19 Nov 2020 17:04:31 +0100 Subject: [PATCH 18/21] fixed crop output wrong shape (+1) --- tvipsconverter/utils/recorder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tvipsconverter/utils/recorder.py b/tvipsconverter/utils/recorder.py index 8c5d69d..59957ea 100644 --- a/tvipsconverter/utils/recorder.py +++ b/tvipsconverter/utils/recorder.py @@ -833,7 +833,9 @@ def calculate_scan_export_indexes( and xmax < sdimx and ymax < sdimy ): - sel = sel[ymin:ymax, xmin:xmax] + sel = sel[ + ymin : ymax + 1, xmin : xmax + 1 + ] # +1 to include final frame else: logger.warning( "Aborting crop due to incorrect given dimensions: {}".format( From 436323e1e866aeb242828ec7c14e3024bd7ee1c1 Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Thu, 19 Nov 2020 18:24:02 +0100 Subject: [PATCH 19/21] auto save scan attrs when creating vbf + on-load --- tvipsconverter/utils/recorder.py | 29 +++++++++++++++++++++++++---- tvipsconverter/widget.ui | 6 +++--- tvipsconverter/widgets.py | 21 ++++++++++++++++++++- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/tvipsconverter/utils/recorder.py b/tvipsconverter/utils/recorder.py index 59957ea..6b879ee 100644 --- a/tvipsconverter/utils/recorder.py +++ b/tvipsconverter/utils/recorder.py @@ -153,6 +153,17 @@ def getOriginalPreviewImage(path, improc, vbfsettings, frame=0): return firstframe +def write_scan_parameters_hdf5(path, **parameters): + """ + Write parameters as attrs to a hdf5 file. + + Parameters are written under h5['Scan'].attrs + """ + with h5py.File(path, "r+") as f: + for key, val in parameters.items(): + f["Scan"].attrs[key] = val + + class Recorder(QThread): increase_progress = pyqtSignal(int) @@ -685,11 +696,21 @@ def get_scan_info(self): logger.debug(f"final rotator index not found, error: {e}") finrot = None try: - dim = round(np.sqrt(finrot), 6) - if not dim == int(dim): - raise Exception + if ( + "scan_dim_x" in self["Scan"].attrs + and "scan_dim_y" in self["Scan"].attrs + ): + dim = ( + self["Scan"].attrs["scan_dim_x"], + self["Scan"].attrs["scan_dim_y"], + ) + else: - dim = int(dim) + dim = round(np.sqrt(finrot), 6) + if not dim == int(dim): + raise Exception + else: + dim = int(dim) except Exception: logger.debug("Could not calculate scan dimensions") dim = None diff --git a/tvipsconverter/widget.ui b/tvipsconverter/widget.ui index 05414d3..3ee1460 100644 --- a/tvipsconverter/widget.ui +++ b/tvipsconverter/widget.ui @@ -82,7 +82,7 @@ <html><head/><body><p><br/></p></body></html> - 0 + 1 @@ -1147,7 +1147,7 @@ - 1 + 0 @@ -1167,7 +1167,7 @@ 0 0 - 925 + 1069 628 diff --git a/tvipsconverter/widgets.py b/tvipsconverter/widgets.py index 9264763..31493fa 100644 --- a/tvipsconverter/widgets.py +++ b/tvipsconverter/widgets.py @@ -117,6 +117,10 @@ def connectUI(self): self.pushButton_11.clicked.connect(self.export_tiffs) # show cropped region self.pushButton_13.clicked.connect(self.show_cropped_region) + # write scan settings (from VBF preview) to hdf5 + # self.pushButton_write_scan_parameters.clicked.connect( + # self.write_scan_parameters_hdf5 + # ) def export_preview(self): try: @@ -458,6 +462,9 @@ def auto_read_hdf5(self): self.update_line(self.lineEdit_3, "?") if star is not None: self.update_line(self.lineEdit_5, str(star)) + logger.info(f"start: {star}") + self.spinBox_15.setValue(star) + self.checkBox_11.setChecked(True) else: self.update_line(self.lineEdit_5, "?") if en is not None: @@ -469,7 +476,15 @@ def auto_read_hdf5(self): else: self.update_line(self.lineEdit_11, "?") if dim is not None: - self.update_line(self.lineEdit_12, f"{str(int(dim))}x{str(int(dim))}") + if isinstance(dim, (list, tuple)): + self.update_line(self.lineEdit_12, str(dim)) + self.spinBox.setValue(dim[0]) + self.spinBox_2.setValue(dim[1]) + self.checkBox_2.setChecked(True) + else: + self.update_line( + self.lineEdit_12, f"{str(int(dim))}x{str(int(dim))}" + ) else: self.update_line(self.lineEdit_12, "?") self.update_line(self.lineEdit_13, f"{str(imdimx)}x{str(imdimy)}") @@ -537,6 +552,7 @@ def update_vbf(self): "hysteresis": hyst, "winding_scan": snakescan, } + # plot the image and store it for further use. First close prior # image if self.fig_vbf is not None: @@ -561,6 +577,9 @@ def update_vbf(self): yshap, xshap = self.vbf_data.shape self.update_line(self.lineEdit_10, f"Size: {xshap}x{yshap}.") f.close() + + # add settings to hdf5 file, do this after file close + rec.write_scan_parameters_hdf5(path_hdf5, **self.vbf_sets) except Exception as e: self.update_line(self.statusedit, f"Error: {e}") From 0a9b4ce2b0a6fa3587764dc3c4853c21805dddac Mon Sep 17 00:00:00 2001 From: Paddy Harrison Date: Fri, 20 Nov 2020 15:47:00 +0100 Subject: [PATCH 20/21] Update cropping. Add checkbox, perform tests --- tvipsconverter/utils/recorder.py | 10 +-- tvipsconverter/widget.ui | 123 ++++++++++++++++--------------- tvipsconverter/widgets.py | 89 ++++++++++++++++++---- 3 files changed, 142 insertions(+), 80 deletions(-) diff --git a/tvipsconverter/utils/recorder.py b/tvipsconverter/utils/recorder.py index 6b879ee..6d82b2d 100644 --- a/tvipsconverter/utils/recorder.py +++ b/tvipsconverter/utils/recorder.py @@ -828,12 +828,10 @@ def calculate_scan_export_indexes( raise Exception("Final frame is out of bounds") if end_frame <= start_frame: raise Exception( - "Final frame index must be larger than " "first frame index" + "Final frame index must be larger than first frame index" ) if end_frame + 1 - start_frame != sdimx * sdimy: - raise Exception( - "Number of custom frames does not match " "scan dimension" - ) + raise Exception("Number of custom frames does not match scan dimension") # just create an index array sel = np.arange(start_frame, end_frame + 1) sel = sel.reshape(sdimy, sdimx) @@ -854,9 +852,7 @@ def calculate_scan_export_indexes( and xmax < sdimx and ymax < sdimy ): - sel = sel[ - ymin : ymax + 1, xmin : xmax + 1 - ] # +1 to include final frame + sel = sel[ymin:ymax, xmin:xmax] # +1 to include final frame else: logger.warning( "Aborting crop due to incorrect given dimensions: {}".format( diff --git a/tvipsconverter/widget.ui b/tvipsconverter/widget.ui index 3ee1460..f76e1a7 100644 --- a/tvipsconverter/widget.ui +++ b/tvipsconverter/widget.ui @@ -1595,16 +1595,6 @@ Crop - - - - <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> - - - Show Cropped Region - - - @@ -1615,18 +1605,8 @@ - - - - <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> - - - ymin - - - - - + + 93 @@ -1636,19 +1616,16 @@ <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> - - 0 - 1000 - 0 + 100 - - + + 93 @@ -1661,46 +1638,49 @@ 1000 - - 100 - - - + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> - xmax + Show Cropped Region - - - - Qt::Horizontal - - + + + - 40 - 20 + 93 + 0 - + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + + + 1000 + + + 100 + + - - + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> - ymax + ymin - - + + 93 @@ -1710,30 +1690,57 @@ <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + + 0 + 1000 - 100 + 0 - - - - - 93 - 0 - + + + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> + + + xmax + + + + <html><head/><body><p>Crop the output file size. Only the pixels within the crop bounds will be exported.</p></body></html> - - 1000 + + ymax + + + + Apply crop? + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + diff --git a/tvipsconverter/widgets.py b/tvipsconverter/widgets.py index 31493fa..c8a74b4 100644 --- a/tvipsconverter/widgets.py +++ b/tvipsconverter/widgets.py @@ -174,8 +174,8 @@ def update_final_frame(self): # update x, y crop box values self.spinBox_22.setValue(0) self.spinBox_20.setValue(0) - self.spinBox_23.setValue(self.spinBox.value() - 1) - self.spinBox_21.setValue(self.spinBox_2.value() - 1) + self.spinBox_23.setValue(self.spinBox.value()) + self.spinBox_21.setValue(self.spinBox_2.value()) if self.checkBox_2.checkState(): # we use self defined size @@ -632,13 +632,20 @@ def write_to_file(self): ) logger.debug("Calculating shape and indexes") - # xmin, xmax, ymin, ymax - crop = ( - self.spinBox_22.value(), - self.spinBox_23.value(), - self.spinBox_20.value(), - self.spinBox_21.value(), - ) + # check crop desired + if self.checkBox_apply_crop.isChecked(): + # check crop valid + if self.check_crop_limits() is None: + self.update_line( + self.statusedit, + f"File not exported, please check cropping limits.", + ) + return + # get crop + crop = self.get_crop_limits() + else: + crop = None + shape, indexes = f.get_blo_export_data( sdimx, sdimy, start_frame, end_frame, hyst, snakescan, crop=crop ) @@ -720,22 +727,70 @@ def hardRepaint(self): self.window.hide() self.window.show() - def show_cropped_region(self): - # check for validity of cropped region + def remove_cropped_region(self): + # try to remove if it is there + label = "crop_rect" + try: + # see if rectangle already plotted and just update + index = [i.get_label() for i in self.fig_vbf.axes[0].patches].index(label) + rect = self.fig_vbf.axes[0].patches[index] + rect.remove() + self.fig_vbf.canvas.draw() + except ValueError: + logger.info("No crop rectange drawn.") + + def get_crop_limits(self): xmin = self.spinBox_22.value() xmax = self.spinBox_23.value() ymin = self.spinBox_20.value() ymax = self.spinBox_21.value() + + return xmin, xmax, ymin, ymax + + def check_crop_limits(self): + xmin, xmax, ymin, ymax = self.get_crop_limits() + + # check scan boundaries, returns None if any test fails + if ( + xmin < 0 + or ymin < 0 + or not xmax <= self.spinBox.value() + or not ymax <= self.spinBox_2.value() + ): + self.update_line( + self.statusedit, "Crop limits should be within scan bounds." + ) + self.remove_cropped_region() + return + + # check cropping dimension > 0 if not xmax - xmin > 0: - logger.warning("Not valid cropping dimensions: x.") + self.update_line( + self.statusedit, f"Crop dimension incorrect. Crop x: {xmax - xmin}.", + ) + self.remove_cropped_region() return if not ymax - ymin > 0: - logger.warning("Not valid cropping dimensions: x.") + self.update_line( + self.statusedit, f"Crop dimension incorrect. Crop y: {ymax - ymin}.", + ) + self.remove_cropped_region() return + # return non-None if passes checks + return True + + def show_cropped_region(self): + if self.check_crop_limits() is None: + return + + # check for validity of cropped region + xmin, xmax, ymin, ymax = self.get_crop_limits() + if self.fig_vbf is None: - logger.warning("No VBF figure plotted yet.") - # no VBF figure plotted yet + self.update_line( + self.statusedit, "No VBF figure plotted yet.", + ) return label = "crop_rect" @@ -762,6 +817,10 @@ def show_cropped_region(self): self.fig_vbf.axes[0].add_patch(rect) self.fig_vbf.canvas.draw() + self.update_line( + self.statusedit, f"Crop dimensions (x, y): ({xmax - xmin}, {ymax - ymin}).", + ) + def main(): app = QApplication([]) From 0f2d0d43ff623c504b5090b34a8c62c4b26c20d6 Mon Sep 17 00:00:00 2001 From: Niels Cautaerts Date: Fri, 20 Nov 2020 17:07:33 +0100 Subject: [PATCH 21/21] updated changelog, bumped version --- README.md | 9 ++++++++- setup.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 72ec841..779468d 100755 --- a/README.md +++ b/README.md @@ -88,13 +88,20 @@ supply the scan information manually. ## Credits and notes **This tool is not an official product of the TVIPS company. Use at your own risk. -I am not responsbile for loss or corruption of data.** The tool derives from python scripts +We are not responsbile for loss or corruption of data.** The tool derives from python scripts originally developed by the company. We have significantly modified these scripts mainly to make the conversion process possible on a computer with regular sized RAM and support loss-less export to hdf5. The GUI is also our addition. ## Changelog +### 0.1.2 +* calculate and store direct beam positions in the HDF5 file +* calculate mean and maximum images +* added choice in how to bin the data +* more options on export image depth +* bugfixes + ### 0.1.1 * Added an option to crop the file along the scan directions * Added an option to limit the conversion between a set number of frames diff --git a/setup.py b/setup.py index 75c8851..8be0544 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="tvipsconverter", - version="0.1.1", + version="0.1.2", description=( "GUI converter for movie data from TVIPS cameras" "into other formats like .blo, .tiff and .hspy."