diff --git a/src/dtscalibration/io/apsensing.py b/src/dtscalibration/io/apsensing.py index 2b73fb5..87f022c 100644 --- a/src/dtscalibration/io/apsensing.py +++ b/src/dtscalibration/io/apsensing.py @@ -1,3 +1,4 @@ +import csv import os import warnings from pathlib import Path @@ -103,6 +104,23 @@ def read_apsensing_files( load_in_memory=load_in_memory, ) + # add .tra data if it is available + tra_exists, tra_filepathlist = check_if_tra_exists(filepathlist) + if tra_exists: + print(".tra files exist and will be read") + data_dict_list = [] + reftemps_dict_list = [] + date_dict_list = [] + for _, tra_file in enumerate(tra_filepathlist): + data_dict, reftemps_dict, date_dict = read_single_tra_file(tra_file) + data_dict_list.append(data_dict) + reftemps_dict_list.append(reftemps_dict) + date_dict_list.append(date_dict) + data_vars = append_to_data_vars_structure(data_vars, + data_dict_list, reftemps_dict_list, date_dict_list) + + + ds = xr.Dataset(data_vars=data_vars, coords=coords, attrs=attrs, **kwargs) return ds @@ -413,3 +431,219 @@ def metakey(meta, dict_to_parse, prefix): doc = doc_["WITSMLComposite"]["wellSet"]["well"]["wellboreSet"] return metakey({}, doc, ""), skip_chars + + + +def find_corresponding_tra_file(timestamp, tra_filelist): + """ + Finds match between timstamp extracted from .xml file and filepathlist containing tra_files + + Parameters + ---------- + timestamp : str + tra_filelist : list of str + + Returns: + -------- + match_exists : boolean + file : str + """ + match_exists = False + for _, file in enumerate(tra_filelist): + tra_timestamp = file[-18:-5] + if tra_timestamp == timestamp: + match_exists = True + break + return match_exists, file + + +def check_if_tra_exists(filepathlist): + """ + Using AP Sensing N4386B both POSC (.xml) export and trace (.tra) export can be used to log measurements. + This function checks, whether both export options were turned on simultaneously. All files .xml and .tra + must be placed in the same directory. + + Parameters + ---------- + filepathlist : list of str + List of paths that point the the .xml files + + Notes: + ------ + All files .xml and .tra must be placed in the same directory. + + Returns: + -------- + tra_available : boolean, + True only, when all .xml files have a corresponding .tra file + ordered_tra_filepathlist . list of str + if tra_available is True: This list contains a list of filepaths for the + .tra file. The list is ordered the same as the input .xml filepath list. + """ + + filedir = os.path.dirname(filepathlist[0]) + # create list of .tra files in directory + dir_content=os.listdir(filedir) + dir_content.sort() + tra_filelist = [] + for _, file in enumerate(dir_content): + if '.tra' in file: + tra_filelist.append(file) + continue + + if len(tra_filelist) < len(filepathlist): # early exit, if less .tra than .xml files available + return False, [] + + tra_available = True # assume it is and prove it wrong if it's not + ordered_tra_filepathlist = [] + for filepath in filepathlist: + filename = os.path.basename(filepath) + # 14 char long timestamp is the always at the end of AP sensing exported filenames + xml_timestamp = filename[-18:-5] + tra_exists, tra_filename = find_corresponding_tra_file(xml_timestamp,tra_filelist) + if not tra_exists: + tra_available = False + break + ordered_tra_filepathlist.append(os.path.join(filedir,tra_filename)) + + return tra_available, ordered_tra_filepathlist + +def read_single_tra_file(tra_filepath): + """ + Using AP Sensing N4386B both POSC (.xml) export and trace (.tra) export can be used to log measurements. + This function reads the .tra data and appends it to the dask array, which was read from the POSC export (.xml) file. + + .tra files contain different data then the .xml files from POSC export + - more metadata + - log_ratio and loss(attenuation) calculated by device + - PT100 sensor data (optional only if sensors are connnected to device) + + + Parameters + ---------- + tra_filepathlist : list of str + List of paths that point the the .tra files + Notes: + ------ + more metadata could be read from the .tra file and stored in the dask array + + Returns: + -------- + data_dict : dict containing time series data as lists of float + reftemps_dict : dict containing PT100 reference as float + date_dict : dict containing time data as str + """ + raw_data = list() + header_end = None + + # Read data from file + with open(tra_filepath) as f: + csvreader = csv.reader(f, delimiter=';') + for row in csvreader: + raw_data.append(row) + + # Find header and footer + # Read temperature sensors + # Read timestep info + footer_idx_found = False + date_dict = {} + reftemps_dict = {} + for ii, line in enumerate(raw_data): + if len(line)>0: + if 'Date.Year' in line[0] or 'Ref.Temperature.Sensor.' in line[0]: + if not footer_idx_found: + data_end = ii + footer_idx_found = True + + if 'Trace.' in line[0]: + header_end = ii + 1 + + elif "Ref.Temperature.Sensor." in line[0]: + idx_ref_temp_sens = int(line[0][-1]) + reftemps_dict[f"PT100_{idx_ref_temp_sens}"] = float(line[1]) + + elif 'Date.Year' in line[0]: + date_dict["year"] = line[1] + elif 'Date.Month' in line[0]: + date_dict["month"] = line[1] + elif 'Date.Day' in line[0]: + date_dict["day"] = line[1] + elif 'Time.Hour' in line[0]: + date_dict["hour"] = line[1] + elif 'Time.Minute' in line[0]: + date_dict["minute"] = line[1] + elif 'Time.Second' in line[0]: + date_dict["second"] = line[1] + + header = raw_data[:header_end] + footer = raw_data[data_end:] + + # self.t_abs=dt.datetime( + # int(date_dict["year"]), + # int(date_dict["month"]), + # int(date_dict["day"]), + # int(date_dict["hour"]), + # int(date_dict["minute"]), + # int(date_dict["second"]) + # ) + + ### Convert data into usable datatype + data_lst = raw_data[header_end:data_end] + data_colnames = ["index", "distance", "t_by_dts", "log_ratio", "loss"] + data_dict = dict() + for ii, _ in enumerate(data_lst[0]): + data_dict.update({data_colnames[ii]:[]}) + for ii, line in enumerate(data_lst): + for jj, elem in enumerate(line): + data_dict[data_colnames[jj]].append(float(elem)) + + return data_dict, reftemps_dict, date_dict + +def append_to_data_vars_structure(data_vars, + data_dict_list, reftemps_dict_list, date_dict_list): + # compose array of format [[value(x1t1).. value(x1tm)] + # .... + # [value(xnt1).. value(xntm)]] + for idx, data_dict in enumerate(data_dict_list): + if idx == 0: + t_by_dts = np.column_stack(np.column_stack(data_dict["t_by_dts"])) + log_ratio = np.column_stack(np.column_stack(data_dict["log_ratio"])) + loss = np.column_stack(np.column_stack(data_dict["loss"])) + else: + t_by_dts = np.concatenate((t_by_dts, np.array(data_dict["t_by_dts"])[:,None]), axis=1) + log_ratio = np.concatenate((log_ratio, np.array(data_dict["log_ratio"])[:,None]), axis=1) + loss = np.concatenate((loss, np.array(data_dict["loss"])[:,None]), axis=1) + # add log_ratio and attenaution to data_vars + data_vars["log_ratio_by_dts"] = (("x", "time"), log_ratio) + data_vars["loss_by_dts"] = (("x", "time"), loss) + + # add reference temp data, if they exist + for idx_ref_temp in range(1,5): + if f"PT100_{idx_ref_temp}" in reftemps_dict_list[0].keys(): + ref_temps = [] + for _, reftemps_dict in enumerate(reftemps_dict_list): + ref_temps.append(reftemps_dict[f"PT100_{idx_ref_temp}"]) + data_vars[f"probe{idx_ref_temp}Temperature"] = (("time",), ref_temps) + + + # check if files match by comparing timestamps and dts temperature + for idx_t in range(0,len(date_dict_list)): + err_msg = f"fatal error in allocation of .xml and .tra data.\nxml file {data_vars['creationDate'][1][idx_t]}\ntra file {date_dict_list[idx_t]}\n\n" + # check timestamps + date_dict = date_dict_list[idx_t] + dd_ts = pd.Timestamp( + int(date_dict["year"]), + int(date_dict["month"]), + int(date_dict["day"]), + int(date_dict["hour"]), + int(date_dict["minute"]), + int(date_dict["second"])) + if not data_vars["creationDate"][1][idx_t] == dd_ts: + raise Exception(err_msg) + + # check dts temperature + for idx_x in [0,2,5]: + if not data_vars["tmp"][1][idx_x][idx_t] == t_by_dts[idx_x][idx_t]: + # fatal error in allocation of .tra and .xml data + raise Exception(err_msg) + return data_vars