new: optional imports

_version.py

@@ -12,5 +12,5 @@
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.1.dev1+g0ea4cc4.d20231129'
-__version_tuple__ = version_tuple = (0, 1, 1, 'dev1', 'g0ea4cc4.d20231129')
+__version__ = version = '0.1.1.dev30+ge4663a3.d20240724'
+__version_tuple__ = version_tuple = (0, 1, 1, 'dev30', 'ge4663a3.d20240724')

atmPy/aerosols/instruments/LAS/LAS.py

@@ -11,10 +11,66 @@
 import numpy as np
 import pandas as pd
 import pylab as plt
-from StringIO import StringIO as io
+from io import StringIO as io
 from scipy.interpolate import UnivariateSpline
 
 from atmPy.aerosols.size_distribution import sizedistribution
+import atmPy.aerosols.instruments.POPS.mie as atmmie
+import atmPy.aerosols.instruments.POPS.calibration as atmpc
+from functools import partial
+
+
+def makeMie(IOR = 1.5,
+              noOfdiameters = 200,
+              diameterRangeInMikroMeter = [0.1, 5],
+              geometry = 'perpendicular'):
+
+    '''
+    def collectionangle(d = 2.5, #mirror diameter
+                    h = 1.0 # mirror scattering center distance
+                   ):
+    sa = np.rad2deg(np.arctan(d/(2*h)))#, np.rad2deg(np.arctan2(d/(2*h)))
+    return float(90-sa), float(90 + sa)
+    collectionangle()
+    '''
+
+    mirrorJetDist=8.106 # (32.962602120353196, 147.0373978796468)
+    mirrorJetDist_hole = 44 #(74.14063373339431, 105.85936626660569)
+    noOfAngles=101 #there was a sampling issue
+    WavelengthInUm=0.633
+    # diameterRange = [LAS_bincenters.LAS_Bin_Centers.min(), LAS_bincenters.LAS_Bin_Centers.max()]
+
+    #scattering of the entire collection mirrow including the part that is lost through the whole
+    d,scatt_woh = atmmie.makeMie_diameter(mirrorJetDist=mirrorJetDist,
+                                        diameterRangeInMikroMeter=diameterRangeInMikroMeter,
+                                        noOfdiameters=noOfdiameters,
+                                        noOfAngles=noOfAngles,
+                                        # POPSdesign='POPS 2',
+                                        IOR=IOR,
+                                        WavelengthInUm=WavelengthInUm,
+                                        geometry=geometry,
+                                        # scale='log',
+                                        # broadened=False,
+                                        # doreturn='tuple',
+                                    )
+    # scattering that is lost through the hole
+    d,scatt_hole = atmmie.makeMie_diameter(mirrorJetDist=mirrorJetDist_hole,
+                                        diameterRangeInMikroMeter=diameterRangeInMikroMeter,
+                                        noOfdiameters=noOfdiameters,
+                                        noOfAngles=noOfAngles,
+                                        # POPSdesign='POPS 2',
+                                        IOR=IOR,
+                                        WavelengthInUm=WavelengthInUm,
+                                        geometry=geometry,
+                                        # scale='log',
+                                        # broadened=False,
+                                        # doreturn='tuple',
+                                    )
+
+    scatt = scatt_woh - scatt_hole
+    return [d, scatt]
+
+simulate_calibration =  partial(atmpc.generate_calibration, makeMie_diameter = makeMie)
 
 
 def read_csv(fname):
@@ -172,7 +228,6 @@ def read_Calibration_fromString(data):
     calibrationInstance = calibration(dataFrame)
     return calibrationInstance
 
-
 def save_Calibration(calibrationInstance, fname):
     """should be saved hier cd ~/data/POPS_calibrations/"""
     calibrationInstance.data.to_csv(fname, index = False)

atmPy/aerosols/instruments/POPS/calibration.py

@@ -29,10 +29,12 @@ def generate_calibration(single_pnt_cali_d=508,
                          no_cal_pts=50,
                          plot=True,
                          raise_error=True,
-                         test=False
+                         test=False,
+                         makeMie_diameter = mie.makeMie_diameter,
+                         geometry = 'perpendicular',
                          ):
     """
-    This function generates a calibration function for the POPS instrument based on its theoretical responds.
+    This function generates a calibration function for the POPS instrument based on its theoretical responds. It furthermore allows for a single point calibration. Here in an additional step the scaling factor will be determined which is nesesarry to account for instrument effects, like laser power and detector sensitivity.
 
     Args:
         single_pnt_cali_d: float [508]
@@ -67,9 +69,10 @@ def generate_calibration(single_pnt_cali_d=508,
         if test: Series instance
     """
     drum = np.array(dr)/1e3
-    d, amp = mie.makeMie_diameter(diameterRangeInMikroMeter=drum,
-                                 noOfdiameters=no_pts,
-                                 IOR=ior)
+    d, amp = makeMie_diameter(diameterRangeInMikroMeter=drum,
+                                noOfdiameters=no_pts,
+                                IOR=ior,
+                                geometry=geometry,)
 
     df = pd.DataFrame({'d': d, 'amp': amp})
     # a calibration function is created with no_cal_pts of calibration points. no_cal_pts is steadyly decreased until
@@ -82,7 +85,7 @@ def generate_calibration(single_pnt_cali_d=508,
             0]) * 0.01  # this is to ensure the first point is not onthe edge ... for cut function used later
 
         mie_cal = df.groupby(pd.cut(df.amp, binedgs)).median()
-        dfstd = df.groupby(pd.cut(df.amp, binedgs)).mad()
+        dfstd = df.groupby(pd.cut(df.amp, binedgs)).std() #202408 std used to be mad which was deprecated in 1.5. I hope this does not change too much
 
         mie_cal.index = mie_cal.d
         dfstd.index = mie_cal.d
@@ -103,37 +106,40 @@ def generate_calibration(single_pnt_cali_d=508,
     cali_inst_pre = Calibration(mie_cal)
 
     # single point calibration
-    ## solve calibration function ot get amp at calibration diameter
+    ## solve calibration function to get amp at calibration diameter
     ### first guess
     dt = mie_cal.index[abs(mie_cal.index - single_pnt_cali_d).argmin()]
     at = mie_cal.loc[dt, 'amp']
-
     # cali_inst_at_single_pnt_calid_d = cali_inst_pre.calibrationFunction(single_pnt_cali_d)
 
-    ### solve
+    ### solve, gives the aplitude at the calibration diameter ... not sure why this is not done by iterpolation?!?
     cali_inst_at_single_pnt_calid_d = optimize.fsolve(lambda x: cali_inst_pre.calibrationFunction(x) - single_pnt_cali_d, at)
-
     ## scale for ior mismatch
     if ior == single_pnt_cali_ior:
-        scale_ioradj = 1
+        # scale_ioradj = 1
         single_pnt_cali_int_pre = cali_inst_at_single_pnt_calid_d
     else:
         # single_pnt_cali_d = 500
         single_pnt_cali_d *= 1e-3
-        dt, mt = mie.makeMie_diameter(diameterRangeInMikroMeter=[single_pnt_cali_d, single_pnt_cali_d + 1e-3],
+        dt, mt = makeMie_diameter(diameterRangeInMikroMeter=[single_pnt_cali_d, 
+                                                             single_pnt_cali_d + 1e-3],
                                       IOR=single_pnt_cali_ior,
-                                      noOfdiameters=2)
-        single_pnt_cali_int_pre = mt[0]
+                                      noOfdiameters=2,
+                                geometry=geometry,)
 
-        scale_ioradj = cali_inst_at_single_pnt_calid_d / single_pnt_cali_int_pre
+        single_pnt_cali_int_pre = mt[0]
 
-    ## scale for instrument calibration
+
+        # scale_ioradj = cali_inst_at_single_pnt_calid_d / single_pnt_cali_int_pre #amp at desired ior divided by amp at cal ior; this is not needed, using it would be double dipping
+
+        single_pnt_cali_d *= 1e3
+
+    ## scale for instrument calibration, this is puly to adjust the intenisty of the mie calculation to the laser intensity and detector efficiency
     scale_instrument = single_pnt_cali_int / single_pnt_cali_int_pre
-
-    ## total scale
-    scale = scale_ioradj * scale_instrument
-
-    mie_cal.loc[:,'amp'] *= scale
+    ## total scale, this makes no sence, applying this would be applying the ior effect twice!!! the effect from ior is already in the mie calculations!
+    # scale = scale_ioradj * scale_instrument
+
+    mie_cal.loc[:,'amp'] *= scale_instrument
 
     if not isinstance(noise_level, type(None)):
         mie_cal.loc[:, 'amp'] += noise_level
@@ -142,22 +148,22 @@ def generate_calibration(single_pnt_cali_d=508,
 
     if plot:
         f, a = plt.subplots()
-        a.plot(df.d * 1e3, df.amp * scale, label='POPS resp.')
+        a.plot(df.d * 1e3, df.amp * scale_instrument, label='POPS resp.')
         cali_inst.plot(ax = a)
         # a.plot(ampm.index * 1e3, ampm.values * scale, label='POPS resp. smooth')
         # g, = a.plot(cali.index, cali.values, label='cali')
         # g.set_linestyle('')
         # g.set_marker('x')
         # g.set_markersize(10)
         # g.set_markeredgewidth(2)
-        g, = a.plot(single_pnt_cali_d * 1e3, single_pnt_cali_int, label='single ptn cal')
+        g, = a.plot(single_pnt_cali_d, single_pnt_cali_int, label='single ptn cal')
         g.set_linestyle('')
         g.set_marker('x')
         g.set_markersize(10)
         g.set_markeredgewidth(2)
         g.set_label('single pt. cali.')
         # # st.plot(ax = a)
-        # a.loglog()
+        a.loglog()
         a.legend()
         # return dft
         # return cali_inst, a
@@ -507,7 +513,7 @@ def plot(self, ax=None, nofpts = 500):
 
         d = cal_function(amp)
 
-        if type(ax).__name__ == 'AxesSubplot':
+        if type(ax).__name__ in ['Axes','AxesSubplot']:
             a = ax
             f = a.get_figure()
         else:

atmPy/aerosols/instruments/POPS/mie.py

@@ -30,6 +30,7 @@
 
 
 ###########################
+
 def makeMie_diameter(diameterRangeInMikroMeter = [0.1,3.],
             noOfdiameters = 200,
             noOfAngles = 100, # number of scatternig angles
@@ -420,7 +421,7 @@ def get_mirror_grid(self):
 #         print "ss"
 #         raw_input(ss.astype(int))
 
-        ArcLengthMatrix[ArcLengthMatrix > ss/2.] = np.NAN
+        ArcLengthMatrix[ArcLengthMatrix > ss/2.] = np.nan
 #         print "ArcLengthMatrix"
 #         raw_input(ArcLengthMatrix.astype(int))
 

atmPy/aerosols/physics/column_optical_properties.py

@@ -11,7 +11,8 @@
 import scipy as _sp
 #import pathlib as _pl
 import xarray as xr
-import statsmodels.nonparametric.smoothers_lowess as smlowess
+# import statsmodels.nonparametric.smoothers_lowess as smlowess
+from atmPy.opt_imports import statsmodels_nonparametric_smoothers_lowess as smlowess
 
 _colors = _plt.rcParams['axes.prop_cycle'].by_key()['color']
 

atmPy/atmosphere/sounding.py

@@ -2,8 +2,9 @@
 from atmPy.general import flightpath as _flightpath
 import xarray as xr
 import pandas as pd
-import metpy
-import metpy.calc
+# import metpy
+# import metpy.calc
+from atmPy.opt_imports import metpy
 
 class BalloonSounding(object):
     def __init__(self, data, column_lat='Lat', column_lon='Lon', column_altitude='Altitude'):

atmPy/data_archives/NOAA_ESRL_GMD_GRAD/baseline/baseline.py

@@ -22,14 +22,14 @@
               'state' :'Samoa',
               'abbreviation': ['SMO',],
               'lon': -170.5644,
-              'lat': 14.2474,
+              'lat': -14.2474,
               'alt' : 42.,
               'timezone': 11},
-              {'name': '',
+              {'name': 'South Pole',
               'state' :'',
-              'abbreviation': ['', ''],
+              'abbreviation': ['SPO', ],
               'lon': 59,
-              'lat': 90.00,
+              'lat': -90.00,
               'alt' : 2840,
               'timezone': -12}]
 

atmPy/data_archives/NOAA_ESRL_GMD_GRAD/surfrad/surfrad.py

@@ -108,53 +108,67 @@ class FilterFunctionError(Exception):
               'lon': -88.37309,
               'lat': 40.05192,
               'alt' :230,
-              'timezone': -6},
+              'timezone': -6,
+              'earthdata_granule': 'h11v05',
+              },
               {'name': 'Sioux Falls',
               'state': 'SD',
               'abbreviation': ['SXF', 'sxf'],
               'lon': -96.62328,
               'lat': 43.73403,
               'alt': 473,
-              'timezone': -6},
+              'timezone': -6,
+              'earthdata_granule': 'h11v04',
+              },
               {'name': 'Table Mountain',
               'state': 'CO',
               'abbreviation': ['TBL', 'tbl'],
               'lon': -105.23680,
               'lat': 40.12498,
               'alt': 1689,
-              'timezone': -7},
+              'timezone': -7,
+              'earthdata_granule': 'h09v04',
+              },
               {'name': 'Desert Rock',
               'state': 'NV',
               'abbreviation': ['DRA', 'dra'],
               'lon': -116.01947,
               'lat': 36.62373,
               'alt': 1007,
               'timezone': -8,
-              'type': 'permanent'},
+              'type': 'permanent',
+              'earthdata_granule': 'h08v05',
+              },
               {'name': 'Fort Peck',
               'state': 'MT',
               'abbreviation': ['FPK', 'fpk', 'FPE', 'fpe'],
               'lon': -105.10170,
               'lat': 48.30783,
               'alt': 634,
               'timezone': -7,
-              'type': 'permanent'},
+              'type': 'permanent',
+              'earthdata_granule': 'h10v04',
+              },
               {'name': 'Goodwin Creek',
               'state': 'MS',
               'abbreviation': ['GWN', 'gwn'],
               'lon': -89.8729,
               'lat': 34.2547,
               'alt': 98,
               'timezone': -6,
-              'type': 'permanent'},
+              'type': 'permanent',
+              'earthdata_granule': 'h10v05',
+              },
               {'name': 'Penn. State Univ.',
               'state': 'PA',
               'abbreviation': ['PSU', 'psu'],
               'lon': -77.93085,
               'lat': 40.72012,
               'alt': 376,
               'timezone': -5,
-              'type': 'permanent'},
+              'type': 'permanent',
+              'earthdata_granule': 'h12v04',
+              },
               # {'name': 'ARM Southern Great Plains Facility',
               # 'state': 'OK',
               # 'abbreviation': ['SGP', 'sgp'],

atmPy/general/data_structure.py

@@ -1,12 +1,15 @@
 import pandas as _pd
 import numpy as _np
 import warnings as _warnings
-try:
-    from statsmodels import robust as _robust
-except ModuleNotFoundError:
-    _warnings.warn('statsmodels not installed. You might encounter some functionality limitations.')
+# try:
+#     from statsmodels import robust as _robust
+# except ModuleNotFoundError:
+#     _warnings.warn('statsmodels not installed. You might encounter some functionality limitations.')
 from functools import wraps as _wraps
 
+# from atmPy.opt_imports import statsmodels as 
+
+
 def close_gaps(ts, verbose = False):
     """This is an older version to deal with gaps ... rather consider using the ones below"""
     ts = ts.copy()
@@ -49,7 +52,8 @@ def detect_gaps(ts, toleranz=1.95, return_all=False):
     dt = (idx[1:] - idx[:-1]) / _np.timedelta64(1, 's')
 
     med = _np.median(dt)
-    mad = _robust.mad(dt)
+    # mad = _robust.mad(dt) # mad was deprecated
+    mad = _np.std(dt) * 0.8 # this is of course only valid for normal distributed data
 
     if mad == 0:
         noofgaps = 0