diff --git a/.coveragerc b/.coveragerc index 4c62180950..04ab2270b8 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,3 +3,4 @@ relative_files = True source = metplus omit = metplus/wrappers/cyclone_plotter_wrapper.py + metplus/produtil/** \ No newline at end of file diff --git a/internal/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile index e9f9c3ecbb..2df83a89c5 100644 --- a/internal/scripts/docker/Dockerfile +++ b/internal/scripts/docker/Dockerfile @@ -58,5 +58,5 @@ RUN if [ ${OBTAIN_SOURCE_CODE} != "none" ]; then \ sed -i 's|MET_INSTALL_DIR = /path/to|MET_INSTALL_DIR = /usr/local|g' parm/metplus_config/*.conf; \ sed -i 's|OUTPUT_BASE = /path/to|OUTPUT_BASE = /data/output|g' parm/metplus_config/*.conf; \ sed -i 's|INPUT_BASE = /path/to|INPUT_BASE = /data/input/METplus_Data|g' parm/metplus_config/*.conf; \ - python3 setup.py install; \ + pip install .; \ fi diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index 1504a3f161..ab3a8ae341 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -3,8 +3,8 @@ sonar.projectKey=METplus sonar.projectName=METplus sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME -sonar.sources=docs,internal,manage_externals,metplus,parm,produtil,ush -sonar.coverage.exclusions=internal/tests/**,parm/**,internal/scripts/**,manage_externals/**,docs/**,produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py +sonar.sources=docs,internal,manage_externals,metplus,parm,ush +sonar.coverage.exclusions=internal/tests/**,parm/**,metplus/parm/**,internal/scripts/**,manage_externals/**,docs/**,metplus/produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 diff --git a/internal/tests/pytests/util/run_util/test_run_util.py b/internal/tests/pytests/util/run_util/test_run_util.py index 6815e0b9d2..9b95e891e5 100644 --- a/internal/tests/pytests/util/run_util/test_run_util.py +++ b/internal/tests/pytests/util/run_util/test_run_util.py @@ -4,7 +4,6 @@ import os import re -import produtil import metplus.util.run_util as ru import metplus.util.wrapper_init as wi from metplus.wrappers.ensemble_stat_wrapper import EnsembleStatWrapper diff --git a/metplus/parm b/metplus/parm new file mode 120000 index 0000000000..302dacab35 --- /dev/null +++ b/metplus/parm @@ -0,0 +1 @@ +../parm \ No newline at end of file diff --git a/produtil/README_produtil.md b/metplus/produtil/README_produtil.md similarity index 100% rename from produtil/README_produtil.md rename to metplus/produtil/README_produtil.md diff --git a/produtil/__init__.py b/metplus/produtil/__init__.py similarity index 100% rename from produtil/__init__.py rename to metplus/produtil/__init__.py diff --git a/produtil/batchsystem.py b/metplus/produtil/batchsystem.py similarity index 100% rename from produtil/batchsystem.py rename to metplus/produtil/batchsystem.py diff --git a/produtil/cluster.py b/metplus/produtil/cluster.py similarity index 100% rename from produtil/cluster.py rename to metplus/produtil/cluster.py diff --git a/produtil/config.py b/metplus/produtil/config.py similarity index 99% rename from produtil/config.py rename to metplus/produtil/config.py index f020838a10..ffffa40621 100644 --- a/produtil/config.py +++ b/metplus/produtil/config.py @@ -11,19 +11,18 @@ # decides what symbols are imported by "from produtil.config import *" __all__=['from_file','confwalker','ProdConfig','ENVIRONMENT','ProdTask'] -import collections,re,string,os,logging,threading +import collections,re,os,logging,threading import os.path,sys import datetime -import produtil.fileop, produtil.datastore -import produtil.numerics, produtil.log +import metplus.produtil.fileop as fileop import configparser from configparser import ConfigParser from io import StringIO -from produtil.datastore import Datastore,Task +from metplus.produtil.datastore import Datastore,Task -from produtil.numerics import to_datetime +from metplus.produtil.numerics import to_datetime, to_datetime_rel, fcst_hr_min from string import Formatter from configparser import NoOptionError,NoSectionError @@ -379,7 +378,7 @@ def get_value(self,key,args,kwargs): v=ap6.strftime(ANL_P6_KEYS[key]) elif '__ftime' in kwargs and '__atime' in kwargs and \ key in TIME_DIFF_KEYS: - (ihours,iminutes)=produtil.numerics.fcst_hr_min( + (ihours,iminutes)=fcst_hr_min( kwargs['__ftime'],kwargs['__atime']) if key=='fahr': v=int(ihours) @@ -702,7 +701,7 @@ def from_args(self,args=None,allow_files=True,allow_options=True, elif not os.path.isfile(path): logger.error(path+': conf file is not a regular file.') sys.exit(2) - elif not produtil.fileop.isnonempty(path) and verbose: + elif not fileop.isnonempty(path) and verbose: logger.warning( path+': conf file is empty. Will continue anyway.') if verbose: logger.info('Conf input: '+repr(path)) @@ -823,7 +822,7 @@ def getdatastore(self): with self: if self._datastore is None: dsfile=self.getstr('config','datastore') - self._datastore=produtil.datastore.Datastore(dsfile, + self._datastore=Datastore(dsfile, logger=self.log('datastore')) return self._datastore @@ -1001,7 +1000,7 @@ def makedirs(self,*args): with self: dirs=[self.getstr('dir',arg) for arg in args] for makeme in dirs: - produtil.fileop.makedirs(makeme) + fileop.makedirs(makeme) def keys(self,sec): """!get options in a section @@ -1107,13 +1106,13 @@ def timestrinterp(self,sec,string,ftime=None,atime=None,**kwargs): @param atime the analysis time or None @param kwargs more variables for string expansion""" if atime is not None: - atime=produtil.numerics.to_datetime(atime) + atime=to_datetime(atime) else: atime=self.cycle if ftime is None: ftime=atime else: - ftime=produtil.numerics.to_datetime_rel(ftime,atime) + ftime=to_datetime_rel(ftime,atime) with self: return self._time_formatter.format(string,__section=sec, __key='__string__',__depth=0,__conf=self._conf,ENV=ENVIRONMENT, @@ -1343,7 +1342,7 @@ def getbool(self,sec,opt,default=None,badtypeok=False,morevars=None,taskvars=Non ######################################################################## -class ProdTask(produtil.datastore.Task): +class ProdTask(Task): """!A subclass of produtil.datastore.Task that provides a variety of convenience functions related to unix conf files and logging.""" diff --git a/produtil/datastore.py b/metplus/produtil/datastore.py similarity index 98% rename from produtil/datastore.py rename to metplus/produtil/datastore.py index cb2a665956..42268fe3f4 100644 --- a/produtil/datastore.py +++ b/metplus/produtil/datastore.py @@ -6,9 +6,10 @@ Datum, which is the base class of anything that can be stored in the Datastore.""" -import sqlite3, threading, collections, re, contextlib, time, random,\ - traceback, datetime, logging, os, time -import produtil.fileop, produtil.locking, produtil.sigsafety, produtil.log +import sqlite3, threading, collections, re, contextlib, datetime, logging, os, time +import metplus.produtil.fileop as fileop +from metplus.produtil.locking import LockFile +from metplus.produtil.log import jlogger ##@var __all__ # Symbols exported by "from produtil.datastore import *" @@ -175,7 +176,7 @@ def __init__(self,filename,logger=None,locking=True): lockfile=filename+'.lock' if logger is not None: logger.debug('Lockfile is %s for database %s'%(lockfile,filename)) - self._file_lock=produtil.locking.LockFile( + self._file_lock=LockFile( lockfile,logger=logger,max_tries=300,sleep_time=0.1,first_warn=50) self._transtack=collections.defaultdict(list) with self.transaction(): @@ -884,7 +885,7 @@ def undeliver(self,delete=True,logger=None): @param logger a logging.Logger for log messages""" loc=self.location if loc and delete: - produtil.fileop.remove_file(filename=loc,logger=logger,info=True) + fileop.remove_file(filename=loc,logger=logger,info=True) self.available=False def deliver(self,location=None,frominfo=None,keep=True,logger=None, copier=None): @@ -917,7 +918,7 @@ def deliver(self,location=None,frominfo=None,keep=True,logger=None, raise UnknownLocation( '%s: no location known when delivering product. Specify a ' 'location to deliver().'%(self.did)) - produtil.fileop.deliver_file(frominfo,loc,keep=keep,logger=logger, + fileop.deliver_file(frominfo,loc,keep=keep,logger=logger, copier=copier) if setloc: self.set_loc_avail(loc,True) @@ -966,7 +967,7 @@ def check(self,frominfo=None,minsize=None,minage=None,logger=None): minsize=int(self.get('minsize',0)) if minage is None: minage=int(self.get('minage',20)) - if not produtil.fileop.check_file(loc,min_size=minsize, + if not fileop.check_file(loc,min_size=minsize, min_mtime_age=minage): if self.available: self.available=False @@ -1093,7 +1094,7 @@ def jlogfile(self): intended to receive only major errors, and per-job start and completion information. This is equivalent to simply accessing produtil.log.jlogger.""" - return produtil.log.jlogger + return jlogger def postmsg(self,message,*args,**kwargs): """!same as produtil.log.jlogger.info() @@ -1103,7 +1104,7 @@ def postmsg(self,message,*args,**kwargs): @param message the message @param args positional arguments for string replacement @param kwargs keyword arguments for string replacement.""" - produtil.log.jlogger.info(message,*args,**kwargs) + jlogger.info(message,*args,**kwargs) def setstate(self,val): """!Sets the state of this job. diff --git a/produtil/dbnalert.py b/metplus/produtil/dbnalert.py similarity index 98% rename from produtil/dbnalert.py rename to metplus/produtil/dbnalert.py index de8343146e..62f9830a2a 100644 --- a/produtil/dbnalert.py +++ b/metplus/produtil/dbnalert.py @@ -6,10 +6,9 @@ __all__=["DBNAlert"] import logging, os -import produtil.run -from produtil.prog import Runner -from produtil.run import checkrun, batchexe, alias, run +from metplus.produtil.prog import Runner +from metplus.produtil.run import batchexe, alias, run # Globals: diff --git a/produtil/externals b/metplus/produtil/externals similarity index 100% rename from produtil/externals rename to metplus/produtil/externals diff --git a/produtil/fileop.py b/metplus/produtil/fileop.py similarity index 99% rename from produtil/fileop.py rename to metplus/produtil/fileop.py index 348daa6006..6853f25a95 100644 --- a/produtil/fileop.py +++ b/metplus/produtil/fileop.py @@ -18,7 +18,7 @@ 'netcdfver','touch'] import os,tempfile,filecmp,stat,shutil,errno,random,time,fcntl,math,logging -import produtil.cluster, produtil.pipeline +import metplus.produtil.cluster as cluster module_logger=logging.getLogger('produtil.fileop') @@ -403,9 +403,9 @@ def deliver_file(infile,outfile,keep=True,verify=False,blocksize=1048576, the temp_file_object is an object that can be used to write to the file. The copier should NOT close the temp_file_object. """ if preserve_group is None: - preserve_group = not produtil.cluster.group_quotas() + preserve_group = not cluster.group_quotas() if copy_acl is None: - copy_acl = produtil.cluster.use_acl_for_rstdata() + copy_acl = cluster.use_acl_for_rstdata() if copier is not None: # Cannot simply do a "move" if we are using an external # function to copy. diff --git a/produtil/locking.py b/metplus/produtil/locking.py similarity index 97% rename from produtil/locking.py rename to metplus/produtil/locking.py index 1cf90fd277..0cd4a7e617 100644 --- a/produtil/locking.py +++ b/metplus/produtil/locking.py @@ -14,9 +14,9 @@ ... the file is now unlocked ... @endcode""" -import fcntl, time, errno, os.path -import produtil.retry as retry -import produtil.fileop +import fcntl, errno, os.path +import metplus.produtil.retry as retry +import metplus.produtil.fileop as fileop ##@var __all__ # Symbols exported by "from produtil.locking import *" @@ -110,7 +110,7 @@ def acquire_impl(self): 'the process was exiting.') thedir=os.path.dirname(self._filename) if thedir: - produtil.fileop.makedirs(thedir) + fileop.makedirs(thedir) if self._fd is None: self._fd=open(self._filename,'wb') try: diff --git a/produtil/log.py b/metplus/produtil/log.py similarity index 99% rename from produtil/log.py rename to metplus/produtil/log.py index 329cab79f2..0d6487f382 100644 --- a/produtil/log.py +++ b/metplus/produtil/log.py @@ -12,7 +12,7 @@ 'MasterLogHandler','JLogHandler','set_jlogfile' ] import logging, os, sys, traceback, threading -import produtil.batchsystem +import metplus.produtil.batchsystem as batchsystem ##@var logthread # string for log messages to indicate thread number/name @@ -379,7 +379,7 @@ def configureLogging(jlogfile=None, # Configure log formatting: jobstr=os.environ.get('job',None) if jobstr is None: - jobstr=produtil.batchsystem.jobname() + jobstr=batchsystem.jobname() jobstr=str(jobstr).replace('(','_').replace(')','_').replace('%','_') # Format for jlogfile domain logging to jlogfile: jformat=JLogFormatter( diff --git a/produtil/mpi_impl/__init__.py b/metplus/produtil/mpi_impl/__init__.py similarity index 89% rename from produtil/mpi_impl/__init__.py rename to metplus/produtil/mpi_impl/__init__.py index b986e7d76c..10923ff839 100644 --- a/produtil/mpi_impl/__init__.py +++ b/metplus/produtil/mpi_impl/__init__.py @@ -125,7 +125,8 @@ # __init__.py on how to modify it to achieve these steps. import logging -import produtil.fileop +import metplus.produtil.fileop +import metplus.produtil.prog ##@var __all__ # An empty list that indicates no symbols are exported by "from @@ -220,8 +221,8 @@ def register_implementations(logger=None): # no_implementation=None is used to detect if # register_implementations was called. global no_implementation - import produtil.mpi_impl.no_mpi - no_implementation=produtil.mpi_impl.no_mpi.Implementation.detect() + from metplus.produtil.mpi_impl.no_mpi import Implementation + no_implementation=Implementation.detect() # Now add each implementation. We need to wrap each around a # try...except so that NCEP Central Operations can delete the @@ -230,64 +231,64 @@ def register_implementations(logger=None): try: # If we have srun, and we're in a pack group... - import produtil.mpi_impl.srun_pack_groups - add_implementation(produtil.mpi_impl.srun_pack_groups.Implementation) + import metplus.produtil.mpi_impl.srun_pack_groups + add_implementation(metplus.produtil.mpi_impl.srun_pack_groups.Implementation) except ImportError: pass try: # This must be after the pack group case. # If we have srun and SLURM resources... - import produtil.mpi_impl.srun - add_implementation(produtil.mpi_impl.srun.Implementation) + import metplus.produtil.mpi_impl.srun + add_implementation(metplus.produtil.mpi_impl.srun.Implementation) except ImportError: pass try: - import produtil.mpi_impl.inside_aprun - add_implementation(produtil.mpi_impl.inside_aprun.Implementation) + import metplus.produtil.mpi_impl.inside_aprun + add_implementation(metplus.produtil.mpi_impl.inside_aprun.Implementation) except ImportError: pass try: - import produtil.mpi_impl.pbs_cray_intel - add_implementation(produtil.mpi_impl.pbs_cray_intel.Implementation) + import metplus.produtil.mpi_impl.pbs_cray_intel + add_implementation(metplus.produtil.mpi_impl.pbs_cray_intel.Implementation) except ImportError: pass try: - import produtil.mpi_impl.lsf_cray_intel - add_implementation(produtil.mpi_impl.lsf_cray_intel.Implementation) + import metplus.produtil.mpi_impl.lsf_cray_intel + add_implementation(metplus.produtil.mpi_impl.lsf_cray_intel.Implementation) except ImportError: pass try: - import produtil.mpi_impl.impi - add_implementation(produtil.mpi_impl.impi.Implementation) + import metplus.produtil.mpi_impl.impi + add_implementation(metplus.produtil.mpi_impl.impi.Implementation) except ImportError: pass try: - import produtil.mpi_impl.mpirun_lsf - add_implementation(produtil.mpi_impl.mpirun_lsf.Implementation) + import metplus.produtil.mpi_impl.mpirun_lsf + add_implementation(metplus.produtil.mpi_impl.mpirun_lsf.Implementation) except ImportError: pass try: - import produtil.mpi_impl.mpiexec_mpt - add_implementation(produtil.mpi_impl.mpiexec_mpt.Implementation) + import metplus.produtil.mpi_impl.mpiexec_mpt + add_implementation(metplus.produtil.mpi_impl.mpiexec_mpt.Implementation) except ImportError: pass try: - import produtil.mpi_impl.mpiexec - add_implementation(produtil.mpi_impl.mpiexec.Implementation) + import metplus.produtil.mpi_impl.mpiexec + add_implementation(metplus.produtil.mpi_impl.mpiexec.Implementation) except ImportError: pass try: - import produtil.mpi_impl.srun_shell - add_implementation(produtil.mpi_impl.srun_shell.Implementation) + import metplus.produtil.mpi_impl.srun_shell + add_implementation(metplus.produtil.mpi_impl.srun_shell.Implementation) except ImportError: pass @@ -315,7 +316,7 @@ def get_mpi(mpi_name=NO_NAME,force=False,logger=None,**kwargs): @raise NotImplementedError if the MPI implementation is unknown, or if the implementation is unavailble on this machine, and force=False""" - + import metplus.produtil.mpi_impl.no_mpi as no_mpi if logger is None: logger=logging.getLogger('mpi_impl') @@ -351,8 +352,8 @@ def get_mpi(mpi_name=NO_NAME,force=False,logger=None,**kwargs): result=detect( force=force,logger=logger,**kwargs) except (Exception, - produtil.fileop.FileOpError, - produtil.prog.ExitStatusException): + metplus.produtil.fileop.FileOpError, + metplus.produtil.prog.ExitStatusException): # Ignore exceptions related to an inability to detect the # MPI implementation. We assume the issue has already # been logged, and we move on to the next implementation's diff --git a/produtil/mpi_impl/mpi_impl_base.py b/metplus/produtil/mpi_impl/mpi_impl_base.py similarity index 98% rename from produtil/mpi_impl/mpi_impl_base.py rename to metplus/produtil/mpi_impl/mpi_impl_base.py index 572bfc63db..6f1bb4411b 100644 --- a/produtil/mpi_impl/mpi_impl_base.py +++ b/metplus/produtil/mpi_impl/mpi_impl_base.py @@ -12,9 +12,8 @@ import tempfile,stat,os, logging, io, re -import produtil.prog -import produtil.pipeline -from produtil.prog import shbackslash +from metplus.produtil.pipeline import Pipeline +from metplus.produtil.prog import shbackslash, Runner, ImmutableRunner module_logger=logging.getLogger('produtil.mpi_impl') @@ -139,8 +138,8 @@ def find_mpiserial(self,mpiserial_path,force): def runsync(self,logger=None): """!Runs the "sync" command as an exe().""" if logger is None: logger=self.logger - sync=produtil.prog.Runner(['/bin/sync']) - produtil.pipeline.Pipeline(sync,capture=True,logger=logger) + sync=Runner(['/bin/sync']) + Pipeline(sync,capture=True,logger=logger) def openmp(self,arg,threads): """!Does nothing. This implementation does not support OpenMP. @@ -168,7 +167,7 @@ def make_bigexe(self,exe,**kwargs): @returns an empty list @param exe The executable to run on compute nodes. @param kwargs Ignored.""" - return produtil.prog.ImmutableRunner([str(exe)],**kwargs) + return ImmutableRunner([str(exe)],**kwargs) class CMDFGen(object): diff --git a/produtil/mpi_impl/no_mpi.py b/metplus/produtil/mpi_impl/no_mpi.py similarity index 88% rename from produtil/mpi_impl/no_mpi.py rename to metplus/produtil/mpi_impl/no_mpi.py index 71dfc1bfd9..bb20f9a2b3 100644 --- a/produtil/mpi_impl/no_mpi.py +++ b/metplus/produtil/mpi_impl/no_mpi.py @@ -6,9 +6,8 @@ # produtil.run.mpiserial functions, providing the implementation # needed to run when MPI is unavailable. -import os, logging -import produtil.prog,produtil.pipeline -from .mpi_impl_base import MPIDisabled,ImplementationBase +import logging +from .mpi_impl_base import ImplementationBase module_logger=logging.getLogger('lsf_cray_intel') class Implementation(ImplementationBase): diff --git a/produtil/mpi_impl/srun.py b/metplus/produtil/mpi_impl/srun.py similarity index 90% rename from produtil/mpi_impl/srun.py rename to metplus/produtil/mpi_impl/srun.py index c3574ab1ee..d4518ed0fa 100644 --- a/produtil/mpi_impl/srun.py +++ b/metplus/produtil/mpi_impl/srun.py @@ -6,12 +6,14 @@ # commands. import os, logging, re -import produtil.fileop,produtil.prog,produtil.mpiprog,produtil.pipeline +from metplus.produtil.fileop import find_exe +from metplus.produtil.prog import Runner, ImmutableRunner +import metplus.produtil.mpiprog as mpiprog +import metplus.produtil.pipeline as pipeline from .mpi_impl_base import MPIMixed,CMDFGen,ImplementationBase, \ MPIThreadsMixed,MPILocalOptsMixed,MPITooManyRanks -from produtil.pipeline import NoMoreProcesses -from produtil.mpiprog import MIXED_VALUES +from metplus.produtil.mpiprog import MIXED_VALUES class Implementation(ImplementationBase): """Adds SLURM srun support to produtil.run @@ -37,12 +39,12 @@ def detect(srun_path=None,mpiserial_path=None,logger=None,force=False,silent=Fal if force: srun_path='srun' else: - srun_path=produtil.fileop.find_exe('srun',raise_missing=True) + srun_path=find_exe('srun',raise_missing=True) if scontrol_path is None: if force: scontrol_path='scontrol' else: - scontrol_path=produtil.fileop.find_exe('scontrol',raise_missing=True) + scontrol_path=find_exe('scontrol',raise_missing=True) if 'SLURM_NODELIST' not in os.environ and not force: return None return Implementation(srun_path,scontrol_path,mpiserial_path,logger,silent,force) @@ -58,8 +60,8 @@ def __init__(self,srun_path,scontrol_path,mpiserial_path,logger,silent,force): def runsync(self,logger=None): """!Runs the "sync" command as an exe().""" if logger is None: logger=self.logger - sync=produtil.prog.Runner(['/bin/sync']) - produtil.pipeline.Pipeline(sync,capture=True,logger=logger) + sync=Runner(['/bin/sync']) + pipeline.Pipeline(sync,capture=True,logger=logger) def openmp(self,arg,threads): """!Adds OpenMP support to the provided object @@ -86,7 +88,7 @@ def make_bigexe(self,exe,**kwargs): @returns an empty list @param exe The executable to run on compute nodes. @param kwargs Ignored.""" - return produtil.prog.ImmutableRunner([str(exe)],**kwargs) + return ImmutableRunner([str(exe)],**kwargs) def mpirunner(self,arg,allranks=False,**kwargs): """!Turns a produtil.mpiprog.MPIRanksBase tree into a produtil.prog.Runner @@ -104,10 +106,10 @@ def mpirunner(self,arg,allranks=False,**kwargs): def _get_available_nodes(self): available_nodes=list() nodeset=set() - scontrol=produtil.prog.Runner([ + scontrol=Runner([ self.scontrol_path,'show','hostnames', os.environ['SLURM_NODELIST']]) - p=produtil.pipeline.Pipeline( + p=pipeline.Pipeline( scontrol,capture=True,logger=self.logger) nodelist=p.to_string() for line in nodelist.splitlines(): @@ -121,7 +123,7 @@ def _get_available_nodes(self): def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False,**kwargs): """!This is the underlying implementation of mpirunner and should not be called directly.""" - assert(isinstance(arg,produtil.mpiprog.MPIRanksBase)) + assert(isinstance(arg,mpiprog.MPIRanksBase)) (serial,parallel)=arg.check_serial() if serial and parallel: raise MPIMixed('Cannot mix serial and parallel MPI ranks in the ' @@ -143,7 +145,7 @@ def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False, srun_args.append('--distribution=block:block') arglist=[ str(a) for a in arg.to_arglist( pre=srun_args,before=[],between=[])] - return produtil.prog.Runner(arglist) + return Runner(arglist) elif allranks: raise MPIAllRanksError( "When using allranks=True, you must provide an mpi program " @@ -151,9 +153,9 @@ def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False, "all ranks).") elif serial: srun_args.append('--distribution=block:block') - arg=produtil.mpiprog.collapse(arg) + arg=mpiprog.collapse(arg) lines=[str(a) for a in arg.to_arglist(to_shell=True,expand=True)] - return produtil.prog.Runner( + return Runner( [self.srun_path,'--ntasks','%s'%(arg.nranks()),self.mpiserial_path], prerun=CMDFGen('serialcmdf',lines,silent=self.silent,**kwargs)) else: @@ -204,4 +206,4 @@ def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False, silent=self.silent,filename_option='--nodelist', next_prerun=prerun,**kwargs) - return produtil.prog.Runner(srun_args,prerun=prerun) + return Runner(srun_args,prerun=prerun) diff --git a/produtil/mpiprog.py b/metplus/produtil/mpiprog.py similarity index 99% rename from produtil/mpiprog.py rename to metplus/produtil/mpiprog.py index e7adf6e1f4..ad7609f445 100644 --- a/produtil/mpiprog.py +++ b/metplus/produtil/mpiprog.py @@ -44,12 +44,9 @@ # Ensure nothing is loaded by "from produtil.mpiprog import *" __all__=[] -import sys - import io import logging -import produtil.prog -from produtil.prog import ProgSyntaxError, shbackslash +from metplus.produtil.prog import ProgSyntaxError, shbackslash, Runner, ImmutableRunner class MPIProgSyntaxError(ProgSyntaxError): """!Base class of syntax errors in MPI program specifications""" @@ -831,7 +828,7 @@ def __init__(self,arg,logger=None): self._localopts=list(arg._localopts) self._turbomode=arg.turbomode self._ranks_per_node=arg.ranks_per_node - elif isinstance(arg,produtil.prog.Runner): + elif isinstance(arg,Runner): if arg.isplainexe(): self._args=[x for x in arg.args()] else: @@ -871,7 +868,7 @@ def delthreads(self): def to_shell(self): """!Return a POSIX sh representation of this MPI rank, if possible.""" - return ' '.join([produtil.prog.shbackslash(x) for x in self._args]) + return ' '.join([shbackslash(x) for x in self._args]) def __getitem__(self,args): """!Adds arguments to this MPI rank's program.""" c=self.copy() @@ -995,8 +992,8 @@ def __init__(self,runner,logger=None): self._ranks_per_node=0 def make_runners_immutable(self): """!Creates a version of self with a produtil.prog.ImmutableRunner child.""" - if not isinstance(self._runner,produtil.prog.ImmutableRunner): - return MPISerial(produtil.prog.ImmutableRunner(self._runner),self._logger) + if not isinstance(self._runner,ImmutableRunner): + return MPISerial(ImmutableRunner(self._runner),self._logger) else: return self def copy(self): diff --git a/produtil/numerics.py b/metplus/produtil/numerics.py similarity index 100% rename from produtil/numerics.py rename to metplus/produtil/numerics.py diff --git a/produtil/pipeline.py b/metplus/produtil/pipeline.py similarity index 100% rename from produtil/pipeline.py rename to metplus/produtil/pipeline.py diff --git a/produtil/prog.py b/metplus/produtil/prog.py similarity index 99% rename from produtil/prog.py rename to metplus/produtil/prog.py index f6f2c6fdfc..441cc6bd35 100644 --- a/produtil/prog.py +++ b/metplus/produtil/prog.py @@ -37,11 +37,9 @@ directly, except for type checking (ie.: to see if your argument is a Runner before passing it to produtil.run.checkrun)..""" -import produtil.sigsafety -import io,select,io,re,time,fcntl,os,logging,signal +import re, os -import produtil.mpi_impl -from produtil.pipeline import launch, manage, PIPE, ERR2OUT +from metplus.produtil.pipeline import ERR2OUT ERR2OUT_FCT_STR = '.err2out()' diff --git a/produtil/retry.py b/metplus/produtil/retry.py similarity index 100% rename from produtil/retry.py rename to metplus/produtil/retry.py diff --git a/produtil/run.py b/metplus/produtil/run.py similarity index 97% rename from produtil/run.py rename to metplus/produtil/run.py index a7e469b890..bdd305ea29 100644 --- a/produtil/run.py +++ b/metplus/produtil/run.py @@ -155,15 +155,14 @@ """ import time, logging -import produtil.mpi_impl -import produtil.sigsafety -import produtil.prog as prog -import produtil.mpiprog as mpiprog -import produtil.pipeline as pipeline +import metplus.produtil.mpi_impl as mpi_impl +import metplus.produtil.prog as prog +import metplus.produtil.mpiprog as mpiprog +import metplus.produtil.pipeline as pipeline # These two were moved to produtil.prog to avoid a cyclic import. # They still need to be available from produtil.run: -from produtil.prog import InvalidRunArgument,ExitStatusException +from metplus.produtil.prog import InvalidRunArgument,ExitStatusException ##@var __all__ # List of symbols exported by "from produtil.run import *" @@ -180,7 +179,7 @@ # The cached return value from detect_mpi() _detected_mpi=None -def make_mpi(mpi_name=produtil.mpi_impl.NO_NAME,**kwargs): +def make_mpi(mpi_name=mpi_impl.NO_NAME,**kwargs): """!Creates an MPI implementation object for the specified MPI implementation. @@ -208,7 +207,7 @@ def make_mpi(mpi_name=produtil.mpi_impl.NO_NAME,**kwargs): # situation does not need any initialization of the produtil.run # module: if mpi_name is None: - return produtil.mpi_impl.get_mpi(None,**kwargs) + return mpi_impl.get_mpi(None,**kwargs) # For anything other than "None," we have to ensure the # produtil.run is initialized to something before running @@ -216,7 +215,7 @@ def make_mpi(mpi_name=produtil.mpi_impl.NO_NAME,**kwargs): detect_mpi() # Next, return the requested implementation: - return produtil.mpi_impl.get_mpi(mpi_name,**kwargs) + return mpi_impl.get_mpi(mpi_name,**kwargs) def detect_mpi(): """!Called by functions inside produtil.run to automatically @@ -253,10 +252,10 @@ def detect_mpi(): # First, set the implementation to None so that the mpi_impl # subclasses can use produtil.run. This will only allow serial, # non-OpenMP programs: - _detected_mpi=produtil.mpi_impl.get_mpi(None) + _detected_mpi=mpi_impl.get_mpi(None) - # Next, ask produtil.mpi_impl to detect the MPI implementation: - detected=produtil.mpi_impl.get_mpi() + # Next, ask mpi_impl to detect the MPI implementation: + detected=mpi_impl.get_mpi() # If detection succeeds, override the selected MPI implementation: if detected: diff --git a/produtil/setup.py b/metplus/produtil/setup.py similarity index 85% rename from produtil/setup.py rename to metplus/produtil/setup.py index 9a58e6a8b0..b97bdc8840 100644 --- a/produtil/setup.py +++ b/metplus/produtil/setup.py @@ -7,9 +7,12 @@ # Lists symbols exported by "from produtil.setup import *" __all__=['setup'] -import logging, threading -import produtil.sigsafety, produtil.log, produtil.dbnalert, produtil.cluster -import produtil.batchsystem +import threading +from metplus.produtil.sigsafety import install_handlers +from metplus.produtil.log import configureLogging +from metplus.produtil.dbnalert import init_module +from metplus.produtil.cluster import set_cluster, where +from metplus.produtil.batchsystem import set_default_name def setup(ignore_hup=False,dbnalert_logger=None,jobname=None,cluster=None, send_dbn=None,thread_logger=False,thread_stack=2**24,**kwargs): @@ -61,19 +64,19 @@ def setup(ignore_hup=False,dbnalert_logger=None,jobname=None,cluster=None, # Set the default jobname. This is usually used for manually-run # scripts to ensure they have a "jobname" in the logging system: if jobname is not None: - produtil.batchsystem.set_default_name(jobname) + set_default_name(jobname) # Configure logging next so that the install_handlers will be able # to log. - produtil.log.configureLogging(thread_logger=thread_logger,**kwargs) + configureLogging(thread_logger=thread_logger,**kwargs) # Install signal handlers, and let the caller configure SIGHUP settings: - produtil.sigsafety.install_handlers(ignore_hup=ignore_hup) + install_handlers(ignore_hup=ignore_hup) # Set up dbnalert: - produtil.dbnalert.init_module(logger=dbnalert_logger,jobname=jobname, + init_module(logger=dbnalert_logger,jobname=jobname, send_dbn=send_dbn) # Set up cluster: if cluster is not None: - produtil.cluster.set_cluster(cluster) + set_cluster(cluster) else: - produtil.cluster.where() # guess cluster + where() # guess cluster diff --git a/produtil/sigsafety.py b/metplus/produtil/sigsafety.py similarity index 97% rename from produtil/sigsafety.py rename to metplus/produtil/sigsafety.py index 301e488031..23699d766d 100644 --- a/produtil/sigsafety.py +++ b/metplus/produtil/sigsafety.py @@ -40,7 +40,8 @@ One can call install_handlers directly, though it is recommended to call produtil.setup.setup instead.""" -import produtil.locking, produtil.pipeline +from metplus.produtil.locking import disable_locking +from metplus.produtil.pipeline import kill_all import signal ##@var defaultsigs @@ -135,7 +136,7 @@ def hup_handler(signum,frame): caught_signal=signum caught_class=HangupSignal - produtil.locking.disable_locking() + disable_locking() raise HangupSignal(signum) def term_handler(signum,frame): @@ -145,8 +146,8 @@ def term_handler(signum,frame): caught_signal=signum caught_class=FatalSignal - produtil.locking.disable_locking() # forbid file locks - produtil.pipeline.kill_all() # kill all subprocesses + disable_locking() # forbid file locks + kill_all() # kill all subprocesses uninstall_handlers() raise FatalSignal(signum) diff --git a/metplus/scripts/run_metplus.py b/metplus/scripts/run_metplus.py new file mode 120000 index 0000000000..116d56b438 --- /dev/null +++ b/metplus/scripts/run_metplus.py @@ -0,0 +1 @@ +../../ush/run_metplus.py \ No newline at end of file diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index c0d25deea6..238b2c10b5 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -11,7 +11,6 @@ import os import re -import sys import logging from datetime import datetime, timezone import time @@ -20,7 +19,7 @@ from pathlib import Path import uuid -from produtil.config import ProdConfig +from metplus.produtil.config import ProdConfig from .constants import RUNTIME_CONFS, MISSING_DATA_VALUE from .string_template_substitution import do_string_sub @@ -63,7 +62,7 @@ # set parm base to METPLUS_BASE/parm unless METPLUS_PARM_BASE env var is set PARM_BASE = os.environ.get('METPLUS_PARM_BASE', - os.path.join(METPLUS_BASE, PARM_DIR)) + os.path.join(METPLUS_BASE, 'metplus', PARM_DIR)) # name of directory under PARM_DIR that contains defaults METPLUS_CONFIG_DIR = 'metplus_config' diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index 145468c5a6..90ace07686 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -6,7 +6,7 @@ from logging import Logger import shlex -from produtil.run import exe, run +from metplus.produtil.run import exe, run from .string_manip import get_logfile_info, log_terminal_includes_info from .system_util import get_user_info, write_list_to_file diff --git a/pyproject.toml b/pyproject.toml index b5a3c468d9..30e5629b73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,47 @@ [build-system] requires = [ - "setuptools>=42", - "wheel" + "setuptools", + "wheel", + "setuptools-git-versioning>=2.0,<3", ] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" + +[project] +name = "metplus" +dynamic = ["version", "dependencies"] +description = "METplus Wrappers" +authors = [ + {name = "METplus"}, +] +requires-python = ">=3.10.4" +readme = "README.md" +license = {text = "MIT"} +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://github.com/dtcenter/METplus" + +[project.scripts] +run_metplus = "metplus.scripts.run_metplus:cli_main" + +[tool.setuptools.packages] +find = {include = ["metplus*"], exclude = ["parm"]} + +[tool.setuptools.package-data] +metplus = [ + "VERSION", "RELEASE_DATE", "PYTHON_VERSION", "PYTHON_VERSION_MIN", "parm/**" +] + +[tool.setuptools.exclude-package-data] +metplus = ["README", "__pycache__", "*~"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} + +[tool.setuptools-git-versioning] +enabled = true +version_file = "metplus/VERSION" diff --git a/requirements.txt b/requirements.txt index 169d9d94cb..2c05e903ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -certifi==2024.7.4 -python-dateutil==2.8.2 -six==1.16.0 +certifi>=2024.7.4 +python-dateutil>=2.8.2 +six>=1.16.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 2114a77efe..0000000000 --- a/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -from setuptools import setup, find_packages -from distutils.util import convert_path -import os - -with open("README.md", "r") as fh: - long_description = fh.read() - -with open("metplus/VERSION", "r") as fh: - version = fh.read().strip() - -with open("metplus/PYTHON_VERSION_MIN", "r") as fh: - python_version_req = fh.read().strip() - -# get list of additional files needed to add to package -data_files = [] -# add version and release date files -data_files.append(('metplus', - ['metplus/VERSION', - 'metplus/RELEASE_DATE', - 'metplus/PYTHON_VERSION', - 'metplus/PYTHON_VERSION_MIN', - ])) - -for root, _, files in os.walk('parm'): - parm_files = [] - for filename in files: - filepath = os.path.join(root, filename) - # skip README, tilda, and pycache files - if ('__pycache__' in filepath or - 'README' in filepath or - filepath.endswith('~')): - continue - parm_files.append(filepath) - if parm_files: - data_files.append((root, parm_files)) - -setup( - name="metplus", - version=version, - author="METplus", - description="METplus Wrappers", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/dtcenter/METplus", - packages=find_packages(), - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], - python_requires=f'>={python_version_req}', - data_files=data_files, - zip_safe=False, -) diff --git a/ush/run_metplus.py b/ush/run_metplus.py index eb374b4427..6f329867ac 100755 --- a/ush/run_metplus.py +++ b/ush/run_metplus.py @@ -25,7 +25,7 @@ # add metplus directory to path so the wrappers and utilities can be found sys.path.insert(0, abspath(join(dirname(realpath(__file__)), pardir))) -import produtil.setup +from metplus.produtil.setup import setup as produtil_setup from metplus.util import pre_run_setup, run_metplus, post_run_cleanup from metplus import __version__ as metplus_version @@ -110,12 +110,15 @@ def get_config_inputs_from_command_line(): return config_inputs -if __name__ == "__main__": +def cli_main(): try: - produtil.setup.setup(send_dbn=False, jobname='run-METplus') + produtil_setup(send_dbn=False, jobname='run-METplus') if not main(): sys.exit(1) except Exception as exc: print(traceback.format_exc()) print('ERROR: run_metplus failed: %s' % exc) sys.exit(2) + +if __name__ == "__main__": + cli_main()