From 843944539672f2756b3ff442ea5d134e003b922c Mon Sep 17 00:00:00 2001 From: Benjamin Auer Date: Fri, 5 Aug 2022 16:06:34 -0400 Subject: [PATCH 01/13] start of fixing #1620 --- pfio/HistoryCollection.F90 | 42 +++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/pfio/HistoryCollection.F90 b/pfio/HistoryCollection.F90 index 7822ea810534..b15b33b4658a 100644 --- a/pfio/HistoryCollection.F90 +++ b/pfio/HistoryCollection.F90 @@ -9,6 +9,8 @@ module pFIO_HistoryCollectionMod use pFIO_FileMetadataMod use pFIO_StringVariableMapMod use pFIO_ConstantsMod + use gFTL_StringVector + use NetCDF implicit none private @@ -17,12 +19,14 @@ module pFIO_HistoryCollectionMod type :: HistoryCollection type (Filemetadata) :: fmd + type (StringVector) :: files_created type (StringNetCDF4_FileFormatterMap) :: formatters contains procedure :: find procedure :: ModifyMetadata procedure :: clear + procedure :: check_if_i_created end type HistoryCollection interface HistoryCollection @@ -51,18 +55,28 @@ function find(this, file_name,rc) result(formatter) type(StringNetCDF4_FileFormatterMapIterator) :: iter integer :: status character(len=*), parameter :: Iam = "HistoryCollection::find()" - logical :: f_exist + logical :: f_exist, i_created iter = this%formatters%find(trim(file_name)) if (iter == this%formatters%end()) then inquire(file=file_name, exist=f_exist) - if(.not. f_exist) then + if(.not. f_exist) then call fm%create(trim(file_name),rc=status) _VERIFY(status) call fm%write(this%fmd, rc=status) _VERIFY(status) + call this%files_created%push_back(file_name) else - call fm%open(trim(file_name), pFIO_WRITE) + i_created = this%check_if_i_created(file_name) + if (i_created) then + call fm%open(trim(file_name), pFIO_WRITE) + else + call fm%create(trim(file_name),mode=NF90_CLOBBER,rc=status) + _VERIFY(status) + call fm%write(this%fmd, rc=status) + _VERIFY(status) + call this%files_created%push_back(file_name) + end if endif call this%formatters%insert( trim(file_name),fm) iter = this%formatters%find(trim(file_name)) @@ -112,6 +126,28 @@ subroutine clear(this, rc) _RETURN(_SUCCESS) end subroutine clear + function check_if_i_created(this,input_file,rc) result(i_created) + logical :: i_created + class (HistoryCollection), intent(inout) :: this + character(len=*), intent(in) :: input_file + integer, optional, intent(out) :: rc + + integer :: status + character(len=:), pointer :: file_name + type(StringVectorIterator) :: iter + + i_created = .false. + iter = this%files_created%begin() + do while (iter /= this%files_created%end()) + file_name => iter%get() + if (file_name == input_file) i_created = .true. + call iter%next() + enddo + + _RETURN(_SUCCESS) + + end function + end module pFIO_HistoryCollectionMod From 97952c38f39146b4270992a27db61427b6a07735 Mon Sep 17 00:00:00 2001 From: Benjamin Auer Date: Mon, 8 Aug 2022 10:03:22 -0400 Subject: [PATCH 02/13] don't need to store a list, just one --- pfio/HistoryCollection.F90 | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pfio/HistoryCollection.F90 b/pfio/HistoryCollection.F90 index b15b33b4658a..d439b39dc494 100644 --- a/pfio/HistoryCollection.F90 +++ b/pfio/HistoryCollection.F90 @@ -19,7 +19,7 @@ module pFIO_HistoryCollectionMod type :: HistoryCollection type (Filemetadata) :: fmd - type (StringVector) :: files_created + character(len=:), allocatable :: file_created type (StringNetCDF4_FileFormatterMap) :: formatters contains @@ -65,7 +65,7 @@ function find(this, file_name,rc) result(formatter) _VERIFY(status) call fm%write(this%fmd, rc=status) _VERIFY(status) - call this%files_created%push_back(file_name) + this%file_created = trim(file_name) else i_created = this%check_if_i_created(file_name) if (i_created) then @@ -75,7 +75,7 @@ function find(this, file_name,rc) result(formatter) _VERIFY(status) call fm%write(this%fmd, rc=status) _VERIFY(status) - call this%files_created%push_back(file_name) + this%file_created=trim(file_name) end if endif call this%formatters%insert( trim(file_name),fm) @@ -133,16 +133,11 @@ function check_if_i_created(this,input_file,rc) result(i_created) integer, optional, intent(out) :: rc integer :: status - character(len=:), pointer :: file_name - type(StringVectorIterator) :: iter i_created = .false. - iter = this%files_created%begin() - do while (iter /= this%files_created%end()) - file_name => iter%get() - if (file_name == input_file) i_created = .true. - call iter%next() - enddo + if (allocated(this%file_created)) then + if (input_file == this%file_created) i_created=.true. + end if _RETURN(_SUCCESS) From ff80042816aabccc788470241a52b49d3d57fbd7 Mon Sep 17 00:00:00 2001 From: Benjamin Auer Date: Mon, 8 Aug 2022 10:15:59 -0400 Subject: [PATCH 03/13] choose beter variable name --- pfio/HistoryCollection.F90 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pfio/HistoryCollection.F90 b/pfio/HistoryCollection.F90 index d439b39dc494..47bf3af2bea1 100644 --- a/pfio/HistoryCollection.F90 +++ b/pfio/HistoryCollection.F90 @@ -19,7 +19,7 @@ module pFIO_HistoryCollectionMod type :: HistoryCollection type (Filemetadata) :: fmd - character(len=:), allocatable :: file_created + character(len=:), allocatable :: last_file_created type (StringNetCDF4_FileFormatterMap) :: formatters contains @@ -65,7 +65,7 @@ function find(this, file_name,rc) result(formatter) _VERIFY(status) call fm%write(this%fmd, rc=status) _VERIFY(status) - this%file_created = trim(file_name) + this%last_file_created = trim(file_name) else i_created = this%check_if_i_created(file_name) if (i_created) then @@ -75,7 +75,7 @@ function find(this, file_name,rc) result(formatter) _VERIFY(status) call fm%write(this%fmd, rc=status) _VERIFY(status) - this%file_created=trim(file_name) + this%last_file_created=trim(file_name) end if endif call this%formatters%insert( trim(file_name),fm) @@ -135,8 +135,8 @@ function check_if_i_created(this,input_file,rc) result(i_created) integer :: status i_created = .false. - if (allocated(this%file_created)) then - if (input_file == this%file_created) i_created=.true. + if (allocated(this%last_file_created)) then + if (input_file == this%last_file_created) i_created=.true. end if _RETURN(_SUCCESS) From 802c5d8d05bfac367b6f842cae2d5ec936f698b6 Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Mon, 8 Aug 2022 11:26:25 -0400 Subject: [PATCH 04/13] overload client message for NOCLOBBER --- pfio/AddHistCollectionMessage.F90 | 4 +++- pfio/HistoryCollection.F90 | 13 ++++++++++--- pfio/ServerThread.F90 | 6 +++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/pfio/AddHistCollectionMessage.F90 b/pfio/AddHistCollectionMessage.F90 index 1511958c2862..9895ca028fe5 100644 --- a/pfio/AddHistCollectionMessage.F90 +++ b/pfio/AddHistCollectionMessage.F90 @@ -13,8 +13,10 @@ module pFIO_AddHistCollectionMessageMod type, extends(AbstractMessage) :: AddHistCollectionMessage type(FileMetadata) :: fmd - ! WY node: -1 : add ( default ) + ! WY node: -1 : add (clobber, default ) ! other : replace + ! overload collection_id of the message + ! -2 : add and no_clobber for these series of files integer :: collection_id = -1 contains procedure, nopass :: get_type_id diff --git a/pfio/HistoryCollection.F90 b/pfio/HistoryCollection.F90 index 47bf3af2bea1..13ab72b32da6 100644 --- a/pfio/HistoryCollection.F90 +++ b/pfio/HistoryCollection.F90 @@ -21,7 +21,7 @@ module pFIO_HistoryCollectionMod type (Filemetadata) :: fmd character(len=:), allocatable :: last_file_created type (StringNetCDF4_FileFormatterMap) :: formatters - + integer :: create_mode contains procedure :: find procedure :: ModifyMetadata @@ -35,12 +35,19 @@ module pFIO_HistoryCollectionMod contains - function new_HistoryCollection(fmd) result(collection) + function new_HistoryCollection(fmd, option) result(collection) type (HistoryCollection) :: collection type (FilemetaData), intent(in) :: fmd + integer, optional, intent(in) :: option collection%fmd = fmd collection%formatters = StringNetCDF4_FileFormatterMap() + collection%create_mode = NF90_CLOBBER + if (present(option)) then + if (option == -2) then + collection%ceate_mode = NF90_NOCLOBBER + endif + endif end function new_HistoryCollection @@ -71,7 +78,7 @@ function find(this, file_name,rc) result(formatter) if (i_created) then call fm%open(trim(file_name), pFIO_WRITE) else - call fm%create(trim(file_name),mode=NF90_CLOBBER,rc=status) + call fm%create(trim(file_name),mode=this%create_mode,rc=status) _VERIFY(status) call fm%write(this%fmd, rc=status) _VERIFY(status) diff --git a/pfio/ServerThread.F90 b/pfio/ServerThread.F90 index ca33c9f8a877..ecf044930e96 100644 --- a/pfio/ServerThread.F90 +++ b/pfio/ServerThread.F90 @@ -531,14 +531,14 @@ subroutine handle_AddHistCollection(this, message, rc) class(AbstractSocket),pointer :: connection if (associated(ioserver_profiler)) call ioserver_profiler%start("add_Histcollection") - if ( message%collection_id == -1 ) then + if ( message%collection_id == -1 .or. message%collection_id == -2 ) then n = this%hist_collections%size()+1 else n = message%collection_id endif - hist_collection = HistoryCollection(message%fmd) - if ( message%collection_id == -1) then + hist_collection = HistoryCollection(message%fmd, message%collection_id) + if ( message%collection_id == -1 .or. message%collection_id == -2 ) then call this%hist_collections%push_back(hist_collection) else call this%hist_collections%set(n,hist_collection) From caf03d7e59e5b34868f268f8da9770f1cc4269ac Mon Sep 17 00:00:00 2001 From: Benjamin Auer Date: Tue, 9 Aug 2022 09:58:01 -0400 Subject: [PATCH 05/13] fixed typo --- pfio/HistoryCollection.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pfio/HistoryCollection.F90 b/pfio/HistoryCollection.F90 index 13ab72b32da6..0225f133af86 100644 --- a/pfio/HistoryCollection.F90 +++ b/pfio/HistoryCollection.F90 @@ -45,7 +45,7 @@ function new_HistoryCollection(fmd, option) result(collection) collection%create_mode = NF90_CLOBBER if (present(option)) then if (option == -2) then - collection%ceate_mode = NF90_NOCLOBBER + collection%create_mode = NF90_NOCLOBBER endif endif From 9974f363f63e83ee0f3b6ccd46b90dd84ab3cfde Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Tue, 9 Aug 2022 14:37:20 -0400 Subject: [PATCH 06/13] pass mode to server --- pfio/AddHistCollectionMessage.F90 | 6 +++--- pfio/ClientManager.F90 | 5 +++-- pfio/ClientThread.F90 | 6 ++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pfio/AddHistCollectionMessage.F90 b/pfio/AddHistCollectionMessage.F90 index 9895ca028fe5..4ce8e17e0a74 100644 --- a/pfio/AddHistCollectionMessage.F90 +++ b/pfio/AddHistCollectionMessage.F90 @@ -31,12 +31,12 @@ module pFIO_AddHistCollectionMessageMod contains - function new_AddHistCollectionMessage(fmd, collection_id) result(message) + function new_AddHistCollectionMessage(fmd, mode) result(message) type (AddHistCollectionMessage) :: message type(FileMetadata), intent(in) :: fmd - integer, optional, intent(in) :: collection_id + integer, optional, intent(in) :: mode message%fmd = fmd - if( present(collection_id)) message%collection_id = collection_id + if( present(mode)) message%collection_id = mode end function new_AddHistCollectionMessage diff --git a/pfio/ClientManager.F90 b/pfio/ClientManager.F90 index 6439a5ed91d2..6d2e70788018 100644 --- a/pfio/ClientManager.F90 +++ b/pfio/ClientManager.F90 @@ -130,18 +130,19 @@ function add_ext_collection(this, template, unusable, rc) result(collection_id) _UNUSED_DUMMY(unusable) end function add_ext_collection - function add_hist_collection(this, fmd, unusable, rc) result(hist_collection_id) + function add_hist_collection(this, fmd, unusable,mode, rc) result(hist_collection_id) integer :: hist_collection_id class (ClientManager), intent(inout) :: this type(FileMetadata),intent(in) :: fmd class (KeywordEnforcer), optional, intent(out) :: unusable + integer, optional, intent(in) :: mode integer, optional, intent(out) :: rc class (ClientThread), pointer :: clientPtr integer :: i do i = 1, this%size() ClientPtr => this%clients%at(i) - hist_collection_id = clientPtr%add_hist_collection(fmd) + hist_collection_id = clientPtr%add_hist_collection(fmd, mode=mode) enddo _RETURN(_SUCCESS) diff --git a/pfio/ClientThread.F90 b/pfio/ClientThread.F90 index 50adf0fff852..089015884d3e 100644 --- a/pfio/ClientThread.F90 +++ b/pfio/ClientThread.F90 @@ -128,10 +128,12 @@ function add_ext_collection(this, template, rc) result(collection_id) _RETURN(_SUCCESS) end function add_ext_collection - function add_hist_collection(this, fmd, rc) result(hist_collection_id) + function add_hist_collection(this, fmd, unusable, mode, rc) result(hist_collection_id) integer :: hist_collection_id class (ClientThread), intent(inout) :: this type(FileMetadata),intent(in) :: fmd + class (KeywordEnforcer), optional, intent(out) :: unusable + integer, optional, intent(in) :: mode integer, optional, intent(out) :: rc class (AbstractMessage), pointer :: message @@ -139,7 +141,7 @@ function add_hist_collection(this, fmd, rc) result(hist_collection_id) integer :: status connection=>this%get_connection() - call connection%send(AddHistCollectionMessage(fmd)) + call connection%send(AddHistCollectionMessage(fmd, mode=mode)) message => connection%receive() select type(message) From 3549a2a84c66dad54a9ebf152c34047fc2dbc1fd Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 10 Aug 2022 12:43:19 -0400 Subject: [PATCH 07/13] pass create_mode to pfio server --- gridcomps/History/MAPL_HistoryGridComp.F90 | 16 ++++++------- pfio/AddHistCollectionMessage.F90 | 16 ++++++------- pfio/ClientManager.F90 | 20 ---------------- pfio/ClientThread.F90 | 28 ---------------------- pfio/HistoryCollection.F90 | 10 +++----- pfio/ServerThread.F90 | 14 +++-------- 6 files changed, 21 insertions(+), 83 deletions(-) diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index c5b68118db15..1e04bd84aeba 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -45,6 +45,7 @@ module MAPL_HistoryGridCompMod use regex_module use MAPL_TimeUtilsMod, only: is_valid_time, is_valid_date use gFTL_StringStringMap + use netcdf !use ESMF_CFIOMOD implicit none @@ -116,7 +117,6 @@ module MAPL_HistoryGridCompMod logical :: integer_time integer :: collectionWriteSplit integer :: serverSizeSplit - logical :: allow_overwrite end type HISTORY_STATE type HISTORY_wrap @@ -428,6 +428,8 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) type(StringStringMap) :: global_attributes character(len=ESMF_MAXSTR) :: name,regrid_method logical :: has_conservative_keyword, has_regrid_keyword + logical :: allow_overwrite + integer :: create_mode ! Begin !------ @@ -538,9 +540,11 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ConfigGetAttribute(config, value=cFileOrder, & label='FileOrder:', default='ABC', rc=status) _VERIFY(STATUS) - call ESMF_ConfigGetAttribute(config, value=intState%allow_overwrite, & + call ESMF_ConfigGetAttribute(config, value=allow_overwrite, & label='Allow_Overwrite:', default=.false., _RC) - + create_mode = NF90_NOCLOBBER ! defaut no overwrite + if (allow_overwrite) create_mode = NF90_CLOBBER + if (trim(cFileOrder) == 'ABC') then intstate%fileOrderAlphabetical = .true. else if (trim(cFileOrder) == 'AddOrder') then @@ -2532,7 +2536,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call list(n)%mGriddedIO%CreateFileMetaData(list(n)%items,list(n)%bundle,list(n)%timeInfo,vdata=list(n)%vdata,global_attributes=global_attributes,rc=status) _VERIFY(status) end if - collection_id = o_Clients%add_hist_collection(list(n)%mGriddedIO%metadata) + collection_id = o_Clients%add_hist_collection(list(n)%mGriddedIO%metadata, mode = create_mode) call list(n)%mGriddedIO%set_param(write_collection_id=collection_id) end if end if @@ -3630,10 +3634,6 @@ subroutine Run ( gc, import, export, clock, rc ) else if( list(n)%unit.eq.0 ) then if (list(n)%format == 'CFIO') then - if (.not.intState%allow_overwrite) then - inquire (file=trim(filename(n)),exist=file_exists) - _ASSERT(.not.file_exists,trim(filename(n))//" being created for History output already exists") - end if call list(n)%mGriddedIO%modifyTime(oClients=o_Clients,rc=status) _VERIFY(status) list(n)%currentFile = filename(n) diff --git a/pfio/AddHistCollectionMessage.F90 b/pfio/AddHistCollectionMessage.F90 index 4ce8e17e0a74..6ea7d07f391f 100644 --- a/pfio/AddHistCollectionMessage.F90 +++ b/pfio/AddHistCollectionMessage.F90 @@ -6,6 +6,7 @@ module pFIO_AddHistCollectionMessageMod use pFIO_UtilitiesMod use pFIO_AbstractMessageMod use pFIO_FileMetadataMod + use netcdf implicit none private @@ -13,11 +14,7 @@ module pFIO_AddHistCollectionMessageMod type, extends(AbstractMessage) :: AddHistCollectionMessage type(FileMetadata) :: fmd - ! WY node: -1 : add (clobber, default ) - ! other : replace - ! overload collection_id of the message - ! -2 : add and no_clobber for these series of files - integer :: collection_id = -1 + integer :: create_mode contains procedure, nopass :: get_type_id procedure :: get_length @@ -36,7 +33,8 @@ function new_AddHistCollectionMessage(fmd, mode) result(message) type(FileMetadata), intent(in) :: fmd integer, optional, intent(in) :: mode message%fmd = fmd - if( present(mode)) message%collection_id = mode + message%create_mode = NF90_NOCLOBBER + if( present(mode)) message%create_mode = mode end function new_AddHistCollectionMessage @@ -49,7 +47,7 @@ integer function get_length(this) result(length) class (AddHistCollectionMessage), intent(in) :: this integer,allocatable :: buffer(:) ! no-op call this%fmd%serialize(buffer) - length = size(buffer) + 1 ! 1 is the collection_id + length = size(buffer) + 1 ! 1 is the create_mode end function get_length @@ -62,7 +60,7 @@ subroutine serialize(this, buffer, rc) integer :: status call this%fmd%serialize(tmp_buffer, status) _VERIFY(status) - buffer = [tmp_buffer,serialize_intrinsic(this%collection_id)] + buffer = [tmp_buffer,serialize_intrinsic(this%create_mode)] _RETURN(_SUCCESS) end subroutine serialize @@ -78,7 +76,7 @@ subroutine deserialize(this, buffer,rc) _VERIFY(status) call deserialize_intrinsic(buffer(n:), length) n = n + length - call deserialize_intrinsic(buffer(n:), this%collection_id) + call deserialize_intrinsic(buffer(n:), this%create_mode) _RETURN(_SUCCESS) end subroutine deserialize diff --git a/pfio/ClientManager.F90 b/pfio/ClientManager.F90 index 6d2e70788018..fc19ebadb6b6 100644 --- a/pfio/ClientManager.F90 +++ b/pfio/ClientManager.F90 @@ -38,7 +38,6 @@ module pFIO_ClientManagerMod contains procedure :: add_ext_collection procedure :: add_hist_collection - procedure :: replace_hist_collection procedure :: modify_metadata procedure :: modify_metadata_all procedure :: prefetch_data @@ -149,25 +148,6 @@ function add_hist_collection(this, fmd, unusable,mode, rc) result(hist_collectio _UNUSED_DUMMY(unusable) end function add_hist_collection - subroutine replace_hist_collection(this, hist_collection_id, fmd, unusable, rc) - class (ClientManager), intent(inout) :: this - integer, intent(in) :: hist_collection_id - type(FileMetadata),intent(in) :: fmd - class (KeywordEnforcer), optional, intent(out) :: unusable - integer, optional, intent(out) :: rc - - class (ClientThread), pointer :: clientPtr - integer :: i - - do i = 1, this%size() - ClientPtr => this%clients%at(i) - call clientPtr%replace_hist_collection(hist_collection_id, fmd) - enddo - - _RETURN(_SUCCESS) - _UNUSED_DUMMY(unusable) - end subroutine replace_hist_collection - subroutine prefetch_data(this, collection_id, file_name, var_name, data_reference, & & unusable, start, rc) class (ClientManager), intent(inout) :: this diff --git a/pfio/ClientThread.F90 b/pfio/ClientThread.F90 index 089015884d3e..0a302bb552e1 100644 --- a/pfio/ClientThread.F90 +++ b/pfio/ClientThread.F90 @@ -56,7 +56,6 @@ module pFIO_ClientThreadMod contains procedure :: add_ext_collection procedure :: add_hist_collection - procedure :: replace_hist_collection procedure :: modify_metadata procedure :: prefetch_data procedure :: stage_data @@ -154,33 +153,6 @@ function add_hist_collection(this, fmd, unusable, mode, rc) result(hist_collect _RETURN(_SUCCESS) end function add_hist_collection - subroutine replace_hist_collection(this,hist_collection_id,fmd, rc) - class (ClientThread), intent(inout) :: this - integer, intent(in) :: hist_collection_id - type(FileMetadata),intent(in) :: fmd - integer, optional, intent(out) :: rc - - integer :: return_id - - class (AbstractMessage), pointer :: message - class(AbstractSocket),pointer :: connection - integer :: status - - connection=>this%get_connection() - call connection%send(AddHistCollectionMessage(fmd,hist_collection_id),_RC) - - message => connection%receive() - select type(message) - type is(IDMessage) - return_id = message%id - class default - _FAIL( " should get id message") - end select - - _ASSERT( return_id == hist_collection_id, "return id should be the same as the collection_id") - _RETURN(_SUCCESS) - end subroutine replace_hist_collection - function prefetch_data(this, collection_id, file_name, var_name, data_reference, & & unusable, start, rc) result(request_id) class (ClientThread), intent(inout) :: this diff --git a/pfio/HistoryCollection.F90 b/pfio/HistoryCollection.F90 index 0225f133af86..e8c4d9aff05a 100644 --- a/pfio/HistoryCollection.F90 +++ b/pfio/HistoryCollection.F90 @@ -35,19 +35,15 @@ module pFIO_HistoryCollectionMod contains - function new_HistoryCollection(fmd, option) result(collection) + function new_HistoryCollection(fmd, create_mode) result(collection) type (HistoryCollection) :: collection type (FilemetaData), intent(in) :: fmd - integer, optional, intent(in) :: option + integer, optional, intent(in) :: create_mode collection%fmd = fmd collection%formatters = StringNetCDF4_FileFormatterMap() collection%create_mode = NF90_CLOBBER - if (present(option)) then - if (option == -2) then - collection%create_mode = NF90_NOCLOBBER - endif - endif + if (present(create_mode)) collection%create_mode = create_mode end function new_HistoryCollection diff --git a/pfio/ServerThread.F90 b/pfio/ServerThread.F90 index ecf044930e96..d067f07c14e1 100644 --- a/pfio/ServerThread.F90 +++ b/pfio/ServerThread.F90 @@ -531,18 +531,10 @@ subroutine handle_AddHistCollection(this, message, rc) class(AbstractSocket),pointer :: connection if (associated(ioserver_profiler)) call ioserver_profiler%start("add_Histcollection") - if ( message%collection_id == -1 .or. message%collection_id == -2 ) then - n = this%hist_collections%size()+1 - else - n = message%collection_id - endif - hist_collection = HistoryCollection(message%fmd, message%collection_id) - if ( message%collection_id == -1 .or. message%collection_id == -2 ) then - call this%hist_collections%push_back(hist_collection) - else - call this%hist_collections%set(n,hist_collection) - endif + n = this%hist_collections%size()+1 + hist_collection = HistoryCollection(message%fmd, message%create_mode) + call this%hist_collections%push_back(hist_collection) connection=>this%get_connection() call connection%send(IdMessage(n),_RC) From e553399650d782547b92581210ca12375221e25c Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang Date: Wed, 10 Aug 2022 12:47:54 -0400 Subject: [PATCH 08/13] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15fabaaf4ca5..a9ad546a31cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add an option to overwrite history output + ### Changed ### Removed From e4b945b7b979f7c8e24f43eb6ec74f8e29efa9ea Mon Sep 17 00:00:00 2001 From: Weiyuan Jiang <52509753+weiyuan-jiang@users.noreply.github.com> Date: Wed, 10 Aug 2022 13:03:10 -0400 Subject: [PATCH 09/13] Update HistoryCollection.F90 --- pfio/HistoryCollection.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pfio/HistoryCollection.F90 b/pfio/HistoryCollection.F90 index e8c4d9aff05a..63f93cb4d544 100644 --- a/pfio/HistoryCollection.F90 +++ b/pfio/HistoryCollection.F90 @@ -42,7 +42,7 @@ function new_HistoryCollection(fmd, create_mode) result(collection) collection%fmd = fmd collection%formatters = StringNetCDF4_FileFormatterMap() - collection%create_mode = NF90_CLOBBER + collection%create_mode = NF90_NOCLOBBER if (present(create_mode)) collection%create_mode = create_mode end function new_HistoryCollection From fd1039fc612e6e19604581a2bf9c351ce83b0b95 Mon Sep 17 00:00:00 2001 From: Benjamin Auer Date: Wed, 10 Aug 2022 16:50:01 -0400 Subject: [PATCH 10/13] define some pfio constants to keep netcdf out of history --- gridcomps/History/MAPL_HistoryGridComp.F90 | 8 +++---- pfio/HistoryCollection.F90 | 2 +- pfio/NetCDF4_FileFormatter.F90 | 27 ++++++++++++++++++---- pfio/pFIO_Constants.F90 | 7 ++++++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index 1e04bd84aeba..733f4e1abd06 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -40,12 +40,12 @@ module MAPL_HistoryGridCompMod use MAPL_GriddedIOitemMod use pFIO_ClientManagerMod, only: o_Clients use pFIO_DownbitMod, only: pFIO_DownBit + use pFIO_ConstantsMod use HistoryTrajectoryMod use MAPL_StringTemplate use regex_module use MAPL_TimeUtilsMod, only: is_valid_time, is_valid_date use gFTL_StringStringMap - use netcdf !use ESMF_CFIOMOD implicit none @@ -542,8 +542,8 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) _VERIFY(STATUS) call ESMF_ConfigGetAttribute(config, value=allow_overwrite, & label='Allow_Overwrite:', default=.false., _RC) - create_mode = NF90_NOCLOBBER ! defaut no overwrite - if (allow_overwrite) create_mode = NF90_CLOBBER + create_mode = PFIO_NOCLOBBER ! defaut no overwrite + if (allow_overwrite) create_mode = PFIO_CLOBBER if (trim(cFileOrder) == 'ABC') then intstate%fileOrderAlphabetical = .true. @@ -3393,7 +3393,7 @@ subroutine Run ( gc, import, export, clock, rc ) integer :: sec ! variables for "backwards" mode - logical :: fwd, file_exists + logical :: fwd logical, allocatable :: Ignore(:) ! ErrLog vars diff --git a/pfio/HistoryCollection.F90 b/pfio/HistoryCollection.F90 index 63f93cb4d544..3d600d3e584e 100644 --- a/pfio/HistoryCollection.F90 +++ b/pfio/HistoryCollection.F90 @@ -42,7 +42,7 @@ function new_HistoryCollection(fmd, create_mode) result(collection) collection%fmd = fmd collection%formatters = StringNetCDF4_FileFormatterMap() - collection%create_mode = NF90_NOCLOBBER + collection%create_mode = PFIO_NOCLOBBER if (present(create_mode)) collection%create_mode = create_mode end function new_HistoryCollection diff --git a/pfio/NetCDF4_FileFormatter.F90 b/pfio/NetCDF4_FileFormatter.F90 index 382921e4b9c3..ace66d4400dd 100644 --- a/pfio/NetCDF4_FileFormatter.F90 +++ b/pfio/NetCDF4_FileFormatter.F90 @@ -150,12 +150,21 @@ subroutine create(this, file, unusable, mode, rc) integer :: status integer :: mode_ + integer :: pfio_mode if (present(mode)) then - mode_=mode + pfio_mode=mode else - mode_=NF90_CLOBBER + pfio_mode=PFIO_CLOBBER end if + + select case (pfio_mode) + case (pFIO_CLOBBER) + mode_ = NF90_CLOBBER + case (pFIO_NOCLOBBER) + mode_ = NF90_NOCLOBBER + end select + !$omp critical status = nf90_create(file, IOR(mode_, NF90_NETCDF4), this%ncid) !$omp end critical @@ -175,17 +184,25 @@ subroutine create_par(this, file, unusable, mode, comm, info, rc) integer, optional, intent(in) :: info integer, optional, intent(out) :: rc - integer :: comm_ + integer :: comm_, integer :: info_ integer :: status integer :: mode_ + integer :: pfio_mode if (present(mode)) then - mode_=mode + pfio_mode=mode else - mode_=NF90_CLOBBER + pfio_mode=PFIO_CLOBBER end if + select case (pfio_mode) + case (pFIO_CLOBBER) + mode_ = NF90_CLOBBER + case (pFIO_NOCLOBBER) + mode_ = NF90_NOCLOBBER + end select + if (present(comm)) then comm_ = comm else diff --git a/pfio/pFIO_Constants.F90 b/pfio/pFIO_Constants.F90 index 1edb709377e5..ce3b8a1eaa9e 100644 --- a/pfio/pFIO_Constants.F90 +++ b/pfio/pFIO_Constants.F90 @@ -24,6 +24,8 @@ module pFIO_ConstantsMod ! IO modes public :: pFIO_WRITE public :: pFIO_READ + public :: pFIO_CLOBBER + public :: pFIO_NOCLOBBER public :: pFIO_s_tag public :: pFIO_m_w_tag public :: pFIO_w_m_tag @@ -58,6 +60,11 @@ module pFIO_ConstantsMod enumerator :: pFIO_WRITE end enum + enum, bind(C) + enumerator :: pFIO_CLOBBER + enumerator :: pFIO_NOCLOBBER + end enum + integer, parameter :: pFIO_s_tag = 9999 integer, parameter :: pFIO_m_w_tag = 8888 integer, parameter :: pFIO_w_m_tag = 7777 From 491f987d06d3544de035a3a2ee93eea28c8dec51 Mon Sep 17 00:00:00 2001 From: Benjamin Auer Date: Wed, 10 Aug 2022 16:55:41 -0400 Subject: [PATCH 11/13] fix bug --- pfio/AddHistCollectionMessage.F90 | 4 ++-- pfio/NetCDF4_FileFormatter.F90 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pfio/AddHistCollectionMessage.F90 b/pfio/AddHistCollectionMessage.F90 index 6ea7d07f391f..d4f813ac4465 100644 --- a/pfio/AddHistCollectionMessage.F90 +++ b/pfio/AddHistCollectionMessage.F90 @@ -6,7 +6,7 @@ module pFIO_AddHistCollectionMessageMod use pFIO_UtilitiesMod use pFIO_AbstractMessageMod use pFIO_FileMetadataMod - use netcdf + use pFIO_ConstantsMod implicit none private @@ -33,7 +33,7 @@ function new_AddHistCollectionMessage(fmd, mode) result(message) type(FileMetadata), intent(in) :: fmd integer, optional, intent(in) :: mode message%fmd = fmd - message%create_mode = NF90_NOCLOBBER + message%create_mode = PFIO_NOCLOBBER if( present(mode)) message%create_mode = mode end function new_AddHistCollectionMessage diff --git a/pfio/NetCDF4_FileFormatter.F90 b/pfio/NetCDF4_FileFormatter.F90 index ace66d4400dd..bddae585035b 100644 --- a/pfio/NetCDF4_FileFormatter.F90 +++ b/pfio/NetCDF4_FileFormatter.F90 @@ -184,7 +184,7 @@ subroutine create_par(this, file, unusable, mode, comm, info, rc) integer, optional, intent(in) :: info integer, optional, intent(out) :: rc - integer :: comm_, + integer :: comm_ integer :: info_ integer :: status integer :: mode_ From db0bb598147bfee41897eca7fe262fde675f0f07 Mon Sep 17 00:00:00 2001 From: Benjamin Auer Date: Wed, 10 Aug 2022 16:58:40 -0400 Subject: [PATCH 12/13] update changelog verbiage --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9ad546a31cb..3ed3ea5442d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Add an option to overwrite history output +- Add an option to PFIO Server to allow for either clobbering or stopping execution if a pre-existing file exists before application starts ### Changed From c0845010390fcf296caf73f41226e431bc3ea0ab Mon Sep 17 00:00:00 2001 From: Benjamin Auer Date: Thu, 11 Aug 2022 10:05:57 -0400 Subject: [PATCH 13/13] change default --- pfio/NetCDF4_FileFormatter.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pfio/NetCDF4_FileFormatter.F90 b/pfio/NetCDF4_FileFormatter.F90 index bddae585035b..740081c517e7 100644 --- a/pfio/NetCDF4_FileFormatter.F90 +++ b/pfio/NetCDF4_FileFormatter.F90 @@ -155,7 +155,7 @@ subroutine create(this, file, unusable, mode, rc) if (present(mode)) then pfio_mode=mode else - pfio_mode=PFIO_CLOBBER + pfio_mode=PFIO_NOCLOBBER end if select case (pfio_mode) @@ -193,7 +193,7 @@ subroutine create_par(this, file, unusable, mode, comm, info, rc) if (present(mode)) then pfio_mode=mode else - pfio_mode=PFIO_CLOBBER + pfio_mode=PFIO_NOCLOBBER end if select case (pfio_mode)