Auto PR - develop → MAPL-v3 - start of fixing #1620

CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Add an option to PFIO Server to allow for either clobbering or stopping execution if a pre-existing file exists before application starts
+
 ### Changed
 
 ### Removed

gridcomps/History/MAPL_HistoryGridComp.F90

@@ -40,6 +40,7 @@ module MAPL_HistoryGridCompMod
   use MAPL_GriddedIOitemMod
   use pFIO_ClientManagerMod, only: o_Clients
   use pFIO_DownbitMod, only: pFIO_DownBit
+  use pFIO_ConstantsMod
   use HistoryTrajectoryMod
   use MAPL_StringTemplate
   use regex_module
@@ -116,7 +117,6 @@ module MAPL_HistoryGridCompMod
      logical                             :: integer_time
      integer                             :: collectionWriteSplit
      integer                             :: serverSizeSplit
-     logical                             :: allow_overwrite
   end type HISTORY_STATE
 
   type HISTORY_wrap
@@ -428,6 +428,8 @@ subroutine Initialize ( gc, import, dumexport, clock, rc )
     type(StringStringMap) :: global_attributes
     character(len=ESMF_MAXSTR) :: name,regrid_method
     logical :: has_conservative_keyword, has_regrid_keyword
+    logical :: allow_overwrite
+    integer :: create_mode
 
 ! Begin
 !------
@@ -538,9 +540,11 @@ subroutine Initialize ( gc, import, dumexport, clock, rc )
     call ESMF_ConfigGetAttribute(config, value=cFileOrder,         &
                                          label='FileOrder:', default='ABC', rc=status)
     _VERIFY(STATUS)
-    call ESMF_ConfigGetAttribute(config, value=intState%allow_overwrite,  &
+    call ESMF_ConfigGetAttribute(config, value=allow_overwrite,  &
                                          label='Allow_Overwrite:', default=.false., _RC)
-
+    create_mode = PFIO_NOCLOBBER ! defaut no overwrite
+    if (allow_overwrite) create_mode = PFIO_CLOBBER
+
     if (trim(cFileOrder) == 'ABC') then
        intstate%fileOrderAlphabetical = .true.
     else if (trim(cFileOrder) == 'AddOrder') then
@@ -2532,7 +2536,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc )
                 call list(n)%mGriddedIO%CreateFileMetaData(list(n)%items,list(n)%bundle,list(n)%timeInfo,vdata=list(n)%vdata,global_attributes=global_attributes,rc=status)
                 _VERIFY(status)
              end if
-             collection_id = o_Clients%add_hist_collection(list(n)%mGriddedIO%metadata)
+             collection_id = o_Clients%add_hist_collection(list(n)%mGriddedIO%metadata, mode = create_mode)
              call list(n)%mGriddedIO%set_param(write_collection_id=collection_id)
           end if
        end if
@@ -3389,7 +3393,7 @@ subroutine Run ( gc, import, export, clock, rc )
     integer                        :: sec
 
 !   variables for "backwards" mode
-    logical                        :: fwd, file_exists
+    logical                        :: fwd
     logical, allocatable           :: Ignore(:)
 
 !   ErrLog vars
@@ -3630,10 +3634,6 @@ subroutine Run ( gc, import, export, clock, rc )
          else
             if( list(n)%unit.eq.0 ) then
                if (list(n)%format == 'CFIO') then
-                  if (.not.intState%allow_overwrite) then
-                     inquire (file=trim(filename(n)),exist=file_exists)
-                     _ASSERT(.not.file_exists,trim(filename(n))//" being created for History output already exists")
-                  end if
                   call list(n)%mGriddedIO%modifyTime(oClients=o_Clients,rc=status)
                   _VERIFY(status)
                   list(n)%currentFile = filename(n)

pfio/AddHistCollectionMessage.F90

@@ -6,16 +6,15 @@ module pFIO_AddHistCollectionMessageMod
    use pFIO_UtilitiesMod
    use pFIO_AbstractMessageMod
    use pFIO_FileMetadataMod
+   use pFIO_ConstantsMod
    implicit none
    private
 
    public :: AddHistCollectionMessage
 
    type, extends(AbstractMessage) :: AddHistCollectionMessage
       type(FileMetadata) :: fmd
-      ! WY node: -1    : add ( default )
-      !          other : replace
-      integer :: collection_id = -1
+      integer :: create_mode
    contains
       procedure, nopass :: get_type_id
       procedure :: get_length
@@ -29,12 +28,13 @@ module pFIO_AddHistCollectionMessageMod
 
 contains
 
-   function new_AddHistCollectionMessage(fmd, collection_id) result(message)
+   function new_AddHistCollectionMessage(fmd, mode) result(message)
       type (AddHistCollectionMessage) :: message
       type(FileMetadata), intent(in) :: fmd
-      integer, optional, intent(in) :: collection_id
+      integer, optional, intent(in) :: mode
       message%fmd = fmd
-      if( present(collection_id)) message%collection_id = collection_id
+      message%create_mode = PFIO_NOCLOBBER
+      if( present(mode)) message%create_mode = mode
    end function new_AddHistCollectionMessage
 
 
@@ -47,7 +47,7 @@ integer function get_length(this) result(length)
       class (AddHistCollectionMessage), intent(in) :: this
       integer,allocatable :: buffer(:) ! no-op
       call this%fmd%serialize(buffer)
-      length = size(buffer) + 1 ! 1 is the collection_id
+      length = size(buffer) + 1 ! 1 is the create_mode
    end function get_length
 
 
@@ -60,7 +60,7 @@ subroutine serialize(this, buffer, rc)
       integer :: status
       call this%fmd%serialize(tmp_buffer, status)
       _VERIFY(status)
-      buffer = [tmp_buffer,serialize_intrinsic(this%collection_id)]
+      buffer = [tmp_buffer,serialize_intrinsic(this%create_mode)]
       _RETURN(_SUCCESS)
    end subroutine serialize
 
@@ -76,7 +76,7 @@ subroutine deserialize(this, buffer,rc)
       _VERIFY(status)
       call deserialize_intrinsic(buffer(n:), length)
       n = n + length 
-      call deserialize_intrinsic(buffer(n:), this%collection_id)
+      call deserialize_intrinsic(buffer(n:), this%create_mode)
       _RETURN(_SUCCESS)
    end subroutine deserialize
 

pfio/ClientManager.F90

@@ -38,7 +38,6 @@ module pFIO_ClientManagerMod
    contains
       procedure :: add_ext_collection
       procedure :: add_hist_collection
-      procedure :: replace_hist_collection
       procedure :: modify_metadata
       procedure :: modify_metadata_all
       procedure :: prefetch_data
@@ -130,43 +129,25 @@ function add_ext_collection(this, template, unusable, rc) result(collection_id)
       _UNUSED_DUMMY(unusable)
    end function add_ext_collection
 
-   function add_hist_collection(this, fmd, unusable, rc) result(hist_collection_id)
+   function add_hist_collection(this, fmd, unusable,mode, rc) result(hist_collection_id)
       integer :: hist_collection_id
       class (ClientManager), intent(inout) :: this
       type(FileMetadata),intent(in) :: fmd
       class (KeywordEnforcer), optional, intent(out) :: unusable
+      integer, optional, intent(in) :: mode
       integer, optional, intent(out) :: rc
       class (ClientThread), pointer :: clientPtr
       integer :: i
 
       do i = 1, this%size()
          ClientPtr => this%clients%at(i)
-         hist_collection_id = clientPtr%add_hist_collection(fmd)
+         hist_collection_id = clientPtr%add_hist_collection(fmd, mode=mode)
       enddo
 
       _RETURN(_SUCCESS)
       _UNUSED_DUMMY(unusable)
    end function add_hist_collection
 
-   subroutine replace_hist_collection(this, hist_collection_id, fmd, unusable, rc)
-      class (ClientManager), intent(inout) :: this
-      integer, intent(in) :: hist_collection_id
-      type(FileMetadata),intent(in) :: fmd
-      class (KeywordEnforcer), optional, intent(out) :: unusable
-      integer, optional, intent(out) :: rc
-
-      class (ClientThread), pointer :: clientPtr
-      integer :: i
-
-      do i = 1, this%size()
-         ClientPtr => this%clients%at(i)
-         call clientPtr%replace_hist_collection(hist_collection_id, fmd)
-      enddo
-
-      _RETURN(_SUCCESS)
-      _UNUSED_DUMMY(unusable)
-   end subroutine replace_hist_collection
-
    subroutine prefetch_data(this, collection_id, file_name, var_name, data_reference, &
         & unusable, start, rc)
       class (ClientManager), intent(inout) :: this

pfio/ClientThread.F90

@@ -56,7 +56,6 @@ module pFIO_ClientThreadMod
    contains
       procedure :: add_ext_collection
       procedure :: add_hist_collection
-      procedure :: replace_hist_collection
       procedure :: modify_metadata
       procedure :: prefetch_data
       procedure :: stage_data
@@ -128,18 +127,20 @@ function add_ext_collection(this, template, rc) result(collection_id)
       _RETURN(_SUCCESS)
    end function add_ext_collection
 
-   function add_hist_collection(this, fmd, rc) result(hist_collection_id)
+   function add_hist_collection(this, fmd, unusable,  mode, rc) result(hist_collection_id)
       integer :: hist_collection_id
       class (ClientThread), intent(inout) :: this
       type(FileMetadata),intent(in) :: fmd
+      class (KeywordEnforcer), optional, intent(out) :: unusable
+      integer, optional, intent(in) :: mode
       integer, optional, intent(out) :: rc
 
       class (AbstractMessage), pointer :: message
       class(AbstractSocket),pointer :: connection
       integer :: status
 
       connection=>this%get_connection()
-      call connection%send(AddHistCollectionMessage(fmd))
+      call connection%send(AddHistCollectionMessage(fmd, mode=mode))
 
       message => connection%receive()
       select type(message)
@@ -152,33 +153,6 @@ function add_hist_collection(this, fmd, rc) result(hist_collection_id)
       _RETURN(_SUCCESS)
    end function add_hist_collection
 
-   subroutine replace_hist_collection(this,hist_collection_id,fmd, rc)
-      class (ClientThread), intent(inout) :: this
-      integer, intent(in) :: hist_collection_id
-      type(FileMetadata),intent(in) :: fmd
-      integer, optional, intent(out) :: rc
-
-      integer :: return_id
-
-      class (AbstractMessage), pointer :: message
-      class(AbstractSocket),pointer :: connection
-      integer :: status
-
-      connection=>this%get_connection()
-      call connection%send(AddHistCollectionMessage(fmd,hist_collection_id),_RC)
-
-      message => connection%receive()
-      select type(message)
-      type is(IDMessage)
-        return_id = message%id
-      class default
-        _FAIL( " should get id message")
-      end select
-
-      _ASSERT( return_id == hist_collection_id, "return id should be the same as the collection_id")
-      _RETURN(_SUCCESS)
-   end subroutine replace_hist_collection
-
    function prefetch_data(this, collection_id, file_name, var_name, data_reference, &
         & unusable, start, rc) result(request_id)
       class (ClientThread), intent(inout) :: this

pfio/HistoryCollection.F90

@@ -9,6 +9,8 @@ module pFIO_HistoryCollectionMod
   use pFIO_FileMetadataMod
   use pFIO_StringVariableMapMod
   use pFIO_ConstantsMod
+  use gFTL_StringVector
+  use NetCDF
   implicit none
   private
 
@@ -17,12 +19,14 @@ module pFIO_HistoryCollectionMod
 
   type :: HistoryCollection
     type (Filemetadata) :: fmd
+    character(len=:), allocatable :: last_file_created
     type (StringNetCDF4_FileFormatterMap) :: formatters
-
+    integer :: create_mode
   contains
     procedure :: find
     procedure :: ModifyMetadata
     procedure :: clear
+    procedure :: check_if_i_created
   end type HistoryCollection
 
   interface HistoryCollection
@@ -31,12 +35,15 @@ module pFIO_HistoryCollectionMod
 
 contains
 
-  function new_HistoryCollection(fmd) result(collection)
+  function new_HistoryCollection(fmd, create_mode) result(collection)
     type (HistoryCollection) :: collection
     type (FilemetaData), intent(in) :: fmd
+    integer, optional, intent(in) :: create_mode
 
     collection%fmd = fmd
     collection%formatters = StringNetCDF4_FileFormatterMap() 
+    collection%create_mode = PFIO_NOCLOBBER
+    if (present(create_mode)) collection%create_mode = create_mode
 
   end function new_HistoryCollection
 
@@ -51,18 +58,28 @@ function find(this, file_name,rc) result(formatter)
     type(StringNetCDF4_FileFormatterMapIterator) :: iter
     integer :: status
     character(len=*), parameter :: Iam = "HistoryCollection::find()"
-    logical :: f_exist
+    logical :: f_exist, i_created
 
     iter = this%formatters%find(trim(file_name))
     if (iter == this%formatters%end()) then
        inquire(file=file_name, exist=f_exist)
-       if(.not. f_exist) then 
+       if(.not. f_exist) then
          call fm%create(trim(file_name),rc=status)
          _VERIFY(status)
          call fm%write(this%fmd, rc=status)
          _VERIFY(status)
+         this%last_file_created = trim(file_name)
        else
-          call fm%open(trim(file_name), pFIO_WRITE)
+          i_created = this%check_if_i_created(file_name)
+          if (i_created) then
+             call fm%open(trim(file_name), pFIO_WRITE)
+          else
+             call fm%create(trim(file_name),mode=this%create_mode,rc=status)
+             _VERIFY(status)
+             call fm%write(this%fmd, rc=status)
+             _VERIFY(status)
+             this%last_file_created=trim(file_name)
+          end if
        endif
        call this%formatters%insert( trim(file_name),fm)
        iter = this%formatters%find(trim(file_name))
@@ -112,6 +129,23 @@ subroutine clear(this, rc)
     _RETURN(_SUCCESS)
   end subroutine clear
 
+  function check_if_i_created(this,input_file,rc) result(i_created)
+    logical :: i_created
+    class (HistoryCollection), intent(inout) :: this
+    character(len=*), intent(in) :: input_file
+    integer, optional, intent(out) :: rc 
+
+    integer :: status
+
+    i_created = .false.
+    if (allocated(this%last_file_created)) then
+       if (input_file == this%last_file_created) i_created=.true.
+    end if
+
+    _RETURN(_SUCCESS)
+
+  end function
+
 end module pFIO_HistoryCollectionMod