diff --git a/Apps/Regrid_Util.F90 b/Apps/Regrid_Util.F90 index 072ab7373b96..9906beed1586 100644 --- a/Apps/Regrid_Util.F90 +++ b/Apps/Regrid_Util.F90 @@ -26,6 +26,7 @@ module regrid_util_support_mod integer :: deflate, shave integer :: quantize_algorithm integer :: quantize_level + integer :: zstandard_level logical :: use_weights contains procedure :: create_grid @@ -98,6 +99,7 @@ subroutine process_command_line(this,rc) this%deflate=0 this%quantize_algorithm=0 this%quantize_level=0 + this%zstandard_level=0 this%use_weights = .false. nargs = command_argument_count() do i=1,nargs @@ -161,6 +163,9 @@ subroutine process_command_line(this,rc) case('-quantize_level') call get_command_argument(i+1,astr) read(astr,*)this%quantize_level + case('-zstandard_level') + call get_command_argument(i+1,astr) + read(astr,*)this%zstandard_level case('-file_weights') this%use_weights = .true. case('--help') @@ -432,7 +437,7 @@ subroutine main() call ESMF_ClockSet(clock,currtime=time,_RC) if (.not. writer_created) then - call newWriter%create_from_bundle(bundle,clock,n_steps=tsteps,time_interval=tint,nbits_to_keep=support%shave,deflate=support%deflate,vertical_data=vertical_data,quantize_algorithm=support%quantize_algorithm,quantize_level=support%quantize_level,_RC) + call newWriter%create_from_bundle(bundle,clock,n_steps=tsteps,time_interval=tint,nbits_to_keep=support%shave,deflate=support%deflate,vertical_data=vertical_data,quantize_algorithm=support%quantize_algorithm,quantize_level=support%quantize_level,zstandard_level=support%zstandard_level,_RC) writer_created=.true. end if diff --git a/CHANGELOG.md b/CHANGELOG.md index df53e03b29fb..db766527e9c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added zstandard compression support + - Note this requires netCDF-C to have been compiled with zstandard support. We have a CMake test to check for this + and enabling zstandard output in History will fail if the library does not support it + ### Changed - ExtDataDriver.x now uses ExtData2G by default diff --git a/Tests/ExtData_Testing_Framework/CMakeLists.txt b/Tests/ExtData_Testing_Framework/CMakeLists.txt index 9a8e46f8ea5b..4ac289cb31a2 100644 --- a/Tests/ExtData_Testing_Framework/CMakeLists.txt +++ b/Tests/ExtData_Testing_Framework/CMakeLists.txt @@ -29,6 +29,16 @@ set(QUANTIZE_TESTS "case34" ) +# We have one test that requires netcdf zstandard support +set(ZSTD_TESTS + "case35" +) + +# We have one test which requires *both* Quantize and ZSTD support +set(QUANTIZE_AND_ZSTD_TESTS + "case36" +) + file(STRINGS "test_cases/extdata_2g_cases.txt" TEST_CASES_2G) foreach(TEST_CASE ${TEST_CASES_2G}) @@ -38,6 +48,16 @@ foreach(TEST_CASE ${TEST_CASES_2G}) continue() endif() + # Skip tests that require ZSTD support if we don't have it + if (NOT NETCDF_HAS_ZSTD AND ${TEST_CASE} IN_LIST ZSTD_TESTS) + continue() + endif() + + # Skip tests that require both Quantize and ZSTD support if we don't have it + if (NOT (NETCDF_HAS_QUANTIZE AND NETCDF_HAS_ZSTD) AND ${TEST_CASE} IN_LIST QUANTIZE_AND_ZSTD_TESTS) + continue() + endif() + if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc) file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs) else() diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM1.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM1.rc new file mode 100644 index 000000000000..83ad27a2c551 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM1.rc @@ -0,0 +1,24 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: GenerateExports + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM2.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM2.rc new file mode 100644 index 000000000000..2e79954523bd --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/AGCM2.rc @@ -0,0 +1,29 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: CompareImports + +IMPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/CAP.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP.rc new file mode 100644 index 000000000000..680d0ffa9c5b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP.rc @@ -0,0 +1,4 @@ +CASES:: +CAP1.rc +CAP2.rc +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/CAP1.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP1.rc new file mode 100644 index 000000000000..ce2690d6937b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP1.rc @@ -0,0 +1,25 @@ +ROOT_NAME: Root +ROOT_CF: AGCM1.rc +HIST_CF: HISTORY1.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20040115 210000 +20040215 210000 +20040315 210000 +20040415 210000 +20040515 210000 +20040615 210000 +20040715 210000 +20040815 210000 +20040915 210000 +20041015 210000 +20041115 210000 +20041215 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/CAP2.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP2.rc new file mode 100644 index 000000000000..4e9e1bb95026 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/CAP2.rc @@ -0,0 +1,15 @@ +ROOT_NAME: Root +ROOT_CF: AGCM2.rc +HIST_CF: HISTORY2.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20041125 210000 +20041126 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/ExtData.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/ExtData.rc new file mode 100644 index 000000000000..a45d1dd13f7f --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/ExtData.rc @@ -0,0 +1,13 @@ +#CASE_SENSITIVE_VARIABLE_NAMES: .false. +Ext_AllowExtrap: .false. +Prefetch: .true. +#DEBUG_LEVEL: 20 + +PrimaryExports%% +VAR2D NA N N 0 none none VAR2D case1.%y4.nc4 +VAR3D NA N N 0 none none VAR3D case1.%y4.nc4 +%% + + +DerivedExports%% +%% diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY1.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY1.rc new file mode 100644 index 000000000000..632008de4f12 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY1.rc @@ -0,0 +1,14 @@ +GRID_LABELS: +:: + +COLLECTIONS: case1 +:: + + case1.template: '%y4.nc4', + case1.format: 'CFIO', + case1.frequency: 010000, + case1.duration: 000000, + case1.zstandard_level: 2, + case1.fields: 'VAR2D', 'Root', + 'VAR3D', 'Root', + :: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY2.rc b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY2.rc new file mode 100644 index 000000000000..2895432e995a --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/HISTORY2.rc @@ -0,0 +1,5 @@ +GRID_LABELS: +:: + +COLLECTIONS: +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/README b/Tests/ExtData_Testing_Framework/test_cases/case35/README new file mode 100644 index 000000000000..9a6d7597262d --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/README @@ -0,0 +1 @@ +Case, 12-month/12 time 2004 file with 2 updates, non-climatology diff --git a/Tests/ExtData_Testing_Framework/test_cases/case35/extdata.yaml b/Tests/ExtData_Testing_Framework/test_cases/case35/extdata.yaml new file mode 100644 index 000000000000..e2ddb90675ab --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case35/extdata.yaml @@ -0,0 +1,5 @@ +Collections: + fstream1: {template: case1.%y4.nc4, valid_range: "2004-01-01/2005-01-01" } +Exports: + VAR2D: {variable: VAR2D, collection: fstream1} + VAR3D: {variable: VAR3D, collection: fstream1} diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM1.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM1.rc new file mode 100644 index 000000000000..83ad27a2c551 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM1.rc @@ -0,0 +1,24 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: GenerateExports + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM2.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM2.rc new file mode 100644 index 000000000000..2e79954523bd --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/AGCM2.rc @@ -0,0 +1,29 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: CompareImports + +IMPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/CAP.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP.rc new file mode 100644 index 000000000000..680d0ffa9c5b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP.rc @@ -0,0 +1,4 @@ +CASES:: +CAP1.rc +CAP2.rc +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/CAP1.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP1.rc new file mode 100644 index 000000000000..ce2690d6937b --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP1.rc @@ -0,0 +1,25 @@ +ROOT_NAME: Root +ROOT_CF: AGCM1.rc +HIST_CF: HISTORY1.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20040115 210000 +20040215 210000 +20040315 210000 +20040415 210000 +20040515 210000 +20040615 210000 +20040715 210000 +20040815 210000 +20040915 210000 +20041015 210000 +20041115 210000 +20041215 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/CAP2.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP2.rc new file mode 100644 index 000000000000..4e9e1bb95026 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/CAP2.rc @@ -0,0 +1,15 @@ +ROOT_NAME: Root +ROOT_CF: AGCM2.rc +HIST_CF: HISTORY2.rc + +BEG_DATE: 20040101 210000 + +JOB_SGMT: 00001200 000000 +HEARTBEAT_DT: 3600 + +#RUN_EXTDATA: .false. +RUN_TIMES:: +20041125 210000 +20041126 210000 +:: + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/ExtData.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/ExtData.rc new file mode 100644 index 000000000000..a45d1dd13f7f --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/ExtData.rc @@ -0,0 +1,13 @@ +#CASE_SENSITIVE_VARIABLE_NAMES: .false. +Ext_AllowExtrap: .false. +Prefetch: .true. +#DEBUG_LEVEL: 20 + +PrimaryExports%% +VAR2D NA N N 0 none none VAR2D case1.%y4.nc4 +VAR3D NA N N 0 none none VAR3D case1.%y4.nc4 +%% + + +DerivedExports%% +%% diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY1.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY1.rc new file mode 100644 index 000000000000..014b2e83e177 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY1.rc @@ -0,0 +1,16 @@ +GRID_LABELS: +:: + +COLLECTIONS: case1 +:: + + case1.template: '%y4.nc4', + case1.format: 'CFIO', + case1.frequency: 010000, + case1.duration: 000000, + case1.zstandard_level: 2, + case1.quantization_algorithm: 'granular_bitround', + case1.quantization_level: 5, + case1.fields: 'VAR2D', 'Root', + 'VAR3D', 'Root', + :: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY2.rc b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY2.rc new file mode 100644 index 000000000000..2895432e995a --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/HISTORY2.rc @@ -0,0 +1,5 @@ +GRID_LABELS: +:: + +COLLECTIONS: +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/README b/Tests/ExtData_Testing_Framework/test_cases/case36/README new file mode 100644 index 000000000000..9a6d7597262d --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/README @@ -0,0 +1 @@ +Case, 12-month/12 time 2004 file with 2 updates, non-climatology diff --git a/Tests/ExtData_Testing_Framework/test_cases/case36/extdata.yaml b/Tests/ExtData_Testing_Framework/test_cases/case36/extdata.yaml new file mode 100644 index 000000000000..e2ddb90675ab --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case36/extdata.yaml @@ -0,0 +1,5 @@ +Collections: + fstream1: {template: case1.%y4.nc4, valid_range: "2004-01-01/2005-01-01" } +Exports: + VAR2D: {variable: VAR2D, collection: fstream1} + VAR3D: {variable: VAR3D, collection: fstream1} diff --git a/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt b/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt index 76be9d4d54f8..b79e4a5c9d29 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt +++ b/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt @@ -32,3 +32,5 @@ case31 case32 case33 case34 +case35 +case36 diff --git a/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md b/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md index f3700c12e5ee..65802345b833 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md +++ b/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md @@ -39,3 +39,6 @@ path_to_script/run_extdatadriver_cases.py --builddir path_to_geos_install/bin -- 32. Case1 with deflate compression and NetCDF bitgroom quantization (only enabled if netcdf built with quantization support) 33. Case1 with deflate compression and NetCDF bitround quantization (only enabled if netcdf built with quantization support) 34. Case1 with deflate compression and NetCDF granular_bitround quantization (only enabled if netcdf built with quantization support) +35. Case1 with zstandard compression (only enabled if netcdf built with zstandard support) +36. Case1 with zstandard compression and NetCDF granular_bitround quantization (only enabled if netcdf built with quantization + support and zstandard support) diff --git a/gridcomps/History/MAPL_HistoryCollection.F90 b/gridcomps/History/MAPL_HistoryCollection.F90 index 39f3c0b723d1..336773e0400d 100644 --- a/gridcomps/History/MAPL_HistoryCollection.F90 +++ b/gridcomps/History/MAPL_HistoryCollection.F90 @@ -83,6 +83,7 @@ module MAPL_HistoryCollectionMod character(len=ESMF_MAXSTR) :: quantize_algorithm_string integer :: quantize_algorithm integer :: quantize_level + integer :: zstandard_level integer :: slices integer :: Root integer :: Psize diff --git a/gridcomps/History/MAPL_HistoryGridComp.F90 b/gridcomps/History/MAPL_HistoryGridComp.F90 index 7aa0afff2a1c..adc88d215dbb 100644 --- a/gridcomps/History/MAPL_HistoryGridComp.F90 +++ b/gridcomps/History/MAPL_HistoryGridComp.F90 @@ -836,6 +836,18 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call ESMF_ConfigGetAttribute ( cfg, list(n)%deflate, default=0, & label=trim(string) // 'deflate:' ,_RC ) + ! We only allow deflate level to be between 0 and 9 + _ASSERT( .not. (list(n)%deflate < 0 .or. list(n)%deflate > 9), 'deflate level must be between 0 and 9') + + call ESMF_ConfigGetAttribute ( cfg, list(n)%zstandard_level, default=0, & + label=trim(string) // 'zstandard_level:' ,_RC ) + + ! We only allow zstandard level to be between 0 and 22 + _ASSERT( .not. (list(n)%zstandard_level < 0 .or. list(n)%zstandard_level > 22), 'zstandard level must be between 0 and 22') + + ! We only allow either deflate or zstandard compression to be used, not both + _ASSERT( .not. (list(n)%deflate > 0 .and. list(n)%zstandard_level > 0), 'deflate and zstandard_level cannot be used together') + call ESMF_ConfigGetAttribute ( cfg, list(n)%quantize_algorithm_string, default='NONE', & label=trim(string) // 'quantize_algorithm:' ,_RC ) @@ -2405,6 +2417,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call list(n)%xsampler%set_param(deflation=list(n)%deflate,_RC) call list(n)%xsampler%set_param(quantize_algorithm=list(n)%quantize_algorithm,_RC) call list(n)%xsampler%set_param(quantize_level=list(n)%quantize_level,_RC) + call list(n)%xsampler%set_param(zstandard_level=list(n)%zstandard_level,_RC) call list(n)%xsampler%set_param(chunking=list(n)%chunkSize,_RC) call list(n)%xsampler%set_param(nbits_to_keep=list(n)%nbits_to_keep,_RC) call list(n)%xsampler%set_param(regrid_method=list(n)%regrid_method,_RC) @@ -2415,6 +2428,7 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) call list(n)%mGriddedIO%set_param(deflation=list(n)%deflate,_RC) call list(n)%mGriddedIO%set_param(quantize_algorithm=list(n)%quantize_algorithm,_RC) call list(n)%mGriddedIO%set_param(quantize_level=list(n)%quantize_level,_RC) + call list(n)%mGriddedIO%set_param(zstandard_level=list(n)%zstandard_level,_RC) call list(n)%mGriddedIO%set_param(chunking=list(n)%chunkSize,_RC) call list(n)%mGriddedIO%set_param(nbits_to_keep=list(n)%nbits_to_keep,_RC) call list(n)%mGriddedIO%set_param(regrid_method=list(n)%regrid_method,_RC) @@ -2493,6 +2507,9 @@ subroutine Initialize ( gc, import, dumexport, clock, rc ) print *, 'Quantize Alg: ', trim(list(n)%quantize_algorithm_string) print *, 'Quantize Lvl: ', list(n)%quantize_level end if + if (list(n)%zstandard_level > 0) then + print *, 'Zstandard Lvl: ', list(n)%zstandard_level + end if if (associated(list(n)%chunksize)) then print *, ' ChunkSize: ', list(n)%chunksize end if diff --git a/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 b/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 index 0fb038bff536..cf051b2b66a8 100644 --- a/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 +++ b/gridcomps/History/Sampler/MAPL_EpochSwathMod.F90 @@ -90,6 +90,7 @@ module MAPL_EpochSwathMod integer :: deflateLevel = 0 integer :: quantizeAlgorithm = 1 integer :: quantizeLevel = 0 + integer :: zstandardLevel = 0 integer, allocatable :: chunking(:) logical :: itemOrderAlphabetical = .true. integer :: fraction @@ -536,11 +537,12 @@ subroutine Create_bundle_RH(this,items,bundle,tunit,timeInfo,vdata,ogrid,rc) end subroutine Create_Bundle_RH - subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,rc) + subroutine set_param(this,deflation,quantize_algorithm,quantize_level,zstandard_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,rc) class (sampler), intent(inout) :: this integer, optional, intent(in) :: deflation integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(in) :: chunking(:) integer, optional, intent(in) :: nbits_to_keep integer, optional, intent(in) :: regrid_method @@ -555,6 +557,7 @@ subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,n if (present(deflation)) this%deflateLevel = deflation if (present(quantize_algorithm)) this%quantizeAlgorithm = quantize_algorithm if (present(quantize_level)) this%quantizeLevel = quantize_level + if (present(zstandard_level)) this%zstandardLevel = zstandard_level if (present(chunking)) then allocate(this%chunking,source=chunking,_STAT) end if diff --git a/griddedio/FieldBundleWrite.F90 b/griddedio/FieldBundleWrite.F90 index 1fb4e134e304..36e7d57f9b59 100644 --- a/griddedio/FieldBundleWrite.F90 +++ b/griddedio/FieldBundleWrite.F90 @@ -29,7 +29,7 @@ module MAPL_ESMFFieldBundleWrite contains - subroutine write_bundle_single_time(bundle,clock,output_file,nbits_to_keep,deflate,quantize_algorithm,quantize_level,rc) + subroutine write_bundle_single_time(bundle,clock,output_file,nbits_to_keep,deflate,quantize_algorithm,quantize_level,zstandard_level,rc) type(ESMF_FieldBundle), intent(inout) :: bundle type(ESMF_Clock), intent(inout) :: clock character(len=*), intent(in) :: output_file @@ -37,20 +37,21 @@ subroutine write_bundle_single_time(bundle,clock,output_file,nbits_to_keep,defla integer, optional, intent(in) :: deflate integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(out) :: rc integer :: status type(FieldBundleWriter) :: newWriter - call newWriter%create_from_bundle(bundle,clock,output_file=output_File,n_steps=1,time_interval=0,nbits_to_keep=nbits_to_keep,deflate=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level,rc=status) + call newWriter%create_from_bundle(bundle,clock,output_file=output_File,n_steps=1,time_interval=0,nbits_to_keep=nbits_to_keep,deflate=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level,zstandard_level=zstandard_level,rc=status) _VERIFY(status) call newWriter%write_to_file(rc=status) _VERIFY(status) _RETURN(_SUCCESS) end subroutine write_bundle_single_time - subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_steps,time_interval,nbits_to_keep,deflate,quantize_algorithm,quantize_level,rc) + subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_steps,time_interval,nbits_to_keep,deflate,quantize_algorithm,quantize_level,zstandard_level,rc) class(FieldBundleWRiter), intent(inout) :: this type(ESMF_FieldBundle), intent(inout) :: bundle type(ESMF_Clock), intent(inout) :: clock @@ -62,6 +63,7 @@ subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_step integer, optional, intent(in) :: deflate integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(out) :: rc type(TimeData) :: time_info @@ -85,7 +87,7 @@ subroutine create_from_bundle(this,bundle,clock,output_file,vertical_data,n_step time_interval_=0 end if - call this%cfio%set_param(nbits_to_keep=nbits_to_keep,deflation=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level) + call this%cfio%set_param(nbits_to_keep=nbits_to_keep,deflation=deflate,quantize_algorithm=quantize_algorithm,quantize_level=quantize_level,zstandard_level=zstandard_level) time_info = TimeData(clock,file_steps,time_interval_,offset) call ESMF_FieldBundleGet(bundle, fieldCount=num_fields,rc=status) _VERIFY(status) diff --git a/griddedio/GriddedIO.F90 b/griddedio/GriddedIO.F90 index 7eb3cf53c7f7..06ec0754460e 100644 --- a/griddedio/GriddedIO.F90 +++ b/griddedio/GriddedIO.F90 @@ -54,6 +54,7 @@ module MAPL_GriddedIOMod integer :: deflateLevel = 0 integer :: quantizeAlgorithm = MAPL_NOQUANTIZE integer :: quantizeLevel = 0 + integer :: zstandardLevel = 0 integer, allocatable :: chunking(:) logical :: itemOrderAlphabetical = .true. integer :: fraction @@ -250,11 +251,12 @@ subroutine destroy(this, rc) end subroutine destroy - subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,regrid_hints,rc) + subroutine set_param(this,deflation,quantize_algorithm,quantize_level,zstandard_level,chunking,nbits_to_keep,regrid_method,itemOrder,write_collection_id,regrid_hints,rc) class (MAPL_GriddedIO), intent(inout) :: this integer, optional, intent(in) :: deflation integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(in) :: chunking(:) integer, optional, intent(in) :: nbits_to_keep integer, optional, intent(in) :: regrid_method @@ -270,6 +272,7 @@ subroutine set_param(this,deflation,quantize_algorithm,quantize_level,chunking,n if (present(deflation)) this%deflateLevel = deflation if (present(quantize_algorithm)) this%quantizeAlgorithm = quantize_algorithm if (present(quantize_level)) this%quantizeLevel = quantize_level + if (present(zstandard_level)) this%zstandardLevel = zstandard_level if (present(chunking)) then allocate(this%chunking,source=chunking,stat=status) _VERIFY(status) @@ -409,7 +412,7 @@ subroutine CreateVariable(this,itemName,rc) _FAIL( 'Unsupported field rank') end if end if - v = Variable(type=PFIO_REAL32,dimensions=vdims,chunksizes=this%chunking,deflation=this%deflateLevel,quantize_algorithm=this%quantizeAlgorithm,quantize_level=this%quantizeLevel) + v = Variable(type=PFIO_REAL32,dimensions=vdims,chunksizes=this%chunking,deflation=this%deflateLevel,quantize_algorithm=this%quantizeAlgorithm,quantize_level=this%quantizeLevel,zstandard_level=this%zstandardLevel) call v%add_attribute('units',trim(units)) call v%add_attribute('long_name',trim(longName)) call v%add_attribute('standard_name',trim(longName)) diff --git a/pfio/CMakeLists.txt b/pfio/CMakeLists.txt index 3c64d0598926..3c6b826eb4f3 100644 --- a/pfio/CMakeLists.txt +++ b/pfio/CMakeLists.txt @@ -114,6 +114,27 @@ else () message(STATUS "netCDF does not have quantize capability") endif () +################################################################ +# Check to see if zstandard capability is present in netcdf-c. # +################################################################ + +check_c_source_compiles(" + #include + #if !NC_HAS_ZSTD + choke me + #endif + int main(void) { + return 0; + } + " + NETCDF_HAS_ZSTD) +if (NETCDF_HAS_ZSTD) + message(STATUS "netCDF has zstandard capability") + add_definitions(-DNF_HAS_ZSTD) +else () + message(STATUS "netCDF does not have zstandard capability") +endif () + if (BUILD_WITH_PFLOGGER) find_package (PFLOGGER REQUIRED) endif () diff --git a/pfio/NetCDF4_FileFormatter.F90 b/pfio/NetCDF4_FileFormatter.F90 index 7ffc315bbbba..1527d8eb5524 100644 --- a/pfio/NetCDF4_FileFormatter.F90 +++ b/pfio/NetCDF4_FileFormatter.F90 @@ -709,6 +709,7 @@ subroutine def_variables(this, cf, unusable, varname, rc) integer :: deflation integer :: quantize_algorithm integer :: quantize_level + integer :: zstandard_level character(len=:), pointer :: var_name character(len=:), pointer :: dim_name class (Variable), pointer :: var @@ -783,6 +784,18 @@ subroutine def_variables(this, cf, unusable, varname, rc) #endif end if + zstandard_level = var%get_zstandard_level() + if (zstandard_level /= 0) then +#ifdef NF_HAS_ZSTD + !$omp critical + status = nf90_def_var_zstandard(this%ncid, varid, zstandard_level) + !$omp end critical + _VERIFY(status) +#else + _FAIL("netcdf was not built with zstandard support") +#endif + end if + call this%put_var_attributes(var, varid, rc=status) _VERIFY(status) diff --git a/pfio/Variable.F90 b/pfio/Variable.F90 index 624954bfd162..84958a172945 100644 --- a/pfio/Variable.F90 +++ b/pfio/Variable.F90 @@ -30,6 +30,7 @@ module pFIO_VariableMod integer :: deflation = 0 ! default no compression integer :: quantize_algorithm = 0 ! default no quantization integer :: quantize_level = 0 ! default no quantize_level + integer :: zstandard_level = 0 ! default no zstandard integer, allocatable :: chunksizes(:) contains procedure :: get_type @@ -51,6 +52,7 @@ module pFIO_VariableMod procedure :: set_deflation procedure :: get_quantize_algorithm procedure :: get_quantize_level + procedure :: get_zstandard_level procedure :: is_attribute_present generic :: operator(==) => equal generic :: operator(/=) => not_equal @@ -69,7 +71,7 @@ module pFIO_VariableMod contains - function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflation, quantize_algorithm, quantize_level, rc) result(var) + function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflation, quantize_algorithm, quantize_level, zstandard_level, rc) result(var) type (Variable) :: var integer, optional, intent(in) :: type class (KeywordEnforcer), optional, intent(in) :: unusable @@ -79,6 +81,7 @@ function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflat integer, optional, intent(in) :: deflation integer, optional, intent(in) :: quantize_algorithm integer, optional, intent(in) :: quantize_level + integer, optional, intent(in) :: zstandard_level integer, optional, intent(out) :: rc integer:: empty(0) @@ -87,6 +90,7 @@ function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflat var%deflation = 0 var%quantize_algorithm = 0 var%quantize_level = 0 + var%zstandard_level = 0 var%chunksizes = empty var%dimensions = StringVector() var%attributes = StringAttributeMap() @@ -120,6 +124,10 @@ function new_Variable(unusable, type, dimensions, chunksizes,const_value, deflat var%quantize_level = quantize_level endif + if (present(zstandard_level)) then + var%zstandard_level = zstandard_level + endif + _RETURN(_SUCCESS) _UNUSED_DUMMY(unusable) contains @@ -313,6 +321,13 @@ function get_quantize_level(this) result(quantizeLevel) quantizeLevel=this%quantize_level end function get_quantize_level + function get_zstandard_level(this) result(zstandardLevel) + class (Variable), target, intent(In) :: this + integer :: zstandardLevel + + zstandardLevel=this%zstandard_level + end function get_zstandard_level + logical function equal(a, b) class (Variable), target, intent(in) :: a type (Variable), target, intent(in) :: b @@ -388,6 +403,7 @@ subroutine serialize(this, buffer, rc) buffer = [buffer, serialize_intrinsic(this%deflation)] buffer = [buffer, serialize_intrinsic(this%quantize_algorithm)] buffer = [buffer, serialize_intrinsic(this%quantize_level)] + buffer = [buffer, serialize_intrinsic(this%zstandard_level)] if( .not. allocated(this%chunksizes)) then buffer =[buffer,[1]] @@ -453,6 +469,9 @@ subroutine deserialize(this, buffer, rc) call deserialize_intrinsic(buffer(n:),this%quantize_level) length = serialize_buffer_length(this%quantize_level) n = n + length + call deserialize_intrinsic(buffer(n:),this%zstandard_level) + length = serialize_buffer_length(this%zstandard_level) + n = n + length call deserialize_intrinsic(buffer(n:),this%chunksizes) _RETURN(_SUCCESS) end subroutine deserialize