GEOS-ESM · mathomp4 · Sep 25, 2024 · Sep 25, 2024 · Sep 25, 2024 · Sep 25, 2024
@@ -26,6 +26,7 @@ module regrid_util_support_mod
       integer :: deflate, shave
       integer :: quantize_algorithm
       integer :: quantize_level
+      integer :: zstandard_level
       logical :: use_weights
    contains
       procedure :: create_grid
@@ -98,6 +99,7 @@ subroutine process_command_line(this,rc)
     this%deflate=0
     this%quantize_algorithm=0
     this%quantize_level=0
+    this%zstandard_level=0
     this%use_weights = .false.
     nargs = command_argument_count()
     do i=1,nargs
@@ -161,6 +163,9 @@ subroutine process_command_line(this,rc)
       case('-quantize_level')
          call get_command_argument(i+1,astr)
          read(astr,*)this%quantize_level
+      case('-zstandard_level')
+         call get_command_argument(i+1,astr)
+         read(astr,*)this%zstandard_level
       case('-file_weights')
          this%use_weights = .true.
       case('--help')
@@ -432,7 +437,7 @@ subroutine main()
 
          call ESMF_ClockSet(clock,currtime=time,_RC)
          if (.not. writer_created) then
-            call newWriter%create_from_bundle(bundle,clock,n_steps=tsteps,time_interval=tint,nbits_to_keep=support%shave,deflate=support%deflate,vertical_data=vertical_data,quantize_algorithm=support%quantize_algorithm,quantize_level=support%quantize_level,_RC)
+            call newWriter%create_from_bundle(bundle,clock,n_steps=tsteps,time_interval=tint,nbits_to_keep=support%shave,deflate=support%deflate,vertical_data=vertical_data,quantize_algorithm=support%quantize_algorithm,quantize_level=support%quantize_level,zstandard_level=support%zstandard_level,_RC)
             writer_created=.true.
          end if
 

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added zstandard compression support
+  - Note this requires netCDF-C to have been compiled with zstandard support. We have a CMake test to check for this
+    and enabling zstandard output in History will fail if the library does not support it
+
 ### Changed
 
 - ExtDataDriver.x now uses ExtData2G by default

@@ -29,6 +29,16 @@ set(QUANTIZE_TESTS
   "case34"
 )
 
+# We have one test that requires netcdf zstandard support
+set(ZSTD_TESTS
+  "case35"
+)
+
+# We have one test which requires *both* Quantize and ZSTD support
+set(QUANTIZE_AND_ZSTD_TESTS
+  "case36"
+)
+
 file(STRINGS "test_cases/extdata_2g_cases.txt" TEST_CASES_2G)
 
 foreach(TEST_CASE ${TEST_CASES_2G})
@@ -38,6 +48,16 @@ foreach(TEST_CASE ${TEST_CASES_2G})
     continue()
   endif()
 
+  # Skip tests that require ZSTD support if we don't have it
+  if (NOT NETCDF_HAS_ZSTD AND ${TEST_CASE} IN_LIST ZSTD_TESTS)
+    continue()
+  endif()
+
+  # Skip tests that require both Quantize and ZSTD support if we don't have it
+  if (NOT (NETCDF_HAS_QUANTIZE AND NETCDF_HAS_ZSTD) AND ${TEST_CASE} IN_LIST QUANTIZE_AND_ZSTD_TESTS)
+    continue()
+  endif()
+
   if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc)
     file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs)
   else()

@@ -0,0 +1,24 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: GenerateExports
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000
@@ -0,0 +1,29 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: CompareImports
+
+IMPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000
@@ -0,0 +1,4 @@
+CASES::
+CAP1.rc
+CAP2.rc
+::
@@ -0,0 +1,25 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM1.rc
+HIST_CF: HISTORY1.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20040115 210000
+20040215 210000
+20040315 210000
+20040415 210000
+20040515 210000
+20040615 210000
+20040715 210000
+20040815 210000
+20040915 210000
+20041015 210000
+20041115 210000
+20041215 210000
+::
+
@@ -0,0 +1,15 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM2.rc
+HIST_CF: HISTORY2.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20041125 210000
+20041126 210000
+::
+
@@ -0,0 +1,13 @@
+#CASE_SENSITIVE_VARIABLE_NAMES: .false.
+Ext_AllowExtrap: .false.
+Prefetch: .true.
+#DEBUG_LEVEL: 20
+
+PrimaryExports%%
+VAR2D NA  N N 0   none     none VAR2D case1.%y4.nc4
+VAR3D NA  N N 0   none     none VAR3D case1.%y4.nc4
+%%
+
+
+DerivedExports%%
+%%
@@ -0,0 +1,14 @@
+GRID_LABELS:
+::
+
+COLLECTIONS: case1
+::
+
+  case1.template:  '%y4.nc4',
+  case1.format:    'CFIO',
+  case1.frequency:  010000,
+  case1.duration: 000000,
+  case1.zstandard_level: 2,
+  case1.fields: 'VAR2D', 'Root',
+                'VAR3D', 'Root',
+                          ::
@@ -0,0 +1,5 @@
+GRID_LABELS:
+::
+
+COLLECTIONS:
+::
@@ -0,0 +1 @@
+Case, 12-month/12 time 2004 file with 2 updates, non-climatology
@@ -0,0 +1,5 @@
+Collections:
+   fstream1: {template: case1.%y4.nc4, valid_range: "2004-01-01/2005-01-01" }
+Exports:
+   VAR2D: {variable: VAR2D, collection: fstream1}
+   VAR3D: {variable: VAR3D, collection: fstream1}
@@ -0,0 +1,24 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: GenerateExports
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000
@@ -0,0 +1,29 @@
+NX: 1
+NY: 1
+
+Root.GRID_TYPE: LatLon
+Root.GRIDNAME: DC90x45-PC
+Root.LM: 3
+Root.IM_WORLD: 90
+Root.JM_WORLD: 45
+Root.POLE: 'PC'
+Root.DATELINE: 'DC'
+
+RUN_MODE: CompareImports
+
+IMPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+EXPORT_STATE::
+VAR2D , time , days , xy , c
+VAR3D , time , days , xyz , c
+::
+
+FILL_DEF::
+VAR2D time
+VAR3D time
+::
+
+REF_TIME: 20040701 000000
@@ -0,0 +1,4 @@
+CASES::
+CAP1.rc
+CAP2.rc
+::
@@ -0,0 +1,25 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM1.rc
+HIST_CF: HISTORY1.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20040115 210000
+20040215 210000
+20040315 210000
+20040415 210000
+20040515 210000
+20040615 210000
+20040715 210000
+20040815 210000
+20040915 210000
+20041015 210000
+20041115 210000
+20041215 210000
+::
+
@@ -0,0 +1,15 @@
+ROOT_NAME: Root
+ROOT_CF: AGCM2.rc
+HIST_CF: HISTORY2.rc
+
+BEG_DATE:     20040101 210000
+
+JOB_SGMT:     00001200 000000
+HEARTBEAT_DT:   3600
+
+#RUN_EXTDATA: .false.
+RUN_TIMES::
+20041125 210000
+20041126 210000
+::
+
@@ -0,0 +1,13 @@
+#CASE_SENSITIVE_VARIABLE_NAMES: .false.
+Ext_AllowExtrap: .false.
+Prefetch: .true.
+#DEBUG_LEVEL: 20
+
+PrimaryExports%%
+VAR2D NA  N N 0   none     none VAR2D case1.%y4.nc4
+VAR3D NA  N N 0   none     none VAR3D case1.%y4.nc4
+%%
+
+
+DerivedExports%%
+%%
@@ -0,0 +1,16 @@
+GRID_LABELS:
+::
+
+COLLECTIONS: case1
+::
+
+  case1.template:  '%y4.nc4',
+  case1.format:    'CFIO',
+  case1.frequency:  010000,
+  case1.duration: 000000,
+  case1.zstandard_level: 2,
+  case1.quantization_algorithm: 'granular_bitround',
+  case1.quantization_level: 5,
+  case1.fields: 'VAR2D', 'Root',
+                'VAR3D', 'Root',
+                          ::
@@ -0,0 +1,5 @@
+GRID_LABELS:
+::
+
+COLLECTIONS:
+::
@@ -0,0 +1 @@
+Case, 12-month/12 time 2004 file with 2 updates, non-climatology
@@ -0,0 +1,5 @@
+Collections:
+   fstream1: {template: case1.%y4.nc4, valid_range: "2004-01-01/2005-01-01" }
+Exports:
+   VAR2D: {variable: VAR2D, collection: fstream1}
+   VAR3D: {variable: VAR3D, collection: fstream1}
@@ -32,3 +32,5 @@ case31
 case32
 case33
 case34
+case35
+case36
@@ -39,3 +39,6 @@ path_to_script/run_extdatadriver_cases.py --builddir path_to_geos_install/bin --
 32. Case1 with deflate compression and NetCDF bitgroom quantization (only enabled if netcdf built with quantization support)
 33. Case1 with deflate compression and NetCDF bitround quantization (only enabled if netcdf built with quantization support)
 34. Case1 with deflate compression and NetCDF granular_bitround quantization (only enabled if netcdf built with quantization support)
+35. Case1 with zstandard compression (only enabled if netcdf built with zstandard support)
+36. Case1 with zstandard compression and NetCDF granular_bitround quantization (only enabled if netcdf built with quantization
+    support and zstandard support)
@@ -83,6 +83,7 @@ module MAPL_HistoryCollectionMod
      character(len=ESMF_MAXSTR)         :: quantize_algorithm_string
      integer                            :: quantize_algorithm
      integer                            :: quantize_level
+     integer                            :: zstandard_level
      integer                            :: slices
      integer                            :: Root
      integer                            :: Psize