diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildhdf5/action.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildhdf5/action.yml new file mode 100644 index 0000000000..4ba086823b --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildhdf5/action.yml @@ -0,0 +1,56 @@ +description: 'Build the HDF5 library' +inputs: + hdf5_version: + description: 'Tag in the HDF5 repository to use' + default: hdf5-1_12_2 + required: False + type: string + enable_logging: + description: 'Enable hdf5 logging (for debugging)' + default: False + required: False + type: boolean + enable_parallel: + description: 'Enable hdf5 parallel build' + default: True + required: False + type: boolean + enable_testing: + description: 'Build HDF5 tests' + default: False + required: False + type: boolean + + install_prefix: + description: 'Install path of hdf5' + default: ${GITHUB_WORKSPACE}/hdf5 + required: False + type: string + mpi_path: + description: 'Path to mpi install' + default: /usr + required: False + type: string +runs: + using: composite + steps: + - uses: actions/checkout@v3 + with: + repository: HDFGroup/hdf5 + path: hdf5-src + ref: ${{ inputs.hdf5_version }} + - id: hdf5-build + shell: bash + run: | + pushd hdf5-src + mkdir hdf5-build + pushd hdf5-build + export PATH=$PATH:${{ inputs.mpi_path }}/bin + cmake -DCMAKE_INSTALL_PREFIX=${{ inputs.install_prefix }} \ + -DHDF5_ENABLE_PARALLEL=${{ inputs.enable_parallel }} \ + -DHDF5_ENABLE_Z_LIB_SUPPORT=ON \ + -DBUILD_TESTING=${{ inputs.enable_testing }} \ + -DHDF5_BUILD_TOOLS=OFF \ + ../ + make + make install diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildmpich/action.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildmpich/action.yml new file mode 100644 index 0000000000..3437a6065f --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildmpich/action.yml @@ -0,0 +1,30 @@ +description: 'Build the MPICH library' +inputs: + mpich_version: + description: 'Tag in the MPICH repository to use' + default: v4.0.3 + required: False + type: string + install_prefix: + description: 'Install path of mpich' + default: $HOME/mpich + required: False + type: string +runs: + using: composite + steps: + - uses: actions/checkout@v3 + with: + repository: pmodels/mpich + path: mpich-src + ref: ${{ inputs.mpich_version }} + submodules: 'recursive' + - id: mpich-build + shell: bash + run: | + pushd mpich-src + ./autogen.sh + ./configure --prefix=${{ inputs.install_prefix }} + make -j4 + make install + popd diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildnetcdf/action.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildnetcdf/action.yml new file mode 100644 index 0000000000..a7d168f657 --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildnetcdf/action.yml @@ -0,0 +1,67 @@ +description: 'Build the netcdf fortran library' +inputs: + netcdf_version: + description: 'Tag in the netcdf repository to use' + default: v4.9.0 + required: False + type: string + shared_libraries: + description: 'Build shared libraries' + default: True + required: False + type: boolean + build_utilities: + description: 'Build netcdf utility tools' + default: False + required: False + type: boolean + enable_examples: + description: 'Build netcdf examples' + default: False + required: False + type: boolean + enable_netcdf4: + description: 'Build netcdf4 (hdf5) libraries' + default: True + required: False + type: boolean + enable_logging: + description: 'Enable netcdf logging (for debugging)' + default: False + required: False + type: boolean + enable_dap: + description: 'Enable netcdf DAP' + default: False + required: False + type: boolean + + install_prefix: + description: 'Install path of netcdf' + default: $HOME/netcdf-c + required: False + type: string + +runs: + using: composite + steps: + - uses: actions/checkout@v3 + with: + repository: Unidata/netcdf-c + path: netcdf-c-src + ref: ${{ inputs.NETCDF_VERSION }} + - id: netcdf-c-build + shell: bash + run: | + pushd $GITHUB_WORKSPACE/netcdf-c-src + autoreconf -i + config_args=--prefix=${{ inputs.install_prefix }} + if [ "${{ inputs.enable_dap }}" = "false" ]; + then + config_args="$config_args --disable-dap" + fi + echo "config_args=$config_args" + ./configure $config_args + make + make install + popd diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildnetcdff/action.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildnetcdff/action.yml new file mode 100644 index 0000000000..4063a11cb7 --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildnetcdff/action.yml @@ -0,0 +1,35 @@ +description: 'Build the netcdf fortran library' +inputs: + netcdf_fortran_version: + description: 'Tag in the netcdf fortran repository to use' + default: v5.6.0 + required: False + type: string + netcdf_c_path: + description: 'Path to the installed netcdf c code root' + default: /usr + required: False + type: string + install_prefix: + description: 'Install path of netcdf-fortran' + default: $HOME/netcdf-fortran + required: False + type: string + +runs: + using: composite + steps: + - uses: actions/checkout@v3 + with: + repository: Unidata/netcdf-fortran + path: netcdf-fortran-src + ref: ${{ inputs.NETCDF_FORTRAN_VERSION }} + - id: netcdf-fortran-build + shell: bash + run: | + pushd netcdf-fortran-src + export CPPFLAGS="$CPPFLAGS -I${{ inputs.netcdf_c_path }}/include" + export LDFLAGS="$LDFLAGS -L${{ inputs.netcdf_c_path }}/lib -lnetcdf" + ./configure --prefix=${{ inputs.install_prefix }} + make + make install diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildopenmpi/action.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildopenmpi/action.yml new file mode 100644 index 0000000000..f5f95fcf75 --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildopenmpi/action.yml @@ -0,0 +1,30 @@ +description: 'Build the OPENMPI library' +inputs: + openmpi_version: + description: 'Tag in the OPENMPI repository to use' + default: v4.1.4 + required: False + type: string + install_prefix: + description: 'Install path of openmpi' + default: ${GITHUB_WORKSPACE}/openmpi + required: False + type: string +runs: + using: composite + steps: + - uses: actions/checkout@v3 + with: + repository: open-mpi/ompi + path: openmpi-src + ref: ${{ inputs.openmpi_version }} + submodules: 'recursive' + - id: openmpi-build + shell: bash + run: | + pushd openmpi-src + perl ./autogen.pl + ./configure --prefix=${{ inputs.install_prefix }} + make -j4 + make install + popd diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildpnetcdf/action.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildpnetcdf/action.yml new file mode 100644 index 0000000000..43f2ad8c05 --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/buildpnetcdf/action.yml @@ -0,0 +1,52 @@ +description: 'Build the pnetcdf library' +inputs: + pnetcdf_version: + description: 'Tag in the pnetcdf repository to use' + default: checkpoint.1.12.3 + required: False + type: string + install_prefix: + description: 'Install path of pnetcdf' + default: $GITHUB_WORKSPACE/pnetcdf + required: False + type: string + enable_shared: + description: 'Enable shared library build' + default: True + required: False + type: boolean + enable_fortran: + description: "Build the fortran library" + default: False + required: False + type: boolean +runs: + using: composite + steps: + - uses: actions/checkout@v3 + with: + repository: Parallel-NetCDF/PnetCDF + path: pnetcdf-src + ref: ${{ inputs.pnetcdf_version }} + - id: Build-PnetCDF + shell: bash + run: | + pushd pnetcdf-src + config_opts="--prefix=${{ inputs.install_prefix }} --disable-cxx " + if [ "${{ inputs.enable_fortran }}" = "false" ]; + then + config_opts="${config_opts} --disable-fortran" + fi + if [ "${{ inputs.enable_shared }}" = "true" ]; + then + config_opts="${config_opts} --enable-shared=yes" + fi + + config_opts="${config_opts}" + echo "config_opts=$config_opts" + autoreconf -i + ./configure $config_opts + + make + make install + popd diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/intelcompilers/action.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/intelcompilers/action.yml new file mode 100644 index 0000000000..5f7c658d66 --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/intelcompilers/action.yml @@ -0,0 +1,23 @@ +description: 'Install Intel Compilers' +runs: + using: composite + steps: + - uses: actions/checkout@v3 + - name: setup repo + shell: bash + run: | + wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + sudo echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list + sudo apt-get update + - name: install + shell: bash + run: | + sudo apt-get install -y intel-oneapi-common-vars + sudo apt-get install -y intel-oneapi-compiler-fortran + sudo apt-get install -y intel-oneapi-mkl + sudo apt-get install -y intel-oneapi-mpi + sudo apt-get install -y intel-oneapi-mpi-devel + source /opt/intel/oneapi/setvars.sh + printenv >> $GITHUB_ENV diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/parallelio_autotools/action.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/parallelio_autotools/action.yml new file mode 100644 index 0000000000..fd85ebe8af --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/parallelio_autotools/action.yml @@ -0,0 +1,131 @@ +description: 'Build the parallelio library using cmake' +inputs: + parallelio_version: + description: 'Tag in the parallelio repository to use' + default: pio2_5_10 + required: False + type: string + shared_libraries: + description: 'Build shared libraries' + default: True + required: False + type: boolean + enable_examples: + description: 'Build parallelio examples' + default: False + required: False + type: boolean + enable_logging: + description: 'Enable parallelio logging (for debugging)' + default: False + required: False + type: boolean + enable_timing: + description: 'Enable parallelio timing library (GPTL)' + default: False + required: False + type: boolean + enable_doc: + description: 'Build parallelio Documentation' + default: False + required: False + type: boolean + enable_coverage: + description: 'Enable parallelio code coverage' + default: False + required: False + type: boolean + enable_netcdf_integration: + description: 'Enable netcdf integration' + default: False + required: False + type: boolean + with_mpi-serial: + description: 'Build with mpi-serial library' + default: + required: False + type: string + with_pnetcdf: + description: 'Build with pnetcdf library' + default: + required: False + type: string + with_netcdf: + description: 'Build with netcdf library' + default: /usr + required: False + type: string + with_netcdff: + description: 'Build with netcdff library' + default: + required: False + type: string + with_valgrind: + description: 'Build the parallelio valgrind leak check ' + default: False + required: False + type: boolean + enable_fortran: + description: 'Build the parallelio Fortran Library ' + default: False + required: False + type: boolean + extra_cflags: + description: 'Additional cflags to use' + default: + required: False + type: string + + extra_fflags: + description: 'Additional fflags to use' + default: + required: False + type: string + with_mpiexec: + description: 'alternate mpiexec command' + default: mpiexec + required: False + type: string + install_prefix: + description: 'Install path of parallelio' + default: ${GITHUB_WORKSPACE}/parallelio + required: False + type: string + src_path: + description: 'Path to parallelio source' + default: ${GITHUB_WORKSPACE}/parallelio-src + required: False + type: string + +runs: + using: composite + steps: + - name: Check if already present + uses: andstor/file-existence-action@v2 + with: + files: ${{ inputs.src_path }} + - name: get parallelio + if: ${{ steps.check_files.outputs.files_exists != 'true' }} + uses: actions/checkout@v3 + with: + repository: NCAR/ParallelIO + path: ${{ inputs.src_path }} + ref: ${{ inputs.parallelio_version }} + - id: parallelio-build + shell: bash + run: | + cd ${{ inputs.src_path }} + autoreconf -i + config_opts='--prefix=${{ inputs.install_prefix }} ' + if [[ -z "${{ inputs.with_pnetcdf }}" ]]; + then + config_opts="${config_opts} --disable-pnetcdf " + fi + if [ "${{ inputs.enable_fortran }}" = "true" ]; + then + config_opts="${config_opts} --enable-fortran=yes " + fi + echo "config_opts = ${config_opts} --with-mpiexec=${{ inputs.with_mpiexec }}" + ./configure ${config_opts} --with-mpiexec="${{ inputs.with_mpiexec }}" + make -j 4 VERBOSE=1 + make install diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/parallelio_cmake/action.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/parallelio_cmake/action.yml new file mode 100644 index 0000000000..5a135fce2c --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/actions/parallelio_cmake/action.yml @@ -0,0 +1,141 @@ +description: 'Build the parallelio library using cmake' +inputs: + parallelio_version: + description: 'Tag in the parallelio repository to use' + default: pio2_5_9 + required: False + type: string + shared_libraries: + description: 'Build shared libraries' + default: True + required: False + type: boolean + enable_examples: + description: 'Build parallelio examples' + default: False + required: False + type: boolean + enable_logging: + description: 'Enable parallelio logging (for debugging)' + default: False + required: False + type: boolean + enable_timing: + description: 'Enable parallelio timing library (GPTL)' + default: False + required: False + type: boolean + enable_doc: + description: 'Build parallelio Documentation' + default: False + required: False + type: boolean + enable_coverage: + description: 'Enable parallelio code coverage' + default: False + required: False + type: boolean + enable_netcdf_integration: + description: 'Enable netcdf integration' + default: False + required: False + type: boolean + with_mpi-serial: + description: 'Build with mpi-serial library' + default: + required: False + type: string + with_valgrind: + description: 'Build the parallelio valgrind leak check ' + default: False + required: False + type: boolean + enable_fortran: + description: 'Build the parallelio Fortran Library ' + default: False + required: False + type: boolean + extra_cflags: + description: 'Additional cflags to use' + default: + required: False + type: string + + extra_ldflags: + description: 'Additional ldflags to use' + default: + required: False + type: string + + extra_fflags: + description: 'Additional fflags to use' + default: + required: False + type: string + mpiexec_flags: + description: 'extra mpiexec flags' + default: + required: False + type: string + netcdf_c_include_dir: + default: /usr/include + required: False + type: string + netcdf_c_library: + default: /usr/lib/x86_64-linux-gnu/libnetcdf.so + required: False + type: string + netcdf_fortran_include_dir: + default: /usr/include + required: False + type: string + netcdf_fortran_library: + default: /usr/lib/x86_64-linux-gnu/libnetcdff.so + required: False + type: string + pnetcdf_include_dir: + default: /usr/include + required: False + type: string + pnetcdf_library: + default: /usr/lib/x86_64-linux-gnu/libpnetcdf.so + required: False + type: string + install_prefix: + description: 'Install path of parallelio' + default: ${GITHUB_WORKSPACE}/parallelio + required: False + type: string + +runs: + using: composite + steps: + - uses: actions/checkout@v3 + with: + repository: NCAR/ParallelIO + path: parallelio-src + ref: ${{ inputs.parallelio_version }} + - id: parallelio-build + shell: bash + run: | + mkdir build + cd build + export CFLAGS="$CFLAGS ${{ inputs.extra_cflags }}" + export FFLAGS="$FFLAGS ${{ inputs.extra_fflags }}" + export LDFLAGS="$LDFLAGS ${{ inputs.extra_ldflags }}" + cmake -Wno-dev -DPIO_ENABLE_LOGGING=${{ inputs.enable_logging }} \ + -DPIO_ENABLE_FORTRAN=${{ inputs.enable_fortran }} \ + -DPIO_ENABLE_EXAMPLES=${{ inputs.enable_examples }} \ + -DPIO_ENABLE_TIMING=${{ inputs.enable_timing }} \ + -DCMAKE_INSTALL_PREFIX=${{ inputs.install_prefix }} \ + -DMPIEXEC_PREFLAGS="${{ inputs.mpiexec_flags }}" \ + -DNetCDF_C_INCLUDE_DIR=${{ inputs.netcdf_c_include_dir }} \ + -DNetCDF_C_LIBRARY=${{ inputs.netcdf_c_library }} \ + -DNetCDF_Fortran_INCLUDE_DIR=${{ inputs.netcdf_fortran_include_dir }} \ + -DNetCDF_Fortran_LIBRARY=${{ inputs.netcdf_fortran_library }} \ + -DPnetCDF_C_INCLUDE_DIR=${{ inputs.pnetcdf_include_dir }} \ + -DPnetCDF_C_LIBRARY=${{ inputs.pnetcdf_library }} \ + $GITHUB_WORKSPACE/parallelio-src + make VERBOSE=1 + #make tests + make install diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/autotools.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/autotools.yml index 8708fed114..eba2d60bfe 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/autotools.yml +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/autotools.yml @@ -1,10 +1,10 @@ -name: autotools_ubuntu_latest +name: autotools ubuntu openmpi latest with Wall on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] jobs: build: @@ -14,22 +14,21 @@ jobs: env: CC: mpicc FC: mpifort - CPPFLAGS: "-I/usr/include -I/usr/local/include -I/home/runner/pnetcdf/include" - LDFLAGS: "-L/home/runner/pnetcdf/lib" - PNETCDF_VERSION: 1.12.3 - FCFLAGS: "-fallow-argument-mismatch" + CFLAGS: "-std=c99 -Wall -Werror" + CPPFLAGS: "-I/usr/include -I/usr/lib/x86_64-linux-gnu/netcdf/mpi/include/ " + LDFLAGS: "-L/usr/lib/x86_64-linux-gnu -lnetcdf_mpi -lpnetcdf" + FCFLAGS: "-Wall -Werror -fallow-argument-mismatch -Wno-conversion" steps: - uses: actions/checkout@v3 - name: Installs run: | set -x - # sudo gem install apt-spy2 - # sudo apt-spy2 check - # sudo apt-spy2 fix --commit - # after selecting a specific mirror, we need to run 'apt-get update' - sudo apt-get update - sudo apt-get install netcdf-bin - sudo apt-get install libnetcdf-dev + sudo apt-get update + sudo apt-get install netcdf-bin + sudo apt-get install libnetcdf-mpi-19 + sudo apt-get install libnetcdf-mpi-dev + sudo apt-get install pnetcdf-bin + sudo apt-get install libpnetcdf-dev sudo apt-get install doxygen sudo apt-get install graphviz sudo apt-get install wget @@ -38,29 +37,21 @@ jobs: sudo apt-get install libz-dev sudo apt-get install openmpi-bin sudo apt-get install libopenmpi-dev - - name: cache-pnetcdf - id: cache-pnetcdf - uses: actions/cache@v3 + cd /usr/lib/x86_64-linux-gnu + sudo ln -fs libnetcdf_mpi.so libnetcdf.so + - name: Build ParallelIO with autotools + uses: ./.github/actions/parallelio_autotools with: - path: ~/pnetcdf - key: pnetcdf-${{ runner.os }}-${{ env.PNETCDF_VERSION }} - - - name: build-pnetcdf - if: steps.cache-pnetcdf.outputs.cache-hit != 'true' + enable_fortran: True + with_pnetcdf: /usr + with_mpiexec: 'mpiexec --oversubscribe' + parallelio_version: ${{ env.GITHUB_SHA }} + src_path: ${GITHUB_WORKSPACE} + - name: make check run: | - set -x - wget https://parallel-netcdf.github.io/Release/pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz &> /dev/null - tar -xzvf pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz - pushd pnetcdf-${{ env.PNETCDF_VERSION }} - ./configure --prefix=/home/runner/pnetcdf --enable-shared --disable-cxx - make - sudo make install - popd + cd $GITHUB_WORKSPACE + make check - - name: autoreconf - run: autoreconf -i - - name: configure - run: ./configure --enable-fortran --with-mpiexec='mpiexec --oversubscribe' - #run: ./configure --enable-fortran --enable-docs --with-mpiexec='mpiexec --oversubscribe' - - name: make check - run: make -j check +# - name: Setup tmate session +# if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/cmake.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/cmake.yml index 8d7443c2c6..e554dd6904 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/cmake.yml +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/cmake.yml @@ -1,10 +1,11 @@ -name: cmake +name: cmake ubuntu openmpi latest +#with AddressSanitizer on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] jobs: build: @@ -14,51 +15,36 @@ jobs: env: CC: mpicc FC: mpifort - CPPFLAGS: "-I/usr/include -I/usr/local/include -I/home/runner/pnetcdf/include" - LDFLAGS: "-L/home/runner/pnetcdf/lib" - LD_LIBRARY_PATH: "/home/runner/pnetcdf/lib" - PNETCDF_VERSION: 1.12.3 FCFLAGS: "-fallow-argument-mismatch" +# LDFLAGS: "-static-libasan" +# ASAN_OPTIONS: "detect_odr_violation=0" steps: - uses: actions/checkout@v3 - name: Installs run: | set -x - #sudo gem install apt-spy2 - #sudo apt-spy2 check - #sudo apt-spy2 fix --commit - # after selecting a specific mirror, we need to run 'apt-get update' sudo apt-get update - sudo apt-get install netcdf-bin libnetcdf-dev doxygen graphviz wget gfortran libjpeg-dev libz-dev openmpi-bin libopenmpi-dev cmake - - - name: cache-pnetcdf - id: cache-pnetcdf - uses: actions/cache@v3 - with: - path: ~/pnetcdf - key: pnetcdf-${{ runner.os }}-${{ env.PNETCDF_VERSION }} - - - name: build-pnetcdf - if: steps.cache-pnetcdf.outputs.cache-hit != 'true' - run: | - set -x - wget https://parallel-netcdf.github.io/Release/pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz &> /dev/null - tar -xzvf pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz - pushd pnetcdf-${{ env.PNETCDF_VERSION }} - ./configure --prefix=/home/runner/pnetcdf --enable-shared --disable-cxx - make - sudo make install - popd + sudo apt-get install netcdf-bin libnetcdf-dev doxygen graphviz wget gfortran \ + libjpeg-dev libz-dev openmpi-bin libopenmpi-dev cmake pnetcdf-bin libpnetcdf-dev libnetcdff-dev + nc-config --all - name: cmake build + uses: ./.github/actions/parallelio_cmake + with: + parallelio_version: ${{ env.GITHUB_SHA }} + enable_fortran: True + enable_logging: True + install_prefix: ${GITHUB_WORKSPACE}/parallelio + mpiexec_flags: --oversubscribe +# extra_cflags: "-g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan" +# extra_fflags: "-g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan" + - name: parallelio tests run: | - set -x - mkdir build - cd build - cmake -Wno-dev -DNetCDF_C_LIBRARY=/usr/lib/x86_64-linux-gnu/libnetcdf.so \ - -DNetCDF_C_INCLUDE_DIR=/usr/include -DPnetCDF_PATH='/home/runner/pnetcdf' \ - -DPIO_ENABLE_FORTRAN=Off \ - -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off -DMPIEXEC_PREFLAGS="--oversubscribe" .. - make VERBOSE=1 + pushd $GITHUB_WORKSPACE/build make tests VERBOSE=1 ctest -VV + popd + +# - name: Setup tmate session +# if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/cmake_ubuntu_latest.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/cmake_ubuntu_latest.yml index d998aae307..eb24571d5e 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/cmake_ubuntu_latest.yml +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/cmake_ubuntu_latest.yml @@ -4,10 +4,10 @@ name: cmake_ubuntu-latest on: push: branches: - - master + - mainnot pull_request: branches: - - master + - mainnot jobs: build: @@ -17,10 +17,10 @@ jobs: env: CC: mpicc FC: mpifort - CPPFLAGS: "-I/usr/include -I/usr/local/include -I/home/runner/pnetcdf/include" - LDFLAGS: "-L/home/runner/pnetcdf/lib" - LD_LIBRARY_PATH: "/home/runner/pnetcdf/lib" - PNETCDF_VERSION: 1.12.3 + CPPFLAGS: "-I/usr/include -I/usr/local/include -I${GITHUB_WORKSPACE}/pnetcdf/include" + LDFLAGS: "-L${GITHUB_WORKSPACE}/pnetcdf/lib -L/usr/lib" + LD_LIBRARY_PATH: "${GITHUB_WORKSPACE}/pnetcdf/lib:/usr/lib" + PNETCDF_VERSION: checkpoint.1.12.3 FCFLAGS: "-fallow-argument-mismatch" steps: - uses: actions/checkout@v3 @@ -41,28 +41,25 @@ jobs: path: ~/pnetcdf key: pnetcdf-${{ runner.os }}-${{ env.PNETCDF_VERSION }} - - name: build-pnetcdf + - name: Build PNetCDF if: steps.cache-pnetcdf.outputs.cache-hit != 'true' - run: | - set -x - wget https://parallel-netcdf.github.io/Release/pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz &> /dev/null - tar -xzvf pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz - pushd pnetcdf-${{ env.PNETCDF_VERSION }} - ./configure --prefix=/home/runner/pnetcdf --enable-shared --disable-cxx - make - sudo make install - popd + uses: ./.github/actions/buildpnetcdf + with: + pnetcdf_version: ${{ env.PNETCDF_VERSION }} + install_prefix: $HOME/pnetcdf - name: cmake build + uses: ./.github/actions/parallelio_cmake + with: + parallelio_version: ${{ env.GITHUB_SHA }} + pnetcdf_include_dir: $GITHUB_WORKSPACE/pnetcdf/include + pnetcdf_library: $GITHUB_WORKSPACE/pnetcdf/lib/libpnetcdf.so + netcdf_include_dir: /usr/include + netcdf_library: /usr/lib/x86_64-linux-gnu/libnetcdf.so + install_prefix: $GITHUB_WORKSPACE/parallelio + with_mpiexec: /usr/bin/mpiexec --oversubscribe + - name: run tests run: | - set -x - mkdir build - cd build - cmake -Wno-dev -DPIO_ENABLE_NETCDF_INTEGRATION=OFF \ - -DNetCDF_C_LIBRARY=/usr/lib/x86_64-linux-gnu/libnetcdf.so \ - -DNetCDF_C_INCLUDE_DIR=/usr/include -DPnetCDF_PATH='/home/runner/pnetcdf' \ - -DPIO_ENABLE_FORTRAN=OFF \ - -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off -DMPIEXEC_PREFLAGS="--oversubscribe" .. - make VERBOSE=1 + pushd $GITHUB_WORKSPACE/parallelio-src make tests VERBOSE=1 ctest -VV diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/intel.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/intel.yml new file mode 100644 index 0000000000..10eb10fcaa --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/intel.yml @@ -0,0 +1,131 @@ +name: Intel OneAPI +# Tests ParallelIO using Intel Compiler and IMPI library. +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + env: + CC: mpicc + FC: mpiifort + # Versions should match the github tag names + PNETCDF_VERSION: checkpoint.1.12.3 + NETCDF_C_VERSION: v4.9.2 + NETCDF_FORTRAN_VERSION: v4.6.1 + HDF5_VERSION: hdf5_1_12_2 + steps: + - uses: actions/checkout@v3 + - name: Installs + run: | + set -x + sudo apt-get update + sudo apt-get install libcurl4-gnutls-dev + sudo apt-get install libz-dev + echo "/opt/intel/oneapi/compiler/2023.2.1/linux/bin/intel64/" >> $GITHUB_PATH + - name: cache intel compiler + id: cache-intel + uses: actions/cache@v3 + with: + path: /opt/intel/oneapi + key: intel-${{ runner.os }} + - name: Install Intel OneAPI + if: steps.cache-intel.outputs.cache-hit != 'true' + uses: ./.github/actions/intelcompilers + - name: Prep Intel OneAPI + if: steps.cache-intel.outputs.cache-hit == 'true' + run: | + source /opt/intel/oneapi/setvars.sh + printenv >> $GITHUB_ENV + + - name: cache-hdf5 + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: ~/work/ParallelIO/ParallelIO/hdf5 + key: hdf5-${{ runner.os }}-${{ env.HDF5_VERSION }}-impi + - name: build-hdf5 + if: steps.cache-hdf5.outputs.cache-hit != 'true' + uses: ./.github/actions/buildhdf5 + with: + install_prefix: ${GITHUB_WORKSPACE}/hdf5 + enable_parallel: True + hdf5_version: ${{ env.HDF5_VERSION }} + mpi_path: /opt/intel/mpi + - name: cache netcdf C + id: cache-netcdf-c + uses: actions/cache@v3 + with: + path: ~/work/ParallelIO/ParallelIO/netcdf-c + key: netcdf-c-${{ runner.os }}-${{ env.NETCDF_C_VERSION }}-impi-hdf5-${{ env.HDF5_VERSION }} + - name: cache netcdf Fortran + id: cache-netcdf-f + uses: actions/cache@v3 + with: + path: ~/work/ParallelIO/ParallelIO/netcdf-f + key: netcdf-f-${{ runner.os }}-${{ env.NETCDF_FORTRAN_VERSION }}-impi-hdf5-${{ env.HDF5_VERSION }} + + - name: prep netcdf-c + run: | + export PATH=$GITHUB_WORKSPACE/hdf5/bin:$GITHUB_WORKSPACE/netcdf/bin:$PATH + export LDFLAGS="$LDFLAGS -L$GITHUB_WORKSPACE/hdf5/lib -L/usr/lib/x86_64-linux-gnu/ -lcurl" + export CPPFLAGS="$CPPFLAGS -I$GITHUB_WORKSPACE/hdf5/include -I$GITHUB_WORKSPACE/netcdf/include" + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/hdf5/lib:/usr/lib/x86_64-linux-gnu/:/opt/intel/oneapi/compiler/2023.2.1/linux/compiler/lib/intel64_lin/" + printenv >> $GITHUB_ENV + - name: build-netcdf-c + if: steps.cache-netcdf-c.outputs.cache-hit != 'true' + uses: ./.github/actions/buildnetcdf + with: + netcdf_version: ${{ env.NETCDF_C_VERSION }} + install_prefix: ${GITHUB_WORKSPACE}/netcdf-c + + - name: Build NetCDF Fortran + if: steps.cache-netcdf-f.outputs.cache-hit != 'true' + uses: ./.github/actions/buildnetcdff + with: + netcdf_fortran_version: ${{ env.NETCDF_FORTRAN_VERSION }} + install_prefix: ${GITHUB_WORKSPACE}/netcdf-f + netcdf_c_path: ${GITHUB_WORKSPACE}/netcdf-c + + - name: cache-pnetcdf + id: cache-pnetcdf + uses: actions/cache@v3 + with: + path: ~/work/ParallelIO/ParallelIO/pnetcdf + key: pnetcdf-${{ runner.os }}-${{ env.PNETCDF_VERSION }}-impi-5 + + - name: Build PNetCDF + if: steps.cache-pnetcdf.outputs.cache-hit != 'true' + uses: ./.github/actions/buildpnetcdf + with: + pnetcdf_version: ${{ env.PNETCDF_VERSION }} + install_prefix: ${GITHUB_WORKSPACE}/pnetcdf + + - name: cmake build + uses: ./.github/actions/parallelio_cmake + with: + parallelio_version: ${{ env.GITHUB_SHA }} + enable_fortran: True + netcdf_c_library: $GITHUB_WORKSPACE/netcdf-c/lib/libnetcdf.so + netcdf_c_include_dir: $GITHUB_WORKSPACE/netcdf-c/include + netcdf_fortran_library: $GITHUB_WORKSPACE/netcdf-f/lib/libnetcdff.so + netcdf_fortran_include_dir: $GITHUB_WORKSPACE/netcdf-f/include + pnetcdf_library: $GITHUB_WORKSPACE/pnetcdf/lib/libpnetcdf.a + pnetcdf_include_dir: $GITHUB_WORKSPACE/pnetcdf/include + install_prefix: $GITHUB_WORKSPACE/parallelio + - name: parallelio tests + run: | + pushd $GITHUB_WORKSPACE/build + make tests + ctest -VV -E test_async_ + popd + # the following can be used by developers to login to the github server in case of errors + # see https://github.com/marketplace/actions/debugging-with-tmate for further details +# - name: Setup tmate session +# if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_hdf5_no_pnetcdf_ncint_mpich.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_hdf5_no_pnetcdf_ncint_mpich.yml index 6a8fb24f05..81b0d0635e 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_hdf5_no_pnetcdf_ncint_mpich.yml +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_hdf5_no_pnetcdf_ncint_mpich.yml @@ -1,119 +1,108 @@ name: netcdf_hdf5_no_pnetcdf_ncint_mpich on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] jobs: build: - runs-on: ubuntu-latest - env: - CPPFLAGS: "-I/home/runner/mpich/include -I/home/runner/hdf5/include -I/home/runner/netcdf-c/include -I/home/runner/netcdf-fortran/include" - LDFLAGS: "-L/home/runner/mpich/lib -L/home/runner/hdf5/lib -L/home/runner/netcdf-c/lib -L/home/runner/netcdf-fortran/lib" + CPPFLAGS: "-I${GITHUB_WORKSPACE}/mpich/include" + LDFLAGS: "-L${GITHUB_WORKSPACE}/mpich/lib " # Note issue https://github.com/NCAR/ParallelIO/issues/1889 netcdf integration currently only works with netcdf 4.7.4 - NETCDF_C_VERSION: 4.7.4 - NETCDF_F_VERSION: 4.5.4 - MPICH_VERSION: 4.0.3 - HDF5_VERSION_MAJOR: 1.12 - HDF5_VERSION_PATCH: 2 + NETCDF_C_VERSION: v4.9.2 + NETCDF_FORTRAN_VERSION: v4.6.1 + MPICH_VERSION: v4.1.2 + HDF5_VERSION: hdf5_1_12_2 FFLAGS: "-fallow-argument-mismatch" FCFLAGS: "-fallow-argument-mismatch" steps: - uses: actions/checkout@v3 - name: Installs run: | - sudo apt-get install doxygen graphviz wget gfortran libjpeg-dev libz-dev + sudo apt-get install doxygen graphviz wget gfortran libjpeg-dev libz-dev libcurl4-gnutls-dev - name: cache-mpich id: cache-mpich uses: actions/cache@v3 with: - path: ~/mpich + path: ~/work/ParallelIO/ParallelIO/mpich key: mpich-${{ runner.os }}-${{ env.MPICH_VERSION }} - name: build-mpich if: steps.cache-mpich.outputs.cache-hit != 'true' - run: | - wget http://www.mpich.org/static/downloads/${{ env.MPICH_VERSION }}/mpich-${{ env.MPICH_VERSION }}.tar.gz &> /dev/null - tar -xzf mpich-${{ env.MPICH_VERSION }}.tar.gz - pushd mpich-${{ env.MPICH_VERSION }} - ./configure --prefix=/home/runner/mpich - make - sudo make install - popd + uses: ./.github/actions/buildmpich + with: + install_prefix: ${GITHUB_WORKSPACE}/mpich + mpich_version: ${{ env.MPICH_VERSION }} + - name: cache-hdf5 id: cache-hdf5 uses: actions/cache@v3 with: - path: ~/hdf5 - key: hdf5-${{ runner.os }}-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}-mpich-${{ env.MPICH_VERSION }} + path: ~/work/ParallelIO/ParallelIO/hdf5 + key: hdf5-${{ runner.os }}-${{ env.HDF5_VERSION }}-mpich-${{ env.MPICH_VERSION }} - name: build-hdf5 if: steps.cache-hdf5.outputs.cache-hit != 'true' - run: | - set -x - export PATH="/home/runner/mpich/bin:$PATH" - export CC=/home/runner/mpich/bin/mpicc - wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${{ env.HDF5_VERSION_MAJOR }}/hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}/src/hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}.tar.gz &> /dev/null - tar -xzf hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}.tar.gz - pushd hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }} - ./configure --prefix=/home/runner/hdf5 --enable-parallel --disable-tools --disable-fortran --disable-cxx --enable-parallel-tests - make - sudo make install - popd - - name: cache-netcdf-c - id: cache-netcdf-c - uses: actions/cache@v3 + uses: ./.github/actions/buildhdf5 with: - path: ~/netcdf-c - key: netcdf-c-${{ runner.os }}-${{ env.NETCDF_C_VERSION }}-mpich-${{ env.MPICH_VERSION }}-hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }} + install_prefix: $GITHUB_WORKSPACE/hdf5 + enable_parallel: True + hdf5_version: ${{ env.HDF5_VERSION }} + mpi_path: $GITHUB_WORKSPACE/mpich - - name: build-netcdf-c - if: steps.cache-netcdf-c.outputs.cache-hit != 'true' + - name: cache-netcdf + id: cache-netcdf + uses: actions/cache@v3 + with: + path: ~/work/ParallelIO/ParallelIO/netcdf + key: netcdf-${{ runner.os }}-${{ env.NETCDF_C_VERSION }}-${{ env.NETCDF_FORTRAN_VERSION }}-mpich-${{ env.MPICH_VERSION }}-hdf5-${{ env.HDF5_VERSION }} + - name: Prep Netcdf run: | - set -x - export PATH="/home/runner/mpich/bin:$PATH" - export CC=/home/runner/mpich/bin/mpicc - wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v${{ env.NETCDF_C_VERSION }}.tar.gz - tar -xzf v${{ env.NETCDF_C_VERSION }}.tar.gz &> /dev/null - pushd netcdf-c-${{ env.NETCDF_C_VERSION }} - ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities - make -j - sudo make install - popd + export CC=mpicc + export FC=mpifort + export PATH="${GITHUB_WORKSPACE}/hdf5/bin:${GITHUB_WORKSPACE}/mpich/bin:${GITHUB_WORKSPACE}/netcdf/bin:$PATH" + export CPPFLAGS="$CPPFLAGS -I${GITHUB_WORKSPACE}/hdf5/include -I${GITHUB_WORKSPACE}/netcdf/include" + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${GITHUB_WORKSPACE}/hdf5/lib:/usr/lib/x86_64-linux-gnu/" + export LDFLAGS="$LDFLAGS -L${GITHUB_WORKSPACE}/hdf5/lib -L${GITHUB_WORKSPACE}/netcdf/lib -L/usr/lib/x86_64-linux-gnu/ -lcurl" + printenv >> $GITHUB_ENV + + + - name: Build NetCDF C + if: steps.cache-netcdf.outputs.cache-hit != 'true' + uses: ./.github/actions/buildnetcdf + with: + netcdf_version: ${{ env.NETCDF_C_VERSION }} + install_prefix: ${GITHUB_WORKSPACE}/netcdf - name: cache-netcdf-fortran id: cache-netcdf-fortran uses: actions/cache@v3 with: path: ~/netcdf-fortran - key: netcdf-fortran-${{ runner.os }}-${{ env.NETCDF_F_VERSION }}-mpich-${{ env.MPICH_VERSION }}-hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }} + key: netcdf-fortran-${{ runner.os }}-${{ env.NETCDF_FORTRAN_VERSION }}-mpich-${{ env.MPICH_VERSION }}-hdf5-${{ env.HDF5_VERSION }} - - name: build-netcdf-fortran - if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' - run: | - set -x - export PATH="/home/runner/mpich/bin:$PATH" - export CC=/home/runner/mpich/bin/mpicc - wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v${{ env.NETCDF_F_VERSION }}.tar.gz - tar -zxf v${{ env.NETCDF_F_VERSION }}.tar.gz &> /dev/null - pushd netcdf-fortran-${{ env.NETCDF_F_VERSION }} - ./configure --prefix=/home/runner/netcdf-fortran - make -j - sudo make install - popd + - name: Build NetCDF Fortran + if: steps.cache-netcdf.outputs.cache-hit != 'true' + uses: ./.github/actions/buildnetcdff + with: + netcdf_fortran_version: ${{ env.NETCDF_FORTRAN_VERSION }} + install_prefix: ${GITHUB_WORKSPACE}/netcdf + netcdf_c_path: ${GITHUB_WORKSPACE}/netcdf - - name: autotools build + - name: Build ParallelIO with autotools + uses: ./.github/actions/parallelio_autotools + with: + enable_fortran: True + enable_netcdf_integration: True + src_path: ${GITHUB_WORKSPACE} + parallelio_version: ${{ env.GITHUB_SHA }} + - name: make check run: | - set -x - gcc --version - export PATH=/home/runner/mpich/bin:/home/runner/netcdf-c/bin:$PATH - export CC=/home/runner/mpich/bin/mpicc - export FC=/home/runner/mpich/bin/mpifort - autoreconf -i - ./configure --enable-fortran --enable-netcdf-integration --disable-pnetcdf + cd $GITHUB_WORKSPACE make -j check - - +# - name: Setup tmate session +# if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_hdf5_pnetcdf_ncint_mpich_asan.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_hdf5_pnetcdf_ncint_mpich_asan.yml index 7ee67f90f2..5b055f0466 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_hdf5_pnetcdf_ncint_mpich_asan.yml +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_hdf5_pnetcdf_ncint_mpich_asan.yml @@ -1,9 +1,9 @@ name: netcdf_hdf5_pnetcdf_ncint_mpich_asan on: push: - branches: [ master ] + branches: [ maintodo ] pull_request: - branches: [ master ] + branches: [ maintodo ] jobs: build: @@ -11,15 +11,12 @@ jobs: runs-on: ubuntu-latest env: - CPPFLAGS: "-I/home/runner/mpich/include -I/home/runner/hdf5/include -I/home/runner/netcdf-c/include -I/home/runner/netcdf-fortran/include -I/home/runner/pnetcdf/include" - LDFLAGS: "-L/home/runner/mpich/lib -L/home/runner/hdf5/lib -L/home/runner/netcdf-c/lib -L/home/runner/netcdf-fortran/lib -L/home/runner/pnetcdf/lib" # Note issue https://github.com/NCAR/ParallelIO/issues/1889 netcdf integration only currently works with netcdf-c 4.7.4 NETCDF_C_VERSION: 4.7.4 NETCDF_F_VERSION: 4.5.4 - PNETCDF_VERSION: 1.12.3 - MPICH_VERSION: 4.0.3 - HDF5_VERSION_MAJOR: 1.12 - HDF5_VERSION_PATCH: 2 + PNETCDF_VERSION: checkpoint.1.12.3 + MPICH_VERSION: v4.0.3 + HDF5_VERSION: hdf5_1_12_2 FCFLAGS: "-fallow-argument-mismatch" FFLAGS: "-fallow-argument-mismatch" steps: @@ -36,54 +33,40 @@ jobs: - name: build-mpich if: steps.cache-mpich.outputs.cache-hit != 'true' - run: | - wget http://www.mpich.org/static/downloads/${{ env.MPICH_VERSION }}/mpich-${{ env.MPICH_VERSION }}.tar.gz &> /dev/null - tar -xzf mpich-${{ env.MPICH_VERSION }}.tar.gz - pushd mpich-${{ env.MPICH_VERSION }} - ./configure --prefix=/home/runner/mpich - make - sudo make install - popd + uses: ./.github/actions/buildmpich + with: + install_prefix: $HOME/mpich + mpich_version: ${{ env.MPICH_VERSION }} + - name: cache-hdf5 id: cache-hdf5 uses: actions/cache@v3 with: path: ~/hdf5 - key: hdf5-${{ runner.os }}-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}-mpich-${{ env.MPICH_VERSION }} + key: hdf5-${{ runner.os }}-${{ env.HDF5_VERSION }}-mpich-${{ env.MPICH_VERSION }} - name: build-hdf5 if: steps.cache-hdf5.outputs.cache-hit != 'true' - run: | - set -x - export PATH="/home/runner/mpich/bin:$PATH" - export CC=/home/runner/mpich/bin/mpicc - wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${{ env.HDF5_VERSION_MAJOR }}/hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}/src/hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}.tar.gz &> /dev/null - tar -xzf hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}.tar.gz - pushd hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }} - ./configure --prefix=/home/runner/hdf5 --enable-parallel --disable-tools --disable-fortran --disable-cxx --enable-parallel-tests - make - sudo make install - popd + uses: ./.github/actions/buildhdf5 + with: + install_prefix: $HOME/hdf5 + enable_parallel: True + hdf5_version: ${{ env.HDF5_VERSION }} + mpi_path: $HOME/mpich - name: cache-netcdf-c id: cache-netcdf-c uses: actions/cache@v3 with: path: ~/netcdf-c - key: netcdf-c-${{ runner.os }}-${{ env.NETCDF_C_VERSION }}-mpich-${{ env.MPICH_VERSION }}-hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }} + key: netcdf-c-${{ runner.os }}-${{ env.NETCDF_C_VERSION }}-mpich-${{ env.MPICH_VERSION }}-hdf5-${{ env.HDF5_VERSION }} - name: build-netcdf-c if: steps.cache-netcdf-c.outputs.cache-hit != 'true' - run: | - set -x - export PATH="/home/runner/mpich/bin:$PATH" - export CC=/home/runner/mpich/bin/mpicc - wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v${{ env.NETCDF_C_VERSION }}.tar.gz - tar -xzf v${{ env.NETCDF_C_VERSION }}.tar.gz - pushd netcdf-c-${{ env.NETCDF_C_VERSION }} - ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities - make -j - sudo make install - popd + uses: ./.github/actions/buildnetcdf + with: + netcdf_version: ${{ env.NETCDF_VERSION }} + install_prefix: $HOME/netcdf-c + - name: cache-netcdf-fortran id: cache-netcdf-fortran uses: actions/cache@v3 @@ -91,19 +74,14 @@ jobs: path: ~/netcdf-fortran key: netcdf-fortran-${{ runner.os }}-${{ env.NETCDF_F_VERSION }}-mpich-${{ env.MPICH_VERSION }}-hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }} - - name: build-netcdf-fortran + - name: Build NetCDF Fortran if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' - run: | - set -x - export PATH="/home/runner/mpich/bin:$PATH" - export CC=/home/runner/mpich/bin/mpicc - wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v${{ env.NETCDF_F_VERSION }}.tar.gz - tar -xzf v${{ env.NETCDF_F_VERSION }}.tar.gz - pushd netcdf-fortran-${{ env.NETCDF_F_VERSION }} - ./configure --prefix=/home/runner/netcdf-fortran - make -j - sudo make install - popd + uses: ./.github/actions/buildnetcdff + with: + netcdf_fortran_version: ${{ env.NETCDF_FORTRAN_VERSION }} + install_prefix: $HOME/netcdf-fortran + netcdf_c_path: $HOME/netcdf-c + - name: cache-pnetcdf id: cache-pnetcdf uses: actions/cache@v3 @@ -111,52 +89,42 @@ jobs: path: ~/pnetcdf key: pnetcdf-${{ runner.os }}-${{ env.PNETCDF_VERSION }}-mpich-${{ env.MPICH_VERSION }} - - name: build-pnetcdf + - name: Build PNetCDF if: steps.cache-pnetcdf.outputs.cache-hit != 'true' + uses: ./.github/actions/buildpnetcdf + with: + pnetcdf_version: ${{ env.PNETCDF_VERSION }} + install_prefix: $HOME/pnetcdf + - name: prep for build run: | - set -x - export PATH="/home/runner/mpich/bin:$PATH" + export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:/home/runner/mpich/lib:/home/runner/hdf5/lib:/home/runner/netcdf-fortran/lib:/home/runner/pnetcdf/lib:$LD_LIBRARY_PATH" + export ASAN_OPTIONS="detect_odr_violation=0" export CC=/home/runner/mpich/bin/mpicc - wget https://parallel-netcdf.github.io/Release/pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz &> /dev/null - tar -xzf pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz - pushd pnetcdf-${{ env.PNETCDF_VERSION }} - ./configure --prefix=/home/runner/pnetcdf --enable-shared --disable-cxx - make - sudo make install - popd + export FC=/home/runner/mpich/bin/mpifort + export PATH=/home/runner/mpich/bin:$PATH + printenv >> $GITHUB_ENV - name: cmake build + uses: ./.github/actions/parallelio_cmake + with: + parallelio_version: ${{ env.GITHUB_SHA }} + with_pnetcdf: $HOME/pnetcdf + with_netcdf: $HOME/netcdf-c + with_netcdff: $HOME/netcdf-fortran + enable_fortran: True + install_prefix: $HOME/parallelio + extra_cflags: "-g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan" + extra_fflags: " -g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan" + extra_ldflags: " -static-libasan" + + - name: run tests run: | - set -x - gcc --version - echo 'export PATH=/home/runner/mpich/bin:$PATH' > .bashrc - source .bashrc - export CC=/home/runner/mpich/bin/mpicc - export FC=/home/runner/mpich/bin/mpifort - export CFLAGS="-g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan" - export FCFLAGS="$FCFLAGS -g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan" - export LDFLAGS="$LDFLAGS -static-libasan" - export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:/home/runner/mpich/lib:/home/runner/hdf5/lib:/home/runner/netcdf-fortran/lib:/home/runner/pnetcdf/lib:$LD_LIBRARY_PATH" - export ASAN_OPTIONS="detect_odr_violation=0" - mkdir build - cd build - cmake -Wno-dev -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include -DPnetCDF_PATH='/home/runner/pnetcdf' -DPIO_ENABLE_FORTRAN=Off -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) - make VERBOSE=1 - make tests VERBOSE=1 -# ctest -VV + cd $GITHUB_WORKSPACE/parallelio-src + make -j check + - name: autotools build run: | set -x gcc --version - echo 'export PATH=/home/runner/mpich/bin:$PATH' > .bashrc - source .bashrc - export CC=/home/runner/mpich/bin/mpicc - export FC=/home/runner/mpich/bin/mpifort - export CFLAGS="-g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan" - export FCFLAGS="$FCFLAGS -g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan" - export LDFLAGS="$LDFLAGS -static-libasan" - export ASAN_OPTIONS="detect_odr_violation=0" autoreconf -i ./configure --enable-fortran --enable-netcdf-integration make -j check - - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_pnetcdf_openmpi.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_pnetcdf_openmpi.yml index 4ab12970de..1b3b6c1b05 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_pnetcdf_openmpi.yml +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/netcdf_pnetcdf_openmpi.yml @@ -2,9 +2,9 @@ name: netcdf_pnetcdf_openmpi on: push: - branches: [ master ] + branches: [ maintodo ] pull_request: - branches: [ master ] + branches: [ maintodo ] jobs: build: @@ -12,15 +12,13 @@ jobs: runs-on: ubuntu-latest env: - CPPFLAGS: "-I/home/runner/openmpi/include -I/home/runner/hdf5/include -I/home/runner/netcdf-c/include -I/home/runner/netcdf-fortran/include -I/home/runner/pnetcdf/include" - LDFLAGS: "-L/home/runner/openmpi/lib -L/home/runner/hdf5/lib -L/home/runner/netcdf-c/lib -L/home/runner/netcdf-fortran/lib -L/home/runner/pnetcdf/lib" - NETCDF_C_VERSION: 4.9.0 - NETCDF_F_VERSION: 4.6.0 - OPENMPI_VERSION_MAJOR: 4.1 - OPENMPI_VERSION_PATCH: 4 - PNETCDF_VERSION: 1.12.3 - HDF5_VERSION_PATCH: 2 - HDF5_VERSION_MAJOR: 1.12 +# CPPFLAGS: "-I${GITHUB_WORKSPACE}/openmpi/include -I${GITHUB_WORKSPACE}/hdf5/include -I${GITHUB_WORKSPACE}/netcdf/include -I${GITHUB_WORKSPACE}/pnetcdf/include" +# LDFLAGS: "-L${GITHUB_WORKSPACE}/openmpi/lib -L${GITHUB_WORKSPACE}/hdf5/lib -L${GITHUB_WORKSPACE}/netcdf/lib -L${GITHUB_WORKSPACE}/pnetcdf/lib" + NETCDF_C_VERSION: v4.9.2 + NETCDF_FORTRAN_VERSION: v4.6.1 + OPENMPI_VERSION: v4.1.5 + PNETCDF_VERSION: checkpoint.1.12.3 + HDF5_VERSION: hdf5_1_12_2 FCFLAGS: "-fallow-argument-mismatch" FFLAGS: "-fallow-argument-mismatch" steps: @@ -32,125 +30,97 @@ jobs: id: cache-openmpi uses: actions/cache@v3 with: - path: ~/openmpi + path: ~/work/ParallelIO/ParallelIO/openmpi key: openmpi-${{ runner.os }}-${{ env.OPENMPI_VERSION }} - name: build-openmpi if: steps.cache-openmpi.outputs.cache-hit != 'true' - run: | - wget https://download.open-mpi.org/release/open-mpi/v${{ env.OPENMPI_VERSION_MAJOR }}/openmpi-${{ env.OPENMPI_VERSION_MAJOR }}.${{ env.OPENMPI_VERSION_PATCH}}.tar.gz &> /dev/null - tar -xzf openmpi-${{ env.OPENMPI_VERSION_MAJOR }}.${{ env.OPENMPI_VERSION_PATCH }}.tar.gz - pushd openmpi-${{ env.OPENMPI_VERSION_MAJOR }}.${{ env.OPENMPI_VERSION_PATCH }} - ./configure --prefix=/home/runner/openmpi - make - sudo make install - popd + uses: ./.github/actions/buildopenmpi + with: + install_prefix: ${GITHUB_WORKSPACE}/openmpi + openmpi_version: ${{ env.OPENMPI_VERSION }} + - name: cache-hdf5 id: cache-hdf5 uses: actions/cache@v3 with: - path: ~/hdf5 - key: hdf5-${{ runner.os }}-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}-openmpi-${{ env.OPENMPI_VERSION_MAJOR }}.${{ env.OPENMPI_VERSION_PATCH }} + path: ~/work/ParallelIO/ParallelIO/hdf5 + key: hdf5-${{ runner.os }}-${{ env.HDF5_VERSION }}-openmpi-${{ env.OPENMPI_VERSION }} - name: build-hdf5 if: steps.cache-hdf5.outputs.cache-hit != 'true' - run: | - set -x - export PATH="/home/runner/openmpi/bin:$PATH" - export CC=/home/runner/openmpi/bin/mpicc - wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${{ env.HDF5_VERSION_MAJOR }}/hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}/src/hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}.tar.gz &> /dev/null - tar -xzf hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }}.tar.gz - pushd hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }} - ./configure --prefix=/home/runner/hdf5 --enable-parallel --disable-tools --disable-fortran --disable-cxx --enable-parallel-tests - make - sudo make install - popd - - name: cache-netcdf-c - id: cache-netcdf-c + uses: ./.github/actions/buildhdf5 + with: + install_prefix: ${GITHUB_WORKSPACE}/hdf5 + enable_parallel: True + hdf5_version: ${{ env.HDF5_VERSION }} + mpi_path: ${GITHUB_WORKSPACE}/openmpi + + - name: cache-netcdf + id: cache-netcdf uses: actions/cache@v3 with: - path: ~/netcdf-c - key: netcdf-c-${{ runner.os }}-${{ env.NETCDF_C_VERSION }}-openmpi-${{ env.OPENMPI_VERSION_MAJOR }}.${{ env.OPENMPI_VERSION_PATCH }}-hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }} + path: ~/work/ParallelIO/ParallelIO/netcdf + key: netcdf-${{ runner.os }}-${{ env.NETCDF_C_VERSION }}-${{ env.NETCDF_FORTRAN_VERSION }}-openmpi-${{ env.OPENMPI_VERSION }}-hdf5-${{ env.HDF5_VERSION }} + - name: build-netcdf-c - if: steps.cache-netcdf-c.outputs.cache-hit != 'true' - run: | - set -x - export PATH="/home/runner/openmpi/bin:$PATH" - export CC=/home/runner/openmpi/bin/mpicc - wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v${{ env.NETCDF_C_VERSION }}.tar.gz - tar -xzf v${{ env.NETCDF_C_VERSION }}.tar.gz &> /dev/null - pushd netcdf-c-${{ env.NETCDF_C_VERSION }} - ./configure --prefix=/home/runner/netcdf-c --disable-dap --disable-utilities - make -j - sudo make install - popd - - name: cache-netcdf-fortran - id: cache-netcdf-fortran - uses: actions/cache@v3 + if: steps.cache-netcdf.outputs.cache-hit != 'true' + uses: ./.github/actions/buildnetcdf with: - path: ~/netcdf-fortran - key: netcdf-fortran-${{ runner.os }}-${{ env.NETCDF_F_VERSION }}-openmpi-${{ env.OPENMPI_VERSION_MAJOR }}.${{ env.OPENMPI_VERSION_PATCH }}-hdf5-${{ env.HDF5_VERSION_MAJOR }}.${{ env.HDF5_VERSION_PATCH }} + netcdf_version: ${{ env.NETCDF_C_VERSION }} + install_prefix: ${GITHUB_WORKSPACE}/netcdf - - name: build-netcdf-fortran - if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' - run: | - set -x - export PATH="/home/runner/openmpi/bin:$PATH" - export CC=/home/runner/openmpi/bin/mpicc - wget https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v${{ env.NETCDF_F_VERSION }}.tar.gz - tar -zxf v${{ env.NETCDF_F_VERSION }}.tar.gz &> /dev/null - pushd netcdf-fortran-${{ env.NETCDF_F_VERSION }} - ./configure --prefix=/home/runner/netcdf-fortran - make -j - sudo make install - popd - - name: cache-pnetcdf - id: cache-pnetcdf - uses: actions/cache@v3 +# - name: Setup tmate session +# if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 + + - name: Build NetCDF Fortran + if: steps.cache-netcdf.outputs.cache-hit != 'true' + uses: ./.github/actions/buildnetcdff with: - path: ~/pnetcdf - key: pnetcdf-${{ runner.os }}-${{ env.PNETCDF_VERSION }}-openmpi-${{ env.OPENMPI_VERSION_MAJOR }}.${{ env.OPENMPI_VERSION_PATCH }} + netcdf_fortran_version: ${{ env.NETCDF_FORTRAN_VERSION }} + install_prefix: ${GITHUB_WORKSPACE}/netcdf + netcdf_c_path: ${GITHUB_WORKSPACE}/netcdf - - name: build-pnetcdf - if: steps.cache-pnetcdf.outputs.cache-hit != 'true' - run: | - set -x - export PATH="/home/runner/openmpi/bin:$PATH" - export CC=/home/runner/openmpi/bin/mpicc - wget https://parallel-netcdf.github.io/Release/pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz &> /dev/null - tar -xzf pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz - pushd pnetcdf-${{ env.PNETCDF_VERSION }} - ./configure --prefix=/home/runner/pnetcdf --enable-shared --disable-cxx - make - sudo make install - popd - - name: autotools build - run: | - set -x - echo 'export PATH=/home/runner/openmpi/bin:$PATH' > .bashrc - source .bashrc - export PATH="/home/runner/openmpi/bin:$PATH" - export CC=/home/runner/openmpi/bin/mpicc - autoreconf -i - ./configure --with-mpiexec='mpiexec --oversubscribe' - cat config.h - make check - - name: cmake build - run: | - set -x - echo 'export PATH=/home/runner/openmpi/bin:$PATH' > .bashrc - source .bashrc - export CC=mpicc - mkdir build - cd build - export LD_LIBRARY_PATH="/home/runner/netcdf-c/lib:/home/runner/pnetcdf/lib:/home/runner/hdf5/lib:/home/runner/openmpi/lib:$LD_LIBRARY_PATH" - cmake -Wno-dev -DWITH_MPIEXEC='/home/runner/openmpi/bin/mpiexec;--oversubscribe' \ - -DNetCDF_C_LIBRARY=/home/runner/netcdf-c/lib/libnetcdf.so \ - -DNetCDF_C_INCLUDE_DIR=/home/runner/netcdf-c/include -DPnetCDF_PATH='/home/runner/pnetcdf' \ - -DPIO_ENABLE_FORTRAN=Off -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On \ - -DPIO_ENABLE_TIMING=Off .. || (cat CMakeFiles/CMakeOutput.log && cat CMakeFiles/CMakeError.log) - cat config.h - make VERBOSE=1 - make tests VERBOSE=1 - ctest -VV + # - name: cache-pnetcdf + # id: cache-pnetcdf + # uses: actions/cache@v3 + # with: + # path: ~/pnetcdf + # key: pnetcdf-${{ runner.os }}-${{ env.PNETCDF_VERSION }}-openmpi-${{ env.OPENMPI_VERSION_MAJOR }}.${{ env.OPENMPI_VERSION_PATCH }} + + # - name: Build PNetCDF + # if: steps.cache-pnetcdf.outputs.cache-hit != 'true' + # uses: ./.github/actions/buildpnetcdf + # with: + # pnetcdf_version: ${{ env.PNETCDF_VERSION }} + # install_prefix: pnetcdf + + # - name: Build ParallelIO with autotools + # uses: ./.github/actions/parallelio_autotools + # with: + # enable_fortran: True + # with_mpiexec: 'mpiexec --oversubscribe' + # parallelio_version: ${{ env.GITHUB_SHA }} + # - name: make check + # run: | + # cd ${GITHUB_WORKSPACE}/parallelio-src + # make -j check + # make distclean + # - name: cmake build + # uses: ./.github/actions/parallelio_cmake + # with: + # parallelio_version: ${{ env.GITHUB_SHA }} + # with_pnetcdf: pnetcdf + # with_netcdf: netcdf + # with_netcdff: netcdf + # enable_fortran: True + # install_prefix: parallelio + # with_mpiexec: "${{ env.GITHUB_WORKSPACE }}/openmpi/bin/mpiexec --oversubscribe" + # - name: run tests + # run: | + # pushd ${GITHUB_WORKSPACE}/build + # make tests VERBOSE=1 + # ctest -VV + # popd diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/strict_autotools_ubuntu_latest.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/strict_autotools_ubuntu_latest.yml index faf74ce40b..952348a5fb 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/strict_autotools_ubuntu_latest.yml +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/strict_autotools_ubuntu_latest.yml @@ -2,9 +2,9 @@ name: strict_autotools_ubuntu_latest on: push: - branches: [ master ] + branches: [ maintodo ] pull_request: - branches: [ master ] + branches: [ maintodo ] jobs: build: @@ -14,19 +14,16 @@ jobs: env: CC: mpicc FC: mpifort - CPPFLAGS: "-I/usr/include -I/usr/local/include -I/home/runner/pnetcdf/include" - LDFLAGS: "-L/home/runner/pnetcdf/lib" - FCFLAGS: "-fallow-argument-mismatch" - PNETCDF_VERSION: 1.12.3 + CPPFLAGS: "-I/usr/include -I/usr/local/include -I${GITHUB_WORKSPACE}/pnetcdf/include" + LDFLAGS: "-L/usr/lib -L${GITHUB_WORKSPACE}/pnetcdf/lib" + CFLAGS: "-std=c99 -Wall" + FCFLAGS: "-fallow-argument-mismatch -Wall" + PNETCDF_VERSION: checkpoint.1.12.3 steps: - uses: actions/checkout@v3 - name: Installs run: | set -x - #sudo gem install apt-spy2 - #sudo apt-spy2 check - #sudo apt-spy2 fix --commit - # after selecting a specific mirror, we need to run 'apt-get update' sudo apt-get update sudo apt-get install netcdf-bin libnetcdf-dev doxygen graphviz wget gfortran libjpeg-dev libz-dev openmpi-bin libopenmpi-dev @@ -34,29 +31,23 @@ jobs: id: cache-pnetcdf uses: actions/cache@v3 with: - path: ~/pnetcdf - key: pnetcdf-${{ runner.os }}-${{ env.PNETCDF_VERSION }} + path: ~/work/ParallelIO/ParallelIO/pnetcdf + key: pnetcdf-${{ runner.os }}-${{ env.PNETCDF_VERSION }}-openmpi - - name: build-pnetcdf + - name: Build PNetCDF if: steps.cache-pnetcdf.outputs.cache-hit != 'true' + uses: ./.github/actions/buildpnetcdf + with: + pnetcdf_version: ${{ env.PNETCDF_VERSION }} + install_prefix: ${GITHUB_WORKSPACE}/pnetcdf + - name: Build ParallelIO with autotools + uses: ./.github/actions/parallelio_autotools + with: + enable_fortran: True + with_mpiexec: 'mpiexec --oversubscribe' + with_pnetcdf: ${GITHUB_WORKSPACE}/pnetcdf + parallelio_version: ${{ env.GITHUB_SHA }} + - name: Run Tests run: | - set -x - wget https://parallel-netcdf.github.io/Release/pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz &> /dev/null - tar -xzvf pnetcdf-${{ env.PNETCDF_VERSION }}.tar.gz - pushd pnetcdf-${{ env.PNETCDF_VERSION }} - ./configure --prefix=/home/runner/pnetcdf --enable-shared --disable-cxx - make - sudo make install - popd - - - name: autoreconf - run: autoreconf -i - - name: build - run: | - set -x - export CFLAGS="-std=c99 -Wall" - export FFLAGS="-Wall" - export FCFLAGS="$FCFLAGS -Wall" - export DISTCHECK_CONFIGURE_FLAGS="--enable-fortran --with-mpiexec='/usr/bin/mpiexec --oversubscribe'" - ./configure + cd ${GITHUB_WORKSPACE}/parallelio-src make -j distcheck diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/withspack.yml b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/withspack.yml new file mode 100644 index 0000000000..998d61ea01 --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/.github/workflows/withspack.yml @@ -0,0 +1,77 @@ +name: Build with Spack +# Tests ParallelIO using spack tools +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Installs + run: | + set -x + sudo apt-get update + sudo apt-get install wget + sudo apt-get install libjpeg-dev + sudo apt-get install libz-dev + sudo apt-get install gfortran + + - name: Get latest spack release tag + run: | + export SPACK_LATEST="$(curl -sL https://github.com/spack/spack/releases/latest |grep 'Release' | awk '{print $2}')" + printenv >> $GITHUB_ENV + - name: cache spack + id: cache-spack + uses: actions/cache@v3 + with: + path: | + ~/work/ParallelIO/ParallelIO/spack + ~/.spack + key: spack-${{ runner.os }}-${{ env.SPACK_LATEST }} + - name: Get Spack + if: steps.cache-spack.outputs.cache-hit != 'true' + uses: actions/checkout@v3 + with: + repository: spack/spack + path: spack + ref: ${{ env.SPACK_LATEST }} + + - name: Prep spack + run: | + source $GITHUB_WORKSPACE/spack/share/spack/setup-env.sh + spack compiler find + # Remove the patch for gfortran, we don't want it + # + sed -i 's/patch(.*$//' $GITHUB_WORKSPACE/spack/var/spack/repos/builtin/packages/parallelio/package.py + - name: Build with spack + run: | + source $GITHUB_WORKSPACE/spack/share/spack/setup-env.sh + mkdir genf90 + pushd genf90 + ln -fs $GITHUB_WORKSPACE/scripts/genf90.pl . + popd + # the || true prevents a fail if parallelio is not installed + # spack uninstall -y parallelio@=develop+pnetcdf+fortran ^mpich || true + # spack dev-build -d $GITHUB_WORKSPACE parallelio@=develop+pnetcdf+fortran ^mpich + spack uninstall -y parallelio@=develop+pnetcdf+fortran ^mpich || true + spack dev-build -d $GITHUB_WORKSPACE parallelio@=develop+pnetcdf+fortran ^mpich + + - name: Test parallelio + run: | + pioblddir=$(ls -td */ | head -1) + pushd $pioblddir + make tests + # Exclude two tests that are timing out. + ctest -VV -LE skipforspack + popd + + + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 diff --git a/src/Infrastructure/IO/PIO/ParallelIO/.gitignore b/src/Infrastructure/IO/PIO/ParallelIO/.gitignore index 5055b0c6a8..7df6a3a7c8 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/.gitignore +++ b/src/Infrastructure/IO/PIO/ParallelIO/.gitignore @@ -29,4 +29,4 @@ m4/ *.nc *.log *.gz -!/decomps/*/*.nc \ No newline at end of file +!/decomps/*/*.nc diff --git a/src/Infrastructure/IO/PIO/ParallelIO/CMakeLists.txt b/src/Infrastructure/IO/PIO/ParallelIO/CMakeLists.txt index 5abb33a588..906727f061 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/CMakeLists.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/CMakeLists.txt @@ -9,9 +9,8 @@ project (PIO C) # The project version number. set(VERSION_MAJOR 2 CACHE STRING "Project major version number.") -set(VERSION_MINOR 5 CACHE STRING "Project minor version number.") -set(VERSION_PATCH 10 CACHE STRING "Project patch version number.") - +set(VERSION_MINOR 6 CACHE STRING "Project minor version number.") +set(VERSION_PATCH 2 CACHE STRING "Project patch version number.") mark_as_advanced(VERSION_MAJOR VERSION_MINOR VERSION_PATCH) # Create version info in autotools parlance for pio_meta.h. @@ -25,6 +24,9 @@ SET(PACKAGE_VERSION ${PIO_VERSION_MAJOR}.${PIO_VERSION_MINOR}.${PIO_VERSION_PATC # This provides cmake_print_variables() function for debugging. include(CMakePrintHelpers) +# This provides check_symbol_exists +include(CheckSymbolExists) + # Determine the configure date. IF(DEFINED ENV{SOURCE_DATE_EPOCH}) EXECUTE_PROCESS( @@ -124,12 +126,6 @@ else() set(NETCDF_INTEGRATION 0) endif() -if(PIO_USE_MPISERIAL) - set(USE_MPI_SERIAL 1) -else() - set(USE_MPI_SERIAL 0) -endif() - #============================================================================== # PREPEND TO CMAKE MODULE PATH #============================================================================== @@ -169,14 +165,16 @@ if (PIO_USE_MPISERIAL) if (MPISERIAL_C_FOUND) set (CMAKE_REQUIRED_INCLUDES ${MPISERIAL_C_INCLUDE_DIRS}) endif () + set(USE_MPI_SERIAL 1) else () find_package (MPI REQUIRED) set (CMAKE_REQUIRED_INCLUDES ${MPI_INCLUDE_PATH}) + set(USE_MPI_SERIAL 0) endif () -#SET(CMAKE_EXTRA_INCLUDE_FILES "mpi.h") -#check_type_size("MPI_Offset" SIZEOF_MPI_OFFSET) -#SET(CMAKE_EXTRA_INCLUDE_FILES) +SET(CMAKE_EXTRA_INCLUDE_FILES "mpi.h") +CHECK_TYPE_SIZE("MPI_Offset" SIZEOF_MPI_OFFSET) +SET(CMAKE_EXTRA_INCLUDE_FILES) #===== Library Variables ===== set (PIO_FILESYSTEM_HINTS IGNORE CACHE STRING "Filesystem hints (lustre or gpfs)") @@ -248,6 +246,7 @@ endif () # Allow argument mismatch in gfortran versions > 10 for mpi library compatibility if (CMAKE_C_COMPILER_NAME STREQUAL "GNU") if ("${CMAKE_Fortran_COMPILER_VERSION}" VERSION_LESS 10) + message (WARNING "gfortran version is ${CMAKE_Fortran_COMPILER_VERSION}") else() set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch") endif() @@ -358,14 +357,13 @@ int main() {return 0;}" HAVE_SZIP_WRITE) ### # Check to see if parallel filters are supported by HDF5/netcdf-c. ### -CHECK_C_SOURCE_COMPILES(" +if (HAVE_NETCDF_PAR) + CHECK_C_SOURCE_COMPILES(" #include <netcdf_meta.h> #if !NC_HAS_PAR_FILTERS - choke me + choke me #endif -int main() {return 0;}" HDF5_HAS_PAR_FILTERS) -if(HDF5_HAS_PAR_FILTERS) - set(HAVE_PAR_FILTERS 1) +int main() {return 0;}" HAVE_PAR_FILTERS) else() set(HAVE_PAR_FILTERS 0) endif() @@ -385,18 +383,21 @@ if (HAVE_472) endif () ### -# Check to see if dispatch table version 2 is supported for netcdf integration. +# Check to see if dispatch table is supported for netcdf integration. ### CHECK_C_SOURCE_COMPILES(" #include <netcdf_meta.h> -#if NC_DISPATCH_VERSION != 2 +#if NC_DISPATCH_VERSION < 2 choke me #endif -int main() {return 0;}" HAVE_DISPATCH2) +#if NC_DISPATCH_VERSION > 5 + choke me +#endif +int main() {return 0;}" HAVE_DISPATCH) if (NETCDF_INTEGRATION) - if (NOT HAVE_DISPATCH2) - message (FATAL_ERROR "Need newer version of netcdf-c for netcdf integration feature, please upgrade your netCDF library") + if (NOT HAVE_DISPATCH) + message (FATAL_ERROR "The netcdf-c netcdf integration layer is incompatible with the one in this ParallelIO version") endif () set(HAVE_NETCDF_INTEGRATION 1) else () @@ -451,16 +452,19 @@ SET(LDFLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS_${CMAKE_BUILD_TY is_disabled(BUILD_SHARED_LIBS enable_static) is_enabled(BUILD_SHARED_LIBS enable_shared) - +is_enabled(HAVE_PAR_FILTERS have_par_filters) is_enabled(USE_SZIP HAS_SZIP_WRITE) is_enabled(STATUS_PNETCDF HAS_PNETCDF) is_enabled(HAVE_H5Z_SZIP HAS_SZLIB) -is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS) is_enabled(HAVE_NETCDF4 HAS_NETCDF4) is_enabled(HAVE_NETCDF_PAR HAS_NETCDF4_PAR) is_enabled(HAVE_NETCDF_INTEGRATION HAS_NETCDF_INTEGRATION) is_enabled(PIO_ENABLE_FORTRAN HAS_PIO_FORTRAN) +if(HAVE_PAR_FILTERS) + SET(PIO_HAS_PAR_FILTERS 1) +endif() + # Generate file from template. CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libpio.settings.in" "${CMAKE_CURRENT_BINARY_DIR}/libpio.settings" @@ -473,12 +477,16 @@ FILE(READ "${CMAKE_CURRENT_BINARY_DIR}/libpio.settings" LIBPIO_SETTINGS) MESSAGE(STATUS ${LIBPIO_SETTINGS}) +# Set RPATH for shared libraries +set(CMAKE_MACOSX_RPATH 1) +set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") + # Install libpio.settings file into same location # as the libraries. INSTALL(FILES "${PIO_BINARY_DIR}/libpio.settings" DESTINATION lib COMPONENT libraries) - + ##### # Create pio_meta.h include file. ##### diff --git a/src/Infrastructure/IO/PIO/ParallelIO/README.md b/src/Infrastructure/IO/PIO/ParallelIO/README.md index 3ecaaa6a05..eb523848ff 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/README.md +++ b/src/Infrastructure/IO/PIO/ParallelIO/README.md @@ -99,8 +99,8 @@ The typical configuration with CMake can be done as follows: CC=mpicc FC=mpif90 cmake [-DOPTION1=value1 -DOPTION2=value2 ...] /path/to/pio/source ``` -Full instructions for the cmake build can be found in the installation -documentation. +Full instructions for the cmake build can be found in the [installation +documentation](https://ncar.github.io/ParallelIO/install.html). # References diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindGPTL.cmake b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindGPTL.cmake index c223c1b346..c4a8b54488 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindGPTL.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindGPTL.cmake @@ -1,6 +1,6 @@ # - Try to find GPTL # -# This can be controlled by setting the GPTL_DIR (or, equivalently, the +# This can be controlled by setting the GPTL_DIR (or, equivalently, the # GPTL environment variable), or GPTL_<lang>_DIR CMake variables, where # <lang> is the COMPONENT language one needs. # @@ -55,7 +55,7 @@ foreach (GPTL_comp IN LISTS GPTL_FIND_VALID_COMPONENTS) set (mpilibs ${MPI_Fortran_LIBRARIES}) set (mpifound ${MPI_Fortran_FOUND}) endif () - + # Search for the package component if (mpifound) initialize_paths (GPTL_${GPTL_comp}_PATHS @@ -68,5 +68,5 @@ foreach (GPTL_comp IN LISTS GPTL_FIND_VALID_COMPONENTS) endif () endif () - + endforeach () diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindHDF5.cmake b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindHDF5.cmake index e918277b1a..d19c423ed7 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindHDF5.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindHDF5.cmake @@ -1,6 +1,6 @@ # - Try to find HDF5 # -# This can be controlled by setting the HDF5_DIR (or, equivalently, the +# This can be controlled by setting the HDF5_DIR (or, equivalently, the # HDF5 environment variable), or HDF5_<lang>_DIR CMake variables, where # <lang> is the COMPONENT language one needs. # @@ -78,14 +78,14 @@ foreach (HDF5_comp IN LISTS HDF5_FIND_VALID_COMPONENTS) # Dependencies if (HDF5_comp STREQUAL C AND NOT HDF5_C_IS_SHARED) - + # DEPENDENCY: LIBZ find_package (LIBZ) if (LIBZ_FOUND) list (APPEND HDF5_C_INCLUDE_DIRS ${LIBZ_INCLUDE_DIRS}) list (APPEND HDF5_C_LIBRARIES ${LIBZ_LIBRARIES}) endif () - + # DEPENDENCY: SZIP (Optional) check_macro (HDF5_C_HAS_SZIP NAME TryHDF5_HAS_SZIP.c @@ -99,20 +99,20 @@ foreach (HDF5_comp IN LISTS HDF5_FIND_VALID_COMPONENTS) list (APPEND HDF5_C_LIBRARIES ${SZIP_LIBRARIES}) endif () endif () - + elseif (NOT HDF5_${HDF5_comp}_IS_SHARED) - + # DEPENDENCY: HDF5 find_package (HDF5 COMPONENTS C) if (HDF5_C_FOUND) list (APPEND HDF5_${HDF5_comp}_INCLUDE_DIRS ${HDF5_C_INCLUDE_DIRS}) list (APPEND HDF5_${HDF5_comp}_LIBRARIES ${HDF5_C_LIBRARIES}) endif () - + endif () endif () - + endif () - + endforeach () diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindLIBRT.cmake b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindLIBRT.cmake index 1f55f9f3f1..211984fcb8 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindLIBRT.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindLIBRT.cmake @@ -1,6 +1,6 @@ # - Try to find LIBRT # -# This can be controlled by setting the LIBRT_DIR (or, equivalently, the +# This can be controlled by setting the LIBRT_DIR (or, equivalently, the # LIBRT environment variable). # # Once done, this will define: @@ -21,7 +21,7 @@ define_package_component (LIBRT # SEARCH FOR PACKAGE if (NOT LIBRT_FOUND) - + # Search for the package find_package_component(LIBRT) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindLIBZ.cmake b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindLIBZ.cmake index 8ebbaefeed..c003eb94ca 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindLIBZ.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindLIBZ.cmake @@ -1,6 +1,6 @@ # - Try to find LIBZ # -# This can be controlled by setting the LIBZ_DIR (or, equivalently, the +# This can be controlled by setting the LIBZ_DIR (or, equivalently, the # LIBZ environment variable). # # Once done, this will define: @@ -22,7 +22,7 @@ define_package_component (LIBZ # SEARCH FOR PACKAGE if (NOT LIBZ_FOUND) - # Manually add the MPI include and library dirs to search paths + # Manually add the MPI include and library dirs to search paths # and search for the package component if (MPI_C_FOUND) initialize_paths (LIBZ_PATHS diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindMPE.cmake b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindMPE.cmake index 5a964172da..cdfc360c23 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindMPE.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindMPE.cmake @@ -13,7 +13,7 @@ # MPE_<lang>_LIBRARY (FILE) - Path to the C library file # MPE_<lang>_LIBRARIES (LIST) - link these to use MPE # -# The available COMPONENTS are: C +# The available COMPONENTS are: C include (LibFind) include (LibCheck) @@ -46,5 +46,5 @@ foreach (NCDFcomp IN LISTS MPE_FIND_VALID_COMPONENTS) endif () endif () - + endforeach () diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindMPISERIAL.cmake b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindMPISERIAL.cmake index 09906eb7a2..47d30077b1 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindMPISERIAL.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindMPISERIAL.cmake @@ -1,6 +1,6 @@ # - Try to find MPISERIAL # -# This can be controlled by setting the MPISERIAL_PATH (or, equivalently, the +# This can be controlled by setting the MPISERIAL_PATH (or, equivalently, the # MPISERIAL environment variable). # # Once done, this will define: @@ -38,7 +38,7 @@ foreach (MPISERIAL_comp IN LISTS MPISERIAL_FIND_VALID_COMPONENTS) # Search for the package find_package_component(MPISERIAL COMPONENT ${MPISERIAL_comp}) - + endif () endforeach () diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindPAPI.cmake b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindPAPI.cmake index dcf1445bc7..369abb7f8c 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindPAPI.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindPAPI.cmake @@ -1,6 +1,6 @@ # - Try to find PAPI # -# This can be controlled by setting the PAPI_DIR (or, equivalently, the +# This can be controlled by setting the PAPI_DIR (or, equivalently, the # PAPI environment variable). # # Once done, this will define: diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindPnetCDF.cmake b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindPnetCDF.cmake index b87d245cd1..7d644a6380 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindPnetCDF.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindPnetCDF.cmake @@ -1,6 +1,6 @@ # - Try to find PnetCDF # -# This can be controlled by setting the PnetCDF_PATH (or, equivalently, the +# This can be controlled by setting the PnetCDF_PATH (or, equivalently, the # PNETCDF environment variable), or PnetCDF_<lang>_PATH CMake variables, where # <lang> is the COMPONENT language one needs. # @@ -54,7 +54,7 @@ foreach (PNCDFcomp IN LISTS PnetCDF_FIND_VALID_COMPONENTS) # Continue only if component found if (PnetCDF_${PNCDFcomp}_FOUND) - + # Check version check_version (PnetCDF NAME "pnetcdf.h" @@ -62,7 +62,7 @@ foreach (PNCDFcomp IN LISTS PnetCDF_FIND_VALID_COMPONENTS) MACRO_REGEX "PNETCDF_VERSION_") endif () - + endif () - + endforeach () diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindSZIP.cmake b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindSZIP.cmake index e65cfe5fd6..ab8c497f59 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindSZIP.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/FindSZIP.cmake @@ -1,6 +1,6 @@ # - Try to find SZIP # -# This can be controlled by setting the SZIP_DIR (or, equivalently, the +# This can be controlled by setting the SZIP_DIR (or, equivalently, the # SZIP environment variable). # # Once done, this will define: @@ -22,7 +22,7 @@ define_package_component (SZIP # SEARCH FOR PACKAGE if (NOT SZIP_FOUND) - # Manually add the MPI include and library dirs to search paths + # Manually add the MPI include and library dirs to search paths # and search for the package component if (MPI_C_FOUND) initialize_paths (SZIP_PATHS diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/LibCheck.cmake b/src/Infrastructure/IO/PIO/ParallelIO/cmake/LibCheck.cmake index 3f12bdf796..2e8cadcbdd 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/LibCheck.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/LibCheck.cmake @@ -101,4 +101,4 @@ function (check_version PKG) endif () -endfunction () \ No newline at end of file +endfunction () diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/LibFind.cmake b/src/Infrastructure/IO/PIO/ParallelIO/cmake/LibFind.cmake index 61cd93aa37..ac13748091 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/LibFind.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/LibFind.cmake @@ -77,7 +77,7 @@ function (define_package_component PKG) else () set (PKGCOMP ${PKG}) endif () - + # Set return values if (${PKG}_COMPONENT) if (${PKG}_DEFAULT) @@ -96,7 +96,7 @@ endfunction () #______________________________________________________________________________ # - Function to find valid package components # -# Assumes pre-defined variables: +# Assumes pre-defined variables: # ${PKG}_FIND_COMPONENTS (LIST) # ${PKG}_DEFAULT_COMPONENT (STRING) # ${PKG}_VALID_COMPONENTS (LIST) @@ -109,7 +109,7 @@ function (find_valid_components PKG) if (NOT ${PKG}_FIND_COMPONENTS) set (${PKG}_FIND_COMPONENTS ${${PKG}_DEFAULT_COMPONENT}) endif () - + set (FIND_VALID_COMPONENTS) foreach (comp IN LISTS ${PKG}_FIND_COMPONENTS) if (";${${PKG}_VALID_COMPONENTS};" MATCHES ";${comp};") @@ -118,7 +118,7 @@ function (find_valid_components PKG) endforeach () set (${PKG}_FIND_VALID_COMPONENTS ${FIND_VALID_COMPONENTS} PARENT_SCOPE) - + endfunction () @@ -137,7 +137,7 @@ function (initialize_paths PATHLIST) # Parse the input arguments set (multiValueArgs INCLUDE_DIRECTORIES LIBRARIES) cmake_parse_arguments (INIT "" "" "${multiValueArgs}" ${ARGN}) - + set (paths) foreach (inc IN LISTS INIT_INCLUDE_DIRECTORIES) list (APPEND paths ${inc}) @@ -156,7 +156,7 @@ function (initialize_paths PATHLIST) list (APPEND paths ${prefx}) endif () endforeach () - + set (${PATHLIST} ${paths} PARENT_SCOPE) endfunction () @@ -180,7 +180,7 @@ function (find_package_component PKG) set (options) set (oneValueArgs COMPONENT) set (multiValueArgs HINTS PATHS) - cmake_parse_arguments (${PKG} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cmake_parse_arguments (${PKG} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) set (COMP ${${PKG}_COMPONENT}) if (COMP) set (PKGCOMP ${PKG}_${COMP}) @@ -189,10 +189,10 @@ function (find_package_component PKG) endif () string (TOUPPER ${PKG} PKGUP) string (TOUPPER ${PKGCOMP} PKGCOMPUP) - + # Only continue if package not found already if (NOT ${PKGCOMP}_FOUND) - + # Handle QUIET and REQUIRED arguments if (${${PKG}_FIND_QUIETLY}) set (${PKGCOMP}_FIND_QUIETLY TRUE) @@ -200,7 +200,7 @@ function (find_package_component PKG) if (${${PKG}_FIND_REQUIRED}) set (${PKGCOMP}_FIND_REQUIRED TRUE) endif () - + # Determine search order set (SEARCH_DIRS) if (${PKG}_HINTS) @@ -234,7 +234,7 @@ function (find_package_component PKG) endforeach () foreach (dir IN LISTS SEARCH_DIRS) - + # Search for include file names in current dirrectory foreach (iname IN LISTS ${PKGCOMP}_INCLUDE_NAMES) if (EXISTS ${dir}/${iname}) @@ -248,7 +248,7 @@ function (find_package_component PKG) break () endif () endforeach () - + # Search for library file names in the found prefix only! if (${PKGCOMP}_PREFIX) find_library (${PKGCOMP}_LIBRARY @@ -256,12 +256,12 @@ function (find_package_component PKG) PATHS ${${PKGCOMP}_PREFIX} PATH_SUFFIXES lib NO_DEFAULT_PATH) - - # If found, check if library is static or dynamic + + # If found, check if library is static or dynamic if (${PKGCOMP}_LIBRARY) is_shared_library (${PKGCOMP}_IS_SHARED ${${PKGCOMP}_LIBRARY}) - - # If we want only shared libraries, and it isn't shared... + + # If we want only shared libraries, and it isn't shared... if (PREFER_SHARED AND NOT ${PKGCOMP}_IS_SHARED) find_shared_library (${PKGCOMP}_SHARED_LIBRARY NAMES ${${PKGCOMP}_LIBRARY_NAMES} @@ -272,7 +272,7 @@ function (find_package_component PKG) set (${PKGCOMP}_LIBRARY ${${PKGCOMP}_SHARED_LIBRARY}) set (${PKGCOMP}_IS_SHARED TRUE) endif () - + # If we want only static libraries, and it is shared... elseif (PREFER_STATIC AND ${PKGCOMP}_IS_SHARED) find_static_library (${PKGCOMP}_STATIC_LIBRARY @@ -286,11 +286,11 @@ function (find_package_component PKG) endif () endif () endif () - + # If include dir and library both found, then we're done if (${PKGCOMP}_INCLUDE_DIR AND ${PKGCOMP}_LIBRARY) break () - + # Otherwise, reset the search variables and continue else () set (${PKGCOMP}_PREFIX ${PKGCOMP}_PREFIX-NOTFOUND) @@ -298,19 +298,19 @@ function (find_package_component PKG) set (${PKGCOMP}_LIBRARY ${PKGCOMP}_LIBRARY-NOTFOUND) endif () endif () - + endforeach () - - # handle the QUIETLY and REQUIRED arguments and + + # handle the QUIETLY and REQUIRED arguments and # set NetCDF_C_FOUND to TRUE if all listed variables are TRUE find_package_handle_standard_args (${PKGCOMP} DEFAULT_MSG - ${PKGCOMP}_LIBRARY + ${PKGCOMP}_LIBRARY ${PKGCOMP}_INCLUDE_DIR) mark_as_advanced (${PKGCOMP}_INCLUDE_DIR ${PKGCOMP}_LIBRARY) - + # HACK For bug in CMake v3.0: set (${PKGCOMP}_FOUND ${${PKGCOMPUP}_FOUND}) - + # Set return variables if (${PKGCOMP}_FOUND) set (${PKGCOMP}_INCLUDE_DIRS ${${PKGCOMP}_INCLUDE_DIR}) @@ -324,10 +324,7 @@ function (find_package_component PKG) set (${PKGCOMP}_LIBRARY ${${PKGCOMP}_LIBRARY} PARENT_SCOPE) set (${PKGCOMP}_LIBRARIES ${${PKGCOMP}_LIBRARIES} PARENT_SCOPE) set (${PKGCOMP}_IS_SHARED ${${PKGCOMP}_IS_SHARED} PARENT_SCOPE) - + endif () endfunction () - - - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake/mpiexec.alcf b/src/Infrastructure/IO/PIO/ParallelIO/cmake/mpiexec.alcf index 48765fd022..62780b898d 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake/mpiexec.alcf +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake/mpiexec.alcf @@ -10,7 +10,7 @@ NP=$1 shift ${BGQ_RUNJOB:-runjob} --np $NP --block $COBALT_PARTNAME \ - --envs GPFSMPIO_NAGG_PSET=16 GPFSMPIO_ONESIDED_ALWAYS_RMW=1 \ + --envs GPFSMPIO_NAGG_PSET=16 GPFSMPIO_ONESIDED_ALWAYS_RMW=1 \ GPFSMPIO_BALANCECONTIG=1 GPFSMPIO_WRITE_AGGMETHOD=2 \ GPFSMPIO_READ_AGGMETHOD=2 PAMID_TYPED_ONESIDED=1 \ PAMID_RMA_PENDING=1M GPFSMPIO_BRIDGERINGAGG=1 : $@ diff --git a/src/Infrastructure/IO/PIO/ParallelIO/cmake_config.h.in b/src/Infrastructure/IO/PIO/ParallelIO/cmake_config.h.in index 586030c202..68b9e8dc4b 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/cmake_config.h.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/cmake_config.h.in @@ -26,6 +26,7 @@ #define USE_VARD @USE_VARD@ +#cmakedefine PIO_HAS_PAR_FILTERS /* Does netCDF support netCDF/HDF5 files? */ #cmakedefine HAVE_NETCDF4 diff --git a/src/Infrastructure/IO/PIO/ParallelIO/configure.ac b/src/Infrastructure/IO/PIO/ParallelIO/configure.ac index 007b1fdedc..1ba2087881 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/configure.ac +++ b/src/Infrastructure/IO/PIO/ParallelIO/configure.ac @@ -2,7 +2,7 @@ ## Ed Hartnett 8/16/17 # Initialize autoconf and automake. -AC_INIT(pio, 2.5.10) +AC_INIT(pio, 2.6.2) AC_CONFIG_SRCDIR(src/clib/pio_darray.c) AM_INIT_AUTOMAKE([foreign serial-tests]) @@ -10,17 +10,17 @@ AM_INIT_AUTOMAKE([foreign serial-tests]) # The PIO version, again. Use AC_SUBST for pio_meta.h and # AC_DEFINE_UNQUOTED for config.h. AC_SUBST([PIO_VERSION_MAJOR]) PIO_VERSION_MAJOR=2 -AC_SUBST([PIO_VERSION_MINOR]) PIO_VERSION_MINOR=5 -AC_SUBST([PIO_VERSION_PATCH]) PIO_VERSION_PATCH=10 - +AC_SUBST([PIO_VERSION_MINOR]) PIO_VERSION_MINOR=6 +AC_SUBST([PIO_VERSION_PATCH]) PIO_VERSION_PATCH=2 AC_DEFINE_UNQUOTED([PIO_VERSION_MAJOR], [$PIO_VERSION_MAJOR], [PIO major version]) AC_DEFINE_UNQUOTED([PIO_VERSION_MINOR], [$PIO_VERSION_MINOR], [PIO minor version]) AC_DEFINE_UNQUOTED([PIO_VERSION_PATCH], [$PIO_VERSION_PATCH], [PIO patch version]) # Once more for the documentation. AC_SUBST([VERSION_MAJOR], [2]) -AC_SUBST([VERSION_MINOR], [5]) -AC_SUBST([VERSION_PATCH], [10]) +AC_SUBST([VERSION_MINOR], [6]) +AC_SUBST([VERSION_PATCH], [2]) + # The m4 directory holds macros for autoconf. AC_CONFIG_MACRO_DIR([m4]) @@ -38,13 +38,11 @@ fi LD=ld # Required for MPE to work. LT_INIT -# Find and learn about install and ranlib. +# Find and learn about install AC_PROG_INSTALL -AC_PROG_RANLIB # Find and learn about the C compiler. AC_PROG_CC -AC_PROG_CC_C99 # Compiler with version information. This consists of the full path # name of the compiler and the reported version number. @@ -292,12 +290,8 @@ AC_MSG_RESULT([${have_szip_write}]) # Do we have parallel filter support? Parallel filters are required # for iotype NETCDF4P to use compression. -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include "netcdf_meta.h"], -[[#if !NC_HAS_PAR_FILTERS -# error -#endif] -])], [have_par_filters=yes], [have_par_filters=no]) AC_MSG_CHECKING([whether netCDF provides parallel filter support]) +AC_CHECK_LIB([netcdf], [nc_inq_filter_avail], [have_par_filters=yes], [have_par_filters=no]) AC_MSG_RESULT([${have_par_filters}]) if test "x$have_par_filters" = xyes ; then AC_DEFINE([HAVE_PAR_FILTERS], [1], [if true, netcdf-c supports filters with parallel I/O]) @@ -319,14 +313,17 @@ fi # Do we have the correct dispatch table version in netcdf-c for netcdf # integration? AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include "netcdf_meta.h"], -[[#if NC_DISPATCH_VERSION != 2 +[[#if NC_DISPATCH_VERSION < 2 +# error +#endif +#if NC_DISPATCH_VERSION > 5 # error #endif] -])], [have_dispatch2=yes], [have_dispatch2=no]) -AC_MSG_CHECKING([whether netcdf-c supports version 2 of dispatch table for netcdf integration]) -AC_MSG_RESULT([${have_dispatch2}]) -if test "x$enable_netcdf_integration" = xyes -a "x$have_dispatch2" = xno; then - AC_MSG_ERROR([NetCDF integration cannot be used with this version of netcdf-c, please upgrade your netCDF version.]) +])], [have_dispatch=yes], [have_dispatch=no]) +AC_MSG_CHECKING([whether netcdf-c supports compatible dispatch table for netcdf integration]) +AC_MSG_RESULT([${have_dispatch}]) +if test "x$enable_netcdf_integration" = xyes -a "x$have_dispatch" = xno; then + AC_MSG_ERROR([NetCDF integration version is not compatible with that in ParallelIO]) fi # Set some build settings for when netcdf-4 is supported. @@ -336,12 +333,12 @@ fi AM_CONDITIONAL(BUILD_NETCDF4, [test "x$have_netcdf_par" = xyes]) # Not working for some reason, so I will just set it... -#AC_CHECK_TYPE([MPI_Offset], [], [], [#include <mpi.h>]) -#if test "x${ac_cv_type_MPI_Offset}" = xyes; then -# AC_CHECK_SIZEOF([MPI_Offset], [], [#include <mpi.h>]) -#else -# AC_MSG_ERROR([Unable to find type MPI_Offset in mpi.h]) -#fi +AC_CHECK_TYPE([MPI_Offset], [], [], [#include <mpi.h>]) +if test "x${ac_cv_type_MPI_Offset}" = xyes; then + AC_CHECK_SIZEOF([MPI_Offset], [], [#include <mpi.h>]) +else + AC_MSG_ERROR([Unable to find type MPI_Offset in mpi.h]) +fi # If we want the timing library, we must find it. if test "x$enable_timing" = xyes; then @@ -446,6 +443,7 @@ AX_SET_META([PIO_HAS_NETCDF_INTEGRATION], [$enable_netcdf_integration],[yes]) # Create output variables from various shell variables, for use in # generating libpio.settings. + AC_SUBST([enable_shared]) AC_SUBST([enable_static]) AC_SUBST([CFLAGS]) @@ -457,13 +455,13 @@ AC_SUBST([FPPFLAGS]) # ignored by autotools AC_SUBST(HAS_PNETCDF,[$enable_pnetcdf]) AC_SUBST(HAS_LOGGING, [$enable_logging]) AC_SUBST(HAS_SZIP_WRITE, [$have_szip_write]) -AC_SUBST([HAS_PAR_FILTERS], [$have_par_filters]) +AC_SUBST([have_par_filters]) AC_SUBST([HAS_NETCDF4], [$have_netcdf4]) AC_SUBST([HAS_NETCDF4_PAR], [$have_netcdf_par]) AC_SUBST([HAS_NETCDF_INTEGRATION], [$enable_netcdf_integration]) AC_SUBST([HAS_PIO_FORTRAN], [$enable_fortran]) -# Create the build summary file. +# Create the build summary file. AC_CONFIG_FILES([libpio.settings src/clib/pio_meta.h ]) @@ -473,30 +471,30 @@ AC_CONFIG_LINKS([tests/unit/input.nl:tests/unit/input.nl]) AC_CONFIG_HEADERS([config.h]) # Create the makefiles. -AC_OUTPUT(Makefile - src/Makefile - src/clib/Makefile - src/ncint/Makefile - src/flib/Makefile - src/gptl/Makefile - tests/Makefile - tests/cunit/Makefile - tests/ncint/Makefile - tests/fncint/Makefile - tests/unit/Makefile - tests/general/Makefile - tests/general/util/Makefile - tests/performance/Makefile - doc/Makefile - doc/source/Makefile - doc/images/Makefile - examples/Makefile - examples/c/Makefile - examples/f03/Makefile - cmake/Makefile - scripts/Makefile) +AC_CONFIG_FILES(Makefile + src/Makefile + src/clib/Makefile + src/ncint/Makefile + src/flib/Makefile + src/gptl/Makefile + tests/Makefile + tests/cunit/Makefile + tests/ncint/Makefile + tests/fncint/Makefile + tests/unit/Makefile + tests/general/Makefile + tests/general/util/Makefile + tests/performance/Makefile + doc/Makefile + doc/source/Makefile + doc/images/Makefile + examples/Makefile + examples/c/Makefile + examples/f03/Makefile + cmake/Makefile + scripts/Makefile) + +AC_OUTPUT() # Show the build summary. cat libpio.settings - - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-alcf.cmake b/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-alcf.cmake index 607076479d..5c1edff260 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-alcf.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-alcf.cmake @@ -1,7 +1,7 @@ #============================================================================== # # This file sets the environment variables needed to configure and build -# on the Argonne Leadership Computing Facility systems +# on the Argonne Leadership Computing Facility systems # (mira/cetus/vesta/cooley). # #============================================================================== diff --git a/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-anlworkstation.cmake b/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-anlworkstation.cmake index 38e5b4b0a9..07ba92a2a7 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-anlworkstation.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-anlworkstation.cmake @@ -23,4 +23,3 @@ endif () if (DEFINED ENV{VALGRIND_CHECK}) set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_VALGRIND_CHECK=ON") endif () - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-ncsa.cmake b/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-ncsa.cmake index 706946ec2b..c09bdf0245 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-ncsa.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-ncsa.cmake @@ -1,7 +1,7 @@ #============================================================================== # # This file sets the environment variables needed to configure and build -# on the NCSA systems +# on the NCSA systems # (Blue Waters). # #============================================================================== diff --git a/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-nersc.cmake b/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-nersc.cmake index 6b1ac8fa79..ac20364297 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-nersc.cmake +++ b/src/Infrastructure/IO/PIO/ParallelIO/ctest/CTestEnvironment-nersc.cmake @@ -1,7 +1,7 @@ #============================================================================== # # This file sets the environment variables needed to configure and build -# on the NERSC systems +# on the NERSC systems # (edison/ corip1). # #============================================================================== diff --git a/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-gnu.sh b/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-gnu.sh index ba6b60a5d8..67c735f29d 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-gnu.sh +++ b/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-gnu.sh @@ -31,7 +31,7 @@ if [ ! -d src ]; then git clone https://github.com/PARALLELIO/ParallelIO src fi cd src -git checkout master -git pull origin master +git checkout main +git pull origin main ctest -S CTestScript.cmake,${model} -VV diff --git a/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-intel.sh b/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-intel.sh index e85d5760c0..3608789405 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-intel.sh +++ b/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-intel.sh @@ -34,7 +34,7 @@ if [ ! -d src ]; then git clone https://github.com/PARALLELIO/ParallelIO src fi cd src -git checkout master -git pull origin master +git checkout main +git pull origin main ctest -S CTestScript.cmake,${model} -VV diff --git a/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-pgi.sh b/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-pgi.sh index 8d9c0cde8b..91a576dcf0 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-pgi.sh +++ b/src/Infrastructure/IO/PIO/ParallelIO/ctest/runcdash-nwscla-pgi.sh @@ -31,7 +31,7 @@ if [ ! -d src ]; then git clone https://github.com/PARALLELIO/ParallelIO src fi cd src -git checkout master -git pull origin master +git checkout main +git pull origin main ctest -S CTestScript.cmake,${model} -VV diff --git a/src/Infrastructure/IO/PIO/ParallelIO/ctest/runctest-ncsa.sh b/src/Infrastructure/IO/PIO/ParallelIO/ctest/runctest-ncsa.sh index c3cd75e300..353066174d 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/ctest/runctest-ncsa.sh +++ b/src/Infrastructure/IO/PIO/ParallelIO/ctest/runctest-ncsa.sh @@ -1,7 +1,7 @@ #!/bin/sh #============================================================================== # -# This script defines how to run CTest on the National Center for +# This script defines how to run CTest on the National Center for # Supercomputing Applications system (blue waters). # # This assumes the CTest model name (e.g., "Nightly") is passed to it when diff --git a/src/Infrastructure/IO/PIO/ParallelIO/ctest/runctest-nersc.sh b/src/Infrastructure/IO/PIO/ParallelIO/ctest/runctest-nersc.sh index a84d26bbeb..3bfd3c8a59 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/ctest/runctest-nersc.sh +++ b/src/Infrastructure/IO/PIO/ParallelIO/ctest/runctest-nersc.sh @@ -45,7 +45,7 @@ case "$NERSC_HOST" in cori) salloc -N 1 -C knl ./runctest.slurm ;; -esac +esac # Wait for the job to complete before exiting #while true; do # status=`squeue -j $jobid` diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/CMakeFiles/CMakeOutput.log b/src/Infrastructure/IO/PIO/ParallelIO/doc/CMakeFiles/CMakeOutput.log index c254632f6b..36520873ae 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/CMakeFiles/CMakeOutput.log +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/CMakeFiles/CMakeOutput.log @@ -1,8 +1,8 @@ The system is: Linux - 3.10.0-123.el7.x86_64 - x86_64 Compiling the C compiler identification source file "CMakeCCompilerId.c" succeeded. -Compiler: /usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc -Build flags: -Id flags: +Compiler: /usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc +Build flags: +Id flags: The output was: 0 @@ -14,9 +14,9 @@ Compilation of the C compiler identification source "CMakeCCompilerId.c" produce The C compiler identification is Intel, found in "/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/3.2.3/CompilerIdC/a.out" Compiling the CXX compiler identification source file "CMakeCXXCompilerId.cpp" succeeded. -Compiler: /usr/bin/c++ -Build flags: -Id flags: +Compiler: /usr/bin/c++ +Build flags: +Id flags: The output was: 0 @@ -38,7 +38,7 @@ Building C object CMakeFiles/cmTryCompileExec1905307408.dir/testCCompiler.c.o icc: command line remark #10148: option '-i-dynamic' not supported Linking C executable cmTryCompileExec1905307408 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec1905307408.dir/link.txt --verbose=1 -/usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc CMakeFiles/cmTryCompileExec1905307408.dir/testCCompiler.c.o -o cmTryCompileExec1905307408 -rdynamic +/usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc CMakeFiles/cmTryCompileExec1905307408.dir/testCCompiler.c.o -o cmTryCompileExec1905307408 -rdynamic icc: command line remark #10148: option '-i-dynamic' not supported gmake[1]: Leaving directory `/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp' @@ -55,7 +55,7 @@ Building C object CMakeFiles/cmTryCompileExec3327212404.dir/CMakeCCompilerABI.c. icc: command line remark #10148: option '-i-dynamic' not supported Linking C executable cmTryCompileExec3327212404 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec3327212404.dir/link.txt --verbose=1 -/usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc -v CMakeFiles/cmTryCompileExec3327212404.dir/CMakeCCompilerABI.c.o -o cmTryCompileExec3327212404 -rdynamic +/usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc -v CMakeFiles/cmTryCompileExec3327212404.dir/CMakeCCompilerABI.c.o -o cmTryCompileExec3327212404 -rdynamic mpicc for MVAPICH2 version 1.8.1 icc: command line remark #10148: option '-i-dynamic' not supported icc version 15.0.2 (gcc version 4.8.3 compatibility) @@ -190,7 +190,7 @@ Building CXX object CMakeFiles/cmTryCompileExec2556829595.dir/testCXXCompiler.cx /usr/bin/c++ -o CMakeFiles/cmTryCompileExec2556829595.dir/testCXXCompiler.cxx.o -c /scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp/testCXXCompiler.cxx Linking CXX executable cmTryCompileExec2556829595 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec2556829595.dir/link.txt --verbose=1 -/usr/bin/c++ CMakeFiles/cmTryCompileExec2556829595.dir/testCXXCompiler.cxx.o -o cmTryCompileExec2556829595 -rdynamic +/usr/bin/c++ CMakeFiles/cmTryCompileExec2556829595.dir/testCXXCompiler.cxx.o -o cmTryCompileExec2556829595 -rdynamic gmake[1]: Leaving directory `/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp' @@ -205,14 +205,14 @@ Building CXX object CMakeFiles/cmTryCompileExec1080422183.dir/CMakeCXXCompilerAB /usr/bin/c++ -o CMakeFiles/cmTryCompileExec1080422183.dir/CMakeCXXCompilerABI.cpp.o -c /home/katec/cmake/cmake-3.2.3/Modules/CMakeCXXCompilerABI.cpp Linking CXX executable cmTryCompileExec1080422183 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec1080422183.dir/link.txt --verbose=1 -/usr/bin/c++ -v CMakeFiles/cmTryCompileExec1080422183.dir/CMakeCXXCompilerABI.cpp.o -o cmTryCompileExec1080422183 -rdynamic +/usr/bin/c++ -v CMakeFiles/cmTryCompileExec1080422183.dir/CMakeCXXCompilerABI.cpp.o -o cmTryCompileExec1080422183 -rdynamic Using built-in specs. COLLECT_GCC=/usr/bin/c++ COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/4.8.3/lto-wrapper Target: x86_64-redhat-linux Configured with: ../configure --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-bootstrap --enable-shared --enable-threads=posix --enable-checking=release --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-linker-hash-style=gnu --enable-languages=c,c++,objc,obj-c++,java,fortran,ada,go,lto --enable-plugin --enable-initfini-array --disable-libgcj --with-isl=/builddir/build/BUILD/gcc-4.8.3-20140911/obj-x86_64-redhat-linux/isl-install --with-cloog=/builddir/build/BUILD/gcc-4.8.3-20140911/obj-x86_64-redhat-linux/cloog-install --enable-gnu-indirect-function --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux Thread model: posix -gcc version 4.8.3 20140911 (Red Hat 4.8.3-9) (GCC) +gcc version 4.8.3 20140911 (Red Hat 4.8.3-9) (GCC) COMPILER_PATH=/usr/local/intel-cluster-15.0.2.164/:/usr/libexec/gcc/x86_64-redhat-linux/4.8.3/:/usr/libexec/gcc/x86_64-redhat-linux/4.8.3/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/4.8.3/:/usr/lib/gcc/x86_64-redhat-linux/ LIBRARY_PATH=/usr/lib/gcc/x86_64-redhat-linux/4.8.3/:/usr/lib/gcc/x86_64-redhat-linux/4.8.3/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/4.8.3/../../../:/lib/:/usr/lib/ COLLECT_GCC_OPTIONS='-v' '-o' 'cmTryCompileExec1080422183' '-rdynamic' '-shared-libgcc' '-mtune=generic' '-march=x86-64' @@ -301,7 +301,7 @@ Building CXX object CMakeFiles/cmTryCompileExec2444100226.dir/feature_tests.cxx. /usr/bin/c++ -std=c++1y -o CMakeFiles/cmTryCompileExec2444100226.dir/feature_tests.cxx.o -c /scratch/cluster/katec/ParallelIO/doc/CMakeFiles/feature_tests.cxx Linking CXX executable cmTryCompileExec2444100226 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec2444100226.dir/link.txt --verbose=1 -/usr/bin/c++ CMakeFiles/cmTryCompileExec2444100226.dir/feature_tests.cxx.o -o cmTryCompileExec2444100226 -rdynamic +/usr/bin/c++ CMakeFiles/cmTryCompileExec2444100226.dir/feature_tests.cxx.o -o cmTryCompileExec2444100226 -rdynamic gmake[1]: Leaving directory `/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp' @@ -375,7 +375,7 @@ Building CXX object CMakeFiles/cmTryCompileExec295155124.dir/feature_tests.cxx.o /usr/bin/c++ -std=c++11 -o CMakeFiles/cmTryCompileExec295155124.dir/feature_tests.cxx.o -c /scratch/cluster/katec/ParallelIO/doc/CMakeFiles/feature_tests.cxx Linking CXX executable cmTryCompileExec295155124 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec295155124.dir/link.txt --verbose=1 -/usr/bin/c++ CMakeFiles/cmTryCompileExec295155124.dir/feature_tests.cxx.o -o cmTryCompileExec295155124 -rdynamic +/usr/bin/c++ CMakeFiles/cmTryCompileExec295155124.dir/feature_tests.cxx.o -o cmTryCompileExec295155124 -rdynamic gmake[1]: Leaving directory `/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp' @@ -449,7 +449,7 @@ Building CXX object CMakeFiles/cmTryCompileExec3307289814.dir/feature_tests.cxx. /usr/bin/c++ -std=c++98 -o CMakeFiles/cmTryCompileExec3307289814.dir/feature_tests.cxx.o -c /scratch/cluster/katec/ParallelIO/doc/CMakeFiles/feature_tests.cxx Linking CXX executable cmTryCompileExec3307289814 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec3307289814.dir/link.txt --verbose=1 -/usr/bin/c++ CMakeFiles/cmTryCompileExec3307289814.dir/feature_tests.cxx.o -o cmTryCompileExec3307289814 -rdynamic +/usr/bin/c++ CMakeFiles/cmTryCompileExec3307289814.dir/feature_tests.cxx.o -o cmTryCompileExec3307289814 -rdynamic gmake[1]: Leaving directory `/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp' diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/Doxyfile.in b/src/Infrastructure/IO/PIO/ParallelIO/doc/Doxyfile.in index 713d3b56aa..3047a79e14 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/Doxyfile.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/Doxyfile.in @@ -291,7 +291,7 @@ OPTIMIZE_OUTPUT_VHDL = NO # Note that for custom extensions you also need to set FILE_PATTERNS otherwise # the files are not read by doxygen. -EXTENSION_MAPPING = f90=Fortran +EXTENSION_MAPPING = f90=Fortran # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments # according to the Markdown format, which allows for more readable @@ -772,7 +772,7 @@ INPUT = @CMAKE_CURRENT_SOURCE_DIR@/../doc/source \ @CMAKE_CURRENT_SOURCE_DIR@/../examples/c \ @CMAKE_CURRENT_SOURCE_DIR@/../examples/f03 \ @FORTRAN_SRC_FILES@ \ - @C_SRC_FILES@ + @C_SRC_FILES@ # This tag can be used to specify the character encoding of the source files diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/DoxygenLayout.xml b/src/Infrastructure/IO/PIO/ParallelIO/doc/DoxygenLayout.xml index 563e9b1df9..c0f0b1cb89 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/DoxygenLayout.xml +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/DoxygenLayout.xml @@ -6,7 +6,7 @@ <tab type="namespacemembers" visible="yes" title="Symbol Index"/> <tab type="modules" visible="yes" title="Modules"/> <tab type="classes" visible="yes" title="Data Types"/> - <tab type="examples" visible="yes" title="Examples"/> + <tab type="examples" visible="yes" title="Examples"/> <tab type="pages" visible="yes" title="Page Index"/> <tab type="namespaces" visible="no" title="Namespaces (JMD)"> <tab type="namespaces" visible="yes" title=""/> diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/customdoxygen.css b/src/Infrastructure/IO/PIO/ParallelIO/doc/customdoxygen.css index 5f14f59d3f..f870af6320 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/customdoxygen.css +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/customdoxygen.css @@ -141,11 +141,11 @@ a.elRef { } a.code, a.code:visited { - color: #4665A2; + color: #4665A2; } a.codeRef, a.codeRef:visited { - color: #4665A2; + color: #4665A2; } /* @end */ @@ -289,7 +289,7 @@ p.formulaDsp { } img.formulaDsp { - + } img.formulaInl { @@ -347,20 +347,20 @@ span.charliteral { color: #008080 } -span.vhdldigit { - color: #ff00ff +span.vhdldigit { + color: #ff00ff } -span.vhdlchar { - color: #000000 +span.vhdlchar { + color: #000000 } -span.vhdlkeyword { - color: #700070 +span.vhdlkeyword { + color: #700070 } -span.vhdllogic { - color: #ff0000 +span.vhdllogic { + color: #ff0000 } blockquote { @@ -555,9 +555,9 @@ table.memberdecls { } .memdoc, dl.reflist dd { - border-bottom: 1px solid #A8B8D9; - border-left: 1px solid #A8B8D9; - border-right: 1px solid #A8B8D9; + border-bottom: 1px solid #A8B8D9; + border-left: 1px solid #A8B8D9; + border-right: 1px solid #A8B8D9; padding: 6px 10px 2px 10px; background-color: #FBFCFD; border-top-width: 0; @@ -609,18 +609,18 @@ dl.reflist dd { .params, .retval, .exception, .tparams { margin-left: 0px; padding-left: 0px; -} +} .params .paramname, .retval .paramname { font-weight: bold; vertical-align: top; } - + .params .paramtype { font-style: italic; vertical-align: top; -} - +} + .params .paramdir { font-family: "courier new",courier,monospace; vertical-align: top; @@ -797,8 +797,8 @@ table.fieldtable { .fieldtable td.fielddoc p:first-child { margin-top: 2px; -} - +} + .fieldtable td.fielddoc p:last-child { margin-bottom: 2px; } @@ -872,7 +872,7 @@ table.fieldtable { color: #283A5D; font-family: 'Lucida Grande',Geneva,Helvetica,Arial,sans-serif; text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); - text-decoration: none; + text-decoration: none; } .navpath li.navelem a:hover @@ -901,7 +901,7 @@ div.summary padding-right: 5px; width: 50%; text-align: right; -} +} div.summary a { @@ -1013,19 +1013,19 @@ dl.section dd { vertical-align: bottom; border-collapse: separate; } - + #projectlogo img -{ +{ border: 0px none; } - + #projectname { font: 300% Tahoma, Arial,sans-serif; margin: 0px; padding: 2px 0px; } - + #projectbrief { font: 120% Tahoma, Arial,sans-serif; @@ -1120,7 +1120,7 @@ div.toc ul { list-style: none outside none; border: medium none; padding: 0px; -} +} div.toc li.level1 { margin-left: 0px; @@ -1181,4 +1181,3 @@ tr.heading h2 { display:inline; } } - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/CAMexample.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/CAMexample.txt index 77bd3df9fb..d6a3c20f3c 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/CAMexample.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/CAMexample.txt @@ -13,7 +13,7 @@ * Documents produced by Doxygen are derivative works derived from the * input used in their production; they are not affected by this license. * - */ + */ /*! \page CAMexample Community Atmosphere Model (CAM) \section cam Implementation of PIO in CAM @@ -37,7 +37,7 @@ initializes PIO in CAM. Init_pio_subsystem calls read_namelist_pio, which calls set_pio_parameters. The main parameters set includes the IO mode (netcdf vs pnetcdf), number of IO tasks, and IO stride. -Cam_pio_createfile and cam_pio_openfile create and open a PIO file, respectively. +Cam_pio_createfile and cam_pio_openfile create and open a PIO file, respectively. Cam_pio_createfile is called from cam_write_restart, h_define (called from wshist, which is called from write_restart_history), and atm_write_srfrest_mct. @@ -72,7 +72,7 @@ decomposition already exists. The routine performing that search is find_iodesc (called from get_phys_decomp and get_dyn_decomp). This capability is supported only when the range of history output is the whole domain. -Get_phys_decomp calls get_phys_ldof (or get_column_ldof), and get_dyn_decomp calls +Get_phys_decomp calls get_phys_ldof (or get_column_ldof), and get_dyn_decomp calls get_dyn_ldof (or get_column_ldof). These routines do the bulk of the work in constructing the IO decompositions. Get_column_ldof is called when the history output is restricted to a subset of the domain. diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Decomp.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Decomp.txt index f47c8bf18e..27dc1f1249 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Decomp.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Decomp.txt @@ -38,21 +38,21 @@ As an example suppose we have a global two dimensional grid of size terms of offset from the initial element ie <pre> - 0 1 2 3 - 4 5 6 7 + 0 1 2 3 + 4 5 6 7 8 9 10 11 12 13 14 15 - 16 17 18 19 + 16 17 18 19 </pre> Now suppose this data is distributed over the compute tasks as follows: <pre> -0: { 0 4 8 12 } -1: { 16 1 5 9 } -2: { 13 17 2 6 } -3: { 10 14 18 3 } -4: { 7 11 15 19 } +0: { 0 4 8 12 } +1: { 16 1 5 9 } +2: { 13 17 2 6 } +3: { 10 14 18 3 } +4: { 7 11 15 19 } </pre> If we have 2 io tasks the Box rearranger would give: @@ -83,4 +83,3 @@ IO task provides a measure of what you might expect the performance of the underlying IO library to be if it were used without PIO. */ - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Error.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Error.txt index 1234c9f7c1..2911446de3 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Error.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Error.txt @@ -1,4 +1,4 @@ -/*! +/*! \page error Error Handling By default, PIO handles errors internally by printing a string diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Examples.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Examples.txt index 932d278957..db7a8c4e8b 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Examples.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Examples.txt @@ -28,11 +28,11 @@ The Fortran examples are in the examples/f03 subdirectory. ### Other Examples - PIO has been implemented in several geophysical component models, including the -Community Atmosphere Model (CAM), the Community Land Model (CLM), the Parallel Ocean Program -(POP), the Community Ice CodE (CICE), and coupler for used by CCSM4.0 (CPL7). We also provide -several simpler example code as well as a test code that is suitable for regression testing and -benchmarking. + PIO has been implemented in several geophysical component models, including the +Community Atmosphere Model (CAM), the Community Land Model (CLM), the Parallel Ocean Program +(POP), the Community Ice CodE (CICE), and coupler for used by CCSM4.0 (CPL7). We also provide +several simpler example code as well as a test code that is suitable for regression testing and +benchmarking. - \subpage CAMexample - \subpage testpio_example diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Installing.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Installing.txt index d0423abc1f..f2e32f622b 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Installing.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Installing.txt @@ -17,7 +17,7 @@ pio-2.4.3.tar.gz. PIO can use NetCDF (version 4.6.1+) and/or PnetCDF (version 1.9.0+) for I/O. Ideally, the NetCDF version should be built with MPI, which requires that it -be linked with an MPI-enabled version of HDF5. Optionally, NetCDF can be +be linked with an MPI-enabled version of HDF5. Optionally, NetCDF can be built with DAP support, which introduces a dependency on CURL. Additionally, HDF5, itself, introduces dependencies on LIBZ and (optionally) SZIP. @@ -91,10 +91,10 @@ where `mpicc` and `mpif90` are the appropriate MPI-enabled compiler wrappers for your system. The `OPTIONS` section typically should consist of pointers to the install -locations for various dependencies, assuming these dependencies are not -located in *canonical* search locations. +locations for various dependencies, assuming these dependencies are not +located in *canonical* search locations. -For each dependency `XXX`, one can specify the location of its +For each dependency `XXX`, one can specify the location of its installation path with the CMake variable `XXX_PATH`. If the `C` and `Fortran` libraries for the dependency are installed in different locations (such as can be done with NetCDF), then you can specify individually @@ -111,7 +111,7 @@ CMake configuration line: This works for the dependencies: `NetCDF`, `PnetCDF`, `HDF5`, `LIBZ`, `SZIP`. -For specific instructions to install on various commonly used super computers, please read the [walk-through guide to PIO Installation](@ref mach_walkthrough). +For specific instructions to install on various commonly used super computers, please read the [walk-through guide to PIO Installation](@ref mach_walkthrough). ### Additional CMake Options ### @@ -125,14 +125,14 @@ libraries are already installed on the system, the user can point PIO to the location of these libraries with the `GPTL_PATH` variable (or, individually, `GPTL_C_PATH` and `GPTL_Fortran_Perf_PATH` variables). However, if these GPTL libraries are not installed on the system, and GPTL cannot be found, -then PIO will build its own internal version of GPTL. +then PIO will build its own internal version of GPTL. If PnetCDF is not installed on the system, the user can disable its use by setting `-DWITH_PNETCDF=OFF`. This will disable the search for PnetCDF on the system and disable the use of PnetCDF from within PIO. If the user wishes to disable the PIO tests, then the user can set the -variable `-DPIO_ENABLE_TESTS=OFF`. This will entirely disable the CTest +variable `-DPIO_ENABLE_TESTS=OFF`. This will entirely disable the CTest testing suite, as well as remove all of the test build targets. If you wish to install PIO in a safe location for use later with other @@ -168,7 +168,7 @@ Once the tests have been built, you may run tests with: _Note: If you have not run `make tests` before you run `ctest`, then you will see all of the tests fail._ -Alternatively, you may build the test executables and then run tests +Alternatively, you may build the test executables and then run tests immediately with: diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Testing.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Testing.txt index 0ef7c48a31..058df9ba78 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Testing.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/Testing.txt @@ -112,17 +112,17 @@ the PIO build directory). The contents of the namelist file should look like: &pioperf - + decompfile = "/u/home/user/piodecomp30tasks01dims06.dat" - + pio_typenames = 'pnetcdf' - + niotasks = 30 - + rearrangers = 1 - + nvars = 2 - + / Here, the second line ("decompfile") points to the path for your diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/api.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/api.txt index 0e36baa82a..435f59f972 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/api.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/api.txt @@ -19,7 +19,7 @@ - \ref PIO_inquire_variable - \ref PIO_inq_att \section metadata Defining Metadata - - \ref PIO_def_dim + - \ref PIO_def_dim - \ref PIO_def_var - \ref PIO_get_att - \ref PIO_put_att diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/base.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/base.txt index d53491e1d8..d81d27f4e4 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/base.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/base.txt @@ -1,4 +1,4 @@ -/*! +/*! @mainpage Parallel I/O Libraries (PIO) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/contributing_code.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/contributing_code.txt index 70e3a0fc46..4b070b3f35 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/contributing_code.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/contributing_code.txt @@ -118,18 +118,18 @@ all issues have been resolved. <ul> <li>Programmers begin work on a feature or fix by branching from -master. +main. <li>When a branch is ready, it is submitted to code review via pull -request. A number of Github tests will be run automatically. +request. A number of Github tests will be run automatically. <li>When code review is complete, and the changes are approved, the PR -is merged into the master branch. +is merged into the main branch. -<li>The master branch is tested periodically by CDash (every ~6 -hours). Any test failures and the merge to master may be rolled back. +<li>The main branch is tested periodically by CDash (every ~6 +hours). Any test failures and the merge to main may be rolled back. -<li>The branch is then deleted by whomever merged it to master. +<li>The branch is then deleted by whomever merged it to main. </ul> diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc index ee58c4bbeb..6a8d49ee28 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc @@ -1,6 +1,6 @@ type (iosystem_desc_t) :: iosystem - integer (i4) :: dims(1) + integer (i4) :: dims(1) integer (kind=PIO_OFFSET) :: start(1), count(1) type (io_desc_t) :: iodesc ... @@ -11,4 +11,3 @@ start(1) = 3 count(1) = 3 call PIO_initdecomp(iosystem,PIO_double,dims,start,count,iodesc) - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr index c919f29cc5..ea1cd843ad 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr @@ -1,6 +1,6 @@ type (iosystem_desc_t) :: iosystem - integer (i4) :: dims(1) + integer (i4) :: dims(1) integer (kind=PIO_OFFSET) :: compstart(1), compcount(1) integer (kind=PIO_OFFSET) :: iostart(1), iocount(1) type (io_desc_t) :: iodesc @@ -14,4 +14,3 @@ iostart(1) = 1 iocount(1) = 4 call PIO_initdecomp(iosystem,PIO_double,dims,compstart,compcount,iodesc,iostart=iostart,iocount=iocount) - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr-pe1 b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr-pe1 index 5399ad8377..9f98f09f34 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr-pe1 +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr-pe1 @@ -1,6 +1,6 @@ type (iosystem_desc_t) :: iosystem - integer (i4) :: dims(1) + integer (i4) :: dims(1) integer (kind=PIO_OFFSET) :: compstart(1), compcount(1) type (io_desc_t) :: iodesc ... @@ -11,4 +11,3 @@ compstart(1) = 1 compcount(1) = 2 call PIO_initdecomp(iosystem,PIO_double,dims,compstart,compcount,iodesc) - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr-pe2 b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr-pe2 index 31880f4f87..917e09ba63 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr-pe2 +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-bc-rearr-pe2 @@ -1,6 +1,6 @@ type (iosystem_desc_t) :: iosystem - integer (i4) :: dims(1) + integer (i4) :: dims(1) integer (kind=PIO_OFFSET) :: compstart(1), compcount(1) integer (kind=PIO_OFFSET) :: iostart(1), iocount(1) type (io_desc_t) :: iodesc @@ -14,4 +14,3 @@ iostart(1) = 5 iocount(1) = 4 call PIO_initdecomp(iosystem,PIO_double,dims,compstart,compcount,iodesc,iostart=iostart,iocount=iocount) - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-dof b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-dof index b53a82d799..e967bf22ef 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-dof +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-dof @@ -1,6 +1,6 @@ type (iosystem_desc_t) :: iosystem - integer (i4) :: dims(1) + integer (i4) :: dims(1) integer (i4) :: compdof type (io_desc_t) :: iodesc ... @@ -10,4 +10,3 @@ dims(1) = 8 compdof = (/2,4,5/) call PIO_initdecomp(iosystem,PIO_double,dims,compdof,iodesc) - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-dof-rearr b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-dof-rearr index af8b9910c2..dd95125153 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-dof-rearr +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/example/simple-dof-rearr @@ -1,6 +1,6 @@ type (iosystem_desc_t) :: iosystem - integer (i4) :: dims(1) + integer (i4) :: dims(1) integer (i4) :: compdof type (io_desc_t) :: iodesc integer (kind=PIO_OFFSET) :: iostart(:),iocount(:) @@ -14,4 +14,3 @@ iostart(1) = 1 iocount(1) = 4 call PIO_initdecomp(iosystem,PIO_double,dims,compdof,iodesc,iostart=iostart,iocount=iocount) - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/faq.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/faq.txt index c3c4bca336..713174004c 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/faq.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/faq.txt @@ -1,14 +1,14 @@ -/*! \page faq Frequently Asked Questions +/*! \page faq Frequently Asked Questions Here is a list of frequently asked questions and their answers. - + <dl> <dt>How do I specify which tasks perform IO?</dt> <dd>This is done in the call to \ref PIO_init which has two interfaces: init_intracom and init_intercom. - + <ul> <li> In the init_intracom interface, use the num_iotasks and @@ -32,7 +32,7 @@ the tunable blocksize parameter, \ref PIO_set_blocksize, determines the actual number of tasks used for a given IO operation. </dd> -<dt>How do I test if PIO is installed and working correctly?</dt> +<dt>How do I test if PIO is installed and working correctly?</dt> <dd>The PIO Library distribution contains tests for PIO. They are run my 'make check'. The tests use mpiexec to run tests on 4, 8, or 16 diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/iosystem.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/iosystem.txt index 5acce7fc92..b4616c84fa 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/iosystem.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/iosystem.txt @@ -21,4 +21,3 @@ for each open IOSystem. @image html PIO_Async.png "PIO Async Mode" */ - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/mach_walkthrough.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/mach_walkthrough.txt index 357b888213..d741cd28e7 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/mach_walkthrough.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/mach_walkthrough.txt @@ -77,13 +77,13 @@ Building PIO requires running the CMake configure and then make. In the PIO_buil <ol> <li>Directory setup -Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. +Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. <li>Modules -Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Edison. +Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Edison. -+ Intel ++ Intel %> module purge<br> %> module load PrgEnv-intel<br> @@ -156,7 +156,7 @@ Building PIO requires running the CMake configure and then make. In the PIO_buil <ol> <li>Directory setup -Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. +Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. <li> Softenv packages and environment variables @@ -187,11 +187,11 @@ Building PIO requires running the CMake configure and then make. In the PIO_buil <ol> <li>Directory setup -Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. +Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. <li>Modules -Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Hobart. +Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Hobart. + Intel @@ -244,11 +244,11 @@ Building PIO requires running the CMake configure and then make. In the PIO_buil <ol> <li>Directory setup -Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. +Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. <li>Modules -Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Hobart. +Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Hobart. + Intel @@ -261,7 +261,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module purge<br> %> module load compiler/nag/6.0<br> %> module load tool/parallel-netcdf/1.6.1/nag/openmpi<br> - + + PGI %> module purge<br> diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/testpio_example.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/testpio_example.txt index c5d4ef7e5c..770ced7528 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/testpio_example.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/testpio_example.txt @@ -1,8 +1,8 @@ /*! \page testpio_example testpio: a regression and benchmarking code -The testpio directory, included with the release package, tests both the accuracy -and performance of reading and writing data -using the pio library. +The testpio directory, included with the release package, tests both the accuracy +and performance of reading and writing data +using the pio library. The testpio directory contains 3 perl scripts that you can use to build and run the testpio.F90 code. <ul> @@ -49,7 +49,7 @@ block, io_nml, contains some general settings: ("bin","pnc","snc"), binary, pnetcdf, or serial netcdf</td> </tr> <tr> - <td>rearr</td> <td>string, type of rearranging to be done + <td>rearr</td> <td>string, type of rearranging to be done ("none","mct","box","boxauto")</td> </tr> <tr> @@ -61,15 +61,15 @@ block, io_nml, contains some general settings: <td>base</td> <td>integer, base pe associated with nprocIO striding</td> </tr> <tr> - <td>stride</td> <td>integer, the stride of io pes across the global pe set. A stride=-1 + <td>stride</td> <td>integer, the stride of io pes across the global pe set. A stride=-1 directs PIO to calculate the stride automatically.</td> </tr> <tr> - <td>num_aggregator</td> <td>integer, mpi-io number of aggregators, only used if no + <td>num_aggregator</td> <td>integer, mpi-io number of aggregators, only used if no pio rearranging is done</td> </tr> <tr> - <td>dir</td> <td>string, directory to write output data, this must exist + <td>dir</td> <td>string, directory to write output data, this must exist before the model starts up</td> </tr> <tr> @@ -85,19 +85,19 @@ block, io_nml, contains some general settings: <td>compdof_input</td> <td>string, setting of the compDOF ('namelist' or a filename)</td> </tr> <tr> - <td>compdof_output</td> <td>string, whether the compDOF is saved to disk + <td>compdof_output</td> <td>string, whether the compDOF is saved to disk ('none' or a filename)</td> </tr> </table> Notes: - the "mct" rearr option is not currently available - - if rearr is set to "none", then the computational decomposition is also + - if rearr is set to "none", then the computational decomposition is also going to be used as the IO decomposition. The computation decomposition must therefore be suited to the underlying I/O methods. - if rearr is set to "box", then pio is going to generate an internal IO decomposition automatically and pio will rearrange to that decomp. - - num_aggregator is used with mpi-io and no pio rearranging. mpi-io is only + - num_aggregator is used with mpi-io and no pio rearranging. mpi-io is only used with binary data. - nprocsIO, base, and stride implementation has some special options - if nprocsIO > 0 and stride > 0, then use input values @@ -123,7 +123,7 @@ blocks are identical in use. ("xyz","xzy","yxz","yzx","zxy","zyx")</td> </tr> <tr> - <td>grddecomp</td> <td>string, sets up the block size with gdx, gdy, and gdz, see + <td>grddecomp</td> <td>string, sets up the block size with gdx, gdy, and gdz, see below, ("x","y","z","xy","xye","xz","xze","yz","yze", "xyz","xyze","setblk")</td> </tr> @@ -167,7 +167,7 @@ are provided below. Testpio writes out several files including summary information to stdout, data files to the namelists directory, and a netcdf -file summarizing the decompositions. The key output information +file summarizing the decompositions. The key output information is written to stdout and contains the timing information. In addition, a netcdf file called gdecomp.nc is written that provides both the block and task ids for each gridcell as computed by the decompositions. @@ -221,24 +221,24 @@ combinations of these cpp flags. The decomposition implementation supports the decomposition of a general 3 dimensional "nx * ny * nz" grid into multiple blocks -of gridcells which are then ordered and assigned to processors. -In general, blocks in the decomposition are rectangular, -"gdx * gdy * gdz" and the same size, although some blocks around -the edges of the domain may be smaller if the decomposition is uneven. -Both gridcells within the block and blocks within the domain can be +of gridcells which are then ordered and assigned to processors. +In general, blocks in the decomposition are rectangular, +"gdx * gdy * gdz" and the same size, although some blocks around +the edges of the domain may be smaller if the decomposition is uneven. +Both gridcells within the block and blocks within the domain can be ordered in any of the possible dimension hierarchies, such as "xyz" -where the first dimension is the fastest. +where the first dimension is the fastest. -The gdx, gdy, and gdz inputs allow the user to specify the size in -any dimension and the grddecomp input specifies which dimensions are -to be further optimized. In general, automatic decomposition generation -of 3 dimensional grids can be done in any of possible combination of +The gdx, gdy, and gdz inputs allow the user to specify the size in +any dimension and the grddecomp input specifies which dimensions are +to be further optimized. In general, automatic decomposition generation +of 3 dimensional grids can be done in any of possible combination of dimensions, (x, y, z, xy, xz, yz, or xyz), with the other dimensions having a fixed block size. The automatic generation of the decomposition is based upon an internal algorithm that tries to determine the most "square" blocks with an additional constraint on minimizing the maximum number of gridcells across processors. If evenly divided grids are -desired, use of the "e" addition to grddecomp specifies that the grid +desired, use of the "e" addition to grddecomp specifies that the grid decomposition must be evenly divided. The setblk option uses the prescibed gdx, gdy, and gdz inputs without further automation. @@ -247,7 +247,7 @@ in mapping blocks to processors, but has a few additional options. "cont1d" (contiguous 1d) basically unwraps the blocks in the order specified by the blkorder input and then decomposes that "1d" list of blocks onto processors by contiguously grouping blocks together and allocating -them to a processor. The number of contiguous blocks that are +them to a processor. The number of contiguous blocks that are allocated to a processor is the maximum of the values of bdx, bdy, and bdz inputs. Contiguous blocks are allocated to each processor in turn in a round robin fashion until all blocks are allocated. The @@ -256,13 +256,13 @@ contiguous blocks are set automatically such that each processor recieves only 1 set of contiguous blocks. The ysym2 and ysym4 blkdecomp2 options modify the original block layout such that the tasks assigned to the blocks are 2-way or 4-way symetric -in the y axis. +in the y axis. The decomposition tool is extremely flexible, but arbitrary inputs will not always yield valid decompositions. If a valid decomposition cannot be computed based on the global grid size, -number of pes, number of blocks desired, and decomposition options, -the model will stop. +number of pes, number of blocks desired, and decomposition options, +the model will stop. As indicated above, the IO decomposition must be suited to the IO methods, so decompositions are even further limited by those @@ -290,7 +290,7 @@ Some decomposition examples: Standard xyz ordering, 2d decomp: note: blkdecomp plays no role since there is 1 block per pe <pre> - nx_global 6 + nx_global 6 ny_global 4 nz_global 1 ______________________________ npes 4 |B3 P3 |B4 P4 | @@ -311,7 +311,7 @@ note: blkdecomp plays no role since there is 1 block per pe Same as above but yxz ordering, 2d decomp note: blkdecomp plays no role since there is 1 block per pe <pre> - nx_global 6 + nx_global 6 ny_global 4 nz_global 1 _____________________________ npes 4 |B2 P2 |B4 P4 | @@ -329,11 +329,11 @@ note: blkdecomp plays no role since there is 1 block per pe bdz 0 </pre> -xyz grid ordering, 1d x decomp +xyz grid ordering, 1d x decomp note: blkdecomp plays no role since there is 1 block per pe note: blkorder plays no role since it's a 1d decomp <pre> - nx_global 8 + nx_global 8 ny_global 4 nz_global 1 _____________________________________ npes 4 |B1 P1 |B2 P2 |B3 P3 |B4 P4 | @@ -353,7 +353,7 @@ xyz grid ordering, 1d x decomp yxz block ordering, 2d grid decomp, 2d block decomp, 4 block per pe <pre> - nx_global 8 + nx_global 8 ny_global 4 nz_global 1 _____________________________________ npes 4 |B4 P2 |B8 P2 |B12 P4 |B16 P4 | diff --git a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/users_guide.txt b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/users_guide.txt index 539aa0cda3..a03198d9d7 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/doc/source/users_guide.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/doc/source/users_guide.txt @@ -1,4 +1,4 @@ -/*! +/*! @page users_guide PIO User's Guide diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/CMakeLists.txt b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/CMakeLists.txt index efa784e01e..01238266a6 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/CMakeLists.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/CMakeLists.txt @@ -7,7 +7,7 @@ ADD_CUSTOM_COMMAND( ) ENDFOREACH() -SET(SRC check_mod.F90 gdecomp_mod.F90 kinds_mod.F90 namelist_mod.F90 +SET(SRC check_mod.F90 gdecomp_mod.F90 kinds_mod.F90 namelist_mod.F90 testpio.F90 utils_mod.F90 ${TEMPSRCF90}) SET(WSSRC wstest.c) @@ -15,7 +15,7 @@ INCLUDE_DIRECTORIES(${PIO_INCLUDE_DIRS}) LINK_DIRECTORIES(${PIO_LIB_DIR}) ADD_EXECUTABLE(testpio ${SRC}) ADD_EXECUTABLE(wstest ${WSSRC}) -if(${PIO_BUILD_TIMING} MATCHES "ON") +if(${PIO_BUILD_TIMING} MATCHES "ON") SET(TIMING_LINK_LIB timing) endif() @@ -25,8 +25,3 @@ else() TARGET_LINK_LIBRARIES(testpio piof pioc ${TIMING_LINK_LIB}) endif() TARGET_LINK_LIBRARIES(wstest pioc ${TIMING_LINK_LIB}) - - - - - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/MPASA30km.csh b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/MPASA30km.csh index a141354fa9..37e7cf7b48 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/MPASA30km.csh +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/MPASA30km.csh @@ -1,10 +1,9 @@ #!/usr/bin/csh set id = `date "+%m%d%y-%H%M"` set host = 'kraken' -#./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 120 --bench MPASA30km -numIO 20 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close +#./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 120 --bench MPASA30km -numIO 20 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close #./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 240 --bench MPASA30km -numIO 40 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close #./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 480 --bench MPASA30km -numIO 80 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close #./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 960 --bench MPASA30km -numIO 160 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close #./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 1920 --bench MPASA30km -numIO 320 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close ./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 3840 --bench MPASA30km -numIO 320 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/MPASA60km.csh b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/MPASA60km.csh index 0c3cd5a2c1..b909a3295b 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/MPASA60km.csh +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/MPASA60km.csh @@ -7,4 +7,3 @@ set host = 'kraken' ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 480 --bench MPASA60km -numIO 80 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 960 --bench MPASA60km -numIO 160 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 1020 --bench MPASA60km -numIO 170 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPB.csh b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPB.csh index 0f9cf55ec8..1103966b15 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPB.csh +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPB.csh @@ -7,4 +7,3 @@ set host = 'kraken' ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 160 --bench POPB --numIO 24 --logfile-suffix trunk_close ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 320 --bench POPB --numIO 48 --logfile-suffix trunk_close ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 640 --bench POPB --numIO 96 --logfile-suffix trunk_close - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPD.csh b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPD.csh index 62d0f1da3e..d36be4c5b2 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPD.csh +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPD.csh @@ -11,4 +11,3 @@ set host = 'kraken' #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 2000 --bench POPD --numIO 320 --log ${host}.2000.pnc.iotask_320.log.${id} #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 4000 --bench POPD --numIO 640 --log ${host}.4000.pnc.iotask_640.log.${id} #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id} - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv0.csh b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv0.csh index d51d7bb31a..0bb7f7951a 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv0.csh +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv0.csh @@ -17,4 +17,3 @@ set host = 'kraken' ./testpio_bench.pl --maxiter 10 --iofmt snc --pecount 4000 --bench POPD --numIO 1 --log ${host}.4000.snc.box.iotask_1.log.${id} #./testpio_bench.pl --maxiter 2 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id} - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv1.csh b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv1.csh index f10ab6f302..a02ab1fff0 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv1.csh +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv1.csh @@ -17,4 +17,3 @@ set id = 012211-1449 ./testpio_bench.pl --maxiter 10 --iofmt snc --pecount 4000 --bench POPD --numIO 640 --log ${host}.4000.snc.box.iotask_640.log.${id} #./testpio_bench.pl --maxiter 2 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id} - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv2.csh b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv2.csh index 57d3d58e47..981e6d288f 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv2.csh +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv2.csh @@ -35,5 +35,3 @@ set host = 'hopper' #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 1600 --bench POPD --numIO 320 --mpi-cb-buffer-size=8388608 --logfile-suffix DnR1 #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 2000 --bench POPD --numIO 640 --mpi-cb-buffer-size=8388608 --logfile-suffix DnR1 #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 4000 --bench POPD --numIO 640 --mpi-cb-buffer-size=8388608 --logfile-suffix DnR1 - - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv3.csh b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv3.csh index 4483057f12..65b32fcf6d 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv3.csh +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv3.csh @@ -13,4 +13,3 @@ set host = 'kraken' ./testpio_bench.pl --maxiter 10 --rearr none --iofmt pnc --pecount 4000 --bench POPD --numIO 4000 --log ${host}.4000.pnc.none.iotask_4000.log.${id} #./testpio_bench.pl --maxiter 2 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id} - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv4.csh b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv4.csh index 918ea7e87b..c7b80fdd79 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv4.csh +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv4.csh @@ -14,4 +14,3 @@ set host = 'kraken' ./testpio_bench.pl --maxiter 10 --iofmt bin --pecount 4000 --bench POPD --numIO 640 --log ${host}.4000.bin.box.iotask_640.log.012211-2233 #./testpio_bench.pl --maxiter 2 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id} - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv5.csh b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv5.csh index c1267dd790..994152107e 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv5.csh +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/POPDv5.csh @@ -14,4 +14,3 @@ set host = 'kraken' #./testpio_bench.pl --maxiter 10 --rearr none --iofmt bin --pecount 4000 --bench POPD --numIO 4000 --log ${host}.4000.bin.none.iotask_4000.log.${id} #./testpio_bench.pl --maxiter 2 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id} - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/README.testpio b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/README.testpio index db914e3c77..1e2f0cc907 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/README.testpio +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/README.testpio @@ -1,7 +1,7 @@ TESTPIO README -Testpio tests both the accuracy and performance of reading and writing data +Testpio tests both the accuracy and performance of reading and writing data using the pio library. The tests are controlled via namelist. There are a set of general namelist and then namelist to setup a computational decomposition and an IO decomposition. The computational decomposition @@ -21,34 +21,34 @@ block, io_nml, contains some general settings: nx_global - integer, global size of "x" dimension ny_global - integer, global size of "y" dimension nz_global - integer, glboal size of "z" dimension - ioFMT - string, type and i/o method of data file + ioFMT - string, type and i/o method of data file ("bin","pnc","snc"), binary, pnetcdf, or serial netcdf - rearr - string, type of rearranging to be done + rearr - string, type of rearranging to be done ("none","mct","box","boxauto") nprocsIO - integer, number of IO processors used only when rearr is not "none", if rearr is "none", then the IO decomposition will be the computational decomposition base - integer, base pe associated with nprocIO striding stride - integer, the stride of io pes across the global pe set - num_aggregator - integer, mpi-io number of aggregators, only used if no + num_aggregator - integer, mpi-io number of aggregators, only used if no pio rearranging is done - dir - string, directory to write output data, this must exist + dir - string, directory to write output data, this must exist before the model starts up num_iodofs - tests either 1dof or 2dof init decomp interfaces (1,2) maxiter - integer, the number of trials for the test DebugLevel - integer, sets the debug level (0,1,2,3) compdof_input - string, setting of the compDOF ('namelist' or a filename) - compdof_output - string, whether the compDOF is saved to disk + compdof_output - string, whether the compDOF is saved to disk ('none' or a filename) Notes: - the "mct" rearr option is not currently available - - if rearr is set to "none", then the computational decomposition is also + - if rearr is set to "none", then the computational decomposition is also going to be used as the IO decomposition. The computation decomposition must therefore be suited to the underlying I/O methods. - if rearr is set to "box", then pio is going to generate an internal IO decomposition automatically and pio will rearrange to that decomp. - - num_aggregator is used with mpi-io and no pio rearranging. mpi-io is only + - num_aggregator is used with mpi-io and no pio rearranging. mpi-io is only used with binary data. - nprocsIO, base, and stride implementation has some special options if nprocsIO > 0 and stride > 0, then use input values @@ -66,7 +66,7 @@ blocks are identical in use. increasing this increases the flexibility of decompositions. grdorder - string, sets the gridcell ordering within the block ("xyz","xzy","yxz","yzx","zxy","zyx") - grddecomp - string, sets up the block size with gdx, gdy, and gdz, see + grddecomp - string, sets up the block size with gdx, gdy, and gdz, see below, ("x","y","z","xy","xye","xz","xze","yz","yze", "xyz","xyze","setblk") gdx - integer, "x" size of block @@ -89,7 +89,7 @@ are provided below. Testpio writes out several files including summary information to stdout, data files to the namelist dir directory, and a netcdf -file summarizing the decompositions. The key output information +file summarizing the decompositions. The key output information is stdout, which contains the timing information. In addition, a netcdf file called gdecomp.nc is written that provides both the block and task ids for each gridcell as computed by the decompositions. @@ -110,7 +110,7 @@ option to testpio_run.pl There are several testpio_in files for the pio test suite. The ones that come with pio test specific things. In general, there are tests for - sn = serial netcdf and no rearrangement + sn = serial netcdf and no rearrangement sb = serial netcdf and box rearrangement pn = parallel netcdf and no rearrangement pb = parallel netcdf and box rearrangement @@ -121,7 +121,7 @@ and the test number (01, etc) is consistent across I/O methods with 02 = simple 2d xy decomp across all pes with all pes active in I/O 03 = all data on root pe, all pes active in I/O 04 = simple 2d xy decomp with yxz ordering and stride=4 pes active in I/O - 05 = 2d xy decomp with 4 blocks/pe, yxz ordering, xy block decomp, and + 05 = 2d xy decomp with 4 blocks/pe, yxz ordering, xy block decomp, and stride=4 pes active in I/O 06 = 3d xy decomp with 4 blocks/pe, yxz ordering, xy block decomp, and stride=4 pes active in I/O @@ -146,24 +146,24 @@ DECOMPOSITION: The decomposition implementation supports the decomposition of a general 3 dimensional "nx * ny * nz" grid into multiple blocks -of gridcells which are then ordered and assigned to processors. -In general, blocks in the decomposition are rectangular, -"gdx * gdy * gdz" and the same size, although some blocks around -the edges of the domain may be smaller if the decomposition is uneven. -Both gridcells within the block and blocks within the domain can be +of gridcells which are then ordered and assigned to processors. +In general, blocks in the decomposition are rectangular, +"gdx * gdy * gdz" and the same size, although some blocks around +the edges of the domain may be smaller if the decomposition is uneven. +Both gridcells within the block and blocks within the domain can be ordered in any of the possible dimension hierarchies, such as "xyz" -where the first dimension is the fastest. +where the first dimension is the fastest. -The gdx, gdy, and gdz inputs allow the user to specify the size in -any dimension and the grddecomp input specifies which dimensions are -to be further optimized. In general, automatic decomposition generation -of 3 dimensional grids can be done in any of possible combination of +The gdx, gdy, and gdz inputs allow the user to specify the size in +any dimension and the grddecomp input specifies which dimensions are +to be further optimized. In general, automatic decomposition generation +of 3 dimensional grids can be done in any of possible combination of dimensions, (x, y, z, xy, xz, yz, or xyz), with the other dimensions having a fixed block size. The automatic generation of the decomposition is based upon an internal algorithm that tries to determine the most "square" blocks with an additional constraint on minimizing the maximum number of gridcells across processors. If evenly divided grids are -desired, use of the "e" addition to grddecomp specifies that the grid +desired, use of the "e" addition to grddecomp specifies that the grid decomposition must be evenly divided. the setblk option uses the prescibed gdx, gdy, and gdz inputs withtout further automation. @@ -172,7 +172,7 @@ in mapping blocks to processors, but has a few additional options. "cont1d" (contiguous 1d) basically unwraps the blocks in the order specified by the blkorder input and then decomposes that "1d" list of blocks onto processors by contiguously grouping blocks together and allocating -them to a processor. The number of contiguous blocks that are +them to a processor. The number of contiguous blocks that are allocated to a processor is the maximum of the values of bdx, bdy, and bdz inputs. Contiguous blocks are allocated to each processor in turn in a round robin fashion until all blocks are allocated. The @@ -181,13 +181,13 @@ contiguous blocks are set automatically such that each processor recieves only 1 set of contiguous blocks. The ysym2 and ysym4 blkdecomp2 options modify the original block layout such that the tasks assigned to the blocks are 2-way or 4-way symetric -in the y axis. +in the y axis. The decomposition tool is extremely flexible, but arbitrary inputs will not always yield valid decompositions. If a valid decomposition cannot be computed based on the global grid size, -number of pes, number of blocks desired, and decomposition options, -the model will stop. +number of pes, number of blocks desired, and decomposition options, +the model will stop. As indicated above, the IO decomposition must be suited to the IO methods, so decompositions are even further limited by those @@ -212,7 +212,7 @@ Some decomposition examples: Standard xyz ordering, 2d decomp: note: blkdecomp plays no role since there is 1 block per pe - nx_global 6 + nx_global 6 ny_global 4 nz_global 1 ______________________________ npes 4 |B3 P3 |B4 P4 | @@ -231,7 +231,7 @@ Standard xyz ordering, 2d decomp: Same as above but yxz ordering, 2d decomp note: blkdecomp plays no role since there is 1 block per pe - nx_global 6 + nx_global 6 ny_global 4 nz_global 1 _____________________________ npes 4 |B2 P2 |B4 P4 | @@ -248,10 +248,10 @@ Same as above but yxz ordering, 2d decomp bdy 0 |______________|______________| bdz 0 -xyz grid ordering, 1d x decomp +xyz grid ordering, 1d x decomp note: blkdecomp plays no role since there is 1 block per pe note: blkorder plays no role since it's a 1d decomp - nx_global 8 + nx_global 8 ny_global 4 nz_global 1 _____________________________________ npes 4 |B1 P1 |B2 P2 |B3 P3 |B4 P4 | @@ -269,7 +269,7 @@ xyz grid ordering, 1d x decomp bdz 0 yxz block ordering, 2d grid decomp, 2d block decomp, 4 block per pe - nx_global 8 + nx_global 8 ny_global 4 nz_global 1 _____________________________________ npes 4 |B4 P2 |B8 P2 |B12 P4 |B16 P4 | @@ -285,4 +285,3 @@ yxz block ordering, 2d grid decomp, 2d block decomp, 4 block per pe bdx 0 | 1 2 | 1 2 | | | bdy 0 |________|_________|________|_________| bdz 0 - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/alloc_mod.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/alloc_mod.F90.in index f70259d6ef..7843c7f69f 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/alloc_mod.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/alloc_mod.F90.in @@ -1,6 +1,6 @@ #define __PIO_FILE__ "alloc_mod.F90.in" !> -!! @file +!! @file !! $Revision$ !! $LastChangedDate$ !! @brief Internal allocation routines for PIO @@ -15,18 +15,18 @@ module alloc_mod !> !! @private -!! PIO internal memory allocation check routines. +!! PIO internal memory allocation check routines. !< public:: alloc_check !> !! @private -!! PIO internal memory allocation check routines. +!! PIO internal memory allocation check routines. !< - public:: dealloc_check + public:: dealloc_check interface alloc_check ! TYPE long,int,real,double ! DIMS 1,2 - module procedure alloc_check_{DIMS}d_{TYPE} + module procedure alloc_check_{DIMS}d_{TYPE} ! TYPE double,long,int,real module procedure alloc_check_0d_{TYPE} end interface @@ -42,19 +42,19 @@ module alloc_mod !> !! @private -!! PIO internal memory allocation check routines. +!! PIO internal memory allocation check routines. !< public :: alloc_print_usage !> !! @private -!! PIO internal memory allocation check routines. +!! PIO internal memory allocation check routines. !< public :: alloc_trace_on !> !! @private -!! PIO internal memory allocation check routines. +!! PIO internal memory allocation check routines. !< public :: alloc_trace_off @@ -66,7 +66,7 @@ contains ! Instantiate all the variations of alloc_check_ and dealloc_check_ ! - ! TYPE long,int,real,double + ! TYPE long,int,real,double subroutine alloc_check_1d_{TYPE} (data,varlen,msg) {VTYPE}, pointer :: data(:) @@ -102,7 +102,7 @@ contains end subroutine alloc_check_1d_{TYPE} - ! TYPE long,int,real,double + ! TYPE long,int,real,double subroutine alloc_check_2d_{TYPE} (data,size1, size2,msg) {VTYPE}, pointer :: data(:,:) @@ -214,7 +214,7 @@ end subroutine dealloc_check_0d_{TYPE} !> !! @private !! @fn alloc_print_usage -!! PIO internal memory allocation check routines. +!! PIO internal memory allocation check routines. !< subroutine alloc_print_usage(rank,msg) #ifndef NO_MPIMOD diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/build_defaults.xml b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/build_defaults.xml index 0bea13fd8e..0bf3766e18 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/build_defaults.xml +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/build_defaults.xml @@ -31,7 +31,7 @@ #BSUB -J testpio_suite #BSUB -W 3:00 " - /> + /> <!-- PNETCDF_PATH="/glade/u/home/jedwards/pnetcdf/svn1163/intel/" --> @@ -65,7 +65,7 @@ #BSUB -J testpio_suite #BSUB -W 1:00 ' - /> + /> <yellowstone_pgi PNETCDF_PATH="${PNETCDF}" @@ -96,7 +96,7 @@ #BSUB -J testpio_suite #BSUB -W 1:00 ' - /> + /> <yellowstone_gnu PNETCDF_PATH="${PNETCDF}" NETCDF_PATH="${NETCDF}" @@ -126,7 +126,7 @@ #BSUB -J testpio_suite #BSUB -W 1:00 ' - /> + /> <janus @@ -155,7 +155,7 @@ #PBS -q janus-short #PBS -V " - /> + /> <carver PNETCDF_PATH="/global/homes/j/jedwards/pnetcdf/carver/intel/" @@ -182,15 +182,15 @@ #PBS -q debug #PBS -V " - /> + /> <intrepid ENV_IBMCMP_INCLUDE="/soft/apps/ibmcmp-jan2013/vac/bg/9.0/include:/soft/apps/ibmcmp-jan2010/vacpp/bg/9.0/include:/soft/apps/ibmcmp-jan2010/xlf/bg/11.1/include:/soft/apps/ibmcmp-jan2010/xlmass/bg/4.4/include:/soft/apps/ibmcmp-jan2010/xlsmp/bg/1.7/include" ADDENV_PATH="/soft/apps/darshan/bin/:/soft/apps/ibmcmp-jan2013/xlf/bg/11.1/bin/:/soft/apps/ibmcmp-jan2010/vac/bg/9.0/bin/" - NETCDF_PATH="/soft/apps/current/netcdf-4.1.3-disable_netcdf_4" - PNETCDF_PATH="/home/robl/soft/pnetcdf-1.3.0pre1-xl" + NETCDF_PATH="/soft/apps/current/netcdf-4.1.3-disable_netcdf_4" + PNETCDF_PATH="/home/robl/soft/pnetcdf-1.3.0pre1-xl" MPIFC = "/software/common/apps/misc-scripts/tmpixlf90" MPICC = "/software/common/apps/misc-scripts/tmpixlc" FC = "/software/common/apps/misc-scripts/tmpixlf90" @@ -216,7 +216,7 @@ MPICC = "mpicc" FC = "gfortran" CC = "gcc" - NETCDF_PATH="" + NETCDF_PATH="" FFLAGS = "-g --free-line-length-none " submit="qsub" run="mpirun" @@ -230,7 +230,7 @@ #PBS -V " testsuites = "mpiio" -/> +/> <cyberstar ADDENV_PATH = "" @@ -240,7 +240,7 @@ MPICC = "mpicc" FC = "pgf90" CC = "gcc" - NETCDF_PATH="" + NETCDF_PATH="" FFLAGS = "-g" submit="qsub" run="mpirun" @@ -254,7 +254,7 @@ #PBS -V " testsuites = "mpiio" -/> +/> <mirage1 @@ -263,7 +263,7 @@ MPICC = "icc" FC = "ifort" CC = "icc" - NETCDF_PATH="/fs/local/apps/netcdf-3.6.2/" + NETCDF_PATH="/fs/local/apps/netcdf-3.6.2/" FFLAGS = "-g " submit="" run="" @@ -272,14 +272,14 @@ corespernode="1" preamble = "" testsuites = "snet " -/> +/> <titan MPIFC = "ftn " MPICC = "cc " FC = "pgf90" CC = "pgcc" - CPPDEFS = " -DFORTRANUNDERSCORE" + CPPDEFS = " -DFORTRANUNDERSCORE" ENV_NETCDF_PATH="$NETCDF_DIR" ENV_PNETCDF_PATH="$PARALLEL_NETCDF_DIR" workdir = "/tmp/work/${USER}/testpio" @@ -322,8 +322,8 @@ <kraken MPIFC = "ftn -L$LIBSCI_BASE_DIR/pgi/lib" MPICC = "cc -L$LIBSCI_BASE_DIR/pgi/lib" - FC = "pgf90" - CC = "pgcc" + FC = "pgf90" + CC = "pgcc" FFLAGS = "-D_USE_FLOW_CONTROL" NETCDF_PATH = "$NETCDF_PATH" PNETCDF_PATH = "$PNETCDF_DIR" @@ -343,8 +343,8 @@ <columbia MPIFC = "mpif90" MPICC = "mpicc" - FC = "ifort" - CC = "icc" + FC = "ifort" + CC = "icc" FFLAGS = "-D_USE_FLOW_CONTROL" NETCDF_PATH = "$NETCDF_DIR" PNETCDF_PATH = "$PNETCDF_DIR" @@ -412,7 +412,7 @@ NETCDF_PATH = "${NETCDF}" LDLIBS="`$NETCDF/bin/nc-config --flibs` -lssl -lcrypto -ldl -lz -L/contrib/libidn/1.19/gnu/lib -lidn -L/opt/cray/pmi/1.0-1.0000.8160.39.1.ss/lib64 -lpmi" workdir = "/ptmp/${USER}/testpio.intel" - run = "aprun" + run = "aprun" corespernode = "12" submit = "qsub " conopts = " --enable-filesystem-hints=lustre " diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/check_mod.F90 b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/check_mod.F90 index fe0d7d4c4d..b9291bb555 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/check_mod.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/check_mod.F90 @@ -1,5 +1,5 @@ module check_mod - + use kinds_mod use pio_types, only : PIO_NOERR ! _EXTERNAL use alloc_mod ! _EXTERNAL @@ -8,11 +8,11 @@ module check_mod use mpi !_EXTERNAL #endif implicit none - private + private #ifdef NO_MPIMOD include 'mpif.h' ! _EXTERNAL #endif - public :: checkpattern + public :: checkpattern interface checkpattern module procedure check_1D_r8, & @@ -23,7 +23,7 @@ module check_mod check_3D_i4 end interface -contains +contains subroutine check_1D_r8(my_comm, fname,wr_array,rd_array,len,iostat) integer, intent(in) :: my_comm @@ -38,7 +38,7 @@ subroutine check_1D_r8(my_comm, fname,wr_array,rd_array,len,iostat) integer(i4) :: ierr,cbad,rank, maxbadloc(1) if(present(iostat)) iostat = PIO_noerr - + call alloc_check(diff,len,' check_1D_r8:diff ') if(len>0) then @@ -62,7 +62,7 @@ subroutine check_1D_r8(my_comm, fname,wr_array,rd_array,len,iostat) wr_array(maxbadloc), rd_array(maxbadloc) if(present(iostat)) iostat = -20 endif - call dealloc_check(diff) + call dealloc_check(diff) end subroutine check_1D_r8 subroutine check_3D_r8(my_comm, fname,wr_array,rd_array) @@ -76,17 +76,17 @@ subroutine check_3D_r8(my_comm, fname,wr_array,rd_array) real(r8) :: lsum,gsum integer(i4) :: ierr,cbad,rank integer(i4) :: len1,len2,len3 - + len1 = SIZE(wr_array,dim=1) len2 = SIZE(wr_array,dim=2) len3 = SIZE(wr_array,dim=3) - + allocate(diff(len1,len2,len3)) - + diff = wr_array - rd_array cbad = COUNT(diff .ne. 0.0) lsum = SUM(diff) - + call MPI_Allreduce(lsum,gsum,1,MPI_REAL8,MPI_SUM,MY_COMM,ierr) call CheckMPIReturn('Call to MPI_Allreduce()',ierr,__FILE__,__LINE__) @@ -96,7 +96,7 @@ subroutine check_3D_r8(my_comm, fname,wr_array,rd_array) if(lsum .ne. 0.0) print *,'IAM: ', rank, 'File: ',TRIM(fname),& ' Error detected for correctness test(3D,R8): ',lsum,' # bad: ',cbad endif - deallocate(diff) + deallocate(diff) end subroutine check_3D_r8 @@ -111,17 +111,17 @@ subroutine check_3D_r4(my_comm, fname,wr_array,rd_array) real(r4) :: lsum,gsum integer(i4) :: ierr,cbad,rank integer(i4) :: len1,len2,len3 - + len1 = SIZE(wr_array,dim=1) len2 = SIZE(wr_array,dim=2) len3 = SIZE(wr_array,dim=3) - + allocate(diff(len1,len2,len3)) - + diff = wr_array - rd_array cbad = COUNT(diff .ne. 0.0) lsum = SUM(diff) - + call MPI_Allreduce(lsum,gsum,1,MPI_REAL,MPI_SUM,MY_COMM,ierr) call CheckMPIReturn('Call to MPI_Allreduce()',ierr,__FILE__,__LINE__) @@ -131,7 +131,7 @@ subroutine check_3D_r4(my_comm, fname,wr_array,rd_array) if(lsum .ne. 0) print *,'IAM: ', rank, 'File: ',TRIM(fname),& ' Error detected for correctness test(3D,R4): ',lsum,' # bad: ',cbad endif - deallocate(diff) + deallocate(diff) end subroutine check_3D_r4 @@ -146,17 +146,17 @@ subroutine check_3D_i4(my_comm, fname,wr_array,rd_array) integer(i4) :: lsum,gsum integer(i4) :: ierr,cbad,rank integer(i4) :: len1,len2,len3 - + len1 = SIZE(wr_array,dim=1) len2 = SIZE(wr_array,dim=2) len3 = SIZE(wr_array,dim=3) - + allocate(diff(len1,len2,len3)) - + diff = wr_array - rd_array cbad = COUNT(diff .ne. 0.0) lsum = SUM(diff) - + call MPI_Allreduce(lsum,gsum,1,MPI_INTEGER,MPI_SUM,MY_COMM,ierr) call CheckMPIReturn('Call to MPI_Allreduce()',ierr,__FILE__,__LINE__) if(gsum .ne. 0.0) then @@ -165,13 +165,13 @@ subroutine check_3D_i4(my_comm, fname,wr_array,rd_array) if(lsum .ne. 0) print *,'IAM: ', rank, 'File: ',TRIM(fname),& ' Error detected for correctness test(3D,I4): ',lsum,' # bad: ',cbad endif - deallocate(diff) + deallocate(diff) end subroutine check_3D_i4 subroutine check_1D_r4(my_comm,fname,wr_array,rd_array,len,iostat) integer, intent(in) :: my_comm - + character(len=*) :: fname real(r4) :: wr_array(:) real(r4) :: rd_array(:) @@ -181,11 +181,11 @@ subroutine check_1D_r4(my_comm,fname,wr_array,rd_array,len,iostat) real(r4) :: lsum,gsum integer(i4) :: ierr,len,cbad,rank - + ! Set default (no error) value for iostat if present) if(present(iostat)) iostat = PIO_noerr - + call alloc_check(diff,len,' check_1D_r4:diff ') if(len>0) then @@ -195,7 +195,7 @@ subroutine check_1D_r4(my_comm,fname,wr_array,rd_array,len,iostat) else lsum = 0 end if - + call MPI_Allreduce(lsum,gsum,1,MPI_REAL,MPI_SUM,MY_COMM,ierr) call CheckMPIReturn('Call to MPI_Allreduce()',ierr,__FILE__,__LINE__) if(abs(gsum) > tiny(gsum)) then @@ -205,7 +205,7 @@ subroutine check_1D_r4(my_comm,fname,wr_array,rd_array,len,iostat) ' Error detected for correctness test(1D,R4): ',lsum,' # bad: ',cbad if(present(iostat)) iostat = -20 endif - deallocate(diff) + deallocate(diff) end subroutine check_1D_r4 @@ -221,11 +221,11 @@ subroutine check_1D_i4(my_comm, fname,wr_array,rd_array,len,iostat) integer(i4) :: lsum,gsum integer(i4) :: ierr,cbad,rank, lloc(1) - + ! Set default (no error) value for iostat if present) if(present(iostat)) iostat = PIO_noerr - + call alloc_check(diff,len,' check_1D_r4:diff ') if(len>0) then diff = wr_array - rd_array @@ -245,7 +245,7 @@ subroutine check_1D_i4(my_comm, fname,wr_array,rd_array,len,iostat) lloc, wr_array(lloc(1)), rd_array(lloc(1)) if(present(iostat)) iostat = -20 endif - deallocate(diff) + deallocate(diff) end subroutine check_1D_i4 diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/config_bench.xml b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/config_bench.xml index 0f8beaf2ea..78e0beb10f 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/config_bench.xml +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/config_bench.xml @@ -75,39 +75,39 @@ <!-- CUBE4K --> <CompConfig nprocs="512" bench_name="CUBE2K" - <ldx>256</ldx> <ldy>256</ldy> <ldz>256</ldz> + <ldx>256</ldx> <ldy>256</ldy> <ldz>256</ldz> </CompConfig> <!-- POPB benchmark --> <CompConfig nprocs="24" bench_name="POPB" - <ldx>80</ldx> <ldy>48</ldy> <ldz>60</ldz> + <ldx>80</ldx> <ldy>48</ldy> <ldz>60</ldz> </CompConfig> <CompConfig nprocs="80" bench_name="POPB" - <ldx>32</ldx> <ldy>48</ldy> <ldz>60</ldz> + <ldx>32</ldx> <ldy>48</ldy> <ldz>60</ldz> </CompConfig> <CompConfig nprocs="160" bench_name="POPB" - <ldx>32</ldx> <ldy>24</ldy> <ldz>60</ldz> + <ldx>32</ldx> <ldy>24</ldy> <ldz>60</ldz> </CompConfig> <CompConfig nprocs="320" bench_name="POPB" - <ldx>16</ldx> <ldy>24</ldy> <ldz>60</ldz> + <ldx>16</ldx> <ldy>24</ldy> <ldz>60</ldz> </CompConfig> <CompConfig nprocs="640" bench_name="POPB" - <ldx>16</ldx> <ldy>12</ldy> <ldz>60</ldz> + <ldx>16</ldx> <ldy>12</ldy> <ldz>60</ldz> </CompConfig> -<!-- WRFB benchmark --> +<!-- WRFB benchmark --> <CompConfig nprocs="125" bench_name="WRFB" - <ldx>90</ldx> <ldy>72</ldy> <ldz>100</ldz> + <ldx>90</ldx> <ldy>72</ldy> <ldz>100</ldz> </CompConfig> <CompConfig nprocs="250" bench_name="WRFB" - <ldx>90</ldx> <ldy>36</ldy> <ldz>100</ldz> + <ldx>90</ldx> <ldy>36</ldy> <ldz>100</ldz> </CompConfig> <CompConfig nprocs="450" bench_name="WRFB" @@ -115,23 +115,23 @@ </CompConfig> <CompConfig nprocs="500" bench_name="WRFB" - <ldx>45</ldx> <ldy>36</ldy> <ldz>100</ldz> + <ldx>45</ldx> <ldy>36</ldy> <ldz>100</ldz> </CompConfig> <CompConfig nprocs="900" bench_name="WRFB" - <ldx>30</ldx> <ldy>30</ldy> <ldz>100</ldz> + <ldx>30</ldx> <ldy>30</ldy> <ldz>100</ldz> </CompConfig> <CompConfig nprocs="1000" bench_name="WRFB" - <ldx>45</ldx> <ldy>18</ldy> <ldz>100</ldz> + <ldx>45</ldx> <ldy>18</ldy> <ldz>100</ldz> </CompConfig> <CompConfig nprocs="2025" bench_name="WRFB" - <ldx>20</ldx> <ldy>40</ldy> <ldz>100</ldz> + <ldx>20</ldx> <ldy>40</ldy> <ldz>100</ldz> </CompConfig> <CompConfig nprocs="4050" bench_name="WRFB" - <ldx>20</ldx> <ldy>20</ldy> <ldz>100</ldz> + <ldx>20</ldx> <ldy>20</ldy> <ldz>100</ldz> </CompConfig> <CompConfig nprocs="8100" bench_name="WRFB" @@ -142,7 +142,7 @@ <ldx>10</ldx> <ldy>10</ldy> <ldz>100</ldz> </CompConfig> -<!-- CAM05 benchmark --> +<!-- CAM05 benchmark --> <CompConfig nprocs="64" bench_name="CAM05" <ldx>576</ldx> <ldy>6</ldy> <ldz>26</ldz> </CompConfig> @@ -312,4 +312,3 @@ </Config> - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/fdepends.awk b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/fdepends.awk index 03bab9769d..2980920cf2 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/fdepends.awk +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/fdepends.awk @@ -19,9 +19,9 @@ BEGIN { IGNORECASE=1 # -# awk reads each line of the filename argument $2 until it finds +# awk reads each line of the filename argument $2 until it finds # a "use" or "#include" -# +# /^[ \t]*use[ \t]+/ { @@ -30,7 +30,7 @@ BEGIN { IGNORECASE=1 if ( $0 ~ /_EXTERNAL/ ) next # Assume the second field is the F90 module name, - # remove any comma at the end of the second field (due to + # remove any comma at the end of the second field (due to # ONLY or rename), and print it in a dependency line. sub(/,$/,"",$2) @@ -49,8 +49,8 @@ BEGIN { IGNORECASE=1 if ( $0 ~ /_EXTERNAL/ ) next # Remove starting or ending quote or angle bracket - sub(/^["<']/,"",$2) - sub(/[">']$/,"",$2) + sub(/^["<']/,"",$2) + sub(/[">']$/,"",$2) print PRLINE $2 - + } diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/gdecomp_mod.F90 b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/gdecomp_mod.F90 index b5b41407ad..d848931d4a 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/gdecomp_mod.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/gdecomp_mod.F90 @@ -32,9 +32,9 @@ module gdecomp_mod character(len=128):: nml_file ! namelist filename if used character(len=16) :: nml_var ! namelist variable if used end type - + character(len=*),parameter :: modname = 'gdecomp_mod' - integer(i4),parameter :: master_task = 0 + integer(i4),parameter :: main_task = 0 !================================================================== contains @@ -51,7 +51,7 @@ subroutine gdecomp_set(gdecomp,nxg,nyg,nzg,gdx,gdy,gdz,bdx,bdy,bdz, & type(gdecomp_type), intent(inout) :: gdecomp ! NOTE: not all of these are optional, but optional allows -! them to be called in arbitrary order +! them to be called in arbitrary order integer(i4),optional :: nxg,nyg,nzg ! global grid size integer(i4),optional :: gdx,gdy,gdz ! block size @@ -171,11 +171,11 @@ subroutine gdecomp_set(gdecomp,nxg,nyg,nzg,gdx,gdy,gdz,bdx,bdy,bdz, & endif if (present(my_task)) then - if (my_task == master_task) call gdecomp_print(gdecomp) + if (my_task == main_task) call gdecomp_print(gdecomp) endif end subroutine gdecomp_set - + !================================================================== subroutine gdecomp_read_nml(gdecomp,nml_file,nml_var,my_task,ntasks,gdims) @@ -260,11 +260,11 @@ subroutine gdecomp_read_nml(gdecomp,nml_file,nml_var,my_task,ntasks,gdims) gdecomp%bdz = bdz if (present(my_task)) then - if (my_task == master_task) call gdecomp_print(gdecomp) + if (my_task == main_task) call gdecomp_print(gdecomp) endif end subroutine gdecomp_read_nml - + !================================================================== subroutine gdecomp_print(gdecomp) @@ -384,7 +384,7 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test) !DBG print *,'IAM: ',my_task,'gdecomp_DOF: point #3 gsiz:',gsiz !DBG print *,'IAM: ',my_task,'gdecomp_DOF: point #3 bsiz:',bsiz - if(wdecomp) then + if(wdecomp) then allocate(blkid(gsiz(1),gsiz(2),gsiz(3))) allocate(tskid(gsiz(1),gsiz(2),gsiz(3))) blkid = -1 @@ -422,7 +422,7 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test) write(6,*) trim(subname),' ERROR: contval must be > 0 ',nbor call piodie(__FILE__,__LINE__) endif - if (my_task == master_task) & + if (my_task == main_task) & write(6,*) trim(subname),' blkdecomp1 = ',trim(gdecomp%blkdecomp1),' contval = ',contval case ('cont1dm') call pad_div(contval,nblks,gnpes) @@ -430,7 +430,7 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test) write(6,*) trim(subname),' ERROR: contval must be > 0 ',nbor call piodie(__FILE__,__LINE__) endif - if (my_task == master_task) & + if (my_task == main_task) & write(6,*) trim(subname),' blkdecomp1 = ',trim(gdecomp%blkdecomp1),' contval = ',contval case default call calcdecomp(gdecomp%blkdecomp1,gnpes,nblk,nbor,ierr) @@ -562,7 +562,7 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test) ! ii = (n3-1)*gsiz(2)*gsiz(1) + (n2-1)*gsiz(1) + n1 nbxyz = ((n3-1)/bsiz(3))*nblk(2)*nblk(1) + ((n2-1)/bsiz(2))*nblk(1) + & ((n1-1)/bsiz(1)) + 1 - if(wdecomp) then + if(wdecomp) then blkid(n1,n2,n3) = bxyzbord(nbxyz) tskid(n1,n2,n3) = bxyzpord(nbxyz) endif @@ -579,7 +579,7 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test) cntmax = maxval(cnta) ! --- map gridcells to dof --- - + if (testonly) then allocate(testdof(cntmax,0:gnpes-1)) testdof = 0 @@ -658,18 +658,18 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test) start(1:3) = pstart(1:3,my_task) count(1:3) = pend(1:3,my_task) - pstart(1:3,my_task) + 1 endif - if (my_task == master_task) & + if (my_task == main_task) & write(6,*) trim(subname),' start and count were computed ',my_task,start,count else start = 1 count = 0 - if (my_task == master_task) & + if (my_task == main_task) & write(6,*) trim(subname),' start and count could NOT be computed ' endif -!------- MASTER TASK WRITE ------------------------------------- +!------- main TASK WRITE ------------------------------------- - if (my_task == master_task) then + if (my_task == main_task) then ! --- write testdof --- @@ -717,7 +717,7 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test) if (first_call) then rcode = nf90_create(ncname,nf90_clobber,ncid) else - rcode = nf90_open(ncname,nf90_write,ncid) + rcode = nf90_open(ncname,nf90_write,ncid) endif rcode = nf90_redef(ncid) dname = trim(gdecomp%nml_var)//'_nx' @@ -738,9 +738,9 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test) endif ! testonly -!------- END MASTER TASK WRITE --------------------------------- +!------- END main TASK WRITE --------------------------------- - if(wdecomp) then + if(wdecomp) then deallocate(blkid,tskid) endif deallocate(cnta,cntb,bxyzbord,bxyzpord,bordpord) @@ -944,7 +944,7 @@ subroutine calcbsiz(npes,gsiz,bsiz,option,ierr) npes2 = npes2/m bs = bs - 1 else - write(6,*) trim(subname),' ERROR: bsiz not allowed ',n,gsiz(n),bsiz(n),m,npes,npes2 + write(6,*) trim(subname),' ERROR: bsiz not allowed ',n,gsiz(n),bsiz(n),m,npes,npes2 call piodie(__FILE__,__LINE__) endif endif @@ -1112,7 +1112,7 @@ end subroutine piodie subroutine mpas_decomp_generator(dim1,dim2,dim3,my_task,fname,dof) integer :: dim1, dim2, dim3 integer, intent(in) :: my_task ! my MPI rank - character(len=*),intent(in) :: fname ! name of MPAS partition file + character(len=*),intent(in) :: fname ! name of MPAS partition file integer(kind=pio_offset_kind), pointer :: dof(:) ! Local variables @@ -1132,7 +1132,7 @@ subroutine mpas_decomp_generator(dim1,dim2,dim3,my_task,fname,dof) ! 1st dimension: vertical ! 2nd dimension: horizontal - gnz = dim1 + gnz = dim1 nCellsGlobal = dim2*dim3 call get_global_id_list(my_task,fname,nCellsSolve,nCellsGlobal,globalIDList) @@ -1197,7 +1197,7 @@ end subroutine get_global_id_list subroutine camlike_decomp_generator(gnx, gny, gnz, myid, ntasks, npr_yz, dof) integer, intent(in) :: gnx, gny, gnz, myid, ntasks, npr_yz(4) - integer(kind=pio_offset_kind), pointer :: dof(:), tdof(:), tchk(:) + integer(kind=pio_offset_kind), pointer :: dof(:), tdof(:), tchk(:) real, pointer :: rdof(:) integer(kind=pio_offset_kind) :: dofsize,tdofsize @@ -1280,7 +1280,7 @@ subroutine camlike_decomp_generator(gnx, gny, gnz, myid, ntasks, npr_yz, dof) end do end do - CALL qsRecursive(1_PIO_OFFSET_KIND, dofsize, dof) !kicks off the recursive + CALL qsRecursive(1_PIO_OFFSET_KIND, dofsize, dof) !kicks off the recursive deallocate(tdof) @@ -1323,7 +1323,7 @@ integer(kind=pio_offset_kind) FUNCTION qsPartition (loin, hiin, list) hi = hi - 1 END DO IF (hi /= lo) then !move the entry indexed by hi to left side of partition - list(lo) = list(hi) + list(lo) = list(hi) lo = lo + 1 END IF DO !move in from the left @@ -1331,7 +1331,7 @@ integer(kind=pio_offset_kind) FUNCTION qsPartition (loin, hiin, list) lo = lo + 1 END DO IF (hi /= lo) then !move the entry indexed by hi to left side of partition - list(hi) = list(lo) + list(hi) = list(lo) hi = hi - 1 END IF END DO diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/namelist_mod.F90 b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/namelist_mod.F90 index fa237a7198..1dbd343cb2 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/namelist_mod.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/namelist_mod.F90 @@ -12,7 +12,7 @@ module namelist_mod use pio_support, only : piodie, CheckMPIReturn ! _EXTERNAL use pio, only : pio_offset_kind - implicit none + implicit none private public :: broadcast_namelist @@ -21,7 +21,7 @@ module namelist_mod integer(kind=i4), public, parameter :: buffer_size_str_len = 20 integer(kind=i4), public, parameter :: true_false_str_len = 6 integer(kind=i4), public, parameter :: romio_str_len = 10 - + logical, public, save :: async integer(i4), public, save :: nx_global,ny_global,nz_global integer(i4), public, save :: rearr_type @@ -49,9 +49,9 @@ module namelist_mod integer(kind=i4), public, save :: set_lustre_values = 0 !! Set to one for true integer(kind=i4), public, save :: lfs_ost_count = 1 - + character(len=80), save, public :: compdof_input - character(len=80), save, public :: iodof_input + character(len=80), save, public :: iodof_input character(len=80), save, public :: compdof_output character(len=256), save, public :: part_input character(len=256), save, public :: casename @@ -125,7 +125,7 @@ subroutine ReadTestPIO_Namelist(device, nprocs, filename, caller, ierror) character(len=*), parameter :: myname_=myname//'ReadPIO_Namelist' !------------------------------------------------- - ! set default values for namelist io_nml variables + ! set default values for namelist io_nml variables !------------------------------------------------- async = .false. @@ -175,14 +175,14 @@ subroutine ReadTestPIO_Namelist(device, nprocs, filename, caller, ierror) open (device, file=filename,status='old',iostat=ierror) - if(ierror /= 0) then + if(ierror /= 0) then write(*,*) caller,'->',myname_,':: Error opening file ',filename, & ' on device ',device,' with iostat=',ierror ierror = -1 else ierror = 1 endif - + do while (ierror > 0) read(device, nml=io_nml, iostat=ierror) enddo @@ -318,7 +318,7 @@ subroutine ReadTestPIO_Namelist(device, nprocs, filename, caller, ierror) stride = (nprocs-base)/num_iotasks endif elseif (nprocsIO <= 0) then -#ifdef BGx +#ifdef BGx ! A negative value for num_iotasks has a special meaning on Blue Gene num_iotasks = nprocsIO #else @@ -333,7 +333,7 @@ subroutine ReadTestPIO_Namelist(device, nprocs, filename, caller, ierror) endif !------------------------------------------------ - ! reset stride if there are not enough processors + ! reset stride if there are not enough processors !------------------------------------------------ if (base + num_iotasks * (stride-1) > nprocs-1) then stride = FLOOR(real((nprocs - 1 - base),kind=r8)/real(num_iotasks,kind=r8)) @@ -342,9 +342,9 @@ subroutine ReadTestPIO_Namelist(device, nprocs, filename, caller, ierror) !------------------------------------------------------- ! If rearrangement is 'none' reset to the proper values !------------------------------------------------------- - if(trim(rearr) == 'none') then + if(trim(rearr) == 'none') then stride = 1 - num_iotasks = nprocs + num_iotasks = nprocs endif write(*,*) trim(string),' n_iotasks = ',num_iotasks,' (updated)' @@ -381,7 +381,7 @@ subroutine Broadcast_Namelist(caller, myID, root, comm, ierror) integer(i4) :: itmp !------------------------------------------ - ! broadcast namelist info to all processors + ! broadcast namelist info to all processors !------------------------------------------ if(async) then diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/ChangeLog b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/ChangeLog index d5bfab8368..a5bbed9f87 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/ChangeLog +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/ChangeLog @@ -6,7 +6,7 @@ Originator(s): erik Date: Sat Jun 13, 2009 One-line Summary: Add %ymd indicator for streams so can do year-month-days -M Streams/Template.pm ---- Add ability to write out %ymd year-month-day +M Streams/Template.pm ---- Add ability to write out %ymd year-month-day for filenames in streams. It assumes a noleap calendar -- could easily be extended to make Gregorian optional. @@ -14,7 +14,7 @@ M t/01.t ---- Change formatting of successful test M t/02.t ---- Add more tests for %ymd, and offset M t/03.t ---- Change formatting of successful test M t/04.t ---- Change formatting of successful test -M t/datm.streams.txt ---------- Add another year and the last-month +M t/datm.streams.txt ---------- Add another year and the last-month to start for testing A t/datm.ymd.streams.txt ------ Add streams test file with %ymd M t/datm.template.streams.xml - Add CPLHIST test section with %ymd @@ -27,7 +27,7 @@ Date: Tue Jun 9, 2009 One-line Summary: add offset support for streams template M Streams/Template.pm - + ============================================================== Tag name: perl5lib_090424 Originator(s): erik @@ -79,7 +79,7 @@ Build/Namelist.pm . Change validate_variable_value() from an object method to a class method, and remove the unused argument. . add fix to _split_namelist_value method to replace embedded newlines by - spaces. + spaces. Build/NamelistDefaults.pm . make the method interfaces case insensitive by converting all variable @@ -146,7 +146,7 @@ Originator(s): erik (KLUZEK ERIK 1326 CGD) Date: Mon Aug 11 10:44:52 MDT 2008 One-line Summary: Turn off printing of file existance if NOT -verbose -M Streams/Template.pm ----------- Turn off printing of file +M Streams/Template.pm ----------- Turn off printing of file checking if NOT $printing; ============================================================== @@ -190,8 +190,8 @@ about needing to do validation as is done now. Change the validate methods a bit and make them more robust. M Build/Config.pm --------------- Add get_valid_values method and use it internally. -M Build/NamelistDefinition.pm --- Add namelist validate_variable_value to validate - method. Add option to return without quotes to +M Build/NamelistDefinition.pm --- Add namelist validate_variable_value to validate + method. Add option to return without quotes to get_valid_values method. M Build/Namelist.pm ------------- Make validate_variable_value more robust. diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Changes b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Changes index d0be5104f7..43f2860475 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Changes +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Changes @@ -24,4 +24,3 @@ Revision history for Perl extension XML::Lite. 0.01 Sat Aug 25 13:31:48 2001 - original version; created by h2xs 1.20 with options -XA -n XML::Lite - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Lite.pm b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Lite.pm index d6aa32e978..5ace82b52e 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Lite.pm +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Lite.pm @@ -35,12 +35,12 @@ my $xml = new XML::Lite( xml => 'a_file.xml' ); =head1 DESCRIPTION -XML::Lite is a lightweight XML parser, with basic element traversing -methods. It is entirely self-contained, pure Perl (i.e. I<not> based on -expat). It provides useful methods for reading most XML files, including -traversing and finding elements, reading attributes and such. It is -designed to take advantage of Perl-isms (Attribute lists are returned as -hashes, rather than, say, lists of objects). It provides only methods +XML::Lite is a lightweight XML parser, with basic element traversing +methods. It is entirely self-contained, pure Perl (i.e. I<not> based on +expat). It provides useful methods for reading most XML files, including +traversing and finding elements, reading attributes and such. It is +designed to take advantage of Perl-isms (Attribute lists are returned as +hashes, rather than, say, lists of objects). It provides only methods for reading a file, currently. =head1 METHODS @@ -50,7 +50,7 @@ The following methods are available: =over 4 =cut - + use XML::Lite::Element; BEGIN { use vars qw( $VERSION @ISA ); @@ -75,17 +75,17 @@ use vars qw( %ERRORS ); =item my $xml = new XML::Lite( xml => $source[, ...] ); Creates a new XML::Lite object. The XML::Lite object acts as the document -object for the $source that is sent to it to parse. This means that you -create a new object for each document (or document sub-section). As the +object for the $source that is sent to it to parse. This means that you +create a new object for each document (or document sub-section). As the objects are lightweight this should not be a performance consideration. The object constructor can take several named parameters. Parameter names -may begin with a '-' (as in the example above) but are not required to. The +may begin with a '-' (as in the example above) but are not required to. The following parameters are recognized. - xml The source XML to parse. This can be a filename, a scalar that + xml The source XML to parse. This can be a filename, a scalar that contains the document (or document fragment), or an IO handle. - + As a convenince, if only on parameter is given, it is assumed to be the source. So you can use this, if you wish: @@ -99,7 +99,7 @@ sub new { my $proto = shift; my %parms; my $class = ref($proto) || $proto; - + # Parse parameters $self->{settings} = {}; if( @_ > 1 ) { @@ -109,7 +109,7 @@ sub new { while( ($k, $v) = each %parms ) { $k =~ s/^-//; # Removed leading '-' if it exists. (Why do Perl programmers use this?) $self->{settings}{$k} = $v; - } # end while + } # end while } else { $self->{settings}{xml} = $_[0]; } # end if; @@ -121,10 +121,10 @@ sub new { $self->{doc} = ''; $self->{_CDATA} = []; $self->{handlers} = {}; - + # Refer to global error messages $self->{ERRORS} = $self->{settings}{error_messages} || \%ERRORS; - + # Now parse the XML document and build look-up tables return undef unless $self->_parse_it(); @@ -181,8 +181,8 @@ sub root_element { Returns a list of all elements that match C<$name>. C<@list> is a list of L<XML::Lite::Element> objects If called in a scalar context, this will return the -first element found that matches (it's more efficient -to call in a scalar context than assign the results +first element found that matches (it's more efficient +to call in a scalar context than assign the results to a list of one scalar). If no matching elements are found then returns C<undef> @@ -201,7 +201,7 @@ sub element_by_name; sub elements_by_name { my $self = shift; my( $name ) = @_; - + if( wantarray ) { my @list = (); foreach( @{$self->{elements}{$name}} ) { @@ -241,7 +241,7 @@ sub elements_by_name { # ---------------------------------------------------------- sub _parse_it { my $self = shift; - + # Get the xml content if( $self->{settings}{xml} =~ /^\s*</ ) { $self->{doc} = $self->{settings}{xml}; @@ -268,26 +268,26 @@ sub _parse_it { $self->{doc_offset} = length $1; # Store the number of removed chars for messages } # end if $self->{doc} =~ s/\s+$//; - - + + # Build lookup tables $self->{elements} = {}; $self->{tree} = []; # - These are used in the building process my $element_list = []; my $current_element = $self->{tree}; - + # Call init handler if defined &{$self->{handlers}{init}}($self) if defined $self->{handlers}{init}; - + # Make a table of offsets to each element start and end point # Table is a hash of element names to lists of offsets: # [start_tag_start, start_tag_end, end_tag_start, end_tag_end] # where tags include the '<' and '>' - - # Also make a tree of linked lists. List contains root element + + # Also make a tree of linked lists. List contains root element # and other nodes. Each node consits of a list ref (the position list) - # and a following list containing the child element. Text nodes are + # and a following list containing the child element. Text nodes are # a list ref (with just two positions). # Find the opening and closing of the XML, giving errors if not well-formed @@ -297,22 +297,22 @@ sub _parse_it { $self->_error( 'ROOT_NOT_CLOSED', $start_pos + $self->{doc_offset} ) if $end_pos == -1; my $doc_end = rindex( $self->{doc}, '>' ); $self->_error( 'ROOT_NOT_CLOSED' ) if $doc_end == -1; - + # Now walk through the document, one tag at a time, building up our # lookup tables while( $end_pos <= $doc_end ) { - + # Get a tag my $tag = substr( $self->{doc}, $start_pos, $end_pos - $start_pos + 1 ); # Get the tag name and see if it's an end tag (starts with </) my( $end, $name ) = $tag =~ m{^<\s*(/?)\s*([^/>\s]+)}; - + if( $end ) { # If there is no start tag for this end tag then throw an error $self->_error( 'NO_START', $start_pos + $self->{doc_offset}, $tag ) unless defined $self->{elements}{$name}; - - # Otherwise, add the end point to the array for the last element in + + # Otherwise, add the end point to the array for the last element in # the by-name lookup hash my( $x, $found ) = (@{$self->{elements}{$name}} - 1, 0); while( $x >= 0 ) { @@ -329,24 +329,24 @@ sub _parse_it { # If we didn't find an open element then throw an error $self->_error( 'NO_START', $start_pos + $self->{doc_offset}, $tag ) unless $found; - + # Call an end-tag handler if defined (not yet exposed) &{$self->{handlers}{end}}($self, $name) if defined $self->{handlers}{end}; - + # Close element in linked list (tree) $current_element = pop @$element_list; - + } else { - # Make a new list in the by-name lookup hash if none found by this name yet + # Make a new list in the by-name lookup hash if none found by this name yet $self->{elements}{$name} = [] unless defined $self->{elements}{$name}; - + # Add start points to the array of positions and push it on the hash my $pos_list = [$start_pos, $end_pos]; push @{$self->{elements}{$name}}, $pos_list; - + # Call start-tag handler if defined (not yet exposed) &{$self->{handlers}{start}}($self, $name) if defined $self->{handlers}{start}; - + # If this is a single-tag element (e.g. <.../>) then close it immediately if( $tag =~ m{/\s*>$} ) { push @$current_element, $pos_list; @@ -364,7 +364,7 @@ sub _parse_it { } # end if } # end if - + # Move the start pointer to beginning of next element $start_pos = index( $self->{doc}, '<', $start_pos + 1 ); last if $start_pos == -1 || $end_pos == $doc_end; @@ -372,16 +372,16 @@ sub _parse_it { # Now $end_pos is end of old tag and $start_pos is start of new # So do things on the data between the tags as needed if( $start_pos - $end_pos > 1 ) { - # Call any character data handler + # Call any character data handler &{$self->{handlers}{char}}($self, substr($self->{doc}, $end_pos + 1, $start_pos - $end_pos - 1)) if defined $self->{handlers}{char}; # Inserting the text into the linked list as well # push @$current_element, [$end_pos + 1, $start_pos - 1]; } # end if - + # Now finish by incrementing the parser to the next element $end_pos = index( $self->{doc}, '>', $start_pos + 1 ); - + # If there is no next element, and we're not at the end of the document, # then throw an error $self->_error( 'ELM_NOT_CLOSED', $start_pos + $self->{doc_offset} ) if $end_pos == -1; @@ -401,7 +401,7 @@ sub _parse_it { # # Returns: Scalar content of $file, undef on error # -# Description: Reads from $file and returns the content. +# Description: Reads from $file and returns the content. # $file may be either a filename or an IO handle # ---------------------------------------------------------- # Date Modification Author @@ -412,7 +412,7 @@ sub _get_a_file { my $self = shift; my $file = shift; my $content = undef; - + # If it's a ref and a handle, then read that if( ref($file) ) { $content = join '', <$file>; @@ -422,12 +422,12 @@ sub _get_a_file { open( XML, $file ) || return undef; $content = join '', <XML>; close XML || return undef; - } + } # Don't know how to handle this type of parameter else { return undef; } # end if - + return $content; } # end _get_a_file @@ -448,10 +448,10 @@ sub _error { my $self = shift; my( $code, @args ) = @_; my $msg = $self->{ERRORS}{$code}; - + # Handle replacement codes $msg =~ s/\%(\d+)/$args[$1]/g; - + # Throw exception die ref($self) . ":$msg\n"; } # end _error @@ -462,7 +462,7 @@ sub _error { # # Args: $content # -# Returns: A reference to the CDATA element, padded to +# Returns: A reference to the CDATA element, padded to # original size. # # Description: Stores the CDATA element in the internal @@ -498,13 +498,13 @@ sub _store_cdata { sub _dump_tree { my $self = shift; my $node = shift || $self->{tree}; - + my $tree = ''; for( my $i = 0; $i < scalar(@$node) && defined $node->[$i]; $i++ ) { if( (scalar(@{$node->[$i]}) == 4) && (defined $node->[$i][2]) ) { $tree .= '[' . join( ',', @{$node->[$i]} ) . "] " - . substr($self->{doc}, $node->[$i][0], $node->[$i][1] - $node->[$i][0] + 1) - . "..." + . substr($self->{doc}, $node->[$i][0], $node->[$i][1] - $node->[$i][0] + 1) + . "..." . substr($self->{doc}, $node->[$i][2], $node->[$i][3] - $node->[$i][2] + 1) . " (child $i)\n"; # Do child list $i++; @@ -530,7 +530,7 @@ END { } =head1 BUGS Lots. This 'parser' (Matt Sergeant takes umbrance to my us of that word) will handle some XML -documents, but not all. +documents, but not all. =head1 VERSION @@ -547,4 +547,3 @@ This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Lite/Element.pm b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Lite/Element.pm index 388511d89a..05debd50e2 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Lite/Element.pm +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/Lite/Element.pm @@ -33,18 +33,18 @@ print $elm->get_attribute( 'attribute_name' ); =head1 DESCRIPTION -C<XML::Lite::Element> objects contain rudimentary methods for querying XML -elements in an XML document as parsed by XML::Lite. Usually these objects +C<XML::Lite::Element> objects contain rudimentary methods for querying XML +elements in an XML document as parsed by XML::Lite. Usually these objects are returned by method calls in XML::Lite. =head1 METHODS -The following methods are available. All methods like 'get_name' can be +The following methods are available. All methods like 'get_name' can be abbeviated as 'name.' =over 4 -=cut +=cut use strict; BEGIN { @@ -63,8 +63,8 @@ use vars qw(); Creates a new XML::Lite::Element object from the XML::Lite object, C<$owner_document>. -Currently, you must not call this manually. You can create an object with one of -the 'factory' methods in XML::Lite, such as C<element_by_name> or C<root_element> +Currently, you must not call this manually. You can create an object with one of +the 'factory' methods in XML::Lite, such as C<element_by_name> or C<root_element> or with one of the XML::Lite::Element 'factory' methods below, like C<get_children>. =cut @@ -77,15 +77,15 @@ sub new { # The arguments are as follows: # $owner_document is an XML::Lite object within which this element lives # \@pointers is a two or four element array ref containing the offsets - # into the original document of the start and end points of + # into the original document of the start and end points of # the opening and closing (when it exists) tags for the element - + # Validate arguments return undef unless @_ >= 2; return undef unless ref($_[0]) && (ref($_[1]) eq 'ARRAY'); - + # Load 'em up - + # The data structure for the ::Element object has these properties # doc A reference to the containing XML::Lite object # node A reference to an array of pointers to our element in the document @@ -94,11 +94,11 @@ sub new { # name The name on our tag # _attrs A string of the attibutes in our tag (unparsed) # attrs A hash ref of attributes in our tag - + $self->{doc} = $_[0]; $self->{node} = $_[1]; - - # Using the pointers, find out tag name, and attribute list from the + + # Using the pointers, find out tag name, and attribute list from the # opening tag (if there are any attributes). my $tag = substr( $self->{doc}{doc}, $self->{node}[0], $self->{node}[1] - $self->{node}[0] + 1 ); if( $tag =~ m{^<\s*([^/>\s]+)\s+([^>]+)\s*/?\s*>$} ) { @@ -111,7 +111,7 @@ sub new { # Should have been caught in the parsing! maybe an assert? $self->{doc}->_error( 'ELM_NOT_CLOSED', $self->{node}[0] + $self->{doc}->{doc_offset} ); } # end if - + # Good. Now returns it. bless ($self, $class); return $self; @@ -142,16 +142,16 @@ sub content; sub get_content { my $self = shift; - # If we don't have any content, then we should return + # If we don't have any content, then we should return # '' right away. return '' unless defined $self->{node}[2]; - + # Using our pointers, find everything between our tags my $content = substr( $self->{doc}{doc}, $self->{node}[1] + 1, $self->{node}[2] - $self->{node}[1] - 1 ); - + # Now, restore any CDATA chunks that may have been pulled out $content =~ s/<!\[CDATA\[(\S+)\s*\]\]\/>/<![CDATA[$self->{doc}{_CDATA}[$1]]]>/g; - + # And return the content return $content; } # end get_content @@ -173,11 +173,11 @@ sub attributes; *attributes = \&get_attributes; sub get_attributes { my $self = shift; - + # Parse the attribute string into a hash of name-value pairs # unless we've already done that. $self->_parse_attrs() unless defined $self->{attrs}; - + # Just return a *copy* of the hash (this is read-only after all!) if ( defined($self->{attrs}) ) { return %{$self->{attrs}}; @@ -202,10 +202,10 @@ sub attribute; sub get_attribute { my $self = shift; my( $name ) = @_; - + # If we haven't parsed the attribute string into a hash, then do that. $self->_parse_attrs() unless defined $self->{attrs}; - + # Now return the requested attribute. If it's not there # then 'undef' is returned return $self->{attrs}{$name}; @@ -233,9 +233,9 @@ sub get_name { =item my @children = $element->get_children() -Returns a list of XML::Lite::Element objects for each element contained -within the current element. This does not return any text or CDATA in -the content of this element. You can parse that through the L<get_content> +Returns a list of XML::Lite::Element objects for each element contained +within the current element. This does not return any text or CDATA in +the content of this element. You can parse that through the L<get_content> method. If no child elements exist then an empty list is returned. @@ -256,7 +256,7 @@ sub get_children { my $self = shift; my @children = (); - # If we don't have any content, then we should return an emtpty + # If we don't have any content, then we should return an emtpty # list right away -- we have no children. return @children unless defined $self->{node}[2]; @@ -264,8 +264,8 @@ sub get_children { # This will also load {children} and {parent} as well $self->_find_self() unless defined $self->{self}; - # Now that we know who we are (if this didn't fail) we can - # iterate through the sub nodes (our child list) and make + # Now that we know who we are (if this didn't fail) we can + # iterate through the sub nodes (our child list) and make # XML::Lite::Elements objects for each child if( defined $self->{children} ) { my $i = 0; @@ -276,7 +276,7 @@ sub get_children { $node = $self->{children}[++$i]; } # end while } # end if - + return @children; } # end get_children @@ -304,14 +304,14 @@ sub get_text { my $self = shift; my $content = ''; - # If we don't have any content, then we should return + # If we don't have any content, then we should return # $content right away -- we have no text return $content unless defined $self->{node}[2]; # Otherwise get out content and children my @children = $self->get_children; my $orig_content = $self->get_content; - + # Then remove the child elements from our content my $start = 0; foreach( @children ) { @@ -320,10 +320,10 @@ sub get_text { $start = ($_->{node}[3] || $_->{node}[1]) - $self->{node}[1]; } # end foreach $content .= substr( $orig_content, $start ) if $start < length($orig_content); - + # Remove the CDATA wrapper, preserving the content $content =~ s/<!\[CDATA\[(.+?)]\]>/$1/g; - + # Return the left-over text return $content; } # end get_text @@ -352,7 +352,7 @@ sub get_text { # ---------------------------------------------------------- sub _parse_attrs { my $self = shift; - + my $attrs = $self->{_attrs}; if ( defined($attrs) ) { $attrs =~ s/^\s+//; @@ -364,7 +364,7 @@ sub _parse_attrs { $attrs =~ s/^\s+//; } # end while } - + return 1; } # end _parse_atttrs @@ -376,7 +376,7 @@ sub _parse_attrs { # Returns: A reference to our node or undef on error # # Description: Traverses the owner document's tree to find -# the node that references the current element. Sets +# the node that references the current element. Sets # $self-{self} as a side-effect. Even if this is already set, # _find_self will traverse again, so don't call unless needed. # ---------------------------------------------------------- @@ -387,8 +387,8 @@ sub _parse_attrs { # ---------------------------------------------------------- sub _find_self { my $self = shift; - - # We actually just call this recusively, so the first + + # We actually just call this recusively, so the first # argument can be a starting point to descend from # but we don't doc that above my $node = shift || $self->{doc}{tree}; @@ -405,10 +405,10 @@ sub _find_self { # If this is our self, then we're done! # NOTE: Since the list references are the same in the by-name hash # and tree objects, we can just do a reference compare here. - # If objects are ever created with non-factory methods then we need to + # If objects are ever created with non-factory methods then we need to # use a _compare_lists call. -# if( _compare_lists( $node->[$i], $self->{node} ) ) { - if( $node->[$i] eq $self->{node} ) { +# if( _compare_lists( $node->[$i], $self->{node} ) ) { + if( $node->[$i] eq $self->{node} ) { $self->{parent} = $node; $self->{self} = $node->[$i]; # If this list has children, then add a pointer to that list @@ -453,16 +453,16 @@ sub _find_self { # ---------------------------------------------------------- sub _compare_lists { my( $rA, $rB ) = @_; - + # Lists are not equal unless same size return 0 unless scalar(@$rA) == scalar(@$rB); - + # Now compare item by item. my $i; for( $i = 0; $i < scalar(@$rA); $i++ ) { return 0 unless $rA->[$i] eq $rB->[$i]; } # end for - + return 1; } # end _compare_lists @@ -488,4 +488,3 @@ This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/README b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/README index 6234a760ce..6f9ebf8456 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/README +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/README @@ -7,7 +7,7 @@ for most things you need to do with XML files. It is not dependent on any other modules or external programs for installation. -NOTE that this parser will do many things that you want with XML but +NOTE that this parser will do many things that you want with XML but not everything. It is not a validating parser! It will not handle international characters (unless run on those systems). Use at your own risk. @@ -16,5 +16,3 @@ Copyright 2001-2003 Wadsack-Allen. All rights reserved. This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. - - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/man3/XML_Lite.3 b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/man3/XML_Lite.3 index f2b3912ed7..f0adb0e00a 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/man3/XML_Lite.3 +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/man3/XML_Lite.3 @@ -148,12 +148,12 @@ use \s-1XML:\s0:Lite; my \f(CW$xml\fR = new \s-1XML:\s0:Lite( xml => 'a_file.xml' ); .SH "DESCRIPTION" .IX Header "DESCRIPTION" -\&\s-1XML:\s0:Lite is a lightweight \s-1XML\s0 parser, with basic element traversing -methods. It is entirely self-contained, pure Perl (i.e. \fInot\fR based on -expat). It provides useful methods for reading most \s-1XML\s0 files, including -traversing and finding elements, reading attributes and such. It is -designed to take advantage of Perl-isms (Attribute lists are returned as -hashes, rather than, say, lists of objects). It provides only methods +\&\s-1XML:\s0:Lite is a lightweight \s-1XML\s0 parser, with basic element traversing +methods. It is entirely self-contained, pure Perl (i.e. \fInot\fR based on +expat). It provides useful methods for reading most \s-1XML\s0 files, including +traversing and finding elements, reading attributes and such. It is +designed to take advantage of Perl-isms (Attribute lists are returned as +hashes, rather than, say, lists of objects). It provides only methods for reading a file, currently. .SH "METHODS" .IX Header "METHODS" @@ -161,16 +161,16 @@ The following methods are available: .Ip "my \f(CW$xml\fR = new \s-1XML:\s0:Lite( xml => \f(CW$source\fR[, ...] );" 4 .IX Item "my $xml = new XML::Lite( xml => $source[, ...] );" Creates a new \s-1XML:\s0:Lite object. The \s-1XML:\s0:Lite object acts as the document -object for the \f(CW$source\fR that is sent to it to parse. This means that you -create a new object for each document (or document sub-section). As the +object for the \f(CW$source\fR that is sent to it to parse. This means that you +create a new object for each document (or document sub-section). As the objects are lightweight this should not be a performance consideration. .Sp The object constructor can take several named parameters. Parameter names -may begin with a '\-' (as in the example above) but are not required to. The +may begin with a '\-' (as in the example above) but are not required to. The following parameters are recognized. .Sp .Vb 2 -\& xml The source XML to parse. This can be a filename, a scalar that +\& xml The source XML to parse. This can be a filename, a scalar that \& contains the document (or document fragment), or an IO handle. .Ve As a convenince, if only on parameter is given, it is assumed to be the source. @@ -190,8 +190,8 @@ Returns \f(CW\*(C`undef\*(C'\fR on errors. Returns a list of all elements that match \f(CW\*(C`$name\*(C'\fR. \&\f(CW\*(C`@list\*(C'\fR is a list of the XML::Lite::Element manpage objects If called in a scalar context, this will return the -first element found that matches (it's more efficient -to call in a scalar context than assign the results +first element found that matches (it's more efficient +to call in a scalar context than assign the results to a list of one scalar). .Sp If no matching elements are found then returns \f(CW\*(C`undef\*(C'\fR @@ -199,7 +199,7 @@ in scalar context or an empty list in array context. .SH "BUGS" .IX Header "BUGS" Lots. This 'parser' (Matt Sergeant takes umbrance to my us of that word) will handle some \s-1XML\s0 -documents, but not all. +documents, but not all. .SH "VERSION" .IX Header "VERSION" 0.14 diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/man3/XML_Lite_Element.3 b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/man3/XML_Lite_Element.3 index 5eaf684214..f31d1336e4 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/man3/XML_Lite_Element.3 +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/perl5lib/XML/man3/XML_Lite_Element.3 @@ -151,19 +151,19 @@ my \f(CW$elm\fR = \f(CW$xml\fR->elements_by_name( 'element_name' ); print \f(CW$elm\fR->get_attribute( 'attribute_name' ); .SH "DESCRIPTION" .IX Header "DESCRIPTION" -\&\f(CW\*(C`XML::Lite::Element\*(C'\fR objects contain rudimentary methods for querying \s-1XML\s0 -elements in an \s-1XML\s0 document as parsed by \s-1XML:\s0:Lite. Usually these objects +\&\f(CW\*(C`XML::Lite::Element\*(C'\fR objects contain rudimentary methods for querying \s-1XML\s0 +elements in an \s-1XML\s0 document as parsed by \s-1XML:\s0:Lite. Usually these objects are returned by method calls in \s-1XML:\s0:Lite. .SH "METHODS" .IX Header "METHODS" -The following methods are available. All methods like 'get_name' can be +The following methods are available. All methods like 'get_name' can be abbeviated as 'name.' .Ip "my \f(CW$element\fR = new \s-1XML:\s0:Lite::Element( \f(CW$owner_document\fR, \e@pointers );" 4 .IX Item "my $element = new XML::Lite::Element( $owner_document, @pointers );" Creates a new \s-1XML:\s0:Lite::Element object from the \s-1XML:\s0:Lite object, \f(CW\*(C`$owner_document\*(C'\fR. .Sp -Currently, you must not call this manually. You can create an object with one of -the 'factory' methods in \s-1XML:\s0:Lite, such as \f(CW\*(C`element_by_name\*(C'\fR or \f(CW\*(C`root_element\*(C'\fR +Currently, you must not call this manually. You can create an object with one of +the 'factory' methods in \s-1XML:\s0:Lite, such as \f(CW\*(C`element_by_name\*(C'\fR or \f(CW\*(C`root_element\*(C'\fR or with one of the \s-1XML:\s0:Lite::Element 'factory' methods below, like \f(CW\*(C`get_children\*(C'\fR. .Ip "my \f(CW$content\fR = \f(CW$element\fR->\fIget_content()\fR" 4 .IX Item "my $content = $element->get_content()" @@ -180,9 +180,9 @@ Returns the value of the named attribute for this element. Returns the name of the element tag .Ip "my \f(CW@children\fR = \f(CW$element\fR->\fIget_children()\fR" 4 .IX Item "my @children = $element->get_children()" -Returns a list of \s-1XML:\s0:Lite::Element objects for each element contained -within the current element. This does not return any text or \s-1CDATA\s0 in -the content of this element. You can parse that through the the get_content manpage +Returns a list of \s-1XML:\s0:Lite::Element objects for each element contained +within the current element. This does not return any text or \s-1CDATA\s0 in +the content of this element. You can parse that through the the get_content manpage method. .Sp If no child elements exist then an empty list is returned. diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testdecomp.F90 b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testdecomp.F90 index 9684e14a19..6ea015fd56 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testdecomp.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testdecomp.F90 @@ -7,7 +7,7 @@ program testdecomp use gdecomp_mod implicit none - + integer, pointer :: compDOF(:), ioDOF(:) integer :: startcomp(3),cntcomp(3) integer :: startio(3),cntio(3),gdims(3) @@ -23,7 +23,7 @@ program testdecomp num_tasks = 192 gdims(1) = 3600 gdims(2) = 2400 - gdims(3) = 40 + gdims(3) = 40 ! call gdecomp_read_nml(gdecomp,fin,'comp',my_task) ! print *,'after gdecomp_read_nml' diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testdecomp.bluefire.run b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testdecomp.bluefire.run index d3f1ddbf79..371d6fc128 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testdecomp.bluefire.run +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testdecomp.bluefire.run @@ -27,6 +27,3 @@ cp -f $srcdir/testdecomp_in ./testdecomp_in mpirun.lsf ./testdecomp >& testdecomp.out.$LID cp testdecomp.out.$LID $srcdir/ - - - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio.F90 b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio.F90 index d93d4e2752..5b4201f0b0 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio.F90 @@ -46,7 +46,7 @@ program testpio integer(i4) :: indx integer(i4) :: mode - integer(i4) :: ip,numPhases + integer(i4) :: ip,numPhases character(len=*), parameter :: TestR8CaseName = 'r8_test' character(len=*), parameter :: TestR4CaseName = 'r4_test' character(len=*), parameter :: TestI4CaseName = 'i4_test' @@ -101,17 +101,17 @@ program testpio real(r8) :: dt_write_r8, dt_write_r4, dt_write_i4 ! individual write times real(r8) :: dt_read_r8, dt_read_r4, dt_read_i4 ! individual read times ! Arrays to hold globally reduced read/write times--one element per time trial - real(r8), dimension(:), pointer :: gdt_write_r8, gdt_write_r4, gdt_write_i4 + real(r8), dimension(:), pointer :: gdt_write_r8, gdt_write_r4, gdt_write_i4 real(r8), dimension(:), pointer :: gdt_read_r8, gdt_read_r4, gdt_read_i4 integer(i4) :: nprocs - integer(i4) :: lLength ! local number of words in the computational decomposition + integer(i4) :: lLength ! local number of words in the computational decomposition integer(i4), parameter :: nml_in = 10 character(len=*), parameter :: nml_filename = 'testpio_in' - integer(i4) :: master_task - logical :: log_master_task + integer(i4) :: main_task + logical :: log_main_task integer(i4) :: nml_error integer(kind=pio_offset_kind) :: sdof,sdof_sum,sdof_min,sdof_max @@ -149,7 +149,7 @@ program testpio call MPI_INIT(ierr) call CheckMPIReturn('Call to MPI_INIT()',ierr,__FILE__,__LINE__) - + ! call enable_abort_on_exit @@ -157,11 +157,11 @@ program testpio call CheckMPIReturn('Call to MPI_COMM_RANK()',ierr,__FILE__,__LINE__) call MPI_COMM_SIZE(MPI_COMM_WORLD,nprocs,ierr) call CheckMPIReturn('Call to MPI_COMM_SIZE()',ierr,__FILE__,__LINE__) - master_task = 0 - if (my_task == master_task) then - log_master_task = .true. + main_task = 0 + if (my_task == main_task) then + log_main_task = .true. else - log_master_task = .false. + log_main_task = .false. endif if(Debug) print *,'testpio: before call to t_initf' @@ -171,7 +171,7 @@ program testpio !--------------------------------------------------------------- if(Debug) print *,'testpio: point #1' call t_initf(nml_filename, logprint=.false., logunit=6, & - mpicom=MPI_COMM_WORLD, MasterTask=log_master_task) + mpicom=MPI_COMM_WORLD, mainTask=log_main_task) if(Debug) print *,'testpio: point #2' call t_startf('testpio_total') @@ -193,14 +193,14 @@ program testpio if (mrss1 - mrss0 > 0) then mb_blk = (8.0_r8)/((mrss1-mrss0)*1.0_r8) endif - if (my_task == master_task) then + if (my_task == main_task) then write(*,*) myname,' 8 MB memory alloc in MB is ',(mrss1-mrss0)*mb_blk write(*,*) myname,' 8 MB memory dealloc in MB is ',(mrss1-mrss2)*mb_blk write(*,*) myname,' Memory block size conversion in bytes is ',mb_blk*1024_r8*1024.0_r8 endif #endif -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -212,19 +212,19 @@ program testpio !---------------------------------------------------------------- if(Debug) print *,'testpio: before call to readTestPIO_Namelist' - if(my_task == master_task) then + if(my_task == main_task) then call ReadTestPIO_Namelist(nml_in, nprocs, nml_filename, myname, nml_error) endif if(Debug) print *,'testpio: before call to broadcast_namelist' call MPI_barrier(MPI_COMM_WORLD,ierr) - call Broadcast_Namelist(myname, my_task, master_task, MPI_COMM_WORLD, ierr) + call Broadcast_Namelist(myname, my_task, main_task, MPI_COMM_WORLD, ierr) if(Debug) print *,'testpio: after call to broadcast_namelist' !------------------------------------- ! Checks (num_iotasks can be negative on BGx) !------------------------------------- -#if !defined(BGx) +#if !defined(BGx) if (num_iotasks <= 0) then write(*,*) trim(myname),' ERROR: ioprocs invalid num_iotasks=',num_iotasks call piodie(__FILE__,__LINE__) @@ -234,7 +234,7 @@ program testpio ! ---------------------------------------------------------------- ! if stride is and num_iotasks is incompatible than reset stride (ignore stride on BGx) ! ---------------------------------------------------------------- -#if !defined(BGx) +#if !defined(BGx) if (base + num_iotasks * (stride-1) > nprocs-1) then write(*,*) trim(myname),' ERROR: num_iotasks, base and stride too large', & ' base=',base,' num_iotasks=',num_iotasks,' stride=',stride,' nprocs=',nprocs @@ -243,7 +243,7 @@ program testpio #endif !-------------------------------------- - ! Initalizes the parallel IO subsystem + ! Initalizes the parallel IO subsystem !-------------------------------------- call PIO_setDebugLevel(DebugLevel) @@ -266,7 +266,7 @@ program testpio call MPI_COMM_SIZE(MPI_COMM_COMPUTE,nprocs,ierr) else -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -279,7 +279,7 @@ program testpio call PIO_init(my_task, MPI_COMM_COMPUTE, num_iotasks, num_aggregator, stride, & rearr_type, PIOSYS, base) -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -289,7 +289,7 @@ program testpio end if - + if(Debug) print *,'testpio: after call to PIO_init', nprocs,mpi_comm_io @@ -339,7 +339,7 @@ program testpio trim(ibm_io_sparse_access)) end if end if -! if(set_lustre_values /= 0) then +! if(set_lustre_values /= 0) then ! call PIO_setnum_OST(PIOSYS,lfs_ost_count) ! endif @@ -355,11 +355,11 @@ program testpio if(index(casename,'CAM')==1) then call camlike_decomp_generator(gdims3d(1),gdims3d(2),gdims3d(3),my_task,nprocs,npr_yz,compDOF) - elseif(index(casename,'MPAS')==1) then + elseif(index(casename,'MPAS')==1) then ! print *,'testpio: before call to mpas_decomp_generator: (',TRIM(part_input),') gdims3d: ',gdims3d call mpas_decomp_generator(gdims3d(1),gdims3d(2),gdims3d(3),my_task,part_input,compDOF) else if (trim(compdof_input) == 'namelist') then -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -369,7 +369,7 @@ program testpio if(Debug) print *,'iam: ',My_task,'testpio: point #1' call gdecomp_read_nml(gdecomp,nml_filename,'comp',my_task,nprocs,gDims3D(1:3)) -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -380,7 +380,7 @@ program testpio if(Debug) print *,'iam: ',My_task,'testpio: point #2' call gdecomp_DOF(gdecomp,My_task,compDOF,start,count) if(Debug) print *,'iam: ',My_task,'testpio: point #3', minval(compdof),maxval(compdof) -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -413,7 +413,7 @@ program testpio endif endif -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -427,7 +427,7 @@ program testpio call pio_writedof(trim(compdof_output),gdims3d, compDOF,MPI_COMM_COMPUTE,75) endif -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -436,17 +436,17 @@ program testpio #endif sdof = sum(compDOF) - call MPI_REDUCE(sdof,sdof_sum,1,MPI_INTEGER8,MPI_SUM,master_task,MPI_COMM_COMPUTE,ierr) + call MPI_REDUCE(sdof,sdof_sum,1,MPI_INTEGER8,MPI_SUM,main_task,MPI_COMM_COMPUTE,ierr) call CheckMPIReturn('Call to MPI_REDUCE SUM',ierr,__FILE__,__LINE__) sdof = minval(compDOF) - call MPI_REDUCE(sdof,sdof_min,1,MPI_INTEGER8,MPI_MIN,master_task,MPI_COMM_COMPUTE,ierr) + call MPI_REDUCE(sdof,sdof_min,1,MPI_INTEGER8,MPI_MIN,main_task,MPI_COMM_COMPUTE,ierr) call CheckMPIReturn('Call to MPI_REDUCE MIN',ierr,__FILE__,__LINE__) sdof = maxval(compDOF) - call MPI_REDUCE(sdof,sdof_max,1,MPI_INTEGER8,MPI_MAX,master_task,MPI_COMM_COMPUTE,ierr) + call MPI_REDUCE(sdof,sdof_max,1,MPI_INTEGER8,MPI_MAX,main_task,MPI_COMM_COMPUTE,ierr) call CheckMPIReturn('Call to MPI_REDUCE MAX',ierr,__FILE__,__LINE__) - if (my_task == master_task) then + if (my_task == main_task) then write(6,*) trim(myname),' total nprocs = ',nprocs write(6,*) trim(myname),' compDOF sum/min/max = ',sdof_sum,sdof_min,sdof_max endif @@ -482,7 +482,7 @@ program testpio else call piodie(__FILE__,__LINE__,' rearr '//trim(rearr)//' not supported') endif -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -505,19 +505,19 @@ program testpio lLength = size(compDOF) !---------------------- - ! allocate and set test arrays + ! allocate and set test arrays !---------------------- - if(TestR8 .or. TestCombo) then + if(TestR8 .or. TestCombo) then call alloc_check(test_r8wr,lLength,'testpio:test_r8wr') endif - if(TestR4 .or. TestCombo) then + if(TestR4 .or. TestCombo) then call alloc_check(test_r4wr,lLength,'testpio:test_r4wr' ) endif - if(TestInt .or. TestCombo) then + if(TestInt .or. TestCombo) then call alloc_check(test_i4wr,lLength,'testpio:test_i4wr') endif - if(TestInt) then + if(TestInt) then call alloc_check(test_i4i ,lLength,'testpio:test_i4i ') call alloc_check(test_i4j ,lLength,'testpio:test_i4j ') call alloc_check(test_i4k ,lLength,'testpio:test_i4k ') @@ -527,27 +527,27 @@ program testpio do n = 1,lLength call c1dto3d(compdof(n),gDims3D(1),gDims3D(2),gDims3D(3),i1,j1,k1) - if(TestInt) then + if(TestInt) then test_i4dof(n) = compdof(n) test_i4i(n) = i1 test_i4j(n) = j1 test_i4k(n) = k1 test_i4m(n) = my_task endif - if(TestR8 .or. TestCombo) then + if(TestR8 .or. TestCombo) then ! test_r8wr(n) = 10.0_r8*cos(20.*real(i1,kind=r8)/real(gDims3D(1),kind=r8))* & ! cos(10.*real(j1,kind=r8)/real(gDims3D(2),kind=r8))* & ! (1.0+1.0*real(j1,kind=r8)/real(gDims3D(2),kind=r8))* & ! cos(25.*real(k1,kind=r8)/real(gDims3D(3),kind=r8)) test_r8wr = compdof endif - if(TestR4 .or. TestCombo) then + if(TestR4 .or. TestCombo) then test_r4wr(n) = 10.0_r4*cos(20.*real(i1,kind=r4)/real(gDims3D(1),kind=r4))* & cos(10.*real(j1,kind=r4)/real(gDims3D(2),kind=r4))* & (1.0+1.0*real(j1,kind=r4)/real(gDims3D(2),kind=r4))* & cos(25.*real(k1,kind=r4)/real(gDims3D(3),kind=r4)) endif - if(TestInt .or. TestCombo) then + if(TestInt .or. TestCombo) then test_i4wr(n) = compdof(n) ! test_i4wr(n) = nint(10.0_r8*cos(20.*real(i1,kind=r8)/real(gDims3D(1),kind=r8))* & ! cos(10.*real(j1,kind=r8)/real(gDims3D(2),kind=r8))* & @@ -558,7 +558,7 @@ program testpio if(Debug) print *,'iam: ',My_task,'testpio: point #10' -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -581,7 +581,7 @@ program testpio !-------------------------------- ! allocate arrays for holding globally-reduced timing information !-------------------------------- -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -595,25 +595,25 @@ program testpio call alloc_check(gdt_write_i4, maxiter, ' testpio:gdt_write_i4 ') call alloc_check(gdt_read_i4, maxiter, ' testpio:gdt_read_i4 ') if(Debug) print *,'iam: ',My_task,'testpio: point #11' -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss print *,__FILE__,__LINE__,'mem=',rss,' it=',it end if #endif - if(splitPhase) then + if(splitPhase) then numPhases = 2 else numPhases = 1 endif do ip=1,numPhases - if(numPhases == 1) then + if(numPhases == 1) then readPhase = .true. writePhase = .true. else - if(ip == 1) then + if(ip == 1) then writePhase = .true. readPhase = .false. else @@ -621,11 +621,11 @@ program testpio readPhase = .true. endif endif - if(log_master_task) print *,'{write,read}Phase: ',writePhase,readPhase + if(log_main_task) print *,'{write,read}Phase: ',writePhase,readPhase + - do it=1,maxiter -#ifdef MEMCHK +#ifdef MEMCHK call GPTLget_memusage(msize, rss, mshare, mtext, mstack) if(rss>lastrss) then lastrss=rss @@ -644,7 +644,7 @@ program testpio IOdesc_r8,iostart=startpio,iocount=countpio) glenr8 = product(gdims3d) if(Debug) print *,'iam: ',My_task,'testpio: point #7.1' - + if(TestR4 .or. TestCombo) then call PIO_initDecomp(PIOSYS,PIO_real, gDims3D,compDOF,& IOdesc_r4,iostart=startpio,iocount=countpio) @@ -675,7 +675,7 @@ program testpio if(TestInt .or. TestCombo) then ! print *,__FILE__,__LINE__,gdims3d ! print *,__FILE__,__LINE__,compdof - + call PIO_initDecomp(PIOSYS,PIO_int, gDims3D,compDOF,& IOdesc_i4) if(Debug) print *,'iam: ',My_task,'testpio: point #8.4' @@ -750,7 +750,7 @@ program testpio endif endif if(Debug) print *,'iam: ',My_task,'testpio: point #9' - + if(Debug) then write(*,'(a,2(a,i8))') myname,':: After call to initDecomp. comp_rank=',my_task, & ' io_rank=',iorank @@ -758,10 +758,10 @@ program testpio call PIO_getnumiotasks(PIOSYS,num_iotasks) !------------ - ! Open file{s} + ! Open file{s} !------------ write(citer,'(i3.3)') it - + fname = TRIM(dir)//'foo.'//citer//'.'//TRIM(Iofmtd) fname_r8 = TRIM(dir)//'foo.r8.'//citer//'.'//TRIM(Iofmtd) fname_r4 = TRIM(dir)//'foo.r4.'//citer//'.'//TRIM(Iofmtd) @@ -772,7 +772,7 @@ program testpio ! print *, __FILE__,__LINE__,'>',fname_r4,'<' mode = pio_64bit_offset - if(writePhase) then + if(writePhase) then if(TestCombo) then if(Debug) write(*,'(2a,i8)') myname,':: Combination Test: Creating File...it=',it ierr = PIO_CreateFile(PIOSYS,File,iotype,trim(fname), mode) @@ -797,21 +797,21 @@ program testpio call check_pioerr(ierr,__FILE__,__LINE__,' i4 createfile') endif - + allocate(vard_r8(nvars), vard_r4(nvars)) - + !--------------------------- - ! Code specifically for netCDF files + ! Code specifically for netCDF files !--------------------------- - if(iotype == iotype_pnetcdf .or. & + if(iotype == iotype_pnetcdf .or. & iotype == iotype_netcdf .or. & iotype == PIO_iotype_netcdf4p .or. & iotype == PIO_iotype_netcdf4c) then - if(TestR8) then + if(TestR8) then !----------------------------------- - ! for the single record real*8 file + ! for the single record real*8 file !----------------------------------- call WriteHeader(File_r8,nx_global,ny_global,nz_global,dimid_x,dimid_y,dimid_z) @@ -829,9 +829,9 @@ program testpio call check_pioerr(iostat,__FILE__,__LINE__,' r8 enddef') endif - if(TestR4) then + if(TestR4) then !----------------------------------- - ! for the single record real*4 file + ! for the single record real*4 file !----------------------------------- call WriteHeader(File_r4,nx_global,ny_global,nz_global,dimid_x,dimid_y,dimid_z) iostat = PIO_def_dim(File_r4,'charlen',strlen,charlen) @@ -846,14 +846,14 @@ program testpio call check_pioerr(iostat,__FILE__,__LINE__,' i4 enddef') endif - if(TestInt) then + if(TestInt) then !----------------------------------- - ! for the single record integer file + ! for the single record integer file !----------------------------------- call WriteHeader(File_i4,nx_global,ny_global,nz_global,dimid_x,dimid_y,dimid_z) - iostat = PIO_def_var(File_i4,'fdof',PIO_int,(/dimid_x,dimid_y,dimid_z/),vard_i4dof) + iostat = PIO_def_var(File_i4,'fdof',PIO_int,(/dimid_x,dimid_y,dimid_z/),vard_i4dof) call check_pioerr(iostat,__FILE__,__LINE__,' i4dof defvar') - + iostat = PIO_def_var(File_i4,'field',PIO_int,(/dimid_x,dimid_y,dimid_z/),vard_i4) call check_pioerr(iostat,__FILE__,__LINE__,' i4 defvar') iostat = PIO_def_var(File_i4,'fi',PIO_int,(/dimid_x,dimid_y,dimid_z/),vard_i4i) @@ -869,10 +869,10 @@ program testpio iostat = PIO_enddef(File_i4) call check_pioerr(iostat,__FILE__,__LINE__,' i4 enddef') endif - - if(TestCombo) then + + if(TestCombo) then !----------------------------------- - ! for the multi record file + ! for the multi record file !----------------------------------- call WriteHeader(File,nx_global,ny_global,nz_global,dimid_x,dimid_y,dimid_z) iostat = PIO_def_var(File,'field_r8',PIO_double,(/dimid_x,dimid_y,dimid_z/),vard_r8c) @@ -902,12 +902,12 @@ program testpio endif !------------------------- - ! Time the parallel write + ! Time the parallel write !------------------------- - + dt_write_r8 = 0. - + if(TestR8) then if(iofmtd .ne. 'bin') then iostat = pio_put_var(file_r8,varfn_r8,fname_r8) @@ -962,7 +962,7 @@ program testpio endif if(Debug) print *,'iam: ',My_task,'testpio: point #13' - if(TestInt) then + if(TestInt) then dt_write_i4 = 0. call MPI_Barrier(MPI_COMM_COMPUTE,ierr) call CheckMPIReturn('Call to MPI_BARRIER()',ierr,__FILE__,__LINE__) @@ -1001,10 +1001,10 @@ program testpio call PIO_write_darray(File_i4,vard_i4m,iodesc_i4,test_i4m,iostat) call check_pioerr(iostat,__FILE__,__LINE__,' i4m write_darray') - call PIO_CloseFile(File_i4) + call PIO_CloseFile(File_i4) endif - if(TestCombo) then + if(TestCombo) then if(iofmtd .ne. 'bin') then iostat = pio_put_var(file,varfn,fname) iostat = pio_put_var(file,varfruit,fruits) @@ -1022,9 +1022,9 @@ program testpio endif if(Debug) then - write(*,'(a,2(a,i8),i8)') myname,':: After calls to PIO_write_darray. comp_rank=',my_task, & + write(*,'(a,2(a,i8),i8)') myname,':: After calls to PIO_write_darray. comp_rank=',my_task, & ' io_rank=',iorank,mpi_comm_io - + endif endif @@ -1033,17 +1033,17 @@ program testpio if(Debug) print *,'iam: ',My_task,'testpio: point #14' - if (readPhase) then + if (readPhase) then !------------------------------------- ! Open the file back up and check data !------------------------------------- - - if(TestR8) then + + if(TestR8) then ierr = PIO_OpenFile(PIOSYS, File_r8, iotype, fname_r8) call check_pioerr(ierr,__FILE__,__LINE__,' r8 openfile') endif if(Debug) print *,'iam: ',My_task,'testpio: point #15' - + if(TestR4) then ierr = PIO_OpenFile(PIOSYS,File_r4,iotype, fname_r4) call check_pioerr(ierr,__FILE__,__LINE__,' r4 openfile') @@ -1059,13 +1059,13 @@ program testpio if(Debug) then write(*,'(2a,i8)') myname,':: After calls to PIO_OpenFile. my_task=',my_task endif - + if(Debug) print *,__FILE__,__LINE__ if(iotype == iotype_pnetcdf .or. & iotype == iotype_netcdf) then do ivar=1,nvars - if(TestR8) then + if(TestR8) then iostat = PIO_inq_varid(file_r8,'filename',varfn_r8) @@ -1073,15 +1073,15 @@ program testpio call check_pioerr(iostat,__FILE__,__LINE__,' r8 inq_varid') endif - if(TestR4) then + if(TestR4) then if(iofmtd(2:3) .eq. 'nc') then iostat = PIO_inq_varid(file_r4,'filename',varfn_r4) end if - iostat = PIO_inq_varid(File_r4,'field00001',vard_r4(ivar)) + iostat = PIO_inq_varid(File_r4,'field00001',vard_r4(ivar)) call check_pioerr(iostat,__FILE__,__LINE__,' r4 inq_varid') endif end do - if(TestInt) then + if(TestInt) then iostat = PIO_inq_varid(File_i4,'field',vard_i4) call check_pioerr(iostat,__FILE__,__LINE__,' i4 inq_varid') endif @@ -1093,7 +1093,7 @@ program testpio ! Time the parallel read !------------------------- dt_read_r8 = 0. - if(TestR8) then + if(TestR8) then if(iofmtd(2:3) .eq. 'nc') then iostat = pio_get_var(file_r8,varfn_r8, fnamechk) if(fnamechk /= fname_r8) then @@ -1116,7 +1116,7 @@ program testpio call t_stopf('testpio_read') #endif et = MPI_Wtime() - dt_read_r8 = dt_read_r8 + (et - st)/nvars + dt_read_r8 = dt_read_r8 + (et - st)/nvars call check_pioerr(iostat,__FILE__,__LINE__,' r8 read_darray') endif @@ -1166,12 +1166,12 @@ program testpio endif !------------------------------- - ! Print the maximum memory usage + ! Print the maximum memory usage !------------------------------- ! call alloc_print_usage(0,'testpio: after calls to PIO_read_darray') #ifdef TESTMEM -! stop +! stop #endif if(Debug) then @@ -1180,7 +1180,7 @@ program testpio endif !------------------- - ! close the file up + ! close the file up !------------------- if(TestR8) call PIO_CloseFile(File_r8) if(TestR4) call PIO_CloseFile(File_r4) @@ -1196,13 +1196,13 @@ program testpio ! endif !----------------------------- - ! Perform correctness testing + ! Perform correctness testing !----------------------------- - if(TestR8 .and. CheckArrays) then + if(TestR8 .and. CheckArrays) then call checkpattern(mpi_comm_compute, fname_r8,test_r8wr,test_r8rd,lLength,iostat) call check_pioerr(iostat,__FILE__,__LINE__,' checkpattern r8 test') endif - + if( TestR4 .and. CheckArrays) then call checkpattern(mpi_comm_compute, fname_r4,test_r4wr,test_r4rd,lLength,iostat) call check_pioerr(iostat,__FILE__,__LINE__,' checkpattern r4 test') @@ -1214,15 +1214,15 @@ program testpio endif if(Debug) print *,'iam: ',My_task,'testpio: point #21' - if(TestCombo .and. CheckArrays) then + if(TestCombo .and. CheckArrays) then !------------------------------------- - ! Open up and read the combined file + ! Open up and read the combined file !------------------------------------- - + ierr = PIO_OpenFile(PIOSYS,File,iotype,fname) call check_pioerr(ierr,__FILE__,__LINE__,' combo test read openfile') - + if(iofmtd(1:2).eq.'nc') then iostat = PIO_inq_varid(File,'field_r8',vard_r8c) call check_pioerr(iostat,__FILE__,__LINE__,' combo test r8 inq_varid') @@ -1256,22 +1256,22 @@ program testpio call PIO_CloseFile(File) if(Debug) print *,'iam: ',My_task,'testpio: point #22a' et = MPI_Wtime() - dt_read_r8 = dt_read_r8 + (et - st)/nvars + dt_read_r8 = dt_read_r8 + (et - st)/nvars !----------------------------- - ! Check the combined file + ! Check the combined file !----------------------------- call checkpattern(mpi_comm_compute, fname,test_r8wr,test_r8rd,lLength,iostat) call check_pioerr(iostat,__FILE__,__LINE__,' checkpattern test_r8 ') - + call checkpattern(mpi_comm_compute, fname,test_r4wr,test_r4rd,lLength,iostat) call check_pioerr(iostat,__FILE__,__LINE__,' checkpattern test_r4 ') - + call checkpattern(mpi_comm_compute, fname,test_i4wr,test_i4rd,lLength,iostat) call check_pioerr(iostat,__FILE__,__LINE__,' checkpattern test_i4 ') - + endif !--------------------------------------- - ! Print out the performance measurements + ! Print out the performance measurements !--------------------------------------- call MPI_Barrier(MPI_COMM_COMPUTE,ierr) endif @@ -1289,7 +1289,7 @@ program testpio if(writePhase) call GetMaxTime(dt_write_r4, gdt_write_r4(it), MPI_COMM_COMPUTE, ierr) endif if(Debug) print *,'iam: ',My_task,'testpio: point #24' - + if(TestInt) then ! Maximum read/write times if(readPhase) call GetMaxTime(dt_read_i4, gdt_read_i4(it), MPI_COMM_COMPUTE, ierr) @@ -1307,30 +1307,30 @@ program testpio !-------------------------------- - ! Clean up initialization memory + ! Clean up initialization memory ! note: make sure DOFs are not used later !-------------------------------- if (My_task >= 0) call dealloc_check(compDOF) !---------------------------------- - ! Print summary bandwidth statistics + ! Print summary bandwidth statistics !---------------------------------- if(Debug) print *,'iam: ',My_task,'testpio: point #26' if(TestR8 .or. TestCombo .and. (iorank == 0) ) then - call WriteTimeTrialsStats(casename,TestR8CaseName, fname_r8, glenr8, gdt_read_r8, gdt_write_r8, maxiter) + call WriteTimeTrialsStats(casename,TestR8CaseName, fname_r8, glenr8, gdt_read_r8, gdt_write_r8, maxiter) endif if(TestR4 .and. (iorank == 0) ) then - call WriteTimeTrialsStats(casename,TestR4CaseName, fname_r4, glenr4, gdt_read_r4, gdt_write_r4, maxiter) + call WriteTimeTrialsStats(casename,TestR4CaseName, fname_r4, glenr4, gdt_read_r4, gdt_write_r4, maxiter) endif if(TestInt .and. (iorank == 0) ) then - call WriteTimeTrialsStats(casename,TestI4CaseName, fname_i4, gleni4, gdt_read_i4, gdt_write_i4, maxiter) + call WriteTimeTrialsStats(casename,TestI4CaseName, fname_i4, gleni4, gdt_read_i4, gdt_write_i4, maxiter) endif !------------------------------- - ! Print timers and memory usage + ! Print timers and memory usage !------------------------------- #ifdef TIMING @@ -1343,7 +1343,7 @@ program testpio lmem(2) = mrss call mpi_gather(lmem,2,MPI_INTEGER,gmem,2,MPI_INTEGER,0,MPI_COMM_COMPUTE,ierr) call CheckMPIReturn('Call to mpi_gather',ierr,__FILE__,__LINE__) - if (my_task == master_task) then + if (my_task == main_task) then do n = 0,nprocs-1 write(*,'(2a,i8,a,2f10.2)') myname,' my_task=',n,' : (hw, usage) memory (MB) = ',gmem(1,n)*mb_blk,gmem(2,n)*mb_blk enddo @@ -1356,8 +1356,8 @@ program testpio call MPI_Barrier(MPI_COMM_COMPUTE,ierr) -! print *,my_task, master_task - if (my_task == master_task) then +! print *,my_task, main_task + if (my_task == main_task) then print *,' ' print *,'testpio completed successfully' print *,' ' @@ -1451,7 +1451,7 @@ end subroutine WriteStats !============================================================================= - subroutine WriteTimeTrialsStats(casename,TestName, FileName, glen, ReadTimes, WriteTimes, nTrials) + subroutine WriteTimeTrialsStats(casename,TestName, FileName, glen, ReadTimes, WriteTimes, nTrials) implicit none @@ -1660,4 +1660,3 @@ end subroutine check_pioerr !============================================================================= end program testpio - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_bench.pl b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_bench.pl index fe3a223964..a47eada67f 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_bench.pl +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_bench.pl @@ -275,7 +275,7 @@ sub usage{ if($attributes{NETCDF_PATH} =~ /netcdf-4/){ $enablenetcdf4="--enable-netcdf4"; } - } + } } if(defined $suites){ @@ -314,7 +314,7 @@ sub usage{ ldz => 0, partfile => 'null', partdir => 'foo', - iofmt => 'pnc', + iofmt => 'pnc', rearr => 'box', numprocsIO => 10, stride => -1, @@ -435,7 +435,7 @@ sub usage{ $configuration{$name}=$value; } $found = 1; - } + } } #my $suffix = $bname . "-" . $pecount; my $suffix = $bname . "_PE-" . $pecount . "_IO-" . $iofmt . "-" . $numIO; @@ -587,8 +587,8 @@ sub usage{ }elsif(/ENV_(.*)/){ print "set $1 $attributes{$_}\n"; print F "\$ENV{$1}=\"$attributes{$_}\"\;\n"; - } - + } + } @@ -659,7 +659,7 @@ sub usage{ my \@testlist = \"$suffix"; # unlink("../pio/Makefile.conf"); # copy("testpio_in","$tstdir"); # copy the namelist file into test directory - + chdir ("$tstdir"); my \$test; my \$run = "$attributes{run}"; @@ -709,7 +709,7 @@ sub usage{ open(LOG,\$log); my \@logout = <LOG>; close(LOG); - + my \$cnt = grep /testpio completed successfully/ , \@logout; open(T,">TestStatus"); if(\$cnt>0){ @@ -724,7 +724,7 @@ sub usage{ close(T); } }else{ - print "suite \$suite FAILED to configure or build\\n"; + print "suite \$suite FAILED to configure or build\\n"; } } print "test complete on $host \$passcnt tests PASS, \$failcnt tests FAIL\\n"; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_build.pl b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_build.pl index 56faccd69f..a447f381e6 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_build.pl +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_build.pl @@ -47,8 +47,8 @@ }elsif(/ENV_(.*)/){ print "set $1 $attributes{$_}\n"; $ENV{$1}="$attributes{$_}"; - } - + } + } @@ -70,4 +70,3 @@ system('gmake clean') if($clean eq 'yes'); system('gmake'); } - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_run.pl b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_run.pl index d0f8900bf8..6c7b2ab606 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_run.pl +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/testpio_run.pl @@ -79,8 +79,8 @@ sub usage{ # }elsif(/ENV_(.*)/){ # print "set $1 $attributes{$_}\n"; # print F "\$ENV{$1}=\"$attributes{$_}\n\""; -# } - +# } + } if(defined $suites){ @@ -233,7 +233,7 @@ sub usage{ # \$ENV{MP_PROCS} = 1; #system("hostname > $tstdir/hostfile"); #\$ENV{MP_HOSTFILE}="$tstdir/hostfile"; - + # } if("$host" eq "yellowstone_pgi") { \$ENV{LD_PRELOAD}="/opt/ibmhpc/pe1304/ppe.pami/gnu/lib64/pami64/libpami.so"; @@ -242,7 +242,7 @@ sub usage{ if("$host" eq "erebus" or "$host" =~ /^yellowstone/){ # \$ENV{MP_PROCS}=\$saveprocs; # delete \$ENV{MP_HOSTFILE}; - } + } } my \$test; @@ -276,9 +276,9 @@ sub usage{ unlink("testpio") if(-e "testpio"); if($twopass){ - copy("$tstdir/testpio.\$suite","testpio"); + copy("$tstdir/testpio.\$suite","testpio"); }else{ - copy("$tstdir/testpio","testpio"); + copy("$tstdir/testpio","testpio"); } chmod 0755,"testpio"; @@ -316,7 +316,7 @@ sub usage{ open(LOG,\$log); my \@logout = <LOG>; close(LOG); - + my \$cnt = grep /testpio completed successfully/ , \@logout; open(T,">TestStatus"); if(\$cnt>0){ @@ -331,7 +331,7 @@ sub usage{ close(T); } }else{ - print "suite \$suite FAILED to configure or build\\n"; + print "suite \$suite FAILED to configure or build\\n"; } } if($twopass && \$thispass==1){ @@ -341,7 +341,7 @@ sub usage{ print "Run ($script) second pass with \$subsys\n"; }else{ exec(\$subsys); - } + } } print "test complete on $host \$passcnt tests PASS, \$failcnt tests FAIL\\n"; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/utils_mod.F90 b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/utils_mod.F90 index 203064da41..6919390a8e 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/utils_mod.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/utils_mod.F90 @@ -11,7 +11,7 @@ module utils_mod !> !! @private -!! @brief Writes netcdf header information for testpio. +!! @brief Writes netcdf header information for testpio. !! @param File @copydoc file_desc_t !! @param nx !! @param ny diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/wstest.c b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/wstest.c index de4f05dcd5..c2fa962855 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/wstest.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/basic/wstest.c @@ -27,7 +27,7 @@ int main(int argc, char *argv[]) PIOc_Init_Intracomm(MPI_COMM_WORLD, npe, 1, 0, PIO_REARR_SUBSET,&iosysid); - // Create a weak scaling test - + // Create a weak scaling test - nx=6; ny=6; nz=2; @@ -52,15 +52,15 @@ int main(int argc, char *argv[]) PIOc_createfile(iosysid, &ncid, &iotype, "wstest.nc", PIO_CLOBBER); // Order of dims in c is slowest first - PIOc_def_dim(ncid, "nx", (PIO_Offset) gdim[2], dimids+2); - PIOc_def_dim(ncid, "ny", (PIO_Offset) gdim[1], dimids+1); + PIOc_def_dim(ncid, "nx", (PIO_Offset) gdim[2], dimids+2); + PIOc_def_dim(ncid, "ny", (PIO_Offset) gdim[1], dimids+1); PIOc_def_dim(ncid, "nz", (PIO_Offset) gdim[0], dimids); PIOc_def_var(ncid, "idof", PIO_INT, 3, dimids, &vid); - + PIOc_enddef(ncid); - + PIOc_write_darray(ncid, vid, iodesc,(PIO_Offset) (nx*ny*nz), iarray, NULL); diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/c/CMakeLists.txt b/src/Infrastructure/IO/PIO/ParallelIO/examples/c/CMakeLists.txt index 4e5576ea66..7edfa4d6f4 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/c/CMakeLists.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/c/CMakeLists.txt @@ -29,6 +29,10 @@ ADD_EXECUTABLE(examplePio EXCLUDE_FROM_ALL examplePio.c) TARGET_LINK_LIBRARIES(examplePio pioc) add_dependencies(tests examplePio) +ADD_EXECUTABLE(piorcw piorcw.c) +TARGET_LINK_LIBRARIES(piorcw pioc) +add_dependencies(tests piorcw) + ADD_EXECUTABLE(example1 example1.c) TARGET_LINK_LIBRARIES(example1 pioc) add_dependencies(tests example1) @@ -47,6 +51,7 @@ if (PIO_USE_MPISERIAL) else () add_mpi_test(examplePio EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/examplePio NUMPROCS 4 TIMEOUT 60) add_mpi_test(example1 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/example1 NUMPROCS 4 TIMEOUT 60) + #add_mpi_test(piorcw EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/piorcw NUMPROCS 16 TIMEOUT 600) #add_mpi_test(darray_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_async NUMPROCS 5 TIMEOUT 60) add_mpi_test(darray_no_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_no_async NUMPROCS 4 TIMEOUT 60) endif () diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/c/darray_async.c b/src/Infrastructure/IO/PIO/ParallelIO/examples/c/darray_async.c index ea5f62ac2d..1186f998c6 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/c/darray_async.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/c/darray_async.c @@ -64,7 +64,7 @@ /* Number of computation components. */ #define COMPONENT_COUNT 1 - + /* Lengths of dimensions. */ int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y}; @@ -114,7 +114,7 @@ char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"}; /* { */ /* char my_dim_name[PIO_MAX_NAME]; */ /* PIO_Offset dimlen; */ - + /* if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen))) */ /* return ret; */ /* if (dimlen != (d ? dim_len[d] : NUM_TIMESTEPS) || strcmp(my_dim_name, dim_name[d])) */ @@ -136,7 +136,7 @@ char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"}; /* for (int t = 0; t < NUM_TIMESTEPS; t++) */ /* { */ /* int varid = 0; /\* There's only one var in sample file. *\/ */ - + /* /\* This is the data we expect for this timestep. *\/ */ /* for (int i = 0; i < elements_per_pe; i++) */ /* buffer[i] = 100 * t + START_DATA_VAL + my_rank; */ @@ -207,7 +207,7 @@ int main(int argc, char* argv[]) if ((ret = GPTLinitialize ())) return ret; #endif - + /* Initialize MPI. */ if ((ret = MPI_Init(&argc, &argv))) MPIERR(ret); @@ -239,7 +239,7 @@ int main(int argc, char* argv[]) /* Num procs for computation. */ int num_procs2[COMPONENT_COUNT] = {4}; - + /* Is the current process a computation task? */ int comp_task = my_rank < NUM_IO_TASKS ? 0 : 1; @@ -258,7 +258,7 @@ int main(int argc, char* argv[]) { /* PIO_Offset elements_per_pe; /\* Array elements per processing unit. *\/ */ /* int ioid; /\* The I/O description ID. *\/ */ - + /* /\* How many elements on each computation task? *\/ */ /* elements_per_pe = DIM_LEN_X * DIM_LEN_Y / NUM_COMP_TASKS; */ @@ -318,7 +318,7 @@ int main(int argc, char* argv[]) /* /\* Create some data for this timestep. *\/ */ /* for (int i = 0; i < elements_per_pe; i++) */ /* buffer[i] = 100 * t + START_DATA_VAL + my_rank; */ - + /* /\* Write data to the file. *\/ */ /* printf("rank: %d Writing sample data...\n", my_rank); */ /* if ((ret = PIOc_setframe(ncid, varid, t))) */ diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/c/examplePio.c b/src/Infrastructure/IO/PIO/ParallelIO/examples/c/examplePio.c index 4f678815ba..3ec01748ab 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/c/examplePio.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/c/examplePio.c @@ -1,10 +1,10 @@ /** - * @file + * @file * @brief A simple C example for the ParallelIO Library. * * This example creates a netCDF output file with one dimension and * one variable. It first writes, then reads the sample file using the - * ParallelIO library. + * ParallelIO library. * * This example can be run in parallel for 1, 2, 4, 8, or 16 * processors. @@ -65,7 +65,7 @@ typedef struct examplePioClass /** Pointer to function that cleans up example memory, and library resources. */ struct examplePioClass* (*cleanUp) (struct examplePioClass*); - + /** Pointer to function that handles errors. */ struct examplePioClass* (*errorHandler) (struct examplePioClass*, const char*, const int); @@ -136,12 +136,12 @@ typedef struct examplePioClass PIO_Offset *compdof; /** The example file name. */ - char *fileName; - + char *fileName; + } examplePioClass; -/** @brief Initialize libraries, create sample data. - +/** @brief Initialize libraries, create sample data. + This function is called as part of the creation of a sample data file for this example. @@ -175,7 +175,7 @@ struct examplePioClass* epc_init( struct examplePioClass* this ) /* ** initialize MPI */ - + MPI_Init(NULL, NULL); MPI_Comm_rank(MPI_COMM_WORLD, &this->myRank); MPI_Comm_size(MPI_COMM_WORLD, &this->ntasks); @@ -198,35 +198,35 @@ struct examplePioClass* epc_init( struct examplePioClass* this ) /* ** set up PIO for rest of example */ - + this->stride = 1; this->numAggregator = 0; this->optBase = 0; this->iotype = PIO_IOTYPE_NETCDF; this->fileName = "examplePio_c.nc"; this->dimLen[0] = LEN; - + this->niotasks = this->ntasks; /* keep things simple - 1 iotask per MPI process */ - + if (this->myRank == 0){ printf("Running with %d MPI processes and %d PIO processes. \n",this->ntasks,this->niotasks); } - + PIOc_Init_Intracomm(MPI_COMM_WORLD, this->niotasks, this->stride, this->optBase, PIO_REARR_SUBSET, &this->pioIoSystem); - + /* ** set up some data that we will write to a netcdf file */ - + this->arrIdxPerPe = LEN / this->ntasks; - + if (this->arrIdxPerPe < 1) { this->errorHandler(this, "Not enough work to distribute among pes",ERR_CODE); } - + this->ista = this->myRank * this->arrIdxPerPe; this->isto = this->ista + (this->arrIdxPerPe - 1); - + this->dataBuffer = (int *)malloc(this->arrIdxPerPe * sizeof (int)); this->readBuffer = (int *)malloc(this->arrIdxPerPe * sizeof (int)); this->compdof = (PIO_Offset *)malloc(this->arrIdxPerPe * sizeof(PIO_Offset)); @@ -234,14 +234,14 @@ struct examplePioClass* epc_init( struct examplePioClass* this ) /* ** assign values to various arrays */ - + localVal = this->ista; for (i = 0; i < this->arrIdxPerPe; i++ ){ - + this->dataBuffer[i] = this->myRank + VAL; this->compdof[i] = localVal + 1; this->readBuffer[i] = 99; - + if (localVal > this->isto) { printf("error, should ABORT \n"); } @@ -331,11 +331,11 @@ struct examplePioClass* epc_createDecomp( struct examplePioClass* this ) Uses the function PIOc_createfile() to create the netCDF output file. The format of the file is created in accordance with the iotype member variable, which specifies one of the following - values: + values: - - PIO_IOTYPE_PNETCDF=1 Parallel Netcdf (parallel) - - PIO_IOTYPE_NETCDF=2 Netcdf3 Classic format (serial) - - PIO_IOTYPE_NETCDF4C=3 NetCDF4 (HDF5) compressed format (serial) + - PIO_IOTYPE_PNETCDF=1 Parallel Netcdf (parallel) + - PIO_IOTYPE_NETCDF=2 Netcdf3 Classic format (serial) + - PIO_IOTYPE_NETCDF4C=3 NetCDF4 (HDF5) compressed format (serial) - PIO_IOTYPE_NETCDF4P=4 NetCDF4 (HDF5) parallel The PIOc_createfile() function has the following parameters: @@ -346,7 +346,7 @@ struct examplePioClass* epc_createDecomp( struct examplePioClass* this ) - the name of the sample file. - the NetCDF file creating mode, PIO_CLOBBER means overwrite any existing file with this name. - + @param [in] this Pointer to self. @retval examplePioClass* Pointer to self. */ @@ -360,7 +360,7 @@ struct examplePioClass* epc_createFile( struct examplePioClass* this ) } /** @brief Define netCDF metadata. - + This function is called as part of the creation of a sample data file for this example. @@ -370,7 +370,7 @@ struct examplePioClass* epc_createFile( struct examplePioClass* this ) All of the functions take the pioFileDesc returned by PIOc_createfile(). This is the ncid of the netCDF file. - + @param [in] this Pointer to self. @retval examplePioClass* Pointer to self. */ @@ -393,7 +393,7 @@ struct examplePioClass* epc_defineVar( struct examplePioClass* this ) The data are written with the PIOc_write_darray() function. After the write is complete, ensure the file is synced for all processes after the write. - + @param [in] this Pointer to self. @retval examplePioClass* Pointer to self. */ @@ -404,7 +404,7 @@ struct examplePioClass* epc_writeVar( struct examplePioClass* this ) PIOc_write_darray(this->pioFileDesc, this->pioVar, this->iodescNCells, (PIO_Offset)this->arrIdxPerPe, this->dataBuffer, NULL); PIOc_sync(this->pioFileDesc); - + return this; } @@ -415,14 +415,14 @@ struct examplePioClass* epc_writeVar( struct examplePioClass* this ) This function reads the data that has been written to the sample data file. The data are read with the PIOc_read_darray() function. - + @param [in] this Pointer to self. @retval examplePioClass* Pointer to self. */ struct examplePioClass* epc_readVar( struct examplePioClass* this ) { int i; - + PIOc_read_darray(this->pioFileDesc, this->pioVar, this->iodescNCells, (PIO_Offset)this->arrIdxPerPe, this->readBuffer); @@ -432,7 +432,7 @@ struct examplePioClass* epc_readVar( struct examplePioClass* this ) this->errorHandler(this, "The data was not what was expected!", ERR_CODE); if (this->verbose) printf("rank: %d Data read matches expected data.\n", this->myRank); - + return this; } @@ -449,7 +449,7 @@ struct examplePioClass* epc_closeFile( struct examplePioClass* this ) if (this->verbose) printf("rank: %d Closing the sample data file...\n", this->myRank); PIOc_closefile(this->pioFileDesc); - + return this; } @@ -459,7 +459,7 @@ struct examplePioClass* epc_closeFile( struct examplePioClass* this ) ParallelIO library function PIOc_freedecomp() to free decomposition resources. Then calles PIOc_free_iosystem() and MPI_finalize() to free library resources. - + @param [in] this Pointer to self. @retval examplePioClass* Pointer to self. */ @@ -470,12 +470,12 @@ struct examplePioClass* epc_cleanUp( struct examplePioClass* this ) free(this->dataBuffer); free(this->readBuffer); free(this->compdof); - + PIOc_freedecomp(this->pioIoSystem, this->iodescNCells); PIOc_free_iosystem(this->pioIoSystem); MPI_Finalize(); - + return this; } @@ -484,7 +484,7 @@ struct examplePioClass* epc_cleanUp( struct examplePioClass* this ) On error, process with rank zero will print error message, the netCDF file will be closed with PIOc_closefile(), and MPI_Abort is called to end the example execution on all processes. - + @param [in] this Pointer to self. @param [in] errMsg an error message @param [in] retVal the non-zero return value that indicated an error @@ -495,17 +495,17 @@ struct examplePioClass* epc_errorHandler(struct examplePioClass* this, const cha /* class(pioExampleClass), intent(inout) :: this character(len=*), intent(in) :: errMsg integer, intent(in) :: retVal*/ - + if (retVal != PIO_NOERR){ - + if (this->myRank == 0){ printf("%d %s\n",retVal,errMsg); } - + PIOc_closefile(this->pioFileDesc); MPI_Abort(MPI_COMM_WORLD, retVal); } - + return this; } @@ -514,16 +514,16 @@ struct examplePioClass* epc_errorHandler(struct examplePioClass* this, const cha This function allocates memory for the struct that contains the code and data for this example. Then pointers are to the functions used in the example. - + @param [in] verbose Non-zero for output to stdout. @retval examplePioClass* Pointer to self. */ struct examplePioClass* epc_new(int verbose) { struct examplePioClass* this = malloc((sizeof(struct examplePioClass))); - + /* assign function pointers to impls */ - + this->init = epc_init; this->createDecomp = epc_createDecomp; this->createFile = epc_createFile; @@ -534,7 +534,7 @@ struct examplePioClass* epc_new(int verbose) this->cleanUp = epc_cleanUp; this->errorHandler = epc_errorHandler; this->verbose = verbose; - + return this; } @@ -571,7 +571,7 @@ struct examplePioClass* epc_new(int verbose) foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45 ; } </pre> - + @param [in] argc argument count (should be zero) @param [in] argv argument array (should be NULL) @retval examplePioClass* Pointer to self. @@ -597,12 +597,12 @@ int main(int argc, char* argv[]) return ret; struct examplePioClass* pioExInst = epc_new(verbose); - -#ifdef TIMING + +#ifdef TIMING /* Initialize the GPTL timing library. */ if ((ret = GPTLinitialize ())) return ret; -#endif +#endif pioExInst->init(pioExInst); pioExInst->createDecomp(pioExInst); @@ -618,12 +618,12 @@ int main(int argc, char* argv[]) /* #endif /\* USE_MPE *\/ */ pioExInst->cleanUp(pioExInst); - -#ifdef TIMING + +#ifdef TIMING /* Finalize the GPTL timing library. */ if ((ret = GPTLfinalize ())) return ret; -#endif +#endif free(pioExInst); return 0; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/c/piorcw.c b/src/Infrastructure/IO/PIO/ParallelIO/examples/c/piorcw.c new file mode 100644 index 0000000000..cc04c25046 --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/c/piorcw.c @@ -0,0 +1,245 @@ +#include <config.h> +#include <mpi.h> +#include <pio.h> +#include <pio_internal.h> + +static int debug = 0; + +double *dvarw, *dvarr; +float *fvarw, *fvarr; +int *ivarw, *ivarr; +char *cvarw, *cvarr; + +typedef struct dimlist +{ + char name[16]; + PIO_Offset value; +} dimlist; + + + +int rcw_write_darray(int iosys, int rank) +{ + int ierr; + int comm_size; + int ncid; + int iotype = PIO_IOTYPE_NETCDF4P; + int ndims; + int *global_dimlen; + int num_tasks; + int *maplen; + int maxmaplen; + int *full_map; + int *dimid; + int varid; + int globalsize; + int ioid; + char dimname[PIO_MAX_NAME]; + char varname[PIO_MAX_NAME]; + + + ierr = MPI_Comm_size(MPI_COMM_WORLD, &comm_size); + ierr = PIOc_createfile(iosys, &ncid, &iotype, "testfile.nc4", PIO_CLOBBER); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + dimid = calloc(ndims,sizeof(int)); + for(int i=0; i<ndims; i++) + { + sprintf(dimname,"dim%4.4d",i); + ierr = PIOc_def_dim(ncid, dimname, (PIO_Offset) global_dimlen[i], &(dimid[i])); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + } + ierr = PIOc_def_var(ncid, varname, PIO_DOUBLE, ndims, dimid, &varid); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + free(dimid); + ierr = PIOc_enddef(ncid); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + PIO_Offset *dofmap; + + if (!(dofmap = malloc(sizeof(PIO_Offset) * maplen[rank]))) + return PIO_ENOMEM; + + /* Copy array into PIO_Offset array. */ + dvarw = malloc(sizeof(double)*maplen[rank]); + for (int e = 0; e < maplen[rank]; e++) + { + dofmap[e] = full_map[rank * maxmaplen + e]+1; + dvarw[e] = dofmap[e]; + } + /* allocated in pioc_read_nc_decomp_int */ + free(full_map); + ierr = PIOc_InitDecomp(iosys, PIO_DOUBLE, ndims, global_dimlen, maplen[rank], + dofmap, &ioid, NULL, NULL, NULL); + + free(global_dimlen); + double dsum=0; + for(int i=0; i < maplen[rank]; i++) + dsum += dvarw[i]; + if(dsum != rank) + printf("%d: dvarwsum = %g\n",rank, dsum); + + ierr = PIOc_write_darray(ncid, varid, ioid, maplen[rank], dvarw, NULL); + free(maplen); + ierr = PIOc_closefile(ncid); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + return ierr; +} + + +int rcw_read_darray(int iosys,int rank) +{ + int ierr; + int comm_size; + int ncid; + int iotype = PIO_IOTYPE_PNETCDF; + int ndims; + int *global_dimlen; + int num_tasks; + int *maplen; + int maxmaplen; + int *full_map; + int *dimid; + int varid; + int globalsize; + int ioid; + int pio_type; + char dimname[PIO_MAX_NAME]; + char varname[PIO_MAX_NAME]; + + int i, nvars, natts, unlimdim; + dimlist *dim; + + ierr = MPI_Comm_size(MPI_COMM_WORLD, &comm_size); + + ierr = PIOc_openfile(iosys, &ncid, &iotype, "testfile.nc", PIO_NOWRITE); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + ierr = PIOc_inq(ncid, &ndims, &nvars, &natts, &unlimdim); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + + dim = (dimlist *) malloc(ndims*sizeof(dimlist)); + for(i=0; i<ndims; i++){ + ierr = PIOc_inq_dim(ncid, i, dim[i].name, &(dim[i].value)); + if(ierr || debug) printf("%d %d i=%d\n",__LINE__,ierr, i); + } + + + + + /* TODO: support multiple variables and types*/ + sprintf(varname,"var%4.4d",0); + ierr = PIOc_inq_varid(ncid, varname, &varid); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + ierr = PIOc_inq_varndims(ncid, varid, &ndims); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + ierr = PIOc_inq_vartype(ncid, varid, &pio_type); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + dimid = calloc(ndims,sizeof(int)); + ierr = PIOc_inq_vardimid(ncid, varid, dimid); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + for(int i=0; i<ndims; i++) + { + PIO_Offset gdimlen; + ierr = PIOc_inq_dimlen(ncid, dimid[i], &gdimlen); + + pioassert(gdimlen == global_dimlen[i], "testfile.nc does not match decomposition file",__FILE__,__LINE__); + } + free(dimid); + PIO_Offset *dofmap; + + if (!(dofmap = malloc(sizeof(PIO_Offset) * maplen[rank]))) + return PIO_ENOMEM; + + for (int e = 0; e < maplen[rank]; e++) + { + dofmap[e] = full_map[rank * maxmaplen + e] + 1; + } + free(full_map); +// PIOc_set_log_level(3); + ierr = PIOc_InitDecomp(iosys, pio_type, ndims, global_dimlen, maplen[rank], + dofmap, &ioid, NULL, NULL, NULL); + free(dofmap); + free(global_dimlen); + switch(pio_type) + { + case PIO_DOUBLE: + dvarr = malloc(sizeof(double)*maplen[rank]); + ierr = PIOc_read_darray(ncid, varid, ioid, maplen[rank], dvarr); + double dsum=0; + for(int i=0; i < maplen[rank]; i++) + dsum += dvarr[i]; + if(dsum != rank) + printf("%d: dsum = %g\n",rank, dsum); + break; + case PIO_INT: + ivarr = malloc(sizeof(int)*maplen[rank]); + ierr = PIOc_read_darray(ncid, varid, ioid, maplen[rank], ivarr); + int isum=0; + for(int i=0; i < maplen[rank]; i++) + isum += ivarr[i]; + printf("%d: isum = %d\n",rank, isum); + break; + case PIO_FLOAT: + fvarr = malloc(sizeof(float)*maplen[rank]); + ierr = PIOc_read_darray(ncid, varid, ioid, maplen[rank], fvarr); + float fsum=0; + for(int i=0; i < maplen[rank]; i++) + fsum += fvarr[i]; + printf("%d: fsum = %f\n",rank, fsum); + break; + case PIO_BYTE: + cvarr = malloc(sizeof(char)*maplen[rank]); + ierr = PIOc_read_darray(ncid, varid, ioid, maplen[rank], cvarr); + int csum=0; + for(int i=0; i < maplen[rank]; i++) + csum += (int) cvarr[i]; + printf("%d: csum = %d\n",rank, csum); + break; + } + + ierr = PIOc_closefile(ncid); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + + free(maplen); + return ierr; + +} + + + +int main(int argc, char *argv[]) +{ + int ierr; + int rank; + int comm_size; + int iosys; + int iotasks; + MPI_Init(&argc, &argv); + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + MPI_Comm_size(MPI_COMM_WORLD, &comm_size); + + iotasks = comm_size/36; + + ierr = PIOc_Init_Intracomm(MPI_COMM_WORLD, iotasks, 36, 0, PIO_REARR_SUBSET, &iosys); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + ierr = rcw_read_darray(iosys, rank); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + ierr = rcw_write_darray(iosys, rank); + if(ierr || debug) printf("%d %d\n",__LINE__,ierr); + + MPI_Finalize(); + +} diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/c/valsupp_example1.supp b/src/Infrastructure/IO/PIO/ParallelIO/examples/c/valsupp_example1.supp index 63f3e07383..ace46e4738 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/c/valsupp_example1.supp +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/c/valsupp_example1.supp @@ -12,4 +12,4 @@ fun:flush_buffer fun:PIOc_sync fun:main -} \ No newline at end of file +} diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/cxx/examplePio.cxx b/src/Infrastructure/IO/PIO/ParallelIO/examples/cxx/examplePio.cxx index b4f03cc13e..73cdd0cab8 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/cxx/examplePio.cxx +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/cxx/examplePio.cxx @@ -13,26 +13,26 @@ class pioExampleClass { pioExampleClass::pioExampleClass(){ // user defined ctor with no arguments - + cout << " pioExampleClass::pioExampleClass() "<< endl; - + } void pioExampleClass::init () { - + cout << " pioExampleClass::init() " << endl; - + } int main () { - + pioExampleClass *pioExInst; - + pioExInst = new pioExampleClass(); - + pioExInst->init(); delete(pioExInst); - + return 0; -} \ No newline at end of file +} diff --git a/src/Infrastructure/IO/PIO/ParallelIO/examples/f03/exampleAsyncPio.F90 b/src/Infrastructure/IO/PIO/ParallelIO/examples/f03/exampleAsyncPio.F90 index 180e3cd794..e8a6f629e6 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/examples/f03/exampleAsyncPio.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/examples/f03/exampleAsyncPio.F90 @@ -235,7 +235,6 @@ subroutine createDecomp(this) implicit none class(pioExampleClass), intent(inout) :: this - integer :: ierr call PIO_initdecomp(this%pioIoSystem(1), PIO_int, this%dimLen, this%compdof(this%ista:this%isto), & this%iodescNCells) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/libpio.settings.in b/src/Infrastructure/IO/PIO/ParallelIO/libpio.settings.in index c3e0904585..444a9e1cbc 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/libpio.settings.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/libpio.settings.in @@ -28,11 +28,10 @@ More Fortran Flags: @FPPFLAGS@ -------- PnetCDF Support: @HAS_PNETCDF@ SZIP Write Support: @HAS_SZIP_WRITE@ -Parallel Filters: @HAS_PAR_FILTERS@ +Parallel Filters: @have_par_filters@ NetCDF/HDF5 Support: @HAS_NETCDF4@ NetCDF/HDF5 Par I/O: @HAS_NETCDF4_PAR@ NetCDF Integration: @HAS_NETCDF_INTEGRATION@ PIO Logging: @HAS_LOGGING@ MPIEXEC: @WITH_MPIEXEC@ Fortran: @HAS_PIO_FORTRAN@ - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/scripts/prune_decomps.pl b/src/Infrastructure/IO/PIO/ParallelIO/scripts/prune_decomps.pl index 58e4f66f0f..52f2781acb 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/scripts/prune_decomps.pl +++ b/src/Infrastructure/IO/PIO/ParallelIO/scripts/prune_decomps.pl @@ -18,7 +18,7 @@ sub rem_dup_decomp_files { my($dirname) = @_; # Find files in current directory that are - # named *piodecomp* - these are the pio + # named *piodecomp* - these are the pio # decomposition files opendir(F,$dirname); #my @decompfiles = grep(/^piodecomp/,readdir(F)); @@ -30,7 +30,7 @@ sub rem_dup_decomp_files #for(my $i=0; $i<$ndecompfile_info; $i++){ # print "File : $decompfile_info[$i]->{FNAME} , size = $decompfile_info[$i]->{SIZE}\n"; #} - + my $rmfile=0; # Compare the decomposition files to find # duplicates - and delete the dups @@ -56,7 +56,7 @@ sub rem_dup_decomp_files my $nline = shift (@file2); # Ignore stack traces when comparing files # The stack traces start with a line containing - # "Obtained" + # "Obtained" # Also, stack trace is the last line being # compared if(($line =~ /${BEGIN_STACK_TRACE}/) @@ -67,7 +67,7 @@ sub rem_dup_decomp_files last; } next if($line eq $nline); - # Files are different, don't remove + # Files are different, don't remove $rmfile = 0; last; } @@ -158,5 +158,3 @@ () if($verbose){ print "Decoding stack traces for decomposition files from : \"", $rundir, "\"\n"; } &decode_stack_traces($rundir, $exe); } - - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/CMakeLists.txt b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/CMakeLists.txt index e1f6a13bd7..609de6f575 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/CMakeLists.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/CMakeLists.txt @@ -9,8 +9,8 @@ project (PIOC C) if (CMAKE_BUILD_TYPE) define_property( SOURCE - PROPERTY COMPILE_FLAGS - INHERITED + PROPERTY COMPILE_FLAGS + INHERITED BRIEF_DOCS "brief-doc" FULL_DOCS "full-doc" ) @@ -34,10 +34,10 @@ add_library (pioc ${src}) # Always use -fPIC set_property(TARGET pioc PROPERTY POSITION_INDEPENDENT_CODE ON) -set_source_files_properties( - pioc_async.c +set_source_files_properties( + pioc_async.c PROPERTIES - COMPILE_FLAGS -O0 + COMPILE_FLAGS -O0 ) @@ -63,7 +63,7 @@ target_compile_definitions (pioc # Compiler-specific compiler options if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - string(APPEND CMAKE_C_FLAGS " -std=c99 " ) + string(APPEND CMAKE_C_FLAGS " -std=gnu99 " ) elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI") string(APPEND CMAKE_C_FLAGS " -c99 ") elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/parallel_sort.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/parallel_sort.c index 76b938c64d..7757106a4f 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/parallel_sort.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/parallel_sort.c @@ -1,5 +1,5 @@ /** - * c.f.: https://raw.githubusercontent.com/rabauke/mpl/master/examples/parallel_sort_mpi.c + * c.f.: https://raw.githubusercontent.com/rabauke/mpl/main/examples/parallel_sort_mpi.c * parallel sort algorithm for distributed memory computers * * algorithm works as follows: @@ -96,8 +96,8 @@ bool is_unique(CVector v) { * @param comm the MPI communicator over which v is distributed * @param v A CVector distributed over comm * @param ierr indicates an error was encountered - * @return A CVector sorted over comm, the size of the new vector may be different - * than v with a worst case of the entire result on one task. + * @return A CVector sorted over comm, the size of the new vector may be different + * than v with a worst case of the entire result on one task. */ CVector parallel_sort(MPI_Comm comm, CVector v, int *ierr) { @@ -130,7 +130,7 @@ CVector parallel_sort(MPI_Comm comm, CVector v, int *ierr) { if(!(pivot_pos = malloc((size + 1) * sizeof(*pivot_pos)))) *ierr = pio_err(NULL, NULL, PIO_ENOMEM, __FILE__,__LINE__); - + pivot_pos[0] = v.data; for (size_t i = 0; i < size - 1; ++i) pivot_pos[i + 1] = partition(pivot_pos[i], v.data + v.N, local_pivots[i]); @@ -189,15 +189,15 @@ CVector parallel_sort(MPI_Comm comm, CVector v, int *ierr) { */ int run_unique_check(MPI_Comm comm, size_t N,datatype *v, bool *has_dups) { - int rank, size, i, r; - int mpierr=MPI_SUCCESS, mpierr2; + int rank, size; + int mpierr=MPI_SUCCESS; int ierr; if ((mpierr = MPI_Comm_rank(comm, &rank))) - check_mpi(NULL, NULL, mpierr2, __FILE__, __LINE__); + check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Comm_size(comm, &size))) - check_mpi(NULL, NULL, mpierr2, __FILE__, __LINE__); - + check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); + srand(time(NULL) * rank); CVector sorted = parallel_sort(comm, (CVector){v, N}, &ierr); @@ -205,13 +205,13 @@ int run_unique_check(MPI_Comm comm, size_t N,datatype *v, bool *has_dups) int i_have_dups = is_unique(sorted) ? 0:1; int global_dups; if ((mpierr = MPI_Allreduce(&i_have_dups, &global_dups, 1, MPI_INT, MPI_MAX, comm))) - check_mpi(NULL, NULL, mpierr2, __FILE__, __LINE__); - + check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); + if(global_dups > 0) *has_dups = true; else *has_dups = false; - + #ifdef DEBUG_PARALLEL_SORT for (r=0; r<size; r++) { diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio.h b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio.h index 3e3c0d6b31..a4b20b7522 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio.h +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio.h @@ -17,6 +17,16 @@ #include <uthash.h> #include <netcdf.h> +#include <netcdf_meta.h> + +#define NETCDF_VERSION_LE(Maj, Min, Pat) \ + (((NC_VERSION_MAJOR == Maj) && (NC_VERSION_MINOR == Min) && (NC_VERSION_PATCH <= Pat)) || \ + ((NC_VERSION_MAJOR == Maj) && (NC_VERSION_MINOR < Min)) || (NC_VERSION_MAJOR < Maj)) + +#define NETCDF_VERSION_GE(Maj, Min, Pat) \ + (((NC_VERSION_MAJOR == Maj) && (NC_VERSION_MINOR == Min) && (NC_VERSION_PATCH >= Pat)) || \ + ((NC_VERSION_MAJOR == Maj) && (NC_VERSION_MINOR > Min)) || (NC_VERSION_MAJOR > Maj)) + /** PIO_OFFSET is an integer type of size sufficient to represent the * size (in bytes) of the largest file supported by MPI. This is not @@ -114,7 +124,7 @@ typedef struct var_desc_t int record; /** ID of each outstanding pnetcdf request for this variable. */ - int *request; +// int *request; /** Number of requests pending with pnetcdf. */ int nreqs; @@ -306,7 +316,7 @@ typedef struct io_desc_t * sort it. */ bool needssort; - /** If the decomp has repeated values it can only be used for reading + /** If the decomp has repeated values it can only be used for reading since it doesn't make sense to write a single value from more than one location. */ bool readonly; @@ -450,13 +460,13 @@ typedef struct iosystem_desc_t * process is not part of the IO communicator. */ int io_rank; - /** Set to MPI_ROOT if this task is the master of IO communicator, 0 + /** Set to MPI_ROOT if this task is the main of IO communicator, 0 * otherwise. */ - int iomaster; + int iomain; - /** Set to MPI_ROOT if this task is the master of comp communicator, 0 + /** Set to MPI_ROOT if this task is the main of comp communicator, 0 * otherwise. */ - int compmaster; + int compmain; /** Rank of IO root task (which is rank 0 in io_comm) in the union * communicator. */ @@ -677,6 +687,13 @@ enum PIO_ERROR_HANDLERS #define PIO_64BIT_OFFSET NC_64BIT_OFFSET /**< Use large (64-bit) file offsets. Mode flag for nc_create(). */ #define PIO_64BIT_DATA NC_64BIT_DATA /**< CDF5 format. */ +#ifdef NC_HAS_QUANTIZE +#define PIO_NOQUANTIZE NC_NOQUANTIZE +#define PIO_QUANTIZE_BITGROOM NC_QUANTIZE_BITGROOM +#define PIO_QUANTIZE_GRANULARBR NC_QUANTIZE_GRANULARBR +#define PIO_QUANTIZE_BITROUND NC_QUANTIZE_BITROUND /**< Use BitRound quantization. */ +#endif + /** Define the netCDF-based error codes. */ #define PIO_NOERR NC_NOERR /**< No Error */ #define PIO_EBADID NC_EBADID /**< Bad ncid */ @@ -767,7 +784,7 @@ enum PIO_ERROR_HANDLERS #define PIO_FILL_UINT64 NC_FILL_UINT64 /**< Default fill value for this type. */ #define PIO_EINDEP (-203) /**< independent access error. */ - +#define PIO_EINSUFFBUF (-219) /**< Insufficient buffer size (pnetcdf only) */ #define PIO_FIRST_ERROR_CODE (-500) /**< The first error code for PIO. */ #define PIO_EBADIOTYPE (-500) /**< Bad IOTYPE error. */ #define PIO_EVARDIMMISMATCH (-501) /**< Variable dimensions do not match in a multivar call. */ @@ -946,6 +963,13 @@ extern "C" { int PIOc_set_fill(int ncid, int fillmode, int *old_modep); int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value); int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep); +#ifdef NC_HAS_BZ2 + int PIOc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp); +#endif +#ifdef NC_HAS_ZSTD + int PIOc_def_var_zstandard(int ncid, int varid, int level); + int PIOc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp); +#endif int PIOc_rename_var(int ncid, int varid, const char *name); /* These variable settings only apply to netCDF-4 files. */ @@ -1244,6 +1268,17 @@ extern "C" { int PIOc_put_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum, const unsigned long long *op); +#ifdef NC_HAS_PAR_FILTERS + int PIOc_def_var_filter(int ncid, int varid,unsigned int id, size_t nparams, unsigned int *params); + int PIOc_inq_var_filter_ids(int ncid, int varid, size_t *nfiltersp, unsigned int *ids); + int PIOc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t *nparamsp, unsigned int *params ); + int PIOc_inq_filter_avail(int ncid, unsigned int id ); +#endif +#ifdef NC_HAS_QUANTIZE + int PIOc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd ); + int PIOc_inq_var_quantize(int ncid, int varid, int *quantize_mode, int *nsdp ); +#endif + /* These functions are for the netCDF integration layer. */ int nc_def_iosystem(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr, int *iosysidp); diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_darray.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_darray.c index eb3066e01a..e7274da3e0 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_darray.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_darray.c @@ -208,35 +208,35 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, char fillvalue_present = fillvalue ? true : false; /* Is fillvalue non-NULL? */ int flushtodisk_int = flushtodisk; /* Need this to be int not boolean. */ - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); /* Send the function parameters and associated informaiton * to the msg handler. */ if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&nvars, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&nvars, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)varids, nvars, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)varids, nvars, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ioid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ioid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&arraylen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&arraylen, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(array, arraylen * iodesc->piotype_size, MPI_CHAR, ios->compmaster, + mpierr = MPI_Bcast(array, arraylen * iodesc->piotype_size, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&frame_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&frame_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr && frame_present) - mpierr = MPI_Bcast((void *)frame, nvars, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)frame, nvars, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&fillvalue_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&fillvalue_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr && fillvalue_present) mpierr = MPI_Bcast((void *)fillvalue, nvars * iodesc->piotype_size, MPI_CHAR, - ios->compmaster, ios->intercomm); + ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&flushtodisk_int, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&flushtodisk_int, 1, MPI_INT, ios->compmain, ios->intercomm); PLOG((2, "PIOc_write_darray_multi file->pio_ncid = %d nvars = %d ioid = %d arraylen = %d " "frame_present = %d fillvalue_present = %d flushtodisk = %d", file->pio_ncid, nvars, ioid, arraylen, frame_present, fillvalue_present, flushtodisk)); @@ -263,14 +263,14 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, /* Determine total size of aggregated data (all vars/records). * For netcdf serial writes we collect the data on io nodes and - * then move that data one node at a time to the io master node + * then move that data one node at a time to the io main node * and write (or read). The buffer size on io task 0 must be as * large as the largest used to accommodate this serial io * method. */ rlen = 0; if (iodesc->llen > 0 || ((file->iotype == PIO_IOTYPE_NETCDF || - file->iotype == PIO_IOTYPE_NETCDF4C) && ios->iomaster)) + file->iotype == PIO_IOTYPE_NETCDF4C) && ios->iomain)) rlen = iodesc->maxiobuflen * nvars; /* Allocate iobuf. */ @@ -895,19 +895,19 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, { int msg = PIO_MSG_READDARRAY; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); /* Send the function parameters and associated informaiton * to the msg handler. */ if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ioid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ioid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&arraylen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&arraylen, 1, MPI_OFFSET, ios->compmain, ios->intercomm); PLOG((2, "PIOc_read_darray ncid %d varid %d ioid %d arraylen %d", ncid, varid, ioid, arraylen)); } @@ -925,8 +925,8 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, pioassert(iodesc->rearranger == PIO_REARR_BOX || iodesc->rearranger == PIO_REARR_SUBSET, "unknown rearranger", __FILE__, __LINE__); - /* iomaster needs max of buflen, others need local len */ - if (ios->iomaster == MPI_ROOT) + /* iomain needs max of buflen, others need local len */ + if (ios->iomain == MPI_ROOT) rlen = iodesc->maxiobuflen; else rlen = iodesc->llen; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_darray_int.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_darray_int.c index 6ef60594a6..084a214293 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_darray_int.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_darray_int.c @@ -427,6 +427,8 @@ write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *vari int rrcnt = 0; /* Number of subarray requests (pnetcdf only). */ PIO_Offset *startlist[num_regions]; /* Array of start arrays for ncmpi_iput_varn(). */ PIO_Offset *countlist[num_regions]; /* Array of count arrays for ncmpi_iput_varn(). */ + + ierr = ncmpi_wait_all(file->fh, NC_REQ_ALL, NULL, NULL); #endif /* _PNETCDF */ /* Process each region of data to be written. */ @@ -455,11 +457,12 @@ write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *vari bufptr = (void *)((char *)iobuf + iodesc->mpitype_size * (nv * llen + region->loffset)); /* Ensure collective access. */ - ierr = nc_var_par_access(file->fh, varids[nv], NC_COLLECTIVE); + if((ierr = nc_var_par_access(file->fh, varids[nv], NC_COLLECTIVE))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); /* Write the data for this variable. */ - if (!ierr) - ierr = nc_put_vara(file->fh, varids[nv], (size_t *)start, (size_t *)count, bufptr); + if((ierr = nc_put_vara(file->fh, varids[nv], (size_t *)start, (size_t *)count, bufptr))) + return pio_err(ios, file, ierr, __FILE__, __LINE__); } break; #endif @@ -642,27 +645,10 @@ write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *vari else /* don't flush yet, accumulate the request size */ vard_llen += llen; #else - if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0) - { - if (!(vdesc->request = realloc(vdesc->request, sizeof(int) * - (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - - for (int i = vdesc->nreqs; i < vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK; i++) - vdesc->request[i] = NC_REQ_NULL; - } - /* Write, in non-blocking fashion, a list of subarrays. */ -// PLOG((3, "about to call ncmpi_iput_varn() varids[%d] = %d rrcnt = %d, llen = %d", -// nv, varids[nv], rrcnt, llen)); -// for(int i=0;i < llen; i++) -// PLOG((3, "bufptr[%d] = %d",i,((int *)bufptr)[i])); ierr = ncmpi_iput_varn(file->fh, varids[nv], rrcnt, startlist, countlist, - bufptr, llen, iodesc->mpitype, &vdesc->request[vdesc->nreqs]); + bufptr, llen, iodesc->mpitype, NULL); - /* keeps wait calls in sync */ - if (vdesc->request[vdesc->nreqs] == NC_REQ_NULL) - vdesc->request[vdesc->nreqs] = PIO_REQ_NULL; vdesc->nreqs++; #endif @@ -1681,11 +1667,11 @@ pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, * ios->num_iotasks is the number of iotasks actually * used in this decomposition. */ if (rtask < ios->num_iotasks && tmp_bufsize > 0){ - + if ((mpierr = MPI_Send(iobuf, tmp_bufsize, iodesc->mpitype, rtask, 4 * ios->num_iotasks + rtask, ios->io_comm))) return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); - } + } } } } @@ -1749,58 +1735,34 @@ flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) * limit, then flush to disk. */ if (force || (usage >= pio_pnetcdf_buffer_size_limit)) { - int rcnt; int maxreq; - int reqcnt; maxreq = 0; - reqcnt = 0; - rcnt = 0; for (int i = 0; i < file->nvars; i++) { if ((ierr = get_var_desc(i, &file->varlist, &vdesc))) return pio_err(NULL, file, ierr, __FILE__, __LINE__); - reqcnt += vdesc->nreqs; if (vdesc->nreqs > 0) maxreq = i; } - int request[reqcnt]; - int status[reqcnt]; - if (file->varlist) { for (int i = 0; i <= maxreq; i++) { if ((ierr = get_var_desc(i, &file->varlist, &vdesc))) return pio_err(NULL, file, ierr, __FILE__, __LINE__); -#ifdef MPIO_ONESIDED - /*onesided optimization requires that all of the requests in a wait_all call represent - a contiguous block of data in the file */ - if (rcnt > 0 && (prev_record != vdesc->record || vdesc->nreqs == 0)) - { - ierr = ncmpi_wait_all(file->fh, rcnt, request, status); - rcnt = 0; - } - prev_record = vdesc->record; -#endif - for (reqcnt = 0; reqcnt < vdesc->nreqs; reqcnt++) - request[rcnt++] = max(vdesc->request[reqcnt], NC_REQ_NULL); - PLOG((3,"flush_output_buffer rcnt=%d",rcnt)); - if (vdesc->request != NULL) - free(vdesc->request); - vdesc->request = NULL; vdesc->nreqs = 0; #ifdef FLUSH_EVERY_VAR - ierr = ncmpi_wait_all(file->fh, rcnt, request, status); + ierr = ncmpi_wait_all(file->fh, NC_REQ_ALL, NULL, NULL); + //ierr = ncmpi_wait_all(file->fh, rcnt, request, status); rcnt = 0; #endif } } - - if (rcnt > 0) - ierr = ncmpi_wait_all(file->fh, rcnt, request, status); + /* make sure all buffers are now clean */ + ierr = ncmpi_wait_all(file->fh, NC_REQ_ALL, NULL, NULL); /* Release resources. */ if (file->iobuf) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_file.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_file.c index 1edf4f42ec..e9edb6af4c 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_file.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_file.c @@ -156,7 +156,7 @@ PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename, PLOG((1, "PIOc_createfile iosysid = %d iotype = %d filename = %s mode = %d", iosysid, *iotype, filename, mode)); - + /* Create the file. */ if ((ret = PIOc_createfile_int(iosysid, ncidp, iotype, filename, mode, 0))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); @@ -235,8 +235,8 @@ PIOc_closefile(int ncid) if (file->writable) PIOc_sync(ncid); - /* If async is in use and this is a comp tasks, then the compmaster - * sends a msg to the pio_msg_handler running on the IO master and + /* If async is in use and this is a comp tasks, then the compmain + * sends a msg to the pio_msg_handler running on the IO main and * waiting for a message. Then broadcast the ncid over the intercomm * to the IO tasks. */ if (ios->async) @@ -245,11 +245,11 @@ PIOc_closefile(int ncid) { int msg = PIO_MSG_CLOSE_FILE; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -276,8 +276,10 @@ PIOc_closefile(int ncid) break; #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - if (file->writable) + if (file->writable){ + ierr = ncmpi_wait_all(file->fh, NC_REQ_ALL, NULL, NULL); ierr = ncmpi_buffer_detach(file->fh); + } ierr = ncmpi_close(file->fh); break; #endif @@ -326,7 +328,7 @@ PIOc_deletefile(int iosysid, const char *filename) if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); - /* If async is in use, send message to IO master task. */ + /* If async is in use, send message to IO main task. */ if (ios->async) { if (!ios->ioproc) @@ -336,9 +338,9 @@ PIOc_deletefile(int iosysid, const char *filename) len = strlen(filename); if (!mpierr) - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "Bcast len = %d filename = %s", len, filename)); } @@ -426,18 +428,18 @@ PIOc_sync(int ncid) } } - /* If async is in use, send message to IO master tasks. */ + /* If async is in use, send message to IO main tasks. */ if (ios->async) { if (!ios->ioproc) { int msg = PIO_MSG_SYNC; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -477,8 +479,8 @@ PIOc_sync(int ncid) } /* Broadcast and check the return code. */ - if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); +// if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) +// return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if (ierr) return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__); diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_getput_int.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_getput_int.c index ce1014b4ad..5ab5904169 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_getput_int.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_getput_int.c @@ -77,30 +77,30 @@ PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype, { int msg = PIO_MSG_PUT_ATT; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); int namelen = strlen(name); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&memtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&memtype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)op, len * memtype_len, MPI_BYTE, ios->compmaster, + mpierr = MPI_Bcast((void *)op, len * memtype_len, MPI_BYTE, ios->compmain, ios->intercomm); PLOG((2, "PIOc_put_att finished bcast ncid = %d varid = %d namelen = %d name = %s " "len = %d atttype_len = %d memtype = %d memtype_len = %d", ncid, varid, namelen, @@ -299,32 +299,32 @@ PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void *ip int msg = PIO_MSG_GET_ATT; PLOG((2, "sending parameters")); - /* Send the message to IO master. */ - if (ios->compmaster == MPI_ROOT) + /* Send the message to IO main. */ + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); /* Send the function parameters. */ if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); int namelen = strlen(name); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&memtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&memtype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->compmain, ios->intercomm); PLOG((2, "Bcast complete ncid = %d varid = %d namelen = %d name = %s iotype = %d " "atttype = %d attlen = %d atttype_len = %d", ncid, varid, namelen, name, file->iotype, atttype, attlen, atttype_len)); @@ -563,35 +563,35 @@ PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset { int msg = PIO_MSG_GET_VARS; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); /* Send the function parameters and associated informaiton * to the msg handler. */ if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr && start_present) - mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr && count_present) - mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr && stride_present) - mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmain, ios->intercomm); PLOG((2, "PIOc_get_vars_tc ncid = %d varid = %d ndims = %d start_present = %d " "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid, ndims, start_present, count_present, stride_present, xtype, num_elem)); @@ -641,9 +641,9 @@ PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset if ((ierr = ncmpi_begin_indep_data(file->fh))) return pio_err(ios, file, ierr, __FILE__, __LINE__); - /* Only the IO master does the IO, so we are not really + /* Only the IO main does the IO, so we are not really * getting parallel IO here. */ - if (ios->iomaster == MPI_ROOT) + if (ios->iomain == MPI_ROOT) { switch(xtype) { @@ -1010,42 +1010,42 @@ PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset { int msg = PIO_MSG_PUT_VARS; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); /* Send the function parameters and associated informaiton * to the msg handler. */ if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr && start_present) - mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr && count_present) - mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr && stride_present) - mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmain, ios->intercomm); PLOG((2, "PIOc_put_vars_tc ncid = %d varid = %d ndims = %d start_present = %d " "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid, ndims, start_present, count_present, stride_present, xtype, num_elem)); /* Send the data. */ if (!mpierr) - mpierr = MPI_Bcast((void *)buf, num_elem * typelen, MPI_BYTE, ios->compmaster, + mpierr = MPI_Bcast((void *)buf, num_elem * typelen, MPI_BYTE, ios->compmain, ios->intercomm); } @@ -1085,95 +1085,38 @@ PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) { - /* Scalars have to be handled differently. */ - if (ndims == 0) - { - /* This is a scalar var. */ - PLOG((2, "pnetcdf writing scalar with ncmpi_put_vars_*() file->fh = %d varid = %d", - file->fh, varid)); - pioassert(!start && !count && !stride, "expected NULLs", __FILE__, __LINE__); - - /* Turn on independent access for pnetcdf file. */ - if ((ierr = ncmpi_begin_indep_data(file->fh))) - return pio_err(ios, file, ierr, __FILE__, __LINE__); - - /* Only the IO master does the IO, so we are not really - * getting parallel IO here. */ - if (ios->iomaster == MPI_ROOT) - { - switch(xtype) - { - case NC_BYTE: - ierr = ncmpi_put_vars_schar(file->fh, varid, start, count, stride, buf); - break; - case NC_CHAR: - ierr = ncmpi_put_vars_text(file->fh, varid, start, count, stride, buf); - break; - case NC_SHORT: - ierr = ncmpi_put_vars_short(file->fh, varid, start, count, stride, buf); - break; - case NC_INT: - ierr = ncmpi_put_vars_int(file->fh, varid, start, count, stride, buf); - break; - case PIO_LONG_INTERNAL: - ierr = ncmpi_put_vars_long(file->fh, varid, start, count, stride, buf); - break; - case NC_FLOAT: - ierr = ncmpi_put_vars_float(file->fh, varid, start, count, stride, buf); - break; - case NC_DOUBLE: - ierr = ncmpi_put_vars_double(file->fh, varid, start, count, stride, buf); - break; - default: - return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__); - } - } - - /* Turn off independent access for pnetcdf file. */ - if ((ierr = ncmpi_end_indep_data(file->fh))) - return pio_err(ios, file, ierr, __FILE__, __LINE__); - } - else + if (ios->iomain == MPI_ROOT) { /* This is not a scalar var. */ var_desc_t *vdesc; - int *request; - + PLOG((2, "PIOc_put_vars_tc calling pnetcdf function")); - flush_output_buffer(file, false, num_elem*typelen); - - /*vdesc = &file->varlist[varid];*/ + if ((ierr = get_var_desc(varid, &file->varlist, &vdesc))) return pio_err(ios, file, ierr, __FILE__, __LINE__); - if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0) - if (!(vdesc->request = realloc(vdesc->request, - sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)))) - return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); - request = vdesc->request + vdesc->nreqs; - PLOG((2, "PIOc_put_vars_tc request = %d size = %d", vdesc->request, num_elem*typelen)); - + switch(xtype) { case NC_BYTE: - ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, fake_stride, buf, request); + ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, fake_stride, buf, NULL); break; case NC_CHAR: - ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, fake_stride, buf, request); + ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, fake_stride, buf, NULL); break; case NC_SHORT: - ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, fake_stride, buf, request); + ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, fake_stride, buf, NULL); break; case NC_INT: - ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, fake_stride, buf, request); + ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, fake_stride, buf, NULL); break; case PIO_LONG_INTERNAL: - ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, fake_stride, buf, request); + ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, fake_stride, buf, NULL); break; case NC_FLOAT: - ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, fake_stride, buf, request); + ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, fake_stride, buf, NULL); break; case NC_DOUBLE: - ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, fake_stride, buf, request); + ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, fake_stride, buf, NULL); break; default: return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__); @@ -1182,13 +1125,11 @@ PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset vdesc->nreqs++; -// flush_output_buffer(file, ierr == PIO_EINSUFFBUF, 0); -// PLOG((2, "PIOc_put_vars_tc flushed output buffer")); if(ierr == PIO_EINVALCOORDS) for(int i=0; i<ndims; i++) PLOG((2,"start[%d] %ld count[%d] %ld\n",i,start[i],i,count[i])); - } /* endif ndims == 0 */ + } /* end if MPI_ROOT */ } #endif /* _PNETCDF */ diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_internal.h b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_internal.h index efd4f8182e..620ce4d18a 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_internal.h +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_internal.h @@ -13,9 +13,14 @@ #include <config.h> #include <pio.h> #include <pio_error.h> +#include <stdint.h> #include <limits.h> #include <math.h> #include <netcdf.h> +#ifdef NC_HAS_PAR_FILTERS +#include <netcdf_filter.h> +#include <netcdf_meta.h> +#endif #ifdef _NETCDF4 #include <netcdf_par.h> #endif @@ -30,6 +35,22 @@ #include <mpe.h> #endif /* USE_MPE */ +/* define an MPI type equivalent to size_t */ +#if SIZE_MAX == UCHAR_MAX + #define PIO_MPI_SIZE_T MPI_UNSIGNED_CHAR +#elif SIZE_MAX == USHRT_MAX + #define PIO_MPI_SIZE_T MPI_UNSIGNED_SHORT +#elif SIZE_MAX == UINT_MAX + #define PIO_MPI_SIZE_T MPI_UNSIGNED +#elif SIZE_MAX == ULONG_MAX + #define PIO_MPI_SIZE_T MPI_UNSIGNED_LONG +#elif SIZE_MAX == ULLONG_MAX + #define PIO_MPI_SIZE_T MPI_UNSIGNED_LONG_LONG +#else + #error "what is happening here?" +#endif + + //#ifndef MPI_OFFSET /** MPI_OFFSET is an integer type of size sufficient to represent the * size (in bytes) of the largest file supported by MPI. In some MPI @@ -65,7 +86,7 @@ /** Some MPI implementations do not allow passing MPI_DATATYPE_NULL to * comm functions even though the send or recv length is 0, in these * cases we use MPI_CHAR, after this issue raised its ugly head again in mpich - * 4.0.0 we decided to use this workaround in all cases. + * 4.0.0 we decided to use this workaround in all cases. * See https://github.com/NCAR/ParallelIO/issues/1945 */ #define PIO_DATATYPE_NULL MPI_CHAR @@ -613,6 +634,7 @@ enum PIO_MSG PIO_MSG_DEF_VAR_DEFLATE, PIO_MSG_INQ_VAR_DEFLATE, PIO_MSG_INQ_VAR_SZIP, + PIO_MSG_DEF_VAR_SZIP, PIO_MSG_DEF_VAR_FLETCHER32, PIO_MSG_INQ_VAR_FLETCHER32, PIO_MSG_DEF_VAR_CHUNKING, @@ -638,7 +660,24 @@ enum PIO_MSG PIO_MSG_GET_ATT, PIO_MSG_PUT_ATT, PIO_MSG_INQ_TYPE, - PIO_MSG_INQ_UNLIMDIMS + PIO_MSG_INQ_UNLIMDIMS, +#ifdef NC_HAS_BZ2 + PIO_MSG_INQ_VAR_BZIP2, + PIO_MSG_DEF_VAR_BZIP2, +#endif +#ifdef NC_HAS_ZSTD + PIO_MSG_INQ_VAR_ZSTANDARD, + PIO_MSG_DEF_VAR_ZSTANDARD, +#endif + PIO_MSG_DEF_VAR_FILTER, + PIO_MSG_INQ_VAR_FILTER_IDS, + PIO_MSG_INQ_VAR_FILTER_INFO, + PIO_MSG_INQ_FILTER_AVAIL, + +#ifdef NC_HAS_QUANTIZE + PIO_MSG_DEF_VAR_QUANTIZE, + PIO_MSG_INQ_VAR_QUANTIZE, +#endif }; #endif /* __PIO_INTERNAL__ */ diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_meta.h.in b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_meta.h.in index a735a89002..b98f10d73d 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_meta.h.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_meta.h.in @@ -22,10 +22,11 @@ #define PIO_VERSION_MINOR @PIO_VERSION_MINOR@ /*!< pio-c minor version. */ #define PIO_VERSION_PATCH @PIO_VERSION_PATCH@ /*!< pio-c patch version. */ #define PIO_VERSION "@PACKAGE_VERSION@" - +#ifndef PIO_HAS_PAR_FILTERS +#define PIO_HAS_PAR_FILTERS @PIO_HAS_PAR_FILTERS@ /*!< NetCDF supports parallel I/O with filters. */ +#endif #define PIO_HAS_SZIP_WRITE @PIO_HAS_SZIP_WRITE@ /*!< szip write support */ #define PIO_HAS_PNETCDF @PIO_HAS_PNETCDF@ /*!< PnetCDF support. */ -#define PIO_HAS_PAR_FILTERS @PIO_HAS_PAR_FILTERS@ /*!< NetCDF supports parallel I/O with filters. */ #define PIO_HAS_NETCDF4 @PIO_HAS_NETCDF4@ /*!< NetCDF-4 supported. */ #define PIO_HAS_NETCDF4_PAR @PIO_HAS_NETCDF4_PAR@ /*!< NetCDF-4 parallel I/O supported. */ #define PIO_HAS_NETCDF_INTEGRATION @PIO_HAS_NETCDF_INTEGRATION@ /*!< NetCDF integration supported. */ diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_msg.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_msg.c index f49384d297..3302ae7a36 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_msg.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_msg.c @@ -14,9 +14,9 @@ * @author Ed Hartnett */ -#include <config.h> #include <pio.h> #include <pio_internal.h> +#include <config.h> #ifdef PIO_ENABLE_LOGGING extern int my_rank; @@ -50,7 +50,7 @@ int inq_type_handler(iosystem_desc_t *ios) PLOG((1, "inq_type_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -94,7 +94,7 @@ int inq_format_handler(iosystem_desc_t *ios) PLOG((1, "inq_format_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -134,7 +134,7 @@ int set_fill_handler(iosystem_desc_t *ios) PLOG((1, "set_fill_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -182,7 +182,7 @@ int create_file_handler(iosystem_desc_t *ios) PLOG((1, "create_file_handler comproot = %d", ios->comproot)); assert(ios); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -252,7 +252,7 @@ int close_file_handler(iosystem_desc_t *ios) PLOG((1, "close_file_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -286,7 +286,7 @@ int inq_handler(iosystem_desc_t *ios) PLOG((1, "inq_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -341,7 +341,7 @@ int inq_unlimdims_handler(iosystem_desc_t *ios) PLOG((1, "inq_unlimdims_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -390,7 +390,7 @@ int inq_dim_handler(iosystem_desc_t *ios, int msg) PLOG((1, "inq_dim_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -436,7 +436,7 @@ int inq_dimid_handler(iosystem_desc_t *ios) PLOG((1, "inq_dimid_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -483,15 +483,15 @@ int inq_att_handler(iosystem_desc_t *ios) PLOG((1, "inq_att_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm))) + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster, + if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, 0, ios->intercomm))) @@ -534,13 +534,13 @@ int inq_attname_handler(iosystem_desc_t *ios) PLOG((1, "inq_att_name_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmaster, ios->intercomm))) + if ((mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmain, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -579,15 +579,15 @@ int inq_attid_handler(iosystem_desc_t *ios) PLOG((1, "inq_attid_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm))) + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm))) + if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -629,15 +629,15 @@ int att_put_handler(iosystem_desc_t *ios) PLOG((1, "att_put_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm))) + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm))) + if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&atttype, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -698,15 +698,15 @@ int att_get_handler(iosystem_desc_t *ios) PLOG((1, "att_get_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm))) + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm))) + if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -765,7 +765,7 @@ int put_vars_handler(iosystem_desc_t *ios) PLOG((1, "put_vars_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -906,7 +906,7 @@ int get_vars_handler(iosystem_desc_t *ios) PLOG((1, "get_vars_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1051,7 +1051,7 @@ int inq_var_handler(iosystem_desc_t *ios) PLOG((1, "inq_var_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1112,7 +1112,7 @@ int inq_var_chunking_handler(iosystem_desc_t *ios) assert(ios); PLOG((1, "inq_var_chunking_handler")); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1142,6 +1142,350 @@ int inq_var_chunking_handler(iosystem_desc_t *ios) return PIO_NOERR; } +#ifdef PIO_HAS_PAR_FILTERS +/** + * Do an inq_var_filter_ids on a netCDF variable. This function is only + * run on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @returns 0 for success, error code otherwise. + */ +int inq_var_filter_ids_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + size_t *nfiltersp=NULL; + unsigned int *ids=NULL; + size_t nfilters; + char nfilters_present; + char ids_present; + size_t idsize=0; + int mpierr; + + assert(ios); + PLOG((1, "inq_var_filter_ids_handler")); + + /* Get the parameters for this function that the the comp main + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&nfilters_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&ids_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if(ids_present){ + if ((mpierr = MPI_Bcast(&idsize, 1, PIO_MPI_SIZE_T, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if (!(ids = malloc(idsize *sizeof(size_t)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + } + + PLOG((2,"inq_var_filter_ids_handler ncid = %d varid = %d nfilters_present = %d ids_present = %d idsize = %d", + ncid, varid, nfilters_present, ids_present, idsize)); + + /* Set the non-NULL pointers. */ + if(nfilters_present) + nfiltersp = &nfilters; + + /* Call the inq function to get the values. */ + PIOc_inq_var_filter_ids(ncid, varid, nfiltersp, ids); + + if(ids_present) + free(ids); + + return PIO_NOERR; +} +#ifdef NC_HAS_BZ2 +/** + * Do an inq_var_bzip2 on a netCDF variable. This function is only + * run on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @returns 0 for success, error code otherwise. + */ +int inq_var_bzip2_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int *hasfilterp=NULL; + int *levelp=NULL; + char hasfilterp_present; + char levelp_present; + int hasfilter; + int level; + int mpierr; + + assert(ios); + PLOG((1, "inq_var_bzip2_handler")); + + /* Get the parameters for this function that the the comp main + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&hasfilterp_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&levelp_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + + PLOG((2,"inq_var_bzip2_handler ncid = %d varid = %d hasfilter_present = %d ", + ncid, varid, hasfilterp_present, levelp_present)); + + /* Set the non-NULL pointers. */ + if(hasfilterp_present) + hasfilterp = &hasfilter; + if(levelp_present) + levelp = &level; + + /* Call the inq function to get the values. */ + PIOc_inq_var_bzip2(ncid, varid, hasfilterp, levelp); + + return PIO_NOERR; +} +#endif + +#ifdef PIO_HAS_PAR_FILTERS +/** + * Do an inq_var_filter_info on a netCDF variable. This function is only + * run on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @returns 0 for success, error code otherwise. + */ +int inq_var_filter_info_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + unsigned int id; + size_t *nparamsp = NULL; + size_t nparams; + unsigned int *params = NULL; + char nparams_present; + char params_present; + size_t paramssize; + int mpierr; + + assert(ios); + PLOG((1, "inq_var_filter_info_handler")); + + /* Get the parameters for this function that the the comp main + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&id, 1, MPI_UNSIGNED, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&nparams_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(¶ms_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if(params_present){ + if ((mpierr = MPI_Bcast(¶mssize, 1, PIO_MPI_SIZE_T, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if (!(params = malloc(paramssize *sizeof(unsigned int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + + } + PLOG((2,"inq_var_filter_info_handler ncid = %d varid = %d nparams_present = %d params_present = %d", + ncid, varid, nparams_present, params_present)); + + /* Set the non-NULL pointers. */ + if (nparams_present) + nparamsp = &nparams; + + /* Call the inq function to get the values. */ + PIOc_inq_var_filter_info(ncid, varid, id, nparamsp, params); + + if(params_present) + free(params); + + return PIO_NOERR; +} +#endif +#ifdef NC_HAS_QUANTIZE +/** + * Do an inq_var_quantize on a netCDF variable. This function is only + * run on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @returns 0 for success, error code otherwise. + */ +int inq_var_quantize_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int *quantize_modep = NULL; + int *nsdp = NULL; + int qmode; + int nsd; + int mpierr; + char qmode_present; + char nsd_present; + + assert(ios); + PLOG((1, "inq_var_chunking_handler")); + + /* Get the parameters for this function that the the comp main + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&qmode_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&nsd_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + + + PLOG((2,"inq_var_handler ncid = %d varid = %d", + ncid, varid)); + + if (qmode_present) + quantize_modep = &qmode; + if(nsd_present) + nsdp = &nsd; + /* Call the inq function to get the values. */ + PIOc_inq_var_quantize(ncid, varid, quantize_modep, nsdp); + + return PIO_NOERR; +} + +/** + * This function is run on the IO tasks to define a netCDF + * variable quantize level. + * + * @param ios pointer to the iosystem_desc_t. + * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code + * from netCDF base function. + * @internal + * @author Jim Edwards, Ed Hartnett + */ +int def_var_quantize_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int mode; + int nsd; + int mpierr; + + PLOG((1, "def_var_quantize_handler comproot = %d", ios->comproot)); + assert(ios); + + /* Get the parameters for this function that the he comp main + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&nsd, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + + PLOG((1, "def_var_quantize_handler got parameters ncid = %d " + "varid = %d mode = %d nsd = %d ", ncid, varid, mode, nsd)); + + /* Call the function. */ + PIOc_def_var_quantize(ncid, varid, mode, nsd); + + + PLOG((1, "def_var_quantize_handler succeeded!")); + return PIO_NOERR; +} +#endif + +#ifdef NC_HAS_ZSTD +/** + * Do an inq_var_bzip2 on a netCDF variable. This function is only + * run on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @returns 0 for success, error code otherwise. + */ +int inq_var_zstandard_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int *hasfilterp=NULL; + int *levelp=NULL; + char hasfilterp_present; + char levelp_present; + int hasfilter; + int level; + int mpierr; + + assert(ios); + PLOG((1, "inq_var_zstandard_handler")); + + /* Get the parameters for this function that the the comp main + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&hasfilterp_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&levelp_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + + PLOG((2,"inq_var_zstandard_handler ncid = %d varid = %d hasfilter_present = %d ", + ncid, varid, hasfilterp_present, levelp_present)); + + /* Set the non-NULL pointers. */ + if(hasfilterp_present) + hasfilterp = &hasfilter; + if(levelp_present) + levelp = &level; + + /* Call the inq function to get the values. */ + PIOc_inq_var_zstandard(ncid, varid, hasfilterp, levelp); + + return PIO_NOERR; +} +/** + * This function is run on the IO tasks to define a netCDF + * variable quantize level. + * + * @param ios pointer to the iosystem_desc_t. + * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code + * from netCDF base function. + * @internal + * @author Jim Edwards, Ed Hartnett + */ +int def_var_zstandard_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int level; + int mpierr; + + PLOG((1, "def_var_zstandard_handler comproot = %d", ios->comproot)); + assert(ios); + + /* Get the parameters for this function that the he comp main + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&level, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + + PLOG((1, "def_var_zstandard_handler got parameters ncid = %d " + "varid = %d level = %d ", ncid, varid, level)); + + /* Call the function. */ + PIOc_def_var_zstandard(ncid, varid, level); + + PLOG((1, "def_var_zstandard_handler succeeded!")); + return PIO_NOERR; +} +#endif +#endif /** * Do an inq_var_fill on a netCDF variable. This function is only @@ -1163,7 +1507,7 @@ int inq_var_fill_handler(iosystem_desc_t *ios) assert(ios); PLOG((1, "inq_var_fill_handler")); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1226,7 +1570,7 @@ int inq_var_endian_handler(iosystem_desc_t *ios) assert(ios); PLOG((1, "inq_var_endian_handler")); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1269,7 +1613,7 @@ int inq_var_deflate_handler(iosystem_desc_t *ios) assert(ios); PLOG((1, "inq_var_deflate_handler")); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1327,7 +1671,7 @@ int inq_varid_handler(iosystem_desc_t *ios) assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1359,7 +1703,7 @@ int sync_file_handler(iosystem_desc_t *ios) PLOG((1, "sync_file_handler")); assert(ios); - /* Get the parameters for this function that the comp master + /* Get the parameters for this function that the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1392,7 +1736,7 @@ int setframe_handler(iosystem_desc_t *ios) PLOG((1, "setframe_handler")); assert(ios); - /* Get the parameters for this function that the comp master + /* Get the parameters for this function that the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1429,7 +1773,7 @@ int advanceframe_handler(iosystem_desc_t *ios) PLOG((1, "advanceframe_handler")); assert(ios); - /* Get the parameters for this function that the comp master + /* Get the parameters for this function that the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1463,7 +1807,7 @@ int change_def_file_handler(iosystem_desc_t *ios, int msg) PLOG((1, "change_def_file_handler")); assert(ios); - /* Get the parameters for this function that the comp master + /* Get the parameters for this function that the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1502,7 +1846,7 @@ int def_var_handler(iosystem_desc_t *ios) PLOG((1, "def_var_handler comproot = %d", ios->comproot)); assert(ios); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1554,7 +1898,7 @@ int def_var_chunking_handler(iosystem_desc_t *ios) assert(ios); PLOG((1, "def_var_chunking_handler comproot = %d", ios->comproot)); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1605,7 +1949,7 @@ int def_var_fill_handler(iosystem_desc_t *ios) assert(ios); PLOG((1, "def_var_fill_handler comproot = %d", ios->comproot)); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1658,7 +2002,7 @@ int def_var_endian_handler(iosystem_desc_t *ios) assert(ios); PLOG((1, "def_var_endian_handler comproot = %d", ios->comproot)); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1695,7 +2039,7 @@ int def_var_deflate_handler(iosystem_desc_t *ios) assert(ios); PLOG((1, "def_var_deflate_handler comproot = %d", ios->comproot)); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1736,7 +2080,7 @@ int set_var_chunk_cache_handler(iosystem_desc_t *ios) assert(ios); PLOG((1, "set_var_chunk_cache_handler comproot = %d", ios->comproot)); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1779,7 +2123,7 @@ int def_dim_handler(iosystem_desc_t *ios) PLOG((1, "def_dim_handler comproot = %d", ios->comproot)); assert(ios); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1820,7 +2164,7 @@ int rename_dim_handler(iosystem_desc_t *ios) PLOG((1, "rename_dim_handler")); assert(ios); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1861,7 +2205,7 @@ int rename_var_handler(iosystem_desc_t *ios) PLOG((1, "rename_var_handler")); assert(ios); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1902,7 +2246,7 @@ int rename_att_handler(iosystem_desc_t *ios) PLOG((1, "rename_att_handler")); assert(ios); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1947,7 +2291,7 @@ int delete_att_handler(iosystem_desc_t *ios) PLOG((1, "delete_att_handler")); assert(ios); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -1992,7 +2336,7 @@ int open_file_handler(iosystem_desc_t *ios) PLOG((1, "open_file_handler comproot = %d", ios->comproot)); assert(ios); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2032,8 +2376,10 @@ int open_file_handler(iosystem_desc_t *ios) } else { +// PIOc_set_log_level(3); PIOc_openfile_retry(ios->iosysid, &ncid, &iotype, filename, mode, 0, use_ext_ncid); +// PIOc_set_log_level(0); } return PIO_NOERR; @@ -2057,7 +2403,7 @@ int delete_file_handler(iosystem_desc_t *ios) PLOG((1, "delete_file_handler comproot = %d", ios->comproot)); assert(ios); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2103,7 +2449,7 @@ int initdecomp_dof_handler(iosystem_desc_t *ios) PLOG((1, "initdecomp_dof_handler called")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2209,7 +2555,7 @@ int write_darray_multi_handler(iosystem_desc_t *ios) PLOG((1, "write_darray_multi_handler")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2310,7 +2656,7 @@ int read_darray_handler(iosystem_desc_t *ios) PLOG((1, "read_darray_handler called")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2351,7 +2697,7 @@ int seterrorhandling_handler(iosystem_desc_t *ios) PLOG((1, "seterrorhandling_handler comproot = %d", ios->comproot)); assert(ios); - /* Get the parameters for this function that the he comp master + /* Get the parameters for this function that the he comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&method, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2392,7 +2738,7 @@ int set_chunk_cache_handler(iosystem_desc_t *ios) PLOG((1, "set_chunk_cache_handler called")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2436,7 +2782,7 @@ int get_chunk_cache_handler(iosystem_desc_t *ios) PLOG((1, "get_chunk_cache_handler called")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2489,7 +2835,7 @@ int get_var_chunk_cache_handler(iosystem_desc_t *ios) PLOG((1, "get_var_chunk_cache_handler called")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2537,7 +2883,7 @@ int freedecomp_handler(iosystem_desc_t *ios) PLOG((1, "freedecomp_handler called")); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2570,7 +2916,7 @@ int finalize_handler(iosystem_desc_t *ios, int index) PLOG((1, "finalize_handler called index = %d", index)); assert(ios); - /* Get the parameters for this function that the the comp master + /* Get the parameters for this function that the the comp main * task is broadcasting. */ if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2592,12 +2938,12 @@ int finalize_handler(iosystem_desc_t *ios, int index) */ int set_loglevel_handler(iosystem_desc_t *ios) { -#if PIO_ENABLE_LOGGING +#if PIO_ENABLE_LOGGING int iosysid; int level; int mpierr; #endif - + PLOG((0, "set_loglevel_handler called")); assert(ios); #if PIO_ENABLE_LOGGING @@ -2609,8 +2955,89 @@ int set_loglevel_handler(iosystem_desc_t *ios) #endif return PIO_NOERR; } +#ifdef PIO_HAS_PAR_FILTERS +/** + * Do an inq_var_filter_avail on a netCDF variable. This function is only + * run on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @returns 0 for success, error code otherwise. + */ +int inq_filter_avail_handler(iosystem_desc_t *ios) +{ + int ncid; + unsigned int id; + int mpierr; + + assert(ios); + PLOG((1, "inq_filter_avail_handler")); + + /* Get the parameters for this function that the the comp main + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&id, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + + PLOG((2,"inq_filter_avail_handler ncid = %d id = %d", + ncid, id)); + /* Call the inq function to get the values. */ + PIOc_inq_filter_avail(ncid, id); + return PIO_NOERR; +} +/** + * This function is run on the IO tasks to define a netCDF + * variable filter. + * + * @param ios pointer to the iosystem_desc_t. + * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code + * from netCDF base function. + * @internal + * @author Jim Edwards, Ed Hartnett + */ +int def_var_filter_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int id; + size_t nparams; + unsigned int *params; + int mpierr; + + PLOG((1, "def_var_filter_handler comproot = %d", ios->comproot)); + assert(ios); + + /* Get the parameters for this function that the he comp main + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&id, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&nparams, 1, PIO_MPI_SIZE_T, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if (!(params = malloc(nparams * sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + + if ((mpierr = MPI_Bcast(params, nparams, MPI_UNSIGNED, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + + PLOG((1, "def_var_filter_handler got parameters ncid = %d " + "varid = %d id = %d nparams = %d ", ncid, varid, id, nparams)); + + /* Call the function. */ + PIOc_def_var_filter(ncid, varid, id, nparams, params); + + /* Free resources. */ + free(params); + + PLOG((1, "def_var_filter_handler succeeded!")); + return PIO_NOERR; +} +#endif /** * This function is called by the IO tasks. This function will not * return, unless there is an error. @@ -2682,7 +3109,7 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys, if ((mpierr = MPI_Bcast(&outcount, 1, MPI_INT, 0, io_comm))) return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); PLOG((3, "outcount MPI_Bcast complete outcount = %d", outcount)); - + for(int creq=0; creq < outcount; creq++) { int idx = index[creq]; @@ -2750,6 +3177,16 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys, case PIO_MSG_DEF_VAR: ret = def_var_handler(my_iosys); break; +#ifdef PIO_HAS_PAR_FILTERS +#ifdef NC_HAS_ZSTD + case PIO_MSG_INQ_VAR_ZSTANDARD: + ret = inq_var_zstandard_handler(my_iosys); + break; + case PIO_MSG_DEF_VAR_ZSTANDARD: + ret = def_var_zstandard_handler(my_iosys); + break; +#endif +#endif case PIO_MSG_DEF_VAR_CHUNKING: ret = def_var_chunking_handler(my_iosys); break; @@ -2852,6 +3289,28 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys, case PIO_MSG_SETLOGLEVEL: ret = set_loglevel_handler(my_iosys); break; +#ifdef PIO_HAS_PAR_FILTERS +#ifdef NC_HAS_QUANTIZE + case PIO_MSG_DEF_VAR_QUANTIZE: + ret = def_var_quantize_handler(my_iosys); + break; + case PIO_MSG_INQ_VAR_QUANTIZE: + ret = inq_var_quantize_handler(my_iosys); + break; +#endif + case PIO_MSG_DEF_VAR_FILTER: + ret = def_var_filter_handler(my_iosys); + break; + case PIO_MSG_INQ_FILTER_AVAIL: + ret = inq_filter_avail_handler(my_iosys); + break; + case PIO_MSG_INQ_VAR_FILTER_IDS: + ret = inq_var_filter_ids_handler(my_iosys); + break; + case PIO_MSG_INQ_VAR_FILTER_INFO: + ret = inq_var_filter_info_handler(my_iosys); + break; +#endif case PIO_MSG_EXIT: finalize++; ret = finalize_handler(my_iosys, idx); diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_nc.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_nc.c index 5f874c4996..2808e45ec3 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_nc.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_nc.c @@ -118,19 +118,19 @@ PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp) char ngatts_present = ngattsp ? true : false; char unlimdimid_present = unlimdimidp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_inq ncid = %d ndims_present = %d nvars_present = %d ngatts_present = %d unlimdimid_present = %d", ncid, ndims_present, nvars_present, ngatts_present, unlimdimid_present)); } @@ -313,15 +313,15 @@ PIOc_inq_unlimdims(int ncid, int *nunlimdimsp, int *unlimdimidsp) char nunlimdimsp_present = nunlimdimsp ? true : false; char unlimdimidsp_present = unlimdimidsp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&nunlimdimsp_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&nunlimdimsp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&unlimdimidsp_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&unlimdimidsp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_inq_unlimdims ncid = %d nunlimdimsp_present = %d unlimdimidsp_present = %d", ncid, nunlimdimsp_present, unlimdimidsp_present)); } @@ -449,17 +449,17 @@ PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) char name_present = name ? true : false; char size_present = sizep ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) @@ -491,7 +491,7 @@ PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) if (name) { int slen; - if (ios->iomaster == MPI_ROOT) + if (ios->iomain == MPI_ROOT) slen = strlen(name); if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); @@ -538,13 +538,13 @@ PIOc_inq_format(int ncid, int *formatp) int msg = PIO_MSG_INQ_FORMAT; char format_present = formatp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -624,18 +624,18 @@ PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) char name_present = name ? true : false; char len_present = lenp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_inq netcdf Bcast name_present = %d", name_present)); if (!mpierr) - mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_inq netcdf Bcast len_present = %d", len_present)); } @@ -676,7 +676,7 @@ PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) { int slen; PLOG((2, "bcasting results my_comm = %d", ios->my_comm)); - if (ios->iomaster == MPI_ROOT) + if (ios->iomain == MPI_ROOT) slen = strlen(name); if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); @@ -771,18 +771,18 @@ PIOc_inq_dimid(int ncid, const char *name, int *idp) int msg = PIO_MSG_INQ_DIMID; char id_present = idp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); int namelen = strlen(name); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -873,23 +873,23 @@ PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, char dimids_present = dimidsp ? true : false; char natts_present = nattsp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d " "dimids_present = %d, natts_present = %d nattsp = %d", name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp)); @@ -967,7 +967,7 @@ PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if (name) { int slen; - if (ios->iomaster == MPI_ROOT) + if (ios->iomain == MPI_ROOT) slen = strlen(name); if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); @@ -1125,17 +1125,17 @@ PIOc_inq_varid(int ncid, const char *name, int *varidp) { int msg = PIO_MSG_INQ_VARID; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); int namelen; namelen = strlen(name); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -1221,23 +1221,23 @@ PIOc_inq_att_eh(int ncid, int varid, const char *name, int eh, char len_present = lenp ? true : false; int namelen = strlen(name); - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&eh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&eh, 1, MPI_INT, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -1382,17 +1382,17 @@ PIOc_inq_attname(int ncid, int varid, int attnum, char *name) int msg = PIO_MSG_INQ_ATTNAME; char name_present = name ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -1482,19 +1482,19 @@ PIOc_inq_attid(int ncid, int varid, const char *name, int *idp) int namelen = strlen(name); char id_present = idp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -1575,17 +1575,17 @@ PIOc_rename_dim(int ncid, int dimid, const char *name) int msg = PIO_MSG_RENAME_DIM; /* Message for async notification. */ int namelen = strlen(name); - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_rename_dim Bcast file->fh = %d dimid = %d namelen = %d name = %s", file->fh, dimid, namelen, name)); } @@ -1664,17 +1664,17 @@ PIOc_rename_var(int ncid, int varid, const char *name) int msg = PIO_MSG_RENAME_VAR; /* Message for async notification. */ int namelen = strlen(name); - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_rename_var Bcast file->fh = %d varid = %d namelen = %d name = %s", file->fh, varid, namelen, name)); } @@ -1758,21 +1758,21 @@ PIOc_rename_att(int ncid, int varid, const char *name, int namelen = strlen(name); int newnamelen = strlen(newname); - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((char *)newname, newnamelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((char *)newname, newnamelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -1847,17 +1847,17 @@ PIOc_del_att(int ncid, int varid, const char *name) int msg = PIO_MSG_DEL_ATT; int namelen = strlen(name); /* Length of name string. */ - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -1928,15 +1928,15 @@ PIOc_set_fill(int ncid, int fillmode, int *old_modep) int old_modep_present = old_modep ? 1 : 0; PLOG((3, "PIOc_set_fill about to send msg %d", msg)); - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&fillmode, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&fillmode, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&old_modep_present, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&old_modep_present, 1, MPI_INT, ios->compmain, ios->intercomm); PLOG((2, "PIOc_set_fill sent ncid = %d fillmode = %d old_modep_present = %d", ncid, fillmode, old_modep_present)); } @@ -2065,18 +2065,18 @@ PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp) int msg = PIO_MSG_DEF_DIM; int namelen = strlen(name); - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmain, ios->intercomm); } @@ -2091,6 +2091,7 @@ PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp) if (ios->ioproc) { #ifdef _PNETCDF + if (file->iotype == PIO_IOTYPE_PNETCDF) ierr = ncmpi_def_dim(file->fh, name, len, idp); #endif /* _PNETCDF */ @@ -2230,21 +2231,21 @@ PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims, int msg = PIO_MSG_DEF_VAR; int namelen = strlen(name); - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&(ncid), 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&(ncid), 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -2381,21 +2382,21 @@ PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_valuep) int msg = PIO_MSG_DEF_VAR_FILL; char fill_value_present = fill_valuep ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&fill_mode, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&fill_mode, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr && fill_value_present) - mpierr = MPI_Bcast((PIO_Offset *)fill_valuep, type_size, MPI_CHAR, ios->compmaster, + mpierr = MPI_Bcast((PIO_Offset *)fill_valuep, type_size, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_def_var_fill ncid = %d varid = %d fill_mode = %d type_size = %d fill_value_present = %d", ncid, varid, fill_mode, type_size, fill_value_present)); @@ -2511,19 +2512,19 @@ PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep) char fill_value_present = fill_valuep ? true : false; PLOG((2, "sending msg type_size = %d", type_size)); - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&no_fill_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&no_fill_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_inq_var_fill ncid = %d varid = %d type_size = %lld no_fill_present = %d fill_value_present = %d", ncid, varid, type_size, no_fill_present, fill_value_present)); } diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_nc4.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_nc4.c index becaf1cdf8..3074bbd95a 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_nc4.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_nc4.c @@ -4,9 +4,9 @@ * * @author Ed Hartnett */ -#include <config.h> #include <pio.h> #include <pio_internal.h> +#include <config.h> /** * Set deflate (zlib) settings for a variable. @@ -49,7 +49,7 @@ PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); PLOG((1, "PIOc_def_var_deflate ncid = %d varid = %d shuffle = %d deflate = %d deflate_level = %d", - ncid, varid, shuffle, deflate, deflate_level)); + ncid, varid, shuffle, deflate, deflate_level)); /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async) @@ -58,19 +58,272 @@ PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, { int msg = PIO_MSG_DEF_VAR_DEFLATE; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&shuffle, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&deflate, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&deflate_level, 1, MPI_INT, ios->compmain, ios->intercomm); + } + + /* Handle MPI errors from computation tasks. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + } + + if (ios->ioproc) + { +#ifdef _NETCDF4 + if (file->do_io) + ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level); +#endif + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + return PIO_NOERR; +} + +/** + * Set szip settings for a variable. + * + * This function only applies to netCDF-4 files. When used with netCDF + * classic files, the error PIO_ENOTNC4 will be returned. + * + * See the <a + * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF + * variable documentation</a> for details about the operation of this + * function. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable. + * @param option_mask The options mask. Can be PIO_SZIP_EC or PIO_SZIP_NN. + * @param pixels_per_block Pixels per block. Must be even and not greater than 32, with typical + * values being 8, 10, 16, or 32. This parameter affects compression + * ratio; the more pixel values vary, the smaller this number should be + * to achieve better performance. If pixels_per_block is bigger than the + * total number of elements in a dataset chunk, NC_EINVAL will be + * returned. + * @return PIO_NOERR for success, otherwise an error code. + * @ingroup PIO_def_var_c + * @author Jim Edwards, Ed Hartnett + */ +int +PIOc_def_var_szip(int ncid, int varid, int options_mask, int pixels_per_block) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + + /* Get the file info. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Only netCDF-4 files can use this feature. */ + if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C) + return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); + + PLOG((1, "PIOc_def_var_szip ncid = %d varid = %d mask = %d ppb = %d", + ncid, varid, options_mask, pixels_per_block)); + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_DEF_VAR_SZIP; + + if (ios->compmain == MPI_ROOT) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&options_mask, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&pixels_per_block, 1, MPI_INT, ios->compmain, ios->intercomm); + } + + /* Handle MPI errors from computation tasks. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + } + + if (ios->ioproc) + { +#ifdef _NETCDF4 + if (file->do_io) + ierr = nc_def_var_szip(file->fh, varid, options_mask, pixels_per_block); +#endif + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + return PIO_NOERR; +} + +#ifdef NC_HAS_BZ2 +/** + * Set bzip2 settings for a variable. + * + * This function only applies to netCDF-4 files. When used with netCDF + * classic files, the error PIO_ENOTNC4 will be returned. + * + * See the <a + * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF + * variable documentation</a> for details about the operation of this + * function. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable. + * @param bzip2_level 1 to 9, with 1 being faster and 9 being more + * compressed. + * @return PIO_NOERR for success, otherwise an error code. + * @ingroup PIO_def_var_c + * @author Jim Edwards, Ed Hartnett + */ +int +PIOc_def_var_bzip2(int ncid, int varid, int level) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + + /* Get the file info. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Only netCDF-4 files can use this feature. */ + if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C) + return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); + + PLOG((1, "PIOc_def_var_bzip2 ncid = %d varid = %d level = %d", + ncid, varid, level)); + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_DEF_VAR_BZIP2; + + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&level, 1, MPI_INT, ios->compmain, ios->intercomm); + } + + /* Handle MPI errors from computation tasks. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + } + + if (ios->ioproc) + { +#ifdef _NETCDF4 + if (file->do_io) + ierr = nc_def_var_bzip2(file->fh, varid, level); +#endif + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + return PIO_NOERR; +} +#endif + +#ifdef NC_HAS_ZSTD +/** + * Set zstandard settings for a variable. + * + * This function only applies to netCDF-4 files. When used with netCDF + * classic files, the error PIO_ENOTNC4 will be returned. + * + * See the <a + * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF + * variable documentation</a> for details about the operation of this + * function. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable. + * @param zstandard_level 1 to 9, with 1 being faster and 9 being more + * compressed. + * @return PIO_NOERR for success, otherwise an error code. + * @ingroup PIO_def_var_c + * @author Jim Edwards, Ed Hartnett + */ +int +PIOc_def_var_zstandard(int ncid, int varid, int level) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + + /* Get the file info. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Only netCDF-4 files can use this feature. */ + if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C) + return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); + + PLOG((1, "PIOc_def_var_zstandard ncid = %d varid = %d level = %d", + ncid, varid, level)); + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_DEF_VAR_ZSTANDARD; + + if (ios->compmain == MPI_ROOT) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!mpierr) - mpierr = MPI_Bcast(&shuffle, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&deflate, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&deflate_level, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&level, 1, MPI_INT, ios->compmain, ios->intercomm); } /* Handle MPI errors from computation tasks. */ @@ -83,8 +336,8 @@ PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, if (ios->ioproc) { #ifdef _NETCDF4 - if (file->do_io) - ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level); + if (file->do_io) + ierr = nc_def_var_zstandard(file->fh, varid, level); #endif } @@ -96,6 +349,7 @@ PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, return PIO_NOERR; } +#endif /** * This function only applies to netCDF-4 files. When used with netCDF @@ -151,25 +405,25 @@ PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep, char deflate_present = deflatep ? true : false; char deflate_level_present = deflate_levelp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&shuffle_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&shuffle_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (shuffle_present && !mpierr) - mpierr = MPI_Bcast(shufflep, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(shufflep, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&deflate_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&deflate_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (deflate_present && !mpierr) - mpierr = MPI_Bcast(deflatep, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(deflatep, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&deflate_level_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&deflate_level_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (deflate_level_present && !mpierr) - mpierr = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->compmain, ios->intercomm); PLOG((2, "PIOc_inq_var_deflate ncid = %d varid = %d shuffle_present = %d deflate_present = %d " "deflate_level_present = %d", ncid, varid, shuffle_present, deflate_present, deflate_level_present)); @@ -271,21 +525,21 @@ PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunks int msg = PIO_MSG_DEF_VAR_CHUNKING; char chunksizes_present = chunksizesp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&storage, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&storage, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr && chunksizes_present) - mpierr = MPI_Bcast((PIO_Offset *)chunksizesp, ndims, MPI_OFFSET, ios->compmaster, + mpierr = MPI_Bcast((PIO_Offset *)chunksizesp, ndims, MPI_OFFSET, ios->compmain, ios->intercomm); PLOG((2, "PIOc_def_var_chunking ncid = %d varid = %d storage = %d ndims = %d chunksizes_present = %d", ncid, varid, storage, ndims, chunksizes_present)); @@ -396,19 +650,19 @@ PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizes char storage_present = storagep ? true : false; char chunksizes_present = chunksizesp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&storage_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&storage_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_inq_var_chunking ncid = %d varid = %d storage_present = %d chunksizes_present = %d", ncid, varid, storage_present, chunksizes_present)); } @@ -511,15 +765,15 @@ PIOc_def_var_endian(int ncid, int varid, int endian) if (!ios->ioproc) { int msg = PIO_MSG_DEF_VAR_ENDIAN; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&endian, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&endian, 1, MPI_INT, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -592,15 +846,15 @@ PIOc_inq_var_endian(int ncid, int varid, int *endianp) int msg = PIO_MSG_INQ_VAR_ENDIAN; char endian_present = endianp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&endian_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&endian_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -685,19 +939,19 @@ PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems { int msg = PIO_MSG_SET_CHUNK_CACHE; /* Message for async notification. */ - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iotype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -789,19 +1043,19 @@ PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nel char nelems_present = nelemsp ? true : false; char preemption_present = preemptionp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iotype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, ios->compmaster, + mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_get_chunk_cache size_present = %d nelems_present = %d " "preemption_present = %d ", size_present, nelems_present, preemption_present)); @@ -906,19 +1160,19 @@ PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems { int msg = PIO_MSG_SET_VAR_CHUNK_CACHE; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -977,7 +1231,7 @@ PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nel int ierr; /* Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ - PLOG((1, "PIOc_get_var_chunk_cache ncid = %d varid = %d")); + PLOG((1, "PIOc_get_var_chunk_cache ncid = %d varid = %d", ncid, varid)); /* Get the file info. */ if ((ierr = pio_get_file(ncid, &file))) @@ -998,19 +1252,19 @@ PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nel char nelems_present = nelemsp ? true : false; char preemption_present = preemptionp ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, ios->compmaster, + mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); PLOG((2, "PIOc_get_var_chunk_cache size_present = %d nelems_present = %d " "preemption_present = %d ", size_present, nelems_present, preemption_present)); @@ -1052,3 +1306,759 @@ PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nel return PIO_NOERR; } +/* use this variable in the NETCDF library (introduced in v4.9.0) to determine if the following + functions are available */ +#ifdef NC_HAS_MULTIFILTERS +/** + * Set the variable filter ids + * + * This function only applies to netCDF-4 files. When used with netCDF + * classic files, the error PIO_ENOTNC4 will be returned. + * + * See the <a + * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF + * variable documentation</a> for details about the operation of this + * function. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable. + * @param id set the filter id. + * @return PIO_NOERR for success, otherwise an error code. + * @ingroup PIO_filters + * @author Jim Edwards/Ed Hartnett + */ +int +PIOc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, unsigned int* params) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + + PLOG((1, "PIOc_def_var_filter ncid = %d varid = %d id = %d nparams = %d", ncid, varid, id, nparams)); +#ifdef DEBUG + for(i=0; i<nparams; i++) + PLOG(1, " param %d %d\n",i, params[i]); +#endif + + /* Get the file info. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Only netCDF-4 files can use this feature. */ + if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C) + return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_DEF_VAR_FILTER; /* Message for async notification. */ + + if (ios->compmain == MPI_ROOT) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&id, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&nparams, 1, PIO_MPI_SIZE_T, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(params, nparams, MPI_UNSIGNED, ios->compmain, ios->intercomm); + + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + if (file->do_io) + ierr = nc_def_var_filter(file->fh, varid, id, nparams, params); + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + return PIO_NOERR; +} +#ifdef PIO_HAS_PAR_FILTERS +/** + * Get the variable filter ids if any + * + * This function only applies to netCDF-4 files. When used with netCDF + * classic files, the error PIO_ENOTNC4 will be returned. + * + * Note that these settings are not part of the data file - they apply + * only to the open file as long as it is open. + * + * See the <a + * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF + * variable documentation</a> for details about the operation of this + * function. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable. + * @param nfiltersp Pointer to the number of filters; may be 0. + * @param ids return the filter ids. + * @return PIO_NOERR for success, otherwise an error code. + * @ingroup PIO_filters + * @author Jim Edwards/Ed Hartnett + */ +int +PIOc_inq_var_filter_ids(int ncid, int varid, size_t *nfiltersp, unsigned int *ids) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + + PLOG((1, "PIOc_inq_var_filter_ids ncid = %d varid = %d", ncid, varid)); + + /* Get the file info. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Only netCDF-4 files can use this feature. */ + if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C) + return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_VAR_FILTER_IDS; /* Message for async notification. */ + char cnt_present = nfiltersp ? true : false; + char ids_present = ids ? true : false; + + if (ios->compmain == MPI_ROOT) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&cnt_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ids_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); + if(!mpierr && ids_present){ + size_t idcnt; + idcnt = sizeof(ids); + mpierr = MPI_Bcast(&idcnt, 1, PIO_MPI_SIZE_T, ios->compmain, ios->intercomm); + } + + PLOG((2, "PIOc_inq_var_filter_ids cnt_present = %d ids_present = %d", + cnt_present, ids_present)); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + if (file->do_io) + ierr = nc_inq_var_filter_ids(file->fh, varid, nfiltersp, ids); + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. */ + if (nfiltersp && !ierr) + if ((mpierr = MPI_Bcast(nfiltersp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if ((*nfiltersp)> 0 && ids && !ierr) + if ((mpierr = MPI_Bcast(ids, *nfiltersp, MPI_UNSIGNED, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + + return PIO_NOERR; +} + +/** + * Get the variable filter info if any + * + * This function only applies to netCDF-4 files. When used with netCDF + * classic files, the error PIO_ENOTNC4 will be returned. + * + * Note that these settings are not part of the data file - they apply + * only to the open file as long as it is open. + * + * See the <a + * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF + * variable documentation</a> for details about the operation of this + * function. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable. + * @param id The filter id of interest + * @param nparamsp (OUT) Storage which will get the number of parameters to the filter + * @param params (OUT) Storage which will get the associated parameters. + * @return PIO_NOERR for success, otherwise an error code. + * @ingroup PIO_filters + * @author Jim Edwards/Ed Hartnett + */ +int +PIOc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t *nparamsp, unsigned int *params ) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + + PLOG((1, "PIOc_inq_var_filter_info ncid = %d varid = %d id=%d", ncid, varid, id)); + + /* Get the file info. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Only netCDF-4 files can use this feature. */ + if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C) + return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_VAR_FILTER_INFO; /* Message for async notification. */ + char nparamsp_present = nparamsp ? true : false; + char params_present = params ? true : false; + + if (ios->compmain == MPI_ROOT) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&id, 1, MPI_UNSIGNED, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&nparamsp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(¶ms_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); + if(!mpierr && params_present){ + size_t paramsize; + paramsize = sizeof(params); + mpierr = MPI_Bcast(¶msize, 1, PIO_MPI_SIZE_T, ios->compmain, ios->intercomm); + } + PLOG((2, "PIOc_inq_var_filter_info nparamsp_present = %d params_present = %d ", + nparamsp_present, params_present)); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + if (file->do_io) + ierr = nc_inq_var_filter_info(file->fh, varid, id, nparamsp, params); + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. */ + if (nparamsp && !ierr) + if ((mpierr = MPI_Bcast(nparamsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if ((*nparamsp)> 0 && params && !ierr) + if ((mpierr = MPI_Bcast(params, *(nparamsp), MPI_UNSIGNED, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + + return PIO_NOERR; +} + +#ifdef NC_HAS_BZ2 +/** + * Get the variable bzip2 filter info if any + * + * This function only applies to netCDF-4 files. When used with netCDF + * classic files, the error PIO_ENOTNC4 will be returned. + * + * + * See the <a + * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF + * variable documentation</a> for details about the operation of this + * function. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable. + * @param hasfilterp (OUT) Pointer that gets a 0 if bzip2 is not in use for this var and a 1 if it is. Ignored if NULL + * @param levelp (OUT) Pointer that gets the level setting (1 - 9) Ignored if NULL + * @return PIO_NOERR for success, otherwise an error code. + * @ingroup PIO_filters + * @author Jim Edwards/Ed Hartnett + */ +int +PIOc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + + PLOG((1, "PIOc_inq_var_bzip2 ncid = %d varid = %d", ncid, varid)); + + /* Get the file info. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Only netCDF-4 files can use this feature. */ + if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C) + return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_VAR_BZIP2; /* Message for async notification. */ + char hasfilterp_present = hasfilterp ? true : false; + char levelp_present = levelp ? true : false; + + if (ios->compmain == MPI_ROOT) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&hasfilterp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&levelp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); + PLOG((2, "PIOc_inq_var_bzip2 hasfilterp_present = %d levelp_present = %d ", + hasfilterp_present, levelp_present)); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + if (file->do_io) + ierr = nc_inq_var_bzip2(file->fh, varid, hasfilterp, levelp); + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. */ + if (hasfilterp && !ierr) + if ((mpierr = MPI_Bcast(hasfilterp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + + if (levelp && !ierr) + if ((mpierr = MPI_Bcast(levelp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + + return PIO_NOERR; +} +#endif +#ifdef NC_HAS_ZSTD +/** + * Get the variable zstandard filter info if any + * + * This function only applies to netCDF-4 files. When used with netCDF + * classic files, the error PIO_ENOTNC4 will be returned. + * + * + * See the <a + * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF + * variable documentation</a> for details about the operation of this + * function. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable. + * @param hasfilterp (OUT) Pointer that gets a 0 if zstandard is not in use for this var and a 1 if it is. Ignored if NULL + * @param levelp (OUT) Pointer that gets the level setting (1 - 9) Ignored if NULL + * @return PIO_NOERR for success, otherwise an error code. + * @ingroup PIO_filters + * @author Jim Edwards/Ed Hartnett + */ +int +PIOc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + + PLOG((1, "PIOc_inq_var_zstandard ncid = %d varid = %d", ncid, varid)); + + /* Get the file info. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Only netCDF-4 files can use this feature. */ + if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C) + return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_VAR_ZSTANDARD; /* Message for async notification. */ + char hasfilterp_present = hasfilterp ? true : false; + char levelp_present = levelp ? true : false; + + if (ios->compmain == MPI_ROOT) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&hasfilterp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&levelp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); + PLOG((2, "PIOc_inq_var_zstandard hasfilterp_present = %d levelp_present = %d ", + hasfilterp_present, levelp_present)); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + if (file->do_io) + ierr = nc_inq_var_zstandard(file->fh, varid, hasfilterp, levelp); + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. */ + if (hasfilterp && !ierr) + if ((mpierr = MPI_Bcast(hasfilterp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + + if (levelp && !ierr) + if ((mpierr = MPI_Bcast(levelp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + + return PIO_NOERR; +} +// NC_HAS_ZSTD +#endif +#endif +#ifdef PIO_HAS_PAR_FILTERS +/** + * + * + * This function only applies to netCDF-4 files. When used with netCDF + * classic files, the error PIO_ENOTNC4 will be returned. + * + * Note that these settings are not part of the data file - they apply + * only to the open file as long as it is open. + * + * See the <a + * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF + * variable documentation</a> for details about the operation of this + * function. + * + * @param ncid the ncid of the open file. + * @param id the filter of interest + * @return PIO_NOERR if the filter is available, PIO_ENOFILTER if unavailable + * @ingroup PIO_filters + * @author Jim Edwards/Ed Hartnett + */ +int +PIOc_inq_filter_avail(int ncid, unsigned int id ) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + + PLOG((1, "PIOc_inq_filter_avail ncid = %d id = %d ", ncid, id)); + + /* Get the file info. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Only netCDF-4 files can use this feature. */ + if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C) + return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_FILTER_AVAIL; /* Message for async notification. */ + + if (ios->compmain == MPI_ROOT) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&id, 1, MPI_INT, ios->compmain, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + if (file->do_io) + ierr = nc_inq_filter_avail(file->fh, id); + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (ierr && ierr !=NC_ENOFILTER) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. */ + + return ierr; +} +// PIO_HAS_PAR_FILTERS +#endif +// NC_HAS_MULTIFILTERS +#endif +#ifdef NC_HAS_QUANTIZE +/** + * Turn on quantization for a variable + * + * This function only applies to netCDF-4 files. When used with netCDF + * classic files, the error PIO_ENOTNC4 will be returned. + * + * Note that these settings are not part of the data file - they apply + * only to the open file as long as it is open. + * + * See the <a + * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF + * variable documentation</a> for details about the operation of this + * function. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable. + * @param quantize_mode + * @param nsd Number of significant digits. + * @return PIO_NOERR for success, otherwise an error code. + * @ingroup PIO_inq_var_c + * @author Jim Edwards/Ed Hartnett + */ +int +PIOc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd ) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + + PLOG((1, "PIOc_def_var_quantize ncid = %d varid = %d quantize_mode=%d nsd=%d", ncid, varid, quantize_mode, nsd)); + + /* Get the file info. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Only netCDF-4 files can use this feature. */ + if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C) + return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_DEF_VAR_QUANTIZE; /* Message for async notification. */ + + if (ios->compmain == MPI_ROOT) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&quantize_mode, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&nsd, 1, MPI_INT, ios->compmain, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + if (file->do_io) + ierr = nc_def_var_quantize(file->fh, varid, quantize_mode, nsd); + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + return PIO_NOERR; +} + +/** + * Learn whether quantization is on for a variable, and, if so, the NSD setting. + * + * This function only applies to netCDF-4 files. When used with netCDF + * classic files, the error PIO_ENOTNC4 will be returned. + * + * Note that these settings are not part of the data file - they apply + * only to the open file as long as it is open. + * + * See the <a + * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF + * variable documentation</a> for details about the operation of this + * function. + * + * @param ncid the ncid of the open file. + * @param varid the ID of the variable. + * @param quantize_modep Pointer that gets a 0 if quantization is not in use for this var, and a 1 if it is. Ignored if NULL. + * @param nsdp Pointer that gets the NSD setting (from 1 to 15), if quantization is in use. Ignored if NULL. + * @return PIO_NOERR for success, otherwise an error code. + * @ingroup PIO_inq_var_c + * @author Jim Edwards/Ed Hartnett + */ +int +PIOc_inq_var_quantize(int ncid, int varid, int *quantize_mode, int *nsdp ) +{ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + + PLOG((1, "PIOc_inq_var_quantize ncid = %d varid = %d ", ncid, varid)); + + /* Get the file info. */ + if ((ierr = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ierr, __FILE__, __LINE__); + ios = file->iosystem; + + /* Only netCDF-4 files can use this feature. */ + if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C) + return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__); + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_VAR_QUANTIZE; /* Message for async notification. */ + char qmode_present = quantize_mode ? true : false; + char nsdp_present = nsdp ? true : false; + + if (ios->compmain == MPI_ROOT) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&qmode_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&nsdp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); + + PLOG((2, "PIOc_inq_var_quantize qmode_present = %d nsdp_present = %d ", + qmode_present, nsdp_present)); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + if (file->do_io) + ierr = nc_inq_var_quantize(file->fh, varid, quantize_mode, nsdp); + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. */ + if (quantize_mode && !ierr) + if ((mpierr = MPI_Bcast(quantize_mode, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + if (nsdp && !ierr) + if ((mpierr = MPI_Bcast(nsdp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); + + return PIO_NOERR; +} +#endif diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_rearrange.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_rearrange.c index fb174c3a3f..2aabacb5bd 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_rearrange.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_rearrange.c @@ -11,7 +11,7 @@ #if PIO_USE_MPISERIAL #define MPI_Type_create_hvector MPI_Type_hvector #endif -#define SIZEOF_MPI_OFFSET sizeof(MPI_Offset) + /** * Convert a 1-D index into a coordinate value in an arbitrary * dimension space. E.g., for index 4 into a array defined as a[3][2], @@ -35,7 +35,7 @@ idx_to_dim_list(int ndims, const int *gdimlen, PIO_Offset idx, /* Check inputs. */ pioassert(ndims >= 0 && gdimlen && idx >= -1 && dim_list, "invalid input", __FILE__, __LINE__); - PLOG((2, "idx_to_dim_list ndims = %d idx = %d", ndims, idx)); + PLOG((3, "idx_to_dim_list ndims = %d idx = %d", ndims, idx)); /* Easiest to start from the right and move left. */ for (int i = ndims - 1; i >= 0; --i) @@ -314,7 +314,7 @@ create_mpi_datatypes(MPI_Datatype mpitype, int msgcnt, if (mindex) { for(int j=0; j<numinds; j++) - PLOG((3,"mindex[%d] = %d",j,mindex[j])); + PLOG((5,"mindex[%d] = %d",j,mindex[j])); if (!(lindex = malloc(numinds * sizeof(PIO_Offset)))) return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); memcpy(lindex, mindex, (size_t)(numinds * sizeof(PIO_Offset))); @@ -476,7 +476,7 @@ define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc) /* The different rearrangers get different values for mfrom. */ int *mfrom = iodesc->rearranger == PIO_REARR_SUBSET ? iodesc->rfrom : NULL; - + /* Create the MPI datatypes. */ PLOG((2, "Calling create_mpi_datatypes at line %d ",__LINE__)); if ((ret = create_mpi_datatypes(iodesc->mpitype, iodesc->nrecvs, iodesc->rindex, @@ -510,7 +510,7 @@ define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc) /* Remember how many types we created for the send side. */ iodesc->num_stypes = ntypes; - + /* Create the MPI data types. */ PLOG((2, "Calling create_mpi_datatypes at line %d",__LINE__)); if ((ret = create_mpi_datatypes(iodesc->mpitype, ntypes, iodesc->sindex, @@ -1113,7 +1113,7 @@ rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, /* Data in sbuf on the ionodes is sent to rbuf on the compute * nodes. */ - + if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, rbuf, recvcounts, rdispls, recvtypes, mycomm, &iodesc->rearr_opts.io2comp))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); @@ -2260,7 +2260,6 @@ subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, PIO_Offset soffset; /* we only want a single copy of each source point in the iobuffer but it may be sent to multiple destinations in a read operation */ - int k=0; // PIO_Offset previomap[ntasks]; // for (i = 0; i < ntasks; i++) // previomap[i] = -1; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_spmd.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_spmd.c index 090e08ff82..68ee05fd1d 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_spmd.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pio_spmd.c @@ -364,12 +364,13 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty * them here. */ if (steps > 0) { - PLOG((2, "Waiting for outstanding msgs")); - if ((mpierr = MPI_Waitall(steps, rcvids, MPI_STATUSES_IGNORE))) - return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); - if (fc->isend) - if ((mpierr = MPI_Waitall(steps, sndids, MPI_STATUSES_IGNORE))) - return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); + MPI_Status statuses[steps]; + PLOG((2, "Waiting for outstanding msgs")); + if ((mpierr = MPI_Waitall(steps, rcvids, statuses))) + return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); + if (fc->isend) + if ((mpierr = MPI_Waitall(steps, sndids, statuses))) + return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__); } return PIO_NOERR; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc.c index 5ff24c7da9..1be5923457 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc.c @@ -207,13 +207,13 @@ PIOc_advanceframe(int ncid, int varid) { int msg = PIO_MSG_ADVANCEFRAME; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -269,15 +269,15 @@ PIOc_setframe(int ncid, int varid, int frame) { int msg = PIO_MSG_SETFRAME; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&frame, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&frame, 1, MPI_INT, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -352,14 +352,8 @@ PIOc_get_local_array_size(int ioid) int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method) { - iosystem_desc_t *ios; int oldmethod; - /* Get the iosystem info. */ - if (iosysid != PIO_DEFAULT) - if (!(ios = pio_get_iosystem_from_id(iosysid))) - piodie("Could not find IO system.", __FILE__, __LINE__); - /* Set the error handler. */ if (PIOc_set_iosystem_error_handling(iosysid, method, &oldmethod)) piodie("Could not set the IOSystem error hanlder", __FILE__, __LINE__); @@ -408,13 +402,13 @@ PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method) int msg = PIO_MSG_SETERRORHANDLING; char old_method_present = old_method ? true : false; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&method, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&method, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&old_method_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&old_method_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -541,37 +535,37 @@ PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int ma char rearranger_present = rearranger ? true : false; char iostart_present = iostart ? true : false; char iocount_present = iocount ? true : false; - if (ios->compmaster == MPI_ROOT){ + if (ios->compmain == MPI_ROOT){ PLOG((1, "about to sent msg %d union_comm %d",msg,ios->union_comm)); mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); } if (!mpierr) - mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&pio_type, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&pio_type, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((int *)gdimlen, ndims, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((int *)gdimlen, ndims, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&maplen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&maplen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((PIO_Offset *)compmap, maplen, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)compmap, maplen, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&rearranger_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&rearranger_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (rearranger_present && !mpierr) - mpierr = MPI_Bcast((int *)rearranger, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((int *)rearranger, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&iostart_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iostart_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (iostart_present && !mpierr) - mpierr = MPI_Bcast((PIO_Offset *)iostart, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)iostart, ndims, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&iocount_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iocount_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (iocount_present && !mpierr) - mpierr = MPI_Bcast((PIO_Offset *)iocount, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)iocount, ndims, MPI_OFFSET, ios->compmain, ios->intercomm); PLOG((2, "PIOc_InitDecomp iosysid = %d pio_type = %d ndims = %d maplen = %d rearranger_present = %d iostart_present = %d " "iocount_present = %d ", iosysid, pio_type, ndims, maplen, rearranger_present, iostart_present, iocount_present)); } @@ -856,37 +850,37 @@ PIOc_InitDecomp_ReadOnly(int iosysid, int pio_type, int ndims, const int *gdimle char rearranger_present = rearranger ? true : false; char iostart_present = iostart ? true : false; char iocount_present = iocount ? true : false; - if (ios->compmaster == MPI_ROOT){ + if (ios->compmain == MPI_ROOT){ PLOG((1, "about to sent msg %d union_comm %d",msg,ios->union_comm)); mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); } if (!mpierr) - mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&pio_type, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&pio_type, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((int *)gdimlen, ndims, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((int *)gdimlen, ndims, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&maplen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&maplen, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((PIO_Offset *)compmap, maplen, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)compmap, maplen, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&rearranger_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&rearranger_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (rearranger_present && !mpierr) - mpierr = MPI_Bcast((int *)rearranger, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((int *)rearranger, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&iostart_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iostart_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (iostart_present && !mpierr) - mpierr = MPI_Bcast((PIO_Offset *)iostart, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)iostart, ndims, MPI_OFFSET, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&iocount_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iocount_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (iocount_present && !mpierr) - mpierr = MPI_Bcast((PIO_Offset *)iocount, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)iocount, ndims, MPI_OFFSET, ios->compmain, ios->intercomm); PLOG((2, "PIOc_InitDecomp iosysid = %d pio_type = %d ndims = %d maplen = %d rearranger_present = %d iostart_present = %d " "iocount_present = %d ", iosysid, pio_type, ndims, maplen, rearranger_present, iostart_present, iocount_present)); } @@ -974,7 +968,6 @@ PIOc_InitDecomp_ReadOnly(int iosysid, int pio_type, int ndims, const int *gdimle return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); } - PLOG((2, "iodesc->rearranger = %d", iodesc->rearranger)); /* Is this the subset rearranger? */ @@ -1140,7 +1133,7 @@ PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int m /* Add 1 to all elements in compmap. */ for (int e = 0; e < maplen; e++) { - PLOG((3, "zero-based compmap[%d] = %d", e, compmap[e])); + PLOG((5, "zero-based compmap[%d] = %d", e, compmap[e])); compmap_1_based[e] = compmap[e] + 1; } @@ -1358,9 +1351,9 @@ PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, for (int i = 0; i < ios->num_comptasks; i++) ios->compranks[i] = i; - /* Is this the comp master? */ + /* Is this the comp main? */ if (ios->comp_rank == 0) - ios->compmaster = MPI_ROOT; + ios->compmain = MPI_ROOT; PLOG((2, "comp_rank = %d num_comptasks = %d", ios->comp_rank, ios->num_comptasks)); /* Create an array that holds the ranks of the tasks to be used @@ -1381,7 +1374,7 @@ PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, /* Identify the task that will be the root of the IO communicator. */ if (ios->comp_rank == ios->ioranks[0]) - ios->iomaster = MPI_ROOT; + ios->iomain = MPI_ROOT; /* Create a group for the computation tasks. */ if ((mpierr = MPI_Comm_group(ios->comp_comm, &compgroup))) @@ -1539,7 +1532,7 @@ PIOc_free_iosystem(int iosysid) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); /* If asynch IO is in use, send the PIO_MSG_EXIT message from the - * comp master to the IO processes. This may be called by + * comp main to the IO processes. This may be called by * componets for other components iosysid. So don't send unless * there is a valid union_comm. */ if (ios->async && ios->union_comm != MPI_COMM_NULL) @@ -1554,12 +1547,12 @@ PIOc_free_iosystem(int iosysid) ios->ioroot, ios->union_comm)); /* Send the message to the message handler. */ - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); /* Send the parameters of the function call. */ if (!mpierr) - mpierr = MPI_Bcast((int *)&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((int *)&iosysid, 1, MPI_INT, ios->compmain, ios->intercomm); } /* Handle MPI errors. */ @@ -1691,7 +1684,6 @@ PIOc_iotype_available(int iotype) #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: return 1; - break; #endif default: return 0; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc_async.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc_async.c index 7927fb34d7..4ac8e68fd6 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc_async.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc_async.c @@ -30,7 +30,6 @@ extern int diosysid; #endif /* NETCDF_INTEGRATION */ extern int default_error_handler; /* defined in pioc.c */ - /** * @defgroup PIO_init_async Initialize an ASYNC IO System * Initialize the IOSystem, including specifying number of IO and @@ -198,9 +197,9 @@ PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, /* Rank of current process in IO communicator. */ int io_rank = -1; - /* Set to MPI_ROOT on master process, MPI_PROC_NULL on other + /* Set to MPI_ROOT on main process, MPI_PROC_NULL on other * processes. */ - int iomaster; + int iomain; /* Create a group for the IO component. */ if ((ret = MPI_Group_incl(world_group, num_io_procs, my_io_proc_list, &io_group))) @@ -228,9 +227,9 @@ PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, PLOG((3, "about to get io rank")); if ((ret = MPI_Comm_rank(io_comm, &io_rank))) return check_mpi(NULL, NULL, ret, __FILE__, __LINE__); - iomaster = !io_rank ? MPI_ROOT : MPI_PROC_NULL; + iomain = !io_rank ? MPI_ROOT : MPI_PROC_NULL; PLOG((3, "intracomm created for io_comm = %d io_rank = %d IO %s", - io_comm, io_rank, iomaster == MPI_ROOT ? "MASTER" : "SERVANT")); + io_comm, io_rank, iomain == MPI_ROOT ? "main" : "SERVANT")); } /* We will create a group for each computational component. */ @@ -379,14 +378,14 @@ PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, if ((ret = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank))) return check_mpi(NULL, NULL, ret, __FILE__, __LINE__); - /* Set comp_rank 0 to be the compmaster. It will have a + /* Set comp_rank 0 to be the compmain. It will have a * setting of MPI_ROOT, all other tasks will have a * setting of MPI_PROC_NULL. */ - my_iosys->compmaster = my_iosys->comp_rank ? MPI_PROC_NULL : MPI_ROOT; + my_iosys->compmain = my_iosys->comp_rank ? MPI_PROC_NULL : MPI_ROOT; PLOG((3, "intracomm created for cmp = %d comp_comm = %d comp_rank = %d comp %s", cmp, my_iosys->comp_comm, my_iosys->comp_rank, - my_iosys->compmaster == MPI_ROOT ? "MASTER" : "SERVANT")); + my_iosys->compmain == MPI_ROOT ? "main" : "SERVANT")); } /* If this is the IO component, make a copy of the IO comm for @@ -397,7 +396,7 @@ PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, if ((ret = MPI_Comm_dup(io_comm, &my_iosys->io_comm))) return check_mpi(NULL, NULL, ret, __FILE__, __LINE__); PLOG((3, "dup of io_comm = %d io_rank = %d", my_iosys->io_comm, io_rank)); - my_iosys->iomaster = iomaster; + my_iosys->iomain = iomain; my_iosys->io_rank = io_rank; my_iosys->ioroot = 0; my_iosys->comp_idx = cmp; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc_support.c b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc_support.c index c3d670450b..9d44d1ee7f 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc_support.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/clib/pioc_support.c @@ -204,10 +204,10 @@ int PIOc_set_global_log_level(int iosysid, int level) if(!ios->ioproc) { int msg = PIO_MSG_SETLOGLEVEL; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmain, ios->intercomm); } } @@ -755,11 +755,10 @@ pio_err(iosystem_desc_t *ios, file_desc_t *file, int err_num, const char *fname, MPI_Abort(MPI_COMM_WORLD, -1); } - /* What should we do here??? */ - if (err_handler == PIO_BCAST_ERROR) - { - /* ??? */ - } + /* Do nothing, error is bcast to all tasks and application will handle */ +// if (err_handler == PIO_BCAST_ERROR) +// { +// } /* If abort was not called, we'll get here. */ return err_num; @@ -1016,13 +1015,13 @@ PIOc_freedecomp(int iosysid, int ioid) { int msg = PIO_MSG_FREEDECOMP; /* Message for async notification. */ - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ioid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ioid, 1, MPI_INT, ios->compmain, ios->intercomm); PLOG((2, "PIOc_freedecomp iosysid = %d ioid = %d", iosysid, ioid)); } @@ -1685,8 +1684,8 @@ pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int **glo return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); PLOG((1, "pioc_read_nc_decomp_int iosysid = %d filename = %s", iosysid, filename)); - - + + /* Open the netCDF decomp file. */ if ((ret = PIOc_open(iosysid, filename, NC_WRITE, &ncid))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); @@ -2093,28 +2092,28 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, /* Send the message to the message handler. */ PLOG((3, "msg %d ios->union_comm %d MPI_COMM_NULL %d", msg, ios->union_comm, MPI_COMM_NULL)); - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); /* Send the parameters of the function call. */ if (!mpierr) - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&mode, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&mode, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&ncidp_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncidp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm); if (ncidp_present) if (!mpierr) - mpierr = MPI_Bcast(ncidp, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(ncidp, 1, MPI_INT, ios->compmain, ios->intercomm); #ifdef NETCDF_INTEGRATION if (!mpierr) - mpierr = MPI_Bcast(&diosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&diosysid, 1, MPI_INT, ios->compmain, ios->intercomm); #endif /* NETCDF_INTEGRATION */ PLOG((2, "len %d filename %s iotype %d mode %d use_ext_ncid %d " "ncidp_present %d", len, filename, file->iotype, mode, @@ -2148,8 +2147,10 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, case PIO_IOTYPE_NETCDF: if (!ios->io_rank) { +// PIOc_set_log_level(3); PLOG((2, "Calling nc_create mode = %d", mode)); ierr = nc_create(filename, mode, &file->fh); + PLOG((2, "Called nc_create mode = %d %d %s", mode, ierr, filename)); } break; #ifdef _PNETCDF @@ -2330,7 +2331,7 @@ inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, /* Check inputs. */ pioassert(rec_var && pio_type && pio_type_size && mpi_type && mpi_type_size, "pointers must be provided", __FILE__, __LINE__); - + ret = PIO_NOERR; /* How many vars in the file? */ if (iotype == PIO_IOTYPE_PNETCDF) { @@ -2384,7 +2385,6 @@ inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars, return pio_err(NULL, file, ret, __FILE__, __LINE__); #endif /* _NETCDF4 */ } - /* Learn the unlimited dimension ID(s), if there are any. */ if (nunlimdims) { @@ -2542,7 +2542,6 @@ find_iotype_from_omode(int mode, int *iotype) else *iotype = PIO_IOTYPE_NETCDF; } - return PIO_NOERR; } @@ -2623,12 +2622,12 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, int *mpi_type_size = NULL; int *ndims = NULL; int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - int ierr = PIO_NOERR; /* Return code from function calls. */ + int ierr; /* Return code from function calls. */ #ifdef USE_MPE pio_start_mpe_log(OPEN); #endif /* USE_MPE */ - + ierr = PIO_NOERR; /* Return code from function calls. */ /* Get the IO system info from the iosysid. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); @@ -2639,8 +2638,8 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, if (*iotype < PIO_IOTYPE_PNETCDF || *iotype > PIO_IOTYPE_NETCDF4P) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); - PLOG((2, "PIOc_openfile_retry iosysid = %d iotype = %d filename = %s mode = %d retry = %d", - iosysid, *iotype, filename, mode, retry)); + PLOG((2, "PIOc_openfile_retry iosysid = %d iotype = %d filename = %s mode = %d retry = %d ierr=%d", + iosysid, *iotype, filename, mode, retry, ierr)); /* Allocate space for the file info. */ if (!(file = calloc(sizeof(*file), 1))) @@ -2667,23 +2666,23 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, if (!ios->ioproc) { /* Send the message to the message handler. */ - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); /* Send the parameters of the function call. */ if (!mpierr) - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&mode, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&mode, 1, MPI_INT, ios->compmain, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, ios->compmain, ios->intercomm); #ifdef NETCDF_INTEGRATION if (!mpierr) - mpierr = MPI_Bcast(&diosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&diosysid, 1, MPI_INT, ios->compmain, ios->intercomm); #endif /* NETCDF_INTEGRATION */ } @@ -2693,6 +2692,8 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, if (mpierr) return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); } + PLOG((2, "%d: PIOc_openfile_retry ierr=%d",__LINE__,ierr)); + /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) @@ -2707,9 +2708,11 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, #else imode = mode | NC_MPIIO; if ((ierr = nc_open_par(filename, imode, ios->io_comm, ios->info, - &file->fh))) - break; + &file->fh))){ + PLOG((2, "%d: PIOc_openfile_retry nc_open_par ierr=%d",__LINE__,ierr)); + break; + } /* Check the vars for valid use of unlim dims. */ if ((ierr = check_unlim_use(file->fh))) break; @@ -2727,11 +2730,15 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, case PIO_IOTYPE_NETCDF4C: if (ios->io_rank == 0) { - if ((ierr = nc_open(filename, mode, &file->fh))) + if ((ierr = nc_open(filename, mode, &file->fh))){ + PLOG((2, "%d: PIOc_openfile_retry ierr=%d filename=%s mode=%d",__LINE__,ierr, filename, mode)); break; + } /* Check the vars for valid use of unlim dims. */ - if ((ierr = check_unlim_use(file->fh))) + if ((ierr = check_unlim_use(file->fh))){ + PLOG((2, "%d: PIOc_openfile_retry ierr=%d",__LINE__,ierr)); break; + } ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF4C, &nvars, &rec_var, &pio_type, &pio_type_size, &mpi_type, @@ -2739,6 +2746,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, PLOG((2, "PIOc_openfile_retry:nc_open for 4C filename = %s mode = %d " "ierr = %d", filename, mode, ierr)); } + PLOG((2, "%d: PIOc_openfile_retry ierr=%d",__LINE__,ierr)); break; #endif /* _NETCDF4 */ @@ -2763,7 +2771,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, // This should only be done with a file opened to append if (ierr == PIO_NOERR && (mode & PIO_WRITE)) { - if (ios->iomaster == MPI_ROOT) + if (ios->iomain == MPI_ROOT) PLOG((2, "%d Setting IO buffer %ld", __LINE__, pio_pnetcdf_buffer_size_limit)); ierr = ncmpi_buffer_attach(file->fh, pio_pnetcdf_buffer_size_limit); @@ -2782,15 +2790,17 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__); } + PLOG((2, "%d: PIOc_openfile_retry ierr=%d",__LINE__,ierr)); + /* If the caller requested a retry, and we failed to open a file due to an incompatible type of NetCDF, try it once with just plain old basic NetCDF. */ if (retry) { PLOG((2, "retry error code ierr = %d io_rank %d", ierr, ios->io_rank)); - if ((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) + if ((ierr == NC_ENOTNC || ierr == NC_EINVAL || ierr == NC_ENOTBUILT) && (file->iotype != PIO_IOTYPE_NETCDF)) { - if (ios->iomaster == MPI_ROOT) + if (ios->iomain == MPI_ROOT) printf("PIO2 pio_file.c retry NETCDF\n"); /* reset ierr on all tasks */ @@ -2821,6 +2831,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, if (ios->ioroot == ios->union_rank) PLOG((2, "Bcasting error code ierr %d ios->ioroot %d ios->my_comm %d", ierr, ios->ioroot, ios->my_comm)); + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(NULL, file, mpierr, __FILE__, __LINE__); PLOG((2, "Bcast openfile_retry error code ierr = %d", ierr)); @@ -3026,13 +3037,13 @@ pioc_change_def(int ncid, int is_enddef) if (!ios->ioproc) { int msg = is_enddef ? PIO_MSG_ENDDEF : PIO_MSG_REDEF; - if (ios->compmaster == MPI_ROOT) + if (ios->compmain == MPI_ROOT) { PLOG((2, "pioc_change_def request sent")); mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); } if (!mpierr) - mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm); PLOG((3, "pioc_change_def ncid = %d mpierr = %d", ncid, mpierr)); } @@ -3100,15 +3111,15 @@ iotype_is_valid(int iotype) ret++; /* Some builds include netCDF-4. */ -#ifdef _NETCDF4 +#ifdef NC_HAS_NC4 if (iotype == PIO_IOTYPE_NETCDF4C || iotype == PIO_IOTYPE_NETCDF4P) ret++; #endif /* _NETCDF4 */ /* Some builds include pnetcdf. */ +#ifdef _PNETCDF if (iotype == PIO_IOTYPE_PNETCDF) ret++; -#ifdef _PNETCDF #endif /* _PNETCDF */ return ret; @@ -3250,7 +3261,7 @@ determine_procs(int num_io_procs, int component_count, int *num_procs_per_comp, /** * Used in check_compmap to sort the compmap in accending order. - * + * * @param a pointer to an offset * @param b pointer to another offset * @returns 0 if offsets are the same or if either pointer is NULL @@ -3264,7 +3275,7 @@ int offsetsort(const void *a,const void *b) /** * check_compmap gathers the entire compmap to comp task 0, sorts it into accending order - * then looks for repeated values > 0. If any repeated values are found the iodesc is marked + * then looks for repeated values > 0. If any repeated values are found the iodesc is marked * read only. * * @param ios pointer to the iosystem_desc_t struct. @@ -3282,11 +3293,11 @@ bool check_compmap(iosystem_desc_t *ios, io_desc_t *iodesc,const PIO_Offset *com int ierr; bool readonly = 0; - if(ios->compproc) + if(ios->compproc) { #ifdef OLDWAY int *gmaplen; - if(ios->compmaster == MPI_ROOT) + if(ios->compmain == MPI_ROOT) gmaplen = malloc(ios->num_comptasks * sizeof(int)); else gmaplen = NULL; @@ -3323,7 +3334,7 @@ bool check_compmap(iosystem_desc_t *ios, io_desc_t *iodesc,const PIO_Offset *com int *displs; int gcompmaplen=0; int *gcompmaps; - if(ios->compmaster == MPI_ROOT) + if(ios->compmain == MPI_ROOT) { displs = malloc(ios->num_comptasks * sizeof(int)); displs[0] = 0; @@ -3332,7 +3343,7 @@ bool check_compmap(iosystem_desc_t *ios, io_desc_t *iodesc,const PIO_Offset *com displs[i] = displs[i-1] + gmaplen[i-1]; } gcompmaplen = displs[ios->num_comptasks-1] + gmaplen[ios->num_comptasks-1]; - gcompmaps = malloc(gcompmaplen * sizeof(PIO_Offset)); + gcompmaps = malloc(gcompmaplen * sizeof(PIO_Offset)); // printf("gcompmaplen %d\n",gcompmaplen); // for(int i=0;i<ios->num_comptasks; i++) // printf("gmaplen=%d displs[%d]=%d\n",gmaplen[i], i,displs[i]); @@ -3342,7 +3353,7 @@ bool check_compmap(iosystem_desc_t *ios, io_desc_t *iodesc,const PIO_Offset *com if ((ierr = MPI_Gatherv(compmap, iodesc->maplen, MPI_OFFSET, gcompmaps, gmaplen, displs, MPI_OFFSET, 0, ios->comp_comm))) return check_mpi(ios, NULL, ierr, __FILE__,__LINE__); - if(ios->compmaster == MPI_ROOT) + if(ios->compmain == MPI_ROOT) { /* sort */ qsort(gcompmaps, gcompmaplen, sizeof(MPI_OFFSET), offsetsort); @@ -3359,7 +3370,7 @@ bool check_compmap(iosystem_desc_t *ios, io_desc_t *iodesc,const PIO_Offset *com free(displs); free(gcompmaps); } - + } MPI_Bcast(&readonly, 1, MPI_CHAR, ios->comproot, ios->my_comm); #else diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/flib/CMakeLists.txt b/src/Infrastructure/IO/PIO/ParallelIO/src/flib/CMakeLists.txt index d168039951..11b60e7db1 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/flib/CMakeLists.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/flib/CMakeLists.txt @@ -200,6 +200,7 @@ else () target_compile_definitions (gptl PUBLIC NO_MPIMOD) endif() + target_compile_definitions (piof PUBLIC NO_MPIMOD) endif () #===== GPTL ===== diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio.F90 b/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio.F90 index fabe61a7e9..f94ed7ce11 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio.F90 @@ -7,6 +7,7 @@ !> !! @defgroup PIO_set_blocksize Box Rearranger Settings !! Set the box rearranger blocksize in Fortran. +#include <netcdf_meta.h> #include "config.h" module pio @@ -16,6 +17,14 @@ module pio use pio_kinds, only : pio_offset_kind + use pionfatt_mod, only : PIO_put_att => put_att, & + PIO_get_att => get_att, & + PIO_inq_var_fill => inq_var_fill + use pionfput_mod, only : PIO_put_var => put_var + use pionfget_mod, only : PIO_get_var => get_var + use pio_support, only: pio_writedof, pio_readdof, pio_write_nc_dof, pio_read_nc_dof + use iso_c_binding + use piolib_mod, only : pio_initdecomp, & pio_openfile, pio_closefile, pio_createfile, pio_setdebuglevel, & pio_seterrorhandling, pio_setframe, pio_init, pio_get_local_array_size, & @@ -41,8 +50,11 @@ module pio pio_max_name, pio_max_var_dims, pio_rearr_subset, pio_rearr_box, & pio_nofill, pio_unlimited, pio_fill_int, pio_fill_double, pio_fill_float, & pio_64bit_offset, pio_64bit_data, pio_fill, & +#ifdef NC_HAS_QUANTIZE + PIO_NOQUANTIZE, PIO_QUANTIZE_BITGROOM, PIO_QUANTIZE_GRANULARBR, PIO_QUANTIZE_BITROUND, & +#endif + ! last line of use clause needs to be outside of macro pio_internal_error, pio_bcast_error, pio_return_error, pio_default - use piodarray, only : pio_read_darray, pio_write_darray, pio_set_buffer_size_limit use pio_nf, only: & @@ -64,7 +76,25 @@ module pio PIO_inq_unlimdim, & PIO_def_dim , & PIO_def_var , & - PIO_def_var_deflate , & +#ifdef PIO_HAS_PAR_FILTERS +#ifdef NC_HAS_BZ + PIO_def_var_bzip2, & +#endif +#ifdef NC_HAS_ZSTD + PIO_def_var_zstandard, & +#endif +#ifdef NC_HAS_QUANTIZE + PIO_def_var_quantize , & + PIO_inq_var_quantize , & +#endif + PIO_inq_var_filter_ids , & + PIO_inq_var_filter_info , & + PIO_inq_filter_avail , & + PIO_def_var_szip, & +#endif + PIO_def_var_deflate ,& + PIO_def_var_chunking, & + PIO_inq_var_chunking, & PIO_redef , & PIO_set_log_level, & PIO_inquire_variable , & @@ -76,13 +106,6 @@ module pio PIO_set_fill, & PIO_strerror - use pionfatt_mod, only : PIO_put_att => put_att, & - PIO_get_att => get_att, & - PIO_inq_var_fill => inq_var_fill - use pionfput_mod, only : PIO_put_var => put_var - use pionfget_mod, only : PIO_get_var => get_var - use pio_support, only: pio_writedof, pio_readdof, pio_write_nc_dof, pio_read_nc_dof - use iso_c_binding implicit none public diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_nf.F90 b/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_nf.F90 index f08ff754cc..db08a7725b 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_nf.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_nf.F90 @@ -1,3 +1,4 @@ +#include <netcdf_meta.h> #include "config.h" !> !! @file @@ -134,6 +135,26 @@ module pio_nf pio_redef , & pio_set_log_level , & pio_strerror , & +#ifdef PIO_HAS_PAR_FILTERS +#ifdef NC_HAS_QUANTIZE + pio_def_var_quantize , & + pio_inq_var_quantize , & +#endif +#ifdef NC_HAS_MULTIFILTERS +#ifdef NC_HAS_BZ + pio_inq_var_bzip2 , & + pio_def_var_bzip2 , & +#endif +#ifdef NC_HAS_ZSTD + pio_inq_var_zstandard , & + pio_def_var_zstandard , & +#endif + pio_def_var_szip , & + pio_inq_var_filter_ids , & + pio_inq_var_filter_info , & + pio_inq_filter_avail , & +#endif +#endif pio_set_fill ! pio_copy_att to be done @@ -151,8 +172,46 @@ module pio_nf end interface pio_def_var_deflate interface pio_def_var_chunking module procedure & - def_var_chunking_desc + def_var_chunking_desc, & + def_var_chunking_int, & + def_var_chunking_vid end interface pio_def_var_chunking +#ifdef PIO_HAS_PAR_FILTERS +#ifdef NC_HAS_BZ + interface pio_def_var_bzip2 + module procedure & + def_var_bzip2_desc, & + def_var_bzip2_int, & + def_var_bzip2_vid + end interface pio_def_var_bzip2 + interface pio_inq_var_bzip2 + module procedure & + inq_var_bzip2_desc , & + inq_var_bzip2_vid , & + inq_var_bzip2_id + end interface pio_inq_var_bzip2 +#endif + interface pio_def_var_szip + module procedure & + def_var_szip_desc, & + def_var_szip_int, & + def_var_szip_vid + end interface pio_def_var_szip +#ifdef NC_HAS_ZSTD + interface pio_def_var_zstandard + module procedure & + def_var_zstandard_desc, & + def_var_zstandard_int, & + def_var_zstandard_vid + end interface pio_def_var_zstandard + interface pio_inq_var_zstandard + module procedure & + inq_var_zstandard_desc , & + inq_var_zstandard_vid , & + inq_var_zstandard_id + end interface pio_inq_var_zstandard +#endif +#endif interface pio_inq_attname module procedure & inq_attname_desc , & @@ -215,6 +274,7 @@ module pio_nf inq_var_deflate_vid , & inq_var_deflate_id end interface pio_inq_var_deflate + interface pio_inq_var_chunking module procedure & inq_var_chunking_desc , & @@ -337,7 +397,35 @@ module pio_nf get_var_chunk_cache_desc , & get_var_chunk_cache_id end interface pio_get_var_chunk_cache - +#ifdef NC_HAS_QUANTIZE + interface pio_def_var_quantize + module procedure & + def_var_quantize_desc , & + def_var_quantize_id + end interface pio_def_var_quantize + interface pio_inq_var_quantize + module procedure & + inq_var_quantize_desc , & + inq_var_quantize_id + end interface pio_inq_var_quantize +#endif +#ifdef PIO_HAS_PAR_FILTERS + interface pio_inq_var_filter_ids + module procedure & + inq_var_filter_ids_desc , & + inq_var_filter_ids_id + end interface pio_inq_var_filter_ids + interface pio_inq_var_filter_info + module procedure & + inq_var_filter_info_desc , & + inq_var_filter_info_id + end interface pio_inq_var_filter_info + interface pio_inq_filter_avail + module procedure & + inq_filter_avail_desc , & + inq_filter_avail_id + end interface pio_inq_filter_avail +#endif contains !> @@ -1460,7 +1548,146 @@ end function PIOc_inq_var_chunking enddo end function inq_var_chunking_id +#ifdef PIO_HAS_PAR_FILTERS +#ifdef NC_HAS_BZ + !> + !! @public + !! @ingroup PIO_inquire_variable + !! Gets metadata information for netcdf file. + !! + !! @param File @copydoc file_desc_t + !! @param vardesc @copydoc var_desc_t + !! @param storage 0 for chunked, 1 for contiguous + !! @param chunksizes Array of chunk sizes. + !! @retval ierr @copydoc error_return + !! @author Ed Hartnett + !< + integer function inq_var_bzip2_desc(File, vardesc, hasfilter, level) result(ierr) + + type (File_desc_t), intent(in) :: File + type (Var_desc_t), intent(in) :: vardesc + logical, intent(out) :: hasfilter + integer, intent(out) :: level + + ierr = pio_inq_var_bzip2(File%fh, vardesc%varid, hasfilter, level) + end function inq_var_bzip2_desc + + !> + !! @public + !! @ingroup PIO_inquire_variable + !! Gets metadata information for netcdf file. + !! @author Ed Hartnett + !< + integer function inq_var_bzip2_vid(File, varid, hasfilter, level) result(ierr) + + type (File_desc_t), intent(in) :: File + integer, intent(in) :: varid + logical, intent(out) :: hasfilter + integer, intent(out) :: level + + ierr = pio_inq_var_bzip2(File%fh, varid, hasfilter, level) + end function inq_var_bzip2_vid + !> + !! @public + !! @ingroup PIO_inquire_variable + !! Gets metadata information for netcdf file. + !! @author Ed Hartnett + !< + integer function inq_var_bzip2_id(ncid, varid, hasfilter, level) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: varid + logical, intent(out) :: hasfilter + integer, intent(out) :: level + + integer :: hasfilterp + interface + integer(C_INT) function PIOc_inq_var_bzip2(ncid, varid, hasfilterp, levelp) & + bind(C, name="PIOc_inq_var_bzip2") + use iso_c_binding + integer(C_INT), value :: ncid + integer(C_INT), value :: varid + integer(C_INT) :: hasfilterp + integer(C_INT) :: levelp + end function PIOc_inq_var_bzip2 + end interface + + ierr = PIOc_inq_var_bzip2(ncid, varid-1, hasfilterp, level) + hasfilter = .false. + if(hasfilterp .ne. 0) hasfilter = .true. + + end function inq_var_bzip2_id +#endif +#ifdef NC_HAS_ZSTD + !> + !! @public + !! @ingroup PIO_inquire_variable + !! Gets metadata information for netcdf file. + !! + !! @param File @copydoc file_desc_t + !! @param vardesc @copydoc var_desc_t + !! @param storage 0 for chunked, 1 for contiguous + !! @param chunksizes Array of chunk sizes. + !! @retval ierr @copydoc error_return + !! @author Ed Hartnett + !< + integer function inq_var_zstandard_desc(File, vardesc, hasfilter, level) result(ierr) + + type (File_desc_t), intent(in) :: File + type (Var_desc_t), intent(in) :: vardesc + logical, intent(out) :: hasfilter + integer, intent(out) :: level + + ierr = pio_inq_var_zstandard(File%fh, vardesc%varid, hasfilter, level) + end function inq_var_zstandard_desc + + !> + !! @public + !! @ingroup PIO_inquire_variable + !! Gets metadata information for netcdf file. + !! @author Ed Hartnett + !< + integer function inq_var_zstandard_vid(File, varid, hasfilter, level) result(ierr) + + type (File_desc_t), intent(in) :: File + integer, intent(in) :: varid + logical, intent(out) :: hasfilter + integer, intent(out) :: level + + ierr = pio_inq_var_zstandard(File%fh, varid, hasfilter, level) + end function inq_var_zstandard_vid + + !> + !! @public + !! @ingroup PIO_inquire_variable + !! Gets metadata information for netcdf file. + !! @author Ed Hartnett + !< + integer function inq_var_zstandard_id(ncid, varid, hasfilter, level) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: varid + logical, intent(out) :: hasfilter + integer, intent(out) :: level + + integer :: hasfilterp + interface + integer(C_INT) function PIOc_inq_var_zstandard(ncid, varid, hasfilterp, levelp) & + bind(C, name="PIOc_inq_var_zstandard") + use iso_c_binding + integer(C_INT), value :: ncid + integer(C_INT), value :: varid + integer(C_INT) :: hasfilterp + integer(C_INT) :: levelp + end function PIOc_inq_var_zstandard + end interface + + ierr = PIOc_inq_var_zstandard(ncid, varid-1, hasfilterp, level) + hasfilter = .false. + if(hasfilterp .ne. 0) hasfilter = .true. + + end function inq_var_zstandard_id +#endif +#endif !> !! @public !! @ingroup PIO_inquire_variable @@ -1931,6 +2158,32 @@ integer function def_var_chunking_desc(file, vardesc, storage, chunksizes) resul type (var_desc_t), intent(in) :: vardesc integer, intent(in) :: storage integer, intent(in) :: chunksizes(:) + + ierr = pio_def_var_chunking(file%fh, vardesc%varid, storage, chunksizes) + end function def_var_chunking_desc + !> + !! @ingroup PIO_def_var_chunking + !! Changes chunking settings for a netCDF-4/HDF5 variable. + !! @author Ed Hartnett + !< + integer function def_var_chunking_vid(file, varid, storage, chunksizes) result(ierr) + type (File_desc_t), intent(in) :: file + integer, intent(in) :: varid + integer, intent(in) :: storage + integer, intent(in) :: chunksizes(:) + + ierr = pio_def_var_chunking(file%fh, varid, storage, chunksizes) + end function def_var_chunking_vid + !> + !! @ingroup PIO_def_var_chunking + !! Changes chunking settings for a netCDF-4/HDF5 variable. + !! @author Ed Hartnett + !< + integer function def_var_chunking_int(ncid, varid, storage, chunksizes) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: varid + integer, intent(in) :: storage + integer, intent(in) :: chunksizes(:) integer(kind=PIO_OFFSET_KIND) :: cchunksizes(PIO_MAX_VAR_DIMS) integer :: ndims, i @@ -1949,9 +2202,156 @@ end function PIOc_def_var_chunking cchunksizes(i) = chunksizes(ndims-i+1) enddo - ierr = PIOc_def_var_chunking(file%fh, vardesc%varid-1, storage, cchunksizes) - end function def_var_chunking_desc + ierr = PIOc_def_var_chunking(ncid, varid-1, storage, cchunksizes) + end function def_var_chunking_int +#ifdef PIO_HAS_PAR_FILTERS +#ifdef NC_HAS_BZ + !> + !! @ingroup PIO_def_var_bzip2 + !! Changes bzip2 settings for a netCDF-4/HDF5 variable. + !! @author Ed Hartnett + !< + integer function def_var_bzip2_desc(file, vardesc, level) result(ierr) + type (File_desc_t), intent(in) :: file + type (var_desc_t), intent(in) :: vardesc + integer, intent(in) :: level + ierr = pio_def_var_bzip2(file%fh, vardesc%varid, level) + end function def_var_bzip2_desc + !> + !! @ingroup PIO_def_var_bzip2 + !! Changes chunking settings for a netCDF-4/HDF5 variable. + !! @author Ed Hartnett + !< + integer function def_var_bzip2_vid(file, varid, level) result(ierr) + type (File_desc_t), intent(in) :: file + integer, intent(in) :: varid + integer, intent(in) :: level + + ierr = pio_def_var_bzip2(file%fh, varid, level) + end function def_var_bzip2_vid + !> + !! @ingroup PIO_def_var_bzip2 + !! Changes chunking settings for a netCDF-4/HDF5 variable. + !! @author Ed Hartnett + !< + integer function def_var_bzip2_int(ncid, varid, level) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: varid + integer, intent(in) :: level + + interface + integer (C_INT) function PIOc_def_var_bzip2(ncid, varid, level) & + bind(c,name="PIOc_def_var_bzip2") + use iso_c_binding + integer(c_int), value :: ncid + integer(c_int), value :: varid + integer(c_int), value :: level + end function PIOc_def_var_bzip2 + end interface + + ierr = PIOc_def_var_bzip2(ncid, varid-1, level) + end function def_var_bzip2_int +#endif +#ifdef NC_HAS_ZSTD + !> + !! @ingroup PIO_def_var_zstandard + !! Changes chunking settings for a netCDF-4/HDF5 variable. + !! @author Ed Hartnett + !< + integer function def_var_zstandard_desc(file, vardesc, level) result(ierr) + type (File_desc_t), intent(in) :: file + type (var_desc_t), intent(in) :: vardesc + integer, intent(in) :: level + + ierr = pio_def_var_zstandard(file%fh, vardesc%varid, level) + end function def_var_zstandard_desc + !> + !! @ingroup PIO_def_var_zstandard + !! Changes zstandard settings for a netCDF-4/HDF5 variable. + !! @author Ed Hartnett + !< + integer function def_var_zstandard_vid(file, varid, level) result(ierr) + type (File_desc_t), intent(in) :: file + integer, intent(in) :: varid + integer, intent(in) :: level + + ierr = pio_def_var_zstandard(file%fh, varid, level) + end function def_var_zstandard_vid + !> + !! @ingroup PIO_def_var_zstandard + !! Changes chunking settings for a netCDF-4/HDF5 variable. + !! @author Ed Hartnett + !< + integer function def_var_zstandard_int(ncid, varid, level) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: varid + integer, intent(in) :: level + + interface + integer (C_INT) function PIOc_def_var_zstandard(ncid, varid, level) & + bind(c,name="PIOc_def_var_zstandard") + use iso_c_binding + integer(c_int), value :: ncid + integer(c_int), value :: varid + integer(c_int), value :: level + end function PIOc_def_var_zstandard + end interface + + ierr = PIOc_def_var_zstandard(ncid, varid-1, level) + end function def_var_zstandard_int +#endif + !> + !! @ingroup PIO_def_var_szip + !! Changes chunking settings for a netCDF-4/HDF5 variable. + !! @author Ed Hartnett + !< + integer function def_var_szip_desc(file, vardesc, mask, ppb) result(ierr) + type (File_desc_t), intent(in) :: file + type (var_desc_t), intent(in) :: vardesc + integer, intent(in) :: mask + integer, intent(in) :: ppb + + ierr = pio_def_var_szip(file%fh, vardesc%varid, mask, ppb) + end function def_var_szip_desc + !> + !! @ingroup PIO_def_var_szip + !! Changes szip settings for a netCDF-4/HDF5 variable. + !! @author Jim Edwards, Ed Hartnett + !< + integer function def_var_szip_vid(file, varid, mask, ppb) result(ierr) + type (File_desc_t), intent(in) :: file + integer, intent(in) :: varid + integer, intent(in) :: mask + integer, intent(in) :: ppb + + ierr = pio_def_var_szip(file%fh, varid, mask, ppb) + end function def_var_szip_vid + !> + !! @ingroup PIO_def_var_szip + !! Changes chunking settings for a netCDF-4/HDF5 variable. + !! @author Ed Hartnett + !< + integer function def_var_szip_int(ncid, varid, mask, ppb) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: varid + integer, intent(in) :: mask + integer, intent(in) :: ppb + + interface + integer (C_INT) function PIOc_def_var_szip(ncid, varid, options_mask, pixels_per_block) & + bind(c,name="PIOc_def_var_szip") + use iso_c_binding + integer(c_int), value :: ncid + integer(c_int), value :: varid + integer(c_int), value :: options_mask + integer(c_int), value :: pixels_per_block + end function PIOc_def_var_szip + end interface + + ierr = PIOc_def_var_szip(ncid, varid-1, mask, ppb) + end function def_var_szip_int +#endif !> !! @ingroup PIO_set_chunk_cache !! Changes chunk cache settings for netCDF-4/HDF5 files created after this call. @@ -2105,5 +2505,189 @@ end function PIOc_get_var_chunk_cache ierr = PIOc_get_var_chunk_cache(file%fh, varid-1, chunk_cache_size, & chunk_cache_nelems, chunk_cache_preemption) end function get_var_chunk_cache_id +#ifdef NC_HAS_QUANTIZE + !> + !! @ingroup PIO_def_var_quantize + !! Set quantize level for a netCDF-4/HDF5 variable + !! @author Jim Edwards, Ed Hartnett + !< + integer function def_var_quantize_desc(file, vardesc, quantize_mode, nsd) result(ierr) + type (File_desc_t), intent(in) :: file + type (var_desc_t), intent(in) :: vardesc + integer, intent(in) :: quantize_mode + integer, intent(in) :: nsd + + ierr = def_var_quantize_id(file%fh, vardesc%varid, quantize_mode, nsd) + end function def_var_quantize_desc + !> + !! @ingroup PIO_def_var_quantize + !! Set quantize level for a netCDF-4/HDF5 variable. + !! @author Jim Edwards, Ed Hartnett + !< + integer function def_var_quantize_id(ncid, varid, quantize_mode , nsd) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: varid + integer, intent(in) :: quantize_mode + integer, intent(in) :: nsd + + interface + integer (C_INT) function PIOc_def_var_quantize(ncid, varid, quantize_mode, nsd) & + bind(c,name="PIOc_def_var_quantize") + use iso_c_binding + integer(c_int), value :: ncid + integer(c_int), value :: varid + integer(c_int), value :: quantize_mode + integer(c_int), value :: nsd + end function PIOc_def_var_quantize + end interface + + ierr = PIOc_def_var_quantize(ncid, varid-1, quantize_mode, nsd) + end function def_var_quantize_id + !> + !! @ingroup PIO_inq_var_quantize + !! Set quantize level for a netCDF-4/HDF5 variable + !! @author Jim Edwards, Ed Hartnett + !< + integer function inq_var_quantize_desc(file, vardesc, quantize_mode, nsd) result(ierr) + type (File_desc_t), intent(in) :: file + type (var_desc_t), intent(in) :: vardesc + integer, intent(out) :: quantize_mode + integer, intent(out) :: nsd + + ierr = inq_var_quantize_id(file%fh, vardesc%varid, quantize_mode, nsd) + end function inq_var_quantize_desc + !> + !! @ingroup PIO_inq_var_quantize + !! Set quantize level for a netCDF-4/HDF5 variable. + !! @author Jim Edwards, Ed Hartnett + !< + integer function inq_var_quantize_id(ncid, varid, quantize_mode , nsd) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: varid + integer, intent(out) :: quantize_mode + integer, intent(out) :: nsd + + interface + integer (C_INT) function PIOc_inq_var_quantize(ncid, varid, quantize_mode, nsd) & + bind(c,name="PIOc_inq_var_quantize") + use iso_c_binding + integer(c_int), value :: ncid + integer(c_int), value :: varid + integer(c_int) :: quantize_mode + integer(c_int) :: nsd + end function PIOc_inq_var_quantize + end interface + + ierr = PIOc_inq_var_quantize(ncid, varid-1, quantize_mode, nsd) + end function inq_var_quantize_id +#endif +#ifdef PIO_HAS_PAR_FILTERS + !> + !! @ingroup PIO_inq_var_filter_ids + !! Inquire filter ids for a netCDF-4/HDF5 variable. + !! @author Jim Edwards, Ed Hartnett + !< + integer function inq_var_filter_ids_id(ncid, varid, nfilters, filterids) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: varid + integer, intent(out) :: nfilters + integer, intent(out) :: filterids(:) + + interface + integer (C_INT) function PIOc_inq_var_filter_ids(ncid, varid, nfiltersp, filterids) & + bind(c,name="PIOc_inq_var_filter_ids") + use iso_c_binding + integer(c_int), value :: ncid + integer(c_int), value :: varid + integer(c_int) :: nfiltersp + integer(c_int) :: filterids(:) + end function PIOc_inq_var_filter_ids + end interface + + ierr = PIOc_inq_var_filter_ids(ncid, varid-1, nfilters, filterids) + end function inq_var_filter_ids_id + !> + !! @ingroup PIO_inq_var_filter_ids + !! Inquire filter ids for a netCDF-4/HDF5 variable. + !! @author Jim Edwards, Ed Hartnett + !< + integer function inq_var_filter_ids_desc(file, vardesc, nfilters, filterids) result(ierr) + type (File_desc_t), intent(in) :: file + type (var_desc_t), intent(in) :: vardesc + integer, intent(out) :: nfilters + integer, intent(out) :: filterids(:) + + ierr = inq_var_filter_ids_id(file%fh, vardesc%varid, nfilters, filterids) + end function inq_var_filter_ids_desc + !> + !! @ingroup PIO_inq_var_filter_info + !! Inquire filter ids for a netCDF-4/HDF5 variable. + !! @author Jim Edwards, Ed Hartnett + !< + integer function inq_var_filter_info_id(ncid, varid, id, params) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: varid + integer, intent(in) :: id + integer, intent(out) :: params(:) + interface + integer (C_INT) function PIOc_inq_var_filter_info(ncid, varid, id, params) & + bind(c,name="PIOc_inq_var_filter_info") + use iso_c_binding + integer(c_int), value :: ncid + integer(c_int), value :: varid + integer(c_int), value :: id + integer(c_int) :: params(:) + end function PIOc_inq_var_filter_info + end interface + + ierr = PIOc_inq_var_filter_info(ncid, varid-1, id, params) + end function inq_var_filter_info_id + !> + !! @ingroup PIO_inq_var_filter_info + !! Inquire filter ids for a netCDF-4/HDF5 variable. + !! @author Jim Edwards, Ed Hartnett + !< + integer function inq_var_filter_info_desc(file, vardesc, id, params) result(ierr) + type (File_desc_t), intent(in) :: file + type (var_desc_t), intent(in) :: vardesc + integer, intent(in) :: id + integer, intent(out) :: params(:) + + ierr = inq_var_filter_info_id(file%fh, vardesc%varid, id, params) + end function inq_var_filter_info_desc +#ifdef PIO_HAS_PAR_FILTERS + !> + !! @ingroup PIO_inq_filter_avail_id + !! Inquire filter available for a netCDF-4/HDF5 file. + !! @author Jim Edwards, Ed Hartnett + !< + integer function inq_filter_avail_id(ncid, id) result(ierr) + integer, intent(in) :: ncid + integer, intent(in) :: id + + interface + integer (C_INT) function PIOc_inq_filter_avail(ncid, id) & + bind(c,name="PIOc_inq_filter_avail") + use iso_c_binding + integer(c_int), value :: ncid + integer(c_int), value :: id + end function PIOc_inq_filter_avail + end interface + + ierr = PIOc_inq_filter_avail(ncid, id) + end function inq_filter_avail_id + !> + !! @ingroup PIO_inq_var_filter_info + !! Inquire filter ids for a netCDF-4/HDF5 variable. + !! @author Jim Edwards, Ed Hartnett + !< + integer function inq_filter_avail_desc(file, id) result(ierr) + type (File_desc_t), intent(in) :: file + integer, intent(in) :: id + + ierr = inq_filter_avail_id(file%fh, id) + end function inq_filter_avail_desc +#endif +#endif end module pio_nf diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_support.F90 b/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_support.F90 index 95405b83c4..773aa18364 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_support.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_support.F90 @@ -93,9 +93,9 @@ subroutine piodie (file,line, msg, ival1, msg2, ival2, msg3, ival3, mpirank) call xl__trbk() #endif - ! passing an argument of 1 to mpi_abort will lead to a STOPALL output + ! passing an argument of 1 to mpi_abort will lead to a STOPALL output ! error code of 257 - call mpi_abort (MPI_COMM_WORLD, 1, ierr) + call mpi_abort (MPI_COMM_WORLD, 1, ierr) #ifdef CPRNAG stop @@ -162,7 +162,7 @@ integer(c_int) function PIOc_writemap_from_f90(file, ndims, gdims, maplen, map, character(C_CHAR), intent(in) :: file integer(C_INT), value, intent(in) :: ndims integer(C_INT), intent(in) :: gdims(*) - integer(C_SIZE_T), value, intent(in) :: maplen + integer(C_SIZE_T), value, intent(in) :: maplen integer(C_SIZE_T), intent(in) :: map(*) integer(C_INT), value, intent(in) :: f90_comm end function PIOc_writemap_from_f90 @@ -195,11 +195,11 @@ subroutine pio_write_nc_dof(ios, filename, cmode, iodesc, ret, title, history, f character(len=*), optional :: title character(len=*), optional :: history logical, optional :: fortran_order - + interface integer(c_int) function PIOc_write_nc_decomp(iosysid, filename, cmode, & ioid, title, history, fortran_order) & - bind(C,name="PIOc_write_nc_decomp") + bind(C,name="PIOc_write_nc_decomp") use iso_c_binding integer(C_INT), value :: iosysid character(kind=c_char) :: filename @@ -213,19 +213,17 @@ end function PIOc_write_nc_decomp character(len=:), allocatable :: ctitle, chistory integer :: nl integer :: forder - integer :: i - if(present(title)) then - ctitle = trim(title)//C_NULL_CHAR + ctitle(1:len_trim(title)+1) = trim(title)//C_NULL_CHAR else - ctitle = C_NULL_CHAR + ctitle(1:1) = C_NULL_CHAR endif if(present(history)) then - chistory = trim(history)//C_NULL_CHAR + chistory(1:len_trim(history)+1) = trim(history)//C_NULL_CHAR else - chistory = C_NULL_CHAR + chistory(1:1) = C_NULL_CHAR endif if(present(fortran_order)) then @@ -237,7 +235,6 @@ end function PIOc_write_nc_decomp endif nl = len_trim(filename) ret = PIOc_write_nc_decomp(ios%iosysid, filename(:nl)//C_NULL_CHAR, cmode, iodesc%ioid, ctitle, chistory, forder) - end subroutine pio_write_nc_dof @@ -266,7 +263,7 @@ subroutine pio_readdof (file, ndims, gdims, DOF, comm) type(C_PTR) :: tgdims, tmap interface integer(C_INT) function PIOc_readmap_from_f90(file, ndims, gdims, maplen, map, f90_comm) & - bind(C,name="PIOc_readmap_from_f90") + bind(C,name="PIOc_readmap_from_f90") use iso_c_binding character(C_CHAR), intent(in) :: file integer(C_INT), intent(out) :: ndims @@ -303,11 +300,11 @@ subroutine pio_read_nc_dof(ios, filename, iodesc, ret, title, history, fortran_o character(len=*), optional :: title character(len=*), optional :: history logical, optional :: fortran_order - + interface integer(c_int) function PIOc_read_nc_decomp(iosysid, filename, ioid, & title, history, fortran_order) & - bind(C,name="PIOc_read_nc_decomp") + bind(C,name="PIOc_read_nc_decomp") use iso_c_binding integer(C_INT), value :: iosysid character(kind=c_char) :: filename @@ -317,17 +314,17 @@ integer(c_int) function PIOc_read_nc_decomp(iosysid, filename, ioid, & integer(c_int), value :: fortran_order end function PIOc_read_nc_decomp end interface - character(len=:), allocatable :: ctitle, chistory integer :: nl integer :: forder nl = len_trim(filename) + forder = 0 ret = PIOc_read_nc_decomp(ios%iosysid, filename(:nl)//C_NULL_CHAR, iodesc%ioid, title, history, forder) if(present(fortran_order)) then if(forder /= 0) then fortran_order = .true. else - fortran_order = .true. + fortran_order = .false. endif endif end subroutine pio_read_nc_dof diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_types.F90 b/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_types.F90 index dad1b49944..7dc88ebec1 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_types.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/flib/pio_types.F90 @@ -1,4 +1,5 @@ #include "config.h" +#include <netcdf_meta.h> !> !! @file !! Derived datatypes and constants for PIO Fortran API. @@ -153,6 +154,26 @@ module pio_types enumerator :: PIO_rearr_comm_p2p = 0 !< do point-to-point communications using mpi send and recv calls. enumerator :: PIO_rearr_comm_coll !< use the MPI_ALLTOALLW function of the mpi library end enum +#ifdef NC_HAS_QUANTIZE + enum, bind(c) + enumerator :: PIO_NOQUANTIZE = 0 + enumerator :: PIO_QUANTIZE_BITGROOM + enumerator :: PIO_QUANTIZE_GRANULARBR + enumerator :: PIO_QUANTIZE_BITROUND + end enum +#endif +#ifdef NC_HAS_MULTIFILTERS + enum, bind(c) + enumerator :: PIO_FILTER_NONE = 0 + enumerator :: PIO_FILTER_DEFLATE + enumerator :: PIO_FILTER_SHUFFLE + enumerator :: PIO_FILTER_FLETCHER32 + enumerator :: PIO_FILTER_SZIP + enumerator :: PIO_FILTER_NBIT + enumerator :: PIO_FILTER_SCALEOFFSET + end ENUM +#endif + !> !! @defgroup PIO_rearr_comm_t Rearranger Communication @@ -206,6 +227,9 @@ module pio_types end type PIO_rearr_opt_t public :: PIO_rearr_comm_p2p, PIO_rearr_comm_coll,& +#ifdef NC_HAS_QUANTIZE + PIO_NOQUANTIZE, PIO_QUANTIZE_BITGROOM, PIO_QUANTIZE_GRANULARBR, PIO_QUANTIZE_BITROUND, & +#endif PIO_rearr_comm_fc_2d_enable, PIO_rearr_comm_fc_1d_comp2io,& PIO_rearr_comm_fc_1d_io2comp, PIO_rearr_comm_fc_2d_disable diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/ChangeLog b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/ChangeLog index 3d11911e6b..8bbbbcfe4f 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/ChangeLog +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/ChangeLog @@ -2,15 +2,15 @@ timing_120921: Add code for cmake build, should not have any affect otherwise timing_120803: Bug fix in setting timing_detail_limit default. [Patrick Worley] timing_120731: Correction in Makefile for serial build [Jim Edwards] -timing_120728: Replace process subset optional parameter in t_prf with - outpe_thispe optional parameter. Change def_perf_outpe_num to 0. +timing_120728: Replace process subset optional parameter in t_prf with + outpe_thispe optional parameter. Change def_perf_outpe_num to 0. [Patrick Worley] timing_120717: Retain timestamp on cp in Makefile [Jim Edwards] timing_120710: Correct issue in Makefile [Jim Edwards] timing_120709: Change for BGP to measure on compute nodes rather than IO nodes only, minor Change in Makefile so that gptl can build seperate from csm_share in cesm [Jim Edwards] -timing_120512: Bug fix in global statistics logic for when a thread has no events +timing_120512: Bug fix in global statistics logic for when a thread has no events to contribute to the merge (mods to gptl.c) [Patrick Worley] timing_120419: Minor changes for mpi-serial compile (jedwards) @@ -18,7 +18,7 @@ timing_120408: Make HAVE_COMM_F2C default to true. (jedwards) timing_120110: Update to GPTL 4.1 source (mods to gptl.c and GPTLprint_memusage) [Jim Rosinski (GPTL 4.1), Patrick Worley] timing_120109: Bug fix (adding shr_kind_i8 to shr_kind_mod list) -timing_111205: Update to gptl 4.0 (introducing CESM customizations); +timing_111205: Update to gptl 4.0 (introducing CESM customizations); support for handles in t_startf/t_stopf; support for restricting output to explicitly named process subsets [Jim Rosinski (gptl 4.0), Patrick Worley] @@ -29,7 +29,7 @@ timing_101210: Fix interface to cesm build system, add workaround for xlf bug timing_101202: updated get_memusage and print_memusage from GPTL version 3.7; adds improved support for MacOS and SLASHPROC [Jim Rosinski, Chuck Bardeen (integrated by P. Worley)] -timing_091021: update to GPTL version 3.5; rewrite of GPTLpr_summary: much faster, merging +timing_091021: update to GPTL version 3.5; rewrite of GPTLpr_summary: much faster, merging events from all processes and all threads (not just process 0/thread 0); miscellaneous fixes [Jim Rosinski (gptl 3.5), Joseph Singh, Patrick Worley] @@ -39,7 +39,7 @@ timing_090929: added explicit support for the GPTL-native token HAVE_MPI (indica timing_081221: restore default assumption that gettimeofday available timing_081028: bug fix in include order in gptl_papi.c timing_081026: change in output format to make postprocessing simpler -timing_081024: support for up to one million processes and writing timing files to +timing_081024: support for up to one million processes and writing timing files to subdirectories timing_081017: updated to gptl version 3_4_2. Changed some defaults. [Jim Rosinski, Patrick Worley] @@ -57,8 +57,8 @@ timing_071023: updated to gptl version 2.16, added support for output of global statistics; removed dependencies on shr and CAM routines; renamed gptlutil.c to GPTLutil.c [Patrick Worley, Jim Rosinski] -timing_071019: modified namelist logic to abort if try to set unknown namelist parameters; - changed default number of reporting processes to 1; +timing_071019: modified namelist logic to abort if try to set unknown namelist parameters; + changed default number of reporting processes to 1; reversed meaning and changed names of CPP tokens to NO_C99_INLINE and NO_VPRINTF [Patrick Worley] timing_071010: modified gptl.c to remove the 'inline' specification unless the @@ -67,75 +67,75 @@ timing_071010: modified gptl.c to remove the 'inline' specification unless the timing_070810: added ChangeLog updated to latest version of GPTL (from Jim Rosinski) modified perf_mod.F90: - - added perf_outpe_num and perf_outpe_stride to perf_inparm + - added perf_outpe_num and perf_outpe_stride to perf_inparm namelist to control which processes output timing data - added perf_papi_enable to perf_inparm namelist to enable - PAPI counters + PAPI counters - added papi_inparm namelist and papi_ctr1,2,3,4 namelist parameters to specify PAPI counters [Patrick Worley, Jim Rosinski] -timing_070525: bug fix in gptl.c +timing_070525: bug fix in gptl.c - unitialized pointer, testing for null pter before traversing [Patrick Worley] timing_070328: modified perf_mod.F90 - deleted HIDE_MPI cpp token [Erik Kluzek] -timing_070327: bug fixes in gptl.c - - testing for null pters before traversing +timing_070327: bug fixes in gptl.c + - testing for null pters before traversing links; added missing type declaration to GPTLallocate for sum - bug fixes in perf_mod.F90 - - fixed OMP-related logic, modified settings reporting, + bug fixes in perf_mod.F90 + - fixed OMP-related logic, modified settings reporting, modified to work when namelist input is missing; moved timer depth logic back into gptl.c [Patrick Worley] -timing_070308: added perf_mod.F90 - - defines all t_xxx entry points - calling gptlxxx directly +timing_070308: added perf_mod.F90 + - defines all t_xxx entry points - calling gptlxxx directly and removing all external gptlxxx dependencies, added detail option as an alternative way to disable event timing, added runtime selection of timing_disable, perf_timer, timer_depth_limit, timing_detail_limit, timing_barrier, perf_single_file via namelist parameters - modified f_wrappers.c - - replaced all t_xxx entry points with gptlxxx entry points, + modified f_wrappers.c + - replaced all t_xxx entry points with gptlxxx entry points, added new gptlxxx entry points, deleted _fcd support - modified gptl.c + modified gptl.c - deleted DISABLE_TIMERS cpp token, modified GPTLpr call and logic to move some of support for concatenating timing output into a single file to perf_mod.F90 - modified gptl.h - - exposed gptlxxx entry points and to add support for choice + modified gptl.h + - exposed gptlxxx entry points and to add support for choice of GPTL timer modified gptl.inc - removed t_xxx entry points and expose gptlxxx entry points [Patrick Worley] -timing_061207: modified gptl.c - - improved event output ordering +timing_061207: modified gptl.c + - improved event output ordering [Jim Edwards] -timing_061124: modified gptl.c +timing_061124: modified gptl.c - modified GPTLpr to add option to concatenate all timing data in a single output file, added GPTL_enable - and GPTL_disable as runtime control of event timing, + and GPTL_disable as runtime control of event timing, process 0-only reporting of timing options - unless DEBUG cpp token defined - modified gptl.h + modified gptl.h - redefined GPTLpr parameters - modified f_wrappers.c - - added t_enablef and t_disablef to call GPTL_enable and + modified f_wrappers.c + - added t_enablef and t_disablef to call GPTL_enable and GPTL_disable, added t_pr_onef, added string.h include - bug fix in f_wrappers.c + bug fix in f_wrappers.c - changed character string size declaration from int to size_t - bug fix in gptl_papi.c + bug fix in gptl_papi.c - modified error message - from Jim Edwards modified private.h - increased maximum event name length [Patrick Worley] -timing_061028: modified f_wrappers.c +timing_061028: modified f_wrappers.c - deleted dependency on cfort.h [Patrick Worley] -timing_060524: modified f_wrappers.c - - added support for CRAY cpp token and fixed routine +timing_060524: modified f_wrappers.c + - added support for CRAY cpp token and fixed routine type declarations [Patrick Worley] -timing_051212: original subversion version +timing_051212: original subversion version - see CAM ChangeLog for earlier history diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLget_memusage.c b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLget_memusage.c index 4ccdef8b2a..4b0d138b2b 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLget_memusage.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLget_memusage.c @@ -4,7 +4,7 @@ ** Author: Jim Rosinski ** Credit to Chuck Bardeen for MACOS section (__APPLE__ ifdef) ** -** get_memusage: +** get_memusage: ** ** Designed to be called from Fortran, returns information about memory ** usage in each of 5 input int* args. On Linux read from the /proc @@ -63,7 +63,7 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack long long total; int node_config; - + /* memory available */ Kernel_GetPersonality(&pers, sizeof(pers)); total = BGP_Personality_DDRSizeMB(&pers); @@ -116,7 +116,7 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack ** arguments, close the file and return. */ - ret = fscanf (fd, "%d %d %d %d %d %d %d", + ret = fscanf (fd, "%d %d %d %d %d %d %d", size, rss, share, text, datastack, &dum, &dum); ret = fclose (fd); return 0; @@ -124,9 +124,9 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack #elif (defined __APPLE__) FILE *fd; - char cmd[60]; + char cmd[60]; int pid = (int) getpid (); - + sprintf (cmd, "ps -o vsz -o rss -o tsiz -p %d | grep -v RSS", pid); fd = popen (cmd, "r"); @@ -145,7 +145,7 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack if (getrusage (RUSAGE_SELF, &usage) < 0) return -1; - + *size = -1; *rss = usage.ru_maxrss; *share = -1; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLprint_memusage.c b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLprint_memusage.c index 5ab873dccb..a185d61100 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLprint_memusage.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLprint_memusage.c @@ -30,13 +30,13 @@ int GPTLprint_memusage (const char *str) static const int nbytes = 1024*1024*10; /* allocate 10 MB */ static double blockstomb; /* convert blocks to MB */ void *space; /* allocated space */ - + if (GPTLget_memusage (&size, &rss, &share, &text, &datastack) < 0) return -1; #if (defined HAVE_SLASHPROC || defined __APPLE__) /* - ** Determine size in bytes of memory usage info presented by the OS. Method: allocate a + ** Determine size in bytes of memory usage info presented by the OS. Method: allocate a ** known amount of memory and see how much bigger the process becomes. */ @@ -47,7 +47,7 @@ int GPTLprint_memusage (const char *str) /* ** Estimate bytes per block, then refine to nearest power of 2. ** The assumption is that the OS presents memory usage info in - ** units that are a power of 2. + ** units that are a power of 2. */ bytesperblock = (int) ((nbytes / (double) (size2 - size)) + 0.5); bytesperblock = nearest_powerof2 (bytesperblock); @@ -57,19 +57,19 @@ int GPTLprint_memusage (const char *str) } free (space); } - + if (bytesperblock > 0) - printf ("%s size=%.1f MB rss=%.1f MB share=%.1f MB text=%.1f MB datastack=%.1f MB\n", - str, size*blockstomb, rss*blockstomb, share*blockstomb, + printf ("%s size=%.1f MB rss=%.1f MB share=%.1f MB text=%.1f MB datastack=%.1f MB\n", + str, size*blockstomb, rss*blockstomb, share*blockstomb, text*blockstomb, datastack*blockstomb); else - printf ("%s size=%d rss=%d share=%d text=%d datastack=%d\n", + printf ("%s size=%d rss=%d share=%d text=%d datastack=%d\n", str, size, rss, share, text, datastack); #else /* - ** Use max rss as returned by getrusage. If someone knows how to + ** Use max rss as returned by getrusage. If someone knows how to ** get the process size under AIX please tell me. */ @@ -85,7 +85,7 @@ int GPTLprint_memusage (const char *str) } /* -** nearest_powerof2: +** nearest_powerof2: ** Determine nearest integer which is a power of 2. ** Note: algorithm can't use anything that requires -lm because this is a library, ** and we don't want to burden the user with having to add extra libraries to the @@ -112,7 +112,7 @@ static int nearest_powerof2 (const int val) delta1 = val - lower; delta2 = higher - val; - + if (delta1 < delta2) return lower; else diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLutil.c b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLutil.c index b1c7cf80df..d9a1a93866 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLutil.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/GPTLutil.c @@ -25,10 +25,10 @@ static int max_error = 500; /* max number of error print msgs */ int GPTLerror (const char *fmt, ...) { va_list args; - + va_start (args, fmt); static int num_error = 0; - + if (fmt != NULL && num_error < max_error) { #ifndef NO_VPRINTF (void) vfprintf (stderr, fmt, args); @@ -39,10 +39,10 @@ int GPTLerror (const char *fmt, ...) (void) fprintf (stderr, "Truncating further error print now after %d msgs", num_error); ++num_error; - } - + } + va_end (args); - + if (abort_on_error) exit (-1); @@ -79,4 +79,3 @@ void *GPTLallocate (const int nbytes) return ptr; } - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/README b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/README index 2f0991da21..f8f3f7f7a0 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/README +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/README @@ -18,7 +18,7 @@ Of course these events can only be enabled if the PAPI counters they require are available on the target architecture. -Using GPTL +Using GPTL ---------- C codes making GPTL library calls should #include <gptl.h>. Fortran codes @@ -63,7 +63,7 @@ GPTLfinalize() can be called to clean up the GPTL environment. All space malloc'ed by the GPTL library will be freed by this call. -Example +Example ------- From "man GPTLstart", a simple example calling sequence to time a couple of @@ -86,7 +86,7 @@ do_work(); /* do some work */ (void) GPTLpr (mympitaskid); /* print the results to timing.<mympitaskid> */ -Auto-instrumentation +Auto-instrumentation -------------------- If the regions to be timed are defined by function entry and exit points, and @@ -128,7 +128,7 @@ Running hex2name.pl converts the function addresses back to human-readable function names. It uses the UNIX "nm" utility to do this. -Multi-processor instrumented codes +Multi-processor instrumented codes ---------------------------------- For instrumented codes which make use of threading and/or MPI, a diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/f_wrappers.c b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/f_wrappers.c index 02f4b75678..b1da29ec4e 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/f_wrappers.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/f_wrappers.c @@ -2,7 +2,7 @@ ** $Id: f_wrappers.c,v 1.56 2010-12-29 18:46:42 rosinski Exp $ ** ** Author: Jim Rosinski -** +** ** Fortran wrappers for timing library routines */ @@ -175,12 +175,12 @@ int gptlsetoption (int *option, int *val); int gptlenable (void); int gptldisable (void); int gptlsetutr (int *option); -int gptlquery (const char *name, int *t, int *count, int *onflg, double *wallclock, - double *usr, double *sys, long long *papicounters_out, int *maxcounters, +int gptlquery (const char *name, int *t, int *count, int *onflg, double *wallclock, + double *usr, double *sys, long long *papicounters_out, int *maxcounters, int nc); int gptlquerycounters (const char *name, int *t, long long *papicounters_out, int nc); int gptlget_wallclock (const char *name, int *t, double *value, int nc); -int gptlget_eventvalue (const char *timername, const char *eventname, int *t, double *value, +int gptlget_eventvalue (const char *timername, const char *eventname, int *t, double *value, int nc1, int nc2); int gptlget_nregions (int *t, int *nregions); int gptlget_regionname (int *t, int *region, char *name, int nc); @@ -258,7 +258,7 @@ int gptlpr_summary (int *fcomm) #endif #else int ccomm = 0; -#endif +#endif return GPTLpr_summary (ccomm); } @@ -278,7 +278,7 @@ int gptlpr_summary_file (int *fcomm, char *file, int nc1) #endif #else int ccomm = 0; -#endif +#endif if ( ! (locfile = (char *) malloc (nc1+1))) return GPTLerror ("gptlpr_summary_file: malloc error\n"); @@ -304,7 +304,7 @@ int gptlbarrier (int *fcomm, char *name, int nc1) #endif #else int ccomm = 0; -#endif +#endif numchars = MIN (nc1, MAX_CHARS); strncpy (cname, name, numchars); @@ -394,8 +394,8 @@ int gptlsetutr (int *option) return GPTLsetutr (*option); } -int gptlquery (const char *name, int *t, int *count, int *onflg, double *wallclock, - double *usr, double *sys, long long *papicounters_out, int *maxcounters, +int gptlquery (const char *name, int *t, int *count, int *onflg, double *wallclock, + double *usr, double *sys, long long *papicounters_out, int *maxcounters, int nc) { char cname[MAX_CHARS+1]; @@ -430,7 +430,7 @@ int gptlget_wallclock (const char *name, int *t, double *value, int nc) return GPTLget_wallclock (cname, *t, value); } -int gptlget_eventvalue (const char *timername, const char *eventname, int *t, double *value, +int gptlget_eventvalue (const char *timername, const char *eventname, int *t, double *value, int nc1, int nc2) { char ctimername[MAX_CHARS+1]; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl.c b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl.c index d06d33f088..a038422fc9 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl.c @@ -17,7 +17,7 @@ #include <assert.h> #ifndef HAVE_C99_INLINE -#define inline +#define inline #endif #ifdef HAVE_PAPI @@ -134,7 +134,7 @@ static char **timerlist; /* list of all timers */ typedef struct { int val; /* depth in calling tree */ int padding[31]; /* padding is to mitigate false cache sharing */ -} Nofalse; +} Nofalse; static Timer ***callstack; /* call stack */ static Nofalse *stackidx; /* index into callstack: */ @@ -260,7 +260,7 @@ int GPTLsetoption (const int option, /* option */ switch (option) { case GPTLcpu: #ifdef HAVE_TIMES - cpustats.enabled = (bool) val; + cpustats.enabled = (bool) val; if (verbose) printf ("%s: cpustats = %d\n", thisfunc, val); #else @@ -268,56 +268,56 @@ int GPTLsetoption (const int option, /* option */ return GPTLerror ("%s: times() not available\n", thisfunc); #endif return 0; - case GPTLwall: - wallstats.enabled = (bool) val; + case GPTLwall: + wallstats.enabled = (bool) val; if (verbose) printf ("%s: boolean wallstats = %d\n", thisfunc, val); return 0; - case GPTLoverhead: - overheadstats.enabled = (bool) val; + case GPTLoverhead: + overheadstats.enabled = (bool) val; if (verbose) printf ("%s: boolean overheadstats = %d\n", thisfunc, val); return 0; - case GPTLdepthlimit: - depthlimit = val; + case GPTLdepthlimit: + depthlimit = val; if (verbose) printf ("%s: depthlimit = %d\n", thisfunc, val); return 0; - case GPTLverbose: - verbose = (bool) val; + case GPTLverbose: + verbose = (bool) val; #ifdef HAVE_PAPI (void) GPTL_PAPIsetoption (GPTLverbose, val); #endif if (verbose) printf ("%s: boolean verbose = %d\n", thisfunc, val); return 0; - case GPTLpercent: - percent = (bool) val; + case GPTLpercent: + percent = (bool) val; if (verbose) printf ("%s: boolean percent = %d\n", thisfunc, val); return 0; - case GPTLdopr_preamble: - dopr_preamble = (bool) val; + case GPTLdopr_preamble: + dopr_preamble = (bool) val; if (verbose) printf ("%s: boolean dopr_preamble = %d\n", thisfunc, val); return 0; - case GPTLdopr_threadsort: - dopr_threadsort = (bool) val; + case GPTLdopr_threadsort: + dopr_threadsort = (bool) val; if (verbose) printf ("%s: boolean dopr_threadsort = %d\n", thisfunc, val); return 0; - case GPTLdopr_multparent: - dopr_multparent = (bool) val; + case GPTLdopr_multparent: + dopr_multparent = (bool) val; if (verbose) printf ("%s: boolean dopr_multparent = %d\n", thisfunc, val); return 0; - case GPTLdopr_collision: - dopr_collision = (bool) val; + case GPTLdopr_collision: + dopr_collision = (bool) val; if (verbose) printf ("%s: boolean dopr_collision = %d\n", thisfunc, val); return 0; case GPTLprint_method: - method = (Method) val; + method = (Method) val; if (verbose) printf ("%s: print_method = %s\n", thisfunc, methodstr (method)); return 0; @@ -338,8 +338,8 @@ int GPTLsetoption (const int option, /* option */ printf ("%s: boolean sync_mpi = %d\n", thisfunc, val); return 0; - /* - ** Allow GPTLmultiplex to fall through because it will be handled by + /* + ** Allow GPTLmultiplex to fall through because it will be handled by ** GPTL_PAPIsetoption() */ @@ -405,7 +405,7 @@ int GPTLsetutr (const int option) ** GPTLinitialize (): Initialization routine must be called from single-threaded ** region before any other timing routines may be called. The need for this ** routine could be eliminated if not targetting timing library for threaded -** capability. +** capability. ** ** return value: 0 (success) or GPTLerror (failure) */ @@ -469,12 +469,12 @@ int GPTLinitialize (void) return GPTLerror ("%s: Failure from GPTL_PAPIinitialize\n", thisfunc); #endif - /* + /* ** Call init routine for underlying timing routine. */ if ((*funclist[funcidx].funcinit)() < 0) { - fprintf (stderr, "%s: Failure initializing %s. Reverting underlying timer to %s\n", + fprintf (stderr, "%s: Failure initializing %s. Reverting underlying timer to %s\n", thisfunc, funclist[funcidx].name, funclist[0].name); funcidx = 0; } @@ -620,12 +620,12 @@ int GPTLstart_instr (void *self) ptr = getentry_instr (hashtable[t], self, &indx); - /* - ** Recursion => increment depth in recursion and return. We need to return + /* + ** Recursion => increment depth in recursion and return. We need to return ** because we don't want to restart the timer. We want the reported time for ** the timer to reflect the outermost layer of recursion. */ - + if (ptr && ptr->onflg) { ++ptr->recurselvl; return 0; @@ -662,7 +662,7 @@ int GPTLstart_instr (void *self) return GPTLerror ("%s: update_ptr error\n", thisfunc); return (0); -} +} /* ** GPTLstart: start a timer @@ -700,15 +700,15 @@ int GPTLstart (const char *name) /* timer name */ return 0; } - /* + /* ** ptr will point to the requested timer in the current list, - ** or NULL if this is a new entry + ** or NULL if this is a new entry */ ptr = getentry (hashtable[t], name, &indx); - /* - ** Recursion => increment depth in recursion and return. We need to return + /* + ** Recursion => increment depth in recursion and return. We need to return ** because we don't want to restart the timer. We want the reported time for ** the timer to reflect the outermost layer of recursion. */ @@ -786,7 +786,7 @@ int GPTLstart_handle (const char *name, /* timer name */ } /* - ** If on input, handle references a non-zero value, assume it's a previously returned Timer* + ** If on input, handle references a non-zero value, assume it's a previously returned Timer* ** passed in by the user. If zero, generate the hash entry and return it to the user. */ @@ -795,9 +795,9 @@ int GPTLstart_handle (const char *name, /* timer name */ } else { ptr = getentry (hashtable[t], name, &indx); } - - /* - ** Recursion => increment depth in recursion and return. We need to return + + /* + ** Recursion => increment depth in recursion and return. We need to return ** because we don't want to restart the timer. We want the reported time for ** the timer to reflect the outermost layer of recursion. */ @@ -869,7 +869,7 @@ static int update_ll_hash (Timer *ptr, const int t, const unsigned int indx) last[t] = ptr; ++hashtable[t][indx].nument; nument = hashtable[t][indx].nument; - + eptr = (Timer **) realloc (hashtable[t][indx].entries, nument * sizeof (Timer *)); if ( ! eptr) return GPTLerror ("update_ll_hash: realloc error\n"); @@ -898,7 +898,7 @@ static inline int update_ptr (Timer *ptr, const int t) if (cpustats.enabled && get_cpustamp (&ptr->cpu.last_utime, &ptr->cpu.last_stime) < 0) return GPTLerror ("update_ptr: get_cpustamp error"); - + if (wallstats.enabled) { tp2 = (*ptr2wtimefunc) (); ptr->wall.last = tp2; @@ -922,9 +922,9 @@ static inline int update_ptr (Timer *ptr, const int t) ** Return value: 0 (success) or GPTLerror (failure) */ -static inline int update_parent_info (Timer *ptr, - Timer **callstackt, - int stackidxt) +static inline int update_parent_info (Timer *ptr, + Timer **callstackt, + int stackidxt) { int n; /* loop index through known parents */ Timer *pptr; /* pointer to parent in callstack */ @@ -941,7 +941,7 @@ static inline int update_parent_info (Timer *ptr, callstackt[stackidxt] = ptr; - /* + /* ** If the region has no parent, bump its orphan count ** (should never happen since "GPTL_ROOT" added). */ @@ -1010,7 +1010,7 @@ int GPTLstop_instr (void *self) return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc); /* Get the timestamp */ - + if (wallstats.enabled) { tp1 = (*ptr2wtimefunc) (); } @@ -1033,7 +1033,7 @@ int GPTLstop_instr (void *self) ptr = getentry_instr (hashtable[t], self, &indx); - if ( ! ptr) + if ( ! ptr) return GPTLerror ("%s: timer for %p had not been started.\n", thisfunc, self); if ( ! ptr->onflg ) @@ -1041,7 +1041,7 @@ int GPTLstop_instr (void *self) ++ptr->count; - /* + /* ** Recursion => decrement depth in recursion and return. We need to return ** because we don't want to stop the timer. We want the reported time for ** the timer to reflect the outermost layer of recursion. @@ -1085,7 +1085,7 @@ int GPTLstop (const char *name) /* timer name */ return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc); /* Get the timestamp */ - + if (wallstats.enabled) { tp1 = (*ptr2wtimefunc) (); } @@ -1114,7 +1114,7 @@ int GPTLstop (const char *name) /* timer name */ ++ptr->count; - /* + /* ** Recursion => decrement depth in recursion and return. We need to return ** because we don't want to stop the timer. We want the reported time for ** the timer to reflect the outermost layer of recursion. @@ -1160,7 +1160,7 @@ int GPTLstop_handle (const char *name, /* timer name */ return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc); /* Get the timestamp */ - + if (wallstats.enabled) { tp1 = (*ptr2wtimefunc) (); } @@ -1182,7 +1182,7 @@ int GPTLstop_handle (const char *name, /* timer name */ } /* - ** If on input, handle references a non-zero value, assume it's a previously returned Timer* + ** If on input, handle references a non-zero value, assume it's a previously returned Timer* ** passed in by the user. If zero, generate the hash entry and return it to the user. */ @@ -1198,7 +1198,7 @@ int GPTLstop_handle (const char *name, /* timer name */ ++ptr->count; - /* + /* ** Recursion => decrement depth in recursion and return. We need to return ** because we don't want to stop the timer. We want the reported time for ** the timer to reflect the outermost layer of recursion. @@ -1224,7 +1224,7 @@ int GPTLstop_handle (const char *name, /* timer name */ } /* -** update_stats: update stats inside ptr. Called by GPTLstop, GPTLstop_instr, +** update_stats: update stats inside ptr. Called by GPTLstop, GPTLstop_instr, ** GPTLstop_handle ** ** Input arguments: @@ -1237,9 +1237,9 @@ int GPTLstop_handle (const char *name, /* timer name */ ** Return value: 0 (success) or GPTLerror (failure) */ -static inline int update_stats (Timer *ptr, - const double tp1, - const long usr, +static inline int update_stats (Timer *ptr, + const double tp1, + const long usr, const long sys, const int t) { @@ -1375,7 +1375,7 @@ int GPTLreset (void) return 0; } -/* +/* ** GPTLpr_set_append: set GPTLpr_file and GPTLpr_summary_file ** to use append mode */ @@ -1386,20 +1386,20 @@ int GPTLpr_set_append (void) return 0; } -/* +/* ** GPTLpr_query_append: query whether GPTLpr_file and GPTLpr_summary_file ** use append mode */ int GPTLpr_query_append (void) { - if (pr_append) + if (pr_append) return 1; - else + else return 0; } -/* +/* ** GPTLpr_set_write: set GPTLpr_file and GPTLpr_summary_file ** to use write mode */ @@ -1410,20 +1410,20 @@ int GPTLpr_set_write (void) return 0; } -/* +/* ** GPTLpr_query_write: query whether GPTLpr_file and GPTLpr_summary_file ** use write mode */ int GPTLpr_query_write (void) { - if (pr_append) + if (pr_append) return 0; - else + else return 1; } -/* +/* ** GPTLpr: Print values of all timers ** ** Input arguments: @@ -1448,7 +1448,7 @@ int GPTLpr (const int id) /* output file will be named "timing.<id>" */ return 0; } -/* +/* ** GPTLpr_file: Print values of all timers ** ** Input arguments: @@ -1500,9 +1500,9 @@ int GPTLpr_file (const char *outfile) /* output file to write */ /* 2 is for "/" plus null */ if (outdir) - totlen = strlen (outdir) + strlen (outfile) + 2; + totlen = strlen (outdir) + strlen (outfile) + 2; else - totlen = strlen (outfile) + 2; + totlen = strlen (outfile) + 2; outpath = (char *) GPTLallocate (totlen); @@ -1619,11 +1619,11 @@ int GPTLpr_file (const char *outfile) /* output file to write */ } sum = (float *) GPTLallocate (nthreads * sizeof (float)); - + for (t = 0; t < nthreads; ++t) { /* - ** Construct tree for printing timers in parent/child form. get_max_depth() must be called + ** Construct tree for printing timers in parent/child form. get_max_depth() must be called ** AFTER construct_tree() because it relies on the per-parent children arrays being complete. */ @@ -1671,7 +1671,7 @@ int GPTLpr_file (const char *outfile) /* output file to write */ printself_andchildren (timers[t], fp, t, -1, tot_overhead); - /* + /* ** Sum of overhead across timers is meaningful. ** Factor of 2 is because there are 2 utr calls per start/stop pair. */ @@ -1721,8 +1721,8 @@ int GPTLpr_file (const char *outfile) /* output file to write */ /* Start at next to skip dummy */ for (ptr = timers[0]->next; ptr; ptr = ptr->next) { - - /* + + /* ** To print sum stats, first create a new timer then copy thread 0 ** stats into it. then sum using "add", and finally print. */ @@ -1874,7 +1874,7 @@ int GPTLpr_file (const char *outfile) /* output file to write */ totmem += gptlmem; fprintf (fp, "\n"); fprintf (fp, "Thread %d total memory usage = %g KB\n", t, gptlmem*.001); - fprintf (fp, " Hashmem = %g KB\n" + fprintf (fp, " Hashmem = %g KB\n" " Regionmem = %g KB (papimem portion = %g KB)\n" " Parent/child arrays = %g KB\n", hashmem*.001, regionmem*.001, papimem*.001, pchmem*.001); @@ -1892,7 +1892,7 @@ int GPTLpr_file (const char *outfile) /* output file to write */ return 0; } -/* +/* ** construct_tree: Build the parent->children tree starting with knowledge of ** parent list for each child. ** @@ -1944,7 +1944,7 @@ int construct_tree (Timer *timerst, Method method) } break; case GPTLfull_tree: - /* + /* ** Careful: this one can create *lots* of output! */ for (n = 0; n < ptr->nparent; ++n) { @@ -1959,7 +1959,7 @@ int construct_tree (Timer *timerst, Method method) return 0; } -/* +/* ** methodstr: Return a pointer to a string which represents the method ** ** Input arguments: @@ -1980,9 +1980,9 @@ static char *methodstr (Method method) return "Unknown"; } -/* +/* ** newchild: Add an entry to the children list of parent. Use function -** is_descendant() to prevent infinite loops. +** is_descendant() to prevent infinite loops. ** ** Input arguments: ** parent: parent node @@ -2017,7 +2017,7 @@ static int newchild (Timer *parent, Timer *child) } /* - ** To guarantee no loops, ensure that proposed parent isn't already a descendant of + ** To guarantee no loops, ensure that proposed parent isn't already a descendant of ** proposed child */ @@ -2040,13 +2040,13 @@ static int newchild (Timer *parent, Timer *child) return 0; } -/* +/* ** get_max_depth: Determine the maximum call tree depth by traversing the ** tree recursively ** ** Input arguments: ** ptr: Starting timer -** startdepth: current depth when function invoked +** startdepth: current depth when function invoked ** ** Return value: maximum depth */ @@ -2064,7 +2064,7 @@ static int get_max_depth (const Timer *ptr, const int startdepth) return maxdepth; } -/* +/* ** num_descendants: Determine the number of descendants of a timer by traversing ** the tree recursively. This function is not currently used. It could be ** useful in a pruning algorithm @@ -2086,7 +2086,7 @@ static int num_descendants (Timer *ptr) return ptr->num_desc; } -/* +/* ** is_descendant: Determine whether node2 is in the descendant list for ** node1 ** @@ -2114,7 +2114,7 @@ static int is_descendant (const Timer *node1, const Timer *node2) return 0; } -/* +/* ** printstats: print a single timer ** ** Input arguments: @@ -2224,7 +2224,7 @@ static void printstats (const Timer *timer, else fprintf (fp, "%13.3e ", timer->nbytes / timer->count); #endif - + #ifdef HAVE_PAPI GPTL_PAPIpr (fp, &timer->aux, t, timer->count, timer->wall.accum); #endif @@ -2232,13 +2232,13 @@ static void printstats (const Timer *timer, fprintf (fp, "\n"); } -/* -** print_multparentinfo: +/* +** print_multparentinfo: ** ** Input arguments: ** Input/output arguments: */ -void print_multparentinfo (FILE *fp, +void print_multparentinfo (FILE *fp, Timer *ptr) { int n; @@ -2263,7 +2263,7 @@ void print_multparentinfo (FILE *fp, fprintf (fp, "%8.1e %-32s\n\n", (float) ptr->count, ptr->name); } -/* +/* ** add: add the contents of tin to tout ** ** Input arguments: @@ -2272,14 +2272,14 @@ void print_multparentinfo (FILE *fp, ** tout: output timer summed into */ -static void add (Timer *tout, +static void add (Timer *tout, const Timer *tin) { tout->count += tin->count; if (wallstats.enabled) { tout->wall.accum += tin->wall.accum; - + tout->wall.max = MAX (tout->wall.max, tin->wall.max); tout->wall.min = MIN (tout->wall.min, tin->wall.min); } @@ -2293,8 +2293,8 @@ static void add (Timer *tout, #endif } -/* -** GPTLpr_summary: Gather and print summary stats across +/* +** GPTLpr_summary: Gather and print summary stats across ** threads and MPI tasks ** ** Input arguments: @@ -2315,10 +2315,10 @@ int GPTLpr_summary (int comm) } #ifdef HAVE_MPI -int GPTLpr_summary_file (MPI_Comm comm, +int GPTLpr_summary_file (MPI_Comm comm, const char *outfile) #else -int GPTLpr_summary_file (int comm, +int GPTLpr_summary_file (int comm, const char *outfile) #endif { @@ -2362,7 +2362,7 @@ int GPTLpr_summary_file (int comm, return GPTLerror ("%s: GPTLinitialize() has not been called\n", thisfunc); /* - ** Each process gathers stats for its threads. + ** Each process gathers stats for its threads. ** Binary tree used combine results. ** Master prints results. */ @@ -2411,7 +2411,7 @@ int GPTLpr_summary_file (int comm, /* allocate storage for data for all timers */ if( !( storage = malloc( sizeof(Summarystats) * count ) ) && count ) return GPTLerror ("%s: memory allocation failed\n", thisfunc); - + if ( (ret = collect_data( iam, comm, &count, &storage) ) != 0 ) return GPTLerror ("%s: master collect_data failed\n", thisfunc); @@ -2526,7 +2526,7 @@ static int merge_thread_data() /* count timers for thread 0 */ count_r = 0; - for (ptr = timers[0]->next; ptr; ptr = ptr->next) count_r++; + for (ptr = timers[0]->next; ptr; ptr = ptr->next) count_r++; timerlist = (char **) GPTLallocate( sizeof (char *)); if( !( timerlist[0] = (char *)malloc( count_r * length * sizeof (char)) ) && count_r) @@ -2551,7 +2551,7 @@ static int merge_thread_data() /* count timers for thread */ count[t] = 0; - for (ptr = timers[t]->next; ptr; ptr = ptr->next) count[t]++; + for (ptr = timers[t]->next; ptr; ptr = ptr->next) count[t]++; if( count[t] > max_count || max_count == 0 ) max_count = count[t]; @@ -2587,24 +2587,24 @@ static int merge_thread_data() k = 0; n = 0; num_newtimers = 0; - while( k < count[0] && n < count[t] ) { + while( k < count[0] && n < count[t] ) { /* linear comparison of timers */ compare = strcmp( sort[0][k], sort[t][n] ); - if( compare == 0 ) { + if( compare == 0 ) { /* both have, nothing needs to be done */ k++; n++; continue; } - if( compare < 0 ) { + if( compare < 0 ) { /* event that only master has, nothing needs to be done */ k++; continue; } - if( compare > 0 ) { + if( compare > 0 ) { /* event that only slave thread has, need to add */ newtimers[num_newtimers] = sort[t][n]; n++; @@ -2612,8 +2612,8 @@ static int merge_thread_data() } } - while( n < count[t] ) { - /* adds any remaining timers, since we know that all the rest + while( n < count[t] ) { + /* adds any remaining timers, since we know that all the rest are new since have checked all master thread timers */ newtimers[num_newtimers] = sort[t][n]; num_newtimers++; @@ -2622,7 +2622,7 @@ static int merge_thread_data() if( num_newtimers ) { /* sorts by memory address to restore original order */ - qsort( newtimers, num_newtimers, sizeof(char*), ncmp ); + qsort( newtimers, num_newtimers, sizeof(char*), ncmp ); /* reallocate memory to hold additional timers */ if( !( sort[0] = realloc( sort[0], (count[0] + num_newtimers) * sizeof (char *)) ) ) @@ -2631,7 +2631,7 @@ static int merge_thread_data() return GPTLerror ("%s: memory reallocation failed\n", thisfunc); k = count[0]; - for (n = 0; n < num_newtimers; n++) { + for (n = 0; n < num_newtimers; n++) { /* add new found timers */ memcpy( timerlist[0] + (count[0] + n) * length, newtimers[n], length * sizeof (char) ); } @@ -2639,7 +2639,7 @@ static int merge_thread_data() count[0] += num_newtimers; /* reassign pointers in sort since realloc will have broken them if it moved the memory. */ - x = 0; + x = 0; for (k = 0; k < count[0]; k++) { sort[0][k] = timerlist[0] + x; x += length; @@ -2649,7 +2649,7 @@ static int merge_thread_data() } } - free(sort[0]); + free(sort[0]); /* don't free timerlist[0], since needed for subsequent steps in gathering global statistics */ for (t = 1; t < nthreads; t++) { free(sort[t]); @@ -2679,14 +2679,14 @@ static int merge_thread_data() */ #ifdef HAVE_MPI -static int collect_data(const int iam, +static int collect_data(const int iam, MPI_Comm comm, - int *count, + int *count, Summarystats **summarystats_cumul ) #else -static int collect_data(const int iam, +static int collect_data(const int iam, int comm, - int *count, + int *count, Summarystats **summarystats_cumul ) #endif { @@ -2809,11 +2809,11 @@ static int collect_data(const int iam, { compare = strcmp(sort_master[k], sort_slave[n]); - if (compare == 0) { + if (compare == 0) { /* matching timers found */ /* find element number of the name in original timerlist so that it can be matched with its summarystats */ - m_index = get_index( timerlist[0], sort_master[k] ); + m_index = get_index( timerlist[0], sort_master[k] ); s_index = get_index( timers_slave, sort_slave[n] ); get_summarystats (&summarystats[m_index], &summarystats_slave[s_index]); @@ -2822,7 +2822,7 @@ static int collect_data(const int iam, continue; } - if (compare > 0) { + if (compare > 0) { /* s1 >s2 . slave has event; master does not */ newtimers[num_newtimers] = sort_slave[n]; num_newtimers++; @@ -2834,7 +2834,7 @@ static int collect_data(const int iam, k++; } - while (n < count_slave) { + while (n < count_slave) { /* add all remaining timers which only the slave has */ newtimers[num_newtimers] = sort_slave[n]; num_newtimers++; @@ -2842,7 +2842,7 @@ static int collect_data(const int iam, } /* sort by memory address to get original order */ - qsort (newtimers, num_newtimers, sizeof(char*), ncmp); + qsort (newtimers, num_newtimers, sizeof(char*), ncmp); /* reallocate to hold new timer names and summary stats from slave */ if (!(timerlist[0] = realloc( timerlist[0], length * (*count + num_newtimers) * sizeof (char) ) )) @@ -2922,7 +2922,7 @@ static int collect_data(const int iam, ** Return value: index of element in list */ -int get_index( const char * list, +int get_index( const char * list, const char * element ) { return (( element - list ) / ( MAX_CHARS + 1 )); @@ -2957,7 +2957,7 @@ static int ncmp( const void *x, const void *y ) GPTLerror("%s: shared memory address between timers\n", thisfunc); } -/* +/* ** get_threadstats: gather stats for timer "name" over all threads ** ** Input arguments: @@ -2967,7 +2967,7 @@ static int ncmp( const void *x, const void *y ) ** summarystats: max/min stats over all threads */ -void get_threadstats (const int iam, +void get_threadstats (const int iam, const char *name, Summarystats *summarystats) { @@ -3019,7 +3019,7 @@ void get_threadstats (const int iam, summarystats->papimax[n] = value; summarystats->papimax_t[n] = t; } - + if (value < summarystats->papimin[n] || summarystats->papimin[n] == 0.) { summarystats->papimin[n] = value; summarystats->papimin_t[n] = t; @@ -3032,7 +3032,7 @@ void get_threadstats (const int iam, if ( summarystats->count ) summarystats->processes = 1; } -/* +/* ** get_summarystats: write max/min stats into mpistats based on comparison ** with summarystats_slave ** @@ -3042,7 +3042,7 @@ void get_threadstats (const int iam, ** summarystats: stats (starts out as master stats) */ -void get_summarystats (Summarystats *summarystats, +void get_summarystats (Summarystats *summarystats, const Summarystats *summarystats_slave) { if (summarystats_slave->count == 0) return; @@ -3053,7 +3053,7 @@ void get_summarystats (Summarystats *summarystats, summarystats->wallmax_t = summarystats_slave->wallmax_t; } - if ((summarystats_slave->wallmin < summarystats->wallmin) || + if ((summarystats_slave->wallmin < summarystats->wallmin) || (summarystats->count == 0)){ summarystats->wallmin = summarystats_slave->wallmin; summarystats->wallmin_p = summarystats_slave->wallmin_p; @@ -3070,7 +3070,7 @@ void get_summarystats (Summarystats *summarystats, summarystats->papimax_t[n] = summarystats_slave->papimax_t[n]; } - if ((summarystats_slave->papimin[n] < summarystats->papimin[n]) || + if ((summarystats_slave->papimin[n] < summarystats->papimin[n]) || (summarystats->count == 0)){ summarystats->papimin[n] = summarystats_slave->papimin[n]; summarystats->papimin_p[n] = summarystats_slave->papimin_p[n]; @@ -3087,7 +3087,7 @@ void get_summarystats (Summarystats *summarystats, summarystats->threads += summarystats_slave->threads; } -/* +/* ** GPTLbarrier: When MPI enabled, set and time an MPI barrier ** ** Input arguments: @@ -3140,10 +3140,10 @@ static inline int get_cpustamp (long *usr, long *sys) } /* -** GPTLquery: return current status info about a timer. If certain stats are not +** GPTLquery: return current status info about a timer. If certain stats are not ** enabled, they should just have zeros in them. If PAPI is not enabled, input ** counter info is ignored. -** +** ** Input args: ** name: timer name ** maxcounters: max number of PAPI counters to get info for @@ -3158,7 +3158,7 @@ static inline int get_cpustamp (long *usr, long *sys) ** papicounters_out: accumulated PAPI counters */ -int GPTLquery (const char *name, +int GPTLquery (const char *name, int t, int *count, int *onflg, @@ -3171,14 +3171,14 @@ int GPTLquery (const char *name, Timer *ptr; /* linked list pointer */ unsigned int indx; /* linked list index returned from getentry (unused) */ static const char *thisfunc = "GPTLquery"; - + if ( ! initialized) return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc); - + /* ** If t is < 0, assume the request is for the current thread */ - + if (t < 0) { if ((t = get_thread_num ()) < 0) return GPTLerror ("%s: get_thread_num failure\n", thisfunc); @@ -3186,7 +3186,7 @@ int GPTLquery (const char *name, if (t >= maxthreads) return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t); } - + ptr = getentry (hashtable[t], name, &indx); if ( !ptr) return GPTLerror ("%s: requested timer %s does not have a name hash\n", thisfunc, name); @@ -3205,7 +3205,7 @@ int GPTLquery (const char *name, /* ** GPTLquerycounters: return current PAPI counters for a timer. ** THIS ROUTINE ID DEPRECATED. USE GPTLget_eventvalue() instead -** +** ** Input args: ** name: timer name ** t: thread number (if < 0, the request is for the current thread) @@ -3214,21 +3214,21 @@ int GPTLquery (const char *name, ** papicounters_out: accumulated PAPI counters */ -int GPTLquerycounters (const char *name, +int GPTLquerycounters (const char *name, int t, long long *papicounters_out) { Timer *ptr; /* linked list pointer */ unsigned int indx; /* hash index returned from getentry */ static const char *thisfunc = "GPTLquery_counters"; - + if ( ! initialized) return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc); - + /* ** If t is < 0, assume the request is for the current thread */ - + if (t < 0) { if ((t = get_thread_num ()) < 0) return GPTLerror ("%s: get_thread_num failure\n", thisfunc); @@ -3236,7 +3236,7 @@ int GPTLquerycounters (const char *name, if (t >= maxthreads) return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t); } - + ptr = getentry (hashtable[t], name, &indx); if ( !ptr) return GPTLerror ("%s: requested timer %s does not have a name hash\n", thisfunc, name); @@ -3250,7 +3250,7 @@ int GPTLquerycounters (const char *name, /* ** GPTLget_wallclock: return wallclock accumulation for a timer. -** +** ** Input args: ** timername: timer name ** t: thread number (if < 0, the request is for the current thread) @@ -3267,17 +3267,17 @@ int GPTLget_wallclock (const char *timername, Timer *ptr; /* linked list pointer */ unsigned int indx; /* hash index returned from getentry (unused) */ static const char *thisfunc = "GPTLget_wallclock"; - + if ( ! initialized) return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc); if ( ! wallstats.enabled) return GPTLerror ("%s: wallstats not enabled\n", thisfunc); - + /* ** If t is < 0, assume the request is for the current thread */ - + if (t < 0) { if ((t = get_thread_num ()) < 0) return GPTLerror ("%s: bad return from get_thread_num\n", thisfunc); @@ -3285,9 +3285,9 @@ int GPTLget_wallclock (const char *timername, if (t >= maxthreads) return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t); } - - /* - ** Don't know whether hashtable entry for timername was generated with + + /* + ** Don't know whether hashtable entry for timername was generated with ** *_instr() or not, so try both possibilities */ @@ -3307,7 +3307,7 @@ int GPTLget_wallclock (const char *timername, /* ** GPTLget_eventvalue: return PAPI-based event value for a timer. All values will be ** returned as doubles, even if the event is not derived. -** +** ** Input args: ** timername: timer name ** eventname: event name (must be currently enabled) @@ -3326,14 +3326,14 @@ int GPTLget_eventvalue (const char *timername, Timer *ptr; /* linked list pointer */ unsigned int indx; /* hash index returned from getentry (unused) */ static const char *thisfunc = "GPTLget_eventvalue"; - + if ( ! initialized) return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc); - + /* ** If t is < 0, assume the request is for the current thread */ - + if (t < 0) { if ((t = get_thread_num ()) < 0) return GPTLerror ("%s: get_thread_num failure\n", thisfunc); @@ -3341,9 +3341,9 @@ int GPTLget_eventvalue (const char *timername, if (t >= maxthreads) return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t); } - - /* - ** Don't know whether hashtable entry for timername was generated with + + /* + ** Don't know whether hashtable entry for timername was generated with ** *_instr() or not, so try both possibilities */ @@ -3359,13 +3359,13 @@ int GPTLget_eventvalue (const char *timername, #ifdef HAVE_PAPI return GPTL_PAPIget_eventvalue (eventname, &ptr->aux, value); #else - return GPTLerror ("%s: PAPI not enabled\n", thisfunc); + return GPTLerror ("%s: PAPI not enabled\n", thisfunc); #endif } /* ** GPTLget_nregions: return number of regions (i.e. timer names) for this thread -** +** ** Input args: ** t: thread number (if < 0, the request is for the current thread) ** @@ -3373,7 +3373,7 @@ int GPTLget_eventvalue (const char *timername, ** nregions: number of regions */ -int GPTLget_nregions (int t, +int GPTLget_nregions (int t, int *nregions) { Timer *ptr; /* walk through linked list */ @@ -3381,11 +3381,11 @@ int GPTLget_nregions (int t, if ( ! initialized) return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc); - + /* ** If t is < 0, assume the request is for the current thread */ - + if (t < 0) { if ((t = get_thread_num ()) < 0) return GPTLerror ("%s: get_thread_num failure\n", thisfunc); @@ -3393,9 +3393,9 @@ int GPTLget_nregions (int t, if (t >= maxthreads) return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t); } - + *nregions = 0; - for (ptr = timers[t]->next; ptr; ptr = ptr->next) + for (ptr = timers[t]->next; ptr; ptr = ptr->next) ++*nregions; return 0; @@ -3403,7 +3403,7 @@ int GPTLget_nregions (int t, /* ** GPTLget_regionname: return region name for this thread -** +** ** Input args: ** t: thread number (if < 0, the request is for the current thread) ** region: region number @@ -3425,11 +3425,11 @@ int GPTLget_regionname (int t, /* thread number */ if ( ! initialized) return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc); - + /* ** If t is < 0, assume the request is for the current thread */ - + if (t < 0) { if ((t = get_thread_num ()) < 0) return GPTLerror ("%s: get_thread_num failure\n", thisfunc); @@ -3437,7 +3437,7 @@ int GPTLget_regionname (int t, /* thread number */ if (t >= maxthreads) return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t); } - + ptr = timers[t]->next; for (i = 0; i < region; i++) { if ( ! ptr) @@ -3448,7 +3448,7 @@ int GPTLget_regionname (int t, /* thread number */ if (ptr) { ncpy = MIN (nc, strlen (ptr->name)); strncpy (name, ptr->name, ncpy); - + /* ** Adding the \0 is only important when called from C */ @@ -3525,7 +3525,7 @@ static inline Timer *getentry (const Hashentry *hashtable, /* hash table */ const unsigned char *c; /* pointer to elements of "name" */ Timer *ptr = 0; /* return value when entry not found */ - /* + /* ** Hash value is sum of: chars times their 1-based position index, modulo tablesize */ @@ -3537,7 +3537,7 @@ static inline Timer *getentry (const Hashentry *hashtable, /* hash table */ *indx %= tablesize; - /* + /* ** If nument exceeds 1 there was a hash collision and we must search ** linearly through an array for a match */ @@ -3725,7 +3725,7 @@ static int init_papitime () return GPTLerror ("%s: not enabled\n", thisfunc); #endif } - + static inline double utr_papitime () { #ifdef HAVE_PAPI @@ -3737,8 +3737,8 @@ static inline double utr_papitime () #endif } -/* -** Probably need to link with -lrt for this one to work +/* +** Probably need to link with -lrt for this one to work */ static int init_clock_gettime () @@ -3833,7 +3833,7 @@ static inline double utr_gettimeofday () #endif } -/* +/* ** Determine underlying timing routine overhead: call it 1000 times. */ @@ -3854,7 +3854,7 @@ static double utr_getoverhead () */ static void printself_andchildren (const Timer *ptr, - FILE *fp, + FILE *fp, const int t, const int depth, const double tot_overhead) @@ -3870,9 +3870,9 @@ static void printself_andchildren (const Timer *ptr, #ifdef ENABLE_PMPI /* -** GPTLgetentry: called ONLY from pmpi.c (i.e. not a public entry point). Returns a pointer to the +** GPTLgetentry: called ONLY from pmpi.c (i.e. not a public entry point). Returns a pointer to the ** requested timer name by calling internal function getentry() -** +** ** Return value: 0 (NULL) or the return value of getentry() */ @@ -3896,7 +3896,7 @@ Timer *GPTLgetentry (const char *name) } /* -** GPTLpr_file_has_been_called: Called ONLY from pmpi.c (i.e. not a public entry point). Return +** GPTLpr_file_has_been_called: Called ONLY from pmpi.c (i.e. not a public entry point). Return ** whether GPTLpr_file has been called. MPI_Finalize wrapper needs ** to know whether it needs to call GPTLpr. */ @@ -3919,7 +3919,7 @@ int GPTLpr_has_been_called (void) ** $Id: gptl.c,v 1.157 2011-03-28 20:55:18 rosinski Exp $ ** ** Author: Jim Rosinski -** +** ** Utility functions handle thread-based GPTL needs. */ @@ -3927,7 +3927,7 @@ int GPTLpr_has_been_called (void) #define MAX_THREADS 128 /**********************************************************************************/ -/* +/* ** 3 sets of routines: OMP threading, PTHREADS, unthreaded */ @@ -3953,13 +3953,13 @@ static int threadinit (void) if (omp_get_thread_num () != 0) return GPTLerror ("OMP %s: MUST only be called by the master thread\n", thisfunc); - /* - ** Allocate the threadid array which maps physical thread IDs to logical IDs + /* + ** Allocate the threadid array which maps physical thread IDs to logical IDs ** For OpenMP this will be just threadid_omp[iam] = iam; */ - if (threadid_omp) - return GPTLerror ("OMP %s: has already been called.\nMaybe mistakenly called by multiple threads?", + if (threadid_omp) + return GPTLerror ("OMP %s: has already been called.\nMaybe mistakenly called by multiple threads?", thisfunc); maxthreads = MAX ((1), (omp_get_max_threads ())); @@ -3977,7 +3977,7 @@ static int threadinit (void) #ifdef VERBOSE printf ("OMP %s: Set maxthreads=%d\n", thisfunc, maxthreads); #endif - + return 0; } @@ -4020,7 +4020,7 @@ static inline int get_thread_num (void) if (t == threadid_omp[t]) return t; - /* + /* ** Thread id not found. Modify threadid_omp with our ID, then start PAPI events if required. ** Due to the setting of threadid_omp, everything below here will only execute once per thread. */ @@ -4051,7 +4051,7 @@ static inline int get_thread_num (void) /* ** nthreads = maxthreads based on setting in threadinit */ - + nthreads = maxthreads; #ifdef VERBOSE printf ("OMP %s: nthreads=%d\n", thisfunc, nthreads); @@ -4071,7 +4071,7 @@ static void print_threadmapping (FILE *fp) } /**********************************************************************************/ -/* +/* ** PTHREADS */ @@ -4098,7 +4098,7 @@ static int threadinit (void) static const char *thisfunc = "threadinit"; /* - ** The following test is not rock-solid, but it's pretty close in terms of guaranteeing that + ** The following test is not rock-solid, but it's pretty close in terms of guaranteeing that ** threadinit gets called by only 1 thread. Problem is, mutex hasn't yet been initialized ** so we can't use it. */ @@ -4114,7 +4114,7 @@ static int threadinit (void) ** Previously, t_mutex = PTHREAD_MUTEX_INITIALIZER on the static declaration line was ** adequate to initialize the mutex. But this failed in programs that invoked ** GPTLfinalize() followed by GPTLinitialize(). - ** "man pthread_mutex_init" indicates that passing NULL as the second argument to + ** "man pthread_mutex_init" indicates that passing NULL as the second argument to ** pthread_mutex_init() should appropriately initialize the mutex, assuming it was ** properly destroyed by a previous call to pthread_mutex_destroy(); */ @@ -4123,16 +4123,16 @@ static int threadinit (void) if ((ret = pthread_mutex_init ((pthread_mutex_t *) &t_mutex, NULL)) != 0) return GPTLerror ("PTHREADS %s: mutex init failure: ret=%d\n", thisfunc, ret); #endif - - /* - ** Allocate the threadid array which maps physical thread IDs to logical IDs + + /* + ** Allocate the threadid array which maps physical thread IDs to logical IDs */ - if (threadid) + if (threadid) return GPTLerror ("PTHREADS %s: threadid not null\n", thisfunc); else if ( ! (threadid = (pthread_t *) GPTLallocate (MAX_THREADS * sizeof (pthread_t)))) return GPTLerror ("PTHREADS %s: malloc failure for %d elements of threadid\n", thisfunc, MAX_THREADS); - + maxthreads = MAX_THREADS; /* @@ -4177,7 +4177,7 @@ static void threadfinalize () ** ** Output results: ** nthreads: Updated number of threads -** threadid: Our thread id added to list on 1st call +** threadid: Our thread id added to list on 1st call ** ** Return value: thread number (success) or GPTLerror (failure) */ @@ -4212,7 +4212,7 @@ static inline int get_thread_num (void) return t; #endif - /* + /* ** Thread id not found. Define a critical region, then start PAPI counters if ** necessary and modify threadid[] with our id. */ @@ -4236,7 +4236,7 @@ static inline int get_thread_num (void) threadid[nthreads] = mythreadid; #ifdef VERBOSE - printf ("PTHREADS %s: 1st call threadid=%lu maps to location %d\n", + printf ("PTHREADS %s: 1st call threadid=%lu maps to location %d\n", thisfunc, (unsigned long) mythreadid, nthreads); #endif @@ -4249,14 +4249,14 @@ static inline int get_thread_num (void) if (GPTLget_npapievents () > 0) { #ifdef VERBOSE - printf ("PTHREADS get_thread_num: Starting EventSet threadid=%lu location=%d\n", + printf ("PTHREADS get_thread_num: Starting EventSet threadid=%lu location=%d\n", (unsigned long) mythreadid, nthreads); #endif if (GPTLcreate_and_start_events (nthreads) < 0) { if (unlock_mutex () < 0) fprintf (stderr, "PTHREADS %s: mutex unlock failure\n", thisfunc); - return GPTLerror ("PTHREADS %s: error from GPTLcreate_and_start_events for thread %d\n", + return GPTLerror ("PTHREADS %s: error from GPTLcreate_and_start_events for thread %d\n", thisfunc, nthreads); } } diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl.inc b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl.inc index 4d9d782a79..2ed2ca5c07 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl.inc +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl.inc @@ -97,7 +97,7 @@ integer gptlstart_handle integer gptlstop integer gptlstop_handle - integer gptlstamp + integer gptlstamp integer gptlpr_set_append integer gptlpr_query_append integer gptlpr_set_write @@ -107,7 +107,7 @@ integer gptlpr_summary integer gptlpr_summary_file integer gptlbarrier - integer gptlreset + integer gptlreset integer gptlfinalize integer gptlget_memusage integer gptlprint_memusage @@ -130,7 +130,7 @@ external gptlstart_handle external gptlstop external gptlstop_handle - external gptlstamp + external gptlstamp external gptlpr_set_append external gptlpr_query_append external gptlpr_set_write @@ -140,7 +140,7 @@ external gptlpr_summary external gptlpr_summary_file external gptlbarrier - external gptlreset + external gptlreset external gptlfinalize external gptlget_memusage external gptlprint_memusage diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl_papi.c b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl_papi.c index a8e42fd132..1f701cb897 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl_papi.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/gptl_papi.c @@ -5,7 +5,7 @@ ** ** Contains routines which interface to PAPI library */ - + #include "private.h" #include "gptl.h" @@ -149,8 +149,8 @@ static const Entry derivedtable [] = { }; static const int nderivedentries = sizeof (derivedtable) / sizeof (Entry); -static int npapievents = 0; /* number of PAPI events: initialize to 0 */ -static int nevents = 0; /* number of events: initialize to 0 */ +static int npapievents = 0; /* number of PAPI events: initialize to 0 */ +static int nevents = 0; /* number of events: initialize to 0 */ static int *EventSet; /* list of events to be counted by PAPI */ static long_long **papicounters; /* counters returned from PAPI */ @@ -171,11 +171,11 @@ static int enable (int); static int getderivedidx (int); /* -** GPTL_PAPIsetoption: enable or disable PAPI event defined by "counter". Called +** GPTL_PAPIsetoption: enable or disable PAPI event defined by "counter". Called ** from GPTLsetoption. Since all events are off by default, val=false degenerates ** to a no-op. Coded this way to be consistent with the rest of GPTL ** -** Input args: +** Input args: ** counter: PAPI counter ** val: true or false for enable or disable ** @@ -219,7 +219,7 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ break; } - /* + /* ** If val is false, return an error if the event has already been enabled. ** Otherwise just warn that attempting to disable a PAPI-based event ** that has already been enabled doesn't work--for now it's just a no-op @@ -238,10 +238,10 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ /* If the event has already been enabled for printing, exit */ if (already_enabled (counter)) - return GPTLerror ("GPTL_PAPIsetoption: counter %d has already been enabled\n", + return GPTLerror ("GPTL_PAPIsetoption: counter %d has already been enabled\n", counter); - /* + /* ** Initialize PAPI if it hasn't already been done. ** From here on down we can assume the intent is to enable (not disable) an option */ @@ -267,7 +267,7 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ pr_event[nevents].numidx = enable (PAPI_TOT_INS); pr_event[nevents].denomidx = enable (PAPI_TOT_CYC); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_TOT_INS / PAPI_TOT_CYC\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_TOT_INS / PAPI_TOT_CYC\n", pr_event[nevents].event.namestr); ++nevents; return 0; @@ -278,18 +278,18 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ pr_event[nevents].numidx = enable (PAPI_FP_OPS); pr_event[nevents].denomidx = enable (PAPI_LST_INS); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_LST_INS\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_LST_INS\n", pr_event[nevents].event.namestr); } else if (canenable2 (PAPI_FP_OPS, PAPI_L1_DCA)) { pr_event[nevents].event = derivedtable[idx]; pr_event[nevents].numidx = enable (PAPI_FP_OPS); pr_event[nevents].denomidx = enable (PAPI_L1_DCA); #ifdef DEBUG - printf ("GPTL_PAPIsetoption: pr_event %d is derived and will be PAPI event %d / %d\n", + printf ("GPTL_PAPIsetoption: pr_event %d is derived and will be PAPI event %d / %d\n", nevents, pr_event[nevents].numidx, pr_event[nevents].denomidx); #endif if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_L1_DCA\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_L1_DCA\n", pr_event[nevents].event.namestr); } else { return GPTLerror ("GPTL_PAPIsetoption: GPTL_CI unavailable\n"); @@ -305,7 +305,7 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ pr_event[nevents].numidx = enable (PAPI_FP_OPS); pr_event[nevents].denomidx = enable (PAPI_TOT_CYC); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_TOT_CYC\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_TOT_CYC\n", pr_event[nevents].event.namestr); ++nevents; return 0; @@ -318,7 +318,7 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ pr_event[nevents].numidx = enable (PAPI_FP_OPS); pr_event[nevents].denomidx = enable (PAPI_TOT_INS); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_TOT_INS\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_TOT_INS\n", pr_event[nevents].event.namestr); ++nevents; return 0; @@ -329,14 +329,14 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ pr_event[nevents].numidx = enable (PAPI_LST_INS); pr_event[nevents].denomidx = enable (PAPI_TOT_INS); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_TOT_INS\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_TOT_INS\n", pr_event[nevents].event.namestr); } else if (canenable2 (PAPI_L1_DCA, PAPI_TOT_INS)) { pr_event[nevents].event = derivedtable[idx]; pr_event[nevents].numidx = enable (PAPI_L1_DCA); pr_event[nevents].denomidx = enable (PAPI_TOT_INS); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_TOT_INS\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_TOT_INS\n", pr_event[nevents].event.namestr); } else { return GPTLerror ("GPTL_PAPIsetoption: GPTL_LSTPI unavailable\n"); @@ -352,7 +352,7 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ pr_event[nevents].numidx = enable (PAPI_L1_DCM); pr_event[nevents].denomidx = enable (PAPI_L1_DCA); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCM / PAPI_L1_DCA\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCM / PAPI_L1_DCA\n", pr_event[nevents].event.namestr); ++nevents; return 0; @@ -363,14 +363,14 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ pr_event[nevents].numidx = enable (PAPI_LST_INS); pr_event[nevents].denomidx = enable (PAPI_L1_DCM); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_L1_DCM\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_L1_DCM\n", pr_event[nevents].event.namestr); } else if (canenable2 (PAPI_L1_DCA, PAPI_L1_DCM)) { pr_event[nevents].event = derivedtable[idx]; pr_event[nevents].numidx = enable (PAPI_L1_DCA); pr_event[nevents].denomidx = enable (PAPI_L1_DCM); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_L1_DCM\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_L1_DCM\n", pr_event[nevents].event.namestr); } else { return GPTLerror ("GPTL_PAPIsetoption: GPTL_LSTPDCM unavailable\n"); @@ -389,7 +389,7 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ pr_event[nevents].numidx = enable (PAPI_L2_TCM); pr_event[nevents].denomidx = enable (PAPI_L2_TCA); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L2_TCM / PAPI_L2_TCA\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L2_TCM / PAPI_L2_TCA\n", pr_event[nevents].event.namestr); ++nevents; return 0; @@ -400,14 +400,14 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ pr_event[nevents].numidx = enable (PAPI_LST_INS); pr_event[nevents].denomidx = enable (PAPI_L2_TCM); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_L2_TCM\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_L2_TCM\n", pr_event[nevents].event.namestr); } else if (canenable2 (PAPI_L1_DCA, PAPI_L2_TCM)) { pr_event[nevents].event = derivedtable[idx]; pr_event[nevents].numidx = enable (PAPI_L1_DCA); pr_event[nevents].denomidx = enable (PAPI_L2_TCM); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_L2_TCM\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_L2_TCM\n", pr_event[nevents].event.namestr); } else { return GPTLerror ("GPTL_PAPIsetoption: GPTL_LSTPL2M unavailable\n"); @@ -423,7 +423,7 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ pr_event[nevents].numidx = enable (PAPI_L3_TCM); pr_event[nevents].denomidx = enable (PAPI_L3_TCR); if (verbose) - printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L3_TCM / PAPI_L3_TCR\n", + printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L3_TCM / PAPI_L3_TCR\n", pr_event[nevents].event.namestr); ++nevents; return 0; @@ -444,11 +444,11 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ pr_event[nevents].numidx = enable (counter); pr_event[nevents].denomidx = -1; /* flag says not derived (no denominator) */ } else { - return GPTLerror ("GPTL_PAPIsetoption: Can't enable event \n", + return GPTLerror ("GPTL_PAPIsetoption: Can't enable event \n", papitable[n].longstr); } if (verbose) - printf ("GPTL_PAPIsetoption: enabling PAPI preset event %s\n", + printf ("GPTL_PAPIsetoption: enabling PAPI preset event %s\n", pr_event[nevents].event.namestr); ++nevents; return 0; @@ -458,9 +458,9 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ /* ** Check native events last: If PAPI_event_code_to_name fails, give up */ - + if ((ret = PAPI_event_code_to_name (counter, eventname)) != PAPI_OK) - return GPTLerror ("GPTL_PAPIsetoption: name not found for counter %d: PAPI_strerror: %s\n", + return GPTLerror ("GPTL_PAPIsetoption: name not found for counter %d: PAPI_strerror: %s\n", counter, PAPI_strerror (ret)); /* @@ -514,12 +514,12 @@ int GPTL_PAPIsetoption (const int counter, /* PAPI counter (or option) */ /* ** canenable: determine whether a PAPI counter can be enabled ** -** Input args: +** Input args: ** counter: PAPI counter ** ** Return value: 0 (success) or non-zero (failure) */ - + int canenable (int counter) { char eventname[PAPI_MAX_STR_LEN]; /* returned from PAPI_event_code_to_name */ @@ -539,13 +539,13 @@ int canenable (int counter) /* ** canenable2: determine whether 2 PAPI counters can be enabled ** -** Input args: +** Input args: ** counter1: PAPI counter ** counter2: PAPI counter ** ** Return value: 0 (success) or non-zero (failure) */ - + int canenable2 (int counter1, int counter2) { char eventname[PAPI_MAX_STR_LEN]; /* returned from PAPI_event_code_to_name */ @@ -573,12 +573,12 @@ int canenable2 (int counter1, int counter2) ** well as output directly. E.g. PAPI_FP_OPS is used to compute ** computational intensity, and floating point ops per instruction. ** -** Input args: +** Input args: ** counter: PAPI counter ** ** Return value: index into papieventlist (success) or negative (not found) */ - + int papievent_is_enabled (int counter) { int n; @@ -591,14 +591,14 @@ int papievent_is_enabled (int counter) /* ** already_enabled: determine whether a PAPI-based event has already been -** enabled for printing. +** enabled for printing. ** -** Input args: +** Input args: ** counter: PAPI or derived counter ** ** Return value: 1 (true) or 0 (false) */ - + int already_enabled (int counter) { int n; @@ -613,12 +613,12 @@ int already_enabled (int counter) ** enable: enable a PAPI event. ASSUMES that canenable() has already determined ** that the event can be enabled. ** -** Input args: +** Input args: ** counter: PAPI counter ** ** Return value: index into papieventlist */ - + int enable (int counter) { int n; @@ -643,7 +643,7 @@ int enable (int counter) /* ** getderivedidx: find the table index of a derived counter ** -** Input args: +** Input args: ** counter: derived counter ** ** Return value: index into derivedtable (success) or GPTLerror (failure) @@ -672,7 +672,7 @@ int GPTL_PAPIlibraryinit () if ((ret = PAPI_is_initialized ()) == PAPI_NOT_INITED) { if ((ret = PAPI_library_init (PAPI_VER_CURRENT)) != PAPI_VER_CURRENT) { - fprintf (stderr, "GPTL_PAPIlibraryinit: ret=%d PAPI_VER_CURRENT=%d\n", + fprintf (stderr, "GPTL_PAPIlibraryinit: ret=%d PAPI_VER_CURRENT=%d\n", ret, (int) PAPI_VER_CURRENT); return GPTLerror ("GPTL_PAPIlibraryinit: PAPI_library_init failure:%s\n", PAPI_strerror (ret)); @@ -683,16 +683,16 @@ int GPTL_PAPIlibraryinit () /* ** GPTL_PAPIinitialize(): Initialize the PAPI interface. Called from GPTLinitialize. -** PAPI_library_init must be called before any other PAPI routines. +** PAPI_library_init must be called before any other PAPI routines. ** PAPI_thread_init is called subsequently if threading is enabled. ** Finally, allocate space for PAPI counters and start them. ** -** Input args: +** Input args: ** maxthreads: number of threads ** ** Return value: 0 (success) or GPTLerror or -1 (failure) */ - + int GPTL_PAPIinitialize (const int maxthreads, /* number of threads */ const bool verbose_flag, /* output verbosity */ int *nevents_out, /* nevents needed by gptl.c */ @@ -748,8 +748,8 @@ int GPTL_PAPIinitialize (const int maxthreads, /* number of threads */ ** Threaded routine to create the "event set" (PAPI terminology) and start ** the counters. This is only done once, and is called from get_thread_num ** for the first time for the thread. -** -** Input args: +** +** Input args: ** t: thread number ** ** Return value: 0 (success) or GPTLerror (failure) @@ -764,7 +764,7 @@ int GPTLcreate_and_start_events (const int t) /* thread number */ /* Create the event set */ if ((ret = PAPI_create_eventset (&EventSet[t])) != PAPI_OK) - return GPTLerror ("GPTLcreate_and_start_events: thread %d failure creating eventset: %s\n", + return GPTLerror ("GPTLcreate_and_start_events: thread %d failure creating eventset: %s\n", t, PAPI_strerror (ret)); if (verbose) @@ -797,20 +797,20 @@ int GPTLcreate_and_start_events (const int t) /* thread number */ if ((ret = PAPI_cleanup_eventset (EventSet[t])) != PAPI_OK) return GPTLerror ("GPTLcreate_and_start_events: %s\n", PAPI_strerror (ret)); - + if ((ret = PAPI_destroy_eventset (&EventSet[t])) != PAPI_OK) return GPTLerror ("GPTLcreate_and_start_events: %s\n", PAPI_strerror (ret)); if ((ret = PAPI_create_eventset (&EventSet[t])) != PAPI_OK) - return GPTLerror ("GPTLcreate_and_start_events: failure creating eventset: %s\n", + return GPTLerror ("GPTLcreate_and_start_events: failure creating eventset: %s\n", PAPI_strerror (ret)); if ((ret = PAPI_multiplex_init ()) != PAPI_OK) - return GPTLerror ("GPTLcreate_and_start_events: failure from PAPI_multiplex_init%s\n", + return GPTLerror ("GPTLcreate_and_start_events: failure from PAPI_multiplex_init%s\n", PAPI_strerror (ret)); if ((ret = PAPI_set_multiplex (EventSet[t])) != PAPI_OK) - return GPTLerror ("GPTLcreate_and_start_events: failure from PAPI_set_multiplex: %s\n", + return GPTLerror ("GPTLcreate_and_start_events: failure from PAPI_set_multiplex: %s\n", PAPI_strerror (ret)); for (n = 0; n < npapievents; n++) { @@ -825,20 +825,20 @@ int GPTLcreate_and_start_events (const int t) /* thread number */ /* Start the event set. It will only be read from now on--never stopped */ if ((ret = PAPI_start (EventSet[t])) != PAPI_OK) - return GPTLerror ("GPTLcreate_and_start_events: failed to start event set: %s\n", + return GPTLerror ("GPTLcreate_and_start_events: failed to start event set: %s\n", PAPI_strerror (ret)); return 0; } /* -** GPTL_PAPIstart: Start the PAPI counters (actually they are just read). +** GPTL_PAPIstart: Start the PAPI counters (actually they are just read). ** Called from GPTLstart. ** -** Input args: +** Input args: ** t: thread number ** -** Output args: +** Output args: ** aux: struct containing the counters ** ** Return value: 0 (success) or GPTLerror (failure) @@ -849,7 +849,7 @@ int GPTL_PAPIstart (const int t, /* thread number */ { int ret; /* return code from PAPI lib calls */ int n; /* loop index */ - + /* If no events are to be counted just return */ if (npapievents == 0) @@ -860,25 +860,25 @@ int GPTL_PAPIstart (const int t, /* thread number */ if ((ret = PAPI_read (EventSet[t], papicounters[t])) != PAPI_OK) return GPTLerror ("GPTL_PAPIstart: %s\n", PAPI_strerror (ret)); - /* + /* ** Store the counter values. When GPTL_PAPIstop is called, the counters ** will again be read, and differenced with the values saved here. */ for (n = 0; n < npapievents; n++) aux->last[n] = papicounters[t][n]; - + return 0; } /* -** GPTL_PAPIstop: Stop the PAPI counters (actually they are just read). +** GPTL_PAPIstop: Stop the PAPI counters (actually they are just read). ** Called from GPTLstop. ** ** Input args: ** t: thread number ** -** Input/output args: +** Input/output args: ** aux: struct containing the counters ** ** Return value: 0 (success) or GPTLerror (failure) @@ -900,8 +900,8 @@ int GPTL_PAPIstop (const int t, /* thread number */ if ((ret = PAPI_read (EventSet[t], papicounters[t])) != PAPI_OK) return GPTLerror ("GPTL_PAPIstop: %s\n", PAPI_strerror (ret)); - - /* + + /* ** Accumulate the difference since timer start in aux. ** Negative accumulation can happen when multiplexing is enabled, so don't ** set count to BADCOUNT in that case. @@ -924,14 +924,14 @@ int GPTL_PAPIstop (const int t, /* thread number */ ** GPTL_PAPIprstr: Print the descriptive string for all enabled PAPI events. ** Called from GPTLpr. ** -** Input args: +** Input args: ** fp: file descriptor */ void GPTL_PAPIprstr (FILE *fp) { int n; - + if (narrowprint) { for (n = 0; n < nevents; n++) { fprintf (fp, "%8.8s ", pr_event[n].event.str8); @@ -957,7 +957,7 @@ void GPTL_PAPIprstr (FILE *fp) ** GPTL_PAPIpr: Print PAPI counter values for all enabled events, including ** derived events. Called from GPTLpr. ** -** Input args: +** Input args: ** fp: file descriptor ** aux: struct containing the counters */ @@ -989,7 +989,7 @@ void GPTL_PAPIpr (FILE *fp, /* file descriptor to write denomidx = pr_event[n].denomidx; #ifdef DEBUG - printf ("GPTL_PAPIpr: derived event: numidx=%d denomidx=%d values = %ld %ld\n", + printf ("GPTL_PAPIpr: derived event: numidx=%d denomidx=%d values = %ld %ld\n", numidx, denomidx, (long) aux->accum[numidx], (long) aux->accum[denomidx]); #endif /* Protect against divide by zero */ @@ -1003,7 +1003,7 @@ void GPTL_PAPIpr (FILE *fp, /* file descriptor to write } else { /* Raw PAPI event */ #ifdef DEBUG - printf ("GPTL_PAPIpr: raw event: numidx=%d value = %ld\n", + printf ("GPTL_PAPIpr: raw event: numidx=%d value = %ld\n", numidx, (long) aux->accum[numidx]); #endif if (aux->accum[numidx] < PRTHRESH) @@ -1055,12 +1055,12 @@ void GPTL_PAPIprintenabled (FILE *fp) fprintf (fp, " %s\n", eventname); fprintf (fp, "\n"); } -} +} /* ** GPTL_PAPIadd: Accumulate PAPI counters. Called from add. ** -** Input/Output args: +** Input/Output args: ** auxout: auxout = auxout + auxin ** ** Input args: @@ -1071,7 +1071,7 @@ void GPTL_PAPIadd (Papistats *auxout, /* output struct */ const Papistats *auxin) /* input struct */ { int n; - + for (n = 0; n < npapievents; n++) if (auxin->accum[n] == BADCOUNT || auxout->accum[n] == BADCOUNT) auxout->accum[n] = BADCOUNT; @@ -1229,7 +1229,7 @@ int GPTLevent_name_to_code (const char *name, int *code) int n; /* loop over derived entries */ /* - ** First check derived events + ** First check derived events */ for (n = 0; n < nderivedentries; ++n) { @@ -1272,7 +1272,7 @@ int GPTLevent_code_to_name (const int code, char *name) int n; /* loop over derived entries */ /* - ** First check derived events + ** First check derived events */ for (n = 0; n < nderivedentries; ++n) { @@ -1323,4 +1323,3 @@ int GPTLevent_code_to_name (int code, char *name) } #endif /* HAVE_PAPI */ - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/perf_mod.F90 b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/perf_mod.F90 index 622d5cf2c1..8490a35d26 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/perf_mod.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/perf_mod.F90 @@ -1,14 +1,14 @@ module perf_mod -!----------------------------------------------------------------------- -! +!----------------------------------------------------------------------- +! ! Purpose: This module is responsible for controlling the performance ! timer logic. -! +! ! Author: P. Worley, January 2007 ! ! $Id$ -! +! !----------------------------------------------------------------------- !----------------------------------------------------------------------- @@ -62,7 +62,7 @@ module perf_mod !----------------------------------------------------------------------- !- include statements -------------------------------------------------- !----------------------------------------------------------------------- -#include <mpif.h> +#include <mpif.h> #include "gptl.inc" !----------------------------------------------------------------------- @@ -93,7 +93,7 @@ module perf_mod integer, parameter :: def_timer_depth_limit = 99999 ! default integer, private :: timer_depth_limit = def_timer_depth_limit ! integer indicating maximum number of levels of - ! timer nesting + ! timer nesting integer, parameter :: def_timing_detail_limit = 1 ! default integer, private :: timing_detail_limit = def_timing_detail_limit @@ -111,19 +111,19 @@ module perf_mod logical, parameter :: def_perf_single_file = .false. ! default logical, private :: perf_single_file = def_perf_single_file ! flag indicating whether the performance timer - ! output should be written to a single file - ! (per component communicator) or to a + ! output should be written to a single file + ! (per component communicator) or to a ! separate file for each process integer, parameter :: def_perf_outpe_num = 0 ! default integer, private :: perf_outpe_num = def_perf_outpe_num - ! maximum number of processes writing out + ! maximum number of processes writing out ! timing data (for this component communicator) integer, parameter :: def_perf_outpe_stride = 1 ! default integer, private :: perf_outpe_stride = def_perf_outpe_stride ! separation between process ids for processes - ! that are writing out timing data + ! that are writing out timing data ! (for this component communicator) logical, parameter :: def_perf_global_stats = .true. ! default @@ -176,9 +176,9 @@ module perf_mod !======================================================================== ! subroutine t_getLogUnit(LogUnit) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Get log unit number. -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments----------------------------- ! @@ -193,9 +193,9 @@ end subroutine t_getLogUnit !======================================================================== ! subroutine t_setLogUnit(LogUnit) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Set log unit number. -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments----------------------------- ! @@ -222,9 +222,9 @@ subroutine perf_defaultopts(timing_disable_out, & perf_single_file_out, & perf_global_stats_out, & perf_papi_enable_out ) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Return default runtime options -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments----------------------------- ! timers disable/enable option @@ -296,16 +296,16 @@ subroutine perf_setopts(mastertask, & perf_single_file_in, & perf_global_stats_in, & perf_papi_enable_in ) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Set runtime options -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments---------------------------- ! ! master process? logical, intent(in) :: mastertask ! Print out to log file? - logical, intent(IN) :: LogPrint + logical, intent(IN) :: LogPrint ! timers disable/enable option logical, intent(in), optional :: timing_disable_in ! performance timer option @@ -337,7 +337,7 @@ subroutine perf_setopts(mastertask, & timing_disable = timing_disable_in if (timing_disable) then ierr = GPTLdisable() - else + else ierr = GPTLenable() endif endif @@ -392,17 +392,17 @@ subroutine perf_setopts(mastertask, & endif ! if (mastertask .and. LogPrint) then - write(p_logunit,*) '(t_initf) Using profile_disable=', timing_disable, & + write(p_logunit,*) '(t_initf) Using profile_disable=', timing_disable, & ' profile_timer=', perf_timer - write(p_logunit,*) '(t_initf) profile_depth_limit=', timer_depth_limit, & + write(p_logunit,*) '(t_initf) profile_depth_limit=', timer_depth_limit, & ' profile_detail_limit=', timing_detail_limit write(p_logunit,*) '(t_initf) profile_barrier=', timing_barrier, & ' profile_outpe_num=', perf_outpe_num write(p_logunit,*) '(t_initf) profile_outpe_stride=', perf_outpe_stride , & ' profile_single_file=', perf_single_file write(p_logunit,*) '(t_initf) profile_global_stats=', perf_global_stats , & - ' profile_papi_enable=', perf_papi_enable - endif + ' profile_papi_enable=', perf_papi_enable + endif ! #ifdef DEBUG else @@ -420,9 +420,9 @@ subroutine papi_defaultopts(papi_ctr1_out, & papi_ctr2_out, & papi_ctr3_out, & papi_ctr4_out ) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Return default runtime PAPI counter options -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments----------------------------- ! PAPI counter option #1 @@ -456,9 +456,9 @@ subroutine papi_setopts(papi_ctr1_in, & papi_ctr2_in, & papi_ctr3_in, & papi_ctr4_in ) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Set runtime PAPI counter options -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments---------------------------- ! @@ -518,12 +518,12 @@ end subroutine papi_setopts !======================================================================== ! logical function t_profile_onf() -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Return flag indicating whether profiling is currently active. ! Part of workaround to implement FVbarrierclock before ! communicators exposed in Pilgrim. Does not check level of ! event nesting. -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- if ((.not. timing_initialized) .or. & @@ -539,10 +539,10 @@ end function t_profile_onf !======================================================================== ! logical function t_barrier_onf() -!----------------------------------------------------------------------- -! Purpose: Return timing_barrier. Part of workaround to implement -! FVbarrierclock before communicators exposed in Pilgrim. -! Author: P. Worley +!----------------------------------------------------------------------- +! Purpose: Return timing_barrier. Part of workaround to implement +! FVbarrierclock before communicators exposed in Pilgrim. +! Author: P. Worley !----------------------------------------------------------------------- t_barrier_onf = timing_barrier @@ -552,10 +552,10 @@ end function t_barrier_onf !======================================================================== ! logical function t_single_filef() -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Return perf_single_file. Used to control output of other ! performance data, only spmdstats currently. -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- t_single_filef = perf_single_file @@ -565,9 +565,9 @@ end function t_single_filef !======================================================================== ! subroutine t_stampf(wall, usr, sys) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Record wallclock, user, and system times (seconds). -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Output arguments----------------------------- ! @@ -596,14 +596,14 @@ end subroutine t_stampf !======================================================================== ! subroutine t_startf(event, handle) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Start an event timer -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments----------------------------- ! ! performance timer event name - character(len=*), intent(in) :: event + character(len=*), intent(in) :: event ! !---------------------------Input/Output arguments---------------------- ! @@ -634,14 +634,14 @@ end subroutine t_startf !======================================================================== ! subroutine t_stopf(event, handle) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Stop an event timer -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments----------------------------- ! ! performance timer event name - character(len=*), intent(in) :: event + character(len=*), intent(in) :: event ! !---------------------------Input/Output arguments---------------------- ! @@ -672,10 +672,10 @@ end subroutine t_stopf !======================================================================== ! subroutine t_enablef() -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Enable t_startf, t_stopf, t_stampf, and t_barrierf. Ignored ! in threaded regions. -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Local workspace----------------------------- ! @@ -709,10 +709,10 @@ end subroutine t_enablef !======================================================================== ! subroutine t_disablef() -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Disable t_startf, t_stopf, t_stampf, and t_barrierf. Ignored ! in threaded regions. -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Local workspace----------------------------- ! @@ -744,9 +744,9 @@ end subroutine t_disablef !======================================================================== ! subroutine t_adj_detailf(detail_adjustment) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Modify current detail level. Ignored in threaded regions. -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments----------------------------- ! @@ -776,11 +776,11 @@ end subroutine t_adj_detailf !======================================================================== ! subroutine t_barrierf(event, mpicom) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Call (and time) mpi_barrier. Ignored inside OpenMP ! threaded regions. Note that barrier executed even if ! event not recorded because of level of timer event nesting. -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments----------------------------- ! mpi communicator id @@ -835,9 +835,9 @@ end subroutine t_barrierf ! subroutine t_prf(filename, mpicom, num_outpe, stride_outpe, & single_file, global_stats, output_thispe) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Write out performance timer data -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments----------------------------- ! @@ -847,7 +847,7 @@ subroutine t_prf(filename, mpicom, num_outpe, stride_outpe, & integer, intent(in), optional :: mpicom ! maximum number of processes writing out timing data integer, intent(in), optional :: num_outpe - ! separation between process ids for processes writing out data + ! separation between process ids for processes writing out data integer, intent(in), optional :: stride_outpe ! enable/disable the writing of data to a single file logical, intent(in), optional :: single_file @@ -862,7 +862,7 @@ subroutine t_prf(filename, mpicom, num_outpe, stride_outpe, & ! all data to a single file logical glb_stats ! flag indicting whether to compute ! global statistics - logical pr_write ! flag indicating whether the current + logical pr_write ! flag indicating whether the current ! GPTL output mode is write logical write_data ! flag indicating whether this process ! should output its timing data @@ -1113,8 +1113,8 @@ end subroutine t_prf !======================================================================== ! subroutine t_initf(NLFilename, LogPrint, LogUnit, mpicom, MasterTask) -!----------------------------------------------------------------------- -! Purpose: Set default values of runtime timing options +!----------------------------------------------------------------------- +! Purpose: Set default values of runtime timing options ! before namelists prof_inparm and papi_inparm are read, ! read namelists (and broadcast, if SPMD), ! then initialize timing library. @@ -1224,12 +1224,12 @@ subroutine t_initf(NLFilename, LogPrint, LogUnit, mpicom, MasterTask) open( unitn, file=trim(NLFilename), status='old', iostat=ierr ) if (ierr .eq. 0) then - ! Look for prof_inparm group name in the input file. + ! Look for prof_inparm group name in the input file. ! If found, leave the file positioned at that namelist group. call find_group_name(unitn, 'prof_inparm', status=ierr) if (ierr == 0) then ! found prof_inparm - read(unitn, nml=prof_inparm, iostat=ierr) + read(unitn, nml=prof_inparm, iostat=ierr) if (ierr /= 0) then call shr_sys_abort( subname//':: namelist read returns an'// & ' error condition for prof_inparm' ) @@ -1291,12 +1291,12 @@ subroutine t_initf(NLFilename, LogPrint, LogUnit, mpicom, MasterTask) ierr = 1 open( unitn, file=trim(NLFilename), status='old', iostat=ierr ) if (ierr .eq. 0) then - ! Look for papi_inparm group name in the input file. + ! Look for papi_inparm group name in the input file. ! If found, leave the file positioned at that namelist group. call find_group_name(unitn, 'papi_inparm', status=ierr) if (ierr == 0) then ! found papi_inparm - read(unitn, nml=papi_inparm, iostat=ierr) + read(unitn, nml=papi_inparm, iostat=ierr) if (ierr /= 0) then call shr_sys_abort( subname//':: namelist read returns an'// & ' error condition for papi_inparm' ) @@ -1355,12 +1355,12 @@ subroutine t_initf(NLFilename, LogPrint, LogUnit, mpicom, MasterTask) !$OMP MASTER ! - ! Set options and initialize timing library. - ! + ! Set options and initialize timing library. + ! ! Set timer if (gptlsetutr (perf_timer) < 0) call shr_sys_abort (subname//':: gptlsetutr') ! - ! For logical settings, 2nd arg 0 + ! For logical settings, 2nd arg 0 ! to gptlsetoption means disable, non-zero means enable ! ! Turn off CPU timing (expensive) @@ -1404,9 +1404,9 @@ end subroutine t_initf !======================================================================== ! subroutine t_finalizef() -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: shut down timing library -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Local workspace----------------------------- ! diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/perf_utils.F90 b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/perf_utils.F90 index b0e722567e..7f0fadbeb9 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/perf_utils.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/perf_utils.F90 @@ -1,15 +1,15 @@ module perf_utils -!----------------------------------------------------------------------- -! +!----------------------------------------------------------------------- +! ! Purpose: This module supplies the csm_share and CAM utilities ! needed by perf_mod.F90 (when the csm_share and CAM utilities ! are not available). -! +! ! Author: P. Worley, October 2007 ! ! $Id$ -! +! !----------------------------------------------------------------------- #ifndef NO_MPIMOD use mpi @@ -49,7 +49,7 @@ module perf_utils !- include statements -------------------------------------------------- !----------------------------------------------------------------------- #ifdef NO_MPIMOD -#include <mpif.h> +#include <mpif.h> #endif #include "gptl.inc" @@ -82,9 +82,9 @@ module perf_utils !======================================================================== ! subroutine perfutils_setunit(LogUnit) -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- ! Purpose: Set log unit number. -! Author: P. Worley +! Author: P. Worley !----------------------------------------------------------------------- !---------------------------Input arguments----------------------------- ! @@ -327,7 +327,7 @@ END SUBROUTINE shr_mpi_bcastl0 ! 2005-Dec-14 - E. Kluzek - creation ! 2007-Oct-21 - P. Worley - dumbed down for use in perf_mod ! -! !INTERFACE: ------------------------------------------------------------------ +! !INTERFACE: ------------------------------------------------------------------ INTEGER FUNCTION shr_file_getUnit () @@ -377,7 +377,7 @@ END FUNCTION shr_file_getUnit ! 2005-Dec-14 - E. Kluzek - creation ! 2007-Oct-21 - P. Worley - dumbed down for use in perf_mod ! -! !INTERFACE: ------------------------------------------------------------------ +! !INTERFACE: ------------------------------------------------------------------ SUBROUTINE shr_file_freeUnit ( unit) @@ -418,12 +418,12 @@ END SUBROUTINE shr_file_freeUnit subroutine find_group_name(unit, group, status) !--------------------------------------------------------------------------------------- -! Purpose: +! Purpose: ! Search a file that contains namelist input for the specified namelist group name. ! Leave the file positioned so that the current record is the first record of the ! input for the specified group. -! -! Method: +! +! Method: ! Read the file line by line. Each line is searched for an '&' which may only ! be preceded by blanks, immediately followed by the group name which is case ! insensitive. If found then backspace the file so the current record is the @@ -489,17 +489,17 @@ end subroutine find_group_name function to_lower(str) -!----------------------------------------------------------------------- -! Purpose: +!----------------------------------------------------------------------- +! Purpose: ! Convert character string to lower case. -! -! Method: +! +! Method: ! Use achar and iachar intrinsics to ensure use of ascii collating sequence. ! ! Author: B. Eaton, July 2001 -! +! ! $Id$ -!----------------------------------------------------------------------- +!----------------------------------------------------------------------- implicit none character(len=*), intent(in) :: str ! String to convert to lower case @@ -518,7 +518,7 @@ function to_lower(str) ctmp = str(i:i) aseq = iachar(ctmp) if ( aseq >= iachar("A") .and. aseq <= iachar("Z") ) & - ctmp = achar(aseq + upper_to_lower) + ctmp = achar(aseq + upper_to_lower) to_lower(i:i) = ctmp end do diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/private.h b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/private.h index 8d14b1479a..c8a52a9f35 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/private.h +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/private.h @@ -32,7 +32,7 @@ /* longest timer name allowed (probably safe to just change) */ #define MAX_CHARS 63 -/* +/* ** max allowable number of PAPI counters, or derived events. For convenience, ** set to max (# derived events, # papi counters required) so "avail" lists ** all available options. @@ -61,7 +61,7 @@ typedef struct { long long last[MAX_AUX]; /* array of saved counters from "start" */ long long accum[MAX_AUX]; /* accumulator for counters */ } Papistats; - + typedef struct { int counter; /* PAPI or Derived counter */ char *namestr; /* PAPI or Derived counter as string */ @@ -84,7 +84,7 @@ typedef struct TIMER { #endif #ifdef HAVE_PAPI Papistats aux; /* PAPI stats */ -#endif +#endif Wallstats wall; /* wallclock stats */ Cpustats cpu; /* cpu stats */ unsigned long count; /* number of start/stop calls */ @@ -127,7 +127,7 @@ extern void __cyg_profile_func_exit (void *, void *); }; #endif -/* +/* ** These are needed for communication between gptl.c and gptl_papi.c */ diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/threadutil.c b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/threadutil.c index 85d9c52d15..86e4681c1c 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/threadutil.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/gptl/threadutil.c @@ -2,7 +2,7 @@ ** $Id: threadutil.c,v 1.28 2009/12/31 01:51:59 rosinski Exp $ ** ** Author: Jim Rosinski -** +** ** Utility functions handle thread-based GPTL needs. */ @@ -21,7 +21,7 @@ static bool first = true; /**********************************************************************************/ -/* +/* ** 3 sets of routines: OMP threading, PTHREADS, unthreaded */ @@ -62,7 +62,7 @@ int threadinit (int *nthreads, int *maxthreads) #ifdef VERBOSE printf ("OMP threadinit: Set *maxthreads=%d *nthreads=%d\n", *maxthreads, *nthreads); #endif - + return 0; } @@ -138,7 +138,7 @@ void print_threadmapping (int nthreads, FILE *fp) } /**********************************************************************************/ -/* +/* ** PTHREADS */ @@ -179,7 +179,7 @@ int threadinit (int *nthreads, int *maxthreads) first = false; /* - ** Initialize nthreads to 0 and define the threadid array now that initialization + ** Initialize nthreads to 0 and define the threadid array now that initialization ** is done. The actual value will be determined as get_thread_num is called. */ @@ -228,8 +228,8 @@ int get_thread_num (int *nthreads, int *maxthreads) return GPTLerror ("get_thread_num: mutex lock failure\n"); /* - ** Loop over known physical thread IDs. When my id is found, map it - ** to logical thread id for indexing. If not found return a negative + ** Loop over known physical thread IDs. When my id is found, map it + ** to logical thread id for indexing. If not found return a negative ** number. ** A critical region is necessary because acess to ** the array threadid must be by only one thread at a time. @@ -251,7 +251,7 @@ int get_thread_num (int *nthreads, int *maxthreads) return GPTLerror ("get_thread_num: nthreads=%d is too big Recompile " "with larger value of MAX_THREADS\n", *nthreads); - } + } threadid[n] = mythreadid; @@ -268,7 +268,7 @@ int get_thread_num (int *nthreads, int *maxthreads) if (GPTLget_npapievents () > 0) { #ifdef VERBOSE - printf ("PTHREADS get_thread_num: Starting EventSet threadid=%lu location=%d\n", + printf ("PTHREADS get_thread_num: Starting EventSet threadid=%lu location=%d\n", (unsigned long) mythreadid, n); #endif if (GPTLcreate_and_start_events (n) < 0) { @@ -286,7 +286,7 @@ int get_thread_num (int *nthreads, int *maxthreads) printf ("PTHREADS get_thread_num: *nthreads=%d\n", *nthreads); #endif } - + if (unlock_mutex () < 0) return GPTLerror ("get_thread_num: mutex unlock failure\n"); diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncint_pio.c b/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncint_pio.c index a8c606dd80..e4a59bad28 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncint_pio.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncint_pio.c @@ -25,14 +25,16 @@ nc_def_iosystem(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int ret; /* Make sure PIO was initialized. */ - if ((ret = PIO_NCINT_initialize())) + if ((ret = PIO_NCINT_initialize())){ + //printf("%s %d ret=%d\n",__FILE__,__LINE__,ret); return ret; - + } /* Call the PIOc_ function to initialize the intracomm. */ if ((ret = PIOc_Init_Intracomm(comp_comm, num_iotasks, stride, base, rearr, - iosysidp))) + iosysidp))){ + //printf("%s %d ret=%d\n",__FILE__,__LINE__,ret); return ret; - + } /* Remember the io system id. */ diosysid = *iosysidp; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncintdispatch.c b/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncintdispatch.c index db442a8c7f..30ec267816 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncintdispatch.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncintdispatch.c @@ -25,18 +25,18 @@ int diosysid; int ncint_initialized = 0; /** Version of dispatch table. */ -#define DISPATCH_VERSION 2 +#define DISPATCH_VERSION NC_DISPATCH_VERSION /* Internal filter actions - copied from nc4internal.h */ -#define NCFILTER_DEF 1 -#define NCFILTER_REMOVE 2 -#define NCFILTER_INQ 3 +#define NCFILTER_DEF 1 +#define NCFILTER_REMOVE 2 +#define NCFILTER_INQ 3 #define NCFILTER_FILTERIDS 4 -#define NCFILTER_INFO 5 -#define NCFILTER_FREESPEC 6 -#define NCFILTER_CLIENT_REG 10 -#define NCFILTER_CLIENT_UNREG 11 -#define NCFILTER_CLIENT_INQ 12 +#define NCFILTER_INFO 5 +#define NCFILTER_FREESPEC 6 +#define NCFILTER_CLIENT_REG 10 +#define NCFILTER_CLIENT_UNREG 11 +#define NCFILTER_CLIENT_INQ 12 /* This is the dispatch object that holds pointers to all the * functions that make up the NCINT dispatch interface. */ @@ -127,7 +127,22 @@ NC_Dispatch NCINT_dispatcher = { NC_NOTNC4_def_var_filter, NC_NOTNC4_set_var_chunk_cache, NC_NOTNC4_get_var_chunk_cache, - PIO_NCINT_filter_actions +#ifdef PIO_HAS_PAR_FILTERS +#if NC_DISPATCH_VERSION == 2 + PIO_NCINT_filter_actions, +#endif +#if NC_DISPATCH_VERSION >= 3 + PIOc_inq_var_filter_ids, + PIOc_inq_var_filter_info, +#endif +#if NC_DISPATCH_VERSION >= 4 + PIOc_def_var_quantize, + PIOc_inq_var_quantize, +#endif +#if NC_DISPATCH_VERSION >= 5 + PIOc_inq_filter_avail, +#endif +#endif }; /** @@ -206,9 +221,8 @@ PIO_NCINT_create(const char *path, int cmode, size_t initialsz, int basepe, if (!(ios = pio_get_iosystem_from_id(diosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); - /* Turn of NC_UDF0 in the mode flag. */ - cmode = cmode & ~NC_UDF0; - + /* Turn off NC_UDF0 in the mode flag. */ + cmode = ((cmode) & ~(NC_UDF0)); /* Find the IOTYPE from the mode flag. */ if ((ret = find_iotype_from_omode(cmode, &iotype))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); @@ -217,10 +231,11 @@ PIO_NCINT_create(const char *path, int cmode, size_t initialsz, int basepe, if ((ret = nc4_file_list_add(ncid, path, cmode, NULL))) return ret; - /* Create the file with PIO. The final parameter tests + /* Create the file with PIO. The final parameter tells * createfile_int to accept the externally assigned ncid. */ if ((ret = PIOc_createfile_int(diosysid, &ncid, &iotype, path, cmode, 1))) return ret; + return PIO_NOERR; } @@ -257,8 +272,8 @@ PIO_NCINT_open(const char *path, int mode, int basepe, size_t *chunksizehintp, if (!(ios = pio_get_iosystem_from_id(diosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); - /* Turn of NC_UDF0 in the mode flag. */ - mode = mode & ~NC_UDF0; + /* Turn off NC_UDF0 in the mode flag. */ + mode = (mode) & ~(NC_UDF0); /* Find the IOTYPE from the mode flag. */ if ((ret = find_iotype_from_omode(mode, &iotype))) @@ -266,11 +281,11 @@ PIO_NCINT_open(const char *path, int mode, int basepe, size_t *chunksizehintp, /* Add necessary structs to hold netcdf-4 file data. */ if ((ret = nc4_file_list_add(ncid, path, mode, NULL))) - return ret; + return ret; /* Open the file with PIO. Tell openfile_retry to accept the * externally assigned ncid. */ - if ((ret = PIOc_openfile_retry(diosysid, &ncid, &iotype, path, mode, 0, 1))) + if ((ret = PIOc_openfile_retry(diosysid, &ncid, &iotype, path, mode, 1, 1))) return ret; return NC_NOERR; @@ -417,8 +432,8 @@ PIO_NCINT_inq_format_extended(int ncid, int *formatp, int *modep) PLOG((2, "%s: ncid 0x%x", __func__, ncid)); - if ((retval = nc4_file_list_get(ncid, NULL, &my_mode, NULL))) - return NC_EBADID; + if ((retval = PIOc_inq_format(ncid, &my_mode))) + return retval; if (modep) *modep = my_mode|NC_UDF0; @@ -883,18 +898,22 @@ PIO_NCINT_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, unsigned int *idp, size_t *nparamsp, unsigned int *params) { int ret; + int format; ret = PIOc_inq_var(ncid, varid, name, xtypep, ndimsp, dimidsp, nattsp); +#ifdef _NETCDF4 + ret = PIOc_inq_format(ncid, &format); + if (!ret && (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) ){ + if (!ret && contiguousp && chunksizesp) + ret = PIOc_inq_var_chunking(ncid, varid, contiguousp, (MPI_Offset *)chunksizesp); - if (!ret) - ret = PIOc_inq_var_chunking(ncid, varid, contiguousp, (MPI_Offset *)chunksizesp); + if (!ret && shufflep && deflatep && deflate_levelp) + ret = PIOc_inq_var_deflate(ncid, varid, shufflep, deflatep, deflate_levelp); - if (!ret) - ret = PIOc_inq_var_deflate(ncid, varid, shufflep, deflatep, deflate_levelp); - - if (!ret) - ret = PIOc_inq_var_endian(ncid, varid, endiannessp); - + if (!ret && endiannessp) + ret = PIOc_inq_var_endian(ncid, varid, endiannessp); + } +#endif return ret; } @@ -993,7 +1012,7 @@ PIO_NCINT_inq_type_equal(int ncid1, nc_type typeid1, int ncid2, */ int PIO_NCINT_def_var_deflate(int ncid, int varid, int shuffle, int deflate, - int deflate_level) + int deflate_level) { return PIOc_def_var_deflate(ncid, varid, shuffle, deflate, deflate_level); } @@ -1028,6 +1047,7 @@ PIO_NCINT_def_var_chunking(int ncid, int varid, int storage, const size_t *chunk return PIOc_def_var_chunking(ncid, varid, storage, (const PIO_Offset *)chunksizesp); } +#if NC_DISPATCH_VERSION == 2 /** * @internal Carry out one of several filter actions * @@ -1043,7 +1063,8 @@ PIO_NCINT_filter_actions(int ncid, int varid, int action, struct NC_Filterobject { if (action == NCFILTER_INFO) { - + } return PIO_NOERR; } +#endif diff --git a/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncintdispatch.h b/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncintdispatch.h index 8934740f55..26764726dc 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncintdispatch.h +++ b/src/Infrastructure/IO/PIO/ParallelIO/src/ncint/ncintdispatch.h @@ -147,15 +147,15 @@ extern "C" { extern int PIO_NCINT_def_var_deflate(int ncid, int varid, int shuffle, int deflate, - int deflate_level); + int deflate_level); extern int PIO_NCINT_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp); - +#if NC_DISPATCH_VERSION == 2 extern int PIO_NCINT_filter_actions(int ncid, int varid, int action, struct NC_Filterobject* spec); - +#endif #if defined(__cplusplus) } #endif diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cperf/CMakeLists.txt b/src/Infrastructure/IO/PIO/ParallelIO/tests/cperf/CMakeLists.txt index fed02b9e33..e82c297357 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cperf/CMakeLists.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cperf/CMakeLists.txt @@ -5,9 +5,10 @@ include_directories("${CMAKE_SOURCE_DIR}/src/clib") include_directories("${CMAKE_BINARY_DIR}") # Compiler-specific compiler options +message("Compiler is ${CMAKE_C_COMPILER_ID}") if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu99") elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -c99") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -c99") @@ -15,8 +16,8 @@ elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99") elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu99") endif() #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0") #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cperf/piodecomptest.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cperf/piodecomptest.c index 63ea7118b1..5efedb0266 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cperf/piodecomptest.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cperf/piodecomptest.c @@ -7,7 +7,7 @@ const char *argp_program_version = "pioperformance 0.1"; const char *argp_program_bug_address = "<https://github.com/NCAR/ParallelIO>"; -static char doc[] = +static char doc[] = "a test of pio for performance and correctness of a given decomposition"; static struct argp_option options[] = { @@ -17,7 +17,7 @@ static struct argp_option options[] = { { 0 } }; -struct arguments +struct arguments { char *args[2]; char *wdecomp_file; @@ -84,12 +84,11 @@ int test_write_darray(int iosys, const char decomp_file[], int rank, const char int *full_map; int *dimid; int varid; - int globalsize; int ioid; char dimname[PIO_MAX_NAME]; char varname[PIO_MAX_NAME]; - ierr = pioc_read_nc_decomp_int(iosys, decomp_file, &ndims, &global_dimlen, &num_tasks, + ierr = pioc_read_nc_decomp_int(iosys, decomp_file, &ndims, &global_dimlen, &num_tasks, &maplen, &maxmaplen, &full_map, NULL, NULL, NULL, NULL, NULL); if(ierr || debug) printf("%d %d\n",__LINE__,ierr); @@ -103,10 +102,10 @@ int test_write_darray(int iosys, const char decomp_file[], int rank, const char ierr = MPI_Abort(MPI_COMM_WORLD, -1); } } - + ierr = PIOc_createfile(iosys, &ncid, &iotype, "testfile.nc", PIO_CLOBBER); if(ierr || debug) printf("%d %d\n",__LINE__,ierr); - + dimid = calloc(ndims,sizeof(int)); for(int i=0; i<ndims; i++) { @@ -116,7 +115,7 @@ int test_write_darray(int iosys, const char decomp_file[], int rank, const char } /* TODO: support multiple variables and types*/ if(myvarname != NULL) - sprintf(varname,"%s",myvarname); + sprintf(varname,"%s",myvarname); else sprintf(varname,"var%4.4d",0); @@ -149,7 +148,7 @@ int test_write_darray(int iosys, const char decomp_file[], int rank, const char for(int i=0; i < maplen[rank]; i++) dsum += dvarw[i]; if(dsum != rank) - printf("%d: dvarwsum = %d\n",rank, dsum); + printf("%d: dvarwsum = %g\n",rank, dsum); ierr = PIOc_write_darray(ncid, varid, ioid, maplen[rank], dvarw, NULL); free(maplen); @@ -174,13 +173,13 @@ int test_read_darray(int iosys,const char decomp_file[], int rank, const char my int *full_map; int *dimid; int varid; - int globalsize; + //int globalsize; int ioid; int pio_type; - char dimname[PIO_MAX_NAME]; + //char dimname[PIO_MAX_NAME]; char varname[PIO_MAX_NAME]; - ierr = pioc_read_nc_decomp_int(iosys, decomp_file, &ndims, &global_dimlen, &num_tasks, + ierr = pioc_read_nc_decomp_int(iosys, decomp_file, &ndims, &global_dimlen, &num_tasks, &maplen, &maxmaplen, &full_map, NULL, NULL, NULL, NULL, NULL); if(ierr || debug) printf("%d %d\n",__LINE__,ierr); @@ -194,13 +193,13 @@ int test_read_darray(int iosys,const char decomp_file[], int rank, const char my ierr = MPI_Abort(MPI_COMM_WORLD, -1); } } - + ierr = PIOc_openfile(iosys, &ncid, &iotype, "testfile.nc", PIO_NOWRITE); if(ierr || debug) printf("%d %d\n",__LINE__,ierr); - + /* TODO: support multiple variables and types*/ if(myvarname != NULL) - sprintf(varname,"%s",myvarname); + sprintf(varname,"%s",myvarname); else sprintf(varname,"var%4.4d",0); ierr = PIOc_inq_varid(ncid, varname, &varid); @@ -220,7 +219,7 @@ int test_read_darray(int iosys,const char decomp_file[], int rank, const char my { PIO_Offset gdimlen; ierr = PIOc_inq_dimlen(ncid, dimid[i], &gdimlen); - + pioassert(gdimlen == global_dimlen[i], "testfile.nc does not match decomposition file",__FILE__,__LINE__); } free(dimid); @@ -248,7 +247,7 @@ int test_read_darray(int iosys,const char decomp_file[], int rank, const char my for(int i=0; i < maplen[rank]; i++) dsum += dvarr[i]; if(dsum != rank) - printf("%d: dsum = %d\n",rank, dsum); + printf("%d: dsum = %g\n",rank, dsum); break; case PIO_INT: ivarr = malloc(sizeof(int)*maplen[rank]); @@ -327,4 +326,3 @@ int main(int argc, char *argv[]) MPI_Finalize(); } - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/CMakeLists.txt b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/CMakeLists.txt index 951773bdf0..dc9239816e 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/CMakeLists.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/CMakeLists.txt @@ -3,6 +3,7 @@ include (LibMPI) include_directories("${CMAKE_SOURCE_DIR}/tests/cunit") include_directories("${CMAKE_SOURCE_DIR}/src/clib") include_directories("${CMAKE_BINARY_DIR}") +include_directories("${CMAKE_BINARY_DIR}/src/clib") # Compiler-specific compiler options if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") @@ -69,6 +70,8 @@ if (NOT PIO_USE_MPISERIAL) target_link_libraries (test_pioc_fill pioc) add_executable (test_darray EXCLUDE_FROM_ALL test_darray.c test_common.c) target_link_libraries (test_darray pioc) + add_executable (test_darray_lossycompress EXCLUDE_FROM_ALL test_darray_lossycompress.c test_common.c) + target_link_libraries (test_darray_lossycompress pioc) add_executable (test_darray_append EXCLUDE_FROM_ALL test_darray_append.c test_common.c) target_link_libraries (test_darray_append pioc) add_executable (test_darray_frame EXCLUDE_FROM_ALL test_darray_frame.c test_common.c) @@ -127,6 +130,7 @@ add_dependencies (tests test_pioc_unlim) add_dependencies (tests test_pioc_putget) add_dependencies (tests test_pioc_fill) add_dependencies (tests test_darray) +add_dependencies (tests test_darray_lossycompress) add_dependencies (tests test_darray_append) add_dependencies (tests test_darray_frame) add_dependencies (tests test_darray_multi) @@ -235,6 +239,9 @@ else () EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_pioc_putget NUMPROCS ${AT_LEAST_FOUR_TASKS} TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + # timeout in github testing - skip this test + set_tests_properties(test_pioc_putget PROPERTIES LABELS "skipforspack") + add_mpi_test(test_pioc_fill EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_pioc_fill NUMPROCS ${AT_LEAST_FOUR_TASKS} @@ -243,6 +250,10 @@ else () EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray NUMPROCS ${AT_LEAST_FOUR_TASKS} TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + add_mpi_test(test_darray_lossycompress + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_lossycompress + NUMPROCS ${AT_LEAST_FOUR_TASKS} + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) add_mpi_test(test_darray_append EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_append NUMPROCS ${AT_LEAST_FOUR_TASKS} @@ -259,6 +270,9 @@ else () EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multivar NUMPROCS ${AT_LEAST_FOUR_TASKS} TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + # timeout in github testing - skip this test + set_tests_properties(test_darray_multivar PROPERTIES LABELS "skipforspack") + add_mpi_test(test_darray_multivar2 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multivar2 NUMPROCS ${AT_LEAST_FOUR_TASKS} diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/Makefile.am b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/Makefile.am index 7a8edd6194..f7afb15a63 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/Makefile.am +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/Makefile.am @@ -19,7 +19,8 @@ test_decomp_uneven test_decomps test_rearr test_darray_async_simple \ test_darray_async test_darray_async_many test_darray_2sync \ test_async_multicomp test_async_multi2 test_async_manyproc \ test_darray_fill test_decomp_frame test_perf2 test_async_perf \ -test_darray_vard test_async_1d test_darray_append test_simple +test_darray_vard test_async_1d test_darray_append test_simple \ +test_darray_lossycompress if RUN_TESTS # Tests will run from a bash script. @@ -41,6 +42,7 @@ test_pioc_unlim_SOURCES = test_pioc_unlim.c test_common.c test_shared.c pio_test test_pioc_putget_SOURCES = test_pioc_putget.c test_common.c test_shared.c pio_tests.h test_pioc_fill_SOURCES = test_pioc_fill.c test_common.c test_shared.c pio_tests.h test_darray_SOURCES = test_darray.c test_common.c pio_tests.h +test_darray_lossycompress_SOURCES = test_darray_lossycompress.c test_common.c pio_tests.h test_darray_append_SOURCES = test_darray_append.c test_common.c pio_tests.h test_darray_multi_SOURCES = test_darray_multi.c test_common.c pio_tests.h test_darray_multivar_SOURCES = test_darray_multivar.c test_common.c pio_tests.h diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_async_mpi.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_async_mpi.c index 79390326e7..698284b640 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_async_mpi.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_async_mpi.c @@ -63,7 +63,7 @@ int resultlen; int get_test_comm(int my_rank, int ntasks, int min_ntasks, int max_ntasks, MPI_Comm *comm) { int ret; - + /* Check that a valid number of processors was specified. */ if (ntasks < min_ntasks) { @@ -102,7 +102,7 @@ int get_test_comm(int my_rank, int ntasks, int min_ntasks, int max_ntasks, MPI_C * This function is called by the IO task. This function will not * return, unless there is an error. * - * @param verbose non-zero to turn on printf statements. + * @param verbose non-zero to turn on printf statements. * @param my_rank rank of this task. * @param io_rank rank of the IO processor in union_comm. * @param component_count number of computation components @@ -260,22 +260,22 @@ int main(int argc, char **argv) MPI_Group world_group; MPI_Comm io_comm; MPI_Group io_group; - int my_io_proc_list[1] = {0}; /* List of processors in IO component. */ + int my_io_proc_list[1] = {0}; /* List of processors in IO component. */ int num_io_procs = 1; int num_procs_per_comp[COMPONENT_COUNT] = {1, 1}; int in_io = my_rank ? 0 : 1; /* Non-zero if this task is in IO. */ int io_rank = -1; /* Rank of current process in IO comm. */ int comp_rank = -1; - int iomaster; /* MPI_ROOT on master IO task, MPI_PROC_NULL otherwise. */ + int iomain; /* MPI_ROOT on main IO task, MPI_PROC_NULL otherwise. */ MPI_Group group[COMPONENT_COUNT]; /* Group with comp tasks. */ MPI_Group union_group[COMPONENT_COUNT]; /* Group with IO and comp tasks. */ int my_proc_list[COMPONENT_COUNT][1] = {{1}, {2}}; /* Tasks for computation components. */ - MPI_Comm comp_comm[COMPONENT_COUNT]; + MPI_Comm comp_comm[COMPONENT_COUNT]; MPI_Comm union_comm[COMPONENT_COUNT]; MPI_Comm intercomm[COMPONENT_COUNT]; int in_cmp[COMPONENT_COUNT] = {0, 0}; /* Is this process in this computation component? */ int verbose = 0; /* Non-zero to turn on printf statements. */ - + /* Create group for world. */ if ((ret = MPI_Comm_group(test_comm, &world_group))) MPIERR(ret); @@ -296,12 +296,12 @@ int main(int argc, char **argv) { if ((ret = MPI_Comm_rank(io_comm, &io_rank))) MPIERR(ret); - iomaster = !io_rank ? MPI_ROOT : MPI_PROC_NULL; + iomain = !io_rank ? MPI_ROOT : MPI_PROC_NULL; } if (verbose) printf("my_rank %d in_io %d io_rank %d IO %s\n", my_rank, in_io, - io_rank, iomaster == MPI_ROOT ? "MASTER" : "SERVANT"); - + io_rank, iomain == MPI_ROOT ? "main" : "SERVANT"); + /* For each computation component. */ for (int cmp = 0; cmp < COMPONENT_COUNT; cmp++) { @@ -317,7 +317,7 @@ int main(int argc, char **argv) int union_rank = -1; int pidx; MPI_Comm io_comm2; - + /* Create a group for this component. */ if ((ret = MPI_Group_incl(world_group, 1, my_proc_list[cmp], &group[cmp]))) MPIERR(ret); @@ -345,7 +345,7 @@ int main(int argc, char **argv) if ((ret = MPI_Comm_create(test_comm, group[cmp], &comp_comm[cmp]))) MPIERR(ret); MPI_Group_free(&group[cmp]); - + if (in_cmp[cmp]) { /* Get the rank in this comp comm. */ @@ -355,7 +355,7 @@ int main(int argc, char **argv) if (verbose) printf("my_rank %d intracomm created for cmp = %d comp_comm[cmp] = %lld comp_rank = %d\n", my_rank, cmp, (long long int)comp_comm[cmp], comp_rank); - + /* If this is the IO component, make a copy of the IO comm for * each computational component. */ if (in_io) @@ -386,7 +386,7 @@ int main(int argc, char **argv) MPIERR(ret); if (verbose) printf("my_rank %d union_rank %d\n", my_rank, union_rank); - + if (in_io) { /* Create the intercomm from IO to computation component. */ @@ -414,7 +414,7 @@ int main(int argc, char **argv) if (in_io) { int comproot[COMPONENT_COUNT] = {1, 1}; - + if ((ret = msg_handler(verbose, my_rank, 0, COMPONENT_COUNT, union_comm, comp_comm, comproot, io_comm))) ERR(ret); @@ -429,7 +429,7 @@ int main(int argc, char **argv) { int ioroot = 0; int msg = MSG_EXIT; - + /* if (verbose) */ /* printf("my_rank %d sending exit message on union_comm %d\n", my_rank, union_comm[cmp]); */ if ((mpierr = MPI_Send(&msg, 1, MPI_INT, ioroot, 1, union_comm[cmp]))) @@ -443,7 +443,7 @@ int main(int argc, char **argv) printf("my_rank %d freeing resources\n", my_rank); for (int cmp = 0; cmp < COMPONENT_COUNT; cmp++) { - if (comp_comm[cmp] != MPI_COMM_NULL) + if (comp_comm[cmp] != MPI_COMM_NULL) MPI_Comm_free(&comp_comm[cmp]); if (union_comm[cmp] != MPI_COMM_NULL) MPI_Comm_free(&union_comm[cmp]); @@ -451,12 +451,12 @@ int main(int argc, char **argv) MPI_Comm_free(&intercomm[cmp]); } MPI_Group_free(&world_group); - if (io_comm != MPI_COMM_NULL) - MPI_Comm_free(&io_comm); + if (io_comm != MPI_COMM_NULL) + MPI_Comm_free(&io_comm); } /* Free the MPI communicator for this test. */ - MPI_Comm_free(&test_comm); + MPI_Comm_free(&test_comm); /* Finalize MPI. */ MPI_Finalize(); diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_common.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_common.c index e6a2b61582..cc9f15c31b 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_common.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_common.c @@ -80,7 +80,7 @@ int pio_type[NUM_PIO_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, P /* Attribute test data. */ signed char byte_att_data[ATT_LEN] = {NC_MAX_BYTE, NC_MIN_BYTE, NC_MAX_BYTE}; -char char_att_data[ATT_LEN] = {NC_MAX_CHAR, 0, NC_MAX_CHAR}; +char char_att_data[ATT_LEN] = {(char) NC_MAX_CHAR, (char) 0, (char) NC_MAX_CHAR}; short short_att_data[ATT_LEN] = {NC_MAX_SHORT, NC_MIN_SHORT, NC_MAX_SHORT}; int int_att_data[ATT_LEN] = {NC_MAX_INT, NC_MIN_INT, NC_MAX_INT}; float float_att_data[ATT_LEN] = {NC_MAX_FLOAT, NC_MIN_FLOAT, NC_MAX_FLOAT}; @@ -95,7 +95,7 @@ unsigned long long uint64_att_data[ATT_LEN] = {NC_MAX_UINT64, 0, NC_MAX_UINT64}; /* Scalar variable test data. */ signed char byte_scalar_data = NC_MAX_BYTE; -char char_scalar_data = NC_MAX_CHAR; +char char_scalar_data = (char) NC_MAX_CHAR; short short_scalar_data = NC_MAX_SHORT; int int_scalar_data = NC_MAX_INT; float float_scalar_data = NC_MAX_FLOAT; @@ -441,13 +441,10 @@ create_nc_sample(int sample, int iosysid, int format, char *filename, int my_ran { case 0: return create_nc_sample_0(iosysid, format, filename, my_rank, ncid); - break; case 1: return create_nc_sample_1(iosysid, format, filename, my_rank, ncid); - break; case 2: return create_nc_sample_2(iosysid, format, filename, my_rank, ncid); - break; } return PIO_EINVAL; } @@ -460,13 +457,10 @@ check_nc_sample(int sample, int iosysid, int format, char *filename, int my_rank { case 0: return check_nc_sample_0(iosysid, format, filename, my_rank, ncid); - break; case 1: return check_nc_sample_1(iosysid, format, filename, my_rank, ncid); - break; case 2: return check_nc_sample_2(iosysid, format, filename, my_rank, ncid); - break; } return PIO_EINVAL; } @@ -602,8 +596,8 @@ create_nc_sample_1(int iosysid, int format, char *filename, int my_rank, int *nc return ret; /* Write some data. For the PIOc_put/get functions, all data must - * be on compmaster before the function is called. Only - * compmaster's arguments are passed to the async msg handler. All + * be on compmain before the function is called. Only + * compmain's arguments are passed to the async msg handler. All * other computation tasks are ignored. */ for (int i = 0; i < DIM_LEN_S1; i++) data[i] = i; @@ -797,8 +791,8 @@ create_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *nc return ret; /* Write some data. For the PIOc_put/get functions, all data must - * be on compmaster before the function is called. Only - * compmaster's arguments are passed to the async msg handler. All + * be on compmain before the function is called. Only + * compmain's arguments are passed to the async msg handler. All * other computation tasks are ignored. */ for (int i = 0; i < DIM_LEN_S2; i++) data[i] = i; @@ -828,7 +822,7 @@ int check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *ncidp) { int ncid; - int ret; + int ret=PIO_NOERR; int ndims, nvars, ngatts, unlimdimid; int ndims2, nvars2, ngatts2, unlimdimid2; int dimid2; @@ -998,7 +992,7 @@ check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *nci return ERR_CHECK; } - return 0; + return ret; } /* Create the decomposition to divide the 3-dimensional sample data @@ -1122,7 +1116,7 @@ int create_nc_sample_3(int iosysid, int iotype, int my_rank, int my_comp_idx, sprintf(var_name, "%s_%d", THREED_VAR_NAME, my_comp_idx); if ((ret = PIOc_def_var(ncid, var_name, PIO_SHORT, NDIM3, dimid, &varid[2]))) ERR(ret); - + /* End define mode. */ if ((ret = PIOc_enddef(ncid))) ERR(ret); @@ -1138,7 +1132,7 @@ int create_nc_sample_3(int iosysid, int iotype, int my_rank, int my_comp_idx, /* Write the 2-D variable with put_var(). */ if ((ret = PIOc_put_var_short(ncid, 1, data_2d))) ERR(ret); - + /* Write the 3D data. */ if (use_darray) { @@ -1156,11 +1150,11 @@ int create_nc_sample_3(int iosysid, int iotype, int my_rank, int my_comp_idx, { PIO_Offset start[NDIM3] = {0, 0, 0}; PIO_Offset count[NDIM3] = {1, DIM_X_LEN, DIM_Y_LEN}; - + /* Write a record of the 3-D variable with put_vara(). */ if ((ret = PIOc_put_vara_short(ncid, varid[2], start, count, data_2d))) ERR(ret); - + /* Write another record of the 3-D variable with put_vara(). */ start[0] = 1; if ((ret = PIOc_put_vara_short(ncid, varid[2], start, count, data_2d))) @@ -1281,14 +1275,14 @@ int check_nc_sample_3(int iosysid, int iotype, int my_rank, int my_comp_idx, { PIO_Offset start[NDIM3] = {0, 0, 0}; PIO_Offset count[NDIM3] = {1, DIM_X_LEN, DIM_Y_LEN}; - + /* Read a record of the 3-D variable with get_vara(). */ if ((ret = PIOc_get_vara_short(ncid, 2, start, count, data_2d))) ERR(ret); for (int i = 0; i < DIM_X_LEN * DIM_Y_LEN; i++) if (data_2d[i] != my_comp_idx + i) ERR(ERR_WRONG); - + /* Read another record of the 3-D variable with get_vara(). */ start[0] = 1; if ((ret = PIOc_get_vara_short(ncid, 2, start, count, data_2d))) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_2sync.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_2sync.c index 657bbdd703..c093165100 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_2sync.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_2sync.c @@ -312,8 +312,8 @@ int darray_fill_test(int iosysid, int my_rank, int num_iotypes, int *iotype, unsigned char data_in[elements_per_pe * NUM_COMPUTATION_PROCS]; if ((ret = PIOc_get_var_uchar(ncid2, 0, data_in))) AERR(ret); - if (my_rank && data_in[0] != 1 && data_in[1] != -1 && data_in[2] != 2 && - data_in[3] != -2 && data_in[4] != 3 && data_in[5] != -3) + if (my_rank && data_in[0] != 1 && data_in[1] != 1 && data_in[2] != 2 && + data_in[3] != 2 && data_in[4] != 3 && data_in[5] != 3) AERR(ERR_WRONG); } break; @@ -322,8 +322,8 @@ int darray_fill_test(int iosysid, int my_rank, int num_iotypes, int *iotype, unsigned short data_in[elements_per_pe * NUM_COMPUTATION_PROCS]; if ((ret = PIOc_get_var_ushort(ncid2, 0, data_in))) AERR(ret); - if (my_rank && data_in[0] != 1 && data_in[1] != -1 && data_in[2] != 2 && - data_in[3] != -2 && data_in[4] != 3 && data_in[5] != -3) + if (my_rank && data_in[0] != 1 && data_in[1] != 1 && data_in[2] != 2 && + data_in[3] != 2 && data_in[4] != 3 && data_in[5] != 3) AERR(ERR_WRONG); } break; @@ -332,8 +332,8 @@ int darray_fill_test(int iosysid, int my_rank, int num_iotypes, int *iotype, unsigned int data_in[elements_per_pe * NUM_COMPUTATION_PROCS]; if ((ret = PIOc_get_var_uint(ncid2, 0, data_in))) AERR(ret); - if (my_rank && data_in[0] != 1 && data_in[1] != -1 && data_in[2] != 2 && - data_in[3] != -2 && data_in[4] != 3 && data_in[5] != -3) + if (my_rank && data_in[0] != 1 && data_in[1] != 1 && data_in[2] != 2 && + data_in[3] != 2 && data_in[4] != 3 && data_in[5] != 3) AERR(ERR_WRONG); } break; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_3d.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_3d.c index dc77f40ee3..476dffbb48 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_3d.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_3d.c @@ -254,7 +254,7 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, for (int fmt = 0; fmt < num_flavors; fmt++) { int ret; /* Return code. */ - + /* Create the filename. */ sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]); diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_fill.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_fill.c index 0dafa22899..f4d6e64a5b 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_fill.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_fill.c @@ -164,7 +164,7 @@ int main(int argc, char **argv) #define NUM_TYPES 11 int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64}; - + #endif /* _NETCDF4 */ /* Determine what data to write. Put value of 42 into diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_lossycompress.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_lossycompress.c new file mode 100644 index 0000000000..041aea633f --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_lossycompress.c @@ -0,0 +1,499 @@ +/* + * Tests for PIO distributed arrays. + * + * @author Ed Hartnett + * @date 2/16/17 + */ +#include <config.h> +#include <pio.h> +#include <pio_internal.h> +#include <pio_tests.h> + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 4 + +/* The minimum number of tasks this test should run on. */ +#define MIN_NTASKS 4 + +/* The name of this test. */ +#define TEST_NAME "test_darray_lossycompress" + +/* Number of processors that will do IO. */ +#define NUM_IO_PROCS 1 + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +/* The number of dimensions in the example data. In this test, we + * are using three-dimensional data. */ +#define NDIM 3 + +/* But sometimes we need arrays of the non-record dimensions. */ +#define NDIM2 2 + +/* The length of our sample data along each dimension. */ +#define X_DIM_LEN 4 +#define Y_DIM_LEN 4 + +/* The number of timesteps of data to write. */ +#define NUM_TIMESTEPS 2 + +/* The names of variables in the netCDF output files. */ +#define VAR_NAME "Billy-Bob" +#define VAR_NAME2 "Sally-Sue" + +/* Test cases relating to PIOc_write_darray_multi(). */ +#define NUM_TEST_CASES_WRT_MULTI 3 + +/* Test with and without specifying a fill value to + * PIOc_write_darray(). */ +#define NUM_TEST_CASES_FILLVALUE 1 + +/* This struct allows us to treat float as uint32_t + * types. */ +union FU { + float f; + uint32_t u; +}; + +/* This struct allows us to treat double points as uint64_t + * types. */ +union DU { + double d; + uint64_t u; +}; + +/* The dimension names. */ +char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"}; + +/* Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; + +/** + * Test the darray functionality. Create a netCDF file with 3 + * dimensions and 1 PIO_INT variable, and use darray to write some + * data. + * + * @param iosysid the IO system ID. + * @param ioid the ID of the decomposition. + * @param num_flavors the number of IOTYPES available in this build. + * @param flavor array of available iotypes. + * @param my_rank rank of this task. + * @param pio_type the type of the data. + * @returns 0 for success, error code otherwise. + */ +int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, + int pio_type) +{ + char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ + int dimids[NDIM]; /* The dimension IDs. */ + int ncid; /* The ncid of the netCDF file. */ + int ncid2; /* The ncid of the re-opened netCDF file. */ + int varid; /* The ID of the netCDF varable. */ + int varid2; /* The ID of a netCDF varable of different type. */ + int wrong_varid = TEST_VAL_42; /* A wrong ID. */ + int ret; /* Return code. */ + MPI_Datatype mpi_type; + int type_size; /* size of a variable of type pio_type */ + int other_type; /* another variable of the same size but different type */ + PIO_Offset arraylen = 4; + void *fillvalue, *ofillvalue; + void *test_data; + void *test_data_in; + int fillvalue_int = NC_FILL_INT; + int test_data_int[arraylen]; + int test_data_int_in[arraylen]; + float fillvalue_float = NC_FILL_FLOAT; + float test_data_float[arraylen]; + float test_data_float_in[arraylen] ; + double fillvalue_double = NC_FILL_DOUBLE; + double test_data_double[arraylen]; + double test_data_double_in[arraylen]; +#ifdef NC_NOQUANTIZE + // union FU fin; + + //xpect[0].u = 0x3f8e3000; + //xpect[1].u = 0x3f800fff; + //xpect[2].u = 0x41200000; + //xpect[3].u = 0x4640efff; +// xpect[4].u = 0x3dfcd000; + //double_xpect[0].u = 0x3ff1c60000000000; + //double_xpect[1].u = 0x3ff001ffffffffff; + //double_xpect[2].u = 0x4023fe0000000000; + //double_xpect[3].u = 0x41d265ffffffffff; +// double_xpect[4].u = 0x42dc120000000000; +#endif + /* Initialize some data. */ + for (int f = 0; f < arraylen; f++) + { + test_data_int[f] = my_rank * 10 + f; + } + test_data_float[0] = 1.11111111; + test_data_float[1] = 1.0; + test_data_float[2] = 9.99999999; + test_data_float[3] = 12345.67; +// test_data_float[4] = .1234567; + + test_data_double[0] = 1.1111111; + test_data_double[1] = 1.0; + test_data_double[2] = 9.999999999; + test_data_double[3] = 1234567890.12345; +// test_data_double[4] = 123456789012345.0; + + + + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + if(flavor[fmt] == PIO_IOTYPE_NETCDF4P || flavor[fmt] == PIO_IOTYPE_NETCDF4C) + { + /* Add a couple of extra tests for the + * PIOc_write_darray_multi() function. */ + for (int test_multi = 0; test_multi < NUM_TEST_CASES_WRT_MULTI; test_multi++) + { + /* Test with/without providing a fill value to PIOc_write_darray(). */ + for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) + { + /* Create the filename. */ + sprintf(filename, "data_%s_iotype_%d_pio_type_%d_test_multi_%d_provide_fill_%d.nc", TEST_NAME, + flavor[fmt], pio_type, test_multi, provide_fill); + /* Select the fill value and data. */ + switch (pio_type) + { + case PIO_INT: + fillvalue = provide_fill ? &fillvalue_int : NULL; + test_data = test_data_int; + test_data_in = test_data_int_in; + break; + case PIO_FLOAT: + fillvalue = provide_fill ? &fillvalue_float : NULL; + test_data = test_data_float; + test_data_in = test_data_float_in; + break; + case PIO_DOUBLE: + fillvalue = provide_fill ? &fillvalue_double : NULL; + test_data = test_data_double; + test_data_in = test_data_double_in; + break; + default: + ERR(ERR_WRONG); + } + + /* Create the netCDF output file. */ + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + for (int d = 0; d < NDIM; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + + /* Define a variable. */ + if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, &varid))) + ERR(ret); +#ifdef NC_NOQUANTIZE + if(pio_type == PIO_REAL || pio_type == PIO_DOUBLE) + { + if ((ret = PIOc_def_var_quantize(ncid, varid, PIO_QUANTIZE_BITGROOM, 3))) + ERR(ret); + } + else + { + /* this should fail */ + if ((ret = PIOc_def_var_quantize(ncid, varid, PIO_QUANTIZE_BITROUND, 5) != PIO_EINVAL )) + ERR(ret); + + } +#endif + if ((ret = PIOc_def_var_deflate(ncid, varid, 0, 1, 1))) + ERR(ret); + + + /* Define a variable with a different type but same size. */ + if ((ret = find_mpi_type(pio_type, &mpi_type, &type_size))) + ERR(ret); + if (type_size == NETCDF_INT_FLOAT_SIZE) + other_type = pio_type == PIO_INT ? PIO_FLOAT : PIO_INT; +// else if(type_size == NETCDF_DOUBLE_INT64_SIZE) +// other_type = pio_type == PIO_INT64 ? PIO_DOUBLE : PIO_INT64; + else + other_type = 0; /* skip the test */ + switch (other_type) + { + case PIO_INT: + ofillvalue = provide_fill ? &fillvalue_int : NULL; + break; + case PIO_FLOAT: + ofillvalue = provide_fill ? &fillvalue_float : NULL; + break; + default: + break; + } + if (other_type && (ret = PIOc_def_var(ncid, VAR_NAME2, other_type, NDIM, dimids, &varid2))) + ERR(ret); + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Set the value of the record dimension. */ + if ((ret = PIOc_setframe(ncid, varid, 0))) + ERR(ret); + if (other_type && (ret = PIOc_setframe(ncid, varid2, 0))) + ERR(ret); + + int frame = 0; + int flushtodisk = test_multi - 1; + if (!test_multi) + { + /* These should not work. */ + if (PIOc_write_darray(ncid + TEST_VAL_42, varid, ioid, arraylen, test_data, fillvalue) != PIO_EBADID) + ERR(ERR_WRONG); + if (PIOc_write_darray(ncid, varid, ioid + TEST_VAL_42, arraylen, test_data, fillvalue) != PIO_EBADID) + ERR(ERR_WRONG); + if (PIOc_write_darray(ncid, varid, ioid, arraylen - 1, test_data, fillvalue) != PIO_EINVAL) + ERR(ERR_WRONG); + if (PIOc_write_darray(ncid, TEST_VAL_42, ioid, arraylen, test_data, fillvalue) != PIO_ENOTVAR) + ERR(ERR_WRONG); + + /* This should work - library type conversion */ + if (other_type && (ret = PIOc_write_darray(ncid, varid2, ioid, arraylen, test_data, ofillvalue))) + ERR(ret); + + /* Write the data. */ +// printf("test_data[0] = %f\n",test_data_float[0]); + if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue))) + ERR(ret); + + } + else + { + int varid_big = PIO_MAX_VARS + TEST_VAL_42; + + /* These will not work. */ + if (PIOc_write_darray_multi(ncid + TEST_VAL_42, &varid, ioid, 1, arraylen, test_data, &frame, + fillvalue, flushtodisk) != PIO_EBADID) + ERR(ERR_WRONG); + if (PIOc_write_darray_multi(ncid, NULL, ioid, 1, arraylen, test_data, &frame, + fillvalue, flushtodisk) != PIO_EINVAL) + ERR(ERR_WRONG); + if (PIOc_write_darray_multi(ncid, &varid, ioid + TEST_VAL_42, 1, arraylen, test_data, &frame, + fillvalue, flushtodisk) != PIO_EBADID) + ERR(ERR_WRONG); + if (PIOc_write_darray_multi(ncid, &varid, ioid, -1, arraylen, test_data, &frame, + fillvalue, flushtodisk) != PIO_EINVAL) + ERR(ERR_WRONG); + if (PIOc_write_darray_multi(ncid, &varid_big, ioid, 1, arraylen, test_data, &frame, + fillvalue, flushtodisk) != PIO_ENOTVAR) + ERR(ERR_WRONG); + if (PIOc_write_darray_multi(ncid, &wrong_varid, ioid, 1, arraylen, test_data, &frame, + fillvalue, flushtodisk) != PIO_ENOTVAR) + ERR(ERR_WRONG); +// pio_setloglevel(0); + + /* This should work - library type conversion */ + if (other_type && (ret = PIOc_write_darray_multi(ncid, &varid2, ioid, 1, arraylen, test_data, &frame, + fillvalue, flushtodisk))) + ERR(ret); + + /* Write the data with the _multi function. */ + if ((ret = PIOc_write_darray_multi(ncid, &varid, ioid, 1, arraylen, test_data, &frame, + fillvalue, flushtodisk))) + ERR(ret); + } + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) + ERR(ret); + + PIO_Offset dimlen; + /* check the unlimited dim size - it should be 1 */ + if ((ret = PIOc_inq_dimlen(ncid2, dimids[0], &dimlen))) + ERR(ret); + if (dimlen != 1) + ERR(ERR_WRONG); + + /* These should not work. */ + if (PIOc_read_darray(ncid2 + TEST_VAL_42, varid, ioid, arraylen, + test_data_in) != PIO_EBADID) + ERR(ERR_WRONG); + if (PIOc_read_darray(ncid2, varid, ioid + TEST_VAL_42, arraylen, + test_data_in) != PIO_EBADID) + ERR(ERR_WRONG); + + /* Set the record number. */ + if ((ret = PIOc_setframe(ncid2, varid, 0))) + ERR(ret); + + /* Read the data. */ + if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in))) + ERR(ret); + + /* /\* Read the data. *\/ */ + /* if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, (void *)test_data_in))) */ + /* ERR(ret); */ + + /* Check the results. */ + /* HOW does one test a lossy compression algorythm? */ +#ifdef NC_NOQUANTIZE + for (int f = 0; f < arraylen; f++) + { + switch (pio_type) + { + case PIO_INT: + if (test_data_int_in[f] != test_data_int[f]) + return ERR_WRONG; + break; + case PIO_FLOAT: + /* + We do not expect an exact match for lossy data - so how do we test? + if (test_data_float_in[f] != test_data_float[f]) + return ERR_WRONG; + */ + break; + case PIO_DOUBLE: + /* + if (test_data_double_in[f] != test_data_double[f]) + return ERR_WRONG; + */ + break; + default: + ERR(ERR_WRONG); + } + } +#else + printf("Quantize support was not available in the netcdf build and thus is not tested\n"); +#endif + + /* Try to write, but it won't work, because we opened file read-only. */ + if (!test_multi) + { + if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data, fillvalue) != PIO_EPERM) + ERR(ERR_WRONG); + } + else + { + if (PIOc_write_darray_multi(ncid2, &varid, ioid, 1, arraylen, test_data, &frame, + fillvalue, flushtodisk) != PIO_EPERM) + ERR(ERR_WRONG); + } + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid2))) + ERR(ret); + } /* next fillvalue test case */ + } /* next test multi */ + }/* only netcdf4 iotypes allowed */ + } /* next iotype */ + + return PIO_NOERR; +} + +/** + * Run all the tests. + * + * @param iosysid the IO system ID. + * @param num_flavors number of available iotypes in the build. + * @param flavor pointer to array of the available iotypes. + * @param my_rank rank of this task. + * @param test_comm the communicator the test is running on. + * @returns 0 for success, error code otherwise. + */ +int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, + MPI_Comm test_comm) +{ +#define NUM_TYPES_TO_TEST 2 + int ioid; + char filename[PIO_MAX_NAME + 1]; + int pio_type[NUM_TYPES_TO_TEST] = {PIO_FLOAT, PIO_DOUBLE}; + int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; + int ret; /* Return code. */ + + for (int t = 0; t < NUM_TYPES_TO_TEST; t++) + { + /* This will be our file name for writing out decompositions. */ + sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank, + *flavor, pio_type[t]); + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, + &ioid, pio_type[t]))) + return ret; + + /* Run a simple darray test. */ + if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t]))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + } + + return PIO_NOERR; +} + +/* Run tests for darray functions. */ +int main(int argc, char **argv) +{ +#define NUM_REARRANGERS_TO_TEST 2 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; + int my_rank; + int ntasks; + int num_flavors; /* Number of PIO netCDF flavors in this build. */ + int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ + MPI_Comm test_comm; /* A communicator for this test. */ + int ret; /* Return code. */ + + /* Initialize test. */ + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, + MIN_NTASKS, -1, &test_comm))) + ERR(ERR_INIT); + + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Only do something on max_ntasks tasks. */ + if (my_rank < TARGET_NTASKS) + { + int iosysid; /* The ID for the parallel I/O system. */ + int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ + int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int ret; /* Return code. */ + + /* Figure out iotypes. */ + if ((ret = get_iotypes(&num_flavors, flavor))) + ERR(ret); + + for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++) + { + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, + ioproc_start, rearranger[r], &iosysid))) + return ret; + + /* Run tests. */ + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_free_iosystem(iosysid))) + return ret; + } /* next rearranger */ + } /* endif my_rank < TARGET_NTASKS */ + + /* Finalize the MPI library. */ + if ((ret = pio_test_finalize(&test_comm))) + return ret; + /* if ((ret = pio_test_finalize2(&test_comm, TEST_NAME))) */ + /* return ret; */ + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + return 0; +} diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_multivar3.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_multivar3.c index 21b2d88724..f44e389765 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_multivar3.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_darray_multivar3.c @@ -214,7 +214,7 @@ int test_multivar_darray(int iosysid, int ioid, int ioid_float, int num_flavors, if (file_fv_int != custom_fillvalue_int) return ERR_WRONG; } - + /* Check the float fill value. */ if ((ret = PIOc_get_var1_float(ncid2, varid[2], idx, &file_fv_float))) return ret; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_decomp_frame.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_decomp_frame.c index 54969a8b4c..3aa893d6e7 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_decomp_frame.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_decomp_frame.c @@ -165,7 +165,7 @@ int main(int argc, char **argv) #define NUM_TYPES 11 int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64}; - + #endif /* _NETCDF4 */ /* Determine what data to write. Put value of 42 into diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_decomp_uneven.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_decomp_uneven.c index 8008c06d2d..c916d3abdf 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_decomp_uneven.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_decomp_uneven.c @@ -128,7 +128,7 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, for (int fmt = 0; fmt < 1; fmt++) { int ret; /* Return code. */ - + /* Create the filename. */ snprintf(filename, PIO_MAX_NAME, "decomp_%s_pio_type_%d_dims_%d_x_%d_x_%d.nc", TEST_NAME, pio_type, dim_len[0], dim_len[1], dim_len[2]); @@ -329,7 +329,7 @@ int main(int argc, char **argv) for (int dc = 0; dc < NUM_DIM_COMBOS_TO_TEST; dc++) { int test_type[NUM_TYPES_TO_TEST] = {PIO_INT}; - + /* What is length of map for this combo? */ int full_maplen = 1; for (int d = 0; d < NDIM3; d++) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_intercomm2.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_intercomm2.c index eb99e0ff86..deb4657afb 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_intercomm2.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_intercomm2.c @@ -338,6 +338,7 @@ int main(int argc, char **argv) sprintf(filename[fmt], "test_intercomm2_%d.nc", flavor[fmt]); /* Create a netCDF file with one dimension and one variable. */ + if ((ret = PIOc_createfile(iosysid[my_comp_idx], &ncid, &flavor[fmt], filename[fmt], NC_CLOBBER))) ERR(ret); @@ -492,8 +493,8 @@ int main(int argc, char **argv) ERR(ret); /* Write some data. For the PIOc_put/get functions, all - * data must be on compmaster before the function is - * called. Only compmaster's arguments are passed to the + * data must be on compmain before the function is + * called. Only compmain's arguments are passed to the * async msg handler. All other computation tasks are * ignored. */ for (int i = 0; i < DIM_LEN; i++) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_iosystem2_simple2.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_iosystem2_simple2.c index d4db1540f7..26f487682f 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_iosystem2_simple2.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_iosystem2_simple2.c @@ -49,7 +49,7 @@ int main(int argc, char **argv) { int num_flavors; /* Number of PIO netCDF flavors in this build. */ int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ - + /* Figure out iotypes. */ if ((ret = get_iotypes(&num_flavors, flavor))) ERR(ret); diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_perf2.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_perf2.c index 523b00227c..ea77cf41d7 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_perf2.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_perf2.c @@ -160,7 +160,7 @@ test_darray(int iosysid, int ioid, int num_flavors, int *flavor, /* How many megabytes will we write? */ num_megabytes = (NUM_TIMESTEPS * X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN * sizeof(int))/(MILLION); - + sprintf(filename, "data_%s_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[fmt], rearranger); /* Create the filename. Use the same filename for all, so we @@ -268,7 +268,7 @@ test_darray(int iosysid, int ioid, int num_flavors, int *flavor, /* Now reopen the file and re-read the data. */ { int *test_data_in; - + if (!(test_data_in = malloc(sizeof(int) * arraylen))) ERR(PIO_ENOMEM); @@ -319,9 +319,9 @@ test_darray(int iosysid, int ioid, int num_flavors, int *flavor, /* Free resources. */ free(test_data_in); - + } /* re-reading file */ - + if (!my_rank) printf("%d,\t%d,\t%s,\t%s,\t%s,\t%8.3f,\t%8.3f,\t%8.1f,\t%8.3f,\t%8.3f\n", ntasks, num_io_procs, (rearranger == 1 ? "box" : "subset"), (provide_fill ? "fill" : "nofill"), diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc.c index a9289c0198..f50f6fbdd2 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc.c @@ -7,6 +7,7 @@ #include <pio.h> #include <pio_internal.h> #include <pio_tests.h> +#include <pio_meta.h> /* The number of tasks this test should run on. */ #define TARGET_NTASKS 4 @@ -1048,7 +1049,7 @@ int test_empty_files(int iosysid, int num_flavors, int *flavor, int my_rank) if ((ret = get_iotype_name(flavor[fmt], iotype_name))) ERR(ret); sprintf(filename, "%s_empty_%s.nc", TEST_NAME, iotype_name); - + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) ERR(ret); @@ -1076,10 +1077,10 @@ int test_empty_files(int iosysid, int num_flavors, int *flavor, int my_rank) /* Check that the fill values are correctly reported by find_var_fill(). * * @param ncid the ID of the open test file. - * @param ntypes the number ot types we are testing. + * @param ntypes the number ot types we are testing. * @param use_custom_fill true if custom fill values were used. * @param my_rank rank of this task. - * @return 0 on success. + * @return 0 on success. */ int check_fillvalues(int ncid, int num_types, int use_custom_fill, int my_rank) { @@ -1128,7 +1129,7 @@ int check_fillvalues(int ncid, int num_types, int use_custom_fill, int my_rank) if ((ret = pio_get_file(ncid, &file))) ERR(ret); - + for (int v = 0; v < num_types; v++) { var_desc_t *vdesc; @@ -1136,7 +1137,7 @@ int check_fillvalues(int ncid, int num_types, int use_custom_fill, int my_rank) /* Get the var info. */ if ((ret = get_var_desc(v, &file->varlist, &vdesc))) ERR(ret); - + /* Check the fill value with this internal function. */ if ((ret = find_var_fillvalue(file, v, vdesc))) ERR(ret); @@ -1194,7 +1195,7 @@ int check_fillvalues(int ncid, int num_types, int use_custom_fill, int my_rank) return PIO_NOERR; } - + /* Test the internal function that determins a var's fillvalue. * * @param iosysid the iosystem ID that will be used for the test. @@ -1571,10 +1572,10 @@ int test_nc4(int iosysid, int num_flavors, int *flavor, int my_rank) ERR(ret); /* Setting deflate works with parallel iotype starting - * with netcdf-c-4.7.4. If present, HAVE_PAR_FILTERS will + * with netcdf-c-4.7.4. If present, PIO_HAS_PAR_FILTERS will * be defined. */ ret = PIOc_def_var_deflate(ncid, 0, 0, 1, 1); -#ifdef HAVE_PAR_FILTERS +#ifdef PIO_HAS_PAR_FILTERS if (ret) ERR(ret); #else @@ -1613,16 +1614,16 @@ int test_nc4(int iosysid, int num_flavors, int *flavor, int my_rank) ERR(ERR_AWFUL); /* For parallel netCDF-4, we turned on deflate above, if - * HAVE_PAR_FILTERS is defined. */ + * PIO_HAS_PAR_FILTERS is defined. */ if (flavor[fmt] == PIO_IOTYPE_NETCDF4P) { -#ifdef HAVE_PAR_FILTERS +#ifdef PIO_HAS_PAR_FILTERS if (shuffle || !deflate || deflate_level != 1) ERR(ERR_AWFUL); #else if (shuffle || deflate) ERR(ERR_AWFUL); -#endif /* HAVE_PAR_FILTERS */ +#endif /* PIO_HAS_PAR_FILTERS */ } /* Check setting the chunk cache for the variable. */ @@ -1826,6 +1827,9 @@ int test_scalar(int iosysid, int num_flavors, int *flavor, int my_rank, int asyn int test_val = TEST_VAL_42; if ((ret = PIOc_put_var_int(ncid, varid, &test_val))) ERR(ret); + /* flush the write buffer */ + if ((ret = PIOc_sync(ncid))) + ERR(ret); /* Check the scalar var. */ if ((ret = check_scalar_var(ncid, varid, flavor[fmt], my_rank))) @@ -2325,7 +2329,6 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te /* This will be our file name for writing out decompositions. */ sprintf(filename, "decomp_%d.txt", my_rank); sprintf(nc_filename, "decomp_%d.nc", my_rank); - /* This is a simple test that just creates the decomp with * async. */ if (async) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc_putget.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc_putget.c index ce2732718f..748ed85c4f 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc_putget.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc_putget.c @@ -823,7 +823,7 @@ int test_read_att(int ncid, int *varid, int flavor) unsigned int uint_array_in[ATT_LEN]; long long int64_array_in[ATT_LEN]; unsigned long long uint64_array_in[ATT_LEN]; - + if ((ret = PIOc_get_att_uchar(ncid, varid[7], UCHAR_ATT_NAME, ubyte_array_in))) return ret; if ((ret = PIOc_get_att_ushort(ncid, varid[8], USHORT_ATT_NAME, ushort_array_in))) @@ -2059,5 +2059,4 @@ int main(int argc, char **argv) return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, -1, TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS); - return 0; } diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc_unlim.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc_unlim.c index 88621523e3..be56ee1d84 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc_unlim.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/cunit/test_pioc_unlim.c @@ -102,24 +102,24 @@ int create_test_file(int iosysid, int ioid, int iotype, int my_rank, int *ncid, /* Create the filename. */ sprintf(filename, "%s_iotype_%d.nc", TEST_NAME, iotype); - + /* Create the netCDF output file. */ if ((ret = PIOc_createfile(iosysid, ncid, &iotype, filename, PIO_CLOBBER))) ERR(ret); - + /* Define netCDF dimensions and variable. */ for (int d = 0; d < NDIM; d++) if ((ret = PIOc_def_dim(*ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) ERR(ret); - + /* Define a variable. */ if ((ret = PIOc_def_var(*ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, varid))) ERR(ret); - + /* End define mode. */ if ((ret = PIOc_enddef(*ncid))) ERR(ret); - + return PIO_NOERR; } @@ -147,11 +147,11 @@ int run_multiple_unlim_test(int iosysid, int ioid, int iotype, int my_rank, /* Add unlimited dimension. */ if ((ret = PIOc_def_dim(ncid, UDIM1_NAME, NC_UNLIMITED, &dimid[0]))) - ERR(ret); + ERR(ret); /* Add another unlimited dimension. */ if ((ret = PIOc_def_dim(ncid, UDIM2_NAME, NC_UNLIMITED, &dimid[1]))) - ERR(ret); + ERR(ret); /* Check for correctness. */ if ((ret = PIOc_inq_unlimdims(ncid, &nunlimdims, unlimdimids))) @@ -168,7 +168,7 @@ int run_multiple_unlim_test(int iosysid, int ioid, int iotype, int my_rank, { int nunlimdims; int unlimdimids[NUM_UNLIM_DIMS]; - + /* These should also work. */ if ((ret = PIOc_inq_unlimdims(ncid, NULL, NULL))) ERR(ret); @@ -191,7 +191,7 @@ int run_multiple_unlim_test(int iosysid, int ioid, int iotype, int my_rank, if (PIOc_def_var(ncid, VAR_NAME2, PIO_INT, NUM_UNLIM_DIMS, unlimdimids, &varid) != PIO_EINVAL) ERR(ERR_WRONG); - + /* Close the file. */ if ((PIOc_closefile(ncid))) return ret; @@ -203,10 +203,10 @@ int run_multiple_unlim_test(int iosysid, int ioid, int iotype, int my_rank, int ncid; int dimids[NUM_UNLIM_DIMS]; int varid; - + if ((ret = nc_create(NETCDF4_UNLIM_FILE_NAME, NC_CLOBBER|NC_NETCDF4, &ncid))) ERR(ret); - + if ((ret = nc_def_dim(ncid, DIM_NAME1, NC_UNLIMITED, &dimids[0]))) ERR(ret); if ((ret = nc_def_dim(ncid, DIM_NAME2, NC_UNLIMITED, &dimids[1]))) @@ -241,7 +241,7 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te if ((ret = MPI_Comm_size(test_comm, &my_test_size))) MPIERR(ret); - + /* This will be our file name for writing out decompositions. */ sprintf(filename, "decomp_%d.txt", my_rank); @@ -273,7 +273,7 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te return ret; if (vdesc->record != 1) return ERR_WRONG; - + if ((PIOc_closefile(ncid))) return ret; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/derecho_testsuite.py b/src/Infrastructure/IO/PIO/ParallelIO/tests/derecho_testsuite.py new file mode 100644 index 0000000000..4f31f59715 --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/derecho_testsuite.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +#PBS -r n +#PBS -j oe +#PBS -S /bin/bash +#PBS -l select=1:ncpus=128:mpiprocs=128:ompthreads=1:mem=230GB +#PBS -N parallelioTest +#PBS -A P93300606 +#PBS -q main +#PBS -l walltime=08:00:00 + +import os, sys, shutil +import subprocess +lmod_root = os.environ["LMOD_ROOT"] +sys.path.append(lmod_root+"/lmod/init/") +from env_modules_python import module + + +compilers = ["cce/15.0.1", "intel/2023.0.0", "intel-oneapi/2023.0.0", "nvhpc/23.5", "gcc/12.2.0"] +#compilers = ["intel/2023.0.0"] +#mpilibs = ["cray-mpich/8.1.25", "intel-mpi/2021.8.0", "mpi-serial/2.3.0"] +mpilibs = ["mpi-serial/2.3.0","cray-mpich/8.1.25"] +netcdf = ["netcdf/4.9.2","netcdf-mpi/4.9.2"] +pnetcdf = ["parallel-netcdf/1.12.3"] + +cmakeopts = [None,["-DPIO_ENABLE_FORTRAN=OFF"],["-DPIO_ENABLE_TIMING=OFF","-DPIO_ENABLE_NETCDF_INTEGRATION=ON"],["-DWITH_PNETCDF=OFF"]] + +piodir = "/glade/work/jedwards/sandboxes/ParallelIO" +bldcnt=0 +module("purge") +module("load", "ncarenv/23.06") +module("load", "cesmdev/1.0") +module("load", "cmake/3.26.3") + +for compiler in compilers: + cmd = " load "+compiler + module("load", compiler) + for mpilib in mpilibs: + module("load", mpilib) + cmakeflags = ["-DPIO_ENABLE_EXAMPLES=OFF"] + for netlib in netcdf: + module("unload", "netcdf") + module("unload", "hdf5") + module("unload", "netcdf-mpi") + module("unload", "hdf5-mpi") + if "mpi-serial" in mpilib: + cc = os.getenv("CC") + ftn = os.getenv("FC") + if not cc: + cc = "cc" + if not ftn: + ftn = "ftn" + mpi_serial = os.environ["NCAR_ROOT_MPI_SERIAL"] + cmakeflags.extend(["-DPIO_USE_MPISERIAL=ON","-DMPISERIAL_PATH="+mpi_serial,"-DPIO_ENABLE_TESTS=OFF","-DPIO_ENABLE_TIMING=OFF"]) + if "mpi" in netlib: + continue + + module("load", netlib) + if not "mpi-serial" in mpilib: + cc = "cc" + ftn = "ftn" + for plib in pnetcdf: + module("load", plib) + cflags = " -g " + fflags = " -g " + if "gcc" in compiler: + fflags += " -ffree-line-length-none -ffixed-line-length-none -fallow-argument-mismatch " + elif "intel" in compiler: + cflags += " -std=gnu99 " + if "mpi-serial" in mpilib: + fflags += " -I$NCAR_INC_MPI_SERIAL " + module("list") + for cmakeopt in cmakeopts: + if cmakeopt and "mpi-serial" in mpilib and "PNETCDF" in cmakeopt: + continue + bldcnt = bldcnt+1 + bld = f"/glade/derecho/scratch/jedwards/piotest/bld{bldcnt:02d}" + os.chdir(piodir) + if os.path.exists(bld): + shutil.rmtree(bld) + os.mkdir(bld) + print(" ") + print(f"bld is {bld}", flush=True) + os.environ["CC"] = cc + os.environ["FC"] = ftn + os.environ["CFLAGS"] = cflags + os.environ["FFLAGS"] = fflags + cmake = ["cmake"] + + if cmakeflags: + cmake.extend(cmakeflags) + if cmakeopt: + cmake.extend(cmakeopt) + cmake.extend([piodir]) + print("Running cmake") + print(f"compiler = {compiler} netcdf={netlib} mpilib={mpilib} cmake = {cmake}", flush=True) + cmakeout = None + try: + cmakeout = subprocess.run(cmake, check=True, cwd=bld, capture_output=True, text=True) +# cmakeout = subprocess.run(cmake, check=True, cwd=bld) + except: + #print(f"cmake process failed {cmakeout.stdout}") + continue + + + printline = False + for line in cmakeout.stdout.split("\n"): + if "PIO Configuration Summary" in line: + printline = True + if printline: + print(line, flush=True) + # print(f"cmakeout is {cmakeout.stdout}") + ctestout = None + print("Running make", flush=True) + if "mpi-serial" in mpilib: + try: + makeout = subprocess.run(["make"], check=True, cwd=bld, capture_output=True, text=True) + except: + print("mpi-serial make process failed") + else: + try: + makeout = subprocess.run(["make", "tests"], check=True, cwd=bld, capture_output=True, text=True) + except: + print("make tests failed") + continue + try: + print("Running ctest", flush=True) + ctestout = subprocess.run(["ctest"], capture_output=True, cwd=bld, text=True) + except: + print("ctest process failed") + #print(f"makeout is {makeout.stdout}") + if ctestout: + for line in ctestout.stdout.split("\n"): + if "Fail" in line or "tests fail" in line: + print(line, flush=True) + diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/doftests/CMakeLists.txt b/src/Infrastructure/IO/PIO/ParallelIO/tests/doftests/CMakeLists.txt index 9e0825978b..4940b32d58 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/doftests/CMakeLists.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/doftests/CMakeLists.txt @@ -16,4 +16,3 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") # target_compile_options (gptl # PRIVATE -mismatch_all) endif () - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/doftests/dofcopy.F90 b/src/Infrastructure/IO/PIO/ParallelIO/tests/doftests/dofcopy.F90 index e676ba58da..1f2bbc54e5 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/doftests/dofcopy.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/doftests/dofcopy.F90 @@ -6,7 +6,7 @@ program dofcopy use mpi #endif use pio - + implicit none #ifdef NO_MPIMOD #include <mpif.h> @@ -36,7 +36,7 @@ program dofcopy endif CALL get_command_argument(1, infile) - + call pio_readdof(trim(infile), ndims, gdims, compmap, MPI_COMM_WORLD) if(mype < npe) then @@ -44,7 +44,7 @@ program dofcopy call PIO_InitDecomp(iosystem, PIO_INT, gdims, compmap, iodesc, rearr=rearr) write(outfile, *) trim(infile)//".nc" - call PIO_write_nc_dof(iosystem, outfile, PIO_64BIT_DATA, iodesc, ierr) + call PIO_write_nc_dof(iosystem, outfile, PIO_64BIT_DATA, iodesc, ierr) call PIO_finalize(iosystem, ierr) endif diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/CMakeLists.txt b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/CMakeLists.txt index 80fa196593..83b6ec7563 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/CMakeLists.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/CMakeLists.txt @@ -24,6 +24,7 @@ SET(GENERATED_SRCS pio_file_simple_tests.F90 pio_decomp_fillval.F90 pio_iosystem_tests.F90 pio_iosystem_async_tests.F90 + pio_filter_tests.F90 pio_iosystem_tests2.F90 pio_iosystem_tests3.F90) @@ -38,7 +39,7 @@ foreach (SRC_FILE IN LISTS GENERATED_SRCS) endforeach () if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") - add_definitions(-ffree-line-length-none) + add_definitions(-ffree-line-length-none -Wno-conversion) endif() if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") @@ -56,7 +57,7 @@ endif() #============================================================================== # Test Timeout (4 min = 240 sec) -set (DEFAULT_TEST_TIMEOUT 480) +set (DEFAULT_TEST_TIMEOUT 600) add_library(pio_tutil util/pio_tutil.F90) target_link_libraries(pio_tutil piof pioc) @@ -100,24 +101,6 @@ else () TIMEOUT ${DEFAULT_TEST_TIMEOUT}) endif () -#===== pio_file_simple_tests ===== -add_executable (pio_file_simple_tests EXCLUDE_FROM_ALL - pio_file_simple_tests.F90) -target_link_libraries (pio_file_simple_tests pio_tutil) -add_dependencies (tests pio_file_simple_tests) - -if (PIO_USE_MPISERIAL) - add_test(NAME pio_file_simple_tests - COMMAND pio_file_simple_tests) - set_tests_properties(pio_file_simple_tests - PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -else () - add_mpi_test(pio_file_simple_tests - EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_file_simple_tests - NUMPROCS 2 - TIMEOUT ${DEFAULT_TEST_TIMEOUT}) -endif () - #===== pio_file_fail ===== add_executable (pio_file_fail EXCLUDE_FROM_ALL pio_file_fail.F90) @@ -140,6 +123,24 @@ else () TIMEOUT ${DEFAULT_TEST_TIMEOUT}) endif () +#===== pio_file_simple_tests ===== +add_executable (pio_file_simple_tests EXCLUDE_FROM_ALL + pio_file_simple_tests.F90) +target_link_libraries (pio_file_simple_tests pio_tutil) +add_dependencies (tests pio_file_simple_tests) + +if (PIO_USE_MPISERIAL) + add_test(NAME pio_file_simple_tests + COMMAND pio_file_simple_tests) + set_tests_properties(pio_file_simple_tests + PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) +else () + add_mpi_test(pio_file_simple_tests + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_file_simple_tests + NUMPROCS 2 + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) +endif () + #===== ncdf_simple_tests ===== add_executable (ncdf_simple_tests EXCLUDE_FROM_ALL ncdf_simple_tests.F90) @@ -255,6 +256,8 @@ else () NUMPROCS 4 TIMEOUT ${DEFAULT_TEST_TIMEOUT}) endif () +# Test times out in github action. +set_tests_properties(pio_rearr_opts PROPERTIES LABELS "skipforspack") #===== pio_rearr_opts2 ===== add_executable (pio_rearr_opts2 EXCLUDE_FROM_ALL @@ -814,6 +817,18 @@ if (NOT PIO_USE_MPISERIAL) TIMEOUT ${DEFAULT_TEST_TIMEOUT}) endif () +add_executable (pio_filter_tests EXCLUDE_FROM_ALL + pio_filter_tests.F90) +target_link_libraries (pio_filter_tests pio_tutil) +add_dependencies (tests pio_filter_tests) + +if (NOT PIO_USE_MPISERIAL AND PIO_USE_PARALLEL_FILTERS) + add_mpi_test(pio_filter_tests + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_filter_tests + NUMPROCS 5 + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) +endif () + #===== pio_iosystems_test2 ===== diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/README.md b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/README.md index 8353978988..0fa10a65c7 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/README.md +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/README.md @@ -5,7 +5,7 @@ * Improved testcase code readability : Most of the boiler plate code is generated by the framework * Better error checking and reporting : User friendly parallel asserts * More testing with less code : Templated tests, auto generation of boiler plate PIO code -* Easy integration with CTest +* Easy integration with CTest # Writing a test @@ -81,7 +81,7 @@ call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc) - ... + ... PIO_TF_AUTO_tEST_SUB_END templated_hellow @@ -123,7 +123,7 @@ * Add the testcase stub source into `GENERATED_SRCS` * Add a CMake build step : To generate the test executable - add_executable (pio_init_finalize EXCLUDE_FROM_ALL + add_executable (pio_init_finalize EXCLUDE_FROM_ALL pio_init_finalize.F90 ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90) target_link_libraries (pio_init_finalize piof) @@ -137,6 +137,6 @@ TIMEOUT ${DEFAULT_TEST_TIMEOUT}) # Debugging failed tests - + * Run the test suite using ctest with the "--verbose" option. This outputs a lot of information from the testing framework (to stdout), including the various tests run and the reason for failure. All information in the output from the PIO testing framework is prepended with the `PIO_TF: ` tag. To run a single failing test use the "-R" option available with ctest (e.g. To only run "pio_iosystem_tests3", ctest --verbose -R pio_iosystem_tests3). * Run the failing test manually (using the MPI job launcher) and debug it. Also consider running the test with PIO log level > 0 (mpiexec -n 4 ./tests/general/pio_iosystem_tests3 --pio-tf-log-level=6) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_fail.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_fail.F90.in index eed98aad6f..74a8afd6ec 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_fail.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_fail.F90.in @@ -52,21 +52,25 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_redef_twice type(file_desc_t) :: pio_file integer :: ret + ret = PIO_NOERR + ! as of netcdf 8.4.0 netcdf4p and netcdf4c do not generate an error when redef is called twice. ret = PIO_openfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname, PIO_write) PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(tgv_fname)) ! A simple redef and then enddef ret = PIO_redef(pio_file) PIO_TF_CHECK_ERR(ret, "Failed to enter redef mode" // trim(tgv_fname)) - ret = PIO_redef(pio_file) - PIO_TF_PASSERT(ret /= PIO_NOERR, "Entering redef twice did not fail as expected") - + if(tgv_iotype == PIO_IOTYPE_PNETCDF .or. tgv_iotype == PIO_IOTYPE_NETCDF) then + PIO_TF_PASSERT(ret /= PIO_NOERR, "Entering redef twice did not fail as expected") + else + PIO_TF_CHECK_ERR(ret, "Entering redef twice did not pass as expected") + endif ret = PIO_enddef(pio_file) PIO_TF_CHECK_ERR(ret, "Failed to end redef mode" // trim(tgv_fname)) - call PIO_closefile(pio_file) + PIO_TF_AUTO_TEST_SUB_END test_redef_twice PIO_TF_TEST_DRIVER_BEGIN diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_inq.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_inq.F90.in index 9745bd1b8e..983612a8a1 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_inq.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_inq.F90.in @@ -65,7 +65,7 @@ SUBROUTINE test_teardown(ret) integer, intent(out) :: ret ret = PIO_NOERR - call PIO_deletefile(pio_tf_iosystem_, tgv_fname) + call PIO_deletefile(pio_tf_iosystem_, tgv_fname) END SUBROUTINE test_teardown SUBROUTINE test_inq_var(pio_file, ret) @@ -118,7 +118,7 @@ SUBROUTINE test_inq_dim(pio_file, ret) PIO_TF_CHECK_ERR(ret, "Failed to inq dimid :"//trim(tgv_fname)) ret = pio_inq_dimname(pio_file, dim_id, dim_name) - PIO_TF_PASSERT(dim_name .eq. tgv_dim_name, "Dim name is not the expected value") + PIO_TF_PASSERT(dim_name .eq. tgv_dim_name, "Dim name is not the expected value") ret = pio_inq_dimlen(pio_file, dim_id, dim_len) PIO_TF_PASSERT(dim_len == TGV_DIM_LEN, "Dim length is not the expected value") @@ -157,7 +157,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_inq character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs integer :: num_iotypes integer :: i, ret - + num_iotypes = 0 call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) do i=1,num_iotypes diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_simple_tests.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_simple_tests.F90.in index 12368f1300..26650e3e3d 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_simple_tests.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/ncdf_simple_tests.F90.in @@ -123,7 +123,10 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_data_conversion dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_ compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps wbuf = pio_tf_world_rank_; - exp_val = pio_tf_world_rank_; + +! Type conversions here to avoid gfortran warnings + + exp_val = pio_tf_world_rank_ ! Set the decomposition for writing data as PIO_int call PIO_initdecomp(pio_tf_iosystem_, PIO_int, dims, compdof, wiodesc) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_frame_tests.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_frame_tests.F90.in index 70edaf6fbb..8b168274a9 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_frame_tests.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_frame_tests.F90.in @@ -1,7 +1,7 @@ #include "config.h" ! Get a 3D column decomposition ! If force_rearrange is FALSE, the decomposition is such that -! # All even procs have VEC_HGT_SZ blocks of +! # All even procs have VEC_HGT_SZ blocks of ! (VEC_COL_SZ rows x VEC_ROW_SZ columns) elements ! # All odd procs have VEC_HGT_SZ blocks of ! (VEC_COL_SZ rows x VEC_ROW_SZ + 1 columns) elements diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_tests.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_tests.F90.in index cb6f939869..deeadcab15 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_tests.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_tests.F90.in @@ -62,7 +62,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_1d_darray filename = "test_pio_decomp_simple_tests.testfile" do i=1,num_iotypes PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) - ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim) @@ -163,7 +163,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_const_buf_sz filename = "test_pio_decomp_simple_tests.testfile" do i=1,num_iotypes PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) - ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim) @@ -230,11 +230,11 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_1d_reuse_decomp do i=1,num_iotypes PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) ierr = PIO_createfile(pio_tf_iosystem_, pio_file1, iotypes(i),& - filename1, PIO_CLOBBER) + filename1, PIO_CLOBBER) PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename1)) ierr = PIO_createfile(pio_tf_iosystem_, pio_file2, iotypes(i),& - filename2, PIO_CLOBBER) + filename2, PIO_CLOBBER) PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename1)) ierr = PIO_def_dim(pio_file1, 'PIO_TF_test_dim', dims(1), pio_dim_file1) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_tests_3d.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_tests_3d.F90.in index 17a20b2d0e..a8f0b5961f 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_tests_3d.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_decomp_tests_3d.F90.in @@ -1,7 +1,7 @@ #include "config.h" ! Get a 3D column decomposition ! If force_rearrange is FALSE, the decomposition is such that -! # All even procs have VEC_HGT_SZ blocks of +! # All even procs have VEC_HGT_SZ blocks of ! (VEC_COL_SZ rows x VEC_ROW_SZ columns) elements ! # All odd procs have VEC_HGT_SZ blocks of ! (VEC_COL_SZ rows x VEC_ROW_SZ + 1 columns) elements @@ -117,7 +117,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_3d_col_decomp nrows = count(1) ncols = count(2) nhgts = count(3) - + allocate(wbuf(nrows, ncols, nhgts)) allocate(compdof(nrows * ncols * nhgts)) do k=1,nhgts @@ -139,7 +139,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_3d_col_decomp nrows = count(1) ncols = count(2) nhgts = count(3) - + allocate(rbuf(nrows, ncols, nhgts)) allocate(compdof(nrows * ncols * nhgts)) allocate(exp_val(nrows, ncols, nhgts)) @@ -163,7 +163,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_3d_col_decomp filename = "test_pio_decomp_simple_tests.testfile" do i=1,num_iotypes PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) - ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1)) @@ -193,7 +193,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_3d_col_decomp PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val") call PIO_closefile(pio_file) - + call PIO_deletefile(pio_tf_iosystem_, filename); end do diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_file_fail.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_file_fail.F90.in index 8ef1063180..e718bde1ba 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_file_fail.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_file_fail.F90.in @@ -37,7 +37,7 @@ PIO_TF_TEST_DRIVER_BEGIN integer :: num_uiotypes num_uiotypes = 0 - call PIO_TF_Get_undef_iotypes(uiotypes, uiotype_descs, num_uiotypes) + call PIO_TF_Get_undef_iotypes(uiotypes, uiotype_descs, num_uiotypes) dummy_file = "test_pio_file_fail.testfile" do i=1,num_uiotypes PIO_TF_TEST_RUN(create_file_always_fail(uiotypes(i), dummy_file), trim(uiotype_descs(i))) @@ -49,4 +49,3 @@ PIO_TF_TEST_DRIVER_BEGIN end if PIO_TF_TEST_DRIVER_END - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_filter_tests.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_filter_tests.F90.in new file mode 100644 index 0000000000..996dd5de74 --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_filter_tests.F90.in @@ -0,0 +1,417 @@ +#include "config.h" +#include <netcdf_meta.h> +! Get a 2D column decomposition +! If force_rearrange is FALSE, the decomposition is such that +! # All even procs have VEC_COL_SZ rows of VEC_ROW_SZ elements +! # All odd procs have VEC_COL_SZ rows of VEC_ROW_SZ + 1 elements +! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2 and ranks 0, 1, 2, +! e.g. 1) |(1,1) (1,2)| |(1,3) (1,4) (1,5)| |(1,6) (1,7)| +! |(2,1) (2,2)|, |(2,3) (2,4) (2,5)|, |(2,6) (2,7)| +! If force_rearrange is TRUE, the decomposition is such that, +! If possible, the even rank "exchanges" elements with the next +! higher ranked odd proc. +! This for example can be used to force rearrangement when reading +! or writing data. +! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2 and ranks 0, 1, 2 +! e.g. 1) |(1,3) (1,4) (1,5)| |(1,1) (1,2)| |(1,6) (1,7)| +! |(2,3) (2,4) (2,5)|, |(2,1) (2,2)|, |(2,6) (2,7)| +SUBROUTINE get_2d_col_decomp_info(rank, sz, dims, start, count, force_rearrange) + integer, parameter :: VEC_ROW_SZ = 7 + integer, parameter :: VEC_COL_SZ = 7 + integer, parameter :: NDIMS = 2 + integer, intent(in) :: rank + integer, intent(in) :: sz + integer, dimension(NDIMS), intent(out) :: dims + integer, dimension(NDIMS), intent(out) :: start + integer, dimension(NDIMS), intent(out) :: count + logical, intent(in) :: force_rearrange + + logical :: is_even_rank + integer :: num_odd_procs, num_even_procs + integer :: iodd, ieven + + is_even_rank = .false. + if (mod(rank, 2) == 0) then + is_even_rank = .true. + end if + num_odd_procs = sz / 2 + num_even_procs = sz - num_odd_procs + dims(1) = VEC_COL_SZ + dims(2) = num_even_procs * VEC_ROW_SZ + num_odd_procs * (VEC_ROW_SZ + 1) + ! Number of odd and even procs before this rank + iodd = rank / 2 + ieven = (rank + 1) / 2 + + ! Rows + start(1) = 1 + count(1) = VEC_COL_SZ + + ! Columns + if(force_rearrange) then + ! Make sure that we force rearrangement + if (is_even_rank) then + if(rank + 1 < sz) then + ! Force rearrangement + count(2) = VEC_ROW_SZ + 1 + start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + (VEC_ROW_SZ) + 1 + else + count(2) = VEC_ROW_SZ + start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1 + end if + else + ! For all odd procs there is an even lower ranked, rank-1, proc + ! So force rearrangement + count(2) = VEC_ROW_SZ + start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) - (VEC_ROW_SZ) + 1 + end if + else + if (is_even_rank) then + count(2) = VEC_ROW_SZ + else + count(2) = VEC_ROW_SZ + 1 + end if + start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1 + end if + +END SUBROUTINE + +! Get a 2D row decomposition +! If force_rearrange is FALSE, the decomposition is such that +! # All even procs have VEC_COL_SZ rows of VEC_ROW_SZ elements +! # All odd procs have VEC_COL_SZ+1 rows of VEC_ROW_SZ elements +! e.g. For VEC_ROW_SZ = 6, VEC_COL_SZ = 1 and ranks 0, 1, 2, +! e.g. 1) |(1,1) (1,2) (1,3) (1,4) (1,5) (1,6)|, +! |(2,1) (2,2) (2,3) (2,4) (2,5) (2,6)| +! |(3,1) (3,2) (3,3) (3,4) (3,5) (3,6)|, +! |(4,1) (4,2) (4,3) (4,4) (4,5) (4,6)| +! If force_rearrange is TRUE, the decomposition is such that, +! If possible, the even rank "exchanges" elements (rows) with the next +! higher ranked odd proc. +! This for example can be used to force rearrangement when reading +! or writing data. +! e.g. For VEC_ROW_SZ = 6, VEC_COL_SZ = 1 and ranks 0, 1, 2 +! e.g. 1) |(2,1) (2,2) (2,3) (2,4) (2,5) (2,6)| +! |(3,1) (3,2) (3,3) (3,4) (3,5) (3,6)|, +! |(1,1) (1,2) (1,3) (1,4) (1,5) (1,6)|, +! |(4,1) (4,2) (4,3) (4,4) (4,5) (4,6)| +SUBROUTINE get_2d_row_decomp_info(rank, sz, dims, start, count, force_rearrange) + integer, parameter :: VEC_COL_SZ = 7 + integer, parameter :: VEC_ROW_SZ = 7 + integer, parameter :: NDIMS = 2 + integer, intent(in) :: rank + integer, intent(in) :: sz + integer, dimension(NDIMS), intent(out) :: dims + integer, dimension(NDIMS), intent(out) :: start + integer, dimension(NDIMS), intent(out) :: count + logical, intent(in) :: force_rearrange + + logical :: is_even_rank + integer :: num_odd_procs, num_even_procs + integer :: iodd, ieven + + is_even_rank = .false. + if (mod(rank, 2) == 0) then + is_even_rank = .true. + end if + num_odd_procs = sz / 2 + num_even_procs = sz - num_odd_procs + dims(1) = num_even_procs * VEC_COL_SZ + num_odd_procs * (VEC_COL_SZ + 1) + dims(2) = VEC_ROW_SZ + ! Number of odd and even procs before this rank + iodd = rank / 2 + ieven = (rank + 1) / 2 + + ! Rows + if(force_rearrange) then + ! Make sure that we force rearrangement + if (is_even_rank) then + if(rank + 1 < sz) then + ! Force rearrangement + count(1) = VEC_COL_SZ + 1 + start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + (VEC_COL_SZ) + 1 + else + count(1) = VEC_COL_SZ + start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + 1 + end if + else + ! For all odd procs there is an even lower ranked, rank-1, proc + ! So force rearrangement + count(1) = VEC_COL_SZ + start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) - (VEC_COL_SZ) + 1 + end if + else + if (is_even_rank) then + count(1) = VEC_COL_SZ + else + count(1) = VEC_COL_SZ + 1 + end if + start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + 1 + end if + + ! Columns + start(2) = 1 + count(2) = VEC_ROW_SZ + +END SUBROUTINE + +! Write with one decomp (to force rearrangement) and read with another (no +! rearrangement) +PIO_TF_TEMPLATE<PIO_TF_PREDEF_TYPENAME PIO_TF_DATA_TYPE, PIO_TF_PREDEF_TYPENAME PIO_TF_FC_DATA_TYPE> +PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_2d_col_decomp + implicit none + type(var_desc_t) :: pio_var(6) ! one for each potential filter type + type(file_desc_t) :: pio_file + integer, parameter :: NDIMS = 2 + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(io_desc_t) :: wr_iodesc, rd_iodesc + integer, dimension(:), allocatable :: compdof + integer, dimension(NDIMS) :: start, count + PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: rbuf, wbuf, exp_val + integer, dimension(NDIMS) :: dims + integer, dimension(NDIMS) :: pio_dims + integer :: i, j, tmp_idx, ierr, nrows, ncols + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + integer, dimension(:), allocatable :: filtertypes + + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: filtertype_descs + character(len=PIO_TF_MAX_STR_LEN) :: msg + integer :: num_iotypes + integer :: num_filtertypes + + ! Set the decomposition for writing data - forcing rearrangement + call get_2d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.) + nrows = count(1) + ncols = count(2) + + allocate(wbuf(nrows, ncols)) + allocate(compdof(nrows * ncols)) + do j=1,ncols + do i=1,nrows + wbuf(i,j) = (start(2) - 1 + j - 1) * nrows + i + tmp_idx = (j - 1) * nrows + i + compdof(tmp_idx) = int(wbuf(i,j)) + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc) + deallocate(compdof) + + ! Set the decomposition for reading data - different from the write decomp + call get_2d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.) + nrows = count(1) + ncols = count(2) + + allocate(rbuf(nrows, ncols)) + allocate(compdof(nrows * ncols)) + allocate(exp_val(nrows, ncols)) + do j=1,ncols + do i=1,nrows + tmp_idx = (j - 1) * nrows + i + compdof(tmp_idx) = (start(2) - 1 + j - 1) * nrows + i + ! Expected value, after reading, is the same as the compdof + exp_val(i,j) = compdof(tmp_idx) + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc) + deallocate(compdof) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + + filename = "test_pio_decomp_filter_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", trim(iotype_descs(i)) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) +#ifdef PIO_HAS_PAR_FILTERS + if(iotypes(i) == PIO_IOTYPE_NETCDF4P .or. iotypes(i) == PIO_IOTYPE_NETCDF4C) then + call PIO_TF_Get_nc4_filtertypes(pio_file, filtertypes, filtertype_descs, num_filtertypes) + print *,__FILE__,__LINE__,num_filtertypes + do j=1, num_filtertypes + print *,__FILE__,__LINE__,trim(filtertype_descs(j)) + enddo + endif +#else + num_filtertypes = 1 +#endif + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1)) + PIO_TF_CHECK_ERR(ierr, "Failed to define dim1") + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2)) + PIO_TF_CHECK_ERR(ierr, "Failed to define dim2") +#ifdef NC_HAS_BZ + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var_bzip2', PIO_TF_DATA_TYPE, pio_dims, pio_var(1)) + PIO_TF_CHECK_ERR(ierr, "Failed to define var") +#endif +#ifdef NC_HAS_ZSTD + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var_zstandard', PIO_TF_DATA_TYPE, pio_dims, pio_var(2)) + PIO_TF_CHECK_ERR(ierr, "Failed to define var") +#endif +#if defined(PIO_HAS_PAR_FILTERS) + if((iotypes(i) == PIO_IOTYPE_NETCDF4P .or. iotypes(i) == PIO_IOTYPE_NETCDF4C) & + .and. (PIO_TF_DATA_TYPE==PIO_double .or. PIO_TF_DATA_TYPE==PIO_real)) then + ierr = PIO_def_var_szip(pio_file, pio_var(1), 32, 32) + write(msg, *) "Failed to define var szip compression. ierr=",ierr + PIO_TF_CHECK_ERR(ierr, msg) + + ierr = PIO_def_var_zstandard(pio_file, pio_var(2), 4) + write(msg, *) "Failed to define var zstandard compression. ierr=",ierr + PIO_TF_CHECK_ERR(ierr, msg) + +! ierr = PIO_def_var_deflate(pio_file, pio_var(2), shuffle, deflate, deflate_level) +! write(msg, *) "Failed to define var deflate compression. ierr=",ierr +! PIO_TF_CHECK_ERR(ierr, msg) + + + + else + PIO_TF_LOG(0,*) 'Do not test compression here ',trim(iotype_descs(i)), PIO_TF_DATA_TYPE + endif +#endif + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + do j=1,2 + ! Write the variable out + call PIO_write_darray(pio_file, pio_var(j), wr_iodesc, wbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + + call PIO_syncfile(pio_file) + + call PIO_read_darray(pio_file, pio_var(j), rd_iodesc, rbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + + PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val") + enddo + call PIO_closefile(pio_file) + + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc) + call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc) + deallocate(exp_val) + deallocate(rbuf) + deallocate(wbuf) +PIO_TF_AUTO_TEST_SUB_END nc_write_read_2d_col_decomp + +! Write with one decomp (to force rearrangement) and read with another (no +! rearrangement) +PIO_TF_TEMPLATE<PIO_TF_PREDEF_TYPENAME PIO_TF_DATA_TYPE, PIO_TF_PREDEF_TYPENAME PIO_TF_FC_DATA_TYPE> +PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_2d_row_decomp + implicit none + type(var_desc_t) :: pio_var + type(file_desc_t) :: pio_file + integer, parameter :: NDIMS = 2 + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(io_desc_t) :: wr_iodesc, rd_iodesc + integer, dimension(:), allocatable :: compdof + integer, dimension(NDIMS) :: start, count + PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: rbuf, wbuf, exp_val + integer, dimension(NDIMS) :: dims + integer, dimension(NDIMS) :: pio_dims + integer :: i, j, tmp_idx, ierr, nrows, ncols + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Set the decomposition for writing data - forcing rearrangement + call get_2d_row_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.) + nrows = count(1) + ncols = count(2) + + allocate(wbuf(nrows, ncols)) + allocate(compdof(nrows * ncols)) + do j=1,ncols + do i=1,nrows + wbuf(i,j) = (start(2) - 1 + j - 1) * dims(1) + start(1) + i - 1 + tmp_idx = (j - 1) * nrows + i + compdof(tmp_idx) = int(wbuf(i,j)) + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc) + deallocate(compdof) + + ! Set the decomposition for reading data - different from the write decomp + call get_2d_row_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.) + nrows = count(1) + ncols = count(2) + + allocate(rbuf(nrows, ncols)) + allocate(compdof(nrows * ncols)) + allocate(exp_val(nrows, ncols)) + do j=1,ncols + do i=1,nrows + tmp_idx = (j - 1) * nrows + i + compdof(tmp_idx) = (start(2) - 1 + j - 1) * dims(1) + start(1) + i - 1 + ! Expected value, after reading, is the same as the compdof + exp_val(i,j) = compdof(tmp_idx) + end do + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc) + deallocate(compdof) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_filter_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", trim(iotype_descs(i)) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1)) + PIO_TF_CHECK_ERR(ierr, "Failed to define dim1") + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2)) + PIO_TF_CHECK_ERR(ierr, "Failed to define dim2") + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var) + PIO_TF_CHECK_ERR(ierr, "Failed to define a var : PIO_TF_test_var") +!#if defined(NC_HAS_QUANTIZE) && PIO_TF_DATA_TYPE==PIO_DOUBLE || PIO_TF_DATA_TYPE==PIO_REAL +! if(iotypes(i) == PIO_IOTYPE_NETCDF4P .or. iotypes(i) == PIO_IOTYPE_NETCDF4C) then +! ierr = PIO_def_var_quantize(pio_file, pio_var, PIO_QUANTIZE_BITROUND, 4) +! PIO_TF_CHECK_ERR(ierr, "Failed to quantize a var : " // trim(filename)) +! else +! PIO_TF_LOG(0,*) 'Do not test compression here ',trim(iotype_descs(i)), PIO_TF_DATA_TYPE +! endif +!#endif + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + ! Write the variable out + call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + + call PIO_syncfile(pio_file) + + call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + + PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val") + call PIO_closefile(pio_file) + + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc) + call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc) + deallocate(exp_val) + deallocate(rbuf) + deallocate(wbuf) +PIO_TF_AUTO_TEST_SUB_END nc_write_read_2d_row_decomp diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_init_finalize.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_init_finalize.F90.in index 27e03f8ea7..571f9587cd 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_init_finalize.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_init_finalize.F90.in @@ -3,5 +3,3 @@ PIO_TF_AUTO_TEST_SUB_BEGIN init_finalize ! The default test driver should initialize and finalize PIO PRINT *, "Hello world" PIO_TF_AUTO_TEST_SUB_END init_finalize - - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_iosystem_async_tests.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_iosystem_async_tests.F90.in index 117ee1de53..e814878807 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_iosystem_async_tests.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_iosystem_async_tests.F90.in @@ -99,12 +99,11 @@ END SUBROUTINE create_file ! Check the contents of file : Check the ! global attribute 'filename' (should be equal to the ! name of the file, fname) -SUBROUTINE check_file(comm, iosys, pio_file, fname, attname, dimname, ret) +SUBROUTINE check_file(comm, pio_file, fname, attname, dimname, ret) use pio_tutil implicit none integer, intent(in) :: comm - type(iosystem_desc_t), intent(in) :: iosys type(file_desc_t), intent(inout) :: pio_file character(len=*), intent(in) :: fname character(len=*), intent(in) :: attname @@ -129,7 +128,7 @@ SUBROUTINE check_file(comm, iosys, pio_file, fname, attname, dimname, ret) PIO_TF_PASSERT(val .eq. fname, comm, "Attribute value is not the expected value") call PIO_SetErrorHandling(pio_file, PIO_BCAST_ERROR, old_eh) -! ret = pio_set_log_level(iosys, 3) +! ret = pio_set_log_level(iosys, 0) ret = PIO_get_att(pio_file, pio_var, "wrongname", ival) write(errstr, *) "Got wrong error ",ret," on getatt in file:", trim(fname) PIO_TF_PASSERT(ret==PIO_ENOTATT, comm, errstr) @@ -155,11 +154,15 @@ SUBROUTINE open_and_check_file(comm, iosys, iotype, pio_file, fname, & character(len=*), intent(in) :: dimname logical, intent(in) :: disable_fclose integer, intent(inout) :: ret - + logical res +! ret = pio_set_log_level(3) + inquire(file=trim(fname), exist=res) + print *,__FILE__,__LINE__,trim(fname), 'res=',res ret = PIO_openfile(iosys, pio_file, iotype, fname, PIO_write) PIO_TF_CHECK_ERR(ret, comm, "Failed to open:" // fname) +! ret = pio_set_log_level(0) - call check_file(comm, iosys, pio_file, fname, attname, dimname, ret) + call check_file(comm, pio_file, fname, attname, dimname, ret) PIO_TF_CHECK_ERR(ret, comm, "Checking contents of file failed:" // fname) if(.not. disable_fclose) then @@ -177,7 +180,6 @@ PIO_TF_AUTO_TEST_SUB_BEGIN two_comps_odd_even_async character(len=PIO_TF_MAX_STR_LEN), target :: fname2 = "pio_iosys_async_test_file2.nc" character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename" character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim" - character(len=PIO_TF_MAX_STR_LEN), pointer :: fname integer, dimension(:), allocatable :: iotypes character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs integer :: i, num_iotypes = 0 @@ -189,9 +191,11 @@ PIO_TF_AUTO_TEST_SUB_BEGIN two_comps_odd_even_async integer :: comp_comm(2), io_comm integer :: all_comp_comm ! comm common to all components integer :: ret + logical :: res ! Split world to odd even and io procs call split_world_odd_even_io(pio_tf_comm_, all_comp_comm, comp_comm, io_comm, ret) + print *,__FILE__,__LINE__,'comp_comm: ',comp_comm call PIO_init(iosys, pio_tf_comm_, comp_comm, io_comm, PIO_REARR_BOX) if(io_comm == MPI_COMM_NULL) then if(comp_comm(1) /= MPI_COMM_NULL) then @@ -207,19 +211,21 @@ PIO_TF_AUTO_TEST_SUB_BEGIN two_comps_odd_even_async ! Create two files to be opened later if(comp_comm(1) /= MPI_COMM_NULL) then is_even = .false. -! print *,__FILE__,__LINE__,'create_file', is_even call create_file(comp_comm(1), iosys(1), iotypes(i), & fname1, attname, dimname, ret) + if (iotypes(i) == PIO_IOTYPE_NETCDF4C) ret = pio_set_log_level(0) + print *,__FILE__,__LINE__,'create_file', is_even, trim(fname1) + inquire(file=trim(fname1), exist=res) + print *,__FILE__,__LINE__,trim(fname1), res PIO_TF_CHECK_ERR(ret, comp_comm(1), "Failed to create file :" // fname1) else is_even = .true. -! print *,__FILE__,__LINE__,'create_file', is_even + print *,__FILE__,__LINE__,'create_file', is_even, trim(fname2) call create_file(comp_comm(2), iosys(2), iotypes(i), & fname2, attname, dimname, ret) PIO_TF_CHECK_ERR(ret, comp_comm(2), "Failed to create file :" // fname2) endif -! print *,__FILE__,__LINE__,'at barrier', is_even call mpi_barrier(all_comp_comm, ret) ! Open file1 from odd processes and file2 from even processes @@ -234,6 +240,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN two_comps_odd_even_async PIO_TF_CHECK_ERR(ret, comp_comm(1), "Checking contents of file failed :" // fname2) call pio_deletefile(iosys(1), fname2) end if + call mpi_barrier(all_comp_comm, ret) end do if (is_even) then call PIO_finalize(iosys(2), ret) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr.F90.in index 202ad10958..53ad53e447 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr.F90.in @@ -197,7 +197,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_rearrs_combs pio_rearr_subset, pio_rearr_subset,& pio_rearr_box, pio_rearr_box/),& (/NUM_REARRANGERS,MAX_PERMS/)& - ) + ) character(len=PIO_TF_MAX_STR_LEN) :: rearrs_perms_info(NUM_REARRANGERS,MAX_PERMS) =& reshape(& (/"PIO_REARR_SUBSET", "PIO_REARR_BOX ",& @@ -248,4 +248,3 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_rearrs_combs deallocate(iotype_descs) end if PIO_TF_AUTO_TEST_SUB_END test_rearrs_combs - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr_opts.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr_opts.F90.in index d209860c2f..2a81603e82 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr_opts.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr_opts.F90.in @@ -281,7 +281,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN write_with_rearr_opts integer, dimension(1), intent(out) :: dims integer, intent(out) :: ret end subroutine create_decomp_and_init_buf - end interface + end interface integer, parameter :: NUM_REARRANGERS = 2 integer :: rearrs(NUM_REARRANGERS) = (/pio_rearr_subset,pio_rearr_box/) @@ -333,7 +333,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN write_with_rearr_opts call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) do i=1,num_iotypes PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i) - ! Create the file and decomp + ! Create the file and decomp call create_file_and_var(iotypes(i), ret) PIO_TF_CHECK_ERR(ret, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname)) @@ -404,7 +404,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN write_with_rearr_opts do cur_enable_hs_i2c=1,num_enable_hs_opts_io2comp pio_rearr_opts%comm_fc_opts_io2comp%enable_hs =& enable_hs_opts(cur_enable_hs_i2c) - + do cur_enable_isend_i2c=1,num_enable_isend_opts_io2comp pio_rearr_opts%comm_fc_opts_io2comp%enable_isend =& enable_isend_opts(cur_enable_isend_i2c) @@ -459,7 +459,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN write_with_rearr_opts end do ! cur_max_pend_req_i2c end do ! cur_enable_isend_i2c end do ! cur_enable_hs_i2c - end do ! cur_max_pend_req_c2i + end do ! cur_max_pend_req_c2i end do ! cur_enable_isend_c2i end do ! cur_enable_hs_c2i end do ! cur_fcd_opt diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr_opts2.F90.in b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr_opts2.F90.in index 0d6a48c8a8..4491f1b766 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr_opts2.F90.in +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/pio_rearr_opts2.F90.in @@ -230,7 +230,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN set_rearr_opts_and_write integer, dimension(1), intent(out) :: dims integer, intent(out) :: ret end subroutine create_decomp_and_init_buf - end interface + end interface type(pio_rearr_opt_t) :: pio_rearr_opts @@ -277,7 +277,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN set_rearr_opts_and_write call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) do i=1,num_iotypes PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i) - ! Create the file and decomp + ! Create the file and decomp call create_file_and_var(iotypes(i), ret) PIO_TF_CHECK_ERR(ret, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname)) @@ -347,7 +347,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN set_rearr_opts_and_write do cur_enable_hs_i2c=1,num_enable_hs_opts_io2comp pio_rearr_opts%comm_fc_opts_io2comp%enable_hs =& enable_hs_opts(cur_enable_hs_i2c) - + do cur_enable_isend_i2c=1,num_enable_isend_opts_io2comp pio_rearr_opts%comm_fc_opts_io2comp%enable_isend =& enable_isend_opts(cur_enable_isend_i2c) @@ -400,7 +400,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN set_rearr_opts_and_write end do ! cur_max_pend_req_i2c end do ! cur_enable_isend_i2c end do ! cur_enable_hs_i2c - end do ! cur_max_pend_req_c2i + end do ! cur_max_pend_req_c2i end do ! cur_enable_isend_c2i end do ! cur_enable_hs_c2i end do ! cur_fcd_opt diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/util/pio_tf_f90gen.pl b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/util/pio_tf_f90gen.pl index 696f50aaae..beb0cb267c 100755 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/util/pio_tf_f90gen.pl +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/util/pio_tf_f90gen.pl @@ -53,13 +53,13 @@ sub init_predef_types { $template_predef_typename_types{"PIO_TF_DATA_TYPE"} = []; $template_predef_typename_types{"PIO_TF_FC_DATA_TYPE"} = []; - push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_int"); + push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_INT"); push(@{$template_predef_typename_types{"PIO_TF_FC_DATA_TYPE"}}, "integer(kind=fc_int)"); - push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_short"); + push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_SHORT"); push(@{$template_predef_typename_types{"PIO_TF_FC_DATA_TYPE"}}, "integer(kind=fc_short)"); - push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_real"); + push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_REAL"); push(@{$template_predef_typename_types{"PIO_TF_FC_DATA_TYPE"}}, "real(kind=fc_real)"); - push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_double"); + push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_DOUBLE"); push(@{$template_predef_typename_types{"PIO_TF_FC_DATA_TYPE"}}, "real(kind=fc_double)"); } @@ -624,7 +624,7 @@ sub get_default_test_main $out_line = $out_line . " CALL MPI_Init(ierr)\n"; $out_line = $out_line . " DO i=1,SIZE(rearrs)\n"; if($test_type eq "async"){ - $out_line = $out_line . " CALL PIO_TF_Init_async_(rearrs(i))\n"; + $out_line = $out_line . " CALL PIO_TF_Init_async_()\n"; }else{ $out_line = $out_line . " CALL PIO_TF_Init_(rearrs(i))\n"; } @@ -698,6 +698,8 @@ sub get_default_test_driver $out_line = $out_line . " END IF\n"; $cur_test_case_num += 1; } + # This line just avoids a gfortran warning + $out_line = $out_line . " mpierr = 0\n"; $out_line = $out_line . " END SUBROUTINE PIO_TF_Test_driver_\n"; return $out_line; } diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/util/pio_tutil.F90 b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/util/pio_tutil.F90 index 71cf930a5a..89a9c5b1d9 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/general/util/pio_tutil.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/general/util/pio_tutil.F90 @@ -1,3 +1,4 @@ +#include "config.h" ! PIO Testing framework utilities module MODULE pio_tutil USE pio @@ -66,6 +67,9 @@ MODULE pio_tutil PUBLIC :: PIO_TF_Init_, PIO_TF_Finalize_, PIO_TF_Passert_ PUBLIC :: PIO_TF_Is_netcdf PUBLIC :: PIO_TF_Get_nc_iotypes, PIO_TF_Get_undef_nc_iotypes +#ifdef NC_HAS_MULTIFILTERS + public :: pio_tf_get_nc4_filtertypes +#endif PUBLIC :: PIO_TF_Get_iotypes, PIO_TF_Get_undef_iotypes PUBLIC :: PIO_TF_Get_data_types PUBLIC :: PIO_TF_Check_val_ @@ -190,7 +194,7 @@ SUBROUTINE PIO_TF_Init_(rearr) END SUBROUTINE PIO_TF_Init_ ! Initialize Testing framework - Internal (Not directly used by unit tests) - SUBROUTINE PIO_TF_Init_async_(rearr) + SUBROUTINE PIO_TF_Init_async_() #ifdef TIMING use perf_mod #endif @@ -199,7 +203,6 @@ SUBROUTINE PIO_TF_Init_async_(rearr) #else include 'mpif.h' #endif - INTEGER, INTENT(IN) :: rearr INTEGER ierr CALL MPI_COMM_DUP(MPI_COMM_WORLD, pio_tf_comm_, ierr); @@ -355,7 +358,6 @@ SUBROUTINE PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) ! pnetcdf num_iotypes = num_iotypes + 1 #endif - ! ALLOCATE with 0 elements ok? ALLOCATE(iotypes(num_iotypes)) ALLOCATE(iotype_descs(num_iotypes)) @@ -385,7 +387,59 @@ SUBROUTINE PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) i = i + 1 #endif END SUBROUTINE - +#ifdef PIO_HAS_PAR_FILTERS + ! Returns a list of defined netcdf4 filter types + ! pio_file : An open file to check + ! filtertypes : After the routine returns contains a list of defined + ! netcdf4 filter types + ! filtertype_descs : After the routine returns contains description of + ! the netcdf4 filter types returned in filtertypes + ! num_filtertypes : After the routine returns contains the number of + ! of defined netcdf4 types, i.e., size of filtertypes and + ! filtertype_descs arrays + SUBROUTINE PIO_TF_Get_nc4_filtertypes(pio_file, filtertypes, filtertype_descs, num_filtertypes) + use pio, only : pio_inq_filter_avail + type(file_desc_t), intent(in) :: pio_file + INTEGER, DIMENSION(:), ALLOCATABLE, INTENT(OUT) :: filtertypes + CHARACTER(LEN=*), DIMENSION(:), ALLOCATABLE, INTENT(OUT) :: filtertype_descs + INTEGER, INTENT(OUT) :: num_filtertypes + INTEGER :: i + integer :: ierr + integer, parameter :: num_possible_filters = 6 + INTEGER :: tmpfiltertypes(num_possible_filters) + + num_filtertypes = 0 + ! First find the number of filter types + + do i=1,num_possible_filters + ierr = pio_inq_filter_avail(pio_file, i) + if(ierr == PIO_NOERR) then + num_filtertypes = num_filtertypes + 1 + tmpfiltertypes(num_filtertypes) = i + endif + enddo + allocate(filtertypes(num_filtertypes)) + allocate(filtertype_descs(num_filtertypes)) + + filtertypes = tmpfiltertypes(1:num_filtertypes) + do i=1,num_filtertypes + select case(filtertypes(i)) + case (1) + filtertype_descs(i) = "DEFLATE" + case (2) + filtertype_descs(i) = "SHUFFLE" + case (3) + filtertype_descs(i) = "FLETCHER32" + case (4) + filtertype_descs(i) = "SZIP" + case (5) + filtertype_descs(i) = "NBIT" + case (6) + filtertype_descs(i) = "SCALEOFFSET" + end select + enddo + END SUBROUTINE PIO_TF_Get_nc4_filtertypes +#endif ! Returns a list of undefined netcdf iotypes ! e.g. This list could be used by a test to make sure that PIO ! fails gracefully for undefined types diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/CMakeLists.txt b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/CMakeLists.txt index f2aa696682..dfdbb832ff 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/CMakeLists.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/CMakeLists.txt @@ -9,7 +9,7 @@ include (LibMPI) include_directories("${CMAKE_SOURCE_DIR}/tests/ncint") include_directories("${CMAKE_BINARY_DIR}") -set (my_tests tst_async_multi tst_ncint_async_perf +set (my_tests tst_async_multi tst_ncint_async_perf tst_ncint_open tst_ncint_perf tst_pio_async tst_pio_udf tst_var_compress) # Test Timeout in seconds. @@ -28,4 +28,3 @@ FOREACH(tst ${my_tests}) NUMPROCS 4 TIMEOUT ${DEFAULT_TEST_TIMEOUT}) ENDFOREACH() - diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/ncint.h b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/ncint.h new file mode 100644 index 0000000000..f5fcb4bb2e --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/ncint.h @@ -0,0 +1,86 @@ +#include <pio.h> + +#if NETCDF_VERSION_LE(4,9,1) +// only netcdf4 formats supported +#define NUM_MODES 4 +#elif defined(HAVE_NETCDF_PAR) +#define NUM_MODES 10 +#else +// need to fix this: netcdf4 not available with netcdf serial bld +#define NUM_MODES 6 +#endif + + +#if NUM_MODES==4 + int cmode[NUM_MODES] = {NC_PIO|NC_NETCDF4, + NC_PIO|NC_NETCDF4|NC_MPIIO, + NC_PIO|NC_NETCDF4|NC_CLASSIC_MODEL, + NC_PIO|NC_NETCDF4|NC_MPIIO|NC_CLASSIC_MODEL}; + char mode_name[NUM_MODES][NC_MAX_NAME] = {"netcdf4 serial ", + "netcdf4 parallel ", + "netcdf4 classic serial ", + "netcdf4 classic parallel"}; + int expected_format[NUM_MODES] = {NC_PIO|NC_FORMAT_NETCDF4, + NC_PIO|NC_FORMAT_NETCDF4, + NC_PIO|NC_FORMAT_NETCDF4_CLASSIC, + NC_PIO|NC_FORMAT_NETCDF4_CLASSIC}; +#endif +#if NUM_MODES==6 +int cmode[NUM_MODES] = {NC_PIO, + NC_PIO|NC_64BIT_OFFSET, + NC_PIO|NC_64BIT_DATA, + NC_PIO|NC_PNETCDF, + NC_PIO|NC_PNETCDF|NC_64BIT_OFFSET, + NC_PIO|NC_PNETCDF|NC_64BIT_DATA}; + + + char mode_name[NUM_MODES][NC_MAX_NAME] = {"classic serial ", + "64bit offset serial ", + "64bit data serial ", + "classic pnetcdf ", + "64bit offset pnetcdf ", + "64bit data pnetcdf "}; + + + int expected_format[NUM_MODES] = {NC_PIO|NC_FORMAT_CLASSIC, + NC_PIO|NC_FORMAT_64BIT_OFFSET, + NC_PIO|NC_FORMAT_64BIT_DATA, + NC_PIO|NC_FORMAT_CLASSIC, + NC_PIO|NC_FORMAT_64BIT_OFFSET, + NC_PIO|NC_FORMAT_64BIT_DATA}; +#endif +#if NUM_MODES==10 +int cmode[NUM_MODES] = {NC_PIO, + NC_PIO|NC_64BIT_OFFSET, + NC_PIO|NC_64BIT_DATA, + NC_PIO|NC_PNETCDF, + NC_PIO|NC_PNETCDF|NC_64BIT_OFFSET, + NC_PIO|NC_PNETCDF|NC_64BIT_DATA, + NC_PIO|NC_NETCDF4, + NC_PIO|NC_NETCDF4|NC_CLASSIC_MODEL, + NC_PIO|NC_NETCDF4|NC_MPIIO, + NC_PIO|NC_NETCDF4|NC_MPIIO|NC_CLASSIC_MODEL}; + + char mode_name[NUM_MODES][NC_MAX_NAME] = {"classic serial ", + "64bit offset serial ", + "64bit data serial ", + "classic pnetcdf ", + "64bit offset pnetcdf ", + "64bit data pnetcdf ", + "netcdf4 serial ", + "netcdf4 classic serial ", + "netcdf4 parallel ", + "netcdf4 classic parallel"}; + + + int expected_format[NUM_MODES] = {NC_PIO|NC_FORMAT_CLASSIC, + NC_PIO|NC_FORMAT_64BIT_OFFSET, + NC_PIO|NC_FORMAT_64BIT_DATA, + NC_PIO|NC_FORMAT_CLASSIC, + NC_PIO|NC_FORMAT_64BIT_OFFSET, + NC_PIO|NC_FORMAT_64BIT_DATA, + NC_PIO|NC_FORMAT_NETCDF4, + NC_PIO|NC_FORMAT_NETCDF4_CLASSIC, + NC_PIO|NC_FORMAT_NETCDF4, + NC_PIO|NC_FORMAT_NETCDF4_CLASSIC}; +#endif diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/pio_err_macros.h b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/pio_err_macros.h index 40ef85b4cc..af7a221ab1 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/pio_err_macros.h +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/pio_err_macros.h @@ -73,7 +73,7 @@ static int total_err = 0, err = 0; } while (0) /* This is also defined in tests/cunit/pio_tests.h. It will reduce - * confusion to use the same value. */ + * confusion to use the same value. */ #define ERR_WRONG 1112 #endif /* _PIO_ERR_MACROS_H */ diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_async_multi.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_async_multi.c index 576b05d165..1e5b12dcbf 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_async_multi.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_async_multi.c @@ -42,7 +42,7 @@ create_file(int file_num, int my_rank, int ntasks, int num_io_procs, /* Create a file with a 3D record var. */ sprintf(file_name, "%s_file_%d.nc", TEST_NAME, file_num); - if (nc_create(file_name, NC_PIO|NC_NETCDF4, &ncid)) PERR; + if (nc_create(file_name, NC_PIO, &ncid)) PERR; if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_ncint_async_perf.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_ncint_async_perf.c index 7a8cb5a698..d0404b5f15 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_ncint_async_perf.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_ncint_async_perf.c @@ -11,6 +11,7 @@ #include <pio.h> #include <sys/time.h> #include "pio_err_macros.h" +#include "ncint.h" #define FILE_NAME "tst_ncint_async_perf.nc" #define VAR_NAME "data_var" @@ -24,7 +25,7 @@ #define NDIM2 2 #define NDIM3 3 #define NUM_TIMESTEPS 1 -#define NUM_MODES 4 +//#define NUM_MODES 4 extern NC_Dispatch NCINT_dispatcher; @@ -95,6 +96,7 @@ main(int argc, char **argv) float num_megabytes = DIM_LEN_X * DIM_LEN_Y * sizeof(int) / (float)1000000 * NUM_TIMESTEPS; float delta_in_sec; float mb_per_sec; +/* int cmode[NUM_MODES] = {NC_PIO, NC_PIO|NC_NETCDF4, NC_PIO|NC_NETCDF4|NC_MPIIO, NC_PIO|NC_PNETCDF}; @@ -102,6 +104,7 @@ main(int argc, char **argv) "netCDF-4 sequential ", "netCDF-4 parallel I/O", "pnetcdf "}; +*/ int t, m; /* Print header. */ diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_ncint_open.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_ncint_open.c new file mode 100644 index 0000000000..676b2f3f0a --- /dev/null +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_ncint_open.c @@ -0,0 +1,99 @@ +/* Test openfile function in ncint layer. + + This test simply makes sure that a file created in any mode can be reopened. + + +*/ +#include "config.h" +#include "pio_err_macros.h" +#include "ncint.h" + +#define FILE_NAME "tst_pio_udf_open_" +#define VAR_NAME "data_var" +#define DIM_NAME_UNLIMITED "dim_unlimited" +#define DIM_NAME_X "dim_x" +#define DIM_NAME_Y "dim_y" +#define DIM_LEN_X 4 +#define DIM_LEN_Y 4 +#define NDIM2 2 +#define NDIM3 3 +#define TEST_VAL_42 42 + +extern NC_Dispatch NCINT_dispatcher; + +int +main(int argc, char **argv) +{ + int my_rank; + int ntasks; + char filename[30]; + + /* Initialize MPI. */ + if (MPI_Init(&argc, &argv)) PERR; + + /* Learn my rank and the total number of processors. */ + if (MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)) PERR; + if (MPI_Comm_size(MPI_COMM_WORLD, &ntasks)) PERR; + + if (!my_rank) + printf("\n*** Testing netCDF integration layer.\n"); + + + if (!my_rank) + printf("*** testing simple use of netCDF integration layer format...\n"); + { + int ncid; + int dimid[NDIM3], varid; + int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y}; + int iosysid; + NC_Dispatch *disp_in; + int n, m; + + /* Turn on logging for PIO library. */ + /* PIOc_set_log_level(3); */ + + /* Initialize the intracomm. */ + if (nc_def_iosystem(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) PERR; + + for( m=0; m < NUM_MODES; m++){ + sprintf(filename, "%s%d.nc",FILE_NAME,m); + + /* Create a file with a 3D record var. */ + if(!my_rank) + printf("\ncreate with: cmode = %d name=%s\n", m,mode_name[m]); + if (nc_create(filename, cmode[m], &ncid)) PERR; + if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; + if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; + if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; + if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR; + if (nc_enddef(ncid)) PERR; + + if (nc_close(ncid)) PERR; + + /* Check that our user-defined format has been added. */ + if (nc_inq_user_format(NC_PIO, &disp_in, NULL)) PERR; + if (disp_in != &NCINT_dispatcher) PERR; + + for(n=0; n < NUM_MODES; n++){ + /* Open the file. */ + if(!my_rank) + printf("open %s with: %d, %s\n", filename, cmode[n],mode_name[n] ); + + if (nc_open(filename, cmode[n], &ncid)) PERR; + + /* Close file. */ + if (nc_close(ncid)) PERR; + } + /* Free resources. */ + + /* delete file. */ + PIOc_deletefile(iosysid, filename); + } + if (nc_free_iosystem(iosysid)) PERR; + } + PSUMMARIZE_ERR; + + /* Finalize MPI. */ + MPI_Finalize(); + PFINAL_RESULTS; +} diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_ncint_perf.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_ncint_perf.c index a5833cc0db..0e8a51d66b 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_ncint_perf.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_ncint_perf.c @@ -11,8 +11,9 @@ #include <pio.h> #include <sys/time.h> #include "pio_err_macros.h" +#include "ncint.h" -#define FILE_NAME "tst_ncint_perf.nc" +#define FILE_PREFIX "tst_ncint_perf" #define VAR_NAME "data_var" #define DIM_NAME_UNLIMITED "dim_unlimited" #define DIM_NAME_X "dim_x" @@ -24,7 +25,7 @@ #define NDIM2 2 #define NDIM3 3 #define NUM_TIMESTEPS 1 -#define NUM_MODES 4 + extern NC_Dispatch NCINT_dispatcher; @@ -45,7 +46,7 @@ main(int argc, char **argv) if (MPI_Comm_size(MPI_COMM_WORLD, &ntasks)) PERR; if (!my_rank) - printf("\n*** Testing netCDF integration PIO performance.\n"); + printf("\n*** Testing netCDF integration PIO performance.\n"); if (!my_rank) printf("*** testing simple intercomm use of netCDF integration layer...\n"); { @@ -58,11 +59,11 @@ main(int argc, char **argv) int *my_data; int num_io_procs; int i; - + int found_format; /* Turn on logging for PIO library. */ - /* PIOc_set_log_level(4); */ - /* if (!my_rank) */ - /* nc_set_log_level(3); */ +/* PIOc_set_log_level(4); + if (!my_rank) + nc_set_log_level(3); */ if (ntasks <= 16) num_io_procs = 1; else if (ntasks <= 64) @@ -87,13 +88,6 @@ main(int argc, char **argv) float num_megabytes = DIM_LEN_X * DIM_LEN_Y * sizeof(int) / (float)1000000 * NUM_TIMESTEPS; float delta_in_sec; float mb_per_sec; - int cmode[NUM_MODES] = {NC_PIO, NC_PIO|NC_NETCDF4, - NC_PIO|NC_NETCDF4|NC_MPIIO, - NC_PIO|NC_PNETCDF}; - char mode_name[NUM_MODES][NC_MAX_NAME + 1] = {"classic sequential ", - "netCDF-4 sequential ", - "netCDF-4 parallel I/O", - "pnetcdf "}; int t, m; /* Print header. */ @@ -104,7 +98,13 @@ main(int argc, char **argv) for (m = 0; m < NUM_MODES; m++) { /* Create a file with a 3D record var. */ - if (nc_create(FILE_NAME, cmode[m], &ncid)) PERR; + char filename[strlen(FILE_PREFIX)+16]; + sprintf(filename,"%s%d.nc",FILE_PREFIX,cmode[m]); + /* Turn on logging for PIO library. */ + if(m==9) PIOc_set_log_level(2); +// if (!my_rank) +// nc_set_log_level(2); + if (nc_create(filename, cmode[m], &ncid)) PERR; if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; @@ -139,8 +139,19 @@ main(int argc, char **argv) /* Write some data with distributed arrays. */ for (t = 0; t < NUM_TIMESTEPS; t++) if (nc_put_vard_int(ncid, varid, ioid, t, my_data)) PERR; - if (nc_close(ncid)) PERR; + /* check the file format */ + if (nc_inq_format_extended(ncid, NULL, &found_format)) PERR; + if (found_format != expected_format[m]) { + printf("expected format 0x%x found format 0x%x\n",expected_format[m], + found_format); + PERR; + } + + if (nc_close(ncid)) PERR; +// PIOc_set_log_level(0); +// if (!my_rank) +// nc_set_log_level(0); /* Stop the clock. */ gettimeofday(&endtime, NULL); @@ -154,9 +165,9 @@ main(int argc, char **argv) printf("%s,\t%d,\t%d,\t%d,\t%8.3f,\t%8.1f,\t%8.3f\n", mode_name[m], ntasks, num_io_procs, 1, delta_in_sec, num_megabytes, mb_per_sec); - - free(my_data); - if (nc_free_decomp(ioid)) PERR; + + free(my_data); + if (nc_free_decomp(ioid)) PERR; } /* next mode flag */ } diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_pio_async.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_pio_async.c index 23f09537b6..13d09d9bb5 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_pio_async.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_pio_async.c @@ -9,6 +9,7 @@ #include "config.h" #include <pio.h> #include "pio_err_macros.h" +#include "ncint.h" #define FILE_NAME "tst_pio_async.nc" #define VAR_NAME "data_var" @@ -41,7 +42,7 @@ main(int argc, char **argv) if (!my_rank) printf("\n*** Testing netCDF integration layer.\n"); if (!my_rank) - printf("*** testing simple async use of netCDF integration layer..."); + printf("*** testing simple async use of netCDF integration layer...\n"); { int ncid, ioid; int dimid[NDIM3], varid; @@ -69,8 +70,12 @@ main(int argc, char **argv) if (my_rank) { + int m; /* Create a file with a 3D record var. */ - if (nc_create(FILE_NAME, NC_PIO|NC_NETCDF4, &ncid)) PERR; + for( m=0; m<NUM_MODES; m++){ + if(my_rank==1) + printf(" cmode = %d\n", cmode[m]); + if (nc_create(FILE_NAME, cmode[m], &ncid)) PERR; if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; @@ -113,7 +118,7 @@ main(int argc, char **argv) size_t dim_len_in; /* Open the file. */ - if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR; + if (nc_open(FILE_NAME, cmode[m], &ncid)) PERR; /* Check the file. */ if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) PERR; @@ -148,6 +153,7 @@ main(int argc, char **argv) free(my_data); if (nc_free_decomp(ioid)) PERR; + } if (nc_free_iosystem(iosysid)) PERR; } } diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_pio_udf.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_pio_udf.c index 0dc68e1cae..f61c9d7a2a 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_pio_udf.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_pio_udf.c @@ -9,6 +9,7 @@ #include "config.h" #include "pio_err_macros.h" #include <pio.h> +#include "ncint.h" #define FILE_NAME "tst_pio_udf.nc" #define VAR_NAME "data_var" @@ -50,7 +51,7 @@ main(int argc, char **argv) PSUMMARIZE_ERR; if (!my_rank) - printf("*** testing simple use of netCDF integration layer format..."); + printf("*** testing simple use of netCDF integration layer format...\n"); { int ncid, ioid; int dimid[NDIM3], varid; @@ -61,7 +62,7 @@ main(int argc, char **argv) size_t *compdof; /* The decomposition mapping. */ int *my_data; int *data_in; - int i; + int i, m; /* Turn on logging for PIO library. */ /* PIOc_set_log_level(3); */ @@ -69,56 +70,61 @@ main(int argc, char **argv) /* Initialize the intracomm. */ if (nc_def_iosystem(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) PERR; - /* Create a file with a 3D record var. */ - if (nc_create(FILE_NAME, NC_PIO, &ncid)) PERR; - if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; - if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; - if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; - if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR; - - /* Calculate a decomposition for distributed arrays. */ - elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks; - if (!(compdof = malloc(elements_per_pe * sizeof(size_t)))) + for( m=0; m < NUM_MODES; m++){ + /* Create a file with a 3D record var. */ + if(!my_rank) + printf(" cmode = %d\n", cmode[m]); + if (nc_create(FILE_NAME, cmode[m], &ncid)) PERR; + if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; + if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; + if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; + if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR; + if (nc_enddef(ncid)) PERR; + + /* Calculate a decomposition for distributed arrays. */ + elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks; + if (!(compdof = malloc(elements_per_pe * sizeof(size_t)))) PERR; - for (i = 0; i < elements_per_pe; i++) + for (i = 0; i < elements_per_pe; i++) compdof[i] = my_rank * elements_per_pe + i; - /* Create the PIO decomposition for this test. */ - if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe, - compdof, &ioid, 1, NULL, NULL)) PERR; - free(compdof); + /* Create the PIO decomposition for this test. */ + if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe, + compdof, &ioid, 1, NULL, NULL)) PERR; + free(compdof); - /* Create some data on this processor. */ - if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR; - for (i = 0; i < elements_per_pe; i++) + /* Create some data on this processor. */ + if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR; + for (i = 0; i < elements_per_pe; i++) my_data[i] = my_rank * 10 + i; - /* Write some data with distributed arrays. */ - if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR; - if (nc_close(ncid)) PERR; + /* Write some data with distributed arrays. */ + if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR; + if (nc_close(ncid)) PERR; - /* Check that our user-defined format has been added. */ - if (nc_inq_user_format(NC_PIO, &disp_in, NULL)) PERR; - if (disp_in != &NCINT_dispatcher) PERR; + /* Check that our user-defined format has been added. */ + if (nc_inq_user_format(NC_PIO, &disp_in, NULL)) PERR; + if (disp_in != &NCINT_dispatcher) PERR; - /* Open the file. */ - if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR; + /* Open the file. */ + if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR; - /* Read distributed arrays. */ - if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR; - if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR; + /* Read distributed arrays. */ + if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR; + if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR; - /* Check results. */ - for (i = 0; i < elements_per_pe; i++) + /* Check results. */ + for (i = 0; i < elements_per_pe; i++) if (data_in[i] != my_data[i]) PERR; - /* Close file. */ - if (nc_close(ncid)) PERR; + /* Close file. */ + if (nc_close(ncid)) PERR; - /* Free resources. */ - free(data_in); - free(my_data); - if (nc_free_decomp(ioid)) PERR; + /* Free resources. */ + free(data_in); + free(my_data); + if (nc_free_decomp(ioid)) PERR; + } if (nc_free_iosystem(iosysid)) PERR; } PSUMMARIZE_ERR; diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_var_compress.c b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_var_compress.c index 8e3060a630..28bb420858 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_var_compress.c +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/ncint/tst_var_compress.c @@ -76,10 +76,10 @@ run_var_compress_test(int my_rank, int ntasks, int iosysid) int endian_in; int d; /* int ret; */ - + /* Open the file. */ if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR; - + /* Check the variable deflate. */ /* if ((ret = nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in))) */ /* NCPERR(ret); */ @@ -95,15 +95,15 @@ run_var_compress_test(int my_rank, int ntasks, int iosysid) /* Check the endianness. */ if (nc_inq_var_endian(ncid, 0, &endian_in)) PERR; if (endian_in != NC_ENDIAN_BIG) PERR; - + /* Read distributed arrays. */ if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR; if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR; - + /* Check results. */ for (i = 0; i < elements_per_pe; i++) if (data_in[i] != my_data[i]) PERR; - + /* Close file. */ if (nc_close(ncid)) PERR; @@ -147,7 +147,7 @@ main(int argc, char **argv) /* Free the iosystem. */ if (nc_free_iosystem(iosysid)) PERR; - + PSUMMARIZE_ERR; #endif /* _NETCDF4 */ diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/CMakeLists.txt b/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/CMakeLists.txt index fce7142430..759c4cb368 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/CMakeLists.txt +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/CMakeLists.txt @@ -7,18 +7,27 @@ add_executable (pioperf EXCLUDE_FROM_ALL target_link_libraries (pioperf piof) add_dependencies (tests pioperf) +# Compiler-specific compile options if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") - target_compile_options (pioperf + target_compile_options (piof PRIVATE -ffree-line-length-none) -endif() - -if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") - set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" ) - # target_compile_options (gptl +elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") + set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all " ) + # target_compile_options (piof # PRIVATE -mismatch_all) -endif () +elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray") + set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ef") +elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Intel") + set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -debug minimal") +endif() +if (CMAKE_BUILD_TYPE STREQUAL "DEBUG") + set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g") +endif() if (PIO_HDF5_LOGGING) target_compile_definitions (pioperf PUBLIC LOGGING) endif () +#if (DEFINED ENV{NCAR_SPACK_HASH_LINARO_FORGE}) +# set (CMAKE_EXE_LINKER_FLAGS "-L. -lmap-sampler-pmpi -lmap-sampler -Wl,-rpath /glade/work/jedwards/sandboxes/ParallelIO/bld/tests/performance") +#endif() diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/Pioperformance.md b/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/Pioperformance.md index 186e489e8d..8a70c7ad9a 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/Pioperformance.md +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/Pioperformance.md @@ -27,16 +27,16 @@ decompfile="ROUNDROBIN", or decompfile="BLOCK" They call init_ideal_dof which internally generates a dof as follows: - if(doftype .eq. 'ROUNDROBIN') then - do i=1,varsize - compmap(i) = (i-1)*npe+mype+1 - enddo - else if(doftype .eq. 'BLOCK') then - do i=1,varsize - compmap(i) = (i+varsize*mype) - enddo + if(doftype .eq. 'ROUNDROBIN') then + do i=1,varsize + compmap(i) = (i-1)*npe+mype+1 + enddo + else if(doftype .eq. 'BLOCK') then + do i=1,varsize + compmap(i) = (i+varsize*mype) + enddo endif - + The size of the variable is npes*varsize where varsize can be set in the namelist. varsize is the variable array size per task. You can add variables by changing nvars in the namelist. @@ -59,4 +59,4 @@ When this is run, output like the following will appear: These are read and write rates in units of MB/s for Box and Subset rearrangers - the time measured is from the call to readdof or writedof to the completion of the close (since writes are buffered the -close needs to be included). \ No newline at end of file +close needs to be included). diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/kt.PIO1.perfmakefile b/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/kt.PIO1.perfmakefile index 67f0d3be11..fdb18e2c85 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/kt.PIO1.perfmakefile +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/kt.PIO1.perfmakefile @@ -1,7 +1,7 @@ all: pioperf pioperfp1 pioperf: pioperformance.o - mpif90 pioperformance.o -o pioperf ../pio_build_int/src/flib/libpiof.a ../pio_build_int/src/clib/libpioc.a ../pio_build_int/src/gptl/libgptl.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdff.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdf.so /glade/apps/opt/pnetcdf/1.6.1/intel/15.0.3/lib/libpnetcdf.a -lirng -ldecimal -lcilkrts -lstdc++ + mpif90 pioperformance.o -o pioperf ../pio_build_int/src/flib/libpiof.a ../pio_build_int/src/clib/libpioc.a ../pio_build_int/src/gptl/libgptl.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdff.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdf.so /glade/apps/opt/pnetcdf/1.6.1/intel/15.0.3/lib/libpnetcdf.a -lirng -ldecimal -lcilkrts -lstdc++ pioperformance.o: pioperformance.F90 mpif90 -DCPRINTEL -DHAVE_MPI -DINCLUDE_CMAKE_FCI -DLINUX -DTIMING -DUSEMPIIO -DUSE_PNETCDF_VARN -DUSE_PNETCDF_VARN_ON_READ -D_NETCDF -D_NETCDF4 -D_PNETCDF -I/glade/p/work/katec/pio_work/ncar_pio2/src/flib -I/glade/p/work/katec/pio_work/pio_build_int/src/flib -I/glade/apps/opt/netcdf-mpi/4.3.2/intel/default/include -I/glade/apps/opt/pnetcdf/1.6.1/intel/15.0.3/include -I/glade/p/work/katec/pio_work/ncar_pio2/src/clib -I/glade/p/work/katec/pio_work/ncar_pio2/src/gptl -I/glade/p/work/katec/pio_work/pio_build_int/src/gptl -c pioperformance.F90 -o pioperformance.o @@ -16,4 +16,4 @@ cleanp1: rm pioperformancep1.o pioperfp1 clean: - rm pioperformance.o pioperf \ No newline at end of file + rm pioperformance.o pioperf diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/pioperformance.F90 b/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/pioperformance.F90 index 35266ce1eb..7f09919ed8 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/pioperformance.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/performance/pioperformance.F90 @@ -1,7 +1,7 @@ #include "config.h" -#define VARINT 1 +!#define VARINT 1 !#define VARREAL 1 -!#define VARDOUBLE 1 +#define VARDOUBLE 1 program pioperformance #ifndef NO_MPIMOD @@ -117,6 +117,7 @@ program pioperformance if(nvars(nv)>0) then call pioperformancetest(decompfile(i), piotypes(1:niotypes), mype, npe, & rearrangers, niotasks, nframes, nvars(nv), varsize(vs),unlimdimindof) + if(mype==0) print * ,' complete' endif enddo endif @@ -124,6 +125,7 @@ program pioperformance enddo call t_finalizef() + if(mype==0) print *, ' calling mpi finalize' call MPI_Finalize(ierr) contains @@ -149,7 +151,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & integer(kind=PIO_Offset_kind) :: maplen, gmaplen integer :: ndims integer, pointer :: gdims(:) - character(len=20) :: fname + character(len=24) :: fname type(var_desc_t) :: vari(nvars), varr(nvars), vard(nvars) type(iosystem_desc_t) :: iosystem integer :: stride, n @@ -170,6 +172,10 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & double precision, parameter :: cd0 = 1.0e30 integer :: nvarmult character(len=*), parameter :: rearr_name(2) = (/' BOX','SUBSET'/) + character(len=8) :: date + character(len=10) :: time + type(var_desc_t) :: rundate + logical, save :: firstpass=.true. nullify(compmap) @@ -260,15 +266,25 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & call pio_init(mype, comm, ntasks, 0, stride, PIO_REARR_SUBSET, iosystem) - write(fname, '(a,i1,a,i4.4,a,i1,a)') 'pioperf.',rearr,'-',ntasks,'-',iotype,'.nc' + write(fname, '(a,i1,a,i6.6,a,i1,a)') 'pioperf.',rearr,'-',ntasks,'-',iotype,'.nc' + call PIO_set_hint(iosystem, "nc_var_align_size", "1") + ierr = PIO_CreateFile(iosystem, File, iotype, trim(fname), mode) - call WriteMetadata(File, gdims, vari, varr, vard, unlimdimindof) + call WriteMetadata(File, gdims, vari, varr, vard, unlimdimindof, rundate) call MPI_Barrier(comm,ierr) call t_stampf(wall(1), usr(1), sys(1)) + if(firstpass) then + firstpass = .false. + else + if(mype==0) print *,'Writing rundate to file ',trim(filename) + call date_and_time(DATE=date, TIME=time) + nvarmult= pio_put_var(File, rundate, date//' '//time(1:4)) + endif + if(.not. unlimdimindof) then #ifdef VARINT call PIO_InitDecomp(iosystem, PIO_INT, gdims, compmap, iodesc_i4, rearr=rearr) @@ -328,9 +344,14 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & #endif endif enddo - call pio_closefile(File) - +! if(modulo(mype,128)==0) then +! call PAT_REGION_BEGIN(1,'pio_closefile') +! endif + call pio_closefile(File) +! if(modulo(mype,128)==0) then +! call PAT_REGION_END(1, 'pio_closefile') +! endif call MPI_Barrier(comm,ierr) call t_stampf(wall(2), usr(2), sys(2)) @@ -348,16 +369,17 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & #ifdef VARDOUBLE nvarmult = nvarmult+2 #endif - write(*,'(a15,a9,i10,i10,i10,f20.10)') & + write(*,'(a15,a9,i10,i10,i10,2f20.10)') & 'RESULT: write ',rearr_name(rearr), piotypes(k), ntasks, nvars, & - nvarmult*nvars*nframes*gmaplen*4.0/(1048576.0*wall(2)) + nvarmult*nvars*nframes*gmaplen*4.0/(1048576.0*wall(2)), wall(2) #ifdef BGQTRY call print_memusage() #endif end if ! Now the Read +#ifdef DOREAD - write(fname, '(a,i1,a,i4.4,a,i1,a)') 'pioperf.',rearr,'-',ntasks,'-',iotype,'.nc' + write(fname, '(a,i1,a,i6.6,a,i1,a)') 'pioperf.',rearr,'-',ntasks,'-',iotype,'.nc' ierr = PIO_OpenFile(iosystem, File, iotype, trim(fname), mode=PIO_NOWRITE); do nv=1,nvars @@ -396,7 +418,6 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & #ifdef VARINT call PIO_setframe(File, vari(nv), frame) call pio_read_darray(File, vari(nv), iodesc_i4, ifld_in(:,nv,frame), ierr) - print *,__FILE__,__LINE__,size(ifld_in),ifld_in(1,1,1) #endif #ifdef VARREAL call PIO_setframe(File, varr(nv), frame) @@ -496,6 +517,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & call print_memusage() #endif end if +#endif ! DOREAD #ifdef VARREAL call PIO_freedecomp(iosystem, iodesc_r4) #endif @@ -555,11 +577,11 @@ subroutine init_ideal_dof(doftype, mype, npe, ndims, gdims, compmap, varsize) end subroutine init_ideal_dof - subroutine WriteMetadata(File, gdims, vari, varr, vard,unlimdimindof) + subroutine WriteMetadata(File, gdims, vari, varr, vard,unlimdimindof, rundate) use pio type(file_desc_t) :: File integer, intent(in) :: gdims(:) - type(var_desc_t),intent(out) :: vari(:), varr(:), vard(:) + type(var_desc_t),intent(out) :: vari(:), varr(:), vard(:), rundate logical, intent(in) :: unlimdimindof integer :: ndims character(len=12) :: dimname @@ -576,6 +598,11 @@ subroutine WriteMetadata(File, gdims, vari, varr, vard,unlimdimindof) endif allocate(dimid(ndims+1)) + iostat = PIO_def_dim(File, 'strlen', int(13, pio_offset_kind), dimid(1)) + varname = ' ' + write(varname,'(a7)') 'rundate' + iostat = PIO_def_var(File, trim(varname), PIO_CHAR, dimid(1:1), rundate) + do i=1,ndims write(dimname,'(a,i6.6)') 'dim',i @@ -583,6 +610,7 @@ subroutine WriteMetadata(File, gdims, vari, varr, vard,unlimdimindof) enddo iostat = PIO_def_dim(File, 'time', PIO_UNLIMITED, dimid(ndims+1)) + do nv=1,nvars #ifdef VARINT write(varname,'(a,i4.4)') 'vari',nv @@ -599,10 +627,10 @@ subroutine WriteMetadata(File, gdims, vari, varr, vard,unlimdimindof) iostat = PIO_def_var(File, varname, PIO_DOUBLE, dimid, vard(nv)) iostat = PIO_put_att(File, vard(nv), "_FillValue", PIO_FILL_DOUBLE); #endif - enddo + enddo iostat = PIO_enddef(File) - + end subroutine WriteMetadata diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/Levy_Notes b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/Levy_Notes index 952c7ab0f5..4fc4a627cf 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/Levy_Notes +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/Levy_Notes @@ -27,7 +27,7 @@ Frankfurt --------- "write_mpiio_int after call to file_set_view:MPI_ERR_ARG: invalid argument of so me other kind - pio_support::pio_die:: myrank= -1 : ERROR: iompi_mod.F90: 223 + pio_support::pio_die:: myrank= -1 : ERROR: iompi_mod.F90: 223 : (no message)" Yellowstone @@ -35,7 +35,7 @@ Yellowstone "write_mpiio_int after call to file_set_view:Invalid argument, error stack: MPI_ FILE_SET_VIEW(57): Invalid displacement argument - pio_support::pio_die:: myrank= -1 : ERROR: iompi_mod.F90: 223 + pio_support::pio_die:: myrank= -1 : ERROR: iompi_mod.F90: 223 : (no message)" 5) For tests where we expect failure (e.g. writing a file that was opened nowrite) diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/basic_tests.F90 b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/basic_tests.F90 index 9289e14f0d..50fb578b3a 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/basic_tests.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/basic_tests.F90 @@ -42,7 +42,7 @@ Subroutine test_create(test_id, err_msg) iotype = iotypes(test_id) ! Delete file before initial create -! if (master_task) call system("rm -f " // trim(filename)) +! if (main_task) call system("rm -f " // trim(filename)) call PIO_deletefile(pio_iosystem, filename) @@ -104,7 +104,7 @@ Subroutine test_create(test_id, err_msg) call mpi_barrier(mpi_comm_world,ret_val) ! Recreate file with NOCLOBBER if (is_netcdf(iotype)) then - if(master_task) write(*,"(6x,A,1x)") "trying to create with noclobber, error expected ... " + if(main_task) write(*,"(6x,A,1x)") "trying to create with noclobber, error expected ... " call mpi_barrier(mpi_comm_world,ret_val) ret_val = PIO_createfile(pio_iosystem, pio_file, iotype, filename, PIO_NOCLOBBER) @@ -159,7 +159,7 @@ Subroutine test_open(test_id, err_msg) iotype = iotypes(test_id) ! Open file that doesn't exist - if(master_task) write(*,"(6x,A)") "trying to open nonexistant file error expected ... " + if(main_task) write(*,"(6x,A)") "trying to open nonexistant file error expected ... " call mpi_barrier(MPI_COMM_WORLD,ret_val) ret_val = PIO_openfile(pio_iosystem, pio_file, iotype, "FAKE.FILE", & PIO_nowrite) @@ -256,7 +256,7 @@ Subroutine test_open(test_id, err_msg) end if ! Try to write (should fail) - if(master_task) write(*,"(6x,A)") "trying to write to readonly file, error expected ... " + if(main_task) write(*,"(6x,A)") "trying to write to readonly file, error expected ... " call mpi_barrier(MPI_COMM_WORLD,ret_val) call PIO_write_darray(pio_file, pio_var, iodesc_nCells, data_buffer, ret_val) @@ -304,7 +304,7 @@ Subroutine test_open(test_id, err_msg) ! Try to open standard binary file as netcdf (if iotype = netcdf) if (is_netcdf(iotype)) then - if(master_task) write(*,"(6x,A,1x)") "trying to open non-netcdf file using netcdf, error expected ... " + if(main_task) write(*,"(6x,A,1x)") "trying to open non-netcdf file using netcdf, error expected ... " call mpi_barrier(MPI_COMM_WORLD,ret_val) ret_val = PIO_openfile(pio_iosystem, pio_file, iotype, & diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/driver.F90 b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/driver.F90 index c26a10291b..6632f3f3a8 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/driver.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/driver.F90 @@ -37,9 +37,9 @@ Program pio_unit_test_driver #endif !! call MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN, ierr) - master_task = my_rank.eq.0 + main_task = my_rank.eq.0 - if (master_task) then + if (main_task) then ltest_netcdf = .false. ltest_netcdf4p = .false. ltest_netcdf4c = .false. @@ -132,31 +132,31 @@ Program pio_unit_test_driver ! Make sure i is a valid test number select case (test_id) case (NETCDF4P) - if (master_task) & + if (main_task) & write(*,"(A)") "Testing PIO's netcdf4 parallel input / output:" case (NETCDF4C) - if (master_task) & + if (main_task) & write(*,"(A)") "Testing PIO's netcdf4 compressed input / output:" case (NETCDF) - if (master_task) & + if (main_task) & write(*,"(A)") "Testing PIO's netcdf input / output:" case (PNETCDF) - if (master_task) & + if (main_task) & write(*,"(A)") "Testing PIO's pnetcdf input / output:" case DEFAULT - if (master_task) & + if (main_task) & write(*,"(A,I0)") "Error, not configured for test #", test_id call MPI_Abort(MPI_COMM_WORLD, 0, ierr) end select ! test_create() - if (master_task) write(*,"(3x,A,1x)") "testing PIO_createfile..." + if (main_task) write(*,"(3x,A,1x)") "testing PIO_createfile..." call test_create(test_id, err_msg) call parse(err_msg, fail_cnt) ! test_open() - if (master_task) write(*,"(3x,A,I3)", advance="no") "testing PIO_openfile...",test_id + if (main_task) write(*,"(3x,A,I3)", advance="no") "testing PIO_openfile...",test_id call test_open(test_id, err_msg) call parse(err_msg, fail_cnt) @@ -164,16 +164,16 @@ Program pio_unit_test_driver ! netcdf-specific tests if (is_netcdf(iotypes(test_id))) then - if (master_task) write(*,"(3x,A,1x)", advance="no") "testing PIO_redef..." + if (main_task) write(*,"(3x,A,1x)", advance="no") "testing PIO_redef..." call test_redef(test_id, err_msg) call parse(err_msg, fail_cnt) - if (master_task) write(*,"(3x,A,1x)", advance="no") "testing PIO_enddef..." + if (main_task) write(*,"(3x,A,1x)", advance="no") "testing PIO_enddef..." call test_enddef(test_id, err_msg) print *, 'err_msg =', err_msg, ' fail_cnt = ', fail_cnt call parse(err_msg, fail_cnt) - if (master_task) write(*,"(3x,A,1x)", advance="no") "testing PIO netCDF-4 functions..." + if (main_task) write(*,"(3x,A,1x)", advance="no") "testing PIO netCDF-4 functions..." print *, 'err_msg =', err_msg, ' fail_cnt = ', fail_cnt call test_nc4(test_id, err_msg) print *, 'err_msg =', err_msg, ' fail_cnt = ', fail_cnt @@ -181,13 +181,13 @@ Program pio_unit_test_driver end if - if (master_task) write(*,*) "" + if (main_task) write(*,*) "" end if ! ltest(test_id) end do - if (master_task) then + if (main_task) then write(*,"(A,I0)") "Total failure count: ", fail_cnt if (fail_cnt.eq.0) then write(*,"(A)") "PASSED unit testing." @@ -214,7 +214,7 @@ Subroutine parse(err_msg, fail_counter) integer, intent(inout) :: fail_counter logical :: test_passed - if (master_task) then + if (main_task) then test_passed = (trim(err_msg).eq."no_error") if (test_passed) then write(*,"(A)") "success!" diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/ftst_vars_chunking.F90 b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/ftst_vars_chunking.F90 index af04112d5f..37d4840eb8 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/ftst_vars_chunking.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/ftst_vars_chunking.F90 @@ -9,13 +9,13 @@ program ftst_vars_chunking use mpi use pio use pio_nf - + integer, parameter :: NUM_IOTYPES = 2 integer, parameter :: NDIM2 = 2 - + type(iosystem_desc_t) :: pio_iosystem type(file_desc_t) :: pio_file - type(var_desc_t) :: pio_var + type(var_desc_t) :: pio_var integer :: my_rank, ntasks integer :: niotasks = 1, stride = 1 character(len=64) :: filename = 'ftst_vars_chunking.nc' @@ -28,7 +28,7 @@ program ftst_vars_chunking integer (kind=PIO_OFFSET_KIND) :: chunksizes_in(NDIM2) integer :: iotype(NUM_IOTYPES) = (/ PIO_iotype_netcdf4c, PIO_iotype_netcdf4p /) integer :: iotype_idx, ierr - + ! Set up MPI call MPI_Init(ierr) call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) @@ -44,7 +44,7 @@ program ftst_vars_chunking PIO_rearr_subset, pio_iosystem, base=1) ! Set error handling for test. - call PIO_seterrorhandling(pio_iosystem, PIO_RETURN_ERROR) + call PIO_seterrorhandling(pio_iosystem, PIO_RETURN_ERROR) call PIO_seterrorhandling(PIO_DEFAULT, PIO_RETURN_ERROR) ! Uncomment (and build with --enable-logging) to turn on logging. @@ -52,32 +52,32 @@ program ftst_vars_chunking ! Try this test for NETCDF4C and NETCDF4P. do iotype_idx = 1, NUM_IOTYPES - + ! Create a file. ierr = PIO_createfile(pio_iosystem, pio_file, iotype(iotype_idx), filename) if (ierr .ne. PIO_NOERR) stop 3 - + ! Define dims. ret_val = PIO_def_dim(pio_file, dim_name_1, dim_len1, dimid1) if (ierr .ne. PIO_NOERR) stop 5 ret_val = PIO_def_dim(pio_file, dim_name_2, dim_len2, dimid2) if (ierr .ne. PIO_NOERR) stop 6 - + ! Define a var. ret_val = PIO_def_var(pio_file, var_name, PIO_int, (/dimid1, dimid2/), pio_var) if (ierr .ne. PIO_NOERR) stop 7 - + ! Define chunking for var. ret_val = PIO_def_var_chunking(pio_file, pio_var, 0, (/chunksize1, chunksize2/)) if (ierr .ne. PIO_NOERR) stop 9 - + ! Close the file. call PIO_closefile(pio_file) - + ! Open the file. - ret_val = PIO_openfile(pio_iosystem, pio_file, iotype(iotype_idx), filename, PIO_nowrite) + ret_val = PIO_openfile(pio_iosystem, pio_file, iotype(iotype_idx), filename, PIO_nowrite) if (ierr .ne. PIO_NOERR) stop 23 - + ! Find var chunksizes using varid. ret_val = PIO_inq_var_chunking(pio_file, 1, storage_in, chunksizes_in) if (ierr .ne. PIO_NOERR) stop 25 @@ -88,11 +88,11 @@ program ftst_vars_chunking call PIO_closefile(pio_file) end do ! next IOTYPE - + ! Finalize PIO. call PIO_finalize(pio_iosystem, ierr) - + if (my_rank .eq. 0) print *,'SUCCESS!' -#endif - call MPI_Finalize(ierr) +#endif + call MPI_Finalize(ierr) end program ftst_vars_chunking diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/global_vars.F90 b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/global_vars.F90 index 79318357c6..78c0710892 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/global_vars.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/global_vars.F90 @@ -20,7 +20,7 @@ module global_vars ! MPI Variables integer :: my_rank, ntasks - logical :: master_task + logical :: main_task ! PIO Variables integer :: stride, niotasks diff --git a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/ncdf_tests.F90 b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/ncdf_tests.F90 index b414e26109..5acd092643 100644 --- a/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/ncdf_tests.F90 +++ b/src/Infrastructure/IO/PIO/ParallelIO/tests/unit/ncdf_tests.F90 @@ -127,7 +127,7 @@ Subroutine test_redef(test_id, err_msg) call PIO_closefile(pio_file) ! Try to enter define mode again - if(master_task) write(*,"(6x,A,1x)") "trying to enter define mode in closed file, error expected ... " + if(main_task) write(*,"(6x,A,1x)") "trying to enter define mode in closed file, error expected ... " call mpi_barrier(MPI_COMM_WORLD,ret_val) ret_val = PIO_redef(pio_file) if (ret_val .eq. PIO_NOERR) then @@ -171,7 +171,7 @@ Subroutine test_enddef(test_id, err_msg) end if ! Enter define mode - if(master_task) write(*,"(6x,A,1x)") "trying to enter define mode in read only file, error expected ... " + if(main_task) write(*,"(6x,A,1x)") "trying to enter define mode in read only file, error expected ... " call mpi_barrier(MPI_COMM_WORLD,ret_val) ret_val = PIO_redef(pio_file) if (ret_val .eq. PIO_NOERR) then @@ -212,7 +212,7 @@ Subroutine test_enddef(test_id, err_msg) end if ! Try to end define mode from data mode - if(master_task) write(*,"(6x,A,1x)") "trying to end define mode in data mode, error expected ... " + if(main_task) write(*,"(6x,A,1x)") "trying to end define mode in data mode, error expected ... " call mpi_barrier(MPI_COMM_WORLD,ret_val) ret_val = PIO_enddef(pio_file) if (ret_val .eq. PIO_NOERR) then @@ -227,7 +227,7 @@ Subroutine test_enddef(test_id, err_msg) call PIO_closefile(pio_file) ! Try to end define mode in un-opened file - if(master_task) write(*,"(6x,A,1x)") "trying to end define mode in closed file, error expected ... " + if(main_task) write(*,"(6x,A,1x)") "trying to end define mode in closed file, error expected ... " call mpi_barrier(MPI_COMM_WORLD,ret_val) ret_val = PIO_enddef(pio_file) if (ret_val .eq. PIO_NOERR) then