Bring up-to-date with main

Author: DennisHeimbigner

.github/workflows/run_tests_osx.yml

@@ -7,7 +7,7 @@
 name: Run macOS-based netCDF Tests
 
 
-on: [pull_request, workflow_dispatch]
+on: [pull_request,workflow_dispatch]
 
 jobs:
 

.github/workflows/run_tests_ubuntu.yml

@@ -1,5 +1,5 @@
 ###
-# Build hdf4, hdf5 dependencies and cache them in a combined directory.
+# Build hdf5 dependencies and cache them in a combined directory.
 ###
 
 name: Run Ubuntu/Linux netCDF Tests
@@ -25,7 +25,7 @@ jobs:
         run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen
 
         ###
-        # Installing libhdf4 and libhdf5
+        # Installing libhdf5
         ###
       - name: Cache libhdf5-${{ matrix.hdf5 }}
         id: cache-hdf5
@@ -39,13 +39,15 @@ jobs:
         if: steps.cache-hdf5.outputs.cache-hit != 'true'
         run: |
           set -x
+
           wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2
           tar -jxf hdf-4.2.15.tar.bz2
           pushd hdf-4.2.15
-          ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib
+          ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-hdf4-xdr
           make -j
           make install -j
           popd
+
           wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(echo ${{ matrix.hdf5 }} | cut -d. -f 1,2)/hdf5-${{ matrix.hdf5 }}/src/hdf5-${{ matrix.hdf5 }}.tar.bz2
           tar -jxf hdf5-${{ matrix.hdf5 }}.tar.bz2
           pushd hdf5-${{ matrix.hdf5 }}
@@ -72,7 +74,7 @@ jobs:
         run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev mpich libmpich-dev 
 
         ###
-        # Installing libhdf4 and libhdf5
+        # Installing libhdf5
         ###
       - name: Cache libhdf5-parallel-${{ matrix.hdf5 }}
         id: cache-hdf5
@@ -86,13 +88,15 @@ jobs:
         if: steps.cache-hdf5.outputs.cache-hit != 'true'
         run: |
           set -x
+
           wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2
           tar -jxf hdf-4.2.15.tar.bz2
           pushd hdf-4.2.15
-          CC=mpicc ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-parallel
+          CC=mpicc ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-parallel --enable-hdf4-xdr
           make -j
           make install -j
           popd
+
           wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(echo ${{ matrix.hdf5 }} | cut -d. -f 1,2)/hdf5-${{ matrix.hdf5 }}/src/hdf5-${{ matrix.hdf5 }}.tar.bz2
           tar -jxf hdf5-${{ matrix.hdf5 }}.tar.bz2
           pushd hdf5-${{ matrix.hdf5 }}
@@ -164,7 +168,7 @@ jobs:
 
       - name: Configure
         shell: bash -l {0}
-        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf4 --enable-hdf5 --enable-dap --disable-dap-remote-tests  --enable-doxygen  --enable-external-server-tests
+        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests  --enable-doxygen  --enable-external-server-tests
         if: ${{ success() }}
 
       - name: Look at config.log if error
@@ -240,7 +244,7 @@ jobs:
 
       - name: Configure
         shell: bash -l {0}
-        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc ./configure --enable-hdf4 --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-parallel-tests --enable-pnetcdf
+        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-parallel-tests --enable-pnetcdf
         if: ${{ success() }}
 
       - name: Look at config.log if error
@@ -322,7 +326,7 @@ jobs:
         run: |
           mkdir build
           cd build
-          LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DENABLE_HDF4=TRUE -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=TRUE -DENABLE_HDF5=TRUE -DENABLE_NCZARR=TRUE -D ENABLE_DAP_LONG_TESTS=TRUE
+          LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=TRUE -DENABLE_HDF5=TRUE -DENABLE_NCZARR=TRUE -D ENABLE_DAP_LONG_TESTS=TRUE
 
       - name: Print Summary
         shell: bash -l {0}
@@ -402,7 +406,7 @@ jobs:
         run: |
           mkdir build
           cd build
-          LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_C_COMPILER=mpicc -DENABLE_HDF4=TRUE -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=TRUE -DENABLE_HDF5=TRUE -DENABLE_NCZARR=TRUE -D ENABLE_DAP_LONG_TESTS=TRUE -DENABLE_PNETCDF=TRUE
+          LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_C_COMPILER=mpicc -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=TRUE -DENABLE_HDF5=TRUE -DENABLE_NCZARR=TRUE -D ENABLE_DAP_LONG_TESTS=TRUE -DENABLE_PNETCDF=TRUE
 
       - name: Print Summary
         shell: bash -l {0}
@@ -458,11 +462,9 @@ jobs:
       - run: echo "LDFLAGS=-L${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV
       - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV
       - run: |
-          echo "ENABLE_HDF4=--disable-hdf4" >> $GITHUB_ENV
           echo "ENABLE_HDF5=--disable-hdf5" >> $GITHUB_ENV
         if: matrix.use_nc4 == 'nc3'
       - run: |
-          echo "ENABLE_HDF4=--enable-hdf4" >> $GITHUB_ENV
           echo "ENABLE_HDF5=--enable-hdf5" >> $GITHUB_ENV
         if: matrix.use_nc4 == 'nc4'
       - run: echo "ENABLE_DAP=--disable-dap" >> $GITHUB_ENV
@@ -499,7 +501,7 @@ jobs:
 
       - name: Configure
         shell: bash -l {0}
-        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure ${ENABLE_HDF4} ${ENABLE_HDF5} ${ENABLE_DAP} ${ENABLE_NCZARR}
+        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure ${ENABLE_HDF5} ${ENABLE_DAP} ${ENABLE_NCZARR}
         if: ${{ success() }}
 
       - name: Look at config.log if error
@@ -526,18 +528,6 @@ jobs:
         run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j
         if: ${{ success() }}
 
-      # - name: Make Distcheck
-      #   shell: bash -l {0}
-      #   run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} DISTCHECK_CONFIGURE_FLAGS="${ENABLE_HDF4} ${ENABLE_HDF5} ${ENABLE_DAP} ${ENABLE_NCZARR}" make distcheck
-      #   if: ${{ success() }}
-
-      #- name: Start SSH Debug
-      #  uses: luchihoratiu/debug-via-ssh@main
-      #  with:
-      #    NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }}
-      #    SSH_PASS: ${{ secrets.SSH_PASS }}
-      #  if: ${{ failure() }}
-
   nc-cmake:
 
     needs: [ nc-cmake-tests-oneoff-serial, nc-ac-tests-oneoff-serial, nc-cmake-tests-oneoff-parallel, nc-ac-tests-oneoff-parallel ]
@@ -564,11 +554,9 @@ jobs:
       - run: echo "CMAKE_PREFIX_PATH=${HOME}/environments/${{ matrix.hdf5 }}/" >> $GITHUB_ENV
       - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV
       - run: |
-          echo "ENABLE_HDF4=OFF" >> $GITHUB_ENV
           echo "ENABLE_HDF5=OFF" >> $GITHUB_ENV
         if: matrix.use_nc4 == 'nc3'
       - run: |
-          echo "ENABLE_HDF4=ON" >> $GITHUB_ENV
           echo "ENABLE_HDF5=ON" >> $GITHUB_ENV
         if: matrix.use_nc4 == 'nc4'
       - run: echo "ENABLE_DAP=OFF" >> $GITHUB_ENV
@@ -605,7 +593,7 @@ jobs:
         run: |
           mkdir build
           cd build
-          LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DENABLE_HDF4=${ENABLE_HDF4} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=${ENABLE_DAP} -DENABLE_HDF5=${ENABLE_HDF5} -DENABLE_NCZARR=${ENABLE_NCZARR}
+          LD_LIBRARY_PATH=${LD_LIBRARY_PATH} cmake .. -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} -DENABLE_DAP=${ENABLE_DAP} -DENABLE_HDF5=${ENABLE_HDF5} -DENABLE_NCZARR=${ENABLE_NCZARR}
 
       - name: Print Summary
         shell: bash -l {0}

.github/workflows/run_tests_win_cygwin.yml

@@ -1,6 +1,6 @@
 name: Run Cygwin-based tests
 
-on: [pull_request, workflow_dispatch]
+on: [pull_request,workflow_dispatch]
 
 env:
   SHELLOPTS: igncr
@@ -29,7 +29,7 @@ jobs:
             git automake libtool autoconf2.5 make libhdf5-devel
             libhdf4-devel zipinfo libxml2-devel perl zlib-devel
             libzstd-devel libbz2-devel libaec-devel libzip-devel
-            libdeflate-devel gcc-core
+            libdeflate-devel gcc-core libcurl-devel libiconv-devel
 
       - name: (Autotools) Run autoconf and friends
         run: |

.github/workflows/run_tests_win_mingw.yml

@@ -9,7 +9,7 @@ name: Run MSYS2, MinGW64-based Tests
 env:
   CPPFLAGS: "-D_BSD_SOURCE"
 
-on: [pull_request, workflow_dispatch]
+on: [pull_request,workflow_dispatch]
 
 jobs:
 

CMakeLists.txt

@@ -519,8 +519,8 @@ OPTION(ENABLE_CDF5 "Enable CDF5 support" ON)
 # Netcdf-4 support (i.e. libsrc4) is required by more than just HDF5 (e.g. NCZarr)
 # So depending on what above formats are enabled, enable netcdf-4
 if(ENABLE_HDF5 OR ENABLE_HDF4 OR ENABLE_NCZARR)
-SET(ENABLE_NETCDF_4 ON CACHE BOOL "Enable netCDF-4 API" FORCE)
-SET(ENABLE_NETCDF4 ON CACHE BOOL "Enable netCDF4 Alias" FORCE)
+  SET(ENABLE_NETCDF_4 ON CACHE BOOL "Enable netCDF-4 API" FORCE)
+  SET(ENABLE_NETCDF4 ON CACHE BOOL "Enable netCDF4 Alias" FORCE)
 endif()
 
 IF(ENABLE_HDF4)
@@ -1062,19 +1062,22 @@ ENDIF()
 IF(ENABLE_DAP)
   SET(USE_DAP ON CACHE BOOL "")
   SET(ENABLE_DAP2 ON CACHE BOOL "")
+ 
   IF(ENABLE_HDF5)
+    MESSAGE(STATUS "Enabling DAP4")
     SET(ENABLE_DAP4 ON CACHE BOOL "")
-  ELSE(ENABLE_HDF5)
+  ELSE()
+    MESSAGE(STATUS "Disabling DAP4")
     SET(ENABLE_DAP4 OFF CACHE BOOL "")
-  ENDIF(NOT ENABLE_HDF5)
+  ENDIF(ENABLE_HDF5)
 
 ELSE()
   SET(ENABLE_DAP2 OFF CACHE BOOL "")
   SET(ENABLE_DAP4 OFF CACHE BOOL "")
 ENDIF()
 
 # Option to support byte-range reading of remote datasets
-OPTION(ENABLE_BYTERANGE "Enable byte-range access to remote datasets.." OFF)
+OPTION(ENABLE_BYTERANGE "Enable byte-range access to remote datasets.." ON)
 
 # Check for the math library so it can be explicitly linked.
 IF(NOT WIN32)
@@ -1311,7 +1314,7 @@ ENDIF()
 
 IF(NOT ENABLE_S3_SDK)
   IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS)
-    message(FATAL_ERROR "S3 support library not found; please specify option DENABLE_NCZARR_S3=NO")
+    message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_NCZARR_S3=NO")
     SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "NCZARR S3 support" FORCE)
     SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
   ENDIF()
@@ -2537,6 +2540,7 @@ is_enabled(ENABLE_ZERO_LENGTH_COORD_BOUND RELAX_COORD_BOUND)
 is_enabled(USE_CDF5 HAS_CDF5)
 is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
 is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
+is_enabled(ENABLE_NCZARR_S3 HAS_NCZARR_S3)
 is_enabled(ENABLE_NCZARR HAS_NCZARR)
 is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS)
 is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS)

RELEASE_NOTES.md

@@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.9.1 - T.B.D.
 
-### 4.9.1 - Release Candidate 2 - TBD
+### 4.9.1 - Release Candidate 2 - November 21, 2022
 
 #### Known Issues
 
@@ -20,6 +20,10 @@ This file contains a high-level description of this package's evolution. Release
 * [Bug Fix] Fix a race condition when testing missing filters. See [Github #2557](https://github.com/Unidata/netcdf-c/pull/2557). 
 * [Bug Fix] Make major changes to libdap4 and dap4_test to update the non-remote DAP4 tests. See [Github #2555](https://github.com/Unidata/netcdf-c/pull/2555).
 * [Bug Fix] Fix some race conditions due to use of a common file in multiple shell scripts . See [Github #2552](https://github.com/Unidata/netcdf-c/pull/2552).
+
+
+### 4.9.1 - Release Candidate 1 - October 24, 2022
+
 * [Enhancement][Documentation] Add Plugins Quick Start Guide.  See [GitHub #2524](https://github.com/Unidata/netcdf-c/pull/2524) for more information.
 * [Enhancement] Add new entries in `netcdf_meta.h`, `NC_HAS_BLOSC` and `NC_HAS_BZ2`. See [Github #2511](https://github.com/Unidata/netcdf-c/issues/2511) and [Github #2512](https://github.com/Unidata/netcdf-c/issues/2512) for more information.
 * [Enhancement] Add new options to `nc-config`: `--has-multifilters`, `--has-stdfilters`, `--has-quantize`, `--plugindir`.  See [Github #2509](https://github.com/Unidata/netcdf-c/pull/2509) for more information.

configure.ac

@@ -97,7 +97,7 @@ AC_CONFIG_LINKS([nc_test4/ref_hdf5_compat3.nc:nc_test4/ref_hdf5_compat3.nc])
 AC_CONFIG_LINKS([hdf4_test/ref_chunked.hdf4:hdf4_test/ref_chunked.hdf4])
 AC_CONFIG_LINKS([hdf4_test/ref_contiguous.hdf4:hdf4_test/ref_contiguous.hdf4])
 AM_INIT_AUTOMAKE([foreign dist-zip subdir-objects])
-
+AM_MAINTAINER_MODE()
 # Check for the existence of this file before proceeding.
 AC_CONFIG_SRCDIR([include/netcdf.h])
 
@@ -1278,9 +1278,9 @@ fi
 # Does the user want to allow reading of remote data via range headers?
 AC_MSG_CHECKING([whether byte range support is enabled])
 AC_ARG_ENABLE([byterange],
-              [AS_HELP_STRING([--enable-byterange],
+              [AS_HELP_STRING([--disable-byterange],
                               [allow byte-range I/O])])
-test "x$enable_byterange" = xyes || enable_byterange=no
+test "x$enable_byterange" = xno || enable_byterange=yes
 AC_MSG_RESULT($enable_byterange)
 # Need curl for byte ranges
 if test "x$found_curl" = xno && test "x$enable_byterange" = xyes ; then
@@ -1925,6 +1925,7 @@ AC_SUBST(HAS_ERANGE_FILL,[$enable_erange_fill])
 AC_SUBST(HAS_BYTERANGE,[$enable_byterange])
 AC_SUBST(RELAX_COORD_BOUND,[yes])
 AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters])
+AC_SUBST(HAS_NCZARR_S3,[$enable_nczarr_s3])
 AC_SUBST(HAS_NCZARR,[$enable_nczarr])
 AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
 AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
@@ -2060,12 +2061,12 @@ AX_SET_META([NC_HAS_SZIP],[$enable_hdf5_szip],[yes])
 AX_SET_META([NC_HAS_ZSTD],[$have_zstd],[yes])
 AX_SET_META([NC_HAS_BLOSC],[$have_blosc],[yes])
 AX_SET_META([NC_HAS_BZ2],[$have_bz2],[yes])
+
 # This is the version of the dispatch table. If the dispatch table is
 # changed, this should be incremented, so that user-defined format
 # applications like PIO can determine whether they have an appropriate
 # dispatch table to submit. If this is changed, make sure the value in
 # CMakeLists.txt also changes to match.
-
 AC_SUBST([NC_DISPATCH_VERSION], [5])
 AC_DEFINE_UNQUOTED([NC_DISPATCH_VERSION], [${NC_DISPATCH_VERSION}], [Dispatch table version.])
 

docs/dispatch.md

@@ -504,6 +504,30 @@ The code in *hdf4var.c* does an *nc_get_vara()* on the HDF4 SD
 dataset. This is all that is needed for all the nc_get_* functions to
 work.
 
+# Appendix A. Changing NC_DISPATCH_VERSION
+
+When new entries are added to the *struct NC_Dispatch* type `located in include/netcdf_dispatch.h.in` it is necessary to do two things.
+
+1. Bump the NC_DISPATCH_VERSION number
+2. Modify the existing dispatch tables to include the new entries.
+It if often the case that the new entries do not mean anything for
+a given dispatch table. In that case, the new entries may be set to
+some variant of *NC_RO_XXX* or *NC_NOTNC4_XXX* *NC_NOTNC3_XXX*.
+
+Modifying the dispatch version requires two steps:
+1. Modify the version number in *netcdf-c/configure.ac*, and
+2. Modify the version number in *netcdf-c/CMakeLists.txt*.
+
+The two should agree in value.
+
+### NC_DISPATCH_VERSION Incompatibility
+
+When dynamically adding a dispatch table
+-- in nc_def_user_format (see libdispatch/dfile.c) --
+the version of the new table is compared with that of the built-in
+NC_DISPATCH_VERSION; if they differ, then an error is returned from
+that function.
+
 # Appendix B. Inferring the Dispatch Table
 
 As mentioned above, the dispatch table is inferred using the following