From 7b09290a3abb378db6673322cd7edf5cf3ae2b14 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 14 May 2022 16:05:48 -0600 Subject: [PATCH 01/10] Improve filter installation process to avoid use of an extra shell script re: https://github.com/Unidata/netcdf-c/issues/2338 re: https://github.com/Unidata/netcdf-c/issues/2294 In issue https://github.com/Unidata/netcdf-c/issues/2338, Ed Hartnett suggested a better way to install filters to a user defined location -- for Automake, anyway. This PR implements that suggestion. It turns out to be more complicated than it appears, so there are fair number of changes; mostly to shell scripts. Most of the change is in plugins/Makefile.am. NOTE: this PR still does NOT address the use of HDF5_PLUGIN_PATH as the default; this turns out to be complex when dealing with NCZarr. So this will be addressed in a subsequent post 4.9.0 PR. ## Misc. Changes 1. Record the occurrences of incomplete codecs in libnczarr so that they can be included in _Codecs attribute correctly. This allows users to see what missing filters are referenced in the Zarr file. Primarily affects libnczarr/zfilter.[ch]. Also required creating a new no-effect filter: H5Zunknown.c. 2. Move the unknown filter test to a separate test file. 3. Incorporates PR https://github.com/Unidata/netcdf-c/pull/2343 --- CMakeLists.txt | 18 +- RELEASE_NOTES.md | 2 + configure.ac | 2 +- dap4_test/Makefile.am | 2 +- examples/C/run_filter.sh | 5 +- libnczarr/zclose.c | 2 +- libnczarr/zfilter.c | 266 +++++++++++------- libnczarr/zfilter.h | 4 +- libnczarr/zinternal.c | 2 +- libnczarr/zinternal.h | 1 + libnczarr/zsync.c | 7 +- nc_test/Makefile.am | 4 +- nc_test/run_pnetcdf_tests.sh.in | 1 + nc_test4/CMakeLists.txt | 1 + nc_test4/Makefile.am | 8 +- nc_test4/findplugin.in | 81 ++++-- nc_test4/noop1.cdl | 83 ++++++ nc_test4/tst_filter.sh | 43 --- nc_test4/tst_unknown.sh | 109 +++++++ nc_test4/unknown.cdl | 83 ++++++ ncdap_test/Makefile.am | 3 +- ncdump/Makefile.am | 2 +- ncdump/ncpathcvt.c | 35 ++- ncdump/test_scope.sh | 1 - ncdump/tst_nccopy3.sh | 1 - ncdump/tst_output.sh | 2 +- nctest/compare_test_files.sh | 2 +- nczarr_test/CMakeLists.txt | 10 + nczarr_test/Makefile.am | 27 +- nczarr_test/ref_multi.cdl | 2 +- ...s.zip => ref_power_901_constants_orig.zip} | Bin .../{ref_quotes.zip => ref_quotes_orig.zip} | Bin nczarr_test/run_filter.sh | 40 +-- nczarr_test/run_interop.sh | 8 +- plugins/CMakeLists.txt | 3 + plugins/H5Znoop.c | 2 +- plugins/H5Zunknown.c | 147 ++++++++++ plugins/Makefile.am | 86 ++++-- plugins/NCZtest.c | 179 ++++++++++++ plugins/stdinstall.in | 44 --- test_common.in | 8 +- 41 files changed, 1004 insertions(+), 322 deletions(-) mode change 100644 => 100755 nc_test4/findplugin.in create mode 100644 nc_test4/noop1.cdl create mode 100755 nc_test4/tst_unknown.sh create mode 100644 nc_test4/unknown.cdl rename nczarr_test/{ref_power_901_constants.zip => ref_power_901_constants_orig.zip} (100%) rename nczarr_test/{ref_quotes.zip => ref_quotes_orig.zip} (100%) create mode 100644 plugins/H5Zunknown.c create mode 100644 plugins/NCZtest.c delete mode 100755 plugins/stdinstall.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 846ceb6416..89f1890cdf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1862,10 +1862,10 @@ ENDIF(ENABLE_MMAP) #CHECK_FUNCTION_EXISTS(alloca HAVE_ALLOCA) # Used in the `configure_file` calls below -SET(ISCMAKE "1") +SET(ISCMAKE "yes") IF(MSVC) -SET(ISMSVC ON CACHE BOOL "" FORCE) -SET(REGEDIT ON CACHE BOOL "" FORCE) +SET(ISMSVC yes CACHE BOOL "" FORCE) +SET(REGEDIT yes CACHE BOOL "" FORCE) # Get windows major version and build number EXECUTE_PROCESS(COMMAND "systeminfo" OUTPUT_VARIABLE WININFO) IF(WININFO STREQUAL "") @@ -2509,25 +2509,15 @@ configure_file( SET(EXTRA_DIST ${EXTRA_DIST} ${CMAKE_CURRENT_SOURCE_DIR}/test_common.in) SET(TOPSRCDIR "${CMAKE_CURRENT_SOURCE_DIR}") SET(TOPBUILDDIR "${CMAKE_CURRENT_BINARY_DIR}") -SET(ISMSVC "${MSVC}") configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY_DIR}/test_common.sh @ONLY NEWLINE_STYLE LF) ##### -# Build nc_test4/findplugin.sh +# Build and copy nc_test4/findplugin.sh to various places ##### configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nc_test4/findplugin.sh @ONLY NEWLINE_STYLE LF) - -IF(ENABLE_NCZARR) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nczarr_test/findplugin.sh @ONLY NEWLINE_STYLE LF) -ENDIF() - -IF(ENABLE_PLUGINS) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/plugins/findplugin.sh @ONLY NEWLINE_STYLE LF) -ENDIF() - -IF(ENABLE_EXAMPLES) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF) -ENDIF() IF(ENABLE_TESTS) ##### diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f876461b14..5f11960e0e 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD +* [Enhancement] Improve filter installation process to avoid use of an extra shell script. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* [Bug Fix] Get "make distcheck" to work See [Github #/2343](https://github.com/Unidata/netcdf-c/pull/2343). * [Enhancement] Update the documentation to match the current filter capabilities See [Github #2249](https://github.com/Unidata/netcdf-c/pull/2249). * [Enhancement] Support installation of pre-built standard filters into user-specified location. See [Github #2318](https://github.com/Unidata/netcdf-c/pull/2318). * [Enhancement] Improve filter support. More specifically (1) add nc_inq_filter_avail to check if a filter is available, (2) add the notion of standard filters, (3) cleanup szip support to fix interaction with NCZarr. See [Github #2245](https://github.com/Unidata/netcdf-c/pull/2245). diff --git a/configure.ac b/configure.ac index c825135dc8..dc24644cae 100644 --- a/configure.ac +++ b/configure.ac @@ -1993,10 +1993,10 @@ AC_CONFIG_FILES(nczarr_test/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(plugins/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in) +AC_CONFIG_FILES([nc_test/run_pnetcdf_tests.sh:nc_test/run_pnetcdf_tests.sh.in],[chmod ugo+x nc_test/run_pnetcdf_tests.sh]) AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in) AC_CONFIG_FILES(dap4_test/pingurl4.c:ncdap_test/pingurl.c) AC_CONFIG_FILES([h5_test/run_par_tests.sh], [chmod ugo+x h5_test/run_par_tests.sh]) -AC_CONFIG_FILES([nc_test/run_pnetcdf_tests.sh], [chmod ugo+x nc_test/run_pnetcdf_tests.sh]) AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh]) AC_CONFIG_FILES([nc_perf/run_par_bm_test.sh], [chmod ugo+x nc_perf/run_par_bm_test.sh]) AC_CONFIG_FILES([nc_perf/run_gfs_test.sh], [chmod ugo+x nc_perf/run_gfs_test.sh]) diff --git a/dap4_test/Makefile.am b/dap4_test/Makefile.am index ec56230948..158a1fcbb8 100644 --- a/dap4_test/Makefile.am +++ b/dap4_test/Makefile.am @@ -19,7 +19,7 @@ LDADD = ${top_builddir}/liblib/libnetcdf.la AM_CPPFLAGS += -I$(top_srcdir)/liblib AM_CPPFLAGS += -I$(top_srcdir)/libdap4 AM_CPPFLAGS += -DTOPSRCDIR=${abs_top_srcdir} -AM_CPPFLAGS += -DTOPBINDIR=${abs_top_bindir} +AM_CPPFLAGS += -DTOPBINDIR=${abs_top_builddir} # Set up the tests; do the .sh first, then .c check_PROGRAMS = diff --git a/examples/C/run_filter.sh b/examples/C/run_filter.sh index 90e0935fe9..d87b60813c 100755 --- a/examples/C/run_filter.sh +++ b/examples/C/run_filter.sh @@ -15,11 +15,14 @@ set -e echo "*** running test_filter example..." . ${builddir}/findplugin.sh +echo "findplugin.sh loaded" # Locate the plugin path and the library names; argument order is critical # Find bzip2 and capture findplugin h5bzip2 -BZIP2PATH="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}" +BZIP2LIB="${HDF5_PLUGIN_LIB}" +BZIP2DIR="${HDF5_PLUGIN_DIR}" +BZIP2PATH="${BZIP2DIR}/${BZIP2LIB}" # Verify if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index f287a2f5a0..f1f3354b1d 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -167,7 +167,7 @@ zclose_vars(NC_GRP_INFO_T* grp) #ifdef ENABLE_NCZARR_FILTERS /* Reclaim filters */ if(var->filters != NULL) { - (void)NCZ_filter_freelist(var); + (void)NCZ_filter_freelists(var); } var->filters = NULL; #endif diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 963a977183..90f05b9652 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -65,6 +65,7 @@ /* Hold the loaded filter plugin information */ typedef struct NCZ_Plugin { + int incomplete; struct HDF5API { const H5Z_class2_t* filter; NCPSharedLib* hdf5lib; /* source of the filter */ @@ -134,11 +135,15 @@ typedef struct NCZ_Filter { # define FLAG_CODEC 4 /* If set, then visbile parameters come from an existing codec string */ # define FLAG_HDF5 8 /* If set, => visible parameters came from nc_def_var_filter */ # define FLAG_NEWVISIBLE 16 /* If set, => visible parameters were modified */ +# define FLAG_INCOMPLETE 32 /* If set, => filter has no complete matching plugin */ NCZ_HDF5 hdf5; NCZ_Codec codec; struct NCZ_Plugin* plugin; /**< Implementation of this filter. */ + int chainindex; /* Position in original chain */ } NCZ_Filter; +#define FILTERINCOMPLETE(f) ((f)->flags & FLAG_INCOMPLETE?1:0) + /* WARNING: GLOBAL DATA */ /* TODO: move to common global state */ @@ -280,12 +285,14 @@ static int pluginnamecheck(const char* name); */ int -NCZ_filter_freelist(NC_VAR_INFO_T* var) +NCZ_filter_freelists(NC_VAR_INFO_T* var) { int i, stat=NC_NOERR; - NClist* filters = (NClist*)var->filters; + NClist* filters = NULL; + NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info; ZTRACE(6,"var=%s",var->hdr.name); + filters = (NClist*)var->filters; if(filters == NULL) goto done; /* Free the filter list elements */ for(i=0;ifilters = NULL; + /* Free the incomplete filters */ + filters = (NClist*)zvar->incompletefilters; + if(filters == NULL) goto done; + /* Free the filter list elements */ + for(i=0;iincompletefilters = NULL; done: return ZUNTRACE(stat); } @@ -337,7 +354,7 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t int stat = NC_NOERR; struct NCZ_Filter* fi = NULL; NCZ_Plugin* plugin = NULL; - NCZ_HDF5 hdf5 = hdf5_empty; + NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info; ZTRACE(6,"file=%s var=%s id=%u nparams=%u params=%p",file->hdr.name,var->hdr.name,id,nparams,params); @@ -345,24 +362,16 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t {stat = NC_EINVAL; goto done;} if(var->filters == NULL) var->filters = (void*)nclistnew(); + if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew(); /* Before anything else, find the matching plugin */ if((stat = NCZ_plugin_loaded(id,&plugin))) goto done; - if(plugin == NULL || plugin->codec.codec == NULL) { /* fail */ + if(plugin == NULL) { ZLOG(NCLOGWARN,"no such plugin: %u",(unsigned)id); stat = NC_ENOFILTER; goto done; } - /* Fill in the hdf5 */ - hdf5 = hdf5_empty; - hdf5.id = id; - /* Capture the visible parameters */ - hdf5.visible.nparams = nparams; - if(nparams > 0) { - if((stat = paramclone(nparams,&hdf5.visible.params,params))) goto done; - } - /* Find the NCZ_Filter */ if((stat=NCZ_filter_lookup(var,id,&fi))) goto done; if(fi != NULL) { @@ -373,19 +382,31 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t if((fi = calloc(1,sizeof(struct NCZ_Filter))) == NULL) {stat = NC_ENOMEM; goto done;} fi->plugin = plugin; - nclistpush((NClist*)var->filters, fi); - } + if(plugin->incomplete) { + fi->flags |= (FLAG_INCOMPLETE); + nclistpush((NClist*)zvar->incompletefilters, fi); + } else + nclistpush((NClist*)var->filters, fi); + } - /* (over)write the HDF5 parameters */ - nullfree(fi->hdf5.visible.params); - nullfree(fi->hdf5.working.params); - fi->hdf5.working.nparams = 0; - fi->hdf5.working.params = NULL; - fi->hdf5 = hdf5; - hdf5 = hdf5_empty; - fi->flags |= FLAG_VISIBLE; + if(!FILTERINCOMPLETE(fi)) { + /* (over)write the HDF5 parameters */ + nullfree(fi->hdf5.visible.params); + nullfree(fi->hdf5.working.params); + /* Fill in the hdf5 */ + fi->hdf5 = hdf5_empty; /* struct copy */ + fi->hdf5.id = id; + /* Capture the visible parameters */ + fi->hdf5.visible.nparams = nparams; + if(nparams > 0) { + if((stat = paramclone(nparams,&fi->hdf5.visible.params,params))) goto done; + } + fi->hdf5.working.nparams = 0; + fi->hdf5.working.params = NULL; + fi->flags |= FLAG_VISIBLE; + } - fi = NULL; /* either way,its in the var->filters list */ + fi = NULL; /* either way,its in a filters list */ done: if(fi) NCZ_filter_free(fi); @@ -422,7 +443,6 @@ NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp int i; NClist* flist = (NClist*)var->filters; - ZTRACE(6,"var=%s id=%u",var->hdr.name,id); if(specp) *specp = NULL; @@ -434,7 +454,7 @@ NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp for(i=0;ihdf5.id == id) { + if(spec->hdf5.id == id && !FILTERINCOMPLETE(spec)) { if(specp) *specp = spec; break; } @@ -606,7 +626,7 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids NC_GRP_INFO_T* grp = NULL; NC_VAR_INFO_T* var = NULL; NClist* flist = NULL; - size_t nfilters = 0; + size_t nfilters; ZTRACE(1,"ncid=%d varid=%d",ncid,varid); @@ -624,12 +644,12 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids flist = var->filters; - nfilters = nclistlength(flist); + nfilters = nclistlength(flist); /* including incomplets */ if(nfilters > 0 && ids != NULL) { int k; for(k=0;khdf5.id; + ids[k] = f->hdf5.id; } } if(nfiltersp) *nfiltersp = nfilters; @@ -710,7 +730,7 @@ NCZ_inq_filter_avail(int ncid, unsigned id) if((stat = NCZ_filter_initialize())) goto done; /* Check the available filters list */ if((stat = NCZ_plugin_loaded((int)id, &plug))) goto done; - if(plug == NULL) + if(plug == NULL || plug->incomplete) stat = NC_ENOFILTER; done: return ZUNTRACE(stat); @@ -770,6 +790,9 @@ NCZ_filter_finalize(void) if(nclistlength(default_libs) > 0) { for(i=0;i>> DEBUGL: NCZ_filter_finalize: reclaim default_lib[i]=%p\n",l); +#endif if(l != NULL) (void)ncpsharedlibfree(l); } } @@ -822,7 +845,9 @@ NCZ_applyfilterchain(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NClist* cha /* Make sure all the filters are loaded && setup */ for(i=0;ihdf5.id > 0 && f->plugin != NULL); + assert(f != NULL); + if(FILTERINCOMPLETE(f)) {stat = NC_ENOFILTER; goto done;} + assert(f->hdf5.id > 0 && f->plugin != NULL); if(!(f->flags & FLAG_WORKING)) {/* working not yet available */ if((stat = ensure_working(var,f))) goto done; } @@ -938,10 +963,9 @@ NCZ_filter_jsonize(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Fil return ZUNTRACEX(stat,"codec=%s",NULLIFY(filter->codec.codec)); } - /* Build filter from parsed Zarr metadata */ int -NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* jfilter) +NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex) { int i,stat = NC_NOERR; NCZ_Filter* filter = NULL; @@ -949,10 +973,12 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j NCZ_Plugin* plugin = NULL; NCZ_Codec codec = codec_empty; NCZ_HDF5 hdf5 = hdf5_empty; + NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info; ZTRACE(6,"file=%s var=%s jfilter=%s",file->hdr.name,var->hdr.name,NCJtrace(jfilter)); if(var->filters == NULL) var->filters = nclistnew(); + if(zvar->incompletefilters == NULL) zvar->incompletefilters = nclistnew(); /* Get the id of this codec filter */ if(NCJdictget(jfilter,"id",&jvalue)<0) {stat = NC_EFILTER; goto done;} @@ -973,6 +999,10 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j {plugin = loaded_plugins[i]; break;} } + /* Will always have a filter; possibly unknown */ + if((filter = calloc(1,sizeof(NCZ_Filter)))==NULL) {stat = NC_ENOMEM; goto done;} + filter->chainindex = chainindex; + if(plugin != NULL) { /* Save the hdf5 id */ hdf5.id = plugin->codec.codec->hdf5id; @@ -984,13 +1014,16 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j #endif if(stat) goto done; } - - if((filter = calloc(1,sizeof(NCZ_Filter)))==NULL) {stat = NC_ENOMEM; goto done;} filter->flags |= FLAG_VISIBLE; filter->hdf5 = hdf5; hdf5 = hdf5_empty; filter->codec = codec; codec = codec_empty; filter->flags |= FLAG_CODEC; filter->plugin = plugin; plugin = NULL; + } else { + /* Create a fake filter so we do not forget about this codec */ + filter->hdf5 = hdf5_empty; + filter->codec = codec; codec = codec_empty; + filter->flags |= (FLAG_INCOMPLETE|FLAG_CODEC); } if(filter != NULL) { @@ -1199,20 +1232,19 @@ NCZ_load_all_plugins(void) } } - /* Expunge all plugins for which we do not have both HDF5 and codec */ + /* Mark all plugins for which we do not have both HDF5 and codec */ { int i; NCZ_Plugin* p; for(i=0;ihdf5.filter == NULL || p->codec.codec == NULL) { + /* mark this entry as incomplete */ + p->incomplete = 1; #ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: Incomplete plugin: expunged: id=%u; reasons: %s %s\n",i, + fprintf(stderr,">>> DEBUGL: Incomplete plugin: id=%u; reasons: %s %s\n",i, (p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":"")); #endif - /* expunge this entry */ - (void)NCZ_unload_plugin(p); - loaded_plugins[i] = NULL; } #ifdef DEBUGL else @@ -1227,6 +1259,7 @@ NCZ_load_all_plugins(void) NCZ_Plugin* p; for(i=0;iincomplete) continue; if(p->hdf5.filter != NULL && p->codec.codec != NULL) { if(p->codec.codec && p->codec.codec->NCZ_codec_initialize) p->codec.codec->NCZ_codec_initialize(); @@ -1347,12 +1380,38 @@ NCZ_load_plugin_dir(const char* path) return ZUNTRACE(stat); } +int +loadcodecdefaults(const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp) +{ + int stat = NC_NOERR; + int lib_used = 0; + + nclistpush(default_libs,lib); + for(;*cp;cp++) { + struct CodecAPI* c0; +#ifdef DEBUGL + fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id); +#endif + c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI)); + if(c0 == NULL) {stat = NC_ENOMEM; goto done;} + c0->codec = *cp; + c0->codeclib = lib; + lib_used = 1; /* remember */ + nclistpush(codec_defaults,c0); c0 = NULL; + } +done: + if(lib_usedp) *lib_usedp = lib_used; + return stat; +} + static int NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) { int stat = NC_NOERR; NCZ_Plugin* plugin = NULL; const H5Z_class2_t* h5class = NULL; + H5PL_type_t h5type = 0; + const NCZ_codec_t** cp = NULL; const NCZ_codec_t* codec = NULL; NCPSharedLib* lib = NULL; int flags = NCP_GLOBAL; @@ -1374,7 +1433,7 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) if((stat = ncpload(lib,path,flags))) goto done; #ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s\n",path); + fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s lib=%p\n",path,lib); #endif /* See what we have */ @@ -1384,49 +1443,27 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info"); const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults"); - if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL) + if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL) {stat = NC_ENOFILTER; goto done;} + /* We can have cpd or we can have (gpt && gpi && npi) but not both sets */ if(cpd != NULL) { - /* Deal with defaults first */ - const NCZ_codec_t** cp = NULL; - nclistpush(default_libs,lib); cp = (const NCZ_codec_t**)cpd(); -#ifdef DEBUGL - fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp); -#endif - for(;*cp;cp++) { - struct CodecAPI* c0; -#ifdef DEBUGL - fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id); -#endif - c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI)); - if(c0 == NULL) {stat = NC_ENOMEM; goto done1;} - c0->codec = *cp; - c0->codeclib = lib; - nclistpush(codec_defaults,c0); c0 = NULL; + } else {/* cpd => !gpt && !gpi && !npi */ + if(gpt != NULL && gpi != NULL) { /* get HDF5 info */ + h5type = gpt(); + h5class = gpi(); + /* Verify */ + if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;} + if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;} + } + if(npi != NULL) {/* get Codec info */ + codec = npi(); + /* Verify */ + if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;} + if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;} } -done1: - lib = NULL; - goto done; - } - - if(gpt != NULL && gpi != NULL) { - /* get HDF5 info */ - H5PL_type_t h5type = gpt(); - h5class = gpi(); - /* Verify */ - if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;} - if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;} - } - - if(npi != NULL) { - /* get Codec info */ - codec = npi(); - /* Verify */ - if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;} - if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;} - } + } } #ifdef DEBUGL @@ -1436,6 +1473,17 @@ if(codec) fprintf(stderr,">>> %u/%s",codec->hdf5id,codec->codecid); fprintf(stderr,">>> \n"); #endif + /* Handle defaults separately */ + if(cp != NULL) { + int used = 0; +#ifdef DEBUGL + fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp); +#endif + if((stat = loadcodecdefaults(path,cp,lib,&used))) goto done; + if(used) lib = NULL; + goto done; + } + if(h5class != NULL && codec != NULL) { /* Verify consistency of the HDF5 and the Codec */ if(h5class->id != codec->hdf5id) goto done; /* ignore */ @@ -1461,33 +1509,29 @@ fprintf(stderr,">>> \n"); } /* Fill in the plugin */ - if(plugin->hdf5.filter == NULL) { + if(h5class != NULL && plugin->hdf5.filter == NULL) { plugin->hdf5.filter = h5class; plugin->hdf5.hdf5lib = lib; lib = NULL; } - if(plugin->codec.codec == NULL) { + if(codec != NULL && plugin->codec.codec == NULL) { plugin->codec.codec = codec; plugin->codec.codeclib = lib; lib = NULL; } - #ifdef DEBUGL - if(plugin) + if(plugin) fprintf(stderr,">>> DEBUGL: load_plugin: %s\n",printplugin(plugin)); #endif - - /* Cleanup */ - if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) - plugin->codec.codeclib = NULL; + if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) /* Works for NULL case also */ + plugin->codec.codeclib = NULL; if((stat=NCZ_plugin_save(h5id,plugin))) goto done; plugin = NULL; done: - if(lib) { - (void)ncpsharedlibfree(lib); - } + if(lib) + (void)ncpsharedlibfree(lib); if(plugin) NCZ_unload_plugin(plugin); return ZUNTRACEX(stat,"plug=%p",*plugp); } @@ -1541,7 +1585,6 @@ pluginnamecheck(const char* name) } #endif - /**************************************************/ /* _Codecs attribute */ @@ -1552,23 +1595,51 @@ NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data) size_t len; char* contents = NULL; NCbytes* buf = NULL; + NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info; NClist* filters = (NClist*)var->filters; - + NClist* incfilters = (NClist*)zvar->incompletefilters; + NCZ_Filter** chain = NULL; + size_t nfilters; + ZTRACE(6,"var=%s",var->hdr.name); - if(nclistlength(filters) == 0) {stat = NC_ENOTATT; goto done;} + + nfilters = nclistlength(filters) + nclistlength(incfilters); + + if(nfilters == 0) + {stat = NC_ENOTATT; goto done;} + + /* Collect all filters in chain order */ + chain = (NCZ_Filter**)calloc(sizeof(NCZ_Filter*),nfilters); + if(chain == NULL) {stat = NC_ENOMEM; goto done;} + + /* Sort the complete and incomplete filters in order in the chain */ + for(i=0;ichainindex] == NULL); + chain[f->chainindex] = f; + } + for(i=0;ichainindex] == NULL); + chain[f->chainindex] = f; + } + + /* Now construct the attribute */ buf = ncbytesnew(); ncbytessetalloc(buf,1024); ncbytescat(buf,"["); - for(i=0;i 0) ncbytescat(buf,","); ncbytescat(buf,spec->codec.codec); } ncbytescat(buf,"]"); + len = ncbyteslength(buf); contents = nclistcontents(buf); if(lenp) *lenp = len; if(data) strncpy((char*)data,contents,len+1); done: + nullfree(chain); ncbytesfree(buf); return ZUNTRACEX(stat,"len=%u data=%p",(unsigned)len,data); } @@ -1577,6 +1648,7 @@ static int ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter) { int stat = NC_NOERR; + if(FILTERINCOMPLETE(filter)) {stat = NC_ENOFILTER; goto done;} if(!(filter->flags & FLAG_WORKING)) { const size_t oldnparams = filter->hdf5.visible.nparams; const unsigned* oldparams = filter->hdf5.visible.params; @@ -1683,7 +1755,9 @@ NCZ_filter_setup(NC_VAR_INFO_T* var) filters = (NClist*)var->filters; for(i=0;iplugin != NULL); + assert(filter != NULL); + if(FILTERINCOMPLETE(filter)) continue; /* ignore these */ + assert(filter->plugin != NULL); assert((filter->flags & FLAG_VISIBLE)); /* Assume visible params are defined */ /* verify */ assert(filter->hdf5.id > 0 && (filter->hdf5.visible.nparams == 0 || filter->hdf5.visible.params != NULL)); diff --git a/libnczarr/zfilter.h b/libnczarr/zfilter.h index 38d8b0c3d8..6df40789bc 100644 --- a/libnczarr/zfilter.h +++ b/libnczarr/zfilter.h @@ -32,11 +32,11 @@ int NCZ_filter_initialize(void); int NCZ_filter_finalize(void); int NCZ_addfilter(NC_FILE_INFO_T*, NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params); int NCZ_filter_setup(NC_VAR_INFO_T* var); -int NCZ_filter_freelist(NC_VAR_INFO_T* var); +int NCZ_filter_freelists(NC_VAR_INFO_T* var); int NCZ_codec_freelist(NCZ_VAR_INFO_T* zvar); int NCZ_applyfilterchain(const NC_FILE_INFO_T*, NC_VAR_INFO_T*, NClist* chain, size_t insize, void* indata, size_t* outlen, void** outdata, int encode); int NCZ_filter_jsonize(const NC_FILE_INFO_T*, const NC_VAR_INFO_T*, struct NCZ_Filter* filter, struct NCjson**); -int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter); +int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex); int NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data); #endif /*ZFILTER_H*/ diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 018842d094..ba1b907c4c 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -358,7 +358,7 @@ close_vars(NC_GRP_INFO_T *grp) /* Reclaim filters */ if(var->filters != NULL) { - (void)NCZ_filter_freelist(var); + (void)NCZ_filter_freelists(var); } var->filters = NULL; diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 3bc4b6b131..0e3cec55a4 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -185,6 +185,7 @@ typedef struct NCZ_VAR_INFO { struct NCZChunkCache* cache; struct NClist* xarray; /* names from _ARRAY_DIMENSIONS */ char dimension_separator; /* '.' | '/' */ + NClist* incompletefilters; } NCZ_VAR_INFO_T; /* Struct to hold ZARR-specific info for a field. */ diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index 7711032499..65c8b16751 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -1446,6 +1446,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) NClist* dimnames = nclistnew(); #ifdef ENABLE_NCZARR_FILTERS NCjson* jfilter = NULL; + int chainindex; #endif zinfo = file->format_file_info; @@ -1614,8 +1615,10 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) /* Do filters key before compressor key so final filter chain is in correct order */ { if(var->filters == NULL) var->filters = (void*)nclistnew(); + if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew(); #ifdef ENABLE_NCZARR_FILTERS { int k; + chainindex = 0; /* track location of filter in the chain */ if((stat = NCZ_filter_initialize())) goto done; if((stat = NCJdictget(jvar,"filters",&jvalue))) goto done; if(jvalue != NULL && NCJsort(jvalue) != NCJ_NULL) { @@ -1625,7 +1628,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) jfilter = NCJith(jvalue,k); if(jfilter == NULL) break; /* done */ if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} - if((stat = NCZ_filter_build(file,var,jfilter))) goto done; + if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done; } } } @@ -1642,7 +1645,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if((stat = NCJdictget(jvar,"compressor",&jfilter))) goto done; if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) { if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} - if((stat = NCZ_filter_build(file,var,jfilter))) goto done; + if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done; } #endif } diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am index b33c341e29..39ef40596f 100644 --- a/nc_test/Makefile.am +++ b/nc_test/Makefile.am @@ -16,7 +16,7 @@ include $(top_srcdir)/lib_flags.am AM_CPPFLAGS += -I$(top_srcdir)/libsrc AM_CPPFLAGS += -DTOPSRCDIR=${abs_top_srcdir} -AM_CPPFLAGS += -DTOPBINDIR=${abs_top_bindir} +AM_CPPFLAGS += -DTOPBINDIR=${abs_top_builddir} LDADD = ${top_builddir}/liblib/libnetcdf.la AM_CPPFLAGS += -I$(top_builddir)/liblib -I$(top_builddir)/include -I$(top_srcdir)/libsrc @@ -107,7 +107,7 @@ endif # The .c files that are generated with m4 are already distributed, but # we also include the original m4 files, plus test scripts data. EXTRA_DIST = test_get.m4 test_put.m4 run_diskless.sh run_diskless2.sh \ -run_diskless5.sh run_mmap.sh run_pnetcdf_test.sh test_read.m4 \ +run_diskless5.sh run_mmap.sh test_read.m4 \ test_write.m4 ref_tst_diskless2.cdl tst_diskless5.cdl \ ref_tst_diskless3_create.cdl ref_tst_diskless3_open.cdl \ run_inmemory.sh run_mmap.sh f03tst_open_mem.nc test_byterange.sh \ diff --git a/nc_test/run_pnetcdf_tests.sh.in b/nc_test/run_pnetcdf_tests.sh.in index b76a6cf0de..9ddb532afe 100755 --- a/nc_test/run_pnetcdf_tests.sh.in +++ b/nc_test/run_pnetcdf_tests.sh.in @@ -3,6 +3,7 @@ # This script runs some PnetCDF I/O tests set -e + echo echo "Testing file created with PnetCDF is modifiable with netCDF..." ./tst_pnetcdf diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index 6c9c8ffc49..f3bf4aef59 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -50,6 +50,7 @@ IF(USE_HDF5 AND ENABLE_FILTER_TESTING) build_bin_test(tst_filter_avail) build_bin_test(test_filter_vlen) ADD_SH_TEST(nc_test4 tst_filter) + ADD_SH_TEST(nc_test4 tst_unknown) ADD_SH_TEST(nc_test4 tst_specific_filters) IF(ENABLE_CLIENTSIDE_FILTERS) add_bin_test(nc_test4 test_filter_reg) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 9524339f3e..ef5cdb4d39 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -83,6 +83,7 @@ extradir = check_PROGRAMS += test_filter test_filter_misc test_filter_order test_filter_repeat test_filter_vlen check_PROGRAMS += tst_multifilter tst_filter_avail TESTS += tst_filter.sh +TESTS += tst_unknown.sh TESTS += tst_specific_filters.sh endif endif # USE_HDF5 @@ -109,8 +110,8 @@ ref_filteredvv.cdl ref_multi.cdl \ ref_ncgenF.cdl ref_nccopyF.cdl \ ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \ ref_filter_order_create.txt ref_filter_order_read.txt \ -ref_any.cdl tst_specific_filters.sh \ -tst_virtual_datasets.c +ref_any.cdl tst_specific_filters.sh tst_unknown.sh \ +tst_virtual_datasets.c noop1.cdl unknown.cdl # The tst_filterinstall test can only be run after an install # occurred with --with-plugin-dir enabled. So there is no point @@ -128,7 +129,8 @@ perftest.txt bigmeta.nc bigvars.nc *.gz MSGCPP_*.nc \ floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \ testfilter_reg.nc filterrepeat.txt tmp_fillonly.nc \ testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1 \ -tmp_*.txt tmp_*.nc tmp*.dump tmp*.cdl tmp*.txt tmp*.tmp tmp_bzip2.c bzip2.nc +tmp_*.txt tmp_*.nc tmp*.dump tmp*.cdl tmp*.txt tmp*.tmp \ +tmp_bzip2.c bzip2.nc noop.nc tmp_*.dmp DISTCLEANFILES = findplugin.sh run_par_test.sh diff --git a/nc_test4/findplugin.in b/nc_test4/findplugin.in old mode 100644 new mode 100755 index a8f2a4e3b8..423221b82b --- a/nc_test4/findplugin.in +++ b/nc_test4/findplugin.in @@ -20,27 +20,43 @@ # is obviated by setting the LIBRARY_OUTPUT_DIRECTORY # variables: see hdf5plugins/CMakeLists.txt -findplugin() { +# Define location of execution +TOPSRCDIR='@abs_top_srcdir@' +TOPBUILDDIR='@abs_top_builddir@' -FP_NAME="$1" +# Need info from test_common +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ${TOPBUILDDIR}/test_common.sh -FP_PLUGIN_LIB= +findpluginext() { + # Infer the expected plugin shared library extension + # Note: will fail if used before plugins is built + # Also assumes that misc filter is always built + # Approach is to use find to see what is in plugins directory. + TSO=`find ${TOPBUILDDIR}/plugins -name '*misc.so'` + TDY=`find ${TOPBUILDDIR}/plugins -name '*misc.0.dylib'` + TCYG=`find ${TOPBUILDDIR}/plugins -name 'cyg*misc.dll'` + TDLL=`find ${TOPBUILDDIR}/plugins -name '*misc.dll'` + if test "x$TSO" != x ; then + FP_PLUGIN_EXT="so" + FP_PLUGIN_PRE="lib" + elif test "x$TDY" != x ; then + FP_PLUGIN_EXT="0.dylib" + FP_PLUGIN_PRE="lib" + elif test "x$TCYG" != x ; then + FP_PLUGIN_EXT="dll" + FP_PLUGIN_PRE="cyg" + elif test "x$TDLL" != x ; then + FP_PLUGIN_EXT="dll" + FP_PLUGIN_PRE="" + else # unknown + unset FP_PLUGIN_EXT + unset FP_PLUGIN_PRE + fi +} + +findplugindir() { FP_PLUGIN_DIR= - -# Figure out the plugin file name -# Test for visual studio before cygwin since both might be true -if test "x$FP_ISMSVC" != x ; then - FP_PLUGIN_LIB="${FP_NAME}.dll" -elif test "x$FP_ISMINGW" != x || test "x$FP_ISMSYS" != x ; then - FP_PLUGIN_LIB="${FP_NAME}.dll" -elif test "x$FP_ISCYGWIN" != x ; then - FP_PLUGIN_LIB="cyg${FP_NAME}.dll" -elif test "x$FP_ISOSX" != x ; then - FP_PLUGIN_LIB="lib${FP_NAME}.so" # Should this include the version number in the name? -else # Presumably some form on *nix" - FP_PLUGIN_LIB="lib${FP_NAME}.so" -fi - # Figure out the path to where the lib is stored # This can probably be simplified @@ -52,15 +68,15 @@ cd ${CURWD} # Case 1: Cmake with Visual Studio if test "x$FP_ISCMAKE" != x -a "x${FP_ISMSVC}" != x ; then # Case 1a: ignore the build type directory - if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then + if test -e "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then FP_PLUGIN_DIR="${FP_PLUGINS}" fi else # Case 2: automake # Case 2a: look in .libs - if test -f "${FP_PLUGINS}/.libs/${FP_PLUGIN_LIB}" ; then + if test -e "${FP_PLUGINS}/.libs" ; then FP_PLUGIN_DIR="${FP_PLUGINS}/.libs" else # Case 2: look in FP_PLUGINS directly - if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then + if test -e "${FP_PLUGINS}" ; then FP_PLUGIN_DIR="${FP_PLUGINS}" fi fi @@ -71,16 +87,35 @@ if test "x$FP_PLUGIN_DIR" = x ; then echo "***Fail: Could not locate a usable HDF5_PLUGIN_DIR" return 1 fi + +# Make local path +FP_PLUGIN_DIR=`${NCPATHCVT} -F $FP_PLUGIN_DIR` +HDF5_PLUGIN_DIR="$FP_PLUGIN_DIR" +} + +findplugin() { + +FP_NAME="$1" + +FP_PLUGIN_LIB= + +# Figure out the plugin file name +FP_PLUGIN_LIB="${FP_PLUGIN_PRE}${FP_NAME}.${FP_PLUGIN_EXT}" + +# Verify if ! test -f "$FP_PLUGIN_DIR/$FP_PLUGIN_LIB" ; then echo "***Fail: Could not locate a usable HDF5_PLUGIN_LIB" return 1 fi -FP_PLUGIN_DIR=`${NCPATHCVT} $FP_PLUGIN_DIR` - # Set the final output variables HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB" HDF5_PLUGIN_DIR="$FP_PLUGIN_DIR" return 0 } + +# init +unset HDF5_PLUGIN_DIR +findpluginext +findplugindir diff --git a/nc_test4/noop1.cdl b/nc_test4/noop1.cdl new file mode 100644 index 0000000000..b36b946516 --- /dev/null +++ b/nc_test4/noop1.cdl @@ -0,0 +1,83 @@ +netcdf bzip2 { +dimensions: + dim0 = 4 ; + dim1 = 4 ; + dim2 = 4 ; + dim3 = 4 ; +variables: + float var(dim0, dim1, dim2, dim3) ; + var:_Storage = "chunked" ; + var:_ChunkSizes = 4, 4, 4, 4 ; + var:_Filter = "40001,17" ; + var:_NoFill = "true" ; + +// global attributes: + :_Format = "netCDF-4" ; +data: + + var = + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, + 28, 29, 30, 31, + 32, 33, 34, 35, + 36, 37, 38, 39, + 40, 41, 42, 43, + 44, 45, 46, 47, + 48, 49, 50, 51, + 52, 53, 54, 55, + 56, 57, 58, 59, + 60, 61, 62, 63, + 64, 65, 66, 67, + 68, 69, 70, 71, + 72, 73, 74, 75, + 76, 77, 78, 79, + 80, 81, 82, 83, + 84, 85, 86, 87, + 88, 89, 90, 91, + 92, 93, 94, 95, + 96, 97, 98, 99, + 100, 101, 102, 103, + 104, 105, 106, 107, + 108, 109, 110, 111, + 112, 113, 114, 115, + 116, 117, 118, 119, + 120, 121, 122, 123, + 124, 125, 126, 127, + 128, 129, 130, 131, + 132, 133, 134, 135, + 136, 137, 138, 139, + 140, 141, 142, 143, + 144, 145, 146, 147, + 148, 149, 150, 151, + 152, 153, 154, 155, + 156, 157, 158, 159, + 160, 161, 162, 163, + 164, 165, 166, 167, + 168, 169, 170, 171, + 172, 173, 174, 175, + 176, 177, 178, 179, + 180, 181, 182, 183, + 184, 185, 186, 187, + 188, 189, 190, 191, + 192, 193, 194, 195, + 196, 197, 198, 199, + 200, 201, 202, 203, + 204, 205, 206, 207, + 208, 209, 210, 211, + 212, 213, 214, 215, + 216, 217, 218, 219, + 220, 221, 222, 223, + 224, 225, 226, 227, + 228, 229, 230, 231, + 232, 233, 234, 235, + 236, 237, 238, 239, + 240, 241, 242, 243, + 244, 245, 246, 247, + 248, 249, 250, 251, + 252, 253, 254, 255 ; +} diff --git a/nc_test4/tst_filter.sh b/nc_test4/tst_filter.sh index b478079092..e777956d0b 100755 --- a/nc_test4/tst_filter.sh +++ b/nc_test4/tst_filter.sh @@ -9,7 +9,6 @@ set -e API=1 NG=1 NCP=1 -UNK=1 NGC=1 MISC=1 MULTI=1 @@ -41,18 +40,6 @@ trimleft() { sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 } -# Hide/unhide the noop filter -hidenoop() { - rm -fr ${HDF5_PLUGIN_DIR}/save - mkdir ${HDF5_PLUGIN_DIR}/save - mv ${NOOPDIR} ${HDF5_PLUGIN_DIR}/save -} - -unhidenoop() { - mv ${HDF5_PLUGIN_DIR}/save/${NOOPLIB} ${HDF5_PLUGIN_DIR} - rm -fr ${HDF5_PLUGIN_DIR}/save -} - # Locate the plugin dir and the library names; argument order is critical # Find bzip2 and capture findplugin h5bzip2 @@ -180,36 +167,6 @@ echo " *** Pass: -F var,none" echo "*** Pass: all nccopy filter tests" fi -if test "x$UNK" = x1 ; then -echo "*** Testing access to filter info when filter dll is not available" -rm -f noop.nc ./tmp_filter.txt -# xfail build noop.nc -hidenoop -if ${NCGEN} -lb -4 -o noop.nc ${srcdir}/noop.cdl ; then - echo "*** FAIL: ncgen" -else - echo "*** XFAIL: ncgen" -fi -unhidenoop -# build noop.nc -${NCGEN} -lb -4 -o noop.nc ${srcdir}/noop.cdl -# Now hide the filter code -hidenoop -rm -f ./tmp_filter.txt -# This will xfail -if ${NCDUMP} -s noop.nc > ./tmp_filter.txt ; then - echo "*** FAIL: ncdump -hs noop.nc" -else - echo "*** XFAIL: ncdump -hs noop.nc" -fi -# Restore the filter code -unhidenoop -# Verify we can see filter when using -h -rm -f ./tmp_filter.txt -${NCDUMP} -hs noop.nc > ./tmp_filter.txt -echo "*** Pass: unknown filter" -fi - if test "x$NGC" = x1 ; then rm -f ./tmp_bzip2.c echo "*** Testing dynamic filters using ncgen with -lc" diff --git a/nc_test4/tst_unknown.sh b/nc_test4/tst_unknown.sh new file mode 100755 index 0000000000..52c81ce1a4 --- /dev/null +++ b/nc_test4/tst_unknown.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +if test "x$TESTNCZARR" = x1 ; then +. "$srcdir/test_nczarr.sh" +fi + +set -e + +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +# Set up HDF5_PLUGIN_PATH +export HDF5_PLUGIN_PATH=${HDF5_PLUGIN_DIR} + +# Test operation with an unknown filter + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { + cat $1 \ + | sed -e '/:_IsNetcdf4/d' \ + | sed -e '/:_Endianness/d' \ + | sed -e '/_NCProperties/d' \ + | sed -e '/_SuperblockVersion/d' \ + | cat > $2 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +sed -e '/var.*:_Filter/p' -ed <$1 >$2 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +sed -e '/var.*:_Codecs/p' -ed <$1 >$2 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + +# Locate the plugin path and the library names; argument order is critical + +# Find noop and capture +findplugin h5unknown +UNKNOWNDIR="${HDF5_PLUGIN_DIR}" +UNKNOWNLIB="${HDF5_PLUGIN_LIB}" +UNKNOWNFILTER="${HDF5_PLUGIN_DIR}/${UNKNOWNLIB}" + +# Verify +if ! test -f ${UNKNOWNFILTER} ; then echo "Unable to locate ${UNKNOWNFILTER}"; exit 1; fi + +testunk() { +zext=$1 +echo "*** Testing access to filter info when filter implementation is not available for map $zext" +if test "x$TESTNCZARR" = x1 ; then +fileargs tmp_known +deletemap $zext $file +else +file="tmp_known_${zfilt}.nc" +rm -f $file +fileurl="$file" +fi +printenv HDF5_PLUGIN_PATH +# build .nc file using unknown +${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/unknown.cdl +# dump and clean file when filter is avail +${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt +# Remove irrelevant -s output +sclean ./tmp_known_$zext.txt tmp_known_$zext.dump +# Hide the filter +rm -fr ${UNKNOWNDIR}/save +mkdir -p ${UNKNOWNDIR}/save +# Figure out all matching libs; make sure to remove .so, so.0, etc +LSRC=`${execdir}/../ncdump/ncpathcvt -F ${UNKNOWNFILTER}` +LDST=`${execdir}/../ncdump/ncpathcvt -F ${UNKNOWNDIR}/save` +mv ${LSRC}* ${LDST} +# Verify that the filter is no longer defined +# Try to read the data; should xfail +if ${NCDUMP} -s $fileurl > ./tmp_unk_$zext.dmp ; then +echo "*** FAIL: filter found" +found=1 +else +echo "*** XFAIL: filter not found" +found=0 +fi +# Restore the filter +LSRC="${LDST}" +LDST=`${execdir}/../ncdump/ncpathcvt -F ${UNKNOWNDIR}` +mv ${LSRC}/* ${LDST} +rm -fr ${UNKNOWNDIR}/save +if test "x$found" = x1 ; then exit 1; fi +} + +if test "x$TESTNCZARR" = x1 ; then + testunk file + if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testunk zip ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then testunk s3 ; fi +else +testunk nc +fi + +exit 0 diff --git a/nc_test4/unknown.cdl b/nc_test4/unknown.cdl new file mode 100644 index 0000000000..f3c4c8f39e --- /dev/null +++ b/nc_test4/unknown.cdl @@ -0,0 +1,83 @@ +netcdf bzip2 { +dimensions: + dim0 = 4 ; + dim1 = 4 ; + dim2 = 4 ; + dim3 = 4 ; +variables: + float var(dim0, dim1, dim2, dim3) ; + var:_Storage = "chunked" ; + var:_ChunkSizes = 4, 4, 4, 4 ; + var:_Filter = "33000" ; + var:_NoFill = "true" ; + +// global attributes: + :_Format = "netCDF-4" ; +data: + + var = + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, + 28, 29, 30, 31, + 32, 33, 34, 35, + 36, 37, 38, 39, + 40, 41, 42, 43, + 44, 45, 46, 47, + 48, 49, 50, 51, + 52, 53, 54, 55, + 56, 57, 58, 59, + 60, 61, 62, 63, + 64, 65, 66, 67, + 68, 69, 70, 71, + 72, 73, 74, 75, + 76, 77, 78, 79, + 80, 81, 82, 83, + 84, 85, 86, 87, + 88, 89, 90, 91, + 92, 93, 94, 95, + 96, 97, 98, 99, + 100, 101, 102, 103, + 104, 105, 106, 107, + 108, 109, 110, 111, + 112, 113, 114, 115, + 116, 117, 118, 119, + 120, 121, 122, 123, + 124, 125, 126, 127, + 128, 129, 130, 131, + 132, 133, 134, 135, + 136, 137, 138, 139, + 140, 141, 142, 143, + 144, 145, 146, 147, + 148, 149, 150, 151, + 152, 153, 154, 155, + 156, 157, 158, 159, + 160, 161, 162, 163, + 164, 165, 166, 167, + 168, 169, 170, 171, + 172, 173, 174, 175, + 176, 177, 178, 179, + 180, 181, 182, 183, + 184, 185, 186, 187, + 188, 189, 190, 191, + 192, 193, 194, 195, + 196, 197, 198, 199, + 200, 201, 202, 203, + 204, 205, 206, 207, + 208, 209, 210, 211, + 212, 213, 214, 215, + 216, 217, 218, 219, + 220, 221, 222, 223, + 224, 225, 226, 227, + 228, 229, 230, 231, + 232, 233, 234, 235, + 236, 237, 238, 239, + 240, 241, 242, 243, + 244, 245, 246, 247, + 248, 249, 250, 251, + 252, 253, 254, 255 ; +} diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index 25b19cde26..8cd39e51a9 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -18,7 +18,7 @@ TEST_EXTENSIONS = .sh LDADD = ${top_builddir}/liblib/libnetcdf.la AM_CPPFLAGS += -I$(top_srcdir)/liblib AM_CPPFLAGS += -DTOPSRCDIR=${abs_top_srcdir} -AM_CPPFLAGS += -DTOPBINDIR=${abs_top_bindir} +AM_CPPFLAGS += -DTOPBINDIR=${abs_top_builddir} # Set up the tests; do the .sh first, then .c check_PROGRAMS = @@ -52,6 +52,7 @@ TESTS += test_partvar if ENABLE_DAP_LONG_TESTS TESTS += tst_longremote3.sh tst_longremote3.log: tst_remote3.log + test_manyurls_SOURCES = test_manyurls.c manyurls.h check_PROGRAMS += test_manyurls TESTS += test_manyurls test_manyurls.log: tst_longremote3.log diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index c381f30d99..bcce244b54 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -245,7 +245,7 @@ tst_roman_szip_unlim.cdl tst_perdimpspecs.nc tmppds.* \ keyword1.nc keyword2.nc keyword3.nc keyword4.nc \ tmp_keyword1.cdl tmp_keyword2.cdl tmp_keyword3.cdl tmp_keyword4.cdl \ type_*.nc copy_type_*.cdl \ -scope_*.nc copy_scope_*.cdl +scope_*.nc copy_scope_*.cdl keyword5.nc # Remove directories clean-local: diff --git a/ncdump/ncpathcvt.c b/ncdump/ncpathcvt.c index 3e14916e56..01beaf812d 100755 --- a/ncdump/ncpathcvt.c +++ b/ncdump/ncpathcvt.c @@ -29,12 +29,13 @@ #include "ncpathmgr.h" static const char* USAGE = -"ncpathcvt [-c|-C|-m|-u|-w] [-h] [-e] [-d ] [-B] [-k] [-p] PATH\n" +"ncpathcvt [-c|-C|-m|-u|-w] [-h] [-e] [-F] [-d ] [-B] [-k] [-p] PATH\n" "Options\n" " -h help" " -e add backslash escapes to '\' and ' '\n" " -d use driveletter when needed; defaults to 'c'\n" -" -B convert occurrences of to ' '\n" +" -B convert occurrences of to blank\n" +" -F convert occurrences of '\\' to '/'" "Output type options:\n" " -c convert to Cygwin form of path\n" " -C return canonical form of path\n" @@ -57,6 +58,7 @@ struct Options { int debug; int canon; int blank; + int slash; int pathkind; } cvtoptions; @@ -93,6 +95,27 @@ escape(const char* path) return epath; } +static char* +slash(const char* path) +{ + size_t slen = strlen(path); + const char* p; + char* q; + char* epath = NULL; + + epath = (char*)malloc(slen + 1); + if(epath == NULL) usage("out of memtory"); + p = path; + q = epath; + for(;*p;p++) { + if(*p == '\\') + *q++ = '/'; + else *q++ = *p; + } + *q = '\0'; + return epath; +} + void printlocalkind(void) { @@ -173,7 +196,7 @@ main(int argc, char** argv) memset((void*)&cvtoptions,0,sizeof(cvtoptions)); cvtoptions.drive = 'c'; - while ((c = getopt(argc, argv, "B:CcD:d:ehkmpuwX")) != EOF) { + while ((c = getopt(argc, argv, "B:CFcD:d:ehkmpuwX")) != EOF) { switch(c) { case 'c': cvtoptions.target = NCPD_CYGWIN; break; case 'd': cvtoptions.drive = optarg[0]; break; @@ -190,6 +213,7 @@ main(int argc, char** argv) usage("Bad -B argument"); break; case 'C': cvtoptions.canon = 1; break; + case 'F': cvtoptions.slash = 1; break; case 'D': sscanf(optarg,"%d",&cvtoptions.debug); break; @@ -245,6 +269,11 @@ main(int argc, char** argv) cvtpath = escape(path); free(path); } + if(cvtpath && cvtoptions.slash) { + char* path = cvtpath; cvtpath = NULL; + cvtpath = slash(path); + free(path); + } printf("%s",cvtpath); done: if(canon) free(canon); diff --git a/ncdump/test_scope.sh b/ncdump/test_scope.sh index 0025eeee88..94468af4a2 100755 --- a/ncdump/test_scope.sh +++ b/ncdump/test_scope.sh @@ -31,7 +31,6 @@ diff -wB ${srcdir}/$1.cdl ${execdir}/copy_$1.cdl } typescope() { -echo ">>>" ls -l ${execdir}/printfqn* ${execdir}/$1.nc ${execdir}/$1_copy.nc REFT=`${execdir}/printfqn -f ${execdir}/$1.nc -v test_variable -t` COPYT=`${execdir}/printfqn -f ${execdir}/$1_copy.nc -v test_variable -t` diff --git a/ncdump/tst_nccopy3.sh b/ncdump/tst_nccopy3.sh index 7dcfc910c5..30b53bb8b1 100755 --- a/ncdump/tst_nccopy3.sh +++ b/ncdump/tst_nccopy3.sh @@ -8,7 +8,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -set -x set -e echo "" diff --git a/ncdump/tst_output.sh b/ncdump/tst_output.sh index 18da9c3e6f..a3fd20a6c3 100755 --- a/ncdump/tst_output.sh +++ b/ncdump/tst_output.sh @@ -4,7 +4,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh # This shell script tests the output from several previous tests. -set -x + set -e echo "" diff --git a/nctest/compare_test_files.sh b/nctest/compare_test_files.sh index f9712adbe1..b0d7b780f6 100755 --- a/nctest/compare_test_files.sh +++ b/nctest/compare_test_files.sh @@ -5,7 +5,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e -WS=`${NCPATHCVT} ${srcdir}` +WS=`${NCPATHCVT} -F ${srcdir}` cmp nctest_classic.nc "$WS/ref_nctest_classic.nc" cmp nctest_64bit_offset.nc "$WS/ref_nctest_64bit_offset.nc" diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index 03db4557fb..3df8b61922 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -26,6 +26,15 @@ FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh DESTINATION ${CMAK FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.1) FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh) +FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_unknown.sh SPSOURCE) +STRING(PREPEND SPSOURCE "#!/bin/bash\n") +STRING(PREPEND SPSOURCE "TESTNCZARR=1\n") +FILE(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.1 "${SPSOURCE}") +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.1 ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.sh @ONLY NEWLINE_STYLE LF) +FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR} FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) +FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.1) +FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_unknown.sh) + FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/ref*.cdl ${CMAKE_CURRENT_SOURCE_DIR}/ref*.txt) @@ -126,6 +135,7 @@ IF(ENABLE_TESTS) build_bin_test(test_filter_avail) ADD_SH_TEST(nczarr_test run_nczfilter) ADD_SH_TEST(nczarr_test run_filter) + ADD_SH_TEST(nczarr_test run_unknown) ADD_SH_TEST(nczarr_test run_specific_filters) ENDIF(ENABLE_FILTER_TESTING) if(ENABLE_NCZARR_ZIP) diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index 6563344861..9175c246ee 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -93,7 +93,8 @@ TESTS += run_nczfilter.sh # Echo filter tests from nc_test4 check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi test_filter_avail -TESTS += run_filter.sh +TESTS += run_filter.sh +TESTS += run_unknown.sh TESTS += run_specific_filters.sh endif #ENABLE_FILTER_TESTING @@ -127,7 +128,8 @@ run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_ncgen4.sh \ run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh run_s3_cleanup.sh \ run_purezarr.sh run_interop.sh run_misc.sh \ run_filter.sh run_specific_filters.sh \ -run_newformat.sh run_nczarr_fill.sh run_quantize.sh +run_newformat.sh run_nczarr_fill.sh run_quantize.sh run_nczfilter.sh \ +run_unknown.sh EXTRA_DIST += \ ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \ @@ -145,14 +147,13 @@ ref_avail1.cdl ref_avail1.dmp ref_avail1.txt \ ref_xarray.cdl ref_purezarr.cdl ref_purezarr_base.cdl ref_nczarr2zarr.cdl \ ref_bzip2.cdl ref_filtered.cdl ref_multi.cdl \ ref_any.cdl ref_oldformat.cdl ref_oldformat.zip ref_newformatpure.cdl \ -ref_quotes.zip ref_quotes.cdl \ ref_groups.h5 ref_byte.zarr.zip ref_byte_fill_value_null.zarr.zip \ ref_groups_regular.cdl ref_byte.cdl ref_byte_fill_value_null.cdl # Interoperability files -EXTRA_DIST += ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes.zip ref_quotes.cdl +EXTRA_DIST += ref_power_901_constants_orig.zip ref_power_901_constants.cdl ref_quotes_orig.zip ref_quotes.cdl -CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl +CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh run_filterinstall.sh test_quantize.c: $(top_srcdir)/nc_test4/tst_quantize.c @@ -165,6 +166,15 @@ test_filter_avail.c: $(top_srcdir)/nc_test4/tst_filter_avail.c echo "#define TESTNCZARR" > $@ cat $(top_srcdir)/nc_test4/tst_filter_avail.c >> $@ +run_unknown.sh: $(top_srcdir)/nc_test4/tst_unknown.sh + rm -f $@ run_unknown.tmp + echo "#!/bin/bash" > run_unknown.tmp + echo "TESTNCZARR=1" >> run_unknown.tmp + cat $(top_srcdir)/nc_test4/tst_unknown.sh >> run_unknown.tmp + tr -d '\r' < run_unknown.tmp > $@ + chmod a+x $@ + rm -f run_unknown.tmp + run_specific_filters.sh: $(top_srcdir)/nc_test4/tst_specific_filters.sh rm -f $@ run_specific_filters.tmp echo "#!/bin/bash" > run_specific_filters.tmp @@ -185,10 +195,11 @@ run_filterinstall.sh: $(top_srcdir)/nc_test4/tst_filterinstall.sh # Remove directories clean-local: - rm -fr tmp*.file results.file results.s3 results.zip - rm -fr rcmiscdir + rm -fr tmp_*.nc tmp_*.zarr tst_quantize*.zarr tmp*.file results.file results.s3 results.zip + rm -fr rcmiscdir ref_power_901_constants.file + -DISTCLEANFILES = findplugin.sh test_quantize.c run_specific_filters.sh +DISTCLEANFILES = findplugin.sh test_quantize.c run_specific_filters.sh run_filterinstall.sh # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ diff --git a/nczarr_test/ref_multi.cdl b/nczarr_test/ref_multi.cdl index a3b41f6d63..b81c63c020 100644 --- a/nczarr_test/ref_multi.cdl +++ b/nczarr_test/ref_multi.cdl @@ -9,7 +9,7 @@ variables: var:_Storage = "chunked" ; var:_ChunkSizes = 4, 4, 4, 4 ; var:_Filter = "307,9|1,2|40000" ; - var:_Codecs = "[{\"id\": \"bz2\",\"level\": \"9\"},{\"id\": \"zlib\",\"level\": \"2\"},{\"id\": \"noop0\"}]" ; + var:_Codecs = "[{\"id\": \"bz2\",\"level\": \"9\"},{\"id\": \"zlib\",\"level\": \"2\"},{\"id\": \"noop\"}]" ; var:_NoFill = "true" ; // global attributes: diff --git a/nczarr_test/ref_power_901_constants.zip b/nczarr_test/ref_power_901_constants_orig.zip similarity index 100% rename from nczarr_test/ref_power_901_constants.zip rename to nczarr_test/ref_power_901_constants_orig.zip diff --git a/nczarr_test/ref_quotes.zip b/nczarr_test/ref_quotes_orig.zip similarity index 100% rename from nczarr_test/ref_quotes.zip rename to nczarr_test/ref_quotes_orig.zip diff --git a/nczarr_test/run_filter.sh b/nczarr_test/run_filter.sh index da653a985c..0bf6437d15 100755 --- a/nczarr_test/run_filter.sh +++ b/nczarr_test/run_filter.sh @@ -12,7 +12,6 @@ testset() { testapi $1 testng $1 testncp $1 -testunk $1 testngc $1 testmisc $1 testmulti $1 @@ -51,21 +50,19 @@ sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 # Find misc and capture findplugin h5misc -MISCDIR="${HDF5_PLUGIN_DIR}/${HDF5_PLUGIN_LIB}" - -# Find noop and capture -findplugin h5noop -NOOPLIB="${HDF5_PLUGIN_LIB}" -NOOPDIR="${HDF5_PLUGIN_DIR}/${NOOPLIB}" +MISCLIB="${HDF5_PLUGIN_LIB}" +MISCDIR="${HDF5_PLUGIN_DIR}" +MISCPATH="${MISCDIR}/${MISCLIB}" # Find bzip2 and capture findplugin h5bzip2 BZIP2LIB="${HDF5_PLUGIN_LIB}" -BZIP2DIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}" +BZIP2DIR="${HDF5_PLUGIN_DIR}" +BZIP2PATH="${BZIP2DIR}/${BZIP2LIB}" # Verify -if ! test -f ${BZIP2DIR} ; then echo "Unable to locate ${BZIP2DIR}"; exit 1; fi -if ! test -f ${MISCDIR} ; then echo "Unable to locate ${MISCDIR}"; exit 1; fi +if ! test -f ${BZIP2path} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi +if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi # Execute the specified tests @@ -132,29 +129,6 @@ diff -b -w ${srcdir}/ref_filtered.cdl ./tmp_ncp_$zext.dump echo " *** Pass: nccopy simple filter for map $zext" } -testunk() { -zext=$1 -echo "*** Testing access to filter info when filter implementation is not available for map $zext" -fileargs tmp_known -deletemap $zext $file -# build noop.nc -${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/noop.cdl -# dump and clean noop.nc header when filter is avail -${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt -# Remove irrelevant -s output -sclean ./tmp_known_$zext.txt tmp_known_$zext.dump -# Now hide the filter code -mv ${NOOPDIR} ./${NOOPLIB}.save -# dump and clean noop.nc header when filter is not avail -${NCDUMP} -hs $fileurl > ./tmp_unk_$zext.txt -# Restore the filter code -mv ./${NOOPLIB}.save ${NOOPDIR} -# Verify that the filter is no longer defined -UNK=`sed -e '/var:_Filter/p' -e d ./tmp_unk_$zext.txt` -test "x$UNK" = x -echo "*** Pass: ncgen dynamic filter for map $zext" -} - testngc() { zext=$1 echo "*** Testing dynamic filters using ncgen with -lc for map $zext" diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index a2ee6f3fc0..f3b9ea232c 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -54,16 +54,14 @@ case "$zext" in file) # need to unpack rm -fr ref_power_901_constants ref_power_901_constants.file - unzip ${srcdir}/ref_power_901_constants.zip > /dev/null + unzip ${srcdir}/ref_power_901_constants_orig.zip > /dev/null mv ref_power_901_constants ref_power_901_constants.file testcasefile ref_power_901_constants zarr metaonly; # test xarray as default ;; zip) # Move into position - if test "x$srcdir" != "x$execdir" ; then - cp ${srcdir}/ref_power_901_constants.zip ${execdir} - cp ${srcdir}/ref_quotes.zip ${execdir} - fi + cp ${srcdir}/ref_power_901_constants_orig.zip ${execdir}/ref_power_901_constants.zip + cp ${srcdir}/ref_quotes_orig.zip ${execdir}/ref_quotes.zip testcasezip ref_power_901_constants xarray metaonly # Test large constant interoperability testcasezip ref_quotes zarr metaonly diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index e606b392a3..d88768ded0 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -18,6 +18,8 @@ SET_SOURCE_FILES_PROPERTIES(H5Znoop.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE SET(h5noop1_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c H5Zutil.c h5noop.h) SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE=1) +SET(h5unknown_SOURCES H5Zunknown.c) + SET(h5shuffle_SOURCES H5Zshuffle.c) SET(h5fletcher32_SOURCES H5Zfletcher32.c H5checksum.c) SET(h5deflate_SOURCES H5Zdeflate.c) @@ -51,6 +53,7 @@ ENDMACRO() buildplugin(h5misc "h5misc") buildplugin(h5noop "h5noop") buildplugin(h5noop1 "h5noop1") +buildplugin(h5unknown "h5unknown") buildplugin(h5shuffle "h5shuffle") buildplugin(h5fletcher32 "h5fletcher32") diff --git a/plugins/H5Znoop.c b/plugins/H5Znoop.c index 03fa6d6bbe..d50fe14233 100644 --- a/plugins/H5Znoop.c +++ b/plugins/H5Znoop.c @@ -154,7 +154,7 @@ static NCZ_codec_t NCZ_noop_codec = {/* NCZ_codec_t codec fields */ NCZ_CODEC_CLASS_VER, /* Struct version number */ NCZ_CODEC_HDF5, /* Struct sort */ #if NOOP_INSTANCE == 0 - "noop0", /* Standard name/id of the codec */ + "noop", /* Standard name/id of the codec */ H5Z_FILTER_NOOP, /* HDF5 alias for noop */ #else "noop1", /* Standard name/id of the codec */ diff --git a/plugins/H5Zunknown.c b/plugins/H5Zunknown.c new file mode 100644 index 0000000000..9cd8af135f --- /dev/null +++ b/plugins/H5Zunknown.c @@ -0,0 +1,147 @@ +#include "config.h" +#include +#include +#include +#include +#include + +#include "netcdf_filter_build.h" + +/* use a temporary filter id*/ +#define H5Z_FILTER_UNKNOWN 33000 + +#undef DEBUG + +static htri_t H5Z_unknown_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z_filter_unknown(unsigned int, size_t, const unsigned int cd_values[], size_t, size_t*, void**); + +static H5Z_class2_t H5Z_UNKNOWN[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + (H5Z_filter_t)(H5Z_FILTER_UNKNOWN), /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "unknown", /* Filter name for debugging */ + (H5Z_can_apply_func_t)H5Z_unknown_can_apply, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + (H5Z_func_t)H5Z_filter_unknown, /* The actual filter function */ +}}; + +/* External Discovery Functions */ +DLLEXPORT +H5PL_type_t +H5PLget_plugin_type(void) +{ + return H5PL_TYPE_FILTER; +} + +DLLEXPORT +const void* +H5PLget_plugin_info(void) +{ + return H5Z_UNKNOWN; +} + +/* Make this explicit */ +/* + * The "can_apply" callback returns positive a valid combination, zero for an + * invalid combination and negative for an error. + */ +static htri_t +H5Z_unknown_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id) +{ + return 1; /* Assume it can always apply */ +} + +static size_t +H5Z_filter_unknown(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf) +{ + void* newbuf; + + if (flags & H5Z_FLAG_REVERSE) { + /* Replace buffer */ + newbuf = H5allocate_memory(*buf_size,0); + if(newbuf == NULL) abort(); + memcpy(newbuf,*buf,*buf_size); + /* reclaim old buffer */ + H5free_memory(*buf); + *buf = newbuf; + + } else { + /* Replace buffer */ + newbuf = H5allocate_memory(*buf_size,0); + if(newbuf == NULL) abort(); + memcpy(newbuf,*buf,*buf_size); + /* reclaim old buffer */ + H5free_memory(*buf); + *buf = newbuf; + + } + return *buf_size; +} + +/**************************************************/ +/* NCZarr Codec API */ + +/* Codec Format +{ +"id": "unknown", +} +*/ + +/* Forward */ +static int NCZ_unknown_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_unknown_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); + +/* Structure for NCZ_PLUGIN_CODEC */ +static NCZ_codec_t NCZ_unknown_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "unknown", /* Standard name/id of the codec */ + H5Z_FILTER_UNKNOWN, /* HDF5 alias for unknown */ + NULL, /*NCZ_unknown_codec_initialize*/ + NULL, /*NCZ_unknown_codec_finalize*/ + NCZ_unknown_codec_to_hdf5, + NCZ_unknown_hdf5_to_codec, + NULL, /*NCZ_unknown_modify_parameters*/ +}; + +/* External Export API */ +DLLEXPORT +const void* +NCZ_get_codec_info(void) +{ + return (void*)&NCZ_unknown_codec; +} + +/* NCZarr Interface Functions */ + +static int +NCZ_unknown_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + + *nparamsp = 0; + *paramsp = NULL; + + return stat; +} + +static int +NCZ_unknown_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + char json[8192]; + + if(nparams != 0 || params != NULL) + {stat = NC_EINVAL; goto done;} + + snprintf(json,sizeof(json),"{\"id\": \"%s\"}",NCZ_unknown_codec.codecid); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} diff --git a/plugins/Makefile.am b/plugins/Makefile.am index ded5b30760..fa839264ff 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -7,8 +7,28 @@ include $(top_srcdir)/lib_flags.am AM_LDFLAGS += -module -avoid-version -shared -export-dynamic \ -rpath ${abs_builddir} ${NOUNDEFINED} -lib_LTLIBRARIES = -noinst_LTLIBRARIES = +# Create an alternate directory if not installing or for noinst installs. +ALTPLUGINDIR = ${abs_top_builddir}/plugins/plugindir + +# This is where the plugins are to be installed +if ENABLE_PLUGIN_DIR +plugindir = @PLUGIN_INSTALL_DIR@ +else +plugindir = ${ALTPLUGINDIR} +endif + +plugin_LTLIBRARIES = + +# Apparently one cannot have plugin_LTLIBRARIES and also noinst_LTLIBRARIES. +# So create a tmp location for "noinst" shared libraries. +tmpdir = ${ALTPLUGINDIR} + +tmp_LTLIBRARIES = + +# This linker flag specifies libtool version info. +# See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning +# for information regarding incrementing `-version-info`. +plugin_version_info = -version-info 0:0:0 if ISMINGW LDADD = ${top_builddir}/liblib/libnetcdf.la @@ -17,7 +37,7 @@ endif EXTRA_DIST = CMakeLists.txt # The HDF5 filter wrappers -EXTRA_DIST += stdinstall.sh \ +EXTRA_DIST += \ H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \ H5Zshuffle.c H5Zdeflate.c H5Zszip.c H5Zszip.h \ H5Zbzip2.c h5bzip2.h H5Zblosc.c H5Zblosc.h H5Zzstd.c H5Zzstd.h @@ -29,55 +49,66 @@ EXTRA_DIST += H5checksum.c if ENABLE_FILTER_TESTING -noinst_LTLIBRARIES += libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la - if ENABLE_NCZARR_FILTERS -noinst_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la +plugin_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la libh5shuffle_la_SOURCES = H5Zshuffle.c libh5fletcher32_la_SOURCES = H5Zfletcher32.c H5checksum.c libh5deflate_la_SOURCES = H5Zdeflate.c +libh5shuffle_la_LDFLAGS = ${plugin_version_info} +libh5deflate_la_LDFLAGS = ${plugin_version_info} +libh5fletcher32_la_LDFLAGS = ${plugin_version_info} + # Need our version of szip if libsz available and we are not using HDF5 if HAVE_SZ -noinst_LTLIBRARIES += libh5szip.la +plugin_LTLIBRARIES += libh5szip.la libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h +libh5szip_la_LDFLAGS = ${plugin_version_info} endif endif # ENABLE_NCZARR_FILTERS if ENABLE_PLUGINS +# The NCZarr codec libraries libnczstdfilters_la_SOURCES = NCZstdfilters.c - libnczhdf5filters_la_SOURCES = NCZhdf5filters.c +plugin_LTLIBRARIES += libnczhdf5filters.la +plugin_LTLIBRARIES += libnczstdfilters.la + if HAVE_BLOSC libh5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h +libh5blosc_la_LDFLAGS = ${plugin_version_info} +plugin_LTLIBRARIES += libh5blosc.la endif if HAVE_ZSTD libh5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h -endif - -noinst_LTLIBRARIES += libnczhdf5filters.la -noinst_LTLIBRARIES += libnczstdfilters.la - -if HAVE_BLOSC -noinst_LTLIBRARIES += libh5blosc.la -endif -if HAVE_ZSTD -noinst_LTLIBRARIES += libh5zstd.la +libh5zstd_la_LDFLAGS = ${plugin_version_info} +plugin_LTLIBRARIES += libh5zstd.la endif endif #ENABLE_PLUGINS -libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h -libnczmisc_la_SOURCES = NCZmisc.c - # The noop filter is to allow testing of multifilters and filter order # Need two distinct instances libh5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h libh5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h +libh5noop_la_LDFLAGS = ${plugin_version_info} +libh5noop1_la_LDFLAGS = ${plugin_version_info} + +# The misc filter is to allow testing of filter arguments +libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h +libh5misc_la_LDFLAGS = ${plugin_version_info} +libnczmisc_la_SOURCES = NCZmisc.c +libnczmisc_la_LDFLAGS = ${plugin_version_info} + +# Provide a filter to test missing filter +libh5unknown_la_SOURCES = H5Zunknown.c +libh5unknown_la_LDFLAGS = ${plugin_version_info} + +tmp_LTLIBRARIES += libh5noop.la libh5noop1.la libh5misc.la libnczmisc.la libh5unknown.la # Bzip2 is used to test more complex filters libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h @@ -86,17 +117,17 @@ EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE if HAVE_LOCAL_BZ2 libh5bzip2_la_SOURCES += ${BZIP2SRC} endif +libh5bzip2_la_LDFLAGS = ${plugin_version_info} +plugin_LTLIBRARIES += libh5bzip2.la endif #ENABLE_FILTER_TESTING BUILT_SOURCES = H5Znoop1.c -DISTCLEANFILES = H5Znoop1.c ncjson.h +DISTCLEANFILES = H5Znoop1.c H5Znoop2.c ncjson.h H5Znoop1.c: Makefile H5Znoop.c echo '#define NOOP_INSTANCE 1' > $@ cat ${srcdir}/H5Znoop.c >> $@ -noinst_LTLIBRARIES += libh5bzip2.la - # Record where bzip2 came from; may be out of date BZIP2VER = 1.0.8 BZIP2DIR = bzip2-${BZIP2VER} @@ -108,6 +139,7 @@ bzip2:: cd ${BZIP2DIR}; cp ${BZIP2SRC} ..; cp LICENSE ../BZIP2_LICENSE ; cd .. rm -fr ./${BZIP2DIR} -# Custom install -install-exec-hook: - sh ./stdinstall.sh +# Custom clean +clean-local: + rm -fr ${ALTPLUGINDIR} + diff --git a/plugins/NCZtest.c b/plugins/NCZtest.c new file mode 100644 index 0000000000..06c6879756 --- /dev/null +++ b/plugins/NCZtest.c @@ -0,0 +1,179 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Robb Matzke + * Friday, August 27, 1999 + */ + +/* Converted to NCZarr support by Dennis Heimbigner 5/1/2021 */ + +#include +#include +#include +#include +#include + +#include "netcdf_json.h" + +#include "netcdf.h" +#include "netcdf_filter.h" +#include "netcdf_filter_build.h" + +#include "h5misc.h" + +/**************************************************/ +/* NCZarr Filter Objects */ +/* Codec Format +{ +"id": "test", +"testcase": "n", +"byte": "", +"ubyte": "", +"short": "", +"ushort": "", +"int": "", +"uint": "", +"float": "", +"double0": "", +"double1": "", +"int640": "", +"int641": "", +"uint640": "", +"uint641": "", +} +*/ + +/* Give unique dict key names for parameters */ +static const char* fields[14] = { +"testcase", +"byte", +"ubyte", +"short", +"ushort", +"int", +"uint", +"float", +"double_0", +"double_1", +"int64_0", +"int64_1", +"uint64_0", +"uint64_1" +}; + +/* Forward */ +static int NCZ_misc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); + +/* Structure for NCZ_PLUGIN_CODEC */ +static NCZ_codec_t NCZ_misc_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "test", /* Standard name/id of the codec */ + H5Z_FILTER_TEST, /* HDF5 alias for misc */ + NULL, /*NCZ_misc_codec_initialize*/ + NULL, /*NCZ_misc_codec_finalize*/ + NCZ_misc_codec_to_hdf5, + NCZ_misc_hdf5_to_codec, + NULL, /*NCZ_misc_modify_parameters*/ +}; + +/* External Export API */ +DLLEXPORT +const void* +NCZ_get_codec_info(void) +{ + return (void*)&NCZ_misc_codec; +} + +/* NCZarr Interface Functions */ + +static int +NCZ_misc_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + NCjson* jcodec = NULL; + NCjson* jtmp = NULL; + size_t i,nparams = 0; + unsigned* params = NULL; + + /* parse the JSON */ + if(NCJparse(codec_json,0,&jcodec)) + {stat = NC_EFILTER; goto done;} + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + + /* Verify the codec ID */ + if(NCJdictget(jcodec,"id",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;} + if(strcmp(NCJstring(jtmp),NCZ_misc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} + + /* The codec will have (2*14 + 1) +1 = 29 dict entries + id*/ + nparams = (2*14 + 1) + 1; + if(NCJlength(jcodec) != nparams) { + fprintf(stderr,"Incorrect no. of codec parameters: need=29 sent=%ld\n",(unsigned long)(nparams-1)); + stat = NC_EINVAL; + goto done; + } + + /* Actual # of parameters is 14 (ignoring the testcase number) */ + nparams = 14; + if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + for(i=0;i NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + params[i] = (unsigned)jc.ival; + } + if(nparamsp) *nparamsp = nparams; + if(paramsp) {*paramsp = params; params = NULL;} + +done: + if(params) free(params); + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int i,stat = NC_NOERR; + char json[4096]; + char value[1024]; + + if(nparams == 0 || params == NULL) + {stat = NC_EINVAL; goto done;} + if(nparams != 14) { + fprintf(stderr,"Incorrect no. of parameters: need=14 sent=%ld\n",(unsigned long)nparams); + stat = NC_EINVAL; + goto done; + } + snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_misc_codec.codecid); + for(i=0;i<14;i++) { + snprintf(value,sizeof(value),", \"%s\": \"%u\"",fields[i],params[i]); + strlcat(json,value,sizeof(json)); + } + strlcat(json,"}",sizeof(json)); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} diff --git a/plugins/stdinstall.in b/plugins/stdinstall.in deleted file mode 100755 index 74a73765a8..0000000000 --- a/plugins/stdinstall.in +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -if test "x$srcdir" = x ; then srcdir=`pwd`; fi -. ../test_common.sh - -set -x - -INSTALLDIR=@PLUGIN_INSTALL_DIR@ - -# Load the findplugins function -. ${builddir}/findplugin.sh -echo "findplugin.sh loaded" - -installplugin() { -PLUG="$1" -# Locate the plugin path and the library name; argument order is critical -findplugin ${PLUG} -if ! test -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" ; then - echo "Not exists: ${HDF5_PLUGIN_DIR}/$HDF5_PLUGIN_LIB ; ignored" - return -fi -if ! test -d "${INSTALLDIR}" ; then - echo "Not exists: ${INSTALLDIR} ; creating" - mkdir ${INSTALLDIR} -fi -echo "Installing: $HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB into $INSTALLDIR" -cp -f "$HDF5_PLUGIN_DIR/$HDF5_PLUGIN_LIB" $INSTALLDIR -} - -if test "x$USEPLUGINS" != x ; then - if test "x$INSTALLDIR" != x ; then - installplugin h5bzip2 - installplugin h5zstd - installplugin h5blosc - if test "x$FEATURE_NCZARR" ; then - installplugin h5fletcher32 - installplugin h5shuffle - installplugin h5deflate - installplugin h5szip - installplugin nczdefaults - installplugin nczszip - fi - fi -fi diff --git a/test_common.in b/test_common.in index eff6baf8e5..6624863a0e 100644 --- a/test_common.in +++ b/test_common.in @@ -15,12 +15,10 @@ FP_ISMSVC=@ISMSVC@ FP_ISCYGWIN=@ISCYGWIN@ FP_ISMINGW=@ISMINGW@ FP_ISMSYS=@ISMSYS@ - FP_ISOSX=@ISOSX@ -FP_ISREGEDIT=@ISREGEDIT@ -FP_USEPLUGINS=@USEPLUGINS@ FP_ISREGEDIT=@ISREGEDIT@ +FP_USEPLUGINS=@USEPLUGINS@ # Feature flags FEATURE_HDF5=@HAS_HDF5@ @@ -156,11 +154,13 @@ ncgen3c0="${top_srcdir}/ncgen3/c0.cdl" ncgenc0="${top_srcdir}/ncgen/c0.cdl" ncgenc04="${top_srcdir}/ncgen/c0_4.cdl" +# Set LC_ALL +if test "x$FP_ISMSVC" = xyes || test "x$FP_ISCYGWIN" = xyes; then export LC_ALL="en_US.utf8"; fi + # Test for filter availability avail() { if test yes = `${execdir}/../ncdump/ncfilteravail $1` ; then return 0 ; else echo "filter $1 not available" ; return 1; fi } -if test "x$FP_ISMSVC" = xyes || test "x$FP_ISCYGWIN" = xyes; then export LC_ALL="en_US.utf8"; fi # Make sure we are in builddir (not execdir) cd $builddir From 5e5b1b5ba4401ce6ed7eb1085a4db1a8cdc5e952 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 14 May 2022 16:08:56 -0600 Subject: [PATCH 02/10] Update release notes --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5f11960e0e..de8dbd4e01 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD -* [Enhancement] Improve filter installation process to avoid use of an extra shell script. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* [Enhancement] Improve filter installation process to avoid use of an extra shell script. See [Github #2348](https://github.com/Unidata/netcdf-c/pull/2348). * [Bug Fix] Get "make distcheck" to work See [Github #/2343](https://github.com/Unidata/netcdf-c/pull/2343). * [Enhancement] Update the documentation to match the current filter capabilities See [Github #2249](https://github.com/Unidata/netcdf-c/pull/2249). * [Enhancement] Support installation of pre-built standard filters into user-specified location. See [Github #2318](https://github.com/Unidata/netcdf-c/pull/2318). From 342c20241f3411cb089e853ec4b9c001586f2aee Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 16 May 2022 13:58:42 -0600 Subject: [PATCH 03/10] swith --- configure.ac | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index dc24644cae..10c359690b 100644 --- a/configure.ac +++ b/configure.ac @@ -1885,14 +1885,35 @@ AC_SUBST(STD_FILTERS,[$std_filters]) # If user wants, then install selected standard filters AC_MSG_CHECKING([whether and where we should install plugins]) AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=], - [Install selected standard filters in specified directory])]) + [Install selected standard filters in specified directory])], + [],[with_plugin_dir=no]) AC_MSG_RESULT([$with_plugin_dir]) -if test "x$with_plugin_dir" = x ; then - AC_MSG_WARN([No plugin directory value specified; option ignored.]) - with_plugin_dir= +if test "x$with_plugin_dir" = xno ; then # option missing|disabled + with_plugin_dir=no with_plugin_dir_setting="N.A." enable_plugin_dir=no -else +elif test "x$with_plugin_dir" = xyes ; then # --with-plugin-dir, no argument + # Default to last dir (lowest search priority) in HDF5_PLUGIN_PATH + PLUGIN_PATH="$HDF5_PLUGIN_PATH" + if test "x${PLUGIN_PATH}" = x ; then + if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then + PLUGIN_PATH="${ALLUSERSPROFILE}\\hdfd5\\lib\\plugin" + else + PLUGIN_PATH="/usr/local/hdf5/lib/plugin" + fi + fi + # Use the lowest priority dir in the path + if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then + PLUGIN_DIR=`echo "$PLUGIN_PATH" | sed -e 's|\([^;]*[;]\)*\(.*\)|\2|'` + else + PLUGIN_DIR=`echo "$PLUGIN_PATH" | sed -e 's|\([^:]*[:]\)*\(.*\)|\2|'` + fi + with_plugin_dir_setting="$PLUGIN_DIR" + # canonical form is all forward slashes + with_plugin_dir=`echo "$PLUGIN_DIR" | tr '\\\\' '/'` + enable_plugin_dir=yes + AC_MSG_NOTICE([Defaulting to --with-plugin-path=$with_plugin_dir]) +else # --with-plugin-dir= with_plugin_dir_setting="$with_plugin_dir" enable_plugin_dir=yes fi From 6ae328970118ce8973176416d1680a74c5704962 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Thu, 19 May 2022 22:00:40 -0600 Subject: [PATCH 04/10] I made a major update to this PR with the following changes: ## Overwriting I think I solved the file overwrite problem by doing light name mangling of the shared library names. With this change the probabilty is very small that installing our filter wrappers in a directory will overwrite code produced by others. ## Default Install Location I have setup the --with-plugin-dir option default to install in the following locations in order of preference 1. If HDF5_PLUGIN_PATH is defined (at build time remember), then the last directory in that path will be where the filter wrapper shared libraries will be installed. 2. Otherwise the default is "/usr/local/hdf5/lib/plugin" (on *nix*) or "%ALLUSERSPROFILE%\\hdf5\\lib\\plugin" for Windows or Mingw. Currently, --with-plugin-dir is disabled by default. I should note that even if I enable it by default, installing netcdf-c will still not run "out of the box" because the hypothetical naive user will not know which compressor libraries need to be pre-installed before netcdf is installed. Nor will that user have any way to find out what needs to be installed. --- CMakeLists.txt | 57 ++++++++++++++++++------ configure.ac | 40 ++++++++++++++--- docs/filters.md | 96 +++++++++++++++++++++++++++++------------ liblib/Makefile.am | 2 +- libnczarr/zfilter.c | 53 +++++++++++++---------- libnczarr/zfilter.h | 8 ++-- nc_test4/findplugin.in | 12 +++--- nc_test4/tst_unknown.sh | 11 +++-- nczarr_test/Makefile.am | 4 +- plugins/CMakeLists.txt | 39 ++++++++++------- plugins/Makefile.am | 67 +++++++++++++++------------- 11 files changed, 256 insertions(+), 133 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 89f1890cdf..05870e762f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,10 +36,12 @@ SET(NC_VERSION_NOTE "-development") SET(netCDF_VERSION ${NC_VERSION_MAJOR}.${NC_VERSION_MINOR}.${NC_VERSION_PATCH}${NC_VERSION_NOTE}) SET(VERSION ${netCDF_VERSION}) SET(NC_VERSION ${netCDF_VERSION}) -SET(netCDF_LIB_VERSION 19) -SET(netCDF_SO_VERSION 19) SET(PACKAGE_VERSION ${VERSION}) +# These values should match those in configure.ac +SET(netCDF_LIB_VERSION 19}) +SET(netCDF_SO_VERSION 19) + # Version of the dispatch table. This must match the value in # configure.ac. SET(NC_DISPATCH_VERSION 5) @@ -1116,21 +1118,52 @@ ELSE() set(STD_FILTERS "${STD_FILTERS} bz2") ENDIF() - # If user wants, then install selected plugins SET(PLUGIN_INSTALL_DIR "" CACHE STRING "Whether and where we should install plugins") -SET(ENABLE_PLUGIN_INSTALL OFF) -if(DEFINED PLUGIN_INSTALL_DIR OR DEFINED CACHE{PLUGIN_INSTALL_DIR}) - IF(PLUGIN_INSTALL_DIR STREQUAL "") - MESSAGE(WARNING "No plugin directory value specified; option ignored.") - UNSET(PLUGIN_INSTALL_DIR) - UNSET(PLUGIN_INSTALL_DIR CACHE) - SET(PLUGIN_INSTALL_DIR_SETTING "N.A.") +SET(ENABLE_PLUGIN_INSTALL NO) +IF(DEFINED PLUGIN_INSTALL_DIR) #OR DEFINED CACHE{PLUGIN_INSTALL_DIR} + STRING(TOLOWER "${PLUGIN_INSTALL_DIR}" LCPID) + IF(LCPID STREQUAL "no") + SET(ENABLE_PLUGIN_INSTALL NO) ELSE() - SET(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}") - SET(ENABLE_PLUGIN_INSTALL ON) + SET(ENABLE_PLUGIN_INSTALL YES) ENDIF() +ENDIF() + +IF(PLUGIN_INSTALL_DIR STREQUAL "" OR PLUGIN_INSTALL_DIR STREQUAL "yes") + SET(PLUGIN_INSTALL_DIR "YES") +ENDIF() + +IF(ENABLE_PLUGIN_INSTALL AND PLUGIN_INSTALL_DIR STREQUAL "YES") + # Default to last dir (lowest search priority) in HDF5_PLUGIN_PATH + IF(DEFINED $ENV{HDF5_PLUGIN_PATH}) + SET(PLUGIN_PATH "$HDF5_PLUGIN_PATH") + ELSE() + IF(ISMSVC OR ISMINGW) + SET(PLUGIN_PATH "$ENV{ALLUSERSPROFILE}\\hdfd5\\lib\\plugin") + ELSE() + SET(PLUGIN_PATH "/usr/local/hdf5/lib/plugin") + ENDIF() + ENDIF() +ENDIF() + +# Assume PLUGIN_PATH is defined +# Use the lowest priority dir in the path +IF(NOT ISMSVC AND NOT ISMINGW) + STRING(REPLACE ":" ";" PATH_LIST ${PLUGIN_PATH}) ELSE() + SET(PATH_LIST ${PLUGIN_PATH}) +ENDIF() + +# Get last element +LIST(GET PATH_LIST -1 PLUGIN_INSTALL_DIR) +MESSAGE(WARNING "Defaulting to -DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}") + +IF(ENABLE_PLUGIN_INSTALL) + SET(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}") +ELSE() # No option specified + UNSET(PLUGIN_INSTALL_DIR) + UNSET(PLUGIN_INSTALL_DIR CACHE) SET(PLUGIN_INSTALL_DIR_SETTING "N.A.") ENDIF() diff --git a/configure.ac b/configure.ac index dc24644cae..7b919b0c6a 100644 --- a/configure.ac +++ b/configure.ac @@ -24,6 +24,13 @@ AC_SUBST([NC_VERSION_MINOR]) NC_VERSION_MINOR=8 AC_SUBST([NC_VERSION_PATCH]) NC_VERSION_PATCH=2 AC_SUBST([NC_VERSION_NOTE]) NC_VERSION_NOTE="-development" +## +# These linker flags specify libtool version info. +# See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning +# for information regarding incrementing `-version-info`. +# These values should match those in CMakeLists.txt +AC_SUBST([netCDF_SO_VERSION]) netCDF_SO_VERSION=19:1:0 + ##### # Set some variables used to generate a libnetcdf.settings file, # pattered after the files generated by libhdf4, libhdf5. @@ -1884,15 +1891,36 @@ AC_SUBST(STD_FILTERS,[$std_filters]) # If user wants, then install selected standard filters AC_MSG_CHECKING([whether and where we should install plugins]) -AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=], - [Install selected standard filters in specified directory])]) + AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=], + [Install selected standard filters in specified directory])], + [],[with_plugin_dir=no]) AC_MSG_RESULT([$with_plugin_dir]) -if test "x$with_plugin_dir" = x ; then - AC_MSG_WARN([No plugin directory value specified; option ignored.]) - with_plugin_dir= +if test "x$with_plugin_dir" = xno ; then # option missing|disabled + with_plugin_dir=no with_plugin_dir_setting="N.A." enable_plugin_dir=no -else +elif test "x$with_plugin_dir" = xyes ; then # --with-plugin-dir, no argument + # Default to last dir (lowest search priority) in HDF5_PLUGIN_PATH + PLUGIN_PATH="$HDF5_PLUGIN_PATH" + if test "x${PLUGIN_PATH}" = x ; then + if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then + PLUGIN_PATH="${ALLUSERSPROFILE}\\hdfd5\\lib\\plugin" + else + PLUGIN_PATH="/usr/local/hdf5/lib/plugin" + fi + fi + # Use the lowest priority dir in the path + if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then + PLUGIN_DIR=`echo "$PLUGIN_PATH" | sed -e 's|\([^;]*[;]\)*\(.*\)|\2|'` + else + PLUGIN_DIR=`echo "$PLUGIN_PATH" | sed -e 's|\([^:]*[:]\)*\(.*\)|\2|'` + fi + with_plugin_dir_setting="$PLUGIN_DIR" + # canonical form is all forward slashes + with_plugin_dir=`echo "$PLUGIN_DIR" | tr '\\\\' '/'` + enable_plugin_dir=yes + AC_MSG_NOTICE([Defaulting to --with-plugin-path=$with_plugin_dir]) +else # --with-plugin-dir= with_plugin_dir_setting="$with_plugin_dir" enable_plugin_dir=yes fi diff --git a/docs/filters.md b/docs/filters.md index 26cffce152..e701667e85 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -1042,44 +1042,77 @@ So aside from those four, the current set of standard filters is as follows. bzip2307https://sourceware.org/bzip2/ +It is important to note that in order to use each standard filter, several additonal libraries must be installed. +Consider the zstandard compressor, which is one of the supported standard filters. +When installing the netcdf library, the following other libraries must be installed. + +1. *libzstd.so* | *zstd.dll* | *libzstd.0.dylib* -- The actual zstandard compressor library; typically installed by using your platform specific package manager. +2. The HDF5 wrapper for *libzstd.so* -- There are several options for obtaining this (see Appendix G.) +3. (Optional) The Zarr wrapper for *libzstd.so* -- you need this if you intend to read/write Zarr datasets that were compressed using zstandard; again see Appendix G. + # Appendix G. Finding Filters -A major problem for filter users is finding an implementation for a filter. -There are several ways to do this. +A major problem for filter users is finding an implementation of an HDF5 filter wrapper and (optionally) +its corresponding NCZarr wrapper. There are several ways to do this. + +* **--with-plugin-dir** — An option to *./configure* that will install the necessary wrappers. + See Appendix H. * **HDF5 Assigned Filter Identifiers Repository [3]** — HDF5 maintains a page of standard filter identifiers along with additional contact information. This often includes a pointer -to source code. +to source code. This will provide only HDF5 wrappers and not NCZarr wrappers. * **Community Codec Repository** — The Community Codec Repository (CCR) project [8] provides filters, including HDF5 wrappers, for a number of filters. +It does not as yet provide Zarr wrappers. You can install this library to get access to these supported filters. It does not currently include the required NCZarr Codec API, so they are only usable with netcdf-4. This will change in the future. -* **NetCDF-C Test Plugins Directory** — -As part of the overall build process, a number of filters are built as shared libraries in the "plugins" directory. -They may be in that directory or the "plugins/.libs" subdirectory. -It may be possible for users to utilize some of those libraries to provide filter support for general use. +# Appendix H. Auto-Install of Filter Wrappers +As part of the overall build process, a number of filter wrappers are built as shared libraries in the "plugins" directory. +These wrappers can be installed as part of the overall netcdf-c installation process. +WARNING: the installer still needs to make sure that the actual filter/compression libraries are installed: e.g. libzstd and/or libblosc. - If the user is using NCZarr filters, then the plugins directory has at least the following shared libraries - * libh5shuffle.so — shuffle filter - * libh5fletcher32.so — fletcher32 checksum - * libh5deflate.so — deflate compression - * libnczdefaults.so — provide NCZarr support for shuffle, fletcher32, and deflate. - * *libh5bzip2.so* — an HDF5 filter for bzip2 compression - * *libh5blosc.so* — an HDF5 filter for blosc compression - * *libh5zstd.so* — an HDF5 filter for zstandard compression - - The shuffle, fletcher32, and deflate filters in this case will - be ignored by HDF5 and only used by the NCZarr code. But in - order to use them, it needs additional Codec capabilities - provided by the libnczdefauts.so shared library. Note also that - if you disable HDF5 support, but leave NCZarr support enabled, - then all of the above filters should continue to work. +The target location into which libraries in the "plugins" directory are installed is specified +using a special *./configure* option +```` +--with-plugin-dir= +or +--with-plugin-dir +```` +or its corresponding *cmake* option. +```` +-DPLUGIN_INSTALL_DIR= +or +-DPLUGIN_INSTALL_DIR=YES +```` +If the option is specified with no argument (automake) or with the value "YES" (CMake), +then it defaults (in order) to the following directories: +1. If the HDF5_PLUGIN_PATH environment variable is defined, then last directory in the list of directories in the path is used. +2. (a) "/usr/local/hdf5/lib/plugin” for linux/unix operating systems (including Cygwin)
+ (b) “%ALLUSERSPROFILE%\\hdf5\\lib\\plugin” for Windows and MinGW + +If NCZarr is enabled, then in addition to wrappers for the standard filters, +additional libraries will be installed to support NCZarr access to filters. +Currently, this list includes the following: +* shuffle — shuffle filter +* fletcher32 — fletcher32 checksum +* deflate — deflate compression +* (optional) szip — szip compression, if libsz is available +* bzip2 — an HDF5 filter for bzip2 compression +* lib__nczh5filters.so — provide NCZarr support for shuffle, fletcher32, deflate, and (optionally) szip. +* lib__nczstdfilters.so — provide NCZarr support for bzip2, (optionally)zstandard, and (optionally) blosc. + +The shuffle, fletcher32, and deflate filters in this case will +be ignored by HDF5 and only used by the NCZarr code. But in +order to use them, it needs additional Codec capabilities +provided by the *lib__nczh5filters.so* shared library. Note also that +if you disable HDF5 support, but leave NCZarr support enabled, +then all of the above filters should continue to work. ## HDF5_PLUGIN_PATH @@ -1092,16 +1125,23 @@ wrappers but also the NCZarr codec wrappers. path-list. That is it is a sequence of absolute directory paths separated by a specific separator character. For Windows, the separator character is a semicolon (';') and for Unix, it is a a -colon (':'). For convenience, NCZarr will also accept the -semicolon separator for Unix. +colon (':'). + +So, if HDF5_PLUGIN_PATH is defined at build time, and +*--with-plugin-dir* is specified with no argument then the last +directory in the path will be the one into which filter wrappers are +installed. Otherwise the default directories are used. + +The important thing to note is that at run-time, there are several cases to consider: -So, the user can add the CCR and/or the plugins directory to -the *HDF5\_PLUGIN\_PATH* environment variable to allow the netcdf-c -library to locate wrappers. +1. HDF5_PLUGIN_PATH is defined and is the same value as it was at build time -- no action needed +2. HDF5_PLUGIN_PATH is defined and is has a different value from build time -- the user is responsible for ensuring that the run-time path includes the same directory used at build time, otherwise this case will fail. +3. HDF5_PLUGIN_DIR is not defined at either run-time or build-time -- no action needed +4. HDF5_PLUGIN_DIR is not defined at run-time but was defined at build-time -- this will probably fail # Point of Contact {#filters_poc} *Author*: Dennis Heimbigner
*Email*: dmh at ucar dot edu
*Initial Version*: 1/10/2018
-*Last Revised*: 3/14/2022 +*Last Revised*: 5/18/2022 diff --git a/liblib/Makefile.am b/liblib/Makefile.am index 7a6f7eb0c7..b456aa439a 100644 --- a/liblib/Makefile.am +++ b/liblib/Makefile.am @@ -18,7 +18,7 @@ lib_LTLIBRARIES = libnetcdf.la # for information regarding incrementing `-version-info`. ## -libnetcdf_la_LDFLAGS = -version-info 19:1:0 ${NOUNDEFINED} +libnetcdf_la_LDFLAGS = -version-info @netCDF_SO_VERSION@ ${NOUNDEFINED} libnetcdf_la_CPPFLAGS = ${AM_CPPFLAGS} libnetcdf_la_LIBADD = diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 90f05b9652..b74e4132ba 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -1145,12 +1145,10 @@ static int NCZ_load_all_plugins(void) { int i,j,ret = NC_NOERR; - const char* pluginroot = NULL; + const char* pluginroots = NULL; struct stat buf; NClist* dirs = nclistnew(); -#ifdef _WIN32 - char pluginpath32[4096]; -#endif + char* defaultpluginpath = NULL; ZTRACE(6,""); @@ -1158,29 +1156,37 @@ NCZ_load_all_plugins(void) fprintf(stderr,">>> DEBUGL: NCZ_load_all_plugins\n"); #endif - /* Find the plugin directory root(s) */ - pluginroot = getenv(plugin_env); - if(pluginroot == NULL || strlen(pluginroot) == 0) { + /* Setup the plugin path default */ + { #ifdef _WIN32 const char* win32_root; - win32_root = getenv(win32_root_env); + char dfalt[4096]; + win32_root = getenv(WIN32_ROOT_ENV); if(win32_root != NULL && strlen(win32_root) > 0) { - snprintf(pluginpath32,sizeof(pluginpath32),plugin_dir_win,win32_root); - pluginroot = pluginpath32; - } else - pluginroot = NULL; + snprintf(dfalt,sizeof(dfalt),PLUGIN_DIR_WIN,win32_root); + defaultpluginpath = strdup(dfalt); + } #else /*!_WIN32*/ - pluginroot = plugin_dir_unix; + defaultpluginpath = strdup(PLUGIN_DIR_UNIX); #endif } - ZTRACEMORE(6,"pluginroot=%s",(pluginroot?pluginroot:"null")); - if(pluginroot == NULL) { - ZLOG(NCLOGERR,"no pluginroot: %s",plugin_env); - ret = NC_ENOFILTER; goto done; + /* Find the plugin directory root(s) */ + pluginroots = getenv(PLUGIN_ENV); /* Usually HDF5_PLUGIN_PATH */ + if(pluginroots != NULL && strlen(pluginroots) == 0) pluginroots = NULL; + if(pluginroots == NULL) { + pluginroots = strdup(defaultpluginpath); } + assert(pluginroots != NULL); + ZTRACEMORE(6,"pluginroots=%s",(pluginroots?pluginroots:"null")); + + if((ret = NCZ_split_plugin_path(pluginroots,dirs))) goto done; - if((ret = NCZ_split_plugin_path(pluginroot,dirs))) goto done; + /* Add the default to end of the dirs list if not already there */ + if(!nclistmatch(dirs,defaultpluginpath,0)) { + nclistpush(dirs,defaultpluginpath); + defaultpluginpath = NULL; + } for(i=0;i>> \n"); h5id = codec->hdf5id; if((stat = NCZ_plugin_loaded(codec->hdf5id,&plugin))) goto done; } + if(plugin == NULL) { /* create new entry */ if((plugin = (NCZ_Plugin*)calloc(1,sizeof(NCZ_Plugin)))==NULL) {stat = NC_ENOMEM; goto done;} - } - + } + /* Fill in the plugin */ if(h5class != NULL && plugin->hdf5.filter == NULL) { plugin->hdf5.filter = h5class; diff --git a/libnczarr/zfilter.h b/libnczarr/zfilter.h index 6df40789bc..4a9ae0b101 100644 --- a/libnczarr/zfilter.h +++ b/libnczarr/zfilter.h @@ -20,10 +20,10 @@ #define ENCODING 1 /* list of environment variables to check for plugin roots */ -#define plugin_env "HDF5_PLUGIN_PATH" -#define plugin_dir_unix "/usr/local/hdf5/plugin" -#define plugin_dir_win "%s/hdf5/lib/plugin" -#define win32_root_env "ALLUSERSPROFILE" +#define PLUGIN_ENV "HDF5_PLUGIN_PATH" +#define PLUGIN_DIR_UNIX "/usr/local/hdf5/plugin" +#define PLUGIN_DIR_WIN "%s/hdf5/lib/plugin" +#define WIN32_ROOT_ENV "ALLUSERSPROFILE" /* Opaque */ struct NCZ_Filter; diff --git a/nc_test4/findplugin.in b/nc_test4/findplugin.in index 423221b82b..7146bb0382 100755 --- a/nc_test4/findplugin.in +++ b/nc_test4/findplugin.in @@ -34,21 +34,21 @@ findpluginext() { # Also assumes that misc filter is always built # Approach is to use find to see what is in plugins directory. TSO=`find ${TOPBUILDDIR}/plugins -name '*misc.so'` - TDY=`find ${TOPBUILDDIR}/plugins -name '*misc.0.dylib'` + TDY=`find ${TOPBUILDDIR}/plugins -name '*misc.dylib'` TCYG=`find ${TOPBUILDDIR}/plugins -name 'cyg*misc.dll'` TDLL=`find ${TOPBUILDDIR}/plugins -name '*misc.dll'` if test "x$TSO" != x ; then FP_PLUGIN_EXT="so" - FP_PLUGIN_PRE="lib" + FP_PLUGIN_PRE="lib__nc" elif test "x$TDY" != x ; then - FP_PLUGIN_EXT="0.dylib" - FP_PLUGIN_PRE="lib" + FP_PLUGIN_EXT="dylib" + FP_PLUGIN_PRE="lib__nc" elif test "x$TCYG" != x ; then FP_PLUGIN_EXT="dll" - FP_PLUGIN_PRE="cyg" + FP_PLUGIN_PRE="cyg__nc" elif test "x$TDLL" != x ; then FP_PLUGIN_EXT="dll" - FP_PLUGIN_PRE="" + FP_PLUGIN_PRE="__nc" else # unknown unset FP_PLUGIN_EXT unset FP_PLUGIN_PRE diff --git a/nc_test4/tst_unknown.sh b/nc_test4/tst_unknown.sh index 52c81ce1a4..76a7b1b6ab 100755 --- a/nc_test4/tst_unknown.sh +++ b/nc_test4/tst_unknown.sh @@ -53,6 +53,8 @@ UNKNOWNDIR="${HDF5_PLUGIN_DIR}" UNKNOWNLIB="${HDF5_PLUGIN_LIB}" UNKNOWNFILTER="${HDF5_PLUGIN_DIR}/${UNKNOWNLIB}" +# Getting the name is especially tricky for dylib, which puts the version before the .dylib + # Verify if ! test -f ${UNKNOWNFILTER} ; then echo "Unable to locate ${UNKNOWNFILTER}"; exit 1; fi @@ -67,7 +69,6 @@ file="tmp_known_${zfilt}.nc" rm -f $file fileurl="$file" fi -printenv HDF5_PLUGIN_PATH # build .nc file using unknown ${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/unknown.cdl # dump and clean file when filter is avail @@ -78,9 +79,9 @@ sclean ./tmp_known_$zext.txt tmp_known_$zext.dump rm -fr ${UNKNOWNDIR}/save mkdir -p ${UNKNOWNDIR}/save # Figure out all matching libs; make sure to remove .so, so.0, etc -LSRC=`${execdir}/../ncdump/ncpathcvt -F ${UNKNOWNFILTER}` +LSRC=`${execdir}/../ncdump/ncpathcvt -F "${UNKNOWNDIR}"` LDST=`${execdir}/../ncdump/ncpathcvt -F ${UNKNOWNDIR}/save` -mv ${LSRC}* ${LDST} +mv ${LSRC}/*unknown* ${LDST} # Verify that the filter is no longer defined # Try to read the data; should xfail if ${NCDUMP} -s $fileurl > ./tmp_unk_$zext.dmp ; then @@ -91,9 +92,7 @@ echo "*** XFAIL: filter not found" found=0 fi # Restore the filter -LSRC="${LDST}" -LDST=`${execdir}/../ncdump/ncpathcvt -F ${UNKNOWNDIR}` -mv ${LSRC}/* ${LDST} +mv ${LDST}/*unknown* ${LSRC} rm -fr ${UNKNOWNDIR}/save if test "x$found" = x1 ; then exit 1; fi } diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index 42219af230..ca43d28b35 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -129,8 +129,8 @@ run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_ncgen4.sh \ run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh run_s3_cleanup.sh \ run_purezarr.sh run_interop.sh run_misc.sh \ run_filter.sh run_specific_filters.sh \ -run_newformat.sh run_nczarr_fill.sh run_quantize.sh run_nczfilter.sh \ -run_jsonconvention.sh run_unknown.sh +run_newformat.sh run_nczarr_fill.sh run_quantize.sh \ +run_jsonconvention.sh run_nczfilter.sh run_unknown.sh EXTRA_DIST += \ ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \ diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index d88768ded0..7634a897be 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -6,6 +6,18 @@ # See netcdf-c/COPYRIGHT file for more info. SET(CMAKE_BUILD_TYPE "") +IF(MSVC) + SET(PLUGINEXT "dll") + SET(PLUGINPRE "__nc") +ELSE() + SET(PLUGINPRE "__nc") # Note that CMake will prepend "lib" + if(APPLE) + SET(PLUGINEXT "dylib") + ELSE() + SET(PLUGINEXT "so") + ENDIF() +ENDIF() + FILE(READ H5Znoop.c NOOP_SOURCE) FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "#define NOOP_INSTANCE 1\n") FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "${NOOP_SOURCE}") @@ -36,10 +48,11 @@ SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG "${CMAKE_CURRENT_BINARY_DIR}") SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE "${CMAKE_CURRENT_BINARY_DIR}") MACRO(buildplugin TARGET TARGETLIB) + SET(MANGLE "${PLUGINPRE}${TARGETLIB}") ADD_LIBRARY(${TARGET} MODULE ${${TARGET}_SOURCES}) - SET_TARGET_PROPERTIES(${TARGET} PROPERTIES LIBRARY_OUTPUT_NAME ${TARGETLIB}) - SET_TARGET_PROPERTIES(${TARGET} PROPERTIES ARCHIVE_OUTPUT_NAME ${TARGETLIB}) - SET_TARGET_PROPERTIES(${TARGET} PROPERTIES RUNTIME_OUTPUT_NAME ${TARGETLIB}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES LIBRARY_OUTPUT_NAME ${MANGLE}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES ARCHIVE_OUTPUT_NAME ${MANGLE}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES RUNTIME_OUTPUT_NAME ${MANGLE}) TARGET_LINK_LIBRARIES(${TARGET} ${ALL_TLL_LIBS};${ARGN}) IF(MSVC) target_compile_options(${TARGET} PRIVATE /Zi) @@ -59,9 +72,9 @@ buildplugin(h5shuffle "h5shuffle") buildplugin(h5fletcher32 "h5fletcher32") buildplugin(h5deflate "h5deflate") -buildplugin(nczmisc "nczmisc") -buildplugin(nczhdf5filters "nczhdf5filters" netcdf) -buildplugin(nczstdfilters "nczstdfilters" netcdf) +buildplugin(nczmisc "zmisc") +buildplugin(nczhdf5filters "zhdf5filters" netcdf) +buildplugin(nczstdfilters "zstdfilters" netcdf) IF(ENABLE_BLOSC) SET(h5blosc_SOURCES H5Zblosc.c) @@ -92,13 +105,9 @@ ENDIF() IF(ENABLE_PLUGIN_INSTALL) MACRO(installplugin PLUG) -IF(MSVC) - SET(BUILD_PLUGIN_LIB "${PLUG}.dll") -ELSE() - SET(BUILD_PLUGIN_LIB "lib${PLUG}.so") -ENDIF() -MESSAGE(STATUS "Installing: ${BUILD_PLUGIN_LIB} into ${PLUGIN_INSTALL_DIR}") -install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${BUILD_PLUGIN_LIB} DESTINATION ${PLUGIN_INSTALL_DIR}) + SET(BUILD_PLUGIN_LIB "${PLUGINPRE}${PLUG}.${PLUGINEXT}") + MESSAGE(STATUS "Installing: ${BUILD_PLUGIN_LIB} into ${PLUGIN_INSTALL_DIR}") + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${BUILD_PLUGIN_LIB} DESTINATION ${PLUGIN_INSTALL_DIR}) ENDMACRO() install(DIRECTORY DESTINATION ${PLUGIN_INSTALL_DIR}) @@ -115,8 +124,8 @@ IF(ENABLE_NCZARR) installplugin(h5fletcher32) installplugin(h5shuffle) installplugin(h5deflate) - installplugin(nczhdf5filters) - installplugin(nczstdfilters) + installplugin(zhdf5filters) + installplugin(zstdfilters) IF(Szip_FOUND) installplugin(h5szip) ENDIF() diff --git a/plugins/Makefile.am b/plugins/Makefile.am index fa839264ff..779d7c8663 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -50,10 +50,14 @@ EXTRA_DIST += H5checksum.c if ENABLE_FILTER_TESTING if ENABLE_NCZARR_FILTERS -plugin_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la -libh5shuffle_la_SOURCES = H5Zshuffle.c -libh5fletcher32_la_SOURCES = H5Zfletcher32.c H5checksum.c -libh5deflate_la_SOURCES = H5Zdeflate.c +plugin_LTLIBRARIES += lib__nch5fletcher32.la lib__nch5shuffle.la lib__nch5deflate.la +lib__nch5shuffle_la_SOURCES = H5Zshuffle.c +lib__nch5fletcher32_la_SOURCES = H5Zfletcher32.c H5checksum.c +lib__nch5deflate_la_SOURCES = H5Zdeflate.c + +lib__nch5shuffle_la_LDFLAGS = ${plugin_version_info} +lib__nch5deflate_la_LDFLAGS = ${plugin_version_info} +lib__nch5fletcher32_la_LDFLAGS = ${plugin_version_info} libh5shuffle_la_LDFLAGS = ${plugin_version_info} libh5deflate_la_LDFLAGS = ${plugin_version_info} @@ -61,9 +65,9 @@ libh5fletcher32_la_LDFLAGS = ${plugin_version_info} # Need our version of szip if libsz available and we are not using HDF5 if HAVE_SZ -plugin_LTLIBRARIES += libh5szip.la -libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h -libh5szip_la_LDFLAGS = ${plugin_version_info} +plugin_LTLIBRARIES += lib__nch5szip.la +lib__nch5szip_la_SOURCES = H5Zszip.c H5Zszip.h +lib__nch5szip_la_LDFLAGS = ${plugin_version_info} endif endif # ENABLE_NCZARR_FILTERS @@ -71,54 +75,57 @@ endif # ENABLE_NCZARR_FILTERS if ENABLE_PLUGINS # The NCZarr codec libraries -libnczstdfilters_la_SOURCES = NCZstdfilters.c -libnczhdf5filters_la_SOURCES = NCZhdf5filters.c +lib__nczstdfilters_la_SOURCES = NCZstdfilters.c +lib__nczhdf5filters_la_SOURCES = NCZhdf5filters.c + +plugin_LTLIBRARIES += lib__nczhdf5filters.la +plugin_LTLIBRARIES += lib__nczstdfilters.la plugin_LTLIBRARIES += libnczhdf5filters.la plugin_LTLIBRARIES += libnczstdfilters.la if HAVE_BLOSC -libh5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h -libh5blosc_la_LDFLAGS = ${plugin_version_info} -plugin_LTLIBRARIES += libh5blosc.la +lib__nch5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h +lib__nch5blosc_la_LDFLAGS = ${plugin_version_info} +plugin_LTLIBRARIES += lib__nch5blosc.la endif if HAVE_ZSTD -libh5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h -libh5zstd_la_LDFLAGS = ${plugin_version_info} -plugin_LTLIBRARIES += libh5zstd.la +lib__nch5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h +lib__nch5zstd_la_LDFLAGS = ${plugin_version_info} +plugin_LTLIBRARIES += lib__nch5zstd.la endif endif #ENABLE_PLUGINS # The noop filter is to allow testing of multifilters and filter order # Need two distinct instances -libh5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h -libh5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h -libh5noop_la_LDFLAGS = ${plugin_version_info} -libh5noop1_la_LDFLAGS = ${plugin_version_info} +lib__nch5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h +lib__nch5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h +lib__nch5noop_la_LDFLAGS = ${plugin_version_info} +lib__nch5noop1_la_LDFLAGS = ${plugin_version_info} # The misc filter is to allow testing of filter arguments -libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h -libh5misc_la_LDFLAGS = ${plugin_version_info} -libnczmisc_la_SOURCES = NCZmisc.c -libnczmisc_la_LDFLAGS = ${plugin_version_info} +lib__nch5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h +lib__nch5misc_la_LDFLAGS = ${plugin_version_info} +lib__nczmisc_la_SOURCES = NCZmisc.c +lib__nczmisc_la_LDFLAGS = ${plugin_version_info} # Provide a filter to test missing filter -libh5unknown_la_SOURCES = H5Zunknown.c -libh5unknown_la_LDFLAGS = ${plugin_version_info} +lib__nch5unknown_la_SOURCES = H5Zunknown.c +lib__nch5unknown_la_LDFLAGS = ${plugin_version_info} -tmp_LTLIBRARIES += libh5noop.la libh5noop1.la libh5misc.la libnczmisc.la libh5unknown.la +tmp_LTLIBRARIES += lib__nch5noop.la lib__nch5noop1.la lib__nch5misc.la lib__nczmisc.la lib__nch5unknown.la # Bzip2 is used to test more complex filters -libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h +lib__nch5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE if HAVE_LOCAL_BZ2 -libh5bzip2_la_SOURCES += ${BZIP2SRC} +lib__nch5bzip2_la_SOURCES += ${BZIP2SRC} endif -libh5bzip2_la_LDFLAGS = ${plugin_version_info} -plugin_LTLIBRARIES += libh5bzip2.la +lib__nch5bzip2_la_LDFLAGS = ${plugin_version_info} +plugin_LTLIBRARIES += lib__nch5bzip2.la endif #ENABLE_FILTER_TESTING From 8037b45f7146c2b3883f3618fa2b879a1eabe9e5 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Thu, 19 May 2022 22:43:08 -0600 Subject: [PATCH 05/10] typo --- ncdump/tst_output.sh | 2 +- plugins/Makefile.am | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/ncdump/tst_output.sh b/ncdump/tst_output.sh index d50564c2af..a3fd20a6c3 100755 --- a/ncdump/tst_output.sh +++ b/ncdump/tst_output.sh @@ -1,4 +1,4 @@ -.#!/bin/sh +#!/bin/sh if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 779d7c8663..777495ff65 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -59,10 +59,6 @@ lib__nch5shuffle_la_LDFLAGS = ${plugin_version_info} lib__nch5deflate_la_LDFLAGS = ${plugin_version_info} lib__nch5fletcher32_la_LDFLAGS = ${plugin_version_info} -libh5shuffle_la_LDFLAGS = ${plugin_version_info} -libh5deflate_la_LDFLAGS = ${plugin_version_info} -libh5fletcher32_la_LDFLAGS = ${plugin_version_info} - # Need our version of szip if libsz available and we are not using HDF5 if HAVE_SZ plugin_LTLIBRARIES += lib__nch5szip.la @@ -81,9 +77,6 @@ lib__nczhdf5filters_la_SOURCES = NCZhdf5filters.c plugin_LTLIBRARIES += lib__nczhdf5filters.la plugin_LTLIBRARIES += lib__nczstdfilters.la -plugin_LTLIBRARIES += libnczhdf5filters.la -plugin_LTLIBRARIES += libnczstdfilters.la - if HAVE_BLOSC lib__nch5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h lib__nch5blosc_la_LDFLAGS = ${plugin_version_info} From aa7d9524e056612b00ecfc86f0f1cab6ca4592e2 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Fri, 20 May 2022 14:08:53 -0600 Subject: [PATCH 06/10] Update documentation --- docs/filters.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/filters.md b/docs/filters.md index e701667e85..f6685577e9 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -8,6 +8,17 @@ NetCDF-4 Filter Support {#filters} \tableofcontents +# Feature Overview {#filters_overview} + +NetCDF-C filters have some features of which the user +should be aware. + +## Auto Install of filters +An option is now provided to automatically install +HDF5 filters into a default location, or optionally +into a user-specified location. This is described in +Appendix H (with supporting information in Appendix G). + # Introduction to Filters {#filters_introduction} The netCDF library supports a general filter mechanism to apply From aaa7be64d742c484838be8e7dfcb6ccdbd0a4ec2 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 22 May 2022 16:16:41 -0600 Subject: [PATCH 07/10] Update docs again --- docs/filters.md | 76 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/docs/filters.md b/docs/filters.md index f6685577e9..a07279c70e 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -13,11 +13,20 @@ NetCDF-4 Filter Support {#filters} NetCDF-C filters have some features of which the user should be aware. -## Auto Install of filters +* ***Auto Install of filters***
An option is now provided to automatically install HDF5 filters into a default location, or optionally into a user-specified location. This is described in -Appendix H (with supporting information in Appendix G). +[Appendix H](#filters_appendixh) +(with supporting information in [Appendix G](#filters_appendixg)). + +* ***NCZarr Filter Support***
+[NCZarr filters](#filters_nczarr) are now supported. +This essentially means that it is possible to specify +Zarr Codecs (Zarr equivalent of filters) in Zarr files +and have them processed using HDF5-style wrapper shared libraries. +Zarr filters can be used even if HDF5 support is disabled +in the netCDF-C library. # Introduction to Filters {#filters_introduction} @@ -59,10 +68,10 @@ A user may encounter an incompatibility if any of the following appears in user * The function *\_nc\_inq\_var\_filter* was returning the error value NC\_ENOFILTER if a variable had no associated filters. It has been reverted to the previous case where it returns NC\_NOERR and the returned filter id was set to zero if the variable had no filters. * The function *nc\_inq\_var\_filterids* was renamed to *nc\_inq\_var\_filter\_ids*. -* Some auxilliary functions for parsing textual filter specifications have been moved to the file *netcdf\_aux.h*. See Appendix A. +* Some auxilliary functions for parsing textual filter specifications have been moved to the file *netcdf\_aux.h*. See [Appendix A](#filters_appendixa). * All of the "filterx" functions have been removed. This is unlikely to cause problems because they had limited visibility. -For additional information, see Appendix B. +For additional information, see [Appendix B](#filters_appendixb). # Enabling A HDF5 Compression Filter {#filters_enable} @@ -303,7 +312,7 @@ The rules for all possible cases of the "-F none" flag are defined by this table # Filter Specification Syntax {#filters_syntax} The utilities ncgen and nccopy, and also the output of *ncdump*, support the specification of filter ids, formats, and parameters in text format. -The BNF specification is defined in Appendix C. +The BNF specification is defined in [Appendix C](#filters_appendixc). Basically, These specifications consist of a filter id, a comma, and then a sequence of comma separated constants representing the parameters. The constants are converted within the utility to a proper set of unsigned int constants (see the parameter encoding section). @@ -341,14 +350,33 @@ Each filter is assumed to be compiled into a separate dynamically loaded library For HDF5 conformant filters, these filter libraries are assumed to be in some specific location. The details for writing such a filter are defined in the HDF5 documentation[1,2]. -## Plugin directory {#filters_Plugindir} +## Plugin directory {#filters_plugindir} + +The HDF5 loader searches for plugins in a number of directories. +This search is contingent on the presence or absence of the environment +variable named ***HDF5_PLUGIN_PATH***. + +As with all other "...PATH" variables, it is a sequence of absolute +directories separated by a separator character. For *nix* operating systems, +this separator is the colon (':') character. For Windows and Mingw, the +separator is the semi-colon (';') character. So for example: +* Linux: export HDF5_PLUGIN_PATH=/usr/lib:/usr/local/lib +* Windows: export HDF5_PLUGIN_PATH=c:\ProgramData\hdf5\plugin;c:\tools\lib + +If HDF5_PLUGIN_PATH is defined, then the loader will search each directory +in the path from left to right looking for shared libraries with specific +exported symbols representing the entry points into the library. -The HDF5 loader expects plugins to be in a specified plugin directory. -The default directory is: -* "/usr/local/hdf5/lib/plugin” for linux/unix operating systems (including Cygwin) -* “%ALLUSERSPROFILE%\\hdf5\\lib\\plugin” for Windows systems, although the code does not appear to explicitly use this path. +If HDF5_PLUGIN_PATH is not defined, the loader defaults to using +these default directories: +* Linux: "/usr/local/hdf5/lib/plugin” +* Windows: “%ALLUSERSPROFILE%\\hdf5\\lib\\plugin” -The default may be overridden using the environment variable *HDF5\_PLUGIN\_PATH*. +It should be noted that there is a difference between the search order +for HDF5 versus NCZarr. The HDF5 loader will search only the directories +specificed in HDF5_PLUGIN_PATH. In NCZarr, the loader +searches HDF5_PLUGIN_PATH and as a last resort, +it also searches the default directory. ## Plugin Library Naming {#filters_Pluginlib} @@ -356,15 +384,17 @@ Given a plugin directory, HDF5 examines every file in that directory that confor
PlatformBasenameExtension
Linuxlib*.so* -
OSXlib*.so* +
OSXlib*.dylib*
Cygwincyg*.dll*
Windows*.dll
## Plugin Verification {#filters_Pluginverify} -For each dynamic library located using the previous patterns, HDF5 attempts to load the library and attempts to obtain information from it. -Specifically, It looks for two functions with the following signatures. +For each dynamic library located using the previous patterns, +HDF5 attempts to load the library and attempts to obtain +information from it. Specifically, It looks for two functions +with the following signatures. 1. *H5PL\_type\_t H5PLget\_plugin\_type(void)* — This function is expected to return the constant value *H5PL\_TYPE\_FILTER* to indicate that this is a filter library. 2. *const void* H5PLget\_plugin\_info(void)* — This function returns a pointer to a table of type *H5Z\_class2\_t*. @@ -502,7 +532,7 @@ The netcdf-c library processes all of the shared libraries by interrogating each Any libraries that do not export one or both of the well-known APIs is ignored. Internally, the netcdf-c library pairs up each HDF5 library API with a corresponding Codec API by invoking the relevant well-known functions -(See Appendix E/a>). +(See [Appendix E](#filters_appendixe). This results in this table for associated codec and hdf5 libraries.
HDF5 APICodec APIAction @@ -535,7 +565,7 @@ The netcdf-c library examines its list of known filters to find one matching the The set of parameters provided is stored internally. Then during writing of data, the corresponding HDF5 filter is invoked to encode the data. -When it comes time to write out the meta-data, the stored HDF5-style parameters are passed to a specific Codec function to obtain the corresponding JSON representation. Again see Appendix E. +When it comes time to write out the meta-data, the stored HDF5-style parameters are passed to a specific Codec function to obtain the corresponding JSON representation. Again see [Appendix E](#filters_appendixe). This resulting JSON is then written in the NCZarr metadata. ### Reading an NCZarr Container @@ -810,7 +840,7 @@ This leads to the following set of rules. Because of the encoding rules, this 8-byte value will be in LE format. 4. The filter must finally do an 8-byte byte-swap on that 8-byte value to convert it to desired BE format. -To support these rules, some utility programs exist and are discussed in Appendix B. +To support these rules, some utility programs exist and are discussed in [Appendix B](#filters_appendixb). # Appendix B. Support Utilities {#filters_appendixb} @@ -1028,7 +1058,7 @@ The list of returned items are used to try to provide defaults for any HDF5 filters that have no corresponding Codec. This is for internal use only. -# Appendix F. Standard Filters +# Appendix F. Standard Filters {#filters_appendixf} Support for a select set of standard filters is built into the NetCDF API. Generally, they are accessed using the following generic API, where XXXX is @@ -1058,16 +1088,16 @@ Consider the zstandard compressor, which is one of the supported standard filter When installing the netcdf library, the following other libraries must be installed. 1. *libzstd.so* | *zstd.dll* | *libzstd.0.dylib* -- The actual zstandard compressor library; typically installed by using your platform specific package manager. -2. The HDF5 wrapper for *libzstd.so* -- There are several options for obtaining this (see Appendix G.) -3. (Optional) The Zarr wrapper for *libzstd.so* -- you need this if you intend to read/write Zarr datasets that were compressed using zstandard; again see Appendix G. +2. The HDF5 wrapper for *libzstd.so* -- There are several options for obtaining this (see [Appendix G](#filters_appendixg).) +3. (Optional) The Zarr wrapper for *libzstd.so* -- you need this if you intend to read/write Zarr datasets that were compressed using zstandard; again see [Appendix G](#filters_appendixg). -# Appendix G. Finding Filters +# Appendix G. Finding Filters {#filters_appendixg} A major problem for filter users is finding an implementation of an HDF5 filter wrapper and (optionally) its corresponding NCZarr wrapper. There are several ways to do this. * **--with-plugin-dir** — An option to *./configure* that will install the necessary wrappers. - See Appendix H. + See [Appendix H](#filters_appendixh). * **HDF5 Assigned Filter Identifiers Repository [3]** — HDF5 maintains a page of standard filter identifiers along with @@ -1082,7 +1112,7 @@ You can install this library to get access to these supported filters. It does not currently include the required NCZarr Codec API, so they are only usable with netcdf-4. This will change in the future. -# Appendix H. Auto-Install of Filter Wrappers +# Appendix H. Auto-Install of Filter Wrappers {#filters_appendixh} As part of the overall build process, a number of filter wrappers are built as shared libraries in the "plugins" directory. These wrappers can be installed as part of the overall netcdf-c installation process. From 7bf06df1c3d9a56e1ac5e16251b64230c3383fd1 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 22 May 2022 16:40:55 -0600 Subject: [PATCH 08/10] rebuild --- docs/filters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/filters.md b/docs/filters.md index a07279c70e..be65aca6b7 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -1,7 +1,7 @@ NetCDF-4 Filter Support ============================ - + NetCDF-4 Filter Support {#filters} ================================== From d7e57d261a0a7e011a67f56d51266213bf6b6bbb Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Tue, 24 May 2022 20:05:19 -0600 Subject: [PATCH 09/10] Update to default --with-plugin-dir to yes --- CMakeLists.txt | 38 ++++++++++++++------------------- configure.ac | 15 +++++++------ docs/filters.md | 12 ++++++++++- nc_test/Makefile.am | 8 +++---- nc_test/run_pnetcdf_tests.sh.in | 1 - nc_test4/tst_filterinstall.sh | 2 ++ nczarr_test/Makefile.am | 8 +++---- plugins/CMakeLists.txt | 17 ++++++++------- 8 files changed, 54 insertions(+), 47 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 05870e762f..6a3629b05d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1118,9 +1118,9 @@ ELSE() set(STD_FILTERS "${STD_FILTERS} bz2") ENDIF() -# If user wants, then install selected plugins -SET(PLUGIN_INSTALL_DIR "" CACHE STRING "Whether and where we should install plugins") -SET(ENABLE_PLUGIN_INSTALL NO) +# If user wants, then install selected plugins (default on) +SET(PLUGIN_INSTALL_DIR "YES" CACHE STRING "Whether and where we should install plugins; defaults to yes") +SET(ENABLE_PLUGIN_INSTALL YES) IF(DEFINED PLUGIN_INSTALL_DIR) #OR DEFINED CACHE{PLUGIN_INSTALL_DIR} STRING(TOLOWER "${PLUGIN_INSTALL_DIR}" LCPID) IF(LCPID STREQUAL "no") @@ -1130,36 +1130,30 @@ IF(DEFINED PLUGIN_INSTALL_DIR) #OR DEFINED CACHE{PLUGIN_INSTALL_DIR} ENDIF() ENDIF() -IF(PLUGIN_INSTALL_DIR STREQUAL "" OR PLUGIN_INSTALL_DIR STREQUAL "yes") - SET(PLUGIN_INSTALL_DIR "YES") -ENDIF() - IF(ENABLE_PLUGIN_INSTALL AND PLUGIN_INSTALL_DIR STREQUAL "YES") # Default to last dir (lowest search priority) in HDF5_PLUGIN_PATH - IF(DEFINED $ENV{HDF5_PLUGIN_PATH}) - SET(PLUGIN_PATH "$HDF5_PLUGIN_PATH") + IF(DEFINED ENV{HDF5_PLUGIN_PATH}) + SET(PLUGIN_PATH "$ENV{HDF5_PLUGIN_PATH}") ELSE() IF(ISMSVC OR ISMINGW) - SET(PLUGIN_PATH "$ENV{ALLUSERSPROFILE}\\hdfd5\\lib\\plugin") + SET(PLUGIN_PATH "$ENV{ALLUSERSPROFILE}\\hdf5\\lib\\plugin") ELSE() SET(PLUGIN_PATH "/usr/local/hdf5/lib/plugin") ENDIF() ENDIF() ENDIF() -# Assume PLUGIN_PATH is defined -# Use the lowest priority dir in the path -IF(NOT ISMSVC AND NOT ISMINGW) - STRING(REPLACE ":" ";" PATH_LIST ${PLUGIN_PATH}) -ELSE() - SET(PATH_LIST ${PLUGIN_PATH}) -ENDIF() - -# Get last element -LIST(GET PATH_LIST -1 PLUGIN_INSTALL_DIR) -MESSAGE(WARNING "Defaulting to -DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}") +IF(ENABLE_PLUGIN_INSTALL) + # Use the lowest priority dir in the path + IF(NOT ISMSVC AND NOT ISMINGW) + STRING(REPLACE ":" ";" PATH_LIST ${PLUGIN_PATH}) + ELSE() + SET(PATH_LIST ${PLUGIN_PATH}) + ENDIF() -IF(ENABLE_PLUGIN_INSTALL) + # Get last element + LIST(GET PATH_LIST -1 PLUGIN_INSTALL_DIR) + MESSAGE("Defaulting to -DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}") SET(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}") ELSE() # No option specified UNSET(PLUGIN_INSTALL_DIR) diff --git a/configure.ac b/configure.ac index 9ce6c6d8a4..cce149b3ec 100644 --- a/configure.ac +++ b/configure.ac @@ -1891,9 +1891,9 @@ AC_SUBST(STD_FILTERS,[$std_filters]) # If user wants, then install selected standard filters AC_MSG_CHECKING([whether and where we should install plugins]) -AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=], - [Install selected standard filters in specified directory])], - [],[with_plugin_dir=no]) +AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=|no|--without-plugin-dir], + [Install selected standard filters in specified or default directory])], + [],[with_plugin_dir=yes]) AC_MSG_RESULT([$with_plugin_dir]) if test "x$with_plugin_dir" = xno ; then # option missing|disabled with_plugin_dir=no @@ -1911,15 +1911,17 @@ elif test "x$with_plugin_dir" = xyes ; then # --with-plugin-dir, no argument fi # Use the lowest priority dir in the path if test "x$ISMSVC" = xyes || test "x$ISMINGW" = xyes; then - PLUGIN_DIR=`echo "$PLUGIN_PATH" | sed -e 's|\([^;]*[;]\)*\(.*\)|\2|'` + PLUGIN_DIR=`echo "$PLUGIN_PATH" | tr ';' ' '` else - PLUGIN_DIR=`echo "$PLUGIN_PATH" | sed -e 's|\([^:]*[:]\)*\(.*\)|\2|'` + PLUGIN_DIR=`echo "$PLUGIN_PATH" | tr ':' ' '` fi + for pp in ${PLUGIN_DIR} ; do last="$pp"; done + PLUGIN_DIR="$last" with_plugin_dir_setting="$PLUGIN_DIR" # canonical form is all forward slashes with_plugin_dir=`echo "$PLUGIN_DIR" | tr '\\\\' '/'` enable_plugin_dir=yes - AC_MSG_NOTICE([Defaulting to --with-plugin-path=$with_plugin_dir]) + AC_MSG_NOTICE([Defaulting to --with-plugin-dir=$with_plugin_dir]) else # --with-plugin-dir= with_plugin_dir_setting="$with_plugin_dir" enable_plugin_dir=yes @@ -2019,7 +2021,6 @@ AC_CONFIG_FILES(test_common.sh:test_common.in) AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(nczarr_test/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(plugins/findplugin.sh:nc_test4/findplugin.in) -AC_CONFIG_FILES(plugins/stdinstall.sh:plugins/stdinstall.in) AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in) AC_CONFIG_FILES([nc_test/run_pnetcdf_tests.sh:nc_test/run_pnetcdf_tests.sh.in],[chmod ugo+x nc_test/run_pnetcdf_tests.sh]) diff --git a/docs/filters.md b/docs/filters.md index be65aca6b7..82c2ad6c3d 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -1087,7 +1087,7 @@ It is important to note that in order to use each standard filter, several addit Consider the zstandard compressor, which is one of the supported standard filters. When installing the netcdf library, the following other libraries must be installed. -1. *libzstd.so* | *zstd.dll* | *libzstd.0.dylib* -- The actual zstandard compressor library; typically installed by using your platform specific package manager. +1. *libzstd.so* | *zstd.dll* | *libzstd.dylib* -- The actual zstandard compressor library; typically installed by using your platform specific package manager. 2. The HDF5 wrapper for *libzstd.so* -- There are several options for obtaining this (see [Appendix G](#filters_appendixg).) 3. (Optional) The Zarr wrapper for *libzstd.so* -- you need this if you intend to read/write Zarr datasets that were compressed using zstandard; again see [Appendix G](#filters_appendixg). @@ -1131,6 +1131,16 @@ or its corresponding *cmake* option. or -DPLUGIN_INSTALL_DIR=YES ```` +This option defaults to the value "yes", which means that filters are +installed by default. This can be disabled by one of the following options. +```` +--without-plugin-dir (automake) +or +--with-plugin-dir=no (automake) +or +-DPLUGIN_INSTALL_DIR=NO (CMake) +```` + If the option is specified with no argument (automake) or with the value "YES" (CMake), then it defaults (in order) to the following directories: 1. If the HDF5_PLUGIN_PATH environment variable is defined, then last directory in the list of directories in the path is used. diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am index e102346c6c..39ef40596f 100644 --- a/nc_test/Makefile.am +++ b/nc_test/Makefile.am @@ -7,10 +7,10 @@ # Ed Hartnett, Dennis Heimbigner, Ward Fisher # Un comment to use a more verbose test driver -SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -TESTS_ENVIRONMENT = export SETX=1; +#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT = export SETX=1; # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am diff --git a/nc_test/run_pnetcdf_tests.sh.in b/nc_test/run_pnetcdf_tests.sh.in index 2908f8d3f3..9ddb532afe 100755 --- a/nc_test/run_pnetcdf_tests.sh.in +++ b/nc_test/run_pnetcdf_tests.sh.in @@ -2,7 +2,6 @@ # This script runs some PnetCDF I/O tests -set -x set -e echo diff --git a/nc_test4/tst_filterinstall.sh b/nc_test4/tst_filterinstall.sh index 465a42141d..9b119d88e9 100755 --- a/nc_test4/tst_filterinstall.sh +++ b/nc_test4/tst_filterinstall.sh @@ -1,6 +1,8 @@ #!/bin/bash # Test the filter install +# This cannot be run as a regular test +# because installation will not have occurred if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index 261d3dcc9f..ca43d28b35 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -10,10 +10,10 @@ TESTS_ENVIRONMENT = #TEST_EXTENSIONS = .sh -SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -TESTS_ENVIRONMENT += export SETX=1; +#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT += export SETX=1; AM_CPPFLAGS += -I${top_srcdir} -I${top_srcdir}/libnczarr AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 7634a897be..8ca75c8eb6 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -10,7 +10,7 @@ IF(MSVC) SET(PLUGINEXT "dll") SET(PLUGINPRE "__nc") ELSE() - SET(PLUGINPRE "__nc") # Note that CMake will prepend "lib" + SET(PLUGINPRE "lib__nc") if(APPLE) SET(PLUGINEXT "dylib") ELSE() @@ -48,11 +48,12 @@ SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG "${CMAKE_CURRENT_BINARY_DIR}") SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE "${CMAKE_CURRENT_BINARY_DIR}") MACRO(buildplugin TARGET TARGETLIB) - SET(MANGLE "${PLUGINPRE}${TARGETLIB}") + SET(MANGLELIB "${PLUGINPRE}${TARGETLIB}") ADD_LIBRARY(${TARGET} MODULE ${${TARGET}_SOURCES}) - SET_TARGET_PROPERTIES(${TARGET} PROPERTIES LIBRARY_OUTPUT_NAME ${MANGLE}) - SET_TARGET_PROPERTIES(${TARGET} PROPERTIES ARCHIVE_OUTPUT_NAME ${MANGLE}) - SET_TARGET_PROPERTIES(${TARGET} PROPERTIES RUNTIME_OUTPUT_NAME ${MANGLE}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES LIBRARY_OUTPUT_NAME ${MANGLELIB}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES ARCHIVE_OUTPUT_NAME ${MANGLELIB}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES RUNTIME_OUTPUT_NAME ${MANGLELIB}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES PREFIX "") # Critical that this be set to "" TARGET_LINK_LIBRARIES(${TARGET} ${ALL_TLL_LIBS};${ARGN}) IF(MSVC) target_compile_options(${TARGET} PRIVATE /Zi) @@ -105,9 +106,9 @@ ENDIF() IF(ENABLE_PLUGIN_INSTALL) MACRO(installplugin PLUG) - SET(BUILD_PLUGIN_LIB "${PLUGINPRE}${PLUG}.${PLUGINEXT}") - MESSAGE(STATUS "Installing: ${BUILD_PLUGIN_LIB} into ${PLUGIN_INSTALL_DIR}") - install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${BUILD_PLUGIN_LIB} DESTINATION ${PLUGIN_INSTALL_DIR}) + SET(INSTALLED_PLUGIN_LIB "${PLUGINPRE}${PLUG}.${PLUGINEXT}") + MESSAGE(STATUS "Installing: ${INSTALLED_PLUGIN_LIB} into ${PLUGIN_INSTALL_DIR}") + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${INSTALLED_PLUGIN_LIB} DESTINATION ${PLUGIN_INSTALL_DIR}) ENDMACRO() install(DIRECTORY DESTINATION ${PLUGIN_INSTALL_DIR}) From bbe0b212a5d0ae0ef08b03ba34baa4c5cfde02ae Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Fri, 27 May 2022 15:45:34 -0600 Subject: [PATCH 10/10] Fix internal bz2 problems --- configure.ac | 1 + libnczarr/zfilter.c | 1 + plugins/NCZstdfilters.c | 6 ------ 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index cce149b3ec..94aaab9348 100644 --- a/configure.ac +++ b/configure.ac @@ -686,6 +686,7 @@ AC_MSG_RESULT([${have_bz2}]) if test "x$have_bz2" = "xno" ; then have_local_bz2=yes + AC_MSG_NOTICE([Defaulting to internal libbz2]) else have_local_bz2=no fi diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index b74e4132ba..72d0e139af 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -995,6 +995,7 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j /* Find the plugin for this filter */ for(i=0;i<=loaded_plugins_max;i++) { if (!loaded_plugins[i]) continue; + if(!loaded_plugins[i] || !loaded_plugins[i]->codec.codec) continue; /* no plugin or no codec */ if(strcmp(NCJstring(jvalue), loaded_plugins[i]->codec.codec->codecid) == 0) {plugin = loaded_plugins[i]; break;} } diff --git a/plugins/NCZstdfilters.c b/plugins/NCZstdfilters.c index eda1865162..734541944b 100644 --- a/plugins/NCZstdfilters.c +++ b/plugins/NCZstdfilters.c @@ -38,10 +38,8 @@ Author: Dennis Heimbigner /* Forward */ -#ifdef HAVE_BZ2 static int NCZ_bzip2_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); static int NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); -#endif #ifdef HAVE_ZSTD static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); @@ -58,7 +56,6 @@ static void NCZ_blosc_codec_finalize(void); /**************************************************/ /* Provide the codec support for bzip2 filter */ -#ifdef HAVE_BZ2 static NCZ_codec_t NCZ_bzip2_codec = {/* NCZ_codec_t codec fields */ NCZ_CODEC_CLASS_VER, /* Struct version number */ NCZ_CODEC_HDF5, /* Struct sort */ @@ -139,7 +136,6 @@ NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) done: return stat; } -#endif /**************************************************/ /* Provide the codec support for zstandard filter */ @@ -438,9 +434,7 @@ NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) /**************************************************/ NCZ_codec_t* NCZ_stdfilters_codecs[] = { -#ifdef HAVE_BZ2 &NCZ_bzip2_codec, -#endif #ifdef HAVE_ZSTD &NCZ_zstd_codec, #endif