From d72085436187cdaf2ec6bfc75a5969e06071d9c2 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Fri, 5 May 2017 09:09:51 +0200 Subject: [PATCH] Fixed HDF5 support in cmake. Fixed fortran. --- CMakeLists.txt | 2 + FindHDF5.cmake | 86 ----------------------------------- external/external_build.cmake | 61 ++++++++++++++++++++----- python/cosmotool/fftw.py | 4 +- src/CMakeLists.txt | 1 - src/fortran.cpp | 8 ++-- src/fortran.hpp | 3 ++ src/loadGadget.cpp | 4 +- 8 files changed, 63 insertions(+), 106 deletions(-) delete mode 100644 FindHDF5.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e5d27f4..f6d3669 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,11 +9,13 @@ include(GetGitRevisionDescription) include(ExternalProject) include(FindPkgConfig) include(FindPackageHandleStandardArgs) +include(color_msg) option(BUILD_SHARED_LIBS "Build shared libraries." OFF) option(BUILD_STATIC_LIBS "Build static libraries." ON) option(ENABLE_SHARP "Enable SHARP support." ON) option(INTERNAL_HDF5 "Build internal version of HDF5" OFF) +option(INTERNAL_EIGEN "Build internal version of Eigen" ON) option(INTERNAL_NETCDF "Build internal version of NETCDF" OFF) option(INTERNAL_BOOST "Build internal version of BOOST" OFF) option(USE_GIT_VERSION "Use GIT version to pad version number" OFF) diff --git a/FindHDF5.cmake b/FindHDF5.cmake deleted file mode 100644 index b226c4f..0000000 --- a/FindHDF5.cmake +++ /dev/null @@ -1,86 +0,0 @@ -# -# To be used by projects that make use of Cmakeified hdf5-1.8.17 -# - -# -# Find the HDF5 includes and get all installed hdf5 library settings from -# HDF5-config.cmake file : Requires a CMake compatible hdf5-1.8.5 or later -# for this feature to work. The following vars are set if hdf5 is found. -# -# HDF5_FOUND - True if found, otherwise all other vars are undefined -# HDF5_INCLUDE_DIR - The include dir for main *.h files -# HDF5_FORTRAN_INCLUDE_DIR - The include dir for fortran modules and headers -# HDF5_VERSION_STRING - full version (e.g. 1.8.17) -# HDF5_VERSION_MAJOR - major part of version (e.g. 1.8) -# HDF5_VERSION_MINOR - minor part (e.g. 17) -# -# The following boolean vars will be defined -# HDF5_ENABLE_PARALLEL - 1 if HDF5 parallel supported -# HDF5_BUILD_FORTRAN - 1 if HDF5 was compiled with fortran on -# HDF5_BUILD_CPP_LIB - 1 if HDF5 was compiled with cpp on -# HDF5_BUILD_TOOLS - 1 if HDF5 was compiled with tools on -# HDF5_BUILD_HL_LIB - 1 if HDF5 was compiled with high level on -# HDF5_BUILD_HL_CPP_LIB - 1 if HDF5 was compiled with high level and cpp on -# -# Target names that are valid (depending on enabled options) -# will be the following -# -# hdf5 : HDF5 C library -# hdf5_tools : the tools library -# hdf5_f90cstub : used by Fortran to C interface -# hdf5_fortran : Fortran HDF5 library -# hdf5_cpp : HDF5 cpp interface library -# hdf5_hl : High Level library -# hdf5_hl_f90cstub : used by Fortran to C interface to High Level library -# hdf5_hl_fortran : Fortran High Level library -# hdf5_hl_cpp : High Level cpp interface library -# -# To aid in finding HDF5 as part of a subproject set -# HDF5_ROOT_DIR_HINT to the location where hdf5-config.cmake lies - -include (SelectLibraryConfigurations) -include (FindPackageHandleStandardArgs) - -# The HINTS option should only be used for values computed from the system. -set (_HDF5_HINTS - $ENV{HOME}/.local - $ENV{HDF5_ROOT} - $ENV{HDF5_ROOT_DIR_HINT} -) -# Hard-coded guesses should still go in PATHS. This ensures that the user -# environment can always override hard guesses. -set (_HDF5_PATHS - $ENV{HOME}/.local - $ENV{HDF5_ROOT} - $ENV{HDF5_ROOT_DIR_HINT} - /usr/lib/hdf5 - /usr/share/hdf5 - /usr/local/hdf5 - /usr/local/hdf5/share -) - -FIND_PATH (HDF5_ROOT_DIR "hdf5-config.cmake" - HINTS ${_HDF5_HINTS} - PATHS ${_HDF5_PATHS} - PATH_SUFFIXES - cmake/hdf5 - lib/cmake/hdf5 - share/cmake/hdf5 -) - -FIND_PATH (HDF5_INCLUDE_DIRS "H5public.h" - HINTS ${_HDF5_HINTS} - PATHS ${_HDF5_PATHS} - PATH_SUFFIXES - include - Include -) - -# For backwards compatibility we set HDF5_INCLUDE_DIR to the value of -# HDF5_INCLUDE_DIRS -set ( HDF5_INCLUDE_DIR "${HDF5_INCLUDE_DIRS}" ) - -if (HDF5_INCLUDE_DIR) - set (HDF5_FOUND "YES") - include (${HDF5_ROOT_DIR}/hdf5-config.cmake) -endif (HDF5_INCLUDE_DIR) diff --git a/external/external_build.cmake b/external/external_build.cmake index e7b3aca..09000d5 100644 --- a/external/external_build.cmake +++ b/external/external_build.cmake @@ -3,7 +3,7 @@ include(FindOpenMP) OPTION(ENABLE_OPENMP "Set to Yes if Healpix and/or you need openMP" OFF) SET(FFTW_URL "http://www.fftw.org/fftw-3.3.3.tar.gz" CACHE URL "URL to download FFTW from") -SET(EIGEN_URL "http://bitbucket.org/eigen/eigen/get/3.1.4.tar.gz" CACHE URL "URL to download Eigen from") +SET(EIGEN_URL "http://bitbucket.org/eigen/eigen/get/3.2.10.tar.gz" CACHE URL "URL to download Eigen from") SET(GENGETOPT_URL "ftp://ftp.gnu.org/gnu/gengetopt/gengetopt-2.22.5.tar.gz" CACHE STRING "URL to download gengetopt from") SET(HDF5_URL "https://support.hdfgroup.org/ftp/HDF5/current18/src/hdf5-1.8.18.tar.bz2" CACHE STRING "URL to download HDF5 from") SET(NETCDF_URL "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-4.1.3.tar.gz" CACHE STRING "URL to download NetCDF from") @@ -12,6 +12,25 @@ SET(GSL_URL "ftp://ftp.gnu.org/gnu/gsl/gsl-1.15.tar.gz" CACHE STRING "URL to dow mark_as_advanced(FFTW_URL EIGEN_URL HDF5_URL NETCDF_URL BOOST_URL GSL_URL) +MACRO(CHECK_CHANGE_STATE VAR) + IF (DEFINED _PREVIOUS_${VAR}) + IF (NOT ${_PREVIOUS_${VAR}} EQUAL ${${VAR}}) + foreach(loopvar ${ARGN}) + UNSET(${loopvar} CACHE) + endforeach() + ENDIF (NOT ${_PREVIOUS_${VAR}} EQUAL ${${VAR}}) + ENDIF (DEFINED _PREVIOUS_${VAR}) + SET(_PREVIOUS_${VAR} ${${VAR}} CACHE INTERNAL "Internal value") +ENDMACRO(CHECK_CHANGE_STATE) + +CHECK_CHANGE_STATE(INTERNAL_BOOST Boost_LIBRARIES Boost_INCLUDE_DIRS) +CHECK_CHANGE_STATE(INTERNAL_EIGEN EIGEN3_INCLUDE_DIRS) +CHECK_CHANGE_STATE(INTERNAL_GSL GSL_LIBRARY GSL_CBLAS_LIBRARY GSL_INCLUDE) +CHECK_CHANGE_STATE(INTERNAL_HDF5 + HDF5_INCLUDE_DIR HDF5_LIBRARIES HDF5_CXX_LIBRARIES HDF5_DIR + HDF5_C_STATIC_LIBRARY HDF5_HL_STATIC_LIBRARY HDF5_CXX_STATIC_LIBRARY) +CHECK_CHANGE_STATE(INTERNAL_DLIB DLIB_INCLUDE_DIR DLIB_LIBRARIES) + IF(ENABLE_OPENMP) IF (NOT OPENMP_FOUND) @@ -75,24 +94,35 @@ if (INTERNAL_HDF5) set(HDF5_LIBRARIES ${HDF5_BIN_DIR}/lib/libhdf5.a CACHE STRING "HDF5 lib" FORCE) set(HDF5_HL_LIBRARIES ${HDF5_BIN_DIR}/lib/libhdf5_hl.a CACHE STRING "HDF5 HL lib" FORCE) set(HDF5_CXX_LIBRARIES ${HDF5_BIN_DIR}/lib/libhdf5_cpp.a CACHE STRING "HDF5 C++ lib" FORCE) - SET(HDF5_INCLUDE_DIR ${HDF5_BIN_DIR}/include CACHE STRING "HDF5 include path" FORCE) - mark_as_advanced(HDF5_LIBRARIES HDF5_CXX_LIBRARIES HDF5_INCLUDE_DIR) + SET(HDF5_INCLUDE_DIRS ${HDF5_BIN_DIR}/include CACHE STRING "HDF5 include path" FORCE) + mark_as_advanced(HDF5_LIBRARIES HDF5_CXX_LIBRARIES HDF5_INCLUDE_DIRS) MESSAGE(STATUS "Internal HDF5 directory: $ENV{HDF5_ROOT}") MESSAGE(STATUS "Libs: ${HDF5_LIBRARIES}") SET(HDF5_FOUND TRUE) else (INTERNAL_HDF5) - mark_as_advanced(CLEAR HDF5_LIBRARIES HDF5_CXX_LIBRARIES HDF5_INCLUDE_DIR) + mark_as_advanced(CLEAR HDF5_LIBRARIES HDF5_CXX_LIBRARIES HDF5_INCLUDE_DIRS) if(HDF5_ROOTDIR) SET(ENV{HDF5_ROOT} ${HDF5_ROOTDIR}) endif(HDF5_ROOTDIR) - find_package(HDF5 COMPONENTS C CXX HL static) - set(HDF5_LIBRARIES ${HDF5_C_STATIC_LIBRARY} CACHE STRING "HDF5 lib" FORCE) - set(HDF5_HL_LIBRARIES ${HDF5_HL_STATIC_LIBRARY} CACHE STRING "HDF5 HL lib" FORCE) - set(HDF5_CXX_LIBRARIES ${HDF5_CXX_STATIC_LIBRARY} CACHE STRING "HDF5 C++ lib" FORCE) + find_package(HDF5 COMPONENTS C CXX HL CONFIG QUIET) + if (NOT HDF5_FOUND) + cmessage(CWARNING "Could not find HDF5 cmake config. Try classical exploration") + find_package(HDF5 COMPONENTS C CXX HL) + cmessage(STATUS "HDF5 lib: ${HDF5_LIBRARIES}") + cmessage(STATUS "HDF5 includes: ${HDF5_INCLUDE_DIRS}") + get_filename_component(HDF5_BIN_DIR ${HDF5_C_LIBRARY_hdf5} DIRECTORY) + cmessage(STATUS "HDF5 libpath: ${HDF5_BIN_DIR}") + else() + set(HDF5_LIBRARIES ${HDF5_C_STATIC_LIBRARY} CACHE STRING "HDF5 lib" FORCE) + set(HDF5_HL_LIBRARIES ${HDF5_HL_STATIC_LIBRARY} CACHE STRING "HDF5 HL lib" FORCE) + set(HDF5_CXX_LIBRARIES ${HDF5_CXX_STATIC_LIBRARY} CACHE STRING "HDF5 C++ lib" FORCE) + get_filename_component(HDF5_BIN_DIR ${HDF5_C_STATIC_LIBRARY} DIRECTORY) + endif() + SET(CONFIGURE_LDFLAGS "${CONFIGURE_LDFLAGS} -L${HDF5_BIN_DIR}") endif (INTERNAL_HDF5) -foreach(include_dir ${HDF5_INCLUDE_DIR}) +foreach(include_dir ${HDF5_INCLUDE_DIRS}) SET(CONFIGURE_CPP_FLAGS "${CONFIGURE_CPP_FLAGS} -I${include_dir}") endforeach(include_dir) @@ -161,8 +191,11 @@ if (INTERNAL_BOOST) set(Boost_DEP boost) ELSE (INTERNAL_BOOST) - find_package(Boost 1.53) + find_package(Boost 1.53 QUIET) set(Boost_DEP) + if (NOT Boost_FOUND) + cmessage(CWARNING "Boost >= 1.53 was not found") + endif() endif (INTERNAL_BOOST) mark_as_advanced(Boost_INCLUDE_DIRS Boost_LIBRARIES) @@ -248,13 +281,17 @@ ENDIF(INTERNAL_FFTW) IF (INTERNAL_EIGEN) ExternalProject_Add(eigen URL ${EIGEN_URL} + URL_HASH SHA256=04f8a4fa4afedaae721c1a1c756afeea20d3cdef0ce3293982cf1c518f178502 PREFIX ${BUILD_PREFIX}/eigen-prefix CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXT_INSTALL} ) - SET(EIGEN_INCLUDE_PATH ${EXT_INSTALL}/include/eigen3) + SET(EIGEN3_INCLUDE_DIRS ${EXT_INSTALL}/include/eigen3) ELSE (INTERNAL_EIGEN) - pkg_check_modules(EIGEN3 eigen3) + pkg_check_modules(EIGEN3 QUIET eigen3 ) + if (NOT EIGEN3_FOUND) + cmessage(CWARNING "Eigen library not found") + endif() ENDIF(INTERNAL_EIGEN) diff --git a/python/cosmotool/fftw.py b/python/cosmotool/fftw.py index b25a85b..da0c094 100644 --- a/python/cosmotool/fftw.py +++ b/python/cosmotool/fftw.py @@ -12,8 +12,8 @@ class CubeFT(object): self.max_cpu = multiprocessing.cpu_count() if max_cpu < 0 else max_cpu self._dhat = pyfftw.n_byte_align_empty((self.N,self.N,self.N/2+1), self.align, dtype='complex64') self._density = pyfftw.n_byte_align_empty((self.N,self.N,self.N), self.align, dtype='float32') - self._irfft = pyfftw.FFTW(self._dhat, self._density, axes=(0,1,2), direction='FFTW_BACKWARD', threads=self.max_cpu, normalize_idft=False) - self._rfft = pyfftw.FFTW(self._density, self._dhat, axes=(0,1,2), threads=self.max_cpu, normalize_idft=False) + self._irfft = pyfftw.FFTW(self._dhat, self._density, axes=(0,1,2), direction='FFTW_BACKWARD', threads=self.max_cpu)#, normalize_idft=False) + self._rfft = pyfftw.FFTW(self._density, self._dhat, axes=(0,1,2), threads=self.max_cpu) #, normalize_idft=False) def rfft(self): return ne.evaluate('c*a', out=self._dhat, local_dict={'c':self._rfft(normalise_idft=False),'a':(self.L/self.N)**3}, casting='unsafe') diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6fff514..ca862c0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -76,7 +76,6 @@ if(YORICK_SUPPORT) include_directories(${NETCDF_INCLUDE_PATH} ${NETCDFCPP_INCLUDE_PATH}) endif(YORICK_SUPPORT) -message(STATUS "HDF5 ? ${HDF5_FOUND}") if (HDF5_FOUND) set(CosmoTool_LIBS ${CosmoTool_LIBS} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES} ${HDF5_CXX_LIBRARIES} ${ZLIB}) include_directories(${HDF5_INCLUDE_DIRS}) diff --git a/src/fortran.cpp b/src/fortran.cpp index 716d57b..678e523 100644 --- a/src/fortran.cpp +++ b/src/fortran.cpp @@ -136,8 +136,9 @@ void UnformattedRead::endCheckpoint(bool autodrop) { if (checkPointRef != checkPointAccum) { - if (!autodrop || checkPointAccum > checkPointRef) + if (!autodrop || checkPointAccum > checkPointRef) { throw InvalidUnformattedAccess(); + } f->seekg(checkPointRef-checkPointAccum, ios::cur); } @@ -145,10 +146,11 @@ void UnformattedRead::endCheckpoint(bool autodrop) checkPointRef = (cSize == Check_32bits) ? 4 : 8; checkPointAccum = 0; - checkPointRef = (cSize == Check_32bits) ? readInt32() : readInt64(); + checkPointRef = (cSize == Check_32bits) ? readUint32() : readInt64(); - if (oldCheckPoint != checkPointRef) + if (oldCheckPoint != checkPointRef) { throw InvalidUnformattedAccess(); + } checkPointAccum = checkPointRef = 0; } diff --git a/src/fortran.hpp b/src/fortran.hpp index 48b4a1d..91136a5 100644 --- a/src/fortran.hpp +++ b/src/fortran.hpp @@ -46,6 +46,9 @@ namespace CosmoTool { class InvalidUnformattedAccess : public Exception { + public: + InvalidUnformattedAccess() + : Exception("Invalid unformatted fortran file format") {} }; class FortranTypes diff --git a/src/loadGadget.cpp b/src/loadGadget.cpp index 4ccbe00..c29012f 100644 --- a/src/loadGadget.cpp +++ b/src/loadGadget.cpp @@ -146,14 +146,14 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, if (GadgetFormat == 2) { int64_t startBlock = 0; char block[5]; - int32_t blocksize; + uint32_t blocksize; try { while (true) { f->beginCheckpoint(); f->readOrderedBuffer(block, 4); block[4] = 0; - blocksize = f->readInt32(); + blocksize = f->readUint32(); f->endCheckpoint(); blockTable[block].position = f->position(); blockTable[block].size = blocksize;