From aa42376cbe9e3ff975b237fcb0c25c8756edcd72 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Fri, 8 Dec 2017 11:11:50 +0100 Subject: [PATCH] Update requirements. Split HDF5 compilation into its own virtual target. --- external/external_build.cmake | 11 +++++++++-- python/CMakeLists.txt | 9 +++++++-- python/_cosmotool.pyx | 12 ++++++++---- python/cosmotool/fftw.py | 2 +- python/cosmotool/grafic.py | 18 ++++++++++++------ src/CMakeLists.txt | 18 ++++++++++-------- src/fortran.cpp | 2 ++ src/hdf5_array.hpp | 10 +++++----- src/loadRamses.cpp | 8 ++++++-- 9 files changed, 60 insertions(+), 30 deletions(-) diff --git a/external/external_build.cmake b/external/external_build.cmake index 6a97597..b14d382 100644 --- a/external/external_build.cmake +++ b/external/external_build.cmake @@ -5,7 +5,7 @@ OPTION(ENABLE_OPENMP "Set to Yes if Healpix and/or you need openMP" OFF) SET(FFTW_URL "http://www.fftw.org/fftw-3.3.3.tar.gz" CACHE URL "URL to download FFTW from") SET(EIGEN_URL "http://bitbucket.org/eigen/eigen/get/3.2.10.tar.gz" CACHE URL "URL to download Eigen from") SET(GENGETOPT_URL "ftp://ftp.gnu.org/gnu/gengetopt/gengetopt-2.22.5.tar.gz" CACHE STRING "URL to download gengetopt from") -SET(HDF5_URL "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.18/src/hdf5-1.8.18.tar.gz" CACHE STRING "URL to download HDF5 from") +SET(HDF5_URL "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.18/src/hdf5-1.8.18.tar.bz2" CACHE STRING "URL to download HDF5 from") SET(NETCDF_URL "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.4.1.1.tar.gz" CACHE STRING "URL to download NetCDF from") SET(NETCDFCXX_URL "https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz" CACHE STRING "URL to download NetCDF-C++ from") SET(BOOST_URL "http://sourceforge.net/projects/boost/files/boost/1.61.0/boost_1_61_0.tar.gz/download" CACHE STRING "URL to download Boost from") @@ -81,6 +81,7 @@ if (INTERNAL_HDF5) ExternalProject_Add(hdf5 PREFIX ${BUILD_PREFIX}/hdf5-prefix URL ${HDF5_URL} + URL_HASH MD5=29117bf488887f89888f9304c8ebea0b CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXT_INSTALL} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} @@ -168,8 +169,10 @@ if (INTERNAL_NETCDF) -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DENABLE_NETCDF4=ON - -DENABLE_DAP=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DENABLE_DAP=OFF -DCMAKE_INSTALL_PREFIX=${NETCDF_BIN_DIR} + -DCMAKE_INSTALL_LIBDIR=lib ) SET(NETCDFCXX_SOURCE_DIR ${BUILD_PREFIX}/netcdf-c++-prefix/src/netcdf-c++) @@ -181,13 +184,17 @@ if (INTERNAL_NETCDF) -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DBUILD_SHARED_LIBS=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${NETCDF_BIN_DIR} + -DCMAKE_INSTALL_LIBDIR=lib ) # SET(CONFIGURE_CPP_LDFLAGS "${CONFIGURE_LDFLAGS}") # SET(EXTRA_NC_FLAGS CPPFLAGS=${CONFIGURE_CPP_FLAGS} LDFLAGS=${CONFIGURE_CPP_LDFLAGS}) SET(cosmotool_DEPS ${cosmotool_DEPS} netcdf netcdf-c++) +# find_library(NETCDF_LIBRARY netcdf NO_DEFAULT_PATH HINTS ${NETCDF_BIN_DIR}/lib ${NETCDF_BIN_DIR}/lib32 ${NETCDF_BIN_DIR}/lib64) +# find_library(NETCDFCPP_LIBRARY netcdf-cxx4 NO_DEFAULT_PATH HINTS ${NETCDF_BIN_DIR}/lib ${NETCDF_BIN_DIR}/lib32 ${NETCDF_BIN_DIR}/lib64) SET(NETCDF_LIBRARY ${NETCDF_BIN_DIR}/lib/libnetcdf.a CACHE STRING "NetCDF lib" FORCE) SET(NETCDFCPP_LIBRARY ${NETCDF_BIN_DIR}/lib/libnetcdf-cxx4.a CACHE STRING "NetCDF-C++ lib" FORCE) SET(NETCDF_INCLUDE_PATH ${NETCDF_BIN_DIR}/include CACHE STRING "NetCDF include" FORCE) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index f5c963d..f3c01b5 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -1,7 +1,8 @@ set(CMAKE_SHARED_MODULE_PREFIX) -include_directories(${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_PATH} ${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src ${CMAKE_SOURCE_DIR}/python) +set(PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_PATH} ${CMAKE_SOURCE_DIR}/python) +include_directories(${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src) IF(CYTHON) add_custom_command( @@ -37,7 +38,11 @@ add_library(_cosmo_power MODULE ${CMAKE_CURRENT_BINARY_DIR}/_cosmo_power.cpp) add_library(_cosmo_cic MODULE ${CMAKE_CURRENT_BINARY_DIR}/_cosmo_cic.cpp) add_library(_fast_interp MODULE ${CMAKE_CURRENT_BINARY_DIR}/_fast_interp.cpp) add_library(_project MODULE ${CMAKE_CURRENT_BINARY_DIR}/_project.cpp) - +target_include_directories(_cosmotool PRIVATE ${PYTHON_INCLUDES}) +target_include_directories(_cosmo_power PRIVATE ${PYTHON_INCLUDES}) +target_include_directories(_cosmo_cic PRIVATE ${PYTHON_INCLUDES}) +target_include_directories(_fast_interp PRIVATE ${PYTHON_INCLUDES}) +target_include_directories(_project PRIVATE ${PYTHON_INCLUDES}) SET(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Bsymbolic-functions") diff --git a/python/_cosmotool.pyx b/python/_cosmotool.pyx index 5ef1c1a..a80c3cd 100644 --- a/python/_cosmotool.pyx +++ b/python/_cosmotool.pyx @@ -515,7 +515,7 @@ def writeGadget(str filename, object simulation): cxx_writeGadget(filename, &simdata) -def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = False, bool loadPosition = True, bool loadVelocity = False, bool loadId = False): +def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = False, bool loadPosition = True, bool loadVelocity = False, bool loadId = False, bool loadMass = False): """ loadRamses(basepath, snapshot_id, cpu_id, doublePrecision = False, loadPosition = True, loadVelocity = False) Loads the indicated snapshot based on the cpu id, snapshot id and basepath. It is important to specify the correct precision in doublePrecision otherwise the loading will fail. There is no way of auto-detecting properly the precision of the snapshot file. @@ -528,7 +528,8 @@ def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = doublePrecision (bool): By default it is False, thus singlePrecision loadPosition (bool): Whether to load positions loadVelocity (bool): Whether to load velocities - loadId (bol): Whether to load identifiers + loadId (bool): Whether to load identifiers + loadMass (bool): Whether to load mass value Returns: An object derived from PySimulationBase_. @@ -544,13 +545,16 @@ def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = flags |= NEED_VELOCITY if loadId: flags |= NEED_GADGET_ID + if loadMass: + flags |= NEED_MASS + encpath = basepath.encode('utf-8') try: - data = loadRamsesSimu(basepath, snapshot_id, cpu_id, doublePrecision, flags) + data = loadRamsesSimu(encpath, snapshot_id, cpu_id, doublePrecision, flags) if data == 0: return None except RuntimeError as e: - raise RuntimeError(e.message + ' (check the float precision in snapshot)') + raise RuntimeError(str(e) + ' (check the float precision in snapshot)') return PySimulationAdaptor(wrap_simudata(data, flags)) diff --git a/python/cosmotool/fftw.py b/python/cosmotool/fftw.py index ec0cb6a..57f43d1 100644 --- a/python/cosmotool/fftw.py +++ b/python/cosmotool/fftw.py @@ -19,7 +19,7 @@ class CubeFT(object): self.align = pyfftw.simd_alignment self.L = float(L) self.max_cpu = multiprocessing.cpu_count() if max_cpu < 0 else max_cpu - self._dhat = pyfftw.n_byte_align_empty((self.N,self.N,self.N/2+1), self.align, dtype=fourier_type) + self._dhat = pyfftw.n_byte_align_empty((self.N,self.N,self.N//2+1), self.align, dtype=fourier_type) self._density = pyfftw.n_byte_align_empty((self.N,self.N,self.N), self.align, dtype=real_type) self._irfft = pyfftw.FFTW(self._dhat, self._density, axes=(0,1,2), direction='FFTW_BACKWARD', threads=self.max_cpu)#, normalize_idft=False) self._rfft = pyfftw.FFTW(self._density, self._dhat, axes=(0,1,2), threads=self.max_cpu) #, normalize_idft=False) diff --git a/python/cosmotool/grafic.py b/python/cosmotool/grafic.py index c23073b..04eb5b6 100644 --- a/python/cosmotool/grafic.py +++ b/python/cosmotool/grafic.py @@ -15,16 +15,22 @@ def readGrafic(filename): * the mean matter density :math:`\Omega_\mathrm{m}` * the dark energy density :math:`\Omega_\Lambda` * the hubble constant, relative to 100 km/s/Mpc + * xoffset + * yoffset + * zoffset """ - with file(filename, mode="rb") as f: + with open(filename, mode="rb") as f: p = struct.unpack("IIIIffffffffI", f.read(4*11 + 2*4)) - checkPoint0, Nx, Ny, Nz, delta, _, _, _, scalefac, omega0, omegalambda0, h, checkPoint1 = p + checkPoint0, Nx, Ny, Nz, delta, xoff, yoff, zoff, scalefac, omega0, omegalambda0, h, checkPoint1 = p if checkPoint0 != checkPoint1 or checkPoint0 != 4*11: raise ValueError("Invalid unformatted access") a = np.empty((Nx,Ny,Nz), dtype=np.float32) BoxSize = delta * Nx * h + xoff *= h + yoff *= h + zoff *= h checkPoint = 4*Ny*Nz for i in range(Nx): @@ -38,11 +44,11 @@ def readGrafic(filename): if checkPoint != 4*Ny*Nz: raise ValueError("Invalid unformatted access") - return a, BoxSize, scalefac, omega0, omegalambda0, h + return a, BoxSize, scalefac, omega0, omegalambda0, h, xoff, yoff,zoff def writeGrafic(filename, field, BoxSize, scalefac, **cosmo): - with file(filename, mode="wb") as f: + with open(filename, mode="wb") as f: checkPoint = 4*11 Nx,Ny,Nz = field.shape @@ -65,7 +71,7 @@ def writeGrafic(filename, field, BoxSize, scalefac, **cosmo): def writeWhitePhase(filename, field): - with file(filename, mode="wb") as f: + with open(filename, mode="wb") as f: Nx,Ny,Nz = field.shape checkPoint = 4*4 @@ -80,7 +86,7 @@ def writeWhitePhase(filename, field): def readWhitePhase(filename): - with file(filename, mode="rb") as f: + with open(filename, mode="rb") as f: _, Nx, Ny, Nz, _, _ = struct.unpack("IIIIII", f.read(4*4+2*4)) a = np.empty((Nx,Ny,Nz), dtype=np.float32) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8b0c90e..ee65f47 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,12 +34,13 @@ ENDIF(YORICK_SUPPORT) if (HDF5_FOUND) - set(CosmoTool_SRCS ${CosmoTool_SRCS} + add_library(CosmoHDF5 OBJECT h5_readFlash.cpp loadFlash.cpp ) + target_include_directories(CosmoHDF5 BEFORE PRIVATE ${HDF5_INCLUDE_DIR}) else(HDF5_FOUND) - set(CosmoTool_SRCS ${CosmoTool_SRCS} + add_library(CosmoHdf5 OBJECT loadFlash_dummy.cpp ) endif (HDF5_FOUND) @@ -78,22 +79,23 @@ endif(YORICK_SUPPORT) if (HDF5_FOUND) set(CosmoTool_LIBS ${CosmoTool_LIBS} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES} ${HDF5_CXX_LIBRARIES} ${ZLIB}) - include_directories(${HDF5_INCLUDE_DIR}) message(STATUS "CosmoTool_LIBS: ${CosmoTool_LIBS}") endif (HDF5_FOUND) set(CosmoTool_LIBS ${CosmoTool_LIBS} PARENT_SCOPE) if (BUILD_SHARED_LIBS) - add_library(CosmoTool SHARED ${CosmoTool_SRCS}) + add_library(CosmoTool SHARED ${CosmoTool_SRCS} $) target_link_libraries(CosmoTool ${CosmoTool_LIBS}) if (BUILD_STATIC_LIBS) - add_library(CosmoTool_static STATIC ${CosmoTool_SRCS}) - set_target_properties(CosmoTool_static PROPERTIES COMPILE_FLAGS "${CMAKE_C_COMPILE_OPTIONS_PIC}") + add_library(CosmoTool_static STATIC ${CosmoTool_SRCS} $) + target_link_libraries(CosmoTool_static) + set_property(TARGET CosmoTool_static PROPERTY POSITION_INDEPENDENT_CODE ON) endif(BUILD_STATIC_LIBS) else (BUILD_SHARED_LIBS) - add_library(CosmoTool STATIC ${CosmoTool_SRCS}) - set_target_properties(CosmoTool PROPERTIES COMPILE_FLAGS "${CMAKE_C_COMPILE_OPTIONS_PIC}") + add_library(CosmoTool STATIC ${CosmoTool_SRCS} $) + target_link_libraries(CosmoTool) + set_property(TARGET CosmoTool PROPERTY POSITION_INDEPENDENT_CODE ON) endif (BUILD_SHARED_LIBS) install(TARGETS CosmoTool diff --git a/src/fortran.cpp b/src/fortran.cpp index 1170a9a..83e9730 100644 --- a/src/fortran.cpp +++ b/src/fortran.cpp @@ -51,6 +51,7 @@ UnformattedRead::UnformattedRead(const string& fname) swapOrdering = false; cSize = Check_32bits; checkPointRef = checkPointAccum = 0; + recordBuffer = 0; } UnformattedRead::UnformattedRead(const char *fname) @@ -130,6 +131,7 @@ void UnformattedRead::beginCheckpoint(bool bufferRecord) throw EndOfFileException(); if (bufferRecord) { + std::cout << "Use fast I/O mode" << std::endl; recordBuffer = new uint8_t[checkPointRef]; f->read(reinterpret_cast(recordBuffer), checkPointRef); } diff --git a/src/hdf5_array.hpp b/src/hdf5_array.hpp index f9835a1..f012bc4 100644 --- a/src/hdf5_array.hpp +++ b/src/hdf5_array.hpp @@ -129,7 +129,7 @@ namespace CosmoTool { //! //!_______________________________________________________________________________________ template - void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name, + void hdf5_write_array(H5::H5Location& fg, const std::string& data_set_name, const ArrayType& data, const hdf5_data_type& datatype, const std::vector& dimensions, @@ -156,7 +156,7 @@ namespace CosmoTool { template - void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name, + void hdf5_write_array(H5::H5Location& fg, const std::string& data_set_name, const ArrayType& data, const hdf5_data_type& datatype, bool doCreate = true, @@ -255,7 +255,7 @@ namespace CosmoTool { }; template - void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name, const ArrayType& data ) + void hdf5_write_array(H5::H5Location& fg, const std::string& data_set_name, const ArrayType& data ) { typedef typename ArrayType::element T; get_hdf5_data_type hdf_data_type; @@ -341,7 +341,7 @@ namespace CosmoTool { // ---------------------------------------------------------------------- template - void hdf5_read_array_typed(H5::CommonFG& fg, const std::string& data_set_name, + void hdf5_read_array_typed(H5::H5Location& fg, const std::string& data_set_name, ArrayType& data, const hdf5_data_type& datatype, bool auto_resize = true, bool useBases = false) { @@ -377,7 +377,7 @@ namespace CosmoTool { } template - void hdf5_read_array(H5::CommonFG& fg, const std::string& data_set_name, ArrayType& data, bool auto_resize = true, + void hdf5_read_array(H5::H5Location& fg, const std::string& data_set_name, ArrayType& data, bool auto_resize = true, bool useBases = false ) { typedef typename ArrayType::element T; diff --git a/src/loadRamses.cpp b/src/loadRamses.cpp index 8e218d2..3e841d5 100644 --- a/src/loadRamses.cpp +++ b/src/loadRamses.cpp @@ -417,6 +417,9 @@ CosmoTool::SimuData *CosmoTool::loadRamsesSimu(const char *basename, int outputI data->Vel[1] = new float[nPar]; data->Vel[2] = new float[nPar]; } + if (flags & NEED_MASS) { + data->Mass = new float[nPar]; + } if (flags & NEED_GADGET_ID) { data->Id = new int64_t[nPar]; @@ -455,9 +458,10 @@ CosmoTool::SimuData *CosmoTool::loadRamsesSimu(const char *basename, int outputI float minMass = INFINITY; infile.beginCheckpoint(); - for (uint32_t i = nPar; i > 0; i--) - { + for (uint32_t i = 0; i < nPar; i++) { float dummyF = dp ? infile.readReal64() : infile.readReal32(); + if (flags & NEED_MASS) + data->Mass[i] = dummyF; if (dummyF < minMass) minMass = dummyF; } infile.endCheckpoint();