diff --git a/external/external_build.cmake b/external/external_build.cmake index c3d63ab..f05e79e 100644 --- a/external/external_build.cmake +++ b/external/external_build.cmake @@ -5,7 +5,7 @@ OPTION(ENABLE_OPENMP "Set to Yes if Healpix and/or you need openMP" OFF) SET(FFTW_URL "http://www.fftw.org/fftw-3.3.3.tar.gz" CACHE URL "URL to download FFTW from") SET(EIGEN_URL "http://bitbucket.org/eigen/eigen/get/3.2.10.tar.gz" CACHE URL "URL to download Eigen from") SET(GENGETOPT_URL "ftp://ftp.gnu.org/gnu/gengetopt/gengetopt-2.22.5.tar.gz" CACHE STRING "URL to download gengetopt from") -SET(HDF5_URL "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.18/src/hdf5-1.8.18.tar.gz" CACHE STRING "URL to download HDF5 from") +SET(HDF5_URL "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.18/src/hdf5-1.8.18.tar.bz2" CACHE STRING "URL to download HDF5 from") SET(NETCDF_URL "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.5.0.tar.gz" CACHE STRING "URL to download NetCDF from") SET(NETCDFCXX_URL "https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz" CACHE STRING "URL to download NetCDF-C++ from") SET(BOOST_URL "http://sourceforge.net/projects/boost/files/boost/1.61.0/boost_1_61_0.tar.gz/download" CACHE STRING "URL to download Boost from") @@ -52,11 +52,14 @@ SET(CONFIGURE_LDFLAGS "") if (ENABLE_SHARP) SET(DEP_BUILD ${BUILD_PREFIX}/sharp-prefix/src/sharp/auto) + IF(NOT ENABLE_OPENMP) + SET(SHARP_OPENMP --disable-openmp) + ENDIF() ExternalProject_Add(sharp URL ${CMAKE_SOURCE_DIR}/external/libsharp-6077806.tar.gz PREFIX ${BUILD_PREFIX}/sharp-prefix BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND autoconf && ./configure "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" --prefix=${DEP_BUILD} + CONFIGURE_COMMAND autoconf && ./configure "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" --prefix=${DEP_BUILD} ${SHARP_OPENMP} BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} INSTALL_COMMAND echo "No install" ) @@ -78,6 +81,7 @@ if (INTERNAL_HDF5) ExternalProject_Add(hdf5 PREFIX ${BUILD_PREFIX}/hdf5-prefix URL ${HDF5_URL} + URL_HASH MD5=29117bf488887f89888f9304c8ebea0b CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXT_INSTALL} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} @@ -168,11 +172,13 @@ if (INTERNAL_NETCDF) -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DENABLE_NETCDF4=ON - -DENABLE_DAP=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DENABLE_DAP=OFF -DCMAKE_INSTALL_PREFIX=${NETCDF_BIN_DIR} -DHDF5_C_LIBRARY=${HDF5_C_STATIC_LIBRARY} -DHDF5_HL_LIBRARY=${HDF5_HL_STATIC_LIBRARY} -DHDF5_INCLUDE_DIR=${HDF5_INCLUDE_DIRS} + -DCMAKE_INSTALL_LIBDIR=lib ) SET(NETCDFCXX_SOURCE_DIR ${BUILD_PREFIX}/netcdf-c++-prefix/src/netcdf-c++) @@ -184,13 +190,17 @@ if (INTERNAL_NETCDF) -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DBUILD_SHARED_LIBS=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${NETCDF_BIN_DIR} + -DCMAKE_INSTALL_LIBDIR=lib ) # SET(CONFIGURE_CPP_LDFLAGS "${CONFIGURE_LDFLAGS}") # SET(EXTRA_NC_FLAGS CPPFLAGS=${CONFIGURE_CPP_FLAGS} LDFLAGS=${CONFIGURE_CPP_LDFLAGS}) SET(cosmotool_DEPS ${cosmotool_DEPS} netcdf netcdf-c++) +# find_library(NETCDF_LIBRARY netcdf NO_DEFAULT_PATH HINTS ${NETCDF_BIN_DIR}/lib ${NETCDF_BIN_DIR}/lib32 ${NETCDF_BIN_DIR}/lib64) +# find_library(NETCDFCPP_LIBRARY netcdf-cxx4 NO_DEFAULT_PATH HINTS ${NETCDF_BIN_DIR}/lib ${NETCDF_BIN_DIR}/lib32 ${NETCDF_BIN_DIR}/lib64) SET(NETCDF_LIBRARY ${NETCDF_BIN_DIR}/lib/libnetcdf.a CACHE STRING "NetCDF lib" FORCE) SET(NETCDFCPP_LIBRARY ${NETCDF_BIN_DIR}/lib/libnetcdf-cxx4.a CACHE STRING "NetCDF-C++ lib" FORCE) SET(NETCDF_INCLUDE_PATH ${NETCDF_BIN_DIR}/include CACHE STRING "NetCDF include" FORCE) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index f5c963d..f3c01b5 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -1,7 +1,8 @@ set(CMAKE_SHARED_MODULE_PREFIX) -include_directories(${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_PATH} ${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src ${CMAKE_SOURCE_DIR}/python) +set(PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_PATH} ${CMAKE_SOURCE_DIR}/python) +include_directories(${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src) IF(CYTHON) add_custom_command( @@ -37,7 +38,11 @@ add_library(_cosmo_power MODULE ${CMAKE_CURRENT_BINARY_DIR}/_cosmo_power.cpp) add_library(_cosmo_cic MODULE ${CMAKE_CURRENT_BINARY_DIR}/_cosmo_cic.cpp) add_library(_fast_interp MODULE ${CMAKE_CURRENT_BINARY_DIR}/_fast_interp.cpp) add_library(_project MODULE ${CMAKE_CURRENT_BINARY_DIR}/_project.cpp) - +target_include_directories(_cosmotool PRIVATE ${PYTHON_INCLUDES}) +target_include_directories(_cosmo_power PRIVATE ${PYTHON_INCLUDES}) +target_include_directories(_cosmo_cic PRIVATE ${PYTHON_INCLUDES}) +target_include_directories(_fast_interp PRIVATE ${PYTHON_INCLUDES}) +target_include_directories(_project PRIVATE ${PYTHON_INCLUDES}) SET(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Bsymbolic-functions") diff --git a/python/_cosmotool.pyx b/python/_cosmotool.pyx index d8865e7..61ba1a0 100644 --- a/python/_cosmotool.pyx +++ b/python/_cosmotool.pyx @@ -515,7 +515,7 @@ def writeGadget(str filename, object simulation): cxx_writeGadget(filename, &simdata) -def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = False, bool loadPosition = True, bool loadVelocity = False, bool loadId = False): +def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = False, bool loadPosition = True, bool loadVelocity = False, bool loadId = False, bool loadMass = False): """ loadRamses(basepath, snapshot_id, cpu_id, doublePrecision = False, loadPosition = True, loadVelocity = False) Loads the indicated snapshot based on the cpu id, snapshot id and basepath. It is important to specify the correct precision in doublePrecision otherwise the loading will fail. There is no way of auto-detecting properly the precision of the snapshot file. @@ -528,7 +528,8 @@ def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = doublePrecision (bool): By default it is False, thus singlePrecision loadPosition (bool): Whether to load positions loadVelocity (bool): Whether to load velocities - loadId (bol): Whether to load identifiers + loadId (bool): Whether to load identifiers + loadMass (bool): Whether to load mass value Returns: An object derived from PySimulationBase_. @@ -544,13 +545,16 @@ def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = flags |= NEED_VELOCITY if loadId: flags |= NEED_GADGET_ID + if loadMass: + flags |= NEED_MASS + encpath = basepath.encode('utf-8') try: - data = loadRamsesSimu(basepath, snapshot_id, cpu_id, doublePrecision, flags) + data = loadRamsesSimu(encpath, snapshot_id, cpu_id, doublePrecision, flags) if data == 0: return None except RuntimeError as e: - raise RuntimeError(e.message + ' (check the float precision in snapshot)') + raise RuntimeError(str(e) + ' (check the float precision in snapshot)') return PySimulationAdaptor(wrap_simudata(data, flags)) diff --git a/python/cosmotool/fftw.py b/python/cosmotool/fftw.py index ec0cb6a..57f43d1 100644 --- a/python/cosmotool/fftw.py +++ b/python/cosmotool/fftw.py @@ -19,7 +19,7 @@ class CubeFT(object): self.align = pyfftw.simd_alignment self.L = float(L) self.max_cpu = multiprocessing.cpu_count() if max_cpu < 0 else max_cpu - self._dhat = pyfftw.n_byte_align_empty((self.N,self.N,self.N/2+1), self.align, dtype=fourier_type) + self._dhat = pyfftw.n_byte_align_empty((self.N,self.N,self.N//2+1), self.align, dtype=fourier_type) self._density = pyfftw.n_byte_align_empty((self.N,self.N,self.N), self.align, dtype=real_type) self._irfft = pyfftw.FFTW(self._dhat, self._density, axes=(0,1,2), direction='FFTW_BACKWARD', threads=self.max_cpu)#, normalize_idft=False) self._rfft = pyfftw.FFTW(self._density, self._dhat, axes=(0,1,2), threads=self.max_cpu) #, normalize_idft=False) diff --git a/python/cosmotool/grafic.py b/python/cosmotool/grafic.py index c23073b..04eb5b6 100644 --- a/python/cosmotool/grafic.py +++ b/python/cosmotool/grafic.py @@ -15,16 +15,22 @@ def readGrafic(filename): * the mean matter density :math:`\Omega_\mathrm{m}` * the dark energy density :math:`\Omega_\Lambda` * the hubble constant, relative to 100 km/s/Mpc + * xoffset + * yoffset + * zoffset """ - with file(filename, mode="rb") as f: + with open(filename, mode="rb") as f: p = struct.unpack("IIIIffffffffI", f.read(4*11 + 2*4)) - checkPoint0, Nx, Ny, Nz, delta, _, _, _, scalefac, omega0, omegalambda0, h, checkPoint1 = p + checkPoint0, Nx, Ny, Nz, delta, xoff, yoff, zoff, scalefac, omega0, omegalambda0, h, checkPoint1 = p if checkPoint0 != checkPoint1 or checkPoint0 != 4*11: raise ValueError("Invalid unformatted access") a = np.empty((Nx,Ny,Nz), dtype=np.float32) BoxSize = delta * Nx * h + xoff *= h + yoff *= h + zoff *= h checkPoint = 4*Ny*Nz for i in range(Nx): @@ -38,11 +44,11 @@ def readGrafic(filename): if checkPoint != 4*Ny*Nz: raise ValueError("Invalid unformatted access") - return a, BoxSize, scalefac, omega0, omegalambda0, h + return a, BoxSize, scalefac, omega0, omegalambda0, h, xoff, yoff,zoff def writeGrafic(filename, field, BoxSize, scalefac, **cosmo): - with file(filename, mode="wb") as f: + with open(filename, mode="wb") as f: checkPoint = 4*11 Nx,Ny,Nz = field.shape @@ -65,7 +71,7 @@ def writeGrafic(filename, field, BoxSize, scalefac, **cosmo): def writeWhitePhase(filename, field): - with file(filename, mode="wb") as f: + with open(filename, mode="wb") as f: Nx,Ny,Nz = field.shape checkPoint = 4*4 @@ -80,7 +86,7 @@ def writeWhitePhase(filename, field): def readWhitePhase(filename): - with file(filename, mode="rb") as f: + with open(filename, mode="rb") as f: _, Nx, Ny, Nz, _, _ = struct.unpack("IIIIII", f.read(4*4+2*4)) a = np.empty((Nx,Ny,Nz), dtype=np.float32) diff --git a/sample/CMakeLists.txt b/sample/CMakeLists.txt index 877f155..0b8d250 100644 --- a/sample/CMakeLists.txt +++ b/sample/CMakeLists.txt @@ -68,7 +68,9 @@ if (ENABLE_SHARP AND SHARP_LIBRARY AND SHARP_INCLUDE_PATH AND EIGEN3_FOUND) include_directories(${SHARP_INCLUDE_PATH}) add_executable(test_healpix_calls test_healpix_calls.cpp) target_link_libraries(test_healpix_calls ${tolink} ${SHARP_LIBRARIES}) - set_target_properties(test_healpix_calls PROPERTIES COMPILE_FLAGS "${OpenMP_CXX_FLAGS}" LINK_FLAGS "${OpenMP_CXX_FLAGS}") + if (ENABLE_OPENMP) + set_target_properties(test_healpix_calls PROPERTIES COMPILE_FLAGS "${OpenMP_CXX_FLAGS}" LINK_FLAGS "${OpenMP_CXX_FLAGS}") + endif() add_dependencies(test_healpix_calls sharp) endif (ENABLE_SHARP AND SHARP_LIBRARY AND SHARP_INCLUDE_PATH AND EIGEN3_FOUND) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8b0c90e..ee65f47 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,12 +34,13 @@ ENDIF(YORICK_SUPPORT) if (HDF5_FOUND) - set(CosmoTool_SRCS ${CosmoTool_SRCS} + add_library(CosmoHDF5 OBJECT h5_readFlash.cpp loadFlash.cpp ) + target_include_directories(CosmoHDF5 BEFORE PRIVATE ${HDF5_INCLUDE_DIR}) else(HDF5_FOUND) - set(CosmoTool_SRCS ${CosmoTool_SRCS} + add_library(CosmoHdf5 OBJECT loadFlash_dummy.cpp ) endif (HDF5_FOUND) @@ -78,22 +79,23 @@ endif(YORICK_SUPPORT) if (HDF5_FOUND) set(CosmoTool_LIBS ${CosmoTool_LIBS} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES} ${HDF5_CXX_LIBRARIES} ${ZLIB}) - include_directories(${HDF5_INCLUDE_DIR}) message(STATUS "CosmoTool_LIBS: ${CosmoTool_LIBS}") endif (HDF5_FOUND) set(CosmoTool_LIBS ${CosmoTool_LIBS} PARENT_SCOPE) if (BUILD_SHARED_LIBS) - add_library(CosmoTool SHARED ${CosmoTool_SRCS}) + add_library(CosmoTool SHARED ${CosmoTool_SRCS} $) target_link_libraries(CosmoTool ${CosmoTool_LIBS}) if (BUILD_STATIC_LIBS) - add_library(CosmoTool_static STATIC ${CosmoTool_SRCS}) - set_target_properties(CosmoTool_static PROPERTIES COMPILE_FLAGS "${CMAKE_C_COMPILE_OPTIONS_PIC}") + add_library(CosmoTool_static STATIC ${CosmoTool_SRCS} $) + target_link_libraries(CosmoTool_static) + set_property(TARGET CosmoTool_static PROPERTY POSITION_INDEPENDENT_CODE ON) endif(BUILD_STATIC_LIBS) else (BUILD_SHARED_LIBS) - add_library(CosmoTool STATIC ${CosmoTool_SRCS}) - set_target_properties(CosmoTool PROPERTIES COMPILE_FLAGS "${CMAKE_C_COMPILE_OPTIONS_PIC}") + add_library(CosmoTool STATIC ${CosmoTool_SRCS} $) + target_link_libraries(CosmoTool) + set_property(TARGET CosmoTool PROPERTY POSITION_INDEPENDENT_CODE ON) endif (BUILD_SHARED_LIBS) install(TARGETS CosmoTool diff --git a/src/fortran.cpp b/src/fortran.cpp index 1170a9a..83e9730 100644 --- a/src/fortran.cpp +++ b/src/fortran.cpp @@ -51,6 +51,7 @@ UnformattedRead::UnformattedRead(const string& fname) swapOrdering = false; cSize = Check_32bits; checkPointRef = checkPointAccum = 0; + recordBuffer = 0; } UnformattedRead::UnformattedRead(const char *fname) @@ -130,6 +131,7 @@ void UnformattedRead::beginCheckpoint(bool bufferRecord) throw EndOfFileException(); if (bufferRecord) { + std::cout << "Use fast I/O mode" << std::endl; recordBuffer = new uint8_t[checkPointRef]; f->read(reinterpret_cast(recordBuffer), checkPointRef); } diff --git a/src/hdf5_array.hpp b/src/hdf5_array.hpp index f9835a1..2e73b9a 100644 --- a/src/hdf5_array.hpp +++ b/src/hdf5_array.hpp @@ -51,6 +51,11 @@ knowledge of the CeCILL license and that you accept its terms. #include namespace CosmoTool { +#if (H5_VERS_MAJOR == 1) && (H5_VERS_MINOR <= 8) + typedef H5::CommonFG H5_CommonFileGroup; +#else + typedef H5::Group H5_CommonFileGroup; +#endif //!_______________________________________________________________________________________ //! @@ -129,7 +134,7 @@ namespace CosmoTool { //! //!_______________________________________________________________________________________ template - void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name, + void hdf5_write_array(H5_CommonFileGroup& fg, const std::string& data_set_name, const ArrayType& data, const hdf5_data_type& datatype, const std::vector& dimensions, @@ -156,7 +161,7 @@ namespace CosmoTool { template - void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name, + void hdf5_write_array(H5_CommonFileGroup& fg, const std::string& data_set_name, const ArrayType& data, const hdf5_data_type& datatype, bool doCreate = true, @@ -165,7 +170,7 @@ namespace CosmoTool { std::vector dimensions(data.shape(), data.shape() + data.num_dimensions()); hdf5_write_array(fg, data_set_name, data, datatype, dimensions, doCreate, useBases); } - + /* HDF5 complex type */ template class hdf5_ComplexType @@ -255,14 +260,14 @@ namespace CosmoTool { }; template - void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name, const ArrayType& data ) + void hdf5_write_array(H5_CommonFileGroup& fg, const std::string& data_set_name, const ArrayType& data ) { typedef typename ArrayType::element T; get_hdf5_data_type hdf_data_type; hdf5_write_array(fg, data_set_name, data, hdf_data_type.type()); } - + // HDF5 array reader // // Author Guilhem Lavaux (May 2014) @@ -341,7 +346,7 @@ namespace CosmoTool { // ---------------------------------------------------------------------- template - void hdf5_read_array_typed(H5::CommonFG& fg, const std::string& data_set_name, + void hdf5_read_array_typed(H5_CommonFileGroup& fg, const std::string& data_set_name, ArrayType& data, const hdf5_data_type& datatype, bool auto_resize = true, bool useBases = false) { @@ -377,7 +382,7 @@ namespace CosmoTool { } template - void hdf5_read_array(H5::CommonFG& fg, const std::string& data_set_name, ArrayType& data, bool auto_resize = true, + void hdf5_read_array(H5_CommonFileGroup& fg, const std::string& data_set_name, ArrayType& data, bool auto_resize = true, bool useBases = false ) { typedef typename ArrayType::element T; diff --git a/src/loadRamses.cpp b/src/loadRamses.cpp index 8e218d2..3e841d5 100644 --- a/src/loadRamses.cpp +++ b/src/loadRamses.cpp @@ -417,6 +417,9 @@ CosmoTool::SimuData *CosmoTool::loadRamsesSimu(const char *basename, int outputI data->Vel[1] = new float[nPar]; data->Vel[2] = new float[nPar]; } + if (flags & NEED_MASS) { + data->Mass = new float[nPar]; + } if (flags & NEED_GADGET_ID) { data->Id = new int64_t[nPar]; @@ -455,9 +458,10 @@ CosmoTool::SimuData *CosmoTool::loadRamsesSimu(const char *basename, int outputI float minMass = INFINITY; infile.beginCheckpoint(); - for (uint32_t i = nPar; i > 0; i--) - { + for (uint32_t i = 0; i < nPar; i++) { float dummyF = dp ? infile.readReal64() : infile.readReal32(); + if (flags & NEED_MASS) + data->Mass[i] = dummyF; if (dummyF < minMass) minMass = dummyF; } infile.endCheckpoint();