guilhem_lavaux/cosmotool
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update requirements. Split HDF5 compilation into its own virtual target.

Browse Source
This commit is contained in:
Guilhem Lavaux 2017-12-08 11:11:50 +01:00
parent 1d261c6251
commit aa42376cbe
9 changed files with 60 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
external/external_build.cmake vendored
View File

@ -5,7 +5,7 @@ OPTION(ENABLE_OPENMP "Set to Yes if Healpix and/or you need openMP" OFF)
SET(FFTW_URL "http://www.fftw.org/fftw-3.3.3.tar.gz" CACHE URL "URL to download FFTW from")
SET(EIGEN_URL "http://bitbucket.org/eigen/eigen/get/3.2.10.tar.gz" CACHE URL "URL to download Eigen from")
SET(GENGETOPT_URL "ftp://ftp.gnu.org/gnu/gengetopt/gengetopt-2.22.5.tar.gz" CACHE STRING "URL to download gengetopt from")
SET(HDF5_URL "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.18/src/hdf5-1.8.18.tar.gz" CACHE STRING "URL to download HDF5 from")
SET(HDF5_URL "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.18/src/hdf5-1.8.18.tar.bz2" CACHE STRING "URL to download HDF5 from")
SET(NETCDF_URL "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.4.1.1.tar.gz" CACHE STRING "URL to download NetCDF from")
SET(NETCDFCXX_URL "https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz" CACHE STRING "URL to download NetCDF-C++ from")
SET(BOOST_URL "http://sourceforge.net/projects/boost/files/boost/1.61.0/boost_1_61_0.tar.gz/download" CACHE STRING "URL to download Boost from")
@ -81,6 +81,7 @@ if (INTERNAL_HDF5)
ExternalProject_Add(hdf5
PREFIX ${BUILD_PREFIX}/hdf5-prefix
URL ${HDF5_URL}
URL_HASH MD5=29117bf488887f89888f9304c8ebea0b
CMAKE_ARGS
-DCMAKE_INSTALL_PREFIX=${EXT_INSTALL}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
@ -168,8 +169,10 @@ if (INTERNAL_NETCDF)
-DBUILD_TESTING=OFF
-DCMAKE_BUILD_TYPE=Release
-DENABLE_NETCDF4=ON
-DENABLE_DAP=OFF
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DENABLE_DAP=OFF
-DCMAKE_INSTALL_PREFIX=${NETCDF_BIN_DIR}
-DCMAKE_INSTALL_LIBDIR=lib
)
SET(NETCDFCXX_SOURCE_DIR ${BUILD_PREFIX}/netcdf-c++-prefix/src/netcdf-c++)
@ -181,13 +184,17 @@ if (INTERNAL_NETCDF)
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DBUILD_SHARED_LIBS=OFF
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DBUILD_TESTING=OFF
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=${NETCDF_BIN_DIR}
-DCMAKE_INSTALL_LIBDIR=lib
)
# SET(CONFIGURE_CPP_LDFLAGS "${CONFIGURE_LDFLAGS}")
# SET(EXTRA_NC_FLAGS CPPFLAGS=${CONFIGURE_CPP_FLAGS} LDFLAGS=${CONFIGURE_CPP_LDFLAGS})
SET(cosmotool_DEPS ${cosmotool_DEPS} netcdf netcdf-c++)
# find_library(NETCDF_LIBRARY netcdf NO_DEFAULT_PATH HINTS ${NETCDF_BIN_DIR}/lib ${NETCDF_BIN_DIR}/lib32 ${NETCDF_BIN_DIR}/lib64)
# find_library(NETCDFCPP_LIBRARY netcdf-cxx4 NO_DEFAULT_PATH HINTS ${NETCDF_BIN_DIR}/lib ${NETCDF_BIN_DIR}/lib32 ${NETCDF_BIN_DIR}/lib64)
SET(NETCDF_LIBRARY ${NETCDF_BIN_DIR}/lib/libnetcdf.a CACHE STRING "NetCDF lib" FORCE)
SET(NETCDFCPP_LIBRARY ${NETCDF_BIN_DIR}/lib/libnetcdf-cxx4.a CACHE STRING "NetCDF-C++ lib" FORCE)
SET(NETCDF_INCLUDE_PATH ${NETCDF_BIN_DIR}/include CACHE STRING "NetCDF include" FORCE)

9
python/CMakeLists.txt
View File

@ -1,7 +1,8 @@
set(CMAKE_SHARED_MODULE_PREFIX)
include_directories(${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_PATH} ${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src ${CMAKE_SOURCE_DIR}/python)
set(PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_PATH} ${CMAKE_SOURCE_DIR}/python)
include_directories(${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src)
IF(CYTHON)
add_custom_command(
@ -37,7 +38,11 @@ add_library(_cosmo_power MODULE ${CMAKE_CURRENT_BINARY_DIR}/_cosmo_power.cpp)
add_library(_cosmo_cic MODULE ${CMAKE_CURRENT_BINARY_DIR}/_cosmo_cic.cpp)
add_library(_fast_interp MODULE ${CMAKE_CURRENT_BINARY_DIR}/_fast_interp.cpp)
add_library(_project MODULE ${CMAKE_CURRENT_BINARY_DIR}/_project.cpp)
target_include_directories(_cosmotool PRIVATE ${PYTHON_INCLUDES})
target_include_directories(_cosmo_power PRIVATE ${PYTHON_INCLUDES})
target_include_directories(_cosmo_cic PRIVATE ${PYTHON_INCLUDES})
target_include_directories(_fast_interp PRIVATE ${PYTHON_INCLUDES})
target_include_directories(_project PRIVATE ${PYTHON_INCLUDES})
SET(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Bsymbolic-functions")

12
python/_cosmotool.pyx
View File

@ -515,7 +515,7 @@ def writeGadget(str filename, object simulation):
cxx_writeGadget(filename, &simdata)
def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = False, bool loadPosition = True, bool loadVelocity = False, bool loadId = False):
def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = False, bool loadPosition = True, bool loadVelocity = False, bool loadId = False, bool loadMass = False):
""" loadRamses(basepath, snapshot_id, cpu_id, doublePrecision = False, loadPosition = True, loadVelocity = False)
Loads the indicated snapshot based on the cpu id, snapshot id and basepath. It is important to specify the correct precision in doublePrecision otherwise the loading will fail. There is no way of auto-detecting properly the precision of the snapshot file.
@ -528,7 +528,8 @@ def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision =
doublePrecision (bool): By default it is False, thus singlePrecision
loadPosition (bool): Whether to load positions
loadVelocity (bool): Whether to load velocities
loadId (bol): Whether to load identifiers
loadId (bool): Whether to load identifiers
loadMass (bool): Whether to load mass value
Returns:
An object derived from PySimulationBase_.
@ -544,13 +545,16 @@ def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision =
flags |= NEED_VELOCITY
if loadId:
flags |= NEED_GADGET_ID
if loadMass:
flags |= NEED_MASS
encpath = basepath.encode('utf-8')
try:
data = loadRamsesSimu(basepath, snapshot_id, cpu_id, doublePrecision, flags)
data = loadRamsesSimu(encpath, snapshot_id, cpu_id, doublePrecision, flags)
if data == <SimuData*>0:
return None
except RuntimeError as e:
raise RuntimeError(e.message + ' (check the float precision in snapshot)')
raise RuntimeError(str(e) + ' (check the float precision in snapshot)')
return PySimulationAdaptor(wrap_simudata(data, flags))

2
python/cosmotool/fftw.py
View File

@ -19,7 +19,7 @@ class CubeFT(object):
self.align = pyfftw.simd_alignment
self.L = float(L)
self.max_cpu = multiprocessing.cpu_count() if max_cpu < 0 else max_cpu
self._dhat = pyfftw.n_byte_align_empty((self.N,self.N,self.N/2+1), self.align, dtype=fourier_type)
self._dhat = pyfftw.n_byte_align_empty((self.N,self.N,self.N//2+1), self.align, dtype=fourier_type)
self._density = pyfftw.n_byte_align_empty((self.N,self.N,self.N), self.align, dtype=real_type)
self._irfft = pyfftw.FFTW(self._dhat, self._density, axes=(0,1,2), direction='FFTW_BACKWARD', threads=self.max_cpu)#, normalize_idft=False)
self._rfft = pyfftw.FFTW(self._density, self._dhat, axes=(0,1,2), threads=self.max_cpu) #, normalize_idft=False)

18
python/cosmotool/grafic.py
View File

@ -15,16 +15,22 @@ def readGrafic(filename):
* the mean matter density :math:`\Omega_\mathrm{m}`
* the dark energy density :math:`\Omega_\Lambda`
* the hubble constant, relative to 100 km/s/Mpc
* xoffset
* yoffset
* zoffset
"""
with file(filename, mode="rb") as f:
with open(filename, mode="rb") as f:
p = struct.unpack("IIIIffffffffI", f.read(4*11 + 2*4))
checkPoint0, Nx, Ny, Nz, delta, _, _, _, scalefac, omega0, omegalambda0, h, checkPoint1 = p
checkPoint0, Nx, Ny, Nz, delta, xoff, yoff, zoff, scalefac, omega0, omegalambda0, h, checkPoint1 = p
if checkPoint0 != checkPoint1 or checkPoint0 != 4*11:
raise ValueError("Invalid unformatted access")
a = np.empty((Nx,Ny,Nz), dtype=np.float32)
BoxSize = delta * Nx * h
xoff *= h
yoff *= h
zoff *= h
checkPoint = 4*Ny*Nz
for i in range(Nx):
@ -38,11 +44,11 @@ def readGrafic(filename):
if checkPoint != 4*Ny*Nz:
raise ValueError("Invalid unformatted access")
return a, BoxSize, scalefac, omega0, omegalambda0, h
return a, BoxSize, scalefac, omega0, omegalambda0, h, xoff, yoff,zoff
def writeGrafic(filename, field, BoxSize, scalefac, **cosmo):
with file(filename, mode="wb") as f:
with open(filename, mode="wb") as f:
checkPoint = 4*11
Nx,Ny,Nz = field.shape
@ -65,7 +71,7 @@ def writeGrafic(filename, field, BoxSize, scalefac, **cosmo):
def writeWhitePhase(filename, field):
with file(filename, mode="wb") as f:
with open(filename, mode="wb") as f:
Nx,Ny,Nz = field.shape
checkPoint = 4*4
@ -80,7 +86,7 @@ def writeWhitePhase(filename, field):
def readWhitePhase(filename):
with file(filename, mode="rb") as f:
with open(filename, mode="rb") as f:
_, Nx, Ny, Nz, _, _ = struct.unpack("IIIIII", f.read(4*4+2*4))
a = np.empty((Nx,Ny,Nz), dtype=np.float32)

18
src/CMakeLists.txt
View File

@ -34,12 +34,13 @@ ENDIF(YORICK_SUPPORT)
if (HDF5_FOUND)
set(CosmoTool_SRCS ${CosmoTool_SRCS}
add_library(CosmoHDF5 OBJECT
h5_readFlash.cpp
loadFlash.cpp
)
target_include_directories(CosmoHDF5 BEFORE PRIVATE ${HDF5_INCLUDE_DIR})
else(HDF5_FOUND)
set(CosmoTool_SRCS ${CosmoTool_SRCS}
add_library(CosmoHdf5 OBJECT
loadFlash_dummy.cpp
)
endif (HDF5_FOUND)
@ -78,22 +79,23 @@ endif(YORICK_SUPPORT)
if (HDF5_FOUND)
set(CosmoTool_LIBS ${CosmoTool_LIBS} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES} ${HDF5_CXX_LIBRARIES} ${ZLIB})
include_directories(${HDF5_INCLUDE_DIR})
message(STATUS "CosmoTool_LIBS: ${CosmoTool_LIBS}")
endif (HDF5_FOUND)
set(CosmoTool_LIBS ${CosmoTool_LIBS} PARENT_SCOPE)
if (BUILD_SHARED_LIBS)
add_library(CosmoTool SHARED ${CosmoTool_SRCS})
add_library(CosmoTool SHARED ${CosmoTool_SRCS} $<TARGET_OBJECTS:CosmoHDF5>)
target_link_libraries(CosmoTool ${CosmoTool_LIBS})
if (BUILD_STATIC_LIBS)
add_library(CosmoTool_static STATIC ${CosmoTool_SRCS})
set_target_properties(CosmoTool_static PROPERTIES COMPILE_FLAGS "${CMAKE_C_COMPILE_OPTIONS_PIC}")
add_library(CosmoTool_static STATIC ${CosmoTool_SRCS} $<TARGET_OBJECTS:CosmoHDF5>)
target_link_libraries(CosmoTool_static)
set_property(TARGET CosmoTool_static PROPERTY POSITION_INDEPENDENT_CODE ON)
endif(BUILD_STATIC_LIBS)
else (BUILD_SHARED_LIBS)
add_library(CosmoTool STATIC ${CosmoTool_SRCS})
set_target_properties(CosmoTool PROPERTIES COMPILE_FLAGS "${CMAKE_C_COMPILE_OPTIONS_PIC}")
add_library(CosmoTool STATIC ${CosmoTool_SRCS} $<TARGET_OBJECTS:CosmoHDF5>)
target_link_libraries(CosmoTool)
set_property(TARGET CosmoTool PROPERTY POSITION_INDEPENDENT_CODE ON)
endif (BUILD_SHARED_LIBS)
install(TARGETS CosmoTool

2
src/fortran.cpp
View File

@ -51,6 +51,7 @@ UnformattedRead::UnformattedRead(const string& fname)
swapOrdering = false;
cSize = Check_32bits;
checkPointRef = checkPointAccum = 0;
recordBuffer = 0;
}
UnformattedRead::UnformattedRead(const char *fname)
@ -130,6 +131,7 @@ void UnformattedRead::beginCheckpoint(bool bufferRecord)
throw EndOfFileException();
if (bufferRecord) {
std::cout << "Use fast I/O mode" << std::endl;
recordBuffer = new uint8_t[checkPointRef];
f->read(reinterpret_cast<char *>(recordBuffer), checkPointRef);
}

10
src/hdf5_array.hpp
View File

@ -129,7 +129,7 @@ namespace CosmoTool {
//!
//!_______________________________________________________________________________________
template<typename ArrayType, typename hdf5_data_type>
void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name,
void hdf5_write_array(H5::H5Location& fg, const std::string& data_set_name,
const ArrayType& data,
const hdf5_data_type& datatype,
const std::vector<hsize_t>& dimensions,
@ -156,7 +156,7 @@ namespace CosmoTool {
template<typename ArrayType, typename hdf5_data_type>
void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name,
void hdf5_write_array(H5::H5Location& fg, const std::string& data_set_name,
const ArrayType& data,
const hdf5_data_type& datatype,
bool doCreate = true,
@ -255,7 +255,7 @@ namespace CosmoTool {
};
template<typename ArrayType>
void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name, const ArrayType& data )
void hdf5_write_array(H5::H5Location& fg, const std::string& data_set_name, const ArrayType& data )
{
typedef typename ArrayType::element T;
get_hdf5_data_type<T> hdf_data_type;
@ -341,7 +341,7 @@ namespace CosmoTool {
// ----------------------------------------------------------------------
template<typename ArrayType, typename hdf5_data_type>
void hdf5_read_array_typed(H5::CommonFG& fg, const std::string& data_set_name,
void hdf5_read_array_typed(H5::H5Location& fg, const std::string& data_set_name,
ArrayType& data,
const hdf5_data_type& datatype, bool auto_resize = true, bool useBases = false)
{
@ -377,7 +377,7 @@ namespace CosmoTool {
}
template<typename ArrayType>
void hdf5_read_array(H5::CommonFG& fg, const std::string& data_set_name, ArrayType& data, bool auto_resize = true,
void hdf5_read_array(H5::H5Location& fg, const std::string& data_set_name, ArrayType& data, bool auto_resize = true,
bool useBases = false )
{
typedef typename ArrayType::element T;

8
src/loadRamses.cpp
View File

@ -417,6 +417,9 @@ CosmoTool::SimuData *CosmoTool::loadRamsesSimu(const char *basename, int outputI
data->Vel[1] = new float[nPar];
data->Vel[2] = new float[nPar];
}
if (flags & NEED_MASS) {
data->Mass = new float[nPar];
}
if (flags & NEED_GADGET_ID)
{
data->Id = new int64_t[nPar];
@ -455,9 +458,10 @@ CosmoTool::SimuData *CosmoTool::loadRamsesSimu(const char *basename, int outputI
float minMass = INFINITY;
infile.beginCheckpoint();
for (uint32_t i = nPar; i > 0; i--)
{
for (uint32_t i = 0; i < nPar; i++) {
float dummyF = dp ? infile.readReal64() : infile.readReal32();
if (flags & NEED_MASS)
data->Mass[i] = dummyF;
if (dummyF < minMass) minMass = dummyF;
}
infile.endCheckpoint();