Merge branch 'master' of bitbucket.org:glavaux/cosmotool

external/external_build.cmake

@@ -5,7 +5,7 @@ OPTION(ENABLE_OPENMP "Set to Yes if Healpix and/or you need openMP" OFF)
 SET(FFTW_URL "http://www.fftw.org/fftw-3.3.3.tar.gz" CACHE URL "URL to download FFTW from")
 SET(EIGEN_URL "http://bitbucket.org/eigen/eigen/get/3.2.10.tar.gz" CACHE URL "URL to download Eigen from")
 SET(GENGETOPT_URL "ftp://ftp.gnu.org/gnu/gengetopt/gengetopt-2.22.5.tar.gz" CACHE STRING "URL to download gengetopt from")
-SET(HDF5_URL "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.18/src/hdf5-1.8.18.tar.gz" CACHE STRING "URL to download HDF5 from")
+SET(HDF5_URL "https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8/hdf5-1.8.18/src/hdf5-1.8.18.tar.bz2" CACHE STRING "URL to download HDF5 from")
 SET(NETCDF_URL "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.5.0.tar.gz" CACHE STRING "URL to download NetCDF from")
 SET(NETCDFCXX_URL "https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz" CACHE STRING "URL to download NetCDF-C++ from")
 SET(BOOST_URL "http://sourceforge.net/projects/boost/files/boost/1.61.0/boost_1_61_0.tar.gz/download" CACHE STRING "URL to download Boost from")
@@ -52,11 +52,14 @@ SET(CONFIGURE_LDFLAGS "")
 
 if (ENABLE_SHARP)
   SET(DEP_BUILD ${BUILD_PREFIX}/sharp-prefix/src/sharp/auto)
+  IF(NOT ENABLE_OPENMP)
+    SET(SHARP_OPENMP --disable-openmp)
+  ENDIF()
   ExternalProject_Add(sharp
     URL ${CMAKE_SOURCE_DIR}/external/libsharp-6077806.tar.gz
     PREFIX ${BUILD_PREFIX}/sharp-prefix
     BUILD_IN_SOURCE 1 
-    CONFIGURE_COMMAND autoconf && ./configure "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" --prefix=${DEP_BUILD}
+    CONFIGURE_COMMAND autoconf && ./configure "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}" --prefix=${DEP_BUILD} ${SHARP_OPENMP}
     BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
     INSTALL_COMMAND echo "No install"
   )
@@ -78,6 +81,7 @@ if (INTERNAL_HDF5)
   ExternalProject_Add(hdf5
     PREFIX ${BUILD_PREFIX}/hdf5-prefix
     URL ${HDF5_URL}
+    URL_HASH MD5=29117bf488887f89888f9304c8ebea0b
     CMAKE_ARGS
        -DCMAKE_INSTALL_PREFIX=${EXT_INSTALL}
        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
@@ -168,11 +172,13 @@ if (INTERNAL_NETCDF)
 		-DBUILD_TESTING=OFF
 		-DCMAKE_BUILD_TYPE=Release
 		-DENABLE_NETCDF4=ON
-                -DENABLE_DAP=OFF
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+    -DENABLE_DAP=OFF
 		-DCMAKE_INSTALL_PREFIX=${NETCDF_BIN_DIR}
                 -DHDF5_C_LIBRARY=${HDF5_C_STATIC_LIBRARY}
                 -DHDF5_HL_LIBRARY=${HDF5_HL_STATIC_LIBRARY}
                 -DHDF5_INCLUDE_DIR=${HDF5_INCLUDE_DIRS}
+    -DCMAKE_INSTALL_LIBDIR=lib
   )
   
   SET(NETCDFCXX_SOURCE_DIR ${BUILD_PREFIX}/netcdf-c++-prefix/src/netcdf-c++)
@@ -184,13 +190,17 @@ if (INTERNAL_NETCDF)
 		-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
 		-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
 		-DBUILD_SHARED_LIBS=OFF
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
 		-DBUILD_TESTING=OFF
 		-DCMAKE_BUILD_TYPE=Release
 		-DCMAKE_INSTALL_PREFIX=${NETCDF_BIN_DIR}
+    -DCMAKE_INSTALL_LIBDIR=lib
   )
 #  SET(CONFIGURE_CPP_LDFLAGS "${CONFIGURE_LDFLAGS}")
 #  SET(EXTRA_NC_FLAGS CPPFLAGS=${CONFIGURE_CPP_FLAGS} LDFLAGS=${CONFIGURE_CPP_LDFLAGS})
   SET(cosmotool_DEPS ${cosmotool_DEPS} netcdf netcdf-c++)
+#  find_library(NETCDF_LIBRARY netcdf NO_DEFAULT_PATH HINTS ${NETCDF_BIN_DIR}/lib ${NETCDF_BIN_DIR}/lib32 ${NETCDF_BIN_DIR}/lib64)
+#  find_library(NETCDFCPP_LIBRARY netcdf-cxx4 NO_DEFAULT_PATH HINTS ${NETCDF_BIN_DIR}/lib ${NETCDF_BIN_DIR}/lib32 ${NETCDF_BIN_DIR}/lib64)
   SET(NETCDF_LIBRARY ${NETCDF_BIN_DIR}/lib/libnetcdf.a CACHE STRING "NetCDF lib" FORCE)
   SET(NETCDFCPP_LIBRARY ${NETCDF_BIN_DIR}/lib/libnetcdf-cxx4.a CACHE STRING "NetCDF-C++ lib" FORCE)
   SET(NETCDF_INCLUDE_PATH ${NETCDF_BIN_DIR}/include CACHE STRING "NetCDF include" FORCE)

python/CMakeLists.txt

@@ -1,7 +1,8 @@
 set(CMAKE_SHARED_MODULE_PREFIX)
 
 
-include_directories(${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_PATH} ${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src ${CMAKE_SOURCE_DIR}/python)
+set(PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_PATH} ${CMAKE_SOURCE_DIR}/python)
+include_directories(${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src)
 
 IF(CYTHON)
   add_custom_command(
@@ -37,7 +38,11 @@ add_library(_cosmo_power MODULE ${CMAKE_CURRENT_BINARY_DIR}/_cosmo_power.cpp)
 add_library(_cosmo_cic MODULE ${CMAKE_CURRENT_BINARY_DIR}/_cosmo_cic.cpp)
 add_library(_fast_interp MODULE ${CMAKE_CURRENT_BINARY_DIR}/_fast_interp.cpp)
 add_library(_project MODULE ${CMAKE_CURRENT_BINARY_DIR}/_project.cpp)
-
+target_include_directories(_cosmotool PRIVATE ${PYTHON_INCLUDES}) 
+target_include_directories(_cosmo_power PRIVATE ${PYTHON_INCLUDES}) 
+target_include_directories(_cosmo_cic PRIVATE ${PYTHON_INCLUDES}) 
+target_include_directories(_fast_interp PRIVATE ${PYTHON_INCLUDES}) 
+target_include_directories(_project PRIVATE ${PYTHON_INCLUDES}) 
 
 
 SET(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Bsymbolic-functions")

python/_cosmotool.pyx

@@ -515,7 +515,7 @@ def writeGadget(str filename, object simulation):
 
   cxx_writeGadget(filename, &simdata)
 
-def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = False, bool loadPosition = True, bool loadVelocity = False, bool loadId = False):
+def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision = False, bool loadPosition = True, bool loadVelocity = False, bool loadId = False, bool loadMass = False):
   """ loadRamses(basepath, snapshot_id, cpu_id, doublePrecision = False, loadPosition = True, loadVelocity = False)
   Loads the indicated snapshot based on the cpu id, snapshot id and basepath. It is important to specify the correct precision in doublePrecision otherwise the loading will fail. There is no way of auto-detecting properly the precision of the snapshot file.
   
@@ -528,7 +528,8 @@ def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision =
     doublePrecision (bool): By default it is False, thus singlePrecision
     loadPosition (bool): Whether to load positions
     loadVelocity (bool): Whether to load velocities
-    loadId (bol): Whether to load identifiers
+    loadId (bool): Whether to load identifiers
+    loadMass (bool): Whether to load mass value 
     
   Returns:
     An object derived from PySimulationBase_. 
@@ -544,13 +545,16 @@ def loadRamses(str basepath, int snapshot_id, int cpu_id, bool doublePrecision =
      flags |= NEED_VELOCITY
   if loadId:
      flags |= NEED_GADGET_ID
+  if loadMass:
+     flags |= NEED_MASS
 
+  encpath = basepath.encode('utf-8') 
   try:
-    data = loadRamsesSimu(basepath, snapshot_id, cpu_id, doublePrecision, flags)
+    data = loadRamsesSimu(encpath, snapshot_id, cpu_id, doublePrecision, flags)
     if data == <SimuData*>0:
       return None
   except RuntimeError as e:
-    raise RuntimeError(e.message + ' (check the float precision in snapshot)')
+    raise RuntimeError(str(e) + ' (check the float precision in snapshot)')
 
   return PySimulationAdaptor(wrap_simudata(data, flags))
   

python/cosmotool/fftw.py

@@ -19,7 +19,7 @@ class CubeFT(object):
       self.align = pyfftw.simd_alignment
       self.L = float(L)
       self.max_cpu = multiprocessing.cpu_count() if max_cpu < 0 else max_cpu
-      self._dhat = pyfftw.n_byte_align_empty((self.N,self.N,self.N/2+1), self.align, dtype=fourier_type)
+      self._dhat = pyfftw.n_byte_align_empty((self.N,self.N,self.N//2+1), self.align, dtype=fourier_type)
       self._density = pyfftw.n_byte_align_empty((self.N,self.N,self.N), self.align, dtype=real_type)
       self._irfft = pyfftw.FFTW(self._dhat, self._density, axes=(0,1,2), direction='FFTW_BACKWARD', threads=self.max_cpu)#, normalize_idft=False)
       self._rfft = pyfftw.FFTW(self._density, self._dhat, axes=(0,1,2), threads=self.max_cpu) #, normalize_idft=False)

python/cosmotool/grafic.py

@@ -15,16 +15,22 @@ def readGrafic(filename):
       * the mean matter density :math:`\Omega_\mathrm{m}`
       * the dark energy density :math:`\Omega_\Lambda`
       * the hubble constant, relative to 100 km/s/Mpc
+      * xoffset
+      * yoffset
+      * zoffset
   """ 
-  with file(filename, mode="rb") as f:
+  with open(filename, mode="rb") as f:
         p = struct.unpack("IIIIffffffffI", f.read(4*11 + 2*4))
-        checkPoint0, Nx, Ny, Nz, delta, _, _, _, scalefac, omega0, omegalambda0, h, checkPoint1 = p
+        checkPoint0, Nx, Ny, Nz, delta, xoff, yoff, zoff, scalefac, omega0, omegalambda0, h, checkPoint1 = p
         if checkPoint0 != checkPoint1 or checkPoint0 != 4*11:
           raise ValueError("Invalid unformatted access")
           
         a = np.empty((Nx,Ny,Nz), dtype=np.float32)
         
         BoxSize = delta * Nx * h
+        xoff *= h
+        yoff *= h
+        zoff *= h
         
         checkPoint = 4*Ny*Nz
         for i in range(Nx):
@@ -38,11 +44,11 @@ def readGrafic(filename):
             if checkPoint != 4*Ny*Nz:
               raise ValueError("Invalid unformatted access")
 
-  return a, BoxSize, scalefac, omega0, omegalambda0, h      
+  return a, BoxSize, scalefac, omega0, omegalambda0, h, xoff, yoff,zoff
 
 def writeGrafic(filename, field, BoxSize, scalefac, **cosmo):
 
-    with file(filename, mode="wb") as f:
+    with open(filename, mode="wb") as f:
         checkPoint = 4*11
         
         Nx,Ny,Nz = field.shape
@@ -65,7 +71,7 @@ def writeGrafic(filename, field, BoxSize, scalefac, **cosmo):
         
 def writeWhitePhase(filename, field):
 
-    with file(filename, mode="wb") as f:
+    with open(filename, mode="wb") as f:
         Nx,Ny,Nz = field.shape
         
         checkPoint = 4*4
@@ -80,7 +86,7 @@ def writeWhitePhase(filename, field):
 
 
 def readWhitePhase(filename):
-    with file(filename, mode="rb") as f:
+    with open(filename, mode="rb") as f:
         _, Nx, Ny, Nz, _, _ = struct.unpack("IIIIII", f.read(4*4+2*4))
         
         a = np.empty((Nx,Ny,Nz), dtype=np.float32)

sample/CMakeLists.txt

@@ -68,7 +68,9 @@ if (ENABLE_SHARP AND SHARP_LIBRARY AND SHARP_INCLUDE_PATH AND EIGEN3_FOUND)
   include_directories(${SHARP_INCLUDE_PATH})
   add_executable(test_healpix_calls test_healpix_calls.cpp)
   target_link_libraries(test_healpix_calls ${tolink} ${SHARP_LIBRARIES})
-  set_target_properties(test_healpix_calls PROPERTIES COMPILE_FLAGS "${OpenMP_CXX_FLAGS}" LINK_FLAGS "${OpenMP_CXX_FLAGS}")
+  if (ENABLE_OPENMP)
+    set_target_properties(test_healpix_calls PROPERTIES COMPILE_FLAGS "${OpenMP_CXX_FLAGS}" LINK_FLAGS "${OpenMP_CXX_FLAGS}")
+  endif()
   add_dependencies(test_healpix_calls sharp)
 endif (ENABLE_SHARP AND SHARP_LIBRARY AND SHARP_INCLUDE_PATH AND EIGEN3_FOUND)
 

src/CMakeLists.txt

@@ -34,12 +34,13 @@ ENDIF(YORICK_SUPPORT)
 
 
 if (HDF5_FOUND)
-  set(CosmoTool_SRCS ${CosmoTool_SRCS}
+  add_library(CosmoHDF5 OBJECT
     h5_readFlash.cpp
     loadFlash.cpp
     )
+  target_include_directories(CosmoHDF5 BEFORE PRIVATE ${HDF5_INCLUDE_DIR})
 else(HDF5_FOUND)
-  set(CosmoTool_SRCS ${CosmoTool_SRCS}
+  add_library(CosmoHdf5  OBJECT
     loadFlash_dummy.cpp
   )
 endif (HDF5_FOUND)
@@ -78,22 +79,23 @@ endif(YORICK_SUPPORT)
 
 if (HDF5_FOUND)
   set(CosmoTool_LIBS ${CosmoTool_LIBS} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES} ${HDF5_CXX_LIBRARIES} ${ZLIB})
-  include_directories(${HDF5_INCLUDE_DIR})
   message(STATUS "CosmoTool_LIBS: ${CosmoTool_LIBS}")
 endif (HDF5_FOUND)
 
 set(CosmoTool_LIBS ${CosmoTool_LIBS} PARENT_SCOPE)
 
 if (BUILD_SHARED_LIBS)
-  add_library(CosmoTool SHARED ${CosmoTool_SRCS})
+  add_library(CosmoTool SHARED ${CosmoTool_SRCS} $<TARGET_OBJECTS:CosmoHDF5>)
   target_link_libraries(CosmoTool ${CosmoTool_LIBS})
   if (BUILD_STATIC_LIBS)
-     add_library(CosmoTool_static STATIC ${CosmoTool_SRCS})
+     add_library(CosmoTool_static STATIC ${CosmoTool_SRCS} $<TARGET_OBJECTS:CosmoHDF5>)
-     set_target_properties(CosmoTool_static PROPERTIES COMPILE_FLAGS "${CMAKE_C_COMPILE_OPTIONS_PIC}")
+     target_link_libraries(CosmoTool_static)
+     set_property(TARGET CosmoTool_static PROPERTY POSITION_INDEPENDENT_CODE ON)
   endif(BUILD_STATIC_LIBS)
 else (BUILD_SHARED_LIBS)
-  add_library(CosmoTool STATIC ${CosmoTool_SRCS})
+  add_library(CosmoTool STATIC ${CosmoTool_SRCS} $<TARGET_OBJECTS:CosmoHDF5>)
-  set_target_properties(CosmoTool PROPERTIES COMPILE_FLAGS "${CMAKE_C_COMPILE_OPTIONS_PIC}")
+  target_link_libraries(CosmoTool)
+  set_property(TARGET CosmoTool PROPERTY POSITION_INDEPENDENT_CODE ON)
 endif (BUILD_SHARED_LIBS)
 
 install(TARGETS CosmoTool 

src/fortran.cpp

@@ -51,6 +51,7 @@ UnformattedRead::UnformattedRead(const string& fname)
   swapOrdering = false;
   cSize = Check_32bits;
   checkPointRef = checkPointAccum = 0;
+  recordBuffer = 0;
 }
 
 UnformattedRead::UnformattedRead(const char *fname)
@@ -130,6 +131,7 @@ void UnformattedRead::beginCheckpoint(bool bufferRecord)
     throw EndOfFileException();
 
   if (bufferRecord) {
+    std::cout << "Use fast I/O mode" << std::endl;
     recordBuffer = new uint8_t[checkPointRef];
     f->read(reinterpret_cast<char *>(recordBuffer), checkPointRef);
   }

src/hdf5_array.hpp

@@ -51,6 +51,11 @@ knowledge of the CeCILL license and that you accept its terms.
 #include <H5Cpp.h>
 
 namespace CosmoTool {
+#if (H5_VERS_MAJOR == 1) && (H5_VERS_MINOR <= 8)
+  typedef H5::CommonFG H5_CommonFileGroup;
+#else
+  typedef H5::Group H5_CommonFileGroup;
+#endif
 
   //!_______________________________________________________________________________________
   //!     
@@ -129,7 +134,7 @@ namespace CosmoTool {
 //!     
 //!_______________________________________________________________________________________
   template<typename ArrayType,  typename hdf5_data_type>
-  void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name, 
+  void hdf5_write_array(H5_CommonFileGroup& fg, const std::string& data_set_name, 
                         const ArrayType& data, 
                         const hdf5_data_type& datatype,
                         const std::vector<hsize_t>& dimensions,
@@ -156,7 +161,7 @@ namespace CosmoTool {
 
 
   template<typename ArrayType,  typename hdf5_data_type>
-  void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name, 
+  void hdf5_write_array(H5_CommonFileGroup& fg, const std::string& data_set_name, 
                         const ArrayType& data, 
                         const hdf5_data_type& datatype,
                         bool doCreate = true,
@@ -165,7 +170,7 @@ namespace CosmoTool {
       std::vector<hsize_t> dimensions(data.shape(), data.shape() + data.num_dimensions());
       hdf5_write_array(fg, data_set_name, data, datatype, dimensions, doCreate, useBases);
   }
-  
+
   /* HDF5 complex type */
   template<typename T>
   class hdf5_ComplexType
@@ -255,14 +260,14 @@ namespace CosmoTool {
   };
 
   template<typename ArrayType>
-  void hdf5_write_array(H5::CommonFG& fg, const std::string& data_set_name, const ArrayType& data )
+  void hdf5_write_array(H5_CommonFileGroup& fg, const std::string& data_set_name, const ArrayType& data )
   {
       typedef typename ArrayType::element T;
       get_hdf5_data_type<T> hdf_data_type;
       
       hdf5_write_array(fg, data_set_name, data, hdf_data_type.type());
   }
-  
+
   // HDF5 array reader
   //
   // Author Guilhem Lavaux (May 2014)
@@ -341,7 +346,7 @@ namespace CosmoTool {
   // ----------------------------------------------------------------------
   
   template<typename ArrayType, typename hdf5_data_type>
-  void hdf5_read_array_typed(H5::CommonFG& fg, const std::string& data_set_name, 
+  void hdf5_read_array_typed(H5_CommonFileGroup& fg, const std::string& data_set_name, 
                              ArrayType& data, 
                              const hdf5_data_type& datatype, bool auto_resize = true, bool useBases = false)
   {
@@ -377,7 +382,7 @@ namespace CosmoTool {
   }
 
   template<typename ArrayType>
-  void hdf5_read_array(H5::CommonFG& fg, const std::string& data_set_name, ArrayType& data, bool auto_resize = true, 
+  void hdf5_read_array(H5_CommonFileGroup& fg, const std::string& data_set_name, ArrayType& data, bool auto_resize = true, 
                       bool useBases = false )
   {
       typedef typename ArrayType::element T;

src/loadRamses.cpp

@@ -417,6 +417,9 @@ CosmoTool::SimuData *CosmoTool::loadRamsesSimu(const char *basename, int outputI
 	  data->Vel[1] = new float[nPar];
 	  data->Vel[2] = new float[nPar];
 	}
+      if (flags & NEED_MASS) {
+        data->Mass = new float[nPar];
+      }
       if (flags & NEED_GADGET_ID)
 	{
 	  data->Id = new int64_t[nPar];
@@ -455,9 +458,10 @@ CosmoTool::SimuData *CosmoTool::loadRamsesSimu(const char *basename, int outputI
       
       float minMass = INFINITY;
       infile.beginCheckpoint();
-      for (uint32_t i = nPar; i > 0; i--)
+      for (uint32_t i = 0; i < nPar; i++) {
-	{
 	  float dummyF = dp ? infile.readReal64() : infile.readReal32();
+          if (flags & NEED_MASS)
+            data->Mass[i] = dummyF;
 	  if (dummyF < minMass) minMass = dummyF;
 	}
       infile.endCheckpoint();