Added (optional for the moment) SDF support in simulation_loaders.

c_tools/mock/loaders/flash_loader.cpp

@@ -15,9 +15,10 @@ private:
   int _num_files;
   SimuData *gadget_header;
   string snapshot_name;
+  SimulationPreprocessor *preproc;
 public:
-  FlashLoader(const string& basename, SimuData *header, int flags, bool singleFile, int _num)
+  FlashLoader(const string& basename, SimuData *header, int flags, bool singleFile, int _num, SimulationPreprocessor *p)
-    : snapshot_name(basename), load_flags(flags), onefile(singleFile), _num_files(_num), gadget_header(header)
+    : snapshot_name(basename), load_flags(flags), onefile(singleFile), _num_files(_num), gadget_header(header), preproc(p)
   {
   }
   
@@ -58,6 +59,7 @@ public:
       }
 
     applyTransformations(d);
+    basicPreprocessing(d, preproc);
 
     return d;
   }
@@ -107,5 +109,5 @@ SimulationLoader *flashLoader(const std::string& snapshot, int flags, Simulation
 	}
     }
     
-  return new FlashLoader(snapshot, header, flags, singleFile, num_files);
+  return new FlashLoader(snapshot, header, flags, singleFile, num_files, p);
 }

c_tools/mock/loaders/ramses_loader.cpp

@@ -16,10 +16,11 @@ private:
   bool double_precision;
   SimuData *ramses_header;
   string snapshot_name;
+  SimulationPreprocessor *preproc;
 public:
-  RamsesLoader(const string& basename, int baseid, bool dp, SimuData *header, int flags, int _num)
+  RamsesLoader(const string& basename, int baseid, bool dp, SimuData *header, int flags, int _num, SimulationPreprocessor *p)
     : snapshot_name(basename), load_flags(flags), _num_files(_num), double_precision(dp),
-      ramses_header(header)
+      ramses_header(header), preproc(p)
   {
   }
   
@@ -56,6 +57,7 @@ public:
       }
 
     applyTransformations(d);
+    basicPreprocessing(d, preproc);
 
     return d;
   }
@@ -76,6 +78,6 @@ SimulationLoader *ramsesLoader(const std::string& snapshot, int baseid, bool dou
       delete d;
     }
 
-  return new RamsesLoader(snapshot, baseid, double_precision, header, flags, num_files);
+  return new RamsesLoader(snapshot, baseid, double_precision, header, flags, num_files, p);
 }
 

c_tools/mock/loaders/sdf_loader.cpp

@@ -0,0 +1,232 @@
+#include <iostream>
+#include <boost/format.hpp>
+#include <vector>
+#include <cassert>
+#include <string>
+#include "sdfloader_internal.hpp"
+#include "simulation_loader.hpp"
+#include <libsdf/mpmy.h>
+#include <libsdf/SDF.h>
+#include <libsdf/error.h>
+#include <libsdf/cosmo.h>
+
+using boost::format;
+
+using namespace std;
+using namespace CosmoTool;
+
+class SDFLoader: public SimulationLoader
+{
+private:
+  int load_flags;
+  bool onefile;
+  int _num_files;
+  double unitMpc;
+  SimuData *sdf_header;
+  SDF *sdfp;
+  SimulationPreprocessor *preproc;
+  int num_splitting;
+public:
+  SDFLoader(SDF *sdf_file, SimuData *header, int flags, int num_splitting,
+            SimulationPreprocessor *p)
+    : sdfp(sdf_file), load_flags(flags),
+      sdf_header(header), preproc(p)
+  {
+    this->num_splitting = num_splitting;
+  }
+  
+  ~SDFLoader()
+  {
+    delete sdf_header;
+  }
+  
+  SimuData *getHeader() {
+    return sdf_header;
+  }
+  
+  int num_files() {
+    return num_splitting;
+  }
+
+
+  int64_t getStart(int id)
+  {
+    return sdf_header->TotalNumPart * int64_t(id) / num_splitting;
+  }
+
+  int64_t getNumberInSplit(int id)
+  {
+    return getStart(id+1)-getStart(id);
+  }
+
+  void rescaleParticles(SimuData *d,
+                        double rescale_position,
+                        double rescale_velocity)
+  {
+    float shift = 0.5*d->BoxSize;
+    rescale_position /= d->time;
+
+    if (d->Pos[0] != 0)
+      {
+        for (int k = 0; k < 3; k++)
+          {
+            for (int64_t i = 0; i < d->NumPart; i++)
+              {
+                d->Pos[k][i] = (d->Pos[k][i]*rescale_position + shift);
+              }
+          }
+      }
+    if (d->Vel[0] != 0)
+      {
+        for (int k = 0; k < 3; k++)
+          {
+            for (int64_t i = 0; i < d->NumPart; i++)
+              {
+                d->Vel[k][i] *= rescale_velocity;
+              }
+          }
+      }
+  }
+
+
+  SimuData *loadFile(int id) {
+    SimuData *d;
+    int64_t starts[7];
+    int nobjs[7];
+    int strides[7];
+    void *addrs[7];
+    const char *names[7];
+    int ret;
+
+    if (id >= num_splitting)
+      return 0;
+
+    d = new SimuData;
+    *d = *sdf_header;
+    d->NumPart = getNumberInSplit(id);
+
+    int64_t numPartToLoad = getNumberInSplit(id);
+    int64_t start = getStart(id);
+    int k = 0;
+#define SETUP_READ(name, vec) \
+    names[k] = name, starts[k] = start, nobjs[k] = numPartToLoad, strides[k] = sizeof(vec[0]), addrs[k] = vec, k++;
+
+    if (load_flags & NEED_POSITION)
+      {
+        const char *name_template[3] = { "x", "y", "z" };
+        for (int q = 0; q < 3; q++)
+          {
+            d->Pos[q] = new float[numPartToLoad];
+            SETUP_READ(name_template[q], d->Pos[q]);
+          }
+      }
+    if (load_flags & NEED_VELOCITY)
+      {
+        const char *name_template[3] = { "x", "y", "z" };
+        for (int q = 0; q < 3; q++)
+          {
+            d->Vel[q] = new float[numPartToLoad];
+            SETUP_READ(name_template[q], d->Vel[q]);
+          }
+      }
+    if (load_flags & NEED_GADGET_ID)
+      {
+        d->Id = new long[numPartToLoad];
+        SETUP_READ("ident", d->Id);
+      }
+#undef SETUP_READ
+
+    ret = SDFseekrdvecsarr(sdfp, k, (char**)names, starts, nobjs, addrs, strides);
+    if (ret != 0)
+      {
+        cerr << format("Splitting %d/%d, SDF read failure: %s") % id % num_splitting % SDFerrstring << endl;
+        abort();
+      }
+
+    if (load_flags & (NEED_POSITION | NEED_VELOCITY))
+      rescaleParticles(d, 0.001*d->Hubble, one_kpc/one_Gyr);
+
+    applyTransformations(d);
+    basicPreprocessing(d, preproc);
+
+    return d;
+  }
+};
+
+SimulationLoader *sdfLoader(const std::string& snapshot, int flags, 
+                            int num_splitting,
+                            SimulationPreprocessor *p)
+{
+  SDF *sdfp;
+  int fileversion;
+  SimuData *hdr;
+  int64_t gnobj;
+
+  sdfp = SDFopen(0, snapshot.c_str());
+  if (sdfp == 0)
+    {
+      return 0;
+    }
+  
+  SDFgetintOrDefault(sdfp, "version", &fileversion, 1);
+  if (fileversion == 1)
+    {
+      SDFgetfloatOrDie(sdfp, "Omega0", &hdr->Omega_M);
+      SDFgetfloatOrDie(sdfp, "Lambda_prime", &hdr->Omega_Lambda);
+      SDFgetfloatOrDie(sdfp, "H0", &hdr->Hubble);
+    }
+  else
+    {
+      float Or, Of;
+      SDFgetfloatOrDie(sdfp, "Omega0_m", &hdr->Omega_M);
+      SDFgetfloatOrDie(sdfp, "Omega0_lambda", &hdr->Omega_Lambda);
+      SDFgetfloatOrDie(sdfp, "Omega0_r", &Or);
+      SDFgetfloatOrDie(sdfp, "Omega0_fld", &Of);
+
+      hdr->Omega_M += Or;
+      hdr->Omega_Lambda += Of;
+      SDFgetfloatOrDie(sdfp, "H0", &hdr->Hubble);
+      
+    }
+  double h0;
+  h0 = hdr->Hubble*10.0*(one_kpc/one_Gyr);
+
+  if (SDFhasname("R0", sdfp))
+    {
+      double R0;
+
+      SDFgetdoubleOrDie(sdfp, "R0", &R0);
+      hdr->BoxSize = 2.0*0.001*R0*h0;
+    }
+
+  if (SDFhasname("Rx", sdfp))
+    {
+      double R0;
+
+      SDFgetdoubleOrDie(sdfp, "Rx", &R0);
+      hdr->BoxSize = 2.0*0.001*R0*h0;
+    }
+
+  if (SDFhasname("redshift", sdfp))
+    {
+      double redshift;
+
+      SDFgetdoubleOrDie(sdfp, "redshift", &redshift);
+      hdr->time = 1/(1+redshift);
+    }
+   
+  if (SDFgetint64(sdfp, "npart", &gnobj))
+    {
+      gnobj = SDFnrecs("x", sdfp);
+      cerr << format("[Warning] No 'npart' found in SDF file '%s', guessing npart=%ld from SDFnrecs") % snapshot % gnobj << endl;
+    }
+  hdr->NumPart = hdr->TotalNumPart = gnobj;
+
+  return new SDFLoader(sdfp, hdr, flags, num_splitting, p);
+}
+
+
+void sdfLoaderInit(int& argc, char **& argv)
+{
+  MPMY_Init(&argc, &argv);
+}

c_tools/mock/loaders/sdfloader_internal.hpp

@@ -0,0 +1,6 @@
+#ifndef __VOID_SDF_LOADER_INTERNAL_HPP
+#define __VOID_SDF_LOADER_INTERNAL_HPP
+
+void sdfLoaderInit(int& argc, char **&argv);
+
+#endif

c_tools/mock/loaders/simulation_loader.cpp

@@ -1,6 +1,9 @@
 #include <cmath>
 #include <CosmoTool/loadSimu.hpp>
 #include "simulation_loader.hpp"
+#ifdef SDF_SUPPORT
+#include "sdfloader_internal.hpp"
+#endif
 
 using std::min;
 using namespace CosmoTool;
@@ -44,30 +47,39 @@ void SimulationLoader::applyTransformations(SimuData *s)
 void SimulationLoader::basicPreprocessing(SimuData *d, 
                                           SimulationPreprocessor *preproc)
 {
-    long numAccepted = 0;
+  if (preproc == 0)
-    bool *accepted = new bool[d->NumPart];
+    return;
-    for (long i = 0; i < d->NumPart; i++)
-      {
-	SingleParticle p;
-
-	for (int k = 0; k < 3; k++)
-	  {
-	    p.Pos[k] = (d->Pos[k]) ? 0 : d->Pos[k][i];
-	    p.Vel[k] = (d->Vel[k]) ? 0 : d->Vel[k][i];
-	  }
-	p.ID = (d->Id) ? 0 : d->Id[i];
-	
-	accepted[i] = preproc->accept(p);
-	numAccepted += accepted[i];
-      }
-    
-    for (int k = 0; k < 3; k++)
-      {
-	filteredCopy(d->Pos[k], accepted, d->NumPart);
-	filteredCopy(d->Vel[k], accepted, d->NumPart);
-      }
-    filteredCopy(d->Id, accepted, d->NumPart);
-    filteredCopy(d->type, accepted, d->NumPart);
-    delete[] accepted;
 
+  long numAccepted = 0;
+  bool *accepted = new bool[d->NumPart];
+  for (long i = 0; i < d->NumPart; i++)
+    {
+      SingleParticle p;
+      
+      for (int k = 0; k < 3; k++)
+        {
+          p.Pos[k] = (d->Pos[k]) ? 0 : d->Pos[k][i];
+          p.Vel[k] = (d->Vel[k]) ? 0 : d->Vel[k][i];
+        }
+      p.ID = (d->Id) ? 0 : d->Id[i];
+      
+      accepted[i] = preproc->accept(p);
+      numAccepted += accepted[i];
+    }
+  
+  for (int k = 0; k < 3; k++)
+    {
+      filteredCopy(d->Pos[k], accepted, d->NumPart);
+      filteredCopy(d->Vel[k], accepted, d->NumPart);
+    }
+  filteredCopy(d->Id, accepted, d->NumPart);
+  filteredCopy(d->type, accepted, d->NumPart);
+  delete[] accepted;
+}
+
+void simulationLoadersInit(int& argc, char **& argv)
+{
+#ifdef SDF_SUPPORT
+  sdfLoaderInit(argc, argv);
+#endif
 }

c_tools/mock/loaders/simulation_loader.hpp

@@ -13,6 +13,17 @@ struct SingleParticle
   long ID;
 };
 
+class SimulationPreprocessor
+{
+public:
+  SimulationPreprocessor() {}
+  virtual ~SimulationPreprocessor() {}
+
+  virtual long getEstimatedPostprocessed(long numParticles) { return numParticles; };
+  virtual bool accept(const SingleParticle& p) { return true; }
+  virtual void reset() {}
+};
+
 class SimulationLoader
 {
 protected:
@@ -41,7 +52,7 @@ protected:
   void reallocSimu(CosmoTool::SimuData *s, long newNumPart);
  
 
-  void basicPreprocessor(SimuData *d, SimulationPreprocessor *preproc);
+  void basicPreprocessing(CosmoTool::SimuData *d, SimulationPreprocessor *preproc);
   void applyTransformations(CosmoTool::SimuData *s);
 
   void copyParticleToSimu(const SingleParticle& p, CosmoTool::SimuData *s, long index)
@@ -91,15 +102,6 @@ public:
   }
 };
 
-class SimulationPreprocessor
-{
-public:
-  SimulationPreprocessor() {}
-  virtual ~SimulationPreprocessor() {}
-
-  virtual long getEstimatedPostprocessed(long numParticles) { return numParticles; };
-  virtual bool accept(const SingleParticle& p) { return true; }
-};
 
 template<typename T>
 void delete_adaptor(void *ptr)
@@ -115,6 +117,9 @@ SimulationLoader *gadgetLoader(const std::string& snapshot, double Mpc_unitLengt
 SimulationLoader *flashLoader(const std::string& snapshot, int flags, SimulationPreprocessor *p);
 SimulationLoader *multidarkLoader(const std::string& snapshot, SimulationPreprocessor *p);
 SimulationLoader *ramsesLoader(const std::string& snapshot, int baseid, bool double_precision, int flags, SimulationPreprocessor *p);
+SimulationLoader *sdfLoader(const std::string& snapshot, int flags, int num_splitting, SimulationPreprocessor *p);
 
+/* This is required for some parallel I/O handler (thus MPI beneath it) */
+void simulationLoadersInit(int& argc, char **& argv);
 
 #endif

external/external_build.cmake

@@ -1,6 +1,7 @@
 include(FindOpenMP)
 
 OPTION(ENABLE_OPENMP "Set to Yes if Healpix and/or you need openMP" OFF)
+OPTION(SDF_SUPPORT "Set to Yes to activate support for SDF" OFF)
 
 IF(ENABLE_OPENMP)
 
@@ -76,7 +77,7 @@ if (INTERNAL_GENGETOPT)
 		CC=${CMAKE_C_COMPILER} 
 		CXX=${CMAKE_CXX_COMPILER}
     BUILD_IN_SOURCE 1
-    INSTALL_COMMAND make install
+    INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} install
   )
   SET(GENGETOPT ${GENGETOPT_BIN_DIR}/bin/gengetopt CACHE FILEPATH "Path GenGetOpt binary")
 else(INTERNAL_GENGETOPT)
@@ -100,7 +101,7 @@ if (INTERNAL_HDF5)
 	   CPPFLAGS=${CONFIGURE_CPP_FLAGS} CC=${CMAKE_C_COMPILER} 
 	   CXX=${CMAKE_CXX_COMPILER}
     BUILD_IN_SOURCE 1
-    INSTALL_COMMAND make install
+    INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} install
   )
   SET(cosmotool_DEPS ${cosmotool_DEPS} hdf5)
   SET(hdf5_built hdf5)
@@ -144,7 +145,7 @@ if (INTERNAL_NETCDF)
          --disable-examples ${EXTRA_NC_FLAGS} CC=${CMAKE_C_COMPILER}
          CXX=${CMAKE_CXX_COMPILER}
     BUILD_IN_SOURCE 1
-    INSTALL_COMMAND make install
+    INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} install
   )
   SET(CONFIGURE_CPP_LDFLAGS "${CONFIGURE_LDFLAGS}")
   SET(EXTRA_NC_FLAGS CPPFLAGS=${CONFIGURE_CPP_FLAGS} LDFLAGS=${CONFIGURE_CPP_LDFLAGS})
@@ -198,8 +199,8 @@ IF(INTERNAL_GSL)
            --with-pic --libdir=${CMAKE_BINARY_DIR}/ext_build/gsl/lib
            CPPFLAGS=${CONFIGURE_CPP_FLAGS} CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER}
     BUILD_IN_SOURCE 1
-    BUILD_COMMAND make
+    BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
-    INSTALL_COMMAND make install
+    INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} install
   )
   SET(GSL_INTERNAL_LIBS ${CMAKE_BINARY_DIR}/ext_build/gsl/lib)
   SET(GSL_LIBRARY ${GSL_INTERNAL_LIBS}/libgsl.a CACHE STRING "GSL internal path" FORCE)
@@ -249,9 +250,9 @@ ExternalProject_Add(cfitsio
 	    CPPFLAGS=${CONFIGURE_CPP_FLAGS} 
 	    CC=${CMAKE_C_COMPILER} 
 	    CXX=${CMAKE_CXX_COMPILER}
-  BUILD_COMMAND make
+  BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} 
   BUILD_IN_SOURCE 1
-  INSTALL_COMMAND make install
+  INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} install
 )
 SET(CFITSIO_PREFIX ${CMAKE_BINARY_DIR}/ext_build/cfitsio)
 SET(CFITSIO_LIBRARY ${CFITSIO_PREFIX}/lib/libcfitsio.a)
@@ -266,7 +267,7 @@ ExternalProject_Add(healpix
   PREFIX ${BUILD_PREFIX}/healpix-prefix
   SOURCE_DIR ${CMAKE_SOURCE_DIR}/external/healpix
   CONFIGURE_COMMAND echo No configure
-  BUILD_COMMAND make
+  BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
       HEALPIX_TARGET=sampler 
       HEALPIX_CC=${CMAKE_C_COMPILER} 
       HEALPIX_CXX=${CMAKE_CXX_COMPILER} 
@@ -315,9 +316,28 @@ endif(INTERNAL_QHULL)
 
 SET(QHULL_LIBRARIES ${QHULL_CPP_LIBRARY} ${QHULL_LIBRARY} )
 
+
+###############
+# Build libSDF
+###############
+IF(SDF_SUPPORT)
+  SET(LIBSDF_ARCH x86_64)
+  SET(LIBSDF_PATH ${CMAKE_SOURCE_DIR}/external/libsdf)
+
+  ExternalProject_Add(libSDF
+    PREFIX ${BUILD_PREFIX}/libSDF-prefix
+    SOURCE_DIR ${LIBSDF_PATH}
+    CONFIGURE_COMMAND echo No configure
+    BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} ARCH=${LIBSDF_ARCH}
+    INSTALL_COMMAND echo No install
+    BUILD_IN_SOURCE 1
+  )
+  SET(LIBSDF_INCLUDE_PATH ${LIBSDF_PATH}/include)
+  SET(LIBSDF_LIBRARY ${LIBSDF_PATH}/Objfiles/${LIBSDF_ARCH}/libsw.a)
+ENDIF(SDF_SUPPORT)
+
 include_directories(${CMAKE_BINARY_DIR}/src 
                     ${NETCDF_INCLUDE_PATH} ${GSL_INCLUDE_PATH} 
                     ${HDF5_INCLUDE_PATH} ${COSMOTOOL_INCLUDE_PATH} 
                     ${Boost_INCLUDE_DIRS}
-		    ${QHULL_INCLUDE_PATH})
+		    ${QHULL_INCLUDE_PATH} ${LIBSDF_INCLUDE_PATH})
-