Merge branch 'master' of https://bitbucket.org/cosmicvoids/vide_public

c_tools/mock/loaders/gadget_loader.cpp

@@ -105,6 +105,8 @@ SimulationLoader *gadgetLoader(const std::string& snapshot, double Mpc_unitLengt
   int num_files;
   SimuData *d;
 
+  cout << " File to load is:" << snapshot.c_str() << endl;
+
   try
     {
       d = loadGadgetMulti(snapshot.c_str(), -1, 0);

c_tools/mock/loaders/simulation_loader.cpp

@@ -95,6 +95,9 @@ void SimulationLoader::basicPreprocessing(SimuData *d,
     }
   filteredCopy(d->Id, accepted, d->NumPart);
   filteredCopy(d->type, accepted, d->NumPart);
+  
+  filterAttribute<long>(d, "uniqueID", accepted, d->NumPart);
+  
   d->NumPart = numAccepted;
   delete[] accepted;
 }

c_tools/mock/loaders/simulation_loader.hpp

@@ -110,17 +110,27 @@ public:
 
     while (i < N)
       {
-	if (!accepted[i])
+        if (!accepted[i])
-	  {
+          {
-	    i++;
+            i++;
-	    continue;
+            continue;
-	  }
+          }
 
-	a[j] = a[i];
+        a[j] = a[i];
-	j++;
+        j++;
-	i++;
+        i++;
       }
   }
+  
+  template<typename T>
+  void filterAttribute(CosmoTool::SimuData *d, const std::string& attrname, bool *accepted, long NumPart)
+  {
+    if (d->attributes.find(attrname) == d->attributes.end())
+      return;
+ 
+    long *l = d->as<T>(attrname);
+    filteredCopy<T>(l, accepted, NumPart);
+  }
 };
 
 

external/external_build.cmake

@@ -37,7 +37,7 @@ IF(INTERNAL_NETCDF)
 ENDIF(INTERNAL_NETCDF)
 
 IF(INTERNAL_BOOST)
-  SET(BOOST_URL "http://sourceforge.net/projects/boost/files/boost/1.49.0/boost_1_49_0.tar.gz/download" CACHE STRING "URL to download Boost from")
+  SET(BOOST_URL "http://sourceforge.net/projects/boost/files/boost/1.56.0/boost_1_56_0.tar.gz/download" CACHE STRING "URL to download Boost from")
   mark_as_advanced(BOOST_URL)
 ELSE(INTERNAL_BOOST)
   find_package(Boost 1.49.0 COMPONENTS format spirit phoenix python FATAL_ERROR)
@@ -138,6 +138,11 @@ if (INTERNAL_NETCDF)
     DEPENDS ${hdf5_built}
     PREFIX ${BUILD_PREFIX}/netcdf-prefix
     URL ${NETCDF_URL}
+    PATCH_COMMAND  ${CMAKE_COMMAND} 
+      -DBUILD_PREFIX=${BUILD_PREFIX}/netcdf-prefix
+      -DPATCH_FILE=${CMAKE_SOURCE_DIR}/external/patch_netcdf
+      -DSOURCE_PREFIX=${BUILD_PREFIX}/netcdf-prefix/src/netcdf/ncgen3
+      -P ${CMAKE_SOURCE_DIR}/external/check_and_apply_patch.cmake
     CONFIGURE_COMMAND ${NETCDF_SOURCE_DIR}/configure
          --prefix=${NETCDF_BIN_DIR} --libdir=${NETCDF_BIN_DIR}/lib
          --enable-netcdf-4  --with-pic --disable-shared --disable-dap 

external/external_python_build.cmake

@@ -62,14 +62,18 @@ ENDIF(RET_VALUE)
 STRING(REGEX REPLACE "(\r?\n)+$" "" PYTHON_LOCAL_SITE_PACKAGE "${PYTHON_LOCAL_SITE_PACKAGE}")
 MESSAGE(STATUS "Python is installing its packages in ${PYTHON_LOCAL_SITE_PACKAGE}") 
 
+SET(BASIC_PYTHON_ENV "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} "
+                     "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
+                     "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
+                     "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}"
+                     "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python")
 
 
 IF(INTERNAL_CYTHON)
   SET(BUILD_ENVIRONMENT 
           ${CMAKE_COMMAND}
-           "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
+          ${BASIC_PYTHON_ENV}
-           "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}"
+          "-P") 
-           "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P") 
   ExternalProject_Add(cython
     DEPENDS ${PREV_PYTHON_BUILD}
     URL ${CYTHON_URL}
@@ -88,13 +92,12 @@ IF(INTERNAL_NETCDF4_PYTHON)
   SET(PYTHON_LDFLAGS -L${NETCDF_BIN_DIR}/lib)
   SET(BUILD_ENVIRONMENT 
           ${CMAKE_COMMAND}
-           "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
+          ${BASIC_PYTHON_ENV}
            "-DPYTHON_CPPFLAGS:STRING=${PYTHON_CPPFLAGS}"
            "-DHDF5_DIR=${HDF5_BIN_DIR}"
            "-DNETCDF4_DIR=${NETCDF_BIN_DIR}"
            "-DPYTHON_LDFLAGS:STRING=${PYTHON_LDFLAGS}"
-           "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}"
+           "-P") 
-           "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P") 
 
   ExternalProject_Add(netcdf4-python
     DEPENDS ${PREV_PYTHON_BUILD} netcdf
@@ -111,16 +114,15 @@ ENDIF(INTERNAL_NETCDF4_PYTHON)
 IF(INTERNAL_HEALPY)
   SET(BUILD_ENVIRONMENT 
           ${CMAKE_COMMAND}
-           "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
+          ${BASIC_PYTHON_ENV}
            "-DPYTHON_CPPFLAGS:STRING=${PYTHON_CPPFLAGS}"
            "-DCFITSIO_EXT_LIB=${CFITSIO_LIBRARY}"
            "-DCFITSIO_EXT_INC=${CFITSIO_INCLUDE_PATH}"
            "-DCFITSIO_EXT_PREFIX=${CFITSIO_PREFIX}"
            "-DNETCDF4_DIR=${NETCDF_BIN_DIR}"
            "-DPYTHON_LDFLAGS:STRING=${PYTHON_LDFLAGS}"
-           "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}"
            "-DSUPPORT_ARCH_NATIVE=${SUPPORT_ARCH_NATIVE}"
-           "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P")
+           "-P")
 
   ExternalProject_Add(healpy
     DEPENDS ${PREV_PYTHON_BUILD}
@@ -136,9 +138,8 @@ ENDIF(INTERNAL_HEALPY)
 IF(INTERNAL_SETUPTOOLS)
   SET(BUILD_ENVIRONMENT 
           ${CMAKE_COMMAND}
-           "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
+          ${BASIC_PYTHON_ENV}
-           "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}"
+           "-P")
-           "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P")
 
   ExternalProject_Add(setuptools
     URL ${SETUPTOOLS_URL}
@@ -168,9 +169,8 @@ ENDIF(INTERNAL_ARGPARSE)
 IF(INTERNAL_SCIPY)
   SET(BUILD_ENVIRONMENT 
           ${CMAKE_COMMAND}
-           "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
+          ${BASIC_PYTHON_ENV}
-           "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}"
+           "-P")
-           "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P")
 
   ExternalProject_Add(scipy
     DEPENDS ${PREV_PYTHON_BUILD}
@@ -187,9 +187,8 @@ ENDIF(INTERNAL_SCIPY)
 IF(INTERNAL_KDTREE_SCIPY)
   SET(BUILD_ENVIRONMENT 
           ${CMAKE_COMMAND}
-           "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
+          ${BASIC_PYTHON_ENV}
-           "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}"
+           "-P")
-           "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P")
 
   ExternalProject_Add(kdtree-scipy
     DEPENDS ${PREV_PYTHON_BUILD}

external/patch_netcdf

@@ -0,0 +1,10 @@
+--- load.c	2011-06-30 18:22:19.000000000 -0500
++++ new_load.c	2014-09-11 04:47:52.057267514 -0500
+@@ -5,6 +5,7 @@
+  *********************************************************************/
+ 
+ #include <stdio.h>
++#include <config.h>
+ #include <stdlib.h>
+ #include <string.h>
+ #include <ctype.h>

external/python_build.cmake

@@ -7,6 +7,8 @@ SET(ENV{PYTHONPATH} ${PYTHON_LOCAL_SITE_PACKAGE}:$ENV{PYTHONPATH})
 SET(ENV{CFITSIO_EXT_INC} ${CFITSIO_EXT_INC})
 SET(ENV{CFITSIO_EXT_LIB} ${CFITSIO_EXT_LIB})
 SET(ENV{CFITSIO_EXT_PREFIX} ${CFITSIO_EXT_PREFIX})
+SET(ENV{CC} ${CMAKE_C_COMPILER})
+SET(ENV{CXX} ${CMAKE_CXX_COMPILER})
 IF (NOT SUPPORT_ARCH_NATIVE)
   SET(ENV{HEALPY_WITHOUT_NATIVE} 1)
 ENDIF(NOT SUPPORT_ARCH_NATIVE)

external/python_install.cmake

@@ -7,6 +7,9 @@ SET(ENV{CFITSIO_EXT_INC} ${CFITSIO_EXT_INC})
 SET(ENV{CFITSIO_EXT_PREFIX} ${CFITSIO_EXT_PREFIX})
 SET(ENV{CFITSIO_EXT_LIB} ${CFITSIO_EXT_LIB})
 SET(ENV{PYTHONPATH} ${PYTHON_LOCAL_SITE_PACKAGE}:$ENV{PYTHONPATH})
+SET(ENV{CC} ${CMAKE_C_COMPILER})
+SET(ENV{CXX} ${CMAKE_CXX_COMPILER})
+
 
 SET(PYTHON_INSTALL_COMMAND ${PYTHON_EXECUTABLE} setup.py install --prefix=${TARGET_PATH} --install-lib=${PYTHON_LOCAL_SITE_PACKAGE})
 

pipeline/datasets/example_simulation.py

@@ -98,8 +98,12 @@ redshifts = ["0.0"]
 # list of desired subsamples - these are in unts of h Mpc^-3!
 subSamples = [1.0, 0.5]
 
-# if True, do the subsampling in preparation (only for sdf and multidark)
+# if True, do the subsampling in preparation (available for sdf and multidark)
-doSubSamplingInPrep = True 
+doSubSamplingInPrep = False
+
+# if 'absolute', subSamples are given in particles per cubic Mpc/h
+# if 'relative', subSamples are given as a fraction of input particles
+subSampleMode = "relative"
 
 # shift the z-coord of sims with redshift
 shiftSimZ = False

python_tools/pipeline_source/defaults.py

@@ -96,7 +96,7 @@ doSubSamplingInPrep = False # do the subsampling in preparation script?
 
 # if 'absolute', subSamples are given in particles per cubic Mpc/h
 # if 'relative', subSamples are given as a fraction of input particles
-subSampleMode = "absolute" 
+subSampleMode = "relative"
 
 # common filename of halo files, leave blank to ignore halos
 haloFileBase = "mf_4s_1G_512_bgc2_NNNNN.sdf"

python_tools/void_python_tools/backend/launchers.py

@@ -271,7 +271,7 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
   if sample.dataType == "observation":
     (boxVol, nbar) = vp.getSurveyProps(sample.maskFile, sample.zRange[0],
      sample.zRange[1], sample.zRange[0], sample.zRange[1], "all", 
-     useLCDM=useComoving)
+     useComoving=useComoving)
   else:
     iX = float(sample.mySubvolume[0])
     iY = float(sample.mySubvolume[1])