From 1ff47d928e8f90637ab54ea91a3d0a9be96ee9d3 Mon Sep 17 00:00:00 2001 From: "P.M. Sutter" Date: Thu, 11 Sep 2014 04:00:53 -0500 Subject: [PATCH 1/9] update to latest version of boost to fix compatibility issues with GCC > v4.7 --- external/external_build.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/external_build.cmake b/external/external_build.cmake index f43be0d..57e595a 100644 --- a/external/external_build.cmake +++ b/external/external_build.cmake @@ -37,7 +37,7 @@ IF(INTERNAL_NETCDF) ENDIF(INTERNAL_NETCDF) IF(INTERNAL_BOOST) - SET(BOOST_URL "http://sourceforge.net/projects/boost/files/boost/1.49.0/boost_1_49_0.tar.gz/download" CACHE STRING "URL to download Boost from") + SET(BOOST_URL "http://sourceforge.net/projects/boost/files/boost/1.56.0/boost_1_56_0.tar.gz/download" CACHE STRING "URL to download Boost from") mark_as_advanced(BOOST_URL) ELSE(INTERNAL_BOOST) find_package(Boost 1.49.0 COMPONENTS format spirit phoenix python FATAL_ERROR) From e8f1449741a584397ca7725b0085a2e9c101d70f Mon Sep 17 00:00:00 2001 From: "P.M. Sutter" Date: Thu, 11 Sep 2014 04:51:53 -0500 Subject: [PATCH 2/9] patch to enable compilation of netcdf with later builds of GCC and OSX --- external/external_build.cmake | 5 +++++ external/patch_netcdf | 10 ++++++++++ 2 files changed, 15 insertions(+) create mode 100644 external/patch_netcdf diff --git a/external/external_build.cmake b/external/external_build.cmake index 57e595a..e35f2d6 100644 --- a/external/external_build.cmake +++ b/external/external_build.cmake @@ -138,6 +138,11 @@ if (INTERNAL_NETCDF) DEPENDS ${hdf5_built} PREFIX ${BUILD_PREFIX}/netcdf-prefix URL ${NETCDF_URL} + PATCH_COMMAND ${CMAKE_COMMAND} + -DBUILD_PREFIX=${BUILD_PREFIX}/netcdf-prefix + -DPATCH_FILE=${CMAKE_SOURCE_DIR}/external/patch_netcdf + -DSOURCE_PREFIX=${BUILD_PREFIX}/netcdf-prefix/src/netcdf/ncgen3 + -P ${CMAKE_SOURCE_DIR}/external/check_and_apply_patch.cmake CONFIGURE_COMMAND ${NETCDF_SOURCE_DIR}/configure --prefix=${NETCDF_BIN_DIR} --libdir=${NETCDF_BIN_DIR}/lib --enable-netcdf-4 --with-pic --disable-shared --disable-dap diff --git a/external/patch_netcdf b/external/patch_netcdf new file mode 100644 index 0000000..43ea559 --- /dev/null +++ b/external/patch_netcdf @@ -0,0 +1,10 @@ +--- load.c 2011-06-30 18:22:19.000000000 -0500 ++++ new_load.c 2014-09-11 04:47:52.057267514 -0500 +@@ -5,6 +5,7 @@ + *********************************************************************/ + + #include ++#include + #include + #include + #include From 15e065a83337a932b34e618eb13bf624eddfe907 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Mon, 13 Oct 2014 17:43:52 +0200 Subject: [PATCH 3/9] Enforce the CC/CXX compiler chosen by cmake --- external/external_python_build.cmake | 35 ++++++++++++++-------------- external/python_build.cmake | 2 ++ external/python_install.cmake | 3 +++ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/external/external_python_build.cmake b/external/external_python_build.cmake index bf1907c..88bd264 100644 --- a/external/external_python_build.cmake +++ b/external/external_python_build.cmake @@ -62,14 +62,18 @@ ENDIF(RET_VALUE) STRING(REGEX REPLACE "(\r?\n)+$" "" PYTHON_LOCAL_SITE_PACKAGE "${PYTHON_LOCAL_SITE_PACKAGE}") MESSAGE(STATUS "Python is installing its packages in ${PYTHON_LOCAL_SITE_PACKAGE}") +SET(BASIC_PYTHON_ENV "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} " + "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" + "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" + "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}" + "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python") IF(INTERNAL_CYTHON) SET(BUILD_ENVIRONMENT ${CMAKE_COMMAND} - "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" - "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}" - "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P") + ${BASIC_PYTHON_ENV} + "-P") ExternalProject_Add(cython DEPENDS ${PREV_PYTHON_BUILD} URL ${CYTHON_URL} @@ -88,13 +92,12 @@ IF(INTERNAL_NETCDF4_PYTHON) SET(PYTHON_LDFLAGS -L${NETCDF_BIN_DIR}/lib) SET(BUILD_ENVIRONMENT ${CMAKE_COMMAND} - "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" + ${BASIC_PYTHON_ENV} "-DPYTHON_CPPFLAGS:STRING=${PYTHON_CPPFLAGS}" "-DHDF5_DIR=${HDF5_BIN_DIR}" "-DNETCDF4_DIR=${NETCDF_BIN_DIR}" "-DPYTHON_LDFLAGS:STRING=${PYTHON_LDFLAGS}" - "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}" - "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P") + "-P") ExternalProject_Add(netcdf4-python DEPENDS ${PREV_PYTHON_BUILD} netcdf @@ -111,16 +114,15 @@ ENDIF(INTERNAL_NETCDF4_PYTHON) IF(INTERNAL_HEALPY) SET(BUILD_ENVIRONMENT ${CMAKE_COMMAND} - "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" + ${BASIC_PYTHON_ENV} "-DPYTHON_CPPFLAGS:STRING=${PYTHON_CPPFLAGS}" "-DCFITSIO_EXT_LIB=${CFITSIO_LIBRARY}" "-DCFITSIO_EXT_INC=${CFITSIO_INCLUDE_PATH}" "-DCFITSIO_EXT_PREFIX=${CFITSIO_PREFIX}" "-DNETCDF4_DIR=${NETCDF_BIN_DIR}" "-DPYTHON_LDFLAGS:STRING=${PYTHON_LDFLAGS}" - "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}" "-DSUPPORT_ARCH_NATIVE=${SUPPORT_ARCH_NATIVE}" - "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P") + "-P") ExternalProject_Add(healpy DEPENDS ${PREV_PYTHON_BUILD} @@ -136,9 +138,8 @@ ENDIF(INTERNAL_HEALPY) IF(INTERNAL_SETUPTOOLS) SET(BUILD_ENVIRONMENT ${CMAKE_COMMAND} - "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" - "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}" - "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P") + ${BASIC_PYTHON_ENV} + "-P") ExternalProject_Add(setuptools URL ${SETUPTOOLS_URL} @@ -168,9 +169,8 @@ ENDIF(INTERNAL_ARGPARSE) IF(INTERNAL_SCIPY) SET(BUILD_ENVIRONMENT ${CMAKE_COMMAND} - "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" - "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}" - "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P") + ${BASIC_PYTHON_ENV} + "-P") ExternalProject_Add(scipy DEPENDS ${PREV_PYTHON_BUILD} @@ -187,9 +187,8 @@ ENDIF(INTERNAL_SCIPY) IF(INTERNAL_KDTREE_SCIPY) SET(BUILD_ENVIRONMENT ${CMAKE_COMMAND} - "-DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}" - "-DPYTHON_LOCAL_SITE_PACKAGE=${PYTHON_LOCAL_SITE_PACKAGE}" - "-DTARGET_PATH=${CMAKE_BINARY_DIR}/ext_build/python" "-P") + ${BASIC_PYTHON_ENV} + "-P") ExternalProject_Add(kdtree-scipy DEPENDS ${PREV_PYTHON_BUILD} diff --git a/external/python_build.cmake b/external/python_build.cmake index f733405..3e10cb3 100644 --- a/external/python_build.cmake +++ b/external/python_build.cmake @@ -7,6 +7,8 @@ SET(ENV{PYTHONPATH} ${PYTHON_LOCAL_SITE_PACKAGE}:$ENV{PYTHONPATH}) SET(ENV{CFITSIO_EXT_INC} ${CFITSIO_EXT_INC}) SET(ENV{CFITSIO_EXT_LIB} ${CFITSIO_EXT_LIB}) SET(ENV{CFITSIO_EXT_PREFIX} ${CFITSIO_EXT_PREFIX}) +SET(ENV{CC} ${CMAKE_C_COMPILER}) +SET(ENV{CXX} ${CMAKE_CXX_COMPILER}) IF (NOT SUPPORT_ARCH_NATIVE) SET(ENV{HEALPY_WITHOUT_NATIVE} 1) ENDIF(NOT SUPPORT_ARCH_NATIVE) diff --git a/external/python_install.cmake b/external/python_install.cmake index ee04a3c..231b7ca 100644 --- a/external/python_install.cmake +++ b/external/python_install.cmake @@ -7,6 +7,9 @@ SET(ENV{CFITSIO_EXT_INC} ${CFITSIO_EXT_INC}) SET(ENV{CFITSIO_EXT_PREFIX} ${CFITSIO_EXT_PREFIX}) SET(ENV{CFITSIO_EXT_LIB} ${CFITSIO_EXT_LIB}) SET(ENV{PYTHONPATH} ${PYTHON_LOCAL_SITE_PACKAGE}:$ENV{PYTHONPATH}) +SET(ENV{CC} ${CMAKE_C_COMPILER}) +SET(ENV{CXX} ${CMAKE_CXX_COMPILER}) + SET(PYTHON_INSTALL_COMMAND ${PYTHON_EXECUTABLE} setup.py install --prefix=${TARGET_PATH} --install-lib=${PYTHON_LOCAL_SITE_PACKAGE}) From eb8c7dd3e328dd712ee7f710b0a01df926ab779d Mon Sep 17 00:00:00 2001 From: "P.M. Sutter" Date: Tue, 11 Nov 2014 11:51:03 -0600 Subject: [PATCH 4/9] reports file name when loading gadget files --- c_tools/mock/loaders/gadget_loader.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/c_tools/mock/loaders/gadget_loader.cpp b/c_tools/mock/loaders/gadget_loader.cpp index 804bb40..cdf7541 100644 --- a/c_tools/mock/loaders/gadget_loader.cpp +++ b/c_tools/mock/loaders/gadget_loader.cpp @@ -105,6 +105,8 @@ SimulationLoader *gadgetLoader(const std::string& snapshot, double Mpc_unitLengt int num_files; SimuData *d; + cout << " File to load is:" << snapshot.c_str() << endl; + try { d = loadGadgetMulti(snapshot.c_str(), -1, 0); From 5972813cd53de68ca37fa7252aa12514d86960ba Mon Sep 17 00:00:00 2001 From: "P.M. Sutter" Date: Wed, 12 Nov 2014 11:27:41 -0600 Subject: [PATCH 5/9] fixed bug in getSurveyProps call --- python_tools/void_python_tools/backend/launchers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_tools/void_python_tools/backend/launchers.py b/python_tools/void_python_tools/backend/launchers.py index f1b4a76..2bd4a30 100644 --- a/python_tools/void_python_tools/backend/launchers.py +++ b/python_tools/void_python_tools/backend/launchers.py @@ -271,7 +271,7 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None, if sample.dataType == "observation": (boxVol, nbar) = vp.getSurveyProps(sample.maskFile, sample.zRange[0], sample.zRange[1], sample.zRange[0], sample.zRange[1], "all", - useLCDM=useComoving) + useComoving=useComoving) else: iX = float(sample.mySubvolume[0]) iY = float(sample.mySubvolume[1]) From 5f332edf02187f52a6969d3ab4b9b187c8252cbb Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Thu, 20 Nov 2014 10:57:17 +0100 Subject: [PATCH 6/9] Filter uniqueID attribute --- c_tools/mock/loaders/simulation_loader.cpp | 3 +++ c_tools/mock/loaders/simulation_loader.hpp | 26 +++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/c_tools/mock/loaders/simulation_loader.cpp b/c_tools/mock/loaders/simulation_loader.cpp index 80e09b2..70c441a 100644 --- a/c_tools/mock/loaders/simulation_loader.cpp +++ b/c_tools/mock/loaders/simulation_loader.cpp @@ -95,6 +95,9 @@ void SimulationLoader::basicPreprocessing(SimuData *d, } filteredCopy(d->Id, accepted, d->NumPart); filteredCopy(d->type, accepted, d->NumPart); + + filterAttribute(d, "uniqueID", accepted, d->NumPart); + d->NumPart = numAccepted; delete[] accepted; } diff --git a/c_tools/mock/loaders/simulation_loader.hpp b/c_tools/mock/loaders/simulation_loader.hpp index 8805702..149ee10 100644 --- a/c_tools/mock/loaders/simulation_loader.hpp +++ b/c_tools/mock/loaders/simulation_loader.hpp @@ -110,17 +110,27 @@ public: while (i < N) { - if (!accepted[i]) - { - i++; - continue; - } + if (!accepted[i]) + { + i++; + continue; + } - a[j] = a[i]; - j++; - i++; + a[j] = a[i]; + j++; + i++; } } + + template + void filterAttribute(CosmoTool::SimuData *d, const std::string& attrname, bool *accepted, long NumPart) + { + if (d->attributes.find(attrname) == d->attributes.end()) + return; + + long *l = d->as(attrname); + filteredCopy(l, accepted, NumPart); + } }; From 8a0e01a3b6643a0f94db95eb30aeaf896ebd7087 Mon Sep 17 00:00:00 2001 From: "P.M. Sutter" Date: Tue, 2 Dec 2014 09:22:41 -0500 Subject: [PATCH 7/9] more information about subsampling --- pipeline/datasets/example_simulation.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pipeline/datasets/example_simulation.py b/pipeline/datasets/example_simulation.py index 148e0a3..f452a47 100644 --- a/pipeline/datasets/example_simulation.py +++ b/pipeline/datasets/example_simulation.py @@ -98,8 +98,12 @@ redshifts = ["0.0"] # list of desired subsamples - these are in unts of h Mpc^-3! subSamples = [1.0, 0.5] -# if True, do the subsampling in preparation (only for sdf and multidark) -doSubSamplingInPrep = True +# if True, do the subsampling in preparation (available for sdf and multidark) +doSubSamplingInPrep = False + +# if 'absolute', subSamples are given in particles per cubic Mpc/h +# if 'relative', subSamples are given as a fraction of input particles +subSampleMode = "absolute" # shift the z-coord of sims with redshift shiftSimZ = False From 6620dd8fd71226c8fd87532ffc6b3b0ba11fcf20 Mon Sep 17 00:00:00 2001 From: "P.M. Sutter" Date: Tue, 2 Dec 2014 09:24:22 -0500 Subject: [PATCH 8/9] changed default subsampling behavior --- python_tools/pipeline_source/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_tools/pipeline_source/defaults.py b/python_tools/pipeline_source/defaults.py index 2833409..96fb27f 100644 --- a/python_tools/pipeline_source/defaults.py +++ b/python_tools/pipeline_source/defaults.py @@ -96,7 +96,7 @@ doSubSamplingInPrep = False # do the subsampling in preparation script? # if 'absolute', subSamples are given in particles per cubic Mpc/h # if 'relative', subSamples are given as a fraction of input particles -subSampleMode = "absolute" +subSampleMode = "relative" # common filename of halo files, leave blank to ignore halos haloFileBase = "mf_4s_1G_512_bgc2_NNNNN.sdf" From 5ed1503c91d59d298cca0db9d8a0c7f574b1f09a Mon Sep 17 00:00:00 2001 From: "P.M. Sutter" Date: Thu, 4 Dec 2014 16:16:47 -0500 Subject: [PATCH 9/9] changed default subsampling behavior in example --- pipeline/datasets/example_simulation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/datasets/example_simulation.py b/pipeline/datasets/example_simulation.py index f452a47..65175b5 100644 --- a/pipeline/datasets/example_simulation.py +++ b/pipeline/datasets/example_simulation.py @@ -103,7 +103,7 @@ doSubSamplingInPrep = False # if 'absolute', subSamples are given in particles per cubic Mpc/h # if 'relative', subSamples are given as a fraction of input particles -subSampleMode = "absolute" +subSampleMode = "relative" # shift the z-coord of sims with redshift shiftSimZ = False