Merge branch 'master' of bitbucket.org:cosmicvoids/void_identification

README

@@ -1,5 +1,36 @@
+\        /   /   |-\    -----
+ \      /    |   |  \   |
+  \    /    /    |   |  |--
+   \  /     |    |  /   |
+    \/      /    |-/    -----
+
+*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*
+
+This is VIDE. The Void IDEntifier pipeline.
+
+
+License/Copyright information
+-----------------------------
+
+Copyright (C) 2010-2013 Guilhem Lavaux, 2011-2013 Paul M. Sutter.
+This software is put under the GNU Public License. Please see LICENSE
+for further information.
+
+
+Parts of the pipeline includes ZOBOV. See zobov/zobov_readme.txt for 
+copyright/license information
+
+Building
+--------
+
+
 After compiling, go to the pipeline directory.
 
+
+Using the pipeline
+------------------
+
+
 Create a dataset parameter file. Look at datasets/multidark.py for 
 an example. Describe the simulation, where to put outputs, how many 
 redshift slices, subvolumes, etc. etc.

build_tools/gather_sources.py

@@ -0,0 +1,112 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./build_tools/gather_sources.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
+import shutil
+import tempfile
+import re
+from git import Repo,Tree,Blob
+
+def apply_license(license, relimit, filename):
+  header = re.sub(r'@FILENAME@', filename, license)
+
+  f = file(filename)
+  lines = f.read()
+  f.close()
+
+  lines = re.sub(relimit, '', lines)
+  lines = header + lines
+
+  with tempfile.NamedTemporaryFile(delete=False) as tmp_sources:
+    tmp_sources.write(lines)
+
+  shutil.move(tmp_sources.name, filename)
+
+
+def apply_python_license(filename):
+  license="""#+
+#   VIDE -- Void IDEntification pipeline -- @FILENAME@
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
+"""
+   
+  print("Shell/Python file: %s" % filename)
+  relimit=r'^#\+\n(#.*\n)*#\+\n'
+  apply_license(license, relimit, filename)
+
+
+def apply_cpp_license(filename):
+  license="""/*+
+    VIDE -- Void IDEntification pipeline -- @FILENAME@
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+"""
+  relimit = r'^(?s)/\*\+.*\+\*/'
+  print("C++ file: %s" % filename)
+  apply_license(license, relimit, filename)
+  
+
+def analyze_tree(prefix, t):
+  for ename,entry in t.items():
+    if ename == 'external' or ename == 'zobov':
+      continue
+    if type(entry) == Tree:
+      analyze_tree(prefix + "/" + ename, entry)
+    elif type(entry) == Blob:
+      if re.match(".*\.(sh|py|pyx)$",ename) != None:
+        fname=prefix+"/"+ename
+        apply_python_license(fname)
+      if re.match('.*\.(cpp|hpp|h)$', ename) != None:
+        fname=prefix+"/"+ename
+        apply_cpp_license(fname)
+
+
+if __name__=="__main__":
+  repo = Repo(".")
+  assert repo.bare == False
+  t = repo.tree()
+  analyze_tree(".", t)

c_tools/analysis/voidOverlap.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/analysis/voidOverlap.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 // =============================================================================
 //
 //   Program: voidOverlap

c_tools/libzobov/contour_pixels.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/libzobov/contour_pixels.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <vector>
 #include <healpix_map.h>
 #include <healpix_map_fitsio.h>

c_tools/libzobov/contour_pixels.hpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/libzobov/contour_pixels.hpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #ifndef  __CONTOUR_PIXELS_HPP
 #define  __CONTOUR_PIXELS_HPP
 

c_tools/libzobov/gslIntegrate.hpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/libzobov/gslIntegrate.hpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #ifndef __MYGSL_INTEGRATE_HPP
 #define __MYGSL_INTEGRATE_HPP
 

c_tools/libzobov/loadZobov.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/libzobov/loadZobov.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <cassert>
 #include <string>
 #include <fstream>

c_tools/libzobov/loadZobov.hpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/libzobov/loadZobov.hpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #ifndef __LOAD_ZOBOV_HPP
 #define __LOAD_ZOBOV_HPP
 

c_tools/libzobov/particleInfo.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/libzobov/particleInfo.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <cstdlib>
 #include <netcdfcpp.h>
 #include <CosmoTool/fortran.hpp>

c_tools/libzobov/particleInfo.hpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/libzobov/particleInfo.hpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #ifndef _PARTICLE_INFO_HEADER_HPP
 #define _PARTICLE_INFO_HEADER_HPP
 

c_tools/libzobov/voidTree.hpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/libzobov/voidTree.hpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #ifndef _VOID_TREE_HPP
 #define _VOID_TREE_HPP
 

c_tools/mock/generateFromCatalog.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/generateFromCatalog.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <healpix_map.h>
 #include <healpix_map_fitsio.h>
 #include <boost/format.hpp>

c_tools/mock/generateMock.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/generateMock.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <gsl/gsl_rng.h>
 #include <boost/function.hpp>
 #include <boost/bind.hpp>
@@ -349,6 +370,8 @@ void buildBox(SimuData *simu, long num_targets, long loaded,
   for (uint32_t i = 0; i < num_targets; i++, loaded++)
     {
       long pid = particle_id[loaded];
+      assert(pid < simu->NumPart);
+      assert(loaded < boxed->NumPart);
       
       for (int j = 0; j < 3; j++)
 	{
@@ -513,7 +536,7 @@ void makeBoxFromParameter(SimuData *simu, SimuData* &boxed, generateMock_info& a
            {
              particle_id[pid_write] = particle_id[pid_read];
              uniqueID[pid_write] = uniqueID[pid_read];
-             expansion_fac[pid_write] = uniqueID[pid_read];
+             expansion_fac[pid_write] = expansion_fac[pid_read];
              pid_write++; 
            }
           pid_read++;
@@ -618,6 +641,12 @@ int main(int argc, char **argv)
     {
       loader = multidarkLoader(args_info.multidark_arg, preselector);      
     }
+#ifdef SDF_SUPPORT
+  else if (args_info.sdf_given)
+    {
+      loader = sdfLoader(args_info.sdf_arg, args_info.sdf_splitting_arg, NEED_POSITION|NEED_VELOCITY|NEED_GADGET_ID, preselector);
+    }
+#endif
   else
     {
       cerr << "A simulation snapshot is required to generate a mock catalog." << endl;

c_tools/mock/generateMock.ggo

@@ -11,6 +11,9 @@ option "gadget"	    -	"Base name of gadget snapshot (without parallel writing ex
 option "flash"	    -   "Base name for FLASH snapshot" string optional
 option "multidark"	    -   "Base name for multidark snapshot" string optional
 
+option "sdf"        -   "SDF snapshot name" string optional
+option "sdf_splitting" - "Number of artificial splitting of the SDF snapshot" int optional default="20"
+
 option "axis"	    -	"Redshift axis (X=0, Y=1, Z=2)" int optional default="2"
 
 option "output"	    -   "Output filename for particles" string required
@@ -28,7 +31,7 @@ option "peculiarVelocities" - "Added peculiar velocities distortion" flag off
 
 option "cosmo"	- "Apply cosmological redshift" flag off
 
-option "subsample" - "Subsample the input simulation by the specified amount" double optional
+option "subsample" - "Subsample the input simulation by the specified amount" double optional default="1.0"
 
 option "inputParameter" - "Input geometry (optional, warning!)" string optional
 option "gadgetUnit" - "Unit of length in gadget file in Mpc/h" double optional default="0.001"
@@ -37,3 +40,4 @@ option "subsample_seed" - "Seed for random number generation to select the subsa
 
 option "resubsample" - "Resubsampling factor compared to the subsampled simulation" double optional 
 option "resubsample_seed" - "Seed for resubsampling from a subsampled simulation" int optional default="20132011"
+

c_tools/mock/generateTestMock.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/generateTestMock.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <iostream>
 #include <cstdlib>
 #include <CosmoTool/fortran.hpp>

c_tools/mock/loaders/basic_loader.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/loaders/basic_loader.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <cassert>
 #include <string>
 #include <iostream>

c_tools/mock/loaders/flash_loader.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/loaders/flash_loader.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <cassert>
 #include <string>
 #include <CosmoTool/loadFlash.hpp>

c_tools/mock/loaders/gadget_loader.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/loaders/gadget_loader.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <vector>
 #include <cassert>
 #include <string>

c_tools/mock/loaders/multidark_loader.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/loaders/multidark_loader.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <cassert>
 #include <string>
 #include <fstream>

c_tools/mock/loaders/ramses_loader.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/loaders/ramses_loader.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <cassert>
 #include <string>
 #include <CosmoTool/loadRamses.hpp>

c_tools/mock/loaders/sdf_loader.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/loaders/sdf_loader.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <iostream>
 #include <boost/format.hpp>
 #include <vector>

c_tools/mock/loaders/sdfloader_internal.hpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/loaders/sdfloader_internal.hpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #ifndef __VOID_SDF_LOADER_INTERNAL_HPP
 #define __VOID_SDF_LOADER_INTERNAL_HPP
 

c_tools/mock/loaders/simulation_loader.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/loaders/simulation_loader.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #include <cmath>
 #include <CosmoTool/loadSimu.hpp>
 #include "simulation_loader.hpp"

c_tools/mock/loaders/simulation_loader.hpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/mock/loaders/simulation_loader.hpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 #ifndef _MOCK_SIMULATION_LOADER_HPP
 #define _MOCK_SIMULATION_LOADER_HPP
 

c_tools/stacking/pruneVoids.cpp

@@ -1,3 +1,24 @@
+/*+
+    VIDE -- Void IDEntification pipeline -- ./c_tools/stacking/pruneVoids.cpp
+    Copyright (C) 2010-2013 Guilhem Lavaux
+    Copyright (C) 2011-2013 P. M. Sutter
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; version 2 of the License.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
++*/
+
+
+
 // Reads in the void catalog and removes any void that could potentially
 //   be affected by a mock particle. It does this by computing the longest
 //   particle distance within each void and comparing it to the distance
@@ -548,7 +569,7 @@ int main(int argc, char **argv) {
     }
   }
   voids.resize(iGood);
-  printf("  1st filter: reiGoodected %d obviously bad\n", numWrong); 
+  printf("  1st filter: rejected %d obviously bad\n", numWrong); 
 
   iGood = 0;
   for (iVoid = 0; iVoid < voids.size(); iVoid++) {
@@ -559,7 +580,7 @@ int main(int argc, char **argv) {
     }
   }
   voids.resize(iGood);
-  printf("  2nd filter: reiGoodected %d too small\n", numTooSmall); 
+  printf("  2nd filter: rejected %d too small\n", numTooSmall); 
 
 
   iGood = 0;
@@ -572,7 +593,7 @@ int main(int argc, char **argv) {
     }
   }
   voids.resize(iGood);
-  printf("  3rd filter: reiGoodected %d too close to high redshift boundaries\n", numNearZ); 
+  printf("  3rd filter: rejected %d too close to high redshift boundaries\n", numNearZ); 
 
   numNearZ = 0;
   iGood = 0;
@@ -586,7 +607,7 @@ int main(int argc, char **argv) {
     }
   }
   voids.resize(iGood);
-  printf("  4th filter: reiGoodected %d too close to low redshift boundaries\n", numNearZ); 
+  printf("  4th filter: rejected %d too close to low redshift boundaries\n", numNearZ); 
 
   for (iVoid = 0; iVoid < voids.size(); iVoid++) {
     if (voids[iVoid].centralDen > args.maxCentralDen_arg) {

crossCompare/analysis/datasetsToAnalyze.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./crossCompare/analysis/datasetsToAnalyze.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 #!/usr/bin/env python
 
 

crossCompare/analysis/mergerTree.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./crossCompare/analysis/mergerTree.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 #!/usr/bin/env python
 
 # plots cumulative distributions of number counts

crossCompare/plotting/datasetsToPlot.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./crossCompare/plotting/datasetsToPlot.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 #!/usr/bin/env python
 
 

crossCompare/plotting/plotCompareDensCon.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./crossCompare/plotting/plotCompareDensCon.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 #!/usr/bin/env python
 
 # plots cumulative distributions of number counts versus density contrast

crossCompare/plotting/plotDensConVsR.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./crossCompare/plotting/plotDensConVsR.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 #!/usr/bin/env python
 
 # plots cumulative distributions of number counts versus density contrast

crossCompare/plotting/plotNumberFunc.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./crossCompare/plotting/plotNumberFunc.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 #!/usr/bin/env python
 
 # plots cumulative distributions of number counts

external/external_build.cmake

@@ -321,7 +321,8 @@ SET(QHULL_LIBRARIES ${QHULL_CPP_LIBRARY} ${QHULL_LIBRARY} )
 # Build libSDF
 ###############
 IF(SDF_SUPPORT)
-  SET(LIBSDF_ARCH x86_64)
+  SET(LIBSDF_ARCH x86_64 CACHE STRING "SDF architecture to activate")
+  mark_as_advanced(LIBSDF_ARCH)
   SET(LIBSDF_PATH ${CMAKE_SOURCE_DIR}/external/libsdf)
 
   ExternalProject_Add(libSDF
@@ -329,7 +330,7 @@ IF(SDF_SUPPORT)
     SOURCE_DIR ${LIBSDF_PATH}
     CONFIGURE_COMMAND echo No configure
     BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} ARCH=${LIBSDF_ARCH}
-    INSTALL_COMMAND echo No install
+    INSTALL_COMMAND ${CMAKE_COMMAND} -DDEST_DIR=${CMAKE_BINARY_DIR}/ext_build/sdf -DLIBSDF_ARCH=${LIBSDF_ARCH} -DLIBSDF_PATH=${LIBSDF_PATH} -P ${CMAKE_SOURCE_DIR}/external/install_sdf.cmake
     BUILD_IN_SOURCE 1
   )
   SET(LIBSDF_INCLUDE_PATH ${LIBSDF_PATH}/include)

external/install_sdf.cmake

@@ -0,0 +1,4 @@
+file(MAKE_DIRECTORY ${DEST_DIR})
+file(MAKE_DIRECTORY ${DEST_DIR}/bin)
+file(INSTALL ${LIBSDF_PATH}/apps/SDFcvt/SDFcvt.${LIBSDF_ARCH} DESTINATION ${DEST_DIR}/bin PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE GROUP_EXECUTE GROUP_READ)
+file(RENAME ${DEST_DIR}/bin/SDFcvt.${LIBSDF_ARCH} ${DEST_DIR}/bin/SDFcvt)

external/libsdf/GNUmakefile

@@ -20,7 +20,7 @@ all: All
 # spell them a little differently in this file...
 include Make-common/Make.generic
 
-subdirs:= libSDF libmpmy libsw
+subdirs:= libSDF libmpmy libsw apps/SDFcvt
 
 All:
 	for dir in $(subdirs); do (cd $$dir; $(MAKE) ARCH=$(ARCH) all); done

external/libsdf/apps/SDFcvt/GNUmakefile

@@ -0,0 +1,15 @@
+TREEHOME=../..
+##### Application-specific stuff goes here
+
+treedir=$(TREEHOME)
+programname=SDFcvt
+src=SDFcvt.c 
+
+#### end of application-specific stuff
+include $(treedir)/Make-common/Make.$(ARCH)
+
+include $(treedir)/Make-common/Make.generic
+
+# DO NOT DELETE THIS LINE -- make depend depends on it.
+
+

external/libsdf/apps/SDFcvt/SDFcvt.c

@@ -0,0 +1,461 @@
+/* Convert SDF to ascii
+   Usage:
+     SDFcvt [-s skip][-n nrecout][-h hdrfile] [-bfv] sdffile name ...
+
+   Not too fancy, but it should handle most SDF files, including
+   non-unit arrcnt.  By using -s and -n is is possible to skip
+   to a place in the file and examine pieces of files
+   that would otherwise be too big to handle. We do NOT read the
+   whole file into memory.  We process output lines one at a time,
+   reading one value of each of the names into memory at a time.
+   Thrashing is possible.  We do nothing to prevent it.
+
+   With -v  it prints a potentially useful pair of header lines
+   starting with '#'.
+
+   CHANGELOG:
+   6/28/94:  Added the FORMAT_NATIVE option, and made it the default.
+       This will write SDF_INTS as ints, SDF_SHORTS as shorts, etc.  It's
+       useful for post-processing by programs that can handle mixed types,
+       e.g., perl, but not AVS.
+
+   10/18/93: Added the -S (Swapping) flag to write binary data swapped.
+   Use limits.h/INT_MAX instead of SunOS values.h/MAXINT.
+
+   8/1/93: add the AT_A_TIME construction.  We no longer process lines
+   	one at a time.  Instead, we read up to AT_A_TIME to beat the
+	excessive overhead in each SDFrdvecs call.  One of these days
+	I'll add a hash table to SDF's symbol-lookup functions.  Until
+	then, reading one element at a time will be extremely slow.
+
+   8/20/93: added the option to produce output in binary.  This is MUCH
+   	faster.  With a little luck, perl (and presumably other programs)
+	can read it MUCH faster	too (perl uses read and unpack).
+
+*/
+#define AT_A_TIME 512		/* read this many values at a time */
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <string.h>
+#include "SDF.h"
+#include "protos.h"
+#include "byteswap.h"
+#include "error.h"
+#include "Malloc.h"
+
+#ifndef __SUN5__
+extern int getopt(int, char**, const char *);
+extern char *optarg;
+extern int optind, opterr;
+#endif
+
+#define FORMAT_FLOAT 1
+#define FORMAT_DOUBLE 2
+#define FORMAT_INT 3
+#define FORMAT_NATIVE 4
+
+int binary = 0;
+int format = FORMAT_NATIVE;
+int swap_out = 0;
+
+int fwrite_maybe_swap(void *p, size_t sz, size_t n, FILE *fp){
+    if( swap_out ) 
+	Byteswap(sz, n, p, p);
+    return fwrite(p, sz, n, fp);
+}
+
+void output_char(short c){
+    float f;
+    double d;
+    int i;
+    
+    if( binary ){
+	switch(format){
+	case FORMAT_DOUBLE:
+	    d = c;
+	    fwrite_maybe_swap(&d, sizeof(d), 1, stdout);
+	    break;
+	case FORMAT_FLOAT:
+	    f = c;
+	    fwrite_maybe_swap(&f, sizeof(f), 1, stdout);
+	    break;
+	case FORMAT_INT:
+	    i = c;
+	    fwrite_maybe_swap(&i, sizeof(i), 1, stdout);
+	    break;
+	case FORMAT_NATIVE:
+	    fwrite_maybe_swap(&c, sizeof(c), 1, stdout);
+	    break;
+	}
+    }else{
+	printf("%d", c);
+    }
+}
+
+void output_short(short s){
+    float f;
+    double d;
+    int i;
+    
+    if( binary ){
+	switch(format){
+	case FORMAT_DOUBLE:
+	    d = s;
+	    fwrite_maybe_swap(&d, sizeof(d), 1, stdout);
+	    break;
+	case FORMAT_FLOAT:
+	    f = s;
+	    fwrite_maybe_swap(&f, sizeof(f), 1, stdout);
+	    break;
+	case FORMAT_INT:
+	    i = s;
+	    fwrite_maybe_swap(&i, sizeof(i), 1, stdout);
+	    break;
+	case FORMAT_NATIVE:
+	    fwrite_maybe_swap(&s, sizeof(s), 1, stdout);
+	    break;
+	}
+    }else{
+	printf("%d", s);
+    }
+}
+
+void output_int(int i){
+    float f;
+    double d;
+
+    if( binary ){
+	switch(format){
+	case FORMAT_DOUBLE:
+	    d = i;
+	    fwrite_maybe_swap(&d, sizeof(d), 1, stdout);
+	    break;
+	case FORMAT_FLOAT:
+	    f = i;
+	    fwrite_maybe_swap(&f, sizeof(f), 1, stdout);
+	    break;
+	case FORMAT_NATIVE:
+	case FORMAT_INT:
+	    fwrite_maybe_swap(&i, sizeof(i), 1, stdout);
+	    break;
+	}
+    }else{
+	printf("%d", i);
+    }
+}
+
+void output_long(long l){
+    float f;
+    double d;
+
+    if( binary ){
+	switch(format){
+	case FORMAT_DOUBLE:
+	    d = l;
+	    fwrite_maybe_swap(&d, sizeof(d), 1, stdout);
+	    break;
+	case FORMAT_FLOAT:
+	    f = l;
+	    fwrite_maybe_swap(&f, sizeof(f), 1, stdout);
+	    break;
+	case FORMAT_NATIVE:
+	case FORMAT_INT:
+	    fwrite_maybe_swap(&l, sizeof(l), 1, stdout);
+	    break;
+	}
+    }else{
+	printf("%ld", l);
+    }
+}
+
+void output_int64(int64_t i64){
+    float f;
+    double d;
+    
+    if( binary ){
+	switch(format){
+	case FORMAT_DOUBLE:
+	    d = i64;
+	    fwrite_maybe_swap(&d, sizeof(d), 1, stdout);
+	    break;
+	case FORMAT_FLOAT:
+	    f = i64;
+	    fwrite_maybe_swap(&f, sizeof(f), 1, stdout);
+	    break;
+	case FORMAT_NATIVE:
+	case FORMAT_INT:
+	    fwrite_maybe_swap(&i64, sizeof(i64), 1, stdout);
+	    break;
+	}
+    }else{
+#if __WORDSIZE==64
+	printf("%ld", i64);
+#else
+	printf("%lld", i64);
+#endif
+    }
+}
+
+
+
+void output_float(float f){
+    double d;
+    int i;
+
+    if( binary ){
+	switch(format){
+	case FORMAT_DOUBLE:
+	    d = f;
+	    fwrite_maybe_swap(&d, sizeof(d), 1, stdout);
+	    break;
+	case FORMAT_FLOAT:
+	case FORMAT_NATIVE:
+	    fwrite_maybe_swap(&f, sizeof(f), 1, stdout);
+	    break;
+	case FORMAT_INT:
+	    i = f;
+	    fwrite_maybe_swap(&i, sizeof(i), 1, stdout);
+	    break;
+	}
+    }else{
+	switch(format){
+	case FORMAT_DOUBLE:
+	    printf("%.15e", f);
+	    break;
+	case FORMAT_FLOAT:
+	case FORMAT_NATIVE:
+	    printf("%.6e", f);
+	    break;
+	case FORMAT_INT:
+	    printf("%.0f", f);
+	    break;
+	}
+    }
+}
+
+void output_double(double d){
+    float f;
+    int i;
+
+    if( binary ){
+	switch(format){
+	case FORMAT_NATIVE:
+	case FORMAT_DOUBLE:
+	    fwrite_maybe_swap(&d, sizeof(d), 1, stdout);
+	    break;
+	case FORMAT_FLOAT:
+	    f = d;
+	    fwrite_maybe_swap(&f, sizeof(f), 1, stdout);
+	    break;
+	case FORMAT_INT:
+	    i = d;
+	    fwrite_maybe_swap(&i, sizeof(i), 1, stdout);
+	    break;
+	}
+    }else{
+	switch(format){
+	case FORMAT_DOUBLE:
+	case FORMAT_NATIVE:
+	    printf("%.15e", d);
+	    break;
+	case FORMAT_FLOAT:
+	    printf("%.6e", d);
+	    break;
+	case FORMAT_INT:
+	    printf("%.0f", d);
+	    break;
+	}
+    }
+}
+
+void output_string(const char *s){
+    fwrite(s, strlen(s), 1, stdout);
+}
+
+void output_separator(void){
+    if( !binary )
+	putchar(' ');
+}
+
+void output_record_terminator(void){
+    if( !binary )
+	putchar('\n');
+}
+
+int main(int argc, char **argv){
+    int c;
+    char *fname, *hdr;
+    SDF *sdfp;
+    int64_t n, start;
+    int nnames, nn, i, j, k, kk;
+    int at_a_time;
+    int64_t nrecs, minnrecs, maxnrecs;
+    char *newname;
+    char **names;
+    void **addrs;
+    int *nread;
+    int *acnt;
+    int *strides;
+    enum SDF_type_enum *types, t;
+    int verbose;
+
+    hdr = NULL;
+    start = 0;
+    n = 0;
+    verbose = 0;
+    at_a_time = 0;
+    while((c=getopt(argc, argv, "a:h:s:n:vbfdiS")) != -1)
+	switch(c){
+	case 'h':
+	    hdr = optarg;
+	    break;
+	case 's':
+	    start = atoll(optarg);
+	    break;
+	case 'n':
+	    n = atoll(optarg);
+	    break;
+	case 'a':
+	    at_a_time = atoi(optarg);
+	    break;
+	case 'v':
+	    verbose = 1;
+	    break;
+	case 'b':
+	    binary = 1;
+	    break;
+	case 'f':
+	    format = FORMAT_FLOAT;
+	    break;
+	case 'S':
+	    swap_out = 1;
+	    break;
+	case 'd':
+	    format = FORMAT_DOUBLE;
+	    break;
+	case 'i':
+	    format = FORMAT_INT;
+	    break;
+	case '?':
+	    fprintf(stderr, "Usage: %s [-s start][-n number][-a at-a-time][-v<erbose>][-h hdrfile][-b<inary>][-f<loat>][-d<ouble>][-i<nt>][-S<wap>] sdffile name ...\n", argv[0]);
+	}
+
+    fname = argv[optind];
+    optind++;
+    sdfp = SDFopen(hdr, fname);
+    if( sdfp == NULL ){
+	Error("Could not SDFopen(%s, %s): %s\n", 
+	      hdr?hdr:"<null>", 
+	      fname?fname:"<null>", 
+	      SDFerrstring);
+    }
+    if( at_a_time == 0 )
+	at_a_time = AT_A_TIME;
+    
+    nnames = argc - optind;
+    names = Malloc(nnames*sizeof(char*));
+    addrs = Malloc(nnames*sizeof(void*));
+    nread = Malloc(nnames*sizeof(int));
+    types = Malloc(nnames*sizeof(enum SDF_type_enum));
+    strides = Malloc(nnames*sizeof(int));
+    acnt = Malloc(nnames*sizeof(int));
+    
+    nn = 0;
+    minnrecs = INT64_MAX;
+    maxnrecs = 0;
+    for(; optind < argc; optind++){
+	newname = argv[optind];
+	if( !SDFhasname(newname, sdfp) ){
+	    Error("%s not in %s\n", newname, fname);
+	}
+	t = SDFtype(newname, sdfp);
+	acnt[nn] = SDFarrcnt(newname, sdfp);
+	addrs[nn] = Malloc(SDFtype_sizes[t] * acnt[nn] * at_a_time);
+	strides[nn] = SDFtype_sizes[t]*acnt[nn];
+	names[nn] = newname;
+	types[nn] = t;
+	nrecs = SDFnrecs(newname, sdfp);
+	if( nrecs < minnrecs )
+	    minnrecs = nrecs;
+	if( nrecs > maxnrecs )
+	    maxnrecs = nrecs;
+	nread[nn] = at_a_time;
+	nn++;
+    }
+    if( n==0 ){
+	n = maxnrecs - start;
+    }
+    if( start+n > minnrecs ){
+	Error("Not enough records to start at %ld\n", start);
+    }
+
+    if( start ){
+	for(j=0; j<nnames; j++){
+	    SDFseek(names[j], start, SDF_SEEK_SET, sdfp);
+	}
+    }
+
+    if( verbose ){
+	putchar('#');
+	for(i=0; i<argc; i++){
+	    printf("%s%c", argv[i], (i<argc-1)?' ':'\n');
+	}
+	putchar('#');
+	for(j=0; j<nnames; j++){
+	    printf("%s%c", names[j], (j<nnames-1)?' ':'\n');
+	}
+    }
+	    
+    while( n>0 ){
+	if( n < at_a_time ){
+	    at_a_time = n;
+	    for(j=0; j<nnames; j++){
+		nread[j] = at_a_time;
+	    }
+	}
+	if( SDFrdvecsarr(sdfp, nnames, names, nread, addrs, strides) )
+	    Error("SDFrdvecsarr failed n=%ld: %s\n", n, SDFerrstring); 
+	n -= at_a_time;
+	for(i=0; i<at_a_time; i++){
+	    for(j=0; j<nnames; j++){
+		for(k=0; k<acnt[j]; k++){
+		    kk = acnt[j]*i + k;
+		    if(j||k)
+			output_separator();
+		    switch(types[j]){
+		    case SDF_CHAR:
+			output_char( ((char *)addrs[j])[kk] );
+			break;
+		    case SDF_SHORT:
+			output_short( ((short *)addrs[j])[kk] );
+			break;
+		    case SDF_INT:
+			output_int( ((int *)addrs[j])[kk] );
+			break;
+		    case SDF_LONG:
+			output_long( ((long *)addrs[j])[kk] );
+			break;
+		    case SDF_INT64:
+			output_int64( ((int64_t *)addrs[j])[kk] );
+			break;
+		    case SDF_FLOAT:
+			output_float( ((float *)addrs[j])[kk] );
+			break;
+		    case SDF_DOUBLE:
+			output_double( ((double *)addrs[j])[kk] );
+			break;
+		    case SDF_STRING:
+			output_string((char *)addrs[j]);
+			break;
+		    default:
+			Error("Unrecognized type\n");
+		    }
+		}
+	    }
+	    output_record_terminator();
+	}
+    }
+    exit(0);
+}
+

pipeline/datasets/mock_dr9mid.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./pipeline/datasets/mock_dr9mid.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 import os
 
 # -----------------------------------------------------------------------------

pipeline/datasets/multidark.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./pipeline/datasets/multidark.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 import os
 
 # -----------------------------------------------------------------------------

pipeline/generateCatalog.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./pipeline/generateCatalog.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 #!/usr/bin/env python
 
 # does full void analysis. Also generates 2d/1d stacked plots and hubble diagram

python_tools/pipeline_source/prepareCatalogs.in.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/pipeline_source/prepareCatalogs.in.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 #!/usr/bin/env python
 
 # prepares input catalogs based on multidark simulations
@@ -62,7 +81,7 @@ def getSampleName(setName, redshift, useVel, iSlice=-1, iVol=-1):
 def writeScript(setName, dataFileNameBase, dataFormat,
                 scriptDir, catalogDir, fileNums, redshifts, numSubvolumes,
                 numSlices, useVel, lbox, minRadius, omegaM, subsample=1.0, 
-                suffix=".dat"):
+                suffix=".dat", dataFileNameList=None):
 
 
   if useVel: setName += "_pv"
@@ -134,7 +153,7 @@ newSample = Sample(dataFile = "{dataFile}",
                    numSubvolumes = {numSubvolumes},
                    mySubvolume = "{mySubvolume}",
                    useLightCone = {useLightCone},
-                   subsample = {subsample})
+                   subsample = "{subsample}")
 dataSampleList.append(newSample)
 newSample.addStack(0.0, 5.0, 20, 25, False, False)
 newSample.addStack(0.0, 5.0, 30, 35, False, False)
@@ -165,32 +184,31 @@ newSample.addStack(0.0, 5.0, 90, 95, False, False)
     for i in xrange(len(zVsDY)):
       zVsDX[i] = vp.angularDiameter(zVsDY[i], Om=Om)
 
-    if useLightCone:
     boxWidthZ = np.interp(vp.angularDiameter(zBox,Om=Om)+100. / \
                 LIGHT_SPEED*lbox, zVsDX, zVsDY)-zBox
-      dzSafe = 0.03
-    else:
-      boxWidthZ = np.interp(vp.angularDiameter(zBox,Om=Om)+100. / \
-                  LIGHT_SPEED*lbox, zVsDX, zVsDY)-zBox
-      #boxWidthZ = lbox*100./LIGHT_SPEED
-      dzSafe = 0.0
 
     for iSlice in xrange(numSlices):
-      sliceMin = zBox + dzSafe + iSlice*(boxWidthZ-2.*dzSafe)/numSlices
-      sliceMax = zBox + dzSafe + (iSlice+1)*(boxWidthZ-2.*dzSafe)/numSlices
 
       if useLightCone:
+        dzSafe = 0.03
+        sliceMin = zBox + dzSafe + iSlice*(boxWidthZ-2.*dzSafe)/numSlices
+        sliceMax = zBox + dzSafe + (iSlice+1)*(boxWidthZ-2.*dzSafe)/numSlices
         sliceMinMpc = sliceMin*LIGHT_SPEED/100.
         sliceMaxMpc = sliceMax*LIGHT_SPEED/100.
       else:
-        sliceMinMpc = LIGHT_SPEED/100.*vp.angularDiameter(sliceMin, Om=Om)
+        sliceMinMpc = zBoxMpc + iSlice*lbox/numSlices
-        sliceMaxMpc = LIGHT_SPEED/100.*vp.angularDiameter(sliceMax, Om=Om)
+        sliceMaxMpc = zBoxMpc + (iSlice+1)*lbox/numSlices
+        sliceMin = np.interp(sliceMinMpc*100./LIGHT_SPEED, zVsDX, zVsDY)
+        sliceMax = np.interp(sliceMaxMpc*100./LIGHT_SPEED, zVsDX, zVsDY)
 
       sliceMin = "%0.2f" % sliceMin
       sliceMax = "%0.2f" % sliceMax
       sliceMinMpc = "%0.1f" % sliceMinMpc
       sliceMaxMpc = "%0.1f" % sliceMaxMpc
 
+      if (dataFileNameList != None):
+        dataFileName = dataFileNameList[iFile]
+      else:
         dataFileName = dataFileNameBase + fileNum + suffix
 
       for iX in xrange(numSubvolumes):
@@ -216,7 +234,7 @@ newSample.addStack(0.0, 5.0, 90, 95, False, False)
                                          numSubvolumes=numSubvolumes,
                                          mySubvolume=mySubvolume,
                                          useLightCone=useLightCone,
-                                         subsample=subsample))
+                                         subsample=str(subsample).strip('[]')))
 
   scriptFile.close()
   return
@@ -233,31 +251,32 @@ if not os.access(catalogDir, os.F_OK): os.mkdir(catalogDir)
 #       ss0.000175 ~ SDSS DR9 mid 
 baseResolution = float(numPart)/lbox/lbox/lbox # particles/Mpc^3
 prevSubSample = -1
-for thisSubSample in sorted(subSamples, reverse=True):
+subSamples = sorted(subSamples, reverse=True)
+for iSubSample in xrange(len(subSamples)):
 
-  keepFraction = float(thisSubSample) / baseResolution
+  subSampleList = subSamples[0:iSubSample+1]
-  maxKeep = keepFraction * numPart
+
+  keepFractionList = []
+  for subSample in subSampleList:
+    keepFractionList.append(float(subSample) / baseResolution)
+  thisSubSample = subSamples[iSubSample]
+  maxKeep = keepFractionList[-1] * numPart
   minRadius = int(np.ceil(lbox/maxKeep**(1./3)))
 
+  partFileList = []
+  for (iRedshift, redshift) in enumerate(redshifts):
+    if particleFileDummy == '':
+      partFileList.append(particleFileBase+fileNums[iRedshift])
+    else:
+      partFileList.append(particleFileBase.replace(particleFileDummy, 
+                                                   fileNums[iRedshift]))
+
   if args.script or args.all:
     print " Doing subsample", thisSubSample, "scripts"
     sys.stdout.flush()
     setName = prefix+"ss"+str(thisSubSample)
-    if dataFormat == "multidark":
+
-      subSampleToUse = 1.0
+    if dataFormat == "random":
-      fileToUse = prefix+"ss"+str(thisSubSample)+"_z"
-      suffix = ".dat"
-    elif dataFormat == "gadget":
-      subSampleToUse = keepFraction
-      #subSampleToUse = thisSubSample
-      fileToUse = particleFileBase
-      suffix = ""
-    elif dataFormat == "lanl":
-      subSampleToUse = keepFraction
-      #subSampleToUse = thisSubSample
-      fileToUse = particleFileBase
-      suffix = ""
-    elif dataFormat == "random":
       subSampleToUse = 1.0
       fileToUse = "ran.ss"+str(thisSubSample)+"_z"
       suffix = ".dat"
@@ -270,6 +289,21 @@ for thisSubSample in sorted(subSamples, reverse=True):
                   scriptDir, catalogDir, fileNums, redshifts, 
                   numSubvolumes, numSlices, True, lbox, minRadius, omegaM, 
                   subsample=subSampleToUse, suffix=suffix)
+    else:
+      subSampleToUse = keepFractionList
+      suffix = ""
+      fileToUse = partFileList[0]
+      writeScript(setName, fileToUse, dataFormat, 
+                  scriptDir, catalogDir, fileNums, redshifts, 
+                  numSubvolumes, numSlices, False, lbox, minRadius, omegaM, 
+                  subsample=subSampleToUse, suffix=suffix,
+                  dataFileNameList=partFileList)
+      writeScript(setName, fileToUse, dataFormat, 
+                  scriptDir, catalogDir, fileNums, redshifts, 
+                  numSubvolumes, numSlices, True, lbox, minRadius, omegaM, 
+                  subsample=subSampleToUse, suffix=suffix,
+                  dataFileNameList=partFileList)
+       
 
   if args.subsample or args.all:
     print " Doing subsample", thisSubSample
@@ -411,13 +445,19 @@ if (args.halos or args.all) and haloFileBase != "":
 
       sampleName = prefix+"halos_min"+str(minHaloMass)+"_z"+fileNums[iRedshift]
       outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
-
       outFile.write("%f\n" %(lbox))
       outFile.write("%s\n" %(omegaM))
       outFile.write("%s\n" %(hubble))
       outFile.write("%s\n" %(redshift))
       outFile.write("%d\n" %(numPart))
 
+      if dataFormat == "sdf":
+        # TODO process halo file with SDFcvt
+        outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
+        outFile.write("-99 -99 -99 -99 -99 -99 -99\n")
+        outFile.close()
+      else:
+        outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
         inFile = open(dataFile, 'r')
         for (iHalo,line) in enumerate(inFile):
           if iHalo < haloFileNumComLines: continue
@@ -434,8 +474,8 @@ if (args.halos or args.all) and haloFileBase != "":
             outFile.write("%d %e %e %e %e %e %e\n" %(iHalo,x,y,z,vz,vy,vx))
 
         outFile.write("-99 -99 -99 -99 -99 -99 -99\n")
-      inFile.close()
         outFile.close()
+        inFile.close()
 
 # -----------------------------------------------------------------------------
 # now the HOD
@@ -510,6 +550,14 @@ if (args.hod or args.all) and haloFileBase != "":
       else:
         haloFile = catalogDir+haloFileBase.replace(haloFileDummy,
                                                  fileNums[iRedshift])
+
+      if dataFormat == "sdf":
+        # TODO process halo file with SDFcvt
+        inFile = haloFile
+        outFile = haloFile+"_temp"
+
+        haloFile = outFile
+
       parFile.write(parFileText.format(omegaM=omegaM,
                                      hubble=hubble,
                                      redshift=redshift,
@@ -535,4 +583,6 @@ if (args.hod or args.all) and haloFileBase != "":
       os.system("mv %s/hod.mock %s" % (catalogDir, outFileName))
       os.system("rm %s/hod.*" % catalogDir)
 
+      os.system("rm %s" % haloFile)
+
 print " Done!"

python_tools/setup.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/setup.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 from distutils.core import setup
 from distutils.extension import Extension
 from Cython.Distutils import build_ext

python_tools/void_python_tools/__init__.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/__init__.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 from void_python_tools.backend import *
 from void_python_tools.apTools import *
 from void_python_tools.plotting import *

python_tools/void_python_tools/apTools/__init__.py

@@ -1,2 +1,21 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/apTools/__init__.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 from chi2 import *
 from profiles import *

python_tools/void_python_tools/apTools/chi2/__init__.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/apTools/chi2/__init__.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 from velocityProfileFitNative import *
 from likelihood import * 
 from cosmologyTools import *

python_tools/void_python_tools/apTools/chi2/cosmologyTools.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/apTools/chi2/cosmologyTools.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 # a suite of functions to compute expansion rates, angular diameter 
 # distances, and expected void stretching
 

python_tools/void_python_tools/apTools/profiles/__init__.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/apTools/profiles/__init__.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 from build import *
 from draw import *
 from fit import *

python_tools/void_python_tools/apTools/profiles/getSurveyProps.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/apTools/profiles/getSurveyProps.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 import numpy as np
 import healpy as healpy
 import scipy.integrate

python_tools/void_python_tools/backend/__init__.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/backend/__init__.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 from classes import *
 from launchers import *
 from catalogPrep import *

python_tools/void_python_tools/backend/classes.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/backend/classes.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 # classes and routines used to support scripts
 
 import os

python_tools/void_python_tools/backend/launchers.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/backend/launchers.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 # -----------------------------------------------------------------------------
 # -----------------------------------------------------------------------------
 # routines which communicate with individual data analysis portions - 
@@ -105,27 +124,33 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
 
     datafile = inputDataDir+"/"+sample.dataFile
 
-    if regenerate:
+    prevSubSample = -1
-      inputParameterFlag = "inputParameter " + zobovDir+"/zobov_slice_"+sampleName+".par"
+    for thisSubSample in sample.subsample.split(', '):
-      outputFile = zobovDir+"/regenerated_zobov_slice_" + sampleName 
+
-    else:
+      if prevSubSample == -1:
         inputParameterFlag = ""
-      outputFile = zobovDir+"/zobov_slice_" + sampleName
+        outputFile = zobovDir+"/zobov_slice_" + sampleName + "_ss" + thisSubSample
-
+        keepFraction = float(thisSubSample)
-    if sample.usePecVel:
+        subSampleLine = "subsample %g" % keepFraction
-      includePecVelString = "peculiarVelocities"
       else:
+        inputParameterFlag = "inputParameter " + zobovDir+"/zobov_slice_"+\
+                             sampleName+"_ss"+prevSubSample+".par"
+        outputFile = zobovDir+"/_zobov_slice_" + sampleName + "_ss" + thisSubSample
+        keepFraction = float(thisSubSample)/float(prevSubSample)
+        subSampleLine = "resubsample %g" % keepFraction
+
       includePecVelString = ""
+      if sample.usePecVel: includePecVelString = "peculiarVelocities"
 
-    if sample.useLightCone:
-      useLightConeString = "cosmo"
-    else:
       useLightConeString = ""
+      if sample.useLightCone: useLightConeString = "cosmo"
 
       if sample.dataFormat == "multidark" or sample.dataFormat == "random":
         dataFileLine = "multidark " + datafile
       elif sample.dataFormat == "gadget":
         dataFileLine = "gadget " + datafile
+      elif sample.dataFormat == "sdf":
+        dataFileLine = "sdf " + datafile
     
       iX = float(sample.mySubvolume[0])
       iY = float(sample.mySubvolume[1])
@@ -148,36 +173,52 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
       rangeY_max %g
       rangeZ_min %g
       rangeZ_max %g
-      subsample %g
+      %s
       %s
       """ % (dataFileLine, outputFile,
-             zobovDir+"/zobov_slice_"+sampleName+".par",
+             outputFile+".par",
              includePecVelString,
              useLightConeString,
              sample.dataUnit,
              xMin, xMax, yMin, yMax,
              sample.zBoundaryMpc[0], sample.zBoundaryMpc[1],
-             sample.subsample,inputParameterFlag)
+             subSampleLine,inputParameterFlag)
 
       parmFile = os.getcwd()+"/generate_"+sample.fullName+".par"
 
       file(parmFile, mode="w").write(conf)
 
-    if regenerate or not (continueRun and jobSuccessful(logFile, "Done!\n")):
+      doneLine = "Done! %5.2e\n" % keepFraction
+      print "TEST", doneLine
+      if not (continueRun and jobSuccessful(logFile, doneLine)):
+        if (prevSubSample == -1):
           cmd = "%s --configFile=%s &> %s" % (binPath,parmFile,logFile)
-      os.system(cmd)
-      if jobSuccessful(logFile, "Done!\n"):
-        print "done"
         else:
+          cmd = "%s --configFile=%s &>> %s" % (binPath,parmFile,logFile)
+        os.system(cmd)
+      
+        if not jobSuccessful(logFile, doneLine):
           print "FAILED!"
           exit(-1)
 
-    else:
+      # remove intermediate files
-      print "already done!"
+      if (prevSubSample != -1):
+       os.unlink(zobovDir+"/zobov_slice_"+sampleName+"_ss"+prevSubSample+".par")
+       os.unlink(zobovDir+"/zobov_slice_"+sampleName+"_ss"+prevSubSample)
+
+      prevSubSample = thisSubSample
+
+    if (continueRun and jobSuccessful(logFile, doneLine)): print "already done!"
+    if jobSuccessful(logFile, doneLine): print "done"
+ 
+    # place the final subsample     
+    os.system("mv %s %s" % (zobovDir+"/zobov_slice_"+sampleName+"_ss"+\
+              prevSubSample, zobovDir+"/zobov_slice_"+sampleName))
+    os.system("mv %s %s" % (zobovDir+"/zobov_slice_"+sampleName+"_ss"+\
+              prevSubSample+".par", zobovDir+"/zobov_slice_"+sampleName+".par"))
 
     if os.access("comoving_distance.txt", os.F_OK):
       os.system("mv %s %s" % ("comoving_distance.txt", zobovDir))
-      #os.system("mv %s %s" % ("sample_info.txt", zobovDir))
 
     if os.access(parmFile, os.F_OK):
       os.unlink(parmFile)
@@ -417,7 +458,7 @@ def launchStack(sample, stack, binPath, thisDataPortion=None, logDir=None,
     maxDen = 0.2*float(maskIndex)/float(totalPart)
   else:
     maskIndex = 999999999
-    maxDen = -0.8
+    maxDen = 0.2
 
   if INCOHERENT:
     nullTestFlag = "INCOHERENT"

python_tools/void_python_tools/plotting/__init__.py

@@ -1,2 +1,21 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/plotting/__init__.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 from plotTools import *
 from plotDefs import *

python_tools/void_python_tools/plotting/plotDefs.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/plotting/plotDefs.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 LIGHT_SPEED = 299792.458 
 
 colorList = ['r', 'b', 'g', 'y', 'c', 'm', 

python_tools/void_python_tools/plotting/plotTools.py

@@ -1,3 +1,22 @@
+#+
+#   VIDE -- Void IDEntification pipeline -- ./python_tools/void_python_tools/plotting/plotTools.py
+#   Copyright (C) 2010-2013 Guilhem Lavaux
+#   Copyright (C) 2011-2013 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
 __all__=['plotRedshiftDistribution', 'plotSizeDistribution', 'plot1dProfiles',
          'plotMarg1d', 'plotNumberDistribution']