Merged in python3 (pull request #5)

Port to python3, large code cleanup

* Fixed command line for cosmotool

* Fix path

* Dump command line is log file

* Fix important typo

* Modify paths for example

* Fix path again

* Use an explicit constructor

* Change file to open (python 2->3)

* python3 fix for xrange in periodic_kdtree.py

* Fixed index for Np, numPart, numZones, numZonesTot, partID, zoneID in catalogUtil.py

python_tools/void_pipeline/__main__.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+#+
+#   VIDE -- Void IDentification and Examination -- ./python_tools/void_pipeline/__init__.py
+#   Copyright (C) 2010-2014 Guilhem Lavaux
+#   Copyright (C) 2011-2014 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
+
+# Stage 1 : generate particles
+# Stage 2 : find voids
+# Stage 3 : prune catalog
+
+from vide.backend import *
+import vide
+import imp
+import os
+import pickle
+
+# ------------------------------------------------------------------------------
+
+if (len(sys.argv) == 1):
+  print("Usage: ./generateCatalog.py parameter_file.py")
+  exit(-1)
+
+if (len(sys.argv) > 1):
+  filename = sys.argv[1]
+  print(" Loading parameters from", filename)
+  if not os.access(filename, os.F_OK):
+    print("  Cannot find parameter file %s!" % filename)
+    exit(-1)
+  parms = imp.load_source("name", filename)
+  regenerateFlag = False
+  globals().update(vars(parms))
+  void_path = os.path.split(vide.__file__)[0]
+  ZOBOV_PATH=f'{void_path}/bin/'
+  CTOOLS_PATH=ZOBOV_PATH
+  print(f"ZOBOV_PATH is  {ZOBOV_PATH}")
+else:
+  print(" Using default parameters")
+
+# ------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
+
+if not os.access(logDir, os.F_OK):
+  os.makedirs(logDir)
+
+if not os.access(figDir, os.F_OK):
+  os.makedirs(figDir)
+
+if not continueRun:
+  print(" Cleaning out log files...")
+
+  if startCatalogStage <= 1 and glob.glob(logDir+"/generate*") != []:
+    os.system("rm %s/generate*" % logDir)
+  if startCatalogStage <= 2 and glob.glob(logDir+"/zobov*") != []:
+    os.system("rm %s/zobov*" % logDir)
+  if startCatalogStage <= 3 and glob.glob(logDir+"/prune*") != []:
+    os.system("rm %s/prune*" % logDir)
+
+for sample in dataSampleList:
+
+  sampleName = sample.fullName
+
+  print(" Working with data set", sampleName, "...")
+  zobovDir = workDir+"/sample_"+sampleName+"/"
+  sample.zobovDir = zobovDir
+
+  if not os.access(zobovDir, os.F_OK):
+    os.makedirs(zobovDir)
+
+# ---------------------------------------------------------------------------
+  if (startCatalogStage <= 1) and (endCatalogStage >= 1) and not sample.isCombo:
+    print("  Extracting tracers from catalog...", end=' ')
+    sys.stdout.flush()
+
+    logFile = logDir+"/generate_"+sampleName+".out"
+
+    if sample.dataType == "observation":
+      GENERATE_PATH = CTOOLS_PATH+"/generateFromCatalog"
+    else:
+      GENERATE_PATH = CTOOLS_PATH+"/generateMock"
+
+    launchGenerate(sample, GENERATE_PATH, workDir=workDir, 
+                   inputDataDir=inputDataDir, zobovDir=zobovDir,
+                   figDir=figDir, logFile=logFile, useComoving=sample.useComoving,
+                   continueRun=continueRun, regenerate=regenerateFlag)
+
+  # --------------------------------------------------------------------------
+  if (startCatalogStage <= 2) and (endCatalogStage >= 2) and not sample.isCombo:
+    print("  Extracting voids with ZOBOV...", end=' ')
+    sys.stdout.flush()
+
+    launchZobov(sample, ZOBOV_PATH, zobovDir=zobovDir, logDir=logDir, 
+                continueRun=continueRun, numZobovDivisions=numZobovDivisions,
+                 numZobovThreads=numZobovThreads)
+
+  # -------------------------------------------------------------------------
+  if (startCatalogStage <= 3) and (endCatalogStage >= 3) and not sample.isCombo:
+
+    print("  Taking data portions", "...", end=' ')
+    sys.stdout.flush()
+        
+    logFile = logDir+"/pruneVoids_"+sampleName+".out"
+
+    PRUNE_PATH = CTOOLS_PATH+"/pruneVoids"
+
+    launchPrune(sample, PRUNE_PATH, 
+                logFile=logFile, zobovDir=zobovDir, 
+                useComoving=sample.useComoving, continueRun=continueRun)
+
+# -------------------------------------------------------------------------
+if (startCatalogStage <= 4) and (endCatalogStage >= 4):
+
+  print("  Plotting...", end=' ')
+  sys.stdout.flush()
+
+  #for thisDataPortion in dataPortions:
+    #plotRedshiftDistribution(workDir, dataSampleList, figDir, showPlot=False, 
+    #                         dataPortion=thisDataPortion, setName=setName)
+    #plotSizeDistribution(workDir, dataSampleList, figDir, showPlot=False, 
+    #                         dataPortion=thisDataPortion, setName=setName)
+    #plotNumberDistribution(workDir, dataSampleList, figDir, showPlot=False, 
+    #                         dataPortion=thisDataPortion, setName=setName)
+    #plotVoidDistribution(workDir, dataSampleList, figDir, showPlot=False, 
+    #                         dataPortion=thisDataPortion, setName=setName)
+
+print("\n Done!")

python_tools/void_pipeline/datasets/example_observation.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+#+
+#   VIDE -- Void IDentification and Examination -- ./python_tools/void_pipeline/datasets/example_observation.py
+#   Copyright (C) 2010-2014 Guilhem Lavaux
+#   Copyright (C) 2011-2014 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
+
+import os
+import numpy as np
+from vide.backend.classes import *
+
+# if True, will scan log files for last known completed state and run from there
+continueRun = False
+
+# stages:
+#   1 : extract redshift slices from data
+#   2 : void extraction using zobov
+#   3 : removal of small voids and voids near the edge 
+startCatalogStage = 1
+endCatalogStage   = 3
+
+basePath = os.path.dirname(os.path.abspath(__file__))
+basePath = os.path.abspath(os.path.join(basePath,"..","..","..","examples"))
+
+# directory for input data files
+inputDataDir = basePath
+
+# void catalog output directory
+workDir = os.path.join(basePath,"example_observation")
+
+# output directory for log files
+logDir = os.path.join(basePath,"logs","example_observation")
+
+# output directory for figures
+figDir = os.path.join(basePath,"figs","example_observation")
+
+# optimization: maximum number of parallel threads to use
+numZobovThreads = 2
+
+# optimization: number of subdivisions of the box
+numZobovDivisions = 2
+
+# don't change this
+dataSampleList = []
+
+# define your volume-limited samples
+newSample = Sample(
+                   # path to galaxy file is inputDataDir+dataFile
+                   dataFile = "example_observation.dat",
+
+                   # full name for this sample
+                   fullName = "example_observation",
+
+                   # a convenient nickname
+                   nickName = "exobs",
+
+                   # don't change this
+                   dataType = "observation",
+
+                   # assume sample is volume-limited?
+                   volumeLimited = True,
+
+                   # HEALpix mask file
+                   maskFile = inputDataDir+"/example_observation_mask.fits",
+
+                   # radial selection function (if not volume limited)
+                   selFunFile = None,
+
+                   # max and min redshifts of galaxies in your sample
+                   zBoundary = (0.0, 0.15),
+
+                   # max and min redshifts where you want to find voids
+                   zRange = (0.1, 0.15),
+ 
+                   # leave this at -1 for mean particle separation, 
+                   # or specify your own in Mpc/h
+                   minVoidRadius = -1,
+
+                   # density of mock particles in cubic Mpc/h
+                   # (make this as high as you can afford)
+                   fakeDensity = 0.05,
+                   
+                   # if true, convert to comoving space using LCDM cosmology
+                   useComoving = True
+
+                   )
+dataSampleList.append(newSample)
+
+# repeat the above block for any other samples

python_tools/void_pipeline/datasets/example_simulation.py

@@ -0,0 +1,167 @@
+#+
+#   VIDE -- Void IDentification and Examination -- ./python_tools/void_pipeline/datasets/example_simulation.py
+#   Copyright (C) 2010-2014 Guilhem Lavaux
+#   Copyright (C) 2011-2014 P. M. Sutter
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; version 2 of the License.
+# 
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program; if not, write to the Free Software Foundation, Inc.,
+#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#+
+import os
+
+PWD=os.getcwd()
+
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+# CONFIGURATION
+
+# if True, will scan log files for last known completed state and run from there
+continueRun = False
+
+# stages:
+#   1 : extract redshift slices from data
+#   2 : void extraction using zobov
+#   3 : removal of small voids and voids near the edge 
+startCatalogStage = 1
+endCatalogStage   = 3
+
+# directory for the input simulation files
+catalogDir = PWD+"/examples/"
+
+# void catalog output directory
+voidOutputDir = PWD+"/examples/example_simulation/"
+
+# output directory for log files
+logDir = PWD+"/logs/example_simulation/"
+
+# output directory for figures
+figDir = PWD+"/figs/example_simulation/"
+
+# where to place the pipeline scripts
+scriptDir = PWD+"/example_simulation/"
+
+# don't change
+dataType = "simulation"
+
+# available formats for simulation: gadget, sdf, multidark
+dataFormat = "multidark"
+
+# units of position in Mpc/h
+dataUnit = 1
+
+# place particles on the lightcone (z-axis in sims)?
+useLightCone = False
+
+# add peculiar velocities?
+doPecVel = False
+
+# optimization: maximum number of parallel threads to use
+numZobovThreads = 2
+
+# optimization: number of subdivisions of the box
+numZobovDivisions = 2
+
+# prefix to give all outputs
+prefix = "sim_"
+
+# how many independent slices along the z-axis?
+numSlices = 1
+
+# how many subdivisions along the x- and y- axis?
+#   ( = 2 will make 4 subvolumes for each slice, = 3 will make 9, etc.)
+numSubvolumes = 1
+
+
+###############################################################################
+# Particles
+
+# common filename of particle files
+particleFileBase = "example_simulation_NNNN.dat"
+
+# this flag will be replaced by values in fileNums list below
+particleFileDummy = 'NNNN'
+
+# list of file numbers for the particle files
+fileNums = ["z0.0"]
+
+# redshift of each file in the above fileNums list
+redshifts = ["0.0"]
+
+# list of desired subsamples - these are in unts of h Mpc^-3!
+subSamples = [1.0]
+
+# if True, do the subsampling in preparation (available for sdf and multidark)
+doSubSamplingInPrep = False
+
+# if 'absolute', subSamples are given in particles per cubic Mpc/h
+# if 'relative', subSamples are given as a fraction of input particles
+subSampleMode = "relative"
+
+# shift the z-coord of sims with redshift
+shiftSimZ = False
+
+###############################################################################
+# Halos
+
+# common filename of halo files, leave blank to ignore halos
+haloFileBase = ""
+#haloFileBase = "mf_4s_1G_1k_bgc2_NNNNN.sdf"
+
+# this flag will be replaced by values in fileNums list above
+haloFileDummy = ''
+#haloFileDummy = 'NNNNN'
+
+# minimum halo mass cuts to apply for the halo catalog
+#   use "none" to get all halos
+minHaloMasses = []
+#minHaloMasses = ["none", 1.2e13]
+
+# locations of data in the halo catalog
+haloFileMCol  = 6    # mass
+haloFileXCol  = 0    # x
+haloFileYCol  = 1    # y
+haloFileZCol  = 2    # z
+haloFileVXCol = 3    # v_x
+haloFileVYCol = 4    # v_y
+haloFileVZCol = 5    # v_z
+haloFileColSep = ',' # separator
+haloFileNumComLines = 0 # number of comments before data
+
+
+###############################################################################
+# simulation information
+
+numPart = 1024*1024*1024
+lbox = 999.983 # Mpc/h
+omegaM = 0.2847979853038958
+hubble = 0.6962 # h_0
+
+
+###############################################################################
+# HOD
+
+# each of the HOD sets will be applied to each halo catalog defined above
+hodParmList = [
+  #{'name'       : "LowRes", #BOSS: Manera et al. 2012, eq. 26
+  # 'Mmin'       : 0.0,
+  # 'M1'         : 1.e14,
+  # 'sigma_logM' : 0.596,
+  # 'alpha'      : 1.0127,
+  # 'Mcut'       : 1.19399e13,
+  # 'galDens'    : 0.0002,
+  #},
+]
+
+# END CONFIGURATION
+# -----------------------------------------------------------------------------
+# -----------------------------------------------------------------------------