number of zobov threads and subdivisions now part of dataset definition

pipeline/datasets/multidark.py

@@ -48,9 +48,16 @@ numSubvolumes = 1
 # prefix to give all outputs
 prefix = "md_"
 
-# list of desired subsamples
+# list of desired subsamples - these are in unts of h Mpc^-3!
+#subSamples = [ 1.0 ]
 subSamples = ((0.1, 0.05, 0.01, 0.002, 0.001, 0.0004, 0.0002))
 
+# adjust these two parameters given the memory contraints on your system:
+#   numZobovDivisions: how many sub-volumes per dimension will zobov process
+#   numZobovThreads: how many sub-volumes to process at once?   
+numZobovDivisions = 4
+numZobovThreads = 2
+
 # simulation information
 numPart = 1024*1024*1024
 lbox = 1000 # Mpc/h

pipeline/generateCatalog.py

@@ -88,7 +88,8 @@ for sample in dataSampleList:
     sys.stdout.flush()
 
     launchZobov(sample, ZOBOV_PATH, zobovDir=zobovDir, logDir=logDir, 
-                continueRun=continueRun)
+                continueRun=continueRun, numZobovDivisions=numZobovDivisions,
+                 numZobovThreads=numZobovThreads)
 
   # -------------------------------------------------------------------------
   if (startCatalogStage <= 3) and (endCatalogStage >= 3) and not sample.isCombo:

pipeline/prepareCatalogs.py

@@ -115,10 +115,15 @@ workDir = "{voidOutputDir}/{setName}/"
 inputDataDir = "{inputDataDir}"
 figDir = "{figDir}/{setName}/"
 logDir = "{logDir}/{setName}/"
+
+numZobovDivisions = {numZobovDivisions}
+numZobovThreads = {numZobovThreads}
                """
   scriptFile.write(dataInfo.format(setName=setName, figDir=figDir,
                                    logDir=logDir, voidOutputDir=voidOutputDir,
-                                   inputDataDir=catalogDir))
+                                   inputDataDir=catalogDir, 
+                                   numZobovDivisions=numZobovDivisions,
+                                   numZobovThreads=numZobovThreads))
 
   sampleInfo = """
 newSample = Sample(dataFile = "{dataFile}",
@@ -139,7 +144,6 @@ newSample = Sample(dataFile = "{dataFile}",
                    usePecVel = {usePecVel},
                    numSubvolumes = {numSubvolumes},
                    mySubvolume = "{mySubvolume}",
-                   numSubDivisions = 4,
                    useLightCone = {useLightCone},
                    subsample = {subsample})
 dataSampleList.append(newSample)

python_tools/void_python_tools/backend/classes.py

@@ -63,7 +63,6 @@ class Sample:
   usePecVel = False
   subsample = 1.0
   useLightCone = True
-  numSubDivisions = 1
   numSubvolumes = 1
   mySubvolume = 1
 
@@ -75,9 +74,9 @@ class Sample:
                minVoidRadius=0, fakeDensity=0.01, volumeLimited=True,
                includeInHubble=True, partOfCombo=False, isCombo=False, 
                comboList=(), profileBinSize=2.0, skyFraction=0.19,
-               dataType="observation", numSubDivisions=2, 
                boxLen=1024, usePecVel=False, omegaM=0.27, 
                numSubvolumes=1, mySubvolume=1, dataFormat="sdss",
+               dataType="observation",
                subsample=1.0, useLightCone=True):
     self.dataFile = dataFile
     self.fullName = fullName
@@ -98,7 +97,6 @@ class Sample:
     self.profileBinSize = profileBinSize
     self.skyFraction = skyFraction
     self.dataType = dataType
-    self.numSubDivisions = numSubDivisions
     self.boxLen = boxLen
     self.usePecVel = usePecVel
     self.omegaM = omegaM

python_tools/void_python_tools/backend/launchers.py

@@ -166,7 +166,8 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
       os.unlink(parmFile)
 
 # -----------------------------------------------------------------------------
-def launchZobov(sample, binPath, zobovDir=None, logDir=None, continueRun=None):
+def launchZobov(sample, binPath, zobovDir=None, logDir=None, continueRun=None,
+                 numZobovDivisions=None, numZobovThreads=None):
 
   sampleName = sample.fullName
 
@@ -197,11 +198,9 @@ def launchZobov(sample, binPath, zobovDir=None, logDir=None, continueRun=None):
     if os.access(zobovDir+"/voidDesc_"+sampleName+".out", os.F_OK):
       os.unlink(zobovDir+"/voidDesc_"+sampleName+".out")
 
-    numThreads = 2
-   
     cmd = "%s/vozinit %s 0.1 1.0 %g %s %g %s %s %s >& %s" % \
-          (binPath, datafile,  sample.numSubDivisions, \
-                      "_"+sampleName, numThreads, \
+          (binPath, datafile,  numZobovDivisions, \
+                      "_"+sampleName, numZobovThreads, \
                       binPath, zobovDir, maskIndex, logFile)
     os.system(cmd)