fixed bug in gadget generation script; prepareGadget now uses separate parameter files; support for random catalogs now enabled

pipeline/generateCatalog.py

@@ -113,10 +113,10 @@ if (startCatalogStage <= 4) and (endCatalogStage >= 4):
 
   for thisDataPortion in dataPortions:
     plotRedshiftDistribution(workDir, dataSampleList, figDir, showPlot=False, 
-                             dataPortion=thisDataPortion, setName=catalogName)
+                             dataPortion=thisDataPortion, setName=setName)
     plotSizeDistribution(workDir, dataSampleList, figDir, showPlot=False, 
-                             dataPortion=thisDataPortion, setName=catalogName)
+                             dataPortion=thisDataPortion, setName=setName)
     plotNumberDistribution(workDir, dataSampleList, figDir, showPlot=False, 
-                             dataPortion=thisDataPortion, setName=catalogName)
+                             dataPortion=thisDataPortion, setName=setName)
 
 print "\n Done!"

pipeline/prepareCatalogs.py

@@ -9,66 +9,17 @@ import os
 import sys
 import void_python_tools as vp
 import argparse
+import imp
 
-# -----------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
-# -----------------------------------------------------------------------------
-# CONFIGURATION
 
-# directory for the input simulation/observational particle files
+def my_import(name):
-catalogDir = os.getenv("HOME")+"/workspace/Voids/catalogs/multidark/"
+    mod = __import__(name)
+    components = name.split('.')
+    for comp in components[1:]:
+        mod = getattr(mod, comp)
+    return mod
 
-# path to HOD code
-hodPath = os.getenv("HOME")+"/projects/Voids/hod/HOD.x"
-
-# where to put the final void catalog, figures, and output logs
-voidOutputDir = os.getenv("HOME")+"/workspace/Voids/multidark/"
-figDir = os.getenv("PWD")+"/../figs/multidark/"
-logDir = os.getenv("PWD")+"/../logs/multidark/"
-
-# where to place the pipeline scripts
-scriptDir = os.getenv("PWD")+"/multidark/"
-
-# simulation or observation?
-dataType = "simulation"
-
-# available formats for simulation: gadget, multidark
-dataFormat = "multidark"
-dataUnit = 1 # as multiple of Mpc/h
-
-# place particles on the lightcone?
-useLightCone = True
-
-# common filename of particle files
-particleFileBase = "mdr1_particles_z"
-
-# list of file numbers for the particle files
-# to get particle file name, we take particleFileBase+fileNum
-fileNums = (("0.0", "0.53", "1.0"))
-
-# redshift of each file in the above list
-redshifts = (("0.0", "0.53", "1.0"))
-
-# how many independent slices along the z-axis?
-numSlices = 4
-
-# how many subdivisions along the x- and y- axis?
-#   ( = 2 will make 4 subvolumes for each slice, = 3 will make 9, etc.)
-numSubvolumes = 1
-
-# prefix to give all outputs
-prefix = "md_"
-
-# list of desired subsamples
-subSamples = ((0.1, 0.05, 0.01, 0.002, 0.001, 0.0004, 0.0002))
-
-# simulation information
-numPart = 1024*1024*1024
-lbox = 1000 # Mpc/h
-omegaM = 0.27
-hubble = 0.70
-
-# END CONFIGURATION
-# -----------------------------------------------------------------------------
 # -----------------------------------------------------------------------------
 
 LIGHT_SPEED = 299792.458
@@ -85,12 +36,25 @@ parser.add_argument('--halos', dest='halos', action='store_const',
                    help='write halos')
 parser.add_argument('--hod', dest='hod', action='store_const',
                    const=True, default=False,
-                   help='write hos')
+                   help='write hod')
 parser.add_argument('--all', dest='all', action='store_const',
                    const=True, default=False,
                    help='write everything')
+parser.add_argument('--parmFile', dest='parmFile',
+                   default="",
+                   help='path to parameter file')
 args = parser.parse_args()
 
+
+filename = args.parmFile
+print " Loading parameters from", filename
+if not os.access(filename, os.F_OK):
+  print "  Cannot find parameter file %s!" % filename
+  exit(-1)
+parms = imp.load_source("name", filename)
+globals().update(vars(parms))
+
+
 #------------------------------------------------------------------------------
 def getSampleName(setName, redshift, useVel, iSlice=-1, iVol=-1):
 
@@ -253,6 +217,7 @@ newSample.addStack({zMin}, {zMax}, {minRadius}+18, {minRadius}+24, True, False)
 #------------------------------------------------------------------------------
 #------------------------------------------------------------------------------
 if not os.access(scriptDir, os.F_OK): os.mkdir(scriptDir)
+if not os.access(catalogDir, os.F_OK): os.mkdir(catalogDir)
 
 #------------------------------------------------------------------------------
 # first the directly downsampled runs
@@ -288,45 +253,73 @@ for thisSubSample in subSamples:
                    redshifts, 
                   numSubvolumes, numSlices, True, lbox, minRadius, omegaM,
                   subsample=thisSubSample, suffix="")
+    elif dataFormat == "random":
+      writeScript(setName, "ran.ss"+str(thisSubSample)+"_z", 
+                  scriptDir, catalogDir, fileNums, 
+                  redshifts, 
+                  numSubvolumes, numSlices, False, lbox, minRadius, omegaM, 
+                  subsample=1.0)
   
   if args.subsample or args.all:
     print " Doing subsample", thisSubSample
 
     for (iRedshift, redshift) in enumerate(redshifts):
       print "   redshift", redshift
-      dataFile = catalogDir+"/"+particleFileBase+fileNums[iRedshift]
-      inFile = open(dataFile, 'r')
   
-      sampleName = "md.ss"+str(thisSubSample)+"_z"+redshift
+      if dataFormat == "multidark":
-      outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
+        dataFile = catalogDir+"/"+particleFileBase+fileNums[iRedshift]
+        inFile = open(dataFile, 'r')
 
-      outFile.write("%f\n" %(lbox))
+        sampleName = "md.ss"+str(thisSubSample)+"_z"+redshift
-      outFile.write("%s" %(omegaM))
+        outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
-      outFile.write("%s" %(hubble))
-      outFile.write("%s\n" %(redshift))
-      outFile.write("%d\n" %(maxKeep))
 
-      numKept = 0
+        outFile.write("%f\n" %(lbox))
-      for line in inFile:
+        outFile.write("%s\n" %(omegaM))
-        if np.random.uniform() > keepFraction: continue
+        outFile.write("%s\n" %(hubble))
-        numKept += 1
+        outFile.write("%s\n" %(redshift))
-        if numKept > maxKeep: break
+        outFile.write("%d\n" %(maxKeep))
-        line = line.split(',')
-        x  = float(line[0])
-        y  = float(line[1])
-        z  = float(line[2])
-        vz = float(line[3])
 
-        outFile.write("%e %e %e %e\n" %(x,y,z,vz))
+        numKept = 0
+        for line in inFile:
+          if np.random.uniform() > keepFraction: continue
+          numKept += 1
+          if numKept > maxKeep: break
+          line = line.split(',')
+          x  = float(line[0])
+          y  = float(line[1])
+          z  = float(line[2])
+          vz = float(line[3])
+
+          outFile.write("%e %e %e %e\n" %(x,y,z,vz))
+
+        outFile.write("-99 -99 -99 -99\n")
+        inFile.close()
+        outFile.close()
+
+      elif dataFormat == "random":
+        sampleName = "ran.ss"+str(thisSubSample)+"_z"+redshift
+        outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
+
+        outFile.write("%f\n" %(lbox))
+        outFile.write("%s\n" %(omegaM))
+        outFile.write("%s\n" %(hubble))
+        outFile.write("%s\n" %(redshift))
+        outFile.write("%d\n" %(maxKeep))
+
+        for i in xrange(int(maxKeep)):
+          x  = np.random.uniform()*lbox
+          y  = np.random.uniform()*lbox
+          z  = np.random.uniform()*lbox
+
+          outFile.write("%e %e %e %e\n" %(x,y,z, 0.))
+
+        outFile.write("-99 -99 -99 -99\n")
+        outFile.close()
 
-      outFile.write("-99 -99 -99 -99\n")
-      print "KEEPING:", numKept, "...predicted:", maxKeep
-      inFile.close()
-      outFile.close()
 
 # -----------------------------------------------------------------------------
 # now halos
-if args.script or args.all:
+if (args.script or args.all) and dataFormat == "multidark":
   print " Doing halo scripts"
 
   dataFile = catalogDir+"/mdr1_halos_z"+fileNums[0]
@@ -362,8 +355,8 @@ if args.halos or args.all:
     outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
 
     outFile.write("%f\n" %(lbox))
-    outFile.write("%s" %(omegaM))
+    outFile.write("%s\n" %(omegaM))
-    outFile.write("%s" %(hubble))
+    outFile.write("%s\n" %(hubble))
     outFile.write("%s\n" %(redshift))
     outFile.write("%d\n" %(numPart))
 
@@ -425,7 +418,7 @@ BOX_SIZE   {boxSize}
 root_filename hod
                """
 
-if args.script or args.all:
+if (args.script or args.all) and dataFormat == "multidark":
   print " Doing DR7 HOD scripts"
   if dataFormat == "multidark":
     setName = prefix+"hod_dr72dim2"
@@ -467,16 +460,16 @@ if args.hod or args.all:
 
 # -----------------------------------------------------------------------------
 # now the BOSS HOD
-if args.script or args.all:
+if (args.script or args.all) and dataFormat == "multidark":
   print " Doing DR9 HOD scripts"
   if dataFormat == "multidark":
     setName = prefix+"hod_dr9mid"
     writeScript(setName, "md.hod_dr9mid_z",
               scriptDir, catalogDir, fileNums, redshifts, 
-               numSubvolumes, numSlices, False, lbox, 5, omegaM)
+               numSubvolumes, numSlices, False, lbox, 15, omegaM)
-    writeScript(prefix, "md.hod_dr9mid_z",
+    writeScript(setName, "md.hod_dr9mid_z",
               scriptDir, catalogDir, fileNums, redshifts, 
-               numSubvolumes, numSlices, True, lbox, 5, omegaM)
+               numSubvolumes, numSlices, True, lbox, 15, omegaM)
 
 if args.hod or args.all:
   print " Doing DR9 HOD"

pipeline/prepareGadgetCatalog.py

@@ -1,246 +0,0 @@
-#!/usr/bin/env python
-
-# script which prepares inputs for gadget-based void run
-
-import numpy as np
-import os
-import sys
-import void_python_tools as vp
-import argparse
-
-# -----------------------------------------------------------------------------
-# -----------------------------------------------------------------------------
-# CONFIGURATION
-
-# directory for the input simulation/observational particle files
-catalogDir = os.getenv("HOME")+"/workspace/Voids/catalogs/gadget/"
-
-# where to put the final void catalog, figures, and output logs
-voidOutputDir = os.getenv("HOME")+"/workspace/Voids/gadget/"
-figDir = os.getenv("PWD")+"/../figs/gadget/"
-logDir = os.getenv("PWD")+"/../logs/gadget/"
-
-# where to place the pipeline scripts
-scriptDir = os.getenv("PWD")+"/gadget/"
-
-# simulation or observation?
-dataType = "simulation"
-
-# available formats for simulation: gadget, multidark
-dataFormat = "gadget"
-dataUnit = 1 # as multiple of Mpc/h
-
-# place particles on the lightcone?
-useLightCone = False 
-
-# common filename of particle files
-particleFileBase = "mdr1_particles_z"
-
-# list of file numbers for the particle files
-# to get particle file name, we take particleFileBase+fileNum
-fileNums = (("0.0", "0.53", "1.0")) 
-
-# redshift of each file in the above list
-redshifts = (("0.0", "0.53", "1.0")) 
-
-# how many independent slices along the z-axis?
-numSlices = 4 
-
-# how many subdivisions along the x- and y- axis?
-#   ( = 2 will make 4 subvolumes for each slice, = 3 will make 9, etc.)
-numSubvolumes = 1 
-
-# prefix to give all outputs
-prefix = "gadget_"
-
-# list of desired subsamples
-subSamples = [ 1.0, 0.1 ]
-
-# simulation information
-numPart = 1024*1024*1024 
-lbox = 1000 # Mpc/h
-omegaM = 0.27
-hubble = 0.70
-
-# END CONFIGURATION
-# -----------------------------------------------------------------------------
-# -----------------------------------------------------------------------------
-
-#------------------------------------------------------------------------------
-LIGHT_SPEED = 299792.458
-
-def getSampleName(setName, redshift, useVel, iSlice=-1, iVol=-1):
-
-  sampleName = setName
-
-  if useVel: setName += "_pv"
-    
-  if iSlice != -1: sampleName += "_s" + str(iSlice)
-
-  if iVol != -1: sampleName += "_d" + iVol
-
-  return sampleName
-
-#------------------------------------------------------------------------------
-# for given dataset parameters, outputs a script for use with analyzeVoids
-def writeScript(setName, dataFileNameBase, 
-                scriptDir, catalogDir, fileNums, redshifts, numSubvolumes,
-                numSlices, useVel, lbox, minRadius, omegaM, subsample=1.0, 
-                suffix=".dat"):
-
-  if useVel: setName += "_pv"
-
-  scriptFileName = scriptDir + "/" + setName + ".py"
-  scriptFile = open(scriptFileName, 'w')
-
-  scriptFile.write("""#!/usr/bin/env/python
-import os
-from void_python_tools.backend.classes import *
-
-continueRun = False
-startCatalogStage = 1
-endCatalogStage   = 4
-               
-startAPStage = 1
-endAPStage = 7
-
-ZOBOV_PATH = os.getenv("PWD")+"/../zobov/"
-CTOOLS_PATH = os.getenv("PWD")+"/../c_tools/"
-freshStack = True
-errorBars = "CALCULATED"
-numIncoherentRuns = 100
-ranSeed = 101010
-useLCDM = False
-bias = 1.16
-
-dataPortions = ["central"]
-dataSampleList = []
-""")
-
-  dataInfo = """
-setName = "{setName}"
-
-workDir = "{voidOutputDir}/{setName}/"
-inputDataDir = "{inputDataDir}"
-figDir = "{figDir}/{setName}/"
-logDir = "{logDir}/{setName}/"
-               """
-  scriptFile.write(dataInfo.format(setName=setName, figDir=figDir, 
-                                   logDir=logDir, voidOutputDir=voidOutputDir,
-                                   inputDataDir=catalogDir))
-
-  sampleInfo = """
-newSample = Sample(dataFile = "{dataFile}",
-                   dataFormat = "{dataFormat}",
-                   dataUnit = {dataUnit},
-                   fullName = "{sampleName}",
-                   nickName = "{sampleName}",
-                   dataType = "simulation",
-                   zBoundary = ({zMin}, {zMax}),
-                   zRange    = ({zMin}, {zMax}),
-                   zBoundaryMpc = ({zMinMpc}, {zMaxMpc}),
-                   omegaM    = {omegaM},
-                   minVoidRadius = {minRadius},
-                   includeInHubble = True,
-                   partOfCombo = False,
-                   isCombo = False,
-                   boxLen = {boxLen},
-                   usePecVel = {usePecVel},
-                   numSubvolumes = {numSubvolumes},
-                   mySubvolume = "{mySubvolume}",
-                   numSubDivisions = 4,
-                   useLightCone = {useLightCone},
-                   subsample = {subsample})
-dataSampleList.append(newSample)
-               """
-
-  for (iFile, redshift) in enumerate(redshifts):
-    fileNum = fileNums[iFile]
-    zBox = float(redshift)
-    Om = float(omegaM)
-    zBoxMpc = LIGHT_SPEED/100.*vp.angularDiameter(zBox, Om=Om)
-    boxMaxMpc = zBoxMpc + lbox
-
-    # converter from redshift to comoving distance   
-    zVsDY = np.linspace(0., zBox+8*lbox*100./LIGHT_SPEED, 10000)  
-    zVsDX = np.zeros(len(zVsDY))
-    for i in xrange(len(zVsDY)):
-      zVsDX[i] = vp.angularDiameter(zVsDY[i], Om=Om)
-
-    if useLightCone:
-      boxWidthZ = np.interp(vp.angularDiameter(zBox,Om=Om)+100. / \
-                  LIGHT_SPEED*lbox, zVsDX, zVsDY)-zBox
-      dzSafe = 0.03
-    else:
-      boxWidthZ = lbox*100./LIGHT_SPEED
-      dzSafe = 0.0
-
-    for iSlice in xrange(numSlices):
-      sliceMin = zBox + dzSafe + iSlice*(boxWidthZ-dzSafe)/numSlices
-      sliceMax = zBox + dzSafe + (iSlice+1)*(boxWidthZ-dzSafe)/numSlices
-
-      sliceMinMpc = sliceMin*LIGHT_SPEED/100.
-      sliceMaxMpc = sliceMax*LIGHT_SPEED/100.
-  
-      sliceMin = "%0.2f" % sliceMin
-      sliceMax = "%0.2f" % sliceMax
-      sliceMinMpc = "%0.1f" % sliceMinMpc
-      sliceMaxMpc = "%0.1f" % sliceMaxMpc
-
-      dataFileName = dataFileNameBase + fileNum + suffix
-
-      for iX in xrange(numSubvolumes):
-        for iY in xrange(numSubvolumes):
-  
-          mySubvolume = "%d%d" % (iX, iY)
-
-          sampleName = getSampleName(setName, redshift, useVel, 
-                                     iSlice=iSlice, iVol=mySubvolume)
-
-          scriptFile.write(sampleInfo.format(dataFile=dataFileName, 
-                                         dataFormat=dataFormat,
-                                         dataUnit=dataUnit,
-                                         sampleName=sampleName,
-                                         zMin=sliceMin,
-                                         zMax=sliceMax,
-                                         zMinMpc=sliceMinMpc,
-                                         zMaxMpc=sliceMaxMpc,
-                                         omegaM=Om,
-                                         boxLen=lbox,
-                                         usePecVel=useVel,
-                                         minRadius=minRadius,
-                                         numSubvolumes=numSubvolumes,
-                                         mySubvolume=mySubvolume,
-                                         useLightCone=useLightCone,
-                                         subsample=subsample))
-
-  scriptFile.close()
-  return
-
-
-#------------------------------------------------------------------------------
-#------------------------------------------------------------------------------
-if not os.access(scriptDir, os.F_OK): os.mkdir(scriptDir)
-
-#------------------------------------------------------------------------------
-# first the directly downsampled runs
-# Note: ss0.002   ~ SDSS DR7 dim2
-#       ss0.0004 ~ SDSS DR9 mid 
-baseResolution = float(numPart)/lbox/lbox/lbox # particles/Mpc^3
-for thisSubSample in subSamples:
-
-  keepFraction = float(thisSubSample) / baseResolution
-  maxKeep = keepFraction * numPart
-  minRadius = int(np.ceil(lbox/maxKeep**(1./3)))
-
-  print " Doing subsample", thisSubSample, " scripts"
-  setName = prefix+"ss"+str(thisSubSample)
-  writeScript(setName, particleFileBase, scriptDir, catalogDir, fileNums, 
-                 redshifts, 
-                 numSubvolumes, numSlices, True, lbox, minRadius, omegaM,
-                 subsample=thisSubSample, suffix="")
-  writeScript(setName, particleFileBase, scriptDir, catalogDir, fileNums, 
-                 redshifts, 
-                 numSubvolumes, numSlices, False, lbox, minRadius, omegaM,
-                 subsample=thisSubSample, suffix="")
-  

python_tools/void_python_tools/backend/launchers.py

@@ -106,7 +106,7 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
     else:
       useLightConeString = ""
 
-    if sample.dataFormat == "multidark":
+    if sample.dataFormat == "multidark" or sample.dataFormat == "random":
       dataFileLine = "multidark " + datafile
     elif sample.dataFormat == "gadget":
       dataFileLine = "gadget " + datafile
@@ -197,8 +197,10 @@ def launchZobov(sample, binPath, zobovDir=None, logDir=None, continueRun=None):
     if os.access(zobovDir+"/voidDesc_"+sampleName+".out", os.F_OK):
       os.unlink(zobovDir+"/voidDesc_"+sampleName+".out")
 
-    cmd = "%s/vozinit %s 0.1 1.0 2 %s %g %s %s %s >& %s" % \
+    numThreads = 2
-          (binPath, datafile,  \
+   
+    cmd = "%s/vozinit %s 0.1 1.0 %g %s %g %s %s %s >& %s" % \
+          (binPath, datafile,  numThreads, \
                       "_"+sampleName, sample.numSubDivisions, \
                       binPath, zobovDir, maskIndex, logFile)
     os.system(cmd)