fixed bug in gadget generation script; prepareGadget now uses separate parameter files; support for random catalogs now enabled

This commit is contained in:
P.M. Sutter 2012-11-15 06:37:59 -06:00
parent 292afacaf5
commit 443b4bdeec
4 changed files with 91 additions and 342 deletions

View file

@ -113,10 +113,10 @@ if (startCatalogStage <= 4) and (endCatalogStage >= 4):
for thisDataPortion in dataPortions:
plotRedshiftDistribution(workDir, dataSampleList, figDir, showPlot=False,
dataPortion=thisDataPortion, setName=catalogName)
dataPortion=thisDataPortion, setName=setName)
plotSizeDistribution(workDir, dataSampleList, figDir, showPlot=False,
dataPortion=thisDataPortion, setName=catalogName)
dataPortion=thisDataPortion, setName=setName)
plotNumberDistribution(workDir, dataSampleList, figDir, showPlot=False,
dataPortion=thisDataPortion, setName=catalogName)
dataPortion=thisDataPortion, setName=setName)
print "\n Done!"

View file

@ -9,66 +9,17 @@ import os
import sys
import void_python_tools as vp
import argparse
import imp
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# CONFIGURATION
# ------------------------------------------------------------------------------
# directory for the input simulation/observational particle files
catalogDir = os.getenv("HOME")+"/workspace/Voids/catalogs/multidark/"
def my_import(name):
mod = __import__(name)
components = name.split('.')
for comp in components[1:]:
mod = getattr(mod, comp)
return mod
# path to HOD code
hodPath = os.getenv("HOME")+"/projects/Voids/hod/HOD.x"
# where to put the final void catalog, figures, and output logs
voidOutputDir = os.getenv("HOME")+"/workspace/Voids/multidark/"
figDir = os.getenv("PWD")+"/../figs/multidark/"
logDir = os.getenv("PWD")+"/../logs/multidark/"
# where to place the pipeline scripts
scriptDir = os.getenv("PWD")+"/multidark/"
# simulation or observation?
dataType = "simulation"
# available formats for simulation: gadget, multidark
dataFormat = "multidark"
dataUnit = 1 # as multiple of Mpc/h
# place particles on the lightcone?
useLightCone = True
# common filename of particle files
particleFileBase = "mdr1_particles_z"
# list of file numbers for the particle files
# to get particle file name, we take particleFileBase+fileNum
fileNums = (("0.0", "0.53", "1.0"))
# redshift of each file in the above list
redshifts = (("0.0", "0.53", "1.0"))
# how many independent slices along the z-axis?
numSlices = 4
# how many subdivisions along the x- and y- axis?
# ( = 2 will make 4 subvolumes for each slice, = 3 will make 9, etc.)
numSubvolumes = 1
# prefix to give all outputs
prefix = "md_"
# list of desired subsamples
subSamples = ((0.1, 0.05, 0.01, 0.002, 0.001, 0.0004, 0.0002))
# simulation information
numPart = 1024*1024*1024
lbox = 1000 # Mpc/h
omegaM = 0.27
hubble = 0.70
# END CONFIGURATION
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
LIGHT_SPEED = 299792.458
@ -85,12 +36,25 @@ parser.add_argument('--halos', dest='halos', action='store_const',
help='write halos')
parser.add_argument('--hod', dest='hod', action='store_const',
const=True, default=False,
help='write hos')
help='write hod')
parser.add_argument('--all', dest='all', action='store_const',
const=True, default=False,
help='write everything')
parser.add_argument('--parmFile', dest='parmFile',
default="",
help='path to parameter file')
args = parser.parse_args()
filename = args.parmFile
print " Loading parameters from", filename
if not os.access(filename, os.F_OK):
print " Cannot find parameter file %s!" % filename
exit(-1)
parms = imp.load_source("name", filename)
globals().update(vars(parms))
#------------------------------------------------------------------------------
def getSampleName(setName, redshift, useVel, iSlice=-1, iVol=-1):
@ -253,6 +217,7 @@ newSample.addStack({zMin}, {zMax}, {minRadius}+18, {minRadius}+24, True, False)
#------------------------------------------------------------------------------
#------------------------------------------------------------------------------
if not os.access(scriptDir, os.F_OK): os.mkdir(scriptDir)
if not os.access(catalogDir, os.F_OK): os.mkdir(catalogDir)
#------------------------------------------------------------------------------
# first the directly downsampled runs
@ -288,45 +253,73 @@ for thisSubSample in subSamples:
redshifts,
numSubvolumes, numSlices, True, lbox, minRadius, omegaM,
subsample=thisSubSample, suffix="")
elif dataFormat == "random":
writeScript(setName, "ran.ss"+str(thisSubSample)+"_z",
scriptDir, catalogDir, fileNums,
redshifts,
numSubvolumes, numSlices, False, lbox, minRadius, omegaM,
subsample=1.0)
if args.subsample or args.all:
print " Doing subsample", thisSubSample
for (iRedshift, redshift) in enumerate(redshifts):
print " redshift", redshift
dataFile = catalogDir+"/"+particleFileBase+fileNums[iRedshift]
inFile = open(dataFile, 'r')
if dataFormat == "multidark":
dataFile = catalogDir+"/"+particleFileBase+fileNums[iRedshift]
inFile = open(dataFile, 'r')
sampleName = "md.ss"+str(thisSubSample)+"_z"+redshift
outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
sampleName = "md.ss"+str(thisSubSample)+"_z"+redshift
outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
outFile.write("%f\n" %(lbox))
outFile.write("%s" %(omegaM))
outFile.write("%s" %(hubble))
outFile.write("%s\n" %(redshift))
outFile.write("%d\n" %(maxKeep))
outFile.write("%f\n" %(lbox))
outFile.write("%s\n" %(omegaM))
outFile.write("%s\n" %(hubble))
outFile.write("%s\n" %(redshift))
outFile.write("%d\n" %(maxKeep))
numKept = 0
for line in inFile:
if np.random.uniform() > keepFraction: continue
numKept += 1
if numKept > maxKeep: break
line = line.split(',')
x = float(line[0])
y = float(line[1])
z = float(line[2])
vz = float(line[3])
numKept = 0
for line in inFile:
if np.random.uniform() > keepFraction: continue
numKept += 1
if numKept > maxKeep: break
line = line.split(',')
x = float(line[0])
y = float(line[1])
z = float(line[2])
vz = float(line[3])
outFile.write("%e %e %e %e\n" %(x,y,z,vz))
outFile.write("%e %e %e %e\n" %(x,y,z,vz))
outFile.write("-99 -99 -99 -99\n")
inFile.close()
outFile.close()
elif dataFormat == "random":
sampleName = "ran.ss"+str(thisSubSample)+"_z"+redshift
outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
outFile.write("%f\n" %(lbox))
outFile.write("%s\n" %(omegaM))
outFile.write("%s\n" %(hubble))
outFile.write("%s\n" %(redshift))
outFile.write("%d\n" %(maxKeep))
for i in xrange(int(maxKeep)):
x = np.random.uniform()*lbox
y = np.random.uniform()*lbox
z = np.random.uniform()*lbox
outFile.write("%e %e %e %e\n" %(x,y,z, 0.))
outFile.write("-99 -99 -99 -99\n")
outFile.close()
outFile.write("-99 -99 -99 -99\n")
print "KEEPING:", numKept, "...predicted:", maxKeep
inFile.close()
outFile.close()
# -----------------------------------------------------------------------------
# now halos
if args.script or args.all:
if (args.script or args.all) and dataFormat == "multidark":
print " Doing halo scripts"
dataFile = catalogDir+"/mdr1_halos_z"+fileNums[0]
@ -362,8 +355,8 @@ if args.halos or args.all:
outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
outFile.write("%f\n" %(lbox))
outFile.write("%s" %(omegaM))
outFile.write("%s" %(hubble))
outFile.write("%s\n" %(omegaM))
outFile.write("%s\n" %(hubble))
outFile.write("%s\n" %(redshift))
outFile.write("%d\n" %(numPart))
@ -425,7 +418,7 @@ BOX_SIZE {boxSize}
root_filename hod
"""
if args.script or args.all:
if (args.script or args.all) and dataFormat == "multidark":
print " Doing DR7 HOD scripts"
if dataFormat == "multidark":
setName = prefix+"hod_dr72dim2"
@ -467,16 +460,16 @@ if args.hod or args.all:
# -----------------------------------------------------------------------------
# now the BOSS HOD
if args.script or args.all:
if (args.script or args.all) and dataFormat == "multidark":
print " Doing DR9 HOD scripts"
if dataFormat == "multidark":
setName = prefix+"hod_dr9mid"
writeScript(setName, "md.hod_dr9mid_z",
scriptDir, catalogDir, fileNums, redshifts,
numSubvolumes, numSlices, False, lbox, 5, omegaM)
writeScript(prefix, "md.hod_dr9mid_z",
numSubvolumes, numSlices, False, lbox, 15, omegaM)
writeScript(setName, "md.hod_dr9mid_z",
scriptDir, catalogDir, fileNums, redshifts,
numSubvolumes, numSlices, True, lbox, 5, omegaM)
numSubvolumes, numSlices, True, lbox, 15, omegaM)
if args.hod or args.all:
print " Doing DR9 HOD"

View file

@ -1,246 +0,0 @@
#!/usr/bin/env python
# script which prepares inputs for gadget-based void run
import numpy as np
import os
import sys
import void_python_tools as vp
import argparse
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# CONFIGURATION
# directory for the input simulation/observational particle files
catalogDir = os.getenv("HOME")+"/workspace/Voids/catalogs/gadget/"
# where to put the final void catalog, figures, and output logs
voidOutputDir = os.getenv("HOME")+"/workspace/Voids/gadget/"
figDir = os.getenv("PWD")+"/../figs/gadget/"
logDir = os.getenv("PWD")+"/../logs/gadget/"
# where to place the pipeline scripts
scriptDir = os.getenv("PWD")+"/gadget/"
# simulation or observation?
dataType = "simulation"
# available formats for simulation: gadget, multidark
dataFormat = "gadget"
dataUnit = 1 # as multiple of Mpc/h
# place particles on the lightcone?
useLightCone = False
# common filename of particle files
particleFileBase = "mdr1_particles_z"
# list of file numbers for the particle files
# to get particle file name, we take particleFileBase+fileNum
fileNums = (("0.0", "0.53", "1.0"))
# redshift of each file in the above list
redshifts = (("0.0", "0.53", "1.0"))
# how many independent slices along the z-axis?
numSlices = 4
# how many subdivisions along the x- and y- axis?
# ( = 2 will make 4 subvolumes for each slice, = 3 will make 9, etc.)
numSubvolumes = 1
# prefix to give all outputs
prefix = "gadget_"
# list of desired subsamples
subSamples = [ 1.0, 0.1 ]
# simulation information
numPart = 1024*1024*1024
lbox = 1000 # Mpc/h
omegaM = 0.27
hubble = 0.70
# END CONFIGURATION
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
#------------------------------------------------------------------------------
LIGHT_SPEED = 299792.458
def getSampleName(setName, redshift, useVel, iSlice=-1, iVol=-1):
sampleName = setName
if useVel: setName += "_pv"
if iSlice != -1: sampleName += "_s" + str(iSlice)
if iVol != -1: sampleName += "_d" + iVol
return sampleName
#------------------------------------------------------------------------------
# for given dataset parameters, outputs a script for use with analyzeVoids
def writeScript(setName, dataFileNameBase,
scriptDir, catalogDir, fileNums, redshifts, numSubvolumes,
numSlices, useVel, lbox, minRadius, omegaM, subsample=1.0,
suffix=".dat"):
if useVel: setName += "_pv"
scriptFileName = scriptDir + "/" + setName + ".py"
scriptFile = open(scriptFileName, 'w')
scriptFile.write("""#!/usr/bin/env/python
import os
from void_python_tools.backend.classes import *
continueRun = False
startCatalogStage = 1
endCatalogStage = 4
startAPStage = 1
endAPStage = 7
ZOBOV_PATH = os.getenv("PWD")+"/../zobov/"
CTOOLS_PATH = os.getenv("PWD")+"/../c_tools/"
freshStack = True
errorBars = "CALCULATED"
numIncoherentRuns = 100
ranSeed = 101010
useLCDM = False
bias = 1.16
dataPortions = ["central"]
dataSampleList = []
""")
dataInfo = """
setName = "{setName}"
workDir = "{voidOutputDir}/{setName}/"
inputDataDir = "{inputDataDir}"
figDir = "{figDir}/{setName}/"
logDir = "{logDir}/{setName}/"
"""
scriptFile.write(dataInfo.format(setName=setName, figDir=figDir,
logDir=logDir, voidOutputDir=voidOutputDir,
inputDataDir=catalogDir))
sampleInfo = """
newSample = Sample(dataFile = "{dataFile}",
dataFormat = "{dataFormat}",
dataUnit = {dataUnit},
fullName = "{sampleName}",
nickName = "{sampleName}",
dataType = "simulation",
zBoundary = ({zMin}, {zMax}),
zRange = ({zMin}, {zMax}),
zBoundaryMpc = ({zMinMpc}, {zMaxMpc}),
omegaM = {omegaM},
minVoidRadius = {minRadius},
includeInHubble = True,
partOfCombo = False,
isCombo = False,
boxLen = {boxLen},
usePecVel = {usePecVel},
numSubvolumes = {numSubvolumes},
mySubvolume = "{mySubvolume}",
numSubDivisions = 4,
useLightCone = {useLightCone},
subsample = {subsample})
dataSampleList.append(newSample)
"""
for (iFile, redshift) in enumerate(redshifts):
fileNum = fileNums[iFile]
zBox = float(redshift)
Om = float(omegaM)
zBoxMpc = LIGHT_SPEED/100.*vp.angularDiameter(zBox, Om=Om)
boxMaxMpc = zBoxMpc + lbox
# converter from redshift to comoving distance
zVsDY = np.linspace(0., zBox+8*lbox*100./LIGHT_SPEED, 10000)
zVsDX = np.zeros(len(zVsDY))
for i in xrange(len(zVsDY)):
zVsDX[i] = vp.angularDiameter(zVsDY[i], Om=Om)
if useLightCone:
boxWidthZ = np.interp(vp.angularDiameter(zBox,Om=Om)+100. / \
LIGHT_SPEED*lbox, zVsDX, zVsDY)-zBox
dzSafe = 0.03
else:
boxWidthZ = lbox*100./LIGHT_SPEED
dzSafe = 0.0
for iSlice in xrange(numSlices):
sliceMin = zBox + dzSafe + iSlice*(boxWidthZ-dzSafe)/numSlices
sliceMax = zBox + dzSafe + (iSlice+1)*(boxWidthZ-dzSafe)/numSlices
sliceMinMpc = sliceMin*LIGHT_SPEED/100.
sliceMaxMpc = sliceMax*LIGHT_SPEED/100.
sliceMin = "%0.2f" % sliceMin
sliceMax = "%0.2f" % sliceMax
sliceMinMpc = "%0.1f" % sliceMinMpc
sliceMaxMpc = "%0.1f" % sliceMaxMpc
dataFileName = dataFileNameBase + fileNum + suffix
for iX in xrange(numSubvolumes):
for iY in xrange(numSubvolumes):
mySubvolume = "%d%d" % (iX, iY)
sampleName = getSampleName(setName, redshift, useVel,
iSlice=iSlice, iVol=mySubvolume)
scriptFile.write(sampleInfo.format(dataFile=dataFileName,
dataFormat=dataFormat,
dataUnit=dataUnit,
sampleName=sampleName,
zMin=sliceMin,
zMax=sliceMax,
zMinMpc=sliceMinMpc,
zMaxMpc=sliceMaxMpc,
omegaM=Om,
boxLen=lbox,
usePecVel=useVel,
minRadius=minRadius,
numSubvolumes=numSubvolumes,
mySubvolume=mySubvolume,
useLightCone=useLightCone,
subsample=subsample))
scriptFile.close()
return
#------------------------------------------------------------------------------
#------------------------------------------------------------------------------
if not os.access(scriptDir, os.F_OK): os.mkdir(scriptDir)
#------------------------------------------------------------------------------
# first the directly downsampled runs
# Note: ss0.002 ~ SDSS DR7 dim2
# ss0.0004 ~ SDSS DR9 mid
baseResolution = float(numPart)/lbox/lbox/lbox # particles/Mpc^3
for thisSubSample in subSamples:
keepFraction = float(thisSubSample) / baseResolution
maxKeep = keepFraction * numPart
minRadius = int(np.ceil(lbox/maxKeep**(1./3)))
print " Doing subsample", thisSubSample, " scripts"
setName = prefix+"ss"+str(thisSubSample)
writeScript(setName, particleFileBase, scriptDir, catalogDir, fileNums,
redshifts,
numSubvolumes, numSlices, True, lbox, minRadius, omegaM,
subsample=thisSubSample, suffix="")
writeScript(setName, particleFileBase, scriptDir, catalogDir, fileNums,
redshifts,
numSubvolumes, numSlices, False, lbox, minRadius, omegaM,
subsample=thisSubSample, suffix="")

View file

@ -106,7 +106,7 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
else:
useLightConeString = ""
if sample.dataFormat == "multidark":
if sample.dataFormat == "multidark" or sample.dataFormat == "random":
dataFileLine = "multidark " + datafile
elif sample.dataFormat == "gadget":
dataFileLine = "gadget " + datafile
@ -197,8 +197,10 @@ def launchZobov(sample, binPath, zobovDir=None, logDir=None, continueRun=None):
if os.access(zobovDir+"/voidDesc_"+sampleName+".out", os.F_OK):
os.unlink(zobovDir+"/voidDesc_"+sampleName+".out")
cmd = "%s/vozinit %s 0.1 1.0 2 %s %g %s %s %s >& %s" % \
(binPath, datafile, \
numThreads = 2
cmd = "%s/vozinit %s 0.1 1.0 %g %s %g %s %s %s >& %s" % \
(binPath, datafile, numThreads, \
"_"+sampleName, sample.numSubDivisions, \
binPath, zobovDir, maskIndex, logFile)
os.system(cmd)