some bug fixes to the a-p analysis scripts; fixed major bug in prunevoids; updated catalog release scripts

This commit is contained in:
P.M. Sutter 2012-11-29 11:14:59 -06:00
parent 06c8ddc26e
commit 022eec19bb
13 changed files with 437 additions and 239 deletions

View file

@ -1,8 +1,6 @@
#!/usr/bin/env python
# prepares input catalogs based on multidark simulations
# (borrows heavily from generateMock, but doesn't hold much in memory)
# also creates necessary analyzeVoids input files
# applies a mask to a given dataset
import numpy as np
import os
@ -32,6 +30,18 @@ parser.add_argument('--scripts', dest='script', action='store_const',
parser.add_argument('--parmFile', dest='parmFile',
default="",
help='path to parameter file')
parser.add_argument('--subsamples', dest='subsample', action='store_const',
const=True, default=False,
help='write subsamples')
parser.add_argument('--halos', dest='halos', action='store_const',
const=True, default=False,
help='write halos')
parser.add_argument('--hod', dest='hod', action='store_const',
const=True, default=False,
help='write hod')
parser.add_argument('--all', dest='all', action='store_const',
const=True, default=False,
help='write everything')
args = parser.parse_args()
@ -57,9 +67,9 @@ def getSampleName(setName, redshift, useVel, iSlice=-1, iVol=-1):
#------------------------------------------------------------------------------
# for given dataset parameters, outputs a script for use with analyzeVoids
def writeScript(setName, dataFileNameBase,
scriptDir, catalogDir, fileNums, redshifts, numSubvolumes,
numSlices, useVel, lbox, minRadius, omegaM, subsample=1.0,
def writeObservationScript(setName, dataFileNameBase, maskFileName,
scriptDir, catalogDir, fileNums, redshifts,
useVel, minRadius, omegaM,
suffix=".dat"):
@ -88,7 +98,7 @@ ranSeed = 101010
useLCDM = False
bias = 1.16
dataPortions = ["central"]
dataPortions = ["central", "all"]
dataSampleList = []
""")
@ -112,25 +122,17 @@ numZobovThreads = {numZobovThreads}
sampleInfo = """
newSample = Sample(dataFile = "{dataFile}",
dataFormat = "{dataFormat}",
dataUnit = {dataUnit},
fullName = "{sampleName}",
nickName = "{sampleName}",
dataType = "simulation",
dataType = "observation",
maskFile = "{maskFileName}",
zBoundary = ({zMin}, {zMax}),
zRange = ({zMin}, {zMax}),
zBoundaryMpc = ({zMinMpc}, {zMaxMpc}),
omegaM = {omegaM},
minVoidRadius = {minRadius},
includeInHubble = True,
partOfCombo = False,
isCombo = False,
boxLen = {boxLen},
usePecVel = {usePecVel},
numSubvolumes = {numSubvolumes},
mySubvolume = "{mySubvolume}",
useLightCone = {useLightCone},
subsample = {subsample})
usePecVel = {usePecVel})
dataSampleList.append(newSample)
newSample.addStack({zMin}, {zMax}, {minRadius} , {minRadius}+2, True, False)
newSample.addStack({zMin}, {zMax}, {minRadius} , {minRadius}+4, True, False)
@ -142,180 +144,249 @@ newSample.addStack({zMin}, {zMax}, {minRadius}+18, {minRadius}+24, True, False)
for (iFile, redshift) in enumerate(redshifts):
fileNum = fileNums[iFile]
zBox = float(redshift)
Om = float(omegaM)
zBoxMpc = LIGHT_SPEED/100.*vp.angularDiameter(zBox, Om=Om)
boxMaxMpc = zBoxMpc + lbox
zMin = zBox
zMax = zBox + 0.1
zMin, zMax = getMaskedZRange(lbox, zBox, zMin, zMax, omegaM)
dataFileName = dataFileNameBase + fileNum + suffix
# converter from redshift to comoving distance
zVsDY = np.linspace(0., zBox+8*lbox*100./LIGHT_SPEED, 10000)
zVsDX = np.zeros(len(zVsDY))
for i in xrange(len(zVsDY)):
zVsDX[i] = vp.angularDiameter(zVsDY[i], Om=Om)
sampleName = getSampleName(setName, redshift, useVel,
iSlice=-1, iVol=-1)
if useLightCone:
boxWidthZ = np.interp(vp.angularDiameter(zBox,Om=Om)+100. / \
LIGHT_SPEED*lbox, zVsDX, zVsDY)-zBox
dzSafe = 0.03
else:
boxWidthZ = lbox*100./LIGHT_SPEED
dzSafe = 0.0
for iSlice in xrange(numSlices):
sliceMin = zBox + dzSafe + iSlice*(boxWidthZ-dzSafe)/numSlices
sliceMax = zBox + dzSafe + (iSlice+1)*(boxWidthZ-dzSafe)/numSlices
sliceMinMpc = sliceMin*LIGHT_SPEED/100.
sliceMaxMpc = sliceMax*LIGHT_SPEED/100.
sliceMin = "%0.2f" % sliceMin
sliceMax = "%0.2f" % sliceMax
sliceMinMpc = "%0.1f" % sliceMinMpc
sliceMaxMpc = "%0.1f" % sliceMaxMpc
dataFileName = dataFileNameBase + fileNum + suffix
for iX in xrange(numSubvolumes):
for iY in xrange(numSubvolumes):
mySubvolume = "%d%d" % (iX, iY)
sampleName = getSampleName(setName, sliceMin, useVel,
iSlice=iSlice, iVol=mySubvolume)
scriptFile.write(sampleInfo.format(dataFile=dataFileName,
scriptFile.write(sampleInfo.format(dataFile=dataFileName,
dataFormat=dataFormat,
dataUnit=dataUnit,
sampleName=sampleName,
zMin=sliceMin,
zMax=sliceMax,
zMinMpc=sliceMinMpc,
zMaxMpc=sliceMaxMpc,
omegaM=Om,
boxLen=lbox,
maskFileName=maskFileName,
zMin=zMin,
zMax=zMax,
usePecVel=useVel,
minRadius=minRadius,
numSubvolumes=numSubvolumes,
mySubvolume=mySubvolume,
useLightCone=useLightCone,
subsample=subsample))
minRadius=minRadius))
scriptFile.close()
return
#------------------------------------------------------------------------------
#------------------------------------------------------------------------------
if not os.access(scriptDir, os.F_OK): os.mkdir(scriptDir)
if not os.access(catalogDir, os.F_OK): os.mkdir(catalogDir)
# -----------------------------------------------------------------------------
# now the SDSS HOD
parFileText = """
% cosmology
OMEGA_M {omegaM}
HUBBLE {hubble}
OMEGA_B 0.0469
SIGMA_8 0.82
SPECTRAL_INDX 0.95
ITRANS 5
REDSHIFT {redshift}
% halo definition
%DELTA_HALO 200
DELTA_HALO 740.74 % 200/Om_m
M_max 1.00E+16
% fit function types
pdfs 11
pdfc 2
EXCLUSION 4
% hod parameters
M_min {Mmin}
GALAXY_DENSITY 0.0111134 % computed automatically if M_min set, use for sanity
M1 {M1}
sigma_logM {sigma_logM}
alpha {alpha}
M_cut {Mcut}
% simulation info
real_space_xi 1
HOD 1
populate_sim 1
HaloFile {haloFile}
RESOLUTION {numPartPerSide}
BOX_SIZE {boxSize}
% output
root_filename hod
"""
print " Doing DR9 HOD"
# these parameters come from Manera et al 2012, eq. 26
parFileName = "./hod.par"
parFile = open(parFileName, 'w')
haloFile = catalogDir+haloFileBase+fileNums[iRedshift]
parFile.write(parFileText.format(omegaM=omegaM,
hubble=hubble,
redshift=redshift,
Mmin=1.23e13,
M1=1.e14,
sigma_logM=0.596,
alpha=1.0127,
Mcut=1.19399e13,
haloFile=haloFile,
numPartPerSide=numPart**(1/3.),
boxSize=lbox))
parFile.close()
os.system(hodPath+" "+parFileName+">& /dev/null")
# now place these particles on a lightcone, restrict redshift range, apply mask
mask = hp.read_map(maskFile)
nside = hp.get_nside(mask)
def applyMask(inFileName, outFileName, maskFileName, lbox, zBox, zMin, zMax, omegaM):
mask = hp.read_map(maskFileName)
nside = hp.get_nside(mask)
inFile = open('hod.mock', 'r')
outFile = open(catalogDir+"/mock.out"))
inFile = open(inFileName, 'r')
outFile = open(outFileName, 'w')
zBox = float(redshiftRange[0])
Om = float(omegaM)
Om = float(omegaM)
# converter from redshift to comoving distance
zVsDY = np.linspace(0., zBox+8*lbox*100./LIGHT_SPEED, 10000)
zVsDX = np.zeros(len(zVsDY))
for i in xrange(len(zVsDY)):
zVsDX[i] = vp.angularDiameter(zVsDY[i], Om=Om)
for line in inFile:
line = line.split(',')
x = float(line[0]) - lbox/2.
y = float(line[1]) - lbox/2.
z = float(line[2]) - lbox/2.
vz = float(line[5])
# converter from redshift to comoving distance
zVsDY = np.linspace(0., zBox+8*lbox*100./LIGHT_SPEED, 10000)
zVsDX = np.zeros(len(zVsDY))
for i in xrange(len(zVsDY)):
zVsDX[i] = vp.angularDiameter(zVsDY[i], Om=Om)
zBoxInMpc = vp.angularDiameter(zBox, Om=Om)
redshift = np.sqrt(x*x + y*y + z*z)
redshift = np.interp(zBoxInMpc+100./LIGHT_SPEED*redshift, zVsDX, zVsDY)
for (iLine,line) in enumerate(inFile):
if iLine < 5:
print >> outFile, line.rstrip()
continue
if redshift < redshiftRange[0] or redshift > redshiftRange[1]: continue
line = line.split(' ')
uniqueID = int(line[0])
x = float(line[1]) - lbox/2.
y = float(line[2]) - lbox/2.
z = float(line[3]) - lbox/2.
vz = float(line[4])
RA = np.atan((y-lbox/2.)/(x-lbox/2.)) * 100/np.pi + 180.
Dec = np.asin((z-lboc/2.)/(redshift*LIGHT_SPEED/100.)) * 180/np.pi
# TODO if usePecvel
phi = np.pi/180. * RA
theta = np.pi/2. - Dec*np.pi/180.
pos = np.zeros((3))
redshift = np.sqrt(x*x + y*y + z*z)
redshift = np.interp(zBoxInMpc+100./LIGHT_SPEED*redshift, zVsDX, zVsDY)
pix = hp.ang2pix(nside, theta, phi)
if mask[pix] <= 0: continue
if redshift < zMin or redshift > zMax: continue
vec = np.array((x,y,z))
theta, phi = hp.vec2ang(vec)
theta = theta[0]
phi = phi[0]
RA = phi*180./np.pi
Dec = 180./np.pi*(np.pi/2.-theta)
pix = hp.vec2pix(nside, x, y, z)
if mask[pix] <= 0.2: continue
print >> outFile, RA, Dec, redshift*LIGHT_SPEED, 0., x, y, z
print >> outFile, RA, Dec, redshift*LIGHT_SPEED, uniqueID, x, y, z
inFile.close()
outFile.close()
inFile.close()
outFile.close()
os.system("rm ./hod.*")
#------------------------------------------------------------------------------
def getMaskedZRange(lbox, zBox, zMin, zMax, omegaM):
if zMin < zBox: zMin = zBox
Om = float(omegaM)
# converter from redshift to comoving distance
zVsDY = np.linspace(0., zBox+8*lbox*100./LIGHT_SPEED, 10000)
zVsDX = np.zeros(len(zVsDY))
for i in xrange(len(zVsDY)):
zVsDX[i] = vp.angularDiameter(zVsDY[i], Om=Om)
zBoxInMpc = vp.angularDiameter(zBox, Om=Om)
boxWidthZ = np.interp(vp.angularDiameter(zBox,Om=Om)+100. / \
LIGHT_SPEED*lbox/2., zVsDX, zVsDY)-zBox
print "RANGE", zBox, zBox+boxWidthZ
if zMax > zBox+boxWidthZ: zMax = zBox+boxWidthZ
return zMin, zMax
#------------------------------------------------------------------------------
#------------------------------------------------------------------------------
inCatalogDir = catalogDir
prefix = "masked_" + prefix
voidOutputDir += "masked/"
figDir += "masked/"
logDir += "masked/"
scriptDir += "masked/"
catalogDir += "masked/"
dataType = "observation"
if not os.access(scriptDir, os.F_OK): os.mkdir(scriptDir)
if not os.access(catalogDir, os.F_OK): os.mkdir(catalogDir)
#------------------------------------------------------------------------------
# first the directly downsampled runs
# Note: ss0.002 ~ SDSS DR7 dim2
# ss0.0004 ~ SDSS DR9 mid
baseResolution = float(numPart)/lbox/lbox/lbox # particles/Mpc^3
for thisSubSample in subSamples:
keepFraction = float(thisSubSample) / baseResolution
maxKeep = keepFraction * numPart
minRadius = int(np.ceil(lbox/maxKeep**(1./3)))
if args.script or args.all:
print " Doing subsample", thisSubSample, " scripts"
setName = prefix+"ss"+str(thisSubSample)
writeObservationScript(setName, "masked_md.ss"+str(thisSubSample)+"_z",
maskFileName,
scriptDir, catalogDir, fileNums,
redshifts, False, minRadius, omegaM)
writeObservationScript(setName, "masked_md.ss"+str(thisSubSample)+"_z",
maskFileName,
scriptDir, catalogDir, fileNums,
redshifts, True, minRadius, omegaM)
if args.subsample or args.all:
print " Doing subsample", thisSubSample
for (iRedshift, redshift) in enumerate(redshifts):
print " redshift", redshift
zMin = float(redshift)
zMax = zMin + 0.1
zMin, zMax = getMaskedZRange(lbox, float(redshift), zMin, zMax, omegaM)
if dataFormat == "multidark" or dataFormat == "random":
inFileName = inCatalogDir+"/md.ss"+str(thisSubSample)+"_z"+redshift+".dat"
outFileName = catalogDir+"/masked_md.ss"+str(thisSubSample)+"_z"+redshift+".dat"
applyMask(inFileName, outFileName, maskFileName, lbox, float(redshift),
zMin, zMax, omegaM)
# -----------------------------------------------------------------------------
# now halos
if (args.script or args.all) and dataFormat == "multidark":
print " Doing halo scripts"
for minHaloMass in minHaloMasses:
if dataFormat == "multidark":
setName = prefix+"halos_min"+str(minHaloMass)
writeScript(setName, "md.halos_min"+str(minHaloMass)+"_z",
scriptDir, catalogDir, fileNums,
redshifts,
numSubvolumes, numSlices, False, lbox, minRadius, omegaM)
writeScript(setName, "md.halos_min"+str(minHaloMass)+"_z",
scriptDir, catalogDir, fileNums,
redshifts,
numSubvolumes, numSlices, True, lbox, minRadius, omegaM)
if args.halos or args.all:
print " Doing halos"
for minHaloMass in minHaloMasses:
print " min halo mass = ", minHaloMass
for (iRedshift, redshift) in enumerate(redshifts):
print " z = ", redshift
dataFile = catalogDir+haloFileBase+fileNums[iRedshift]
inFile = open(dataFile, 'r')
numPart = 0
for line in inFile:
line = line.split(',')
if minHaloMass == "none" or float(line[6]) > minHaloMass:
numPart += 1
inFile.close()
sampleName = "md.halos_min"+str(minHaloMass)+"_z"+redshift
outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
# -----------------------------------------------------------------------------
# now the SDSS HOD
if (args.script or args.all) and dataFormat == "multidark":
print " Doing DR7 HOD scripts"
if dataFormat == "multidark":
setName = prefix+"hod_dr72dim2"
writeScript(setName, "md.hod_dr72dim2_z",
scriptDir, catalogDir, fileNums, redshifts,
numSubvolumes, numSlices, False, lbox, 5, omegaM)
writeScript(setName, "md.hod_dr72dim2_z",
scriptDir, catalogDir, fileNums, redshifts,
numSubvolumes, numSlices, True, lbox, 5, omegaM)
if args.hod or args.all:
print " Doing DR7 HOD"
for (iRedshift, redshift) in enumerate(redshifts):
print " z = ", redshift
sampleName = getSampleName("md.hod_dr72dim2", redshift, False)
outFileName = catalogDir+"/"+sampleName+".dat"
# -----------------------------------------------------------------------------
# now the BOSS HOD
if (args.script or args.all) and dataFormat == "multidark":
print " Doing DR9 HOD scripts"
if dataFormat == "multidark":
setName = prefix+"hod_dr9mid"
writeScript(setName, "md.hod_dr9mid_z",
scriptDir, catalogDir, fileNums, redshifts,
numSubvolumes, numSlices, False, lbox, 15, omegaM)
writeScript(setName, "md.hod_dr9mid_z",
scriptDir, catalogDir, fileNums, redshifts,
numSubvolumes, numSlices, True, lbox, 15, omegaM)
if args.hod or args.all:
print " Doing DR9 HOD"
for (iRedshift, redshift) in enumerate(redshifts):
print " z = ", redshift
outFileName = catalogDir+"/"+sampleName+".dat"
if dataFormat == "multidark" or dataFormat == "random":
sampleName = getSampleName("md.hod_dr9mid", redshift, False)
inFileName = inCatalogDir+"/"+sampleName+".dat"
outFileName = catalogDir+"/masked_"+sampleName+".dat"
zMin = float(redshift)
zMax = zMin + 0.1
zMin, zMax = getMaskedZRange(lbox, float(redshift), zMin, zMax, omegaM)
applyMask(inFileName, outFileName, maskFileName, lbox, float(redshift),
zMin, zMax, omegaM)

View file

@ -49,7 +49,7 @@ numSubvolumes = 1
prefix = "md_"
# list of desired subsamples - these are in unts of h Mpc^-3!
#subSamples = [ 1.0 ]
#subSamples = [0.0004]
subSamples = ((0.1, 0.05, 0.01, 0.002, 0.001, 0.0004, 0.0002))
# common filename of halo files
@ -62,7 +62,7 @@ minHaloMasses = (("none", 2e12, 1.23e13))
# adjust these two parameters given the memory contraints on your system:
# numZobovDivisions: how many sub-volumes per dimension will zobov process
# numZobovThreads: how many sub-volumes to process at once?
numZobovDivisions = 4
numZobovDivisions = 2
numZobovThreads = 2
# simulation information
@ -71,6 +71,9 @@ lbox = 1000 # Mpc/h
omegaM = 0.27
hubble = 0.70
# the mask file which is used by applyMaskToMock
maskFileName = os.getenv("HOME")+"/workspace/Voids/catalogs/boss/boss_mask_final.fits"
# END CONFIGURATION
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------

View file

@ -9,13 +9,6 @@ import pickle
# ------------------------------------------------------------------------------
def my_import(name):
mod = __import__(name)
components = name.split('.')
for comp in components[1:]:
mod = getattr(mod, comp)
return mod
if (len(sys.argv) == 0):
print "Usage: ./anylyzeVoids.py parameter_file.py"
exit(-1)

View file

@ -11,15 +11,6 @@ import void_python_tools as vp
import argparse
import imp
# ------------------------------------------------------------------------------
def my_import(name):
mod = __import__(name)
components = name.split('.')
for comp in components[1:]:
mod = getattr(mod, comp)
return mod
# -----------------------------------------------------------------------------
LIGHT_SPEED = 299792.458
@ -83,7 +74,7 @@ def writeScript(setName, dataFileNameBase,
import os
from void_python_tools.backend.classes import *
continueRun = True # set to True to enable restarting aborted jobs
continueRun = False # set to True to enable restarting aborted jobs
startCatalogStage = 1
endCatalogStage = 4
@ -133,6 +124,7 @@ newSample = Sample(dataFile = "{dataFile}",
zBoundaryMpc = ({zMinMpc}, {zMaxMpc}),
omegaM = {omegaM},
minVoidRadius = {minRadius},
profileBinSize = 1.0,
includeInHubble = True,
partOfCombo = False,
isCombo = False,
@ -143,12 +135,12 @@ newSample = Sample(dataFile = "{dataFile}",
useLightCone = {useLightCone},
subsample = {subsample})
dataSampleList.append(newSample)
newSample.addStack({zMin}, {zMax}, {minRadius} , {minRadius}+2, True, False)
newSample.addStack({zMin}, {zMax}, {minRadius} , {minRadius}+4, True, False)
newSample.addStack({zMin}, {zMax}, {minRadius}+2, {minRadius}+6, True, False)
newSample.addStack({zMin}, {zMax}, {minRadius}+6, {minRadius}+10, True, False)
newSample.addStack({zMin}, {zMax}, {minRadius}+10, {minRadius}+18, True, False)
newSample.addStack({zMin}, {zMax}, {minRadius}+18, {minRadius}+24, True, False)
newSample.addStack({zMin}, {zMax}, 2*{minRadius} , 2*{minRadius}+2, True, False)
newSample.addStack({zMin}, {zMax}, 2*{minRadius} , 2*{minRadius}+4, True, False)
newSample.addStack({zMin}, {zMax}, 2*{minRadius}+2, 2*{minRadius}+6, True, False)
newSample.addStack({zMin}, {zMax}, 2*{minRadius}+6, 2*{minRadius}+10, True, False)
newSample.addStack({zMin}, {zMax}, 2*{minRadius}+10, 2*{minRadius}+18, True, False)
newSample.addStack({zMin}, {zMax}, 2*{minRadius}+18, 2*{minRadius}+24, True, False)
"""
for (iFile, redshift) in enumerate(redshifts):
fileNum = fileNums[iFile]