diff --git a/pipeline/generateCatalog.py b/pipeline/generateCatalog.py index 55a05ce..cd8408e 100755 --- a/pipeline/generateCatalog.py +++ b/pipeline/generateCatalog.py @@ -113,10 +113,10 @@ if (startCatalogStage <= 4) and (endCatalogStage >= 4): for thisDataPortion in dataPortions: plotRedshiftDistribution(workDir, dataSampleList, figDir, showPlot=False, - dataPortion=thisDataPortion, setName=catalogName) + dataPortion=thisDataPortion, setName=setName) plotSizeDistribution(workDir, dataSampleList, figDir, showPlot=False, - dataPortion=thisDataPortion, setName=catalogName) + dataPortion=thisDataPortion, setName=setName) plotNumberDistribution(workDir, dataSampleList, figDir, showPlot=False, - dataPortion=thisDataPortion, setName=catalogName) + dataPortion=thisDataPortion, setName=setName) print "\n Done!" diff --git a/pipeline/prepareCatalogs.py b/pipeline/prepareCatalogs.py index 451d2d4..d23d36e 100755 --- a/pipeline/prepareCatalogs.py +++ b/pipeline/prepareCatalogs.py @@ -9,66 +9,17 @@ import os import sys import void_python_tools as vp import argparse +import imp -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# CONFIGURATION +# ------------------------------------------------------------------------------ -# directory for the input simulation/observational particle files -catalogDir = os.getenv("HOME")+"/workspace/Voids/catalogs/multidark/" +def my_import(name): + mod = __import__(name) + components = name.split('.') + for comp in components[1:]: + mod = getattr(mod, comp) + return mod -# path to HOD code -hodPath = os.getenv("HOME")+"/projects/Voids/hod/HOD.x" - -# where to put the final void catalog, figures, and output logs -voidOutputDir = os.getenv("HOME")+"/workspace/Voids/multidark/" -figDir = os.getenv("PWD")+"/../figs/multidark/" -logDir = os.getenv("PWD")+"/../logs/multidark/" - -# where to place the pipeline scripts -scriptDir = os.getenv("PWD")+"/multidark/" - -# simulation or observation? -dataType = "simulation" - -# available formats for simulation: gadget, multidark -dataFormat = "multidark" -dataUnit = 1 # as multiple of Mpc/h - -# place particles on the lightcone? -useLightCone = True - -# common filename of particle files -particleFileBase = "mdr1_particles_z" - -# list of file numbers for the particle files -# to get particle file name, we take particleFileBase+fileNum -fileNums = (("0.0", "0.53", "1.0")) - -# redshift of each file in the above list -redshifts = (("0.0", "0.53", "1.0")) - -# how many independent slices along the z-axis? -numSlices = 4 - -# how many subdivisions along the x- and y- axis? -# ( = 2 will make 4 subvolumes for each slice, = 3 will make 9, etc.) -numSubvolumes = 1 - -# prefix to give all outputs -prefix = "md_" - -# list of desired subsamples -subSamples = ((0.1, 0.05, 0.01, 0.002, 0.001, 0.0004, 0.0002)) - -# simulation information -numPart = 1024*1024*1024 -lbox = 1000 # Mpc/h -omegaM = 0.27 -hubble = 0.70 - -# END CONFIGURATION -# ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- LIGHT_SPEED = 299792.458 @@ -85,12 +36,25 @@ parser.add_argument('--halos', dest='halos', action='store_const', help='write halos') parser.add_argument('--hod', dest='hod', action='store_const', const=True, default=False, - help='write hos') + help='write hod') parser.add_argument('--all', dest='all', action='store_const', const=True, default=False, help='write everything') +parser.add_argument('--parmFile', dest='parmFile', + default="", + help='path to parameter file') args = parser.parse_args() + +filename = args.parmFile +print " Loading parameters from", filename +if not os.access(filename, os.F_OK): + print " Cannot find parameter file %s!" % filename + exit(-1) +parms = imp.load_source("name", filename) +globals().update(vars(parms)) + + #------------------------------------------------------------------------------ def getSampleName(setName, redshift, useVel, iSlice=-1, iVol=-1): @@ -253,6 +217,7 @@ newSample.addStack({zMin}, {zMax}, {minRadius}+18, {minRadius}+24, True, False) #------------------------------------------------------------------------------ #------------------------------------------------------------------------------ if not os.access(scriptDir, os.F_OK): os.mkdir(scriptDir) +if not os.access(catalogDir, os.F_OK): os.mkdir(catalogDir) #------------------------------------------------------------------------------ # first the directly downsampled runs @@ -288,45 +253,73 @@ for thisSubSample in subSamples: redshifts, numSubvolumes, numSlices, True, lbox, minRadius, omegaM, subsample=thisSubSample, suffix="") + elif dataFormat == "random": + writeScript(setName, "ran.ss"+str(thisSubSample)+"_z", + scriptDir, catalogDir, fileNums, + redshifts, + numSubvolumes, numSlices, False, lbox, minRadius, omegaM, + subsample=1.0) if args.subsample or args.all: print " Doing subsample", thisSubSample for (iRedshift, redshift) in enumerate(redshifts): print " redshift", redshift - dataFile = catalogDir+"/"+particleFileBase+fileNums[iRedshift] - inFile = open(dataFile, 'r') + + if dataFormat == "multidark": + dataFile = catalogDir+"/"+particleFileBase+fileNums[iRedshift] + inFile = open(dataFile, 'r') - sampleName = "md.ss"+str(thisSubSample)+"_z"+redshift - outFile = open(catalogDir+"/"+sampleName+".dat", 'w') + sampleName = "md.ss"+str(thisSubSample)+"_z"+redshift + outFile = open(catalogDir+"/"+sampleName+".dat", 'w') - outFile.write("%f\n" %(lbox)) - outFile.write("%s" %(omegaM)) - outFile.write("%s" %(hubble)) - outFile.write("%s\n" %(redshift)) - outFile.write("%d\n" %(maxKeep)) + outFile.write("%f\n" %(lbox)) + outFile.write("%s\n" %(omegaM)) + outFile.write("%s\n" %(hubble)) + outFile.write("%s\n" %(redshift)) + outFile.write("%d\n" %(maxKeep)) - numKept = 0 - for line in inFile: - if np.random.uniform() > keepFraction: continue - numKept += 1 - if numKept > maxKeep: break - line = line.split(',') - x = float(line[0]) - y = float(line[1]) - z = float(line[2]) - vz = float(line[3]) + numKept = 0 + for line in inFile: + if np.random.uniform() > keepFraction: continue + numKept += 1 + if numKept > maxKeep: break + line = line.split(',') + x = float(line[0]) + y = float(line[1]) + z = float(line[2]) + vz = float(line[3]) - outFile.write("%e %e %e %e\n" %(x,y,z,vz)) + outFile.write("%e %e %e %e\n" %(x,y,z,vz)) + + outFile.write("-99 -99 -99 -99\n") + inFile.close() + outFile.close() + + elif dataFormat == "random": + sampleName = "ran.ss"+str(thisSubSample)+"_z"+redshift + outFile = open(catalogDir+"/"+sampleName+".dat", 'w') + + outFile.write("%f\n" %(lbox)) + outFile.write("%s\n" %(omegaM)) + outFile.write("%s\n" %(hubble)) + outFile.write("%s\n" %(redshift)) + outFile.write("%d\n" %(maxKeep)) + + for i in xrange(int(maxKeep)): + x = np.random.uniform()*lbox + y = np.random.uniform()*lbox + z = np.random.uniform()*lbox + + outFile.write("%e %e %e %e\n" %(x,y,z, 0.)) + + outFile.write("-99 -99 -99 -99\n") + outFile.close() - outFile.write("-99 -99 -99 -99\n") - print "KEEPING:", numKept, "...predicted:", maxKeep - inFile.close() - outFile.close() # ----------------------------------------------------------------------------- # now halos -if args.script or args.all: +if (args.script or args.all) and dataFormat == "multidark": print " Doing halo scripts" dataFile = catalogDir+"/mdr1_halos_z"+fileNums[0] @@ -362,8 +355,8 @@ if args.halos or args.all: outFile = open(catalogDir+"/"+sampleName+".dat", 'w') outFile.write("%f\n" %(lbox)) - outFile.write("%s" %(omegaM)) - outFile.write("%s" %(hubble)) + outFile.write("%s\n" %(omegaM)) + outFile.write("%s\n" %(hubble)) outFile.write("%s\n" %(redshift)) outFile.write("%d\n" %(numPart)) @@ -425,7 +418,7 @@ BOX_SIZE {boxSize} root_filename hod """ -if args.script or args.all: +if (args.script or args.all) and dataFormat == "multidark": print " Doing DR7 HOD scripts" if dataFormat == "multidark": setName = prefix+"hod_dr72dim2" @@ -467,16 +460,16 @@ if args.hod or args.all: # ----------------------------------------------------------------------------- # now the BOSS HOD -if args.script or args.all: +if (args.script or args.all) and dataFormat == "multidark": print " Doing DR9 HOD scripts" if dataFormat == "multidark": setName = prefix+"hod_dr9mid" writeScript(setName, "md.hod_dr9mid_z", scriptDir, catalogDir, fileNums, redshifts, - numSubvolumes, numSlices, False, lbox, 5, omegaM) - writeScript(prefix, "md.hod_dr9mid_z", + numSubvolumes, numSlices, False, lbox, 15, omegaM) + writeScript(setName, "md.hod_dr9mid_z", scriptDir, catalogDir, fileNums, redshifts, - numSubvolumes, numSlices, True, lbox, 5, omegaM) + numSubvolumes, numSlices, True, lbox, 15, omegaM) if args.hod or args.all: print " Doing DR9 HOD" diff --git a/pipeline/prepareGadgetCatalog.py b/pipeline/prepareGadgetCatalog.py deleted file mode 100755 index 51f67cc..0000000 --- a/pipeline/prepareGadgetCatalog.py +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env python - -# script which prepares inputs for gadget-based void run - -import numpy as np -import os -import sys -import void_python_tools as vp -import argparse - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# CONFIGURATION - -# directory for the input simulation/observational particle files -catalogDir = os.getenv("HOME")+"/workspace/Voids/catalogs/gadget/" - -# where to put the final void catalog, figures, and output logs -voidOutputDir = os.getenv("HOME")+"/workspace/Voids/gadget/" -figDir = os.getenv("PWD")+"/../figs/gadget/" -logDir = os.getenv("PWD")+"/../logs/gadget/" - -# where to place the pipeline scripts -scriptDir = os.getenv("PWD")+"/gadget/" - -# simulation or observation? -dataType = "simulation" - -# available formats for simulation: gadget, multidark -dataFormat = "gadget" -dataUnit = 1 # as multiple of Mpc/h - -# place particles on the lightcone? -useLightCone = False - -# common filename of particle files -particleFileBase = "mdr1_particles_z" - -# list of file numbers for the particle files -# to get particle file name, we take particleFileBase+fileNum -fileNums = (("0.0", "0.53", "1.0")) - -# redshift of each file in the above list -redshifts = (("0.0", "0.53", "1.0")) - -# how many independent slices along the z-axis? -numSlices = 4 - -# how many subdivisions along the x- and y- axis? -# ( = 2 will make 4 subvolumes for each slice, = 3 will make 9, etc.) -numSubvolumes = 1 - -# prefix to give all outputs -prefix = "gadget_" - -# list of desired subsamples -subSamples = [ 1.0, 0.1 ] - -# simulation information -numPart = 1024*1024*1024 -lbox = 1000 # Mpc/h -omegaM = 0.27 -hubble = 0.70 - -# END CONFIGURATION -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - -#------------------------------------------------------------------------------ -LIGHT_SPEED = 299792.458 - -def getSampleName(setName, redshift, useVel, iSlice=-1, iVol=-1): - - sampleName = setName - - if useVel: setName += "_pv" - - if iSlice != -1: sampleName += "_s" + str(iSlice) - - if iVol != -1: sampleName += "_d" + iVol - - return sampleName - -#------------------------------------------------------------------------------ -# for given dataset parameters, outputs a script for use with analyzeVoids -def writeScript(setName, dataFileNameBase, - scriptDir, catalogDir, fileNums, redshifts, numSubvolumes, - numSlices, useVel, lbox, minRadius, omegaM, subsample=1.0, - suffix=".dat"): - - if useVel: setName += "_pv" - - scriptFileName = scriptDir + "/" + setName + ".py" - scriptFile = open(scriptFileName, 'w') - - scriptFile.write("""#!/usr/bin/env/python -import os -from void_python_tools.backend.classes import * - -continueRun = False -startCatalogStage = 1 -endCatalogStage = 4 - -startAPStage = 1 -endAPStage = 7 - -ZOBOV_PATH = os.getenv("PWD")+"/../zobov/" -CTOOLS_PATH = os.getenv("PWD")+"/../c_tools/" -freshStack = True -errorBars = "CALCULATED" -numIncoherentRuns = 100 -ranSeed = 101010 -useLCDM = False -bias = 1.16 - -dataPortions = ["central"] -dataSampleList = [] -""") - - dataInfo = """ -setName = "{setName}" - -workDir = "{voidOutputDir}/{setName}/" -inputDataDir = "{inputDataDir}" -figDir = "{figDir}/{setName}/" -logDir = "{logDir}/{setName}/" - """ - scriptFile.write(dataInfo.format(setName=setName, figDir=figDir, - logDir=logDir, voidOutputDir=voidOutputDir, - inputDataDir=catalogDir)) - - sampleInfo = """ -newSample = Sample(dataFile = "{dataFile}", - dataFormat = "{dataFormat}", - dataUnit = {dataUnit}, - fullName = "{sampleName}", - nickName = "{sampleName}", - dataType = "simulation", - zBoundary = ({zMin}, {zMax}), - zRange = ({zMin}, {zMax}), - zBoundaryMpc = ({zMinMpc}, {zMaxMpc}), - omegaM = {omegaM}, - minVoidRadius = {minRadius}, - includeInHubble = True, - partOfCombo = False, - isCombo = False, - boxLen = {boxLen}, - usePecVel = {usePecVel}, - numSubvolumes = {numSubvolumes}, - mySubvolume = "{mySubvolume}", - numSubDivisions = 4, - useLightCone = {useLightCone}, - subsample = {subsample}) -dataSampleList.append(newSample) - """ - - for (iFile, redshift) in enumerate(redshifts): - fileNum = fileNums[iFile] - zBox = float(redshift) - Om = float(omegaM) - zBoxMpc = LIGHT_SPEED/100.*vp.angularDiameter(zBox, Om=Om) - boxMaxMpc = zBoxMpc + lbox - - # converter from redshift to comoving distance - zVsDY = np.linspace(0., zBox+8*lbox*100./LIGHT_SPEED, 10000) - zVsDX = np.zeros(len(zVsDY)) - for i in xrange(len(zVsDY)): - zVsDX[i] = vp.angularDiameter(zVsDY[i], Om=Om) - - if useLightCone: - boxWidthZ = np.interp(vp.angularDiameter(zBox,Om=Om)+100. / \ - LIGHT_SPEED*lbox, zVsDX, zVsDY)-zBox - dzSafe = 0.03 - else: - boxWidthZ = lbox*100./LIGHT_SPEED - dzSafe = 0.0 - - for iSlice in xrange(numSlices): - sliceMin = zBox + dzSafe + iSlice*(boxWidthZ-dzSafe)/numSlices - sliceMax = zBox + dzSafe + (iSlice+1)*(boxWidthZ-dzSafe)/numSlices - - sliceMinMpc = sliceMin*LIGHT_SPEED/100. - sliceMaxMpc = sliceMax*LIGHT_SPEED/100. - - sliceMin = "%0.2f" % sliceMin - sliceMax = "%0.2f" % sliceMax - sliceMinMpc = "%0.1f" % sliceMinMpc - sliceMaxMpc = "%0.1f" % sliceMaxMpc - - dataFileName = dataFileNameBase + fileNum + suffix - - for iX in xrange(numSubvolumes): - for iY in xrange(numSubvolumes): - - mySubvolume = "%d%d" % (iX, iY) - - sampleName = getSampleName(setName, redshift, useVel, - iSlice=iSlice, iVol=mySubvolume) - - scriptFile.write(sampleInfo.format(dataFile=dataFileName, - dataFormat=dataFormat, - dataUnit=dataUnit, - sampleName=sampleName, - zMin=sliceMin, - zMax=sliceMax, - zMinMpc=sliceMinMpc, - zMaxMpc=sliceMaxMpc, - omegaM=Om, - boxLen=lbox, - usePecVel=useVel, - minRadius=minRadius, - numSubvolumes=numSubvolumes, - mySubvolume=mySubvolume, - useLightCone=useLightCone, - subsample=subsample)) - - scriptFile.close() - return - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -if not os.access(scriptDir, os.F_OK): os.mkdir(scriptDir) - -#------------------------------------------------------------------------------ -# first the directly downsampled runs -# Note: ss0.002 ~ SDSS DR7 dim2 -# ss0.0004 ~ SDSS DR9 mid -baseResolution = float(numPart)/lbox/lbox/lbox # particles/Mpc^3 -for thisSubSample in subSamples: - - keepFraction = float(thisSubSample) / baseResolution - maxKeep = keepFraction * numPart - minRadius = int(np.ceil(lbox/maxKeep**(1./3))) - - print " Doing subsample", thisSubSample, " scripts" - setName = prefix+"ss"+str(thisSubSample) - writeScript(setName, particleFileBase, scriptDir, catalogDir, fileNums, - redshifts, - numSubvolumes, numSlices, True, lbox, minRadius, omegaM, - subsample=thisSubSample, suffix="") - writeScript(setName, particleFileBase, scriptDir, catalogDir, fileNums, - redshifts, - numSubvolumes, numSlices, False, lbox, minRadius, omegaM, - subsample=thisSubSample, suffix="") - diff --git a/python_tools/void_python_tools/backend/launchers.py b/python_tools/void_python_tools/backend/launchers.py index 7237408..33ef904 100755 --- a/python_tools/void_python_tools/backend/launchers.py +++ b/python_tools/void_python_tools/backend/launchers.py @@ -106,7 +106,7 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None, else: useLightConeString = "" - if sample.dataFormat == "multidark": + if sample.dataFormat == "multidark" or sample.dataFormat == "random": dataFileLine = "multidark " + datafile elif sample.dataFormat == "gadget": dataFileLine = "gadget " + datafile @@ -197,8 +197,10 @@ def launchZobov(sample, binPath, zobovDir=None, logDir=None, continueRun=None): if os.access(zobovDir+"/voidDesc_"+sampleName+".out", os.F_OK): os.unlink(zobovDir+"/voidDesc_"+sampleName+".out") - cmd = "%s/vozinit %s 0.1 1.0 2 %s %g %s %s %s >& %s" % \ - (binPath, datafile, \ + numThreads = 2 + + cmd = "%s/vozinit %s 0.1 1.0 %g %s %g %s %s %s >& %s" % \ + (binPath, datafile, numThreads, \ "_"+sampleName, sample.numSubDivisions, \ binPath, zobovDir, maskIndex, logFile) os.system(cmd)