guilhem_lavaux/vide_public
0
0
Fork 0
mirror of https://bitbucket.org/cosmicvoids/vide_public.git synced 2025-07-04 23:31:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

can now do subsampling of sdf in prepareCatalogs

Browse source
This commit is contained in:
P.M. Sutter 2013-03-20 07:21:02 -05:00
parent 7c172d5642
commit 25a6ead259
1 changed files with 84 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

110
python_tools/pipeline_source/prepareCatalogs.in.py
View file

@ -285,24 +285,39 @@ for iSubSample in xrange(len(subSamples)):
scriptDir, catalogDir, fileNums, redshifts, scriptDir, catalogDir, fileNums, redshifts,
numSubvolumes, numSlices, False, lbox, minRadius, omegaM, numSubvolumes, numSlices, False, lbox, minRadius, omegaM,
subsample=subSampleToUse, suffix=suffix) subsample=subSampleToUse, suffix=suffix)
writeScript(setName, fileToUse, dataFormat, if doPecVel:
scriptDir, catalogDir, fileNums, redshifts, writeScript(setName, fileToUse, dataFormat,
numSubvolumes, numSlices, True, lbox, minRadius, omegaM, scriptDir, catalogDir, fileNums, redshifts,
subsample=subSampleToUse, suffix=suffix) numSubvolumes, numSlices, True, lbox, minRadius, omegaM,
subsample=subSampleToUse, suffix=suffix)
else: else:
subSampleToUse = keepFractionList if doSubSampling:
suffix = "" # prepare scripts using our own format
fileToUse = partFileList[0] dataFormatToUse = "multidark"
writeScript(setName, fileToUse, dataFormat, subSampleToUse = 1.0
fileToUse = prefix+"ss"+str(thisSubSample)+"_z"
partFileList = []
suffix = ".dat"
for (iRedshift, redshift) in enumerate(redshifts):
sampleName = getSampleName(setName, redshift, False)
outFileName = sampleName+".dat"
partFileList.append(outFileName)
else:
dataFormatToUse = dataFormat
subSampleToUse = keepFractionList
suffix = ""
fileToUse = partFileList[0]
writeScript(setName, fileToUse, dataFormatToUse,
scriptDir, catalogDir, fileNums, redshifts, scriptDir, catalogDir, fileNums, redshifts,
numSubvolumes, numSlices, False, lbox, minRadius, omegaM, numSubvolumes, numSlices, False, lbox, minRadius, omegaM,
subsample=subSampleToUse, suffix=suffix, subsample=subSampleToUse, suffix=suffix,
dataFileNameList=partFileList) dataFileNameList=partFileList)
writeScript(setName, fileToUse, dataFormat, if doPecVel:
scriptDir, catalogDir, fileNums, redshifts, writeScript(setName, fileToUse, dataFormatToUse,
numSubvolumes, numSlices, True, lbox, minRadius, omegaM, scriptDir, catalogDir, fileNums, redshifts,
subsample=subSampleToUse, suffix=suffix, numSubvolumes, numSlices, True, lbox, minRadius, omegaM,
dataFileNameList=partFileList) subsample=subSampleToUse, suffix=suffix,
dataFileNameList=partFileList)
if args.subsample or args.all: if args.subsample or args.all:
@ -313,17 +328,31 @@ for iSubSample in xrange(len(subSamples)):
print " redshift", redshift print " redshift", redshift
sys.stdout.flush() sys.stdout.flush()
if dataFormat == "multidark": if dataFormat == "multidark" or dataFormat == "sdf":
# reuse previous subamples in order to: # reuse previous subamples in order to:
# - preserve unique IDs across multiple subsamples # - preserve unique IDs across multiple subsamples
# - reuse smaller files for faster processing # - reuse smaller files for faster processing
if prevSubSample == -1: if prevSubSample == -1:
dataFile = catalogDir+"/"+particleFileBase+fileNums[iRedshift] if particleFileDummy == '':
dataFile = catalogDir+"/"+particleFileBase+fileNums[iRedshift]
else:
dataFile = particleFileBase.replace(particleFileDummy,
fileNums[iRedshift])
dataFile = catalogDir+"/"+dataFile
keepFraction = float(thisSubSample) / baseResolution
else: else:
sampleName = prefix+"ss"+str(prevSubSample)+"_z"+redshift sampleName = prefix+"ss"+str(prevSubSample)+"_z"+redshift
dataFile = catalogDir+"/"+sampleName+".dat" dataFile = catalogDir+"/"+sampleName+".dat"
keepFraction = float(thisSubSample) / float(prevSubSample) keepFraction = float(thisSubSample) / float(prevSubSample)
if prevSubSample == -1 and dataFormat == "sdf":
convertedFile = dataFile + "_temp"
SDFcvt_PATH = "@CMAKE_BINARY_DIR@/external/libsdf/apps/SDFcvt/SDFcvt.x86_64"
command = "%s %s x y z vz vy vx > %s" % (SDFcvt_PATH, dataFile,
convertedFile )
os.system(command)
dataFile = convertedFile
inFile = open(dataFile, 'r') inFile = open(dataFile, 'r')
sampleName = prefix+"ss"+str(thisSubSample)+"_z"+redshift sampleName = prefix+"ss"+str(thisSubSample)+"_z"+redshift
@ -335,6 +364,11 @@ for iSubSample in xrange(len(subSamples)):
outFile.write("%s\n" %(redshift)) outFile.write("%s\n" %(redshift))
outFile.write("%d\n" %(maxKeep)) outFile.write("%d\n" %(maxKeep))
if dataFormat == "sdf":
splitter = ' '
if dataFormat == "multidark":
splitter = ','
numKept = 0 numKept = 0
for (i,line) in enumerate(inFile): for (i,line) in enumerate(inFile):
if (prevSubSample != -1 and i < 5): continue # skip header if (prevSubSample != -1 and i < 5): continue # skip header
@ -344,7 +378,7 @@ for iSubSample in xrange(len(subSamples)):
#if numKept > maxKeep: break #if numKept > maxKeep: break
if (prevSubSample == -1): if (prevSubSample == -1):
line = line.split(',') line = line.split(splitter)
x = float(line[0]) x = float(line[0])
y = float(line[1]) y = float(line[1])
z = float(line[2]) z = float(line[2])
@ -360,6 +394,9 @@ for iSubSample in xrange(len(subSamples)):
inFile.close() inFile.close()
outFile.close() outFile.close()
if prevSubSample == -1 and dataFormat == "sdf":
os.unlink(dataFile)
elif dataFormat == "random": elif dataFormat == "random":
sampleName = "ran.ss"+str(thisSubSample)+"_z"+redshift sampleName = "ran.ss"+str(thisSubSample)+"_z"+redshift
outFile = open(catalogDir+"/"+sampleName+".dat", 'w') outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
@ -409,14 +446,25 @@ if (args.script or args.all) and haloFileBase != "":
minRadies = 10 minRadies = 10
setName = prefix+"halos_min"+str(minHaloMass) setName = prefix+"halos_min"+str(minHaloMass)
fileList = []
for (iRedshift, redshift) in enumerate(redshifts):
sampleName = getSampleName(setName, redshift, False)
outFileName = sampleName+".dat"
fileList.append(outFileName)
writeScript(setName, prefix+"halos_min"+str(minHaloMass)+"_z", "multidark", writeScript(setName, prefix+"halos_min"+str(minHaloMass)+"_z", "multidark",
scriptDir, catalogDir, fileNums, scriptDir, catalogDir, fileNums,
redshifts, redshifts,
numSubvolumes, numSlices, False, lbox, minRadius, omegaM) numSubvolumes, numSlices, False, lbox, minRadius, omegaM,
writeScript(setName, prefix+"halos_min"+str(minHaloMass)+"_z", "multidark", dataFileNameList = fileList)
scriptDir, catalogDir, fileNums,
redshifts, if doPecVel:
numSubvolumes, numSlices, True, lbox, minRadius, omegaM) writeScript(setName, prefix+"halos_min"+str(minHaloMass)+"_z",
"multidark",
scriptDir, catalogDir, fileNums,
redshifts,
numSubvolumes, numSlices, True, lbox, minRadius, omegaM,
dataFileNameList = fileList)
if (args.halos or args.all) and haloFileBase != "": if (args.halos or args.all) and haloFileBase != "":
print " Doing halos" print " Doing halos"
@ -451,7 +499,7 @@ if (args.halos or args.all) and haloFileBase != "":
numPart += 1 numPart += 1
inFile.close() inFile.close()
sampleName = prefix+"halos_min"+str(minHaloMass)+"_z"+fileNums[iRedshift] sampleName = prefix+"halos_min"+str(minHaloMass)+"_z"+redshifts[iRedshift]
outFileName = catalogDir+"/"+sampleName+".dat" outFileName = catalogDir+"/"+sampleName+".dat"
outFile = open(outFileName, 'w') outFile = open(outFileName, 'w')
outFile.write("%f\n" %(lbox)) outFile.write("%f\n" %(lbox))
@ -536,15 +584,25 @@ root_filename {workDir}/hod
if (args.script or args.all) and haloFileBase != "": if (args.script or args.all) and haloFileBase != "":
print " Doing HOD scripts" print " Doing HOD scripts"
sys.stdout.flush() sys.stdout.flush()
for thisHod in hodParmList: for thisHod in hodParmList:
fileList = []
for (iRedshift, redshift) in enumerate(redshifts):
sampleName = getSampleName(prefix+"hod_"+thisHod['name'], redshift, False)
outFileName = sampleName+".dat"
fileList.append(outFileName)
print " ", thisHod['name'] print " ", thisHod['name']
setName = prefix+"hod_"+thisHod['name'] setName = prefix+"hod_"+thisHod['name']
writeScript(setName, prefix+"hod_"+thisHod['name']+"_z", "multidark", writeScript(setName, prefix+"hod_"+thisHod['name']+"_z", "multidark",
scriptDir, catalogDir, fileNums, redshifts, scriptDir, catalogDir, fileNums, redshifts,
numSubvolumes, numSlices, False, lbox, 15, omegaM) numSubvolumes, numSlices, False, lbox, 15, omegaM,
writeScript(setName, prefix+"hod_"+thisHod['name']+"_z", "multidark", dataFileNameList = fileList)
scriptDir, catalogDir, fileNums, redshifts, if doPecVel:
numSubvolumes, numSlices, True, lbox, 15, omegaM) writeScript(setName, prefix+"hod_"+thisHod['name']+"_z", "multidark",
scriptDir, catalogDir, fileNums, redshifts,
numSubvolumes, numSlices, True, lbox, 15, omegaM,
dataFileNameList = fileList)
if (args.hod or args.all) and haloFileBase != "": if (args.hod or args.all) and haloFileBase != "":
print " Doing HOD" print " Doing HOD"