diff --git a/c_tools/analysis/voidOverlap.cpp b/c_tools/analysis/voidOverlap.cpp index f62409c..6fb3b87 100644 --- a/c_tools/analysis/voidOverlap.cpp +++ b/c_tools/analysis/voidOverlap.cpp @@ -286,7 +286,7 @@ int main(int argc, char **argv) { rdist = catalog1.voids[iVoid1].matches[0].dist; rdist /= catalog1.voids[iVoid1].radius; - fprintf(fp, "%d %.2f %.2f %.2f %.2f %.2f %d %d\n", voidID, + fprintf(fp, "%d %.4f %.4f %.4f %.4f %.4f %d %d\n", voidID, catalog1.voids[iVoid1].radiusMpc, rRatio, commonVolRatio, diff --git a/c_tools/mock/generateMock.cpp b/c_tools/mock/generateMock.cpp index c015749..72670f6 100644 --- a/c_tools/mock/generateMock.cpp +++ b/c_tools/mock/generateMock.cpp @@ -370,7 +370,7 @@ void buildBox(SimuData *simu, long num_targets, long loaded, for (uint32_t i = 0; i < num_targets; i++, loaded++) { long pid = particle_id[loaded]; - assert(pid < simu->NumPart); + //assert(pid < simu->NumPart); assert(loaded < boxed->NumPart); for (int j = 0; j < 3; j++) @@ -545,6 +545,7 @@ void makeBoxFromParameter(SimuData *simu, SimuData* &boxed, generateMock_info& a } boxed->NumPart = pid_write; delete preselect; + cout << "Num accepted here = " << pid_write << " / input = " << pid_read << endl; } void makeBoxFromSimulation(SimulationLoader *loader, SimuData* &boxed, MetricFunctor metric, generateMock_info& args_info) @@ -712,7 +713,10 @@ int main(int argc, char **argv) generateOutput(simuOut, args_info.axis_arg, args_info.output_arg); delete preselector; - - printf("Done!\n"); + + double subsample = 1.0; + if (args_info.subsample_given) subsample = args_info.subsample_arg; + if (args_info.resubsample_given) subsample = args_info.resubsample_arg; + printf("Done! %5.2e\n", subsample); return 0; } diff --git a/crossCompare/analysis/datasetsToAnalyze.py b/crossCompare/analysis/datasetsToAnalyze.py index 9c8e5d2..72db90b 100644 --- a/crossCompare/analysis/datasetsToAnalyze.py +++ b/crossCompare/analysis/datasetsToAnalyze.py @@ -21,15 +21,23 @@ workDir = "/home/psutter2/workspace/Voids/" -dataDir = "/home/psutter2/workspace/Voids/multidark/crossCompare/mergerTree/" +dataDir = "/home/psutter2/workspace/Voids/crossCompare/mergerTree/" CTOOLS_PATH = "../../c_tools/" -baseSampleDir = "multidark/md_ss0.000175/sample_md_ss0.000175_z0.56_d00/" +baseSampleDir = "multidark/md_ss0.05/sample_md_ss0.05_z0.56_d00/" sampleDirList = [ - "multidark/md_ss0.0004/sample_md_ss0.0004_z0.56_d00/", + "multidark/md_ss1e-05/sample_md_ss1e-05_z0.56_d00/", + #"multidark/md_ss0.000175/sample_md_ss0.000175_z0.56_d00/", + #"multidark/md_ss0.0004/sample_md_ss0.0004_z0.56_d00/", + #"multidark/md_ss0.001/sample_md_ss0.001_z0.56_d00/", + #"multidark/md_ss0.002/sample_md_ss0.002_z0.56_d00/", + "multidark/md_ss0.01/sample_md_ss0.01_z0.56_d00/", + "multidark/md_hod_dr72dim2/sample_md_hod_dr72dim2_z0.56_d00/", + "multidark/md_hod_dr9mid/sample_md_hod_dr9mid_z0.56_d00/", + "multidark/md_halos_min1.2e+13/sample_md_halos_min1.2e+13_z0.56_d00/", ] dataPortion = "central" diff --git a/crossCompare/analysis/mergerTree.py b/crossCompare/analysis/mergerTree.py index ec70217..4868858 100644 --- a/crossCompare/analysis/mergerTree.py +++ b/crossCompare/analysis/mergerTree.py @@ -63,6 +63,7 @@ for (iSample, sampleDir) in enumerate(sampleDirList): sample = pickle.load(input) print " Working with", sample.fullName, "...", + sys.stdout.flush() sampleName = sample.fullName @@ -72,11 +73,12 @@ for (iSample, sampleDir) in enumerate(sampleDirList): sampleName + "_" #stepOutputFileName = os.getcwd()+"/data/overlap_" - launchVoidOverlap(baseSample, sample, workDir+baseSampleDir, - workDir+sampleDir, binPath, + launchVoidOverlap(sample, baseSample, workDir+sampleDir, + workDir+baseSampleDir, binPath, thisDataPortion="central", logFile=logFile, continueRun=False, workDir=workDir, - outputFile=stepOutputFileName) + outputFile=stepOutputFileName, + matchMethod="proximity") # attach columns to summary file #if iSample == 1: diff --git a/pipeline/datasets/multidark.py b/pipeline/datasets/multidark.py index 79df229..2c6637d 100644 --- a/pipeline/datasets/multidark.py +++ b/pipeline/datasets/multidark.py @@ -30,12 +30,12 @@ catalogDir = os.getenv("HOME")+"/workspace/Voids/catalogs/multidark/" hodPath = os.getenv("HOME")+"/projects/Voids/hod/HOD.x" # where to put the final void catalog, figures, and output logs -voidOutputDir = os.getenv("HOME")+"/workspace/Voids/multidark_nocone/" -figDir = os.getenv("PWD")+"/../figs/multidark_nocone/" -logDir = os.getenv("PWD")+"/../logs/multidark_nocone/" +voidOutputDir = os.getenv("HOME")+"/workspace/Voids/multidark/" +figDir = os.getenv("PWD")+"/../figs/multidark/" +logDir = os.getenv("PWD")+"/../logs/multidark/" # where to place the pipeline scripts -scriptDir = os.getenv("PWD")+"/multidark_nocone/" +scriptDir = os.getenv("PWD")+"/multidark/" # simulation or observation? dataType = "simulation" @@ -49,15 +49,19 @@ useLightCone = False # common filename of particle files particleFileBase = "mdr1_particles_z" +particleFileDummy = '' # list of file numbers for the particle files # to get particle file name, we take particleFileBase+fileNum #fileNums = ["0.53"] -fileNums = ["0.0", "0.53", "1.0"] +fileNums = ["0.0"] +#fileNums = ["0.0", "0.53", "1.0"] # redshift of each file in the above list #redshifts = ["0.53"] -redshifts = ["0.0", "0.53", "1.0"] +redshifts = ["0.0"] +#redshifts = ["0.0", "0.53"] +#redshifts = ["0.0", "0.53", "1.0"] # how many independent slices along the z-axis? numSlices = 1 @@ -71,7 +75,7 @@ numSubvolumes = 1 prefix = "md_" # list of desired subsamples - these are in unts of h Mpc^-3! -#subSamples = [0.0004] +#subSamples = [0.01] subSamples = [0.1, 0.05, 0.01, 0.002, 0.001, 0.0004, 0.000175, 0.00001] # common filename of halo files, leave blank to ignore halos @@ -97,11 +101,12 @@ haloFileNumComLines = 0 # adjust these two parameters given the memory contraints on your system: # numZobovDivisions: how many sub-volumes per dimension will zobov process # numZobovThreads: how many sub-volumes to process at once? -numZobovDivisions = 4 +numZobovDivisions = 2 numZobovThreads = 2 # simulation information -numPart = 2048*2048*2048 +numPart = 100000000 +#numPart = 2048*2048*2048 lbox = 1000 # Mpc/h omegaM = 0.27 hubble = 0.70 diff --git a/pipeline/generateCatalog.py b/pipeline/generateCatalog.py old mode 100644 new mode 100755 index f5473f8..0321fa6 --- a/pipeline/generateCatalog.py +++ b/pipeline/generateCatalog.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python #+ # VIDE -- Void IDEntification pipeline -- ./pipeline/generateCatalog.py # Copyright (C) 2010-2013 Guilhem Lavaux @@ -17,7 +18,6 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. #+ -#!/usr/bin/env python # does full void analysis. Also generates 2d/1d stacked plots and hubble diagram @@ -88,7 +88,7 @@ for sample in dataSampleList: if (sample.dataType == "simulation"): fp.write("Particles placed on lightcone: %g\n" % sample.useLightCone) fp.write("Peculiar velocities included: %g\n" % sample.usePecVel) - fp.write("Additional subsampling fraction: %g\n" % sample.subsample) + fp.write("Additional subsampling fraction: %s\n" % sample.subsample[-1]) fp.write("Simulation box length (Mpc/h): %g\n" % sample.boxLen) fp.write("Simulation Omega_M: %g\n" % sample.omegaM) fp.write("Number of simulation subvolumes: %s\n" % sample.numSubvolumes) diff --git a/python_tools/pipeline_source/prepareCatalogs.in.py b/python_tools/pipeline_source/prepareCatalogs.in.py index 612629b..78d426b 100644 --- a/python_tools/pipeline_source/prepareCatalogs.in.py +++ b/python_tools/pipeline_source/prepareCatalogs.in.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python #+ # VIDE -- Void IDEntification pipeline -- ./python_tools/pipeline_source/prepareCatalogs.in.py # Copyright (C) 2010-2013 Guilhem Lavaux @@ -17,7 +18,6 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. #+ -#!/usr/bin/env python # prepares input catalogs based on multidark simulations # (borrows heavily from generateMock, but doesn't hold much in memory) @@ -453,6 +453,7 @@ if (args.halos or args.all) and haloFileBase != "": if dataFormat == "sdf": # TODO process halo file with SDFcvt + SDFcvt_PATH = "@CMAKE_BINARY_DIR@/ep_build/sdf/bin/SDFcvt/" outFile = open(catalogDir+"/"+sampleName+".dat", 'w') outFile.write("-99 -99 -99 -99 -99 -99 -99\n") outFile.close() diff --git a/python_tools/void_python_tools/backend/launchers.py b/python_tools/void_python_tools/backend/launchers.py index b619601..1a0fb20 100644 --- a/python_tools/void_python_tools/backend/launchers.py +++ b/python_tools/void_python_tools/backend/launchers.py @@ -124,20 +124,31 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None, datafile = inputDataDir+"/"+sample.dataFile + # check if the final subsampling is done + lastSample = sample.subsample.split(', ')[-1] + doneLine = "Done! %5.2e\n" % float(lastSample) + if (continueRun and jobSuccessful(logFile, doneLine)): + print "already done!" + return + prevSubSample = -1 for thisSubSample in sample.subsample.split(', '): if prevSubSample == -1: inputParameterFlag = "" - outputFile = zobovDir+"/zobov_slice_" + sampleName + "_ss" + thisSubSample + outputFile = zobovDir+"/zobov_slice_" + sampleName + "_ss" + \ + thisSubSample keepFraction = float(thisSubSample) subSampleLine = "subsample %g" % keepFraction + resubSampleLine = "" else: inputParameterFlag = "inputParameter " + zobovDir+"/zobov_slice_"+\ sampleName+"_ss"+prevSubSample+".par" - outputFile = zobovDir+"/_zobov_slice_" + sampleName + "_ss" + thisSubSample + outputFile = zobovDir+"/zobov_slice_" + sampleName + "_ss" + \ + thisSubSample keepFraction = float(thisSubSample)/float(prevSubSample) - subSampleLine = "resubsample %g" % keepFraction + subSampleLine = "subsample %g" % keepFraction + resubSampleLine = "resubsample %g" % keepFraction includePecVelString = "" if sample.usePecVel: includePecVelString = "peculiarVelocities" @@ -175,6 +186,7 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None, rangeZ_max %g %s %s + %s """ % (dataFileLine, outputFile, outputFile+".par", includePecVelString, @@ -182,33 +194,30 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None, sample.dataUnit, xMin, xMax, yMin, yMax, sample.zBoundaryMpc[0], sample.zBoundaryMpc[1], - subSampleLine,inputParameterFlag) + subSampleLine,resubSampleLine,inputParameterFlag) parmFile = os.getcwd()+"/generate_"+sample.fullName+".par" file(parmFile, mode="w").write(conf) - doneLine = "Done! %5.2e\n" % keepFraction - print "TEST", doneLine - if not (continueRun and jobSuccessful(logFile, doneLine)): - if (prevSubSample == -1): - cmd = "%s --configFile=%s &> %s" % (binPath,parmFile,logFile) - else: - cmd = "%s --configFile=%s &>> %s" % (binPath,parmFile,logFile) - os.system(cmd) - - if not jobSuccessful(logFile, doneLine): - print "FAILED!" - exit(-1) - + if (prevSubSample == -1): + cmd = "%s --configFile=%s &> %s" % (binPath,parmFile,logFile) + else: + cmd = "%s --configFile=%s &>> %s" % (binPath,parmFile,logFile) + os.system(cmd) + # remove intermediate files if (prevSubSample != -1): os.unlink(zobovDir+"/zobov_slice_"+sampleName+"_ss"+prevSubSample+".par") os.unlink(zobovDir+"/zobov_slice_"+sampleName+"_ss"+prevSubSample) + doneLine = "Done! %5.2e\n" % keepFraction + if not jobSuccessful(logFile, doneLine): + print "FAILED!" + exit(-1) + prevSubSample = thisSubSample - if (continueRun and jobSuccessful(logFile, doneLine)): print "already done!" if jobSuccessful(logFile, doneLine): print "done" # place the final subsample @@ -690,7 +699,7 @@ def launchCombine(sample, stack, voidDir=None, logFile=None, shutil.copy(workDir+"/sample_"+comboName+"/galaxies.txt", zobovDir) elif not doneGalUpdate: dataTemp = file(workDir+"/sample_"+comboName+"/galaxies.txt", -"r").read() + "r").read() file(zobovDir+"/galaxies.txt", "a").write(dataTemp) sourceStackDir = workDir+"/sample_"+comboName+"/stacks_"+\ @@ -758,8 +767,9 @@ def launchCombine(sample, stack, voidDir=None, logFile=None, idxTemp = file(sourceStackDir+"/z_void_indexes.txt", "r").\ readlines() idxTemp = np.array(idxTemp, dtype='i') - dataTemp = (NetCDFFile(voidDir+"/z_posx.nc").\ - variables['array'])[0:] + fp = NetCDFFile(voidDir+"/z_posx.nc") + dataTemp = fp.variables['array'][0:] + fp.close() idxTemp[:] += len(dataTemp) fp = open(voidDir+"/z_void_indexes.txt", "a") for idx in idxTemp: @@ -809,8 +819,9 @@ def launchCombine(sample, stack, voidDir=None, logFile=None, idxTemp = file(sourceStackDir+"/void_indexes.txt", "r").\ readlines() idxTemp = np.array(idxTemp, dtype='i') - dataTemp = (NetCDFFile(voidDir+"/posx.nc").\ - variables['array'])[0:] + fp = NetCDFFile(voidDir+"/posx.nc") + dataTemp = fp.variables['array'][0:] + fp.close() idxTemp[:] += len(dataTemp) fp = open(voidDir+"/void_indexes.txt", "a") for idx in idxTemp: