Merge branch 'master' of bitbucket.org:cosmicvoids/void_identification

c_tools/analysis/voidOverlap.cpp

@@ -286,7 +286,7 @@ int main(int argc, char **argv) {
       rdist = catalog1.voids[iVoid1].matches[0].dist;
       rdist /= catalog1.voids[iVoid1].radius;
 
-      fprintf(fp, "%d %.2f %.2f %.2f %.2f %.2f %d %d\n", voidID, 
+      fprintf(fp, "%d %.4f %.4f %.4f %.4f %.4f %d %d\n", voidID, 
                                    catalog1.voids[iVoid1].radiusMpc,
                                    rRatio, 
                                    commonVolRatio,

c_tools/mock/generateMock.cpp

@@ -370,7 +370,7 @@ void buildBox(SimuData *simu, long num_targets, long loaded,
   for (uint32_t i = 0; i < num_targets; i++, loaded++)
     {
       long pid = particle_id[loaded];
-      assert(pid < simu->NumPart);
+      //assert(pid < simu->NumPart);
       assert(loaded < boxed->NumPart);
       
       for (int j = 0; j < 3; j++)
@@ -545,6 +545,7 @@ void makeBoxFromParameter(SimuData *simu, SimuData* &boxed, generateMock_info& a
     }  
   boxed->NumPart = pid_write; 
   delete preselect;
+  cout << "Num accepted here = " << pid_write <<  " / input = " << pid_read << endl; 
 }
 
 void makeBoxFromSimulation(SimulationLoader *loader, SimuData* &boxed, MetricFunctor metric, generateMock_info& args_info)
@@ -713,6 +714,9 @@ int main(int argc, char **argv)
                  args_info.output_arg);
   delete preselector;
  
-  printf("Done!\n"); 
+  double subsample = 1.0;
+  if (args_info.subsample_given) subsample = args_info.subsample_arg;
+  if (args_info.resubsample_given) subsample = args_info.resubsample_arg;
+  printf("Done! %5.2e\n", subsample); 
   return 0;
 }

crossCompare/analysis/datasetsToAnalyze.py

@@ -21,15 +21,23 @@
 
 
 workDir = "/home/psutter2/workspace/Voids/"
-dataDir = "/home/psutter2/workspace/Voids/multidark/crossCompare/mergerTree/"
+dataDir = "/home/psutter2/workspace/Voids/crossCompare/mergerTree/"
  
 
 CTOOLS_PATH = "../../c_tools/"
 
-baseSampleDir = "multidark/md_ss0.000175/sample_md_ss0.000175_z0.56_d00/"
+baseSampleDir = "multidark/md_ss0.05/sample_md_ss0.05_z0.56_d00/"
 
 sampleDirList = [ 
-                  "multidark/md_ss0.0004/sample_md_ss0.0004_z0.56_d00/",
+                  "multidark/md_ss1e-05/sample_md_ss1e-05_z0.56_d00/",
+                  #"multidark/md_ss0.000175/sample_md_ss0.000175_z0.56_d00/",
+                  #"multidark/md_ss0.0004/sample_md_ss0.0004_z0.56_d00/",
+                  #"multidark/md_ss0.001/sample_md_ss0.001_z0.56_d00/",
+                  #"multidark/md_ss0.002/sample_md_ss0.002_z0.56_d00/",
+                  "multidark/md_ss0.01/sample_md_ss0.01_z0.56_d00/",
+                  "multidark/md_hod_dr72dim2/sample_md_hod_dr72dim2_z0.56_d00/",
+                  "multidark/md_hod_dr9mid/sample_md_hod_dr9mid_z0.56_d00/",
+                  "multidark/md_halos_min1.2e+13/sample_md_halos_min1.2e+13_z0.56_d00/",
                 ]
 
 dataPortion = "central"

crossCompare/analysis/mergerTree.py

@@ -63,6 +63,7 @@ for (iSample, sampleDir) in enumerate(sampleDirList):
     sample = pickle.load(input)
 
   print " Working with", sample.fullName, "...",
+  sys.stdout.flush()
 
   sampleName = sample.fullName
 
@@ -72,11 +73,12 @@ for (iSample, sampleDir) in enumerate(sampleDirList):
                        sampleName + "_"
   #stepOutputFileName = os.getcwd()+"/data/overlap_"
 
-  launchVoidOverlap(baseSample, sample, workDir+baseSampleDir, 
+  launchVoidOverlap(sample, baseSample, workDir+sampleDir, 
-                    workDir+sampleDir, binPath, 
+                    workDir+baseSampleDir, binPath, 
                     thisDataPortion="central", logFile=logFile,
                     continueRun=False, workDir=workDir,
-                    outputFile=stepOutputFileName)
+                    outputFile=stepOutputFileName,
+                    matchMethod="proximity")
 
   # attach columns to summary file
   #if iSample == 1:

pipeline/datasets/multidark.py

@@ -30,12 +30,12 @@ catalogDir = os.getenv("HOME")+"/workspace/Voids/catalogs/multidark/"
 hodPath = os.getenv("HOME")+"/projects/Voids/hod/HOD.x"
 
 # where to put the final void catalog, figures, and output logs
-voidOutputDir = os.getenv("HOME")+"/workspace/Voids/multidark_nocone/"
+voidOutputDir = os.getenv("HOME")+"/workspace/Voids/multidark/"
-figDir = os.getenv("PWD")+"/../figs/multidark_nocone/"
+figDir = os.getenv("PWD")+"/../figs/multidark/"
-logDir = os.getenv("PWD")+"/../logs/multidark_nocone/"
+logDir = os.getenv("PWD")+"/../logs/multidark/"
 
 # where to place the pipeline scripts
-scriptDir = os.getenv("PWD")+"/multidark_nocone/"
+scriptDir = os.getenv("PWD")+"/multidark/"
 
 # simulation or observation?
 dataType = "simulation"
@@ -49,15 +49,19 @@ useLightCone = False
 
 # common filename of particle files
 particleFileBase = "mdr1_particles_z"
+particleFileDummy = ''
 
 # list of file numbers for the particle files
 # to get particle file name, we take particleFileBase+fileNum
 #fileNums = ["0.53"]
-fileNums = ["0.0", "0.53", "1.0"]
+fileNums = ["0.0"]
+#fileNums = ["0.0", "0.53", "1.0"]
 
 # redshift of each file in the above list
 #redshifts = ["0.53"]
-redshifts = ["0.0", "0.53", "1.0"]
+redshifts = ["0.0"]
+#redshifts = ["0.0", "0.53"]
+#redshifts = ["0.0", "0.53", "1.0"]
 
 # how many independent slices along the z-axis?
 numSlices = 1
@@ -71,7 +75,7 @@ numSubvolumes = 1
 prefix = "md_"
 
 # list of desired subsamples - these are in unts of h Mpc^-3!
-#subSamples = [0.0004]
+#subSamples = [0.01]
 subSamples = [0.1, 0.05, 0.01, 0.002, 0.001, 0.0004, 0.000175, 0.00001]
 
 # common filename of halo files, leave blank to ignore halos
@@ -97,11 +101,12 @@ haloFileNumComLines = 0
 # adjust these two parameters given the memory contraints on your system:
 #   numZobovDivisions: how many sub-volumes per dimension will zobov process
 #   numZobovThreads: how many sub-volumes to process at once?   
-numZobovDivisions = 4
+numZobovDivisions = 2
 numZobovThreads = 2
 
 # simulation information
-numPart = 2048*2048*2048
+numPart = 100000000
+#numPart = 2048*2048*2048
 lbox = 1000 # Mpc/h
 omegaM = 0.27
 hubble = 0.70

pipeline/generateCatalog.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 #+
 #   VIDE -- Void IDEntification pipeline -- ./pipeline/generateCatalog.py
 #   Copyright (C) 2010-2013 Guilhem Lavaux
@@ -17,7 +18,6 @@
 #   with this program; if not, write to the Free Software Foundation, Inc.,
 #   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #+
-#!/usr/bin/env python
 
 # does full void analysis. Also generates 2d/1d stacked plots and hubble diagram
 
@@ -88,7 +88,7 @@ for sample in dataSampleList:
   if (sample.dataType == "simulation"):
     fp.write("Particles placed on lightcone: %g\n" % sample.useLightCone)
     fp.write("Peculiar velocities included: %g\n" % sample.usePecVel)
-    fp.write("Additional subsampling fraction: %g\n" % sample.subsample)
+    fp.write("Additional subsampling fraction: %s\n" % sample.subsample[-1])
     fp.write("Simulation box length (Mpc/h): %g\n" % sample.boxLen)
     fp.write("Simulation Omega_M: %g\n" % sample.omegaM)
     fp.write("Number of simulation subvolumes: %s\n" % sample.numSubvolumes)

python_tools/pipeline_source/prepareCatalogs.in.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 #+
 #   VIDE -- Void IDEntification pipeline -- ./python_tools/pipeline_source/prepareCatalogs.in.py
 #   Copyright (C) 2010-2013 Guilhem Lavaux
@@ -17,7 +18,6 @@
 #   with this program; if not, write to the Free Software Foundation, Inc.,
 #   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #+
-#!/usr/bin/env python
 
 # prepares input catalogs based on multidark simulations
 #   (borrows heavily from generateMock, but doesn't hold much in memory)
@@ -453,6 +453,7 @@ if (args.halos or args.all) and haloFileBase != "":
 
       if dataFormat == "sdf":
         # TODO process halo file with SDFcvt
+        SDFcvt_PATH = "@CMAKE_BINARY_DIR@/ep_build/sdf/bin/SDFcvt/"
         outFile = open(catalogDir+"/"+sampleName+".dat", 'w')
         outFile.write("-99 -99 -99 -99 -99 -99 -99\n")
         outFile.close()

python_tools/void_python_tools/backend/launchers.py

@@ -124,20 +124,31 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
 
     datafile = inputDataDir+"/"+sample.dataFile
 
+    # check if the final subsampling is done
+    lastSample = sample.subsample.split(', ')[-1]
+    doneLine = "Done! %5.2e\n" % float(lastSample)
+    if (continueRun and jobSuccessful(logFile, doneLine)): 
+      print "already done!"
+      return
+
     prevSubSample = -1
     for thisSubSample in sample.subsample.split(', '):
 
       if prevSubSample == -1:
         inputParameterFlag = ""
-        outputFile = zobovDir+"/zobov_slice_" + sampleName + "_ss" + thisSubSample
+        outputFile = zobovDir+"/zobov_slice_" + sampleName + "_ss" + \
+                     thisSubSample
         keepFraction = float(thisSubSample)
         subSampleLine = "subsample %g" % keepFraction
+        resubSampleLine = ""
       else:
         inputParameterFlag = "inputParameter " + zobovDir+"/zobov_slice_"+\
                              sampleName+"_ss"+prevSubSample+".par"
-        outputFile = zobovDir+"/_zobov_slice_" + sampleName + "_ss" + thisSubSample
+        outputFile = zobovDir+"/zobov_slice_" + sampleName + "_ss" + \
+                     thisSubSample
         keepFraction = float(thisSubSample)/float(prevSubSample)
-        subSampleLine = "resubsample %g" % keepFraction
+        subSampleLine = "subsample %g" % keepFraction
+        resubSampleLine = "resubsample %g" % keepFraction
 
       includePecVelString = ""
       if sample.usePecVel: includePecVelString = "peculiarVelocities"
@@ -175,6 +186,7 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
       rangeZ_max %g
       %s
       %s
+      %s
       """ % (dataFileLine, outputFile,
              outputFile+".par",
              includePecVelString,
@@ -182,33 +194,30 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
              sample.dataUnit,
              xMin, xMax, yMin, yMax,
              sample.zBoundaryMpc[0], sample.zBoundaryMpc[1],
-             subSampleLine,inputParameterFlag)
+             subSampleLine,resubSampleLine,inputParameterFlag)
 
       parmFile = os.getcwd()+"/generate_"+sample.fullName+".par"
 
       file(parmFile, mode="w").write(conf)
 
-      doneLine = "Done! %5.2e\n" % keepFraction
+      if (prevSubSample == -1):
-      print "TEST", doneLine
+        cmd = "%s --configFile=%s &> %s" % (binPath,parmFile,logFile)
-      if not (continueRun and jobSuccessful(logFile, doneLine)):
+      else:
-        if (prevSubSample == -1):
+        cmd = "%s --configFile=%s &>> %s" % (binPath,parmFile,logFile)
-          cmd = "%s --configFile=%s &> %s" % (binPath,parmFile,logFile)
+      os.system(cmd)
-        else:
-          cmd = "%s --configFile=%s &>> %s" % (binPath,parmFile,logFile)
-        os.system(cmd)
-      
-        if not jobSuccessful(logFile, doneLine):
-          print "FAILED!"
-          exit(-1)
  
       # remove intermediate files
       if (prevSubSample != -1):
        os.unlink(zobovDir+"/zobov_slice_"+sampleName+"_ss"+prevSubSample+".par")
        os.unlink(zobovDir+"/zobov_slice_"+sampleName+"_ss"+prevSubSample)
 
+      doneLine = "Done! %5.2e\n" % keepFraction
+      if not jobSuccessful(logFile, doneLine):
+        print "FAILED!"
+        exit(-1)
+
       prevSubSample = thisSubSample
 
-    if (continueRun and jobSuccessful(logFile, doneLine)): print "already done!"
     if jobSuccessful(logFile, doneLine): print "done"
  
     # place the final subsample     
@@ -690,7 +699,7 @@ def launchCombine(sample, stack, voidDir=None, logFile=None,
         shutil.copy(workDir+"/sample_"+comboName+"/galaxies.txt", zobovDir)
       elif not doneGalUpdate:
         dataTemp = file(workDir+"/sample_"+comboName+"/galaxies.txt", 
-"r").read()
+                        "r").read()
         file(zobovDir+"/galaxies.txt", "a").write(dataTemp)
 
       sourceStackDir = workDir+"/sample_"+comboName+"/stacks_"+\
@@ -758,8 +767,9 @@ def launchCombine(sample, stack, voidDir=None, logFile=None,
         idxTemp = file(sourceStackDir+"/z_void_indexes.txt", "r").\
                   readlines()
         idxTemp = np.array(idxTemp, dtype='i')
-        dataTemp = (NetCDFFile(voidDir+"/z_posx.nc").\
+        fp = NetCDFFile(voidDir+"/z_posx.nc")
-                   variables['array'])[0:]
+        dataTemp = fp.variables['array'][0:]
+        fp.close()
         idxTemp[:] += len(dataTemp)
         fp = open(voidDir+"/z_void_indexes.txt", "a")
         for idx in idxTemp:
@@ -809,8 +819,9 @@ def launchCombine(sample, stack, voidDir=None, logFile=None,
         idxTemp = file(sourceStackDir+"/void_indexes.txt", "r").\
                   readlines()
         idxTemp = np.array(idxTemp, dtype='i')
-        dataTemp = (NetCDFFile(voidDir+"/posx.nc").\
+        fp = NetCDFFile(voidDir+"/posx.nc")
-                   variables['array'])[0:]
+        dataTemp = fp.variables['array'][0:]
+        fp.close()
         idxTemp[:] += len(dataTemp)
         fp = open(voidDir+"/void_indexes.txt", "a")
         for idx in idxTemp: