Added support for regeneration (to finely test impact of peculiar velocities)

pipeline/generateCatalog.py

@@ -20,6 +20,7 @@ if (len(sys.argv) > 1):
     print "  Cannot find parameter file %s!" % filename
     exit(-1)
   parms = imp.load_source("name", filename)
+  regenerateFlag = False
   globals().update(vars(parms))
 else:
   print " Using default parameters"
@@ -90,7 +91,7 @@ for sample in dataSampleList:
     launchGenerate(sample, GENERATE_PATH, workDir=workDir, 
                    inputDataDir=inputDataDir, zobovDir=zobovDir,
                    figDir=figDir, logFile=logFile, useLCDM=useLCDM,
-                   continueRun=continueRun)
+                   continueRun=continueRun, regenerate=regenerateFlag)
 
   # --------------------------------------------------------------------------
   if (startCatalogStage <= 2) and (endCatalogStage >= 2) and not sample.isCombo:

python_tools/pipeline_source/prepareCatalogs.in.py

@@ -81,6 +81,7 @@ endCatalogStage   = 4
 startAPStage = 1
 endAPStage = 1
 
+regenerateFlag = False
 ZOBOV_PATH = "@CMAKE_BINARY_DIR@/zobov/"
 CTOOLS_PATH = "@CMAKE_BINARY_DIR@/c_tools/"
 freshStack = True

python_tools/void_python_tools/backend/launchers.py

@@ -24,11 +24,18 @@ ncFloat = 'f8' # Double precision
 # -----------------------------------------------------------------------------
 def launchGenerate(sample, binPath, workDir=None, inputDataDir=None, 
                    zobovDir=None, figDir=None, logFile=None, useLCDM=False,
-                   continueRun=None):
+                   continueRun=None,regenerate=False):
 
   if sample.dataType == "observation":
     sampleName = sample.fullName
 
+    if regenerate:
+      inputParameterFlag = "inputParameter " + zobovDir+"/zobov_slice_"+sampleName+".par"
+      outputFile = zobovDir+"/regenerated_zobov_slice_" + sampleName
+    else:
+      inputParameterFlag = ""
+      outputFile = zobovDir+"/zobov_slice_" + sampleName
+
     if sample.dataFile == "":
       datafile = inputDataDir+"/"+sampleName
     else:
@@ -50,16 +57,17 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
       zMax %g
       density_fake %g
       %s
-      """ % (datafile, maskFile, zobovDir+"/zobov_slice_"+sampleName,
+      %s
+      """ % (datafile, maskFile, outputFile,
              zobovDir+"/zobov_slice_"+sampleName+".par",
              sample.zBoundary[0], sample.zBoundary[1], sample.fakeDensity,
-             useLCDMFlag)
+             useLCDMFlag, inputParameterFlag)
 
     parmFile = os.getcwd()+"/generate_"+sample.fullName+".par"
 
     file(parmFile, mode="w").write(conf)
 
-    if not (continueRun and jobSuccessful(logFile, "Done!\n")):
+    if regenerate or not (continueRun and jobSuccessful(logFile, "Done!\n")):
       cmd = "%s --configFile=%s &> %s" % (binPath,parmFile,logFile)
       os.system(cmd)
       if jobSuccessful(logFile, "Done!\n"):
@@ -97,6 +105,13 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
 
     datafile = inputDataDir+"/"+sample.dataFile
 
+    if regenerate:
+      inputParameterFlag = "inputParameter " + zobovDir+"/zobov_slice_"+sampleName+".par"
+      outputFile = zobovDir+"/regenerated_zobov_slice_" + sampleName 
+    else:
+      inputParameterFlag = ""
+      outputFile = zobovDir+"/zobov_slice_" + sampleName
+
     if sample.usePecVel:
       includePecVelString = "peculiarVelocities"
     else:
@@ -134,20 +149,21 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None,
       rangeZ_min %g
       rangeZ_max %g
       subsample %g
-      """ % (dataFileLine, zobovDir+"/zobov_slice_"+sampleName,
+      %s
+      """ % (dataFileLine, outputFile,
              zobovDir+"/zobov_slice_"+sampleName+".par",
              includePecVelString,
              useLightConeString,
              sample.dataUnit,
              xMin, xMax, yMin, yMax,
              sample.zBoundaryMpc[0], sample.zBoundaryMpc[1],
-             sample.subsample)
+             sample.subsample,inputParameterFlag)
 
     parmFile = os.getcwd()+"/generate_"+sample.fullName+".par"
 
     file(parmFile, mode="w").write(conf)
 
-    if not (continueRun and jobSuccessful(logFile, "Done!\n")):
+    if regenerate or not (continueRun and jobSuccessful(logFile, "Done!\n")):
       cmd = "%s --configFile=%s &> %s" % (binPath,parmFile,logFile)
       os.system(cmd)
       if jobSuccessful(logFile, "Done!\n"):
@@ -258,6 +274,7 @@ def launchPrune(sample, binPath, thisDataPortion=None,
        sample.boxLen <= 1.e-1: 
       periodicLine += "z"
     periodicLine += "' "
+    periodicLine = ""
 
   if not (continueRun and jobSuccessful(logFile, "NetCDF: Not a valid ID\n")):
     cmd = binPath
@@ -297,6 +314,9 @@ def launchPrune(sample, binPath, thisDataPortion=None,
                           str(thisDataPortion)+"_"+\
                           str(sampleName)+".out"
     cmd += " &> " + logFile
+    f=file("run_prune.sh",mode="w")
+    f.write(cmd)
+    f.close()
     os.system(cmd)
 
     if jobSuccessful(logFile, "NetCDF: Not a valid ID\n") or \
@@ -380,14 +400,14 @@ def launchStack(sample, stack, binPath, thisDataPortion=None, logDir=None,
                 voidDir=None, freshStack=True, runSuffix=None,
                 zobovDir=None,
                 INCOHERENT=False, ranSeed=None, summaryFile=None, 
-                continueRun=None, dataType=None):
+                continueRun=None, dataType=None, prefixRun=""):
 
   sampleName = sample.fullName
 
   runSuffix = getStackSuffix(stack.zMin, stack.zMax, stack.rMin,
                              stack.rMax, thisDataPortion)
 
-  logFile = logDir+"/stack_"+sampleName+"_"+runSuffix+".out"
+  logFile = logDir+("/%sstack_"%prefixRun)+sampleName+"_"+runSuffix+".out"
  
   treeFile = voidDir+"/tree.data"
 
@@ -414,7 +434,7 @@ def launchStack(sample, stack, binPath, thisDataPortion=None, logDir=None,
     maxDen = 0.2*float(maskIndex)/float(totalPart)
   else:
     maskIndex = 999999999
-    maxDen = 0.2
+    maxDen = -0.8
 
   if INCOHERENT:
     nullTestFlag = "INCOHERENT"
@@ -455,7 +475,7 @@ def launchStack(sample, stack, binPath, thisDataPortion=None, logDir=None,
    zobovDir+"/vol_"+sampleName+".dat",
    stack.rMin,
    stack.rMax,
-   zobovDir+"/zobov_slice_"+sampleName,
+   zobovDir+("/%szobov_slice_"%prefixRun)+sampleName,
    zobovDir+"/zobov_slice_"+sampleName+".par",
    maxDen,
    centralRadius,
@@ -471,7 +491,7 @@ def launchStack(sample, stack, binPath, thisDataPortion=None, logDir=None,
    zobovDir+"/boundaryDistances_"+thisDataPortion+"_"+sampleName+".out",
    rescaleFlag)
 
-  parmFile = os.getcwd()+"/stack_"+sample.fullName+".par"
+  parmFile = os.getcwd()+("/%sstack_"%prefixRun)+sample.fullName+".par"
 
   fp = file(parmFile, mode="w")
   fp.write(conf)