From 5e36b11505a5e6108927f20f9f778c5954b635be Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Tue, 19 Mar 2013 09:29:05 -0400 Subject: [PATCH 01/11] Made SHARP building optional in CosmoTool. Disabled in VIDE building makefiles --- external/external_build.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/external/external_build.cmake b/external/external_build.cmake index 401abaf..2885053 100644 --- a/external/external_build.cmake +++ b/external/external_build.cmake @@ -234,6 +234,7 @@ ExternalProject_Add(cosmotool -DGSLCBLAS_LIBRARY=${GSLCBLAS_LIBRARY} -DNETCDF_LIBRARY=${NETCDF_LIBRARY} -DNETCDFCPP_LIBRARY=${NETCDFCPP_LIBRARY} + -DENABLE_SHARP=OFF ) SET(COSMOTOOL_LIBRARY ${CMAKE_BINARY_DIR}/ext_build/cosmotool/lib/libCosmoTool.a) set(COSMOTOOL_INCLUDE_PATH ${CMAKE_BINARY_DIR}/ext_build/cosmotool/include) From 0a20f4a36a7825e4022807719d184ed634afbc85 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Tue, 19 Mar 2013 09:48:21 -0400 Subject: [PATCH 02/11] Fixed compiler warnings in Zobov (hopefully fixing some potential bugs too) --- zobov/jozov.c | 54 +++++++++++++++++++++++++------------------------- zobov/voz.h | 6 ++++-- zobov/voz1b1.c | 8 ++++---- zobov/voztie.c | 6 +++--- 4 files changed, 38 insertions(+), 36 deletions(-) diff --git a/zobov/jozov.c b/zobov/jozov.c index e5ebf7f..697d039 100644 --- a/zobov/jozov.c +++ b/zobov/jozov.c @@ -85,7 +85,7 @@ int main(int argc,char **argv) { printf("Bad density threshold.\n"); exit(0); } - if (sscanf(argv[7],"%f",&mockIndex) == 0) { + if (sscanf(argv[7],"%d",&mockIndex) == 0) { printf("Bad mock galaxy index.\n"); exit(0); } @@ -108,29 +108,29 @@ int main(int argc,char **argv) { p = (PARTICLE *)malloc(np*sizeof(PARTICLE)); /* Adjacencies*/ for (i=0;i 0) - p[i].adj = (int *)malloc(p[i].nadj*sizeof(int)); + p[i].adj = (pid_t *)malloc(p[i].nadj*sizeof(pid_t)); else p[i].adj = 0; p[i].ncnt = 0; /* Temporarily, it's an adj counter */ } for (i=0;i 0) for (k=0;k 0) - fwrite(adjs[i].adj,adjs[i].nadj,sizeof(int),out); + fwrite(adjs[i].adj,adjs[i].nadj,sizeof(pid_t),out); else printf("0"); } fclose(out); diff --git a/zobov/voztie.c b/zobov/voztie.c index cc8c006..b629e81 100644 --- a/zobov/voztie.c +++ b/zobov/voztie.c @@ -18,6 +18,8 @@ int main(int argc, char *argv[]) { int nvp,npnotdone,nvpmax, nvpsum, *orig; double avgnadj, avgvol; + int numRemoved = 0; + // PMS int mockIndex; // END PMS @@ -157,8 +159,6 @@ int main(int argc, char *argv[]) { adjs[i].nadj = 0; } - int numRemoved = 0; - // unlink particles adjacent to mock galaxies for (i = 0; i < mockIndex; i++) { for (j = 0; j < adjs[i].nadj; j++) { @@ -217,7 +217,7 @@ printf("\n"); avgvol += (double)(vols[p]); } if (npnotdone > 0) - printf("%d particles not done!\n"); + printf("%d particles not done!\n", npnotdone); printf("%d particles done more than once.\n",nvpsum-np); avgnadj /= (double)np; avgvol /= (double)np; From 17546f2a1ea7c96f3becd85bbef06ee4adcca40b Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Tue, 19 Mar 2013 10:00:25 -0400 Subject: [PATCH 03/11] Reenforce periodic boundaries (to deal with round-off error in float) --- c_tools/mock/loaders/sdf_loader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_tools/mock/loaders/sdf_loader.cpp b/c_tools/mock/loaders/sdf_loader.cpp index b716536..cab171d 100644 --- a/c_tools/mock/loaders/sdf_loader.cpp +++ b/c_tools/mock/loaders/sdf_loader.cpp @@ -190,7 +190,7 @@ public: if (load_flags & (NEED_POSITION | NEED_VELOCITY)) rescaleParticles(d, d->Hubble*1e-5, one_kpc/one_Gyr); -// enforceBoxSize(d); + enforceBoxSize(d); applyTransformations(d); basicPreprocessing(d, preproc); From 61b016c72fc6f677d5b5c99f81f23029177ad0ae Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Tue, 19 Mar 2013 10:07:15 -0400 Subject: [PATCH 04/11] Use more of the new type 'pid_t' to specify indexes that needs to handle particle indexes/ids --- zobov/voz1b1.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zobov/voz1b1.c b/zobov/voz1b1.c index 17b77ad..93fda36 100644 --- a/zobov/voz1b1.c +++ b/zobov/voz1b1.c @@ -12,7 +12,7 @@ int posread(char *posfile, float ***p, float fact); int main(int argc, char *argv[]) { int exitcode; - int i, j, np; + pid_t i, j, np; float **r; coordT rtemp[3], *parts; coordT deladjs[3*MAXVERVER], points[3*MAXVERVER]; @@ -26,7 +26,8 @@ int main(int argc, char *argv[]) { int isitinbuf; char isitinmain, d; - int numdiv, nvp, nvpall, nvpbuf; + int numdiv; + pid_t nvp, nvpall, nvpbuf; float width, width2, totwidth, totwidth2, bf, s, g; float border, boxsize; float c[3]; From bfe2475b516aba51950caf22aa422465f68fb4d8 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Tue, 19 Mar 2013 10:17:27 -0400 Subject: [PATCH 05/11] Use negated isitmain to be sure the selection is done the same way (float comparison). More pid_t --- zobov/voz1b1.c | 9 +++++---- zobov/voztie.c | 16 ++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/zobov/voz1b1.c b/zobov/voz1b1.c index 93fda36..5f321a8 100644 --- a/zobov/voz1b1.c +++ b/zobov/voz1b1.c @@ -22,7 +22,7 @@ int main(int argc, char *argv[]) { PARTADJ *adjs; float *vols; float predict, xmin,xmax,ymin,ymax,zmin,zmax; - int *orig; + pid_t *orig; int isitinbuf; char isitinmain, d; @@ -183,15 +183,16 @@ int main(int argc, char *argv[]) { nvpbuf = nvp; for (i=0; i 0.5) rtemp[d] --; if (rtemp[d] < -0.5) rtemp[d] ++; isitinbuf = isitinbuf && (fabs(rtemp[d]) 0) && - ((fabs(rtemp[0])>width2)||(fabs(rtemp[1])>width2)||(fabs(rtemp[2])>width2))) { - + if (isitinbuf && !isitinmain) { /*printf("%3.3f ",sqrt(rtemp[0]*rtemp[0] + rtemp[1]*rtemp[1] + rtemp[2]*rtemp[2])); printf("|%2.2f,%2.2f,%2.2f,%f,%f",r[i][0],r[i][1],r[i][2],width2,totwidth2);*/ diff --git a/zobov/voztie.c b/zobov/voztie.c index b629e81..8e40aa3 100644 --- a/zobov/voztie.c +++ b/zobov/voztie.c @@ -14,8 +14,8 @@ int main(int argc, char *argv[]) { int numdiv,np,np2,na; - int i,j,k,p,nout; - int nvp,npnotdone,nvpmax, nvpsum, *orig; + pid_t i,j,k,p,nout; + pid_t nvp,npnotdone,nvpmax, nvpsum, *orig; double avgnadj, avgvol; int numRemoved = 0; @@ -89,7 +89,7 @@ int main(int argc, char *argv[]) { if (adjs == NULL) printf("Couldn't allocate adjs.\n"); vols = (float *)malloc(np*sizeof(float)); if (vols == NULL) printf("Couldn't allocate vols.\n"); - orig = (int *)malloc(nvpmax*sizeof(int)); + orig = (pid_t *)malloc(nvpmax*sizeof(pid_t)); if (orig == NULL) printf("Couldn't allocate orig.\n"); if ((cnt_adj==NULL) || (vols == NULL) || (orig == NULL) || (adjs == NULL)) { printf("Not enough memory to allocate. Exiting.\n"); @@ -113,7 +113,7 @@ int main(int argc, char *argv[]) { nvpsum += nvp; - fread(orig,nvp,sizeof(int),part); + fread(orig,nvp,sizeof(pid_t),part); for (p=0;p -1.) @@ -133,12 +133,12 @@ int main(int argc, char *argv[]) { fread(&na,1,sizeof(int),part); if (na > 0) { adjs[orig[p]].nadj = na; - adjs[orig[p]].adj = (int *)malloc(na*sizeof(int)); + adjs[orig[p]].adj = (pid_t *)malloc(na*sizeof(pid_t)); if (adjs[orig[p]].adj == NULL) { printf("Couldn't allocate adjs[orig[%d]].adj.\n",p); exit(0); } - fread(adjs[orig[p]].adj,na,sizeof(int),part); + fread(adjs[orig[p]].adj,na,sizeof(pid_t),part); } else { printf("0"); fflush(stdout); } @@ -272,9 +272,9 @@ printf("\n"); for (j=0;j i) nout++; fwrite(&nout,1,sizeof(int),adj); for (j=0;j i) - fwrite(&(adjs[i].adj[j]),1,sizeof(int),adj); + fwrite(&id,1,sizeof(pid_t),adj); } } From a0d5b171647d4e373fe420674550c4ff19988bf8 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Tue, 19 Mar 2013 10:45:33 -0400 Subject: [PATCH 06/11] Have an output that is more informative. --- c_tools/mock/generateMock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_tools/mock/generateMock.cpp b/c_tools/mock/generateMock.cpp index 628dca1..cd92cce 100644 --- a/c_tools/mock/generateMock.cpp +++ b/c_tools/mock/generateMock.cpp @@ -274,7 +274,7 @@ void selectBox(SimuData *simu, std::vector& targets, generateMock_info& ar numAccepted++; } } - cout << "Num accepted here = " << numAccepted << " / input = " << simu->NumPart << endl; + cout << "SELECTBOX: Num accepted here = " << numAccepted << " / input = " << simu->NumPart << " (after resubsampling)" << endl; } class PreselectParticles: public SimulationPreprocessor From e63a9f5021065d9c9785ace0b50bbbaa8d465561 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Tue, 19 Mar 2013 10:47:46 -0400 Subject: [PATCH 07/11] More pid_t --- zobov/voztie.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zobov/voztie.c b/zobov/voztie.c index 8e40aa3..5130449 100644 --- a/zobov/voztie.c +++ b/zobov/voztie.c @@ -107,8 +107,8 @@ int main(int argc, char *argv[]) { printf("Unable to open file %s.\n\n",partfile); exit(0); } - fread(&np2,1,sizeof(int),part); - fread(&nvp,1,sizeof(int),part); + fread(&np2,1,sizeof(pid_t),part); + fread(&nvp,1,sizeof(pid_t),part); /*printf("nvp = %d\n",nvp);fflush(stdout);*/ nvpsum += nvp; From dcce9c115616eb01d2f3b542e32e8149d5c70c11 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Tue, 19 Mar 2013 16:20:19 +0100 Subject: [PATCH 08/11] Reset nvpsum --- zobov/voztie.c | 1 + 1 file changed, 1 insertion(+) diff --git a/zobov/voztie.c b/zobov/voztie.c index 8e40aa3..7c334a3 100644 --- a/zobov/voztie.c +++ b/zobov/voztie.c @@ -98,6 +98,7 @@ int main(int argc, char *argv[]) { for (p=0;p Date: Tue, 19 Mar 2013 20:35:42 +0100 Subject: [PATCH 09/11] Added explanatory exception instead of a cryptic error for sample.dataFormat --- python_tools/void_python_tools/backend/launchers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python_tools/void_python_tools/backend/launchers.py b/python_tools/void_python_tools/backend/launchers.py index 1a0fb20..e056d84 100644 --- a/python_tools/void_python_tools/backend/launchers.py +++ b/python_tools/void_python_tools/backend/launchers.py @@ -162,6 +162,8 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None, dataFileLine = "gadget " + datafile elif sample.dataFormat == "sdf": dataFileLine = "sdf " + datafile + else: + raise ValueError("unknown dataFormat '%s'" % sample.dataFormat) iX = float(sample.mySubvolume[0]) iY = float(sample.mySubvolume[1]) From 189fa22c6bd968f791d0eaa40ab8192d0c94340f Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Tue, 19 Mar 2013 16:36:26 -0400 Subject: [PATCH 10/11] Generate a graceful error message instead of a Segmentation Fault --- c_tools/mock/generateMock.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/c_tools/mock/generateMock.cpp b/c_tools/mock/generateMock.cpp index cd92cce..5bb1a50 100644 --- a/c_tools/mock/generateMock.cpp +++ b/c_tools/mock/generateMock.cpp @@ -395,6 +395,11 @@ void saveBox(SimuData *&boxed, const std::string& outbox) int num_snapshots = *boxed->as("num_snapshots"); long *uniqueID = boxed->as("uniqueID"); + if (!f.is_valid()) + { + cerr << "Could not create parameter file '" << outbox << "'. Aborting." << endl; + exit(1); + } f.add_att("range_x_min", ranges[0]); f.add_att("range_x_max", ranges[1]); f.add_att("range_y_min", ranges[2]); From 5cbf68e0726bdfb117ba12f109765dbaed9af3f6 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Wed, 20 Mar 2013 04:55:59 -0400 Subject: [PATCH 11/11] Added a check that the subsampling value is the adequate one. --- c_tools/mock/generateMock.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/c_tools/mock/generateMock.cpp b/c_tools/mock/generateMock.cpp index 5bb1a50..bfd2b94 100644 --- a/c_tools/mock/generateMock.cpp +++ b/c_tools/mock/generateMock.cpp @@ -385,7 +385,7 @@ void buildBox(SimuData *simu, long num_targets, long loaded, } } -void saveBox(SimuData *&boxed, const std::string& outbox) +void saveBox(SimuData *&boxed, const std::string& outbox, generateMock_info& args_info) { double *ranges = boxed->as("ranges"); NcFile f(outbox.c_str(), NcFile::Replace, 0, 0, NcFile::Netcdf4); @@ -408,6 +408,7 @@ void saveBox(SimuData *&boxed, const std::string& outbox) f.add_att("range_z_max", ranges[5]); f.add_att("mask_index", -1); f.add_att("is_observation", 0); + f.add_att("data_subsampling", args_info.subsample_arg); NcDim *NumPart_dim = f.add_dim("numpart_dim", boxed->NumPart); NcDim *NumSnap_dim = f.add_dim("numsnap_dim", num_snapshots); @@ -451,6 +452,13 @@ void makeBoxFromParameter(SimuData *simu, SimuData* &boxed, generateMock_info& a boxed->time = simu->time; boxed->BoxSize = simu->BoxSize; + NcAtt *d_sub = f.get_att("data_subsampling"); + if (d_sub == 0 || d_sub->as_double(0) != args_info.subsample_arg) + { + cerr << "Parameter file was not generated with the same simulation subsampling argument. Particles will be different. Stop here." << endl; + exit(1); + } + NcVar *v_id = f.get_var("particle_ids"); NcVar *v_snap = f.get_var("snapshot_split"); long *edges1; @@ -715,7 +723,7 @@ int main(int argc, char **argv) delete[] efac; } - saveBox(simuOut, args_info.outputParameter_arg); + saveBox(simuOut, args_info.outputParameter_arg, args_info); generateOutput(simuOut, args_info.axis_arg, args_info.output_arg); delete preselector;