From 90fd74d907cf66f8a0f88bf22e04c0ad50b22aff Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Tue, 26 Jan 2016 23:04:01 +0100 Subject: [PATCH 01/10] Added Gadget2 support (untested!) --- c_tools/mock/generateMock.cpp | 6 +++++- c_tools/mock/generateMock.ggo | 1 + c_tools/mock/loaders/gadget_loader.cpp | 19 ++++++++++--------- c_tools/mock/loaders/simulation_loader.hpp | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/c_tools/mock/generateMock.cpp b/c_tools/mock/generateMock.cpp index bbc0b86..0a49265 100644 --- a/c_tools/mock/generateMock.cpp +++ b/c_tools/mock/generateMock.cpp @@ -695,7 +695,11 @@ int main(int argc, char **argv) } else if (args_info.gadget_given) { - loader = gadgetLoader(args_info.gadget_arg, 1/args_info.gadgetUnit_arg, NEED_POSITION|NEED_VELOCITY|NEED_GADGET_ID, preselector); + loader = gadgetLoader(args_info.gadget_arg, 1/args_info.gadgetUnit_arg, NEED_POSITION|NEED_VELOCITY|NEED_GADGET_ID, 1, preselector); + } + else if (args_info.gadget2_given) + { + loader = gadgetLoader(args_info.gadget_arg, 1/args_info.gadgetUnit_arg, NEED_POSITION|NEED_VELOCITY|NEED_GADGET_ID, 2, preselector); } else if (args_info.flash_given) { diff --git a/c_tools/mock/generateMock.ggo b/c_tools/mock/generateMock.ggo index 5041272..f907f59 100644 --- a/c_tools/mock/generateMock.ggo +++ b/c_tools/mock/generateMock.ggo @@ -8,6 +8,7 @@ option "ramsesBase" - "Base directory for ramses" string optional option "ramsesId" - "Ramses snapshot id" int optional option "gadget" - "Base name of gadget snapshot (without parallel writing extension)" string optional +option "gadget2" - "Base name of gadget snapshot (version 2, without parallel writing extension)" string optional option "flash" - "Base name for FLASH snapshot" string optional option "multidark" - "Base name for multidark snapshot" string optional diff --git a/c_tools/mock/loaders/gadget_loader.cpp b/c_tools/mock/loaders/gadget_loader.cpp index cdf7541..c2ca185 100644 --- a/c_tools/mock/loaders/gadget_loader.cpp +++ b/c_tools/mock/loaders/gadget_loader.cpp @@ -36,12 +36,13 @@ private: bool onefile; int _num_files; double unitMpc; + int gadgetFormat; SimuData *gadget_header; string snapshot_name; SimulationPreprocessor *preproc; public: - GadgetLoader(const string& basename, SimuData *header, int flags, bool singleFile, int _num, double unit, SimulationPreprocessor *p) - : snapshot_name(basename), load_flags(flags), onefile(singleFile), _num_files(_num), unitMpc(1/unit), gadget_header(header), preproc(p) + GadgetLoader(const string& basename, SimuData *header, int flags, bool singleFile, int _num, double unit, int gadgetFormat, SimulationPreprocessor *p) + : snapshot_name(basename), load_flags(flags), onefile(singleFile), _num_files(_num), unitMpc(1/unit), gadget_header(header), gadgetFormat(gadgetFormat), preproc(p) { } @@ -67,9 +68,9 @@ public: return 0; if (onefile) - d = loadGadgetMulti(snapshot_name.c_str(), -1, load_flags); + d = loadGadgetMulti(snapshot_name.c_str(), -1, load_flags, gadgetFormat); else - d = loadGadgetMulti(snapshot_name.c_str(), id, load_flags); + d = loadGadgetMulti(snapshot_name.c_str(), id, load_flags, gadgetFormat); if (d->Id != 0) { @@ -99,7 +100,7 @@ public: }; -SimulationLoader *gadgetLoader(const std::string& snapshot, double Mpc_unitLength, int flags, SimulationPreprocessor *p) +SimulationLoader *gadgetLoader(const std::string& snapshot, double Mpc_unitLength, int flags, int gadgetFormat, SimulationPreprocessor *p) { bool singleFile = false; int num_files; @@ -109,7 +110,7 @@ SimulationLoader *gadgetLoader(const std::string& snapshot, double Mpc_unitLengt try { - d = loadGadgetMulti(snapshot.c_str(), -1, 0); + d = loadGadgetMulti(snapshot.c_str(), -1, 0, gadgetFormat); singleFile = true; num_files = 1; } @@ -117,7 +118,7 @@ SimulationLoader *gadgetLoader(const std::string& snapshot, double Mpc_unitLengt { try { - d = loadGadgetMulti(snapshot.c_str(), 0, 0); + d = loadGadgetMulti(snapshot.c_str(), 0, 0, gadgetFormat); num_files = 0; } catch(const NoSuchFileException& e) @@ -135,7 +136,7 @@ SimulationLoader *gadgetLoader(const std::string& snapshot, double Mpc_unitLengt { try { - while ((d = loadGadgetMulti(snapshot.c_str(), num_files, 0)) != 0) + while ((d = loadGadgetMulti(snapshot.c_str(), num_files, 0, gadgetFormat)) != 0) { num_files++; delete d; @@ -146,5 +147,5 @@ SimulationLoader *gadgetLoader(const std::string& snapshot, double Mpc_unitLengt } } - return new GadgetLoader(snapshot, header, flags, singleFile, num_files, Mpc_unitLength, p); + return new GadgetLoader(snapshot, header, flags, singleFile, num_files, Mpc_unitLength, gadgetFormat, p); } diff --git a/c_tools/mock/loaders/simulation_loader.hpp b/c_tools/mock/loaders/simulation_loader.hpp index 149ee10..635b964 100644 --- a/c_tools/mock/loaders/simulation_loader.hpp +++ b/c_tools/mock/loaders/simulation_loader.hpp @@ -144,7 +144,7 @@ void delete_adaptor(void *ptr) // Unit length is the size of one Mpc in the simulation units -SimulationLoader *gadgetLoader(const std::string& snapshot, double Mpc_unitLength, int flags, SimulationPreprocessor *p); +SimulationLoader *gadgetLoader(const std::string& snapshot, double Mpc_unitLength, int flags, int gadgetFormat, SimulationPreprocessor *p); SimulationLoader *flashLoader(const std::string& snapshot, int flags, SimulationPreprocessor *p); SimulationLoader *multidarkLoader(const std::string& snapshot, SimulationPreprocessor *p); SimulationLoader *ramsesLoader(const std::string& snapshot, int baseid, bool double_precision, int flags, SimulationPreprocessor *p); From 555e0a4d5a0e63b672f4f270993b127b62e06bc5 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Tue, 26 Jan 2016 23:07:47 +0100 Subject: [PATCH 02/10] Imported newer version of some cosmotool files --- external/cosmotool/src/fortran.cpp | 30 +++- external/cosmotool/src/fortran.hpp | 13 +- external/cosmotool/src/loadGadget.cpp | 211 +++++++++++++++++++++----- external/cosmotool/src/loadGadget.hpp | 2 +- external/cosmotool/src/loadSimu.hpp | 76 +++++----- external/cosmotool/src/load_data.hpp | 8 +- 6 files changed, 260 insertions(+), 80 deletions(-) diff --git a/external/cosmotool/src/fortran.cpp b/external/cosmotool/src/fortran.cpp index 79641ac..e497ff1 100644 --- a/external/cosmotool/src/fortran.cpp +++ b/external/cosmotool/src/fortran.cpp @@ -1,5 +1,5 @@ /*+ -This is CosmoTool (./src/fortran.cpp) -- Copyright (C) Guilhem Lavaux (2007-2013) +This is CosmoTool (./src/fortran.cpp) -- Copyright (C) Guilhem Lavaux (2007-2014) guilhem.lavaux@gmail.com @@ -71,6 +71,16 @@ UnformattedRead::~UnformattedRead() delete f; } +int64_t UnformattedRead::position() const +{ + return f->tellg(); +} + +void UnformattedRead::seek(int64_t pos) +{ + f->seekg(pos, istream::beg); +} + // Todo implement primitive description void UnformattedRead::setOrdering(Ordering o) { @@ -112,7 +122,7 @@ void UnformattedRead::beginCheckpoint() checkPointRef = (cSize == Check_32bits) ? 4 : 8; checkPointAccum = 0; - checkPointRef = (cSize == Check_32bits) ? readInt32() : readInt64(); + checkPointRef = (cSize == Check_32bits) ? readUint32() : readInt64(); checkPointAccum = 0; if (f->eof()) @@ -144,7 +154,7 @@ void UnformattedRead::endCheckpoint(bool autodrop) void UnformattedRead::readOrderedBuffer(void *buffer, int size) throw (InvalidUnformattedAccess) { - if ((checkPointAccum+size) > checkPointRef) + if ((checkPointAccum+(uint64_t)size) > checkPointRef) throw InvalidUnformattedAccess(); f->read((char *)buffer, size); @@ -186,6 +196,20 @@ float UnformattedRead::readReal32() return a.f; } +uint32_t UnformattedRead::readUint32() + throw (InvalidUnformattedAccess) +{ + union + { + char b[4]; + uint32_t i; + } a; + + readOrderedBuffer(&a, 4); + + return a.i; +} + int32_t UnformattedRead::readInt32() throw (InvalidUnformattedAccess) { diff --git a/external/cosmotool/src/fortran.hpp b/external/cosmotool/src/fortran.hpp index d63e8d3..48b4a1d 100644 --- a/external/cosmotool/src/fortran.hpp +++ b/external/cosmotool/src/fortran.hpp @@ -1,5 +1,5 @@ /*+ -This is CosmoTool (./src/fortran.hpp) -- Copyright (C) Guilhem Lavaux (2007-2013) +This is CosmoTool (./src/fortran.hpp) -- Copyright (C) Guilhem Lavaux (2007-2014) guilhem.lavaux@gmail.com @@ -73,6 +73,8 @@ namespace CosmoTool // Todo implement primitive description void setOrdering(Ordering o); void setCheckpointSize(CheckpointSize cs); + + uint64_t getBlockSize() const { return checkPointRef; } void beginCheckpoint() throw (InvalidUnformattedAccess,EndOfFileException); @@ -83,6 +85,8 @@ namespace CosmoTool throw (InvalidUnformattedAccess); float readReal32() throw (InvalidUnformattedAccess); + uint32_t readUint32() + throw (InvalidUnformattedAccess); int32_t readInt32() throw (InvalidUnformattedAccess); int64_t readInt64() @@ -91,6 +95,11 @@ namespace CosmoTool void skip(int64_t off) throw (InvalidUnformattedAccess); + int64_t position() const; + void seek(int64_t pos); + + void readOrderedBuffer(void *buffer, int size) + throw (InvalidUnformattedAccess); protected: bool swapOrdering; CheckpointSize cSize; @@ -98,8 +107,6 @@ namespace CosmoTool uint64_t checkPointAccum; std::ifstream *f; - void readOrderedBuffer(void *buffer, int size) - throw (InvalidUnformattedAccess); }; class UnformattedWrite: public FortranTypes diff --git a/external/cosmotool/src/loadGadget.cpp b/external/cosmotool/src/loadGadget.cpp index f9f1072..9f77bb4 100644 --- a/external/cosmotool/src/loadGadget.cpp +++ b/external/cosmotool/src/loadGadget.cpp @@ -1,5 +1,5 @@ /*+ -This is CosmoTool (./src/loadGadget.cpp) -- Copyright (C) Guilhem Lavaux (2007-2013) +This is CosmoTool (./src/loadGadget.cpp) -- Copyright (C) Guilhem Lavaux (2007-2014) guilhem.lavaux@gmail.com @@ -38,6 +38,8 @@ knowledge of the CeCILL license and that you accept its terms. #include #include #include +#include +#include #include "load_data.hpp" #include "loadGadget.hpp" #include "fortran.hpp" @@ -65,9 +67,16 @@ void loadGadgetHeader(UnformattedRead *f, GadgetHeader& h, SimuData *data, int i data->Omega_M = h.Omega0 = f->readReal64(); data->Omega_Lambda = h.OmegaLambda = f->readReal64(); data->Hubble = h.HubbleParam = f->readReal64(); + (int)f->readInt32(); // stellarage + (int)f->readInt32(); // metals + for (int i = 0; i < 6; i++) + h.npartTotal[i] |= ((unsigned long)f->readInt32()) << 32; + (int)f->readInt32(); // entropy instead of u + h.flag_doubleprecision = f->readInt32(); + f->endCheckpoint(true); - long NumPart = 0, NumPartTotal = 0; + ssize_t NumPart = 0, NumPartTotal = 0; for(int k=0; k<6; k++) { NumPart += h.npart[k]; @@ -77,6 +86,12 @@ void loadGadgetHeader(UnformattedRead *f, GadgetHeader& h, SimuData *data, int i data->TotalNumPart = NumPartTotal; } +template +T myRead64(UnformattedRead *f) { return f->readReal64(); } + +template +T myRead32(UnformattedRead *f) { return f->readReal32(); } + SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, int loadflags, int GadgetFormat, SimuFilter filter) @@ -86,6 +101,14 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, UnformattedRead *f; GadgetHeader h; float velmul; + boost::function0 readToDouble; + boost::function0 readToSingle; + long float_size; + + if (GadgetFormat > 2) { + cerr << "Unknown gadget format" << endl; + return 0; + } if (id >= 0) { int k = snprintf(0, 0, "%s.%d", fname, id)+1; @@ -96,7 +119,7 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, if (f == 0) return 0; - delete out_fname; + delete[] out_fname; } else { @@ -106,27 +129,68 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, } + typedef std::map BlockMap; + BlockMap blockTable; + + if (GadgetFormat == 2) { + int64_t startBlock = 0; + char block[5]; + int32_t blocksize; + + try { + while (true) { + f->beginCheckpoint(); + f->readOrderedBuffer(block, 4); + block[4] = 0; + blocksize = f->readInt32(); + f->endCheckpoint(); + blockTable[block] = f->position(); + f->skip(blocksize); + } + } catch (EndOfFileException&) {} + + f->seek(startBlock); + } + data = new SimuData; if (data == 0) { delete f; return 0; } - long NumPart = 0, NumPartTotal = 0; + ssize_t NumPart = 0, NumPartTotal = 0; +#define ENSURE(name) { \ + if (GadgetFormat == 2) { \ + BlockMap::iterator iter = blockTable.find(name); \ + if (iter == blockTable.end()) { \ + std::cerr << "GADGET2: Cannot find block named '" << name << "'" << endl; \ + if (data) delete data; \ + delete f; \ + return 0; \ + } \ + f->seek(iter->second); \ + } \ + } + try { + ENSURE("HEAD"); loadGadgetHeader(f, h, data, id); - if (GadgetFormat == 1) - velmul = sqrt(h.time); - else if (GadgetFormat == 2) - velmul = 1/(h.time); - else { - cerr << "unknown gadget format" << endl; - abort(); + velmul = sqrt(h.time); + + if (h.flag_doubleprecision) { + //cout << "Gadget format with double precision" << endl; + readToDouble = boost::bind(myRead64, f); + readToSingle = boost::bind(myRead64, f); + float_size = sizeof(double); + } else { + //cout << "Gadget format with single precision" << endl; + readToDouble = boost::bind(myRead32, f); + readToSingle = boost::bind(myRead32, f); + float_size = sizeof(float); } - NumPart = data->NumPart; NumPartTotal = data->TotalNumPart; } @@ -138,34 +202,47 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, return 0; } + if (loadflags & NEED_TYPE) { int p = 0; - + data->type = new int[data->NumPart]; for (int k = 0; k < 6; k++) - for (int n = 0; n < h.npart[k]; n++,p++) - data->type[p] = k; + for (int n = 0; n < h.npart[k]; n++,p++) + data->type[p] = k; } if (loadflags & NEED_POSITION) { for (int i = 0; i < 3; i++) { - data->Pos[i] = new float[data->NumPart]; - if (data->Pos[i] == 0) { - delete data; - return 0; - } + data->Pos[i] = new float[data->NumPart]; + if (data->Pos[i] == 0) { + delete data; + return 0; } + } try { + ENSURE("POS "); f->beginCheckpoint(); + if (f->getBlockSize() != NumPart*float_size*3) { + // Check that single would work + if (f->getBlockSize() == NumPart*sizeof(float)*3) { + // Change to single + cout << "Change back to single. Buggy header." << endl; + readToDouble = boost::bind(myRead32, f); + readToSingle = boost::bind(myRead32, f); + float_size = sizeof(float); + } + } for(int k = 0, p = 0; k < 6; k++) { for(int n = 0; n < h.npart[k]; n++) { - data->Pos[0][p] = f->readReal32(); - data->Pos[1][p] = f->readReal32(); - data->Pos[2][p] = f->readReal32(); +// if ((n%1000000)==0) cout << n << endl; + data->Pos[0][p] = readToSingle(); + data->Pos[1][p] = readToSingle(); + data->Pos[2][p] = readToSingle(); p++; } } @@ -173,15 +250,16 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, } catch (const InvalidUnformattedAccess& e) { - cerr << "Invalid format while reading positions" << endl; - delete f; - delete data; - return 0; + cerr << "Invalid format while reading positions" << endl; + delete f; + delete data; + return 0; } } else { + long float_size = (h.flag_doubleprecision) ? sizeof(double) : sizeof(float); // Skip positions - f->skip(NumPart * 3 * sizeof(float) + 2*4); + f->skip(NumPart * 3 * float_size + 2*4); } if (loadflags & NEED_VELOCITY) { @@ -198,13 +276,15 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, try { + ENSURE("VEL "); f->beginCheckpoint(); for(int k = 0, p = 0; k < 6; k++) { for(int n = 0; n < h.npart[k]; n++) { // THIS IS GADGET 1 - data->Vel[0][p] = f->readReal32()*velmul; - data->Vel[1][p] = f->readReal32()*velmul; - data->Vel[2][p] = f->readReal32()*velmul; +// if ((n%1000000)==0) cout << n << endl; + data->Vel[0][p] = readToSingle()*velmul; + data->Vel[1][p] = readToSingle()*velmul; + data->Vel[2][p] = readToSingle()*velmul; p++; } } @@ -222,13 +302,14 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, /// // TODO: FIX THE UNITS OF THESE FUNKY VELOCITIES !!! } else { // Skip velocities - f->skip(NumPart*3*sizeof(float)+2*4); + f->skip(NumPart*3*float_size+2*4); } // Skip ids if (loadflags & NEED_GADGET_ID) { try { + ENSURE("ID "); f->beginCheckpoint(); data->Id = new long[data->NumPart]; if (data->Id == 0) @@ -261,11 +342,66 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, f->skip(h.npart[k]*4); } + if (loadflags & NEED_MASS) { + bool do_load = false; + + for (int k = 0; k < 6; k++) + { + do_load = do_load || ((h.mass[k] == 0)&&(h.npart[k]>0)); + } + + try + { + long l = 0; + if (do_load) { + ENSURE("MASS"); + f->beginCheckpoint(); + } + data->Mass = new float[NumPart]; + for (int k = 0; k < 6; k++) + { + if (h.mass[k] == 0) { + for(int n = 0; n < h.npart[k]; n++) + { +// if ((n%1000000)==0) cout << n << endl; + data->Mass[l++] = readToSingle(); + } + } else { + for(int n = 0; n < h.npart[k]; n++) + { + if ((n%1000000)==0) cout << n << endl; + data->Mass[l++] = h.mass[k]; + } + } + } + if (do_load) + f->endCheckpoint(); + } + catch (const InvalidUnformattedAccess& e) + { + cerr << "Invalid unformatted access while reading ID" << endl; + delete f; + delete data; + return 0; + } + catch (const EndOfFileException& e) + { + for (int k = 0; k < 6; k++) + cerr << "mass[" << k << "] = " << h.mass[k] << endl; + } + } else { + f->skip(2*4); + for (int k = 0; k < 6; k++) + if (h.mass[k] == 0) + f->skip(h.npart[k]*4); + } + delete f; return data; } +#undef ENSURE void CosmoTool::writeGadget(const std::string& fname, SimuData *data, int GadgetFormat) @@ -274,12 +410,16 @@ void CosmoTool::writeGadget(const std::string& fname, SimuData *data, int Gadget int npart[6], npartTotal[6]; float mass[6]; - if (data->Pos[0] == 0 || data->Vel[0] == 0 || data->Id == 0) + if (data->Pos[0] == 0 || data->Vel[0] == 0 || data->Id == 0) { + cerr << "Invalid simulation data array" << endl; return; + } f = new UnformattedWrite(fname); - if (f == 0) + if (f == 0) { + cerr << "Cannot create file" << endl; return; + } for (int i = 0; i < 6; i++) { @@ -321,8 +461,7 @@ void CosmoTool::writeGadget(const std::string& fname, SimuData *data, int Gadget f->endCheckpoint(); float velmul = 1.0; - if (GadgetFormat == 1) - velmul = sqrt(data->time); + velmul = sqrt(data->time); f->beginCheckpoint(); for(int n = 0; n < data->NumPart; n++) { diff --git a/external/cosmotool/src/loadGadget.hpp b/external/cosmotool/src/loadGadget.hpp index e05a187..adbc818 100644 --- a/external/cosmotool/src/loadGadget.hpp +++ b/external/cosmotool/src/loadGadget.hpp @@ -1,5 +1,5 @@ /*+ -This is CosmoTool (./src/loadGadget.hpp) -- Copyright (C) Guilhem Lavaux (2007-2013) +This is CosmoTool (./src/loadGadget.hpp) -- Copyright (C) Guilhem Lavaux (2007-2014) guilhem.lavaux@gmail.com diff --git a/external/cosmotool/src/loadSimu.hpp b/external/cosmotool/src/loadSimu.hpp index 1c360a4..2eb5ac8 100644 --- a/external/cosmotool/src/loadSimu.hpp +++ b/external/cosmotool/src/loadSimu.hpp @@ -1,5 +1,5 @@ /*+ -This is CosmoTool (./src/loadSimu.hpp) -- Copyright (C) Guilhem Lavaux (2007-2013) +This is CosmoTool (./src/loadSimu.hpp) -- Copyright (C) Guilhem Lavaux (2007-2014) guilhem.lavaux@gmail.com @@ -36,6 +36,7 @@ knowledge of the CeCILL license and that you accept its terms. #ifndef __COSMOTOOLBOX_HPP #define __COSMOTOOLBOX_HPP +#include #include #include @@ -45,6 +46,8 @@ namespace CosmoTool static const int NEED_POSITION = 2; static const int NEED_VELOCITY = 4; static const int NEED_TYPE = 8; + static const int NEED_MASS = 16; + static const int NEED_DOUBLE_PRECISION = 32; struct SimuParticle { @@ -64,6 +67,8 @@ namespace CosmoTool typedef void (*FreeFunction)(void *); typedef std::map > AttributeMap; + bool noAuto; + float BoxSize; float time; float Hubble; @@ -71,59 +76,60 @@ namespace CosmoTool float Omega_M; float Omega_Lambda; - long NumPart; - long TotalNumPart; + ssize_t NumPart; + ssize_t TotalNumPart; long *Id; float *Pos[3]; float *Vel[3]; + float *Mass; int *type; AttributeMap attributes; public: - SimuData() : Id(0),NumPart(0),type(0) { Pos[0]=Pos[1]=Pos[2]=0; Vel[0]=Vel[1]=Vel[2]=0; } + SimuData() : Mass(0), Id(0),NumPart(0),type(0),noAuto(false) { Pos[0]=Pos[1]=Pos[2]=0; Vel[0]=Vel[1]=Vel[2]=0; } ~SimuData() { - for (int j = 0; j < 3; j++) - { - if (Pos[j]) - delete[] Pos[j]; - if (Vel[j]) - delete[] Vel[j]; - } - if (type) - delete[] type; - if (Id) - delete[] Id; - - for (AttributeMap::iterator i = attributes.begin(); - i != attributes.end(); - ++i) - { - if (i->second.second) - i->second.second(i->second.first); - } + if (!noAuto) { + for (int j = 0; j < 3; j++) { + if (Pos[j]) + delete[] Pos[j]; + if (Vel[j]) + delete[] Vel[j]; + } + if (type) + delete[] type; + if (Id) + delete[] Id; + if (Mass) + delete[] Mass; + } + for (AttributeMap::iterator i = attributes.begin(); + i != attributes.end(); + ++i) { + if (i->second.second) + i->second.second(i->second.first); + } } template T *as(const std::string& n) { - AttributeMap::iterator i = attributes.find(n); - if (i == attributes.end()) - return 0; - - return reinterpret_cast(i->second.first); + AttributeMap::iterator i = attributes.find(n); + if (i == attributes.end()) + return 0; + + return reinterpret_cast(i->second.first); } void new_attribute(const std::string& n, void *p, FreeFunction free_func) { - AttributeMap::iterator i = attributes.find(n); - if (i != attributes.end()) - { - if (i->second.second) - i->second.second(i->second.first); - } - attributes[n] = std::make_pair(p, free_func); + AttributeMap::iterator i = attributes.find(n); + if (i != attributes.end()) { + if (i->second.second) + i->second.second(i->second.first); + } + attributes[n] = std::make_pair(p, free_func); } }; diff --git a/external/cosmotool/src/load_data.hpp b/external/cosmotool/src/load_data.hpp index 6be3bbb..36a2422 100644 --- a/external/cosmotool/src/load_data.hpp +++ b/external/cosmotool/src/load_data.hpp @@ -1,5 +1,5 @@ /*+ -This is CosmoTool (./src/load_data.hpp) -- Copyright (C) Guilhem Lavaux (2007-2013) +This is CosmoTool (./src/load_data.hpp) -- Copyright (C) Guilhem Lavaux (2007-2014) guilhem.lavaux@gmail.com @@ -55,7 +55,11 @@ namespace CosmoTool { double Omega0; double OmegaLambda; double HubbleParam; - char fill[256- 6*4- 6*8- 2*8- 2*4- 6*4- 2*4 - 4*8]; /* fills to 256 Bytes */ + int flag_doubleprecision; + int flag_ic_info; + float lpt_scalingfactor; + char fill[18]; /*!< fills to 256 Bytes */ + char names[15][2]; }; struct ParticleState From 75229b865ada8d8ba8bf1177cd108a79500417a3 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Wed, 27 Jan 2016 10:50:05 +0100 Subject: [PATCH 03/10] Added gadget2 in python tools --- python_tools/void_python_tools/backend/launchers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python_tools/void_python_tools/backend/launchers.py b/python_tools/void_python_tools/backend/launchers.py index 47d9120..90cb0b0 100644 --- a/python_tools/void_python_tools/backend/launchers.py +++ b/python_tools/void_python_tools/backend/launchers.py @@ -165,6 +165,8 @@ def launchGenerate(sample, binPath, workDir=None, inputDataDir=None, dataFileLine = "multidark " + datafile elif sample.dataFormat == "gadget": dataFileLine = "gadget " + datafile + elif sample.dataFormat == "gadget2": + dataFileLine = "gadget2 " + datafile elif sample.dataFormat == "ahf": dataFileLine = "gadget " + datafile elif sample.dataFormat == "sdf": From 859e82cafdd705d8d6d95733ca7e5db7dba9bde5 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Wed, 27 Jan 2016 11:04:32 +0100 Subject: [PATCH 04/10] Fixed argument passingin mock --- c_tools/mock/generateMock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c_tools/mock/generateMock.cpp b/c_tools/mock/generateMock.cpp index 0a49265..e5d525d 100644 --- a/c_tools/mock/generateMock.cpp +++ b/c_tools/mock/generateMock.cpp @@ -699,7 +699,7 @@ int main(int argc, char **argv) } else if (args_info.gadget2_given) { - loader = gadgetLoader(args_info.gadget_arg, 1/args_info.gadgetUnit_arg, NEED_POSITION|NEED_VELOCITY|NEED_GADGET_ID, 2, preselector); + loader = gadgetLoader(args_info.gadget2_arg, 1/args_info.gadgetUnit_arg, NEED_POSITION|NEED_VELOCITY|NEED_GADGET_ID, 2, preselector); } else if (args_info.flash_given) { From 2fc406af458980e1969c5edb346aae6cb54f2544 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Wed, 27 Jan 2016 14:59:50 +0100 Subject: [PATCH 05/10] Fixlets to have cosmotool compile --- external/cosmotool/CMakeLists.txt | 4 ++++ external/cosmotool/src/CMakeLists.txt | 2 +- external/external_build.cmake | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/external/cosmotool/CMakeLists.txt b/external/cosmotool/CMakeLists.txt index 7faca8a..2cd377c 100644 --- a/external/cosmotool/CMakeLists.txt +++ b/external/cosmotool/CMakeLists.txt @@ -47,6 +47,10 @@ if (ENABLE_SHARP) endif (ENABLE_SHARP) +find_package(Boost 1.53) +mark_as_advanced(Boost_INCLUDE_DIRS Boost_LIBRARIES) + + set(HDF5_FIND_COMPONENTS HL CXX) if(HDF5_ROOTDIR) SET(ENV{HDF5_ROOT} ${HDF5_ROOTDIR}) diff --git a/external/cosmotool/src/CMakeLists.txt b/external/cosmotool/src/CMakeLists.txt index 8901027..e28a2d8 100644 --- a/external/cosmotool/src/CMakeLists.txt +++ b/external/cosmotool/src/CMakeLists.txt @@ -55,7 +55,7 @@ SET(CosmoTool_SRCS ${CosmoTool_SRCS} growthFactor.hpp ) -include_directories(${GSL_INCLUDE_PATH} ${NETCDF_INCLUDE_PATH} ${NETCDFCPP_INCLUDE_PATH} ${CMAKE_BINARY_DIR}/src) +include_directories(${Boost_INCLUDE_DIRS} ${GSL_INCLUDE_PATH} ${NETCDF_INCLUDE_PATH} ${NETCDFCPP_INCLUDE_PATH} ${CMAKE_BINARY_DIR}/src) set(CosmoTool_LIBS ${NETCDFCPP_LIBRARY} ${NETCDF_LIBRARY} ${GSL_LIBRARIES}) if (HDF5_FOUND) diff --git a/external/external_build.cmake b/external/external_build.cmake index bce5995..88ce453 100644 --- a/external/external_build.cmake +++ b/external/external_build.cmake @@ -241,6 +241,8 @@ ExternalProject_Add(cosmotool -DNETCDF_LIBRARY=${NETCDF_LIBRARY} -DNETCDFCPP_LIBRARY=${NETCDFCPP_LIBRARY} -DENABLE_SHARP=OFF + -DBOOST_INCLUDEDIR=${Boost_INCLUDE_DIRS} + ) SET(COSMOTOOL_LIBRARY ${CMAKE_BINARY_DIR}/ext_build/cosmotool/lib/libCosmoTool.a) set(COSMOTOOL_INCLUDE_PATH ${CMAKE_BINARY_DIR}/ext_build/cosmotool/include) From 3ce4178322d5f7063e0a4aadf3804acf65420716 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Wed, 27 Jan 2016 15:05:22 +0100 Subject: [PATCH 06/10] Verbose block parsing --- external/cosmotool/src/loadGadget.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/external/cosmotool/src/loadGadget.cpp b/external/cosmotool/src/loadGadget.cpp index 9f77bb4..5fb9f09 100644 --- a/external/cosmotool/src/loadGadget.cpp +++ b/external/cosmotool/src/loadGadget.cpp @@ -146,6 +146,7 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, f->endCheckpoint(); blockTable[block] = f->position(); f->skip(blocksize); + cout << "GADGET: Block '" << block << "', position is " << blockTable[block] << endl; } } catch (EndOfFileException&) {} From 8e774afb05b2818319f7239fc4040ea2bdbdf67d Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Thu, 28 Jan 2016 16:09:42 +0100 Subject: [PATCH 07/10] Fix in cosmotool for Gadget2 file format --- external/cosmotool/src/fortran.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/external/cosmotool/src/fortran.cpp b/external/cosmotool/src/fortran.cpp index e497ff1..679f409 100644 --- a/external/cosmotool/src/fortran.cpp +++ b/external/cosmotool/src/fortran.cpp @@ -79,6 +79,7 @@ int64_t UnformattedRead::position() const void UnformattedRead::seek(int64_t pos) { f->seekg(pos, istream::beg); + checkPointRef = checkPointAccum = 0; } // Todo implement primitive description From f7b9f4493df6e5e47ea166ec0e18b9298c07744c Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Fri, 29 Jan 2016 14:57:55 +0100 Subject: [PATCH 08/10] Clear IOStream error bits to allow I/O operations again --- external/cosmotool/src/fortran.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/external/cosmotool/src/fortran.cpp b/external/cosmotool/src/fortran.cpp index 679f409..716d57b 100644 --- a/external/cosmotool/src/fortran.cpp +++ b/external/cosmotool/src/fortran.cpp @@ -78,6 +78,7 @@ int64_t UnformattedRead::position() const void UnformattedRead::seek(int64_t pos) { + f->clear(); f->seekg(pos, istream::beg); checkPointRef = checkPointAccum = 0; } From 959f631b9ce2a593a0e3db6b8f61b767565c7c76 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Mon, 1 Feb 2016 17:52:15 +0100 Subject: [PATCH 09/10] Removed verbose header print in Gadget loader --- external/cosmotool/src/loadGadget.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/external/cosmotool/src/loadGadget.cpp b/external/cosmotool/src/loadGadget.cpp index 5fb9f09..9f77bb4 100644 --- a/external/cosmotool/src/loadGadget.cpp +++ b/external/cosmotool/src/loadGadget.cpp @@ -146,7 +146,6 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, f->endCheckpoint(); blockTable[block] = f->position(); f->skip(blocksize); - cout << "GADGET: Block '" << block << "', position is " << blockTable[block] << endl; } } catch (EndOfFileException&) {} From 9b80d84804f5b5d1b595f4240f733d72578725b1 Mon Sep 17 00:00:00 2001 From: Guilhem Lavaux Date: Fri, 5 Feb 2016 17:12:48 +0100 Subject: [PATCH 10/10] Support 64 bits ID in Gadget --- external/cosmotool/src/loadGadget.cpp | 40 ++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/external/cosmotool/src/loadGadget.cpp b/external/cosmotool/src/loadGadget.cpp index 9f77bb4..171e7d8 100644 --- a/external/cosmotool/src/loadGadget.cpp +++ b/external/cosmotool/src/loadGadget.cpp @@ -92,6 +92,16 @@ T myRead64(UnformattedRead *f) { return f->readReal64(); } template T myRead32(UnformattedRead *f) { return f->readReal32(); } +template +T myReadI64(UnformattedRead *f) { return f->readInt64(); } + +template +T myReadI32(UnformattedRead *f) { return f->readInt32(); } + +struct BlockInfo { + int64_t position, size; +}; + SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, int loadflags, int GadgetFormat, SimuFilter filter) @@ -103,6 +113,7 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, float velmul; boost::function0 readToDouble; boost::function0 readToSingle; + boost::function0 readID; long float_size; if (GadgetFormat > 2) { @@ -129,7 +140,7 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, } - typedef std::map BlockMap; + typedef std::map BlockMap; BlockMap blockTable; if (GadgetFormat == 2) { @@ -144,7 +155,8 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, block[4] = 0; blocksize = f->readInt32(); f->endCheckpoint(); - blockTable[block] = f->position(); + blockTable[block].position = f->position(); + blockTable[block].size = blocksize; f->skip(blocksize); } } catch (EndOfFileException&) {} @@ -159,19 +171,22 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, } ssize_t NumPart = 0, NumPartTotal = 0; -#define ENSURE(name) { \ +#define ENSURE2(name,out_sz) { \ if (GadgetFormat == 2) { \ BlockMap::iterator iter = blockTable.find(name); \ + int64_t sz; \ if (iter == blockTable.end()) { \ std::cerr << "GADGET2: Cannot find block named '" << name << "'" << endl; \ if (data) delete data; \ delete f; \ return 0; \ } \ - f->seek(iter->second); \ + f->seek(iter->second.position); \ + sz = iter->second.size; \ + out_sz = sz;\ } \ } - +#define ENSURE(name) ENSURE2(name,sz); try { @@ -309,7 +324,18 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, if (loadflags & NEED_GADGET_ID) { try { - ENSURE("ID "); + int64_t idSize; + ENSURE2("ID ", idSize); + + if (idSize / data->NumPart == 8) { + readID = boost::bind(myReadI64, f); + } else + if (idSize / data->NumPart == 4) { + readID = boost::bind(myReadI32, f); + } else { + throw InvalidUnformattedAccess(); + } + f->beginCheckpoint(); data->Id = new long[data->NumPart]; if (data->Id == 0) @@ -323,7 +349,7 @@ SimuData *CosmoTool::loadGadgetMulti(const char *fname, int id, { for(int n = 0; n < h.npart[k]; n++) { - data->Id[p] = f->readInt32(); + data->Id[p] = readID(); p++; } }