Add cleaner paths (#43)

* ignore venv

* ignore pylians if unavaible

* Edit loading paths

* Edit paths

* Stop having default paths

* Add nb

* Add glamdring paths

---------

Co-authored-by: Richard Stiskalek <dc-stis1@login8b.pri.cosma7.alces.network>
Co-authored-by: Richard Stiskalek <dc-stis1@login8a.pri.cosma7.alces.network>

.gitignore

@@ -16,3 +16,4 @@ csiborgtools.egg-info/*
 Pylians3/*
 scripts/plot_correlation.ipynb
 scripts/*.sh
+venv/

csiborgtools/__init__.py

@@ -13,3 +13,10 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 from csiborgtools import (read, match, utils, units, fits, field, clustering)  # noqa
+
+# Arguments to csiborgtools.read.CSiBORGPaths.
+paths_glamdring = {
+    "srcdir": "/mnt/extraspace/hdesmond/",
+    "dumpdir": "/mnt/extraspace/rstiskalek/csiborg/",
+    "mmain_path": "/mnt/zfsusers/hdesmond/Mmain",
+    "initmatch_path": "/mnt/extraspace/rstiskalek/csiborg/initmatch/"}

csiborgtools/field/__init__.py

@@ -12,4 +12,9 @@
 # You should have received a copy of the GNU General Public License along
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-from .density import DensityField  # noqa
+from warnings import warn
+try:
+    import MAS_library as MASL
+    from .density import DensityField
+except ImportError:
+    warn("MAS_library not found, `DensityField` will not be available", UserWarning)  # noqa

csiborgtools/read/halo_cat.py

@@ -99,9 +99,7 @@ class HaloCatalogue:
         Loads the data, merges with mmain, does various coordinate transforms.
         """
         # Load the processed data
-        fname = "ramses_out_{}_{}.npy".format(
+        data = numpy.load(self.paths.hcat_path(self.nsim))
-            str(self.nsim).zfill(5), str(self.nsnap).zfill(5))
-        data = numpy.load(join(self.paths.dumpdir, fname))
 
         # Load the mmain file and add it to the data
         mmain = read_mmain(self.nsim, self.paths.mmain_path)

csiborgtools/read/readsim.py

@@ -35,13 +35,13 @@ class CSiBORGPaths:
 
     Parameters
     ----------
-    srcdir : str, optional
+    srcdir : str
         Path to the folder where CSiBORG simulations are stored.
-    dumpdir : str, optional
+    dumpdir : str
         Path to the folder where files from `run_fit_halos` are stored.
-    mmain_path : str, optional
+    mmain_path : str
         Path to folder where mmain files are stored.
-    initmatch_path : str, optional
+    initmatch_path : str
         Path to the folder where particle ID match between the first and final
         snapshot is stored.
     """
@@ -50,16 +50,12 @@ class CSiBORGPaths:
     _mmain_path = None
     _initmatch_path = None
 
-    def __init__(self, srcdir="/mnt/extraspace/hdesmond/",
+    def __init__(self, srcdir=None, dumpdir=None, mmain_path=None,
-                 dumpdir="/mnt/extraspace/rstiskalek/csiborg/",
+                 initmatch_path=None):
-                 mmain_path="/mnt/zfsusers/hdesmond/Mmain",
+        self.srcdir = srcdir
-                 initmatch_path="/mnt/extraspace/rstiskalek/csiborg/initmatch/"):  # noqa
+        self.dumpdir = dumpdir
-        for path in [srcdir, dumpdir, mmain_path, initmatch_path]:
+        self.mmain_path = mmain_path
-            self._check_directory(path)
+        self.initmatch_path = initmatch_path
-        self._srcdir = srcdir
-        self._dumpdir = dumpdir
-        self._mmain_path = mmain_path
-        self._initmatch_path = initmatch_path
 
     @staticmethod
     def _check_directory(path):
@@ -75,8 +71,17 @@ class CSiBORGPaths:
         -------
         path : str
         """
+        if self._srcdir is None:
+            raise ValueError("`srcdir` is not set!")
         return self._srcdir
 
+    @srcdir.setter
+    def srcdir(self, path):
+        if path is None:
+            return
+        self._check_directory(path)
+        self._srcdir = path
+
     @property
     def dumpdir(self):
         """
@@ -86,8 +91,17 @@ class CSiBORGPaths:
         -------
         path : str
         """
+        if self._dumpdir is None:
+            raise ValueError("`dumpdir` is not set!")
         return self._dumpdir
 
+    @dumpdir.setter
+    def dumpdir(self, path):
+        if path is None:
+            return
+        self._check_directory(path)
+        self._dumpdir = path
+
     @property
     def temp_dumpdir(self):
         """
@@ -111,8 +125,17 @@ class CSiBORGPaths:
         -------
         path : str
         """
+        if self._mmain_path is None:
+            raise ValueError("`mmain_path` is not set!")
         return self._mmain_path
 
+    @mmain_path.setter
+    def mmain_path(self, path):
+        if path is None:
+            return
+        self._check_directory(path)
+        self._mmain_path = path
+
     @property
     def initmatch_path(self):
         """
@@ -123,8 +146,17 @@ class CSiBORGPaths:
         -------
         path : str
         """
+        if self._initmatch_path is None:
+            raise ValueError("`initmatch_path` is not set!")
         return self._initmatch_path
 
+    @initmatch_path.setter
+    def initmatch_path(self, path):
+        if path is None:
+            return
+        self._check_directory(path)
+        self._initmatch_path = path
+
     def ic_ids(self, tonew):
         """
         CSiBORG IC realisation IDs from the list of folders in `self.srcdir`.
@@ -211,7 +243,7 @@ class CSiBORGPaths:
         cdir = join(self.dumpdir, "initmatch")
         return join(cdir, "clump_{}_{}.npy".format(nsim, "particles"))
 
-    def snapshot_path(self, nsnap, nsim, tonew=False):
+    def snapshot_path(self, nsnap, nsim):
         """
         Path to a CSiBORG IC realisation snapshot.
 
@@ -221,16 +253,33 @@ class CSiBORGPaths:
             Snapshot index.
         nsim : int
             IC realisation index.
-        tonew : bool, optional
-            Whether to return the path to the '_new' IC realisation.
 
         Returns
         -------
         snappath : str
         """
+        if nsnap == 1:
+            tonew = True
         simpath = self.ic_path(nsim, tonew=tonew)
         return join(simpath, "output_{}".format(str(nsnap).zfill(5)))
 
+    def hcat_path(self, nsim):
+        """
+        Path to the final snapshot halo catalogue from `fit_halos.py`.
+
+        Parameters
+        ----------
+        nsim : int
+            IC realisation index.
+
+        Returns
+        -------
+        path : str
+        """
+        nsnap = str(max(self.get_snapshots(nsim))).zfill(5)
+        fname = "ramses_out_{}_{}.npy".format(str(self.nsim).zfill(5), nsnap)
+        return join(self.dumpdir, fname)
+
 
 ###############################################################################
 #                          Fortran readers                                    #
@@ -248,7 +297,7 @@ class ParticleReader:
     _paths = None
 
     def __init__(self, paths):
-        assert isinstance(paths, CSiBORGPaths)
+#        assert isinstance(paths, CSiBORGPaths)
         self._paths = paths
 
     @property
@@ -306,7 +355,7 @@ class ParticleReader:
             Opened part files.
         """
         snappath = self.paths.snapshot_path(nsnap, nsim)
-        ncpu = int(self.read_info()["ncpu"])
+        ncpu = int(self.read_info(nsnap, nsim)["ncpu"])
         nsnap = str(nsnap).zfill(5)
         if verbose:
             print("Reading in output `{}` with ncpu = `{}`."

scripts/crosspk.py

@@ -46,7 +46,7 @@ rank = comm.Get_rank()
 nproc = comm.Get_size()
 MAS = "CIC"  # mass asignment scheme
 
-paths = csiborgtools.read.CSiBORGPaths()
+paths = csiborgtools.read.CSiBORGPaths(**csiborgtools.paths_glamdring)
 box = csiborgtools.units.BoxUnits(paths)
 reader = csiborgtools.read.ParticleReader(paths)
 ics = paths.ic_ids(tonew=False)

scripts/fieldprop.py

@@ -60,7 +60,7 @@ fperm = join(dumpdir, "fields", fname + ".npy")
 dtype = {"names": ["delta", "phi"], "formats": [numpy.float32] * 2}
 
 # CSiBORG simulation paths
-paths = csiborgtools.read.CSiBORGPaths()
+paths = csiborgtools.read.CSiBORGPaths(**csiborgtools.paths_glamdring)
 ics = paths.ic_ids(tonew=False)
 nsims = len(ics)
 

scripts/fit_halos.py

@@ -34,7 +34,7 @@ comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
 nproc = comm.Get_size()
 
-paths = csiborgtools.read.CSiBORGPaths()
+paths = csiborgtools.read.CSiBORGPaths(**csiborgtools.paths_glamdring)
 dumpdir = "/mnt/extraspace/rstiskalek/csiborg/"
 loaddir = join(dumpdir, "temp")
 cols_collect = [("npart", numpy.int64), ("totpartmass", numpy.float64),
@@ -118,8 +118,7 @@ for i, nsim in enumerate(paths.ic_ids(tonew=False)):
         out_collected = csiborgtools.read.combine_splits(
             utils.Nsplits, nsnap, nsim, partreader, cols_collect,
             remove_splits=True, verbose=False)
-        fname = join(paths.dumpdir, "ramses_out_{}_{}.npy"
+        fname = paths.hcat_path(nsim)
-                     .format(str(nsim).zfill(5), str(nsnap).zfill(5)))
         print("Saving results to `{}`.".format(fname))
         numpy.save(fname, out_collected)
 

scripts/initmatch.py

@@ -44,25 +44,23 @@ parser = ArgumentParser()
 parser.add_argument("--dump_clumps", type=lambda x: bool(strtobool(x)))
 args = parser.parse_args()
 
-paths = csiborgtools.read.CSiBORGPaths()
+paths = csiborgtools.read.CSiBORGPaths(**csiborgtools.paths_glamdring)
 nsims = paths.ic_ids(tonew=True)
 
 # Output files
-dumpdir = "/mnt/extraspace/rstiskalek/csiborg/"
+ftemp = join(paths.dumpdir, "temp_initmatch", "temp_{}_{}_{}.npy")
-ftemp = join(dumpdir, "temp_initmatch", "temp_{}_{}_{}.npy")
+fpermcm = join(paths.dumpdir, "initmatch", "clump_{}_cm.npy")
-fpermcm = join(dumpdir, "initmatch", "clump_{}_cm.npy")
+fpermpart = join(paths.dumpdir, "initmatch", "clump_{}_particles.npy")
-fpermpart = join(dumpdir, "initmatch", "clump_{}_particles.npy")
 
 for nsim in nsims:
     if rank == 0:
         print("{}: reading simulation {}.".format(datetime.now(), nsim),
               flush=True)
-    nsnap_min = min(paths.get_snapshots(nsim))
     nsnap_max = max(paths.get_snapshots(nsim))
     reader = csiborgtools.read.ParticleReader(paths)
 
     # Read and sort the initial particle files by their particle IDs
-    part0 = reader.read_particle(nsnap_min, nsim, ["x", "y", "z", "M", "ID"],
+    part0 = reader.read_particle(1, nsim, ["x", "y", "z", "M", "ID"],
                                  verbose=False)
     part0 = part0[numpy.argsort(part0["ID"])]
 

scripts/knn_auto.py

@@ -60,7 +60,7 @@ ics = [7444, 7468, 7492, 7516, 7540, 7564, 7588, 7612, 7636, 7660, 7684,
        9820, 9844]
 dumpdir = "/mnt/extraspace/rstiskalek/csiborg/knn"
 fout = join(dumpdir, "auto", "knncdf_{}_{}.p")
-paths = csiborgtools.read.CSiBORGPaths()
+paths = csiborgtools.read.CSiBORGPaths(**csiborgtools.paths_glamdring)
 knncdf = csiborgtools.clustering.kNN_CDF()
 
 ###############################################################################

scripts/knn_cross.py

@@ -58,7 +58,7 @@ ics = [7444, 7468, 7492, 7516, 7540, 7564, 7588, 7612, 7636, 7660, 7684,
        9292, 9316, 9340, 9364, 9388, 9412, 9436, 9460, 9484, 9508, 9532,
        9556, 9580, 9604, 9628, 9652, 9676, 9700, 9724, 9748, 9772, 9796,
        9820, 9844]
-paths = csiborgtools.read.CSiBORGPaths()
+paths = csiborgtools.read.CSiBORGPaths(**csiborgtools.paths_glamdring)
 dumpdir = "/mnt/extraspace/rstiskalek/csiborg/knn"
 fout = join(dumpdir, "cross", "knncdf_{}_{}_{}.p")
 knncdf = csiborgtools.clustering.kNN_CDF()

scripts/singlematch.py

@@ -35,7 +35,7 @@ parser.add_argument("--sigma", type=float)
 args = parser.parse_args()
 
 # File paths
-paths = csiborgtools.read.CSiBORGPaths()
+paths = csiborgtools.read.CSiBORGPaths(**csiborgtools.paths_glamdring)
 fout = join(utils.dumpdir, "overlap",
             "cross_{}_{}.npz".format(args.nsim0, args.nsimx))
 smooth_kwargs = {"sigma": args.sigma, "mode": "constant", "cval": 0.0}

scripts/split_halos.py

@@ -32,7 +32,7 @@ comm = MPI.COMM_WORLD
 rank = comm.Get_rank()
 nproc = comm.Get_size()
 
-paths = csiborgtools.read.CSiBORGPaths()
+paths = csiborgtools.read.CSiBORGPaths(**csiborgtools.paths_glamdring)
 sims = paths.ic_ids(False)
 partcols = ["x", "y", "z", "vx", "vy", "vz", "M", "level"]