Fix paths (#95)

* Fix args

* Fix paths

* Get rid of quijote backup

csiborgtools/read/paths.py

@@ -17,8 +17,9 @@ CSiBORG paths manager.
 """
 from glob import glob
 from os import makedirs
-from os.path import basename, isdir, join
+from os.path import isdir, join
 from warnings import warn
+from re import search
 
 import numpy
 
@@ -66,7 +67,7 @@ class Paths:
 
         self.postdir = postdir
 
-    def get_ics(self, simname, from_quijote_backup=False):
+    def get_ics(self, simname):
         """
         Get available IC realisation IDs for a given simulation.
 
@@ -74,97 +75,33 @@ class Paths:
         ----------
         simname : str
             Simulation name.
-        from_quijote_backup : bool, optional
-            Whether to return the ICs from the Quijote backup.
 
         Returns
         -------
         ids : 1-dimensional array
         """
-        if simname == "csiborg":
+        if simname == "csiborg1":
             files = glob(join(self.csiborg1_srcdir, "chain_*"))
-            files = [int(basename(f).replace("chain_", "") for f in files)]
+            files = [int(search(r'chain_(\d+)', f).group(1)) for f in files]
         elif simname == "csiborg2_main":
             files = glob(join(self.csiborg2_main_srcdir, "chain_*"))
-            files = [int(basename(f).replace("chain_", "") for f in files)]
+            files = [int(search(r'chain_(\d+)', f).group(1)) for f in files]
         elif simname == "csiborg2_random":
-            raise NotImplementedError("TODO")
+            files = glob(join(self.csiborg2_random_srcdir, "chain_*"))
+            files = [int(search(r'chain_(\d+)', f).group(1)) for f in files]
         elif simname == "csiborg2_varysmall":
-            raise NotImplementedError("TODO")
+            files = glob(join(self.csiborg2_varysmall_srcdir, "chain_*"))
-        elif simname == "quijote" or simname == "quijote_full":
+            files = [int(search(r'chain_16417(\d+)', f).group(1))
-            if from_quijote_backup:
+                     for f in files]
-                files = glob(join(self.quijote_dir, "halos_backup", "*"))
+        elif simname == "quijote":
-            else:
+            files = glob(join(self.quijote_dir, "fiducial_processed",
-                warn(("Taking only the snapshots that also have "
+                              "chain_*"))
-                     "a FoF catalogue!"))
+            files = [int(search(r'chain_(\d+)', f).group(1)) for f in files]
-                files = glob(join(self.quijote_dir, "Snapshots_fiducial", "*"))
-            files = [int(f.split("/")[-1]) for f in files]
-            files = [f for f in files if f < 100]
         else:
             raise ValueError(f"Unknown simulation name `{simname}`.")
 
         return numpy.sort(files)
 
-    def snapshots(self, nsim, simname):
-        if simname == "csiborg":
-            fname = "ramses_out_{}"
-            if tonew:
-                fname += "_new"
-                return join(self.postdir, "output", fname.format(nsim))
-            return join(self.csiborg1_srcdir, fname.format(nsim))
-        elif simname == "csiborg2_main":
-            return join(self.csiborg2_main_srcdir, f"chain_{nsim}", "output")
-        elif simname == "csiborg2_random":
-            raise NotImplementedError("TODO")
-        elif simname == "csiborg2_varysmall":
-            raise NotImplementedError("TODO")
-        elif simname == "quijote":
-            return join(self.quijote_dir, "Snapshots_fiducial", str(nsim))
-        else:
-            raise ValueError(f"Unknown simulation name `{simname}`.")
-
-    def snapshot(self, nsnap, nsim, simname):
-        """
-        Path to an IC realisation snapshot.
-
-        Parameters
-        ----------
-        nsnap : int
-            Snapshot index. For Quijote, `-1` indicates the IC snapshot.
-        nsim : inlot
-            IC realisation index.
-        simname : str
-            Simulation name.
-
-        Returns
-        -------
-        str
-        """
-        if simname == "csiborg":
-            return join(self.csiborg1_srcdir, f"chain_{nsim}",
-                        f"snapshot_{str(nsnap).zfill(5)}")
-        elif simname == "csiborg2_main":
-            return join(self.csiborg1_srcdir, f"chain_{nsim}",
-                        f"snapshot_{str(nsnap).zfill(5)}")
-        elif simname == "csiborg2_random":
-            raise NotImplementedError("TODO")
-        elif simname == "csiborg2_varysmall":
-            raise NotImplementedError("TODO")
-        elif simname == "quijote":
-            raise NotImplementedError("TODO")
-        else:
-            raise ValueError(f"Unknown simulation name `{simname}`.")
-
-
-        # simpath = self.snapshots(nsim, simname, tonew=nsnap == 1)
-        # if simname == "csiborg":
-        #     return join(simpath, f"output_{str(nsnap).zfill(5)}")
-        # else:
-        #     if nsnap == -1:
-        #         return join(simpath, "ICs", "ics")
-        #     nsnap = str(nsnap).zfill(3)
-        #     return join(simpath, f"snapdir_{nsnap}", f"snap_{nsnap}")
-
     def get_snapshots(self, nsim, simname):
         """
         List of available snapshots of simulation.
@@ -180,51 +117,79 @@ class Paths:
         -------
         snapshots : 1-dimensional array
         """
-        simpath = self.snapshots(nsim, simname, tonew=False)
+        # simpath = self.snapshots(nsim, simname, tonew=False)
 
-        if simname == "csiborg":
+        if simname == "csiborg1":
-            # Get all files in simpath that start with output_
+            snaps = glob(join(self.csiborg1_srcdir, f"chain_{nsim}",
-            snaps = glob(join(simpath, "output_*"))
+                              "snapshot_*"))
-            # Take just the last _00XXXX from each file  and strip zeros
+            snaps = [int(search(r'snapshot_(\d+)', f).group(1)) for f in snaps]
-            snaps = [int(snap.split("_")[-1].lstrip("0")) for snap in snaps]
+            snaps = sorted(snaps)
         elif simname == "csiborg2_main":
-            snaps = glob(join(simpath, "snapshot_*"))
+            snaps = glob(join(self.csiborg2_main_srcdir, f"chain_{nsim}",
-            snaps = [basename(snap) for snap in snaps]
+                              "output", "snapshot_*"))
-            snaps = [int(snap.split("_")[1]) for snap in snaps]
+            snaps = [int(search(r'snapshot_(\d+)', f).group(1))
+                     for f in snaps]
+            snaps = sorted(snaps)
         elif simname == "csiborg2_random":
-            raise NotImplementedError("TODO")
+            snaps = glob(join(self.csiborg2_random_srcdir, f"chain_{nsim}",
+                              "output", "snapshot_*"))
+            snaps = [int(search(r'snapshot_(\d+)', f).group(1))
+                     for f in snaps]
+            snaps = sorted(snaps)
         elif simname == "csiborg2_varysmall":
-            raise NotImplementedError("TODO")
+            snaps = glob(join(self.csiborg2_random_srcdir, f"chain_{nsim}",
+                              "snapshot_*"))
+            snaps = [int(search(r'snapshot_16417_(\d+)', f).group(1))
+                     for f in snaps]
+            snaps = sorted(snaps)
         elif simname == "quijote":
-            snaps = glob(join(simpath, "snapdir_*"))
+            snaps = glob(join(self.quijote_dir, "fiducial_processed",
-            snaps = [int(snap.split("/")[-1].split("snapdir_")[-1])
+                              f"chain_{nsim}", "snapshot_*"))
-                     for snap in snaps]
+            has_ics = any("snapshot_ICs" in f for f in snaps)
+            snaps = [int(match.group(1))
+                     for f in snaps if (match := search(r'snapshot_(\d+)', f))]
+            snaps = sorted(snaps)
+            if has_ics:
+                snaps.insert(0, "ICs")
         else:
             raise ValueError(f"Unknown simulation name `{simname}`.")
-        return numpy.sort(snaps)
+        return snaps
 
-    @staticmethod
+    def snapshot(self, nsnap, nsim, simname):
-    def quijote_fiducial_nsim(nsim, nobs=None):
         """
-        Fiducial Quijote simulation ID. Combines the IC realisation and
+        Path to a simulation snapshot.
-        observer placement.
 
         Parameters
         ----------
+        nsnap : int
+            Snapshot index. For Quijote, `ICs` indicates the IC snapshot.
         nsim : int
             IC realisation index.
-        nobs : int, optional
+        simname : str
-            Fiducial observer index.
+            Simulation name.
 
         Returns
         -------
         str
         """
-        if nobs is None:
+        if simname == "csiborg1":
-            assert isinstance(nsim, str)
+            return join(self.csiborg1_srcdir, f"chain_{nsim}",
-            assert len(nsim) == 5
+                        f"snapshot_{str(nsnap).zfill(5)}.hdf5")
-            return nsim
+        elif simname == "csiborg2_main":
-        return f"{str(nobs).zfill(2)}{str(nsim).zfill(3)}"
+            return join(self.csiborg2_main_srcdir, f"chain_{nsim}",
+                        f"snapshot_{str(nsnap).zfill(3)}.hdf5")
+        elif simname == "csiborg2_random":
+            return join(self.csiborg2_random_srcdir, f"chain_{nsim}",
+                        f"snapshot_{str(nsnap).zfill(3)}.hdf5")
+        elif simname == "csiborg2_varysmall":
+            return join(self.csiborg2_varysmall_srcdir, f"chain_{nsim}",
+                        f"snapshot_{str(nsnap).zfill(3)}.hdf5")
+        elif simname == "quijote":
+            return join(self.quijote_dir, "fiducial_processed",
+                        f"chain_{nsim}",
+                        f"snapshot_{str(nsnap).zfill(3)}.hdf5")
+        else:
+            raise ValueError(f"Unknown simulation name `{simname}`.")
 
     def overlap(self, simname, nsim0, nsimx, min_logmass, smoothed):
         """

scripts/field_sample.py

@@ -188,8 +188,8 @@ if __name__ == "__main__":
                         help="Field in RSP?")
     parser.add_argument("--nrand", type=int, required=True,
                         help="Number of rand. positions to evaluate the field")
-    parser.add_argument("--simname", type=str, default="csiborg",
+    parser.add_argument("--simname", type=str, default="csiborg1",
-                        choices=["csiborg"], help="Simulation name")
+                        choices=["csiborg1"], help="Simulation name")
     args = parser.parse_args()
 
     paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)

scripts/utils.py

@@ -38,13 +38,8 @@ def get_nsims(args, paths):
     """
     Get simulation indices from the command line arguments.
     """
-    try:
-        from_quijote_backup = args.from_quijote_backup
-    except AttributeError:
-        from_quijote_backup = False
-
     if args.nsims is None or args.nsims[0] == -1:
-        nsims = paths.get_ics(args.simname, from_quijote_backup)
+        nsims = paths.get_ics(args.simname)
     else:
         nsims = args.nsims
     return list(nsims)