Switch to h5py format (#52)

* Edit the particle paths

* Remove script

* Add h5py to dumping

* Minor adjustments

* add h5py support

* remove split path

* h5py support

* Type

* Edit initmatch paths

* Shorten func

* dist_centmass to work with 2D arrays

* Forgot to return the centre of mass

* Fixed code

* Fix halo bug

* Start MPI broadcasting

* Mini bug

* Remove commenting

* Remove test statement

* Fix index

* Printing from rank 0 only

* Move where clump index stored

* Add dtype options

* Add dtype options

csiborgtools/fits/halo.py

@@ -40,8 +40,7 @@ class BaseStructure(ABC):
 
     @particles.setter
     def particles(self, particles):
-        pars = ["x", "y", "z", "M"]
-        assert all(p in particles.dtype.names for p in pars)
+        assert particles.ndim == 2 and particles.shape[1] == 7
         self._particles = particles
 
     @property
@@ -256,24 +255,14 @@ class BaseStructure(ABC):
             return numpy.nan, numpy.nan
         return rs[k], cmass[k]
 
-    @property
-    def keys(self):
-        """
-        Particle array keys.
-
-        Returns
-        -------
-        key : list of str
-        """
-        return self.particles.dtype.names
-
     def __getitem__(self, key):
+        keys = ['x', 'y', 'z', 'vx', 'vy', 'vz', 'M']
         if key not in self.keys:
-            raise RuntimeError("Invalid key `{}`!".format(key))
-        return self.particles[key]
+            raise RuntimeError(f"Invalid key `{key}`!")
+        return self.particles[:, keys.index(key)]
 
     def __len__(self):
-        return self.particles.size
+        return self.particles.shape[0]
 
 
 class Clump(BaseStructure):

csiborgtools/match/match.py

@@ -827,8 +827,8 @@ def dist_centmass(clump):
 
     Parameters
     ----------
-    clump : structurered arrays
-        Clump structured array. Keyes must include `x`, `y`, `z` and `M`.
+    clump : 2-dimensional array of shape (n_particles, 7)
+        Particle array. The first four columns must be `x`, `y`, `z` and `M`.
 
     Returns
     -------
@@ -838,16 +838,8 @@ def dist_centmass(clump):
         Center of mass coordinates.
     """
     # CM along each dimension
-    cmx, cmy, cmz = [numpy.average(clump[p], weights=clump["M"])
-                     for p in ("x", "y", "z")]
-    # Particle distance from the CM
-    dist = numpy.sqrt(
-        numpy.square(clump["x"] - cmx)
-        + numpy.square(clump["y"] - cmy)
-        + numpy.square(clump["z"] - cmz)
-        )
-
-    return dist, numpy.asarray([cmx, cmy, cmz])
+    cm = numpy.average(clump[:, :3], weights=clump[:, 3], axis=0)
+    return numpy.linalg.norm(clump[:, :3] - cm, axis=1), cm
 
 
 def dist_percentile(dist, qs, distmax=0.075):

csiborgtools/read/paths.py

@@ -132,40 +132,19 @@ class CSiBORGPaths:
         nsim : int
             IC realisation index.
         kind : str
-            Type of match.  Can be either `fit` or `particles`.
+            Type of match. Must be one of `["particles", "fit", "halomap"]`.
 
         Returns
         -------
         path : str
         """
-        assert kind in ["fit", "particles"]
+        assert kind in ["particles", "fit", "halomap"]
+        ftype = "npy" if kind == "fit" else "h5"
         fdir = join(self.postdir, "initmatch")
         if not isdir(fdir):
             mkdir(fdir)
             warn(f"Created directory `{fdir}`.", UserWarning, stacklevel=1)
-        return join(fdir, f"{kind}_{str(nsim).zfill(5)}.npy")
-
-    def split_path(self, nsnap, nsim):
-        """
-        Path to the `split` files from `pre_splithalos`.
-
-        Parameters
-        ----------
-        nsnap : int
-            Snapshot index.
-        nsim : int
-            IC realisation index.
-
-        Returns
-        -------
-        path : str
-        """
-        fdir = join(self.postdir, "split")
-        if not isdir(fdir):
-            mkdir(fdir)
-            warn(f"Created directory `{fdir}`.", UserWarning, stacklevel=1)
-        return join(
-            fdir, f"clumps_{str(nsim).zfill(5)}_{str(nsnap).zfill(5)}.npz")
+        return join(fdir, f"{kind}_{str(nsim).zfill(5)}.{ftype}")
 
     def get_ics(self, tonew):
         """
@@ -326,30 +305,37 @@ class CSiBORGPaths:
         fname = f"radpos_{str(nsim).zfill(5)}_{str(nsnap).zfill(5)}.npz"
         return join(fdir, fname)
 
-    def particle_h5py_path(self, nsim, with_vel):
+    def particle_h5py_path(self, nsim, kind=None, dtype="float32"):
         """
-        Path to the files containing all particles in a `.hdf5` file. Used for
-        the SPH calculation.
+        Path to the file containing all particles in a `.h5` file.
 
         Parameters
         ----------
         nsim : int
             IC realisation index.
-        with_vel : bool
-            Whether velocities are included.
+        kind : str
+            Type of output. Must be one of `[None, 'pos', 'clumpmap']`.
+        dtype : str
+            Data type. Must be one of `['float32', 'float64']`.
 
         Returns
         -------
         path : str
         """
-        fdir = join(self.postdir, "environment")
+        assert kind in [None, "pos", "clumpmap"]
+        assert dtype in ["float32", "float64"]
+        fdir = join(self.postdir, "particles")
         if not isdir(fdir):
             makedirs(fdir)
             warn(f"Created directory `{fdir}`.", UserWarning, stacklevel=1)
-        if with_vel:
+        if kind is None:
             fname = f"parts_{str(nsim).zfill(5)}.h5"
         else:
-            fname = f"parts_pos_{str(nsim).zfill(5)}.h5"
+            fname = f"parts_{kind}_{str(nsim).zfill(5)}.h5"
+
+        if dtype == "float64":
+            fname = fname.replace(".h5", "_f64.h5")
+
         return join(fdir, fname)
 
     def density_field_path(self, mas, nsim):