Add SWIFT SPH (#147)

* Add new params

* Add manticore params

* Add SWIFT reader

* Update script

* Simplify

* Add comment

* Update H0 values for the void

* Update queue

csiborgtools/flow/flow_model.py

@@ -610,6 +610,8 @@ def sample_calibration(Vext_min, Vext_max, Vmono_min, Vmono_max, beta_min,
 
     if no_Vext:
         Vext = jnp.zeros(3)
+        # 840 in the direction of (l, b) = (117, 4)
+        # Vext = jnp.asarray([338.9478154 , -11.45056064, 768.49415294])
     else:
         Vext = sample("Vext", Uniform(Vext_min, Vext_max).expand([3]))
 

csiborgtools/params.py

@@ -71,6 +71,7 @@ def simname2boxsize(simname):
          "csiborg2_varysmall": 676.6,
          "csiborg2_random": 676.6,
          "csiborg2X": 681.1,
+         "manticore_2MPP_N128_DES_V1": 681.1,
          "borg1": 677.7,
          "borg2": 676.6,
          "borg2_all": 676.6,
@@ -97,6 +98,7 @@ def simname2Omega_m(simname):
          "csiborg2_random": 0.3111,
          "csiborg2_varysmall": 0.3111,
          "csiborg2X": 0.306,
+         "manticore_2MPP_N128_DES_V1": 0.306,
          "borg1": 0.307,
          "borg2": 0.3111,
          "borg2_all": 0.3111,
@@ -131,4 +133,5 @@ paths_glamdring = {
     "tng300_1_dir": "/mnt/extraspace/rstiskalek/TNG300-1/",
     "aux_cat_dir": "/mnt/extraspace/rstiskalek/catalogs",
     "CF4_dir": "/mnt/extraspace/rstiskalek/catalogs/CF4",
+    "manticore_dir": "/mnt/extraspace/rstiskalek/MANTICORE",
     }

csiborgtools/read/paths.py

@@ -50,6 +50,8 @@ class Paths:
         Path to the CSiBORG2 random simulation directory.
     csiborg2_varysmall_srcdir : str
         Path to the CSiBORG2 varysmall simulation directory.
+    manticore_srcdir : str
+        Path to the MANTICORE simulation directory.
     postdir : str
         Path to the CSiBORG post-processing directory.
     quijote_dir : str
@@ -70,6 +72,7 @@ class Paths:
                  csiborg2_main_srcdir,
                  csiborg2_random_srcdir,
                  csiborg2_varysmall_srcdir,
+                 manticore_dir,
                  postdir,
                  quijote_dir,
                  borg1_dir,
@@ -82,6 +85,7 @@ class Paths:
         self.csiborg2_main_srcdir = csiborg2_main_srcdir
         self.csiborg2_random_srcdir = csiborg2_random_srcdir
         self.csiborg2_varysmall_srcdir = csiborg2_varysmall_srcdir
+        self.manticore_dir = manticore_dir
         self.quijote_dir = quijote_dir
         self.borg1_dir = borg1_dir
         self.borg2_dir = borg2_dir
@@ -108,6 +112,12 @@ class Paths:
             files = glob(join(self.csiborg2_varysmall_srcdir, "chain_*"))
             files = [int(search(r'chain_16417_(\d+)', f).group(1))
                      for f in files]
+        elif "manticore" in simname:
+            fdir = self.manticore_dir
+            if simname == "manticore_2MPP_N128_DES_V1":
+                fdir = join(fdir, "2MPP_N128_DES_V1", "resimulations", "R512")
+                files = glob(join(fdir, "mcmc_*"))
+                files = [int(search(r'mcmc_(\d+)', f).group(1)) for f in files]
         elif simname == "csiborg2X":
             # NOTE this too is preliminary
             fname = "/mnt/extraspace/rstiskalek/MANTICORE/resimulations/fields/2MPP_N128_DES_PROD/R512"  # noqa
@@ -173,6 +183,8 @@ class Paths:
             snaps = sorted(snaps)
         elif simname == "csiborg2X":
             raise ValueError("Snapshots not available for CSiBORG2X.")
+        elif "manticore" in simname:
+            raise ValueError("Snapshots not available for MANTICORE.")
         elif simname == "quijote":
             snaps = glob(join(self.quijote_dir, "fiducial_processed",
                               f"chain_{nsim}", "snapshot_*"))
@@ -203,6 +215,8 @@ class Paths:
                          f"snapshot_{str(nsnap).zfill(3)}.hdf5")
         elif simname == "csiborg2X":
             raise ValueError("Snapshots not available for CSiBORG2X.")
+        elif "manticore" in simname:
+            raise ValueError("Snapshots not available for MANTICORE.")
         elif simname == "quijote":
             fpath = join(self.quijote_dir, "fiducial_processed",
                          f"chain_{nsim}",

scripts/field_prop/field_los_indranil_void.py

@@ -28,7 +28,13 @@ from tqdm import trange
 
 def interpolate_indranil_void(kind, nsims, RA, dec, rmax, dr, dump_folder,
                               catalogue):
-    if kind not in ["exp", "gauss", "mb"]:
+    if kind == "mb":
+        h = 0.7615
+    elif kind == "gauss":
+        h = 0.7724
+    elif kind == "exp":
+        h = 0.7725
+    else:
         raise ValueError(f"Unknown void kind: `{kind}`.")
 
     kind = kind.upper()
@@ -48,11 +54,10 @@ def interpolate_indranil_void(kind, nsims, RA, dec, rmax, dr, dump_folder,
         fname = join(fdir, f"v_pec_{kind}profile_rLG_{nsim}.dat")
         data = np.loadtxt(fname)
 
-        # The grid is in Mpc
+        # The grid is in Mpc from 0 to 251 but we convert to Mpc / h
-        r_grid = np.arange(0, 251)
+        r_grid = np.arange(0, 251) * h
         phi_grid = np.arange(0, 181)
-        # The input is in Mpc/h, so we need to convert to Mpc
+        r_eval = np.arange(0, rmax, dr).astype(float)
-        r_eval = np.arange(0, rmax, dr).astype(float) / 0.674
 
         model_axis = SkyCoord(l=117, b=4, frame='galactic', unit='deg').icrs
         coords = SkyCoord(ra=RA, dec=dec, unit='deg').icrs
@@ -76,7 +81,7 @@ def interpolate_indranil_void(kind, nsims, RA, dec, rmax, dr, dump_folder,
         # Write the output, homogenous density.
         density = np.ones_like(result)
         with File(fname_out, 'a') as f_out:
-            f_out.create_dataset(f"rdist_{k}", data=r_eval * 0.674)
+            f_out.create_dataset(f"rdist_{k}", data=r_eval)
             f_out.create_dataset(f"density_{k}", data=density)
             f_out.create_dataset(f"velocity_{k}", data=result)
 

scripts/field_prop/field_los_indranil_void.sh

@@ -1,7 +1,7 @@
 nthreads=1
-memory=7
+memory=4
 on_login=${1}
-queue="berg"
+queue="cmb"
 env="/mnt/users/rstiskalek/csiborgtools/venv_csiborg/bin/python"
 file="field_los_indranil_void.py"
 

scripts_independent/field_sph_gadget.py

@@ -107,6 +107,33 @@ def prepara_gadget2(snapshot_path, temporary_output_path):
     return boxsize
 
 
+def prepare_swift(snapshot_path, temporary_output_path):
+    """
+    Read in SWIFT Manticore files from Stuart.
+    """
+    # Get some basic information about the snapshots.
+    with File(snapshot_path, 'r') as src, File(temporary_output_path, 'w') as target: # noqa
+        h = src["Cosmology"].attrs["h"][0]
+        npart = src["Header"].attrs["NumPart_Total"][1]
+
+        # Convert to Msun / h
+        mpart = src["Header"].attrs["InitialMassTable"][1] * 1e10 * h
+        # Convert to Mpc / h
+        boxsize = src["Header"].attrs["BoxSize"][0] * h
+
+        print(f"{'Boxsize':<20}: {boxsize}")
+        print(f"{'Npart':<20}: {npart}")
+        print(f"{'Mpart':<20}: {mpart}")
+
+        dset = target.create_dataset("particles", (npart, 7), dtype=np.float32)
+
+        dset[:, :3] = src["DMParticles/Coordinates"][:] * h  # Mpc / h
+        dset[:, 3:6] = src["DMParticles/Velocities"][:]      # km/s
+        dset[:, 6] = np.ones(npart, dtype=np.float32) * mpart
+
+    return boxsize
+
+
 def run_sph_filter(particles_path, output_path, boxsize, resolution,
                    SPH_executable):
     """
@@ -171,7 +198,7 @@ def main(snapshot_path, output_path, resolution, scratch_space, SPH_executable,
     [1] https://bitbucket.org/glavaux/cosmotool/src/master/sample/simple3DFilter.cpp  # noqa
     """
     # First get the temporary file path.
-    if snapshot_kind in ["gadget2", "gadget4"]:
+    if snapshot_kind in ["gadget2", "gadget4", "swift"]:
         temporary_output_path = join(
             scratch_space, generate_unique_id(snapshot_path))
     elif snapshot_kind == "ramses":
@@ -206,10 +233,12 @@ def main(snapshot_path, output_path, resolution, scratch_space, SPH_executable,
         print(f"{now()}: preparing snapshot...", flush=True)
         boxsize = prepara_gadget2(snapshot_path, temporary_output_path)
         print(f"{now()}: wrote temporary data to {temporary_output_path}.")
+    elif snapshot_kind == "swift":
+        print(f"{now()}: preparing snapshot...", flush=True)
+        boxsize = prepare_swift(snapshot_path, temporary_output_path)
+        print(f"{now()}: wrote temporary data to {temporary_output_path}.")
     else:
-        boxsize = 677.7  # Mpc/h
+        raise RuntimeError(f"Snapshot kind `{snapshot_kind}` not implemented.")
-        print(f"{now()}: CAREFUL, forcefully setting the boxsize to {boxsize} Mpc / h.",  # noqa
-              flush=True)
 
     # Run the SPH filter.
     run_sph_filter(temporary_output_path, output_path, boxsize, resolution,
@@ -237,7 +266,7 @@ if __name__ == "__main__":
     parser.add_argument("--SPH_executable", type=str, required=True,
                         help="Path to the `simple3DFilter` executable.")
     parser.add_argument("--snapshot_kind", type=str, required=True,
-                        choices=["gadget4", "gadget2", "ramses"],
+                        choices=["gadget4", "gadget2", "ramses", "swift"],
                         help="Kind of the simulation snapshot.")
     args = parser.parse_args()
 

scripts_independent/field_sph_gadget.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-nthreads=28
+nthreads=12
 memory=7
 on_login=${1}
 queue="berg"
@@ -11,9 +11,10 @@ resolution=1024
 SPH_executable="/mnt/users/rstiskalek/cosmotool/bld/sample/simple3DFilter"
 scratch_space="/mnt/extraspace/rstiskalek/dump/"
 
-snapshot_kind="gadget2"
+# CLONES settings
-snapshot_path="/mnt/extraspace/rstiskalek/CLONES/s8/cf2gvpecc1pt5elmo73_sig6distribsbvoldi_RZA3Derrv2_512_500_ss8_zinit60_000"
+# snapshot_kind="gadget2"
-output_path="/mnt/extraspace/rstiskalek/CLONES/s8/cf2gvpecc1pt5elmo73_sig6distribsbvoldi_RZA3Derrv2_512_500_ss8_zinit60_000.hdf5"
+# snapshot_path="/mnt/extraspace/rstiskalek/CLONES/s8/cf2gvpecc1pt5elmo73_sig6distribsbvoldi_RZA3Derrv2_512_500_ss8_zinit60_000"
+# output_path="/mnt/extraspace/rstiskalek/CLONES/s8/cf2gvpecc1pt5elmo73_sig6distribsbvoldi_RZA3Derrv2_512_500_ss8_zinit60_000.hdf5"
 
 
 # Check if `on_login` is either 0 or 1
@@ -26,14 +27,36 @@ fi
 export OMP_NUM_THREADS={nthreads}
 export OMP_NESTED=true
 
-pythoncm="$env $file --snapshot_path $snapshot_path --output_path $output_path --resolution $resolution --scratch_space $scratch_space --SPH_executable $SPH_executable --snapshot_kind $snapshot_kind"
+# pythoncm="$env $file --snapshot_path $snapshot_path --output_path $output_path --resolution $resolution --scratch_space $scratch_space --SPH_executable $SPH_executable --snapshot_kind $snapshot_kind"
-if [ $on_login -eq 1 ]; then
+# if [ $on_login -eq 1 ]; then
-    echo $pythoncm
+#     echo $pythoncm
-    $pythoncm
+#     $pythoncm
-else
+# else
-    cm="addqueue -s -q $queue -n 1x$nthreads -m $memory $pythoncm"
+#     cm="addqueue -s -q $queue -n 1x$nthreads -m $memory $pythoncm"
-    echo "Submitting:"
+#     echo "Submitting:"
-    echo $cm
+#     echo $cm
-    echo
+#     echo
-    eval $cm
+#     eval $cm
-fi
+# fi
+
+
+# Manticore SWIFT submission loop
+snapshot_kind="swift"
+for k in {0..40}; do
+    snapshot_path="/mnt/extraspace/rstiskalek/MANTICORE/2MPP_N128_DES_V1/resimulations/R512/mcmc_$k/swift_monofonic/snap_0001/snap_0001.hdf5"
+    output_path="/mnt/extraspace/rstiskalek/MANTICORE/2MPP_N128_DES_V1/fields/R512/SPH_$k.hdf5"
+
+    pythoncm="$env $file --snapshot_path $snapshot_path --output_path $output_path --resolution $resolution --scratch_space $scratch_space --SPH_executable $SPH_executable --snapshot_kind $snapshot_kind"
+    if [ $on_login -eq 1 ]; then
+        echo $pythoncm
+        $pythoncm
+    else
+        cm="addqueue -s -q $queue -n 1x$nthreads -m $memory $pythoncm"
+        echo "Submitting:"
+        echo $cm
+        echo
+        eval $cm
+    fi
+
+    sleep 0.05
+done