Add SWIFT SPH (#147)

* Add new params * Add manticore params * Add SWIFT reader * Update script * Simplify * Add comment * Update H0 values for the void * Update queue
2024-12-22 05:48:02 +00:00 · 2024-09-19 14:34:22 +02:00 · 2024-09-19 14:34:22 +02:00 · c2bc5a6398
commit c2bc5a6398
parent 1fc9ca9306
7 changed files with 104 additions and 28 deletions
--- a/csiborgtools/flow/flow_model.py
+++ b/csiborgtools/flow/flow_model.py
@ -610,6 +610,8 @@ def sample_calibration(Vext_min, Vext_max, Vmono_min, Vmono_max, beta_min,

    if no_Vext:
        Vext = jnp.zeros(3)
+        # 840 in the direction of (l, b) = (117, 4)
+        # Vext = jnp.asarray([338.9478154 , -11.45056064, 768.49415294])
    else:
        Vext = sample("Vext", Uniform(Vext_min, Vext_max).expand([3]))

--- a/csiborgtools/params.py
+++ b/csiborgtools/params.py
@ -71,6 +71,7 @@ def simname2boxsize(simname):
         "csiborg2_varysmall": 676.6,
         "csiborg2_random": 676.6,
         "csiborg2X": 681.1,
+         "manticore_2MPP_N128_DES_V1": 681.1,
         "borg1": 677.7,
         "borg2": 676.6,
         "borg2_all": 676.6,
@ -97,6 +98,7 @@ def simname2Omega_m(simname):
         "csiborg2_random": 0.3111,
         "csiborg2_varysmall": 0.3111,
         "csiborg2X": 0.306,
+         "manticore_2MPP_N128_DES_V1": 0.306,
         "borg1": 0.307,
         "borg2": 0.3111,
         "borg2_all": 0.3111,
@ -131,4 +133,5 @@ paths_glamdring = {
    "tng300_1_dir": "/mnt/extraspace/rstiskalek/TNG300-1/",
    "aux_cat_dir": "/mnt/extraspace/rstiskalek/catalogs",
    "CF4_dir": "/mnt/extraspace/rstiskalek/catalogs/CF4",
+    "manticore_dir": "/mnt/extraspace/rstiskalek/MANTICORE",
    }
--- a/csiborgtools/read/paths.py
+++ b/csiborgtools/read/paths.py
@ -50,6 +50,8 @@ class Paths:
        Path to the CSiBORG2 random simulation directory.
    csiborg2_varysmall_srcdir : str
        Path to the CSiBORG2 varysmall simulation directory.
+    manticore_srcdir : str
+        Path to the MANTICORE simulation directory.
    postdir : str
        Path to the CSiBORG post-processing directory.
    quijote_dir : str
@ -70,6 +72,7 @@ class Paths:
                 csiborg2_main_srcdir,
                 csiborg2_random_srcdir,
                 csiborg2_varysmall_srcdir,
+                 manticore_dir,
                 postdir,
                 quijote_dir,
                 borg1_dir,
@ -82,6 +85,7 @@ class Paths:
        self.csiborg2_main_srcdir = csiborg2_main_srcdir
        self.csiborg2_random_srcdir = csiborg2_random_srcdir
        self.csiborg2_varysmall_srcdir = csiborg2_varysmall_srcdir
+        self.manticore_dir = manticore_dir
        self.quijote_dir = quijote_dir
        self.borg1_dir = borg1_dir
        self.borg2_dir = borg2_dir
@ -108,6 +112,12 @@ class Paths:
            files = glob(join(self.csiborg2_varysmall_srcdir, "chain_*"))
            files = [int(search(r'chain_16417_(\d+)', f).group(1))
                     for f in files]
+        elif "manticore" in simname:
+            fdir = self.manticore_dir
+            if simname == "manticore_2MPP_N128_DES_V1":
+                fdir = join(fdir, "2MPP_N128_DES_V1", "resimulations", "R512")
+                files = glob(join(fdir, "mcmc_*"))
+                files = [int(search(r'mcmc_(\d+)', f).group(1)) for f in files]
        elif simname == "csiborg2X":
            # NOTE this too is preliminary
            fname = "/mnt/extraspace/rstiskalek/MANTICORE/resimulations/fields/2MPP_N128_DES_PROD/R512"  # noqa
@ -173,6 +183,8 @@ class Paths:
            snaps = sorted(snaps)
        elif simname == "csiborg2X":
            raise ValueError("Snapshots not available for CSiBORG2X.")
+        elif "manticore" in simname:
+            raise ValueError("Snapshots not available for MANTICORE.")
        elif simname == "quijote":
            snaps = glob(join(self.quijote_dir, "fiducial_processed",
                              f"chain_{nsim}", "snapshot_*"))
@ -203,6 +215,8 @@ class Paths:
                         f"snapshot_{str(nsnap).zfill(3)}.hdf5")
        elif simname == "csiborg2X":
            raise ValueError("Snapshots not available for CSiBORG2X.")
+        elif "manticore" in simname:
+            raise ValueError("Snapshots not available for MANTICORE.")
        elif simname == "quijote":
            fpath = join(self.quijote_dir, "fiducial_processed",
                         f"chain_{nsim}",
--- a/scripts/field_prop/field_los_indranil_void.py
+++ b/scripts/field_prop/field_los_indranil_void.py
@ -28,7 +28,13 @@ from tqdm import trange

 def interpolate_indranil_void(kind, nsims, RA, dec, rmax, dr, dump_folder,
                              catalogue):
-    if kind not in ["exp", "gauss", "mb"]:
+    if kind == "mb":
+        h = 0.7615
+    elif kind == "gauss":
+        h = 0.7724
+    elif kind == "exp":
+        h = 0.7725
+    else:
        raise ValueError(f"Unknown void kind: `{kind}`.")

    kind = kind.upper()
@ -48,11 +54,10 @@ def interpolate_indranil_void(kind, nsims, RA, dec, rmax, dr, dump_folder,
        fname = join(fdir, f"v_pec_{kind}profile_rLG_{nsim}.dat")
        data = np.loadtxt(fname)

-        # The grid is in Mpc
-        r_grid = np.arange(0, 251)
+        # The grid is in Mpc from 0 to 251 but we convert to Mpc / h
+        r_grid = np.arange(0, 251) * h
        phi_grid = np.arange(0, 181)
-        # The input is in Mpc/h, so we need to convert to Mpc
-        r_eval = np.arange(0, rmax, dr).astype(float) / 0.674
+        r_eval = np.arange(0, rmax, dr).astype(float)

        model_axis = SkyCoord(l=117, b=4, frame='galactic', unit='deg').icrs
        coords = SkyCoord(ra=RA, dec=dec, unit='deg').icrs
@ -76,7 +81,7 @@ def interpolate_indranil_void(kind, nsims, RA, dec, rmax, dr, dump_folder,
        # Write the output, homogenous density.
        density = np.ones_like(result)
        with File(fname_out, 'a') as f_out:
-            f_out.create_dataset(f"rdist_{k}", data=r_eval * 0.674)
+            f_out.create_dataset(f"rdist_{k}", data=r_eval)
            f_out.create_dataset(f"density_{k}", data=density)
            f_out.create_dataset(f"velocity_{k}", data=result)

--- a/scripts/field_prop/field_los_indranil_void.sh
+++ b/scripts/field_prop/field_los_indranil_void.sh
@ -1,7 +1,7 @@
 nthreads=1
-memory=7
+memory=4
 on_login=${1}
-queue="berg"
+queue="cmb"
 env="/mnt/users/rstiskalek/csiborgtools/venv_csiborg/bin/python"
 file="field_los_indranil_void.py"

--- a/scripts_independent/field_sph_gadget.py
+++ b/scripts_independent/field_sph_gadget.py
@ -107,6 +107,33 @@ def prepara_gadget2(snapshot_path, temporary_output_path):
    return boxsize


+def prepare_swift(snapshot_path, temporary_output_path):
+    """
+    Read in SWIFT Manticore files from Stuart.
+    """
+    # Get some basic information about the snapshots.
+    with File(snapshot_path, 'r') as src, File(temporary_output_path, 'w') as target: # noqa
+        h = src["Cosmology"].attrs["h"][0]
+        npart = src["Header"].attrs["NumPart_Total"][1]
+
+        # Convert to Msun / h
+        mpart = src["Header"].attrs["InitialMassTable"][1] * 1e10 * h
+        # Convert to Mpc / h
+        boxsize = src["Header"].attrs["BoxSize"][0] * h
+
+        print(f"{'Boxsize':<20}: {boxsize}")
+        print(f"{'Npart':<20}: {npart}")
+        print(f"{'Mpart':<20}: {mpart}")
+
+        dset = target.create_dataset("particles", (npart, 7), dtype=np.float32)
+
+        dset[:, :3] = src["DMParticles/Coordinates"][:] * h  # Mpc / h
+        dset[:, 3:6] = src["DMParticles/Velocities"][:]      # km/s
+        dset[:, 6] = np.ones(npart, dtype=np.float32) * mpart
+
+    return boxsize
+
+
 def run_sph_filter(particles_path, output_path, boxsize, resolution,
                   SPH_executable):
    """
@ -171,7 +198,7 @@ def main(snapshot_path, output_path, resolution, scratch_space, SPH_executable,
    [1] https://bitbucket.org/glavaux/cosmotool/src/master/sample/simple3DFilter.cpp  # noqa
    """
    # First get the temporary file path.
-    if snapshot_kind in ["gadget2", "gadget4"]:
+    if snapshot_kind in ["gadget2", "gadget4", "swift"]:
        temporary_output_path = join(
            scratch_space, generate_unique_id(snapshot_path))
    elif snapshot_kind == "ramses":
@ -206,10 +233,12 @@ def main(snapshot_path, output_path, resolution, scratch_space, SPH_executable,
        print(f"{now()}: preparing snapshot...", flush=True)
        boxsize = prepara_gadget2(snapshot_path, temporary_output_path)
        print(f"{now()}: wrote temporary data to {temporary_output_path}.")
+    elif snapshot_kind == "swift":
+        print(f"{now()}: preparing snapshot...", flush=True)
+        boxsize = prepare_swift(snapshot_path, temporary_output_path)
+        print(f"{now()}: wrote temporary data to {temporary_output_path}.")
    else:
-        boxsize = 677.7  # Mpc/h
-        print(f"{now()}: CAREFUL, forcefully setting the boxsize to {boxsize} Mpc / h.",  # noqa
-              flush=True)
+        raise RuntimeError(f"Snapshot kind `{snapshot_kind}` not implemented.")

    # Run the SPH filter.
    run_sph_filter(temporary_output_path, output_path, boxsize, resolution,
@ -237,7 +266,7 @@ if __name__ == "__main__":
    parser.add_argument("--SPH_executable", type=str, required=True,
                        help="Path to the `simple3DFilter` executable.")
    parser.add_argument("--snapshot_kind", type=str, required=True,
-                        choices=["gadget4", "gadget2", "ramses"],
+                        choices=["gadget4", "gadget2", "ramses", "swift"],
                        help="Kind of the simulation snapshot.")
    args = parser.parse_args()

--- a/scripts_independent/field_sph_gadget.sh
+++ b/scripts_independent/field_sph_gadget.sh
@ -1,5 +1,5 @@
 #!/bin/bash
-nthreads=28
+nthreads=12
 memory=7
 on_login=${1}
 queue="berg"
@ -11,9 +11,10 @@ resolution=1024
 SPH_executable="/mnt/users/rstiskalek/cosmotool/bld/sample/simple3DFilter"
 scratch_space="/mnt/extraspace/rstiskalek/dump/"

-snapshot_kind="gadget2"
-snapshot_path="/mnt/extraspace/rstiskalek/CLONES/s8/cf2gvpecc1pt5elmo73_sig6distribsbvoldi_RZA3Derrv2_512_500_ss8_zinit60_000"
-output_path="/mnt/extraspace/rstiskalek/CLONES/s8/cf2gvpecc1pt5elmo73_sig6distribsbvoldi_RZA3Derrv2_512_500_ss8_zinit60_000.hdf5"
+# CLONES settings
+# snapshot_kind="gadget2"
+# snapshot_path="/mnt/extraspace/rstiskalek/CLONES/s8/cf2gvpecc1pt5elmo73_sig6distribsbvoldi_RZA3Derrv2_512_500_ss8_zinit60_000"
+# output_path="/mnt/extraspace/rstiskalek/CLONES/s8/cf2gvpecc1pt5elmo73_sig6distribsbvoldi_RZA3Derrv2_512_500_ss8_zinit60_000.hdf5"


 # Check if `on_login` is either 0 or 1
@ -26,14 +27,36 @@ fi
 export OMP_NUM_THREADS={nthreads}
 export OMP_NESTED=true

-pythoncm="$env $file --snapshot_path $snapshot_path --output_path $output_path --resolution $resolution --scratch_space $scratch_space --SPH_executable $SPH_executable --snapshot_kind $snapshot_kind"
-if [ $on_login -eq 1 ]; then
+# pythoncm="$env $file --snapshot_path $snapshot_path --output_path $output_path --resolution $resolution --scratch_space $scratch_space --SPH_executable $SPH_executable --snapshot_kind $snapshot_kind"
+# if [ $on_login -eq 1 ]; then
+#     echo $pythoncm
+#     $pythoncm
+# else
+#     cm="addqueue -s -q $queue -n 1x$nthreads -m $memory $pythoncm"
+#     echo "Submitting:"
+#     echo $cm
+#     echo
+#     eval $cm
+# fi
+
+
+# Manticore SWIFT submission loop
+snapshot_kind="swift"
+for k in {0..40}; do
+    snapshot_path="/mnt/extraspace/rstiskalek/MANTICORE/2MPP_N128_DES_V1/resimulations/R512/mcmc_$k/swift_monofonic/snap_0001/snap_0001.hdf5"
+    output_path="/mnt/extraspace/rstiskalek/MANTICORE/2MPP_N128_DES_V1/fields/R512/SPH_$k.hdf5"
+
+    pythoncm="$env $file --snapshot_path $snapshot_path --output_path $output_path --resolution $resolution --scratch_space $scratch_space --SPH_executable $SPH_executable --snapshot_kind $snapshot_kind"
+    if [ $on_login -eq 1 ]; then
        echo $pythoncm
        $pythoncm
-else
+    else
        cm="addqueue -s -q $queue -n 1x$nthreads -m $memory $pythoncm"
        echo "Submitting:"
        echo $cm
        echo
        eval $cm
-fi
+    fi
+
+    sleep 0.05
+done