Scripts for CSiBORG1 SPH calculation (#106)

* Rename files

* Delet efile

* Add new submission script

* Update README

* Add submit script

* Renaming file

* Correct typos

* Update README

* Add all nsims

README.md

@@ -9,9 +9,10 @@ Tools for analysing the suite of Constrained Simulations in BORG (CSiBORG) simul
 - [ ] Improve the storage system for overlaps and calculate it for all simulations.
 
 ### Enviromental dependence of galaxy properties
-- [ ] Calculate the SPH density field for CSiBORG1.
+- [ ] Prepare the CSiBORG one particle files for SPH.
+- [ ] Transfer, calculate the SPH density field for CSiBORG1 and transfer back.
 - [x] Check that the velocity-field flipping of x and z coordinates is correct.
-- [ ] Evaluate and share the density field for SDSS & SDSSxALFALFA for both CSiBORG2 and random fields.
+- [x] Evaluate and share the density field for SDSS and SDSSxALFALFA for both CSiBORG2 and random fields.
 - [ ] Check and verify the density field of galaxy colours (cannot do this now! Glamdring is super slow.)
 
 #### Calculated data

scripts_independent/field_sph.sh

@@ -1,40 +0,0 @@
-#!/bin/sh
-
-#SBATCH --ntasks-per-node=1
-#SBATCH --nodes=1
-#SBATCH --cpus-per-task=16
-#SBATCH --mem-per-cpu=7000
-#SBATCH -J SPH
-#SBATCH -o output_%J.out
-#SBATCH -e error_%J.err
-#SBATCH -p cosma8-serial
-#SBATCH -A dp016
-#SBATCH -t 04:00:00
-#SBATCH --mail-type=BEGIN,END,FAIL
-#SBATCH --mail-user=richard.stiskalek@physics.ox.ac.uk
-
-
-module purge
-module load intel_comp/2019
-module load intel_mpi/2019
-module load hdf5
-module load fftw
-module load gsl
-module load cmake
-module load python/3.10.12
-module list
-
-source /cosma/home/dp016/dc-stis1/csiborgtools/venv_csiborgtools/bin/activate
-export OMP_NUM_THREADS=16
-export OMP_NESTED=true
-
-# ADD CHAINS HERE
-snapshot_path="/cosma8/data/dp016/dc-stis1/csiborg2_main/chain_15517/output/snapshot_099_full.hdf5"
-output_path="/cosma8/data/dp016/dc-stis1/csiborg2_main/field/chain_15517.hdf5"
-resolution=256
-scratch_space="/cosma8/data/dp016/dc-stis1/csiborg2_main/field"
-SPH_executable="/cosma8/data/dp016/dc-stis1/cosmotool/bld2/sample/simple3DFilter"
-snapshot_kind="gadget4"
-
-
-python3 field_sph.py --snapshot_path $snapshot_path --output_path $output_path --resolution $resolution --scratch_space $scratch_space --SPH_executable $SPH_executable --snapshot_kind $snapshot_kind

scripts_independent/field_sph_ramses.py

@@ -0,0 +1,113 @@
+# Copyright (C) 2023 Richard Stiskalek
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 3 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+Script to construct the density and velocity fields for a simulation snapshot.
+The SPH filter is implemented in the cosmotool package.
+"""
+from argparse import ArgumentParser
+from os.path import join
+
+import numpy as np
+from h5py import File
+
+from field_sph_gadget import now, run_sph_filter
+from process_snapshot import CSiBORG1Reader
+
+
+def prepare_csiborg1(nsim, output_path):
+    """
+    Prepare a RAMSES snapshot for the SPH filter.
+
+    Parameters
+    ----------
+    nsim : int
+        Simulation index.
+    output_path : str
+        Path to the output HDF5 file.
+
+    Returns
+    -------
+    None
+    """
+    reader = CSiBORG1Reader(nsim, "final")
+
+    with File(output_path, 'w') as target:
+        print(f"{now()}: loading positions.")
+        pos = reader.read_snapshot("pos")
+        print(f"{now()}: loading velocities.")
+        vel = reader.read_snapshot("vel")
+        print(f"{now()}: loading masses.")
+        mass = reader.read_snapshot("mass")
+
+        print(f"Writing {len(pos)} particles to {output_path}.")
+        dset = target.create_dataset("particles", (len(pos), 7),
+                                     dtype=np.float32)
+
+        dset[:, :3] = pos
+        print(f"{now()}: written positions.")
+        dset[:, 3:6] = vel
+        print(f"{now()}: written velocities.")
+        dset[:, 6] = mass
+        print(f"{now()}: written masses.")
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description="Generate SPH density and velocity field.")  # noqa
+    parser.add_argument("--nsim", type=int, required=True,
+                        help="Simulation index")
+    parser.add_argument("--mode", type=str, required=True,
+                        choices=["prepare", "run"], help="Mode of operation.")
+    parser.add_argument("--output_folder", type=str, required=True,
+                        help="Path to the output HDF5 file.")
+    parser.add_argument("--resolution", type=int, required=True,
+                        help="Resolution of the density and velocity field.")
+    parser.add_argument("--scratch_space", type=str, required=True,
+                        help="Path to a folder where temporary files can be stored.")  # noqa
+    parser.add_argument("--SPH_executable", type=str, required=True,
+                        help="Path to the `simple3DFilter` executable.")
+    parser.add_argument("--snapshot_kind", type=str, required=True,
+                        choices=["ramses"],
+                        help="Kind of the simulation snapshot.")
+    args = parser.parse_args()
+
+    if args.snapshot_kind != "ramses":
+        raise NotImplementedError("Only RAMSES snapshots are supported.")
+
+    particles_path = join(args.scratch_space,
+                          f"ramses_{str(args.nsim).zfill(5)}.hdf5")
+    output_path = join(args.output_folder,
+                       f"sph_ramses_{str(args.nsim).zfill(5)}.hdf5")
+
+    print("---------- SPH Density & Velocity Field Job Information ----------")
+    print(f"Mode:              {args.mode}")
+    print(f"Simulation index:  {args.nsim}")
+    print(f"Paticles path:     {particles_path}")
+    print(f"Output path:       {output_path}")
+    print(f"Resolution:        {args.resolution}")
+    print(f"SPH executable:    {args.SPH_executable}")
+    print(f"Snapshot kind:     {args.snapshot_kind}")
+    print("------------------------------------------------------------------")
+    print(flush=True)
+
+    if args.mode == "prepare":
+        prepare_csiborg1(args.nsim, particles_path)
+    elif args.mode == "run":
+        output_path = join(args.output_folder,
+                           f"sph_ramses_{str(args.nsim).zfill(5)}.hdf5")
+        boxsize = 677.7
+        run_sph_filter(particles_path, output_path, boxsize, args.resolution,
+                       args.SPH_executable)
+    else:
+        raise NotImplementedError(f"Mode `{args.mode}` is not supported.")

scripts_independent/run_field_sph_gadget.py

@@ -14,7 +14,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 """
 Script to write the SLURM submission script and submit it to the queue to
-calculate the SPH density & velocity field.
+calculate the SPH density & velocity field for GADGET.
 """
 from os import system
 
@@ -60,7 +60,7 @@ scratch_space="/snap8/scratch/dp016/dc-stis1/"
 SPH_executable="/cosma8/data/dp016/dc-stis1/cosmotool/bld2/sample/simple3DFilter"
 snapshot_kind="gadget4"
 
-python3 field_sph.py --snapshot_path $snapshot_path --output_path $output_path --resolution $resolution --scratch_space $scratch_space --SPH_executable $SPH_executable --snapshot_kind $snapshot_kind
+python3 field_sph_gadget.py --snapshot_path $snapshot_path --output_path $output_path --resolution $resolution --scratch_space $scratch_space --SPH_executable $SPH_executable --snapshot_kind $snapshot_kind
 """
     fname = f"submit_SPH_{kind}_{chain_index}.sh"
     print(f"Writing file:  `{fname}`.")

scripts_independent/run_field_sph_ramses_prepare.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+nthreads=1  # Keep this at 1!!
+memory=32
+on_login=${1}
+queue="berg"
+env="/mnt/zfsusers/rstiskalek/csiborgtools/venv_csiborg/bin/python"
+file="field_sph_ramses.py"
+
+
+# nsims=(7444)
+nsims=(7444 7468 7492 7516 7540 7564 7588 7612 7636 7660 7684 7708 7732 7756 7780 7804 7828 7852 7876 7900 7924 7948 7972 7996 8020 8044 8068 8092 8116 8140 8164 8188 8212 8236 8260 8284 8308 8332 8356 8380 8404 8428 8452 8476 8500 8524 8548 8572 8596 8620 8644 8668 8692 8716 8740 8764 8788 8812 8836 8860 8884 8908 8932 8956 8980 9004 9028 9052 9076 9100 9124 9148 9172 9196 9220 9244 9268 9292 9316 9340 9364 9388 9412 9436 9460 9484 9508 9532 9556 9580 9604 9628 9652 9676 9700 9724 9748 9772 9796 9820 9844)
+mode="prepare"
+output_folder="/mnt/extraspace/rstiskalek/dump/"
+resolution=1024
+scratch_space="/mnt/extraspace/rstiskalek/dump/"
+SPH_executable="NaN"
+snapshot_kind="ramses"
+
+
+for nsim in "${nsims[@]}"; do
+    pythoncm="$env $file --nsim $nsim --mode $mode --output_folder $output_folder --resolution $resolution --scratch_space $scratch_space --SPH_executable $SPH_executable --snapshot_kind $snapshot_kind"
+    if [ $on_login -eq 1 ]; then
+        echo $pythoncm
+        $pythoncm
+    else
+        cm="addqueue -q $queue -n $nthreads -m $memory $pythoncm"
+        echo "Submitting:"
+        echo $cm
+        echo
+        eval $cm
+    fi
+done

scripts_independent/run_field_sph_ramses_run.py

@@ -0,0 +1,76 @@
+# Copyright (C) 2023 Richard Stiskalek
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 3 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+Script to write the SLURM submission script and submit it to the queue to
+calculate the SPH density & velocity field for RAMSES.
+"""
+from os import system
+
+
+def write_submit(chain_index, resolution, nthreads):
+    txt = f"""#!/bin/sh
+
+#SBATCH --ntasks-per-node=1
+#SBATCH --nodes=1
+#SBATCH --cpus-per-task={nthreads}
+#SBATCH --mem-per-cpu=7000
+#SBATCH -J SPH_{chain_index}
+#SBATCH -o output_{chain_index}_%J.out
+#SBATCH -e error_{chain_index}_%J.err
+#SBATCH -p cosma8-serial
+#SBATCH -A dp016
+#SBATCH -t 16:00:00
+#SBATCH --mail-type=BEGIN,END,FAIL
+#SBATCH --mail-user=richard.stiskalek@physics.ox.ac.uk
+
+
+module purge
+module load intel_comp/2019
+module load intel_mpi/2019
+module load hdf5
+module load fftw
+module load gsl
+module load cmake
+module load python/3.10.12
+module list
+
+source /cosma/home/dp016/dc-stis1/csiborgtools/venv_csiborgtools/bin/activate
+export OMP_NUM_THREADS={nthreads}
+export OMP_NESTED=true
+
+
+output_folder="/cosma8/data/dp016/dc-stis1/csiborg1_sph"
+SPH_executable="/cosma8/data/dp016/dc-stis1/cosmotool/bld2/sample/simple3DFilter"
+
+python3 field_sph_ramses.py --nsim {chain_index} --mode run --output_folder $output_folder --resolution {resolution} --scratch_space $output_folder --SPH_executable $SPH_executable --snapshot_kind ramses
+"""
+    fname = f"submit_SPH_csiborg1_{chain_index}.sh"
+    print(f"Writing file:  `{fname}`.")
+    with open(fname, "w") as txtfile:
+        txtfile.write(txt)
+    # Make the file executable
+    system(f"chmod +x {fname}")
+    return fname
+
+
+if __name__ == "__main__":
+    chains = [7444]
+
+    resolution = 1024
+    nthreads = 32
+
+    for chain_index in chains:
+        fname = write_submit(chain_index, resolution, nthreads)
+        system(f"sbatch {fname}")