Move files

scripts/old/field_observer_velocity.py

@@ -0,0 +1,116 @@
+# Copyright (C) 2023 Richard Stiskalek
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 3 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+
+"""
+from argparse import ArgumentParser
+from datetime import datetime
+from os.path import join
+from warnings import warn
+
+import csiborgtools
+import numpy as np
+from astropy.coordinates import SkyCoord
+from mpi4py import MPI
+from taskmaster import work_delegation
+
+from utils import get_nsims
+
+FDIR = "/mnt/extraspace/rstiskalek/csiborg_postprocessing/peculiar_velocity/observer"  # noqa
+
+
+def t():
+    return datetime.now()
+
+
+def read_velocity_field(args, nsim):
+    if args.simname == "csiborg1":
+        reader = csiborgtools.read.CSiBORG1Field(nsim)
+        return reader.velocity_field("SPH", 1024)
+    elif "csiborg2" in args.simname:
+        kind = args.simname.split("_")[-1]
+        reader = csiborgtools.read.CSiBORG2Field(nsim, kind)
+        return reader.velocity_field("SPH", 1024)
+    elif args.simname == "Carrick2015":
+        folder = "/mnt/extraspace/rstiskalek/catalogs"
+        warn(f"Using local paths from `{folder}`.", RuntimeWarning)
+        fpath = join(folder, "twompp_velocity_carrick2015.npy")
+        field = np.load(fpath).astype(np.float32)
+
+        # Because the Carrick+2015 data is in the following form:
+        # "The velocities are predicted peculiar velocities in the CMB
+        # frame in Galactic Cartesian coordinates, generated from the
+        # \(\delta_g^*\) field with \(\beta^* = 0.43\) and an external
+        # dipole \(V_\mathrm{ext} = [89,-131,17]\) (Carrick et al Table 3)
+        # has already been added.""
+        field[0] -= 89
+        field[1] -= -131
+        field[2] -= 17
+        field /= 0.43
+        return field
+    else:
+        raise ValueError(f"Unknown simname: `{args.simname}`.")
+
+
+def main(smooth_scales, nsim, args):
+    velocity_field = read_velocity_field(args, nsim)
+    boxsize = csiborgtools.simname2boxsize(args.simname)
+
+    if smooth_scales is None:
+        smooth_scales = [0]
+    smooth_scales = np.asanyarray(smooth_scales) / boxsize
+
+    vobs = csiborgtools.field.observer_peculiar_velocity(
+        velocity_field, smooth_scales=smooth_scales, observer=None,
+        verbose=False)
+
+    # For Carrick+2015 the velocity vector is in the Galactic frame, so we
+    # need to convert it to RA/dec
+    if args.simname == "Carrick2015":
+        coord = SkyCoord(vobs, unit='kpc', frame='galactic',
+                         representation_type='cartesian').transform_to("icrs")
+        vobs = coord.cartesian.xyz.value.T
+
+    fname = join(FDIR, f"{args.simname}_{nsim}_observer_velocity.npz")
+    print(f"Saving to `{fname}`.")
+    np.savez(fname, vobs=vobs, smooth_scales=smooth_scales * boxsize)
+
+
+###############################################################################
+#                       Main & command line interface                         #
+###############################################################################
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("--simname", type=str, help="Simulation name.",
+                        choices=["csiborg1", "csiborg2_main", "csiborg2_varysmall", "csiborg2_random", "Carrick2015"])  # noqa
+    args = parser.parse_args()
+    args.nsims = [-1]
+    comm = MPI.COMM_WORLD
+
+    smooth_scales = [0, 0.5, 1.0, 2.0, 4.0, 40.0]
+    paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
+    nsims = get_nsims(args, paths)
+
+    def main_(nsim):
+        main(smooth_scales, nsim, args)
+
+    work_delegation(main_, nsims, comm, master_verbose=True)
+
+    comm.Barrier()
+
+    if comm.Get_rank() == 0:
+        print("All finished.", flush=True)

scripts/old/field_observer_velocity.sh

@@ -0,0 +1,20 @@
+nthreads=5
+memory=40
+on_login=0
+queue="berg"
+env="/mnt/zfsusers/rstiskalek/csiborgtools/venv_csiborg/bin/python"
+file="field_observer_velocity.py"
+
+simname=${1}
+
+pythoncm="$env $file --simname $simname"
+if [ $on_login -eq 1 ]; then
+    echo $pythoncm
+    $pythoncm
+else
+    cm="addqueue -q $queue -n $nthreads -m $memory $pythoncm"
+    echo "Submitting:"
+    echo $cm
+    echo
+    eval $cm
+fi

scripts/old/match_knn.py

@@ -0,0 +1,165 @@
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 3 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+Script to find the nearest neighbour of each halo in a given halo catalogue
+from the remaining catalogues in the suite (CSIBORG or Quijote). The script is
+MPI parallelized over the reference simulations.
+"""
+from argparse import ArgumentParser
+from datetime import datetime
+from distutils.util import strtobool
+from os import remove
+
+import numpy
+import yaml
+from mpi4py import MPI
+from taskmaster import work_delegation
+from tqdm import trange
+
+from utils import open_catalogues
+
+try:
+    import csiborgtools
+except ModuleNotFoundError:
+    import sys
+
+    sys.path.append("../")
+    import csiborgtools
+
+
+def find_neighbour(args, nsim, cats, paths, comm, save_kind):
+    """
+    Find the nearest neighbour of each halo in the given catalogue.
+
+    Parameters
+    ----------
+    args : argparse.Namespace
+        Command line arguments.
+    nsim : int
+        Simulation index.
+    cats : dict
+        Dictionary of halo catalogues. Keys are simulation indices, values are
+        the catalogues.
+    paths : csiborgtools.paths.Paths
+        Paths object.
+    comm : mpi4py.MPI.Comm
+        MPI communicator.
+    save_kind : str
+        Kind of data to save. Must be either `dist` or `bin_dist`.
+
+    Returns
+    -------
+    None
+    """
+    assert save_kind in ["dist", "bin_dist"]
+    ndist, cross_hindxs = csiborgtools.match.find_neighbour(nsim, cats)
+    mass_key = "totpartmass" if args.simname == "csiborg" else "group_mass"
+    cat0 = cats[nsim]
+    rdist = cat0.radial_distance(in_initial=False)
+
+    # Distance is saved optionally, whereas binned distance is always saved.
+    if save_kind == "dist":
+        out = {"ndist": ndist,
+               "cross_hindxs": cross_hindxs,
+               "mass": cat0[mass_key],
+               "ref_hindxs": cat0["index"],
+               "rdist": rdist}
+        fout = paths.cross_nearest(args.simname, args.run, "dist", nsim)
+        if args.verbose:
+            print(f"Rank {comm.Get_rank()} writing to `{fout}`.", flush=True)
+        numpy.savez(fout, **out)
+
+    paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
+    reader = csiborgtools.summary.NearestNeighbourReader(
+        paths=paths, **csiborgtools.neighbour_kwargs)
+    counts = numpy.zeros((reader.nbins_radial, reader.nbins_neighbour),
+                         dtype=numpy.float32)
+    counts = reader.count_neighbour(counts, ndist, rdist)
+    out = {"counts": counts}
+    fout = paths.cross_nearest(args.simname, args.run, "bin_dist", nsim)
+    if args.verbose:
+        print(f"Rank {comm.Get_rank()} writing to `{fout}`.", flush=True)
+    numpy.savez(fout, **out)
+
+
+def collect_dist(args, paths):
+    """
+    Collect the binned nearest neighbour distances into a single file.
+
+    Parameters
+    ----------
+    args : argparse.Namespace
+        Command line arguments.
+    paths : csiborgtools.paths.Paths
+        Paths object.
+
+    Returns
+    -------
+    """
+    fnames = paths.cross_nearest(args.simname, args.run, "bin_dist")
+
+    if args.verbose:
+        print("Collecting counts into a single file.", flush=True)
+
+    for i in trange(len(fnames)) if args.verbose else range(len(fnames)):
+        fname = fnames[i]
+        data = numpy.load(fname)
+        if i == 0:
+            out = data["counts"]
+        else:
+            out += data["counts"]
+        remove(fname)
+
+    fout = paths.cross_nearest(args.simname, args.run, "tot_counts",
+                               nsim=0, nobs=0)
+    if args.verbose:
+        print(f"Writing the summed counts to `{fout}`.", flush=True)
+    numpy.savez(fout, tot_counts=out)
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("--run", type=str, help="Run name")
+    parser.add_argument("--simname", type=str, choices=["csiborg", "quijote"],
+                        help="Simulation name")
+    parser.add_argument("--nsims", type=int, nargs="+", default=None,
+                        help="Indices of simulations to cross. If `-1` processes all simulations.")  # noqa
+    parser.add_argument("--Rmax", type=float, default=155/0.705,
+                        help="High-resolution region radius")
+    parser.add_argument("--verbose", type=lambda x: bool(strtobool(x)),
+                        default=False)
+    args = parser.parse_args()
+    with open("./match_finsnap.yml", "r") as file:
+        config = yaml.safe_load(file)
+
+    if args.simname == "csiborg":
+        save_kind = "dist"
+    else:
+        save_kind = "bin_dist"
+
+    comm = MPI.COMM_WORLD
+    rank = comm.Get_rank()
+    paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
+    cats = open_catalogues(args, config, paths, comm)
+
+    def do_work(nsim):
+        return find_neighbour(args, nsim, cats, paths, comm, save_kind)
+
+    work_delegation(do_work, list(cats.keys()), comm,
+                    master_verbose=args.verbose)
+
+    comm.Barrier()
+    if rank == 0:
+        collect_dist(args, paths)
+        print(f"{datetime.now()}: all finished. Quitting.")

scripts/old/match_knn.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+nthreads=50
+memory=7
+queue="cmb"
+env="/mnt/zfsusers/rstiskalek/csiborgtools/venv_csiborg/bin/python"
+file="match_finsnap.py"
+verbose="true"
+nsims="-1"
+onlogin=false
+
+# for run in "mass001" "mass003" "mass005" "mass007" "mass009"
+for run in "mass002" "mass004" "mass006" "mass008"
+do
+for simname in "csiborg"
+do
+pythoncm="$env $file --simname $simname --run $run --nsims $nsims --verbose $verbose"
+
+if $onlogin
+then
+    $pythoncm
+else
+    cm="addqueue -q $queue -n $nthreads -m $memory $pythoncm"
+    echo "Submitting:"
+    echo $cm
+    echo
+    $cm
+fi
+done
+done

scripts/old/match_knn.yml

@@ -0,0 +1,86 @@
+rmin: 0.1
+rmax: 100
+nneighbours: 8
+nsamples: 1.e+7
+batch_size: 1.e+6
+neval: 10000
+seed: 42
+nbins_marks: 10
+
+
+################################################################################
+#                                 totpartmass                                 #
+################################################################################
+
+
+"mass001":
+  primary:
+    name:
+    - totpartmass
+    - group_mass
+    min: 12.4
+    islog: true
+
+"mass002":
+  primary:
+    name:
+    - totpartmass
+    - group_mass
+    min: 12.6
+    islog: true
+
+"mass003":
+  primary:
+    name:
+    - totpartmass
+    - group_mass
+    min: 12.8
+    islog: true
+
+"mass004":
+  primary:
+    name:
+    - totpartmass
+    - group_mass
+    min: 13.0
+    islog: true
+
+"mass005":
+  primary:
+    name:
+    - totpartmass
+    - group_mass
+    min: 13.2
+    islog: true
+
+"mass006":
+  primary:
+    name:
+    - totpartmass
+    - group_mass
+    min: 13.4
+    islog: true
+
+"mass007":
+  primary:
+    name:
+    - totpartmass
+    - group_mass
+    min: 13.6
+    islog: true
+
+"mass008":
+  primary:
+    name:
+    - totpartmass
+    - group_mass
+    min: 13.8
+    islog: true
+
+"mass009":
+  primary:
+    name:
+    - totpartmass
+    - group_mass
+    min: 14.0
+    islog: true