LSS projected basics (#140)

* Move files

* Move files

* Add galactic to RA/dec

* Update sky maps

* Add projected fields

* Remove old import

* Quick update

* Add IO

* Add imports

* Update imports

* Add basic file
This commit is contained in:
Richard Stiskalek 2024-08-14 13:02:38 +02:00 committed by GitHub
parent 3b46f17ead
commit d578c71b83
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
36 changed files with 365 additions and 231 deletions

View file

@ -0,0 +1,128 @@
# Copyright (C) 2022 Richard Stiskalek
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""
Script to calculate the particle centre of mass and Lagrangian patch size in
the initial snapshot. The initial snapshot particles are read from the sorted
files.
"""
from argparse import ArgumentParser
from datetime import datetime
import csiborgtools
import numpy
from mpi4py import MPI
from taskmaster import work_delegation
from tqdm import tqdm
from utils import get_nsims
def _main(nsim, simname, verbose):
"""
Calculate and save the Lagrangian halo centre of mass and Lagrangian patch
size in the initial snapshot.
Parameters
----------
nsim : int
IC realisation index.
simname : str
Simulation name.
verbose : bool
Verbosity flag.
Returns
-------
None
"""
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
cols = [("index", numpy.int32),
("x", numpy.float32),
("y", numpy.float32),
("z", numpy.float32),
("lagpatch_size", numpy.float32),
("lagpatch_ncells", numpy.int32),]
if simname == "csiborg1":
snap = csiborgtools.read.CSiBORG1Snapshot(nsim, 1, paths,
keep_snapshot_open=True)
cat = csiborgtools.read.CSiBORG1Catalogue(nsim, paths, snapshot=snap)
fout = f"/mnt/extraspace/rstiskalek/csiborg1/chain_{nsim}/initial_lagpatch.npy" # noqa
elif "csiborg2" in simname:
kind = simname.split("_")[-1]
snap = csiborgtools.read.CSiBORG2Snapshot(nsim, 0, kind, paths,
keep_snapshot_open=True)
cat = csiborgtools.read.CSiBORG2Catalogue(nsim, 99, kind, paths,
snapshot=snap)
fout = f"/mnt/extraspace/rstiskalek/csiborg2_{kind}/catalogues/initial_lagpatch_{nsim}.npy" # noqa
elif simname == "quijote":
snap = csiborgtools.read.QuijoteSnapshot(nsim, "ICs", paths,
keep_snapshot_open=True)
cat = csiborgtools.read.QuijoteHaloCatalogue(nsim, paths,
snapshot=snap)
fout = f"/mnt/extraspace/rstiskalek/quijote/fiducial_processed/chain_{nsim}/initial_lagpatch.npy" # noqa
else:
raise ValueError(f"Unknown simulation name `{simname}`.")
boxsize = csiborgtools.simname2boxsize(simname)
# Initialise the overlapper.
if simname == "csiborg" or "csiborg2" in simname:
kwargs = {"box_size": 2048, "bckg_halfsize": 512}
else:
kwargs = {"box_size": 512, "bckg_halfsize": 256}
overlapper = csiborgtools.match.ParticleOverlap(**kwargs)
out = csiborgtools.read.cols_to_structured(len(cat), cols)
for i, hid in enumerate(tqdm(cat["index"]) if verbose else cat["index"]):
out["index"][i] = hid
pos = snap.halo_coordinates(hid)
mass = snap.halo_masses(hid)
# Calculate the centre of mass and the Lagrangian patch size.
cm = csiborgtools.center_of_mass(pos, mass, boxsize=boxsize)
distances = csiborgtools.periodic_distance(pos, cm, boxsize=boxsize)
out["x"][i], out["y"][i], out["z"][i] = cm
out["lagpatch_size"][i] = numpy.percentile(distances, 99)
pos /= boxsize # need to normalize the positions to be [0, 1).
# Calculate the number of cells with > 0 density.
delta = overlapper.make_delta(pos, mass, subbox=True)
out["lagpatch_ncells"][i] = csiborgtools.delta2ncells(delta)
# Now save it
if verbose:
print(f"{datetime.now()}: dumping fits to .. `{fout}`.", flush=True)
with open(fout, "wb") as f:
numpy.save(f, out)
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("--simname", type=str,
choices=["csiborg1", "csiborg2_main", "csiborg2_random", "csiborg2_varysmall", "quijote"], # noqa
help="Simulation name")
parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="IC realisations. If `-1` processes all.")
args = parser.parse_args()
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsims = get_nsims(args, paths)
def main(nsim):
_main(nsim, args.simname, MPI.COMM_WORLD.Get_size() == 1)
work_delegation(main, nsims, MPI.COMM_WORLD)

View file

@ -0,0 +1,23 @@
nthreads=6
memory=16
on_login=${1}
queue="berg"
env="/mnt/zfsusers/rstiskalek/csiborgtools/venv_csiborg/bin/python"
file="fit_init.py"
simname="csiborg2_varysmall"
nsims="-1"
pythoncm="$env $file --nsims $nsims --simname $simname"
if [ $on_login -eq 1 ]; then
echo $pythoncm
$pythoncm
else
cm="addqueue -q $queue -n $nthreads -m $memory $pythoncm"
echo "Submitting:"
echo $cm
echo
eval $cm
fi

View file

@ -0,0 +1,88 @@
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""A script to match all IC pairs of a simulation."""
import warnings
from argparse import ArgumentParser
from distutils.util import strtobool
from itertools import combinations
from random import Random
from mpi4py import MPI
from taskmaster import work_delegation
import csiborgtools
from match_overlap_single import pair_match, pair_match_max
def get_combs(simname):
"""
Get the list of all pairs of IC indices and permute them with a fixed
seed.
"""
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
combs = list(combinations(paths.get_ics(simname), 2))
Random(42).shuffle(combs)
return combs
def main(comb, kind, simname, min_logmass, sigma, mult, verbose):
"""
Match a pair of simulations.
"""
nsim0, nsimx = comb
if kind == "overlap":
pair_match(nsim0, nsimx, simname, min_logmass, sigma, verbose)
elif args.kind == "max":
pair_match_max(nsim0, nsimx, simname, min_logmass, mult, verbose)
else:
raise ValueError(f"Unknown matching kind: `{kind}`.")
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("--kind", type=str, required=True,
choices=["overlap", "max"], help="Kind of matching.")
parser.add_argument("--simname", type=str, required=True,
help="Simulation name.",
choices=["csiborg1", "quijote", "csiborg2_main",
"csiborg2_random", "csiborg2_varysmall"])
parser.add_argument("--nsim0", type=int, default=None,
help="Reference IC for Max's matching method.")
parser.add_argument("--min_logmass", type=float, required=True,
help="Minimum log halo mass.")
parser.add_argument("--sigma", type=float, default=0,
help="Smoothing scale in number of grid cells.")
parser.add_argument("--mult", type=float, default=5,
help="Search radius multiplier for Max's method.")
parser.add_argument("--verbose", type=lambda x: bool(strtobool(x)),
default=False, help="Verbosity flag.")
args = parser.parse_args()
if args.kind == "overlap":
combs = get_combs(args.simname)
else:
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
combs = [(args.nsim0, nsimx) for nsimx in paths.get_ics(args.simname)
if nsimx != args.nsim0]
def _main(comb):
with warnings.catch_warnings():
warnings.filterwarnings("ignore",
"invalid value encountered in cast",
RuntimeWarning)
main(comb, args.kind, args.simname, args.min_logmass, args.sigma,
args.mult, args.verbose)
work_delegation(_main, combs, MPI.COMM_WORLD)

View file

@ -0,0 +1,24 @@
#!/bin/bash
nthreads=41
memory=12
queue="berg"
env="/mnt/zfsusers/rstiskalek/csiborgtools/venv_csiborg/bin/python"
file="match_overlap_all.py"
simname=${1}
min_logmass=12.25
sigma=1
kind="overlap"
mult=10 # Only for Max's method
nsim0=0 # Only for Max's method
verbose="false"
pythoncm="$env $file --kind $kind --simname $simname --nsim0 $nsim0 --min_logmass $min_logmass --mult $mult --sigma $sigma --verbose $verbose"
# $pythoncm
cm="addqueue -q $queue -n $nthreads -m $memory $pythoncm"
echo "Submitting:"
echo $cm
echo
$cm

View file

@ -0,0 +1,218 @@
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""
A script to calculate overlap between two IC realisations of the same
simulation.
"""
from argparse import ArgumentParser
from copy import deepcopy
from datetime import datetime
from distutils.util import strtobool
import numpy
from scipy.ndimage import gaussian_filter
import csiborgtools
def pair_match_max(nsim0, nsimx, simname, min_logmass, mult, verbose):
"""
Match a pair of simulations using the Max method.
Parameters
----------
nsim0, nsimx : int
The reference and cross simulation IC index.
simname : str
Simulation name.
min_logmass : float
Minimum log halo mass.
mult : float
Multiplicative factor for search radius.
verbose : bool
Verbosity flag.
"""
if simname == "csiborg1":
maxdist = 155
periodic = False
bounds = {"dist": (0, maxdist), "totmass": (10**min_logmass, None)}
cat0 = csiborgtools.read.CSiBORG1Catalogue(nsim0, bounds=bounds)
catx = csiborgtools.read.CSiBORG1Catalogue(nsimx, bounds=bounds)
elif "csiborg2" in simname:
raise RuntimeError("CSiBORG2 currently not implemented..")
elif simname == "quijote":
maxdist = None
periodic = True
bounds = {"totmass": (10**min_logmass, None)}
cat0 = csiborgtools.read.QuijoteCatalogue(nsim0, bounds=bounds)
catx = csiborgtools.read.QuijoteHaloCatalogue(nsimx, bounds=bounds)
else:
raise ValueError(f"Unknown simulation `{simname}`.")
reader = csiborgtools.summary.PairOverlap(cat0, catx, min_logmass, maxdist)
out = csiborgtools.match.matching_max(
cat0, catx, "totmass", mult=mult, periodic=periodic,
overlap=reader.overlap(from_smoothed=True),
match_indxs=reader["match_indxs"], verbose=verbose)
fout = cat0.paths.match_max(simname, nsim0, nsimx, min_logmass, mult)
if verbose:
print(f"{datetime.now()}: saving to ... `{fout}`.", flush=True)
numpy.savez(fout, **{p: out[p] for p in out.dtype.names})
def pair_match(nsim0, nsimx, simname, min_logmass, sigma, verbose):
"""
Calculate overlaps between two simulations.
Parameters
----------
nsim0 : int
The reference simulation IC index.
nsimx : int
The cross simulation IC index.
simname : str
Simulation name.
min_logmass : float
Minimum log halo mass.
sigma : float
Smoothing scale in number of grid cells.
verbose : bool
Verbosity flag.
Returns
-------
None
"""
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
smooth_kwargs = {"sigma": sigma, "mode": "constant", "cval": 0}
bounds = {"lagpatch_radius": (0, None)}
if simname == "csiborg1":
overlapper_kwargs = {"box_size": 2048, "bckg_halfsize": 512}
bounds |= {"dist": (0, 135), "totmass": (10**min_logmass, None)}
# Reference simulation.
snap0 = csiborgtools.read.CSiBORG1Snapshot(
nsim0, 1, keep_snapshot_open=True)
cat0 = csiborgtools.read.CSiBORG1Catalogue(
nsim0, snapshot=snap0, bounds=bounds)
# Cross simulation.
snapx = csiborgtools.read.CSiBORG1Snapshot(
nsimx, 1, keep_snapshot_open=True)
catx = csiborgtools.read.CSiBORG1Catalogue(
nsimx, snapshot=snapx, bounds=bounds)
elif "csiborg2" in simname:
kind = simname.split("_")[-1]
overlapper_kwargs = {"box_size": 2048, "bckg_halfsize": 512}
bounds |= {"dist": (0, 135), "totmass": (10**min_logmass, None)}
# Reference simulation.
snap0 = csiborgtools.read.CSiBORG2Snapshot(
nsim0, 99, kind, keep_snapshot_open=True)
cat0 = csiborgtools.read.CSiBORG2Catalogue(
nsim0, 99, kind, snapshot=snap0, bounds=bounds)
# Cross simulation.
snapx = csiborgtools.read.CSiBORG2Snapshot(
nsimx, 99, kind, keep_snapshot_open=True)
catx = csiborgtools.read.CSiBORG2Catalogue(
nsimx, 99, kind, snapshot=snapx, bounds=bounds)
elif simname == "quijote":
overlapper_kwargs = {"box_size": 512, "bckg_halfsize": 256}
bounds |= {"totmass": (10**min_logmass, None)}
# Reference simulation.
snap0 = csiborgtools.read.QuijoteSnapshot(
nsim0, "ICs", keep_snapshot_open=True)
cat0 = csiborgtools.read.QuijoteCatalogue(
nsim0, snapshot=snap0, bounds=bounds)
# Cross simulation.
snapx = csiborgtools.read.QuijoteSnapshot(
nsimx, "ICs", keep_snapshot_open=True)
catx = csiborgtools.read.QuijoteCatalogue(
nsimx, snapshot=snapx, bounds=bounds)
else:
raise ValueError(f"Unknown simulation name: `{simname}`.")
overlapper = csiborgtools.match.ParticleOverlap(**overlapper_kwargs)
delta_bckg = overlapper.make_bckg_delta(cat0, verbose=verbose)
delta_bckg = overlapper.make_bckg_delta(catx, delta=delta_bckg,
verbose=verbose)
matcher = csiborgtools.match.RealisationsMatcher(**overlapper_kwargs)
match_indxs, ngp_overlap = matcher.cross(cat0, catx, delta_bckg,
verbose=verbose)
# We want to store the halo IDs of the matches, not their array positions
# in the catalogues.
match_hids = deepcopy(match_indxs)
for i, matches in enumerate(match_indxs):
for j, match in enumerate(matches):
match_hids[i][j] = catx["index"][match]
fout = paths.overlap(simname, nsim0, nsimx, min_logmass, smoothed=False)
if verbose:
print(f"{datetime.now()}: saving to ... `{fout}`.", flush=True)
numpy.savez(fout, ref_hids=cat0["index"], match_hids=match_hids,
ngp_overlap=ngp_overlap)
if not sigma > 0:
return
if verbose:
print(f"{datetime.now()}: smoothing the background field.", flush=True)
gaussian_filter(delta_bckg, output=delta_bckg, **smooth_kwargs)
# We calculate the smoothed overlap for the pairs whose NGP overlap is > 0.
smoothed_overlap = matcher.smoothed_cross(cat0, catx, delta_bckg,
match_indxs, smooth_kwargs,
verbose=verbose)
fout = paths.overlap(simname, nsim0, nsimx, min_logmass, smoothed=True)
if verbose:
print(f"{datetime.now()}: saving to ... `{fout}`.", flush=True)
numpy.savez(fout, smoothed_overlap=smoothed_overlap, sigma=sigma)
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("--kind", type=str, required=True,
choices=["overlap", "max"], help="Kind of matching.")
parser.add_argument("--nsim0", type=int, required=True,
help="Reference simulation IC index.")
parser.add_argument("--nsimx", type=int, required=True,
help="Cross simulation IC index.")
parser.add_argument("--simname", type=str, required=True,
help="Simulation name.")
parser.add_argument("--min_logmass", type=float, required=True,
help="Minimum log halo mass.")
parser.add_argument("--mult", type=float, default=5,
help="Search radius multiplier for Max's method.")
parser.add_argument("--sigma", type=float, default=0,
help="Smoothing scale in number of grid cells.")
parser.add_argument("--verbose", type=lambda x: bool(strtobool(x)),
default=False, help="Verbosity flag.")
args = parser.parse_args()
if args.kind == "overlap":
pair_match(args.nsim0, args.nsimx, args.simname, args.min_logmass,
args.sigma, args.verbose)
elif args.kind == "max":
pair_match_max(args.nsim0, args.nsimx, args.simname, args.min_logmass,
args.mult, args.verbose)
else:
raise ValueError(f"Unknown matching kind: `{args.kind}`.")

View file

@ -0,0 +1,50 @@
#!/bin/bash
nthreads=1
memory=7
queue="berg"
env="/mnt/zfsusers/rstiskalek/csiborgtools/venv_csiborg/bin/python"
verbose="true"
file="match_overlap_single.py"
simname="csiborg2_main"
kind="overlap"
min_logmass=12
mult=5 # Only for Max's method
sigma=1
# sims=(7444 7468)
sims=(16417 16517)
# sims=(0 1)
# sims=(7468 7588)
nsims=${#sims[@]}
for i in $(seq 0 $((nsims-1)))
do
for j in $(seq 0 $((nsims-1)))
do
if [ $i -eq $j ]
then
continue
elif [ $i -gt $j ]
then
continue
else
:
fi
nsim0=${sims[$i]}
nsimx=${sims[$j]}
pythoncm="$env $file --kind $kind --nsim0 $nsim0 --nsimx $nsimx --simname $simname --min_logmass $min_logmass --sigma $sigma --mult $mult --verbose $verbose"
$pythoncm
# cm="addqueue -q $queue -n 1x1 -m $memory $pythoncm"
# echo "Submitting:"
# echo $cm
# echo
# $cm
# sleep 0.05
done
done