Aquila-Consortium/csiborgtools
0
0
Fork 0
mirror of https://github.com/Richard-Sti/csiborgtools_public.git synced 2025-05-13 14:11:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improving halo fits (#76)

Browse source 
* Add periodic distances

* Little corrections

* Fix little bug

* Modernise the script

* Small updates

* Remove clump

* Add new halo routines

* Fix weights

* Modernise the script

* Add check ups on convergence

* More convergence check ups

* Edit bounds

* Add default argument

* Update fit heuristic and NaNs

* Change maxiter

* Switch NFW minimization to log-sapce

* Remove print statement

* Turn convert_from_box abstract property required for all boxes.

* Move files

* Simplify script

* Improve the argument parser

* Remove optinal argument

* Improve argument parser

* Add a minimum concentration limit
This commit is contained in:
Richard Stiskalek 2023-07-25 16:12:58 +02:00 • committed by GitHub
parent eb8d070fff
commit e08c741fc8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 460 additions and 735 deletions
Download patch file Download diff file Expand all files Collapse all files

10
scripts/cluster_tpcf_auto.py
View file

@ -60,9 +60,9 @@ if __name__ == "__main__":
parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="Indices of simulations to cross. If `-1` processes all simulations.") # noqa
parser.add_argument("--Rmax", type=float, default=155/0.705,
help="High-resolution region radius") # noqa
help="High-resolution region radius.")
parser.add_argument("--verbose", type=lambda x: bool(strtobool(x)),
default=False)
default=False, help="Verbosity flag.")
args = parser.parse_args()
with open("./cluster_tpcf_auto.yml", "r") as file:
@ -79,8 +79,4 @@ if __name__ == "__main__":
return do_auto(args, config, cats, nsim, paths)
nsims = list(cats.keys())
work_delegation(do_work, nsims, comm, master_verbose=args.verbose)
comm.Barrier()
if comm.Get_rank() == 0:
print(f"{datetime.now()}: all finished. Quitting.")
work_delegation(do_work, nsims, comm)

151
scripts/fit_halos.py
View file

@ -13,14 +13,15 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""
A script to fit FoF halos (concentration, ...). The particle array of each
CSiBORG realisation must have been processed in advance by `pre_dumppart.py`.
A script to fit FoF halos (concentration, ...). The CSiBORG particle array of
each realisation must have been processed in advance by `pre_dumppart.py`.
Quijote is not supported yet
"""
from argparse import ArgumentParser
from datetime import datetime
import numpy
from mpi4py import MPI
from taskmaster import work_delegation
from tqdm import trange
from utils import get_nsims
@ -33,72 +34,67 @@ except ModuleNotFoundError:
sys.path.append("../")
import csiborgtools
# Get MPI things
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nproc = comm.Get_size()
verbose = nproc == 1
parser = ArgumentParser()
parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="IC realisations. If `-1` processes all simulations.")
args = parser.parse_args()
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
partreader = csiborgtools.read.ParticleReader(paths)
nfwpost = csiborgtools.fits.NFWPosterior()
nsims = get_nsims(args, paths)
def fit_halo(particles, box):
"""
Fit a single halo from the particle array.
cols_collect = [
("index", numpy.int32),
("npart", numpy.int32),
("totpartmass", numpy.float32),
("vx", numpy.float32),
("vy", numpy.float32),
("vz", numpy.float32),
("conc", numpy.float32),
("rho0", numpy.float32),
("r200c", numpy.float32),
("r500c", numpy.float32),
("m200c", numpy.float32),
("m500c", numpy.float32),
("lambda200c", numpy.float32),
("r200m", numpy.float32),
("m200m", numpy.float32),
("r500m", numpy.float32),
("m500m", numpy.float32),
]
Parameters
----------
particles : 2-dimensional array of shape `(n_particles, 3)`
Particle array. The columns must be `x`, `y`, `z`, `vx`, `vy`, `vz`,
`M`.
box : object derived from :py:class`csiborgtools.read.BaseBox`
Box object.
def fit_halo(particles, clump_info, box):
obj = csiborgtools.fits.Clump(particles, clump_info, box)
Returns
-------
out : dict
"""
halo = csiborgtools.fits.Halo(particles, box)
out = {}
out["npart"] = len(obj)
out["totpartmass"] = numpy.sum(obj["M"])
out["npart"] = len(halo)
out["totpartmass"] = numpy.sum(halo["M"])
for i, v in enumerate(["vx", "vy", "vz"]):
out[v] = numpy.average(obj.vel[:, i], weights=obj["M"])
# Overdensity masses
for n in [200, 500]:
out[f"r{n}c"], out[f"m{n}c"] = obj.spherical_overdensity_mass(
n, kind="crit", npart_min=10)
out[f"r{n}m"], out[f"m{n}m"] = obj.spherical_overdensity_mass(
n, kind="matter", npart_min=10)
# NFW fit
if out["npart"] > 10 and numpy.isfinite(out["r200c"]):
Rs, rho0 = nfwpost.fit(obj)
out["conc"] = out["r200c"] / Rs
out["rho0"] = rho0
# Spin within R200c
if numpy.isfinite(out["r200c"]):
out["lambda200c"] = obj.lambda_bullock(out["r200c"])
out[v] = numpy.average(halo.vel[:, i], weights=halo["M"])
m200c, r200c, cm = halo.spherical_overdensity_mass(200, kind="crit",
maxiter=100)
out["m200c"] = m200c
out["r200c"] = r200c
out["lambda200c"] = halo.lambda_bullock(cm, r200c)
out["conc"] = halo.nfw_concentration(cm, r200c)
return out
# We MPI loop over all simulations.
jobs = csiborgtools.fits.split_jobs(len(nsims), nproc)[rank]
for nsim in [nsims[i] for i in jobs]:
print(f"{datetime.now()}: rank {rank} calculating simulation `{nsim}`.",
flush=True)
def _main(nsim, simname, verbose):
"""
Fit the FoF halos.
Parameters
----------
nsim : int
IC realisation index.
simname : str
Simulation name.
verbose : bool
Verbosity flag.
"""
if simname == "quijote":
raise NotImplementedError("Quijote not implemented yet.")
cols = [("index", numpy.int32),
("npart", numpy.int32),
("totpartmass", numpy.float32),
("vx", numpy.float32),
("vy", numpy.float32),
("vz", numpy.float32),
("conc", numpy.float32),
("r200c", numpy.float32),
("m200c", numpy.float32),
("lambda200c", numpy.float32),]
nsnap = max(paths.get_snapshots(nsim))
box = csiborgtools.read.CSiBORGBox(nsnap, nsim, paths)
@ -106,29 +102,44 @@ for nsim in [nsims[i] for i in jobs]:
f = csiborgtools.read.read_h5(paths.particles(nsim))
particles = f["particles"]
halo_map = f["halomap"]
hid2map = {clid: i for i, clid in enumerate(halo_map[:, 0])}
hid2map = {hid: i for i, hid in enumerate(halo_map[:, 0])}
cat = csiborgtools.read.CSiBORGHaloCatalogue(
nsim, paths, with_lagpatch=False, load_initial=False, rawdata=True,
load_fitted=False)
# Even if we are calculating parent halo this index runs over all clumps.
out = csiborgtools.read.cols_to_structured(len(cat), cols_collect)
indxs = cat["index"]
out = csiborgtools.read.cols_to_structured(len(cat), cols)
for i in trange(len(cat)) if verbose else range(len(cat)):
hid = cat["index"][i]
out["index"][i] = hid
part = csiborgtools.read.load_halo_particles(hid, particles, halo_map,
hid2map)
# We fit the particles if there are any. If not we assign the index,
# otherwise it would be NaN converted to integers (-2147483648) and
# yield an error further down.
# Skip if no particles.
if part is None:
continue
_out = fit_halo(part, cat[i], box)
_out = fit_halo(part, box)
for key in _out.keys():
out[key][i] = _out[key]
fout = paths.structfit(nsnap, nsim)
print(f"Saving to `{fout}`.", flush=True)
if verbose:
print(f"Saving to `{fout}`.", flush=True)
numpy.save(fout, out)
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("--simname", type=str, default="csiborg",
choices=["csiborg", "quijote", "quijote_full"],
help="Simulation name")
parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="IC realisations. If `-1` processes all.")
args = parser.parse_args()
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsims = get_nsims(args, paths)
def main(nsim):
_main(nsim, args.simname, MPI.COMM_WORLD.Get_size() == 1)
work_delegation(main, nsims, MPI.COMM_WORLD)

14
scripts/fit_hmf.py
View file

@ -94,17 +94,13 @@ if __name__ == "__main__":
parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="Indices of simulations to cross. If `-1` processes all simulations.") # noqa
parser.add_argument("--Rmax", type=float, default=155/0.705,
help="High-resolution region radius")
help="High-resolution region radius. Ignored for `quijote_full`.") # noqa
parser.add_argument("--bw", type=float, default=0.2,
help="Bin width in dex")
help="Bin width in dex.")
parser.add_argument("--verbose", type=lambda x: bool(strtobool(x)),
default=False)
default=False, help="Verbosity flag.")
parser_args = parser.parse_args()
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nproc = comm.Get_size()
verbose = nproc == 1
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsims = get_nsims(parser_args, paths)
bins = numpy.arange(11., 16., parser_args.bw, dtype=numpy.float32)
@ -112,4 +108,4 @@ if __name__ == "__main__":
def do_work(nsim):
get_counts(nsim, bins, paths, parser_args)
work_delegation(do_work, nsims, comm, master_verbose=parser_args.verbose)
work_delegation(do_work, nsims, MPI.COMM_WORLD)

93
scripts/fit_init.py
View file

@ -22,6 +22,7 @@ from datetime import datetime
import numpy
from mpi4py import MPI
from taskmaster import work_delegation
from tqdm import tqdm
from utils import get_nsims
@ -35,73 +36,83 @@ except ModuleNotFoundError:
import csiborgtools
# Get MPI things
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nproc = comm.Get_size()
verbose = nproc == 1
def _main(nsim, simname, verbose):
"""
Calculate the Lagrangian halo centre of mass and Lagrangian patch size in
the initial snapshot.
# Argument parser
parser = ArgumentParser()
parser.add_argument("--simname", type=str, default="csiborg",
choices=["csiborg", "quijote"],
help="Simulation name")
parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="IC realisations. If `-1` processes all simulations.")
args = parser.parse_args()
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
partreader = csiborgtools.read.ParticleReader(paths)
Parameters
----------
nsim : int
IC realisation index.
simname : str
Simulation name.
verbose : bool
Verbosity flag.
"""
if simname == "quijote":
raise NotImplementedError("Quijote not implemented yet.")
nsims = get_nsims(args, paths)
cols_collect = [("index", numpy.int32),
("x", numpy.float32),
("y", numpy.float32),
("z", numpy.float32),
("lagpatch_size", numpy.float32),
("lagpatch_ncells", numpy.int32),]
# MPI loop over simulations
jobs = csiborgtools.fits.split_jobs(len(nsims), nproc)[rank]
for nsim in [nsims[i] for i in jobs]:
nsnap = max(paths.get_snapshots(nsim))
overlapper = csiborgtools.match.ParticleOverlap()
print(f"{datetime.now()}: rank {rank} calculating simulation `{nsim}`.",
flush=True)
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
cols = [("index", numpy.int32),
("x", numpy.float32),
("y", numpy.float32),
("z", numpy.float32),
("lagpatch_size", numpy.float32),
("lagpatch_ncells", numpy.int32),]
parts = csiborgtools.read.read_h5(paths.initmatch(nsim, "particles"))
parts = parts['particles']
halo_map = csiborgtools.read.read_h5(paths.particles(nsim))
halo_map = halo_map["halomap"]
cat = csiborgtools.read.CSiBORGHaloCatalogue(
nsim, paths, rawdata=True, load_fitted=False, load_initial=False)
hid2map = {hid: i for i, hid in enumerate(halo_map[:, 0])}
out = csiborgtools.read.cols_to_structured(len(cat), cols_collect)
out = csiborgtools.read.cols_to_structured(len(cat), cols)
for i, hid in enumerate(tqdm(cat["index"]) if verbose else cat["index"]):
out["index"][i] = hid
part = csiborgtools.read.load_halo_particles(hid, parts, halo_map,
hid2map)
# Skip if the halo is too small.
# Skip if the halo has no particles or is too small.
if part is None or part.size < 100:
continue
pos, mass = part[:, :3], part[:, 3]
# Calculate the centre of mass and the Lagrangian patch size.
dist, cm = csiborgtools.fits.dist_centmass(part)
# We enforce a maximum patchsize of 0.075 in box coordinates.
patchsize = min(numpy.percentile(dist, 99), 0.075)
cm = csiborgtools.fits.center_of_mass(pos, mass, boxsize=1.0)
distances = csiborgtools.fits.periodic_distance(pos, cm, boxsize=1.0)
out["x"][i], out["y"][i], out["z"][i] = cm
out["lagpatch_size"][i] = patchsize
out["lagpatch_size"][i] = numpy.percentile(distances, 99)
# Calculate the number of cells with > 0 density.
delta = overlapper.make_delta(part[:, :3], part[:, 3], subbox=True)
overlapper = csiborgtools.match.ParticleOverlap()
delta = overlapper.make_delta(pos, mass, subbox=True)
out["lagpatch_ncells"][i] = csiborgtools.fits.delta2ncells(delta)
# Now save it
fout = paths.initmatch(nsim, "fit")
print(f"{datetime.now()}: dumping fits to .. `{fout}`.",
flush=True)
if verbose:
print(f"{datetime.now()}: dumping fits to .. `{fout}`.", flush=True)
with open(fout, "wb") as f:
numpy.save(f, out)
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("--simname", type=str, default="csiborg",
choices=["csiborg", "quijote"],
help="Simulation name")
parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="IC realisations. If `-1` processes all.")
args = parser.parse_args()
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsims = get_nsims(args, paths)
def main(nsim):
_main(nsim, args.simname, MPI.COMM_WORLD.Get_size() == 1)
work_delegation(main, nsims, MPI.COMM_WORLD)

4
scripts/mv_fofmembership.py
View file

@ -146,6 +146,4 @@ if __name__ == "__main__":
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsims = get_nsims(args, paths)
comm = MPI.COMM_WORLD
work_delegation(main, nsims, comm)
work_delegation(main, nsims, MPI.COMM_WORLD)

0
scripts/pre_mmain.py → scripts/old/pre_mmain.py
View file

14
scripts/old/pre_mmain.sh Normal file
View file

@ -0,0 +1,14 @@
nthreads=102
memory=5
queue="cmb"
env="/mnt/zfsusers/rstiskalek/csiborgtools/venv_csiborg/bin/python"
file="pre_mmain.py"
# pythoncm="$env $file"
# $pythoncm
cm="addqueue -q $queue -n $nthreads -m $memory $env $file"
echo "Submitting:"
echo $cm
$cm

4
scripts/pre_dumppart.py
View file

@ -169,7 +169,7 @@ if __name__ == "__main__":
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsims = get_nsims(args, paths)
def _main(nsim, verbose=MPI.COMM_WORLD.nproc == 1):
main(nsim, args.simname, verbose=verbose)
def _main(nsim):
main(nsim, args.simname, verbose=MPI.COMM_WORLD.Get_size() == 1)
work_delegation(_main, nsims, MPI.COMM_WORLD)

2
scripts/pre_sortinit.py
View file

@ -95,6 +95,6 @@ if __name__ == "__main__":
nsims = get_nsims(args, paths)
def main(nsim):
_main(nsim, args.simname, MPI.COMM_WORLD.size == 1)
_main(nsim, args.simname, MPI.COMM_WORLD.Get_size() == 1)
work_delegation(main, nsims, MPI.COMM_WORLD)