mirror of
https://github.com/Richard-Sti/csiborgtools_public.git
synced 2025-05-12 05:38:42 +00:00
New filesystem, basic neighbours calculation (#10)
* simplify Planck catalogue * add MCXC and base survey * Add 2MPP classes * move match to MCXC to Planck * add halo catalogue * rm comment * rm unused imports * Move conversion to box * add min mass * Run on all simulations * rm old function * add combined catalogue * add halo positions * add knn neighbors * set to 5 sims for testing * add docstring * Switch to neighbours in a catalogue * rename io to read * fix indentation * rename to read * io -> read * add import * add RealisationMatcher * io -> read * add docstrings * add search_sim_indiices * update todo * keep make_cat at 10 for now * update nb
This commit is contained in:
Richard Stiskalek
GitHub
parent
f5bf04c46e
commit
c748c87e45
17 changed files with 3515 additions and 549 deletions
File diff suppressed because one or more lines are too long
|
|
@ -32,9 +32,6 @@ import utils
|
|||
F64 = numpy.float64
|
||||
I64 = numpy.int64
|
||||
|
||||
# Simulations and their snapshot to analyze
|
||||
Nsims = [9844]
|
||||
Nsnap = 1016
|
||||
|
||||
# Get MPI things
|
||||
comm = MPI.COMM_WORLD
|
||||
|
|
@ -50,68 +47,76 @@ cols_collect = [("npart", I64), ("totpartmass", F64), ("Rs", F64),
|
|||
("rmax", F64), ("r200", F64), ("r500", F64),
|
||||
("m200", F64), ("m500", F64)]
|
||||
|
||||
# NOTE later loop over sims too
|
||||
Nsim = Nsims[0]
|
||||
simpath = csiborgtools.io.get_sim_path(Nsim)
|
||||
box = csiborgtools.units.BoxUnits(Nsnap, simpath)
|
||||
Nsims = csiborgtools.read.get_csiborg_ids("/mnt/extraspace/hdesmond")
|
||||
for i, Nsim in enumerate(Nsims):
|
||||
if rank == 0:
|
||||
print("{}: calculating {}th simulation.".format(datetime.now(), i))
|
||||
|
||||
jobs = csiborgtools.fits.split_jobs(utils.Nsplits, nproc)[rank]
|
||||
for icount, Nsplit in enumerate(jobs):
|
||||
print("{}: rank {} working {} / {} jobs.".format(datetime.now(), rank,
|
||||
icount + 1, len(jobs)))
|
||||
parts, part_clumps, clumps = csiborgtools.fits.load_split_particles(
|
||||
Nsplit, loaddir, Nsim, Nsnap, remove_split=False)
|
||||
simpath = csiborgtools.read.get_sim_path(Nsim)
|
||||
Nsnap = csiborgtools.read.get_maximum_snapshot(simpath)
|
||||
box = csiborgtools.units.BoxUnits(Nsnap, simpath)
|
||||
|
||||
N = clumps.size
|
||||
cols = [("index", I64), ("npart", I64), ("totpartmass", F64),
|
||||
("Rs", F64), ("rho0", F64), ("conc", F64),
|
||||
("vx", F64), ("vy", F64), ("vz", F64),
|
||||
("rmin", F64), ("rmax", F64),
|
||||
("r200", F64), ("r500", F64), ("m200", F64), ("m500", F64)]
|
||||
out = csiborgtools.utils.cols_to_structured(N, cols)
|
||||
out["index"] = clumps["index"]
|
||||
jobs = csiborgtools.fits.split_jobs(utils.Nsplits, nproc)[rank]
|
||||
for Nsplit in jobs:
|
||||
parts, part_clumps, clumps = csiborgtools.fits.load_split_particles(
|
||||
Nsplit, loaddir, Nsim, Nsnap, remove_split=False)
|
||||
|
||||
for n in range(N):
|
||||
# Pick clump and its particles
|
||||
xs = csiborgtools.fits.pick_single_clump(n, parts, part_clumps, clumps)
|
||||
clump = csiborgtools.fits.Clump.from_arrays(*xs, rhoc=box.box_rhoc)
|
||||
out["npart"][n] = clump.Npart
|
||||
out["rmin"][n] = clump.rmin
|
||||
out["rmax"][n] = clump.rmax
|
||||
out["totpartmass"][n] = clump.total_particle_mass
|
||||
out["vx"] = numpy.average(clump.vel[:, 0], weights=clump.m)
|
||||
out["vy"] = numpy.average(clump.vel[:, 1], weights=clump.m)
|
||||
out["vz"] = numpy.average(clump.vel[:, 2], weights=clump.m)
|
||||
N = clumps.size
|
||||
cols = [("index", I64), ("npart", I64), ("totpartmass", F64),
|
||||
("Rs", F64), ("rho0", F64), ("conc", F64),
|
||||
("vx", F64), ("vy", F64), ("vz", F64),
|
||||
("rmin", F64), ("rmax", F64),
|
||||
("r200", F64), ("r500", F64), ("m200", F64), ("m500", F64)]
|
||||
out = csiborgtools.utils.cols_to_structured(N, cols)
|
||||
out["index"] = clumps["index"]
|
||||
|
||||
# Spherical overdensity radii and masses
|
||||
rs, ms = clump.spherical_overdensity_mass([200, 500])
|
||||
out["r200"][n] = rs[0]
|
||||
out["r500"][n] = rs[1]
|
||||
out["m200"][n] = ms[0]
|
||||
out["m500"][n] = ms[1]
|
||||
for n in range(N):
|
||||
# Pick clump and its particles
|
||||
xs = csiborgtools.fits.pick_single_clump(n, parts, part_clumps,
|
||||
clumps)
|
||||
clump = csiborgtools.fits.Clump.from_arrays(*xs, rhoc=box.box_rhoc)
|
||||
out["npart"][n] = clump.Npart
|
||||
out["rmin"][n] = clump.rmin
|
||||
out["rmax"][n] = clump.rmax
|
||||
out["totpartmass"][n] = clump.total_particle_mass
|
||||
out["vx"] = numpy.average(clump.vel[:, 0], weights=clump.m)
|
||||
out["vy"] = numpy.average(clump.vel[:, 1], weights=clump.m)
|
||||
out["vz"] = numpy.average(clump.vel[:, 2], weights=clump.m)
|
||||
|
||||
# NFW profile fit
|
||||
if clump.Npart > 10 and numpy.isfinite(out["r200"][n]):
|
||||
nfwpost = csiborgtools.fits.NFWPosterior(clump)
|
||||
logRs, __ = nfwpost.maxpost_logRs()
|
||||
Rs = 10**logRs
|
||||
if not numpy.isnan(logRs):
|
||||
out["Rs"][n] = Rs
|
||||
out["rho0"][n] = nfwpost.rho0_from_Rs(Rs)
|
||||
out["conc"][n] = out["r200"][n] / Rs
|
||||
# Spherical overdensity radii and masses
|
||||
rs, ms = clump.spherical_overdensity_mass([200, 500])
|
||||
out["r200"][n] = rs[0]
|
||||
out["r500"][n] = rs[1]
|
||||
out["m200"][n] = ms[0]
|
||||
out["m500"][n] = ms[1]
|
||||
|
||||
csiborgtools.io.dump_split(out, Nsplit, Nsim, Nsnap, dumpdir)
|
||||
# NFW profile fit
|
||||
if clump.Npart > 10 and numpy.isfinite(out["r200"][n]):
|
||||
nfwpost = csiborgtools.fits.NFWPosterior(clump)
|
||||
logRs, __ = nfwpost.maxpost_logRs()
|
||||
Rs = 10**logRs
|
||||
if not numpy.isnan(logRs):
|
||||
out["Rs"][n] = Rs
|
||||
out["rho0"][n] = nfwpost.rho0_from_Rs(Rs)
|
||||
out["conc"][n] = out["r200"][n] / Rs
|
||||
|
||||
csiborgtools.read.dump_split(out, Nsplit, Nsim, Nsnap, dumpdir)
|
||||
|
||||
# Wait until all jobs finished before moving to another simulation
|
||||
comm.Barrier()
|
||||
|
||||
# Use the rank 0 to combine outputs for this CSiBORG realisation
|
||||
if rank == 0:
|
||||
print("Collecting results!")
|
||||
out_collected = csiborgtools.read.combine_splits(
|
||||
utils.Nsplits, Nsim, Nsnap, utils.dumpdir, cols_collect,
|
||||
remove_splits=True, verbose=False)
|
||||
fname = join(utils.dumpdir, "ramses_out_{}_{}.npy"
|
||||
.format(str(Nsim).zfill(5), str(Nsnap).zfill(5)))
|
||||
print("Saving results to `{}`.".format(fname))
|
||||
numpy.save(fname, out_collected)
|
||||
|
||||
comm.Barrier()
|
||||
|
||||
# Force all ranks to wait
|
||||
comm.Barrier()
|
||||
# Use the rank 0 to combine outputs for this CSiBORG realisation
|
||||
if rank == 0:
|
||||
print("Collecting results!")
|
||||
out_collected = csiborgtools.io.combine_splits(
|
||||
utils.Nsplits, Nsim, Nsnap, utils.dumpdir, cols_collect,
|
||||
remove_splits=True, verbose=False)
|
||||
fname = join(utils.dumpdir, "ramses_out_{}_{}.npy"
|
||||
.format(str(Nsim).zfill(5), str(Nsnap).zfill(5)))
|
||||
print("Saving results to `{}`.".format(fname))
|
||||
numpy.save(fname, out_collected)
|
||||
print("All finished! See ya!")
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ comm = MPI.COMM_WORLD
|
|||
rank = comm.Get_rank()
|
||||
nproc = comm.Get_size()
|
||||
|
||||
Nsims = csiborgtools.io.get_csiborg_ids("/mnt/extraspace/hdesmond")
|
||||
Nsims = csiborgtools.read.get_csiborg_ids("/mnt/extraspace/hdesmond")
|
||||
partcols = ["x", "y", "z", "vx", "vy", "vz", "M", "level"]
|
||||
dumpdir = join(utils.dumpdir, "temp")
|
||||
|
||||
|
|
@ -43,16 +43,16 @@ for icount, sim_index in enumerate(jobs):
|
|||
print("{}: rank {} working {} / {} jobs.".format(datetime.now(), rank,
|
||||
icount + 1, len(jobs)))
|
||||
Nsim = Nsims[sim_index]
|
||||
simpath = csiborgtools.io.get_sim_path(Nsim)
|
||||
Nsnap = csiborgtools.io.get_maximum_snapshot(simpath)
|
||||
simpath = csiborgtools.read.get_sim_path(Nsim)
|
||||
Nsnap = csiborgtools.read.get_maximum_snapshot(simpath)
|
||||
# Load the clumps, particles' clump IDs and particles.
|
||||
clumps = csiborgtools.io.read_clumps(Nsnap, simpath)
|
||||
particle_clumps = csiborgtools.io.read_clumpid(Nsnap, simpath,
|
||||
verbose=False)
|
||||
particles = csiborgtools.io.read_particle(partcols, Nsnap, simpath,
|
||||
verbose=False)
|
||||
clumps = csiborgtools.read.read_clumps(Nsnap, simpath)
|
||||
particle_clumps = csiborgtools.read.read_clumpid(Nsnap, simpath,
|
||||
verbose=False)
|
||||
particles = csiborgtools.read.read_particle(partcols, Nsnap, simpath,
|
||||
verbose=False)
|
||||
# Drop all particles whose clump index is 0 (not assigned to any halo)
|
||||
particle_clumps, particles = csiborgtools.io.drop_zero_indx(
|
||||
particle_clumps, particles = csiborgtools.read.drop_zero_indx(
|
||||
particle_clumps, particles)
|
||||
# Dump it!
|
||||
csiborgtools.fits.dump_split_particles(particles, particle_clumps, clumps,
|
||||
|
|
|
|||
|
|
@ -16,16 +16,14 @@
|
|||
Notebook utility functions.
|
||||
"""
|
||||
|
||||
# import numpy
|
||||
# from os.path import join
|
||||
|
||||
import numpy
|
||||
from os.path import join
|
||||
from astropy.cosmology import FlatLambdaCDM
|
||||
|
||||
try:
|
||||
import csiborgtools
|
||||
except ModuleNotFoundError:
|
||||
import sys
|
||||
sys.path.append("../")
|
||||
# try:
|
||||
# import csiborgtools
|
||||
# except ModuleNotFoundError:
|
||||
# import sys
|
||||
# sys.path.append("../")
|
||||
|
||||
|
||||
Nsplits = 200
|
||||
|
|
@ -42,52 +40,3 @@ _virgo = {"RA": (12 + 27 / 60) * 15,
|
|||
"COMDIST": 16.5}
|
||||
|
||||
specific_clusters = {"Coma": _coma, "Virgo": _virgo}
|
||||
|
||||
|
||||
def load_processed(Nsim, Nsnap):
|
||||
simpath = csiborgtools.io.get_sim_path(Nsim)
|
||||
outfname = join(
|
||||
dumpdir, "ramses_out_{}_{}.npy".format(str(Nsim).zfill(5),
|
||||
str(Nsnap).zfill(5)))
|
||||
data = numpy.load(outfname)
|
||||
# Add mmain
|
||||
mmain = csiborgtools.io.read_mmain(Nsim, "/mnt/zfsusers/hdesmond/Mmain")
|
||||
data = csiborgtools.io.merge_mmain_to_clumps(data, mmain)
|
||||
csiborgtools.utils.flip_cols(data, "peak_x", "peak_z")
|
||||
# Cut on numbre of particles and finite m200
|
||||
data = data[(data["npart"] > 100) & numpy.isfinite(data["m200"])]
|
||||
|
||||
# Do unit conversion
|
||||
boxunits = csiborgtools.units.BoxUnits(Nsnap, simpath)
|
||||
convert_cols = ["m200", "m500", "totpartmass", "mass_mmain",
|
||||
"r200", "r500", "Rs", "rho0", "peak_x", "peak_y", "peak_z"]
|
||||
data = csiborgtools.units.convert_from_boxunits(
|
||||
data, convert_cols, boxunits)
|
||||
# Now calculate spherical coordinates
|
||||
d, ra, dec = csiborgtools.units.cartesian_to_radec(data)
|
||||
data = csiborgtools.utils.add_columns(
|
||||
data, [d, ra, dec], ["dist", "ra", "dec"])
|
||||
return data, boxunits
|
||||
|
||||
|
||||
def load_planck2015(max_comdist=214):
|
||||
cosmo = FlatLambdaCDM(H0=70.5, Om0=0.307, Tcmb0=2.728)
|
||||
fpath = "../data/HFI_PCCS_SZ-union_R2.08.fits"
|
||||
return csiborgtools.io.read_planck2015(fpath, cosmo, max_comdist)
|
||||
|
||||
|
||||
def load_mcxc(max_comdist=214):
|
||||
cosmo = FlatLambdaCDM(H0=70.5, Om0=0.307, Tcmb0=2.728)
|
||||
fpath = ("../data/mcxc.fits")
|
||||
return csiborgtools.io.read_mcxc(fpath, cosmo, max_comdist)
|
||||
|
||||
|
||||
def load_2mpp():
|
||||
cosmo = FlatLambdaCDM(H0=70.5, Om0=0.307, Tcmb0=2.728)
|
||||
return csiborgtools.io.read_2mpp("../data/2M++_galaxy_catalog.dat", cosmo)
|
||||
|
||||
|
||||
def load_2mpp_groups():
|
||||
cosmo = FlatLambdaCDM(H0=70.5, Om0=0.307, Tcmb0=2.728)
|
||||
return csiborgtools.io.read_2mpp_groups(
|
||||
"../data/../data/2M++_group_catalog.dat", cosmo)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue