CSiBORG FoF switch (#75)

* Add moving FoF membership files

* add FoF membership path

* Add notes where its PHEW

* Add FoF catalogue path

* Correct typo

* Add more functionalities

* Make work with halo IDs from FoF

* Edit print statement

* Fix copy bug

* copy

* Add FoF catalogue reading

* Clean up script

* Fix typo

* Little edits

* Fix naming convention

* Rename key

* Remove loading substructure particles

* Rename CSiBORG Cat

* Rename clumps cat

* Rename cat

* Remove misplaced import

* Switch to halos

* rm import

* structfit of only halos

* Add FoF halo reading

* Add a short comment

* Fix __getitem__ to work with int

* Fix problems

* Improve __getitem__

* Add more conversion

* Fix indexing

* Fix __getitem__ assertion

* Fix numbers

* Rename

* Fix verbosity flags

* Add full Quijote HMF option

* Add plot of Quijote only

* Add quijote full paths

* Fix the fit_init script

* Renam arg

* Update .gitignore

* add default argument name

* Change default verbosity flag

* Modernise script structure

* Fix dictionary

* Fix reading to include m200c

* Modernise script

* Add args
This commit is contained in:
Richard Stiskalek 2023-07-24 14:10:21 +02:00 committed by GitHub
parent fcd1a6b321
commit eb8d070fff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 659 additions and 466 deletions

View file

@ -24,7 +24,7 @@ from numba import jit
from scipy.ndimage import gaussian_filter
from tqdm import tqdm, trange
from ..read import load_parent_particles
from ..read import load_halo_particles
BCKG_HALFSIZE = 475
BOX_SIZE = 2048
@ -36,7 +36,7 @@ BOX_SIZE = 2048
class RealisationsMatcher:
"""
A tool to match halos between IC realisations.
A tool to match haloes between IC realisations.
Parameters
----------
@ -112,7 +112,7 @@ class RealisationsMatcher:
"""
return self._overlapper
def cross(self, cat0, catx, particles0, particlesx, clump_map0, clump_mapx,
def cross(self, cat0, catx, particles0, particlesx, halo_map0, halo_mapx,
delta_bckg, cache_size=10000, verbose=True):
r"""
Find all neighbours whose CM separation is less than `nmult` times the
@ -122,9 +122,9 @@ class RealisationsMatcher:
Parameters
----------
cat0 : :py:class:`csiborgtools.read.HaloCatalogue`
cat0 : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue of the reference simulation.
catx : :py:class:`csiborgtools.read.HaloCatalogue`
catx : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue of the cross simulation.
particles0 : 2-dimensional array
Array of particles in box units in the reference simulation.
@ -132,13 +132,13 @@ class RealisationsMatcher:
particlesx : 2-dimensional array
Array of particles in box units in the cross simulation.
The columns must be `x`, `y`, `z` and `M`.
clump_map0 : 2-dimensional array
Clump map of the reference simulation.
clump_mapx : 2-dimensional array
Clump map of the cross simulation.
halo_map0 : 2-dimensional array
Halo map of the reference simulation.
halo_mapx : 2-dimensional array
Halo map of the cross simulation.
delta_bckg : 3-dimensional array
Summed background density field of the reference and cross
simulations calculated with particles assigned to halos at the
simulations calculated with particles assigned to haloes at the
final snapshot. Assumed to only be sampled in cells
:math:`[512, 1536)^3`.
cache_size : int, optional
@ -174,15 +174,14 @@ class RealisationsMatcher:
aratio = numpy.abs(numpy.log10(catx[p][indx] / cat0[p][i]))
match_indxs[i] = match_indxs[i][aratio < self.dlogmass]
clid2map0 = {clid: i for i, clid in enumerate(clump_map0[:, 0])}
clid2mapx = {clid: i for i, clid in enumerate(clump_mapx[:, 0])}
hid2map0 = {hid: i for i, hid in enumerate(halo_map0[:, 0])}
hid2mapx = {hid: i for i, hid in enumerate(halo_mapx[:, 0])}
# We will cache the halos from the cross simulation to speed up the I/O
@lru_cache(maxsize=cache_size)
def load_cached_halox(hid):
return load_processed_halo(hid, particlesx, clump_mapx, clid2mapx,
catx.clumps_cat, nshift=0,
ncells=BOX_SIZE)
return load_processed_halo(hid, particlesx, halo_mapx, hid2mapx,
nshift=0, ncells=BOX_SIZE)
if verbose:
print(f"{datetime.now()}: calculating overlaps.", flush=True)
@ -196,8 +195,7 @@ class RealisationsMatcher:
# Next, we find this halo's particles, total mass, minimum and
# maximum cells and convert positions to cells.
pos0, mass0, totmass0, mins0, maxs0 = load_processed_halo(
k0, particles0, clump_map0, clid2map0, cat0.clumps_cat,
nshift=0, ncells=BOX_SIZE)
k0, particles0, halo_map0, hid2map0, nshift=0, ncells=BOX_SIZE)
# We now loop over matches of this halo and calculate their
# overlap, storing them in `_cross`.
@ -221,8 +219,8 @@ class RealisationsMatcher:
cross = numpy.asanyarray(cross, dtype=object)
return match_indxs, cross
def smoothed_cross(self, cat0, catx, particles0, particlesx, clump_map0,
clump_mapx, delta_bckg, match_indxs, smooth_kwargs,
def smoothed_cross(self, cat0, catx, particles0, particlesx, halo_map0,
halo_mapx, delta_bckg, match_indxs, smooth_kwargs,
cache_size=10000, verbose=True):
r"""
Calculate the smoothed overlaps for pair previously identified via
@ -230,9 +228,9 @@ class RealisationsMatcher:
Parameters
----------
cat0 : :py:class:`csiborgtools.read.ClumpsCatalogue`
cat0 : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue of the reference simulation.
catx : :py:class:`csiborgtools.read.ClumpsCatalogue`
catx : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue of the cross simulation.
particles0 : 2-dimensional array
Array of particles in box units in the reference simulation.
@ -240,10 +238,10 @@ class RealisationsMatcher:
particlesx : 2-dimensional array
Array of particles in box units in the cross simulation.
The columns must be `x`, `y`, `z` and `M`.
clump_map0 : 2-dimensional array
Clump map of the reference simulation.
clump_mapx : 2-dimensional array
Clump map of the cross simulation.
halo_map0 : 2-dimensional array
Halo map of the reference simulation.
halo_mapx : 2-dimensional array
Halo map of the cross simulation.
delta_bckg : 3-dimensional array
Smoothed summed background density field of the reference and cross
simulations calculated with particles assigned to halos at the
@ -263,14 +261,13 @@ class RealisationsMatcher:
overlaps : 1-dimensional array of arrays
"""
nshift = read_nshift(smooth_kwargs)
clid2map0 = {clid: i for i, clid in enumerate(clump_map0[:, 0])}
clid2mapx = {clid: i for i, clid in enumerate(clump_mapx[:, 0])}
hid2map0 = {hid: i for i, hid in enumerate(halo_map0[:, 0])}
hid2mapx = {hid: i for i, hid in enumerate(halo_mapx[:, 0])}
@lru_cache(maxsize=cache_size)
def load_cached_halox(hid):
return load_processed_halo(hid, particlesx, clump_mapx, clid2mapx,
catx.clumps_cat, nshift=nshift,
ncells=BOX_SIZE)
return load_processed_halo(hid, particlesx, halo_mapx, hid2mapx,
nshift=nshift, ncells=BOX_SIZE)
if verbose:
print(f"{datetime.now()}: calculating smoothed overlaps.",
@ -279,8 +276,8 @@ class RealisationsMatcher:
cross = [numpy.asanyarray([], dtype=numpy.float32)] * match_indxs.size
for i, k0 in enumerate(tqdm(indxs) if verbose else indxs):
pos0, mass0, __, mins0, maxs0 = load_processed_halo(
k0, particles0, clump_map0, clid2map0, cat0.clumps_cat,
nshift=nshift, ncells=BOX_SIZE)
k0, particles0, halo_map0, hid2map0, nshift=nshift,
ncells=BOX_SIZE)
# Now loop over the matches and calculate the smoothed overlap.
_cross = numpy.full(match_indxs[i].size, numpy.nan, numpy.float32)
@ -337,7 +334,7 @@ class ParticleOverlap:
Gaussian smoothing.
"""
def make_bckg_delta(self, particles, clump_map, clid2map, halo_cat,
def make_bckg_delta(self, particles, halo_map, hid2map, halo_cat,
delta=None, verbose=False):
"""
Calculate a NGP density field of particles belonging to halos of a
@ -349,12 +346,12 @@ class ParticleOverlap:
----------
particles : 2-dimensional array
Array of particles.
clump_map : 2-dimensional array
halo_map : 2-dimensional array
Array containing start and end indices in the particle array
corresponding to each clump.
clid2map : dict
Dictionary mapping clump IDs to `clump_map` array positions.
halo_cat: :py:class:`csiborgtools.read.HaloCatalogue`
corresponding to each halo.
hid2map : dict
Dictionary mapping halo IDs to `halo_map` array positions.
halo_cat: :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue.
delta : 3-dimensional array, optional
Array to store the density field in. If `None` a new array is
@ -375,12 +372,9 @@ class ParticleOverlap:
else:
assert ((delta.shape == (ncells,) * 3)
& (delta.dtype == numpy.float32))
from tqdm import tqdm
clumps_cat = halo_cat.clumps_cat
for hid in tqdm(halo_cat["index"]) if verbose else halo_cat["index"]:
pos = load_parent_particles(hid, particles, clump_map, clid2map,
clumps_cat)
pos = load_halo_particles(hid, particles, halo_map, hid2map)
if pos is None:
continue
@ -396,8 +390,8 @@ class ParticleOverlap:
def make_delta(self, pos, mass, mins=None, maxs=None, subbox=False,
smooth_kwargs=None):
"""
Calculate a NGP density field of a halo on a cubic grid. Optionally can
be smoothed with a Gaussian kernel.
Calculate a NGP density field of a halo on a regular grid. Optionally
can be smoothed with a Gaussian kernel.
Parameters
----------
@ -409,7 +403,7 @@ class ParticleOverlap:
Minimun and maximum cell numbers along each dimension.
subbox : bool, optional
Whether to calculate the density field on a grid strictly enclosing
the clump.
the halo.
smooth_kwargs : kwargs, optional
Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`.
If `None` no smoothing is applied.
@ -458,10 +452,10 @@ class ParticleOverlap:
mass2 : 1-dimensional array
Particle masses of the second halo.
mins1, maxs1 : 1-dimensional arrays of shape `(3,)`
Minimun and maximum cell numbers along each dimension of `clump1`.
Minimun and maximum cell numbers along each dimension of `halo1`.
Optional.
mins2, maxs2 : 1-dimensional arrays of shape `(3,)`
Minimun and maximum cell numbers along each dimension of `clump2`.
Minimun and maximum cell numbers along each dimension of `halo2`.
Optional.
smooth_kwargs : kwargs, optional
Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`.
@ -470,11 +464,11 @@ class ParticleOverlap:
Returns
-------
delta1, delta2 : 3-dimensional arrays
Density arrays of `clump1` and `clump2`, respectively.
Density arrays of `halo1` and `halo2`, respectively.
cellmins : len-3 tuple
Tuple of left-most cell ID in the full box.
nonzero : 2-dimensional array
Indices where the lower mass clump has a non-zero density.
Indices where the lower mass halo has a non-zero density.
Calculated only if no smoothing is applied, otherwise `None`.
"""
nshift = read_nshift(smooth_kwargs)
@ -509,7 +503,7 @@ class ParticleOverlap:
delta1 = numpy.zeros(ncells, dtype=numpy.float32)
delta2 = numpy.zeros(ncells, dtype=numpy.float32)
# If no smoothing figure out the nonzero indices of the smaller clump
# If no smoothing figure out the nonzero indices of the smaller halo
if smooth_kwargs is None:
if pos1.shape[0] > pos2.shape[0]:
fill_delta(delta1, xc1, yc1, zc1, *cellmins, mass1)
@ -533,12 +527,12 @@ class ParticleOverlap:
mins1=None, maxs1=None, mins2=None, maxs2=None,
totmass1=None, totmass2=None, smooth_kwargs=None):
"""
Calculate overlap between `clump1` and `clump2`. See
Calculate overlap between `halo1` and `halo2`. See
`calculate_overlap(...)` for further information. Be careful so that
the background density field is calculated with the same
`smooth_kwargs`. If any smoothing is applied then loops over the full
density fields, otherwise only over the non-zero cells of the lower
mass clump.
mass halo.
Parameters
----------
@ -558,13 +552,13 @@ class ParticleOverlap:
final snapshot. Assumed to only be sampled in cells
:math:`[512, 1536)^3`.
mins1, maxs1 : 1-dimensional arrays of shape `(3,)`
Minimun and maximum cell numbers along each dimension of `clump1`.
Minimun and maximum cell numbers along each dimension of `halo1`.
Optional.
mins2, maxs2 : 1-dimensional arrays of shape `(3,)`
Minimum and maximum cell numbers along each dimension of `clump2`,
Minimum and maximum cell numbers along each dimension of `halo2`,
optional.
totmass1, totmass2 : floats, optional
Total mass of `clump1` and `clump2`, respectively. Must be provided
Total mass of `halo1` and `halo2`, respectively. Must be provided
if `loop_nonzero` is `True`.
smooth_kwargs : kwargs, optional
Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`.
@ -708,7 +702,7 @@ def get_halolims(pos, ncells, nshift=None):
ncells : int
Number of grid cells of the box along a single dimension.
nshift : int, optional
Lower and upper shift of the clump limits.
Lower and upper shift of the halo limits.
Returns
-------
@ -754,7 +748,7 @@ def calculate_overlap(delta1, delta2, cellmins, delta_bckg):
-------
overlap : float
"""
totmass = 0.0 # Total mass of clump 1 and clump 2
totmass = 0.0 # Total mass of halo 1 and halo 2
intersect = 0.0 # Weighted intersecting mass
i0, j0, k0 = cellmins # Unpack things
bckg_size = 2 * BCKG_HALFSIZE
@ -785,7 +779,7 @@ def calculate_overlap(delta1, delta2, cellmins, delta_bckg):
def calculate_overlap_indxs(delta1, delta2, cellmins, delta_bckg, nonzero,
mass1, mass2):
r"""
Overlap between two clumps whose density fields are evaluated on the
Overlap between two haloes whose density fields are evaluated on the
same grid and `nonzero1` enumerates the non-zero cells of `delta1. This is
a JIT implementation, hence it is outside of the main class.
@ -802,10 +796,10 @@ def calculate_overlap_indxs(delta1, delta2, cellmins, delta_bckg, nonzero,
calculated with particles assigned to halos at the final snapshot.
Assumed to only be sampled in cells :math:`[512, 1536)^3`.
nonzero : 2-dimensional array of shape `(n_cells, 3)`
Indices of cells that are non-zero of the lower mass clump. Expected to
Indices of cells that are non-zero of the lower mass halo. Expected to
be precomputed from `fill_delta_indxs`.
mass1, mass2 : floats, optional
Total masses of the two clumps, respectively. Optional. If not provided
Total masses of the two haloes, respectively. Optional. If not provided
calculcated directly from the density field.
Returns
@ -837,8 +831,7 @@ def calculate_overlap_indxs(delta1, delta2, cellmins, delta_bckg, nonzero,
return intersect / (mass1 + mass2 - intersect)
def load_processed_halo(hid, particles, clump_map, clid2map, clumps_cat,
ncells, nshift):
def load_processed_halo(hid, particles, halo_map, hid2map, ncells, nshift):
"""
Load a processed halo from the `.h5` file. This is to be wrapped by a
cacher.
@ -850,13 +843,11 @@ def load_processed_halo(hid, particles, clump_map, clid2map, clumps_cat,
particles : 2-dimensional array
Array of particles in box units. The columns must be `x`, `y`, `z`
and `M`.
clump_map : 2-dimensional array
halo_map : 2-dimensional array
Array containing start and end indices in the particle array
corresponding to each clump.
clid2map : dict
Dictionary mapping clump IDs to `clump_map` array positions.
clumps_cat : :py:class:`csiborgtools.read.ClumpsCatalogue`
Clumps catalogue.
corresponding to each halo.
hid2map : dict
Dictionary mapping halo IDs to `halo_map` array positions.
ncells : int
Number of cells in the original density field. Typically 2048.
nshift : int
@ -875,8 +866,7 @@ def load_processed_halo(hid, particles, clump_map, clid2map, clumps_cat,
maxs : len-3 tuple
Maximum cell indices of the halo.
"""
pos = load_parent_particles(hid, particles, clump_map, clid2map,
clumps_cat)
pos = load_halo_particles(hid, particles, halo_map, hid2map)
pos, mass = pos[:, :3], pos[:, 3]
pos = pos2cell(pos, ncells)
totmass = numpy.sum(mass)
@ -898,9 +888,9 @@ def radius_neighbours(knn, X, radiusX, radiusKNN, nmult=1.0,
Array of shape `(n_samples, 3)`, where the latter axis represents
`x`, `y` and `z`.
radiusX: 1-dimensional array of shape `(n_samples, )`
Patch radii corresponding to clumps in `X`.
Patch radii corresponding to haloes in `X`.
radiusKNN : 1-dimensional array
Patch radii corresponding to clumps used to train `knn`.
Patch radii corresponding to haloes used to train `knn`.
nmult : float, optional
Multiple of the sum of two radii below which to consider a match.
enforce_int32 : bool, optional

View file

@ -13,8 +13,7 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
from .box_units import CSiBORGBox, QuijoteBox # noqa
from .halo_cat import (ClumpsCatalogue, HaloCatalogue, # noqa
QuijoteHaloCatalogue, fiducial_observers)
from .halo_cat import (CSiBORGHaloCatalogue, QuijoteHaloCatalogue, fiducial_observers) # noqa
from .knn_summary import kNNCDFReader # noqa
from .nearest_neighbour_summary import NearestNeighbourReader # noqa
from .obs import (SDSS, MCXCClusters, PlanckClusters, TwoMPPGalaxies, # noqa
@ -25,8 +24,8 @@ from .overlap_summary import (NPairsOverlap, PairOverlap, # noqa
from .paths import Paths # noqa
from .pk_summary import PKReader # noqa
from .readsim import (MmainReader, ParticleReader, halfwidth_mask, # noqa
load_clump_particles, load_parent_particles, read_initcm)
load_halo_particles, read_initcm) # noqa
from .tpcf_summary import TPCFReader # noqa
from .utils import (M200_to_R200, cartesian_to_radec, # noqa
cols_to_structured, radec_to_cartesian, read_h5,
real2redshift)
real2redshift) # noqa

View file

@ -26,11 +26,11 @@ from .readsim import ParticleReader
# Map of CSiBORG unit conversions
CONV_NAME = {
"length": ["x", "y", "z", "peak_x", "peak_y", "peak_z", "Rs", "rmin",
"rmax", "r200c", "r500c", "r200m", "x0", "y0", "z0",
"rmax", "r200c", "r500c", "r200m", "r500m", "x0", "y0", "z0",
"lagpatch_size"],
"velocity": ["vx", "vy", "vz"],
"mass": ["mass_cl", "totpartmass", "m200c", "m500c", "mass_mmain", "M",
"m200m"],
"m200m", "m500m"],
"density": ["rho0"]}

View file

@ -14,8 +14,8 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""
Simulation catalogues:
- CSiBORG: halo and clump catalogue.
- Quijote: halo catalogue.
- CSiBORG: FoF halo catalogue.
- Quijote: FoF halo catalogue.
"""
from abc import ABC, abstractproperty
from copy import deepcopy
@ -25,7 +25,6 @@ from math import floor
from os.path import join
import numpy
from readfof import FoF_catalog
from sklearn.neighbors import NearestNeighbors
@ -369,6 +368,9 @@ class BaseCatalogue(ABC):
return self.data.dtype.names
def __getitem__(self, key):
if isinstance(key, (int, numpy.integer)):
assert key >= 0
return self.data[key]
if key not in self.keys:
raise KeyError(f"Key '{key}' not in catalogue.")
return self.data[key]
@ -377,111 +379,14 @@ class BaseCatalogue(ABC):
return self.data.size
###############################################################################
# CSiBORG base catalogue #
###############################################################################
class BaseCSiBORG(BaseCatalogue):
"""
Base CSiBORG catalogue class.
"""
@property
def nsnap(self):
return max(self.paths.get_snapshots(self.nsim))
@property
def box(self):
"""
CSiBORG box object handling unit conversion.
Returns
-------
box : instance of :py:class:`csiborgtools.units.BaseBox`
"""
return CSiBORGBox(self.nsnap, self.nsim, self.paths)
###############################################################################
# CSiBORG clumps catalogue #
###############################################################################
class ClumpsCatalogue(BaseCSiBORG):
r"""
Clumps catalogue, defined in the final snapshot.
Parameters
----------
sim : int
IC realisation index.
paths : py:class`csiborgtools.read.Paths`
Paths object.
bounds : dict
Parameter bounds to apply to the catalogue. The keys are the parameter
names and the items are a len-2 tuple of (min, max) values. In case of
no minimum or maximum, use `None`. For radial distance from the origin
use `dist`.
load_fitted : bool, optional
Whether to load fitted quantities.
rawdata : bool, optional
Whether to return the raw data. In this case applies no cuts and
transformations.
"""
def __init__(self, nsim, paths, bounds={"dist": (0, 155.5 / 0.705)},
load_fitted=True, rawdata=False):
self.nsim = nsim
self.paths = paths
# Read in the clumps from the final snapshot
partreader = ParticleReader(self.paths)
cols = ["index", "parent", "x", "y", "z", "mass_cl"]
self._data = partreader.read_clumps(self.nsnap, self.nsim, cols=cols)
# Overwrite the parent with the ultimate parent
mmain = numpy.load(self.paths.mmain(self.nsnap, self.nsim))
self._data["parent"] = mmain["ultimate_parent"]
if load_fitted:
fits = numpy.load(paths.structfit(self.nsnap, nsim, "clumps"))
cols = [col for col in fits.dtype.names if col != "index"]
X = [fits[col] for col in cols]
self._data = add_columns(self._data, X, cols)
# If the raw data is not required, then start applying transformations
# and cuts.
if not rawdata:
flip_cols(self._data, "x", "z")
for p in ("x", "y", "z"):
self._data[p] -= 0.5
names = ["x", "y", "z", "mass_cl", "totpartmass", "rho0", "r200c",
"r500c", "m200c", "m500c", "r200m", "m200m",
"vx", "vy", "vz"]
self._data = self.box.convert_from_box(self._data, names)
if bounds is not None:
self.apply_bounds(bounds)
@property
def ismain(self):
"""
Whether the clump is a main halo.
Returns
-------
ismain : 1-dimensional array
"""
return self["index"] == self["parent"]
###############################################################################
# CSiBORG halo catalogue #
###############################################################################
class HaloCatalogue(BaseCSiBORG):
class CSiBORGHaloCatalogue(BaseCatalogue):
r"""
Halo catalogue, i.e. parent halos with summed substructure, defined in the
final snapshot.
CSiBORG FoF halo catalogue.
Parameters
----------
@ -504,22 +409,17 @@ class HaloCatalogue(BaseCSiBORG):
Whether to return the raw data. In this case applies no cuts and
transformations.
"""
_clumps_cat = None
def __init__(self, nsim, paths, bounds={"dist": (0, 155.5 / 0.705)},
with_lagpatch=True, load_fitted=True, load_initial=True,
load_clumps_cat=False, rawdata=False):
rawdata=False):
self.nsim = nsim
self.paths = paths
# Read in the mmain catalogue of summed substructure
mmain = numpy.load(self.paths.mmain(self.nsnap, self.nsim))
self._data = mmain["mmain"]
# We will also need the clumps catalogue
if load_clumps_cat:
self._clumps_cat = ClumpsCatalogue(nsim, paths, rawdata=True,
load_fitted=False)
reader = ParticleReader(paths)
self._data = reader.read_fof_halos(self.nsim)
if load_fitted:
fits = numpy.load(paths.structfit(self.nsnap, nsim, "halos"))
fits = numpy.load(paths.structfit(self.nsnap, nsim))
cols = [col for col in fits.dtype.names if col != "index"]
X = [fits[col] for col in cols]
self._data = add_columns(self._data, X, cols)
@ -538,19 +438,25 @@ class HaloCatalogue(BaseCSiBORG):
self._data = add_columns(self._data, X, cols)
if not rawdata:
if rawdata:
for p in ('x', 'y', 'z'):
self._data[p] = self.box.mpc2box(self._data[p]) + 0.5
else:
if with_lagpatch:
self._data = self._data[numpy.isfinite(self["lagpatch_size"])]
# Flip positions and convert from code units to cMpc. Convert M too
flip_cols(self._data, "x", "z")
for p in ("x", "y", "z"):
self._data[p] -= 0.5
names = ["x", "y", "z", "M", "totpartmass", "rho0", "r200c",
"r500c", "m200c", "m500c", "r200m", "m200m",
"vx", "vy", "vz"]
self._data = self.box.convert_from_box(self._data, names)
if load_fitted:
flip_cols(self._data, "vx", "vz")
names = ["totpartmass", "rho0", "r200c",
"r500c", "m200c", "m500c", "r200m", "m200m",
"r500m", "m500m", "vx", "vy", "vz"]
self._data = self.box.convert_from_box(self._data, names)
if load_initial:
flip_cols(self._data, "x0", "z0")
for p in ("x0", "y0", "z0"):
self._data[p] -= 0.5
names = ["x0", "y0", "z0", "lagpatch_size"]
self._data = self.box.convert_from_box(self._data, names)
@ -558,17 +464,19 @@ class HaloCatalogue(BaseCSiBORG):
self.apply_bounds(bounds)
@property
def clumps_cat(self):
def nsnap(self):
return max(self.paths.get_snapshots(self.nsim))
@property
def box(self):
"""
The raw clumps catalogue.
CSiBORG box object handling unit conversion.
Returns
-------
clumps_cat : :py:class:`csiborgtools.read.ClumpsCatalogue`
box : instance of :py:class:`csiborgtools.units.BaseBox`
"""
if self._clumps_cat is None:
raise ValueError("`clumps_cat` is not loaded.")
return self._clumps_cat
return CSiBORGBox(self.nsnap, self.nsim, self.paths)
###############################################################################
@ -578,7 +486,7 @@ class HaloCatalogue(BaseCSiBORG):
class QuijoteHaloCatalogue(BaseCatalogue):
"""
Quijote halo catalogue.
Quijote FoF halo catalogue.
Parameters
----------

View file

@ -32,9 +32,9 @@ class PairOverlap:
Parameters
----------
cat0 : :py:class:`csiborgtools.read.HaloCatalogue`
cat0 : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue corresponding to the reference simulation.
catx : :py:class:`csiborgtools.read.HaloCatalogue`
catx : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue corresponding to the cross simulation.
paths : py:class`csiborgtools.read.Paths`
CSiBORG paths object.
@ -58,9 +58,9 @@ class PairOverlap:
Parameters
----------
cat0 : :py:class:`csiborgtools.read.HaloCatalogue`
cat0 : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue corresponding to the reference simulation.
catx : :py:class:`csiborgtools.read.HaloCatalogue`
catx : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue corresponding to the cross simulation.
paths : py:class`csiborgtools.read.Paths`
CSiBORG paths object.
@ -557,9 +557,9 @@ class NPairsOverlap:
Parameters
----------
cat0 : :py:class:`csiborgtools.read.HaloCatalogue`
cat0 : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Single reference simulation halo catalogue.
catxs : list of :py:class:`csiborgtools.read.HaloCatalogue`
catxs : list of :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
List of cross simulation halo catalogues.
paths : py:class`csiborgtools.read.Paths`
CSiBORG paths object.

View file

@ -186,6 +186,42 @@ class Paths:
"""
return join(self.borg_dir, "mcmc", f"mcmc_{nsim}.h5")
def fof_membership(self, nsim, sorted=False):
"""
Path to the file containing the FoF particle membership.
Parameters
----------
nsim : int
IC realisation index.
sorted : bool, optional
Whether to return path to the file that is sorted in the same
order as the PHEW output.
"""
fdir = join(self.postdir, "FoF_membership", )
if not isdir(fdir):
mkdir(fdir)
warn(f"Created directory `{fdir}`.", UserWarning, stacklevel=1)
fout = join(fdir, f"fof_membership_{nsim}.npy")
if sorted:
fout = fout.replace(".npy", "_sorted.npy")
return fout
def fof_cat(self, nsim):
"""
Path to the FoF halo catalogue file.
Parameters
----------
nsim : int
IC realisation index.
"""
fdir = join(self.postdir, "FoF_membership", )
if not isdir(fdir):
mkdir(fdir)
warn(f"Created directory `{fdir}`.", UserWarning, stacklevel=1)
return join(fdir, f"halo_catalog_{nsim}_FOF.txt")
def mmain(self, nsnap, nsim):
"""
Path to the `mmain` CSiBORG files of summed substructure.
@ -246,7 +282,7 @@ class Paths:
-------
ids : 1-dimensional array
"""
assert simname in ["csiborg", "quijote"]
assert simname in ["csiborg", "quijote", "quijote_full"]
if simname == "csiborg":
files = glob(join(self.srcdir, "ramses_out*"))
files = [f.split("/")[-1] for f in files] # Only file names
@ -260,6 +296,7 @@ class Paths:
pass
return numpy.sort(ids)
else:
# TODO here later read this from the catalogues instead.
return numpy.arange(100, dtype=int)
def ic_path(self, nsim, tonew=False):
@ -323,7 +360,7 @@ class Paths:
simpath = self.ic_path(nsim, tonew=tonew)
return join(simpath, f"output_{str(nsnap).zfill(5)}")
def structfit(self, nsnap, nsim, kind):
def structfit(self, nsnap, nsim):
"""
Path to the clump or halo catalogue from `fit_halos.py`. Only CSiBORG
is supported.
@ -334,19 +371,16 @@ class Paths:
Snapshot index.
nsim : int
IC realisation index.
kind : str
Type of catalogue. Can be either `clumps` or `halos`.
Returns
-------
path : str
"""
assert kind in ["clumps", "halos"]
fdir = join(self.postdir, "structfit")
if not isdir(fdir):
mkdir(fdir)
warn(f"Created directory `{fdir}`.", UserWarning, stacklevel=1)
fname = f"{kind}_out_{str(nsim).zfill(5)}_{str(nsnap).zfill(5)}.npy"
fname = f"out_{str(nsim).zfill(5)}_{str(nsnap).zfill(5)}.npy"
return join(fdir, fname)
def overlap(self, nsim0, nsimx, smoothed):
@ -441,7 +475,7 @@ class Paths:
Parameters
----------
simname : str
Simulation name. Must be `csiborg` or `quijote`.
Simulation name. Must be `csiborg`, `quijote` or `quijote_full`.
nsim : int
IC realisation index.

View file

@ -32,7 +32,7 @@ from .utils import cols_to_structured
class ParticleReader:
"""
Shortcut to read in particle files along with their corresponding clumps.
Object to read in particle files along with their corresponding haloes.
Parameters
----------
@ -203,7 +203,7 @@ class ParticleReader:
nsim : int
IC realisation index.
pars_extract : list of str
Parameters to be extacted.
Parameters to be extracted.
return_structured : bool, optional
Whether to return a structured array or a 2-dimensional array. If
the latter, then the order of the columns is the same as the order
@ -280,7 +280,7 @@ class ParticleReader:
def open_unbinding(self, nsnap, nsim, cpu):
"""
Open particle files to a given CSiBORG simulation. Note that to be
Open particle files of a given CSiBORG simulation. Note that to be
consistent CPU is incremented by 1.
Parameters
@ -305,7 +305,8 @@ class ParticleReader:
def read_clumpid(self, nsnap, nsim, verbose=True):
"""
Read clump IDs of particles from unbinding files.
Read PHEW clump IDs of particles from unbinding files. This halo finder
was used when running the catalogue.
Parameters
----------
@ -332,14 +333,13 @@ class ParticleReader:
j = nparts[cpu]
ff = self.open_unbinding(nsnap, nsim, cpu)
clumpid[i:i + j] = ff.read_ints()
ff.close()
return clumpid
def read_clumps(self, nsnap, nsim, cols=None):
"""
Read in a clump file `clump_xxXXX.dat`.
Read in a PHEW clump file `clump_xxXXX.dat`.
Parameters
----------
@ -387,6 +387,54 @@ class ParticleReader:
out[col] = data[:, clump_cols[col][0]]
return out
def read_fof_hids(self, nsim):
"""
Read in the FoF particle halo membership IDs that are sorted to match
the PHEW output.
Parameters
----------
nsim : int
IC realisation index.
Returns
-------
hids : 1-dimensional array
Halo IDs of particles.
"""
return numpy.load(self.paths.fof_membership(nsim, sorted=True))
def read_fof_halos(self, nsim):
"""
Read in the FoF halo catalogue.
Parameters
----------
nsim : int
IC realisation index.
Returns
-------
cat : structured array
"""
fpath = self.paths.fof_cat(nsim)
hid = numpy.genfromtxt(fpath, usecols=0, dtype=numpy.int32)
pos = numpy.genfromtxt(fpath, usecols=(1, 2, 3), dtype=numpy.float32)
totmass = numpy.genfromtxt(fpath, usecols=4, dtype=numpy.float32)
m200c = numpy.genfromtxt(fpath, usecols=5, dtype=numpy.float32)
dtype = {"names": ["index", "x", "y", "z", "fof_totpartmass",
"fof_m200c"],
"formats": [numpy.int32] + [numpy.float32] * 5}
out = numpy.full(hid.size, numpy.nan, dtype=dtype)
out["index"] = hid
out["x"] = pos[:, 0]
out["y"] = pos[:, 1]
out["z"] = pos[:, 2]
out["fof_totpartmass"] = totmass * 1e11
out["fof_m200c"] = m200c * 1e11
return out
###############################################################################
# Summed substructure catalogue #
@ -457,7 +505,7 @@ class MmainReader:
"""
Make the summed substructure catalogue for a final snapshot. Includes
the position of the parent, the summed mass and the fraction of mass in
substructure.
substructure. Corresponds to the PHEW Halo finder.
Parameters
----------
@ -552,39 +600,10 @@ def halfwidth_mask(pos, hw):
return numpy.all((0.5 - hw < pos) & (pos < 0.5 + hw), axis=1)
def load_clump_particles(clid, particles, clump_map, clid2map):
def load_halo_particles(hid, particles, halo_map, hid2map):
"""
Load a clump's particles from a particle array. If it is not there, i.e
clump has no associated particles, return `None`.
Parameters
----------
clid : int
Clump ID.
particles : 2-dimensional array
Array of particles.
clump_map : 2-dimensional array
Array containing start and end indices in the particle array
corresponding to each clump.
clid2map : dict
Dictionary mapping clump IDs to `clump_map` array positions.
Returns
-------
clump_particles : 2-dimensional array
Particle array of this clump.
"""
try:
k0, kf = clump_map[clid2map[clid], 1:]
return particles[k0:kf + 1, :]
except KeyError:
return None
def load_parent_particles(hid, particles, clump_map, clid2map, clumps_cat):
"""
Load a parent halo's particles from a particle array. If it is not there,
return `None`.
Load a halo's particles from a particle array. If it is not there, i.e
halo has no associated particles, return `None`.
Parameters
----------
@ -592,27 +611,19 @@ def load_parent_particles(hid, particles, clump_map, clid2map, clumps_cat):
Halo ID.
particles : 2-dimensional array
Array of particles.
clump_map : 2-dimensional array
halo_map : 2-dimensional array
Array containing start and end indices in the particle array
corresponding to each clump.
clid2map : dict
Dictionary mapping clump IDs to `clump_map` array positions.
clumps_cat : :py:class:`csiborgtools.read.ClumpsCatalogue`
Clumps catalogue.
corresponding to each halo.
hid2map : dict
Dictionary mapping halo IDs to `halo_map` array positions.
Returns
-------
halo : 2-dimensional array
halo_particles : 2-dimensional array
Particle array of this halo.
"""
clids = clumps_cat["index"][clumps_cat["parent"] == hid]
# We first load the particles of each clump belonging to this parent
# and then concatenate them for further analysis.
clumps = []
for clid in clids:
parts = load_clump_particles(clid, particles, clump_map, clid2map)
if parts is not None:
clumps.append(parts)
if len(clumps) == 0:
try:
k0, kf = halo_map[hid2map[hid], 1:]
return particles[k0:kf + 1, :]
except KeyError:
return None
return numpy.concatenate(clumps)