Aquila-Consortium/csiborgtools
0
0
Fork 0
mirror of https://github.com/Richard-Sti/csiborgtools.git synced 2024-10-18 18:25:06 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

CSiBORG FoF switch (#75)

Browse Source 
* Add moving FoF membership files

* add FoF membership path

* Add notes where its PHEW

* Add FoF catalogue path

* Correct typo

* Add more functionalities

* Make work with halo IDs from FoF

* Edit print statement

* Fix copy bug

* copy

* Add FoF catalogue reading

* Clean up script

* Fix typo

* Little edits

* Fix naming convention

* Rename key

* Remove loading substructure particles

* Rename CSiBORG Cat

* Rename clumps cat

* Rename cat

* Remove misplaced import

* Switch to halos

* rm import

* structfit of only halos

* Add FoF halo reading

* Add a short comment

* Fix __getitem__ to work with int

* Fix problems

* Improve __getitem__

* Add more conversion

* Fix indexing

* Fix __getitem__ assertion

* Fix numbers

* Rename

* Fix verbosity flags

* Add full Quijote HMF option

* Add plot of Quijote only

* Add quijote full paths

* Fix the fit_init script

* Renam arg

* Update .gitignore

* add default argument name

* Change default verbosity flag

* Modernise script structure

* Fix dictionary

* Fix reading to include m200c

* Modernise script

* Add args
This commit is contained in:
Richard Stiskalek 2023-07-24 14:10:21 +02:00 committed by GitHub
parent fcd1a6b321
commit eb8d070fff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 659 additions and 466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -23,3 +23,4 @@ scripts_plots/python.sh
scripts_plots/submit.sh scripts_plots/submit.sh
scripts_plots/*.out scripts_plots/*.out
scripts_plots/*.sh scripts_plots/*.sh
notebooks/test.ipynb

134
csiborgtools/match/match.py
View File

@ -24,7 +24,7 @@ from numba import jit
from scipy.ndimage import gaussian_filter from scipy.ndimage import gaussian_filter
from tqdm import tqdm, trange from tqdm import tqdm, trange
from ..read import load_parent_particles from ..read import load_halo_particles
BCKG_HALFSIZE = 475 BCKG_HALFSIZE = 475
BOX_SIZE = 2048 BOX_SIZE = 2048
@ -36,7 +36,7 @@ BOX_SIZE = 2048
class RealisationsMatcher: class RealisationsMatcher:
""" """
A tool to match halos between IC realisations. A tool to match haloes between IC realisations.
Parameters Parameters
---------- ----------
@ -112,7 +112,7 @@ class RealisationsMatcher:
""" """
return self._overlapper return self._overlapper
def cross(self, cat0, catx, particles0, particlesx, clump_map0, clump_mapx, def cross(self, cat0, catx, particles0, particlesx, halo_map0, halo_mapx,
delta_bckg, cache_size=10000, verbose=True): delta_bckg, cache_size=10000, verbose=True):
r""" r"""
Find all neighbours whose CM separation is less than `nmult` times the Find all neighbours whose CM separation is less than `nmult` times the
@ -122,9 +122,9 @@ class RealisationsMatcher:
Parameters Parameters
---------- ----------
cat0 : :py:class:`csiborgtools.read.HaloCatalogue` cat0 : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue of the reference simulation. Halo catalogue of the reference simulation.
catx : :py:class:`csiborgtools.read.HaloCatalogue` catx : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue of the cross simulation. Halo catalogue of the cross simulation.
particles0 : 2-dimensional array particles0 : 2-dimensional array
Array of particles in box units in the reference simulation. Array of particles in box units in the reference simulation.
@ -132,13 +132,13 @@ class RealisationsMatcher:
particlesx : 2-dimensional array particlesx : 2-dimensional array
Array of particles in box units in the cross simulation. Array of particles in box units in the cross simulation.
The columns must be `x`, `y`, `z` and `M`. The columns must be `x`, `y`, `z` and `M`.
clump_map0 : 2-dimensional array halo_map0 : 2-dimensional array
Clump map of the reference simulation. Halo map of the reference simulation.
clump_mapx : 2-dimensional array halo_mapx : 2-dimensional array
Clump map of the cross simulation. Halo map of the cross simulation.
delta_bckg : 3-dimensional array delta_bckg : 3-dimensional array
Summed background density field of the reference and cross Summed background density field of the reference and cross
simulations calculated with particles assigned to halos at the simulations calculated with particles assigned to haloes at the
final snapshot. Assumed to only be sampled in cells final snapshot. Assumed to only be sampled in cells
:math:`[512, 1536)^3`. :math:`[512, 1536)^3`.
cache_size : int, optional cache_size : int, optional
@ -174,15 +174,14 @@ class RealisationsMatcher:
aratio = numpy.abs(numpy.log10(catx[p][indx] / cat0[p][i])) aratio = numpy.abs(numpy.log10(catx[p][indx] / cat0[p][i]))
match_indxs[i] = match_indxs[i][aratio < self.dlogmass] match_indxs[i] = match_indxs[i][aratio < self.dlogmass]
clid2map0 = {clid: i for i, clid in enumerate(clump_map0[:, 0])} hid2map0 = {hid: i for i, hid in enumerate(halo_map0[:, 0])}
clid2mapx = {clid: i for i, clid in enumerate(clump_mapx[:, 0])} hid2mapx = {hid: i for i, hid in enumerate(halo_mapx[:, 0])}
# We will cache the halos from the cross simulation to speed up the I/O # We will cache the halos from the cross simulation to speed up the I/O
@lru_cache(maxsize=cache_size) @lru_cache(maxsize=cache_size)
def load_cached_halox(hid): def load_cached_halox(hid):
return load_processed_halo(hid, particlesx, clump_mapx, clid2mapx, return load_processed_halo(hid, particlesx, halo_mapx, hid2mapx,
catx.clumps_cat, nshift=0, nshift=0, ncells=BOX_SIZE)
ncells=BOX_SIZE)
if verbose: if verbose:
print(f"{datetime.now()}: calculating overlaps.", flush=True) print(f"{datetime.now()}: calculating overlaps.", flush=True)
@ -196,8 +195,7 @@ class RealisationsMatcher:
# Next, we find this halo's particles, total mass, minimum and # Next, we find this halo's particles, total mass, minimum and
# maximum cells and convert positions to cells. # maximum cells and convert positions to cells.
pos0, mass0, totmass0, mins0, maxs0 = load_processed_halo( pos0, mass0, totmass0, mins0, maxs0 = load_processed_halo(
k0, particles0, clump_map0, clid2map0, cat0.clumps_cat, k0, particles0, halo_map0, hid2map0, nshift=0, ncells=BOX_SIZE)
nshift=0, ncells=BOX_SIZE)
# We now loop over matches of this halo and calculate their # We now loop over matches of this halo and calculate their
# overlap, storing them in `_cross`. # overlap, storing them in `_cross`.
@ -221,8 +219,8 @@ class RealisationsMatcher:
cross = numpy.asanyarray(cross, dtype=object) cross = numpy.asanyarray(cross, dtype=object)
return match_indxs, cross return match_indxs, cross
def smoothed_cross(self, cat0, catx, particles0, particlesx, clump_map0, def smoothed_cross(self, cat0, catx, particles0, particlesx, halo_map0,
clump_mapx, delta_bckg, match_indxs, smooth_kwargs, halo_mapx, delta_bckg, match_indxs, smooth_kwargs,
cache_size=10000, verbose=True): cache_size=10000, verbose=True):
r""" r"""
Calculate the smoothed overlaps for pair previously identified via Calculate the smoothed overlaps for pair previously identified via
@ -230,9 +228,9 @@ class RealisationsMatcher:
Parameters Parameters
---------- ----------
cat0 : :py:class:`csiborgtools.read.ClumpsCatalogue` cat0 : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue of the reference simulation. Halo catalogue of the reference simulation.
catx : :py:class:`csiborgtools.read.ClumpsCatalogue` catx : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue of the cross simulation. Halo catalogue of the cross simulation.
particles0 : 2-dimensional array particles0 : 2-dimensional array
Array of particles in box units in the reference simulation. Array of particles in box units in the reference simulation.
@ -240,10 +238,10 @@ class RealisationsMatcher:
particlesx : 2-dimensional array particlesx : 2-dimensional array
Array of particles in box units in the cross simulation. Array of particles in box units in the cross simulation.
The columns must be `x`, `y`, `z` and `M`. The columns must be `x`, `y`, `z` and `M`.
clump_map0 : 2-dimensional array halo_map0 : 2-dimensional array
Clump map of the reference simulation. Halo map of the reference simulation.
clump_mapx : 2-dimensional array halo_mapx : 2-dimensional array
Clump map of the cross simulation. Halo map of the cross simulation.
delta_bckg : 3-dimensional array delta_bckg : 3-dimensional array
Smoothed summed background density field of the reference and cross Smoothed summed background density field of the reference and cross
simulations calculated with particles assigned to halos at the simulations calculated with particles assigned to halos at the
@ -263,14 +261,13 @@ class RealisationsMatcher:
overlaps : 1-dimensional array of arrays overlaps : 1-dimensional array of arrays
""" """
nshift = read_nshift(smooth_kwargs) nshift = read_nshift(smooth_kwargs)
clid2map0 = {clid: i for i, clid in enumerate(clump_map0[:, 0])} hid2map0 = {hid: i for i, hid in enumerate(halo_map0[:, 0])}
clid2mapx = {clid: i for i, clid in enumerate(clump_mapx[:, 0])} hid2mapx = {hid: i for i, hid in enumerate(halo_mapx[:, 0])}
@lru_cache(maxsize=cache_size) @lru_cache(maxsize=cache_size)
def load_cached_halox(hid): def load_cached_halox(hid):
return load_processed_halo(hid, particlesx, clump_mapx, clid2mapx, return load_processed_halo(hid, particlesx, halo_mapx, hid2mapx,
catx.clumps_cat, nshift=nshift, nshift=nshift, ncells=BOX_SIZE)
ncells=BOX_SIZE)
if verbose: if verbose:
print(f"{datetime.now()}: calculating smoothed overlaps.", print(f"{datetime.now()}: calculating smoothed overlaps.",
@ -279,8 +276,8 @@ class RealisationsMatcher:
cross = [numpy.asanyarray([], dtype=numpy.float32)] * match_indxs.size cross = [numpy.asanyarray([], dtype=numpy.float32)] * match_indxs.size
for i, k0 in enumerate(tqdm(indxs) if verbose else indxs): for i, k0 in enumerate(tqdm(indxs) if verbose else indxs):
pos0, mass0, __, mins0, maxs0 = load_processed_halo( pos0, mass0, __, mins0, maxs0 = load_processed_halo(
k0, particles0, clump_map0, clid2map0, cat0.clumps_cat, k0, particles0, halo_map0, hid2map0, nshift=nshift,
nshift=nshift, ncells=BOX_SIZE) ncells=BOX_SIZE)
# Now loop over the matches and calculate the smoothed overlap. # Now loop over the matches and calculate the smoothed overlap.
_cross = numpy.full(match_indxs[i].size, numpy.nan, numpy.float32) _cross = numpy.full(match_indxs[i].size, numpy.nan, numpy.float32)
@ -337,7 +334,7 @@ class ParticleOverlap:
Gaussian smoothing. Gaussian smoothing.
""" """
def make_bckg_delta(self, particles, clump_map, clid2map, halo_cat, def make_bckg_delta(self, particles, halo_map, hid2map, halo_cat,
delta=None, verbose=False): delta=None, verbose=False):
""" """
Calculate a NGP density field of particles belonging to halos of a Calculate a NGP density field of particles belonging to halos of a
@ -349,12 +346,12 @@ class ParticleOverlap:
---------- ----------
particles : 2-dimensional array particles : 2-dimensional array
Array of particles. Array of particles.
clump_map : 2-dimensional array halo_map : 2-dimensional array
Array containing start and end indices in the particle array Array containing start and end indices in the particle array
corresponding to each clump. corresponding to each halo.
clid2map : dict hid2map : dict
Dictionary mapping clump IDs to `clump_map` array positions. Dictionary mapping halo IDs to `halo_map` array positions.
halo_cat: :py:class:`csiborgtools.read.HaloCatalogue` halo_cat: :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue. Halo catalogue.
delta : 3-dimensional array, optional delta : 3-dimensional array, optional
Array to store the density field in. If `None` a new array is Array to store the density field in. If `None` a new array is
@ -375,12 +372,9 @@ class ParticleOverlap:
else: else:
assert ((delta.shape == (ncells,) * 3) assert ((delta.shape == (ncells,) * 3)
& (delta.dtype == numpy.float32)) & (delta.dtype == numpy.float32))
from tqdm import tqdm
clumps_cat = halo_cat.clumps_cat
for hid in tqdm(halo_cat["index"]) if verbose else halo_cat["index"]: for hid in tqdm(halo_cat["index"]) if verbose else halo_cat["index"]:
pos = load_parent_particles(hid, particles, clump_map, clid2map, pos = load_halo_particles(hid, particles, halo_map, hid2map)
clumps_cat)
if pos is None: if pos is None:
continue continue
@ -396,8 +390,8 @@ class ParticleOverlap:
def make_delta(self, pos, mass, mins=None, maxs=None, subbox=False, def make_delta(self, pos, mass, mins=None, maxs=None, subbox=False,
smooth_kwargs=None): smooth_kwargs=None):
""" """
Calculate a NGP density field of a halo on a cubic grid. Optionally can Calculate a NGP density field of a halo on a regular grid. Optionally
be smoothed with a Gaussian kernel. can be smoothed with a Gaussian kernel.
Parameters Parameters
---------- ----------
@ -409,7 +403,7 @@ class ParticleOverlap:
Minimun and maximum cell numbers along each dimension. Minimun and maximum cell numbers along each dimension.
subbox : bool, optional subbox : bool, optional
Whether to calculate the density field on a grid strictly enclosing Whether to calculate the density field on a grid strictly enclosing
the clump. the halo.
smooth_kwargs : kwargs, optional smooth_kwargs : kwargs, optional
Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`. Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`.
If `None` no smoothing is applied. If `None` no smoothing is applied.
@ -458,10 +452,10 @@ class ParticleOverlap:
mass2 : 1-dimensional array mass2 : 1-dimensional array
Particle masses of the second halo. Particle masses of the second halo.
mins1, maxs1 : 1-dimensional arrays of shape `(3,)` mins1, maxs1 : 1-dimensional arrays of shape `(3,)`
Minimun and maximum cell numbers along each dimension of `clump1`. Minimun and maximum cell numbers along each dimension of `halo1`.
Optional. Optional.
mins2, maxs2 : 1-dimensional arrays of shape `(3,)` mins2, maxs2 : 1-dimensional arrays of shape `(3,)`
Minimun and maximum cell numbers along each dimension of `clump2`. Minimun and maximum cell numbers along each dimension of `halo2`.
Optional. Optional.
smooth_kwargs : kwargs, optional smooth_kwargs : kwargs, optional
Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`. Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`.
@ -470,11 +464,11 @@ class ParticleOverlap:
Returns Returns
------- -------
delta1, delta2 : 3-dimensional arrays delta1, delta2 : 3-dimensional arrays
Density arrays of `clump1` and `clump2`, respectively. Density arrays of `halo1` and `halo2`, respectively.
cellmins : len-3 tuple cellmins : len-3 tuple
Tuple of left-most cell ID in the full box. Tuple of left-most cell ID in the full box.
nonzero : 2-dimensional array nonzero : 2-dimensional array
Indices where the lower mass clump has a non-zero density. Indices where the lower mass halo has a non-zero density.
Calculated only if no smoothing is applied, otherwise `None`. Calculated only if no smoothing is applied, otherwise `None`.
""" """
nshift = read_nshift(smooth_kwargs) nshift = read_nshift(smooth_kwargs)
@ -509,7 +503,7 @@ class ParticleOverlap:
delta1 = numpy.zeros(ncells, dtype=numpy.float32) delta1 = numpy.zeros(ncells, dtype=numpy.float32)
delta2 = numpy.zeros(ncells, dtype=numpy.float32) delta2 = numpy.zeros(ncells, dtype=numpy.float32)
# If no smoothing figure out the nonzero indices of the smaller clump # If no smoothing figure out the nonzero indices of the smaller halo
if smooth_kwargs is None: if smooth_kwargs is None:
if pos1.shape[0] > pos2.shape[0]: if pos1.shape[0] > pos2.shape[0]:
fill_delta(delta1, xc1, yc1, zc1, *cellmins, mass1) fill_delta(delta1, xc1, yc1, zc1, *cellmins, mass1)
@ -533,12 +527,12 @@ class ParticleOverlap:
mins1=None, maxs1=None, mins2=None, maxs2=None, mins1=None, maxs1=None, mins2=None, maxs2=None,
totmass1=None, totmass2=None, smooth_kwargs=None): totmass1=None, totmass2=None, smooth_kwargs=None):
""" """
Calculate overlap between `clump1` and `clump2`. See Calculate overlap between `halo1` and `halo2`. See
`calculate_overlap(...)` for further information. Be careful so that `calculate_overlap(...)` for further information. Be careful so that
the background density field is calculated with the same the background density field is calculated with the same
`smooth_kwargs`. If any smoothing is applied then loops over the full `smooth_kwargs`. If any smoothing is applied then loops over the full
density fields, otherwise only over the non-zero cells of the lower density fields, otherwise only over the non-zero cells of the lower
mass clump. mass halo.
Parameters Parameters
---------- ----------
@ -558,13 +552,13 @@ class ParticleOverlap:
final snapshot. Assumed to only be sampled in cells final snapshot. Assumed to only be sampled in cells
:math:`[512, 1536)^3`. :math:`[512, 1536)^3`.
mins1, maxs1 : 1-dimensional arrays of shape `(3,)` mins1, maxs1 : 1-dimensional arrays of shape `(3,)`
Minimun and maximum cell numbers along each dimension of `clump1`. Minimun and maximum cell numbers along each dimension of `halo1`.
Optional. Optional.
mins2, maxs2 : 1-dimensional arrays of shape `(3,)` mins2, maxs2 : 1-dimensional arrays of shape `(3,)`
Minimum and maximum cell numbers along each dimension of `clump2`, Minimum and maximum cell numbers along each dimension of `halo2`,
optional. optional.
totmass1, totmass2 : floats, optional totmass1, totmass2 : floats, optional
Total mass of `clump1` and `clump2`, respectively. Must be provided Total mass of `halo1` and `halo2`, respectively. Must be provided
if `loop_nonzero` is `True`. if `loop_nonzero` is `True`.
smooth_kwargs : kwargs, optional smooth_kwargs : kwargs, optional
Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`. Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`.
@ -708,7 +702,7 @@ def get_halolims(pos, ncells, nshift=None):
ncells : int ncells : int
Number of grid cells of the box along a single dimension. Number of grid cells of the box along a single dimension.
nshift : int, optional nshift : int, optional
Lower and upper shift of the clump limits. Lower and upper shift of the halo limits.
Returns Returns
------- -------
@ -754,7 +748,7 @@ def calculate_overlap(delta1, delta2, cellmins, delta_bckg):
------- -------
overlap : float overlap : float
""" """
totmass = 0.0 # Total mass of clump 1 and clump 2 totmass = 0.0 # Total mass of halo 1 and halo 2
intersect = 0.0 # Weighted intersecting mass intersect = 0.0 # Weighted intersecting mass
i0, j0, k0 = cellmins # Unpack things i0, j0, k0 = cellmins # Unpack things
bckg_size = 2 * BCKG_HALFSIZE bckg_size = 2 * BCKG_HALFSIZE
@ -785,7 +779,7 @@ def calculate_overlap(delta1, delta2, cellmins, delta_bckg):
def calculate_overlap_indxs(delta1, delta2, cellmins, delta_bckg, nonzero, def calculate_overlap_indxs(delta1, delta2, cellmins, delta_bckg, nonzero,
mass1, mass2): mass1, mass2):
r""" r"""
Overlap between two clumps whose density fields are evaluated on the Overlap between two haloes whose density fields are evaluated on the
same grid and `nonzero1` enumerates the non-zero cells of `delta1. This is same grid and `nonzero1` enumerates the non-zero cells of `delta1. This is
a JIT implementation, hence it is outside of the main class. a JIT implementation, hence it is outside of the main class.
@ -802,10 +796,10 @@ def calculate_overlap_indxs(delta1, delta2, cellmins, delta_bckg, nonzero,
calculated with particles assigned to halos at the final snapshot. calculated with particles assigned to halos at the final snapshot.
Assumed to only be sampled in cells :math:`[512, 1536)^3`. Assumed to only be sampled in cells :math:`[512, 1536)^3`.
nonzero : 2-dimensional array of shape `(n_cells, 3)` nonzero : 2-dimensional array of shape `(n_cells, 3)`
Indices of cells that are non-zero of the lower mass clump. Expected to Indices of cells that are non-zero of the lower mass halo. Expected to
be precomputed from `fill_delta_indxs`. be precomputed from `fill_delta_indxs`.
mass1, mass2 : floats, optional mass1, mass2 : floats, optional
Total masses of the two clumps, respectively. Optional. If not provided Total masses of the two haloes, respectively. Optional. If not provided
calculcated directly from the density field. calculcated directly from the density field.
Returns Returns
@ -837,8 +831,7 @@ def calculate_overlap_indxs(delta1, delta2, cellmins, delta_bckg, nonzero,
return intersect / (mass1 + mass2 - intersect) return intersect / (mass1 + mass2 - intersect)
def load_processed_halo(hid, particles, clump_map, clid2map, clumps_cat, def load_processed_halo(hid, particles, halo_map, hid2map, ncells, nshift):
ncells, nshift):
""" """
Load a processed halo from the `.h5` file. This is to be wrapped by a Load a processed halo from the `.h5` file. This is to be wrapped by a
cacher. cacher.
@ -850,13 +843,11 @@ def load_processed_halo(hid, particles, clump_map, clid2map, clumps_cat,
particles : 2-dimensional array particles : 2-dimensional array
Array of particles in box units. The columns must be `x`, `y`, `z` Array of particles in box units. The columns must be `x`, `y`, `z`
and `M`. and `M`.
clump_map : 2-dimensional array halo_map : 2-dimensional array
Array containing start and end indices in the particle array Array containing start and end indices in the particle array
corresponding to each clump. corresponding to each halo.
clid2map : dict hid2map : dict
Dictionary mapping clump IDs to `clump_map` array positions. Dictionary mapping halo IDs to `halo_map` array positions.
clumps_cat : :py:class:`csiborgtools.read.ClumpsCatalogue`
Clumps catalogue.
ncells : int ncells : int
Number of cells in the original density field. Typically 2048. Number of cells in the original density field. Typically 2048.
nshift : int nshift : int
@ -875,8 +866,7 @@ def load_processed_halo(hid, particles, clump_map, clid2map, clumps_cat,
maxs : len-3 tuple maxs : len-3 tuple
Maximum cell indices of the halo. Maximum cell indices of the halo.
""" """
pos = load_parent_particles(hid, particles, clump_map, clid2map, pos = load_halo_particles(hid, particles, halo_map, hid2map)
clumps_cat)
pos, mass = pos[:, :3], pos[:, 3] pos, mass = pos[:, :3], pos[:, 3]
pos = pos2cell(pos, ncells) pos = pos2cell(pos, ncells)
totmass = numpy.sum(mass) totmass = numpy.sum(mass)
@ -898,9 +888,9 @@ def radius_neighbours(knn, X, radiusX, radiusKNN, nmult=1.0,
Array of shape `(n_samples, 3)`, where the latter axis represents Array of shape `(n_samples, 3)`, where the latter axis represents
`x`, `y` and `z`. `x`, `y` and `z`.
radiusX: 1-dimensional array of shape `(n_samples, )` radiusX: 1-dimensional array of shape `(n_samples, )`
Patch radii corresponding to clumps in `X`. Patch radii corresponding to haloes in `X`.
radiusKNN : 1-dimensional array radiusKNN : 1-dimensional array
Patch radii corresponding to clumps used to train `knn`. Patch radii corresponding to haloes used to train `knn`.
nmult : float, optional nmult : float, optional
Multiple of the sum of two radii below which to consider a match. Multiple of the sum of two radii below which to consider a match.
enforce_int32 : bool, optional enforce_int32 : bool, optional

7
csiborgtools/read/__init__.py
View File

@ -13,8 +13,7 @@
# with this program; if not, write to the Free Software Foundation, Inc., # with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
from .box_units import CSiBORGBox, QuijoteBox # noqa from .box_units import CSiBORGBox, QuijoteBox # noqa
from .halo_cat import (ClumpsCatalogue, HaloCatalogue, # noqa from .halo_cat import (CSiBORGHaloCatalogue, QuijoteHaloCatalogue, fiducial_observers) # noqa
QuijoteHaloCatalogue, fiducial_observers)
from .knn_summary import kNNCDFReader # noqa from .knn_summary import kNNCDFReader # noqa
from .nearest_neighbour_summary import NearestNeighbourReader # noqa from .nearest_neighbour_summary import NearestNeighbourReader # noqa
from .obs import (SDSS, MCXCClusters, PlanckClusters, TwoMPPGalaxies, # noqa from .obs import (SDSS, MCXCClusters, PlanckClusters, TwoMPPGalaxies, # noqa
@ -25,8 +24,8 @@ from .overlap_summary import (NPairsOverlap, PairOverlap, # noqa
from .paths import Paths # noqa from .paths import Paths # noqa
from .pk_summary import PKReader # noqa from .pk_summary import PKReader # noqa
from .readsim import (MmainReader, ParticleReader, halfwidth_mask, # noqa from .readsim import (MmainReader, ParticleReader, halfwidth_mask, # noqa
load_clump_particles, load_parent_particles, read_initcm) load_halo_particles, read_initcm) # noqa
from .tpcf_summary import TPCFReader # noqa from .tpcf_summary import TPCFReader # noqa
from .utils import (M200_to_R200, cartesian_to_radec, # noqa from .utils import (M200_to_R200, cartesian_to_radec, # noqa
cols_to_structured, radec_to_cartesian, read_h5, cols_to_structured, radec_to_cartesian, read_h5,
real2redshift) real2redshift) # noqa

4
csiborgtools/read/box_units.py
View File

@ -26,11 +26,11 @@ from .readsim import ParticleReader
# Map of CSiBORG unit conversions # Map of CSiBORG unit conversions
CONV_NAME = { CONV_NAME = {
"length": ["x", "y", "z", "peak_x", "peak_y", "peak_z", "Rs", "rmin", "length": ["x", "y", "z", "peak_x", "peak_y", "peak_z", "Rs", "rmin",
"rmax", "r200c", "r500c", "r200m", "x0", "y0", "z0", "rmax", "r200c", "r500c", "r200m", "r500m", "x0", "y0", "z0",
"lagpatch_size"], "lagpatch_size"],
"velocity": ["vx", "vy", "vz"], "velocity": ["vx", "vy", "vz"],
"mass": ["mass_cl", "totpartmass", "m200c", "m500c", "mass_mmain", "M", "mass": ["mass_cl", "totpartmass", "m200c", "m500c", "mass_mmain", "M",
"m200m"], "m200m", "m500m"],
"density": ["rho0"]} "density": ["rho0"]}

160
csiborgtools/read/halo_cat.py
View File

@ -14,8 +14,8 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
""" """
Simulation catalogues: Simulation catalogues:
- CSiBORG: halo and clump catalogue. - CSiBORG: FoF halo catalogue.
- Quijote: halo catalogue. - Quijote: FoF halo catalogue.
""" """
from abc import ABC, abstractproperty from abc import ABC, abstractproperty
from copy import deepcopy from copy import deepcopy
@ -25,7 +25,6 @@ from math import floor
from os.path import join from os.path import join
import numpy import numpy
from readfof import FoF_catalog from readfof import FoF_catalog
from sklearn.neighbors import NearestNeighbors from sklearn.neighbors import NearestNeighbors
@ -369,6 +368,9 @@ class BaseCatalogue(ABC):
return self.data.dtype.names return self.data.dtype.names
def __getitem__(self, key): def __getitem__(self, key):
if isinstance(key, (int, numpy.integer)):
assert key >= 0
return self.data[key]
if key not in self.keys: if key not in self.keys:
raise KeyError(f"Key '{key}' not in catalogue.") raise KeyError(f"Key '{key}' not in catalogue.")
return self.data[key] return self.data[key]
@ -377,111 +379,14 @@ class BaseCatalogue(ABC):
return self.data.size return self.data.size
###############################################################################
# CSiBORG base catalogue #
###############################################################################
class BaseCSiBORG(BaseCatalogue):
"""
Base CSiBORG catalogue class.
"""
@property
def nsnap(self):
return max(self.paths.get_snapshots(self.nsim))
@property
def box(self):
"""
CSiBORG box object handling unit conversion.
Returns
-------
box : instance of :py:class:`csiborgtools.units.BaseBox`
"""
return CSiBORGBox(self.nsnap, self.nsim, self.paths)
###############################################################################
# CSiBORG clumps catalogue #
###############################################################################
class ClumpsCatalogue(BaseCSiBORG):
r"""
Clumps catalogue, defined in the final snapshot.
Parameters
----------
sim : int
IC realisation index.
paths : py:class`csiborgtools.read.Paths`
Paths object.
bounds : dict
Parameter bounds to apply to the catalogue. The keys are the parameter
names and the items are a len-2 tuple of (min, max) values. In case of
no minimum or maximum, use `None`. For radial distance from the origin
use `dist`.
load_fitted : bool, optional
Whether to load fitted quantities.
rawdata : bool, optional
Whether to return the raw data. In this case applies no cuts and
transformations.
"""
def __init__(self, nsim, paths, bounds={"dist": (0, 155.5 / 0.705)},
load_fitted=True, rawdata=False):
self.nsim = nsim
self.paths = paths
# Read in the clumps from the final snapshot
partreader = ParticleReader(self.paths)
cols = ["index", "parent", "x", "y", "z", "mass_cl"]
self._data = partreader.read_clumps(self.nsnap, self.nsim, cols=cols)
# Overwrite the parent with the ultimate parent
mmain = numpy.load(self.paths.mmain(self.nsnap, self.nsim))
self._data["parent"] = mmain["ultimate_parent"]
if load_fitted:
fits = numpy.load(paths.structfit(self.nsnap, nsim, "clumps"))
cols = [col for col in fits.dtype.names if col != "index"]
X = [fits[col] for col in cols]
self._data = add_columns(self._data, X, cols)
# If the raw data is not required, then start applying transformations
# and cuts.
if not rawdata:
flip_cols(self._data, "x", "z")
for p in ("x", "y", "z"):
self._data[p] -= 0.5
names = ["x", "y", "z", "mass_cl", "totpartmass", "rho0", "r200c",
"r500c", "m200c", "m500c", "r200m", "m200m",
"vx", "vy", "vz"]
self._data = self.box.convert_from_box(self._data, names)
if bounds is not None:
self.apply_bounds(bounds)
@property
def ismain(self):
"""
Whether the clump is a main halo.
Returns
-------
ismain : 1-dimensional array
"""
return self["index"] == self["parent"]
############################################################################### ###############################################################################
# CSiBORG halo catalogue # # CSiBORG halo catalogue #
############################################################################### ###############################################################################
class HaloCatalogue(BaseCSiBORG): class CSiBORGHaloCatalogue(BaseCatalogue):
r""" r"""
Halo catalogue, i.e. parent halos with summed substructure, defined in the CSiBORG FoF halo catalogue.
final snapshot.
Parameters Parameters
---------- ----------
@ -504,22 +409,17 @@ class HaloCatalogue(BaseCSiBORG):
Whether to return the raw data. In this case applies no cuts and Whether to return the raw data. In this case applies no cuts and
transformations. transformations.
""" """
_clumps_cat = None
def __init__(self, nsim, paths, bounds={"dist": (0, 155.5 / 0.705)}, def __init__(self, nsim, paths, bounds={"dist": (0, 155.5 / 0.705)},
with_lagpatch=True, load_fitted=True, load_initial=True, with_lagpatch=True, load_fitted=True, load_initial=True,
load_clumps_cat=False, rawdata=False): rawdata=False):
self.nsim = nsim self.nsim = nsim
self.paths = paths self.paths = paths
# Read in the mmain catalogue of summed substructure reader = ParticleReader(paths)
mmain = numpy.load(self.paths.mmain(self.nsnap, self.nsim)) self._data = reader.read_fof_halos(self.nsim)
self._data = mmain["mmain"]
# We will also need the clumps catalogue
if load_clumps_cat:
self._clumps_cat = ClumpsCatalogue(nsim, paths, rawdata=True,
load_fitted=False)
if load_fitted: if load_fitted:
fits = numpy.load(paths.structfit(self.nsnap, nsim, "halos")) fits = numpy.load(paths.structfit(self.nsnap, nsim))
cols = [col for col in fits.dtype.names if col != "index"] cols = [col for col in fits.dtype.names if col != "index"]
X = [fits[col] for col in cols] X = [fits[col] for col in cols]
self._data = add_columns(self._data, X, cols) self._data = add_columns(self._data, X, cols)
@ -538,19 +438,25 @@ class HaloCatalogue(BaseCSiBORG):
self._data = add_columns(self._data, X, cols) self._data = add_columns(self._data, X, cols)
if not rawdata: if rawdata:
for p in ('x', 'y', 'z'):
self._data[p] = self.box.mpc2box(self._data[p]) + 0.5
else:
if with_lagpatch: if with_lagpatch:
self._data = self._data[numpy.isfinite(self["lagpatch_size"])] self._data = self._data[numpy.isfinite(self["lagpatch_size"])]
# Flip positions and convert from code units to cMpc. Convert M too # Flip positions and convert from code units to cMpc. Convert M too
flip_cols(self._data, "x", "z") flip_cols(self._data, "x", "z")
for p in ("x", "y", "z"): if load_fitted:
self._data[p] -= 0.5 flip_cols(self._data, "vx", "vz")
names = ["x", "y", "z", "M", "totpartmass", "rho0", "r200c", names = ["totpartmass", "rho0", "r200c",
"r500c", "m200c", "m500c", "r200m", "m200m", "r500c", "m200c", "m500c", "r200m", "m200m",
"vx", "vy", "vz"] "r500m", "m500m", "vx", "vy", "vz"]
self._data = self.box.convert_from_box(self._data, names) self._data = self.box.convert_from_box(self._data, names)
if load_initial: if load_initial:
flip_cols(self._data, "x0", "z0")
for p in ("x0", "y0", "z0"):
self._data[p] -= 0.5
names = ["x0", "y0", "z0", "lagpatch_size"] names = ["x0", "y0", "z0", "lagpatch_size"]
self._data = self.box.convert_from_box(self._data, names) self._data = self.box.convert_from_box(self._data, names)
@ -558,17 +464,19 @@ class HaloCatalogue(BaseCSiBORG):
self.apply_bounds(bounds) self.apply_bounds(bounds)
@property @property
def clumps_cat(self): def nsnap(self):
return max(self.paths.get_snapshots(self.nsim))
@property
def box(self):
""" """
The raw clumps catalogue. CSiBORG box object handling unit conversion.
Returns Returns
------- -------
clumps_cat : :py:class:`csiborgtools.read.ClumpsCatalogue` box : instance of :py:class:`csiborgtools.units.BaseBox`
""" """
if self._clumps_cat is None: return CSiBORGBox(self.nsnap, self.nsim, self.paths)
raise ValueError("`clumps_cat` is not loaded.")
return self._clumps_cat
############################################################################### ###############################################################################
@ -578,7 +486,7 @@ class HaloCatalogue(BaseCSiBORG):
class QuijoteHaloCatalogue(BaseCatalogue): class QuijoteHaloCatalogue(BaseCatalogue):
""" """
Quijote halo catalogue. Quijote FoF halo catalogue.
Parameters Parameters
---------- ----------

12
csiborgtools/read/overlap_summary.py
View File

@ -32,9 +32,9 @@ class PairOverlap:
Parameters Parameters
---------- ----------
cat0 : :py:class:`csiborgtools.read.HaloCatalogue` cat0 : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue corresponding to the reference simulation. Halo catalogue corresponding to the reference simulation.
catx : :py:class:`csiborgtools.read.HaloCatalogue` catx : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue corresponding to the cross simulation. Halo catalogue corresponding to the cross simulation.
paths : py:class`csiborgtools.read.Paths` paths : py:class`csiborgtools.read.Paths`
CSiBORG paths object. CSiBORG paths object.
@ -58,9 +58,9 @@ class PairOverlap:
Parameters Parameters
---------- ----------
cat0 : :py:class:`csiborgtools.read.HaloCatalogue` cat0 : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue corresponding to the reference simulation. Halo catalogue corresponding to the reference simulation.
catx : :py:class:`csiborgtools.read.HaloCatalogue` catx : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Halo catalogue corresponding to the cross simulation. Halo catalogue corresponding to the cross simulation.
paths : py:class`csiborgtools.read.Paths` paths : py:class`csiborgtools.read.Paths`
CSiBORG paths object. CSiBORG paths object.
@ -557,9 +557,9 @@ class NPairsOverlap:
Parameters Parameters
---------- ----------
cat0 : :py:class:`csiborgtools.read.HaloCatalogue` cat0 : :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
Single reference simulation halo catalogue. Single reference simulation halo catalogue.
catxs : list of :py:class:`csiborgtools.read.HaloCatalogue` catxs : list of :py:class:`csiborgtools.read.CSiBORGHaloCatalogue`
List of cross simulation halo catalogues. List of cross simulation halo catalogues.
paths : py:class`csiborgtools.read.Paths` paths : py:class`csiborgtools.read.Paths`
CSiBORG paths object. CSiBORG paths object.

48
csiborgtools/read/paths.py
View File

@ -186,6 +186,42 @@ class Paths:
""" """
return join(self.borg_dir, "mcmc", f"mcmc_{nsim}.h5") return join(self.borg_dir, "mcmc", f"mcmc_{nsim}.h5")
def fof_membership(self, nsim, sorted=False):
"""
Path to the file containing the FoF particle membership.
Parameters
----------
nsim : int
IC realisation index.
sorted : bool, optional
Whether to return path to the file that is sorted in the same
order as the PHEW output.
"""
fdir = join(self.postdir, "FoF_membership", )
if not isdir(fdir):
mkdir(fdir)
warn(f"Created directory `{fdir}`.", UserWarning, stacklevel=1)
fout = join(fdir, f"fof_membership_{nsim}.npy")
if sorted:
fout = fout.replace(".npy", "_sorted.npy")
return fout
def fof_cat(self, nsim):
"""
Path to the FoF halo catalogue file.
Parameters
----------
nsim : int
IC realisation index.
"""
fdir = join(self.postdir, "FoF_membership", )
if not isdir(fdir):
mkdir(fdir)
warn(f"Created directory `{fdir}`.", UserWarning, stacklevel=1)
return join(fdir, f"halo_catalog_{nsim}_FOF.txt")
def mmain(self, nsnap, nsim): def mmain(self, nsnap, nsim):
""" """
Path to the `mmain` CSiBORG files of summed substructure. Path to the `mmain` CSiBORG files of summed substructure.
@ -246,7 +282,7 @@ class Paths:
------- -------
ids : 1-dimensional array ids : 1-dimensional array
""" """
assert simname in ["csiborg", "quijote"] assert simname in ["csiborg", "quijote", "quijote_full"]
if simname == "csiborg": if simname == "csiborg":
files = glob(join(self.srcdir, "ramses_out*")) files = glob(join(self.srcdir, "ramses_out*"))
files = [f.split("/")[-1] for f in files] # Only file names files = [f.split("/")[-1] for f in files] # Only file names
@ -260,6 +296,7 @@ class Paths:
pass pass
return numpy.sort(ids) return numpy.sort(ids)
else: else:
# TODO here later read this from the catalogues instead.
return numpy.arange(100, dtype=int) return numpy.arange(100, dtype=int)
def ic_path(self, nsim, tonew=False): def ic_path(self, nsim, tonew=False):
@ -323,7 +360,7 @@ class Paths:
simpath = self.ic_path(nsim, tonew=tonew) simpath = self.ic_path(nsim, tonew=tonew)
return join(simpath, f"output_{str(nsnap).zfill(5)}") return join(simpath, f"output_{str(nsnap).zfill(5)}")
def structfit(self, nsnap, nsim, kind): def structfit(self, nsnap, nsim):
""" """
Path to the clump or halo catalogue from `fit_halos.py`. Only CSiBORG Path to the clump or halo catalogue from `fit_halos.py`. Only CSiBORG
is supported. is supported.
@ -334,19 +371,16 @@ class Paths:
Snapshot index. Snapshot index.
nsim : int nsim : int
IC realisation index. IC realisation index.
kind : str
Type of catalogue. Can be either `clumps` or `halos`.
Returns Returns
------- -------
path : str path : str
""" """
assert kind in ["clumps", "halos"]
fdir = join(self.postdir, "structfit") fdir = join(self.postdir, "structfit")
if not isdir(fdir): if not isdir(fdir):
mkdir(fdir) mkdir(fdir)
warn(f"Created directory `{fdir}`.", UserWarning, stacklevel=1) warn(f"Created directory `{fdir}`.", UserWarning, stacklevel=1)
fname = f"{kind}_out_{str(nsim).zfill(5)}_{str(nsnap).zfill(5)}.npy" fname = f"out_{str(nsim).zfill(5)}_{str(nsnap).zfill(5)}.npy"
return join(fdir, fname) return join(fdir, fname)
def overlap(self, nsim0, nsimx, smoothed): def overlap(self, nsim0, nsimx, smoothed):
@ -441,7 +475,7 @@ class Paths:
Parameters Parameters
---------- ----------
simname : str simname : str
Simulation name. Must be `csiborg` or `quijote`. Simulation name. Must be `csiborg`, `quijote` or `quijote_full`.
nsim : int nsim : int
IC realisation index. IC realisation index.

123
csiborgtools/read/readsim.py
View File

@ -32,7 +32,7 @@ from .utils import cols_to_structured
class ParticleReader: class ParticleReader:
""" """
Shortcut to read in particle files along with their corresponding clumps. Object to read in particle files along with their corresponding haloes.
Parameters Parameters
---------- ----------
@ -203,7 +203,7 @@ class ParticleReader:
nsim : int nsim : int
IC realisation index. IC realisation index.
pars_extract : list of str pars_extract : list of str
Parameters to be extacted. Parameters to be extracted.
return_structured : bool, optional return_structured : bool, optional
Whether to return a structured array or a 2-dimensional array. If Whether to return a structured array or a 2-dimensional array. If
the latter, then the order of the columns is the same as the order the latter, then the order of the columns is the same as the order
@ -280,7 +280,7 @@ class ParticleReader:
def open_unbinding(self, nsnap, nsim, cpu): def open_unbinding(self, nsnap, nsim, cpu):
""" """
Open particle files to a given CSiBORG simulation. Note that to be Open particle files of a given CSiBORG simulation. Note that to be
consistent CPU is incremented by 1. consistent CPU is incremented by 1.
Parameters Parameters
@ -305,7 +305,8 @@ class ParticleReader:
def read_clumpid(self, nsnap, nsim, verbose=True): def read_clumpid(self, nsnap, nsim, verbose=True):
""" """
Read clump IDs of particles from unbinding files. Read PHEW clump IDs of particles from unbinding files. This halo finder
was used when running the catalogue.
Parameters Parameters
---------- ----------
@ -332,14 +333,13 @@ class ParticleReader:
j = nparts[cpu] j = nparts[cpu]
ff = self.open_unbinding(nsnap, nsim, cpu) ff = self.open_unbinding(nsnap, nsim, cpu)
clumpid[i:i + j] = ff.read_ints() clumpid[i:i + j] = ff.read_ints()
ff.close() ff.close()
return clumpid return clumpid
def read_clumps(self, nsnap, nsim, cols=None): def read_clumps(self, nsnap, nsim, cols=None):
""" """
Read in a clump file `clump_xxXXX.dat`. Read in a PHEW clump file `clump_xxXXX.dat`.
Parameters Parameters
---------- ----------
@ -387,6 +387,54 @@ class ParticleReader:
out[col] = data[:, clump_cols[col][0]] out[col] = data[:, clump_cols[col][0]]
return out return out
def read_fof_hids(self, nsim):
"""
Read in the FoF particle halo membership IDs that are sorted to match
the PHEW output.
Parameters
----------
nsim : int
IC realisation index.
Returns
-------
hids : 1-dimensional array
Halo IDs of particles.
"""
return numpy.load(self.paths.fof_membership(nsim, sorted=True))
def read_fof_halos(self, nsim):
"""
Read in the FoF halo catalogue.
Parameters
----------
nsim : int
IC realisation index.
Returns
-------
cat : structured array
"""
fpath = self.paths.fof_cat(nsim)
hid = numpy.genfromtxt(fpath, usecols=0, dtype=numpy.int32)
pos = numpy.genfromtxt(fpath, usecols=(1, 2, 3), dtype=numpy.float32)
totmass = numpy.genfromtxt(fpath, usecols=4, dtype=numpy.float32)
m200c = numpy.genfromtxt(fpath, usecols=5, dtype=numpy.float32)
dtype = {"names": ["index", "x", "y", "z", "fof_totpartmass",
"fof_m200c"],
"formats": [numpy.int32] + [numpy.float32] * 5}
out = numpy.full(hid.size, numpy.nan, dtype=dtype)
out["index"] = hid
out["x"] = pos[:, 0]
out["y"] = pos[:, 1]
out["z"] = pos[:, 2]
out["fof_totpartmass"] = totmass * 1e11
out["fof_m200c"] = m200c * 1e11
return out
############################################################################### ###############################################################################
# Summed substructure catalogue # # Summed substructure catalogue #
@ -457,7 +505,7 @@ class MmainReader:
""" """
Make the summed substructure catalogue for a final snapshot. Includes Make the summed substructure catalogue for a final snapshot. Includes
the position of the parent, the summed mass and the fraction of mass in the position of the parent, the summed mass and the fraction of mass in
substructure. substructure. Corresponds to the PHEW Halo finder.
Parameters Parameters
---------- ----------
@ -552,39 +600,10 @@ def halfwidth_mask(pos, hw):
return numpy.all((0.5 - hw < pos) & (pos < 0.5 + hw), axis=1) return numpy.all((0.5 - hw < pos) & (pos < 0.5 + hw), axis=1)
def load_clump_particles(clid, particles, clump_map, clid2map): def load_halo_particles(hid, particles, halo_map, hid2map):
""" """
Load a clump's particles from a particle array. If it is not there, i.e Load a halo's particles from a particle array. If it is not there, i.e
clump has no associated particles, return `None`. halo has no associated particles, return `None`.
Parameters
----------
clid : int
Clump ID.
particles : 2-dimensional array
Array of particles.
clump_map : 2-dimensional array
Array containing start and end indices in the particle array
corresponding to each clump.
clid2map : dict
Dictionary mapping clump IDs to `clump_map` array positions.
Returns
-------
clump_particles : 2-dimensional array
Particle array of this clump.
"""
try:
k0, kf = clump_map[clid2map[clid], 1:]
return particles[k0:kf + 1, :]
except KeyError:
return None
def load_parent_particles(hid, particles, clump_map, clid2map, clumps_cat):
"""
Load a parent halo's particles from a particle array. If it is not there,
return `None`.
Parameters Parameters
---------- ----------
@ -592,27 +611,19 @@ def load_parent_particles(hid, particles, clump_map, clid2map, clumps_cat):
Halo ID. Halo ID.
particles : 2-dimensional array particles : 2-dimensional array
Array of particles. Array of particles.
clump_map : 2-dimensional array halo_map : 2-dimensional array
Array containing start and end indices in the particle array Array containing start and end indices in the particle array
corresponding to each clump. corresponding to each halo.
clid2map : dict hid2map : dict
Dictionary mapping clump IDs to `clump_map` array positions. Dictionary mapping halo IDs to `halo_map` array positions.
clumps_cat : :py:class:`csiborgtools.read.ClumpsCatalogue`
Clumps catalogue.
Returns Returns
------- -------
halo : 2-dimensional array halo_particles : 2-dimensional array
Particle array of this halo. Particle array of this halo.
""" """
clids = clumps_cat["index"][clumps_cat["parent"] == hid] try:
# We first load the particles of each clump belonging to this parent k0, kf = halo_map[hid2map[hid], 1:]
# and then concatenate them for further analysis. return particles[k0:kf + 1, :]
clumps = [] except KeyError:
for clid in clids:
parts = load_clump_particles(clid, particles, clump_map, clid2map)
if parts is not None:
clumps.append(parts)
if len(clumps) == 0:
return None return None
return numpy.concatenate(clumps)

2
notebooks/perseus.ipynb
View File

@ -82,7 +82,7 @@
"matches = np.full(len(nsims), np.nan)\n", "matches = np.full(len(nsims), np.nan)\n",
"\n", "\n",
"for ii in trange(101):\n", "for ii in trange(101):\n",
" cat = csiborgtools.read.HaloCatalogue(nsims[ii], paths, minmass=('M', 1e13))\n", " cat = csiborgtools.read.CSiBORGHaloCatalogue(nsims[ii], paths, minmass=('M', 1e13))\n",
" dist, ind = cat.angular_neighbours(Xclust, ang_radius=5, rad_tolerance=10)\n", " dist, ind = cat.angular_neighbours(Xclust, ang_radius=5, rad_tolerance=10)\n",
" dist = dist[0]\n", " dist = dist[0]\n",
" ind = ind[0]\n", " ind = ind[0]\n",

88
scripts/fit_halos.py
View File

@ -13,15 +13,17 @@
# with this program; if not, write to the Free Software Foundation, Inc., # with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
""" """
A script to fit halos (concentration, ...). The particle array of each CSiBORG A script to fit FoF halos (concentration, ...). The particle array of each
realisation must have been split in advance by `runsplit_halos`. CSiBORG realisation must have been processed in advance by `pre_dumppart.py`.
""" """
from argparse import ArgumentParser from argparse import ArgumentParser
from datetime import datetime from datetime import datetime
import numpy import numpy
from mpi4py import MPI from mpi4py import MPI
from tqdm import tqdm from tqdm import trange
from utils import get_nsims
try: try:
import csiborgtools import csiborgtools
@ -38,18 +40,13 @@ nproc = comm.Get_size()
verbose = nproc == 1 verbose = nproc == 1
parser = ArgumentParser() parser = ArgumentParser()
parser.add_argument("--kind", type=str, choices=["halos", "clumps"]) parser.add_argument("--nsims", type=int, nargs="+", default=None,
parser.add_argument("--ics", type=int, nargs="+", default=None,
help="IC realisations. If `-1` processes all simulations.") help="IC realisations. If `-1` processes all simulations.")
args = parser.parse_args() args = parser.parse_args()
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring) paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
partreader = csiborgtools.read.ParticleReader(paths) partreader = csiborgtools.read.ParticleReader(paths)
nfwpost = csiborgtools.fits.NFWPosterior() nfwpost = csiborgtools.fits.NFWPosterior()
nsims = get_nsims(args, paths)
if args.ics is None or args.ics[0] == -1:
ics = paths.get_ics("csiborg")
else:
ics = args.ics
cols_collect = [ cols_collect = [
("index", numpy.int32), ("index", numpy.int32),
@ -67,13 +64,12 @@ cols_collect = [
("lambda200c", numpy.float32), ("lambda200c", numpy.float32),
("r200m", numpy.float32), ("r200m", numpy.float32),
("m200m", numpy.float32), ("m200m", numpy.float32),
("r500m", numpy.float32),
("m500m", numpy.float32),
] ]
def fit_clump(particles, clump_info, box): def fit_halo(particles, clump_info, box):
"""
Fit an object. Can be eithe a clump or a parent halo.
"""
obj = csiborgtools.fits.Clump(particles, clump_info, box) obj = csiborgtools.fits.Clump(particles, clump_info, box)
out = {} out = {}
@ -82,16 +78,15 @@ def fit_clump(particles, clump_info, box):
for i, v in enumerate(["vx", "vy", "vz"]): for i, v in enumerate(["vx", "vy", "vz"]):
out[v] = numpy.average(obj.vel[:, i], weights=obj["M"]) out[v] = numpy.average(obj.vel[:, i], weights=obj["M"])
# Overdensity masses # Overdensity masses
out["r200c"], out["m200c"] = obj.spherical_overdensity_mass(200, for n in [200, 500]:
kind="crit") out[f"r{n}c"], out[f"m{n}c"] = obj.spherical_overdensity_mass(
out["r500c"], out["m500c"] = obj.spherical_overdensity_mass(500, n, kind="crit", npart_min=10)
kind="crit") out[f"r{n}m"], out[f"m{n}m"] = obj.spherical_overdensity_mass(
out["r200m"], out["m200m"] = obj.spherical_overdensity_mass(200, n, kind="matter", npart_min=10)
kind="matter")
# NFW fit # NFW fit
if out["npart"] > 10 and numpy.isfinite(out["r200c"]): if out["npart"] > 10 and numpy.isfinite(out["r200c"]):
Rs, rho0 = nfwpost.fit(obj) Rs, rho0 = nfwpost.fit(obj)
out["conc"] = Rs / out["r200c"] out["conc"] = out["r200c"] / Rs
out["rho0"] = rho0 out["rho0"] = rho0
# Spin within R200c # Spin within R200c
if numpy.isfinite(out["r200c"]): if numpy.isfinite(out["r200c"]):
@ -100,8 +95,8 @@ def fit_clump(particles, clump_info, box):
# We MPI loop over all simulations. # We MPI loop over all simulations.
jobs = csiborgtools.fits.split_jobs(len(ics), nproc)[rank] jobs = csiborgtools.fits.split_jobs(len(nsims), nproc)[rank]
for nsim in [ics[i] for i in jobs]: for nsim in [nsims[i] for i in jobs]:
print(f"{datetime.now()}: rank {rank} calculating simulation `{nsim}`.", print(f"{datetime.now()}: rank {rank} calculating simulation `{nsim}`.",
flush=True) flush=True)
nsnap = max(paths.get_snapshots(nsim)) nsnap = max(paths.get_snapshots(nsim))
@ -110,49 +105,30 @@ for nsim in [ics[i] for i in jobs]:
# Particle archive # Particle archive
f = csiborgtools.read.read_h5(paths.particles(nsim)) f = csiborgtools.read.read_h5(paths.particles(nsim))
particles = f["particles"] particles = f["particles"]
clump_map = f["clumpmap"] halo_map = f["halomap"]
clid2map = {clid: i for i, clid in enumerate(clump_map[:, 0])} hid2map = {clid: i for i, clid in enumerate(halo_map[:, 0])}
clumps_cat = csiborgtools.read.ClumpsCatalogue(nsim, paths, rawdata=True, cat = csiborgtools.read.CSiBORGHaloCatalogue(
load_fitted=False) nsim, paths, with_lagpatch=False, load_initial=False, rawdata=True,
# We check whether we fit halos or clumps, will be indexing over different load_fitted=False)
# iterators.
if args.kind == "halos":
ismain = clumps_cat.ismain
else:
ismain = numpy.ones(len(clumps_cat), dtype=bool)
# Even if we are calculating parent halo this index runs over all clumps. # Even if we are calculating parent halo this index runs over all clumps.
out = csiborgtools.read.cols_to_structured(len(clumps_cat), cols_collect) out = csiborgtools.read.cols_to_structured(len(cat), cols_collect)
indxs = clumps_cat["index"] indxs = cat["index"]
for i, clid in enumerate(tqdm(indxs)) if verbose else enumerate(indxs): for i in trange(len(cat)) if verbose else range(len(cat)):
clid = clumps_cat["index"][i] hid = cat["index"][i]
out["index"][i] = clid out["index"][i] = hid
# If we are fitting halos and this clump is not a main, then continue.
if args.kind == "halos" and not ismain[i]:
continue
if args.kind == "halos":
part = csiborgtools.read.load_parent_particles(
clid, particles, clump_map, clid2map, clumps_cat)
else:
part = csiborgtools.read.load_clump_particles(clid, particles,
clump_map, clid2map)
part = csiborgtools.read.load_halo_particles(hid, particles, halo_map,
hid2map)
# We fit the particles if there are any. If not we assign the index, # We fit the particles if there are any. If not we assign the index,
# otherwise it would be NaN converted to integers (-2147483648) and # otherwise it would be NaN converted to integers (-2147483648) and
# yield an error further down. # yield an error further down.
if part is None: if part is None:
continue continue
_out = fit_clump(part, clumps_cat[i], box) _out = fit_halo(part, cat[i], box)
for key in _out.keys(): for key in _out.keys():
out[key][i] = _out[key] out[key][i] = _out[key]
# Finally, we save the results. If we were analysing main halos, then fout = paths.structfit(nsnap, nsim)
# remove array indices that do not correspond to parent halos.
if args.kind == "halos":
out = out[ismain]
fout = paths.structfit(nsnap, nsim, args.kind)
print(f"Saving to `{fout}`.", flush=True) print(f"Saving to `{fout}`.", flush=True)
numpy.save(fout, out) numpy.save(fout, out)

17
scripts/fit_hmf.py
View File

@ -58,7 +58,9 @@ def get_counts(nsim, bins, paths, parser_args):
bounds = {"dist": (0, parser_args.Rmax)} bounds = {"dist": (0, parser_args.Rmax)}
if simname == "csiborg": if simname == "csiborg":
cat = csiborgtools.read.HaloCatalogue(nsim, paths, bounds=bounds) cat = csiborgtools.read.CSiBORGHaloCatalogue(
nsim, paths, bounds=bounds, with_lagpatch=False,
load_initial=False)
logmass = numpy.log10(cat["totpartmass"]) logmass = numpy.log10(cat["totpartmass"])
counts = csiborgtools.fits.number_counts(logmass, bins) counts = csiborgtools.fits.number_counts(logmass, bins)
elif simname == "quijote": elif simname == "quijote":
@ -71,6 +73,12 @@ def get_counts(nsim, bins, paths, parser_args):
cat = cat0.pick_fiducial_observer(nobs, rmax=parser_args.Rmax) cat = cat0.pick_fiducial_observer(nobs, rmax=parser_args.Rmax)
logmass = numpy.log10(cat["group_mass"]) logmass = numpy.log10(cat["group_mass"])
counts[nobs, :] = csiborgtools.fits.number_counts(logmass, bins) counts[nobs, :] = csiborgtools.fits.number_counts(logmass, bins)
elif simname == "quijote_full":
cat = csiborgtools.read.QuijoteHaloCatalogue(nsim, paths, nsnap=4)
logmass = numpy.log10(cat["group_mass"])
counts = csiborgtools.fits.number_counts(logmass, bins)
else:
raise ValueError(f"Unknown simulation name `{simname}`.")
fout = paths.halo_counts(simname, nsim) fout = paths.halo_counts(simname, nsim)
if parser_args.verbose: if parser_args.verbose:
@ -80,12 +88,15 @@ def get_counts(nsim, bins, paths, parser_args):
if __name__ == "__main__": if __name__ == "__main__":
parser = ArgumentParser() parser = ArgumentParser()
parser.add_argument("--simname", type=str, choices=["csiborg", "quijote"], parser.add_argument("--simname", type=str,
choices=["csiborg", "quijote", "quijote_full"],
help="Simulation name") help="Simulation name")
parser.add_argument("--nsims", type=int, nargs="+", default=None, parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="Indices of simulations to cross. If `-1` processes all simulations.") # noqa help="Indices of simulations to cross. If `-1` processes all simulations.") # noqa
parser.add_argument("--Rmax", type=float, default=155/0.705, parser.add_argument("--Rmax", type=float, default=155/0.705,
help="High-resolution region radius") help="High-resolution region radius")
parser.add_argument("--bw", type=float, default=0.2,
help="Bin width in dex")
parser.add_argument("--verbose", type=lambda x: bool(strtobool(x)), parser.add_argument("--verbose", type=lambda x: bool(strtobool(x)),
default=False) default=False)
@ -96,7 +107,7 @@ if __name__ == "__main__":
verbose = nproc == 1 verbose = nproc == 1
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring) paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsims = get_nsims(parser_args, paths) nsims = get_nsims(parser_args, paths)
bins = numpy.arange(11., 16., 0.2, dtype=numpy.float32) bins = numpy.arange(11., 16., parser_args.bw, dtype=numpy.float32)
def do_work(nsim): def do_work(nsim):
get_counts(nsim, bins, paths, parser_args) get_counts(nsim, bins, paths, parser_args)

39
scripts/fit_init.py
View File

@ -24,6 +24,8 @@ import numpy
from mpi4py import MPI from mpi4py import MPI
from tqdm import tqdm from tqdm import tqdm
from utils import get_nsims
try: try:
import csiborgtools import csiborgtools
except ModuleNotFoundError: except ModuleNotFoundError:
@ -41,16 +43,16 @@ verbose = nproc == 1
# Argument parser # Argument parser
parser = ArgumentParser() parser = ArgumentParser()
parser.add_argument("--ics", type=int, nargs="+", default=None, parser.add_argument("--simname", type=str, default="csiborg",
choices=["csiborg", "quijote"],
help="Simulation name")
parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="IC realisations. If `-1` processes all simulations.") help="IC realisations. If `-1` processes all simulations.")
args = parser.parse_args() args = parser.parse_args()
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring) paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
partreader = csiborgtools.read.ParticleReader(paths) partreader = csiborgtools.read.ParticleReader(paths)
if args.ics is None or args.ics[0] == -1: nsims = get_nsims(args, paths)
ics = paths.get_ics("csiborg")
else:
ics = args.ics
cols_collect = [("index", numpy.int32), cols_collect = [("index", numpy.int32),
("x", numpy.float32), ("x", numpy.float32),
@ -61,8 +63,8 @@ cols_collect = [("index", numpy.int32),
# MPI loop over simulations # MPI loop over simulations
jobs = csiborgtools.fits.split_jobs(len(ics), nproc)[rank] jobs = csiborgtools.fits.split_jobs(len(nsims), nproc)[rank]
for nsim in [ics[i] for i in jobs]: for nsim in [nsims[i] for i in jobs]:
nsnap = max(paths.get_snapshots(nsim)) nsnap = max(paths.get_snapshots(nsim))
overlapper = csiborgtools.match.ParticleOverlap() overlapper = csiborgtools.match.ParticleOverlap()
print(f"{datetime.now()}: rank {rank} calculating simulation `{nsim}`.", print(f"{datetime.now()}: rank {rank} calculating simulation `{nsim}`.",
@ -70,22 +72,18 @@ for nsim in [ics[i] for i in jobs]:
parts = csiborgtools.read.read_h5(paths.initmatch(nsim, "particles")) parts = csiborgtools.read.read_h5(paths.initmatch(nsim, "particles"))
parts = parts['particles'] parts = parts['particles']
clump_map = csiborgtools.read.read_h5(paths.particles(nsim)) halo_map = csiborgtools.read.read_h5(paths.particles(nsim))
clump_map = clump_map["clumpmap"] halo_map = halo_map["halomap"]
clumps_cat = csiborgtools.read.ClumpsCatalogue(nsim, paths, rawdata=True, cat = csiborgtools.read.CSiBORGHaloCatalogue(
load_fitted=False) nsim, paths, rawdata=True, load_fitted=False, load_initial=False)
clid2map = {clid: i for i, clid in enumerate(clump_map[:, 0])} hid2map = {hid: i for i, hid in enumerate(halo_map[:, 0])}
ismain = clumps_cat.ismain
out = csiborgtools.read.cols_to_structured(len(clumps_cat), cols_collect) out = csiborgtools.read.cols_to_structured(len(cat), cols_collect)
indxs = clumps_cat["index"] for i, hid in enumerate(tqdm(cat["index"]) if verbose else cat["index"]):
for i, hid in enumerate(tqdm(indxs) if verbose else indxs):
out["index"][i] = hid out["index"][i] = hid
if not ismain[i]: part = csiborgtools.read.load_halo_particles(hid, parts, halo_map,
continue hid2map)
part = csiborgtools.read.load_parent_particles(hid, parts, clump_map,
clid2map, clumps_cat)
# Skip if the halo is too small. # Skip if the halo is too small.
if part is None or part.size < 100: if part is None or part.size < 100:
continue continue
@ -101,7 +99,6 @@ for nsim in [ics[i] for i in jobs]:
delta = overlapper.make_delta(part[:, :3], part[:, 3], subbox=True) delta = overlapper.make_delta(part[:, :3], part[:, 3], subbox=True)
out["lagpatch_ncells"][i] = csiborgtools.fits.delta2ncells(delta) out["lagpatch_ncells"][i] = csiborgtools.fits.delta2ncells(delta)
out = out[ismain]
# Now save it # Now save it
fout = paths.initmatch(nsim, "fit") fout = paths.initmatch(nsim, "fit")
print(f"{datetime.now()}: dumping fits to .. `{fout}`.", print(f"{datetime.now()}: dumping fits to .. `{fout}`.",

10
scripts/match_singlematch.py
View File

@ -30,7 +30,7 @@ except ModuleNotFoundError:
def pair_match(nsim0, nsimx, sigma, smoothen, verbose): def pair_match(nsim0, nsimx, sigma, smoothen, verbose):
from csiborgtools.read import HaloCatalogue, read_h5 from csiborgtools.read import CSiBORGHaloCatalogue, read_h5
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring) paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
smooth_kwargs = {"sigma": sigma, "mode": "constant", "cval": 0.0} smooth_kwargs = {"sigma": sigma, "mode": "constant", "cval": 0.0}
@ -40,10 +40,10 @@ def pair_match(nsim0, nsimx, sigma, smoothen, verbose):
# Load the raw catalogues (i.e. no selection) including the initial CM # Load the raw catalogues (i.e. no selection) including the initial CM
# positions and the particle archives. # positions and the particle archives.
bounds = {"totpartmass": (1e12, None)} bounds = {"totpartmass": (1e12, None)}
cat0 = HaloCatalogue(nsim0, paths, load_initial=True, bounds=bounds, cat0 = CSiBORGHaloCatalogue(nsim0, paths, load_initial=True, bounds=bounds,
with_lagpatch=True, load_clumps_cat=True) with_lagpatch=True, load_clumps_cat=True)
catx = HaloCatalogue(nsimx, paths, load_initial=True, bounds=bounds, catx = CSiBORGHaloCatalogue(nsimx, paths, load_initial=True, bounds=bounds,
with_lagpatch=True, load_clumps_cat=True) with_lagpatch=True, load_clumps_cat=True)
clumpmap0 = read_h5(paths.particles(nsim0))["clumpmap"] clumpmap0 = read_h5(paths.particles(nsim0))["clumpmap"]
parts0 = read_h5(paths.initmatch(nsim0, "particles"))["particles"] parts0 = read_h5(paths.initmatch(nsim0, "particles"))["particles"]

151
scripts/mv_fofmembership.py Normal file
View File

@ -0,0 +1,151 @@
# Copyright (C) 2022 Richard Stiskalek
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""
Short script to move and change format of the CSiBORG FoF membership files
calculated by Julien. Additionally, also orders the particles in the same way
as the PHEW halo finder output.
"""
from argparse import ArgumentParser
from datetime import datetime
from gc import collect
from os.path import join
from shutil import copy
import numpy
from mpi4py import MPI
from taskmaster import work_delegation
from tqdm import trange
from utils import get_nsims
try:
import csiborgtools
except ModuleNotFoundError:
import sys
sys.path.append("../")
import csiborgtools
def copy_membership(nsim, verbose=True):
"""
Copy the FoF particle halo membership to the CSiBORG directory and write it
as a NumPy array instead of a text file.
Parameters
----------
nsim : int
IC realisation index.
verbose : bool, optional
Verbosity flag.
"""
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
fpath = join("/mnt/extraspace/jeg/greenwhale/Constrained_Sims",
f"sim_{nsim}/particle_membership_{nsim}_FOF.txt")
if verbose:
print(f"Loading from ... `{fpath}`.")
data = numpy.genfromtxt(fpath, dtype=int)
fout = paths.fof_membership(nsim)
if verbose:
print(f"Saving to ... `{fout}`.")
numpy.save(fout, data)
def copy_catalogue(nsim, verbose=True):
"""
Move the FoF catalogue to the CSiBORG directory.
Parameters
----------
nsim : int
IC realisation index.
verbose : bool, optional
Verbosity flag.
"""
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
source = join("/mnt/extraspace/jeg/greenwhale/Constrained_Sims",
f"sim_{nsim}/halo_catalog_{nsim}_FOF.txt")
dest = paths.fof_cat(nsim)
if verbose:
print("Copying`{}` to `{}`.".format(source, dest))
copy(source, dest)
def sort_fofid(nsim, verbose=True):
"""
Read the FoF particle halo membership and sort the halo IDs to the ordering
of particles in the PHEW clump IDs.
Parameters
----------
nsim : int
IC realisation index.
verbose : bool, optional
Verbosity flag.
"""
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsnap = max(paths.get_snapshots(nsim))
fpath = paths.fof_membership(nsim)
if verbose:
print(f"{datetime.now()}: loading from ... `{fpath}`.")
# Columns are halo ID, particle ID.
fof = numpy.load(fpath)
reader = csiborgtools.read.ParticleReader(paths)
pars_extract = ["x"] # Dummy variable
__, pids = reader.read_particle(nsnap, nsim, pars_extract,
return_structured=False, verbose=verbose)
del __
collect()
# Map the particle IDs in pids to their corresponding PHEW array index
if verbose:
print(f"{datetime.now()}: mapping particle IDs to their indices.")
pids_idx = {pid: i for i, pid in enumerate(pids)}
if verbose:
print(f"{datetime.now()}: mapping FoF HIDs to their array indices.")
# Unassigned particle IDs are assigned a halo ID of 0. Same as PHEW.
fof_hids = numpy.zeros(pids.size, dtype=numpy.int32)
for i in trange(fof.shape[0]) if verbose else range(fof.shape[0]):
hid, pid = fof[i]
fof_hids[pids_idx[pid]] = hid
fout = paths.fof_membership(nsim, sorted=True)
if verbose:
print(f"Saving the sorted data to ... `{fout}`")
numpy.save(fout, fof_hids)
def main(nsim, verbose=True):
copy_membership(nsim, verbose=verbose)
copy_catalogue(nsim, verbose=verbose)
sort_fofid(nsim, verbose=verbose)
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("--simname", type=str, default="csiborg",
choices=["csiborg", "quijote"],
help="Simulation name")
parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="Indices of simulations to cross. If `-1` processes all simulations.") # noqa
args = parser.parse_args()
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsims = get_nsims(args, paths)
comm = MPI.COMM_WORLD
work_delegation(main, nsims, comm)

151
scripts/pre_dumppart.py
View File

@ -12,12 +12,12 @@
# with this program; if not, write to the Free Software Foundation, Inc., # with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
""" """
Script to load in the simulation particles, load them by their clump ID and Script to load in the simulation particles, sort them by their FoF halo ID and
dump into a HDF5 file. Stores the first and last index of each clump in the dump into a HDF5 file. Stores the first and last index of each halo in the
particle array. This can be used for fast slicing of the array to acces particle array. This can be used for fast slicing of the array to acces
particles of a single clump. particles of a single clump.
""" """
from argparse import ArgumentParser
from datetime import datetime from datetime import datetime
from gc import collect from gc import collect
@ -25,8 +25,11 @@ import h5py
import numba import numba
import numpy import numpy
from mpi4py import MPI from mpi4py import MPI
from taskmaster import work_delegation
from tqdm import trange from tqdm import trange
from utils import get_nsims
try: try:
import csiborgtools import csiborgtools
except ModuleNotFoundError: except ModuleNotFoundError:
@ -35,80 +38,79 @@ except ModuleNotFoundError:
sys.path.append("../") sys.path.append("../")
import csiborgtools import csiborgtools
from argparse import ArgumentParser
# We set up the MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nproc = comm.Get_size()
# And next parse all the arguments and set up CSiBORG objects
parser = ArgumentParser()
parser.add_argument("--ics", type=int, nargs="+", default=None,
help="IC realisations. If `-1` processes all simulations.")
args = parser.parse_args()
verbose = nproc == 1
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
partreader = csiborgtools.read.ParticleReader(paths)
# Keep "ID" as the last column!
pars_extract = ['x', 'y', 'z', 'vx', 'vy', 'vz', 'M', "ID"]
if args.ics is None or args.ics[0] == -1:
ics = paths.get_ics("csiborg")
else:
ics = args.ics
@numba.jit(nopython=True) @numba.jit(nopython=True)
def minmax_clump(clid, clump_ids, start_loop=0): def minmax_halo(hid, halo_ids, start_loop=0):
""" """
Find the start and end index of a clump in a sorted array of clump IDs. Find the start and end index of a halo in a sorted array of halo IDs.
This is much faster than using `numpy.where` and then `numpy.min` and This is much faster than using `numpy.where` and then `numpy.min` and
`numpy.max`. `numpy.max`.
""" """
start = None start = None
end = None end = None
for i in range(start_loop, clump_ids.size): for i in range(start_loop, halo_ids.size):
n = clump_ids[i] n = halo_ids[i]
if n == clid: if n == hid:
if start is None: if start is None:
start = i start = i
end = i end = i
elif n > clid: elif n > hid:
break break
return start, end return start, end
# MPI loop over individual simulations. We read in the particles from RAMSES def main(nsim, simname, verbose):
# files and dump them to a HDF5 file. """
jobs = csiborgtools.fits.split_jobs(len(ics), nproc)[rank] Read in the snapshot particles, sort them by their FoF halo ID and dump
for i in jobs: into a HDF5 file. Stores the first and last index of each halo in the
nsim = ics[i] particle array for fast slicing of the array to acces particles of a single
halo.
Parameters
----------
nsim : int
IC realisation index.
simname : str
Simulation name.
verbose : bool
Verbosity flag.
Returns
-------
None
"""
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
partreader = csiborgtools.read.ParticleReader(paths)
if simname == "quijote":
raise NotImplementedError("Not implemented for Quijote yet.")
# Keep "ID" as the last column!
pars_extract = ['x', 'y', 'z', 'vx', 'vy', 'vz', 'M', "ID"]
nsnap = max(paths.get_snapshots(nsim)) nsnap = max(paths.get_snapshots(nsim))
fname = paths.particles(nsim) fname = paths.particles(nsim)
# We first read in the clump IDs of the particles and infer the sorting. # We first read in the halo IDs of the particles and infer the sorting.
# Right away we dump the clump IDs to a HDF5 file and clear up memory. # Right away we dump the halo IDs to a HDF5 file and clear up memory.
print(f"{datetime.now()}: rank {rank} loading particles {nsim}.", if verbose:
flush=True) print(f"{datetime.now()}: loading particles {nsim}.", flush=True)
part_cids = partreader.read_clumpid(nsnap, nsim, verbose=verbose) part_hids = partreader.read_fof_hids(nsim)
sort_indxs = numpy.argsort(part_cids).astype(numpy.int32) sort_indxs = numpy.argsort(part_hids).astype(numpy.int32)
part_cids = part_cids[sort_indxs] part_hids = part_hids[sort_indxs]
with h5py.File(fname, "w") as f: with h5py.File(fname, "w") as f:
f.create_dataset("clump_ids", data=part_cids) f.create_dataset("halo_ids", data=part_hids)
f.close() f.close()
del part_cids del part_hids
collect() collect()
# Next we read in the particles and sort them by their clump ID. # Next we read in the particles and sort them by their halo ID.
# We cannot directly read this as an unstructured array because the float32 # We cannot directly read this as an unstructured array because the float32
# precision is insufficient to capture the clump IDs. # precision is insufficient to capture the halo IDs.
parts, pids = partreader.read_particle( parts, pids = partreader.read_particle(
nsnap, nsim, pars_extract, return_structured=False, verbose=verbose) nsnap, nsim, pars_extract, return_structured=False, verbose=verbose)
# Now we in two steps save the particles and particle IDs. # Now we in two steps save the particles and particle IDs.
print(f"{datetime.now()}: rank {rank} dumping particles from {nsim}.", if verbose:
flush=True) print(f"{datetime.now()}: dumping particles from {nsim}.", flush=True)
parts = parts[sort_indxs] parts = parts[sort_indxs]
pids = pids[sort_indxs] pids = pids[sort_indxs]
del sort_indxs del sort_indxs
@ -126,29 +128,48 @@ for i in jobs:
del parts del parts
collect() collect()
print(f"{datetime.now()}: rank {rank} creating clump mapping for {nsim}.", if verbose:
flush=True) print(f"{datetime.now()}: creating halo map for {nsim}.", flush=True)
# Load clump IDs back to memory # Load clump IDs back to memory
with h5py.File(fname, "r") as f: with h5py.File(fname, "r") as f:
part_cids = f["clump_ids"][:] part_hids = f["halo_ids"][:]
# We loop over the unique clump IDs. # We loop over the unique clump IDs.
unique_clump_ids = numpy.unique(part_cids) unique_halo_ids = numpy.unique(part_hids)
clump_map = numpy.full((unique_clump_ids.size, 3), numpy.nan, halo_map = numpy.full((unique_halo_ids.size, 3), numpy.nan,
dtype=numpy.int32) dtype=numpy.int32)
start_loop = 0 start_loop = 0
niters = unique_clump_ids.size niters = unique_halo_ids.size
for i in trange(niters) if verbose else range(niters): for i in trange(niters) if verbose else range(niters):
clid = unique_clump_ids[i] hid = unique_halo_ids[i]
k0, kf = minmax_clump(clid, part_cids, start_loop=start_loop) k0, kf = minmax_halo(hid, part_hids, start_loop=start_loop)
clump_map[i, 0] = clid halo_map[i, 0] = hid
clump_map[i, 1] = k0 halo_map[i, 1] = k0
clump_map[i, 2] = kf halo_map[i, 2] = kf
start_loop = kf start_loop = kf
# We save the mapping to a HDF5 file # We save the mapping to a HDF5 file
with h5py.File(paths.particles(nsim), "r+") as f: with h5py.File(paths.particles(nsim), "r+") as f:
f.create_dataset("clumpmap", data=clump_map) f.create_dataset("halomap", data=halo_map)
f.close() f.close()
del part_cids del part_hids
collect() collect()
if __name__ == "__main__":
# And next parse all the arguments and set up CSiBORG objects
parser = ArgumentParser()
parser.add_argument("--simname", type=str, default="csiborg",
choices=["csiborg", "quijote"],
help="Simulation name")
parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="IC realisations. If `-1` processes all .")
args = parser.parse_args()
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsims = get_nsims(args, paths)
def _main(nsim, verbose=MPI.COMM_WORLD.nproc == 1):
main(nsim, args.simname, verbose=verbose)
work_delegation(_main, nsims, MPI.COMM_WORLD)

80
scripts/pre_sortinit.py
View File

@ -14,7 +14,7 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
""" """
Script to sort the initial snapshot particles according to their final Script to sort the initial snapshot particles according to their final
snapshot ordering, which is sorted by the clump IDs. snapshot ordering, which is sorted by the halo IDs.
""" """
from argparse import ArgumentParser from argparse import ArgumentParser
from datetime import datetime from datetime import datetime
@ -23,6 +23,9 @@ from gc import collect
import h5py import h5py
import numpy import numpy
from mpi4py import MPI from mpi4py import MPI
from taskmaster import work_delegation
from utils import get_nsims
try: try:
import csiborgtools import csiborgtools
@ -33,42 +36,37 @@ except ModuleNotFoundError:
import csiborgtools import csiborgtools
# Get MPI things def _main(nsim, simname, verbose):
comm = MPI.COMM_WORLD """
rank = comm.Get_rank() Sort the initial snapshot particles according to their final snapshot
nproc = comm.Get_size() ordering and dump them into a HDF5 file.
verbose = nproc == 1
# Argument parser Parameters
parser = ArgumentParser() ----------
parser.add_argument("--ics", type=int, nargs="+", default=None, nsim : int
help="IC realisations. If `-1` processes all simulations.") IC realisation index.
args = parser.parse_args() simname : str
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring) Simulation name.
partreader = csiborgtools.read.ParticleReader(paths) verbose : bool
# NOTE: ID has to be the last column. Verbosity flag.
pars_extract = ["x", "y", "z", "M", "ID"] """
if simname == "quijote":
raise NotImplementedError("Quijote not implemented yet.")
if args.ics is None or args.ics[0] == -1: paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
ics = paths.get_ics("csiborg") partreader = csiborgtools.read.ParticleReader(paths)
else:
ics = args.ics
# We loop over simulations. Each simulation is then procesed with MPI, rank 0 if verbose:
# loads the data and broadcasts it to other ranks. print(f"{datetime.now()}: reading and processing simulation {nsim}.",
jobs = csiborgtools.fits.split_jobs(len(ics), nproc)[rank] flush=True)
for i in jobs:
nsim = ics[i]
nsnap = max(paths.get_snapshots(nsim))
print(f"{datetime.now()}: reading and processing simulation {nsim}.",
flush=True)
# We first load the particle IDs in the final snapshot. # We first load the particle IDs in the final snapshot.
pidf = csiborgtools.read.read_h5(paths.particles(nsim)) pidf = csiborgtools.read.read_h5(paths.particles(nsim))
pidf = pidf["particle_ids"] pidf = pidf["particle_ids"]
# Then we load the particles in the initil snapshot and make sure that # Then we load the particles in the initil snapshot and make sure that
# their particle IDs are sorted as in the final snapshot. # their particle IDs are sorted as in the final snapshot. Again, because of
# Again, because of precision this must be read as structured. # precision this must be read as structured.
# NOTE: ID has to be the last column.
pars_extract = ["x", "y", "z", "M", "ID"]
part0, pid0 = partreader.read_particle( part0, pid0 = partreader.read_particle(
1, nsim, pars_extract, return_structured=False, verbose=verbose) 1, nsim, pars_extract, return_structured=False, verbose=verbose)
# First enforce them to already be sorted and then apply reverse # First enforce them to already be sorted and then apply reverse
@ -77,6 +75,26 @@ for i in jobs:
del pid0 del pid0
collect() collect()
part0 = part0[numpy.argsort(numpy.argsort(pidf))] part0 = part0[numpy.argsort(numpy.argsort(pidf))]
print(f"{datetime.now()}: dumping particles for {nsim}.", flush=True) if verbose:
print(f"{datetime.now()}: dumping particles for {nsim}.", flush=True)
with h5py.File(paths.initmatch(nsim, "particles"), "w") as f: with h5py.File(paths.initmatch(nsim, "particles"), "w") as f:
f.create_dataset("particles", data=part0) f.create_dataset("particles", data=part0)
if __name__ == "__main__":
# Argument parser
parser = ArgumentParser()
parser.add_argument("--simname", type=str, default="csiborg",
choices=["csiborg", "quijote"],
help="Simulation name")
parser.add_argument("--nsims", type=int, nargs="+", default=None,
help="IC realisations. If `-1` processes all.")
args = parser.parse_args()
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsims = get_nsims(args, paths)
def main(nsim):
_main(nsim, args.simname, MPI.COMM_WORLD.size == 1)
work_delegation(main, nsims, MPI.COMM_WORLD)

4
scripts/utils.py
View File

@ -81,7 +81,7 @@ def read_single_catalogue(args, config, nsim, run, rmax, paths, nobs=None):
Returns Returns
------- -------
cat : csiborgtools.read.HaloCatalogue or csiborgtools.read.QuijoteHaloCatalogue # noqa cat : csiborgtools.read.CSiBORGHaloCatalogue or csiborgtools.read.QuijoteHaloCatalogue # noqa
Halo catalogue with selection criteria applied. Halo catalogue with selection criteria applied.
""" """
selection = config.get(run, None) selection = config.get(run, None)
@ -89,7 +89,7 @@ def read_single_catalogue(args, config, nsim, run, rmax, paths, nobs=None):
raise KeyError(f"No configuration for run {run}.") raise KeyError(f"No configuration for run {run}.")
# We first read the full catalogue without applying any bounds. # We first read the full catalogue without applying any bounds.
if args.simname == "csiborg": if args.simname == "csiborg":
cat = csiborgtools.read.HaloCatalogue(nsim, paths) cat = csiborgtools.read.CSiBORGHaloCatalogue(nsim, paths)
else: else:
cat = csiborgtools.read.QuijoteHaloCatalogue(nsim, paths, nsnap=4) cat = csiborgtools.read.QuijoteHaloCatalogue(nsim, paths, nsnap=4)
if nobs is not None: if nobs is not None:

90
scripts_plots/plot_data.py
View File

@ -46,11 +46,11 @@ def open_csiborg(nsim):
Returns Returns
------- -------
cat : csiborgtools.read.HaloCatalogue cat : csiborgtools.read.CSiBORGHaloCatalogue
""" """
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring) paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
bounds = {"totpartmass": (None, None), "dist": (0, 155/0.705)} bounds = {"totpartmass": (None, None), "dist": (0, 155/0.705)}
return csiborgtools.read.HaloCatalogue(nsim, paths, bounds=bounds) return csiborgtools.read.CSiBORGHaloCatalogue(nsim, paths, bounds=bounds)
def open_quijote(nsim, nobs=None): def open_quijote(nsim, nobs=None):
@ -118,7 +118,7 @@ def plot_mass_vs_ncells(nsim, pdf=False):
def plot_hmf(pdf=False): def plot_hmf(pdf=False):
""" """
Plot the (ultimate paretn) halo mass function of CSiBORG and Quijote. Plot the FoF halo mass function of CSiBORG and Quijote.
Parameters Parameters
---------- ----------
@ -132,7 +132,8 @@ def plot_hmf(pdf=False):
print("Plotting the HMF...", flush=True) print("Plotting the HMF...", flush=True)
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring) paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
csiborg_nsims = paths.get_ics("csiborg") # csiborg_nsims = paths.get_ics("csiborg")
csiborg_nsims = [7444]
print("Loading CSiBORG halo counts.", flush=True) print("Loading CSiBORG halo counts.", flush=True)
for i, nsim in enumerate(tqdm(csiborg_nsims)): for i, nsim in enumerate(tqdm(csiborg_nsims)):
data = numpy.load(paths.halo_counts("csiborg", nsim)) data = numpy.load(paths.halo_counts("csiborg", nsim))
@ -141,6 +142,8 @@ def plot_hmf(pdf=False):
csiborg_counts = numpy.full((len(csiborg_nsims), len(bins) - 1), csiborg_counts = numpy.full((len(csiborg_nsims), len(bins) - 1),
numpy.nan, dtype=numpy.float32) numpy.nan, dtype=numpy.float32)
csiborg_counts[i, :] = data["counts"] csiborg_counts[i, :] = data["counts"]
print(data["counts"])
print(csiborg_counts)
csiborg_counts /= numpy.diff(bins).reshape(1, -1) csiborg_counts /= numpy.diff(bins).reshape(1, -1)
print("Loading Quijote halo counts.", flush=True) print("Loading Quijote halo counts.", flush=True)
@ -211,6 +214,76 @@ def plot_hmf(pdf=False):
plt.close() plt.close()
def plot_hmf_quijote_full(pdf=False):
"""
Plot the FoF halo mass function of Quijote full run.
Parameters
----------
pdf : bool, optional
Whether to save the figure as a PDF file.
Returns
-------
None
"""
print("Plotting the HMF...", flush=True)
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
print("Loading Quijote halo counts.", flush=True)
quijote_nsims = paths.get_ics("quijote")
for i, nsim in enumerate(tqdm(quijote_nsims)):
data = numpy.load(paths.halo_counts("quijote_full", nsim))
if i == 0:
bins = data["bins"]
counts = numpy.full((len(quijote_nsims), len(bins) - 1), numpy.nan,
dtype=numpy.float32)
counts[i, :] = data["counts"]
counts /= numpy.diff(bins).reshape(1, -1)
counts /= 1000**3
x = 10**(0.5 * (bins[1:] + bins[:-1]))
# Edit lower and upper limits
counts[:, x < 10**(12.4)] = numpy.nan
counts[:, x > 4e15] = numpy.nan
with plt.style.context(plt_utils.mplstyle):
cols = plt.rcParams["axes.prop_cycle"].by_key()["color"]
fig, ax = plt.subplots(nrows=2, sharex=True,
figsize=(3.5, 2.625 * 1.25),
gridspec_kw={"height_ratios": [1, 0.65]})
fig.subplots_adjust(hspace=0, wspace=0)
# Upper panel data
mean = numpy.mean(counts, axis=0)
std = numpy.std(counts, axis=0)
ax[0].plot(x, mean)
ax[0].fill_between(x, mean - std, mean + std, alpha=0.5)
# Lower panel data
for i in range(counts.shape[0]):
ax[1].plot(x, counts[i, :] / mean, c=cols[0])
# Labels and accesories
ax[1].axhline(1, color="k", ls=plt.rcParams["lines.linestyle"],
lw=0.5 * plt.rcParams["lines.linewidth"], zorder=0)
ax[0].set_ylabel(r"$\frac{\mathrm{d}^2 n}{\mathrm{d}\log M_{\rm h} \mathrm{d} V}~[\mathrm{dex}^{-1} (\mathrm{Mpc / h})^{-3}]$", # noqa
fontsize="small")
ax[1].set_xlabel(r"$M_{\rm h}$ [$M_\odot$]")
ax[1].set_ylabel(r"$\mathrm{HMF} / \langle \mathrm{HMF} \rangle$",
fontsize="small")
ax[0].set_xscale("log")
ax[0].set_yscale("log")
ax[0].legend()
fig.tight_layout(h_pad=0, w_pad=0)
for ext in ["png"] if pdf is False else ["png", "pdf"]:
fout = join(plt_utils.fout, f"hmf_quijote_full.{ext}")
print(f"Saving to `{fout}`.")
fig.savefig(fout, dpi=plt_utils.dpi, bbox_inches="tight")
plt.close()
def load_field(kind, nsim, grid, MAS, in_rsp=False, smooth_scale=None): def load_field(kind, nsim, grid, MAS, in_rsp=False, smooth_scale=None):
r""" r"""
Load a single field. Load a single field.
@ -516,7 +589,8 @@ def plot_sky_distribution(field, nsim, grid, nside, smooth_scale=None,
if plot_halos is not None: if plot_halos is not None:
bounds = {"dist": (dmin, dmax), bounds = {"dist": (dmin, dmax),
"totpartmass": (plot_halos, None)} "totpartmass": (plot_halos, None)}
cat = csiborgtools.read.HaloCatalogue(nsim, paths, bounds=bounds) cat = csiborgtools.read.CSiBORGHaloCatalogue(nsim, paths,
bounds=bounds)
X = cat.position(cartesian=False) X = cat.position(cartesian=False)
healpy.projscatter(numpy.deg2rad(X[:, 2] + 90), healpy.projscatter(numpy.deg2rad(X[:, 2] + 90),
numpy.deg2rad(X[:, 1]), numpy.deg2rad(X[:, 1]),
@ -560,6 +634,9 @@ if __name__ == "__main__":
if False: if False:
plot_hmf(pdf=False) plot_hmf(pdf=False)
if True:
plot_hmf_quijote_full(pdf=False)
if False: if False:
kind = "overdensity" kind = "overdensity"
grid = 1024 grid = 1024
@ -567,7 +644,7 @@ if __name__ == "__main__":
plot_groups=False, dmin=45, dmax=60, plot_groups=False, dmin=45, dmax=60,
plot_halos=5e13, volume_weight=True) plot_halos=5e13, volume_weight=True)
if True: if False:
kind = "overdensity" kind = "overdensity"
grid = 256 grid = 256
smooth_scale = 0 smooth_scale = 0
@ -589,4 +666,3 @@ if __name__ == "__main__":
plt.tight_layout() plt.tight_layout()
plt.savefig("../plots/velocity_distribution.png", dpi=450, plt.savefig("../plots/velocity_distribution.png", dpi=450,
bbox_inches="tight") bbox_inches="tight")

4
scripts_plots/plot_match.py
View File

@ -51,11 +51,11 @@ def open_cat(nsim):
Returns Returns
------- -------
cat : csiborgtools.read.HaloCatalogue cat : csiborgtools.read.CSiBORGHaloCatalogue
""" """
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring) paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
bounds = {"totpartmass": (1e12, None)} bounds = {"totpartmass": (1e12, None)}
return csiborgtools.read.HaloCatalogue(nsim, paths, bounds=bounds) return csiborgtools.read.CSiBORGHaloCatalogue(nsim, paths, bounds=bounds)
def plot_mass_vs_pairoverlap(nsim0, nsimx): def plot_mass_vs_pairoverlap(nsim0, nsimx):