Aquila-Consortium/csiborgtools
0
0
Fork 0
mirror of https://github.com/Richard-Sti/csiborgtools.git synced 2024-12-22 10:58:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Gaussian smoothing of density fields (#33)

Browse source 
* Simplify smoothing support and looping over nonzero

* Simplify comments

* add now()

* add cat length

* add smoothed calculation

* add smoothing

* Add sorting

* Edit what is ignored

* Move notebooks

* Add nonsymmetric smoothed overlap

* Update NB

* Add support for reading in the smoothed overlap

* Switch to the true overlap definition

* Reader of the true overlap

* rem occups

* Import moved to a class

* Move definition

* Edit submission script

* Update to account for the new definition

* backup nb

* Switch back to properly initialising arrays

* Fix addition bug

* Update NB

* Fix little bug

* Update nb
This commit is contained in:
Richard Stiskalek 2023-03-27 09:22:03 +01:00 committed by GitHub
parent 9b524db617
commit 5dd8c668fa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 10587 additions and 441 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -14,7 +14,5 @@ scripts/*.out
build/*
.eggs/*
csiborgtools.egg-info/*
scripts/playground_*
scripts/*.ipynb
Pylians3/*
scripts/plot_correlation.ipynb

574
csiborgtools/match/match.py
View file

@ -16,11 +16,11 @@
import numpy
from scipy.ndimage import gaussian_filter
from tqdm import (tqdm, trange)
from datetime import datetime
from astropy.coordinates import SkyCoord
from numba import jit
from gc import collect
from ..read import (concatenate_clumps, clumps_pos2cell)
from ..read import concatenate_clumps
from ..utils import now
def brute_spatial_separation(c1, c2, angular=False, N=None, verbose=False):
@ -92,23 +92,19 @@ class RealisationsMatcher:
mass associated with a halo.
overlapper_kwargs : dict, optional
Keyword arguments passed to `ParticleOverlapper`.
remove_nooverlap : bool, optional
Whether to remove pairs with exactly zero overlap. By default `True`.
"""
_nmult = None
_dlogmass = None
_mass_kind = None
_overlapper = None
_remove_nooverlap = None
def __init__(self, nmult=1., dlogmass=2., mass_kind="totpartmass",
overlapper_kwargs={}, remove_nooverlap=True):
overlapper_kwargs={}):
self.nmult = nmult
self.dlogmass = dlogmass
self.mass_kind = mass_kind
self._overlapper = ParticleOverlap(**overlapper_kwargs)
self.remove_nooverlap = remove_nooverlap
@property
def nmult(self):
@ -163,23 +159,6 @@ class RealisationsMatcher:
assert isinstance(mass_kind, str)
self._mass_kind = mass_kind
@property
def remove_nooverlap(self):
"""
Whether to remove pairs with exactly zero overlap.
Returns
-------
remove_nooverlap : bool
"""
return self._remove_nooverlap
@remove_nooverlap.setter
def remove_nooverlap(self, remove_nooverlap):
"""Set `remove_nooverlap`."""
assert isinstance(remove_nooverlap, bool)
self._remove_nooverlap = remove_nooverlap
@property
def overlapper(self):
"""
@ -191,61 +170,48 @@ class RealisationsMatcher:
"""
return self._overlapper
@staticmethod
def _cat2clump_mapping(cat_indxs, clump_indxs):
"""
Create a mapping from a catalogue array index to a clump array index.
Parameters
----------
cat_indxs : 1-dimensional array
Clump indices in the catalogue array.
clump_indxs : 1-dimensional array
Clump indices in the clump array.
Returns
-------
mapping : 1-dimensional array
Mapping. The array indices match catalogue array and values are
array positions in the clump array.
"""
mapping = numpy.full(cat_indxs.size, numpy.nan, dtype=int)
__, ind1, ind2 = numpy.intersect1d(clump_indxs, cat_indxs,
return_indices=True)
mapping[ind2] = ind1
return mapping
def cross(self, cat0, catx, overlap=False, verbose=True):
def cross(self, cat0, catx, clumps0, clumpsx, delta_bckg, verbose=True):
r"""
Find all neighbours whose CM separation is less than `nmult` times the
sum of their initial Lagrangian patch sizes. Enforces that the
neighbours' are similar in mass up to `dlogmass` dex.
sum of their initial Lagrangian patch sizes and optionally calculate
their overlap. Enforces that the neighbours' are similar in mass up to
`dlogmass` dex.
Parameters
----------
cat0, catx: :py:class:`csiborgtools.read.HaloCatalogue`
Halo catalogues corresponding to the reference and cross
simulations.
overlap : bool, optional
whether to calculate overlap between clumps in the initial
snapshot. by default `false`. this operation is slow.
cat0 : :py:class:`csiborgtools.read.HaloCatalogue`
Halo catalogue of the reference simulation.
catx : :py:class:`csiborgtools.read.HaloCatalogue`
Halo catalogue of the cross simulation.
clumps0 : list of structured arrays
List of clump structured arrays of the reference simulation, keys
must include `x`, `y`, `z` and `M`. The positions must already be
converted to cell numbers.
clumpsx : list of structured arrays
List of clump structured arrays of the cross simulation, keys must
include `x`, `y`, `z` and `M`. The positions must already be
converted to cell numbers.
delta_bcgk : 3-dimensional array
Summed background density field of the reference and cross
simulations calculated with particles assigned to halos at the
final snapshot. Assumed to only be sampled in cells
:math:`[512, 1536)^3`.
verbose : bool, optional
iterator verbosity flag. by default `true`.
Returns
-------
ref_indxs : 1-dimensional array
Indices of halos in the reference catalogue.
Halo IDs in the reference catalogue.
cross_indxs : 1-dimensional array
Indices of halos in the cross catalogue.
Halo IDs in the cross catalogue.
match_indxs : 1-dimensional array of arrays
Indices of halo counterparts in the cross catalogue.
overlaps : 1-dimensional array of arrays
Overlaps with the cross catalogue.
"""
# Query the KNN
if verbose:
print("{}: querying the KNN.".format(datetime.now()), flush=True)
verbose and print("{}: querying the KNN.".format(now()), flush=True)
match_indxs = radius_neighbours(
catx.knn(select_initial=True), cat0.positions0,
radiusX=cat0["lagpatch"], radiusKNN=catx["lagpatch"],
@ -253,74 +219,125 @@ class RealisationsMatcher:
# Remove neighbours whose mass is too large/small
if self.dlogmass is not None:
for j, indx in enumerate(match_indxs):
for i, indx in enumerate(match_indxs):
# |log(M1 / M2)|
p = self.mass_kind
aratio = numpy.abs(numpy.log10(catx[p][indx] / cat0[p][j]))
match_indxs[j] = match_indxs[j][aratio < self.dlogmass]
aratio = numpy.abs(numpy.log10(catx[p][indx] / cat0[p][i]))
match_indxs[i] = match_indxs[i][aratio < self.dlogmass]
# Min and max cells along each axis for each halo
limkwargs = {"ncells": self.overlapper.inv_clength,
"nshift": self.overlapper.nshift}
mins0, maxs0 = get_clumplims(clumps0, **limkwargs)
minsx, maxsx = get_clumplims(clumpsx, **limkwargs)
# Mapping from a halo index to the list of clumps
hid2clumps0 = {hid: n for n, hid in enumerate(clumps0["ID"])}
hid2clumpsx = {hid: n for n, hid in enumerate(clumpsx["ID"])}
# Initialise the array outside in case `overlap` is `False`
cross = [numpy.asanyarray([], dtype=numpy.float32)] * match_indxs.size
if overlap:
if verbose:
print("Loading the clump particles", flush=True)
with open(cat0.paths.clump0_path(cat0.n_sim), "rb") as f:
clumps0 = numpy.load(f, allow_pickle=True)
with open(catx.paths.clump0_path(catx.n_sim), 'rb') as f:
clumpsx = numpy.load(f, allow_pickle=True)
# Loop only over halos that have neighbours
iters = numpy.arange(len(cat0))[[x.size > 0 for x in match_indxs]]
for i in tqdm(iters) if verbose else iters:
match0 = hid2clumps0[cat0["index"][i]]
# The clump, its mass and mins & maxs
cl0 = clumps0["clump"][match0]
mass0 = numpy.sum(cl0['M'])
mins0_current, maxs0_current = mins0[match0], maxs0[match0]
# Convert 3D positions to particle IDs
clumps_pos2cell(clumps0, self.overlapper)
clumps_pos2cell(clumpsx, self.overlapper)
# Preallocate arrays to store overlap information
_cross = numpy.full(match_indxs[i].size, numpy.nan,
dtype=numpy.float32)
# Loop over matches of this halo from the other simulation
for j, ind in enumerate(match_indxs[i]):
matchx = hid2clumpsx[catx["index"][ind]]
clx = clumpsx["clump"][matchx]
_cross[j] = self.overlapper(
cl0, clx, delta_bckg, mins0_current, maxs0_current,
minsx[matchx], maxsx[matchx], mass1=mass0,
mass2=numpy.sum(clx['M']))
cross[i] = _cross
# Calculate the particle field
if verbose:
print("Creating and smoothing the crossed field.", flush=True)
delta_bckg0 = self.overlapper.make_background_delta(
clumps0, to_smooth=False)
delta_bckgx = self.overlapper.make_background_delta(
clumpsx, to_smooth=False)
# Remove matches with exactly 0 overlap
mask = cross[i] > 0
match_indxs[i] = match_indxs[i][mask]
cross[i] = cross[i][mask]
# Min and max cells along each axis for each halo
limkwargs = {"ncells": self.overlapper.inv_clength,
"nshift": self.overlapper.nshift}
mins0, maxs0 = get_clumplims(clumps0, **limkwargs)
minsx, maxsx = get_clumplims(clumpsx, **limkwargs)
# Mapping from a catalogue halo index to the list of clumps
cat2clumps0 = self._cat2clump_mapping(cat0["index"], clumps0["ID"])
cat2clumpsx = self._cat2clump_mapping(catx["index"], clumpsx["ID"])
# Loop only over halos that have neighbours
wneigbours = numpy.where([ii.size > 0 for ii in match_indxs])[0]
for k in tqdm(wneigbours) if verbose else wneigbours:
match0 = cat2clumps0[k] # Clump pos matching this halo
# The clump, its mass and mins & maxs
cl0 = clumps0["clump"][match0]
mass0 = numpy.sum(cl0['M'])
mins0_current, maxs0_current = mins0[match0], maxs0[match0]
# Array to store overlaps of this halo
crosses = numpy.full(match_indxs[k].size, numpy.nan,
numpy.float32)
# Loop over matches of this halo from the other simulation
for ii, ind in enumerate(match_indxs[k]):
matchx = cat2clumpsx[ind] # Clump pos matching this halo
clx = clumpsx["clump"][matchx]
crosses[ii] = self.overlapper(
cl0, clx, delta_bckg0, delta_bckgx, mins0_current,
maxs0_current, minsx[matchx], maxsx[matchx],
mass1=mass0, mass2=numpy.sum(clx['M']))
cross[k] = crosses
# Optionally remove matches with exactly 0 overlap
if self.remove_nooverlap:
mask = cross[k] > 0
match_indxs[k] = match_indxs[k][mask]
cross[k] = cross[k][mask]
# Sort the matches by overlap
ordering = numpy.argsort(cross[i])[::-1]
match_indxs[i] = match_indxs[i][ordering]
cross[i] = cross[i][ordering]
cross = numpy.asanyarray(cross, dtype=object)
return cat0["index"], catx["index"], match_indxs, cross
def smoothed_cross(self, clumps0, clumpsx, delta_bckg, ref_indxs,
cross_indxs, match_indxs, smooth_kwargs, verbose=True):
r"""
Calculate the smoothed overlaps for pair previously identified via
`self.cross(...)` to have a non-zero overlap.
Parameters
----------
clumps0 : list of structured arrays
List of clump structured arrays of the reference simulation, keys
must include `x`, `y`, `z` and `M`. The positions must already be
converted to cell numbers.
clumpsx : list of structured arrays
List of clump structured arrays of the cross simulation, keys must
include `x`, `y`, `z` and `M`. The positions must already be
converted to cell numbers.
delta_bcgk : 3-dimensional array
Smoothed summed background density field of the reference and cross
simulations calculated with particles assigned to halos at the
final snapshot. Assumed to only be sampled in cells
:math:`[512, 1536)^3`.
ref_indxs : 1-dimensional array
Halo IDs in the reference catalogue.
cross_indxs : 1-dimensional array
Halo IDs in the cross catalogue.
match_indxs : 1-dimensional array of arrays
Indices of halo counterparts in the cross catalogue.
smooth_kwargs : kwargs
Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`.
verbose : bool, optional
Iterator verbosity flag. By default `True`.
Returns
-------
overlaps : 1-dimensional array of arrays
"""
# Min and max cells along each axis for each halo
limkwargs = {"ncells": self.overlapper.inv_clength,
"nshift": self.overlapper.nshift}
mins0, maxs0 = get_clumplims(clumps0, **limkwargs)
minsx, maxsx = get_clumplims(clumpsx, **limkwargs)
hid2clumps0 = {hid: n for n, hid in enumerate(clumps0["ID"])}
hid2clumpsx = {hid: n for n, hid in enumerate(clumpsx["ID"])}
# Preallocate arrays to store the overlap information
cross = [numpy.asanyarray([], dtype=numpy.float32)] * match_indxs.size
for i, ref_ind in enumerate(tqdm(ref_indxs) if verbose else ref_indxs):
match0 = hid2clumps0[ref_ind]
# The reference clump, its mass and mins & maxs
cl0 = clumps0["clump"][match0]
mins0_current, maxs0_current = mins0[match0], maxs0[match0]
# Preallocate
nmatches = match_indxs[i].size
_cross = numpy.full(nmatches, numpy.nan, numpy.float32)
for j, match_ind in enumerate(match_indxs[i]):
matchx = hid2clumpsx[cross_indxs[match_ind]]
clx = clumpsx["clump"][matchx]
_cross[j] = self.overlapper(
cl0, clx, delta_bckg, mins0_current,
maxs0_current, minsx[matchx], maxsx[matchx],
smooth_kwargs=smooth_kwargs)
cross[i] = _cross
return numpy.asanyarray(cross, dtype=object)
###############################################################################
# Matching statistics #
@ -360,72 +377,21 @@ def cosine_similarity(x, y):
class ParticleOverlap:
r"""
Class to calculate overlap between two halos from different simulations.
The density field calculation is based on the nearest grid position
particle assignment scheme, with optional additional Gaussian smoothing.
Parameters
----------
nshift : int, optional
Number of cells by which to shift the subbox from the outside-most
cell containing a particle. By default 5.
smooth_scale : float or integer, optional
Optional Gaussian smoothing scale to by applied to the fields. By
default no smoothing is applied. Otherwise the scale is to be
specified in the number of cells (i.e. in units of `self.cellsize`).
Class to calculate halo overlaps. The density field calculation is based on
the nearest grid position particle assignment scheme, with optional
Gaussian smoothing.
"""
_inv_clength = None
_smooth_scale = None
_clength = None
_nshift = None
def __init__(self, smooth_scale=None, nshift=5):
def __init__(self):
# Inverse cell length in box units. By default :math:`2^11`, which
# matches the initial RAMSES grid resolution.
self.inv_clength = 2**11
self.smooth_scale = smooth_scale
self.nshift = nshift
@property
def inv_clength(self):
"""
Inverse cell length.
Returns
-------
inv_clength : float
"""
return self._inv_clength
@inv_clength.setter
def inv_clength(self, inv_clength):
"""Sets `inv_clength`."""
assert inv_clength > 0, "`inv_clength` must be positive."
assert isinstance(inv_clength, int), "`inv_clength` must be integer."
self._inv_clength = int(inv_clength)
# Also set the inverse and number of cells
self.nshift = 5 # Hardcode this too to force consistency
self._clength = 1 / self.inv_clength
@property
def smooth_scale(self):
"""
The smoothing scale in units of `self.cellsize`. If not set `None`.
Returns
-------
smooth_scale : int or float
"""
return self._smooth_scale
@smooth_scale.setter
def smooth_scale(self, smooth_scale):
"""Sets `smooth_scale`."""
if smooth_scale is None:
self._smooth_scale = None
else:
assert smooth_scale > 0
self._smooth_scale = smooth_scale
def pos2cell(self, pos):
"""
Convert position to cell number. If `pos` is in
@ -435,6 +401,7 @@ class ParticleOverlap:
Parameters
----------
pos : 1-dimensional array
Array of positions along an axis in the box.
Returns
-------
@ -445,18 +412,55 @@ class ParticleOverlap:
return pos
return numpy.floor(pos * self.inv_clength).astype(int)
def make_background_delta(self, clumps, to_smooth=True):
def clumps_pos2cell(self, clumps):
"""
Convert clump positions directly to cell IDs. Useful to speed up
subsequent calculations. Overwrites the passed in arrays.
Parameters
----------
clumps : array of arrays
Array of clump structured arrays whose `x`, `y`, `z` keys will be
converted.
Returns
-------
None
"""
# Check if clumps are probably already in cells
if any(clumps[0][0].dtype[p].char in numpy.typecodes["AllInteger"]
for p in ('x', 'y', 'z')):
raise ValueError("Positions appear to already be converted cells.")
# Get the new dtype that replaces float for int for positions
names = clumps[0][0].dtype.names # Take the first one, doesn't matter
formats = [descr[1] for descr in clumps[0][0].dtype.descr]
for i in range(len(names)):
if names[i] in ('x', 'y', 'z'):
formats[i] = numpy.int32
dtype = numpy.dtype({"names": names, "formats": formats})
# Loop switch positions for cells IDs and change dtype
for n in range(clumps.size):
for p in ('x', 'y', 'z'):
clumps[n][0][p] = self.pos2cell(clumps[n][0][p])
clumps[n][0] = clumps[n][0].astype(dtype)
def make_bckg_delta(self, clumps, delta=None):
"""
Calculate a NGP density field of clumps within the central
:math:`1/2^3` region of the simulation.
:math:`1/2^3` region of the simulation. Smoothing must be applied
separately.
Parameters
----------
clumps : list of structured arrays
List of clump structured array, keys must include `x`, `y`, `z`
and `M`.
to_smooth : bool, optional
Explicit control over whether to smooth. By default `True`.
delta : 3-dimensional array, optional
Array to store the density field in. If `None` a new array is
created.
Returns
-------
@ -480,30 +484,34 @@ class ParticleOverlap:
cells = [c[mask] for c in cells]
mass = mass[mask]
# Preallocate and fill the array
delta = numpy.zeros((ncells,) * 3, dtype=numpy.float32)
# Prepare the density field or check it is of the right shape
if delta is None:
delta = numpy.zeros((ncells,) * 3, dtype=numpy.float32)
else:
assert ((delta.shape == (ncells,) * 3)
& (delta.dtype == numpy.float32))
fill_delta(delta, *cells, *(cellmin,) * 3, mass)
if to_smooth and self.smooth_scale is not None:
gaussian_filter(delta, self.smooth_scale, output=delta)
return delta
def make_delta(self, clump, mins=None, maxs=None, subbox=False,
to_smooth=True):
smooth_kwargs=None):
"""
Calculate a NGP density field of a halo on a cubic grid.
Calculate a NGP density field of a halo on a cubic grid. Optionally can
be smoothed with a Gaussian kernel.
Parameters
----------
clump: structurered arrays
clump : structurered arrays
Clump structured array, keys must include `x`, `y`, `z` and `M`.
mins, maxs : 1-dimensional arrays of shape `(3,)`
Minimun and maximum cell numbers along each dimension.
subbox : bool, optional
Whether to calculate the density field on a grid strictly enclosing
the clump.
to_smooth : bool, optional
Explicit control over whether to smooth. By default `True`.
smooth_kwargs : kwargs, optional
Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`.
If `None` no smoothing is applied.
Returns
-------
@ -534,15 +542,15 @@ class ParticleOverlap:
delta = numpy.zeros((ncells,) * 3, dtype=numpy.float32)
fill_delta(delta, *cells, *mins, clump['M'])
if to_smooth and self.smooth_scale is not None:
gaussian_filter(delta, self.smooth_scale, output=delta)
if smooth_kwargs is not None:
gaussian_filter(delta, output=delta, **smooth_kwargs)
return delta
def make_deltas(self, clump1, clump2, mins1=None, maxs1=None,
mins2=None, maxs2=None, return_nonzero1=False):
mins2=None, maxs2=None, smooth_kwargs=None):
"""
Calculate a NGP density fields of two halos on a grid that encloses
them both.
them both. Optionally can be smoothed with a Gaussian kernel.
Parameters
----------
@ -555,9 +563,9 @@ class ParticleOverlap:
mins2, maxs2 : 1-dimensional arrays of shape `(3,)`
Minimun and maximum cell numbers along each dimension of `clump2`.
Optional.
return_nonzero1 : bool, optional
Whether to return the indices where the contribution of `clump1` is
non-zero.
smooth_kwargs : kwargs, optional
Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`.
If `None` no smoothing is applied.
Returns
-------
@ -565,9 +573,9 @@ class ParticleOverlap:
Density arrays of `clump1` and `clump2`, respectively.
cellmins : len-3 tuple
Tuple of left-most cell ID in the full box.
nonzero1 : 2-dimensional array
Indices where `delta1` has a non-zero density. If `return_nonzero1`
is `False` return `None` instead.
nonzero : 2-dimensional array
Indices where the lower mass clump has a non-zero density.
Calculated only if no smoothing is applied, otherwise `None`.
"""
xc1, yc1, zc1 = (self.pos2cell(clump1[p]) for p in ('x', 'y', 'z'))
xc2, yc2, zc2 = (self.pos2cell(clump2[p]) for p in ('x', 'y', 'z'))
@ -597,26 +605,37 @@ class ParticleOverlap:
# Preallocate and fill the arrays
delta1 = numpy.zeros((ncells,)*3, dtype=numpy.float32)
delta2 = numpy.zeros((ncells,)*3, dtype=numpy.float32)
if return_nonzero1:
nonzero1 = fill_delta_indxs(
delta1, xc1, yc1, zc1, *cellmins, clump1['M'])
# If no smoothing figure out the nonzero indices of the smaller clump
if smooth_kwargs is None:
if clump1.size > clump2.size:
fill_delta(delta1, xc1, yc1, zc1, *cellmins, clump1['M'])
nonzero = fill_delta_indxs(delta2, xc2, yc2, zc2, *cellmins,
clump2['M'])
else:
nonzero = fill_delta_indxs(delta1, xc1, yc1, zc1, *cellmins,
clump1['M'])
fill_delta(delta2, xc2, yc2, zc2, *cellmins, clump2['M'])
else:
fill_delta(delta1, xc1, yc1, zc1, *cellmins, clump1['M'])
nonzero1 = None
fill_delta(delta2, xc2, yc2, zc2, *cellmins, clump2['M'])
fill_delta(delta2, xc2, yc2, zc2, *cellmins, clump2['M'])
nonzero = None
if self.smooth_scale is not None:
gaussian_filter(delta1, self.smooth_scale, output=delta1)
gaussian_filter(delta2, self.smooth_scale, output=delta2)
if smooth_kwargs is not None:
gaussian_filter(delta1, output=delta1, **smooth_kwargs)
gaussian_filter(delta2, output=delta2, **smooth_kwargs)
return delta1, delta2, cellmins, nonzero
return delta1, delta2, cellmins, nonzero1
def __call__(self, clump1, clump2, delta1_bckg, delta2_bckg,
mins1=None, maxs1=None, mins2=None, maxs2=None,
mass1=None, mass2=None, loop_nonzero=True):
def __call__(self, clump1, clump2, delta_bckg, mins1=None, maxs1=None,
mins2=None, maxs2=None, mass1=None, mass2=None,
smooth_kwargs=None):
"""
Calculate overlap between `clump1` and `clump2`. See
`calculate_overlap(...)` for further information.
`calculate_overlap(...)` for further information. Be careful so that
the background density field is calculated with the same
`smooth_kwargs`. If any smoothing is applied then loops over the full
density fields, otherwise only over the non-zero cells of the lower
mass clump.
Parameters
----------
@ -625,40 +644,39 @@ class ParticleOverlap:
must include `x`, `y`, `z` and `M`.
cellmins : len-3 tuple
Tuple of left-most cell ID in the full box.
delta1_bcgk, delta2_bckg : 3-dimensional arrays
Background density fields of the reference and cross boxes
calculated with particles assigned to halos at the final snapshot.
Assumed to only be sampled in cells :math:`[512, 1536)^3`.
delta_bcgk : 3-dimensional array
Summed background density field of the reference and cross
simulations calculated with particles assigned to halos at the
final snapshot. Assumed to only be sampled in cells
:math:`[512, 1536)^3`.
mins1, maxs1 : 1-dimensional arrays of shape `(3,)`
Minimun and maximum cell numbers along each dimension of `clump1`.
Optional.
mins2, maxs2 : 1-dimensional arrays of shape `(3,)`
Minimun and maximum cell numbers along each dimension of `clump2`.
Optional.
Minimum and maximum cell numbers along each dimension of `clump2`,
optional.
mass1, mass2 : floats, optional
Total mass of `clump1` and `clump2`, respectively. Must be provided
if `loop_nonzero` is `True`.
loop_nonzer : bool, optional
Whether to only loop over cells where `clump1` has non-zero
density. By default `True`.
smooth_kwargs : kwargs, optional
Kwargs to be passed to :py:func:`scipy.ndimage.gaussian_filter`.
If `None` no smoothing is applied.
Returns
-------
overlap : float
"""
delta1, delta2, cellmins, nonzero1 = self.make_deltas(
delta1, delta2, cellmins, nonzero = self.make_deltas(
clump1, clump2, mins1, maxs1, mins2, maxs2,
return_nonzero1=loop_nonzero)
if not loop_nonzero:
return calculate_overlap(delta1, delta2, cellmins,
delta1_bckg, delta2_bckg)
smooth_kwargs=smooth_kwargs)
if smooth_kwargs is not None:
return calculate_overlap(delta1, delta2, cellmins, delta_bckg)
# Calculate masses not given
mass1 = numpy.sum(clump1['M']) if mass1 is None else mass1
mass2 = numpy.sum(clump2['M']) if mass2 is None else mass2
return calculate_overlap_indxs(delta1, delta2, cellmins, delta1_bckg,
delta2_bckg, nonzero1, mass1, mass2)
return calculate_overlap_indxs(delta1, delta2, cellmins, delta_bckg,
nonzero, mass1, mass2)
@jit(nopython=True)
@ -760,29 +778,31 @@ def get_clumplims(clumps, ncells, nshift=None):
@jit(nopython=True)
def calculate_overlap(delta1, delta2, cellmins, delta1_bckg, delta2_bckg):
def calculate_overlap(delta1, delta2, cellmins, delta_bckg):
r"""
Overlap between two clumps whose density fields are evaluated on the
Overlap between two halos whose density fields are evaluated on the
same grid. This is a JIT implementation, hence it is outside of the main
class.
Parameters
----------
delta1, delta2 : 3-dimensional arrays
Clumps density fields.
delta1: 3-dimensional array
Density field of the first halo.
delta2 : 3-dimensional array
Density field of the second halo.
cellmins : len-3 tuple
Tuple of left-most cell ID in the full box.
delta1_bcgk, delta2_bckg : 3-dimensional arrays
Background density fields of the reference and cross boxes calculated
with particles assigned to halos at the final snapshot. Assumed to only
be sampled in cells :math:`[512, 1536)^3`.
delta_bcgk : 3-dimensional array
Summed background density field of the reference and cross simulations
calculated with particles assigned to halos at the final snapshot.
Assumed to only be sampled in cells :math:`[512, 1536)^3`.
Returns
-------
overlap : float
"""
totmass = 0. # Total mass of clump 1 and clump 2
intersect = 0. # Mass of pixels that are non-zero in both clumps
intersect = 0. # Weighted intersecting mass
i0, j0, k0 = cellmins # Unpack things
bckg_offset = 512 # Offset of the background density field
bckg_size = 1024
@ -790,34 +810,27 @@ def calculate_overlap(delta1, delta2, cellmins, delta1_bckg, delta2_bckg):
for i in range(imax):
ii = i0 + i - bckg_offset
flag = 0 <= ii < bckg_size
ishighres = 0 <= ii < bckg_size
for j in range(jmax):
jj = j0 + j - bckg_offset
flag &= 0 <= jj < bckg_size
ishighres &= 0 <= jj < bckg_size
for k in range(kmax):
kk = k0 + k - bckg_offset
flag &= 0 <= kk < bckg_size
ishighres &= 0 <= kk < bckg_size
cell1, cell2 = delta1[i, j, k], delta2[i, j, k]
# If both are zero then skip
if cell1 * cell2 > 0:
if flag:
weight1 = cell1 / delta1_bckg[ii, jj, kk]
weight2 = cell2 / delta2_bckg[ii, jj, kk]
else:
weight1 = 1.
weight2 = 1.
# Average weighted mass in the cell
intersect += 0.5 * (weight1 * cell1 + weight2 * cell2)
totmass += cell1 + cell2
m1, m2 = delta1[i, j, k], delta2[i, j, k]
totmass += m1 + m2
prod = 2 * m1 * m2
if prod > 0: # If both cells are non-zero
bcgk = delta_bckg[ii, jj, kk] if ishighres else m1 + m2
intersect += prod / bcgk if bcgk > 0 else prod / (m1 + m2)
return intersect / (totmass - intersect)
@jit(nopython=True)
def calculate_overlap_indxs(delta1, delta2, cellmins, delta1_bckg, delta2_bckg,
nonzero1, mass1, mass2):
def calculate_overlap_indxs(delta1, delta2, cellmins, delta_bckg, nonzero,
mass1, mass2):
r"""
Overlap between two clumps whose density fields are evaluated on the
same grid and `nonzero1` enumerates the non-zero cells of `delta1. This is
@ -825,17 +838,19 @@ def calculate_overlap_indxs(delta1, delta2, cellmins, delta1_bckg, delta2_bckg,
Parameters
----------
delta1, delta2 : 3-dimensional arrays
Clumps density fields.
delta1: 3-dimensional array
Density field of the first halo.
delta2 : 3-dimensional array
Density field of the second halo.
cellmins : len-3 tuple
Tuple of left-most cell ID in the full box.
delta1_bcgk, delta2_bckg : 3-dimensional arrays
Background density fields of the reference and cross boxes calculated
with particles assigned to halos at the final snapshot. Assumed to only
be sampled in cells :math:`[512, 1536)^3`.
nonzero1 : 2-dimensional array of shape `(n_cells, 3)`
Indices of cells that are non-zero in `delta1`. Expected to be
precomputed from `fill_delta_indxs`.
delta_bcgk : 3-dimensional array
Summed background density field of the reference and cross simulations
calculated with particles assigned to halos at the final snapshot.
Assumed to only be sampled in cells :math:`[512, 1536)^3`.
nonzero : 2-dimensional array of shape `(n_cells, 3)`
Indices of cells that are non-zero of the lower mass clump. Expected to
be precomputed from `fill_delta_indxs`.
mass1, mass2 : floats, optional
Total masses of the two clumps, respectively. Optional. If not provided
calculcated directly from the density field.
@ -844,34 +859,27 @@ def calculate_overlap_indxs(delta1, delta2, cellmins, delta1_bckg, delta2_bckg,
-------
overlap : float
"""
intersect = 0. # Mass of pixels that are non-zero in both clumps
intersect = 0. # Weighted intersecting mass
i0, j0, k0 = cellmins # Unpack cell minimas
bckg_offset = 512 # Offset of the background density field
bckg_size = 1024 # Size of the background density field array
for n in range(nonzero1.shape[0]):
i, j, k = nonzero1[n, :]
cell2 = delta2[i, j, k]
for n in range(nonzero.shape[0]):
i, j, k = nonzero[n, :]
m1, m2 = delta1[i, j, k], delta2[i, j, k]
prod = 2 * m1 * m2
if cell2 > 0: # We already know that cell1 is non-zero
cell1 = delta1[i, j, k] # Now unpack cell1 as well
if prod > 0:
ii = i0 + i - bckg_offset # Indices of this cell in the
jj = j0 + j - bckg_offset # background density field.
kk = k0 + k - bckg_offset
flag = 0 <= ii < bckg_size # Whether this cell is in the high
flag &= 0 <= jj < bckg_size # resolution region for which the
flag &= 0 <= kk < bckg_size # background density is calculated.
ishighres = 0 <= ii < bckg_size # Is this cell is in the high
ishighres &= 0 <= jj < bckg_size # resolution region for which the
ishighres &= 0 <= kk < bckg_size # background field is calculated.
if flag:
weight1 = cell1 / delta1_bckg[ii, jj, kk]
weight2 = cell2 / delta2_bckg[ii, jj, kk]
else:
weight1 = 1.
weight2 = 1.
# Average weighted mass in the cell
intersect += 0.5 * (weight1 * cell1 + weight2 * cell2)
bckg = delta_bckg[ii, jj, kk] if ishighres else m1 + m2
intersect += prod / bckg if bckg > 0 else prod / (m1 + m2)
return intersect / (mass1 + mass2 - intersect)

2
csiborgtools/read/__init__.py
View file

@ -14,7 +14,7 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
from .readsim import (CSiBORGPaths, ParticleReader, read_mmain, read_initcm, halfwidth_select) # noqa
from .make_cat import (HaloCatalogue, concatenate_clumps, clumps_pos2cell) # noqa
from .make_cat import (HaloCatalogue, concatenate_clumps) # noqa
from .readobs import (PlanckClusters, MCXCClusters, TwoMPPGalaxies, # noqa
TwoMPPGroups, SDSS) # noqa
from .outsim import (dump_split, combine_splits, make_ascii_powmes) # noqa

41
csiborgtools/read/make_cat.py
View file

@ -340,6 +340,9 @@ class HaloCatalogue:
raise RuntimeError("Initial positions are not set!")
return self._data[key]
def __len__(self):
return self.data.size
def concatenate_clumps(clumps):
"""
@ -377,41 +380,3 @@ def concatenate_clumps(clumps):
start = end
return particles
def clumps_pos2cell(clumps, overlapper):
"""
Convert clump positions directly to cell IDs. Useful to speed up subsequent
calculations. Overwrites the passed in arrays.
Parameters
----------
clumps : array of arrays
Array of clump structured arrays whose `x`, `y`, `z` keys will be
converted.
overlapper : py:class:`csiborgtools.match.ParticleOverlapper`
`ParticleOverlapper` handling the cell assignment.
Returns
-------
None
"""
# Check if clumps are probably already in cells
if any(clumps[0][0].dtype[p].char in numpy.typecodes["AllInteger"]
for p in ('x', 'y', 'z')):
raise ValueError("Positions appear to already be converted cells.")
# Get the new dtype that replaces float for int for positions
names = clumps[0][0].dtype.names # Take the first one, doesn't matter
formats = [descr[1] for descr in clumps[0][0].dtype.descr]
for i in range(len(names)):
if names[i] in ('x', 'y', 'z'):
formats[i] = numpy.int32
dtype = numpy.dtype({"names": names, "formats": formats})
# Loop switch positions for cells IDs and change dtype
for n in range(clumps.size):
for p in ('x', 'y', 'z'):
clumps[n][0][p] = overlapper.pos2cell(clumps[n][0][p])
clumps[n][0] = clumps[n][0].astype(dtype)

157
csiborgtools/read/summaries.py
View file

@ -213,25 +213,29 @@ class PairOverlap:
.format(cat0.n_sim, catx.n_sim))
# We can set catalogues already now even if inverted
data = numpy.load(fpath, allow_pickle=True)
d = numpy.load(fpath, allow_pickle=True)
ngp_overlap = d["ngp_overlap"]
smoothed_overlap = d["smoothed_overlap"]
match_indxs = d["match_indxs"]
if is_inverted:
inv_match_indxs, inv_overlap = self._invert_match(
data["match_indxs"], data["overlap"],
data["cross_indxs"].size,)
self._data = {
"index": data["cross_indxs"],
"match_indxs": inv_match_indxs,
"overlap": inv_overlap}
indxs = d["cross_indxs"]
# Invert the matches
match_indxs, ngp_overlap, smoothed_overlap = self._invert_match(
match_indxs, ngp_overlap, smoothed_overlap, indxs.size,)
else:
self._data = {
"index": data["ref_indxs"],
"match_indxs": data["match_indxs"],
"overlap": data["overlap"]}
indxs = d["ref_indxs"]
self._data = {
"index": indxs,
"match_indxs": match_indxs,
"ngp_overlap": ngp_overlap,
"smoothed_overlap": smoothed_overlap,
}
self._make_refmask(min_mass, max_dist)
@staticmethod
def _invert_match(match_indxs, overlap, cross_size):
def _invert_match(match_indxs, ngp_overlap, smoothed_overlap, cross_size):
"""
Invert reference and cross matching, possible since the overlap
definition is symmetric.
@ -241,9 +245,12 @@ class PairOverlap:
match_indxs : array of 1-dimensional arrays
Indices of halos from the original cross catalogue matched to the
reference catalogue.
overlap : array of 1-dimensional arrays
Pair overlap of halos between the originla reference and cross
ngp_overlap : array of 1-dimensional arrays
NGP pair overlap of halos between the original reference and cross
simulations.
smoothed_overlap : array of 1-dimensional arrays
Smoothed pair overlap of halos between the original reference and
cross simulations.
cross_size : int
The size of the cross catalogue.
@ -251,35 +258,46 @@ class PairOverlap:
-------
inv_match_indxs : array of 1-dimensional arrays
The inverted match indices.
ind_overlap : array of 1-dimensional arrays
The corresponding overlaps to `inv_match_indxs`.
ind_ngp_overlap : array of 1-dimensional arrays
The corresponding NGP overlaps to `inv_match_indxs`.
ind_smoothed_overlap : array of 1-dimensional arrays
The corresponding smoothed overlaps to `inv_match_indxs`.
"""
# 1. Invert the match. Each reference halo has a list of counterparts
# so loop over those to each counterpart assign a reference halo
# and at the same time also add the overlaps
inv_match_indxs = [[] for __ in range(cross_size)]
inv_overlap = [[] for __ in range(cross_size)]
inv_ngp_overlap = [[] for __ in range(cross_size)]
inv_smoothed_overlap = [[] for __ in range(cross_size)]
for ref_id in range(match_indxs.size):
for cross_id, cross in zip(match_indxs[ref_id], overlap[ref_id]):
for cross_id, ngp_cross, smoothed_cross in zip(match_indxs[ref_id],
ngp_overlap[ref_id],
smoothed_overlap[ref_id]): # noqa
inv_match_indxs[cross_id].append(ref_id)
inv_overlap[cross_id].append(cross)
inv_ngp_overlap[cross_id].append(ngp_cross)
inv_smoothed_overlap[cross_id].append(smoothed_cross)
# 2. Convert the cross matches and overlaps to proper numpy arrays
# and ensure that the overlaps are ordered.
for n in range(len(inv_match_indxs)):
inv_match_indxs[n] = numpy.asanyarray(inv_match_indxs[n],
dtype=numpy.int32)
inv_overlap[n] = numpy.asanyarray(inv_overlap[n],
dtype=numpy.float32)
inv_ngp_overlap[n] = numpy.asanyarray(inv_ngp_overlap[n],
dtype=numpy.float32)
inv_smoothed_overlap[n] = numpy.asanyarray(inv_smoothed_overlap[n],
dtype=numpy.float32)
ordering = numpy.argsort(inv_overlap[n])[::-1]
ordering = numpy.argsort(inv_ngp_overlap[n])[::-1]
inv_match_indxs[n] = inv_match_indxs[n][ordering]
inv_overlap[n] = inv_overlap[n][ordering]
inv_ngp_overlap[n] = inv_ngp_overlap[n][ordering]
inv_smoothed_overlap[n] = inv_smoothed_overlap[n][ordering]
inv_match_indxs = numpy.asarray(inv_match_indxs, dtype=object)
inv_overlap = numpy.asarray(inv_overlap, dtype=object)
inv_ngp_overlap = numpy.asarray(inv_ngp_overlap, dtype=object)
inv_smoothed_overlap = numpy.asarray(inv_smoothed_overlap,
dtype=object)
return inv_match_indxs, inv_overlap
return inv_match_indxs, inv_ngp_overlap, inv_smoothed_overlap
def _make_refmask(self, min_mass, max_dist):
r"""
@ -304,34 +322,63 @@ class PairOverlap:
m = ((self.cat0()["totpartmass"] > min_mass)
& (self.cat0()["dist"] < max_dist))
# Now remove indices that are below this cut
self._data["index"] = self._data["index"][m]
self._data["match_indxs"] = self._data["match_indxs"][m]
self._data["overlap"] = self._data["overlap"][m]
for p in ("index", "match_indxs", "ngp_overlap", "smoothed_overlap"):
self._data[p] = self._data[p][m]
self._data["refmask"] = m
def summed_overlap(self):
def overlap(self, from_smoothed):
"""
Pair overlap of matched halos between the reference and cross
simulations.
Parameters
----------
from_smoothed : bool
Whether to use the smoothed overlap.
Returns
-------
overlap : 1-dimensional array of arrays
"""
if from_smoothed:
return self["smoothed_overlap"]
return self["ngp_overlap"]
def summed_overlap(self, from_smoothed):
"""
Summed overlap of each halo in the reference simulation with the cross
simulation.
Parameters
----------
from_smoothed : bool
Whether to use the smoothed overlap or not.
Returns
-------
summed_overlap : 1-dimensional array of shape `(nhalos, )`
"""
return numpy.array([numpy.sum(cross) for cross in self["overlap"]])
overlap = self.overlap(from_smoothed)
return numpy.array([numpy.sum(cross)for cross in overlap])
def prob_nomatch(self):
def prob_nomatch(self, from_smoothed):
"""
Probability of no match for each halo in the reference simulation with
the cross simulation. Defined as a product of 1 - overlap with other
halos.
Parameters
----------
from_smoothed : bool
Whether to use the smoothed overlap or not.
Returns
-------
prob_nomatch : 1-dimensional array of shape `(nhalos, )`
"""
return numpy.array(
[numpy.product(1 - overlap) for overlap in self["overlap"]])
overlap = self.overlap(from_smoothed)
return numpy.array([numpy.product(1 - overlap) for overlap in overlap])
def dist(self, in_initial, norm_kind=None):
"""
@ -414,14 +461,16 @@ class PairOverlap:
ratio[i] = numpy.abs(ratio[i])
return numpy.array(ratio, dtype=object)
def counterpart_mass(self, overlap_threshold=0., in_log=False,
mass_kind="totpartmass"):
def counterpart_mass(self, from_smoothed, overlap_threshold=0.,
in_log=False, mass_kind="totpartmass"):
"""
Calculate the expected counterpart mass of each halo in the reference
simulation from the crossed simulation.
Parameters
-----------
from_smoothed : bool
Whether to use the smoothed overlap or not.
overlap_threshold : float, optional
Minimum overlap required for a halo to be considered a match. By
default 0.0, i.e. no threshold.
@ -440,8 +489,8 @@ class PairOverlap:
mean = numpy.full(len(self), numpy.nan, dtype=numpy.float32)
std = numpy.full(len(self), numpy.nan, dtype=numpy.float32)
massx = self.catx(mass_kind) # Create references to the arrays here
overlap = self["overlap"] # to speed up the loop below.
massx = self.catx(mass_kind) # Create references to speed
overlap = self.overlap(from_smoothed) # up the loop below
for i, match_ind in enumerate(self["match_indxs"]):
# Skip if no match
@ -538,9 +587,11 @@ class PairOverlap:
def __getitem__(self, key):
"""
Must be one of `index`, `match_indxs`, `overlap` or `refmask`.
Must be one of `index`, `match_indxs`, `ngp_overlap`,
`smoothed_overlap` or `refmask`.
"""
assert key in ("index", "match_indxs", "overlap", "refmask")
assert key in ("index", "match_indxs", "ngp_overlap",
"smoothed_overlap", "refmask")
return self._data[key]
def __len__(self):
@ -574,14 +625,17 @@ class NPairsOverlap:
self._pairs = [PairOverlap(cat0, catx, fskel=fskel, min_mass=min_mass,
max_dist=max_dist) for catx in catxs]
def summed_overlap(self, verbose=False):
def summed_overlap(self, from_smoothed, verbose=False):
"""
Summed overlap of each halo in the reference simulation with the cross
simulations.
Parameters
----------
from_smoothed : bool
Whether to use the smoothed overlap or not.
verbose : bool, optional
Verbosity flag.
Returns
-------
@ -589,17 +643,20 @@ class NPairsOverlap:
"""
out = [None] * len(self)
for i, pair in enumerate(tqdm(self.pairs) if verbose else self.pairs):
out[i] = pair.summed_overlap()
out[i] = pair.summed_overlap(from_smoothed)
return numpy.vstack(out).T
def prob_nomatch(self, verbose=False):
def prob_nomatch(self, from_smoothed, verbose=False):
"""
Probability of no match for each halo in the reference simulation with
the cross simulation.
Parameters
----------
from_smoothed : bool
Whether to use the smoothed overlap or not.
verbose : bool, optional
Verbosity flag.
Returns
-------
@ -607,18 +664,20 @@ class NPairsOverlap:
"""
out = [None] * len(self)
for i, pair in enumerate(tqdm(self.pairs) if verbose else self.pairs):
out[i] = pair.prob_nomatch()
out[i] = pair.prob_nomatch(from_smoothed)
return numpy.vstack(out).T
def counterpart_mass(self, overlap_threshold=0., in_log=False,
mass_kind="totpartmass", return_full=True,
verbose=False):
def counterpart_mass(self, from_smoothed, overlap_threshold=0.,
in_log=False, mass_kind="totpartmass",
return_full=True, verbose=False):
"""
Calculate the expected counterpart mass of each halo in the reference
simulation from the crossed simulation.
Parameters
-----------
from_smoothed : bool
Whether to use the smoothed overlap or not.
overlap_threshold : float, optional
Minimum overlap required for a halo to be considered a match. By
default 0.0, i.e. no threshold.
@ -647,11 +706,12 @@ class NPairsOverlap:
mus, stds = [None] * len(self), [None] * len(self)
for i, pair in enumerate(tqdm(self.pairs) if verbose else self.pairs):
mus[i], stds[i] = pair.counterpart_mass(
from_smoothed=from_smoothed,
overlap_threshold=overlap_threshold, in_log=in_log,
mass_kind=mass_kind)
mus, stds = numpy.vstack(mus).T, numpy.vstack(stds).T
probmatch = 1 - self.prob_nomatch() # Prob of > 0 matches
probmatch = 1 - self.prob_nomatch(from_smoothed) # Prob of > 0 matches
# Normalise it for weighted sums etc.
norm_probmatch = numpy.apply_along_axis(
lambda x: x / numpy.sum(x), axis=1, arr=probmatch)
@ -659,6 +719,7 @@ class NPairsOverlap:
# Mean and standard deviation of weighted stacked Gaussians
mu = numpy.sum(norm_probmatch * mus, axis=1)
std = numpy.sum(norm_probmatch * (mus**2 + stds**2), axis=1) - mu**2
std **= 0.5
if return_full:
return mu, std, mus, stds

6
csiborgtools/utils/__init__.py
View file

@ -13,6 +13,12 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
from datetime import datetime
from .recarray_manip import (cols_to_structured, add_columns, rm_columns, # noqa
list_to_ndarray, array_to_structured, # noqa
flip_cols, extract_from_structured) # noqa
def now(tz=None):
"""Shortcut to `datetime.datetime.now`."""
return datetime.now(tz=tz)

1097
meetings/220317_comboverlap.ipynb
View file

File diff suppressed because one or more lines are too long

2474
notebooks/playground_field.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

1785
notebooks/playground_matching.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

4827
notebooks/plot_correlation.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

0
scripts/plot_galaxy_distribution.ipynb → notebooks/plot_galaxy_distribution.ipynb
View file

0
scripts/plot_mass_function.ipynb → notebooks/plot_mass_function.ipynb
View file

63
scripts/run_singlematch.py
View file

@ -12,14 +12,12 @@
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""
Script to test running the CSiBORG realisations matcher.
"""
import numpy
from argparse import ArgumentParser
from distutils.util import strtobool
from datetime import datetime
"""A script to calculate overlap between two CSiBORG realisations."""
from os.path import join
from argparse import ArgumentParser
from datetime import datetime
import numpy
from scipy.ndimage import gaussian_filter
try:
import csiborgtools
except ModuleNotFoundError:
@ -33,26 +31,59 @@ parser = ArgumentParser()
parser.add_argument("--nsim0", type=int)
parser.add_argument("--nsimx", type=int)
parser.add_argument("--nmult", type=float)
parser.add_argument("--overlap", type=lambda x: bool(strtobool(x)))
parser.add_argument("--sigma", type=float)
args = parser.parse_args()
# File paths
fout = join(
utils.dumpdir, "overlap", "cross_{}_{}.npz".format(args.nsim0, args.nsimx))
fout = join(utils.dumpdir, "overlap",
"cross_{}_{}.npz".format(args.nsim0, args.nsimx))
smooth_kwargs = {"sigma": args.sigma, "mode": "constant", "cval": 0.0}
overlapper = csiborgtools.match.ParticleOverlap()
print("{}: loading catalogues.".format(datetime.now()), flush=True)
# Load catalogues
print("{}: loading catalogues {} and {}."
.format(datetime.now(), args.nsim0, args.nsimx), flush=True)
cat0 = csiborgtools.read.HaloCatalogue(args.nsim0)
catx = csiborgtools.read.HaloCatalogue(args.nsimx)
matcher = csiborgtools.match.RealisationsMatcher()
print("{}: loading simulation {} and converting positions to cell numbers."
.format(datetime.now(), args.nsim0), flush=True)
with open(cat0.paths.clump0_path(args.nsim0), "rb") as f:
clumps0 = numpy.load(f, allow_pickle=True)
overlapper.clumps_pos2cell(clumps0)
print("{}: loading simulation {} and converting positions to cell numbers."
.format(datetime.now(), args.nsimx), flush=True)
with open(catx.paths.clump0_path(args.nsimx), 'rb') as f:
clumpsx = numpy.load(f, allow_pickle=True)
overlapper.clumps_pos2cell(clumpsx)
print("{}: generating the background density fields.".format(datetime.now()),
flush=True)
delta_bckg = overlapper.make_bckg_delta(clumps0)
delta_bckg = overlapper.make_bckg_delta(clumpsx, delta=delta_bckg)
print("{}: crossing the simulations.".format(datetime.now()), flush=True)
ref_indxs, cross_indxs, match_indxs, overlap = matcher.cross(
cat0, catx, overlap=args.overlap)
matcher = csiborgtools.match.RealisationsMatcher()
ref_indxs, cross_indxs, match_indxs, ngp_overlap = matcher.cross(
cat0, catx, clumps0, clumpsx, delta_bckg)
print("{}: smoothing the background field.".format(datetime.now()), flush=True)
gaussian_filter(delta_bckg, output=delta_bckg, **smooth_kwargs)
print("{}: calculating smoothed overlaps.".format(datetime.now()), flush=True)
smoothed_overlap = matcher.smoothed_cross(clumps0, clumpsx, delta_bckg,
ref_indxs, cross_indxs, match_indxs,
smooth_kwargs)
# Dump the result
print("Saving results to `{}`.".format(fout), flush=True)
with open(fout, "wb") as f:
numpy.savez(fout, ref_indxs=ref_indxs, cross_indxs=cross_indxs,
match_indxs=match_indxs, overlap=overlap)
match_indxs=match_indxs, ngp_overlap=ngp_overlap,
smoothed_overlap=smoothed_overlap, sigma=args.sigma)
print("All finished.", flush=True)