Aquila-Consortium/csiborgtools
0
0
Fork 0
mirror of https://github.com/Richard-Sti/csiborgtools.git synced 2024-10-18 16:35:06 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Environmental properties (#20)

Browse Source 
* rm get_positions

* Add comment

* add halfwidth func

* Update docs

* Add imprt

* Evaluate multiple fields simulatenously

* add halfwidth selection

* Change order of grav field and tensor field

* Add gravitational field norm

* Add eigenvalue calculation

* Sorted eigenvalues

* add init script

* add progress

* Add surveys

* Add more survey flexibility

* Minor changes

* add survey names

* rm name

* Fix list bug

* Fig bugs when running the script

* add phi to dtype

* fix dump bug

* Add comment

* Add smoothing options

* Add further comment

* Update TODO
This commit is contained in:
Richard Stiskalek 2022-12-31 17:46:05 +00:00 committed by GitHub
parent 65059f3798
commit 2e99b901ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 302 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -18,7 +18,7 @@
### TODO
- [ ] Add gradient and Hessian of the overdensity field.
- [ ] Write a script to smoothen an overdensity field, calculate the derived fields and evaluate them at the galaxy positions.
- [x] Write a script to smoothen an overdensity field, calculate the derived fields and evaluate them at the galaxy positions.
### Questions

172
csiborgtools/field/density.py
View File

@ -139,7 +139,7 @@ class DensityField:
x = x.astype(numpy.float32)
return x
def density_field(self, grid, verbose=True):
def density_field(self, grid, smooth_scale=None, verbose=True):
"""
Calculate the density field using a Pylians routine [1, 2]. Enforces
float32 precision.
@ -148,6 +148,9 @@ class DensityField:
----------
grid : int
The grid size.
smooth_scale : float, optional
Scale to smoothen the density field, in units matching
`self.boxsize`. By default `None`, i.e. no smoothing is applied.
verbose : float, optional
A verbosity flag. By default `True`.
@ -170,9 +173,11 @@ class DensityField:
# Pre-allocate and do calculations
rho = numpy.zeros((grid, grid, grid), dtype=numpy.float32)
MASL.MA(pos, rho, self.boxsize, self.MAS, W=weights, verbose=verbose)
if smooth_scale is not None:
rho = self.smooth_field(rho, smooth_scale)
return rho
def overdensity_field(self, grid, verbose=True):
def overdensity_field(self, grid, smooth_scale=None, verbose=True):
r"""
Calculate the overdensity field using Pylians routines.
Defined as :math:`\rho/ <\rho> - 1`.
@ -181,6 +186,9 @@ class DensityField:
----------
grid : int
The grid size.
smooth_scale : float, optional
Scale to smoothen the density field, in units matching
`self.boxsize`. By default `None`, i.e. no smoothing is applied.
verbose : float, optional
A verbosity flag. By default `True`.
@ -190,12 +198,12 @@ class DensityField:
Overdensity field.
"""
# Get the overdensity
delta = self.density_field(grid, verbose)
delta = self.density_field(grid, smooth_scale, verbose)
delta /= delta.mean()
delta -= 1
return delta
def potential_field(self, grid, verbose=True):
def potential_field(self, grid, smooth_scale=None, verbose=True):
"""
Calculate the potential field using Pylians routines.
@ -203,6 +211,9 @@ class DensityField:
----------
grid : int
The grid size.
smooth_scale : float, optional
Scale to smoothen the original density field, in units matching
`self.boxsize`. By default `None`, i.e. no smoothing is applied.
verbose : float, optional
A verbosity flag. By default `True`.
@ -211,42 +222,24 @@ class DensityField:
potential : 3-dimensional array of shape `(grid, grid, grid)`.
Potential field.
"""
delta = self.overdensity_field(grid, verbose)
delta = self.overdensity_field(grid, smooth_scale, verbose)
if verbose:
print("Calculating potential from the overdensity..")
return MASL.potential(
delta, self.box._omega_m, self.box._aexp, self.MAS)
def tensor_field(self, grid, verbose=True):
def gravitational_field(self, grid, smooth_scale=None, verbose=True):
"""
Calculate the tidal tensor field.
Parameters
----------
grid : int
The grid size.
verbose : float, optional
A verbosity flag. By default `True`.
Returns
-------
tidal_tensor : :py:class:`MAS_library.tidal_tensor`
Tidal tensor object, whose attributes `tidal_tensor.Tij` contain
the relevant tensor components.
"""
delta = self.overdensity_field(grid, verbose)
return MASL.tidal_tensor(
delta, self.box._omega_m, self.box._aexp, self.MAS)
def gravitational_field(self, grid, verbose=True):
"""
Calculate the gravitational tensor field. Note that this method is
only defined in fork of `Pylians`.
Calculate the gravitational vector field. Note that this method is
only defined in a fork of `Pylians`.
Parameters
----------
grid : int
The grid size.
smooth_scale : float, optional
Scale to smoothen the original density field, in units matching
`self.boxsize`. By default `None`, i.e. no smoothing is applied.
verbose : float, optional
A verbosity flag. By default `True`.
@ -256,11 +249,35 @@ class DensityField:
Tidal tensor object, whose attributes `grav_field_tensor.gi`
contain the relevant tensor components.
"""
delta = self.overdensity_field(grid, verbose)
delta = self.overdensity_field(grid, smooth_scale, verbose)
return MASL.grav_field_tensor(
delta, self.box._omega_m, self.box._aexp, self.MAS)
def auto_powerspectrum(self, grid, verbose=True):
def tensor_field(self, grid, smooth_scale=None, verbose=True):
"""
Calculate the tidal tensor field.
Parameters
----------
grid : int
The grid size.
smooth_scale : float, optional
Scale to smoothen the original density field, in units matching
`self.boxsize`. By default `None`, i.e. no smoothing is applied.
verbose : float, optional
A verbosity flag. By default `True`.
Returns
-------
tidal_tensor : :py:class:`MAS_library.tidal_tensor`
Tidal tensor object, whose attributes `tidal_tensor.Tij` contain
the relevant tensor components.
"""
delta = self.overdensity_field(grid, smooth_scale, verbose)
return MASL.tidal_tensor(
delta, self.box._omega_m, self.box._aexp, self.MAS)
def auto_powerspectrum(self, grid, smooth_scale, verbose=True):
"""
Calculate the auto 1-dimensional power spectrum.
@ -268,6 +285,9 @@ class DensityField:
----------
grid : int
The grid size.
smooth_scale : float, optional
Scale to smoothen the original density field, in units matching
`self.boxsize`. By default `None`, i.e. no smoothing is applied.
verbose : float, optional
A verbosity flag. By default `True`.
@ -276,7 +296,7 @@ class DensityField:
pk : py:class`Pk_library.Pk`
Power spectrum object.
"""
delta = self.overdensity_field(grid, verbose)
delta = self.overdensity_field(grid, smooth_scale, verbose)
return PKL.Pk(
delta, self.boxsize, axis=1, MAS=self.MAS, threads=1,
verbose=verbose)
@ -305,48 +325,51 @@ class DensityField:
W_k = SL.FT_filter(self.boxsize, scale, grid, Filter, threads)
return SL.field_smoothing(field, W_k, threads)
def evaluate_field(self, pos, field):
def evaluate_field(self, *field, pos):
"""
Evaluate the field at Cartesian coordinates.
Parameters
----------
field : (list of) 3-dimensional array of shape `(grid, grid, grid)`
The density field that is to be interpolated.
pos : 2-dimensional array of shape `(n_samples, 3)`
Positions to evaluate the density field. The coordinates span range
of [0, boxsize].
field : 3-dimensional array of shape `(grid, grid, grid)`
The density field that is to be interpolated.
Returns
-------
interp_field : 1-dimensional array of shape `(n_samples,).
Interpolated field at `pos`.
interp_field : (list of) 1-dimensional array of shape `(n_samples,).
Interpolated fields at `pos`.
"""
self._force_f32(pos, "pos")
density_interpolated = numpy.zeros(pos.shape[0], dtype=numpy.float32)
MASL.CIC_interp(field, self.boxsize, pos, density_interpolated)
return density_interpolated
def evaluate_sky(self, pos, field, isdeg=True):
interp_field = [numpy.zeros(pos.shape[0], dtype=numpy.float32)
for __ in range(len(field))]
for i, f in enumerate(field):
MASL.CIC_interp(f, self.boxsize, pos, interp_field[i])
return interp_field
def evaluate_sky(self, *field, pos, isdeg=True):
"""
Evaluate the field at given distance, right ascension and declination.
Assumes that the observed is in the centre of the box.
Parameters
----------
field : (list of) 3-dimensional array of shape `(grid, grid, grid)`
The density field that is to be interpolated. Assumed to be defined
on a Cartesian grid.
pos : 2-dimensional array of shape `(n_samples, 3)`
Spherical coordinates to evaluate the field. Should be distance,
right ascension, declination, respectively.
field : 3-dimensional array of shape `(grid, grid, grid)`
The density field that is to be interpolated. Assumed to be defined
on a Cartesian grid.
isdeg : bool, optional
Whether `ra` and `dec` are in degres. By default `True`.
Returns
-------
interp_field : 1-dimensional array of shape `(n_samples,).
Interpolated field at `pos`.
interp_field : (list of) 1-dimensional array of shape `(n_samples,).
Interpolated fields at `pos`.
"""
self._force_f32(pos, "pos")
X = numpy.vstack(
@ -354,7 +377,58 @@ class DensityField:
X = X.astype(numpy.float32)
# Place the observer at the center of the box
X += 0.5 * self.boxsize
return self.evaluate_field(X, field)
return self.evaluate_field(*field, pos=X)
@staticmethod
def gravitational_field_norm(gx, gy, gz):
"""
Calculate the norm (magnitude) of a gravitational field.
Parameters
----------
gx, gy, gz : 1-dimensional arrays of shape `(n_samples,)`
Gravitational field components.
Returns
-------
g : 1-dimensional array of shape `(n_samples,)`
Gravitational field norm.
"""
return numpy.sqrt(gx * gx + gy * gy + gz * gz)
@staticmethod
def tensor_field_eigvals(T00, T01, T02, T11, T12, T22):
"""
Calculate the eigenvalues of a symmetric tensor field. Eigenvalues are
sorted in increasing order.
Parameters
----------
T00, T01, T02, T11, T12, T22 : 1-dim arrays of shape `(n_samples,)`
Tensor field upper components evaluated for each sample.
Returns
-------
eigvals : 2-dimensional array of shape `(n_samples, 3)`
Eigenvalues of each sample.
"""
n_samples = T00.size
# Fill array of shape `(n_samples, 3, 3)` to calculate eigvals
Teval = numpy.full((n_samples, 3, 3), numpy.nan, dtype=numpy.float32)
Teval[:, 0, 0] = T00
Teval[:, 0, 1] = T01
Teval[:, 0, 2] = T02
Teval[:, 1, 1] = T11
Teval[:, 1, 2] = T12
Teval[:, 2, 2] = T22
# Calculate the eigenvalues
eigvals = numpy.full((n_samples, 3), numpy.nan, dtype=numpy.float32)
for i in range(n_samples):
eigvals[i, :] = numpy.linalg.eigvalsh(Teval[i, ...], 'U')
eigvals[i, :] = numpy.sort(eigvals[i, :])
return eigvals
def make_sky_map(self, ra, dec, field, dist_marg, isdeg=True,
verbose=True):
@ -364,6 +438,8 @@ class DensityField:
position evaluates the field at distances `dist_marg` and sums these
interpolated values of the field.
NOTE: Supports only scalar fields.
Parameters
----------
ra, dec : 1-dimensional arrays of shape `(n_pos, )`
@ -400,6 +476,6 @@ class DensityField:
dec_loop[:] = pos[i, 1]
pos_loop[:] = numpy.vstack([dist_marg, ra_loop, dec_loop]).T
# Evaluate and sum it up
out[i] = numpy.sum(self.evaluate_sky(pos_loop, field, isdeg))
out[i] = numpy.sum(self.evaluate_sky(field, pos_loop, isdeg)[0, :])
return out

2
csiborgtools/read/__init__.py
View File

@ -13,7 +13,7 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
from .readsim import (CSiBORGPaths, ParticleReader, read_mmain, read_initcm, get_positions) # noqa
from .readsim import (CSiBORGPaths, ParticleReader, read_mmain, read_initcm, halfwidth_select) # noqa
from .make_cat import (HaloCatalogue, CombinedHaloCatalogue) # noqa
from .readobs import (PlanckClusters, MCXCClusters, TwoMPPGalaxies, # noqa
TwoMPPGroups, SDSS) # noqa

5
csiborgtools/read/readobs.py
View File

@ -95,6 +95,7 @@ class TwoMPPGalaxies(TextSurvey):
[3] Improving NASA/IPAC Extragalactic Database Redshift Calculations
(2021); Anthony Carr and Tamara Davis
"""
name = "2M++_galaxies"
def __init__(self, fpath=None):
if fpath is None:
@ -143,6 +144,7 @@ class TwoMPPGroups(TextSurvey):
[3] Improving NASA/IPAC Extragalactic Database Redshift Calculations
(2021); Anthony Carr and Tamara Davis
"""
name = "2M++_groups"
def __init__(self, fpath):
if fpath is None:
@ -411,6 +413,7 @@ class PlanckClusters(FitsSurvey):
----------
[1] https://heasarc.gsfc.nasa.gov/W3Browse/all/plancksz2.html
"""
name = "Planck_clusters"
_hdata = 0.7 # little h value of the data
def __init__(self, fpath=None, h=0.7, sel_steps=None):
@ -514,6 +517,7 @@ class MCXCClusters(FitsSurvey):
[2] https://heasarc.gsfc.nasa.gov/W3Browse/rosat/mcxc.html
[3] https://cdsarc.cds.unistra.fr/viz-bin/cat/J/A+A/534/A109#/article
"""
name = "MCXC"
_hdata = 0.7 # Little h of the catalogue
def __init__(self, fpath=None, h=0.7, sel_steps=None):
@ -571,6 +575,7 @@ class SDSS(FitsSurvey):
----------
[1] https://www.sdss.org/dr13/manga/manga-target-selection/nsa/
"""
name = "SDSS"
def __init__(self, fpath=None, h=1, sel_steps=None):
if fpath is None:

61
csiborgtools/read/readsim.py
View File

@ -18,12 +18,11 @@ Functions to read in the particle and clump files.
import numpy
from scipy.io import FortranFile
import gc
from os.path import (join, isfile, isdir)
from glob import glob
from tqdm import tqdm
from warnings import warn
from ..utils import (cols_to_structured, extract_from_structured)
from ..utils import (cols_to_structured)
F16 = numpy.float16
@ -883,49 +882,31 @@ def read_initcm(n, srcdir, fname="clump_{}_cm.npy"):
return None
def get_positions(paths, get_clumpid, verbose=True):
def halfwidth_select(hw, particles):
"""
Shortcut to get particle IDs, positions, masses and optionally clump
indices.
Select particles that in a cube of size `2 hw`, centered at the origin.
Note that this directly modifies the original array and throws away
particles outside the central region.
Parameters
----------
paths : py:class`csiborgtools.read.CSiBORGPaths`
CSiBORG paths-handling object with set `n_sim` and `n_snap`.
get_clumpid : bool
Whether to also return the clump indices.
verbose : bool, optional
Verbosity flag. By default `True`.
hw : float
Central region halfwidth.
particles : structured array
Particle array with keys `x`, `y`, `z`.
Returns
-------
particle_ids : 1-dimensional array
Particle IDs of shape `(n_particles, )`.
particle_pos : 2-dimensional array
Particle box coordinates of shape `(n_particles, 3)`.
particle_mass : 1-dimensional array
Particle mass of shape `(n_particles, )`.
clump_ids : 1-dimensional array, optional
Particles' clump IDs of shape `(n_particles, )`. Returned only if
`get_clumpid` is `True`.
particles : structured array
The modified particle array.
"""
# Extract particles
reader = ParticleReader(paths)
pars_extract = ["ID", "x", "y", "z", "M"]
# Read particles and unpack
particles = reader.read_particle(pars_extract, verbose)
pids = extract_from_structured(particles, "ID")
ppos = extract_from_structured(particles, ["x", "y", "z"])
pmass = extract_from_structured(particles, "M")
# Force early memory release
del particles
gc.collect()
out = (pids, ppos, pmass)
if get_clumpid:
out += (reader.read_clumpid(verbose),)
return out
assert 0 < hw < 0.5
mask = ((0.5 - hw < particles['x']) & (particles['x'] < 0.5 + hw)
& (0.5 - hw < particles['y']) & (particles['y'] < 0.5 + hw)
& (0.5 - hw < particles['z']) & (particles['z'] < 0.5 + hw))
# Subselect the particles
particles = particles[mask]
# Rescale to range [0, 1]
for p in ('x', 'y', 'z'):
particles[p] = (particles[p] - 0.5 + hw) / (2 * hw)
return particles

19
scripts/run_crosspk.py
View File

@ -70,27 +70,18 @@ for n in jobs:
particles = reader.read_particle(["x", "y", "z", "M"], verbose=False)
# Halfwidth -- particle selection
if args.halfwidth < 0.5:
hw = args.halfwidth
mask = ((0.5 - hw < particles['x']) & (particles['x'] < 0.5 + hw)
& (0.5 - hw < particles['y']) & (particles['y'] < 0.5 + hw)
& (0.5 - hw < particles['z']) & (particles['z'] < 0.5 + hw))
# Subselect the particles
particles = particles[mask]
# Rescale to range [0, 1]
for p in ('x', 'y', 'z'):
particles[p] = (particles[p] - 0.5 + hw) / (2 * hw)
length = box.box2mpc(2 * hw) * box.h
particles = csiborgtools.read.halfwidth_select(
args.halfwidth, particles)
length = box.box2mpc(2 * args.halfwidth) * box.h # Mpc/h
else:
mask = None
length = box.box2mpc(1) * box.h
length = box.box2mpc(1) * box.h # Mpc/h
# Calculate the overdensity field
field = csiborgtools.field.DensityField(particles, length, box, MAS)
delta = field.overdensity_field(args.grid, verbose=False)
aexp = box._aexp
# Try to clean up memory
del field, particles, box, reader, mask
del field, particles, box, reader
collect()
# Dump the results

120
scripts/run_fieldprop.py Normal file
View File

@ -0,0 +1,120 @@
# Copyright (C) 2022 Richard Stiskalek
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""
MPI script to evaluate field properties at the galaxy positions.
NOTE:
- Calculate for the entire box or just for a smaller region?
- Add argparser for different options.
- In the argparser add options to smoothen the field.
"""
import numpy
from datetime import datetime
from mpi4py import MPI
from os.path import join
from os import remove
try:
import csiborgtools
except ModuleNotFoundError:
import sys
sys.path.append("../")
import csiborgtools
import utils
halfwidth = 0.5
MAS = "CIC"
grid = 256
# Get MPI things
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
nproc = comm.Get_size()
# Galaxy positions
survey = "SDSS"
survey = utils.surveys[survey]()()
pos = numpy.vstack([survey[p] for p in ("DIST", "RA", "DEC")]).T
pos = pos.astype(numpy.float32)
# File paths
ftemp = join(utils.dumpdir, "temp_fields", "out_" + survey.name + "_{}.npy")
fperm = join(utils.dumpdir, "fields", "out_{}.npy".format(survey.name))
# Edit depending on what is calculated
dtype = {"names": ["delta", "phi"], "formats": [numpy.float32] * 2}
# CSiBORG simulation paths
paths = csiborgtools.read.CSiBORGPaths()
ics = paths.ic_ids[:10]
n_sims = len(ics)
for n in csiborgtools.fits.split_jobs(n_sims, nproc)[rank]:
print("Rank {}@{}: working on {}th IC.".format(rank, datetime.now(), n),
flush=True)
# Set the paths
n_sim = ics[n]
paths.set_info(n_sim, paths.get_maximum_snapshot(n_sim))
# Set reader and the box
reader = csiborgtools.read.ParticleReader(paths)
box = csiborgtools.units.BoxUnits(paths)
# Read particles and select a subset of them
particles = reader.read_particle(["x", "y", "z", "M"], verbose=False)
if halfwidth < 0.5:
particles = csiborgtools.read.halfwidth_select(halfwidth, particles)
length = box.box2mpc(2 * halfwidth) * box.h # Mpc/h
else:
length = box.box2mpc(1) * box.h # Mpc/h
# Initialise the field object and output array
field = csiborgtools.field.DensityField(particles, length, box, MAS)
out = numpy.full(pos.shape[0], numpy.nan, dtype=dtype)
# Calculate the overdensity field and interpolate at galaxy positions
feval = field.overdensity_field(grid, verbose=False)
out["delta"] = field.evaluate_sky(feval, pos=pos, isdeg=True)[0]
# Potential
feval = field.potential_field(grid, verbose=False)
out["phi"] = field.evaluate_sky(feval, pos=pos, isdeg=True)[0]
# Calculate the remaining fields
# ...
# ...
# Dump the results
with open(ftemp.format(n_sim), "wb") as f:
numpy.save(f, out)
# Wait for all ranks to finish
comm.Barrier()
if rank == 0:
print("Collecting files...", flush=True)
out = numpy.full((n_sims, pos.shape[0]), numpy.nan, dtype=dtype)
for n in range(n_sims):
n_sim = ics[n]
with open(ftemp.format(n_sim), "rb") as f:
fin = numpy.load(f, allow_pickle=True)
for name in dtype["names"]:
out[name][n, ...] = fin[name]
# Remove the temporary file
remove(ftemp.format(n_sim))
print("Saving results to `{}`.".format(fperm), flush=True)
with open(fperm, "wb") as f:
numpy.save(f, out)

30
scripts/utils.py
View File

@ -18,11 +18,11 @@ Notebook utility functions.
# from os.path import join
# try:
# import csiborgtools
# except ModuleNotFoundError:
# import sys
# sys.path.append("../")
try:
import csiborgtools
except ModuleNotFoundError:
import sys
sys.path.append("../")
Nsplits = 200
@ -39,3 +39,23 @@ _virgo = {"RA": (12 + 27 / 60) * 15,
"COMDIST": 16.5}
specific_clusters = {"Coma": _coma, "Virgo": _virgo}
###############################################################################
# Surveys #
###############################################################################
class SDSS:
@staticmethod
def steps(cls):
return [(lambda x: cls[x], ("IN_DR7_LSS",)),
(lambda x: cls[x] < 17.6, ("ELPETRO_APPMAG_r", )),
(lambda x: cls[x] < 155, ("DIST", ))
]
def __call__(self):
return csiborgtools.read.SDSS(h=1, sel_steps=self.steps)
surveys = {"SDSS": SDSS}