Attempt at absolute calibration and more ()

* Update nb

* Update defaults

* Add absolute calibration to fpath

* Add basic absolute calibration

* Switch to log density

* Feed in r directly

* Add more complex spacing option

* Update script

* Comment out abs calibration for now

* Update script

* Add MB profile

* Add script

* Update submit for all profiles

* Update

* Add option

* Add iterator over sims

* Update defaults on alpha

* Update script

* Parallelise script

* Switch to feasible init

* Switch to median init

* Add vext option

* Remove file if exists

* Add optional sample_Vext

* Add param

* rm print

* Add more iterator

* Update script

* Update nb

* Update nb
This commit is contained in:
Richard Stiskalek 2024-09-16 12:12:32 +02:00 committed by GitHub
parent 32e36afdc3
commit 9c1aa26428
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 568 additions and 160 deletions

View file

@ -198,7 +198,7 @@ def observer_peculiar_velocity(velocity_field, smooth_scales=None,
###############################################################################
def evaluate_los(*fields, sky_pos, boxsize, rmax, dr, smooth_scales=None,
def evaluate_los(*fields, sky_pos, boxsize, rdist, smooth_scales=None,
interpolation_method="cic", verbose=False):
"""
Interpolate the fields for a set of lines of sights from the observer
@ -212,10 +212,8 @@ def evaluate_los(*fields, sky_pos, boxsize, rmax, dr, smooth_scales=None,
Query positions in spherical coordinates (RA, dec) in degrees.
boxsize : float
Box size in `Mpc / h`.
rmax : float
Maximum radial distance in `Mpc / h`.
dr : float
Radial distance step in `Mpc / h`.
rdist : 1-dimensional array
Radial positions in `Mpc / h` where the fields are evaluated.
smooth_scales : (list of) float, optional
Smoothing scales in `Mpc / h`.
interpolation_method : str, optional
@ -238,12 +236,10 @@ def evaluate_los(*fields, sky_pos, boxsize, rmax, dr, smooth_scales=None,
raise ValueError("`sky_pos` must be a 2D array.")
nsamples = len(sky_pos)
rmax = np.max(rdist)
if interpolation_method == "cic" and rmax > 0.5 * boxsize:
raise ValueError("The maximum radius must be within the box.")
# Radial positions to evaluate for each line of sight.
rdist = np.arange(0, rmax, dr, dtype=fields[0].dtype)
# Create an array of radial positions and sky coordinates of each line of
# sight.
pos = np.empty((nsamples * len(rdist), 3), dtype=fields[0].dtype)

View file

@ -363,12 +363,11 @@ def predict_zobs(dist, beta, Vext_radial, vpec_radial, Omega_m):
###############################################################################
def ptilde_wo_bias(xrange, mu, err_squared, r_squared_xrange):
def log_ptilde_wo_bias(xrange, mu, err_squared, log_r_squared_xrange):
"""Calculate `ptilde(r)` without imhomogeneous Malmquist bias."""
ptilde = jnp.exp(-0.5 * (xrange - mu)**2 / err_squared)
ptilde /= jnp.sqrt(2 * np.pi * err_squared)
ptilde *= r_squared_xrange
return ptilde
return (-0.5 * (xrange - mu)**2 / err_squared
- 0.5 * jnp.log(2 * np.pi * err_squared)
+ log_r_squared_xrange)
def likelihood_zobs(zobs, zobs_pred, e2_cz):
@ -406,8 +405,7 @@ class BaseFlowValidationModel(ABC):
setattr(self, f"{name}", jnp.asarray(value))
def _set_calibration_params(self, calibration_params):
names = []
values = []
names, values = [], []
for key, value in calibration_params.items():
names.append(key)
values.append(value)
@ -421,6 +419,28 @@ class BaseFlowValidationModel(ABC):
self._setattr_as_jax(names, values)
def _set_abs_calibration_params(self, abs_calibration_params):
self.with_absolute_calibration = abs_calibration_params is not None
if abs_calibration_params is None:
self.with_absolute_calibration = False
return
self.calibration_distmod = jnp.asarray(
abs_calibration_params["calibration_distmod"][..., 0])
self.calibration_edistmod = jnp.asarray(
abs_calibration_params["calibration_distmod"][..., 1])
self.data_with_calibration = jnp.asarray(
abs_calibration_params["data_with_calibration"])
self.data_wo_calibration = ~self.data_with_calibration
# Calculate the log of the number of calibrators. Where there is no
# calibrator set the number of calibrators to 1 to avoid log(0) and
# this way only zeros are being added.
length_calibration = abs_calibration_params["length_calibration"]
length_calibration[length_calibration == 0] = 1
self.log_length_calibration = jnp.log(length_calibration)
def _set_radial_spacing(self, r_xrange, Omega_m):
cosmo = FlatLambdaCDM(H0=H0, Om0=Omega_m)
@ -428,21 +448,19 @@ class BaseFlowValidationModel(ABC):
r2_xrange = r_xrange**2
r2_xrange /= r2_xrange.mean()
self.r_xrange = r_xrange
self.r2_xrange = r2_xrange
self.log_r2_xrange = jnp.log(r2_xrange)
# Require `zmin` < 0 because the first radial step is likely at 0.
z_xrange = z_at_value(
cosmo.comoving_distance, r_xrange * u.Mpc, zmin=-0.01)
mu_xrange = cosmo.distmod(z_xrange).value
# In case the first distance is 0 and its distance modulus is infinite.
if not np.isfinite(mu_xrange[0]):
mu_xrange[0] = mu_xrange[1] - 1
self.z_xrange = jnp.asarray(z_xrange)
self.mu_xrange = jnp.asarray(mu_xrange)
# Get the stepsize, we need it to be constant for Simpson's rule.
dr = np.diff(r_xrange)
if not np.all(np.isclose(dr, dr[0], atol=1e-5)):
raise ValueError("The radial step size must be constant.")
self.dr = dr[0]
@property
def ndata(self):
"""Number of PV objects in the catalogue."""
@ -451,7 +469,7 @@ class BaseFlowValidationModel(ABC):
@property
def num_sims(self):
"""Number of simulations."""
return len(self.los_density)
return len(self.log_los_density)
@abstractmethod
def __call__(self, **kwargs):
@ -580,26 +598,38 @@ def sample_simple(e_mu_min, e_mu_max, dmu_min, dmu_max, alpha_min, alpha_max,
def sample_calibration(Vext_min, Vext_max, Vmono_min, Vmono_max, beta_min,
beta_max, sigma_v_min, sigma_v_max, sample_Vmono,
sample_beta):
beta_max, sigma_v_min, sigma_v_max, h_min, h_max,
sample_Vext, sample_Vmono, sample_beta, sample_h):
"""Sample the flow calibration."""
sigma_v = sample("sigma_v", Uniform(sigma_v_min, sigma_v_max))
Vext = sample("Vext", Uniform(Vext_min, Vext_max).expand([3]))
if sample_beta:
beta = sample("beta", Uniform(beta_min, beta_max))
else:
beta = 1.0
if sample_Vext:
Vext = sample("Vext", Uniform(Vext_min, Vext_max).expand([3]))
else:
Vext = jnp.zeros(3)
if sample_Vmono:
Vmono = sample("Vmono", Uniform(Vmono_min, Vmono_max))
else:
Vmono = 0.0
if sample_h:
h = sample("h", Uniform(h_min, h_max))
else:
h = 1.0
return {"Vext": Vext,
"Vmono": Vmono,
"sigma_v": sigma_v,
"beta": beta}
"beta": beta,
"h": h,
"sample_h": sample_h,
}
###############################################################################
@ -630,10 +660,12 @@ class PV_LogLikelihood(BaseFlowValidationModel):
Observed redshifts.
e_zobs : 1-dimensional array of shape (n_objects)
Errors on the observed redshifts.
calibration_params: dict
calibration_params : dict
Calibration parameters of each object.
abs_calibration_params : dict
Absolute calibration parameters.
mag_selection : dict
Magnitude selection parameters.
Magnitude selection parameters, optional.
r_xrange : 1-dimensional array
Radial distances where the field was interpolated for each object.
Omega_m : float
@ -645,22 +677,23 @@ class PV_LogLikelihood(BaseFlowValidationModel):
"""
def __init__(self, los_density, los_velocity, RA, dec, z_obs, e_zobs,
calibration_params, mag_selection, r_xrange, Omega_m,
kind, name):
calibration_params, abs_calibration_params, mag_selection,
r_xrange, Omega_m, kind, name):
if e_zobs is not None:
e2_cz_obs = jnp.asarray((SPEED_OF_LIGHT * e_zobs)**2)
else:
e2_cz_obs = jnp.zeros_like(z_obs)
# Convert RA/dec to radians.
RA = np.deg2rad(RA)
dec = np.deg2rad(dec)
RA, dec = np.deg2rad(RA), np.deg2rad(dec)
names = ["los_density", "los_velocity", "RA", "dec", "z_obs",
names = ["log_los_density", "los_velocity", "RA", "dec", "z_obs",
"e2_cz_obs"]
values = [los_density, los_velocity, RA, dec, z_obs, e2_cz_obs]
values = [jnp.log(los_density), los_velocity, RA, dec, z_obs,
e2_cz_obs]
self._setattr_as_jax(names, values)
self._set_calibration_params(calibration_params)
self._set_abs_calibration_params(abs_calibration_params)
self._set_radial_spacing(r_xrange, Omega_m)
self.kind = kind
@ -765,6 +798,10 @@ class PV_LogLikelihood(BaseFlowValidationModel):
mu = distmod_SN(
mag_true, x1_true, c_true, mag_cal, alpha_cal, beta_cal)
if field_calibration_params["sample_h"]:
raise NotImplementedError("H0 for SN not implemented.")
elif self.kind == "TFR":
a = distmod_params["a"]
b = distmod_params["b"]
@ -834,6 +871,11 @@ class PV_LogLikelihood(BaseFlowValidationModel):
e2_mu = jnp.ones_like(mag_true) * e_mu**2
mu = distmod_TFR(mag_true, eta_true, a, b, c)
if field_calibration_params["sample_h"]:
raise NotImplementedError("H0 for TFR not implemented.")
# mu -= 5 * jnp.log10(field_calibration_params["h"])
elif self.kind == "simple":
dmu = distmod_params["dmu"]
@ -852,19 +894,25 @@ class PV_LogLikelihood(BaseFlowValidationModel):
e2_mu = jnp.ones_like(mag_true) * e_mu**2
mu = mu_true + dmu
if field_calibration_params["sample_h"]:
raise NotImplementedError("H0 for simple not implemented.")
else:
raise ValueError(f"Unknown kind: `{self.kind}`.")
# Calculate p(r) (Malmquist bias). Shape is (ndata, nxrange)
ptilde = ptilde_wo_bias(
log_ptilde = log_ptilde_wo_bias(
self.mu_xrange[None, :], mu[:, None], e2_mu[:, None],
self.r2_xrange[None, :])
self.log_r2_xrange[None, :])
# Inhomogeneous Malmquist bias. Shape is (n_sims, ndata, nxrange)
alpha = distmod_params["alpha"]
ptilde = ptilde[None, ...] * self.los_density**alpha
log_ptilde = log_ptilde[None, ...] + alpha * self.log_los_density
ptilde = jnp.exp(log_ptilde)
# Normalization of p(r). Shape is (n_sims, ndata)
pnorm = simpson(ptilde, dx=self.dr, axis=-1)
pnorm = simpson(ptilde, x=self.r_xrange, axis=-1)
# Calculate z_obs at each distance. Shape is (n_sims, ndata, nxrange)
vrad = field_calibration_params["beta"] * self.los_velocity
@ -872,10 +920,34 @@ class PV_LogLikelihood(BaseFlowValidationModel):
zobs = (1 + self.z_xrange[None, None, :]) * (1 + vrad / SPEED_OF_LIGHT)
zobs -= 1.
# Shape remains (n_sims, ndata, nxrange)
ptilde *= likelihood_zobs(
self.z_obs[None, :, None], zobs, e2_cz[None, :, None])
ll = jnp.log(simpson(ptilde, dx=self.dr, axis=-1)) - jnp.log(pnorm)
if self.with_absolute_calibration:
raise NotImplementedError("Absolute calibration not implemented.")
# Absolute calibration likelihood, the shape is now
# (ndata_with_calibration, ncalib, nxrange)
# ll_calibration = normal_logpdf(
# self.mu_xrange[None, None, :],
# self.calibration_distmod[..., None],
# self.calibration_edistmod[..., None])
# # Average the likelihood over the calibration points. The shape
# is
# # now (ndata, nxrange)
# ll_calibration = logsumexp(
# jnp.nan_to_num(ll_calibration, nan=-jnp.inf), axis=1)
# # This is the normalisation because we want the *average*.
# ll_calibration -= self.log_length_calibration[:, None]
# ptilde = ptilde.at[:, self.data_with_calibration, :].
# multiply(jnp.exp(ll_calibration))
# Integrate over the radial distance. Shape is (n_sims, ndata)
ll = jnp.log(simpson(ptilde, x=self.r_xrange, axis=-1))
ll -= jnp.log(pnorm)
return ll0 + jnp.sum(logsumexp(ll, axis=0)) + self.norm
@ -923,7 +995,54 @@ def PV_validation_model(models, distmod_hyperparams_per_model,
###############################################################################
def get_model(loader, zcmb_min=None, zcmb_max=None, mag_selection=None):
def read_absolute_calibration(kind, data_length, calibration_fpath):
"""
Read the absolute calibration for the CF4 TFR sample from LEDA but
preprocessed by me.
Parameters
----------
kind : str
Calibration kind: `Cepheids`, `TRGB`, `SBF`, ...
data_length : int
Number of samples in CF4 TFR (should be 9,788).
calibration_fpath : str
Path to the preprocessed calibration file.
Returns
-------
data : 3-dimensional array of shape (data_length, max_calib, 2)
Absolute calibration data.
with_calibration : 1-dimensional array of shape (data_length)
Whether the sample has a calibration.
length_calibration : 1-dimensional array of shape (data_length)
Number of calibration points per sample.
"""
data = {}
with File(calibration_fpath, 'r') as f:
for key in f[kind].keys():
x = f[kind][key][:]
# Get rid of points without uncertainties
x = x[~np.isnan(x[:, 1])]
data[key] = x
max_calib = max(len(val) for val in data.values())
out = np.full((data_length, max_calib, 2), np.nan)
with_calibration = np.full(data_length, False)
length_calibration = np.full(data_length, 0)
for i in data.keys():
out[int(i), :len(data[i]), :] = data[i]
with_calibration[int(i)] = True
length_calibration[int(i)] = len(data[i])
return out, with_calibration, length_calibration
def get_model(loader, zcmb_min=None, zcmb_max=None, mag_selection=None,
absolute_calibration=None, calibration_fpath=None):
"""
Get a model and extract the relevant data from the loader.
@ -937,6 +1056,9 @@ def get_model(loader, zcmb_min=None, zcmb_max=None, mag_selection=None):
Maximum observed redshift in the CMB frame to include.
mag_selection : dict, optional
Magnitude selection parameters.
add_absolute_calibration : bool, optional
Whether to add an absolute calibration for CF4 TFRs.
calibration_fpath : str, optional
Returns
-------
@ -949,6 +1071,9 @@ def get_model(loader, zcmb_min=None, zcmb_max=None, mag_selection=None):
los_velocity = loader.los_radial_velocity
kind = loader._catname
if absolute_calibration is not None and "CF4_TFR_" not in kind:
raise ValueError("Absolute calibration supported only for the CF4 TFR sample.") # noqa
if kind in ["LOSS", "Foundation"]:
keys = ["RA", "DEC", "z_CMB", "mB", "x1", "c", "e_mB", "e_x1", "e_c"]
RA, dec, zCMB, mag, x1, c, e_mag, e_x1, e_c = (
@ -963,7 +1088,8 @@ def get_model(loader, zcmb_min=None, zcmb_max=None, mag_selection=None):
model = PV_LogLikelihood(
los_overdensity[:, mask], los_velocity[:, mask],
RA[mask], dec[mask], zCMB[mask], e_zCMB, calibration_params,
mag_selection, loader.rdist, loader._Omega_m, "SN", name=kind)
None, mag_selection, loader.rdist, loader._Omega_m, "SN",
name=kind)
elif "Pantheon+" in kind:
keys = ["RA", "DEC", "zCMB", "mB", "x1", "c", "biasCor_m_b", "mBERR",
"x1ERR", "cERR", "biasCorErr_m_b", "zCMB_SN", "zCMB_Group",
@ -991,7 +1117,8 @@ def get_model(loader, zcmb_min=None, zcmb_max=None, mag_selection=None):
model = PV_LogLikelihood(
los_overdensity[:, mask], los_velocity[:, mask],
RA[mask], dec[mask], zCMB[mask], e_zCMB[mask], calibration_params,
mag_selection, loader.rdist, loader._Omega_m, "SN", name=kind)
None, mag_selection, loader.rdist, loader._Omega_m, "SN",
name=kind)
elif kind in ["SFI_gals", "2MTF", "SFI_gals_masked"]:
keys = ["RA", "DEC", "z_CMB", "mag", "eta", "e_mag", "e_eta"]
RA, dec, zCMB, mag, eta, e_mag, e_eta = (loader.cat[k] for k in keys)
@ -1001,7 +1128,7 @@ def get_model(loader, zcmb_min=None, zcmb_max=None, mag_selection=None):
"e_mag": e_mag[mask], "e_eta": e_eta[mask]}
model = PV_LogLikelihood(
los_overdensity[:, mask], los_velocity[:, mask],
RA[mask], dec[mask], zCMB[mask], None, calibration_params,
RA[mask], dec[mask], zCMB[mask], None, calibration_params, None,
mag_selection, loader.rdist, loader._Omega_m, "TFR", name=kind)
elif "CF4_TFR_" in kind:
# The full name can be e.g. "CF4_TFR_not2MTForSFI_i" or "CF4_TFR_i".
@ -1039,12 +1166,34 @@ def get_model(loader, zcmb_min=None, zcmb_max=None, mag_selection=None):
else:
mask &= Qs == 5
# Read the absolute calibration
if absolute_calibration is not None:
CF4_length = len(RA)
distmod, with_calibration, length_calibration = read_absolute_calibration( # noqa
"Cepheids", CF4_length, calibration_fpath)
distmod = distmod[mask]
with_calibration = with_calibration[mask]
length_calibration = length_calibration[mask]
fprint(f"found {np.sum(with_calibration)} galaxies with absolute calibration.") # noqa
distmod = distmod[with_calibration]
length_calibration = length_calibration[with_calibration]
abs_calibration_params = {
"calibration_distmod": distmod,
"data_with_calibration": with_calibration,
"length_calibration": length_calibration}
else:
abs_calibration_params = None
calibration_params = {"mag": mag[mask], "eta": eta[mask],
"e_mag": e_mag[mask], "e_eta": e_eta[mask]}
model = PV_LogLikelihood(
los_overdensity[:, mask], los_velocity[:, mask],
RA[mask], dec[mask], z_obs[mask], None, calibration_params,
mag_selection, loader.rdist, loader._Omega_m, "TFR", name=kind)
abs_calibration_params, mag_selection, loader.rdist,
loader._Omega_m, "TFR", name=kind)
elif kind in ["CF4_GroupAll"]:
# Note, this for some reason works terribly.
keys = ["RA", "DE", "Vcmb", "DMzp", "eDM"]
@ -1059,7 +1208,7 @@ def get_model(loader, zcmb_min=None, zcmb_max=None, mag_selection=None):
calibration_params = {"mu": mu[mask], "e_mu": e_mu[mask]}
model = PV_LogLikelihood(
los_overdensity[:, mask], los_velocity[:, mask],
RA[mask], dec[mask], zCMB[mask], None, calibration_params,
RA[mask], dec[mask], zCMB[mask], None, calibration_params, None,
mag_selection, loader.rdist, loader._Omega_m, "simple",
name=kind)
else:

View file

@ -106,6 +106,7 @@ def simname2Omega_m(simname):
"Lilow2024": 0.3175,
"IndranilVoid_exp": 0.3,
"IndranilVoid_gauss": 0.3,
"IndranilVoid_mb": 0.3,
"no_field": 0.3,
"CLONES": 0.307115,
}

View file

@ -129,7 +129,7 @@ class Paths:
files = [0]
elif "IndranilVoid" in simname:
kind = simname.split("_")[-1]
if kind not in ["exp", "gauss"]:
if kind not in ["exp", "gauss", "mb"]:
raise ValueError(f"Unknown void kind `{simname}`.")
kind = kind.upper()
@ -649,7 +649,7 @@ class Paths:
"""
return self.tng300_1_dir
def field_los(self, simnname, catalogue):
def field_los(self, simname, catalogue):
"""
Path to the files containing the line-of-sight fields.
@ -666,13 +666,14 @@ class Paths:
"""
fdir = join(self.postdir, "field_los")
try_create_directory(fdir)
return join(fdir, f"los_{catalogue}_{simnname}.hdf5")
return join(fdir, f"los_{catalogue}_{simname}.hdf5")
def flow_validation(self, fdir, simname, catalogue, inference_method,
smooth=None, nsim=None, zcmb_min=None, zcmb_max=None,
mag_selection=None, sample_alpha=False,
sample_beta=False, sample_Vmono=False,
sample_mag_dipole=False, sample_curvature=False):
sample_beta=False, sample_Vext=None,
sample_Vmono=False, sample_mag_dipole=False,
sample_curvature=False, absolute_calibration=None):
"""Flow validation file path."""
if isinstance(catalogue, list) and len(catalogue) == 1:
catalogue = catalogue[0]
@ -686,11 +687,12 @@ class Paths:
fname = f"samples_{simname}_{catalogue}_{inference_method}_"
keys = ["smooth", "nsim", "zcmb_min", "zcmb_max", "mag_selection",
"sample_alpha", "sample_beta", "sample_Vmono",
"sample_mag_dipole", "sample_curvature"]
"sample_alpha", "sample_beta", "sample_Vext", "sample_Vmono",
"sample_mag_dipole", "sample_curvature",
"absolute_calibration"]
values = [smooth, nsim, zcmb_min, zcmb_max, mag_selection,
sample_alpha, sample_beta, sample_Vmono, sample_mag_dipole,
sample_curvature]
sample_alpha, sample_beta, sample_Vext, sample_Vmono,
sample_mag_dipole, sample_curvature, absolute_calibration]
for key, value in zip(keys, values):

File diff suppressed because one or more lines are too long

View file

@ -23,16 +23,19 @@ from os import makedirs, remove, rmdir
from os.path import exists, join
from warnings import warn
import csiborgtools
import numpy as np
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.cosmology import FlatLambdaCDM
from astropy.io import fits
from h5py import File
from mpi4py import MPI
from numba import jit
from scipy.interpolate import interp1d
from taskmaster import work_delegation # noqa
import csiborgtools
sys.path.append("../")
from utils import get_nsims # noqa
@ -41,6 +44,35 @@ from utils import get_nsims # noqa
###############################################################################
def make_spacing(rmax, dr, dense_mu_min, dense_mu_max, dmu, Om0):
"""
Make radial spacing that at low distance is with constant spacing in
distance modulus and at higher distances is with constant spacing in
comoving distance.
"""
# Create interpolant to go from distance modulus to comoving distance.
cosmo = FlatLambdaCDM(H0=100, Om0=Om0)
z_range = np.linspace(0, 0.1, 1000000)[1:]
r_range = cosmo.comoving_distance(z_range).value
mu_range = cosmo.distmod(z_range).value
mu2r = interp1d(mu_range, r_range, kind='cubic')
# Create the spacing in distance modulus.
mu = np.arange(dense_mu_min, dense_mu_max, dmu)
rmin_dense = mu2r(np.min(mu))
rmax_dense = mu2r(np.max(mu))
# Create the spacing in comoving distance below and above.
rlow = np.arange(0, rmin_dense, dr)
rmed = mu2r(mu)
rhigh = np.arange(rmax_dense, rmax, dr)[1:]
# Combine the spacings.
return np.hstack([rlow, rmed, rhigh])
def get_los(catalogue_name, simname, comm):
"""
Get the line of sight RA/dec coordinates for the given catalogue.
@ -296,8 +328,8 @@ def replace_nan_with_last_finite(x, y, apply_decay):
return y, rmax
def interpolate_field(pos, simname, nsim, MAS, grid, dump_folder, rmax,
dr, smooth_scales, verbose=False):
def interpolate_field(pos, simname, nsim, MAS, grid, dump_folder, r,
smooth_scales, verbose=False):
"""
Interpolate the density and velocity fields along the line of sight.
@ -315,10 +347,8 @@ def interpolate_field(pos, simname, nsim, MAS, grid, dump_folder, rmax,
Grid resolution.
dump_folder : str
Folder where the temporary files are stored.
rmax : float
Maximum distance along the line of sight.
dr : float
Distance spacing along the line of sight.
r : 1-dimensional array
Radial spacing.
smooth_scales : list
Smoothing scales.
@ -335,7 +365,7 @@ def interpolate_field(pos, simname, nsim, MAS, grid, dump_folder, rmax,
flush=True)
density = get_field(simname, nsim, "density", MAS, grid)
rdist, finterp = csiborgtools.field.evaluate_los(
density, sky_pos=pos, boxsize=boxsize, rmax=rmax, dr=dr,
density, sky_pos=pos, boxsize=boxsize, rdist=r,
smooth_scales=smooth_scales, verbose=verbose,
interpolation_method="linear")
@ -361,9 +391,8 @@ def interpolate_field(pos, simname, nsim, MAS, grid, dump_folder, rmax,
velocity = get_field(simname, nsim, "velocity", MAS, grid)
rdist, finterp = csiborgtools.field.evaluate_los(
velocity[0], velocity[1], velocity[2],
sky_pos=pos, boxsize=boxsize, rmax=rmax, dr=dr,
smooth_scales=smooth_scales, verbose=verbose,
interpolation_method="linear")
sky_pos=pos, boxsize=boxsize, rdist=r, smooth_scales=smooth_scales,
verbose=verbose, interpolation_method="linear")
rmax_velocity = np.full((3, len(pos), len(smooth_scales)), np.nan)
for k in range(3):
@ -398,16 +427,15 @@ if __name__ == "__main__":
parser.add_argument("--grid", type=int, help="Grid resolution.")
args = parser.parse_args()
rmax = 200
if args.catalogue == "CF4_GroupAll":
dr = 1
else:
dr = 0.75
Om0 = csiborgtools.simname2Omega_m(args.simname)
# r = make_spacing(200, 0.75, 23.25, 34, 0.01, Om0)
r = np.arange(0, 200, 0.75)
# smooth_scales = [0, 2, 4, 6, 8]
smooth_scales = [0]
print(f"Running catalogue {args.catalogue} for simulation {args.simname}.")
print(f"Running catalogue {args.catalogue} for simulation {args.simname} "
f"with {len(r)} radial points.")
comm = MPI.COMM_WORLD
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
@ -429,7 +457,7 @@ if __name__ == "__main__":
def main(nsim):
interpolate_field(pos, args.simname, nsim, args.MAS, args.grid,
dump_folder, rmax, dr, smooth_scales,
dump_folder, r, smooth_scales,
verbose=comm.Get_size() == 1)
work_delegation(main, nsims, comm, master_verbose=True)

View file

@ -19,9 +19,10 @@ fi
# for simname in "csiborg1" "csiborg2_main" "csiborg2X" "Lilow2024" "Carrick2015" "CF4"; do
for simname in "CLONES"; do
for simname in "Carrick2015"; do
# for catalogue in "2MTF" "SFI_gals" "CF4_TFR"; do
for catalogue in "Foundation" "2MTF" "SFI_gals" "CF4_TFR"; do
# for catalogue in "Foundation" "2MTF" "SFI_gals" "CF4_TFR"; do
for catalogue in "CF4_TFR"; do
pythoncm="$env $file --catalogue $catalogue --nsims $nsims --simname $simname --MAS $MAS --grid $grid"
if [ $on_login -eq 1 ]; then
echo $pythoncm

View file

@ -12,29 +12,35 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
"""Script to interpolate the Indranil void profiles for lines of sight."""
from os.path import join
from argparse import ArgumentParser
from os.path import join, exists
from os import remove
import csiborgtools
import numpy as np
from astropy.coordinates import SkyCoord, angular_separation
from field_los import get_los
from h5py import File
from mpi4py import MPI
from scipy.interpolate import RegularGridInterpolator
from tqdm import trange
from field_los import get_los
def interpolate_indranil_void(kind, nsims, RA, dec, rmax, dr, dump_folder,
catalogue):
if kind not in ["exp", "gauss"]:
if kind not in ["exp", "gauss", "mb"]:
raise ValueError(f"Unknown void kind: `{kind}`.")
kind = kind.upper()
fdir = join("/mnt/extraspace/rstiskalek/catalogs", "IndranilVoid",
f"{kind}profile")
fname_out = join(dump_folder, f"los_{catalogue}_IndranilVoid_{kind}.hdf5")
fname_out = join(
dump_folder, f"los_{catalogue}_IndranilVoid_{kind.lower()}.hdf5")
if exists(fname_out):
print(f"Fname `{fname_out}` already exists. Removing.")
remove(fname_out)
print(f"Writing to `{fname_out}`.")
for k in trange(len(nsims), desc="LG observers"):
nsim = nsims[k]
@ -69,7 +75,7 @@ def interpolate_indranil_void(kind, nsims, RA, dec, rmax, dr, dump_folder,
# Write the output, homogenous density.
density = np.ones_like(result)
with File(fname_out, 'w') as f_out:
with File(fname_out, 'a') as f_out:
f_out.create_dataset(f"rdist_{k}", data=r_eval * 0.674)
f_out.create_dataset(f"density_{k}", data=density)
f_out.create_dataset(f"velocity_{k}", data=result)
@ -81,19 +87,21 @@ def interpolate_indranil_void(kind, nsims, RA, dec, rmax, dr, dump_folder,
if __name__ == "__main__":
kind = "gauss"
parser = ArgumentParser()
parser.add_argument("--kind", type=str, choices=["exp", "gauss", "mb"],
help="Kind of void profile.")
parser.add_argument("--catalogue", type=str, help="Catalogue name.")
args = parser.parse_args()
rmax = 165
dr = 1
dr = 0.75
comm = MPI.COMM_WORLD
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
nsims = paths.get_ics(f"IndranilVoid_{kind}")
nsims = paths.get_ics(f"IndranilVoid_{args.kind}")
out_folder = "/mnt/extraspace/rstiskalek/csiborg_postprocessing/field_los"
for catalogue in ["LOSS", "Foundation", "2MTF", "SFI_gals", "CF4_TFR"]:
print(f"Running kind `{kind}` for catalogue `{catalogue}`.")
RA, dec = get_los(catalogue, "", comm).T
interpolate_indranil_void(
kind, nsims, RA, dec, rmax, dr, out_folder, catalogue)
print(f"Running kind `{args.kind}` for catalogue `{args.catalogue}`.")
RA, dec = get_los(args.catalogue, "", comm).T
interpolate_indranil_void(
args.kind, nsims, RA, dec, rmax, dr, out_folder, args.catalogue)

View file

@ -0,0 +1,32 @@
nthreads=1
memory=7
on_login=${1}
queue="berg"
env="/mnt/users/rstiskalek/csiborgtools/venv_csiborg/bin/python"
file="field_los_indranil_void.py"
if [ "$on_login" != "1" ] && [ "$on_login" != "0" ]
then
echo "'on_login' (1) must be either 0 or 1."
exit 1
fi
for kind in "exp" "gauss" "mb"; do
for catalogue in "2MTF" "SFI_gals" "CF4_TFR"; do
pythoncm="$env $file --kind $kind --catalogue $catalogue"
if [ $on_login -eq 1 ]; then
echo $pythoncm
eval $pythoncm
else
cm="addqueue -q $queue -n $nthreads -m $memory $pythoncm"
echo "Submitting:"
echo $cm
echo
eval $cm
fi
done
done

View file

@ -22,6 +22,19 @@ from argparse import ArgumentParser, ArgumentTypeError
def none_or_int(value):
if value.lower() == "none":
return None
if "_" in value:
args = value.split("_")
if len(args) == 2:
k0, kf = args
dk = 1
elif len(args) == 3:
k0, kf, dk = args
else:
raise ArgumentTypeError(f"Invalid length of arguments: `{value}`.")
return [int(k) for k in range(int(k0), int(kf), int(dk))]
try:
return int(value)
except ValueError:
@ -73,17 +86,17 @@ def print_variables(names, variables):
print(flush=True)
def get_models(get_model_kwargs, mag_selection, verbose=True):
def get_models(ksim, get_model_kwargs, mag_selection, verbose=True):
"""Load the data and create the NumPyro models."""
paths = csiborgtools.read.Paths(**csiborgtools.paths_glamdring)
folder = "/mnt/extraspace/rstiskalek/catalogs/"
nsims = paths.get_ics(ARGS.simname)
if ARGS.ksim is None:
if ksim is None:
nsim_iterator = [i for i in range(len(nsims))]
else:
nsim_iterator = [ARGS.ksim]
nsims = [nsims[ARGS.ksim]]
nsim_iterator = [ksim]
nsims = [nsims[ksim]]
if verbose:
print(f"{'Simulation:':<20} {ARGS.simname}")
@ -114,7 +127,7 @@ def get_models(get_model_kwargs, mag_selection, verbose=True):
models[i] = csiborgtools.flow.get_model(
loader, mag_selection=mag_selection[i], **get_model_kwargs)
print(f"\n{'Num. radial steps':<20} {len(loader.rdist)}\n", flush=True)
fprint(f"num. radial steps is {len(loader.rdist)}")
return models
@ -143,7 +156,7 @@ def run_model(model, nsteps, nburn, model_kwargs, out_folder,
raise AttributeError("The models must have an attribute `ndata` "
"indicating the number of data points.") from e
nuts_kernel = NUTS(model, init_strategy=init_to_median(num_samples=1000))
nuts_kernel = NUTS(model, init_strategy=init_to_median(num_samples=10000))
mcmc = MCMC(nuts_kernel, num_warmup=nburn, num_samples=nsteps)
rng_key = jax.random.PRNGKey(42)
@ -283,11 +296,14 @@ if __name__ == "__main__":
num_epochs = 50
inference_method = "mike"
mag_selection = None
sample_alpha = True
sample_alpha = False if "IndranilVoid_" in ARGS.simname else True
sample_beta = None
sample_Vext = None
sample_Vmono = False
sample_mag_dipole = False
absolute_calibration = None
calculate_harmonic = False if inference_method == "bayes" else True
sample_h = True if absolute_calibration is not None else False
fname_kwargs = {"inference_method": inference_method,
"smooth": ARGS.ksmooth,
@ -297,8 +313,10 @@ if __name__ == "__main__":
"mag_selection": mag_selection,
"sample_alpha": sample_alpha,
"sample_beta": sample_beta,
"sample_Vext": sample_Vext,
"sample_Vmono": sample_Vmono,
"sample_mag_dipole": sample_mag_dipole,
"absolute_calibration": absolute_calibration,
}
main_params = {"nsteps": nsteps, "nburn": nburn,
@ -310,6 +328,8 @@ if __name__ == "__main__":
"num_epochs": num_epochs,
"inference_method": inference_method,
"sample_mag_dipole": sample_mag_dipole,
"absolute_calibration": absolute_calibration,
"sample_h": sample_h,
}
print_variables(main_params.keys(), main_params.values())
@ -335,8 +355,11 @@ if __name__ == "__main__":
"Vmono_min": -1000, "Vmono_max": 1000,
"beta_min": -10.0, "beta_max": 10.0,
"sigma_v_min": 1.0, "sigma_v_max": 750.,
"h_min": 0.01, "h_max": 5.0,
"sample_Vext": True if sample_Vext is None else sample_Vext, # noqa
"sample_Vmono": sample_Vmono,
"sample_beta": sample_beta,
"sample_h": sample_h,
}
print_variables(
calibration_hyperparams.keys(), calibration_hyperparams.values())
@ -353,17 +376,34 @@ if __name__ == "__main__":
###########################################################################
get_model_kwargs = {"zcmb_min": zcmb_min, "zcmb_max": zcmb_max}
models = get_models(get_model_kwargs, mag_selection)
model_kwargs = {
"models": models,
"field_calibration_hyperparams": calibration_hyperparams,
"distmod_hyperparams_per_model": distmod_hyperparams_per_catalogue,
"inference_method": inference_method,
get_model_kwargs = {
"zcmb_min": zcmb_min,
"zcmb_max": zcmb_max,
"absolute_calibration": absolute_calibration,
"calibration_fpath": "/mnt/extraspace/rstiskalek/catalogs/PV/CF4/CF4_TF_calibration.hdf5", # noqa
}
model = csiborgtools.flow.PV_validation_model
# In case we want to run multiple simulations independently.
if not isinstance(ARGS.ksim, list):
ksim_iterator = [ARGS.ksim]
else:
ksim_iterator = ARGS.ksim
run_model(model, nsteps, nburn, model_kwargs, out_folder,
calculate_harmonic, nchains_harmonic, num_epochs, kwargs_print,
fname_kwargs)
for i, ksim in enumerate(ksim_iterator):
if len(ksim_iterator) > 1:
print(f"{'Current simulation:':<20} {i + 1} ({ksim}) out of {len(ksim_iterator)}.") # noqa
fname_kwargs["nsim"] = ksim
models = get_models(ksim, get_model_kwargs, mag_selection)
model_kwargs = {
"models": models,
"field_calibration_hyperparams": calibration_hyperparams,
"distmod_hyperparams_per_model": distmod_hyperparams_per_catalogue,
"inference_method": inference_method,
}
model = csiborgtools.flow.PV_validation_model
run_model(model, nsteps, nburn, model_kwargs, out_folder,
calculate_harmonic, nchains_harmonic, num_epochs,
kwargs_print, fname_kwargs)

View file

@ -37,42 +37,36 @@ else
fi
# for simname in "CLONES"; do
# for simname in "csiborg2_main" "csiborg2X" ; do
# for simname in "carrick2015" "lilow2024" "csiborg2_main" "csiborg2x" "cf4" "clones"; do
for simname in "Carrick2015" "csiborg2_main"; do
# for simname in "Carrick2015" "csiborg2X" "csiborg2_main"; do
# for simname in "Carrick2015"; do
# for catalogue in "LOSS" "Foundation" "2MTF" "SFI_gals" "CF4_TFR_i" "CF4_TFR_w1"; do
for catalogue in "SFI_gals" "2MTF" "CF4_TFR_i"; do
for simname in "IndranilVoid_exp" "IndranilVoid_gauss" "IndranilVoid_mb"; do
# for catalogue in "2MTF" "SFI_gals" "CF4_TFR_i" "CF4_TFR_w1"; do
# for catalogue in "CF4_TFR_i" "CF4_TFR_w1"; do
# for catalogue in "2MTF" "SFI" "CF4_TFR_not2MTForSFI_i"; do
# for catalogue in "2MTF" "SFI_gals" "CF4_TFR_i"; do
# for catalogue in "CF4_TFR_w1"; do
# for catalogue in "CF4_GroupAll"; do
for ksmooth in 1 2 3 4; do
for ksim in "none"; do
for catalogue in "2MTF" "SFI_gals" "CF4_TFR_i" "CF4_TFR_w1"; do
# for ksim in "none"; do
# for ksim in 0; do
# for ksim in $(seq 0 5 500); do
for ksim in "0_100_5" "100_200_5" "200_300_5" "300_400_5" "400_500_5"; do
# for ksim in {0..500}; do
pythoncm="$env $file --catalogue $catalogue --simname $simname --ksim $ksim --ksmooth $ksmooth --ndevice $ndevice --device $device"
for ksmooth in 0; do
pythoncm="$env $file --catalogue $catalogue --simname $simname --ksim $ksim --ksmooth $ksmooth --ndevice $ndevice --device $device"
if [ "$on_login" == "1" ]; then
echo $pythoncm
eval $pythoncm
else
if [ "$device" == "gpu" ]; then
cm="addqueue -q $queue -s -m $memory --gpus 1 --gputype $gputype $pythoncm"
if [ "$on_login" == "1" ]; then
echo $pythoncm
eval $pythoncm
else
cm="addqueue -s -q $queue -n 1 -m $memory $pythoncm"
if [ "$device" == "gpu" ]; then
cm="addqueue -q $queue -s -m $memory --gpus 1 --gputype $gputype $pythoncm"
else
cm="addqueue -s -q $queue -n 1 -m $memory $pythoncm"
fi
echo "Submitting:"
echo $cm
eval $cm
fi
echo "Submitting:"
echo $cm
eval $cm
fi
echo
sleep 0.01
echo
sleep 0.001
done
done
done
done
done