Aquila-Consortium/borg_public
32
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Initial import

Browse source
This commit is contained in:
Guilhem Lavaux 2023-05-29 10:41:03 +02:00
commit 56a50eead3
820 changed files with 192077 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

358
extra/hmclet/example/2mpp.ini.txt Normal file
View file

@ -0,0 +1,358 @@
[system]
console_output=logares.txt
mask_precision=0.1
VERBOSE_LEVEL = 2
N0 = 32
N1 = 32
N2 = 32
L0 = 677.7
L1 = 677.7
L2 = 677.7
corner0 = -338.85
corner1 = -338.85
corner2 = -338.85
NUM_MODES=100
N_MC=1000
test_mode=true
# If true, the initial power spectrum of the chain is set to the cosmological one
seed_cpower=true
bias_0_sampler_generic_blocked=false
bias_1_sampler_generic_blocked=false
bias_2_sampler_generic_blocked=false
bias_3_sampler_generic_blocked=false
bias_4_sampler_generic_blocked=true
# Indicate which samplers should be blocked for testing purposes
[block_loop]
# Indicate which samplers should be blocked for testing purposes
#messenger_signal_blocked=false
power_sampler_a_blocked=true
power_sampler_b_blocked=true
power_sampler_c_blocked=true
#bias_sampler_blocked=false
hades_sampler_blocked=false
ares_heat=1.0
[gravity]
#model=HADES_PT
model=LPT_CIC
supersampling=2
forcesampling=2
pm_nsteps=30
pm_start_z=69
lightcone=false
do_rsd=false
[mcmc]
number_to_generate=100
random_ic=false
init_random_scaling=1.0
[julia]
likelihood_path=test_like.jl
likelihood_module=julia_test
bias_sampler_type=hmclet
ic_in_julia=true
#hmclet_matrix=QN_DIAGONAL
hmclet_matrix=DIAGONAL
hmclet_frozen=true
#hmclet_burnin=400
#hmclet_burnin_memory=50
hmclet_maxEpsilon=0.1
#hmclet_maxEpsilon=0.01
hmclet_maxNtime=100
hmclet_massScale = 0
#hmclet_correlationLimiter=0.5
[hades]
likelihood=GENERIC_POISSON_BROKEN_POWERLAW_BIAS
#likelihood=BORG_POISSON
algorithm=HMC
max_epsilon=0.01
max_timesteps=50
mixing=1
[run]
NCAT = 1
[cosmology]
fnl = 0
omega_r = 0
omega_k = 0
omega_m = 0.3175
omega_b = 0.049
omega_q = 0.6825
w = -1
wprime = 0
n_s = 0.9624
sigma8 = 0.8344
h100 = 0.6711
beta = 1.5
z0 = 0
# 11.5 mag cut
[catalog_0]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
#maskdata = one.fits
#maskdata = zero.fits
bias=100,1,0,0.005
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -21.50
galaxy_faint_absolute_magnitude_cut = -21.00
refbias = false
nmean=1
[catalog_1]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -22.00
galaxy_faint_absolute_magnitude_cut = -21.50
refbias = false
nmean=1
[catalog_2]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -22.50
galaxy_faint_absolute_magnitude_cut = -22.00
refbias = false
nmean=1
[catalog_3]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -23.00
galaxy_faint_absolute_magnitude_cut = -22.50
refbias = false
nmean=1
[catalog_4]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -23.50
galaxy_faint_absolute_magnitude_cut = -23.00
refbias = false
nmean=1
[catalog_5]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -24.00
galaxy_faint_absolute_magnitude_cut = -23.50
refbias = false
nmean=1
[catalog_6]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -24.50
galaxy_faint_absolute_magnitude_cut = -24.00
refbias = false
nmean=1
[catalog_7]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -25.00
galaxy_faint_absolute_magnitude_cut = -24.50
refbias = false
nmean=1
# 11.5 - 12.5 mag cut
[catalog_8]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -21.50
galaxy_faint_absolute_magnitude_cut = -21.00
refbias = false
nmean=1
[catalog_9]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -22.00
galaxy_faint_absolute_magnitude_cut = -21.50
refbias = false
nmean=1.
[catalog_10]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -22.50
galaxy_faint_absolute_magnitude_cut = -22.00
refbias = false
nmean=1
[catalog_11]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -23.00
galaxy_faint_absolute_magnitude_cut = -22.50
refbias = false
nmean=1
[catalog_12]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -23.50
galaxy_faint_absolute_magnitude_cut = -23.00
refbias = false
nmean=1
[catalog_13]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -24.00
galaxy_faint_absolute_magnitude_cut = -23.50
refbias = false
nmean=1
[catalog_14]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -24.50
galaxy_faint_absolute_magnitude_cut = -24.00
refbias = false
nmean=1
[catalog_15]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -25.00
galaxy_faint_absolute_magnitude_cut = -24.50
refbias = false
nmean=1

334
extra/hmclet/example/2mpp_TF.ini.txt Normal file
View file

@ -0,0 +1,334 @@
[system]
console_output=logares.txt
mask_precision=0.7
VERBOSE_LEVEL = 3
N0 = 32
N1 = 32
N2 = 32
L0 = 677.7
L1 = 677.7
L2 = 677.7
corner0 = -338.85
corner1 = -338.85
corner2 = -338.85
NUM_MODES=100
N_MC=100
borg_supersampling=1
borg_forcesampling=1
borg_pm_nsteps=30
borg_pm_start_z=69
borg_lightcone=false
borg_do_rsd=false
hades_forward_model=LPT_CIC
hades_likelihood=BORG_POISSON
seed = 1234
test_mode=true
# If true, the initial power spectrum of the chain is set to the cosmological one
seed_cpower=true
# Indicate which samplers should be blocked for testing purposes
#messenger_signal_blocked=false
power_sampler_a_blocked=true
power_sampler_b_blocked=true
power_sampler_c_blocked=true
#bias_sampler_blocked=false
hades_sampler_blocked=false
hades_max_epsilon=0.05
hades_max_timesteps=20
hades_mixing=1
savePeriodicity=10
[julia]
likelihood_path=test_likelihood_TF.jl
likelihood_module=network
bias_sampler_type=hmclet
[run]
NCAT = 1
[cosmology]
omega_r = 0
omega_k = 0
omega_m = 0.3175
omega_b = 0.049
omega_q = 0.6825
w = -1
wprime = 0
n_s = 0.9624
sigma8 = 0.8344
h100 = 0.6711
beta = 1.5
z0 = 0
# 11.5 mag cut
[catalog_0]
datafile = 2MPP.txt
#maskdata = completeness_11_5.fits.gz
maskdata = one.fits
#maskdata = zero.fits
bias=1
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 15.5
galaxy_bright_absolute_magnitude_cut = -25.50
galaxy_faint_absolute_magnitude_cut = -11.00
refbias = false
nmean=1
[catalog_1]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -22.00
galaxy_faint_absolute_magnitude_cut = -21.50
refbias = false
nmean=1
[catalog_2]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -22.50
galaxy_faint_absolute_magnitude_cut = -22.00
refbias = false
nmean=1
[catalog_3]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -23.00
galaxy_faint_absolute_magnitude_cut = -22.50
refbias = false
nmean=1
[catalog_4]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -23.50
galaxy_faint_absolute_magnitude_cut = -23.00
refbias = false
nmean=1
[catalog_5]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -24.00
galaxy_faint_absolute_magnitude_cut = -23.50
refbias = false
nmean=1
[catalog_6]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -24.50
galaxy_faint_absolute_magnitude_cut = -24.00
refbias = false
nmean=1
[catalog_7]
datafile = 2MPP.txt
maskdata = completeness_11_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 9
galaxy_faint_apparent_magnitude_cut = 11.5
galaxy_bright_absolute_magnitude_cut = -25.00
galaxy_faint_absolute_magnitude_cut = -24.50
refbias = false
nmean=1
# 11.5 - 12.5 mag cut
[catalog_8]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -21.50
galaxy_faint_absolute_magnitude_cut = -21.00
refbias = false
nmean=1
[catalog_9]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -22.00
galaxy_faint_absolute_magnitude_cut = -21.50
refbias = false
nmean=1.
[catalog_10]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -22.50
galaxy_faint_absolute_magnitude_cut = -22.00
refbias = false
nmean=1
[catalog_11]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -23.00
galaxy_faint_absolute_magnitude_cut = -22.50
refbias = false
nmean=1
[catalog_12]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -23.50
galaxy_faint_absolute_magnitude_cut = -23.00
refbias = false
nmean=1
[catalog_13]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -24.00
galaxy_faint_absolute_magnitude_cut = -23.50
refbias = false
nmean=1
[catalog_14]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -24.50
galaxy_faint_absolute_magnitude_cut = -24.00
refbias = false
nmean=1
[catalog_15]
datafile = 2MPP.txt
maskdata = completeness_12_5.fits.gz
radial_selection = schechter
schechter_mstar = -23.28
schechter_alpha = -0.94
schechter_sampling_rate = 1000
schechter_dmax = 700
galaxy_bright_apparent_magnitude_cut = 11.5
galaxy_faint_apparent_magnitude_cut = 12.5
galaxy_bright_absolute_magnitude_cut = -25.00
galaxy_faint_absolute_magnitude_cut = -24.50
refbias = false
nmean=1

187
extra/hmclet/example/test_like.jl Normal file
View file

@ -0,0 +1,187 @@
#+
# ARES/HADES/BORG Package -- ./extra/hmclet/example/test_like.jl
# Copyright (C) 2014-2020 Guilhem Lavaux <guilhem.lavaux@iap.fr>
# Copyright (C) 2009-2020 Jens Jasche <jens.jasche@fysik.su.se>
#
# Additional contributions from:
# Guilhem Lavaux <guilhem.lavaux@iap.fr> (2023)
#
#+
module julia_test
using ..libLSS
using NPZ
import ..libLSS.State
import ..libLSS.GhostPlanes, ..libLSS.get_ghost_plane
import ..libLSS.print, ..libLSS.LOG_INFO, ..libLSS.LOG_VERBOSE, ..libLSS.LOG_DEBUG
apply_transform(bias_tilde) = exp.(bias_tilde)
apply_inv_transform(bias) = log.(bias)
function initialize(state)
print(LOG_INFO, "Likelihood initialization in Julia")
NCAT = libLSS.get(state, "NCAT", Int64)
print(LOG_VERBOSE, "Found " *repr(NCAT) * " catalogues")
for catalog in 0:(NCAT-1)
# galaxies = libLSS.get_galaxy_descriptor(state, catalog)
# print(LOG_VERBOSE, repr(size(galaxies)))
# all_spin = getfield.(galaxies, :spin)
bias = libLSS.resize_array(state, "galaxy_bias_"*repr(catalog), 2, Float64)
bias[1] = 1
bias[2] = 0.01
bias .= apply_inv_transform(bias)
end
end
function get_required_planes(state::State)
print(LOG_INFO, "Check required planes")
return Array{UInt64,1}([])
end
function likelihood(state::State, ghosts::GhostPlanes, array::AbstractArray{Float64,3})
print(LOG_VERBOSE, "Likelihood evaluation in Julia")
N0 = libLSS.get(state, "N0", Int64)
NCAT = libLSS.get(state, "NCAT", Int64)
L = Float64(0)
for catalog in 0:(NCAT-1)
sc = repr(catalog)
b = libLSS.get_array_1d(state, "galaxy_bias_"*sc, Float64)
L += likelihood_bias(state, ghosts, array, catalog, b)
end
print(LOG_VERBOSE, "Likelihood is " * repr(L))
return L
end
function generate_mock_data(state::State, ghosts::GhostPlanes, array::AbstractArray{Float64,3})
print(LOG_INFO, "Generate mock")
NCAT = libLSS.get(state, "NCAT", Int64)
for cat in 0:(NCAT-1)
sc = repr(cat)
data = libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64)
b = apply_transform(libLSS.get_array_1d(state, "galaxy_bias_"*sc, Float64))
print(LOG_VERBOSE, "Bias for mock is $(b)")
S = libLSS.get_array_3d(state, "galaxy_sel_window_$(sc)", Float64)
s = size(data)
print(LOG_INFO, "Shape is " * repr(size(data)) * " and " * repr(size(array)))
print(LOG_INFO, "Number of threads " * repr(Threads.nthreads()))
N0=s[1]
N1=s[2]
N2=s[3]
noise = sqrt(b[2])
print(LOG_INFO, "Noise is $(noise)")
bias = b[1]
for i=1:N0,j=1:N1,k=1:N2
data[i,j,k] = S[i,j,k]*(1+bias*array[i,j,k]) + sqrt(S[i,j,k])*noise*libLSS.gaussian(state)
end
end
end
function adjoint_gradient(state::State, array::AbstractArray{Float64,3}, ghosts::GhostPlanes, ag::AbstractArray{Float64,3})
print(LOG_VERBOSE, "Adjoint gradient in Julia")
N0 = libLSS.get(state, "N0", Int64)
NCAT = libLSS.get(state, "NCAT", Int64)
L = Float64(0)
ag[:,:,:] .= 0
for catalog in 0:(NCAT-1)
sc = repr(catalog)
data = libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64)
b = apply_transform(libLSS.get_array_1d(state, "galaxy_bias_"*sc, Float64))
S = libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64)
noise = b[2]
bias = b[1]
Smask = findall(S.>0)
ag[Smask] += -(data[Smask] .- S[Smask].*(1 .+ bias*array[Smask]))*bias/noise
end
end
# 1/2 sum( (data - S (1 + b rho))^2 / (S*n) )
# There is a change of variable to map [-infinity, infinity] to [0, infinity]
# y = exp(x) (x is the bias_tilde, y is the bias params)
# we know the function in terms of y though, but the posterior must be in terms of x
# probability conservation:
# f_tilde(x) dx = f(y) dy
#
# f_tilde(x) = f(y) dy/dx = f(y) exp(x) -> -log(f_tilde) = -log(f) - y
# -dlog(f_tilde(x))/dx = -dlog(f_tilde)/dy dy/dx = (-dlog(f)/dy - 1) * dy/dx
# dy/dx = exp(x) = y
function likelihood_bias(state::State, ghosts::GhostPlanes, array, catalog_id, catalog_bias_tilde)
catalog_bias = apply_transform(catalog_bias_tilde)
sc = string(catalog_id)
print(LOG_VERBOSE,"Catalog id is " * sc * " bias is " * repr(catalog_bias))
data = libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64)
S = libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64)
Smask = findall(S.>0)
noise = catalog_bias[2]
bias = catalog_bias[1]
prior_bias = catalog_bias_tilde[1] # Not the bias-tilde-2
return 0.5*sum(
(data[Smask] .- S[Smask].*(1 .+ bias.*array[Smask])).^2 ./ (S[Smask].*noise)
) + 0.5*size(Smask)[1]*log(noise) - prior_bias
end
function get_step_hint(state, catalog_id, bias_id)
return 0.1
end
function log_prior_bias(state, catalog_id, bias_tilde)
# Change of variable bias = exp(bias_tilde)
return 0
end
function adjoint_bias(state::State, ghosts::GhostPlanes,
array, catalog_id, catalog_bias_tilde, adjoint_gradient_bias)
catalog_bias = apply_transform(catalog_bias_tilde)
print(LOG_VERBOSE,"ADJOINT: Catalog id is $(catalog_id), bias is $(catalog_bias), bias_tilde is $(catalog_bias_tilde)")
sc = string(catalog_id)
data = libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64)
S = libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64)
Smask = findall(S.>0)
noise = catalog_bias[2]
bias = catalog_bias[1]
delta = (data[Smask] .- S[Smask].*(1 .+ bias*array[Smask]))
adjoint_gradient_bias[1] = -sum(delta.*array[Smask]) ./noise
adjoint_gradient_bias[2] = -0.5*sum(delta.^2 ./ (S[Smask])) /(noise^2) + 0.5 * size(Smask)[1]/noise
adjoint_gradient_bias .*= catalog_bias
adjoint_gradient_bias[1] -= 1 # Derivative of the prior
print(LOG_VERBOSE,"ADJOINT: -> $(adjoint_gradient_bias)")
end
function fill_diagonal_mass_matrix(state::State)
return [1e3,1e3]
# return [1e-5,1e-5]
# return [1e-7,1e-7]
end
function generate_ic(state::State)
print(LOG_INFO, "Generate special IC for the chain")
b = libLSS.get_array(state, "galaxy_bias_0", Float64, d1d)
b[1] = 1.
b[2] = 1.
# sref = npzread("velmass_ic_500Mpc_32.npz")["arr_0"]
# s = libLSS.get_array_3d(state, "s_field", Float64)
# s .*= 0.01
# startN0 = libLSS.get(state, "startN0", Int64)
# localN0,N1,N2 = size(s)
# print(LOG_INFO, "Dims = [$(startN0):$(startN0+localN0)]x$(N1)x$(N2)")
# for i=1:localN0, j=1:N1,k=1:N2
# s[i,j,k] = sref[k,j,i+startN0]
##### # 0.01*cos(2*pi*(i-1)/N)*sin(2*pi*(j-1)/N)
# end
# print(LOG_INFO, "DONE DONE")
end
end

139
extra/hmclet/example/test_like_TF.jl Normal file
View file

@ -0,0 +1,139 @@
#+
# ARES/HADES/BORG Package -- ./extra/hmclet/example/test_like_TF.jl
# Copyright (C) 2014-2020 Guilhem Lavaux <guilhem.lavaux@iap.fr>
# Copyright (C) 2009-2020 Jens Jasche <jens.jasche@fysik.su.se>
#
# Additional contributions from:
# Guilhem Lavaux <guilhem.lavaux@iap.fr> (2023)
#
#+
module network
using ..libLSS
import ..libLSS.State
import ..libLSS.GhostPlanes, ..libLSS.get_ghost_plane
import ..libLSS.print, ..libLSS.LOG_INFO, ..libLSS.LOG_VERBOSE, ..libLSS.LOG_DEBUG
using TensorFlow
using PyPlot
sess = Session(allow_growth = true)
adgrad = nothing
wgrad = nothing
function setup(N0, N1, N2)
global adgrad, wgrad
p = [TensorFlow.placeholder(Float64, shape = (), name = "bias"), TensorFlow.placeholder(Float64, shape = (), name = "noise")]
δ = TensorFlow.placeholder(Float64, shape = Int64[N0, N1, N2], name = "density")
g = TensorFlow.placeholder(Float64, shape = Int64[N0, N1, N2], name = "galaxy")
s = TensorFlow.placeholder(Float64, shape = Int64[N0, N1, N2], name = "selection")
gaussian = TensorFlow.placeholder(Float64, shape = Int64[N0, N1, N2], name = "gaussian_field")
mask = TensorFlow.placeholder(Bool, shape = Int64[N0, N1, N2], name = "mask")
mask_ = TensorFlow.reshape(mask, N0 * N1 * N2, name = "flat_mask")
g_ = TensorFlow.identity(TensorFlow.boolean_mask(TensorFlow.reshape(g, N0 * N1 * N2), mask_), name = "flat_masked_galaxy")
s_ = TensorFlow.identity(TensorFlow.boolean_mask(TensorFlow.reshape(s, N0 * N1 * N2), mask_), name = "flat_masked_selection")
output = TensorFlow.add(1., TensorFlow.multiply(p[1], δ), name = "biased_density")
mock = TensorFlow.multiply(s, output, name = "selected_biased_density")
mock_ = TensorFlow.identity(TensorFlow.boolean_mask(TensorFlow.reshape(mock, N0 * N1 * N2), mask_), name = "flat_masked_selected_biased_density")
mock_galaxy = TensorFlow.add(mock, TensorFlow.multiply(TensorFlow.multiply(TensorFlow.sqrt(TensorFlow.exp(p[2])), TensorFlow.sqrt(s)), gaussian), name = "mock_galaxy")
ms = TensorFlow.reduce_sum(TensorFlow.cast(mask, Float64), name = "number_of_voxels")
loss = TensorFlow.identity(TensorFlow.add(TensorFlow.multiply(0.5, TensorFlow.reduce_sum(TensorFlow.square(g_ - mock_) / TensorFlow.multiply(TensorFlow.exp(p[2]), s_))), TensorFlow.multiply(0.5, TensorFlow.multiply(ms, p[2]))) - TensorFlow.exp(p[1]) - TensorFlow.exp(p[2]), name = "loss")
adgrad = TensorFlow.gradients(loss, δ)
wgrad = [TensorFlow.gradients(loss, p[i]) for i in range(1, length = size(p)[1])]
end
function initialize(state)
print(LOG_INFO, "Likelihood initialization in Julia")
setup(libLSS.get(state, "N0", Int64, synchronous=true), libLSS.get(state, "N1", Int64, synchronous=true), libLSS.get(state, "N2", Int64, synchronous=true))
bias = libLSS.resize_array(state, "galaxy_bias_0", 2, Float64)
bias[:] .= log(1.)
print(LOG_VERBOSE, "Found " *repr(libLSS.get(state, "NCAT", Int64, synchronous=true)) * " catalogues")
end
function get_required_planes(state::State)
print(LOG_INFO, "Check required planes")
return Array{UInt64,1}([])
end
function likelihood(state::State, ghosts::GhostPlanes, array::AbstractArray{Float64,3})
print(LOG_INFO, "Likelihood evaluation in Julia")
L = Float64(0.)
for catalog=1:libLSS.get(state, "NCAT", Int64, synchronous=true)
L += run(sess, TensorFlow.get_tensor_by_name("loss"),
Dict(TensorFlow.get_tensor_by_name("bias")=>libLSS.get_array_1d(state, "galaxy_bias_"*repr(catalog - 1), Float64)[1],
TensorFlow.get_tensor_by_name("noise")=>libLSS.get_array_1d(state, "galaxy_bias_"*repr(catalog - 1), Float64)[2],
TensorFlow.get_tensor_by_name("density")=>array,
TensorFlow.get_tensor_by_name("galaxy")=>libLSS.get_array_3d(state, "galaxy_data_"*repr(catalog - 1), Float64),
TensorFlow.get_tensor_by_name("selection")=>libLSS.get_array_3d(state, "galaxy_sel_window_"*repr(catalog - 1), Float64),
TensorFlow.get_tensor_by_name("mask")=>libLSS.get_array_3d(state, "galaxy_sel_window_"*repr(catalog - 1), Float64).>0.))
end
print(LOG_VERBOSE, "Likelihood is " * repr(L))
return L
end
function generate_mock_data(state::State, ghosts::GhostPlanes, array::AbstractArray{Float64,3})
print(LOG_INFO, "Generate mock")
for catalog in 1:libLSS.get(state, "NCAT", Int64, synchronous=true)
gaussian_field = Array{Float64}(undef, size(array)[1], size(array)[2], size(array)[3])
data = libLSS.get_array_3d(state, "galaxy_data_"*repr(catalog - 1), Float64)
for i=1:size(array)[1],j=1:size(array)[2],k=1:size(array)[3]
gaussian_field[i,j,k] = libLSS.gaussian(state)
end
data[:, :, :] = run(sess, TensorFlow.get_tensor_by_name("mock_galaxy"),
Dict(TensorFlow.get_tensor_by_name("bias")=>libLSS.get_array_1d(state, "galaxy_bias_"*repr(catalog - 1), Float64)[1],
TensorFlow.get_tensor_by_name("noise")=>libLSS.get_array_1d(state, "galaxy_bias_"*repr(catalog - 1), Float64)[2],
TensorFlow.get_tensor_by_name("density")=>array,
TensorFlow.get_tensor_by_name("selection")=>libLSS.get_array_3d(state, "galaxy_sel_window_"*repr(catalog - 1), Float64),
TensorFlow.get_tensor_by_name("gaussian_field")=>gaussian_field))
print(LOG_INFO, "Plotting generated mock from catalog "*repr(catalog - 1)*" as ./plots/generate_mock_data_"*repr(catalog - 1)*".png")
imshow(dropdims(sum(data, dims = 3), dims = 3))
colorbar()
savefig("plots/generated_mock_data_"*repr(catalog - 1)*".png")
close()
end
end
function adjoint_gradient(state::State, array::AbstractArray{Float64,3}, ghosts::GhostPlanes, ag::AbstractArray{Float64,3})
print(LOG_VERBOSE, "Adjoint gradient in Julia")
ag[:,:,:] .= 0
for catalog=1:libLSS.get(state, "NCAT", Int64, synchronous=true)
Smask = libLSS.get_array_3d(state, "galaxy_sel_window_"*repr(catalog - 1), Float64).>0.
ag[Smask] += run(sess, adgrad,
Dict(TensorFlow.get_tensor_by_name("bias")=>libLSS.get_array_1d(state, "galaxy_bias_"*repr(catalog - 1), Float64)[1],
TensorFlow.get_tensor_by_name("noise")=>libLSS.get_array_1d(state, "galaxy_bias_"*repr(catalog - 1), Float64)[2],
TensorFlow.get_tensor_by_name("density")=>array, TensorFlow.get_tensor_by_name("galaxy")=>libLSS.get_array_3d(state, "galaxy_data_"*repr(catalog - 1), Float64),
TensorFlow.get_tensor_by_name("selection")=>libLSS.get_array_3d(state, "galaxy_sel_window_"*repr(catalog - 1), Float64),
TensorFlow.get_tensor_by_name("mask")=>Smask))[Smask]
end
end
function likelihood_bias(state::State, ghosts::GhostPlanes, array, catalog_id, catalog_bias)
print(LOG_VERBOSE, "Likelihood bias in Julia")
return run(sess, TensorFlow.get_tensor_by_name("loss"),
Dict(TensorFlow.get_tensor_by_name("bias")=>catalog_bias[1],
TensorFlow.get_tensor_by_name("noise")=>catalog_bias[2],
TensorFlow.get_tensor_by_name("density")=>array,
TensorFlow.get_tensor_by_name("galaxy")=>libLSS.get_array_3d(state, "galaxy_data_"*string(catalog_id), Float64),
TensorFlow.get_tensor_by_name("selection")=>libLSS.get_array_3d(state, "galaxy_sel_window_"*string(catalog_id), Float64),
TensorFlow.get_tensor_by_name("mask")=>libLSS.get_array_3d(state, "galaxy_sel_window_"*string(catalog_id), Float64) .> 0.))
end
function get_step_hint(state, catalog_id, bias_id)
return 0.1
end
function log_prior_bias(state, catalog_id, bias_tilde)
return 0.
end
function adjoint_bias(state::State, ghosts::GhostPlanes, array, catalog_id, catalog_bias, adjoint_gradient_bias)
print(LOG_VERBOSE, "Adjoint gradient of bias in Julia")
adjoint_gradient_bias .= run(sess, wgrad,
Dict(TensorFlow.get_tensor_by_name("bias")=>catalog_bias[1],
TensorFlow.get_tensor_by_name("noise")=>catalog_bias[2],
TensorFlow.get_tensor_by_name("density")=>array,
TensorFlow.get_tensor_by_name("galaxy")=>libLSS.get_array_3d(state, "galaxy_data_"*string(catalog_id), Float64),
TensorFlow.get_tensor_by_name("selection")=>libLSS.get_array_3d(state, "galaxy_sel_window_"*string(catalog_id), Float64),
TensorFlow.get_tensor_by_name("mask")=>libLSS.get_array_3d(state, "galaxy_sel_window_"*string(catalog_id), Float64) .> 0.))
end
end