Aquila-Consortium/borg_public
32
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Initial import

Browse source
This commit is contained in:
Guilhem Lavaux 2023-05-29 10:41:03 +02:00
commit 56a50eead3
820 changed files with 192077 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

118
extra/hmclet/libLSS/tests/convHMC.jl Normal file
View file

@ -0,0 +1,118 @@
module convHMC
using TensorFlow
sess = 0; p = 0; δ = 0; g = 0; s = 0; n = 0; sel = 0; loss = 0; error = 0; ag = 0; output = 0;
function isotropic_weights(params, C0, C1, C2)
out_edge = stack([params[4], params[3], params[4]])
out_face = stack([params[3], params[2], params[3]])
inner = stack([params[2], params[1], params[2]])
face = stack([out_edge, out_face, out_edge])
middle = stack([out_face, inner, out_face])
return reshape(stack([face, middle, face]), (C0, C1, C2, 1, 1))
end
function get_isotropic_weights(num_layers, kernel)
w = Array{Any}(num_layers)
b = Array{Any}(num_layers)
for i = 1:num_layers
w[i] = isotropic_weights(p[(i - 1) * 5 + 1: (i - 1) * 5 + 4], kernel[1], kernel[2], kernel[3])
b[i] = p[i * 5]
end
return w, b
end
function get_3d_conv(num_layers, kernel)
w = Array{Any}(num_layers)
b = Array{Any}(num_layers)
for i = 1:num_layers
w[i] = reshape(p[(i - 1) * 28 + 1: (i - 1) * 28 + 27], (kernel[1], kernel[2], kernel[3], 1, 1))
b[i] = p[i * 28]
end
return w, b
end
function convolutional_network(x, w, b, num_layers, N0, N1, N2)
for i = 1:num_layers
x = nn.relu(nn.conv3d(x, w[i], strides = [1, 1, 1, 1, 1], padding = "SAME") + b[i]) + x
end
x = nn.relu(x)
return reshape(x, (N0, N1, N2))
end
function mse(x, g_, s_, n_, sel_, loss_params)
N0 = loss_params[1]
N1 = loss_params[2]
N2 = loss_params[3]
x = boolean_mask(reshape(x, N0 * N1 * N2), sel_)
return reduce_sum(0.5 * (multiply(x, s_) - g_)^2. / multiply(n_, s_) + 0.5 * log(n_))
end
function get_poisson_bias(_, __)
return -99, -99
end
function no_network(x, _, __, ___, ____, _____, ______)
return x
end
function poisson_bias(x, g_, s_, n_, sel_, loss_params)
N0 = loss_params[1]
N1 = loss_params[2]
N2 = loss_params[3]
x = boolean_mask(reshape(x, N0 * N1 * N2), sel_)
return reduce_sum((g_ .- s_ .* ( .- p[1] .* x)).^2. / (s_ .* n_))
end
function setup(num_layers, N0, N1, N2, num_params, extras, loss_params, network, get_variables, Λ)
global sess, p, δ, g, s, n, sel, output, loss, ag, error
sess = Session();
p = placeholder(Float64, shape = [num_params])
δ = placeholder(Float64, shape = [N0, N1, N2])
δ_ = reshape(δ, (1, N0, N1, N2, 1))
sel = placeholder(Bool, shape = [N0, N1, N2])
sel_ = reshape(sel, N0 * N1 * N2)
g = placeholder(Float64, shape = [N0, N1, N2])
g_ = boolean_mask(reshape(g, N0 * N1 * N2), sel_)
s = placeholder(Float64, shape = [N0, N1, N2])
s_ = boolean_mask(reshape(s, N0 * N1 * N2), sel_)
n = placeholder(Float64, shape = [1])
n_ = n[1]
w, b = get_variables(num_layers, extras)
output = network(δ_, w, b, num_layers, N0, N1, N2)
loss = Λ(output, g_, s_, n_, sel_, loss_params)
ag = gradients(loss, δ)
#error = gradients(loss, p)
run(sess, global_variables_initializer())
end
function evaluate(params, field, galaxy, selection, noise, mask)
return run(sess, loss, Dict(p => params, δ => field, g => galaxy, s => selection, n => [noise], sel => mask))
end
function adjointGradient(params, field, galaxy, selection, noise, mask)
return run(sess, ag, Dict(p => params, δ => field, g => galaxy, s => selection, n => [noise], sel => mask))
end
#function adjointNetworkGradient(params, field, galaxy, selection, noise, mask)
# gradient = run(sess, error, Dict(p => params, δ => field, g => galaxy, s => selection, n => [noise], sel => mask))
# params_gradient = gradient.values[gradient.indices]
# #println(params_gradient)
# #params_gradient = Array{Float64}(tot_num_conv * 5);
# #for i = 1:tot_num_conv
# # for j = 1:4
# # ind = find(x -> x == j, gradient[(i - 1) * 2 + 1].indices);
# # params_gradient[(i - 1) * 5 + j] = sum(gradient[(i - 1) * 2 + 1].values[ind]);
# # end
# # params_gradient[i * 5] = gradient[i * 2];
# #end
# return params_gradient
#end
function get_field(params, field)
return run(sess, output, Dict(p => params, δ => field));
end
end

43
extra/hmclet/libLSS/tests/network.jl Normal file
View file

@ -0,0 +1,43 @@
using TensorFlow
using Distributions
sess = Session(Graph());
inputs = 3;
θ = placeholder(Float32, shape = [nothing, inputs])
m2lnL = placeholder(Float32, shape = [nothing])
layers = 2;
neurons_per_layer = 50;
α = 0.1;
function network(θ, layers, neurons_per_layer, α)
x = θ
weights = Array{Any}(layers + 1)
biases = Array{Any}(layers + 1)
for i=1:layers
if i == 1
weights[i] = get_variable("layer_" * string(i) * "_weights", [3, neurons_per_layer], Float32, initializer=Normal(0., sqrt(2./3.)))
biases[i] = get_variable("layer_" * string(i) * "_biases", [neurons_per_layer], Float32)
elseif i == layers
weights[i] = get_variable("layer_" * string(i) * "_weights", [neurons_per_layer, 1], Float32, initializer=Normal(0., sqrt(2./neurons_per_layer)))
biases[i] = get_variable("layer_" * string(i) * "_biases", [1], Float32)
else
weights[i] = get_variable("layer_" * string(i) * "_weights", [neurons_per_layer, neurons_per_layer], Float32, initializer=Normal(0., sqrt(2./neurons_per_layer)))
biases[i] = get_variable("layer_" * string(i) * "_biases", [neurons_per_layer], Float32)
end
x = x * weights[i] + biases[i]
x = max(α * x, x)
end
x = reshape(x, (-1))
return x, weights, biases
end
output, weights, biases = network(θ, layers, neurons_per_layer, α)
loss = mean(0.5 * (output / m2lnL - 1)^2)
gradient = gradients(loss, θ);
weight_gradients = [gradients(loss, weights[i]) for i=1:layers];
bias_gradients = [gradients(loss, biases[i]) for i=1:layers];

171
extra/hmclet/libLSS/tests/network/TF_conv.jl Normal file
View file

@ -0,0 +1,171 @@
#+
# ARES/HADES/BORG Package -- ./extra/hmclet/libLSS/tests/network/TF_conv.jl
# Copyright (C) 2014-2020 Guilhem Lavaux <guilhem.lavaux@iap.fr>
# Copyright (C) 2009-2020 Jens Jasche <jens.jasche@fysik.su.se>
#
# Additional contributions from:
# Guilhem Lavaux <guilhem.lavaux@iap.fr> (2023)
#
#+
module network
using libLSS
import libLSS.State
import libLSS.GhostPlanes, libLSS.get_ghost_plane
import libLSS.print, libLSS.LOG_INFO, libLSS.LOG_VERBOSE, libLSS.LOG_DEBUG
using TensorFlow
sess = Session(Graph())
p = nothing
#new_p = nothing
#assign_p = nothing
δ = nothing
g = nothing
s = nothing
mask = nothing
output = nothing
mock = nothing
loss = nothing
adgrad = nothing
wgrad = nothing
function setup(N0, number_of_parameters)
global p, new_p, assign_p, δ, g, s, mask, output, mock, loss, adgrad, wgrad
p = Array{Any}(number_of_parameters)
#new_p = Array{Any}(number_of_parameters)
#assign_p = Array{Any}(number_of_parameters)
for i=1:number_of_parameters
p[i] = placeholder(Float64, shape = [])
#p[i] = Variable(zeros(Float64, 1))
#new_p[i] = placeholder(Float64, shape = [])
#assign_p[i] = assign(p[i], expand_dims(new_p[i], 1))
end
δ = placeholder(Float64, shape = [N0, N0, N0])
g = placeholder(Float64, shape = [N0, N0, N0])
s = placeholder(Float64, shape = [N0, N0, N0])
mask = placeholder(Bool, shape = [N0, N0, N0])
output = build_network(δ, p)
mock = output .* s
loss = 0.5 * sum((boolean_mask(reshape(g, N0^3), reshape(mask, N0^3)) .- boolean_mask(reshape(s, N0^3), reshape(mask, N0^3)) .* boolean_mask(reshape(output, N0^3), reshape(mask, N0^3))).^2. ./(boolean_mask(reshape(s, N0^3), reshape(mask, N0^3)))) + 0.5 * sum(cast(mask, Float64))
adgrad = gradients(loss, δ)
wgrad = Array{Any}(number_of_parameters)
for i=1:number_of_parameters
wgrad[i]= gradients(loss, p[i])
end
run(sess, global_variables_initializer())
end
function build_network(input_tensor, weights)
α = Float64(0.01)
x = nn.conv3d(expand_dims(expand_dims(input_tensor, 4), 5), expand_dims(expand_dims(expand_dims(expand_dims(expand_dims(weights[1], 1), 2), 3), 4), 5), strides = [1, 1, 1, 1, 1], padding = "VALID")
x = x .+ weights[2]
x = max(α .* x, x)
x = nn.conv3d(x, expand_dims(expand_dims(expand_dims(expand_dims(expand_dims(weights[3], 1), 2), 3), 4), 5), strides = [1, 1, 1, 1, 1], padding = "VALID")
x = x .+ weights[4]
x = x + expand_dims(expand_dims(input_tensor, 4), 5)
x = max(α .* x, x)
x_ = nn.conv3d(x, expand_dims(expand_dims(expand_dims(expand_dims(expand_dims(weights[5], 1), 2), 3), 4), 5), strides = [1, 1, 1, 1, 1], padding = "VALID")
x_ = x_ .+ weights[6]
x_ = max(α .* x_, x_)
x_ = nn.conv3d(x_, expand_dims(expand_dims(expand_dims(expand_dims(expand_dims(weights[7], 1), 2), 3), 4), 5), strides = [1, 1, 1, 1, 1], padding = "VALID")
x_ = x_ .+ weights[8]
x_ = x_ + x
x_ = max(α .* x_, x_)
return squeeze(x_)
end
#number_of_parameters = 8
#N0 = 32
#setup(N0, number_of_parameters)
#using Distributions
#δ_ = reshape(rand(Normal(0., 1.), 32 * 32 * 32), (32, 32, 32));
#g_ = reshape(rand(Normal(0., 1.), 32 * 32 * 32), (32, 32, 32));
#p_ = zeros(number_of_parameters);
#s_ = reshape(rand(0:1, 32 * 32 * 32), (32, 32, 32));
#s_mask = s_.>0;
#using PyPlot
#imshow(squeeze(sum(δ_, 3), 3))
#imshow(squeeze(sum(g_, 3), 3))
#imshow(squeeze(sum(run(sess, output, Dict(δ=>δ_, p=>p_)), 3), (3)))
#imshow(squeeze(sum(run(sess, mock, Dict(δ=>δ_, p=>p_, s=>s_)), 3), (3)))
#loss_ = run(sess, loss, Dict(δ=>δ_, p=>p_, s=>s_, g=>g_, mask=>s_mask))
#adgrad_ = run(sess, adgrad, Dict(δ=>δ_, p=>p_, s=>s_, g=>g_, mask=>s_mask))
#wgrad_ = run(sess, wgrad, Dict(δ=>δ_, p=>p_, s=>s_, g=>g_, mask=>s_mask))
function initialize(state::State)
print(LOG_INFO, "Likelihood initialization in Julia")
number_of_parameters = 8
N0 = libLSS.get(state, "N0", Int64, synchronous=true)
NCAT = libLSS.get(state, "NCAT", Int64, synchronous=true)
setup(N0, number_of_parameters)
print(LOG_VERBOSE, "Found " *repr(NCAT) * " catalogues")
bias = libLSS.resize_array(state, "galaxy_bias_0", number_of_parameters, Float64)
bias[:] = 0
end
function get_required_planes(state::State)
print(LOG_INFO, "Check required planes")
return Array{UInt64,1}([])
end
function likelihood(state::State, ghosts::GhostPlanes, array::AbstractArray{Float64,3})
print(LOG_INFO, "Likelihood evaluation in Julia")
NCAT = libLSS.get(state, "NCAT", Int64, synchronous=true)
L = Float64(0.)
for catalog=1:NCAT
sc = repr(catalog - 1)
L += run(sess, loss, Dict(p=>libLSS.get_array_1d(state, "galaxy_bias_"*sc, Float64), δ=>array, g=>libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64), s=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64), mask=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64).>0.))
end
print(LOG_VERBOSE, "Likelihood is " * repr(L))
return L
end
function generate_mock_data(state::State, ghosts::GhostPlanes, array::AbstractArray{Float64,3})
print(LOG_INFO, "Generate mock")
sc = "0"
data = run(sess, mock, Dict(p=>libLSS.get_array_1d(state, "galaxy_bias_"*sc, Float64), δ=>array, s=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64)))
data = libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64)
print(LOG_INFO, "Shape is " * repr(size(data)) * " and " * repr(size(array)))
print(LOG_INFO, "Number of threads " * repr(Threads.nthreads()))
print(LOG_INFO, "Noise is not included")
print(LOG_INFO, "Max val is " * repr(maximum(array)) * " and data " * repr(maximum(data)))
end
function adjoint_gradient(state::State, array::AbstractArray{Float64,3}, ghosts::GhostPlanes, ag::AbstractArray{Float64,3})
print(LOG_VERBOSE, "Adjoint gradient in Julia")
NCAT = libLSS.get(state, "NCAT", Int64, synchronous=true)
ag[:,:,:] = 0
for catalog=1:NCAT
sc = repr(catalog - 1)
Smask = libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64).>0.
ag[Smask] += run(sess, adgrad, Dict(p=>libLSS.get_array_1d(state, "galaxy_bias_"*sc, Float64), δ=>array, g=>libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64), s=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64), mask=>Smask))[Smask]
end
end
function likelihood_bias(state::State, ghosts::GhostPlanes, array, catalog_id, catalog_bias)
sc = repr(catalog_id)
return run(sess, loss, Dict(p=>catalog_bias, δ=>array, g=>libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64), s=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64), mask=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64).>0.))
end
function get_step_hint(state, catalog_id)
return 0.1
end
function log_prior_bias(state, catalog_id, bias)
if bias[2] < 0
return Inf
end
return 0
end
function adjoint_bias(state::State, ghosts::GhostPlanes,
array, catalog_id, catalog_bias, adjoint_gradient_bias)
sc = repr(catalog_id)
adjoint_gradient_bias = run(sess, wgrad, Dict(p=>catalog_bias, δ=>array, g=>libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64), s=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64), mask=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64).>0.))
end
end

140
extra/hmclet/libLSS/tests/network/TF_likelihood.jl Normal file
View file

@ -0,0 +1,140 @@
#+
# ARES/HADES/BORG Package -- ./extra/hmclet/libLSS/tests/network/TF_likelihood.jl
# Copyright (C) 2014-2020 Guilhem Lavaux <guilhem.lavaux@iap.fr>
# Copyright (C) 2009-2020 Jens Jasche <jens.jasche@fysik.su.se>
#
# Additional contributions from:
# Guilhem Lavaux <guilhem.lavaux@iap.fr> (2023)
#
#+
module network
using libLSS
import libLSS.State
import libLSS.GhostPlanes, libLSS.get_ghost_plane
import libLSS.print, libLSS.LOG_INFO, libLSS.LOG_VERBOSE, libLSS.LOG_DEBUG
using TensorFlow
sess = Session(Graph())
p = nothing
new_p = nothing
assign_p = nothing
δ = nothing
g = nothing
s = nothing
mask = nothing
loss = nothing
adgrad = nothing
wgrad = nothing
function setup(N0, number_of_parameters)
global p, new_p, assign_p, δ, g, s, mask, loss, adgrad, wgrad
p = Variable(zeros(number_of_parameters))
new_p = placeholder(Float64, shape = [number_of_parameters])
assign_p = assign(p, new_p)
δ = placeholder(Float64, shape = [N0, N0, N0])
g = placeholder(Float64, shape = [N0, N0, N0])
s = placeholder(Float64, shape = [N0, N0, N0])
mask = placeholder(Bool, shape = [N0, N0, N0])
loss = 0.5 * sum((boolean_mask(reshape(g, N0^3), reshape(mask, N0^3)) .- boolean_mask(reshape(s, N0^3), reshape(mask, N0^3)) .* (1. .- p[1] .* boolean_mask(reshape(δ, N0^3), reshape(mask, N0^3)))).^2. ./(boolean_mask(reshape(s, N0^3), reshape(mask, N0^3)) .* p[2])) + 0.5 * sum(cast(mask, Float64)) .* log(p[2])
adgrad = gradients(loss, δ)
wgrad_slice = gradients(loss, p)
wgrad = [wgrad_slice.values, wgrad_slice.indices]
end
function initialize(state::State)
print(LOG_INFO, "Likelihood initialization in Julia")
number_of_parameters = 2
N0 = libLSS.get(state, "N0", Int64, synchronous=true)
NCAT = libLSS.get(state, "NCAT", Int64, synchronous=true)
setup(N0, number_of_parameters)
run(sess, global_variables_initializer())
print(LOG_VERBOSE, "Found " *repr(NCAT) * " catalogues")
bias = libLSS.resize_array(state, "galaxy_bias_0", number_of_parameters, Float64)
bias[:] = 1
run(sess, assign_p, Dict(new_p=>bias))
end
function get_required_planes(state::State)
print(LOG_INFO, "Check required planes")
return Array{UInt64,1}([])
end
function likelihood(state::State, ghosts::GhostPlanes, array::AbstractArray{Float64,3})
print(LOG_INFO, "Likelihood evaluation in Julia")
NCAT = libLSS.get(state, "NCAT", Int64, synchronous=true)
L = Float64(0.)
for catalog=1:NCAT
sc = repr(catalog - 1)
run(sess, assign_p, Dict(new_p=>libLSS.get_array_1d(state, "galaxy_bias_"*sc, Float64)))
L += run(sess, loss, Dict(δ=>array, g=>libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64), s=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64), mask=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64).>0.))
end
print(LOG_VERBOSE, "Likelihood is " * repr(L))
return L
end
function generate_mock_data(state::State, ghosts::GhostPlanes, array::AbstractArray{Float64,3})
print(LOG_INFO, "Generate mock")
sc = "0"
data = libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64)
b = libLSS.get_array_1d(state, "galaxy_bias_"*sc, Float64)
S = libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64)
s = size(data)
print(LOG_INFO, "Shape is " * repr(size(data)) * " and " * repr(size(array)))
print(LOG_INFO, "Number of threads " * repr(Threads.nthreads()))
N0=s[1]
N1=s[2]
N2=s[3]
noise = sqrt(b[1])
print(LOG_INFO, "Noise is " * repr(noise))
bias = b[2]
for i=1:N0,j=1:N1,k=1:N2
data[i,j,k] = S[i,j,k]*(1+bias*array[i,j,k]) + sqrt(S[i,j,k])*noise*libLSS.gaussian(state)
end
print(LOG_INFO, "Max val is " * repr(maximum(array)) * " and data " * repr(maximum(data)))
end
function adjoint_gradient(state::State, array::AbstractArray{Float64,3}, ghosts::GhostPlanes, ag::AbstractArray{Float64,3})
print(LOG_VERBOSE, "Adjoint gradient in Julia")
NCAT = libLSS.get(state, "NCAT", Int64, synchronous=true)
ag[:,:,:] = 0
for catalog=1:NCAT
sc = repr(catalog - 1)
run(sess, assign_p, Dict(new_p=>libLSS.get_array_1d(state, "galaxy_bias_"*sc, Float64)))
Smask = libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64).>0.
ag[Smask] += run(sess, adgrad, Dict(δ=>array, g=>libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64), s=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64), mask=>Smask))[Smask]
end
end
function likelihood_bias(state::State, ghosts::GhostPlanes, array, catalog_id, catalog_bias)
sc = repr(catalog_id)
run(sess, assign_p, Dict(new_p=>catalog_bias))
return run(sess, loss, Dict(δ=>array, g=>libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64), s=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64), mask=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64).>0.))
end
function get_step_hint(state, catalog_id)
return 0.1
end
function log_prior_bias(state, catalog_id, bias)
if bias[2] < 0
return Inf
end
return 0
end
function adjoint_bias(state::State, ghosts::GhostPlanes,
array, catalog_id, catalog_bias, adjoint_gradient_bias)
sc = repr(catalog_id)
run(sess, assign_p, Dict(new_p=>catalog_bias))
error = run(sess, wgrad, Dict(δ=>array, g=>libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64), s=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64), mask=>libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64).>0.))
for i=1:number_of_parameters
adjoint_gradient_bias[i] = sum(error[1][error[2] .== i])
end
end
end

103
extra/hmclet/libLSS/tests/test_conv_like.jl Normal file
View file

@ -0,0 +1,103 @@
module test_conv_like
include("convHMC.jl")
using libLSS
import libLSS.State
import libLSS.GhostPlanes, libLSS.get_ghost_plane
import libLSS.print, libLSS.LOG_INFO, libLSS.LOG_VERBOSE, libLSS.LOG_DEBUG
#import test_conv_like.convHMC.initialise
function initialize(state::State)
print(LOG_INFO, "Likelihood initialization in Julia")
NCAT = libLSS.get(state, "NCAT", Int64, synchronous=true)
print(LOG_VERBOSE, "Found " *repr(NCAT) * " catalogues")
N0 = libLSS.get(state, "localN0", Int64, synchronous=true)
N1 = 32
N2 = 32
num_layers = 1
C0 = 3
C1 = 3
C2 = 3
bias = libLSS.resize_array(state, "galaxy_bias_0", num_layers * 5 + 1, Float64)
#bias = libLSS.resize_array(state, "galaxy_bias_0", 29, Float64)
bias[:] = 0
bias[1] = 1
bias[6] = 100
#bias[28] = 1
#bias[29] = 100
#bias[11] = 1
#bias[16] = 1
#bias[21] = 1
#bias[26] = 100
test_conv_like.convHMC.setup(num_layers, N0, N1, N2, 5 * num_layers, [C0, C1, C2], [N0, N1, N2], test_conv_like.convHMC.convolutional_network, test_conv_like.convHMC.get_isotropic_weights, test_conv_like.convHMC.mse)
#test_conv_like.convHMC.setup(num_layers, N0, N1, N2, 28, [C0, C1, C2], [N0, N1, N2], test_conv_like.convHMC.convolutional_network, test_conv_like.convHMC.get_3d_conv, test_conv_like.convHMC.mse)
#bias = libLSS.resize_array(state, "galaxy_bias_0", 2, Float64)
#bias[1] = 100
#bias[2] = 1
#test_conv_like.convHMC.setup(num_layers, N0, N1, N2, 1, -99, [N0, N1, N2], test_conv_like.convHMC.no_network, test_conv_like.convHMC.get_poisson_bias, test_conv_like.convHMC.poisson_bias)
end
function get_required_planes(state::State)
print(LOG_INFO, "Check required planes")
return []
end
function likelihood(state::State, ghosts::GhostPlanes, array::AbstractArray{Float64,3})
print(LOG_INFO, "Likelihood evaluation in Julia")
NCAT = libLSS.get(state, "NCAT", Int64, synchronous=true)
L = Float64(0)
for catalog in 0:(NCAT-1)
sc = repr(catalog)
data = libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64)
params = libLSS.get_array_1d(state, "galaxy_bias_"*sc, Float64)
S = libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64)
Smask = S.>0
L += test_conv_like.convHMC.evaluate(params[1:end-1], array, data, S, params[end], Smask)
end
print(LOG_VERBOSE, "Likelihood is " * repr(L))
return L
end
function generate_mock_data(state::State, ghosts::GhostPlanes, array::AbstractArray{Float64,3})
#sc = "0"
#data = libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64)
#b = libLSS.get_array_1d(state, "galaxy_bias_"*sc, Float64)
#S = libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64)
#s = size(data)
#print(LOG_INFO, "Shape is " * repr(size(data)) * " and " * repr(size(array)))
#print(LOG_INFO, "Number of threads " * repr(Threads.nthreads()))
#N0=s[1]
#N1=s[2]
#N2=s[3]
#noise = sqrt(b[1])
#bias = b[2]
#for i=1:N0,j=1:N1,k=1:N2
# data[i,j,k] = S[i,j,k]*(1+bias*array[i,j,k] + noise*libLSS.gaussian(state))
#end
print(LOG_INFO, "Generate mock")
params = libLSS.get_array_1d(state, "galaxy_bias_0", Float64)
S = libLSS.get_array_3d(state, "galaxy_sel_window_0", Float64)
data = test_conv_like.convHMC.get_field(params[1:end-1], array) .* S
print(LOG_INFO, "Max val is " * repr(maximum(array)) * " and data " * repr(maximum(data)))
end
function adjoint_gradient(state::State, array::AbstractArray{Float64,3}, ghosts::GhostPlanes, ag::AbstractArray{Float64,3})
print(LOG_VERBOSE, "Adjoint gradient in Julia")
N0 = libLSS.get(state, "N0", Int64, synchronous=true)
NCAT = libLSS.get(state, "NCAT", Int64, synchronous=true)
L = Float64(0)
ag[:, :, :] = 0
for catalog in 0:(NCAT-1)
sc = repr(catalog)
data = libLSS.get_array_3d(state, "galaxy_data_"*sc, Float64)
params = libLSS.get_array_1d(state, "galaxy_bias_0", Float64)
S = libLSS.get_array_3d(state, "galaxy_sel_window_"*sc, Float64)
Smask = S.>0
ag += test_conv_like.convHMC.adjointGradient(params[1:end-1], array, data, S, params[end], Smask)
end
end
end

74
extra/hmclet/libLSS/tests/test_dense_mass.cpp Normal file
View file

@ -0,0 +1,74 @@
/*+
ARES/HADES/BORG Package -- ./extra/hmclet/libLSS/tests/test_dense_mass.cpp
Copyright (C) 2014-2020 Guilhem Lavaux <guilhem.lavaux@iap.fr>
Copyright (C) 2009-2020 Jens Jasche <jens.jasche@fysik.su.se>
Additional contributions from:
Guilhem Lavaux <guilhem.lavaux@iap.fr> (2023)
+*/
#define BOOST_TEST_MODULE mass_matrix
#define BOOST_TEST_NO_MAIN
#define BOOST_TEST_ALTERNATIVE_INIT_API
#include <boost/test/included/unit_test.hpp>
#include <boost/test/data/test_case.hpp>
#include "libLSS/tools/console.hpp"
#include "libLSS/tools/static_init.hpp"
#include "libLSS/samplers/core/random_number.hpp"
#include "libLSS/samplers/rgen/gsl_random_number.hpp"
#include "libLSS/mpi/generic_mpi.hpp"
#include <CosmoTool/algo.hpp>
#include <memory>
#include <H5Cpp.h>
#include "libLSS/hmclet/dense_mass.hpp"
#include "libLSS/samplers/core/random_number.hpp"
#include "libLSS/samplers/rgen/gsl_random_number.hpp"
namespace utf = boost::unit_test;
using namespace LibLSS;
BOOST_AUTO_TEST_CASE(dense_mass) {
MPI_Communication *comm = MPI_Communication::instance();
RandomNumberMPI<GSL_RandomNumber> rgen(comm, -1);
HMCLet::DenseMassMatrix M(3);
boost::multi_array<double, 1> numbers(boost::extents[3]);
auto numbers_w = fwrap(numbers);
double a[3];
auto& cons = Console::instance();
for (int i = 0; i < 20; i++) {
a[0] = rgen.gaussian();
a[1] = rgen.gaussian();
a[2] = rgen.gaussian();
numbers[0] = (a[0]+a[2])/std::sqrt(2.0);
numbers[1] = (a[0]-a[2])/std::sqrt(2.0);
numbers[2] = a[1];
M.addMass(numbers);
M.computeMainComponents();
auto C = M.components();
auto mean = M.getMean();
cons.format<LOG_DEBUG>("c00 = %g, c01 = %g, c02 = %g", C(0,0), C(0,1), C(0,2));
cons.format<LOG_DEBUG>("c10 = %g, c11 = %g, c12 = %g", C(1,0), C(1,1), C(1,2));
cons.format<LOG_DEBUG>("c20 = %g, c21 = %g, c22 = %g", C(2,0), C(2,1), C(2,2));
cons.format<LOG_DEBUG>("mean = %g,%g,%g", mean(0), mean(1), mean(2));
}
}
int main(int argc, char *argv[]) {
setupMPI(argc, argv);
StaticInit::execute();
int ret = utf::unit_test_main(&init_unit_test, argc, argv);
StaticInit::finalize();
doneMPI();
return ret;
}

146
extra/hmclet/libLSS/tests/test_hmclet.cpp Normal file
View file

@ -0,0 +1,146 @@
/*+
ARES/HADES/BORG Package -- ./extra/hmclet/libLSS/tests/test_hmclet.cpp
Copyright (C) 2014-2020 Guilhem Lavaux <guilhem.lavaux@iap.fr>
Copyright (C) 2009-2020 Jens Jasche <jens.jasche@fysik.su.se>
Additional contributions from:
Guilhem Lavaux <guilhem.lavaux@iap.fr> (2023)
+*/
#define BOOST_TEST_MODULE julia_bind
#define BOOST_TEST_NO_MAIN
#define BOOST_TEST_ALTERNATIVE_INIT_API
#include <boost/test/included/unit_test.hpp>
#include <boost/test/data/test_case.hpp>
#include "libLSS/tools/console.hpp"
#include "libLSS/tools/static_init.hpp"
#include "libLSS/samplers/core/random_number.hpp"
#include "libLSS/samplers/rgen/gsl_random_number.hpp"
#include "libLSS/mpi/generic_mpi.hpp"
#include <CosmoTool/algo.hpp>
#include <memory>
#include <H5Cpp.h>
#include <CosmoTool/hdf5_array.hpp>
#include "libLSS/hmclet/hmclet.hpp"
#include "libLSS/hmclet/hmclet_qnhmc.hpp"
#include "libLSS/hmclet/diagonal_mass.hpp"
namespace utf = boost::unit_test;
using CosmoTool::square;
using namespace LibLSS;
using namespace LibLSS::HMCLet;
static const double C[2][2] = { { 9. , 1.}, {1., 4.}};
static const double inv_C[2][2] = { { 0.11428571428571427 , -0.028571428571428574}, {-0.028571428571428574, 0.2571428571428572}};
class TestPosterior : virtual public JointPosterior {
public:
TestPosterior() : JointPosterior() {}
virtual ~TestPosterior() {}
virtual size_t getNumberOfParameters() const { return 2; }
virtual double evaluate(VectorType const &params) {
double const u0 = params[0] - 1;
double const u1 = params[1] - 4;
return 0.5 * (u0*u0 * inv_C[0][0] + 2*u0*u1*inv_C[1][0] + u1*u1*inv_C[1][1]);
}
virtual void
adjointGradient(VectorType const &params, VectorType &params_gradient) {
double const u0 = params[0] - 1;
double const u1 = params[1] - 4;
params_gradient[0] = u0 * inv_C[0][0] + 2*u1 * inv_C[0][1];
params_gradient[1] = u1 * inv_C[1][1] + 2*u0* inv_C[0][1];
}
};
BOOST_AUTO_TEST_CASE(hmclet_launch) {
auto posterior_ptr = std::make_shared<TestPosterior>();
SimpleSampler<DiagonalMassMatrix> sampler(posterior_ptr);
MPI_Communication *comm = MPI_Communication::instance();
RandomNumberMPI<GSL_RandomNumber> rgen(comm, -1);
boost::multi_array<double, 1> init_params(boost::extents[2]);
boost::multi_array<double, 1> init_step(boost::extents[2]);
init_params[0] = 100;
init_params[1] = 100;
init_step[0] = 1;
init_step[1] = 1;
boost::multi_array<double, 1> initMass(boost::extents[2]);
initMass[0] = 1;
initMass[1] = 1;
sampler.getMass().setInitialMass(initMass);
sampler.getMass().freeze();
// sampler.calibrate(comm, rgen, 2, init_params, init_step);
boost::multi_array<double, 2> p(boost::extents[10000][2]);
for (size_t i = 0; i < p.size(); i++) {
sampler.newSample(comm, rgen, init_params);
p[i][0] = init_params[0];
p[i][1] = init_params[1];
}
H5::H5File ff("test_sample.h5", H5F_ACC_TRUNC);
CosmoTool::hdf5_write_array(ff, "hmclet", p);
}
BOOST_AUTO_TEST_CASE(qnhmclet_launch) {
auto posterior_ptr = std::make_shared<TestPosterior>();
QNHMCLet::Sampler<DiagonalMassMatrix,QNHMCLet::BDense> sampler(posterior_ptr);
MPI_Communication *comm = MPI_Communication::instance();
RandomNumberMPI<GSL_RandomNumber> rgen(comm, -1);
boost::multi_array<double, 1> init_params(boost::extents[2]);
boost::multi_array<double, 1> init_step(boost::extents[2]);
boost::multi_array<double, 1> initMass(boost::extents[2]);
initMass[0] = 1;
initMass[1] = 1;
sampler.getMass().setInitialMass(initMass);
sampler.getMass().freeze();
init_params[0] = 100;
init_params[1] = 100;
init_step[0] = 1;
init_step[1] = 1;
boost::multi_array<double, 2> p(boost::extents[10000][2]);
H5::H5File ff("test_sample_qn.h5", H5F_ACC_TRUNC);
for (size_t i = 0; i < p.size(); i++) {
sampler.newSample(comm, rgen, init_params);
p[i][0] = init_params[0];
p[i][1] = init_params[1];
// auto gg = ff.createGroup(boost::str(boost::format("B_%d") % i));
// sampler.getB().save(gg);
}
CosmoTool::hdf5_write_array(ff, "qn_hmclet", p);
}
int main(int argc, char *argv[]) {
setupMPI(argc, argv);
StaticInit::execute();
int ret = utf::unit_test_main(&init_unit_test, argc, argv);
StaticInit::finalize();
doneMPI();
return ret;
}

60
extra/hmclet/libLSS/tests/test_julia.jl Normal file
View file

@ -0,0 +1,60 @@
#+
# ARES/HADES/BORG Package -- ./extra/hmclet/libLSS/tests/test_julia.jl
# Copyright (C) 2014-2020 Guilhem Lavaux <guilhem.lavaux@iap.fr>
# Copyright (C) 2009-2020 Jens Jasche <jens.jasche@fysik.su.se>
#
# Additional contributions from:
# Guilhem Lavaux <guilhem.lavaux@iap.fr> (2023)
#
#+
module TestLikelihood
using ..libLSS
import ..libLSS.State, ..libLSS.GhostPlanes, ..libLSS.get_ghost_plane
import ..libLSS.print, ..libLSS.LOG_INFO, ..libLSS.LOG_VERBOSE, ..libLSS.LOG_DEBUG
import ..libLSS.BadGradient
function initialize(state::State)
print(LOG_VERBOSE, "Likelihood initialization in Julia")
# bias = libLSS.resize_array(state, "galaxy_bias_0", 1, Float64)
# bias[1] = 1
end
function get_required_planes(state::State)
return Array{UInt64,1}([])
end
function likelihood(state::State, ghosts::GhostPlanes, array::AbstractArray{Float64,3})
print(LOG_DEBUG, "my likelihood")
return 0
end
function get_step_hint(state, catalog_id, bias_id)
print(LOG_DEBUG, "get_step_hint")
return 0.1
end
function log_prior_bias(state, catalog_id, bias_tilde)
print(LOG_DEBUG, "log_prior_bias")
# Change of variable bias = exp(bias_tilde)
return sum(bias_tilde.^2)
end
function generate_mock_data(state::State, ghosts::GhostPlanes, array)
end
function likelihood_bias(state::State, ghosts::GhostPlanes, array, catalog_id, catalog_bias_tilde)
return 0
end
function adjoint_gradient(state::State, array, ghosts, ag)
end
function adjoint_bias(state::State, ghosts::GhostPlanes,
array, catalog_id, catalog_bias_tilde, adjoint_gradient_bias)
print(LOG_DEBUG,"Entering ag bias")
throw(BadGradient())
end
end

96
extra/hmclet/libLSS/tests/test_julia_hmclet.cpp Normal file
View file

@ -0,0 +1,96 @@
/*+
ARES/HADES/BORG Package -- ./extra/hmclet/libLSS/tests/test_julia_hmclet.cpp
Copyright (C) 2014-2020 Guilhem Lavaux <guilhem.lavaux@iap.fr>
Copyright (C) 2009-2020 Jens Jasche <jens.jasche@fysik.su.se>
Additional contributions from:
Guilhem Lavaux <guilhem.lavaux@iap.fr> (2023)
+*/
#define BOOST_TEST_MODULE julia_hmclet
#define BOOST_TEST_NO_MAIN
#define BOOST_TEST_ALTERNATIVE_INIT_API
#include <boost/test/included/unit_test.hpp>
#include <boost/test/data/test_case.hpp>
#include "libLSS/julia/julia.hpp"
#include "libLSS/julia/julia_mcmc.hpp"
#include "libLSS/mcmc/global_state.hpp"
#include "libLSS/mcmc/state_element.hpp"
#include "libLSS/tools/static_init.hpp"
#include "libLSS/tools/console.hpp"
#include "libLSS/tests/setup_hades_test_run.hpp"
#include "libLSS/samplers/julia/julia_likelihood.hpp"
#include "libLSS/physics/forwards/borg_lpt.hpp"
#include "libLSS/tools/string_tools.hpp"
#include "libLSS/hmclet/julia_hmclet.hpp"
namespace utf = boost::unit_test;
using namespace LibLSS;
using namespace LibLSS_test;
struct JuliaFixture {
static MPI_Communication *comm;
static MarkovState *state;
static BoxModel box;
JuliaFixture() {
LIBLSS_AUTO_CONTEXT(LOG_DEBUG, ctx);
state = new MarkovState();
setup_hades_test_run(comm, 32, 600., *state);
setup_box(*state, box);
ObjectStateElement<BORGForwardModel, true> *model_elt =
new ObjectStateElement<BORGForwardModel, true>();
state->newScalar<bool>("bias_sampler_blocked", false);
state->newScalar<long>("MCMC_STEP", 0);
double ai = state->getScalar<double>("borg_a_initial");
model_elt->obj =
new BorgLptModel<>(comm, box, box, false, 1, 2.0, ai, 1.0, false);
state->newElement("BORG_model", model_elt);
}
~JuliaFixture() { Console::instance().print<LOG_DEBUG>("Destroying state."); delete state; }
};
MPI_Communication *JuliaFixture::comm = 0;
MarkovState *JuliaFixture::state;
BoxModel JuliaFixture::box;
BOOST_GLOBAL_FIXTURE(JuliaFixture);
BOOST_AUTO_TEST_CASE(julia_hmclet_fail) {
LikelihoodInfo info;
LibLSS_test::setup_likelihood_info(
*JuliaFixture::state, info);
Console::instance().print<LOG_DEBUG>(boost::format("Comm is %p") % JuliaFixture::comm);
auto density = std::make_shared<JuliaDensityLikelihood>(
JuliaFixture::comm, info, TEST_JULIA_LIKELIHOOD_CODE, "TestLikelihood");
return;
JuliaHmcletMeta meta(JuliaFixture::comm, density, "TestLikelihood", JuliaHmclet::types::DIAGONAL, 10, 10, 0.5, true);
density->initializeLikelihood(*JuliaFixture::state);
meta.init_markov(*JuliaFixture::state);
meta.sample(*JuliaFixture::state);
}
int main(int argc, char *argv[]) {
JuliaFixture::comm = setupMPI(argc, argv);
StaticInit::execute();
Console::instance().outputToFile(
"test_julia_hmclet.txt_" +
to_string(MPI_Communication::instance()->rank()));
int ret = utf::unit_test_main(&init_unit_test, argc, argv);
StaticInit::finalize();
doneMPI();
return ret;
}

88
extra/hmclet/libLSS/tests/test_network.cpp Normal file
View file

@ -0,0 +1,88 @@
/*+
ARES/HADES/BORG Package -- ./extra/hmclet/libLSS/tests/test_network.cpp
Copyright (C) 2014-2020 Guilhem Lavaux <guilhem.lavaux@iap.fr>
Copyright (C) 2009-2020 Jens Jasche <jens.jasche@fysik.su.se>
Additional contributions from:
Guilhem Lavaux <guilhem.lavaux@iap.fr> (2023)
+*/
#define BOOST_TEST_MODULE julia_bind
#define BOOST_TEST_NO_MAIN
#define BOOST_TEST_ALTERNATIVE_INIT_API
#include <boost/test/included/unit_test.hpp>
#include <boost/test/data/test_case.hpp>
#include "libLSS/tools/console.hpp"
#include "libLSS/tools/static_init.hpp"
#include "libLSS/samplers/core/random_number.hpp"
#include "libLSS/samplers/rgen/gsl_random_number.hpp"
#include "libLSS/mpi/generic_mpi.hpp"
#include <CosmoTool/algo.hpp>
#include <memory>
#include <H5Cpp.h>
#include <CosmoTool/hdf5_array.hpp>
#include "libLSS/hmclet/hmclet.hpp"
namespace utf = boost::unit_test;
using CosmoTool::square;
using namespace LibLSS;
using namespace LibLSS::HMCLet;
class TestPosterior : virtual public JointPosterior {
public:
TestPosterior() : JointPosterior() {}
virtual ~TestPosterior() {}
virtual size_t getNumberOfParameters() const { return 2; }
virtual double evaluate(VectorType const &params) {
return 0.5 * square(params[0] - 1) / 10. + 0.5 * square(params[1] - 4) / 2.;
}
virtual void
adjointGradient(VectorType const &params, VectorType &params_gradient) {
params_gradient[0] = (params[0] - 1) / 10.;
params_gradient[1] = (params[1] - 4) / 2.;
}
};
BOOST_AUTO_TEST_CASE(hmclet_launch) {
auto posterior_ptr = std::make_shared<TestPosterior>();
SimpleSampler sampler(posterior_ptr);
MPI_Communication *comm = MPI_Communication::instance();
RandomNumberMPI<GSL_RandomNumber> rgen(comm, -1);
boost::multi_array<double, 1> init_params(boost::extents[2]);
boost::multi_array<double, 1> init_step(boost::extents[2]);
init_params[0] = 100;
init_params[1] = 100;
init_step[0] = 1;
init_step[1] = 1;
sampler.calibrate(comm, rgen, 10, init_params, init_step);
boost::multi_array<double, 2> p(boost::extents[1000][2]);
for (size_t i = 0; i < p.size(); i++) {
sampler.newSample(comm, rgen, init_params);
p[i][0] = init_params[0];
p[i][1] = init_params[1];
}
H5::H5File ff("test_sample.h5", H5F_ACC_TRUNC);
CosmoTool::hdf5_write_array(ff, "hmclet", p);
}
int main(int argc, char *argv[]) {
setupMPI(argc, argv);
StaticInit::execute();
int ret = utf::unit_test_main(&init_unit_test, argc, argv);
StaticInit::finalize();
doneMPI();
return ret;
}

27
extra/hmclet/libLSS/tests/tests.cmake Normal file
View file

@ -0,0 +1,27 @@
SET(EXTRA_HMCLET ${CMAKE_SOURCE_DIR}/extra/hmclet/libLSS/tests)
SET(TEST_hmclet_LIST
hmclet
dense_mass
#conv_hmc
#weights
#conv_hmc_julia
)
#SET(TEST_weights_LIBS ${JULIA_LIBRARY})
#SET(TEST_conv_hmc_julia_LIBS ${JULIA_LIBRARY})
IF(BUILD_JULIA)
SET(TEST_hmclet_LIST ${TEST_hmclet_LIST} julia_hmclet)
set_property(
SOURCE ${EXTRA_HMCLET}/test_julia_hmclet.cpp
APPEND PROPERTY COMPILE_DEFINITIONS
TEST_JULIA_LIKELIHOOD_CODE="${EXTRA_HMCLET}/test_julia.jl"
)
SET(TEST_julia_hmclet_LIBS ${JULIA_LIBRARY})
add_test(NAME julia_hmclet COMMAND ${CURRENT_CMAKE_BINARY_DIR}/test_julia_hmclet)
ENDIF()