csiborgtools/scripts_independent/PV_process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copyright (C) 2024 Richard Stiskalek\n",
    "# This program is free software; you can redistribute it and/or modify it\n",
    "# under the terms of the GNU General Public License as published by the\n",
    "# Free Software Foundation; either version 3 of the License, or (at your\n",
    "# option) any later version.\n",
    "# This program is distributed in the hope that it will be useful, but\n",
    "# WITHOUT ANY WARRANTY; without even the implied warranty of\n",
    "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General\n",
    "# Public License for more details.\n",
    "#\n",
    "# You should have received a copy of the GNU General Public License along\n",
    "# with this program; if not, write to the Free Software Foundation, Inc.,\n",
    "# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.\n",
    "from os.path import join\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from h5py import File\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Supernovae data"
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 2,
=======
   "execution_count": 8,
>>>>>>> 4af2cc0 (Add script)
   "metadata": {},
   "outputs": [],
   "source": [
    "a2dir = \"/Users/richard/Data/PV/A2_paper_data/A2\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LOSS data set"
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 3,
=======
   "execution_count": 9,
>>>>>>> 4af2cc0 (Add script)
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [\"z_CMB\", \"mB\", \"x1\", \"c\", \"e_mB\", \"e_x1\", \"e_c\", \"RA\", \"DEC\"]\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(a2dir, \"loss.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[5 + n for n in range(len(names))])\n",
    "\n",
    "loss_data = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    loss_data[n] = data[:, i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
<<<<<<< HEAD
    "## Foundation data set "
=======
    "### Foundation data set "
>>>>>>> 4af2cc0 (Add script)
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 4,
=======
   "execution_count": 10,
>>>>>>> 4af2cc0 (Add script)
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [\"z_CMB\", \"RA\", \"DEC\", \"x1\", \"mB\", \"c\", \"peak\", \"e_peak\", \"e_x1\", \"e_mB\", \"e_c\"]\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(a2dir, \"foundation.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[3 + n for n in range(len(names))])\n",
    "\n",
    "foundation_data = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    foundation_data[n] = data[:, i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
<<<<<<< HEAD
=======
    "### Pantheon+"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnYAAAHWCAYAAAD6oMSKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApuUlEQVR4nO3de3RU1d3/8c+E3CiQieEyQ8olEbkroIAQb1RMiRQVSrwutWh51PoEqFAvpAoID8tErUClAaQLQ/UppaVV0IcWlIh4IdyCFBG5VTDRMINYkwFqJlnk/P7oj6kjAZPJTM5k5/1a6yyZffbs+R62CZ+151wclmVZAgAAQLMXY3cBAAAACA+CHQAAgCEIdgAAAIYg2AEAABiCYAcAAGAIgh0AAIAhCHYAAACGINgBAAAYItbuAiKttrZW5eXlateunRwOh93lAAAANIhlWTpx4oRSU1MVE3P+NTnjg115ebm6du1qdxkAAACNUlZWpi5dupy3j/HBrl27dpL+/ZeRlJRkczUAAAAN4/P51LVr10CmOR/jg92Zr1+TkpIIdgAAoNmqzyllXDwBAABgCIIdAACAIQh2AAAAhiDYAQAAGIJgBwAAYAiCHQAAgCEIdgAAAIYg2AEAABiCYAcAAGAIgh0AAIAhCHYAAACGINgBAAAYgmAHAABgCIIdAACAIQh2AAAAhiDYAQAAGCLW7gIQndKmrw3LOEfyx4RlHAAA8N1YsQMAADAEwQ4AAMAQBDsAAABDEOwAAAAMQbADAAAwBMEOAADAEAQ7AAAAQxDsAAAADEGwAwAAMATBDgAAwBAEOwAAAEMQ7AAAAAxBsAMAADAEwQ4AAMAQBDsAAABDEOwAAAAMYWuwS0tLk8PhOGvLycmRJFVVVSknJ0ft27dX27ZtlZ2dLa/Xa2fJAAAAUcvWYLd9+3YdPXo0sL355puSpFtuuUWSNHXqVL3++utatWqVNm3apPLyco0fP97OkgEAAKJWrJ0f3rFjx6DX+fn56tGjh0aMGKHKykotW7ZMK1as0MiRIyVJhYWF6tu3r7Zs2aLhw4fbUTIAAEDUippz7Kqrq/W///u/+ulPfyqHw6GSkhLV1NQoMzMz0KdPnz7q1q2biouLzzmO3++Xz+cL2gAAAFoCW1fsvmn16tWqqKjQPffcI0nyeDyKj49XcnJyUD+XyyWPx3POcfLy8jR79uwIVoqmljZ9bVjGOZI/JizjAAAQraJmxW7ZsmUaPXq0UlNTGzVObm6uKisrA1tZWVmYKgQAAIhuUbFi9+mnn2rDhg165ZVXAm1ut1vV1dWqqKgIWrXzer1yu93nHCshIUEJCQmRLBcAACAqRcWKXWFhoTp16qQxY/7zVdngwYMVFxenoqKiQNv+/ftVWlqqjIwMO8oEAACIarav2NXW1qqwsFATJkxQbOx/ynE6nZo4caKmTZumlJQUJSUlafLkycrIyOCKWAAAgDrYHuw2bNig0tJS/fSnPz1r3/z58xUTE6Ps7Gz5/X5lZWVp0aJFNlQJE3ARBgDAdLYHu1GjRsmyrDr3JSYmqqCgQAUFBU1cFQAAQPMTFefYAQAAoPEIdgAAAIYg2AEAABiCYAcAAGAIgh0AAIAhCHYAAACGINgBAAAYgmAHAABgCIIdAACAIQh2AAAAhiDYAQAAGIJgBwAAYAiCHQAAgCEIdgAAAIYg2AEAABiCYAcAAGAIgh0AAIAhCHYAAACGINgBAAAYgmAHAABgCIIdAACAIQh2AAAAhiDYAQAAGIJgBwAAYAiCHQAAgCEIdgAAAIYg2AEAABiCYAcAAGAIgh0AAIAhCHYAAACGiLW7ABOkTV8blnGO5I8JyzgAAKBlYsUOAADAEAQ7AAAAQxDsAAAADEGwAwAAMATBDgAAwBAEOwAAAEMQ7AAAAAxBsAMAADAEwQ4AAMAQBDsAAABD2B7sPv/8c911111q3769WrdurUsuuUQ7duwI7LcsSzNnzlTnzp3VunVrZWZm6uDBgzZWDAAAEJ1sDXZfffWVrrzySsXFxelvf/ub9u7dq+eee04XXHBBoM8zzzyj559/XkuWLNHWrVvVpk0bZWVlqaqqysbKAQAAok+snR/+9NNPq2vXriosLAy0paenB/5sWZYWLFigJ554QmPHjpUkvfTSS3K5XFq9erVuv/32Jq8ZAAAgWtm6Yvfaa69pyJAhuuWWW9SpUyddeuml+u1vfxvYf/jwYXk8HmVmZgbanE6nhg0bpuLiYjtKBgAAiFq2BrtPPvlEixcvVs+ePbV+/Xo9+OCDmjJlin73u99JkjwejyTJ5XIFvc/lcgX2fZvf75fP5wvaAAAAWgJbv4qtra3VkCFD9NRTT0mSLr30Uu3Zs0dLlizRhAkTQhozLy9Ps2fPDmeZAAAAzYKtK3adO3dWv379gtr69u2r0tJSSZLb7ZYkeb3eoD5erzew79tyc3NVWVkZ2MrKyiJQOQAAQPSxNdhdeeWV2r9/f1DbgQMH1L17d0n/vpDC7XarqKgosN/n82nr1q3KyMioc8yEhAQlJSUFbQAAAC2BrV/FTp06VVdccYWeeuop3Xrrrdq2bZuWLl2qpUuXSpIcDoceeughzZ07Vz179lR6erpmzJih1NRUjRs3zs7SAQAAoo6twW7o0KF69dVXlZubqzlz5ig9PV0LFizQnXfeGejz6KOP6tSpU7r//vtVUVGhq666SuvWrVNiYqKNlQMAAEQfW4OdJN1www264YYbzrnf4XBozpw5mjNnThNWBQAA0PzY/kgxAAAAhAfBDgAAwBAEOwAAAEMQ7AAAAAxBsAMAADAEwQ4AAMAQBDsAAABDEOwAAAAMQbADAAAwhO1PngBaorTpa8MyzpH8MWEZBwBgBlbsAAAADEGwAwAAMATBDgAAwBAEOwAAAEMQ7AAAAAxBsAMAADAEwQ4AAMAQBDsAAABDEOwAAAAMQbADAAAwBMEOAADAEAQ7AAAAQxDsAAAADEGwAwAAMATBDgAAwBCxdhcANDdp09faXQIAAHVixQ4AAMAQBDsAAABDEOwAAAAMQbADAAAwBMEOAADAEAQ7AAAAQxDsAAAADEGwAwAAMATBDgAAwBAEOwAAAEMQ7AAAAAxBsAMAADAEwQ4AAMAQBDsAAABDEOwAAAAMEWvnhz/55JOaPXt2UFvv3r21b98+SVJVVZV+8YtfaOXKlfL7/crKytKiRYvkcrnsKDfi0qavbfQYR/LHhKESAADQHNm+Yte/f38dPXo0sL333nuBfVOnTtXrr7+uVatWadOmTSovL9f48eNtrBYAACB62bpiJ0mxsbFyu91ntVdWVmrZsmVasWKFRo4cKUkqLCxU3759tWXLFg0fPrypSwUAAIhqtq/YHTx4UKmpqbrwwgt15513qrS0VJJUUlKimpoaZWZmBvr26dNH3bp1U3FxsV3lAgAARC1bV+yGDRum5cuXq3fv3jp69Khmz56tq6++Wnv27JHH41F8fLySk5OD3uNyueTxeM45pt/vl9/vD7z2+XyRKh8AACCq2BrsRo8eHfjzgAEDNGzYMHXv3l1/+tOf1Lp165DGzMvLO+uCDAAAgJbA9q9ivyk5OVm9evXSoUOH5Ha7VV1drYqKiqA+Xq+3znPyzsjNzVVlZWVgKysri3DVAAAA0SGqgt3Jkyf1j3/8Q507d9bgwYMVFxenoqKiwP79+/ertLRUGRkZ5xwjISFBSUlJQRsAAEBLYOtXsQ8//LBuvPFGde/eXeXl5Zo1a5ZatWqlO+64Q06nUxMnTtS0adOUkpKipKQkTZ48WRkZGVwRCwAAUAdbg91nn32mO+64Q19++aU6duyoq666Slu2bFHHjh0lSfPnz1dMTIyys7ODblAMAACAs9ka7FauXHne/YmJiSooKFBBQUETVQQAANB8RdU5dgAAAAgdwQ4AAMAQBDsAAABDEOwAAAAMYevFEzBf2vS1dpcAAECLwYodAACAIQh2AAAAhiDYAQAAGIJgBwAAYAiCHQAAgCEIdgAAAIYg2AEAABiCYAcAAGAIgh0AAIAhCHYAAACGINgBAAAYgmAHAABgCIIdAACAIQh2AAA
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpath = \"/Users/richard/Data/PV/Pantheon+SH0ES.dat\"\n",
    "\n",
    "data = np.genfromtxt(fpath, names=True, dtype=None, encoding=None)\n",
    "data = data[data[\"zCMB\"] < 0.1]\n",
    "\n",
    "keys = [\"zCMB\", \"mB\", \"mBERR\", \"x1\", \"x1ERR\", \"c\", \"cERR\", \"RA\", \"DEC\", \"VPEC\", \"VPECERR\", \"biasCor_m_b\", \"biasCorErr_m_b\"]\n",
    "\n",
    "plt.figure()\n",
    "plt.hist(data[\"zCMB\"], bins=30)\n",
    "plt.xlim(0)\n",
    "plt.xlabel(r\"$z_{\\rm CMB}$\")\n",
    "plt.ylabel(r\"Counts\")\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "pantheon_data = data[keys]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tully-Fisher galaxies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_folder = \"/Users/richard/Data/PV/A2_paper_data/tf\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SFI galaxies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [\"RA\", \"DEC\", \"z_CMB\", \"mag\", \"eta\", \"e_mag\", \"e_eta\"]\n",
    "\n",
    "\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(tf_folder, \"sfi_gals_tf.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[2 + n for n in range(len(names))])\n",
    "\n",
    "sfi_gals = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    sfi_gals[n] = data[:, i]\n",
    "\n",
    "# sfi_gals = sfi_gals[sfi_gals[\"z_CMB\"] < 0.06]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SFI galaxies masked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [\"RA\", \"DEC\", \"z_CMB\", \"mag\", \"eta\", \"e_mag\", \"e_eta\"]\n",
    "\n",
    "\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(tf_folder, \"sfi_gals_tf_masked.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[2 + n for n in range(len(names))])\n",
    "\n",
    "sfi_gals_masked = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    sfi_gals_masked[n] = data[:, i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SFI Groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [\"RA\", \"DEC\", \"z_CMB\", \"r_hMpc\", \"e_r_hMpc\"]\n",
    "\n",
    "\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(tf_folder, \"sfi_grps.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[1 + n for n in range(len(names))])\n",
    "\n",
    "sfi_groups = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    sfi_groups[n] = data[:, i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2MTF "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [\"RA\", \"DEC\", \"mag\", \"e_mag\", \"z_CMB\", \"r_hMpc\", \"e_rhMpc\", \"M\", \"eta\", \"e_eta\"]\n",
    "\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(tf_folder, \"twomtf_k.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[2 + n for n in range(len(names))])\n",
    "\n",
    "twomtf_gals = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    twomtf_gals[n] = data[:, i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
>>>>>>> 4af2cc0 (Add script)
    "## Write output as HDF5 file"
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 5,
=======
   "execution_count": 18,
>>>>>>> 4af2cc0 (Add script)
   "metadata": {},
   "outputs": [],
   "source": [
    "outdir = \"/Users/richard/Downloads\"\n",
    "fname = \"PV_compilation_Supranta2019.hdf5\"\n",
    "\n",
    "with File(join(outdir, fname), 'w') as f:\n",
    "    # Write LOSS\n",
    "    grp = f.create_group(\"LOSS\")\n",
    "    for name in loss_data.dtype.names:\n",
    "        grp.create_dataset(name, data=loss_data[name])\n",
    "\n",
    "    # Write Foundation\n",
    "    grp = f.create_group(\"Foundation\")\n",
    "    for name in foundation_data.dtype.names:\n",
<<<<<<< HEAD
    "        grp.create_dataset(name, data=foundation_data[name])\n"
=======
    "        grp.create_dataset(name, data=foundation_data[name])\n",
    "\n",
    "    # Write SFI gals\n",
    "    grp = f.create_group(\"SFI_gals\")\n",
    "    for name in sfi_gals.dtype.names:\n",
    "        grp.create_dataset(name, data=sfi_gals[name])\n",
    "    \n",
    "    # Write SFI gals masked\n",
    "    grp = f.create_group(\"SFI_gals_masked\")\n",
    "    for name in sfi_gals_masked.dtype.names:\n",
    "        grp.create_dataset(name, data=sfi_gals_masked[name])\n",
    "\n",
    "    # Write SFI groups\n",
    "    grp = f.create_group(\"SFI_groups\")\n",
    "    for name in sfi_groups.dtype.names:\n",
    "        grp.create_dataset(name, data=sfi_groups[name])\n",
    "\n",
    "    # Write 2MTF gals\n",
    "    grp = f.create_group(\"2MTF\")\n",
    "    for name in twomtf_gals.dtype.names:\n",
    "        grp.create_dataset(name, data=twomtf_gals[name])\n",
    "\n",
    "    # Write Pantheon\n",
    "    grp = f.create_group(\"Pantheon+\")\n",
    "    for name in pantheon_data.dtype.names:\n",
    "        grp.create_dataset(name, data=pantheon_data[name])\n",
    "\n"
>>>>>>> 4af2cc0 (Add script)
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}