csiborgtools/notebooks/flow/process_PV.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process PV catalogues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os.path import join\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from h5py import File\n",
    "from astropy.coordinates import match_coordinates_sky, SkyCoord\n",
    "from astropy import units as u\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "SPEED_OF_LIGHT = 299_792.458"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Supernovae data\n",
    "\n",
    "### LOSS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "a2dir = \"/mnt/extraspace/rstiskalek/catalogs/PV/PV_Supranta/A2\"\n",
    "\n",
    "names = [\"z_CMB\", \"mB\", \"x1\", \"c\", \"e_mB\", \"e_x1\", \"e_c\", \"RA\", \"DEC\"]\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(a2dir, \"loss.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[5 + n for n in range(len(names))])\n",
    "\n",
    "loss_data = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    loss_data[n] = data[:, i]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Foundation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [\"z_CMB\", \"RA\", \"DEC\", \"x1\", \"mB\", \"c\", \"peak\", \"e_peak\", \"e_x1\", \"e_mB\", \"e_c\"]\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(a2dir, \"foundation.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[3 + n for n in range(len(names))])\n",
    "\n",
    "foundation_data = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    foundation_data[n] = data[:, i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pantheon+, all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<KeysViewHDF5 ['DEGdeg', 'DEHdeg', 'DEJ2000', 'Host', 'IAUC', 'RAGdeg', 'RAHdeg', 'RAJ2000', 'SNID', 'SimbadName', 'e_vpec', 'e_zHD', 'e_zhel', 'recno', 'vpec', 'vpecG', 'zGHD', 'zGcmb', 'zGhel', 'zHD', 'zcmb', 'zhel']>\n"
     ]
    }
   ],
   "source": [
    "fpath_full = \"/mnt/extraspace/rstiskalek/catalogs/PV/Pantheon+SH0ES.dat\"\n",
    "fpath_group = \"/mnt/extraspace/rstiskalek/catalogs/PV/pantheon+_groups.hdf5\"\n",
    "\n",
    "data_full = np.genfromtxt(fpath_full, names=True, dtype=None, encoding=None)\n",
    "data_group = {}\n",
    "\n",
    "# Read in the groups\n",
    "with File(fpath_group, \"r\") as f:\n",
    "    print(f.keys())\n",
    "    for key in f.keys():\n",
    "        try:\n",
    "            data_group[key] = f[key][...]\n",
    "        except IndexError:\n",
    "            print(f\"Failed to read {key}\")\n",
    "        # data[key] = f[key][...]\n",
    "\n",
    "# data = data[data[\"zCMB\"] < 0.1]\n",
    "\n",
    "keys = [\"zCMB\", \"zCMBERR\", \"mB\", \"mBERR\", \"x1\", \"x1ERR\", \"c\", \"cERR\", \"RA\",\n",
    "        \"DEC\", \"VPEC\", \"VPECERR\", \"biasCor_m_b\", \"biasCorErr_m_b\"]\n",
    "pantheon_data = {}\n",
    "for key in keys:\n",
    "    pantheon_data[key] = data_full[key]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum 2D separation is: 0.0005450609539575544 deg\n"
     ]
    }
   ],
   "source": [
    "match_coord = SkyCoord(data_full[\"RA\"], data_full[\"DEC\"], unit=\"deg\")\n",
    "catalogue_coord = SkyCoord(data_group[\"RAJ2000\"], data_group[\"DEJ2000\"],\n",
    "                           unit=\"deg\")\n",
    "\n",
    "idxs, sep2d, sep3d = match_coordinates_sky(match_coord, catalogue_coord, nthneighbor=1)\n",
    "print(f\"Maximum 2D separation is: {sep2d.max()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "zcmb_supernovae = data_group[\"zcmb\"][idxs]\n",
    "zcmb_group = data_group[\"zGcmb\"][idxs]\n",
    "\n",
    "pantheon_data[\"zCMB_SN\"] = zcmb_supernovae\n",
    "pantheon_data[\"zCMB_Group\"] = zcmb_group\n",
    "\n",
    "mask = pantheon_data[\"zCMB\"] < 0.1\n",
    "for key in pantheon_data.keys():\n",
    "    pantheon_data[key] = pantheon_data[key][mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAoi0lEQVR4nO3dfUxVd4L/8c8FBFynQCsjFEqhpW7drTw4Vh60WZkpKU7YUDbZlZL4MMZ0d8iumjA1lcbVTua3QTd1a60GY9PZacP4MG66rivGBnGYtAq1Ag5rZ2qmZIoWChZnAYdBsHD2D3/cchWQi/fe8733vl/JyamH77l8z4nt/fT76LAsyxIAAIDBQuyuAAAAwL0QWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxguzuwKeMDo6qs7OTj3wwANyOBx2VwcAAEyDZVm6ceOGEhISFBIydRtKQASWzs5OJSUl2V0NAAAwA1evXtUjjzwyZZmACCwPPPCApNsPHBUVZXNtAADAdPT39yspKcn5PT6VgAgsY91AUVFRBBYAAPzMdIZzMOgWAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAwJSqG9u1bMcZVTe221YHAgsAAJhSVX2bOnoHVVXfZlsdCCwAAGBKZXmpSoyZrbK8VNvq4LAsy7Ltt3tIf3+/oqOj1dfXp6ioKLurAwAApsGd729aWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgCA4UxYuM1uBBYAAAxnwsJtdptRYNm3b59SUlIUGRmp7OxsnT9/fsryR48e1YIFCxQZGam0tDSdPHnS5ec/+MEP5HA4XI4VK1bMpGoAAAQcExZus5vbgeXIkSMqLy/X9u3b1dzcrIyMDBUUFOjatWsTlj937pxKS0u1fv16tbS0qLi4WMXFxbp06ZJLuRUrVujLL790HocOHZrZEwEAEGBW5STr7JbvaVVOst1VsY3bK91mZ2dryZIl2rt3ryRpdHRUSUlJ2rBhg7Zs2XJX+ZKSEg0MDOjEiRPOazk5OcrMzNT+/fsl3W5h6e3t1bFjx2b0EKx0CwCA//HaSrfDw8NqampSfn7+Nx8QEqL8/Hw1NDRMeE9DQ4NLeUkqKCi4q3x9fb3mzZunJ598UmVlZbp+/fqk9RgaGlJ/f7/LAQAAApdbgaWnp0cjIyOKi4tzuR4XF6eurq4J7+nq6rpn+RUrVujdd99VXV2ddu7cqV/96lf6/ve/r5GRkQk/s7KyUtHR0c4jKSnJnccAAAD3YNrMJCNmCb3wwgsqKipSWlqaiouLdeLECX388ceqr6+fsHxFRYX6+vqcx9WrV31bYQAAApxpM5PcCiyxsbEKDQ1Vd3e3y/Xu7m7Fx8dPeE98fLxb5SXp8ccfV2xsrD777LMJfx4REaGoqCiXAwAAeI5pM5PcCizh4eFavHix6urqnNdGR0dVV1en3NzcCe/Jzc11KS9JtbW1k5aXpC+++ELXr1/Xww8/7E71AACAh5g2M8ntLqHy8nK99dZbeuedd/Tb3/5WZWVlGhgY0Lp16yRJa9asUUVFhbP8pk2bdOrUKe3atUuffvqpXn31VV24cEH/9E//JEn64x//qM2bN6uxsVGff/656urq9Pzzz+uJJ55QQUGBhx4TAAD4szB3bygpKdFXX32lbdu2qaurS5mZmTp16pRzYO2VK1cUEvJNDlq6dKkOHjyorVu36pVXXtH8+fN17NgxLVy4UJIUGhqq1tZWvfPOO+rt7VVCQoKee+45/eQnP1FERISHHhMAAPgzt9dhMRHrsAAA4H+8tg4LAACAHQgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQUAABiPwAIAAIxHYAEAAMYjsAAAAOMRWAAAgPEILAAAwHgEFgAAYDwCCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAAB+obqxXct2nFF1Y7vdVYENCCwAAL9QVd+mjt5BVdW32V0V2IDAAgDwC2V5qUqMma2yvFS7qwIbOCzLsuyuxP3q7+9XdHS0+vr6FBUVZXd1AADANLjz/U0LCwAAMB6BBQAAGI/AAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAADGI7AAAADjEVgAAIDxCCwAANtVN7Zr2Y4zqm5sd+u+jYdalFpRo42HWrxUM5iCwAIAsF1VfZs6egdVVd/m1n01rZ0asW6fEdgILAAA2y1OflChjttndxSmJyjUcfuMwBZmdwUAAGhq/1+NWLfP7thTukh7Shd5qVYwCS0sAADbleWlKjFmtsryUu2uCgzlsCzLsrsS96u/v1/R0dHq6+tTVFSU3dUBAADT4M73Ny0sAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABANwXlseHLxBYAAD3heXx4QsEFgDAfWF5fPgCC8cBAABbsHAcAAAIKDMKLPv27VNKSooiIyOVnZ2t8+fPT1n+6NGjWrBggSIjI5WWlqaTJ09OWvaHP/yhHA6Hdu/ePZOqAQCAAOR2YDly5IjKy8u1fft2NTc3KyMjQwUFBbp27dqE5c+dO6fS0lKtX79eLS0tKi4uVnFxsS5dunRX2f/8z/9UY2OjEhLoBwUAAN9wO7D827/9m1588UWtW7dOf/mXf6n9+/frz/7sz/TTn/50wvJvvPGGVqxYoc2bN+sv/uIv9JOf/ETf+c53tHfvXpdyHR0d2rBhg37+859r1qxZM3saAAAQkNwKLMPDw2pqalJ+fv43HxASovz8fDU0NEx4T0NDg0t5SSooKHApPzo6qtWrV2vz5s166qmn3KkSAAAIAmHuFO7p6dHIyIji4uJcrsfFxenTTz+d8J6urq4Jy3d1dTn/vHPnToWFhWnjxo3TqsfQ0JCGhoacf+7v75/uIwAAAD9k+yyhpqYmvfHGG/rZz34mh8MxrXsqKysVHR3tPJKSkrxcSwAAYCe3AktsbKxCQ0PV3d3tcr27u1vx8fET3hMfHz9l+Q8++EDXrl3To48+qrCwMIWFham9vV0/+tGPlJKSMuFnVlRUqK+vz3lcvXrVnccAAAB+xq3AEh4ersWLF6uurs55bXR0VHV1dcrNzZ3wntzcXJfyklRbW+ssv3r1arW2turixYvOIyEhQZs3b9b7778/4WdGREQoKirK5QAAAIHLrTEsklReXq61a9fq6aefVlZWlnbv3q2BgQGtW7dOkrRmzRolJiaqsrJSkrRp0yYtX75cu3btUmFhoQ4fPqwLFy7owIEDkqS5c+dq7ty5Lr9j1qxZio+P15NPPnm/zwcAmKHqxnZV1bepLC9Vq3KS7a4OgpzbgaWkpERfffWVtm3bpq6uLmVmZurUqVPOgbVXrlxRSMg3DTdLly7VwYMHtXXrVr3yyiuaP3++jh07poULF3ruKQAAHldV36aO3kFV1bcRWGA79hICAEyIFhZ4mzvf3wQWAABgCzY/BAAAAYXAAgAAjEdgAQAAxiOwAAAA4xFYAACA8QgsAADAeAQWAABgPAILAAAwHoEFAAAYj8ACAACMR2ABAD+z8VCLUitqtPFQi91VAXyGwAIAfqamtVMj1u0zECwILADgZwrTExTquH0GggW7NQMAAFuwWzMAAAgoBBYAAGA8AgsAADAegQUAfITpyMDMEVgAwEeYjgzMHIEFAHyE6cjAzDGtGQAA2IJpzQAAIKAQWAAAgPEILAAAwHgEFgAIYtWN7Vq244yqG9vtrgowJQILAASxqvo2dfQOqqq+ze6qAFMisACAAexq6SjLS1VizGyV5aX69PcC7mJaMwAYYNmOM+roHVRizGyd3fI9u6sD+ATTmgHAz9DSAUyNFhYAAGALWlgAAEBAIbAAAADjEVgAAIDxCCwAAMB4BBYAAGA8AgsAADAegQU
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.scatter(pantheon_data[\"zCMB_SN\"], pantheon_data[\"zCMB_Group\"], s=1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tully-Fisher galaxies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SFI++ galaxies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_folder = \"/mnt/extraspace/rstiskalek/catalogs/PV/PV_Supranta/tf\"\n",
    "\n",
    "names = [\"RA\", \"DEC\", \"z_CMB\", \"mag\", \"eta\", \"e_mag\", \"e_eta\"]\n",
    "\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(tf_folder, \"sfi_gals_tf.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[2 + n for n in range(len(names))])\n",
    "\n",
    "sfi_gals = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    sfi_gals[n] = data[:, i]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SFI++ galaxies masked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [\"RA\", \"DEC\", \"z_CMB\", \"mag\", \"eta\", \"e_mag\", \"e_eta\"]\n",
    "\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(tf_folder, \"sfi_gals_tf_masked.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[2 + n for n in range(len(names))])\n",
    "\n",
    "sfi_gals_masked = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    sfi_gals_masked[n] = data[:, i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SFI++ groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [\"RA\", \"DEC\", \"z_CMB\", \"r_hMpc\", \"e_r_hMpc\"]\n",
    "\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(tf_folder, \"sfi_grps.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[1 + n for n in range(len(names))])\n",
    "\n",
    "sfi_groups = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    sfi_groups[n] = data[:, i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross $\\texttt{SFI++ galaxies}$ and $\\texttt{SFI++ groups}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "[$3^\\circ40{}^\\prime57.6210022{}^{\\prime\\prime}$\n",
       " $0^\\circ45{}^\\prime09.98210907{}^{\\prime\\prime}$\n",
       " $1^\\circ47{}^\\prime48.02945137{}^{\\prime\\prime}$ ...\n",
       " $0^\\circ59{}^\\prime48.39719296{}^{\\prime\\prime}$\n",
       " $1^\\circ27{}^\\prime20.76476097{}^{\\prime\\prime}$\n",
       " $2^\\circ01{}^\\prime03.90094757{}^{\\prime\\prime}$]"
      ],
      "text/plain": [
       "<Angle [3.6826725, 0.7527728, 1.7966748, ..., 0.996777 , 1.455768 ,\n",
       "        2.0177503] deg>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "match_coord = SkyCoord(sfi_gals[\"RA\"], sfi_gals[\"DEC\"], unit=\"deg\")\n",
    "catalogue_coord = SkyCoord(sfi_groups[\"RA\"], sfi_groups[\"DEC\"],\n",
    "                           unit=\"deg\")\n",
    "\n",
    "idxs, sep2d, sep3d = match_coordinates_sky(match_coord, catalogue_coord, nthneighbor=1)\n",
    "sep2d.to(u.degree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnYAAAHWCAYAAAD6oMSKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAr9UlEQVR4nO3deXTTdb7/8VdKF5a2qWXpAoWWfd+hVh1EqLQFUaT3Kg7XU5haHWxR6EGlKpvjPSguMDooeq4Djld0ZFQQVLxYVrHsmzhQpQMXmC5AmTYUpa3t9/eHl/wmw9raNOmnz8c5OYfk+03yTr8nznO+328Sm2VZlgAAANDg+Xh6AAAAANQNwg4AAMAQhB0AAIAhCDsAAABDEHYAAACGIOwAAAAMQdgBAAAYgrADAAAwhK+nB/AG1dXVys/PV1BQkGw2m6fHAQAAcLIsS+fOnVNkZKR8fK6+T46wk5Sfn6+oqChPjwEAAHBFJ06cULt27a66DmEnKSgoSNLPf7Dg4GAPTwMAAPD/ORwORUVFOXvlagg7yXn4NTg4mLADAABe6XpOF+PDEwAAAIYg7AAAAAxB2AEAABiCsAMAADAEYQcAAGAIwg4AAMAQhB0AAIAhCDsAAABDEHYAAACGIOwAAAAMQdgBAAAYgrADAAAwBGEHAABgCMIOAADAEIQdAACAIQg7AAAAQ/h6egDUneiZn9Zo/WPPjXHTJAAAwBPYYwcAAGAIwg4AAMAQhB0AAIAhOMeuEeOcPAAAzMIeOwAAAEMQdgAAAIYg7AAAAAxB2AEAABiCsAMAADAEYQcAAGAIwg4AAMAQhB0AAIAhCDsAAABDEHYAAACGIOwAAAAMQdgBAAAYgrADAAAwBGEHAABgCMIOAADAEIQdAACAIQg7AAAAQxB2AAAAhvBo2M2fP19DhgxRUFCQ2rRpo3Hjxik3N9dlnQsXLig9PV0tW7ZUYGCgkpOTVVRU5LLO8ePHNWbMGDVv3lxt2rTRY489pp9++qk+XwoAAIDHeTTsNm3apPT0dG3btk3r1q1TZWWlRo0apfPnzzvXmT59ulavXq0VK1Zo06ZNys/P1/jx453Lq6qqNGbMGFVUVOjrr7/W22+/rWXLlmn27NmeeEkAAAAeY7Msy/L0EBedPn1abdq00aZNmzRs2DCVlpaqdevWWr58uf7t3/5NknT48GH16NFDOTk5uvHGG/X555/rjjvuUH5+vsLCwiRJS5Ys0RNPPKHTp0/L39//ms/rcDhkt9tVWlqq4OBgt75Gd4qe+albH//Yc2Pc+vgAAOBSNekUrzrHrrS0VJIUGhoqSdq9e7cqKysVHx/vXKd79+5q3769cnJyJEk5OTnq06ePM+okKSEhQQ6HQ99+++1ln6e8vFwOh8PlAgAA0NB5TdhVV1dr2rRpuvnmm9W7d29JUmFhofz9/RUSEuKyblhYmAoLC53r/HPUXVx+cdnlzJ8/X3a73XmJioqq41cDAABQ/7wm7NLT03Xw4EG9//77bn+urKwslZaWOi8nTpxw+3MCAAC4m6+nB5CkjIwMrVmzRps3b1a7du2ct4eHh6uiokIlJSUue+2KiooUHh7uXGfHjh0uj3fxU7MX1/lXAQEBCggIqONXAQAA4Fke3WNnWZYyMjL08ccfa/369YqJiXFZPmjQIPn5+Sk7O9t5W25uro4fP664uDhJUlxcnL755hudOnXKuc66desUHBysnj171s8LAQAA8AIe3WOXnp6u5cuXa9WqVQoKCnKeE2e329WsWTPZ7XalpqYqMzNToaGhCg4O1tSpUxUXF6cbb7xRkjRq1Cj17NlT999/vxYsWKDCwkI9/fTTSk9PZ68cAABoVDwadq+//rokafjw4S63L126VJMmTZIkLVy4UD4+PkpOTlZ5ebkSEhL02muvOddt0qSJ1qxZoylTpiguLk4tWrRQSkqKnnnmmfp6GQAAAF7Bq77HzlP4Hrvrw/fYAQBQ/xrs99gBAACg9gg7AAAAQxB2AAAAhiDsAAAADEHYAQAAGIKwAwAAMARhBwAAYAjCDgAAwBCEHQAAgCEIOwAAAEMQdgAAAIYg7AAAAAxB2AEAABiCsAMAADAEYQcAAGAIX08PgIYjeuanNVr/2HNj3DQJAAC4HPbYAQAAGIKwAwAAMARhBwAAYAjCDgAAwBCEHQAAgCEIOwAAAEMQdgAAAIYg7AAAAAxB2AEAABiCsAMAADAEYQcAAGAIwg4AAMAQhB0AAIAhCDsAAABDEHYAAACGIOwAAAAMQdgBAAAYgrADAAAwBGEHAABgCMIOAADAEIQdAACAIQg7AAAAQxB2AAAAhiDsAAAADEHYAQAAGIKwAwAAMARhBwAAYAjCDgAAwBCEHQAAgCEIOwAAAEMQdgAAAIYg7AAAAAxB2AEAABiCsAMAADAEYQcAAGAIwg4AAMAQhB0AAIAhCDsAAABDEHYAAACGIOwAAAAM4evpAYCLomd+WqP1jz03xk2TAADQMLHHDgAAwBCEHQAAgCEIOwAAAEMQdgAAAIYg7AAAAAxB2AEAABiCsAMAADAEYQcAAGAIwg4AAMAQhB0AAIAhCDsAAABDEHYAAACGIOwAAAAMQdgBAAAYgrADAAAwBGEHAABgCMIOAADAEIQdAACAIQg7AAAAQ/h6egBcWfTMTz09AgAAaEDYYwcAAGAIwg4AAMAQHg27zZs3a+zYsYqMjJTNZtPKlStdlk+aNEk2m83lkpiY6LLO2bNnNXHiRAUHByskJESpqakqKyurx1cBAADgHTx6jt358+fVr18//eY3v9H48eMvu05iYqKWLl3qvB4QEOCyfOLEiSooKNC6detUWVmpyZMn68EHH9Ty5cvdOjs8r6bnIB57boybJgEAwDt4NOySkpKUlJR01XUCAgIUHh5+2WWHDh3S2rVrtXPnTg0ePFiS9Oqrr2r06NF68cUXFRkZWeczAwAAeCuvP8du48aNatOmjbp166YpU6aouLjYuSwnJ0chISHOqJOk+Ph4+fj4aPv27Vd8zPLycjkcDpcLAABAQ+fVYZeYmKg//elPys7O1vPPP69NmzYpKSlJVVVVkqTCwkK1adPG5T6+vr4KDQ1VYWHhFR93/vz5stvtzktUVJRbXwcAAEB98OrvsZswYYLz33369FHfvn3VqVMnbdy4USNHjqz142ZlZSkzM9N53eFwEHcAAKDB8+o9dv+qY8eOatWqlY4cOSJJCg8P16lTp1zW+emnn3T27Nkrnpcn/XzeXnBwsMsFAACgoWtQYXfy5EkVFxcrIiJCkhQXF6eSkhLt3r3buc769etVXV2t2NhYT40JAADgER49FFtWVubc+yZJR48e1b59+xQaGqrQ0FDNmzdPycnJCg8PV15enh5//HF17txZCQkJkqQePXooMTFRaWlpWrJkiSorK5WRkaEJEybwiVgAANDoeHSP3a5duzRgwAANGDBAkpSZmakBAwZo9uzZatKkiQ4cOKA777xTXbt2VWpqqgYNGqQtW7a4fJfdu+++q+7du2vkyJEaPXq0brnlFr355pueekkAAAAe49E9dsOHD5dlWVdc/sUXX1zzMUJDQ/kyYgAAADWwc+wAAABwZYQdAACAIQg7AAAAQxB2AAAAhvDqX55AwxY981NPjwAAQKPCHjsAAABDEHYAAACGIOwAAAAMQdgBAAAYgrADAAAwBGEHAABgCMIOAADAEIQdAACAIQg7AAAAQxB2AAAAhiDsAAAADEHYAQAAGIKwAwAAMARhBwAAYAjCDgAAwBCEHQAAgCEIOwAAAEMQdgAAAIYg7AAAAAxB2AEAABiCsAMAADAEYQcAAGAIwg4AAMAQhB0AAIAhCDsAAABDEHYAAACGIOwAAAAMQdgBAAAYgrADAAAwBGEHAABgCMIOAADAEIQdAACAIXw9PUBjEj3zU0+PAAAADMYeOwAAAEPUKuz27Nmjb775xnl91apVGjdunJ588klVVFTU2XAAAAC4frU6FPvQQw9p5syZ6tOnj/72t79pwoQJuvvuu7VixQr98MMPWrRoUR2PCfxyNT0Ufuy5MW6aBAAA96jVHrvvvvtO/fv
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "m = sep2d.value < 0.5\n",
    "plt.figure()\n",
    "plt.hist(sep2d, bins=\"auto\")\n",
    "plt.xlabel(\"Separation [deg]\")\n",
    "plt.ylabel(\"Counts\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"../../plots/sfi_gals_to_sfi_groups.png\", bbox_inches=\"tight\",\n",
    "            dpi=450)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2MTF "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [\"RA\", \"DEC\", \"mag\", \"e_mag\", \"z_CMB\", \"r_hMpc\", \"e_rhMpc\", \"M\", \"eta\", \"e_eta\"]\n",
    "\n",
    "dtype = [(n, np.float32) for n in names]\n",
    "data = np.genfromtxt(join(tf_folder, \"twomtf_k.csv\"), delimiter=\",\", skip_header=1,\n",
    "                     usecols=[2 + n for n in range(len(names))])\n",
    "\n",
    "twomtf_gals = np.empty(len(data), dtype=dtype)\n",
    "for i, n in enumerate(names):\n",
    "    twomtf_gals[n] = data[:, i]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write to HDF5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "outdir = \"/mnt/extraspace/rstiskalek/catalogs\"\n",
    "fname = \"PV_compilation.hdf5\"\n",
    "\n",
    "with File(join(outdir, fname), 'w') as f:\n",
    "    # Write LOSS\n",
    "    grp = f.create_group(\"LOSS\")\n",
    "    for name in loss_data.dtype.names:\n",
    "        grp.create_dataset(name, data=loss_data[name])\n",
    "\n",
    "    # Write Foundation\n",
    "    grp = f.create_group(\"Foundation\")\n",
    "    for name in foundation_data.dtype.names:\n",
    "        grp.create_dataset(name, data=foundation_data[name])\n",
    "\n",
    "    # Write SFI gals\n",
    "    grp = f.create_group(\"SFI_gals\")\n",
    "    for name in sfi_gals.dtype.names:\n",
    "        grp.create_dataset(name, data=sfi_gals[name])\n",
    "\n",
    "    # Write SFI gals masked\n",
    "    grp = f.create_group(\"SFI_gals_masked\")\n",
    "    for name in sfi_gals_masked.dtype.names:\n",
    "        grp.create_dataset(name, data=sfi_gals_masked[name])\n",
    "\n",
    "    # Write SFI groups\n",
    "    grp = f.create_group(\"SFI_groups\")\n",
    "    for name in sfi_groups.dtype.names:\n",
    "        grp.create_dataset(name, data=sfi_groups[name])\n",
    "\n",
    "    # Write 2MTF gals\n",
    "    grp = f.create_group(\"2MTF\")\n",
    "    for name in twomtf_gals.dtype.names:\n",
    "        grp.create_dataset(name, data=twomtf_gals[name])\n",
    "\n",
    "    # Write Pantheon\n",
    "    grp = f.create_group(\"Pantheon+\")\n",
    "    for name in pantheon_data.keys():\n",
    "        grp.create_dataset(name, data=pantheon_data[name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv_csiborg",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}