read-str-columns-with-lsdb.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6d460d03-24e2-4e49-83fe-c07382d591d9",
   "metadata": {},
   "source": [
    "### Compare how different `lsdb.read_hipscat` kwargs affect the performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "04b0ae45-650b-4b31-9c39-451c23df4174",
   "metadata": {},
   "outputs": [],
   "source": [
    "import lsdb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5fd19cf5-e7a2-4e55-8e62-ed2b7f88a630",
   "metadata": {},
   "source": [
    "### Get data (PSC path is used), use \"band\" column only for most of the tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f469721d-731c-44c2-b7c7-6feb7529a6e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = '/ocean/projects/phy210048p/shared/hipscat/catalogs/ztf_axs/ztf_source'\n",
    "COLUMNS = ['band']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7512b223-2796-47e6-8559-8d1486d18e67",
   "metadata": {},
   "source": [
    "### Load a single column, default (numpy) Series backend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b6f26679-c6ac-46d1-ab32-0285ad4cc610",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "540 ms ± 1.36 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit catalog = lsdb.read_hipscat(PATH, columns=COLUMNS)\n",
    "catalog._ddf.partitions[0]['band'].max().compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53843229-2b4f-4017-b5de-4fe347c8300c",
   "metadata": {},
   "source": [
    "### Load a single column, pyarrow Series backend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "050ed494-ae79-4525-a8ad-0d228e04de2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "394 ms ± 2.33 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit catalog = lsdb.read_hipscat(PATH, columns=COLUMNS, dtype_backend='pyarrow')\n",
    "catalog._ddf.partitions[0]['band'].max().compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f2a59a9-3d63-48ec-a5c2-2148a39644ca",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Load all the columns, default Series backend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "fc9020f2-77a0-4fd2-9e62-ade787d4dc48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "569 ms ± 13.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit catalog = lsdb.read_hipscat(PATH)\n",
    "catalog._ddf.partitions[0]['band'].max().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f698b54-82e4-4552-a074-5895c60682b3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tape-venv-python3.9",
   "language": "python",
   "name": "tape-venv-python3.9"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}