cb3_trial_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "98feac9f-da52-49a1-86ec-3616e227380d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask\n",
    "import socket\n",
    "from dask_jobqueue import SLURMCluster\n",
    "from  distributed import Client\n",
    "import subprocess\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d3468739-55e6-4ea8-b80f-61185608fcbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# packages in environment at /home/arjunp/.conda/envs/r_env:\r\n",
      "#\r\n",
      "# Name                    Version                   Build  Channel\r\n",
      "_libgcc_mutex             0.1                        main  \r\n",
      "_openmp_mutex             5.1                       1_gnu  \r\n",
      "_r-mutex                  1.0.0               anacondar_1  \r\n",
      "_sysroot_linux-64_curr_repodata_hack 3                   haa98f57_10  \r\n",
      "anyio                     4.2.0            py39h06a4308_0  \r\n",
      "argon2-cffi               21.3.0             pyhd3eb1b0_0  \r\n",
      "argon2-cffi-bindings      21.2.0           py39h7f8727e_0  \r\n",
      "asttokens                 2.0.5              pyhd3eb1b0_0  \r\n",
      "async-lru                 2.0.4            py39h06a4308_0  \r\n",
      "attrs                     23.1.0           py39h06a4308_0  \r\n",
      "babel                     2.11.0           py39h06a4308_0  \r\n",
      "backcall                  0.2.0              pyhd3eb1b0_0  \r\n",
      "beautifulsoup4            4.12.3           py39h06a4308_0  \r\n",
      "binutils_impl_linux-64    2.38                 h2a08ee3_1  \r\n",
      "binutils_linux-64         2.38.0               hc2dff05_0  \r\n",
      "blas                      1.0                    openblas  \r\n",
      "bleach                    4.1.0              pyhd3eb1b0_0  \r\n",
      "brotli-python             1.0.9            py39h6a678d5_8  \r\n",
      "bwidget                   1.9.16               h9eba36c_0  \r\n",
      "bzip2                     1.0.8                h5eee18b_6  \r\n",
      "c-ares                    1.19.1               h5eee18b_0  \r\n",
      "ca-certificates           2024.7.2             h06a4308_0  \r\n",
      "cairo                     1.16.0               hb05425b_5  \r\n",
      "certifi                   2024.7.4         py39h06a4308_0  \r\n",
      "cffi                      1.16.0           py39h5eee18b_1  \r\n",
      "charset-normalizer        2.0.4              pyhd3eb1b0_0  \r\n",
      "childespy                 1.0.1                    pypi_0    pypi\r\n",
      "click                     8.1.7                    pypi_0    pypi\r\n",
      "cloudpickle               3.0.0                    pypi_0    pypi\r\n",
      "comm                      0.2.1            py39h06a4308_0  \r\n",
      "contourpy                 1.2.1                    pypi_0    pypi\r\n",
      "curl                      8.7.1                hdbd6064_0  \r\n",
      "cycler                    0.12.1                   pypi_0    pypi\r\n",
      "cyrus-sasl                2.1.28               h52b45da_1  \r\n",
      "dask                      2024.7.0                 pypi_0    pypi\r\n",
      "dask-jobqueue             0.8.5                    pypi_0    pypi\r\n",
      "dbus                      1.13.18              hb2f20db_0  \r\n",
      "debugpy                   1.6.7            py39h6a678d5_0  \r\n",
      "decorator                 5.1.1              pyhd3eb1b0_0  \r\n",
      "defusedxml                0.7.1              pyhd3eb1b0_0  \r\n",
      "distributed               2024.7.0                 pypi_0    pypi\r\n",
      "exceptiongroup            1.2.0            py39h06a4308_0  \r\n",
      "executing                 0.8.3              pyhd3eb1b0_0  \r\n",
      "expat                     2.6.2                h6a678d5_0  \r\n",
      "fontconfig                2.14.1               h4c34cd2_2  \r\n",
      "fonttools                 4.53.1                   pypi_0    pypi\r\n",
      "freetype                  2.12.1               h4a9f257_0  \r\n",
      "fribidi                   1.0.10               h7b6447c_0  \r\n",
      "fsspec                    2024.6.1                 pypi_0    pypi\r\n",
      "gcc_impl_linux-64         11.2.0               h1234567_1  \r\n",
      "gcc_linux-64              11.2.0               h5c386dc_0  \r\n",
      "gfortran_impl_linux-64    11.2.0               h1234567_1  \r\n",
      "gfortran_linux-64         11.2.0               hc2dff05_0  \r\n",
      "glib                      2.78.4               h6a678d5_0  \r\n",
      "glib-tools                2.78.4               h6a678d5_0  \r\n",
      "graphite2                 1.3.14               h295c915_1  \r\n",
      "gst-plugins-base          1.14.1               h6a678d5_1  \r\n",
      "gstreamer                 1.14.1               h5eee18b_1  \r\n",
      "gxx_impl_linux-64         11.2.0               h1234567_1  \r\n",
      "gxx_linux-64              11.2.0               hc2dff05_0  \r\n",
      "harfbuzz                  4.3.0                hf52aaf7_2  \r\n",
      "icu                       73.1                 h6a678d5_0  \r\n",
      "idna                      3.7              py39h06a4308_0  \r\n",
      "importlib-metadata        7.0.1            py39h06a4308_0  \r\n",
      "importlib-resources       6.4.0                    pypi_0    pypi\r\n",
      "importlib_metadata        7.0.1                hd3eb1b0_0  \r\n",
      "ipykernel                 6.28.0           py39h06a4308_0  \r\n",
      "ipython                   8.15.0           py39h06a4308_0  \r\n",
      "ipywidgets                8.1.2            py39h06a4308_0  \r\n",
      "jedi                      0.18.1           py39h06a4308_1  \r\n",
      "jinja2                    3.1.4            py39h06a4308_0  \r\n",
      "jpeg                      9e                   h5eee18b_1  \r\n",
      "json5                     0.9.6              pyhd3eb1b0_0  \r\n",
      "jsonschema                4.19.2           py39h06a4308_0  \r\n",
      "jsonschema-specifications 2023.7.1         py39h06a4308_0  \r\n",
      "jupyter                   1.0.0            py39h06a4308_9  \r\n",
      "jupyter-lsp               2.2.0            py39h06a4308_0  \r\n",
      "jupyter_client            8.6.0            py39h06a4308_0  \r\n",
      "jupyter_console           6.6.3            py39h06a4308_0  \r\n",
      "jupyter_core              5.7.2            py39h06a4308_0  \r\n",
      "jupyter_events            0.10.0           py39h06a4308_0  \r\n",
      "jupyter_server            2.14.1           py39h06a4308_0  \r\n",
      "jupyter_server_terminals  0.4.4            py39h06a4308_1  \r\n",
      "jupyterlab                4.0.11           py39h06a4308_0  \r\n",
      "jupyterlab_pygments       0.1.2                      py_0  \r\n",
      "jupyterlab_server         2.25.1           py39h06a4308_0  \r\n",
      "jupyterlab_widgets        3.0.10           py39h06a4308_0  \r\n",
      "kernel-headers_linux-64   3.10.0              h57e8cba_10  \r\n",
      "kiwisolver                1.4.5                    pypi_0    pypi\r\n",
      "krb5                      1.20.1               h143b758_1  \r\n",
      "ld_impl_linux-64          2.38                 h1181459_1  \r\n",
      "lerc                      3.0                  h295c915_0  \r\n",
      "libclang                  14.0.6          default_hc6dbbc7_1  \r\n",
      "libclang13                14.0.6          default_he11475f_1  \r\n",
      "libcups                   2.4.2                h2d74bed_1  \r\n",
      "libcurl                   8.7.1                h251f7ec_0  \r\n",
      "libdeflate                1.17                 h5eee18b_1  \r\n",
      "libedit                   3.1.20230828         h5eee18b_0  \r\n",
      "libev                     4.33                 h7f8727e_1  \r\n",
      "libffi                    3.4.4                h6a678d5_1  \r\n",
      "libgcc-devel_linux-64     11.2.0               h1234567_1  \r\n",
      "libgcc-ng                 11.2.0               h1234567_1  \r\n",
      "libgfortran-ng            11.2.0               h00389a5_1  \r\n",
      "libgfortran5              11.2.0               h1234567_1  \r\n",
      "libglib                   2.78.4               hdc74915_0  \r\n",
      "libgomp                   11.2.0               h1234567_1  \r\n",
      "libiconv                  1.16                 h5eee18b_3  \r\n",
      "libllvm14                 14.0.6               hdb19cb5_3  \r\n",
      "libnghttp2                1.57.0               h2d74bed_0  \r\n",
      "libopenblas               0.3.21               h043d6bf_0  \r\n",
      "libpng                    1.6.39               h5eee18b_0  \r\n",
      "libpq                     12.17                hdbd6064_0  \r\n",
      "libsodium                 1.0.18               h7b6447c_0  \r\n",
      "libssh2                   1.11.0               h251f7ec_0  \r\n",
      "libstdcxx-devel_linux-64  11.2.0               h1234567_1  \r\n",
      "libstdcxx-ng              11.2.0               h1234567_1  \r\n",
      "libtiff                   4.5.1                h6a678d5_0  \r\n",
      "libuuid                   1.41.5               h5eee18b_0  \r\n",
      "libwebp-base              1.3.2                h5eee18b_0  \r\n",
      "libxcb                    1.15                 h7f8727e_0  \r\n",
      "libxkbcommon              1.0.1                h5eee18b_1  \r\n",
      "libxml2                   2.10.4               hfdd30dd_2  \r\n",
      "locket                    1.0.0                    pypi_0    pypi\r\n",
      "lz4-c                     1.9.4                h6a678d5_1  \r\n",
      "make                      4.2.1                h1bed415_1  \r\n",
      "markupsafe                2.1.3            py39h5eee18b_0  \r\n",
      "matplotlib                3.9.1                    pypi_0    pypi\r\n",
      "matplotlib-inline         0.1.6            py39h06a4308_0  \r\n",
      "mistune                   2.0.4            py39h06a4308_0  \r\n",
      "msgpack                   1.0.8                    pypi_0    pypi\r\n",
      "mysql                     5.7.24               h721c034_2  \r\n",
      "nbclient                  0.8.0            py39h06a4308_0  \r\n",
      "nbconvert                 7.10.0           py39h06a4308_0  \r\n",
      "nbformat                  5.9.2            py39h06a4308_0  \r\n",
      "ncurses                   6.4                  h6a678d5_0  \r\n",
      "nest-asyncio              1.6.0            py39h06a4308_0  \r\n",
      "notebook                  7.0.8            py39h06a4308_2  \r\n",
      "notebook-shim             0.2.3            py39h06a4308_0  \r\n",
      "numpy                     2.0.0                    pypi_0    pypi\r\n",
      "openssl                   3.0.14               h5eee18b_0  \r\n",
      "overrides                 7.4.0            py39h06a4308_0  \r\n",
      "packaging                 24.1             py39h06a4308_0  \r\n",
      "pandas                    2.2.2                    pypi_0    pypi\r\n",
      "pandoc                    2.12                 h06a4308_3  \r\n",
      "pandocfilters             1.5.0              pyhd3eb1b0_0  \r\n",
      "pango                     1.50.7               h05da053_0  \r\n",
      "parso                     0.8.3              pyhd3eb1b0_0  \r\n",
      "partd                     1.4.2                    pypi_0    pypi\r\n",
      "pcre2                     10.42                hebb0a14_1  \r\n",
      "pexpect                   4.8.0              pyhd3eb1b0_3  \r\n",
      "pickleshare               0.7.5           pyhd3eb1b0_1003  \r\n",
      "pillow                    10.4.0                   pypi_0    pypi\r\n",
      "pip                       24.0             py39h06a4308_0  \r\n",
      "pixman                    0.40.0               h7f8727e_1  \r\n",
      "platformdirs              3.10.0           py39h06a4308_0  \r\n",
      "ply                       3.11             py39h06a4308_0  \r\n",
      "prometheus_client         0.14.1           py39h06a4308_0  \r\n",
      "prompt-toolkit            3.0.43           py39h06a4308_0  \r\n",
      "prompt_toolkit            3.0.43               hd3eb1b0_0  \r\n",
      "psutil                    5.9.0            py39h5eee18b_0  \r\n",
      "ptyprocess                0.7.0              pyhd3eb1b0_2  \r\n",
      "pure_eval                 0.2.2              pyhd3eb1b0_0  \r\n",
      "pycparser                 2.21               pyhd3eb1b0_0  \r\n",
      "pygments                  2.15.1           py39h06a4308_1  \r\n",
      "pyparsing                 3.1.2                    pypi_0    pypi\r\n",
      "pyqt                      5.15.10          py39h6a678d5_0  \r\n",
      "pyqt5-sip                 12.13.0          py39h5eee18b_0  \r\n",
      "pysocks                   1.7.1            py39h06a4308_0  \r\n",
      "python                    3.9.19               h955ad1f_1  \r\n",
      "python-dateutil           2.9.0post0       py39h06a4308_2  \r\n",
      "python-fastjsonschema     2.16.2           py39h06a4308_0  \r\n",
      "python-json-logger        2.0.7            py39h06a4308_0  \r\n",
      "pytz                      2024.1           py39h06a4308_0  \r\n",
      "pyyaml                    6.0.1            py39h5eee18b_0  \r\n",
      "pyzmq                     25.1.2           py39h6a678d5_0  \r\n",
      "qt-main                   5.15.2              h53bd1ea_10  \r\n",
      "qtconsole                 5.5.1            py39h06a4308_0  \r\n",
      "qtpy                      2.4.1            py39h06a4308_0  \r\n",
      "r-askpass                 1.2.0             r43h76d94ec_0  \r\n",
      "r-backports               1.4.1             r43h76d94ec_0  \r\n",
      "r-base                    4.3.1                h1ae530e_0  \r\n",
      "r-base64enc               0.1_3             r43h76d94ec_4  \r\n",
      "r-bit                     4.0.5             r43h76d94ec_0  \r\n",
      "r-bit64                   4.0.5             r43h76d94ec_0  \r\n",
      "r-blob                    1.2.4             r43h6115d3f_0  \r\n",
      "r-boot                    1.3_28.1          r43h6115d3f_0  \r\n",
      "r-broom                   1.0.5             r43h6115d3f_0  \r\n",
      "r-bslib                   0.5.1             r43h6115d3f_0  \r\n",
      "r-cachem                  1.0.8             r43h76d94ec_0  \r\n",
      "r-callr                   3.7.3             r43h6115d3f_0  \r\n",
      "r-caret                   6.0_94            r43h76d94ec_0  \r\n",
      "r-cellranger              1.1.0             r43h6115d3f_0  \r\n",
      "r-class                   7.3_22            r43h76d94ec_0  \r\n",
      "r-cli                     3.6.1             r43h884c59f_0  \r\n",
      "r-clipr                   0.8.0             r43h6115d3f_0  \r\n",
      "r-clock                   0.7.0             r43h884c59f_0  \r\n",
      "r-cluster                 2.1.4             r43h640688f_0  \r\n",
      "r-codetools               0.2_19            r43h6115d3f_0  \r\n",
      "r-colorspace              2.1_0             r43h76d94ec_0  \r\n",
      "r-commonmark              1.9.0             r43h76d94ec_0  \r\n",
      "r-conflicted              1.2.0             r43h142f84f_0  \r\n",
      "r-cpp11                   0.4.6             r43h6115d3f_0  \r\n",
      "r-crayon                  1.5.2             r43h6115d3f_0  \r\n",
      "r-curl                    5.1.0             r43h76d94ec_0  \r\n",
      "r-data.table              1.14.8            r43h76d94ec_0  \r\n",
      "r-dbi                     1.1.3             r43h6115d3f_0  \r\n",
      "r-dbplyr                  2.3.4             r43h6115d3f_0  \r\n",
      "r-diagram                 1.6.5             r43h6115d3f_0  \r\n",
      "r-digest                  0.6.33            r43h884c59f_0  \r\n",
      "r-dplyr                   1.1.3             r43h884c59f_0  \r\n",
      "r-dtplyr                  1.3.1             r43h6115d3f_0  \r\n",
      "r-e1071                   1.7_13            r43h884c59f_0  \r\n",
      "r-ellipsis                0.3.2             r43h76d94ec_0  \r\n",
      "r-essentials              4.3.1                     r43_0  \r\n",
      "r-evaluate                0.22              r43h6115d3f_0  \r\n",
      "r-fansi                   1.0.5             r43h76d94ec_0  \r\n",
      "r-farver                  2.1.1             r43h884c59f_0  \r\n",
      "r-fastmap                 1.1.1             r43h884c59f_0  \r\n",
      "r-fontawesome             0.5.2             r43h6115d3f_0  \r\n",
      "r-forcats                 1.0.0             r43h6115d3f_0  \r\n",
      "r-foreach                 1.5.2             r43h6115d3f_0  \r\n",
      "r-foreign                 0.8_85            r43h76d94ec_0  \r\n",
      "r-formatr                 1.14              r43h6115d3f_0  \r\n",
      "r-fs                      1.6.3             r43h884c59f_0  \r\n",
      "r-future                  1.33.0            r43h6115d3f_0  \r\n",
      "r-future.apply            1.11.0            r43h6115d3f_0  \r\n",
      "r-gargle                  1.5.2             r43h142f84f_0  \r\n",
      "r-generics                0.1.3             r43h142f84f_0  \r\n",
      "r-ggplot2                 3.4.4             r43h6115d3f_0  \r\n",
      "r-glmnet                  4.1_8             r43hb5eb8f6_0  \r\n",
      "r-globals                 0.16.2            r43h142f84f_0  \r\n",
      "r-glue                    1.6.2             r43h76d94ec_0  \r\n",
      "r-googledrive             2.1.1             r43h6115d3f_0  \r\n",
      "r-googlesheets4           1.1.1             r43h6115d3f_0  \r\n",
      "r-gower                   1.0.1             r43h76d94ec_0  \r\n",
      "r-gtable                  0.3.4             r43h6115d3f_0  \r\n",
      "r-hardhat                 1.3.0             r43h6115d3f_0  \r\n",
      "r-haven                   2.5.3             r43h884c59f_0  \r\n",
      "r-highr                   0.10              r43h6115d3f_0  \r\n",
      "r-hms                     1.1.3             r43h6115d3f_0  \r\n",
      "r-htmltools               0.5.6.1           r43h76d94ec_0  \r\n",
      "r-httpuv                  1.6.11            r43h884c59f_0  \r\n",
      "r-httr                    1.4.7             r43h6115d3f_0  \r\n",
      "r-ids                     1.0.1             r43h142f84f_0  \r\n",
      "r-ipred                   0.9_14            r43h76d94ec_0  \r\n",
      "r-irdisplay               1.1               r43h6115d3f_0  \r\n",
      "r-irkernel                1.3.2                     r43_0  \r\n",
      "r-isoband                 0.2.7             r43h884c59f_0  \r\n",
      "r-iterators               1.0.14            r43h6115d3f_0  \r\n",
      "r-jquerylib               0.1.4             r43h6115d3f_0  \r\n",
      "r-jsonlite                1.8.7             r43h76d94ec_0  \r\n",
      "r-kernsmooth              2.23_22           r43h640688f_0  \r\n",
      "r-knitr                   1.44              r43h6115d3f_0  \r\n",
      "r-labeling                0.4.3             r43h6115d3f_0  \r\n",
      "r-later                   1.3.1             r43h884c59f_0  \r\n",
      "r-lattice                 0.22_5            r43h76d94ec_0  \r\n",
      "r-lava                    1.7.2.1           r43h6115d3f_0  \r\n",
      "r-lifecycle               1.0.3             r43h142f84f_0  \r\n",
      "r-listenv                 0.9.0             r43h142f84f_0  \r\n",
      "r-lubridate               1.9.3             r43h76d94ec_0  \r\n",
      "r-magrittr                2.0.3             r43h76d94ec_0  \r\n",
      "r-mass                    7.3_60            r43h76d94ec_0  \r\n",
      "r-matrix                  1.6_1.1           r43h76d94ec_0  \r\n",
      "r-memoise                 2.0.1             r43h6115d3f_0  \r\n",
      "r-mgcv                    1.9_0             r43h76d94ec_0  \r\n",
      "r-mime                    0.12              r43h76d94ec_0  \r\n",
      "r-modelmetrics            1.2.2.2           r43h884c59f_0  \r\n",
      "r-modelr                  0.1.11            r43h6115d3f_0  \r\n",
      "r-munsell                 0.5.0             r43h6115d3f_0  \r\n",
      "r-nlme                    3.1_163           r43h640688f_0  \r\n",
      "r-nnet                    7.3_19            r43h76d94ec_0  \r\n",
      "r-numderiv                2016.8_1.1        r43h6115d3f_0  \r\n",
      "r-openssl                 2.1.1             r43h76d94ec_0  \r\n",
      "r-parallelly              1.36.0            r43h6115d3f_0  \r\n",
      "r-pbdzmq                  0.3_10            r43h884c59f_0  \r\n",
      "r-pillar                  1.9.0             r43h6115d3f_0  \r\n",
      "r-pkgconfig               2.0.3             r43h6115d3f_0  \r\n",
      "r-plyr                    1.8.9             r43h884c59f_0  \r\n",
      "r-prettyunits             1.2.0             r43h142f84f_0  \r\n",
      "r-proc                    1.18.4            r43h884c59f_0  \r\n",
      "r-processx                3.8.2             r43h76d94ec_0  \r\n",
      "r-prodlim                 2023.08.28        r43h884c59f_0  \r\n",
      "r-progress                1.2.2             r43h142f84f_0  \r\n",
      "r-progressr               0.14.0            r43h6115d3f_0  \r\n",
      "r-promises                1.2.1             r43h884c59f_0  \r\n",
      "r-proxy                   0.4_27            r43h76d94ec_0  \r\n",
      "r-ps                      1.7.5             r43h76d94ec_0  \r\n",
      "r-purrr                   1.0.2             r43h76d94ec_0  \r\n",
      "r-quantmod                0.4.25            r43h6115d3f_0  \r\n",
      "r-r6                      2.5.1             r43h6115d3f_0  \r\n",
      "r-ragg                    1.2.6             r43h884c59f_0  \r\n",
      "r-randomforest            4.7_1.1           r43h640688f_0  \r\n",
      "r-rappdirs                0.3.3             r43h76d94ec_0  \r\n",
      "r-rcolorbrewer            1.1_3             r43h6115d3f_1  \r\n",
      "r-rcpp                    1.0.11            r43h884c59f_0  \r\n",
      "r-rcppeigen               0.3.3.9.3         r43h884c59f_0  \r\n",
      "r-readr                   2.1.4             r43h884c59f_0  \r\n",
      "r-readxl                  1.4.3             r43h884c59f_0  \r\n",
      "r-recipes                 1.0.8             r43h6115d3f_0  \r\n",
      "r-recommended             4.3.1                     r43_0  \r\n",
      "r-rematch                 2.0.0             r43h6115d3f_0  \r\n",
      "r-rematch2                2.1.2             r43h142f84f_0  \r\n",
      "r-repr                    1.1.6             r43h6115d3f_0  \r\n",
      "r-reprex                  2.0.2             r43h6115d3f_0  \r\n",
      "r-reshape2                1.4.4             r43h884c59f_0  \r\n",
      "r-rlang                   1.1.1             r43h884c59f_0  \r\n",
      "r-rmarkdown               2.25              r43h6115d3f_0  \r\n",
      "r-rpart                   4.1.21            r43h76d94ec_0  \r\n",
      "r-rstudioapi              0.15.0            r43h6115d3f_0  \r\n",
      "r-rvest                   1.0.3             r43h6115d3f_0  \r\n",
      "r-sass                    0.4.7             r43h884c59f_0  \r\n",
      "r-scales                  1.2.1             r43h6115d3f_0  \r\n",
      "r-selectr                 0.4_2             r43h6115d3f_0  \r\n",
      "r-shape                   1.4.6             r43h142f84f_0  \r\n",
      "r-shiny                   1.7.5.1           r43h6115d3f_0  \r\n",
      "r-sourcetools             0.1.7_1           r43h884c59f_0  \r\n",
      "r-spatial                 7.3_17            r43h76d94ec_0  \r\n",
      "r-squarem                 2021.1            r43h142f84f_0  \r\n",
      "r-stringi                 1.7.12            r43h884c59f_0  \r\n",
      "r-stringr                 1.5.0             r43h6115d3f_0  \r\n",
      "r-survival                3.5_7             r43h76d94ec_0  \r\n",
      "r-sys                     3.4.2             r43h76d94ec_0  \r\n",
      "r-systemfonts             1.0.5             r43h884c59f_0  \r\n",
      "r-textshaping             0.3.7             r43h884c59f_0  \r\n",
      "r-tibble                  3.2.1             r43h76d94ec_0  \r\n",
      "r-tidyr                   1.3.0             r43h884c59f_0  \r\n",
      "r-tidyselect              1.2.0             r43h6115d3f_0  \r\n",
      "r-tidyverse               2.0.0             r43h6115d3f_0  \r\n",
      "r-timechange              0.2.0             r43h884c59f_0  \r\n",
      "r-timedate                4022.108          r43h6115d3f_0  \r\n",
      "r-tinytex                 0.48              r43h142f84f_0  \r\n",
      "r-ttr                     0.24.3            r43h76d94ec_0  \r\n",
      "r-tzdb                    0.4.0             r43h884c59f_0  \r\n",
      "r-utf8                    1.2.4             r43h76d94ec_0  \r\n",
      "r-uuid                    1.1_1             r43h76d94ec_0  \r\n",
      "r-vctrs                   0.6.4             r43h884c59f_0  \r\n",
      "r-viridislite             0.4.2             r43h6115d3f_0  \r\n",
      "r-vroom                   1.6.4             r43h884c59f_0  \r\n",
      "r-withr                   2.5.1             r43h6115d3f_0  \r\n",
      "r-xfun                    0.40              r43h76d94ec_0  \r\n",
      "r-xml2                    1.3.5             r43h884c59f_0  \r\n",
      "r-xtable                  1.8_4             r43h6115d3f_0  \r\n",
      "r-xts                     0.13.1            r43h76d94ec_0  \r\n",
      "r-yaml                    2.3.7             r43h76d94ec_0  \r\n",
      "r-zoo                     1.8_12            r43h76d94ec_0  \r\n",
      "readline                  8.2                  h5eee18b_0  \r\n",
      "referencing               0.30.2           py39h06a4308_0  \r\n",
      "requests                  2.32.2           py39h06a4308_0  \r\n",
      "rfc3339-validator         0.1.4            py39h06a4308_0  \r\n",
      "rfc3986-validator         0.1.1            py39h06a4308_0  \r\n",
      "rpds-py                   0.10.6           py39hb02cf49_0  \r\n",
      "rpy2                      3.5.16                   pypi_0    pypi\r\n",
      "seaborn                   0.13.2                   pypi_0    pypi\r\n",
      "send2trash                1.8.2            py39h06a4308_0  \r\n",
      "setuptools                69.5.1           py39h06a4308_0  \r\n",
      "sip                       6.7.12           py39h6a678d5_0  \r\n",
      "six                       1.16.0             pyhd3eb1b0_1  \r\n",
      "sniffio                   1.3.0            py39h06a4308_0  \r\n",
      "sortedcontainers          2.4.0                    pypi_0    pypi\r\n",
      "soupsieve                 2.5              py39h06a4308_0  \r\n",
      "sqlite                    3.45.3               h5eee18b_0  \r\n",
      "stack_data                0.2.0              pyhd3eb1b0_0  \r\n",
      "sysroot_linux-64          2.17                h57e8cba_10  \r\n",
      "tblib                     3.0.0                    pypi_0    pypi\r\n",
      "terminado                 0.17.1           py39h06a4308_0  \r\n",
      "tinycss2                  1.2.1            py39h06a4308_0  \r\n",
      "tk                        8.6.14               h39e8969_0  \r\n",
      "tktable                   2.10                 h3d55465_1  \r\n",
      "tomli                     2.0.1            py39h06a4308_0  \r\n",
      "toolz                     0.12.1                   pypi_0    pypi\r\n",
      "tornado                   6.4.1            py39h5eee18b_0  \r\n",
      "traitlets                 5.14.3           py39h06a4308_0  \r\n",
      "typing-extensions         4.11.0           py39h06a4308_0  \r\n",
      "typing_extensions         4.11.0           py39h06a4308_0  \r\n",
      "tzdata                    2024.1                   pypi_0    pypi\r\n",
      "tzlocal                   5.2                      pypi_0    pypi\r\n",
      "urllib3                   2.2.2            py39h06a4308_0  \r\n",
      "wcwidth                   0.2.5              pyhd3eb1b0_0  \r\n",
      "webencodings              0.5.1            py39h06a4308_1  \r\n",
      "websocket-client          1.8.0            py39h06a4308_0  \r\n",
      "wheel                     0.43.0           py39h06a4308_0  \r\n",
      "widgetsnbextension        4.0.10           py39h06a4308_0  \r\n",
      "xz                        5.4.6                h5eee18b_1  \r\n",
      "yaml                      0.2.5                h7b6447c_0  \r\n",
      "zeromq                    4.3.5                h6a678d5_0  \r\n",
      "zict                      3.0.0                    pypi_0    pypi\r\n",
      "zipp                      3.17.0           py39h06a4308_0  \r\n",
      "zlib                      1.2.13               h5eee18b_1  \r\n",
      "zstd                      1.5.5                hc292b87_2  \r\n"
     ]
    }
   ],
   "source": [
    "!conda list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d4680e99-6135-45a1-a8ea-4dbe98127eef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "node072\r\n"
     ]
    }
   ],
   "source": [
    "!hostname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "30ac9d06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting dask-jobqueue\n",
      "  Using cached dask_jobqueue-0.8.5-py2.py3-none-any.whl.metadata (1.6 kB)\n",
      "Requirement already satisfied: dask>=2022.02.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from dask-jobqueue) (2023.6.0)\n",
      "Requirement already satisfied: distributed>=2022.02.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from dask-jobqueue) (2023.6.0)\n",
      "Requirement already satisfied: click>=8.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from dask>=2022.02.0->dask-jobqueue) (8.1.7)\n",
      "Requirement already satisfied: cloudpickle>=1.5.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from dask>=2022.02.0->dask-jobqueue) (2.2.1)\n",
      "Requirement already satisfied: fsspec>=2021.09.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from dask>=2022.02.0->dask-jobqueue) (2023.4.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from dask>=2022.02.0->dask-jobqueue) (23.1)\n",
      "Requirement already satisfied: partd>=1.2.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from dask>=2022.02.0->dask-jobqueue) (1.4.0)\n",
      "Requirement already satisfied: pyyaml>=5.3.1 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from dask>=2022.02.0->dask-jobqueue) (6.0.1)\n",
      "Requirement already satisfied: toolz>=0.10.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from dask>=2022.02.0->dask-jobqueue) (0.12.0)\n",
      "Requirement already satisfied: importlib-metadata>=4.13.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from dask>=2022.02.0->dask-jobqueue) (6.0.0)\n",
      "Requirement already satisfied: jinja2>=2.10.3 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from distributed>=2022.02.0->dask-jobqueue) (3.1.2)\n",
      "Requirement already satisfied: locket>=1.0.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from distributed>=2022.02.0->dask-jobqueue) (1.0.0)\n",
      "Requirement already satisfied: msgpack>=1.0.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from distributed>=2022.02.0->dask-jobqueue) (1.0.3)\n",
      "Requirement already satisfied: psutil>=5.7.2 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from distributed>=2022.02.0->dask-jobqueue) (5.9.0)\n",
      "Requirement already satisfied: sortedcontainers>=2.0.5 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from distributed>=2022.02.0->dask-jobqueue) (2.4.0)\n",
      "Requirement already satisfied: tblib>=1.6.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from distributed>=2022.02.0->dask-jobqueue) (1.7.0)\n",
      "Requirement already satisfied: tornado>=6.0.4 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from distributed>=2022.02.0->dask-jobqueue) (6.3.3)\n",
      "Requirement already satisfied: urllib3>=1.24.3 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from distributed>=2022.02.0->dask-jobqueue) (1.26.18)\n",
      "Requirement already satisfied: zict>=2.2.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from distributed>=2022.02.0->dask-jobqueue) (3.0.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from importlib-metadata>=4.13.0->dask>=2022.02.0->dask-jobqueue) (3.11.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages (from jinja2>=2.10.3->distributed>=2022.02.0->dask-jobqueue) (2.1.1)\n",
      "Using cached dask_jobqueue-0.8.5-py2.py3-none-any.whl (47 kB)\n",
      "Installing collected packages: dask-jobqueue\n",
      "Successfully installed dask-jobqueue-0.8.5\n"
     ]
    }
   ],
   "source": [
    "#!pip install dask-jobqueue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c7a174d8-19a9-45ff-85f7-166976fde4a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arjunp/.local/lib/python3.11/site-packages/dask_jobqueue/core.py:266: FutureWarning: job_extra has been renamed to job_extra_directives. You are still using it (even if only set to []; please also check config files). If you did not set job_extra_directives yet, job_extra will be respected for now, but it will be removed in a future release. If you already set job_extra_directives, job_extra is ignored and you can remove it.\n",
      "  warnings.warn(warn, FutureWarning)\n",
      "/cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages/distributed/node.py:182: UserWarning: Port 8787 is already in use.\n",
      "Perhaps you already have a cluster running?\n",
      "Hosting the HTTP server on port 42599 instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#!/usr/bin/env bash\n",
      "\n",
      "#SBATCH -J dask-worker\n",
      "#SBATCH -p normal\n",
      "#SBATCH -A cpl\n",
      "#SBATCH -n 1\n",
      "#SBATCH --cpus-per-task=8\n",
      "#SBATCH --mem=15G\n",
      "#SBATCH -t 01:00:00\n",
      "#SBATCH --gres=gpu:QUADRORTX6000:1\n",
      "source /etc/profile.d/modules.sh\n",
      "module load openmind8/anaconda/3-2023.09-0\n",
      "module load openmind/gcc/11.1.0\n",
      "module load openmind/ffmpeg/20160310\n",
      "source ~/.bashrc\n",
      "conda activate codebase3\n",
      "cd /om/user/arjunp/ashraf_repo/whisper-diarization\n",
      "conda list\n",
      "/cm/shared/openmind8/anaconda/3-2023.09-0/bin/python -m distributed.cli.dask_worker tcp://172.16.20.80:46205 --name dummy-name --nthreads 2 --memory-limit 3.73GiB --nworkers 4 --nanny --death-timeout 60\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cluster2 = SLURMCluster(cores=8,\n",
    "                       processes=4,\n",
    "                       memory=\"16GB\",\n",
    "                       account=\"cpl\",\n",
    "                       walltime=\"01:00:00\",\n",
    "                       queue=\"normal\",\n",
    "                       job_script_prologue  =[\n",
    "                        'source /etc/profile.d/modules.sh' ,\n",
    "                        'module load openmind8/anaconda/3-2023.09-0',\n",
    "                        'module load openmind/gcc/11.1.0',\n",
    "                        'module load openmind/ffmpeg/20160310' , 'source ~/.bashrc',\n",
    "                        'conda activate codebase3',\n",
    "                        'cd /om/user/arjunp/ashraf_repo/whisper-diarization',\n",
    "                        'conda list'\n",
    "                                              ],\n",
    "                       job_extra=['--gres=gpu:QUADRORTX6000:1'] \n",
    "                       )\n",
    "cluster2.scale(1)\n",
    "print(cluster2.job_script())\n",
    "client2 = Client(cluster2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "24c1d286-8ec2-4a30-a5c1-713ec9283408",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)\r\n",
      "          37425834    normal dask-wor   arjunp  R       0:04      1 node079\r\n",
      "          37419856    normal     bash   arjunp  R    1:34:46      1 node098\r\n"
     ]
    }
   ],
   "source": [
    "!squeue -u arjunp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8dba9413",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!scancel 37425828"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "05109fe7-4128-461e-a4fc-42aeada8ad6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_cb3_script(**kwargs):\n",
    "     \n",
    "    command = ['python', '/om/user/arjunp/ashraf_repo/whisper-diarization/diarize.py','-a','/om/user/arjunp/first_min_030327.wav']  \n",
    "    \n",
    "    result = subprocess.run(command, capture_output=True, text=True)\n",
    "     \n",
    "    print(\"STDOUT:\", result.stdout)\n",
    "    print(\"STDERR:\", result.stderr)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1fafa54e-f877-4970-b994-48fac43507b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "future2 = client2.submit(run_cb3_script)  \n",
    "result2 = future2.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "8d55f0bc-5bc7-42be-8aee-d3e28278ca33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected model is a bag of 1 models. You will see that many progress bars per track.\n",
      "Separated tracks will be stored in /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/htdemucs\n",
      "Separating track /om/user/arjunp/first_min_030327.wav\n",
      "GPU Name: Quadro RTX 6000\n",
      "CUDA Compute Capability: (7, 5)\n",
      "Your GPU supports efficient float16 computation.\n",
      "Model was trained with pyannote.audio 0.0.1, yours is 3.1.1. Bad things might happen unless you revert pyannote.audio to 0.x.\n",
      "Model was trained with torch 1.10.0+cu102, yours is 2.1.2+cu121. Bad things might happen unless you revert torch to 1.x.\n",
      "[NeMo I 2024-07-24 13:49:03 msdd_models:1092] Loading pretrained diar_msdd_telephonic model from NGC\n",
      "[NeMo I 2024-07-24 13:49:03 cloud:58] Found existing object /home/arjunp/.cache/torch/NeMo/NeMo_1.20.0/diar_msdd_telephonic/3c3697a0a46f945574fa407149975a13/diar_msdd_telephonic.nemo.\n",
      "[NeMo I 2024-07-24 13:49:03 cloud:64] Re-using file from: /home/arjunp/.cache/torch/NeMo/NeMo_1.20.0/diar_msdd_telephonic/3c3697a0a46f945574fa407149975a13/diar_msdd_telephonic.nemo\n",
      "[NeMo I 2024-07-24 13:49:03 common:913] Instantiating model from pre-trained checkpoint\n",
      "[NeMo I 2024-07-24 13:49:04 features:289] PADDING: 16\n",
      "[NeMo I 2024-07-24 13:49:06 features:289] PADDING: 16\n",
      "[NeMo I 2024-07-24 13:49:07 save_restore_connector:249] Model EncDecDiarLabelModel was successfully restored from /home/arjunp/.cache/torch/NeMo/NeMo_1.20.0/diar_msdd_telephonic/3c3697a0a46f945574fa407149975a13/diar_msdd_telephonic.nemo.\n",
      "[NeMo I 2024-07-24 13:49:07 features:289] PADDING: 16\n",
      "[NeMo I 2024-07-24 13:49:08 clustering_diarizer:127] Loading pretrained vad_multilingual_marblenet model from NGC\n",
      "[NeMo I 2024-07-24 13:49:08 cloud:58] Found existing object /home/arjunp/.cache/torch/NeMo/NeMo_1.20.0/vad_multilingual_marblenet/670f425c7f186060b7a7268ba6dfacb2/vad_multilingual_marblenet.nemo.\n",
      "[NeMo I 2024-07-24 13:49:08 cloud:64] Re-using file from: /home/arjunp/.cache/torch/NeMo/NeMo_1.20.0/vad_multilingual_marblenet/670f425c7f186060b7a7268ba6dfacb2/vad_multilingual_marblenet.nemo\n",
      "[NeMo I 2024-07-24 13:49:08 common:913] Instantiating model from pre-trained checkpoint\n",
      "[NeMo I 2024-07-24 13:49:08 features:289] PADDING: 16\n",
      "[NeMo I 2024-07-24 13:49:08 save_restore_connector:249] Model EncDecClassificationModel was successfully restored from /home/arjunp/.cache/torch/NeMo/NeMo_1.20.0/vad_multilingual_marblenet/670f425c7f186060b7a7268ba6dfacb2/vad_multilingual_marblenet.nemo.\n",
      "[NeMo I 2024-07-24 13:49:08 msdd_models:864] Multiscale Weights: [1, 1, 1, 1, 1]\n",
      "[NeMo I 2024-07-24 13:49:08 msdd_models:865] Clustering Parameters: {\n",
      "        \"oracle_num_speakers\": false,\n",
      "        \"max_num_speakers\": 8,\n",
      "        \"enhanced_count_thres\": 80,\n",
      "        \"max_rp_threshold\": 0.25,\n",
      "        \"sparse_search_volume\": 30,\n",
      "        \"maj_vote_spk_count\": false,\n",
      "        \"chunk_cluster_count\": 50,\n",
      "        \"embeddings_per_chunk\": 10000\n",
      "    }\n",
      "[NeMo I 2024-07-24 13:49:08 speaker_utils:93] Number of files to diarize: 1\n",
      "[NeMo I 2024-07-24 13:49:08 clustering_diarizer:309] Split long audio file to avoid CUDA memory issue\n",
      "[NeMo I 2024-07-24 13:49:30 classification_models:272] Perform streaming frame-level VAD\n",
      "[NeMo I 2024-07-24 13:49:30 collections:301] Filtered duration for loading collection is  0.00 hours.\n",
      "[NeMo I 2024-07-24 13:49:30 collections:302] Dataset loaded with 2 items, total duration of  0.02 hours.\n",
      "[NeMo I 2024-07-24 13:49:30 collections:304] # 2 files loaded accounting to # 1 labels\n",
      "[NeMo I 2024-07-24 13:49:31 clustering_diarizer:250] Generating predictions with overlapping input segments\n",
      "[NeMo I 2024-07-24 13:49:31 clustering_diarizer:262] Converting frame level prediction to speech/no-speech segment in start and end times format.\n",
      "[NeMo I 2024-07-24 13:49:31 clustering_diarizer:287] Subsegmentation for embedding extraction: scale0, /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/subsegments_scale0.json\n",
      "[NeMo I 2024-07-24 13:49:31 clustering_diarizer:343] Extracting embeddings for Diarization\n",
      "[NeMo I 2024-07-24 13:49:32 collections:301] Filtered duration for loading collection is  0.00 hours.\n",
      "[NeMo I 2024-07-24 13:49:32 collections:302] Dataset loaded with 30 items, total duration of  0.00 hours.\n",
      "[NeMo I 2024-07-24 13:49:32 collections:304] # 30 files loaded accounting to # 1 labels\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:389] Saved embedding files to /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/embeddings\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:287] Subsegmentation for embedding extraction: scale1, /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/subsegments_scale1.json\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:343] Extracting embeddings for Diarization\n",
      "[NeMo I 2024-07-24 13:49:32 collections:301] Filtered duration for loading collection is  0.00 hours.\n",
      "[NeMo I 2024-07-24 13:49:32 collections:302] Dataset loaded with 32 items, total duration of  0.00 hours.\n",
      "[NeMo I 2024-07-24 13:49:32 collections:304] # 32 files loaded accounting to # 1 labels\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:389] Saved embedding files to /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/embeddings\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:287] Subsegmentation for embedding extraction: scale2, /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/subsegments_scale2.json\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:343] Extracting embeddings for Diarization\n",
      "[NeMo I 2024-07-24 13:49:32 collections:301] Filtered duration for loading collection is  0.00 hours.\n",
      "[NeMo I 2024-07-24 13:49:32 collections:302] Dataset loaded with 34 items, total duration of  0.00 hours.\n",
      "[NeMo I 2024-07-24 13:49:32 collections:304] # 34 files loaded accounting to # 1 labels\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:389] Saved embedding files to /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/embeddings\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:287] Subsegmentation for embedding extraction: scale3, /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/subsegments_scale3.json\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:343] Extracting embeddings for Diarization\n",
      "[NeMo I 2024-07-24 13:49:32 collections:301] Filtered duration for loading collection is  0.00 hours.\n",
      "[NeMo I 2024-07-24 13:49:32 collections:302] Dataset loaded with 40 items, total duration of  0.01 hours.\n",
      "[NeMo I 2024-07-24 13:49:32 collections:304] # 40 files loaded accounting to # 1 labels\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:389] Saved embedding files to /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/embeddings\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:287] Subsegmentation for embedding extraction: scale4, /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/subsegments_scale4.json\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:343] Extracting embeddings for Diarization\n",
      "[NeMo I 2024-07-24 13:49:32 collections:301] Filtered duration for loading collection is  0.00 hours.\n",
      "[NeMo I 2024-07-24 13:49:32 collections:302] Dataset loaded with 56 items, total duration of  0.01 hours.\n",
      "[NeMo I 2024-07-24 13:49:32 collections:304] # 56 files loaded accounting to # 1 labels\n",
      "[NeMo I 2024-07-24 13:49:32 clustering_diarizer:389] Saved embedding files to /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/embeddings\n",
      "[NeMo I 2024-07-24 13:49:33 clustering_diarizer:464] Outputs are saved in /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs directory\n",
      "[NeMo I 2024-07-24 13:49:33 msdd_models:960] Loading embedding pickle file of scale:0 at /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/embeddings/subsegments_scale0_embeddings.pkl\n",
      "[NeMo I 2024-07-24 13:49:33 msdd_models:960] Loading embedding pickle file of scale:1 at /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/embeddings/subsegments_scale1_embeddings.pkl\n",
      "[NeMo I 2024-07-24 13:49:33 msdd_models:960] Loading embedding pickle file of scale:2 at /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/embeddings/subsegments_scale2_embeddings.pkl\n",
      "[NeMo I 2024-07-24 13:49:33 msdd_models:960] Loading embedding pickle file of scale:3 at /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/embeddings/subsegments_scale3_embeddings.pkl\n",
      "[NeMo I 2024-07-24 13:49:33 msdd_models:960] Loading embedding pickle file of scale:4 at /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/embeddings/subsegments_scale4_embeddings.pkl\n",
      "[NeMo I 2024-07-24 13:49:33 msdd_models:938] Loading cluster label file from /weka/scratch/weka/cpl/arjunp/ashraf_repo/whisper-diarization/temp_outputs/speaker_outputs/subsegments_scale4_cluster.label\n",
      "[NeMo I 2024-07-24 13:49:33 collections:617] Filtered duration for loading collection is 0.000000.\n",
      "[NeMo I 2024-07-24 13:49:33 collections:620] Total 1 session files loaded accounting to # 1 audio clips\n",
      "[NeMo I 2024-07-24 13:49:33 msdd_models:1403]      [Threshold: 0.7000] [use_clus_as_main=False] [diar_window=50]\n",
      "[NeMo I 2024-07-24 13:49:33 speaker_utils:93] Number of files to diarize: 1\n",
      "[NeMo I 2024-07-24 13:49:33 speaker_utils:93] Number of files to diarize: 1\n",
      "[NeMo I 2024-07-24 13:49:33 speaker_utils:93] Number of files to diarize: 1\n",
      "[NeMo I 2024-07-24 13:49:33 speaker_utils:93] Number of files to diarize: 1\n",
      "[NeMo I 2024-07-24 13:49:33 msdd_models:1431]   \n",
      "    \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(result2.stdout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "928c01af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[NeMo W 2024-07-24 13:48:30 nemo_logging:349] /home/arjunp/.conda/envs/codebase3/lib/python3.8/site-packages/pyannote/audio/core/io.py:43: UserWarning: torchaudio._backend.set_audio_backend has been deprecated. With dispatcher enabled, this function is no-op. You can remove the function call.\\n      torchaudio.set_audio_backend(\"soundfile\")\\n    \\ntorchvision is not available - cannot save figures\\n/home/arjunp/.conda/envs/codebase3/lib/python3.8/site-packages/demucs/api.py:27: UserWarning: TorchAudio\\'s global backend is now deprecated. Please enable distpatcher by setting `TORCHAUDIO_USE_BACKEND_DISPATCHER=1`, and specify backend when calling load/info/save function.\\n  import torchaudio as ta\\n\\n  0%|                                                                                  | 0.0/64.35 [00:00<?, ?seconds/s]\\n  9%|██████▋                                                                  | 5.85/64.35 [00:02<00:29,  1.99seconds/s]\\n 18%|█████████████▎                                                           | 11.7/64.35 [00:03<00:11,  4.51seconds/s]\\n 27%|████████████████                                           | 17.549999999999997/64.35 [00:03<00:06,  7.63seconds/s]\\n 36%|██████████████████████████▌                                              | 23.4/64.35 [00:03<00:03, 11.37seconds/s]\\n 45%|████████████████████████████████▋                                       | 29.25/64.35 [00:03<00:02, 15.57seconds/s]\\n 55%|████████████████████████████████▏                          | 35.099999999999994/64.35 [00:03<00:01, 20.07seconds/s]\\n 64%|█████████████████████████████████████▌                     | 40.949999999999996/64.35 [00:03<00:00, 24.58seconds/s]\\n 73%|█████████████████████████████████████████████████████                    | 46.8/64.35 [00:03<00:00, 28.83seconds/s]\\n 82%|██████████████████████████████████████████████████████████▉             | 52.65/64.35 [00:03<00:00, 32.61seconds/s]\\n 91%|██████████████████████████████████████████████████████████████████▎      | 58.5/64.35 [00:04<00:00, 35.78seconds/s]\\n100%|████████████████████████████████████████████████████████████████████████| 64.35/64.35 [00:04<00:00, 38.73seconds/s]\\n100%|████████████████████████████████████████████████████████████████████████| 64.35/64.35 [00:04<00:00, 15.17seconds/s]\\nLightning automatically upgraded your loaded checkpoint from v1.5.4 to v1.9.4. To apply the upgrade to your files permanently, run `python -m pytorch_lightning.utilities.upgrade_checkpoint --file ../../../../../../../home/arjunp/.cache/torch/whisperx-vad-segmentation.bin`\\n[NeMo W 2024-07-24 13:49:04 modelPT:161] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.\\n    Train config : \\n    manifest_filepath: null\\n    emb_dir: null\\n    sample_rate: 16000\\n    num_spks: 2\\n    soft_label_thres: 0.5\\n    labels: null\\n    batch_size: 15\\n    emb_batch_size: 0\\n    shuffle: true\\n    \\n[NeMo W 2024-07-24 13:49:04 modelPT:168] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). \\n    Validation config : \\n    manifest_filepath: null\\n    emb_dir: null\\n    sample_rate: 16000\\n    num_spks: 2\\n    soft_label_thres: 0.5\\n    labels: null\\n    batch_size: 15\\n    emb_batch_size: 0\\n    shuffle: false\\n    \\n[NeMo W 2024-07-24 13:49:04 modelPT:174] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).\\n    Test config : \\n    manifest_filepath: null\\n    emb_dir: null\\n    sample_rate: 16000\\n    num_spks: 2\\n    soft_label_thres: 0.5\\n    labels: null\\n    batch_size: 15\\n    emb_batch_size: 0\\n    shuffle: false\\n    seq_eval_mode: false\\n    \\n[NeMo W 2024-07-24 13:49:08 modelPT:161] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.\\n    Train config : \\n    manifest_filepath: /manifests/ami_train_0.63.json,/manifests/freesound_background_train.json,/manifests/freesound_laughter_train.json,/manifests/fisher_2004_background.json,/manifests/fisher_2004_speech_sampled.json,/manifests/google_train_manifest.json,/manifests/icsi_all_0.63.json,/manifests/musan_freesound_train.json,/manifests/musan_music_train.json,/manifests/musan_soundbible_train.json,/manifests/mandarin_train_sample.json,/manifests/german_train_sample.json,/manifests/spanish_train_sample.json,/manifests/french_train_sample.json,/manifests/russian_train_sample.json\\n    sample_rate: 16000\\n    labels:\\n    - background\\n    - speech\\n    batch_size: 256\\n    shuffle: true\\n    is_tarred: false\\n    tarred_audio_filepaths: null\\n    tarred_shard_strategy: scatter\\n    augmentor:\\n      shift:\\n        prob: 0.5\\n        min_shift_ms: -10.0\\n        max_shift_ms: 10.0\\n      white_noise:\\n        prob: 0.5\\n        min_level: -90\\n        max_level: -46\\n        norm: true\\n      noise:\\n        prob: 0.5\\n        manifest_path: /manifests/noise_0_1_musan_fs.json\\n        min_snr_db: 0\\n        max_snr_db: 30\\n        max_gain_db: 300.0\\n        norm: true\\n      gain:\\n        prob: 0.5\\n        min_gain_dbfs: -10.0\\n        max_gain_dbfs: 10.0\\n        norm: true\\n    num_workers: 16\\n    pin_memory: true\\n    \\n[NeMo W 2024-07-24 13:49:08 modelPT:168] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). \\n    Validation config : \\n    manifest_filepath: /manifests/ami_dev_0.63.json,/manifests/freesound_background_dev.json,/manifests/freesound_laughter_dev.json,/manifests/ch120_moved_0.63.json,/manifests/fisher_2005_500_speech_sampled.json,/manifests/google_dev_manifest.json,/manifests/musan_music_dev.json,/manifests/mandarin_dev.json,/manifests/german_dev.json,/manifests/spanish_dev.json,/manifests/french_dev.json,/manifests/russian_dev.json\\n    sample_rate: 16000\\n    labels:\\n    - background\\n    - speech\\n    batch_size: 256\\n    shuffle: false\\n    val_loss_idx: 0\\n    num_workers: 16\\n    pin_memory: true\\n    \\n[NeMo W 2024-07-24 13:49:08 modelPT:174] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).\\n    Test config : \\n    manifest_filepath: null\\n    sample_rate: 16000\\n    labels:\\n    - background\\n    - speech\\n    batch_size: 128\\n    shuffle: false\\n    test_loss_idx: 0\\n    \\n\\nsplitting manifest:   0%|          | 0/1 [00:00<?, ?it/s]\\nsplitting manifest: 100%|██████████| 1/1 [00:22<00:00, 22.18s/it]\\nsplitting manifest: 100%|██████████| 1/1 [00:22<00:00, 22.18s/it]\\n\\nvad:   0%|          | 0/2 [00:00<?, ?it/s]\\nvad:  50%|█████     | 1/2 [00:00<00:00,  1.83it/s]\\nvad: 100%|██████████| 2/2 [00:00<00:00,  3.23it/s]\\n\\ngenerating preds:   0%|          | 0/1 [00:00<?, ?it/s]\\ngenerating preds: 100%|██████████| 1/1 [00:00<00:00,  1.68it/s]\\n                                                               \\n\\ncreating speech segments:   0%|          | 0/1 [00:00<?, ?it/s]\\ncreating speech segments: 100%|██████████| 1/1 [00:00<00:00,  7.47it/s]\\ncreating speech segments: 100%|██████████| 1/1 [00:00<00:00,  7.46it/s]\\n\\n[1/5] extract embeddings:   0%|          | 0/1 [00:00<?, ?it/s]\\n[1/5] extract embeddings: 100%|██████████| 1/1 [00:00<00:00,  6.13it/s]\\n[1/5] extract embeddings: 100%|██████████| 1/1 [00:00<00:00,  6.12it/s]\\n\\n[2/5] extract embeddings:   0%|          | 0/1 [00:00<?, ?it/s]\\n[2/5] extract embeddings: 100%|██████████| 1/1 [00:00<00:00, 14.81it/s]\\n\\n[3/5] extract embeddings:   0%|          | 0/1 [00:00<?, ?it/s]\\n[3/5] extract embeddings: 100%|██████████| 1/1 [00:00<00:00, 14.21it/s]\\n\\n[4/5] extract embeddings:   0%|          | 0/1 [00:00<?, ?it/s]\\n[4/5] extract embeddings: 100%|██████████| 1/1 [00:00<00:00, 12.94it/s]\\n\\n[5/5] extract embeddings:   0%|          | 0/1 [00:00<?, ?it/s]\\n[5/5] extract embeddings: 100%|██████████| 1/1 [00:00<00:00, 12.78it/s]\\n\\nclustering:   0%|          | 0/1 [00:00<?, ?it/s]\\nclustering: 100%|██████████| 1/1 [00:00<00:00,  2.44it/s]\\nclustering: 100%|██████████| 1/1 [00:00<00:00,  2.44it/s]\\n[NeMo W 2024-07-24 13:49:33 der:185] Check if each ground truth RTTMs were present in the provided manifest file. Skipping calculation of Diariazation Error Rate\\n\\n  0%|          | 0/1 [00:00<?, ?it/s]\\n100%|██████████| 1/1 [00:00<00:00,  8.70it/s]\\n100%|██████████| 1/1 [00:00<00:00,  8.67it/s]\\n[NeMo W 2024-07-24 13:49:33 der:185] Check if each ground truth RTTMs were present in the provided manifest file. Skipping calculation of Diariazation Error Rate\\n[NeMo W 2024-07-24 13:49:33 der:185] Check if each ground truth RTTMs were present in the provided manifest file. Skipping calculation of Diariazation Error Rate\\n[NeMo W 2024-07-24 13:49:33 der:185] Check if each ground truth RTTMs were present in the provided manifest file. Skipping calculation of Diariazation Error Rate\\n'"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result2.stderr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9fa604f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# conda environments:\r\n",
      "#\r\n",
      "base                     /cm/shared/openmind8/anaconda/3-2023.09-0\r\n",
      "jax                      /cm/shared/openmind8/anaconda/3-2023.09-0/envs/jax\r\n",
      "nemo2                    /cm/shared/openmind8/anaconda/3-2023.09-0/envs/nemo2\r\n",
      "test                     /cm/shared/openmind8/anaconda/3-2023.09-0/envs/test\r\n",
      "test1                    /cm/shared/openmind8/anaconda/3-2023.09-0/envs/test1\r\n",
      "codebase3                /home/arjunp/.conda/envs/codebase3\r\n",
      "r_env                 *  /home/arjunp/.conda/envs/r_env\r\n",
      "torch_gpu                /home/arjunp/.conda/envs/torch_gpu\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!conda  info --envs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bef3ad8b",
   "metadata": {},
   "source": [
    "# Trying to set up CB1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6f056a41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/om/user/arjunp\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "352ca7c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arjunp/.local/lib/python3.11/site-packages/dask_jobqueue/core.py:266: FutureWarning: job_extra has been renamed to job_extra_directives. You are still using it (even if only set to []; please also check config files). If you did not set job_extra_directives yet, job_extra will be respected for now, but it will be removed in a future release. If you already set job_extra_directives, job_extra is ignored and you can remove it.\n",
      "  warnings.warn(warn, FutureWarning)\n",
      "/home/arjunp/.local/lib/python3.11/site-packages/dask_jobqueue/core.py:266: FutureWarning: job_extra has been renamed to job_extra_directives. You are still using it (even if only set to []; please also check config files). If you did not set job_extra_directives yet, job_extra will be respected for now, but it will be removed in a future release. If you already set job_extra_directives, job_extra is ignored and you can remove it.\n",
      "  warnings.warn(warn, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "cluster4 = SLURMCluster(cores=4,\n",
    "                       processes=2,\n",
    "                       memory=\"16GB\",\n",
    "                       account=\"cpl\",\n",
    "                       walltime=\"01:00:00\",\n",
    "                       queue=\"normal\",\n",
    "                       job_script_prologue  =[\n",
    "                        'source /etc/profile.d/modules.sh' ,\n",
    "                        'module load openmind8/anaconda/3-2023.09-0',\n",
    "                        'module load openmind/gcc/11.1.0',\n",
    "                        'module load openmind/ffmpeg/20160310' , \n",
    "                        'source ~/.bashrc',\n",
    "                        'export MKL_THREADING_LAYER=GNU',\n",
    "                        'conda activate torch_gpu'\n",
    "                                              ],\n",
    "                       job_extra=['--gres=gpu:QUADRORTX6000:1'] \n",
    "                       )\n",
    "cluster4.scale(1)\n",
    "#print(cluster2.job_script())\n",
    "#client3 = Client(cluster3)\n",
    "client4 = Client(cluster4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "7562b710",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)\r\n",
      "          37435051    normal dask-wor   arjunp  R       0:03      1 node078\r\n",
      "          37435035    normal dask-wor   arjunp  R      12:03      1 node092\r\n",
      "          37434728    normal     bash   arjunp  R    1:11:38      1 node080\r\n"
     ]
    }
   ],
   "source": [
    "!squeue -u arjunp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ea895864",
   "metadata": {},
   "outputs": [],
   "source": [
    "!scancel 37434780"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "ee57c5a2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8d94699a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_cb1_script(**kwargs):\n",
    "     \n",
    "    command = ['python', 'yinruiqing_trial.py', 'first_min_030327.wav']  \n",
    "    \n",
    "    result = subprocess.run(command, capture_output=True, text=True)\n",
    "     \n",
    "    print(\"STDOUT:\", result.stdout)\n",
    "    print(\"STDERR:\", result.stderr)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "530e0b48",
   "metadata": {},
   "outputs": [],
   "source": [
    "future4 = client4.submit(run_cb1_script)  \n",
    "result4 = future4.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "a8f8c54c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running CB1: yinruiqing\n",
      "Model was trained with pyannote.audio 0.0.1, yours is 3.3.1. Bad things might happen unless you revert pyannote.audio to 0.x.\n",
      "Model was trained with torch 1.10.0+cu102, yours is 2.3.1+cu118. Bad things might happen unless you revert torch to 1.x.\n",
      "cuda\n",
      "0.00 2.00 SPEAKER_00  You have to press down harder though, honey.\n",
      "2.00 4.00 SPEAKER_00  Whoa!\n",
      "4.00 6.00 SPEAKER_00  Look!\n",
      "6.00 8.00 SPEAKER_00  Look at this! This is cute.\n",
      "8.00 10.00 SPEAKER_00  May I have a piece of paper?\n",
      "10.00 12.00 SPEAKER_00  This one, mom.\n",
      "12.00 14.00 SPEAKER_00  This one's for me?\n",
      "14.00 16.00 SPEAKER_00  This one's for me.\n",
      "16.00 18.00 SPEAKER_00  Alright, you have to press down like this. Watch.\n",
      "18.00 20.00 SPEAKER_00  To make the colors show.\n",
      "20.00 34.00 SPEAKER_00  A L E X Spouse X Spouse what?\n",
      "34.00 42.00 SPEAKER_00  X He's a snow Wolf Hey, mom.\n",
      "42.00 44.00 SPEAKER_00  I see you.\n",
      "44.00 46.00 SPEAKER_00  Can I play with this?\n",
      "46.00 48.00 SPEAKER_00  Which one?\n",
      "48.00 50.00 SPEAKER_00  I see you.\n",
      "50.00 52.00 SPEAKER_00  Mommy, where is this?\n",
      "52.00 54.00 SPEAKER_00  Na na na na na.\n",
      "54.00 56.00 SPEAKER_00  How about this one?\n",
      "56.00 58.00 SPEAKER_00  This one's red.\n",
      "58.00 60.00 SPEAKER_00  I wanna draw a yellow.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(result3.stdout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "398f59fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running CB1: yinruiqing\n",
      "Model was trained with pyannote.audio 0.0.1, yours is 3.3.1. Bad things might happen unless you revert pyannote.audio to 0.x.\n",
      "Model was trained with torch 1.10.0+cu102, yours is 2.3.1+cu118. Bad things might happen unless you revert torch to 1.x.\n",
      "cuda\n",
      "0.00 2.00 SPEAKER_00  You have to press down harder though honey.\n",
      "2.00 4.00 SPEAKER_00  Oh my...\n",
      "4.00 6.00 SPEAKER_00  Look!\n",
      "6.00 8.00 SPEAKER_00  Look at this! This is cute.\n",
      "8.00 10.00 SPEAKER_00  May I have a piece of paper?\n",
      "10.00 12.00 SPEAKER_00  Oh this one mom. This one's for me?\n",
      "12.00 14.00 SPEAKER_00  This one's for me. Alright, may I have a color paper?\n",
      "14.00 16.00 SPEAKER_00  Oh this one! Alright, you have to press down like this.\n",
      "16.00 18.00 SPEAKER_00  Watch.\n",
      "18.00 20.00 SPEAKER_00  To make the colors show.\n",
      "20.00 34.00 SPEAKER_00  A L E X Spouse X Spouse what?\n",
      "34.00 42.00 SPEAKER_00  X He's a snow Wolf Hey mom!\n",
      "42.00 44.00 SPEAKER_00  I see you.\n",
      "44.00 46.00 SPEAKER_00  Can I play with this?\n",
      "46.00 48.00 SPEAKER_00  Which one?\n",
      "48.00 50.00 SPEAKER_00  I see you.\n",
      "50.00 52.00 SPEAKER_00  Mommy!\n",
      "52.00 54.00 SPEAKER_00  Na na na na na.\n",
      "54.00 56.00 SPEAKER_00  How about this one?\n",
      "56.00 58.00 SPEAKER_00  This one's red.\n",
      "58.00 60.00 SPEAKER_00  I wanna draw a yellow.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(result4.stdout)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfedd2f2",
   "metadata": {},
   "source": [
    "## Trying CB2 with the same env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1783fb5a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arjunp/.local/lib/python3.11/site-packages/dask_jobqueue/core.py:266: FutureWarning: job_extra has been renamed to job_extra_directives. You are still using it (even if only set to []; please also check config files). If you did not set job_extra_directives yet, job_extra will be respected for now, but it will be removed in a future release. If you already set job_extra_directives, job_extra is ignored and you can remove it.\n",
      "  warnings.warn(warn, FutureWarning)\n",
      "/cm/shared/openmind8/anaconda/3-2023.09-0/lib/python3.11/site-packages/distributed/node.py:182: UserWarning: Port 8787 is already in use.\n",
      "Perhaps you already have a cluster running?\n",
      "Hosting the HTTP server on port 42847 instead\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "cluster4 = SLURMCluster(cores=4,\n",
    "                       processes=2,\n",
    "                       memory=\"16GB\",\n",
    "                       account=\"cpl\",\n",
    "                       walltime=\"01:00:00\",\n",
    "                       queue=\"normal\",\n",
    "                       job_script_prologue  =[\n",
    "                        'source /etc/profile.d/modules.sh' ,\n",
    "                        'module load openmind8/anaconda/3-2023.09-0',\n",
    "                        'module load openmind/gcc/11.1.0',\n",
    "                        'module load openmind/ffmpeg/20160310' , \n",
    "                        'source ~/.bashrc',\n",
    "                        'export MKL_THREADING_LAYER=GNU',\n",
    "                        'conda activate torch_gpu'\n",
    "                                              ],\n",
    "                       job_extra=['--gres=gpu:QUADRORTX6000:1'] \n",
    "                       )\n",
    "cluster4.scale(1)\n",
    "#print(cluster2.job_script())\n",
    "#client3 = Client(cluster3)\n",
    "client4 = Client(cluster4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b5abe1c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_cb2_script(**kwargs):\n",
    "     \n",
    "    command = ['python', 'speechbox_trial.py', 'first_min_030327.wav']  \n",
    "    \n",
    "    result = subprocess.run(command, capture_output=True, text=True)\n",
    "     \n",
    "    print(\"STDOUT:\", result.stdout)\n",
    "    print(\"STDERR:\", result.stderr)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "bd44f310",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)\r\n",
      "          37441287    normal dask-wor   arjunp  R       0:09      1 node088\r\n",
      "          37441265    normal     bash   arjunp  R      25:40      1 node072\r\n",
      "          37441264    normal     bash   arjunp  R      27:51      1 node097\r\n"
     ]
    }
   ],
   "source": [
    "!squeue -u arjunp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "dc782402",
   "metadata": {},
   "outputs": [],
   "source": [
    "future5 = client4.submit(run_cb2_script)  \n",
    "result5 = future5.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d763bc24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model was trained with pyannote.audio 0.0.1, yours is 3.3.1. Bad things might happen unless you revert pyannote.audio to 0.x.\n",
      "Model was trained with torch 1.10.0+cu102, yours is 2.3.1+cu118. Bad things might happen unless you revert torch to 1.x.\n",
      "Running CB2: speechbox\n",
      "[{'speaker': 'SPEAKER_00', 'text': ' You have to press down like this, watch. To make the colors show. A L E E X Spouse A Ex-spouse. Ex-spouse. Ex-spouse. Yes. Ex-spouse. Yes. Ex-spouse. Ex-spouse. Ex-spouse. Ex-spouse. Ex-spouse. Ex-spouse. Ex-spouse. Ex-spouse. Ex-spouse. Ex-spouse. Ex-spouse. Ex-spouse. Ex-spouse. Ex-spouse. Hey, Mom, first you eat, I see you. For me, for me, what is this? Which one?', 'timestamp': (0.0, 70.0)}, {'speaker': 'SPEAKER_01', 'text': ' I see you, I see you, I see you, I see you.', 'timestamp': (70.0, 74.0)}, {'speaker': 'SPEAKER_00', 'text': ' Mommy, what is this?', 'timestamp': (74.0, 77.0)}, {'speaker': 'SPEAKER_01', 'text': ' M-m-m-m-m-m.', 'timestamp': (79.0, 81.0)}, {'speaker': 'SPEAKER_00', 'text': \" How about this one? This one, a red. I don't want to draw a yellow.\", 'timestamp': (81.0, 89.0)}]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(result5.stdout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "14c95857",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)\r\n",
      "          37441271    normal dask-wor   arjunp  R      18:57      1 node088\r\n",
      "          37441265    normal     bash   arjunp  R      25:02      1 node072\r\n",
      "          37441264    normal     bash   arjunp  R      27:13      1 node097\r\n"
     ]
    }
   ],
   "source": [
    "!squeue -u arjunp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7328ca02",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a87a26c5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}