Merge pull request #81 from ContextLab/paper-rev

major update to paper, some bug fixes, new readme into, updates for new colab version

Author: paxtonfitzpatrick

.github/workflows/ci-tests-colab.yml

@@ -28,7 +28,7 @@ defaults:
 
 jobs:
   run-tests:
-    name: "Run Colab CI Tests (Python 3.7, IPython 5.5.0)"
+    name: "Run Colab CI Tests (Python 3.7, IPython 7.9.0)"
     runs-on: ubuntu-latest
     # run if triggered by any of the following:
     #   - a workflow_dispatch event (manual or from trigger-colab-tests-pr workflow)
@@ -46,7 +46,7 @@ jobs:
       GMAIL_ADDRESS: ${{ secrets.DAVOS_GMAIL_ADDRESS }}
       GMAIL_PASSWORD: ${{ secrets.DAVOS_GMAIL_PASSWORD }}
       HEAD_FORK: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.fork || github.repository_owner }}
-      IPYTHON_VERSION: 5.5.0
+      IPYTHON_VERSION: 7.9.0
       NOTEBOOK_TYPE: colab
       PYTHON_VERSION: 3.7
       RECOVERY_GMAIL_ADDRESS: ${{ secrets.DAVOS_RECOVERY_GMAIL_ADDRESS }}

.github/workflows/ci-tests-jupyter.yml

@@ -78,7 +78,7 @@ jobs:
           sudo apt-get install firefox
 
           # install python packages
-          pip install pytest==6.2 "selenium>=3.141" geckodriver-autoinstaller
+          pip install pytest==6.2 "selenium==3.141" geckodriver-autoinstaller
 
           # install geckodriver
           driver_path=$(python -c '
@@ -112,7 +112,7 @@ jobs:
               pip install typing-extensions
           fi
           [[ "$PYTHON_VERSION" == "3.6" ]] || pip install numpy==1.20.3	
-          pip install "ipykernel==5.0.0" ipython-genutils requests scipy fastdtw==0.3.4 tqdm==4.41.1
+          pip install "ipykernel==5.0.0" ipython-genutils requests "scipy<=1.7.3" fastdtw==0.3.4 tqdm==4.41.1
           if [[ "$IPYTHON_VERSION" == "latest" ]]; then
               pip install --upgrade IPython
           else

.gitignore

@@ -1,20 +1,20 @@
 # Mac Finder display config
 *.DS_Store
 
-# PyCharm project config dir
-*.idea
+# PyCharm & VS Code project config dirs
+*.idea/
+*.vscode/
 
 # Python bytecode
-*__pycache__
+*__pycache__/
 
 # IPython notebook checkpoints
-*.ipynb_checkpoints
+*.ipynb_checkpoints/
 
-# Python package distribution egg
-davos.egg-info
-
-# build dir for non-editable installs
-build
+# Package distribution dirs for wheels & eggs
+dist/
+build/
+davos.egg-info/
 
 # Mypy daemon status file
 .dmypy.json
@@ -24,7 +24,18 @@ tests/geckodriver
 **/geckodriver.log
 
 # scripts & notebooks for scratch/debugging
-scratch
-
-# Illustrator source for snippets
-paper/snippets/snippets.ai
+scratch/
+
+# LaTeX generated files
+paper/**/*.aux
+paper/**/*.bbl
+paper/**/*.blg
+paper/**/*.log
+paper/**/*.out
+paper/**/*.spl
+paper/**/*.synctex.gz
+
+# Illustrator source files
+paper/figs/*.ai
+paper/main.fdb_latexmk
+paper/main.fls

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "paper/CDL-bibliography"]
+	path = paper/CDL-bibliography
+	url = https://github.com/ContextLab/CDL-bibliography.git

README.md

@@ -153,13 +153,13 @@ def __init__(self):
             self._environment = 'Python'
         else:
             import IPython
-            if IPython.version_info[0] < 7:
+            if IPython.version_info[0] >= 7:
                 if 'google.colab' in str(self._ipython_shell):
                     self._environment = 'Colaboratory'
                 else:
-                    self._environment = 'IPython<7.0'
+                    self._environment = 'IPython>=7.0'
             else:
-                self._environment = 'IPython>=7.0'
+                self._environment = 'IPython<7.0'
         self._conda_avail = None
         self._conda_envs_dirs = None
         self._ipy_showsyntaxerror_orig = None
@@ -215,11 +215,12 @@ def __init__(self):
                     self._pip_executable = f'{sys.executable} -m pip'
             else:
                 self._pip_executable = pip_exe
+        self._pip_executable_orig = self._pip_executable
 
     def __repr__(self):
         cls_name = self.__class__.__name__
         base_indent = len(cls_name) + 1
-        attrs_in_repr = ['active', 'auto_rerun', 'conda_avail']
+        attrs_in_repr = ['active', 'auto_rerun']
         if self._conda_avail is not None:
             attrs_in_repr.append('conda_avail')
             if self._conda_avail is True:

davos/core/config.py

@@ -17,7 +17,7 @@
 import importlib
 import itertools
 import sys
-from contextlib import redirect_stdout
+from contextlib import contextmanager, redirect_stdout
 from io import StringIO
 from pathlib import Path
 from subprocess import CalledProcessError
@@ -247,6 +247,69 @@ def get_previously_imported_pkgs(install_cmd_stdout, installer):
     return prev_imported_pkgs
 
 
+@contextmanager
+def handle_alternate_pip_executable(installed_name):
+    """
+    Context manager that makes it possible to load packages installed
+    into a different Python environment by changing the `pip` executable
+    (`davos.config.pip_executable`).
+
+    Parameters
+    ----------
+    installed_name : str
+        Package name as passed to the `pip install` command. This is the
+        value of self.install_name for the Onion object for the
+        just-installed package.
+
+    Notes
+    -----
+    super-duper edge case not handled by this: user has used alternate
+    pip_executable to smuggle local package from CWD and supplies
+    relative path in onion comment (i.e., "# pip: . <args>").
+    """
+    if '/' in installed_name:
+        # handle local paths, local/remote VCS, PEP 440 direct ref, etc.
+        dist_name = installed_name.split('@')[0].split('#')[0].split('/')[-1]
+    else:
+        # common case
+        dist_name = installed_name
+
+    # get install location from `pip show ___` command.
+    # In most cases, this will be davos.config.pip_executable with
+    # trailing 'bin/pip' replaced with python<major.minor>/site-packages
+    # but since this runs only if non-default pip_executable is set,
+    # it's worth the extra run time to check the safer way in order to
+    # handle various edge cases (e.g., installing from VCS, etc.)
+    pip_show_cmd = f'{config._pip_executable} show {dist_name}'
+    try:
+        pip_show_stdout = run_shell_command(pip_show_cmd, live_stdout=False)
+        location_line = next(
+            l for l in pip_show_stdout.strip().splitlines()
+            if l.startswith('Location')
+        )
+    except (CalledProcessError, StopIteration) as e:
+        msg = (
+            "Unable to locate package installed installed with non-default "
+            f"'pip' executable: {dist_name}. Package has been successfully "
+            "installed but not loaded."
+        )
+        raise SmugglerError(msg) from e
+
+    install_location = location_line.split(': ', maxsplit=1)[1].strip()
+    sys.path.insert(0, install_location)
+    try:
+        # reload pkg_resources so import system will search dir
+        # containing just-installed package
+        importlib.reload(sys.modules['pkg_resources'])
+        yield
+    finally:
+        # remove temporary dir from path
+        sys.path.pop(0)
+        # reload pkg_resources again so import system no longer searches
+        # dir used by alternate pip_executable
+        importlib.reload(sys.modules['pkg_resources'])
+
+
 def import_name(name):
     """
     Import an object by its qualified name.
@@ -458,6 +521,8 @@ def install_cmd(self):
             args = self.args_str.replace("<", "'<'").replace(">", "'>'")
         if self.installer == 'pip':
             install_exe = config._pip_executable
+            if config.noninteractive:
+                args = f'{args} --no-input'
         else:
             install_exe = self.installer
 
@@ -945,7 +1010,11 @@ def smuggle(
             else:
                 prompt_restart_rerun_buttons(failed_reloads)
 
-        smuggled_obj = import_name(name)
+        if config._pip_executable != config._pip_executable_orig:
+            with handle_alternate_pip_executable(onion.install_name):
+                smuggled_obj = import_name(name)
+        else:
+            smuggled_obj = import_name(name)
 
     # add the object name/alias to the notebook's global namespace
     if as_ is None:

davos/core/core.py

@@ -25,6 +25,7 @@ from typing import (
 __all__: list[
     Literal[
         'capture_stdout',
+        'check_conda',
         'Onion',
         'parse_line',
         'prompt_input',

davos/core/core.pyi

@@ -70,8 +70,8 @@
 
     _set_custom_showsyntaxerror()
 
-    if import_environment == 'IPython>=7.0':
-        from davos.implementations.ipython_post7 import (
+    if import_environment == 'IPython<7.0':
+        from davos.implementations.ipython_pre7 import (
             _activate_helper,
             _deactivate_helper,
             generate_parser_func
@@ -81,7 +81,7 @@
             prompt_restart_rerun_buttons
         )
     else:
-        from davos.implementations.ipython_pre7 import (
+        from davos.implementations.ipython_post7 import (
             _activate_helper,
             _deactivate_helper,
             generate_parser_func

davos/implementations/__init__.py

@@ -221,8 +221,8 @@ def _showsyntaxerror_davos(
     The `running_compiled_code` argument was added in `IPython` 6.1.0,
     and setting it to `True` accomplishes (something close to) the same
     thing this workaround does. However, since `davos` needs to support
-    `IPython` versions back to v5.5.0 (which is used by Colab), we can't
-    rely on it being available.
+    `IPython` versions back to v5.5.0, we can't rely on it being
+    available.
     """
     etype, value, tb = ipy_shell._get_exc_info()
     if issubclass(etype, DavosParserError):

davos/implementations/ipython_common.py

@@ -48,18 +48,18 @@ def auto_restart_rerun(pkgs):
     1. The message displayed before restarting the kernel can be
        silenced by setting `davos.config.suppress_stdout` to `True`.
     2. After calling `JS_FUNCTIONS.jupyter.restartRunCellsAbove`, this
-       function sleeps until until the kernel restarts to prevent any
-       further code in the current cell or other queued cells from
-       executing. Restarting the kernel is often not instantaneous;
-       there's generally a 1-2s delay while the kernel sends & receives
-       various shutdown messages, but it can take significantly
-       longer on a slow machine, with older Python/Jupyter/ipykernel
-       versions, if a large amount of data was loaded into memory, if
-       multiple notebook kernels are running at once, etc. If this
-       function returned immediately, it's likely subsequent lines of
-       code would be run before the kernel disconnected. This can cause
-       problems if those lines of code use the package(s) that prompted
-       the restart, or have effects that persist across kernel sessions.
+       function sleeps until the kernel restarts to prevent any further
+       code in the current cell or other queued cells from executing.
+       Restarting the kernel is often not instantaneous; there's
+       generally a 1-2s delay while the kernel sends & receives various
+       shutdown messages, but it can take significantly longer on a slow
+       machine, with older Python/Jupyter/ipykernel versions, if a large
+       amount of data was loaded into memory, if multiple notebook
+       kernels are running at once, etc. If this function returned
+       immediately, it's likely subsequent lines of code would be run
+       before the kernel disconnected. This can cause problems if those
+       lines of code use the package(s) that prompted the restart, or
+       have effects that persist across kernel sessions.
     """
     msg = (
         "Restarting kernel and rerunning cells (required to smuggle "