Add support to splitgraph.spec for building multi-file executables

- Pass `--onedir` argument intead of `-F` when calling pyinstaller

Since spec file is just a Python file, add conditional logic
to use Python to create the appropriate `EXE()` and `COLLECT()`
TOCs in the spec file based on presence of `--onedir` flag.

Existing behavior is retained by default and will always build
a single-file executable, except when the `--onedir` flag is
contained in `sys.argv` of the `pyinstaller` command that is
processing the spec.

Author: milesrichardson

.github/workflows/build_and_test_and_release.yml

@@ -8,8 +8,8 @@ jobs:
     runs-on: ubuntu-18.04
     if: "!contains(github.event.head_commit.message, '[skip ci]')"
     env:
-      COMPOSE_VERSION: '1.25.4'
-      POETRY_VERSION: '1.1.6'
+      COMPOSE_VERSION: "1.25.4"
+      POETRY_VERSION: "1.1.6"
       DOCKER_REPO: splitgraph
       DOCKER_ENGINE_IMAGE: engine
       DOCKER_TAG: development
@@ -23,7 +23,7 @@ jobs:
       - name: Setup Python 3.8
         uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
+          python-version: "3.8"
       - uses: actions/cache@v1
         with:
           path: ~/.cache/pip
@@ -212,7 +212,16 @@ jobs:
         with:
           name: sgr-osx
           path: dist/sgr
-
+      - name: Build the multi-file binary
+        run: |
+          pyinstaller --clean --noconfirm --onedir splitgraph.osx.spec
+          dist/sgr-pkg/sgr --version
+          cd dist/sgr-pkg && tar zcvf ../sgr.tgz .
+      - name: Upload multi-file binary.gz as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: sgr-osx.tgz
+          path: dist/sgr.tgz
 
   upload_release:
     runs-on: ubuntu-18.04

install.sh

@@ -77,13 +77,11 @@ _install_binary () {
   _check_sgr_exists
 
   URL="https://github.com/splitgraph/splitgraph/releases/download/v${SGR_VERSION}"/$BINARY
-  echo "Installing the sgr binary from $URL into $INSTALL_DIR"
-  mkdir -p "$INSTALL_DIR"
-  curl -fsL "$URL" > "$INSTALL_DIR/sgr"
-  chmod +x "$INSTALL_DIR/sgr"
-  "$INSTALL_DIR/sgr" --version
-  echo "sgr binary installed."
-  echo
+  # on OS X, splitgraph.spec is called with --onedir to output .tgz of exe and shlibs
+  if [ "$BINARY" == "sgr-osx-x86_64.tgz" ] ; then
+    echo "Installing the compressed sgr binary and deps from $URL into $INSTALL_DIR"
+    echo "Installing sgr binary and deps into $INSTALL_DIR/pkg"
+  fi
 
 }
 

splitgraph.spec

@@ -3,10 +3,23 @@
 # * LD_LIBRARY_PATH=`echo $(python3-config --prefix)/lib` pyinstaller -F splitgraph.spec produces a single sgr binary in the dist/ folder
 #   with libc being the only dynamic dependency (python interpreter included)
 # * can also do poetry install && poetry run pyinstaller -F splitgraph.spec to build the binary inside of the poetry's venv.
+# * specifying `--onedir` instead of `-F` will compile a multi-file executable with COLLECT()
+# * e.g. : pyinstaller --clean --noconfirm --onedir splitgraph.spec
 
+import sys
 import os
 import importlib
 
+# Pass --onedir or -D to build a multi-file executable (dir with shlibs + exe)
+# Note: This is the same flag syntax as pyinstaller uses, but when using a
+#       .spec file with pyinstaller, the flag is normally ignored, so we
+#       explicitly check for it here. This way we can pass different arguments
+#       to EXE() and conditionally call COLLECTION() without duplicating code.
+MAKE_EXE_COLLECTION = False
+if "--onedir" in sys.argv or "-D" in sys.argv:
+    print("splitgraph.spec : --onedir was specified. Will build a multi-file executable...")
+    MAKE_EXE_COLLECTION = True
+
 block_cipher = None
 
 datas = []
@@ -43,18 +56,36 @@ a = Analysis(
 a.datas += Tree("./splitgraph/resources", "splitgraph/resources")
 
 pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
-exe = EXE(
-    pyz,
-    a.scripts,
-    a.binaries,
-    a.zipfiles,
-    a.datas,
-    [],
-    name="sgr",
-    debug=False,
-    bootloader_ignore_signals=False,
-    strip=False,
-    upx=True,
-    runtime_tmpdir=None,
-    console=True,
-)
+
+# Note: `exe` global is injected by pyinstaller. Can see the code for EXE at:
+# https://github.com/pyinstaller/pyinstaller/blob/15f23a8a89be5453b3520df8fc3e667346e103a6/PyInstaller/building/api.py
+
+# TOCS to include in EXE() when in single-file mode (default, i.e. no --onedir flag)
+all_tocs = [a.scripts, a.binaries, a.zipfiles, a.datas]
+# TOCS to include in EXE() when in multi-file mode (i.e., --onedir flag)
+exe_tocs = [a.scripts]
+# TOCS to include in COLL() when in multi-file mode (i.e., --onedir flag)
+coll_tocs = [a.binaries, a.zipfiles, a.datas]
+
+# When compiling single-file executable, we include every TOC in the EXE
+# When compiling multi-file executable, include some TOC in EXE, and rest in COLL
+exe_args = [pyz, *exe_tocs, []] if MAKE_EXE_COLLECTION else [pyz, *all_tocs, []]
+
+exe_kwargs_base = {
+    "name": "sgr",
+    "debug": False,
+    "bootloader_ignore_signals": False,
+    "strip_binaries": False,
+    "runtime_tmpdir": None,
+    "console": True,
+}
+# In multi-file mode, we exclude_binaries from EXE since they will be in COLL
+exe_kwargs_onedir = {**exe_kwargs_base, "upx": False, "exclude_binaries": True}
+# In single-file mode, we set upx: true because it works. (It might actually work in multi-file mode too)
+exe_kwargs_onefile = {**exe_kwargs_base, "upx": True, "exclude_binaries": False}
+exe_kwargs = exe_kwargs_onedir if MAKE_EXE_COLLECTION else exe_kwargs_onefile
+
+exe = EXE(*exe_args, **exe_kwargs)
+
+if MAKE_EXE_COLLECTION:
+    coll = COLLECT(exe, *coll_tocs, name="sgr-pkg", strip=False, upx=False)