Run tests in CI in two batches: first, tests that don't require {pg,mongo,mysql}origin databases to be up (whilst they're starting up), then the remainder of the tests.

Author: mildbyte

.travis.yml

@@ -6,7 +6,13 @@ python:
 install:
   - poetry install -E ingestion --develop splitgraph
 script:
-  - poetry run pytest -v
+  # The majority of the test suite doesn't depend on {pg,mongo,mysql}origin being up and they
+  # take a lot of time to start up (especially the first time). So, whilst the 3 DBs are busy starting up,
+  # we run the core tests against the local/remote engines and then run the rest of the tests (about 10% of them)
+  # with the databases up, merging coverage.
+  - poetry run pytest -v -m "not mounting"
+  - ./wait-for-test-architecture.sh --mounting
+  - poetry run pytest -v -m "mounting" --cov-append
 
 services:
   - docker

test/architecture/wait-for-architecture.sh

@@ -5,9 +5,16 @@ TEST_DIR="${THIS_DIR}/.."
 REPO_ROOT_DIR="${TEST_DIR}/.."
 DEFAULT_SG_CONFIG_FILE="${TEST_DIR}/resources/.sgconfig"
 
+if [[ "$1" == "--mounting" ]]; then
+    echo "Wait for mounting architecture to be up"
+    HEALTHCHECK="import test.splitgraph.conftest as c; c.healthcheck_mounting()"
+else
+    echo "Wait for core architecture to be up"
+    HEALTHCHECK="import test.splitgraph.conftest as c; c.healthcheck()"
+fi
+
 export SG_CONFIG_FILE=${SG_CONFIG_FILE-"${DEFAULT_SG_CONFIG_FILE}"}
 
-echo "Wait for architecture to be up"
 echo "Using config file at $SG_CONFIG_FILE ..."
 
 _init_engines() {
@@ -20,7 +27,7 @@ _init_engines() {
 
 _run_health_check() {
     pushd "$REPO_ROOT_DIR" \
-        && python -c "import test.splitgraph.conftest as c; c.healthcheck()" \
+        && python -c "$HEALTHCHECK" \
         && return 0
 
     return 1

test/splitgraph/commands/test_commit_diff.py

@@ -315,6 +315,7 @@ def test_commit_on_empty(snap_only, pg_repo_local):
     assert OUTPUT.diff('test', image_1=OUTPUT.head.image_hash, image_2=None) == []
 
 
+@pytest.mark.mounting
 @pytest.mark.parametrize("mode", _COMMIT_MODES)
 def test_multiple_mountpoint_commit_diff(mode, pg_repo_local, mg_repo_local):
     pg_repo_local.run_sql("""INSERT INTO fruits VALUES (3, 'mayonnaise');

test/splitgraph/commands/test_mounting.py

@@ -1,5 +1,7 @@
 from datetime import datetime as dt
 
+import pytest
+
 from splitgraph import get_engine
 from splitgraph.core.repository import Repository
 from test.splitgraph.conftest import _mount_postgres, _mount_mysql, _mount_mongo
@@ -9,19 +11,22 @@
 MYSQL_MNT = Repository.from_schema("test/mysql_mount")
 
 
+@pytest.mark.mounting
 def test_mount_unmount(local_engine_empty):
     _mount_postgres(PG_MNT)
     assert (1, 'apple') in get_engine().run_sql("""SELECT * FROM "test/pg_mount".fruits""")
     PG_MNT.delete()
     assert not get_engine().schema_exists(PG_MNT.to_schema())
 
 
+@pytest.mark.mounting
 def test_mount_partial(local_engine_empty):
     _mount_postgres(PG_MNT, tables=['fruits'])
     assert get_engine().table_exists(PG_MNT.to_schema(), 'fruits')
     assert not get_engine().table_exists(PG_MNT.to_schema(), 'vegetables')
 
 
+@pytest.mark.mounting
 def test_mount_mysql(local_engine_empty):
     try:
         _mount_mysql(MYSQL_MNT)
@@ -34,6 +39,7 @@ def test_mount_mysql(local_engine_empty):
         MYSQL_MNT.delete()
 
 
+@pytest.mark.mounting
 def test_cross_joins(local_engine_empty):
     _mount_postgres(PG_MNT)
     _mount_mongo(MG_MNT)

test/splitgraph/commands/test_provenance.py

@@ -1,3 +1,5 @@
+import pytest
+
 from splitgraph.splitfile import execute_commands
 from splitgraph.splitfile.execution import rebuild_image
 from test.splitgraph.conftest import OUTPUT, load_splitfile
@@ -40,6 +42,7 @@ def test_splitfile_recreate_custom_from(local_engine_empty, pg_repo_remote_multi
                                   "JOIN vegetables                                ON fruit_id = vegetable_id"]
 
 
+@pytest.mark.mounting
 def test_splitfile_incomplete_provenance(local_engine_empty, pg_repo_remote_multitag):
     # This splitfile has a MOUNT as its first command. We're expecting image_hash_to_splitfile to base the result
     # on the image created by MOUNT and not regenerate the MOUNT command.

test/splitgraph/conftest.py

@@ -57,10 +57,16 @@ def _mount_mysql(repository):
 
 
 def healthcheck():
-    # A pre-flight check before we run the tests to make sure the test architecture has been brought up:
-    # the local_engine and the two origins (tested by mounting). There's still an implicit race condition
-    # here since we don't touch the remote_engine but we don't run any tests against it until later on,
-    # so it should have enough time to start up.
+    # A pre-flight check for most tests: check that the local and the remote engine fixtures are up and
+    # running.
+    get_current_repositories(get_engine())
+    get_current_repositories(get_engine('remote_engine'))
+
+
+def healthcheck_mounting():
+    # A pre-flight check for heavier tests that also ensures the three origin databases that we mount for FDW tests
+    # are up. Tests that require one of these databases to be up are marked with @pytest.mark.mounting and can be
+    # excluded with `poetry run pytest -m "not mounting"`.
     for mountpoint in [PG_MNT, MG_MNT, MYSQL_MNT]:
         mountpoint.delete()
     _mount_postgres(PG_MNT)

test/splitgraph/splitfile/test_execution.py

@@ -28,6 +28,7 @@ def test_splitfile_preprocessor_escaping():
     assert 'tag-v1-whatever' in commands
 
 
+@pytest.mark.mounting
 def test_basic_splitfile(pg_repo_local, mg_repo_local):
     execute_commands(load_splitfile('create_table.splitfile'), output=OUTPUT)
     log = list(reversed(OUTPUT.head.get_log()))
@@ -42,6 +43,7 @@ def test_basic_splitfile(pg_repo_local, mg_repo_local):
     assert OUTPUT.run_sql("SELECT * FROM my_fruits") == [(1, 'pineapple'), (2, 'banana')]
 
 
+@pytest.mark.mounting
 def test_update_without_import_splitfile(pg_repo_local, mg_repo_local):
     # Test that correct commits are produced by executing an splitfile (both against newly created and already
     # existing tables on an existing mountpoint)
@@ -55,6 +57,7 @@ def test_update_without_import_splitfile(pg_repo_local, mg_repo_local):
     assert OUTPUT.run_sql("SELECT * FROM my_fruits") == [(1, 'pineapple')]
 
 
+@pytest.mark.mounting
 def test_local_import_splitfile(pg_repo_local, mg_repo_local):
     execute_commands(load_splitfile('import_local.splitfile'), output=OUTPUT)
     head = OUTPUT.head
@@ -69,6 +72,7 @@ def test_local_import_splitfile(pg_repo_local, mg_repo_local):
     assert not OUTPUT.engine.table_exists(OUTPUT.to_schema(), 'fruits')
 
 
+@pytest.mark.mounting
 def test_advanced_splitfile(pg_repo_local, mg_repo_local):
     execute_commands(load_splitfile('import_local_multiple_with_queries.splitfile'), output=OUTPUT)
 
@@ -86,6 +90,7 @@ def test_advanced_splitfile(pg_repo_local, mg_repo_local):
     assert OUTPUT.run_sql("SELECT * FROM my_fruits") == [(2, 'orange')]
 
 
+@pytest.mark.mounting
 def test_splitfile_cached(pg_repo_local, mg_repo_local):
     # Check that no new commits/snaps are created if we rerun the same splitfile
     execute_commands(load_splitfile('import_local_multiple_with_queries.splitfile'), output=OUTPUT)
@@ -144,6 +149,7 @@ def test_import_updating_splitfile_with_uploading(local_engine_empty, remote_eng
     assert OUTPUT.run_sql("SELECT fruit_id, name FROM my_fruits") == [(1, 'apple'), (2, 'orange'), (3, 'mayonnaise')]
 
 
+@pytest.mark.mounting
 def test_splitfile_end_to_end_with_uploading(local_engine_empty, remote_engine,
                                              pg_repo_remote_multitag, mg_repo_remote):
     # An end-to-end test:
@@ -171,6 +177,7 @@ def test_splitfile_end_to_end_with_uploading(local_engine_empty, remote_engine,
     assert stage_2.run_sql("SELECT id, name, fruit, vegetable FROM diet") == [(2, 'James', 'orange', 'carrot')]
 
 
+@pytest.mark.mounting
 def test_splitfile_schema_changes(pg_repo_local, mg_repo_local):
     execute_commands(load_splitfile('schema_changes.splitfile'), output=OUTPUT)
     old_output_head = OUTPUT.head
@@ -189,6 +196,7 @@ def test_splitfile_schema_changes(pg_repo_local, mg_repo_local):
     assert OUTPUT.run_sql("SELECT * FROM spirit_fruits") == [('James', 'orange', 12), ('Alex', 'mayonnaise', 22)]
 
 
+@pytest.mark.mounting
 def test_import_with_custom_query(pg_repo_local, mg_repo_local):
     # Mostly a lazy way for the test to distinguish between the importer storing the results of a query as a snap
     # and pointing the commit history to the diff for unchanged objects.
@@ -224,6 +232,7 @@ def test_import_with_custom_query(pg_repo_local, mg_repo_local):
             assert OUTPUT.objects.get_object_meta(objects)[objects[0]].parent_id is not None
 
 
+@pytest.mark.mounting
 def test_import_mount(local_engine_empty):
     execute_commands(load_splitfile('import_from_mounted_db.splitfile'), output=OUTPUT)
 
@@ -246,6 +255,7 @@ def test_import_mount(local_engine_empty):
         assert OUTPUT.objects.get_object_meta(objects)[objects[0]].parent_id is None
 
 
+@pytest.mark.mounting
 def test_import_all(local_engine_empty):
     execute_commands(load_splitfile('import_all_from_mounted.splitfile'), output=OUTPUT)
 
@@ -331,6 +341,7 @@ def test_from_local(pg_repo_local):
     assert OUTPUT.run_sql("SELECT * FROM join_table") == [(1, 'apple', 'potato'), (2, 'orange', 'carrot')]
 
 
+@pytest.mark.mounting
 def test_splitfile_with_external_sql(pg_repo_local, pg_repo_remote, mg_repo_local):
     # Tests are running from root so we pass in the path to the SQL manually to the splitfile.
     execute_commands(load_splitfile('external_sql.splitfile'),

test/splitgraph/test_commandline.py

@@ -353,6 +353,7 @@ def test_commandline_tag_checkout(pg_repo_local):
     assert pg_repo_local.images.by_tag('v1', raise_on_none=False) is None
 
 
+@pytest.mark.mounting
 def test_misc_mountpoint_management(pg_repo_local, mg_repo_local):
     runner = CliRunner()
 
@@ -388,6 +389,7 @@ def test_misc_mountpoint_management(pg_repo_local, mg_repo_local):
     assert mg_repo_local.run_sql("SELECT duration from stuff WHERE name = 'James'") == [(Decimal(2),)]
 
 
+@pytest.mark.mounting
 def test_import(pg_repo_local, mg_repo_local):
     runner = CliRunner()
     head = pg_repo_local.head
@@ -508,6 +510,7 @@ def test_splitfile_rebuild_update(local_engine_empty, pg_repo_remote_multitag):
     assert OUTPUT.images() == curr_commits
 
 
+@pytest.mark.mounting
 def test_mount_and_import(local_engine_empty):
     runner = CliRunner()
     try:

test/splitgraph/test_object_hashing.py

@@ -2,6 +2,8 @@
 from functools import reduce
 from hashlib import sha256
 
+import pytest
+
 from splitgraph import SPLITGRAPH_META_SCHEMA, execute_commands, Repository
 from splitgraph.core.fragment_manager import Digest
 from test.splitgraph.conftest import OUTPUT, PG_DATA, load_splitfile
@@ -277,6 +279,7 @@ def test_diff_fragment_hashing_reused_twice(pg_repo_local):
            [(1, 'apple'), (2, 'mustard'), (3, 'kumquat')]
 
 
+@pytest.mark.mounting
 def test_import_splitfile_reuses_hash(local_engine_empty):
     # Create two repositories and run the same Splitfile that loads some data from a mounted database.
     # Check that the same contents result in the same hash and no extra objects being created

wait-for-test-architecture.sh

@@ -3,4 +3,4 @@
 REPO_ROOT_DIR=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
 TEST_DIR="${REPO_ROOT_DIR}/test"
 
-exec "$TEST_DIR"/architecture/wait-for-architecture.sh
+exec "$TEST_DIR"/architecture/wait-for-architecture.sh $1