Remediate AWS integration CI (closes #30)

.github/workflows/aws-integration-tests.yml

@@ -31,6 +31,12 @@ jobs:
     # Only run if AWS credentials are configured
     if: github.event_name == 'workflow_dispatch' || github.repository == 'scttfrdmn/starburst'
 
+    # Effective test suite: the dispatch input when triggered manually, or
+    # 'quick' on the monthly schedule (where inputs are empty) so the cron run
+    # performs a real, near-zero-cost drift check instead of a no-op.
+    env:
+      SUITE: ${{ github.event.inputs.test_suite || 'quick' }}
+
     permissions:
       id-token: write  # For OIDC
       contents: read
@@ -85,11 +91,13 @@ jobs:
         run: |
           Rscript -e "
             devtools::load_all()
-            starburst_setup(region = 'us-east-1', force = TRUE)
+            # build_image = FALSE: provision resources + write config without the
+            # multi-minute image build. Suites that launch workers build it lazily.
+            starburst_setup(region = 'us-east-1', force = TRUE, build_image = FALSE)
           "
 
       - name: Run quick smoke tests
-        if: github.event.inputs.test_suite == 'quick' || github.event.inputs.test_suite == 'all'
+        if: env.SUITE == 'quick' || env.SUITE == 'all'
         run: |
           Rscript -e "
             devtools::load_all()
@@ -114,7 +122,7 @@ jobs:
           "
 
       - name: Run detached session tests
-        if: github.event.inputs.test_suite == 'detached-sessions' || github.event.inputs.test_suite == 'all'
+        if: env.SUITE == 'detached-sessions' || env.SUITE == 'all'
         env:
           RUN_INTEGRATION_TESTS: "TRUE"  # opt-in gate for tests that launch real Fargate workers
         run: |
@@ -130,7 +138,7 @@ jobs:
           "
 
       - name: Run integration example tests
-        if: github.event.inputs.test_suite == 'integration-examples' || github.event.inputs.test_suite == 'all'
+        if: env.SUITE == 'integration-examples' || env.SUITE == 'all'
         env:
           RUN_INTEGRATION_TESTS: "TRUE"
         run: |
@@ -146,7 +154,7 @@ jobs:
           "
 
       - name: Run EC2 integration tests
-        if: github.event.inputs.test_suite == 'ec2' || github.event.inputs.test_suite == 'all'
+        if: env.SUITE == 'ec2' || env.SUITE == 'all'
         run: |
           Rscript -e "
             devtools::load_all()
@@ -161,7 +169,7 @@ jobs:
           "
 
       - name: Run cleanup tests
-        if: github.event.inputs.test_suite == 'cleanup' || github.event.inputs.test_suite == 'all'
+        if: env.SUITE == 'cleanup' || env.SUITE == 'all'
         run: |
           Rscript -e "
             devtools::load_all()

R/images.R

@@ -230,9 +230,6 @@ get_base_image_uri <- function(region) {
           account_id, region, base_tag)
 }
 
-#' Build base Docker image with common dependencies
-#'
-#' @keywords internal
 #' Ensure a buildx builder with the docker-container driver exists and is usable
 #'
 #' Idempotent across repeated runs and cross-platform (Windows/macOS/Linux).
@@ -252,11 +249,16 @@ get_base_image_uri <- function(region) {
 #' @return TRUE if the named builder is usable, FALSE otherwise
 #' @keywords internal
 ensure_buildx_builder <- function(builder_name = "starburst-builder") {
+  # stdout/stderr are captured (not discarded) so that when a buildx call fails
+  # safe_system()'s stop() carries the real stderr, instead of the failure being
+  # swallowed here and surfacing later as an opaque "no builder found" at the
+  # buildx build step (GitHub #24/#30).
+
   # 1. Does it already exist? inspect returns non-zero (-> error) if not;
   #    --bootstrap also boots an existing-but-stopped builder.
   exists <- tryCatch({
     safe_system("docker", c("buildx", "inspect", "--bootstrap", builder_name),
-                stdout = FALSE, stderr = FALSE)
+                stdout = TRUE, stderr = TRUE)
     TRUE
   }, error = function(e) FALSE)
 
@@ -270,7 +272,7 @@ ensure_buildx_builder <- function(builder_name = "starburst-builder") {
       "docker",
       c("buildx", "create", "--name", builder_name,
         "--driver", "docker-container", "--bootstrap"),
-      stdout = FALSE, stderr = FALSE
+      stdout = TRUE, stderr = TRUE
     )
     TRUE
   }, error = function(e) {
@@ -283,14 +285,25 @@ ensure_buildx_builder <- function(builder_name = "starburst-builder") {
     return(FALSE)
   }
 
-  # 3. Confirm it is now usable (defensive; create + bootstrap should suffice).
+  # 3. Confirm it booted and is selectable. 'inspect' verifies it is usable;
+  #    'use' registers it as current so the explicit --builder reference in
+  #    buildx build resolves (guards against the create not persisting).
   tryCatch({
     safe_system("docker", c("buildx", "inspect", "--bootstrap", builder_name),
-                stdout = FALSE, stderr = FALSE)
+                stdout = TRUE, stderr = TRUE)
+    safe_system("docker", c("buildx", "use", builder_name),
+                stdout = TRUE, stderr = TRUE)
     TRUE
-  }, error = function(e) FALSE)
+  }, error = function(e) {
+    cat_warn(sprintf("Warning: buildx builder '%s' created but not usable: %s\n",
+                     builder_name, conditionMessage(e)))
+    FALSE
+  })
 }
 
+#' Build base Docker image with common dependencies
+#'
+#' @keywords internal
 build_base_image <- function(region) {
   cat_info("[Docker] Building staRburst base image...\n")
 

R/setup.R

@@ -11,6 +11,11 @@
 #'   This prevents surprise costs if you stop using staRburst.
 #'   Recommended: 30 days for regular users, 7 days for occasional users.
 #'   When images are deleted, they will be rebuilt on next use (adds 3-5 min).
+#' @param build_image Build the worker environment image during setup (default: TRUE).
+#'   Set to FALSE to provision AWS resources (S3/ECR/ECS/VPC), write config, and
+#'   check quotas without triggering the multi-minute Docker image build. The
+#'   image is then built lazily on first worker launch via
+#'   \code{ensure_environment()}. Useful for CI / connectivity checks.
 #'
 #' @return Invisibly returns the configuration list.
 #' @export
@@ -26,9 +31,13 @@
 #'
 #'   # Use private base images with 7-day cleanup
 #'   starburst_setup(use_public_base = FALSE, ecr_image_ttl_days = 7)
+#'
+#'   # Provision resources without building the image (fast; CI / connectivity checks)
+#'   starburst_setup(build_image = FALSE)
 #' }
 #' }
-starburst_setup <- function(region = "us-east-1", force = FALSE, use_public_base = TRUE, ecr_image_ttl_days = NULL) {
+starburst_setup <- function(region = "us-east-1", force = FALSE, use_public_base = TRUE,
+                            ecr_image_ttl_days = NULL, build_image = TRUE) {
 
   cat_header("[Start] staRburst Setup\n")
 
@@ -152,12 +161,17 @@ starburst_setup <- function(region = "us-east-1", force = FALSE, use_public_base
     cat_success(sprintf("[OK] Quota is sufficient (%d vCPUs)\n", quota_info$limit))
   }
 
-  # Build initial environment
-  cat_info("\n[Build] Building initial R environment...\n")
-  cat_info("This may take 5-10 minutes on first run\n")
+  # Build initial environment (skippable: image is built lazily on first launch)
+  if (build_image) {
+    cat_info("\n[Build] Building initial R environment...\n")
+    cat_info("This may take 5-10 minutes on first run\n")
 
-  env_hash <- build_initial_environment(region)
-  cat_success("[OK] Environment built and cached\n")
+    env_hash <- build_initial_environment(region)
+    cat_success("[OK] Environment built and cached\n")
+  } else {
+    cat_info("\n[Build] Skipping environment image build (build_image = FALSE)\n")
+    cat_info("  The worker image will be built automatically on first launch.\n")
+  }
 
   # Final message
   cat_success("\n[OK] staRburst setup complete!\n")

tests/testthat/test-docker.R

@@ -186,6 +186,12 @@ test_that("ensure_buildx_builder creates builder when missing", {
   expect_true(all(c("--driver", "docker-container", "--bootstrap")
                   %in% create_call[[1]]))
   expect_true(all(c("--name", "starburst-builder") %in% create_call[[1]]))
+
+  # After create, the builder must be explicitly selected so a later
+  # `buildx build --builder starburst-builder` resolves (GitHub #30).
+  use_call <- Filter(function(a) "use" %in% a, calls)
+  expect_length(use_call, 1)
+  expect_true("starburst-builder" %in% use_call[[1]])
 })
 
 test_that("ensure_buildx_builder returns FALSE when create fails", {
@@ -238,3 +244,37 @@ test_that("build_environment_image aborts when builder is unusable", {
     "starburst-builder"
   )
 })
+
+# Helper: stub starburst_setup's collaborators so it runs offline, returning
+# whether build_initial_environment was invoked.
+run_setup_capturing_build <- function(build_image) {
+  built <- FALSE
+  mockery::stub(starburst_setup, "is_setup_complete", function() FALSE)
+  mockery::stub(starburst_setup, "check_aws_credentials", function() TRUE)
+  mockery::stub(starburst_setup, "get_aws_account_id", function() "123456789012")
+  mockery::stub(starburst_setup, "create_starburst_bucket", function(...) "starburst-test")
+  mockery::stub(starburst_setup, "create_ecr_repository", function(...) list(repositoryUri = "uri"))
+  mockery::stub(starburst_setup, "create_ecs_cluster", function(...) list(clusterName = "starburst-cluster"))
+  mockery::stub(starburst_setup, "setup_vpc_resources",
+    function(...) list(vpc_id = "vpc-1", subnets = list(), security_groups = list()))
+  mockery::stub(starburst_setup, "save_config", function(...) invisible())
+  mockery::stub(starburst_setup, "check_fargate_quota",
+    function(...) list(limit = 1000, used = 0, available = 1000, increase_pending = FALSE))
+  mockery::stub(starburst_setup, "build_initial_environment", function(...) built <<- TRUE)
+  starburst_setup(force = TRUE, build_image = build_image)
+  built
+}
+
+test_that("starburst_setup skips image build when build_image = FALSE", {
+  skip_on_cran()
+  skip_if_not_installed("mockery")
+
+  expect_false(run_setup_capturing_build(build_image = FALSE))
+})
+
+test_that("starburst_setup builds image when build_image = TRUE (default)", {
+  skip_on_cran()
+  skip_if_not_installed("mockery")
+
+  expect_true(run_setup_capturing_build(build_image = TRUE))
+})