From ac3f757b948a7c65f4a0d9214968ee0f4369f34e Mon Sep 17 00:00:00 2001 From: scttfrdmn <3011922+scttfrdmn@users.noreply.github.com> Date: Tue, 2 Jun 2026 17:41:27 -0700 Subject: [PATCH 1/4] feat: add build_image param to starburst_setup (refs #30) starburst_setup() always ended by building the multi-platform worker image (~4 min). Add build_image = TRUE (trailing, backward compatible); set FALSE to provision S3/ECR/ECS/VPC + write config + check quotas without the image build. The image is then built lazily on first worker launch via ensure_environment(). Enables a fast, cheap CI connectivity check. Refs #30 --- R/setup.R | 26 ++++++++++++++++++++------ man/starburst_setup.Rd | 12 +++++++++++- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/R/setup.R b/R/setup.R index 832be9d..5dbb0fb 100644 --- a/R/setup.R +++ b/R/setup.R @@ -11,6 +11,11 @@ #' This prevents surprise costs if you stop using staRburst. #' Recommended: 30 days for regular users, 7 days for occasional users. #' When images are deleted, they will be rebuilt on next use (adds 3-5 min). +#' @param build_image Build the worker environment image during setup (default: TRUE). +#' Set to FALSE to provision AWS resources (S3/ECR/ECS/VPC), write config, and +#' check quotas without triggering the multi-minute Docker image build. The +#' image is then built lazily on first worker launch via +#' \code{ensure_environment()}. Useful for CI / connectivity checks. #' #' @return Invisibly returns the configuration list. #' @export @@ -26,9 +31,13 @@ #' #' # Use private base images with 7-day cleanup #' starburst_setup(use_public_base = FALSE, ecr_image_ttl_days = 7) +#' +#' # Provision resources without building the image (fast; CI / connectivity checks) +#' starburst_setup(build_image = FALSE) #' } #' } -starburst_setup <- function(region = "us-east-1", force = FALSE, use_public_base = TRUE, ecr_image_ttl_days = NULL) { +starburst_setup <- function(region = "us-east-1", force = FALSE, use_public_base = TRUE, + ecr_image_ttl_days = NULL, build_image = TRUE) { cat_header("[Start] staRburst Setup\n") @@ -152,12 +161,17 @@ starburst_setup <- function(region = "us-east-1", force = FALSE, use_public_base cat_success(sprintf("[OK] Quota is sufficient (%d vCPUs)\n", quota_info$limit)) } - # Build initial environment - cat_info("\n[Build] Building initial R environment...\n") - cat_info("This may take 5-10 minutes on first run\n") + # Build initial environment (skippable: image is built lazily on first launch) + if (build_image) { + cat_info("\n[Build] Building initial R environment...\n") + cat_info("This may take 5-10 minutes on first run\n") - env_hash <- build_initial_environment(region) - cat_success("[OK] Environment built and cached\n") + env_hash <- build_initial_environment(region) + cat_success("[OK] Environment built and cached\n") + } else { + cat_info("\n[Build] Skipping environment image build (build_image = FALSE)\n") + cat_info(" The worker image will be built automatically on first launch.\n") + } # Final message cat_success("\n[OK] staRburst setup complete!\n") diff --git a/man/starburst_setup.Rd b/man/starburst_setup.Rd index 409c19c..3984bd5 100644 --- a/man/starburst_setup.Rd +++ b/man/starburst_setup.Rd @@ -8,7 +8,8 @@ starburst_setup( region = "us-east-1", force = FALSE, use_public_base = TRUE, - ecr_image_ttl_days = NULL + ecr_image_ttl_days = NULL, + build_image = TRUE ) } \arguments{ @@ -24,6 +25,12 @@ AWS will automatically delete images older than this many days. This prevents surprise costs if you stop using staRburst. Recommended: 30 days for regular users, 7 days for occasional users. When images are deleted, they will be rebuilt on next use (adds 3-5 min).} + +\item{build_image}{Build the worker environment image during setup (default: TRUE). +Set to FALSE to provision AWS resources (S3/ECR/ECS/VPC), write config, and +check quotas without triggering the multi-minute Docker image build. The +image is then built lazily on first worker launch via +\code{ensure_environment()}. Useful for CI / connectivity checks.} } \value{ Invisibly returns the configuration list. @@ -42,6 +49,9 @@ if (starburst_is_configured()) { # Use private base images with 7-day cleanup starburst_setup(use_public_base = FALSE, ecr_image_ttl_days = 7) + + # Provision resources without building the image (fast; CI / connectivity checks) + starburst_setup(build_image = FALSE) } } } From 41f00cbb0b00252e2c992e438bf4415ee1360f27 Mon Sep 17 00:00:00 2001 From: scttfrdmn <3011922+scttfrdmn@users.noreply.github.com> Date: Tue, 2 Jun 2026 17:41:40 -0700 Subject: [PATCH 2/4] fix: make ensure_buildx_builder surface errors and select builder (refs #30) The inspect/create calls discarded stdout/stderr, so a failed builder setup was swallowed and surfaced later as an opaque 'no builder found' at buildx build. Capture stdout/stderr (so safe_system's stop() carries the real stderr) and add 'docker buildx use ' after create so the explicit --builder reference resolves even if the create didn't register it as current. Refs #30 --- R/images.R | 29 +++++++++++++++++++++-------- man/ensure_buildx_builder.Rd | 30 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 man/ensure_buildx_builder.Rd diff --git a/R/images.R b/R/images.R index 4514583..e7c9864 100644 --- a/R/images.R +++ b/R/images.R @@ -230,9 +230,6 @@ get_base_image_uri <- function(region) { account_id, region, base_tag) } -#' Build base Docker image with common dependencies -#' -#' @keywords internal #' Ensure a buildx builder with the docker-container driver exists and is usable #' #' Idempotent across repeated runs and cross-platform (Windows/macOS/Linux). @@ -252,11 +249,16 @@ get_base_image_uri <- function(region) { #' @return TRUE if the named builder is usable, FALSE otherwise #' @keywords internal ensure_buildx_builder <- function(builder_name = "starburst-builder") { + # stdout/stderr are captured (not discarded) so that when a buildx call fails + # safe_system()'s stop() carries the real stderr, instead of the failure being + # swallowed here and surfacing later as an opaque "no builder found" at the + # buildx build step (GitHub #24/#30). + # 1. Does it already exist? inspect returns non-zero (-> error) if not; # --bootstrap also boots an existing-but-stopped builder. exists <- tryCatch({ safe_system("docker", c("buildx", "inspect", "--bootstrap", builder_name), - stdout = FALSE, stderr = FALSE) + stdout = TRUE, stderr = TRUE) TRUE }, error = function(e) FALSE) @@ -270,7 +272,7 @@ ensure_buildx_builder <- function(builder_name = "starburst-builder") { "docker", c("buildx", "create", "--name", builder_name, "--driver", "docker-container", "--bootstrap"), - stdout = FALSE, stderr = FALSE + stdout = TRUE, stderr = TRUE ) TRUE }, error = function(e) { @@ -283,14 +285,25 @@ ensure_buildx_builder <- function(builder_name = "starburst-builder") { return(FALSE) } - # 3. Confirm it is now usable (defensive; create + bootstrap should suffice). + # 3. Confirm it booted and is selectable. 'inspect' verifies it is usable; + # 'use' registers it as current so the explicit --builder reference in + # buildx build resolves (guards against the create not persisting). tryCatch({ safe_system("docker", c("buildx", "inspect", "--bootstrap", builder_name), - stdout = FALSE, stderr = FALSE) + stdout = TRUE, stderr = TRUE) + safe_system("docker", c("buildx", "use", builder_name), + stdout = TRUE, stderr = TRUE) TRUE - }, error = function(e) FALSE) + }, error = function(e) { + cat_warn(sprintf("Warning: buildx builder '%s' created but not usable: %s\n", + builder_name, conditionMessage(e))) + FALSE + }) } +#' Build base Docker image with common dependencies +#' +#' @keywords internal build_base_image <- function(region) { cat_info("[Docker] Building staRburst base image...\n") diff --git a/man/ensure_buildx_builder.Rd b/man/ensure_buildx_builder.Rd new file mode 100644 index 0000000..7c0fab2 --- /dev/null +++ b/man/ensure_buildx_builder.Rd @@ -0,0 +1,30 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/images.R +\name{ensure_buildx_builder} +\alias{ensure_buildx_builder} +\title{Ensure a buildx builder with the docker-container driver exists and is usable} +\usage{ +ensure_buildx_builder(builder_name = "starburst-builder") +} +\arguments{ +\item{builder_name}{Name of the buildx builder (default "starburst-builder")} +} +\value{ +TRUE if the named builder is usable, FALSE otherwise +} +\description{ +Idempotent across repeated runs and cross-platform (Windows/macOS/Linux). +Probes for an existing builder via \code{docker buildx inspect}; creates it +only when missing; bootstraps it so it is ready to build. A docker-container +driver is required for multi-platform (\code{linux/amd64,linux/arm64}) +builds. Does not mutate the user's default buildx context (no \code{--use}); +the build pins the builder explicitly via \code{--builder}. +} +\details{ +Returns TRUE if the named builder is usable, FALSE otherwise, and never +throws -- callers decide policy. This fixes the failure mode where +\code{buildx create} errored on an already-existing builder, the error was +swallowed, and the subsequent \code{buildx build} failed with "existing +instance for but no append mode" (GitHub #24). +} +\keyword{internal} From c0e79dd2866359c5ae635200ec6b210fa1582995 Mon Sep 17 00:00:00 2001 From: scttfrdmn <3011922+scttfrdmn@users.noreply.github.com> Date: Tue, 2 Jun 2026 17:41:40 -0700 Subject: [PATCH 3/4] test: cover build_image=FALSE/TRUE and buildx use selection (refs #30) - starburst_setup(build_image=FALSE) does not call build_initial_environment; build_image=TRUE (default) does. - ensure_buildx_builder selects the builder via 'buildx use' after create. Refs #30 --- tests/testthat/test-docker.R | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/testthat/test-docker.R b/tests/testthat/test-docker.R index 1d1b5e8..6510fa9 100644 --- a/tests/testthat/test-docker.R +++ b/tests/testthat/test-docker.R @@ -186,6 +186,12 @@ test_that("ensure_buildx_builder creates builder when missing", { expect_true(all(c("--driver", "docker-container", "--bootstrap") %in% create_call[[1]])) expect_true(all(c("--name", "starburst-builder") %in% create_call[[1]])) + + # After create, the builder must be explicitly selected so a later + # `buildx build --builder starburst-builder` resolves (GitHub #30). + use_call <- Filter(function(a) "use" %in% a, calls) + expect_length(use_call, 1) + expect_true("starburst-builder" %in% use_call[[1]]) }) test_that("ensure_buildx_builder returns FALSE when create fails", { @@ -238,3 +244,37 @@ test_that("build_environment_image aborts when builder is unusable", { "starburst-builder" ) }) + +# Helper: stub starburst_setup's collaborators so it runs offline, returning +# whether build_initial_environment was invoked. +run_setup_capturing_build <- function(build_image) { + built <- FALSE + mockery::stub(starburst_setup, "is_setup_complete", function() FALSE) + mockery::stub(starburst_setup, "check_aws_credentials", function() TRUE) + mockery::stub(starburst_setup, "get_aws_account_id", function() "123456789012") + mockery::stub(starburst_setup, "create_starburst_bucket", function(...) "starburst-test") + mockery::stub(starburst_setup, "create_ecr_repository", function(...) list(repositoryUri = "uri")) + mockery::stub(starburst_setup, "create_ecs_cluster", function(...) list(clusterName = "starburst-cluster")) + mockery::stub(starburst_setup, "setup_vpc_resources", + function(...) list(vpc_id = "vpc-1", subnets = list(), security_groups = list())) + mockery::stub(starburst_setup, "save_config", function(...) invisible()) + mockery::stub(starburst_setup, "check_fargate_quota", + function(...) list(limit = 1000, used = 0, available = 1000, increase_pending = FALSE)) + mockery::stub(starburst_setup, "build_initial_environment", function(...) built <<- TRUE) + starburst_setup(force = TRUE, build_image = build_image) + built +} + +test_that("starburst_setup skips image build when build_image = FALSE", { + skip_on_cran() + skip_if_not_installed("mockery") + + expect_false(run_setup_capturing_build(build_image = FALSE)) +}) + +test_that("starburst_setup builds image when build_image = TRUE (default)", { + skip_on_cran() + skip_if_not_installed("mockery") + + expect_true(run_setup_capturing_build(build_image = TRUE)) +}) From 46080b158368cbb0ae814eafc64a0c5575b478ea Mon Sep 17 00:00:00 2001 From: scttfrdmn <3011922+scttfrdmn@users.noreply.github.com> Date: Tue, 2 Jun 2026 17:41:40 -0700 Subject: [PATCH 4/4] ci: default scheduled suite to quick + skip image build in setup (closes #30) - Add job env SUITE = inputs.test_suite || 'quick' and gate the 5 suite steps on env.SUITE, so the monthly schedule (empty inputs) runs the quick smoke suite instead of being a no-op. - Configure staRburst step now uses build_image=FALSE so provisioning is fast for every run; suites that launch workers build the image lazily. Closes #30 --- .github/workflows/aws-integration-tests.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/aws-integration-tests.yml b/.github/workflows/aws-integration-tests.yml index 989dc29..c7ee398 100644 --- a/.github/workflows/aws-integration-tests.yml +++ b/.github/workflows/aws-integration-tests.yml @@ -31,6 +31,12 @@ jobs: # Only run if AWS credentials are configured if: github.event_name == 'workflow_dispatch' || github.repository == 'scttfrdmn/starburst' + # Effective test suite: the dispatch input when triggered manually, or + # 'quick' on the monthly schedule (where inputs are empty) so the cron run + # performs a real, near-zero-cost drift check instead of a no-op. + env: + SUITE: ${{ github.event.inputs.test_suite || 'quick' }} + permissions: id-token: write # For OIDC contents: read @@ -85,11 +91,13 @@ jobs: run: | Rscript -e " devtools::load_all() - starburst_setup(region = 'us-east-1', force = TRUE) + # build_image = FALSE: provision resources + write config without the + # multi-minute image build. Suites that launch workers build it lazily. + starburst_setup(region = 'us-east-1', force = TRUE, build_image = FALSE) " - name: Run quick smoke tests - if: github.event.inputs.test_suite == 'quick' || github.event.inputs.test_suite == 'all' + if: env.SUITE == 'quick' || env.SUITE == 'all' run: | Rscript -e " devtools::load_all() @@ -114,7 +122,7 @@ jobs: " - name: Run detached session tests - if: github.event.inputs.test_suite == 'detached-sessions' || github.event.inputs.test_suite == 'all' + if: env.SUITE == 'detached-sessions' || env.SUITE == 'all' env: RUN_INTEGRATION_TESTS: "TRUE" # opt-in gate for tests that launch real Fargate workers run: | @@ -130,7 +138,7 @@ jobs: " - name: Run integration example tests - if: github.event.inputs.test_suite == 'integration-examples' || github.event.inputs.test_suite == 'all' + if: env.SUITE == 'integration-examples' || env.SUITE == 'all' env: RUN_INTEGRATION_TESTS: "TRUE" run: | @@ -146,7 +154,7 @@ jobs: " - name: Run EC2 integration tests - if: github.event.inputs.test_suite == 'ec2' || github.event.inputs.test_suite == 'all' + if: env.SUITE == 'ec2' || env.SUITE == 'all' run: | Rscript -e " devtools::load_all() @@ -161,7 +169,7 @@ jobs: " - name: Run cleanup tests - if: github.event.inputs.test_suite == 'cleanup' || github.event.inputs.test_suite == 'all' + if: env.SUITE == 'cleanup' || env.SUITE == 'all' run: | Rscript -e " devtools::load_all()