diff --git a/.dockerignore b/.dockerignore index c9f9e5414..167eddca4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,3 @@ target -Dockerfile -arroyo-console/node_modules +docker/Dockerfile +webui/node_modules diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a9cacd64..df55e28ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,23 +17,9 @@ jobs: args: [--frozen-lockfile, --strict-peer-dependencies] - name: Run prettier run: | - cd arroyo-console + cd webui pnpm check - check-dockerfiles: - runs-on: ubuntu-latest - steps: - - name: Check out - uses: actions/checkout@v3 - - name: Set up Ruby - uses: ruby/setup-ruby@ec02537da5712d66d4d50a0f33b7eb52773b5ed1 - with: - ruby-version: '3.1' - - name: Install dependencies - run: gem install toml-rb - - name: Check Dockerfiles - run: ruby docker/check_dockerfiles.rb - build-rust: runs-on: ubicloud-standard-16 env: @@ -54,12 +40,16 @@ jobs: with: toolchain: stable override: true + - name: Check Formatting + run: cargo fmt -- --check - name: Setup npm uses: actions/setup-node@v3 + with: + node-version: '21' - name: Install OpenAPI Generator run: | npm install @openapitools/openapi-generator-cli -g - openapi-generator-cli version + cd crates/arroyo-openapi && openapi-generator-cli version - name: Setup Postgres run: | sudo apt-get update @@ -72,7 +62,7 @@ jobs: tar xvfz refinery*.tar.gz mv /tmp/refinery*-musl/refinery /tmp popd - /tmp/refinery migrate -e DATABASE_URL -p arroyo-api/migrations + /tmp/refinery migrate -e DATABASE_URL -p crates/arroyo-api/migrations - name: Install dependencies run: | curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin @@ -88,14 +78,12 @@ jobs: mkdir /tmp/kraft-combined-logs kafka_*/bin/kafka-storage.sh format -t 9v5PspiySuWU2l5NjTgRuA -c kafka_*/config/kraft/server.properties kafka_*/bin/kafka-server-start.sh -daemon kafka_*/config/kraft/server.properties - - name: Check Formatting - run: cargo fmt -- --check - name: Build run: cargo build --all-features - name: Validate API run: | npm install --global ibm-openapi-validator - cd arroyo-openapi + cd crates/arroyo-openapi lint-openapi --errors-only api-spec.json - name: Test run: cargo nextest run --jobs 4 --all-features @@ -117,5 +105,5 @@ jobs: args: [--frozen-lockfile, --strict-peer-dependencies] - name: Build console run: | - cd arroyo-console + cd webui pnpm build diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 790e88280..38c4c2f46 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -4,6 +4,7 @@ on: push: branches: - master + - dev - '*docker*' tags: - 'v[0-9]+.[0-9]+.[0-9]+*' # Semver matching pattern with optional suffix @@ -18,103 +19,112 @@ jobs: platform: - linux/amd64 - linux/arm64 - image_type: - - worker - - services - - compiler - - single include: - platform: linux/amd64 - mold_arch: x86_64 - proto_arch: x86_64 - prom_arch: amd64 - runs_on: buildjet-8vcpu-ubuntu-2204 + runs_on: buildjet-32vcpu-ubuntu-2204 + arch: amd64 - platform: linux/arm64 - mold_arch: aarch64 - proto_arch: aarch_64 - prom_arch: arm64 - runs_on: buildjet-8vcpu-ubuntu-2204-arm - - image_type: worker - dockerfile: docker/cluster/worker/Dockerfile - - image_type: services - dockerfile: docker/cluster/services/Dockerfile - - image_type: compiler - dockerfile: docker/cluster/compiler/Dockerfile - - image_type: single - dockerfile: docker/single/Dockerfile + runs_on: buildjet-32vcpu-ubuntu-2204-arm + arch: arm64 runs-on: ${{ matrix.runs_on }} steps: - # Get the repository's code - - name: Checkout - uses: actions/checkout@v2 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GHCR if: github.event_name != 'pull_request' - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Docker meta - id: arroyo-docker # you'll use this in the next step - uses: docker/metadata-action@v3 + id: meta + uses: docker/metadata-action@v5 with: - images: | - ghcr.io/arroyosystems/arroyo-${{ matrix.image_type }} - tags: | - type=schedule - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha - flavor: | - latest=false - prefix=${{ matrix.prom_arch }}- - - name: Set env - run: echo "GIT_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV - - name: Build and push - uses: docker/build-push-action@v2 + images: ghcr.io/arroyosystems/arroyo-single + + - name: Build and push single + id: build-single + uses: docker/build-push-action@v5 + with: + file: docker/Dockerfile + platforms: ${{ matrix.platform }} + build-args: | + GIT_SHA=${{ github.sha }} + push: ${{ github.event_name != 'pull_request' }} + cache-from: type=registry,ref=ghcr.io/arroyosystems/arroyo-builder:buildcache-${{ matrix.arch }} + cache-to: type=registry,ref=ghcr.io/arroyosystems/arroyo-builder:buildcache-${{ matrix.arch }} + target: arroyo-single + outputs: type=image,name=ghcr.io/arroyosystems/arroyo-single,push-by-digest=true,name-canonical=true,push=true,store=true + + - name: Docker meta + id: meta-arroyo + uses: docker/metadata-action@v5 + with: + images: ghcr.io/arroyosystems/arroyo + + - name: Build and push arroyo + id: build-arroyo + uses: docker/build-push-action@v5 with: - context: . - file: ${{ matrix.dockerfile }} + file: docker/Dockerfile platforms: ${{ matrix.platform }} build-args: | - MOLD_ARCH=${{ matrix.mold_arch }} - PROTO_ARCH=${{ matrix.proto_arch }} - PROM_ARCH=${{ matrix.prom_arch }} - GIT_SHA=${{ env.GIT_SHA }} + GIT_SHA=${{ github.sha }} push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.arroyo-docker.outputs.tags }} - labels: ${{ steps.arroyo-docker.outputs.labels }} + cache-from: type=registry,ref=ghcr.io/arroyosystems/arroyo-builder:buildcache-${{ matrix.arch }} + cache-to: type=registry,ref=ghcr.io/arroyosystems/arroyo-builder:buildcache-${{ matrix.arch }} + target: arroyo + outputs: type=image,name=ghcr.io/arroyosystems/arroyo,push-by-digest=true,name-canonical=true,push=true,store=true + + - name: Export digest + run: | + mkdir -p /tmp/digests/arroyo-single + digest="${{ steps.build-single.outputs.digest }}" + touch "/tmp/digests/arroyo-single/${digest#sha256:}" + + mkdir -p /tmp/digests/arroyo + digest="${{ steps.build-arroyo.outputs.digest }}" + touch "/tmp/digests/arroyo/${digest#sha256:}" + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + manifest: needs: build strategy: matrix: - image_type: - - worker - - services - - compiler - - single + image_name: + - arroyo + - arroyo-single runs-on: ubuntu-latest steps: - - name: Login to GHCR - uses: docker/login-action@v1 + - + name: Download digests + uses: actions/download-artifact@v4 with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Docker meta - id: arroyo-docker # you'll use this in the next step - uses: docker/metadata-action@v3 + path: /tmp/digests + pattern: digests-* + merge-multiple: true + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 with: - images: | - ghcr.io/arroyosystems/arroyo-${{ matrix.image_type }} + images: ghcr.io/arroyosystems/${{ matrix.image_name }} tags: | type=schedule type=ref,event=branch @@ -123,20 +133,28 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} type=sha - - name: Create and push Docker manifest + - name: Login to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Create manifest list and push + working-directory: /tmp/digests/${{ matrix.image_name }} + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf 'ghcr.io/arroyosystems/${{ matrix.image_name }}@sha256:%s ' *) + - + name: Inspect image run: | - TAG=${{ steps.arroyo-docker.outputs.version }} - IMAGE=ghcr.io/arroyosystems/arroyo-${{ matrix.image_type }} - docker manifest create ${IMAGE}:${TAG} ${IMAGE}:arm64-${TAG} ${IMAGE}:amd64-${TAG} - docker manifest annotate ${IMAGE}:${TAG} ${IMAGE}:arm64-${TAG} --arch arm64 - docker manifest annotate ${IMAGE}:${TAG} ${IMAGE}:amd64-${TAG} --arch amd64 - docker manifest push ${IMAGE}:${TAG} - - name: Push latest to tip tag. - if: github.ref == 'refs/heads/master' + docker buildx imagetools inspect ghcr.io/arroyosystems/${{ matrix.image_name }}:${{ steps.meta.outputs.version }} + # TODO: change this to tip once merged into master + - name: Push to dev tag. + working-directory: /tmp/digests/${{ matrix.image_name }} + # change to dev before merging + if: github.ref == 'refs/heads/df-docker' run: | - TAG=${{ steps.arroyo-docker.outputs.version }} - IMAGE=ghcr.io/arroyosystems/arroyo-${{ matrix.image_type }} - docker manifest create ${IMAGE}:tip ${IMAGE}:arm64-${TAG} ${IMAGE}:amd64-${TAG} - docker manifest annotate ${IMAGE}:tip ${IMAGE}:arm64-${TAG} --arch arm64 - docker manifest annotate ${IMAGE}:tip ${IMAGE}:amd64-${TAG} --arch amd64 - docker manifest push ${IMAGE}:tip \ No newline at end of file + docker buildx imagetools create --tag ghcr.io/arroyosystems/${{ matrix.image_name }}:dev \ + $(printf 'ghcr.io/arroyosystems/${{ matrix.image_name }}@sha256:%s ' *) diff --git a/.gitignore b/.gitignore index 28b7a41e2..274a8d4cc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ target/ .idea .env .vscode/ -arroyo-openapi/api-spec.json -arroyo-openapi/client +crates/arroyo-openapi/api-spec.json +crates/arroyo-openapi/client +openapitools.json diff --git a/Cargo.lock b/Cargo.lock index c2f39852a..4910e0b5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -723,6 +723,7 @@ dependencies = [ name = "arroyo-bin" version = "0.10.0-dev" dependencies = [ + "anyhow", "arroyo-api", "arroyo-compiler-service", "arroyo-connectors", @@ -733,6 +734,7 @@ dependencies = [ "clap 4.4.18", "deadpool-postgres", "postgres-types", + "refinery", "serde", "serde_json", "tokio", @@ -1088,6 +1090,7 @@ dependencies = [ "tower-http", "tracing", "tracing-appender", + "tracing-log", "tracing-logfmt", "tracing-subscriber", "vergen", @@ -6946,9 +6949,9 @@ dependencies = [ [[package]] name = "refinery" -version = "0.8.11" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "529664dbccc0a296947615c997a857912d72d1c44be1fafb7bae54ecfa7a8c24" +checksum = "431fd1854421586de5e83614052cb0d2d82025647829db6d9499b27385e58a6b" dependencies = [ "refinery-core", "refinery-macros", diff --git a/Cargo.toml b/Cargo.toml index 8920b64e4..aafaaf583 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,28 +1,28 @@ [workspace] members = [ - "arroyo", - "arroyo-api", - "arroyo-bin", - "arroyo-compiler-service", - "arroyo-controller", - "arroyo-connectors", - "arroyo-datastream", - "arroyo-df", - "arroyo-formats", - "arroyo-macro", - "arroyo-metrics", - "arroyo-node", - "arroyo-openapi", - "arroyo-operator", - "arroyo-rpc", - "arroyo-sql-testing", - "arroyo-server-common", - "arroyo-state", - "arroyo-storage", - "arroyo-types", - "arroyo-worker", - "copy-artifacts", - "integ", + "crates/arroyo", + "crates/arroyo-api", + "crates/arroyo-bin", + "crates/arroyo-compiler-service", + "crates/arroyo-controller", + "crates/arroyo-connectors", + "crates/arroyo-datastream", + "crates/arroyo-df", + "crates/arroyo-formats", + "crates/arroyo-macro", + "crates/arroyo-metrics", + "crates/arroyo-node", + "crates/arroyo-openapi", + "crates/arroyo-operator", + "crates/arroyo-rpc", + "crates/arroyo-sql-testing", + "crates/arroyo-server-common", + "crates/arroyo-state", + "crates/arroyo-storage", + "crates/arroyo-types", + "crates/arroyo-worker", + "crates/copy-artifacts", + "crates/integ", ] resolver = "2" diff --git a/README.md b/README.md index 90c77b536..1a6450e0a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

- Arroyo + Arroyo

@@ -36,7 +36,7 @@ on both bounded and unbounded sources, emitting results as soon as they are avai In short: Arroyo lets you ask complex questions of high-volume real-time data with subsecond results. -![running job](docs/images/header_image.png) +![running job](images/header_image.png) ## Features diff --git a/arroyo-console/.env b/arroyo-console/.env deleted file mode 100644 index da8112756..000000000 --- a/arroyo-console/.env +++ /dev/null @@ -1,4 +0,0 @@ -# allow the API server to replace this with the correct endpoint -VITE_API_ENDPOINT="{{API_ENDPOINT}}" -VITE_CLUSTER_ID="{{CLUSTER_ID}}" -VITE_DISABLE_TELEMETRY="{{DISABLE_TELEMETRY}}" diff --git a/arroyo-console/openapitools.json b/arroyo-console/openapitools.json deleted file mode 100644 index cd53ff4ca..000000000 --- a/arroyo-console/openapitools.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json", - "spaces": 2, - "generator-cli": { - "version": "6.6.0" - } -} diff --git a/arroyo-openapi/openapitools.json b/arroyo-openapi/openapitools.json deleted file mode 100644 index cd53ff4ca..000000000 --- a/arroyo-openapi/openapitools.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json", - "spaces": 2, - "generator-cli": { - "version": "6.6.0" - } -} diff --git a/arroyo-sql-testing/outputs/.gitignore b/arroyo-sql-testing/outputs/.gitignore deleted file mode 100644 index 72e8ffc0d..000000000 --- a/arroyo-sql-testing/outputs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -* diff --git a/arroyo-api/Cargo.toml b/crates/arroyo-api/Cargo.toml similarity index 100% rename from arroyo-api/Cargo.toml rename to crates/arroyo-api/Cargo.toml diff --git a/arroyo-api/build.rs b/crates/arroyo-api/build.rs similarity index 100% rename from arroyo-api/build.rs rename to crates/arroyo-api/build.rs diff --git a/arroyo-api/migrations/V10__merge_definitions.sql b/crates/arroyo-api/migrations/V10__merge_definitions.sql similarity index 100% rename from arroyo-api/migrations/V10__merge_definitions.sql rename to crates/arroyo-api/migrations/V10__merge_definitions.sql diff --git a/arroyo-api/migrations/V11__delete_cascade.sql b/crates/arroyo-api/migrations/V11__delete_cascade.sql similarity index 100% rename from arroyo-api/migrations/V11__delete_cascade.sql rename to crates/arroyo-api/migrations/V11__delete_cascade.sql diff --git a/arroyo-api/migrations/V12__add_committing.sql b/crates/arroyo-api/migrations/V12__add_committing.sql similarity index 100% rename from arroyo-api/migrations/V12__add_committing.sql rename to crates/arroyo-api/migrations/V12__add_committing.sql diff --git a/arroyo-api/migrations/V13__backfill_pub_id.sql b/crates/arroyo-api/migrations/V13__backfill_pub_id.sql similarity index 100% rename from arroyo-api/migrations/V13__backfill_pub_id.sql rename to crates/arroyo-api/migrations/V13__backfill_pub_id.sql diff --git a/arroyo-api/migrations/V14__unify_job_ids.sql b/crates/arroyo-api/migrations/V14__unify_job_ids.sql similarity index 100% rename from arroyo-api/migrations/V14__unify_job_ids.sql rename to crates/arroyo-api/migrations/V14__unify_job_ids.sql diff --git a/arroyo-api/migrations/V15__rename_connections.sql b/crates/arroyo-api/migrations/V15__rename_connections.sql similarity index 100% rename from arroyo-api/migrations/V15__rename_connections.sql rename to crates/arroyo-api/migrations/V15__rename_connections.sql diff --git a/arroyo-api/migrations/V16__restart_failed.sql b/crates/arroyo-api/migrations/V16__restart_failed.sql similarity index 100% rename from arroyo-api/migrations/V16__restart_failed.sql rename to crates/arroyo-api/migrations/V16__restart_failed.sql diff --git a/arroyo-api/migrations/V17__udfs_table.sql b/crates/arroyo-api/migrations/V17__udfs_table.sql similarity index 100% rename from arroyo-api/migrations/V17__udfs_table.sql rename to crates/arroyo-api/migrations/V17__udfs_table.sql diff --git a/arroyo-api/migrations/V18__udfs_drop_language.sql b/crates/arroyo-api/migrations/V18__udfs_drop_language.sql similarity index 100% rename from arroyo-api/migrations/V18__udfs_drop_language.sql rename to crates/arroyo-api/migrations/V18__udfs_drop_language.sql diff --git a/arroyo-api/migrations/V19__case_insensitive_connection_names.sql b/crates/arroyo-api/migrations/V19__case_insensitive_connection_names.sql similarity index 100% rename from arroyo-api/migrations/V19__case_insensitive_connection_names.sql rename to crates/arroyo-api/migrations/V19__case_insensitive_connection_names.sql diff --git a/arroyo-api/migrations/V1__initial.sql b/crates/arroyo-api/migrations/V1__initial.sql similarity index 100% rename from arroyo-api/migrations/V1__initial.sql rename to crates/arroyo-api/migrations/V1__initial.sql diff --git a/crates/arroyo-api/migrations/V20__pipeline_v2.sql b/crates/arroyo-api/migrations/V20__pipeline_v2.sql new file mode 100644 index 000000000..e37ce866a --- /dev/null +++ b/crates/arroyo-api/migrations/V20__pipeline_v2.sql @@ -0,0 +1,2 @@ +ALTER TABLE pipelines +ADD COLUMN proto_version INT NOT NULL DEFAULT 1; \ No newline at end of file diff --git a/arroyo-api/migrations/V2__add_raw_json.sql b/crates/arroyo-api/migrations/V2__add_raw_json.sql similarity index 100% rename from arroyo-api/migrations/V2__add_raw_json.sql rename to crates/arroyo-api/migrations/V2__add_raw_json.sql diff --git a/arroyo-api/migrations/V3__add_udfs.sql b/crates/arroyo-api/migrations/V3__add_udfs.sql similarity index 100% rename from arroyo-api/migrations/V3__add_udfs.sql rename to crates/arroyo-api/migrations/V3__add_udfs.sql diff --git a/arroyo-api/migrations/V4__add_sources.sql b/crates/arroyo-api/migrations/V4__add_sources.sql similarity index 100% rename from arroyo-api/migrations/V4__add_sources.sql rename to crates/arroyo-api/migrations/V4__add_sources.sql diff --git a/arroyo-api/migrations/V5__add_http_connection.sql b/crates/arroyo-api/migrations/V5__add_http_connection.sql similarity index 100% rename from arroyo-api/migrations/V5__add_http_connection.sql rename to crates/arroyo-api/migrations/V5__add_http_connection.sql diff --git a/arroyo-api/migrations/V6__add_cluster_table.sql b/crates/arroyo-api/migrations/V6__add_cluster_table.sql similarity index 100% rename from arroyo-api/migrations/V6__add_cluster_table.sql rename to crates/arroyo-api/migrations/V6__add_cluster_table.sql diff --git a/arroyo-api/migrations/V7__add_job_log_messages.sql b/crates/arroyo-api/migrations/V7__add_job_log_messages.sql similarity index 100% rename from arroyo-api/migrations/V7__add_job_log_messages.sql rename to crates/arroyo-api/migrations/V7__add_job_log_messages.sql diff --git a/arroyo-api/migrations/V8__connector_redesign.sql b/crates/arroyo-api/migrations/V8__connector_redesign.sql similarity index 100% rename from arroyo-api/migrations/V8__connector_redesign.sql rename to crates/arroyo-api/migrations/V8__connector_redesign.sql diff --git a/arroyo-api/migrations/V9__add_pub_ids.sql b/crates/arroyo-api/migrations/V9__add_pub_ids.sql similarity index 100% rename from arroyo-api/migrations/V9__add_pub_ids.sql rename to crates/arroyo-api/migrations/V9__add_pub_ids.sql diff --git a/arroyo-api/queries/api_queries.sql b/crates/arroyo-api/queries/api_queries.sql similarity index 99% rename from arroyo-api/queries/api_queries.sql rename to crates/arroyo-api/queries/api_queries.sql index daa9ca9d7..507ba91cf 100644 --- a/arroyo-api/queries/api_queries.sql +++ b/crates/arroyo-api/queries/api_queries.sql @@ -131,8 +131,8 @@ WHERE organization_id = :organization_id AND pub_id = :pub_id; --: DbPipeline (state?, ttl_micros?) --! create_pipeline(udfs?, textual_repr?) -INSERT INTO pipelines (pub_id, organization_id, created_by, name, type, textual_repr, udfs, program) -VALUES (:pub_id, :organization_id, :created_by, :name, :type, :textual_repr, :udfs, :program) +INSERT INTO pipelines (pub_id, organization_id, created_by, name, type, textual_repr, udfs, program, proto_version) +VALUES (:pub_id, :organization_id, :created_by, :name, :type, :textual_repr, :udfs, :program, :proto_version) RETURNING id; --! get_pipelines : DbPipeline diff --git a/arroyo-api/src/cloud.rs b/crates/arroyo-api/src/cloud.rs similarity index 100% rename from arroyo-api/src/cloud.rs rename to crates/arroyo-api/src/cloud.rs diff --git a/arroyo-api/src/connection_profiles.rs b/crates/arroyo-api/src/connection_profiles.rs similarity index 100% rename from arroyo-api/src/connection_profiles.rs rename to crates/arroyo-api/src/connection_profiles.rs diff --git a/arroyo-api/src/connection_tables.rs b/crates/arroyo-api/src/connection_tables.rs similarity index 100% rename from arroyo-api/src/connection_tables.rs rename to crates/arroyo-api/src/connection_tables.rs diff --git a/arroyo-api/src/connectors.rs b/crates/arroyo-api/src/connectors.rs similarity index 100% rename from arroyo-api/src/connectors.rs rename to crates/arroyo-api/src/connectors.rs diff --git a/arroyo-api/src/jobs.rs b/crates/arroyo-api/src/jobs.rs similarity index 100% rename from arroyo-api/src/jobs.rs rename to crates/arroyo-api/src/jobs.rs diff --git a/arroyo-api/src/lib.rs b/crates/arroyo-api/src/lib.rs similarity index 100% rename from arroyo-api/src/lib.rs rename to crates/arroyo-api/src/lib.rs diff --git a/arroyo-api/src/metrics.rs b/crates/arroyo-api/src/metrics.rs similarity index 100% rename from arroyo-api/src/metrics.rs rename to crates/arroyo-api/src/metrics.rs diff --git a/arroyo-api/src/pipelines.rs b/crates/arroyo-api/src/pipelines.rs similarity index 99% rename from arroyo-api/src/pipelines.rs rename to crates/arroyo-api/src/pipelines.rs index 9a22c4355..71c1bc85f 100644 --- a/arroyo-api/src/pipelines.rs +++ b/crates/arroyo-api/src/pipelines.rs @@ -369,6 +369,7 @@ pub(crate) async fn create_pipeline<'a>( &text, &udfs.map(|t| serde_json::to_value(t).unwrap()), &program_bytes, + &2, ) .one() .await diff --git a/arroyo-api/src/rest.rs b/crates/arroyo-api/src/rest.rs similarity index 97% rename from arroyo-api/src/rest.rs rename to crates/arroyo-api/src/rest.rs index bf65b7737..2c44ff08a 100644 --- a/arroyo-api/src/rest.rs +++ b/crates/arroyo-api/src/rest.rs @@ -63,11 +63,10 @@ pub async fn api_fallback() -> impl IntoResponse { } pub fn create_rest_app(pool: Pool, controller_addr: &str) -> Router { - let asset_dir = env::var(ASSET_DIR_ENV).unwrap_or_else(|_| "arroyo-console/dist".to_string()); + let asset_dir = env::var(ASSET_DIR_ENV).unwrap_or_else(|_| "webui/dist".to_string()); static INDEX_HTML: Lazy = Lazy::new(|| { - let asset_dir = - env::var(ASSET_DIR_ENV).unwrap_or_else(|_| "arroyo-console/dist".to_string()); + let asset_dir = env::var(ASSET_DIR_ENV).unwrap_or_else(|_| "webui/dist".to_string()); let endpoint = env::var(API_ENDPOINT_ENV).unwrap_or_else(|_| String::new()); diff --git a/arroyo-api/src/rest_utils.rs b/crates/arroyo-api/src/rest_utils.rs similarity index 100% rename from arroyo-api/src/rest_utils.rs rename to crates/arroyo-api/src/rest_utils.rs diff --git a/arroyo-api/src/udfs.rs b/crates/arroyo-api/src/udfs.rs similarity index 100% rename from arroyo-api/src/udfs.rs rename to crates/arroyo-api/src/udfs.rs diff --git a/arroyo-bin/Cargo.toml b/crates/arroyo-bin/Cargo.toml similarity index 89% rename from arroyo-bin/Cargo.toml rename to crates/arroyo-bin/Cargo.toml index 9d0b4ff08..7fc2df30c 100644 --- a/arroyo-bin/Cargo.toml +++ b/crates/arroyo-bin/Cargo.toml @@ -5,7 +5,6 @@ edition = "2021" [features] kafka-sasl = ["arroyo-connectors/kafka-sasl", "arroyo-worker/kafka-sal"] -k8s = ["arroyo-controller/k8s"] [dependencies] arroyo-types = { path ="../arroyo-types" } @@ -26,3 +25,5 @@ postgres-types = { version = "*", features = ["derive"] } tokio-postgres = { version = "*", features = ["with-serde_json-1", "with-time-0_3", "with-uuid-1"] } deadpool-postgres = { version = "0.10" } uuid = { version = "1.7.0", features = ["v4"] } +refinery = { version = "=0.8.9" , features = ["tokio-postgres"] } +anyhow = { version = "1.0.79", features = [] } \ No newline at end of file diff --git a/arroyo-bin/src/main.rs b/crates/arroyo-bin/src/main.rs similarity index 56% rename from arroyo-bin/src/main.rs rename to crates/arroyo-bin/src/main.rs index 4b9b46659..99301427b 100644 --- a/arroyo-bin/src/main.rs +++ b/crates/arroyo-bin/src/main.rs @@ -1,3 +1,4 @@ +use anyhow::{anyhow, bail}; use arroyo_compiler_service; use arroyo_server_common::shutdown::Shutdown; use arroyo_server_common::{log_event, start_admin_server}; @@ -6,9 +7,12 @@ use arroyo_worker::WorkerServer; use clap::{Parser, Subcommand}; use deadpool_postgres::{ManagerConfig, Pool, RecyclingMethod}; use serde_json::json; +use std::process::exit; use std::time::Duration; -use tokio_postgres::NoTls; -use tracing::error; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::time::timeout; +use tokio_postgres::{Client, Connection, NoTls}; +use tracing::{debug, error, info}; use uuid::Uuid; #[derive(Parser)] @@ -34,6 +38,13 @@ enum Commands { /// Starts an Arroyo compiler Compiler {}, + + /// Runs database migrations on the configure Postgres database + Migrate { + /// If set, waits for the specified number of seconds until Postgres is ready before running migrations + #[arg(long)] + wait: Option, + }, } #[derive(Debug, Eq, PartialEq, Copy, Clone)] @@ -75,37 +86,39 @@ async fn main() { Commands::Worker { .. } => { start_worker().await; } + Commands::Migrate { wait } => { + if let Err(e) = migrate(*wait).await { + error!("{}", e); + exit(1); + } + } }; } async fn db_pool() -> Pool { let config = DatabaseConfig::load(); let mut cfg = deadpool_postgres::Config::new(); - cfg.dbname = Some(config.name); - cfg.host = Some(config.host); + cfg.dbname = Some(config.name.clone()); + cfg.host = Some(config.host.clone()); cfg.port = Some(config.port); - cfg.user = Some(config.user); - cfg.password = Some(config.password); + cfg.user = Some(config.user.clone()); + cfg.password = Some(config.password.clone()); cfg.manager = Some(ManagerConfig { recycling_method: RecyclingMethod::Fast, }); let pool = cfg .create_pool(Some(deadpool_postgres::Runtime::Tokio1), NoTls) .unwrap_or_else(|e| { - panic!( - "Unable to connect to database {:?}@{:?}:{:?}/{:?} {}", - cfg.user, cfg.host, cfg.port, cfg.dbname, e - ) + error!("Unable to connect to database {}: {:?}", config, e); + exit(1); }); match pool .get() .await .unwrap_or_else(|e| { - panic!( - "Unable to create database connection for {:?}@{:?}:{:?}/{:?} {}", - cfg.user, cfg.host, cfg.port, cfg.dbname, e - ) + error!("Unable to create database connection for {} {}", config, e); + exit(1); }) .query_one("select id from cluster_info", &[]) .await @@ -122,6 +135,88 @@ async fn db_pool() -> Pool { pool } +mod migrations { + use refinery::embed_migrations; + embed_migrations!("../arroyo-api/migrations"); +} + +async fn connect( + retry: bool, +) -> anyhow::Result<( + Client, + Connection, +)> { + let config = DatabaseConfig::load(); + + loop { + match tokio_postgres::config::Config::new() + .host(&config.host) + .port(config.port) + .user(&config.user) + .password(&config.password) + .dbname(&config.name) + .connect(NoTls) + .await + { + Ok(r) => { + return Ok(r); + } + Err(e) => { + if !e.to_string().contains("authentication") && retry { + debug!("Received error from database while waiting: {}", e); + tokio::time::sleep(Duration::from_millis(500)).await; + continue; + } + + bail!("Failed to connect to database {}: {}", config, e); + } + } + } +} + +async fn migrate(wait: Option) -> anyhow::Result<()> { + let _guard = arroyo_server_common::init_logging("migrate"); + + let (mut client, connection) = if let Some(wait) = wait { + info!("Waiting for database to be ready to run migrations"); + timeout(Duration::from_secs(wait as u64), connect(true)) + .await + .map_err(|e| anyhow!("Timed out waiting for database to connect after {}", e))?? + } else { + connect(false).await? + }; + + tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("connection error: {}", e); + } + }); + + info!("Running migrations on database {}", DatabaseConfig::load()); + + let report = migrations::migrations::runner() + .run_async(&mut client) + .await + .map_err(|e| { + anyhow!( + "Failed to run migrations on {}: {:?}", + DatabaseConfig::load(), + e + ) + })?; + + for migration in report.applied_migrations() { + info!("Applying V{} {}", migration.version(), migration.name()); + } + + info!( + "Successfully applied {} migration(s)", + report.applied_migrations().len() + ); + + Ok(()) +} + async fn start_control_plane(service: CPService) { let _guard = arroyo_server_common::init_logging(service.name()); diff --git a/arroyo-compiler-service/Cargo.toml b/crates/arroyo-compiler-service/Cargo.toml similarity index 100% rename from arroyo-compiler-service/Cargo.toml rename to crates/arroyo-compiler-service/Cargo.toml diff --git a/arroyo-compiler-service/src/lib.rs b/crates/arroyo-compiler-service/src/lib.rs similarity index 100% rename from arroyo-compiler-service/src/lib.rs rename to crates/arroyo-compiler-service/src/lib.rs diff --git a/arroyo-connectors/Cargo.toml b/crates/arroyo-connectors/Cargo.toml similarity index 100% rename from arroyo-connectors/Cargo.toml rename to crates/arroyo-connectors/Cargo.toml diff --git a/arroyo-connectors/build.rs b/crates/arroyo-connectors/build.rs similarity index 100% rename from arroyo-connectors/build.rs rename to crates/arroyo-connectors/build.rs diff --git a/arroyo-connectors/resources/nexmark.svg b/crates/arroyo-connectors/resources/nexmark.svg similarity index 100% rename from arroyo-connectors/resources/nexmark.svg rename to crates/arroyo-connectors/resources/nexmark.svg diff --git a/arroyo-connectors/src/blackhole/blackhole.svg b/crates/arroyo-connectors/src/blackhole/blackhole.svg similarity index 100% rename from arroyo-connectors/src/blackhole/blackhole.svg rename to crates/arroyo-connectors/src/blackhole/blackhole.svg diff --git a/arroyo-connectors/src/blackhole/mod.rs b/crates/arroyo-connectors/src/blackhole/mod.rs similarity index 100% rename from arroyo-connectors/src/blackhole/mod.rs rename to crates/arroyo-connectors/src/blackhole/mod.rs diff --git a/arroyo-connectors/src/blackhole/operator.rs b/crates/arroyo-connectors/src/blackhole/operator.rs similarity index 100% rename from arroyo-connectors/src/blackhole/operator.rs rename to crates/arroyo-connectors/src/blackhole/operator.rs diff --git a/arroyo-connectors/src/confluent/confluent.svg b/crates/arroyo-connectors/src/confluent/confluent.svg similarity index 100% rename from arroyo-connectors/src/confluent/confluent.svg rename to crates/arroyo-connectors/src/confluent/confluent.svg diff --git a/arroyo-connectors/src/confluent/mod.rs b/crates/arroyo-connectors/src/confluent/mod.rs similarity index 100% rename from arroyo-connectors/src/confluent/mod.rs rename to crates/arroyo-connectors/src/confluent/mod.rs diff --git a/arroyo-connectors/src/confluent/profile.json b/crates/arroyo-connectors/src/confluent/profile.json similarity index 100% rename from arroyo-connectors/src/confluent/profile.json rename to crates/arroyo-connectors/src/confluent/profile.json diff --git a/arroyo-connectors/src/delta.rs b/crates/arroyo-connectors/src/delta.rs similarity index 100% rename from arroyo-connectors/src/delta.rs rename to crates/arroyo-connectors/src/delta.rs diff --git a/arroyo-connectors/src/filesystem/filesystem.svg b/crates/arroyo-connectors/src/filesystem/filesystem.svg similarity index 100% rename from arroyo-connectors/src/filesystem/filesystem.svg rename to crates/arroyo-connectors/src/filesystem/filesystem.svg diff --git a/arroyo-connectors/src/filesystem/mod.rs b/crates/arroyo-connectors/src/filesystem/mod.rs similarity index 100% rename from arroyo-connectors/src/filesystem/mod.rs rename to crates/arroyo-connectors/src/filesystem/mod.rs diff --git a/arroyo-connectors/src/filesystem/sink/arrow.rs b/crates/arroyo-connectors/src/filesystem/sink/arrow.rs similarity index 100% rename from arroyo-connectors/src/filesystem/sink/arrow.rs rename to crates/arroyo-connectors/src/filesystem/sink/arrow.rs diff --git a/arroyo-connectors/src/filesystem/sink/delta.rs b/crates/arroyo-connectors/src/filesystem/sink/delta.rs similarity index 100% rename from arroyo-connectors/src/filesystem/sink/delta.rs rename to crates/arroyo-connectors/src/filesystem/sink/delta.rs diff --git a/arroyo-connectors/src/filesystem/sink/json.rs b/crates/arroyo-connectors/src/filesystem/sink/json.rs similarity index 100% rename from arroyo-connectors/src/filesystem/sink/json.rs rename to crates/arroyo-connectors/src/filesystem/sink/json.rs diff --git a/arroyo-connectors/src/filesystem/sink/local.rs b/crates/arroyo-connectors/src/filesystem/sink/local.rs similarity index 100% rename from arroyo-connectors/src/filesystem/sink/local.rs rename to crates/arroyo-connectors/src/filesystem/sink/local.rs diff --git a/arroyo-connectors/src/filesystem/sink/mod.rs b/crates/arroyo-connectors/src/filesystem/sink/mod.rs similarity index 100% rename from arroyo-connectors/src/filesystem/sink/mod.rs rename to crates/arroyo-connectors/src/filesystem/sink/mod.rs diff --git a/arroyo-connectors/src/filesystem/sink/parquet.rs b/crates/arroyo-connectors/src/filesystem/sink/parquet.rs similarity index 100% rename from arroyo-connectors/src/filesystem/sink/parquet.rs rename to crates/arroyo-connectors/src/filesystem/sink/parquet.rs diff --git a/arroyo-connectors/src/filesystem/sink/two_phase_committer.rs b/crates/arroyo-connectors/src/filesystem/sink/two_phase_committer.rs similarity index 100% rename from arroyo-connectors/src/filesystem/sink/two_phase_committer.rs rename to crates/arroyo-connectors/src/filesystem/sink/two_phase_committer.rs diff --git a/arroyo-connectors/src/filesystem/source.rs b/crates/arroyo-connectors/src/filesystem/source.rs similarity index 100% rename from arroyo-connectors/src/filesystem/source.rs rename to crates/arroyo-connectors/src/filesystem/source.rs diff --git a/arroyo-connectors/src/filesystem/table.json b/crates/arroyo-connectors/src/filesystem/table.json similarity index 100% rename from arroyo-connectors/src/filesystem/table.json rename to crates/arroyo-connectors/src/filesystem/table.json diff --git a/arroyo-connectors/src/fluvio/fluvio.svg b/crates/arroyo-connectors/src/fluvio/fluvio.svg similarity index 100% rename from arroyo-connectors/src/fluvio/fluvio.svg rename to crates/arroyo-connectors/src/fluvio/fluvio.svg diff --git a/arroyo-connectors/src/fluvio/mod.rs b/crates/arroyo-connectors/src/fluvio/mod.rs similarity index 100% rename from arroyo-connectors/src/fluvio/mod.rs rename to crates/arroyo-connectors/src/fluvio/mod.rs diff --git a/arroyo-connectors/src/fluvio/sink.rs b/crates/arroyo-connectors/src/fluvio/sink.rs similarity index 100% rename from arroyo-connectors/src/fluvio/sink.rs rename to crates/arroyo-connectors/src/fluvio/sink.rs diff --git a/arroyo-connectors/src/fluvio/source.rs b/crates/arroyo-connectors/src/fluvio/source.rs similarity index 100% rename from arroyo-connectors/src/fluvio/source.rs rename to crates/arroyo-connectors/src/fluvio/source.rs diff --git a/arroyo-connectors/src/fluvio/table.json b/crates/arroyo-connectors/src/fluvio/table.json similarity index 100% rename from arroyo-connectors/src/fluvio/table.json rename to crates/arroyo-connectors/src/fluvio/table.json diff --git a/arroyo-connectors/src/impulse/impulse.svg b/crates/arroyo-connectors/src/impulse/impulse.svg similarity index 100% rename from arroyo-connectors/src/impulse/impulse.svg rename to crates/arroyo-connectors/src/impulse/impulse.svg diff --git a/arroyo-connectors/src/impulse/mod.rs b/crates/arroyo-connectors/src/impulse/mod.rs similarity index 100% rename from arroyo-connectors/src/impulse/mod.rs rename to crates/arroyo-connectors/src/impulse/mod.rs diff --git a/arroyo-connectors/src/impulse/operator.rs b/crates/arroyo-connectors/src/impulse/operator.rs similarity index 100% rename from arroyo-connectors/src/impulse/operator.rs rename to crates/arroyo-connectors/src/impulse/operator.rs diff --git a/arroyo-connectors/src/impulse/table.json b/crates/arroyo-connectors/src/impulse/table.json similarity index 100% rename from arroyo-connectors/src/impulse/table.json rename to crates/arroyo-connectors/src/impulse/table.json diff --git a/arroyo-connectors/src/kafka/kafka.svg b/crates/arroyo-connectors/src/kafka/kafka.svg similarity index 100% rename from arroyo-connectors/src/kafka/kafka.svg rename to crates/arroyo-connectors/src/kafka/kafka.svg diff --git a/arroyo-connectors/src/kafka/mod.rs b/crates/arroyo-connectors/src/kafka/mod.rs similarity index 100% rename from arroyo-connectors/src/kafka/mod.rs rename to crates/arroyo-connectors/src/kafka/mod.rs diff --git a/arroyo-connectors/src/kafka/profile.json b/crates/arroyo-connectors/src/kafka/profile.json similarity index 100% rename from arroyo-connectors/src/kafka/profile.json rename to crates/arroyo-connectors/src/kafka/profile.json diff --git a/arroyo-connectors/src/kafka/sink/mod.rs b/crates/arroyo-connectors/src/kafka/sink/mod.rs similarity index 100% rename from arroyo-connectors/src/kafka/sink/mod.rs rename to crates/arroyo-connectors/src/kafka/sink/mod.rs diff --git a/arroyo-connectors/src/kafka/sink/test.rs b/crates/arroyo-connectors/src/kafka/sink/test.rs similarity index 100% rename from arroyo-connectors/src/kafka/sink/test.rs rename to crates/arroyo-connectors/src/kafka/sink/test.rs diff --git a/arroyo-connectors/src/kafka/source/mod.rs b/crates/arroyo-connectors/src/kafka/source/mod.rs similarity index 100% rename from arroyo-connectors/src/kafka/source/mod.rs rename to crates/arroyo-connectors/src/kafka/source/mod.rs diff --git a/arroyo-connectors/src/kafka/source/test.rs b/crates/arroyo-connectors/src/kafka/source/test.rs similarity index 100% rename from arroyo-connectors/src/kafka/source/test.rs rename to crates/arroyo-connectors/src/kafka/source/test.rs diff --git a/arroyo-connectors/src/kafka/table.json b/crates/arroyo-connectors/src/kafka/table.json similarity index 100% rename from arroyo-connectors/src/kafka/table.json rename to crates/arroyo-connectors/src/kafka/table.json diff --git a/arroyo-connectors/src/kinesis/kinesis.svg b/crates/arroyo-connectors/src/kinesis/kinesis.svg similarity index 100% rename from arroyo-connectors/src/kinesis/kinesis.svg rename to crates/arroyo-connectors/src/kinesis/kinesis.svg diff --git a/arroyo-connectors/src/kinesis/mod.rs b/crates/arroyo-connectors/src/kinesis/mod.rs similarity index 100% rename from arroyo-connectors/src/kinesis/mod.rs rename to crates/arroyo-connectors/src/kinesis/mod.rs diff --git a/arroyo-connectors/src/kinesis/sink.rs b/crates/arroyo-connectors/src/kinesis/sink.rs similarity index 100% rename from arroyo-connectors/src/kinesis/sink.rs rename to crates/arroyo-connectors/src/kinesis/sink.rs diff --git a/arroyo-connectors/src/kinesis/source.rs b/crates/arroyo-connectors/src/kinesis/source.rs similarity index 100% rename from arroyo-connectors/src/kinesis/source.rs rename to crates/arroyo-connectors/src/kinesis/source.rs diff --git a/arroyo-connectors/src/kinesis/table.json b/crates/arroyo-connectors/src/kinesis/table.json similarity index 100% rename from arroyo-connectors/src/kinesis/table.json rename to crates/arroyo-connectors/src/kinesis/table.json diff --git a/arroyo-connectors/src/lib.rs b/crates/arroyo-connectors/src/lib.rs similarity index 100% rename from arroyo-connectors/src/lib.rs rename to crates/arroyo-connectors/src/lib.rs diff --git a/arroyo-connectors/src/nexmark.rs b/crates/arroyo-connectors/src/nexmark.rs similarity index 100% rename from arroyo-connectors/src/nexmark.rs rename to crates/arroyo-connectors/src/nexmark.rs diff --git a/arroyo-connectors/src/polling_http/connector.rs b/crates/arroyo-connectors/src/polling_http/connector.rs similarity index 100% rename from arroyo-connectors/src/polling_http/connector.rs rename to crates/arroyo-connectors/src/polling_http/connector.rs diff --git a/arroyo-connectors/src/polling_http/http.svg b/crates/arroyo-connectors/src/polling_http/http.svg similarity index 100% rename from arroyo-connectors/src/polling_http/http.svg rename to crates/arroyo-connectors/src/polling_http/http.svg diff --git a/arroyo-connectors/src/polling_http/mod.rs b/crates/arroyo-connectors/src/polling_http/mod.rs similarity index 100% rename from arroyo-connectors/src/polling_http/mod.rs rename to crates/arroyo-connectors/src/polling_http/mod.rs diff --git a/arroyo-connectors/src/polling_http/table.json b/crates/arroyo-connectors/src/polling_http/table.json similarity index 100% rename from arroyo-connectors/src/polling_http/table.json rename to crates/arroyo-connectors/src/polling_http/table.json diff --git a/arroyo-connectors/src/preview/mod.rs b/crates/arroyo-connectors/src/preview/mod.rs similarity index 100% rename from arroyo-connectors/src/preview/mod.rs rename to crates/arroyo-connectors/src/preview/mod.rs diff --git a/arroyo-connectors/src/preview/operator.rs b/crates/arroyo-connectors/src/preview/operator.rs similarity index 100% rename from arroyo-connectors/src/preview/operator.rs rename to crates/arroyo-connectors/src/preview/operator.rs diff --git a/arroyo-connectors/src/redis/mod.rs b/crates/arroyo-connectors/src/redis/mod.rs similarity index 100% rename from arroyo-connectors/src/redis/mod.rs rename to crates/arroyo-connectors/src/redis/mod.rs diff --git a/arroyo-connectors/src/redis/operator/mod.rs b/crates/arroyo-connectors/src/redis/operator/mod.rs similarity index 100% rename from arroyo-connectors/src/redis/operator/mod.rs rename to crates/arroyo-connectors/src/redis/operator/mod.rs diff --git a/arroyo-connectors/src/redis/operator/sink.rs b/crates/arroyo-connectors/src/redis/operator/sink.rs similarity index 100% rename from arroyo-connectors/src/redis/operator/sink.rs rename to crates/arroyo-connectors/src/redis/operator/sink.rs diff --git a/arroyo-connectors/src/redis/profile.json b/crates/arroyo-connectors/src/redis/profile.json similarity index 100% rename from arroyo-connectors/src/redis/profile.json rename to crates/arroyo-connectors/src/redis/profile.json diff --git a/arroyo-connectors/src/redis/redis.svg b/crates/arroyo-connectors/src/redis/redis.svg similarity index 100% rename from arroyo-connectors/src/redis/redis.svg rename to crates/arroyo-connectors/src/redis/redis.svg diff --git a/arroyo-connectors/src/redis/table.json b/crates/arroyo-connectors/src/redis/table.json similarity index 100% rename from arroyo-connectors/src/redis/table.json rename to crates/arroyo-connectors/src/redis/table.json diff --git a/arroyo-connectors/src/single_file/mod.rs b/crates/arroyo-connectors/src/single_file/mod.rs similarity index 100% rename from arroyo-connectors/src/single_file/mod.rs rename to crates/arroyo-connectors/src/single_file/mod.rs diff --git a/arroyo-connectors/src/single_file/sink.rs b/crates/arroyo-connectors/src/single_file/sink.rs similarity index 100% rename from arroyo-connectors/src/single_file/sink.rs rename to crates/arroyo-connectors/src/single_file/sink.rs diff --git a/arroyo-connectors/src/single_file/source.rs b/crates/arroyo-connectors/src/single_file/source.rs similarity index 100% rename from arroyo-connectors/src/single_file/source.rs rename to crates/arroyo-connectors/src/single_file/source.rs diff --git a/arroyo-connectors/src/single_file/table.json b/crates/arroyo-connectors/src/single_file/table.json similarity index 100% rename from arroyo-connectors/src/single_file/table.json rename to crates/arroyo-connectors/src/single_file/table.json diff --git a/arroyo-connectors/src/sse/mod.rs b/crates/arroyo-connectors/src/sse/mod.rs similarity index 100% rename from arroyo-connectors/src/sse/mod.rs rename to crates/arroyo-connectors/src/sse/mod.rs diff --git a/arroyo-connectors/src/sse/operator.rs b/crates/arroyo-connectors/src/sse/operator.rs similarity index 100% rename from arroyo-connectors/src/sse/operator.rs rename to crates/arroyo-connectors/src/sse/operator.rs diff --git a/arroyo-connectors/src/sse/sse.svg b/crates/arroyo-connectors/src/sse/sse.svg similarity index 100% rename from arroyo-connectors/src/sse/sse.svg rename to crates/arroyo-connectors/src/sse/sse.svg diff --git a/arroyo-connectors/src/sse/table.json b/crates/arroyo-connectors/src/sse/table.json similarity index 100% rename from arroyo-connectors/src/sse/table.json rename to crates/arroyo-connectors/src/sse/table.json diff --git a/arroyo-connectors/src/webhook/mod.rs b/crates/arroyo-connectors/src/webhook/mod.rs similarity index 100% rename from arroyo-connectors/src/webhook/mod.rs rename to crates/arroyo-connectors/src/webhook/mod.rs diff --git a/arroyo-connectors/src/webhook/operator.rs b/crates/arroyo-connectors/src/webhook/operator.rs similarity index 100% rename from arroyo-connectors/src/webhook/operator.rs rename to crates/arroyo-connectors/src/webhook/operator.rs diff --git a/arroyo-connectors/src/webhook/table.json b/crates/arroyo-connectors/src/webhook/table.json similarity index 100% rename from arroyo-connectors/src/webhook/table.json rename to crates/arroyo-connectors/src/webhook/table.json diff --git a/arroyo-connectors/src/webhook/webhook.svg b/crates/arroyo-connectors/src/webhook/webhook.svg similarity index 100% rename from arroyo-connectors/src/webhook/webhook.svg rename to crates/arroyo-connectors/src/webhook/webhook.svg diff --git a/arroyo-connectors/src/websocket/mod.rs b/crates/arroyo-connectors/src/websocket/mod.rs similarity index 100% rename from arroyo-connectors/src/websocket/mod.rs rename to crates/arroyo-connectors/src/websocket/mod.rs diff --git a/arroyo-connectors/src/websocket/operator.rs b/crates/arroyo-connectors/src/websocket/operator.rs similarity index 100% rename from arroyo-connectors/src/websocket/operator.rs rename to crates/arroyo-connectors/src/websocket/operator.rs diff --git a/arroyo-connectors/src/websocket/table.json b/crates/arroyo-connectors/src/websocket/table.json similarity index 100% rename from arroyo-connectors/src/websocket/table.json rename to crates/arroyo-connectors/src/websocket/table.json diff --git a/arroyo-connectors/src/websocket/websocket.svg b/crates/arroyo-connectors/src/websocket/websocket.svg similarity index 100% rename from arroyo-connectors/src/websocket/websocket.svg rename to crates/arroyo-connectors/src/websocket/websocket.svg diff --git a/arroyo-controller/Cargo.toml b/crates/arroyo-controller/Cargo.toml similarity index 87% rename from arroyo-controller/Cargo.toml rename to crates/arroyo-controller/Cargo.toml index e9aadb3f0..8ffe7f25c 100644 --- a/arroyo-controller/Cargo.toml +++ b/crates/arroyo-controller/Cargo.toml @@ -6,7 +6,6 @@ edition = "2021" [features] default = [] kafka-sasl = [] -k8s = ["kube", "k8s-openapi", "serde_yaml"] [dependencies] arroyo-types = { path = "../arroyo-types" } @@ -43,9 +42,9 @@ serde = "1" anyhow = "1.0.70" # Kubernetes -kube = { version = "0.84", features = ["runtime", "derive"], optional = true } -k8s-openapi = { version = "0.18.0", features = ["v1_26"], optional = true } -serde_yaml = {version = "0.9", optional = true} +kube = { version = "0.84", features = ["runtime", "derive"] } +k8s-openapi = { version = "0.18.0", features = ["v1_26"] } +serde_yaml = {version = "0.9"} # json-schema support serde_json = "1.0" diff --git a/arroyo-controller/build.rs b/crates/arroyo-controller/build.rs similarity index 100% rename from arroyo-controller/build.rs rename to crates/arroyo-controller/build.rs diff --git a/arroyo-controller/queries/controller_queries.sql b/crates/arroyo-controller/queries/controller_queries.sql similarity index 97% rename from arroyo-controller/queries/controller_queries.sql rename to crates/arroyo-controller/queries/controller_queries.sql index 4a568ffb9..4d8574a6d 100644 --- a/arroyo-controller/queries/controller_queries.sql +++ b/crates/arroyo-controller/queries/controller_queries.sql @@ -38,7 +38,7 @@ SET state = :state, WHERE id = :job_id; --! get_program -SELECT program FROM pipelines WHERE id = :id; +SELECT program, proto_version FROM pipelines WHERE id = :id; --! mark_checkpoints_compacted UPDATE checkpoints diff --git a/arroyo-controller/src/compiler.rs b/crates/arroyo-controller/src/compiler.rs similarity index 100% rename from arroyo-controller/src/compiler.rs rename to crates/arroyo-controller/src/compiler.rs diff --git a/arroyo-controller/src/job_controller/checkpointer.rs b/crates/arroyo-controller/src/job_controller/checkpointer.rs similarity index 100% rename from arroyo-controller/src/job_controller/checkpointer.rs rename to crates/arroyo-controller/src/job_controller/checkpointer.rs diff --git a/arroyo-controller/src/job_controller/mod.rs b/crates/arroyo-controller/src/job_controller/mod.rs similarity index 100% rename from arroyo-controller/src/job_controller/mod.rs rename to crates/arroyo-controller/src/job_controller/mod.rs diff --git a/arroyo-controller/src/lib.rs b/crates/arroyo-controller/src/lib.rs similarity index 98% rename from arroyo-controller/src/lib.rs rename to crates/arroyo-controller/src/lib.rs index 6a99e3b22..981146d1c 100644 --- a/arroyo-controller/src/lib.rs +++ b/crates/arroyo-controller/src/lib.rs @@ -498,13 +498,8 @@ impl ControllerServer { Arc::new(NodeScheduler::new()) } Some("kubernetes") | Some("k8s") => { - #[cfg(feature = "k8s")] - { - info!("Using kubernetes scheduler"); - Arc::new(schedulers::kubernetes::KubernetesScheduler::from_env().await) - } - #[cfg(not(feature = "k8s"))] - panic!("Kubernetes not enabled -- compile with `--features k8s` to enable") + info!("Using kubernetes scheduler"); + Arc::new(schedulers::kubernetes::KubernetesScheduler::from_env().await) } Some("embedded") => { info!("Using embedded scheduler"); diff --git a/arroyo-controller/src/schedulers/embedded.rs b/crates/arroyo-controller/src/schedulers/embedded.rs similarity index 100% rename from arroyo-controller/src/schedulers/embedded.rs rename to crates/arroyo-controller/src/schedulers/embedded.rs diff --git a/arroyo-controller/src/schedulers/kubernetes.rs b/crates/arroyo-controller/src/schedulers/kubernetes.rs similarity index 98% rename from arroyo-controller/src/schedulers/kubernetes.rs rename to crates/arroyo-controller/src/schedulers/kubernetes.rs index 08f85abae..ca8a396b3 100644 --- a/arroyo-controller/src/schedulers/kubernetes.rs +++ b/crates/arroyo-controller/src/schedulers/kubernetes.rs @@ -63,10 +63,7 @@ impl KubernetesScheduler { client, namespace: string_config(K8S_NAMESPACE_ENV, "default"), name: format!("{}-worker", string_config(K8S_WORKER_NAME_ENV, "arroyo")), - image: string_config( - K8S_WORKER_IMAGE_ENV, - "ghcr.io/arroyosystems/arroyo-worker:latest", - ), + image: string_config(K8S_WORKER_IMAGE_ENV, "ghcr.io/arroyosystems/arroyo:latest"), labels: yaml_config(K8S_WORKER_LABELS_ENV, BTreeMap::new()), annotations: yaml_config(K8S_WORKER_ANNOTATIONS_ENV, BTreeMap::new()), image_pull_policy: string_config(K8S_WORKER_IMAGE_PULL_POLICY_ENV, "IfNotPresent"), @@ -194,6 +191,7 @@ impl KubernetesScheduler { { "name": "worker", "image": self.image, + "command": ["/app/arroyo-bin", "worker"], "imagePullPolicy": self.image_pull_policy, "resources": self.resources, "ports": [ diff --git a/arroyo-controller/src/schedulers/mod.rs b/crates/arroyo-controller/src/schedulers/mod.rs similarity index 99% rename from arroyo-controller/src/schedulers/mod.rs rename to crates/arroyo-controller/src/schedulers/mod.rs index affbc108f..53744b6c7 100644 --- a/arroyo-controller/src/schedulers/mod.rs +++ b/crates/arroyo-controller/src/schedulers/mod.rs @@ -25,9 +25,7 @@ use tokio::process::Command; use tokio::sync::{oneshot, Mutex}; use tonic::{Request, Status}; use tracing::{info, warn}; - pub mod embedded; -#[cfg(feature = "k8s")] pub mod kubernetes; lazy_static! { diff --git a/arroyo-controller/src/states/checkpoint_stopping.rs b/crates/arroyo-controller/src/states/checkpoint_stopping.rs similarity index 100% rename from arroyo-controller/src/states/checkpoint_stopping.rs rename to crates/arroyo-controller/src/states/checkpoint_stopping.rs diff --git a/arroyo-controller/src/states/compiling.rs b/crates/arroyo-controller/src/states/compiling.rs similarity index 100% rename from arroyo-controller/src/states/compiling.rs rename to crates/arroyo-controller/src/states/compiling.rs diff --git a/arroyo-controller/src/states/finishing.rs b/crates/arroyo-controller/src/states/finishing.rs similarity index 100% rename from arroyo-controller/src/states/finishing.rs rename to crates/arroyo-controller/src/states/finishing.rs diff --git a/arroyo-controller/src/states/mod.rs b/crates/arroyo-controller/src/states/mod.rs similarity index 89% rename from arroyo-controller/src/states/mod.rs rename to crates/arroyo-controller/src/states/mod.rs index c464b6df2..01bf4de5c 100644 --- a/arroyo-controller/src/states/mod.rs +++ b/crates/arroyo-controller/src/states/mod.rs @@ -13,7 +13,7 @@ use tokio::sync::mpsc::{channel, Receiver, Sender}; use tracing::{error, info, warn}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use crate::job_controller::JobController; use crate::queries::controller_queries; @@ -536,24 +536,13 @@ async fn execute_state<'a>( pub async fn run_to_completion( config: Arc>, + mut program: LogicalProgram, mut status: JobStatus, mut state: Box, pool: Pool, mut rx: Receiver, scheduler: Arc, ) { - let c = pool.get().await.unwrap(); - let id = config.read().unwrap().pipeline_id; - let mut program: LogicalProgram = { - let res = controller_queries::get_program() - .bind(&c, &id) - .one() - .await - .unwrap(); - - ArrowProgram::decode(&res[..]).unwrap().try_into().unwrap() - }; - let mut ctx = JobContext { config: config.read().unwrap().clone(), status: &mut status, @@ -606,6 +595,38 @@ impl StateMachine { this } + fn decode_program(bs: &[u8]) -> anyhow::Result { + Ok(ArrowProgram::decode(&bs[..]) + .map_err(|e| anyhow!("Failed to decode program: {:?}", e))? + .try_into() + .map_err(|e| anyhow!("Failed to construct graph from program: {:?}", e))?) + } + + async fn get_program( + pool: &Pool, + job_id: &str, + id: i64, + ) -> anyhow::Result> { + let c = pool.get().await?; + let res = controller_queries::get_program() + .bind(&c, &id) + .one() + .await + .map_err(|e| anyhow!("Failed to fetch program from database: {:?}", e))?; + + Ok(if res.proto_version == 2 { + match Self::decode_program(&res.program) { + Ok(p) => Some(p), + Err(e) => { + warn!("Failed to start {}: {}", job_id, e); + None + } + } + } else { + None + }) + } + async fn start(&mut self, mut status: JobStatus, shutdown_guard: ShutdownGuard) { if !self.done() { // we're already running, don't do anything @@ -647,12 +668,34 @@ impl StateMachine { let config = self.config.clone(); let pool = self.pool.clone(); let scheduler = self.scheduler.clone(); - shutdown_guard.into_spawn_task(async move { - let id = { config.read().unwrap().id.clone() }; - info!(message = "starting state machine", job_id = id); - run_to_completion(config, status, initial_state, pool, rx, scheduler).await; - info!(message = "finished state machine", job_id = id); - }); + + let pipeline_id = config.read().unwrap().pipeline_id; + match Self::get_program(&pool, &status.id, pipeline_id).await { + Ok(Some(program)) => { + shutdown_guard.into_spawn_task(async move { + let id = { config.read().unwrap().id.clone() }; + info!(message = "starting state machine", job_id = id); + run_to_completion( + config, + program, + status, + initial_state, + pool, + rx, + scheduler, + ) + .await; + info!(message = "finished state machine", job_id = id); + }); + } + Ok(None) => { + // this is a bad/old pipeline, skip it + } + Err(e) => { + // something went wrong, we'll retry on the next go around + warn!("Failed to start {}: {:?}", status.id, e); + } + } } self.tx = Some(tx); diff --git a/arroyo-controller/src/states/recovering.rs b/crates/arroyo-controller/src/states/recovering.rs similarity index 100% rename from arroyo-controller/src/states/recovering.rs rename to crates/arroyo-controller/src/states/recovering.rs diff --git a/arroyo-controller/src/states/rescaling.rs b/crates/arroyo-controller/src/states/rescaling.rs similarity index 100% rename from arroyo-controller/src/states/rescaling.rs rename to crates/arroyo-controller/src/states/rescaling.rs diff --git a/arroyo-controller/src/states/restarting.rs b/crates/arroyo-controller/src/states/restarting.rs similarity index 100% rename from arroyo-controller/src/states/restarting.rs rename to crates/arroyo-controller/src/states/restarting.rs diff --git a/arroyo-controller/src/states/running.rs b/crates/arroyo-controller/src/states/running.rs similarity index 100% rename from arroyo-controller/src/states/running.rs rename to crates/arroyo-controller/src/states/running.rs diff --git a/arroyo-controller/src/states/scheduling.rs b/crates/arroyo-controller/src/states/scheduling.rs similarity index 100% rename from arroyo-controller/src/states/scheduling.rs rename to crates/arroyo-controller/src/states/scheduling.rs diff --git a/arroyo-controller/src/states/stopping.rs b/crates/arroyo-controller/src/states/stopping.rs similarity index 100% rename from arroyo-controller/src/states/stopping.rs rename to crates/arroyo-controller/src/states/stopping.rs diff --git a/arroyo-datastream/Cargo.toml b/crates/arroyo-datastream/Cargo.toml similarity index 100% rename from arroyo-datastream/Cargo.toml rename to crates/arroyo-datastream/Cargo.toml diff --git a/arroyo-datastream/src/lib.rs b/crates/arroyo-datastream/src/lib.rs similarity index 100% rename from arroyo-datastream/src/lib.rs rename to crates/arroyo-datastream/src/lib.rs diff --git a/arroyo-datastream/src/logical.rs b/crates/arroyo-datastream/src/logical.rs similarity index 100% rename from arroyo-datastream/src/logical.rs rename to crates/arroyo-datastream/src/logical.rs diff --git a/arroyo-df/Cargo.toml b/crates/arroyo-df/Cargo.toml similarity index 100% rename from arroyo-df/Cargo.toml rename to crates/arroyo-df/Cargo.toml diff --git a/arroyo-df/src/external.rs b/crates/arroyo-df/src/external.rs similarity index 100% rename from arroyo-df/src/external.rs rename to crates/arroyo-df/src/external.rs diff --git a/arroyo-df/src/json.rs b/crates/arroyo-df/src/json.rs similarity index 100% rename from arroyo-df/src/json.rs rename to crates/arroyo-df/src/json.rs diff --git a/arroyo-df/src/lib.rs b/crates/arroyo-df/src/lib.rs similarity index 100% rename from arroyo-df/src/lib.rs rename to crates/arroyo-df/src/lib.rs diff --git a/arroyo-df/src/logical.rs b/crates/arroyo-df/src/logical.rs similarity index 100% rename from arroyo-df/src/logical.rs rename to crates/arroyo-df/src/logical.rs diff --git a/arroyo-df/src/physical.rs b/crates/arroyo-df/src/physical.rs similarity index 100% rename from arroyo-df/src/physical.rs rename to crates/arroyo-df/src/physical.rs diff --git a/arroyo-df/src/plan_graph.rs b/crates/arroyo-df/src/plan_graph.rs similarity index 100% rename from arroyo-df/src/plan_graph.rs rename to crates/arroyo-df/src/plan_graph.rs diff --git a/arroyo-df/src/rewriters.rs b/crates/arroyo-df/src/rewriters.rs similarity index 100% rename from arroyo-df/src/rewriters.rs rename to crates/arroyo-df/src/rewriters.rs diff --git a/arroyo-df/src/schemas.rs b/crates/arroyo-df/src/schemas.rs similarity index 100% rename from arroyo-df/src/schemas.rs rename to crates/arroyo-df/src/schemas.rs diff --git a/arroyo-df/src/tables.rs b/crates/arroyo-df/src/tables.rs similarity index 100% rename from arroyo-df/src/tables.rs rename to crates/arroyo-df/src/tables.rs diff --git a/arroyo-df/src/test.rs b/crates/arroyo-df/src/test.rs similarity index 100% rename from arroyo-df/src/test.rs rename to crates/arroyo-df/src/test.rs diff --git a/arroyo-df/src/types.rs b/crates/arroyo-df/src/types.rs similarity index 100% rename from arroyo-df/src/types.rs rename to crates/arroyo-df/src/types.rs diff --git a/arroyo-df/src/watermark_node.rs b/crates/arroyo-df/src/watermark_node.rs similarity index 100% rename from arroyo-df/src/watermark_node.rs rename to crates/arroyo-df/src/watermark_node.rs diff --git a/arroyo-formats/Cargo.toml b/crates/arroyo-formats/Cargo.toml similarity index 100% rename from arroyo-formats/Cargo.toml rename to crates/arroyo-formats/Cargo.toml diff --git a/arroyo-formats/src/avro.rs b/crates/arroyo-formats/src/avro.rs similarity index 100% rename from arroyo-formats/src/avro.rs rename to crates/arroyo-formats/src/avro.rs diff --git a/arroyo-formats/src/json.rs b/crates/arroyo-formats/src/json.rs similarity index 100% rename from arroyo-formats/src/json.rs rename to crates/arroyo-formats/src/json.rs diff --git a/arroyo-formats/src/json_schema.rs b/crates/arroyo-formats/src/json_schema.rs similarity index 100% rename from arroyo-formats/src/json_schema.rs rename to crates/arroyo-formats/src/json_schema.rs diff --git a/arroyo-formats/src/lib.rs b/crates/arroyo-formats/src/lib.rs similarity index 100% rename from arroyo-formats/src/lib.rs rename to crates/arroyo-formats/src/lib.rs diff --git a/arroyo-formats/src/serialize/mod.rs b/crates/arroyo-formats/src/serialize/mod.rs similarity index 100% rename from arroyo-formats/src/serialize/mod.rs rename to crates/arroyo-formats/src/serialize/mod.rs diff --git a/arroyo-macro/Cargo.toml b/crates/arroyo-macro/Cargo.toml similarity index 100% rename from arroyo-macro/Cargo.toml rename to crates/arroyo-macro/Cargo.toml diff --git a/arroyo-macro/src/lib.rs b/crates/arroyo-macro/src/lib.rs similarity index 100% rename from arroyo-macro/src/lib.rs rename to crates/arroyo-macro/src/lib.rs diff --git a/arroyo-metrics/Cargo.toml b/crates/arroyo-metrics/Cargo.toml similarity index 100% rename from arroyo-metrics/Cargo.toml rename to crates/arroyo-metrics/Cargo.toml diff --git a/arroyo-metrics/src/lib.rs b/crates/arroyo-metrics/src/lib.rs similarity index 100% rename from arroyo-metrics/src/lib.rs rename to crates/arroyo-metrics/src/lib.rs diff --git a/arroyo-node/Cargo.toml b/crates/arroyo-node/Cargo.toml similarity index 100% rename from arroyo-node/Cargo.toml rename to crates/arroyo-node/Cargo.toml diff --git a/arroyo-node/src/main.rs b/crates/arroyo-node/src/main.rs similarity index 100% rename from arroyo-node/src/main.rs rename to crates/arroyo-node/src/main.rs diff --git a/arroyo-openapi/.spectral.js b/crates/arroyo-openapi/.spectral.js similarity index 100% rename from arroyo-openapi/.spectral.js rename to crates/arroyo-openapi/.spectral.js diff --git a/arroyo-openapi/Cargo.toml b/crates/arroyo-openapi/Cargo.toml similarity index 100% rename from arroyo-openapi/Cargo.toml rename to crates/arroyo-openapi/Cargo.toml diff --git a/arroyo-openapi/build.rs b/crates/arroyo-openapi/build.rs similarity index 100% rename from arroyo-openapi/build.rs rename to crates/arroyo-openapi/build.rs diff --git a/arroyo-openapi/src/lib.rs b/crates/arroyo-openapi/src/lib.rs similarity index 100% rename from arroyo-openapi/src/lib.rs rename to crates/arroyo-openapi/src/lib.rs diff --git a/arroyo-operator/Cargo.toml b/crates/arroyo-operator/Cargo.toml similarity index 100% rename from arroyo-operator/Cargo.toml rename to crates/arroyo-operator/Cargo.toml diff --git a/arroyo-operator/src/connector.rs b/crates/arroyo-operator/src/connector.rs similarity index 100% rename from arroyo-operator/src/connector.rs rename to crates/arroyo-operator/src/connector.rs diff --git a/arroyo-operator/src/context.rs b/crates/arroyo-operator/src/context.rs similarity index 100% rename from arroyo-operator/src/context.rs rename to crates/arroyo-operator/src/context.rs diff --git a/arroyo-operator/src/inq_reader.rs b/crates/arroyo-operator/src/inq_reader.rs similarity index 100% rename from arroyo-operator/src/inq_reader.rs rename to crates/arroyo-operator/src/inq_reader.rs diff --git a/arroyo-operator/src/lib.rs b/crates/arroyo-operator/src/lib.rs similarity index 100% rename from arroyo-operator/src/lib.rs rename to crates/arroyo-operator/src/lib.rs diff --git a/arroyo-operator/src/metrics.rs b/crates/arroyo-operator/src/metrics.rs similarity index 100% rename from arroyo-operator/src/metrics.rs rename to crates/arroyo-operator/src/metrics.rs diff --git a/arroyo-operator/src/operator.rs b/crates/arroyo-operator/src/operator.rs similarity index 100% rename from arroyo-operator/src/operator.rs rename to crates/arroyo-operator/src/operator.rs diff --git a/arroyo-rpc/Cargo.toml b/crates/arroyo-rpc/Cargo.toml similarity index 100% rename from arroyo-rpc/Cargo.toml rename to crates/arroyo-rpc/Cargo.toml diff --git a/arroyo-rpc/build.rs b/crates/arroyo-rpc/build.rs similarity index 100% rename from arroyo-rpc/build.rs rename to crates/arroyo-rpc/build.rs diff --git a/arroyo-rpc/proto/api.proto b/crates/arroyo-rpc/proto/api.proto similarity index 100% rename from arroyo-rpc/proto/api.proto rename to crates/arroyo-rpc/proto/api.proto diff --git a/arroyo-rpc/proto/rpc.proto b/crates/arroyo-rpc/proto/rpc.proto similarity index 100% rename from arroyo-rpc/proto/rpc.proto rename to crates/arroyo-rpc/proto/rpc.proto diff --git a/arroyo-rpc/src/api_types/checkpoints.rs b/crates/arroyo-rpc/src/api_types/checkpoints.rs similarity index 100% rename from arroyo-rpc/src/api_types/checkpoints.rs rename to crates/arroyo-rpc/src/api_types/checkpoints.rs diff --git a/arroyo-rpc/src/api_types/connections.rs b/crates/arroyo-rpc/src/api_types/connections.rs similarity index 100% rename from arroyo-rpc/src/api_types/connections.rs rename to crates/arroyo-rpc/src/api_types/connections.rs diff --git a/arroyo-rpc/src/api_types/metrics.rs b/crates/arroyo-rpc/src/api_types/metrics.rs similarity index 100% rename from arroyo-rpc/src/api_types/metrics.rs rename to crates/arroyo-rpc/src/api_types/metrics.rs diff --git a/arroyo-rpc/src/api_types/mod.rs b/crates/arroyo-rpc/src/api_types/mod.rs similarity index 100% rename from arroyo-rpc/src/api_types/mod.rs rename to crates/arroyo-rpc/src/api_types/mod.rs diff --git a/arroyo-rpc/src/api_types/pipelines.rs b/crates/arroyo-rpc/src/api_types/pipelines.rs similarity index 100% rename from arroyo-rpc/src/api_types/pipelines.rs rename to crates/arroyo-rpc/src/api_types/pipelines.rs diff --git a/arroyo-rpc/src/api_types/udfs.rs b/crates/arroyo-rpc/src/api_types/udfs.rs similarity index 100% rename from arroyo-rpc/src/api_types/udfs.rs rename to crates/arroyo-rpc/src/api_types/udfs.rs diff --git a/arroyo-rpc/src/df.rs b/crates/arroyo-rpc/src/df.rs similarity index 100% rename from arroyo-rpc/src/df.rs rename to crates/arroyo-rpc/src/df.rs diff --git a/arroyo-rpc/src/formats.rs b/crates/arroyo-rpc/src/formats.rs similarity index 100% rename from arroyo-rpc/src/formats.rs rename to crates/arroyo-rpc/src/formats.rs diff --git a/arroyo-rpc/src/lib.rs b/crates/arroyo-rpc/src/lib.rs similarity index 100% rename from arroyo-rpc/src/lib.rs rename to crates/arroyo-rpc/src/lib.rs diff --git a/arroyo-rpc/src/public_ids.rs b/crates/arroyo-rpc/src/public_ids.rs similarity index 100% rename from arroyo-rpc/src/public_ids.rs rename to crates/arroyo-rpc/src/public_ids.rs diff --git a/arroyo-rpc/src/schema_resolver.rs b/crates/arroyo-rpc/src/schema_resolver.rs similarity index 100% rename from arroyo-rpc/src/schema_resolver.rs rename to crates/arroyo-rpc/src/schema_resolver.rs diff --git a/arroyo-rpc/src/var_str.rs b/crates/arroyo-rpc/src/var_str.rs similarity index 100% rename from arroyo-rpc/src/var_str.rs rename to crates/arroyo-rpc/src/var_str.rs diff --git a/arroyo-server-common/Cargo.toml b/crates/arroyo-server-common/Cargo.toml similarity index 97% rename from arroyo-server-common/Cargo.toml rename to crates/arroyo-server-common/Cargo.toml index 07a1af77c..2d8b4e0ee 100644 --- a/arroyo-server-common/Cargo.toml +++ b/crates/arroyo-server-common/Cargo.toml @@ -11,6 +11,7 @@ tracing = "0.1" tracing-logfmt = "0.2.0" tracing-subscriber = {version = "0.3", features = [ "env-filter" ]} tracing-appender = "0.2" +tracing-log = "0.2" # middleware tower = "0.4" diff --git a/arroyo-server-common/build.rs b/crates/arroyo-server-common/build.rs similarity index 100% rename from arroyo-server-common/build.rs rename to crates/arroyo-server-common/build.rs diff --git a/arroyo-server-common/src/lib.rs b/crates/arroyo-server-common/src/lib.rs similarity index 98% rename from arroyo-server-common/src/lib.rs rename to crates/arroyo-server-common/src/lib.rs index d22f68af0..8f736e3f0 100644 --- a/arroyo-server-common/src/lib.rs +++ b/crates/arroyo-server-common/src/lib.rs @@ -36,6 +36,7 @@ use tracing_subscriber::EnvFilter; use tracing_subscriber::Registry; use tracing_appender::non_blocking::WorkerGuard; +use tracing_log::LogTracer; pub const BUILD_TIMESTAMP: &str = env!("VERGEN_BUILD_TIMESTAMP"); pub const GIT_SHA: &str = env!("VERGEN_GIT_SHA"); @@ -47,6 +48,10 @@ const PYROSCOPE_SERVER_ADDRESS_ENV: &str = "PYROSCOPE_SERVER_ADDRESS"; static CLUSTER_ID: OnceCell = OnceCell::new(); pub fn init_logging(name: &str) -> Option { + if let Err(e) = LogTracer::init() { + eprintln!("Failed to initialize log tracer {:?}", e); + } + let stdout_log = tracing_subscriber::fmt::layer() .with_line_number(false) .with_file(false) diff --git a/arroyo-server-common/src/shutdown.rs b/crates/arroyo-server-common/src/shutdown.rs similarity index 100% rename from arroyo-server-common/src/shutdown.rs rename to crates/arroyo-server-common/src/shutdown.rs diff --git a/arroyo-sql-testing/Cargo.toml b/crates/arroyo-sql-testing/Cargo.toml similarity index 100% rename from arroyo-sql-testing/Cargo.toml rename to crates/arroyo-sql-testing/Cargo.toml diff --git a/arroyo-sql-testing/golden_outputs/aggregates.json b/crates/arroyo-sql-testing/golden_outputs/aggregates.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/aggregates.json rename to crates/arroyo-sql-testing/golden_outputs/aggregates.json diff --git a/arroyo-sql-testing/golden_outputs/async_udf.json b/crates/arroyo-sql-testing/golden_outputs/async_udf.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/async_udf.json rename to crates/arroyo-sql-testing/golden_outputs/async_udf.json diff --git a/arroyo-sql-testing/golden_outputs/double_negative_udf.json b/crates/arroyo-sql-testing/golden_outputs/double_negative_udf.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/double_negative_udf.json rename to crates/arroyo-sql-testing/golden_outputs/double_negative_udf.json diff --git a/arroyo-sql-testing/golden_outputs/filter_updating_aggregates.json b/crates/arroyo-sql-testing/golden_outputs/filter_updating_aggregates.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/filter_updating_aggregates.json rename to crates/arroyo-sql-testing/golden_outputs/filter_updating_aggregates.json diff --git a/arroyo-sql-testing/golden_outputs/hourly_by_event_type.json b/crates/arroyo-sql-testing/golden_outputs/hourly_by_event_type.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/hourly_by_event_type.json rename to crates/arroyo-sql-testing/golden_outputs/hourly_by_event_type.json diff --git a/arroyo-sql-testing/golden_outputs/month_loose_watermark.json b/crates/arroyo-sql-testing/golden_outputs/month_loose_watermark.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/month_loose_watermark.json rename to crates/arroyo-sql-testing/golden_outputs/month_loose_watermark.json diff --git a/arroyo-sql-testing/golden_outputs/most_active_driver_last_hour.json b/crates/arroyo-sql-testing/golden_outputs/most_active_driver_last_hour.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/most_active_driver_last_hour.json rename to crates/arroyo-sql-testing/golden_outputs/most_active_driver_last_hour.json diff --git a/arroyo-sql-testing/golden_outputs/most_active_driver_last_hour_unaligned.json b/crates/arroyo-sql-testing/golden_outputs/most_active_driver_last_hour_unaligned.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/most_active_driver_last_hour_unaligned.json rename to crates/arroyo-sql-testing/golden_outputs/most_active_driver_last_hour_unaligned.json diff --git a/arroyo-sql-testing/golden_outputs/offset_impulse_join.json b/crates/arroyo-sql-testing/golden_outputs/offset_impulse_join.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/offset_impulse_join.json rename to crates/arroyo-sql-testing/golden_outputs/offset_impulse_join.json diff --git a/arroyo-sql-testing/golden_outputs/outer_join.json b/crates/arroyo-sql-testing/golden_outputs/outer_join.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/outer_join.json rename to crates/arroyo-sql-testing/golden_outputs/outer_join.json diff --git a/arroyo-sql-testing/golden_outputs/select_star.json b/crates/arroyo-sql-testing/golden_outputs/select_star.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/select_star.json rename to crates/arroyo-sql-testing/golden_outputs/select_star.json diff --git a/arroyo-sql-testing/golden_outputs/session_window.json b/crates/arroyo-sql-testing/golden_outputs/session_window.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/session_window.json rename to crates/arroyo-sql-testing/golden_outputs/session_window.json diff --git a/arroyo-sql-testing/golden_outputs/suspicious_drivers.json b/crates/arroyo-sql-testing/golden_outputs/suspicious_drivers.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/suspicious_drivers.json rename to crates/arroyo-sql-testing/golden_outputs/suspicious_drivers.json diff --git a/arroyo-sql-testing/golden_outputs/tight_watermark.json b/crates/arroyo-sql-testing/golden_outputs/tight_watermark.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/tight_watermark.json rename to crates/arroyo-sql-testing/golden_outputs/tight_watermark.json diff --git a/arroyo-sql-testing/golden_outputs/udaf.json b/crates/arroyo-sql-testing/golden_outputs/udaf.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/udaf.json rename to crates/arroyo-sql-testing/golden_outputs/udaf.json diff --git a/arroyo-sql-testing/golden_outputs/union.json b/crates/arroyo-sql-testing/golden_outputs/union.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/union.json rename to crates/arroyo-sql-testing/golden_outputs/union.json diff --git a/arroyo-sql-testing/golden_outputs/updating_full_join.json b/crates/arroyo-sql-testing/golden_outputs/updating_full_join.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/updating_full_join.json rename to crates/arroyo-sql-testing/golden_outputs/updating_full_join.json diff --git a/arroyo-sql-testing/golden_outputs/updating_inner_join.json b/crates/arroyo-sql-testing/golden_outputs/updating_inner_join.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/updating_inner_join.json rename to crates/arroyo-sql-testing/golden_outputs/updating_inner_join.json diff --git a/arroyo-sql-testing/golden_outputs/updating_left_join.json b/crates/arroyo-sql-testing/golden_outputs/updating_left_join.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/updating_left_join.json rename to crates/arroyo-sql-testing/golden_outputs/updating_left_join.json diff --git a/arroyo-sql-testing/golden_outputs/updating_right_join.json b/crates/arroyo-sql-testing/golden_outputs/updating_right_join.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/updating_right_join.json rename to crates/arroyo-sql-testing/golden_outputs/updating_right_join.json diff --git a/arroyo-sql-testing/golden_outputs/windowed_inner_join.json b/crates/arroyo-sql-testing/golden_outputs/windowed_inner_join.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/windowed_inner_join.json rename to crates/arroyo-sql-testing/golden_outputs/windowed_inner_join.json diff --git a/arroyo-sql-testing/golden_outputs/windowed_outer_join.json b/crates/arroyo-sql-testing/golden_outputs/windowed_outer_join.json similarity index 100% rename from arroyo-sql-testing/golden_outputs/windowed_outer_join.json rename to crates/arroyo-sql-testing/golden_outputs/windowed_outer_join.json diff --git a/arroyo-sql-testing/inputs/cars.json b/crates/arroyo-sql-testing/inputs/cars.json similarity index 100% rename from arroyo-sql-testing/inputs/cars.json rename to crates/arroyo-sql-testing/inputs/cars.json diff --git a/arroyo-sql-testing/inputs/impulse.json b/crates/arroyo-sql-testing/inputs/impulse.json similarity index 100% rename from arroyo-sql-testing/inputs/impulse.json rename to crates/arroyo-sql-testing/inputs/impulse.json diff --git a/arroyo-sql-testing/inputs/sorted_cars.json b/crates/arroyo-sql-testing/inputs/sorted_cars.json similarity index 100% rename from arroyo-sql-testing/inputs/sorted_cars.json rename to crates/arroyo-sql-testing/inputs/sorted_cars.json diff --git a/arroyo-sql-testing/src/full_query_tests.rs b/crates/arroyo-sql-testing/src/full_query_tests.rs similarity index 100% rename from arroyo-sql-testing/src/full_query_tests.rs rename to crates/arroyo-sql-testing/src/full_query_tests.rs diff --git a/arroyo-sql-testing/src/lib.rs b/crates/arroyo-sql-testing/src/lib.rs similarity index 100% rename from arroyo-sql-testing/src/lib.rs rename to crates/arroyo-sql-testing/src/lib.rs diff --git a/arroyo-sql-testing/src/smoke_tests.rs b/crates/arroyo-sql-testing/src/smoke_tests.rs similarity index 100% rename from arroyo-sql-testing/src/smoke_tests.rs rename to crates/arroyo-sql-testing/src/smoke_tests.rs diff --git a/arroyo-state/Cargo.toml b/crates/arroyo-state/Cargo.toml similarity index 100% rename from arroyo-state/Cargo.toml rename to crates/arroyo-state/Cargo.toml diff --git a/arroyo-state/src/checkpoint_state.rs b/crates/arroyo-state/src/checkpoint_state.rs similarity index 100% rename from arroyo-state/src/checkpoint_state.rs rename to crates/arroyo-state/src/checkpoint_state.rs diff --git a/arroyo-state/src/committing_state.rs b/crates/arroyo-state/src/committing_state.rs similarity index 100% rename from arroyo-state/src/committing_state.rs rename to crates/arroyo-state/src/committing_state.rs diff --git a/arroyo-state/src/lib.rs b/crates/arroyo-state/src/lib.rs similarity index 100% rename from arroyo-state/src/lib.rs rename to crates/arroyo-state/src/lib.rs diff --git a/arroyo-state/src/metrics.rs b/crates/arroyo-state/src/metrics.rs similarity index 100% rename from arroyo-state/src/metrics.rs rename to crates/arroyo-state/src/metrics.rs diff --git a/arroyo-state/src/parquet.rs b/crates/arroyo-state/src/parquet.rs similarity index 100% rename from arroyo-state/src/parquet.rs rename to crates/arroyo-state/src/parquet.rs diff --git a/arroyo-state/src/schemas/mod.rs b/crates/arroyo-state/src/schemas/mod.rs similarity index 100% rename from arroyo-state/src/schemas/mod.rs rename to crates/arroyo-state/src/schemas/mod.rs diff --git a/arroyo-state/src/tables/expiring_time_key_map.rs b/crates/arroyo-state/src/tables/expiring_time_key_map.rs similarity index 100% rename from arroyo-state/src/tables/expiring_time_key_map.rs rename to crates/arroyo-state/src/tables/expiring_time_key_map.rs diff --git a/arroyo-state/src/tables/global_keyed_map.rs b/crates/arroyo-state/src/tables/global_keyed_map.rs similarity index 100% rename from arroyo-state/src/tables/global_keyed_map.rs rename to crates/arroyo-state/src/tables/global_keyed_map.rs diff --git a/arroyo-state/src/tables/mod.rs b/crates/arroyo-state/src/tables/mod.rs similarity index 100% rename from arroyo-state/src/tables/mod.rs rename to crates/arroyo-state/src/tables/mod.rs diff --git a/arroyo-state/src/tables/table_manager.rs b/crates/arroyo-state/src/tables/table_manager.rs similarity index 100% rename from arroyo-state/src/tables/table_manager.rs rename to crates/arroyo-state/src/tables/table_manager.rs diff --git a/arroyo-storage/Cargo.toml b/crates/arroyo-storage/Cargo.toml similarity index 100% rename from arroyo-storage/Cargo.toml rename to crates/arroyo-storage/Cargo.toml diff --git a/arroyo-storage/src/aws.rs b/crates/arroyo-storage/src/aws.rs similarity index 100% rename from arroyo-storage/src/aws.rs rename to crates/arroyo-storage/src/aws.rs diff --git a/arroyo-storage/src/lib.rs b/crates/arroyo-storage/src/lib.rs similarity index 100% rename from arroyo-storage/src/lib.rs rename to crates/arroyo-storage/src/lib.rs diff --git a/arroyo-types/Cargo.toml b/crates/arroyo-types/Cargo.toml similarity index 100% rename from arroyo-types/Cargo.toml rename to crates/arroyo-types/Cargo.toml diff --git a/arroyo-types/src/lib.rs b/crates/arroyo-types/src/lib.rs similarity index 99% rename from arroyo-types/src/lib.rs rename to crates/arroyo-types/src/lib.rs index 3e4d900e5..e3e50dfff 100644 --- a/arroyo-types/src/lib.rs +++ b/crates/arroyo-types/src/lib.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::convert::TryFrom; use std::env; -use std::fmt::Debug; +use std::fmt::{Debug, Display, Formatter}; use std::hash::Hash; use std::ops::{Range, RangeInclusive}; use std::str::FromStr; @@ -186,6 +186,12 @@ pub struct DatabaseConfig { pub password: String, } +impl Display for DatabaseConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}@{}:{}/{}", self.user, self.host, self.port, self.name) + } +} + impl DatabaseConfig { pub fn load() -> Self { DatabaseConfig { diff --git a/arroyo-worker/Cargo.lock b/crates/arroyo-worker/Cargo.lock similarity index 100% rename from arroyo-worker/Cargo.lock rename to crates/arroyo-worker/Cargo.lock diff --git a/arroyo-worker/Cargo.toml b/crates/arroyo-worker/Cargo.toml similarity index 100% rename from arroyo-worker/Cargo.toml rename to crates/arroyo-worker/Cargo.toml diff --git a/arroyo-worker/build.rs b/crates/arroyo-worker/build.rs similarity index 100% rename from arroyo-worker/build.rs rename to crates/arroyo-worker/build.rs diff --git a/arroyo-worker/src/arrow/join_with_expiration.rs b/crates/arroyo-worker/src/arrow/join_with_expiration.rs similarity index 100% rename from arroyo-worker/src/arrow/join_with_expiration.rs rename to crates/arroyo-worker/src/arrow/join_with_expiration.rs diff --git a/arroyo-worker/src/arrow/mod.rs b/crates/arroyo-worker/src/arrow/mod.rs similarity index 100% rename from arroyo-worker/src/arrow/mod.rs rename to crates/arroyo-worker/src/arrow/mod.rs diff --git a/arroyo-worker/src/arrow/session_aggregating_window.rs b/crates/arroyo-worker/src/arrow/session_aggregating_window.rs similarity index 100% rename from arroyo-worker/src/arrow/session_aggregating_window.rs rename to crates/arroyo-worker/src/arrow/session_aggregating_window.rs diff --git a/arroyo-worker/src/arrow/sliding_aggregating_window.rs b/crates/arroyo-worker/src/arrow/sliding_aggregating_window.rs similarity index 100% rename from arroyo-worker/src/arrow/sliding_aggregating_window.rs rename to crates/arroyo-worker/src/arrow/sliding_aggregating_window.rs diff --git a/arroyo-worker/src/arrow/sync/mod.rs b/crates/arroyo-worker/src/arrow/sync/mod.rs similarity index 100% rename from arroyo-worker/src/arrow/sync/mod.rs rename to crates/arroyo-worker/src/arrow/sync/mod.rs diff --git a/arroyo-worker/src/arrow/sync/streams.rs b/crates/arroyo-worker/src/arrow/sync/streams.rs similarity index 100% rename from arroyo-worker/src/arrow/sync/streams.rs rename to crates/arroyo-worker/src/arrow/sync/streams.rs diff --git a/arroyo-worker/src/arrow/tumbling_aggregating_window.rs b/crates/arroyo-worker/src/arrow/tumbling_aggregating_window.rs similarity index 100% rename from arroyo-worker/src/arrow/tumbling_aggregating_window.rs rename to crates/arroyo-worker/src/arrow/tumbling_aggregating_window.rs diff --git a/arroyo-worker/src/connectors/filesystem/source/mod.rs b/crates/arroyo-worker/src/connectors/filesystem/source/mod.rs similarity index 100% rename from arroyo-worker/src/connectors/filesystem/source/mod.rs rename to crates/arroyo-worker/src/connectors/filesystem/source/mod.rs diff --git a/arroyo-worker/src/connectors/kinesis/mod.rs b/crates/arroyo-worker/src/connectors/kinesis/mod.rs similarity index 100% rename from arroyo-worker/src/connectors/kinesis/mod.rs rename to crates/arroyo-worker/src/connectors/kinesis/mod.rs diff --git a/arroyo-worker/src/connectors/mod.rs b/crates/arroyo-worker/src/connectors/mod.rs similarity index 100% rename from arroyo-worker/src/connectors/mod.rs rename to crates/arroyo-worker/src/connectors/mod.rs diff --git a/arroyo-worker/src/connectors/nexmark/mod.rs b/crates/arroyo-worker/src/connectors/nexmark/mod.rs similarity index 100% rename from arroyo-worker/src/connectors/nexmark/mod.rs rename to crates/arroyo-worker/src/connectors/nexmark/mod.rs diff --git a/arroyo-worker/src/connectors/nexmark/test.rs b/crates/arroyo-worker/src/connectors/nexmark/test.rs similarity index 100% rename from arroyo-worker/src/connectors/nexmark/test.rs rename to crates/arroyo-worker/src/connectors/nexmark/test.rs diff --git a/arroyo-worker/src/engine.rs b/crates/arroyo-worker/src/engine.rs similarity index 100% rename from arroyo-worker/src/engine.rs rename to crates/arroyo-worker/src/engine.rs diff --git a/arroyo-worker/src/lib.rs b/crates/arroyo-worker/src/lib.rs similarity index 100% rename from arroyo-worker/src/lib.rs rename to crates/arroyo-worker/src/lib.rs diff --git a/arroyo-worker/src/network_manager.rs b/crates/arroyo-worker/src/network_manager.rs similarity index 100% rename from arroyo-worker/src/network_manager.rs rename to crates/arroyo-worker/src/network_manager.rs diff --git a/arroyo-worker/src/operators/async_map.rs b/crates/arroyo-worker/src/operators/async_map.rs similarity index 100% rename from arroyo-worker/src/operators/async_map.rs rename to crates/arroyo-worker/src/operators/async_map.rs diff --git a/arroyo-worker/src/operators/mod.rs b/crates/arroyo-worker/src/operators/mod.rs similarity index 100% rename from arroyo-worker/src/operators/mod.rs rename to crates/arroyo-worker/src/operators/mod.rs diff --git a/arroyo-worker/src/operators/watermark_generator.rs b/crates/arroyo-worker/src/operators/watermark_generator.rs similarity index 100% rename from arroyo-worker/src/operators/watermark_generator.rs rename to crates/arroyo-worker/src/operators/watermark_generator.rs diff --git a/arroyo/Cargo.toml b/crates/arroyo/Cargo.toml similarity index 100% rename from arroyo/Cargo.toml rename to crates/arroyo/Cargo.toml diff --git a/arroyo/src/main.rs b/crates/arroyo/src/main.rs similarity index 100% rename from arroyo/src/main.rs rename to crates/arroyo/src/main.rs diff --git a/connector-schemas/nexmark/table.json b/crates/connector-schemas/nexmark/table.json similarity index 100% rename from connector-schemas/nexmark/table.json rename to crates/connector-schemas/nexmark/table.json diff --git a/copy-artifacts/.gitignore b/crates/copy-artifacts/.gitignore similarity index 100% rename from copy-artifacts/.gitignore rename to crates/copy-artifacts/.gitignore diff --git a/copy-artifacts/Cargo.toml b/crates/copy-artifacts/Cargo.toml similarity index 100% rename from copy-artifacts/Cargo.toml rename to crates/copy-artifacts/Cargo.toml diff --git a/copy-artifacts/src/main.rs b/crates/copy-artifacts/src/main.rs similarity index 100% rename from copy-artifacts/src/main.rs rename to crates/copy-artifacts/src/main.rs diff --git a/integ/Cargo.toml b/crates/integ/Cargo.toml similarity index 100% rename from integ/Cargo.toml rename to crates/integ/Cargo.toml diff --git a/integ/src/main.rs b/crates/integ/src/main.rs similarity index 100% rename from integ/src/main.rs rename to crates/integ/src/main.rs diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..67c2fcf4a --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,83 @@ +# syntax=docker/dockerfile:1.4 + +FROM rust:1-bookworm AS builder +WORKDIR /app +ARG GIT_SHA +ARG TARGETPLATFORM + +RUN apt-get update && \ + apt-get -y install curl pkg-config unzip build-essential libssl-dev openssl \ + cmake clang wget postgresql postgresql-client supervisor python3 python-is-python3 sudo bash libsasl2-dev && \ + cargo install refinery_cli + +# Install node & pnpm +RUN mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | \ + sudo tee /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && apt-get install -y nodejs && \ + curl -fsSL https://get.pnpm.io/install.sh | env PNPM_VERSION=8.15.1 SHELL=$(which bash) bash - + +COPY docker/install_deps.sh /install_deps.sh +RUN sh /install_deps.sh build + +COPY webui/pnpm-lock.yaml webui/pnpm-lock.yaml +RUN bash -c "cd webui && source ~/.bashrc && /root/.local/share/pnpm/pnpm fetch" + +COPY Cargo.toml Cargo.toml +COPY Cargo.lock Cargo.lock + +COPY crates crates +COPY webui webui + +# set git hash +ENV VERGEN_GIT_SHA=$GIT_SHA +RUN bash -c "cd webui && source ~/.bashrc && /root/.local/share/pnpm/pnpm install && /root/.local/share/pnpm/pnpm build" + +COPY docker/refinery.toml refinery.toml + +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=${TARGETPLATFORM} \ + --mount=type=cache,target=/app/target,id=${TARGETPLATFORM} \ + service postgresql start && \ + sudo -u postgres psql -c "CREATE USER arroyo WITH PASSWORD 'arroyo' SUPERUSER;" && \ + sudo -u postgres createdb arroyo && \ + refinery migrate -c refinery.toml -p crates/arroyo-api/migrations && \ + CARGO_PROFILE_RELEASE_DEBUG=false cargo build --release --bin arroyo-bin --all-features && \ + mv target/release/arroyo-bin /arroyo-bin + +FROM debian:bookworm-slim AS arroyo-single +WORKDIR /app +RUN apt-get update && \ + apt-get -y install libssl-dev openssl supervisor libsasl2-dev curl postgresql postgresql-client sudo + +COPY docker/install_deps.sh /install_deps.sh +RUN sh /install_deps.sh run + +COPY --from=builder /arroyo-bin ./ +COPY --from=builder /app/webui/dist ./dist + +COPY docker/supervisord.conf /supervisord.conf +COPY docker/entrypoint.sh /entrypoint.sh +COPY docker/prometheus.yml /etc/prometheus/prometheus.yml + +ENV PRODUCTION=true \ + ASSET_DIR="/app/dist" +RUN chmod +x /entrypoint.sh + +EXPOSE 8000 +ENTRYPOINT [ "/entrypoint.sh" ] + +FROM debian:bookworm-slim as arroyo +WORKDIR /app + +RUN apt-get update && \ + apt-get -y install libsasl2-2 ca-certificates + +COPY --from=builder /arroyo-bin ./ +COPY --from=builder /app/webui/dist ./dist + +ENV PRODUCTION=true +ENV ASSET_DIR="/app/dist" + +EXPOSE 8000 +ENTRYPOINT [ "/app/arroyo-bin" ] \ No newline at end of file diff --git a/docker/build_base/.gitignore b/docker/build_base/.gitignore deleted file mode 100644 index ae9f0d11c..000000000 --- a/docker/build_base/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -Cargo.lock -wasm-fns/Cargo.lock diff --git a/docker/build_base/Cargo.toml b/docker/build_base/Cargo.toml deleted file mode 100644 index 130619a87..000000000 --- a/docker/build_base/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[workspace] -resolver = "2" - -members = [ - "pipeline", - "types", -] - -exclude = [ - "wasm-fns", -] -[workspace.dependencies] -arrow = { version = "46.0.0" } -arrow-buffer = { version = "46.0.0" } -arrow-array = { version = "46.0.0" } -arrow-schema = { version = "46.0.0" } -parquet = { version = "46.0.0" } - - -[patch.crates-io] -parquet = {git = 'https://github.com/ArroyoSystems/arrow-rs', branch = '46.0.0/parquet_bytes'} -arrow = {git = 'https://github.com/ArroyoSystems/arrow-rs', branch = '46.0.0/parquet_bytes'} -arrow-buffer = {git = 'https://github.com/ArroyoSystems/arrow-rs', branch = '46.0.0/parquet_bytes'} -arrow-array = {git = 'https://github.com/ArroyoSystems/arrow-rs', branch = '46.0.0/parquet_bytes'} -arrow-schema = {git = 'https://github.com/ArroyoSystems/arrow-rs', branch = '46.0.0/parquet_bytes'} -object_store = {git = 'https://github.com/ArroyoSystems/arrow-rs', branch = 'object_store/put_part_api'} diff --git a/docker/build_base/pipeline/Cargo.toml b/docker/build_base/pipeline/Cargo.toml deleted file mode 100644 index 44921df5f..000000000 --- a/docker/build_base/pipeline/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ - -[package] -name = "pipeline" -version = "1.0.0" -edition = "2021" - -[features] -default = [] -kafka-sasl = ["arroyo-worker/kafka-sasl"] - -[dependencies] -types = { path = "../types" } -udfs = { path = "../udfs_dir/udfs" } -petgraph = "0.6" -bincode = "=2.0.0-rc.3" -bincode_derive = "=2.0.0-rc.3" -chrono = "0.4" -serde = "1.0" -serde_json = "1.0" -anyhow = "1.0" -arrow = { workspace = true} -arrow-array = { workspace = true} -arroyo-types = { path = "/opt/arroyo/src/arroyo-types" } -arroyo-worker = { path = "/opt/arroyo/src/arroyo-worker", features = ["kafka-sasl"] } -arroyo-formats = { path = "/opt/arroyo/src/arroyo-formats" } -tokio = { version = "1", features = ["full", "tracing"] } - -[package.metadata.wasm-pack.profile.release] -wasm-opt = false diff --git a/docker/build_base/pipeline/src/main.rs b/docker/build_base/pipeline/src/main.rs deleted file mode 100644 index c27a5b89e..000000000 --- a/docker/build_base/pipeline/src/main.rs +++ /dev/null @@ -1,27 +0,0 @@ -use arroyo_worker::connectors; -use arroyo_worker::engine::{Program, SubtaskNode}; -use arroyo_worker::operators::joins::*; -use arroyo_worker::operators::sinks; -use arroyo_worker::operators::sinks::*; -use arroyo_worker::operators::windows::*; -use arroyo_worker::operators::*; -use arroyo_worker::{LogicalEdge, LogicalNode}; -use chrono; -use petgraph::graph::DiGraph; -use serde::{Deserialize, Serialize}; -use std::future::Future; -use std::str::FromStr; -use std::time::SystemTime; -use tokio::time::{timeout, Duration}; -use types::*; - -pub fn make_graph() -> DiGraph { - todo!(); -} - -pub fn main() { - let graph = make_graph(); - arroyo_worker::WorkerServer::new("test", "dpw9h0g9sczp7b2b", graph) - .start() - .unwrap(); -} diff --git a/docker/build_base/types/Cargo.toml b/docker/build_base/types/Cargo.toml deleted file mode 100644 index 32624cbd4..000000000 --- a/docker/build_base/types/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ - -[package] -name = "types" -version = "1.0.0" -edition = "2021" - -[dependencies] -bincode = "=2.0.0-rc.3" -bincode_derive = "=2.0.0-rc.3" -arroyo-types = { path = "/opt/arroyo/src/arroyo-types" } -[package.metadata.wasm-pack.profile.release] -wasm-opt = false diff --git a/docker/build_base/types/src/lib.rs b/docker/build_base/types/src/lib.rs deleted file mode 100644 index 8b1378917..000000000 --- a/docker/build_base/types/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/docker/build_base/udfs_dir/udfs/Cargo.toml b/docker/build_base/udfs_dir/udfs/Cargo.toml deleted file mode 100644 index d55fa3a2e..000000000 --- a/docker/build_base/udfs_dir/udfs/Cargo.toml +++ /dev/null @@ -1,4 +0,0 @@ -[package] -name = "udfs" -version = "1.0.0" -edition = "2021" diff --git a/docker/build_base/udfs_dir/udfs/src/lib.rs b/docker/build_base/udfs_dir/udfs/src/lib.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/docker/build_base/wasm-fns/Cargo.toml b/docker/build_base/wasm-fns/Cargo.toml deleted file mode 100644 index e5a4e16f0..000000000 --- a/docker/build_base/wasm-fns/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ - -[package] -name = "wasm-fns" -version = "1.0.0" -edition = "2021" - -[lib] -crate-type = ["cdylib"] - -[dependencies] -types = { path = "../types" } - -bincode = "=2.0.0-rc.3" -bincode_derive = "=2.0.0-rc.3" -arroyo-types = { path = "/opt/arroyo/src/arroyo-types" } -wasm-bindgen = "0.2" -serde_json = "1.0" - -[package.metadata.wasm-pack.profile.release] -wasm-opt = false diff --git a/docker/build_base/wasm-fns/src/lib.rs b/docker/build_base/wasm-fns/src/lib.rs deleted file mode 100644 index 35cbb7235..000000000 --- a/docker/build_base/wasm-fns/src/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -use std::time::SystemTime; -use types::*; -extern "C" { - pub fn send(s: *const u8, length: u32) -> i32; - pub fn fetch_data(s: *const u8) -> i32; -} -fn get_record(len: u32) -> arroyo_types::Record { - let mut data = vec![0u8; len as usize]; - unsafe { fetch_data(data.as_mut_ptr()) }; - let (record, _): (arroyo_types::Record, usize) = - bincode::decode_from_slice(&data[..], bincode::config::standard()).unwrap(); - record -} -fn send_record(record: arroyo_types::Record) { - let encoded = bincode::encode_to_vec(record, bincode::config::standard()).unwrap(); - unsafe { send(encoded.as_ptr(), encoded.len() as u32) }; -} diff --git a/docker/check_dockerfiles.rb b/docker/check_dockerfiles.rb deleted file mode 100755 index ab8579e95..000000000 --- a/docker/check_dockerfiles.rb +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env ruby - -# Checks that the Dockerfiles include all of the workspace members - -require 'toml-rb' - -DOCKERFILES = [ - "docker/cluster/compiler/Dockerfile", - "docker/cluster/services/Dockerfile", - "docker/cluster/worker/Dockerfile", - "docker/single/Dockerfile" -] - -# Get the current script directory -script_dir = File.dirname(__FILE__) - -# Parse the Cargo.toml file and get all of the workspace members -toml = TomlRB.load_file(File.join(script_dir, '..', 'Cargo.toml')) -members = toml['workspace']['members'] - -# Check that each Dockerfile includes all of the workspace members -DOCKERFILES.each do |dockerfile| - File.open(dockerfile, 'r') do |f| - body = f.read - members.each do |member| - if !body.include?(" #{member} ") - puts "ERROR: #{dockerfile} does not include #{member}" - exit 1 - end - end - end -end - -puts "All Dockerfiles include all workspace members" diff --git a/docker/cluster/compiler/Dockerfile b/docker/cluster/compiler/Dockerfile deleted file mode 100644 index 0eb9236a9..000000000 --- a/docker/cluster/compiler/Dockerfile +++ /dev/null @@ -1,72 +0,0 @@ -FROM rust:1-slim-bookworm -WORKDIR /opt/arroyo - -ARG PROTO_ARCH -ARG MOLD_ARCH -ARG GIT_SHA - -RUN apt-get update && \ - apt-get -y install curl git pkg-config unzip build-essential libssl-dev libsasl2-dev openssl cmake clang wget && \ - # Install mold - wget https://github.com/rui314/mold/releases/download/v1.11.0/mold-1.11.0-${MOLD_ARCH}-linux.tar.gz && \ - tar xvfz mold*.tar.gz && \ - mv mold*-linux/bin/* /usr/bin && \ - mv mold*-linux/libexec/* /usr/libexec && \ - rm -rf mold* && \ - # Install protoc - wget https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protoc-21.8-linux-${PROTO_ARCH}.zip && \ - unzip protoc*.zip && \ - mv bin/protoc /usr/local/bin && \ - cargo install wasm-pack - -COPY docker/config.toml $CARGO_HOME/config - -RUN mkdir -p query-compiler/src/ - -# copy the source code -COPY arroyo-types src/arroyo-types -COPY arroyo-worker src/arroyo-worker -COPY arroyo-rpc src/arroyo-rpc -COPY arroyo-macro src/arroyo-macro -COPY arroyo-server-common src/arroyo-server-common -COPY arroyo-state src/arroyo-state -COPY arroyo-metrics src/arroyo-metrics -COPY arroyo-compiler-service src/arroyo-compiler-service -COPY arroyo-connectors src/arroyo-connectors -COPY arroyo-api src/arroyo-api -COPY arroyo-datastream src/arroyo-datastream -COPY arroyo-node src/arroyo-node -COPY arroyo-openapi src/arroyo-openapi -COPY arroyo-operator src/arroyo-operator -COPY arroyo-df src/arroyo-df -COPY arroyo-sql-macro src/arroyo-sql-macro -COPY arroyo-sql-testing src/arroyo-sql-testing -COPY arroyo-controller src/arroyo-controller -COPY arroyo-console src/arroyo-console -COPY arroyo-storage src/arroyo-storage -COPY arroyo-formats src/arroyo-formats -COPY connector-schemas src/connector-schemas -COPY arroyo-bin src/arroyo-bin -COPY arroyo src/arroyo - -COPY copy-artifacts src/copy-artifacts -COPY integ src/integ - -COPY Cargo.toml src/Cargo.toml -COPY Cargo.lock src/Cargo.lock - -# set git hash -ENV VERGEN_GIT_SHA=$GIT_SHA -# build the compiler service -RUN cd src && cargo build --release -p arroyo-compiler-service && mv target/release/arroyo-compiler-service /usr/bin - -# Copy and build the base pipeline -COPY docker/build_base pipeline/ -RUN cd pipeline && cargo build --release && cargo build --release && \ - cd wasm-fns && wasm-pack build - -ENV BUILD_DIR=/opt/arroyo/pipeline -EXPOSE 9000 - -ENTRYPOINT [ "/usr/bin/arroyo-compiler-service" ] -CMD [ "start" ] diff --git a/docker/cluster/services/Dockerfile b/docker/cluster/services/Dockerfile deleted file mode 100644 index c05678842..000000000 --- a/docker/cluster/services/Dockerfile +++ /dev/null @@ -1,83 +0,0 @@ -FROM rust:1-bookworm AS builder -WORKDIR /app -ARG PROTO_ARCH -ARG GIT_SHA - -RUN apt-get update && \ - apt-get -y install curl pkg-config unzip build-essential libssl-dev openssl \ - cmake clang wget postgresql postgresql-client supervisor python3 python-is-python3 sudo bash libsasl2-dev && \ - cargo install refinery_cli && \ - cargo install wasm-pack && \ - wget https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protoc-21.8-linux-${PROTO_ARCH}.zip && \ - unzip protoc*.zip && \ - mv bin/protoc /usr/local/bin - -# Install node & pnpm -RUN mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | \ - sudo tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get update && apt-get install -y nodejs && \ - curl -fsSL https://get.pnpm.io/install.sh | env PNPM_VERSION=8.10.4 SHELL=$(which bash) bash - - - -COPY Cargo.toml Cargo.toml -COPY Cargo.lock Cargo.lock - -COPY arroyo-types arroyo-types -COPY arroyo-worker arroyo-worker -COPY arroyo-rpc arroyo-rpc -COPY arroyo-formats arroyo-formats -COPY arroyo-macro arroyo-macro -COPY arroyo-server-common arroyo-server-common -COPY arroyo-state arroyo-state -COPY arroyo-metrics arroyo-metrics -COPY arroyo-compiler-service arroyo-compiler-service -COPY arroyo-connectors arroyo-connectors -COPY arroyo-api arroyo-api -COPY arroyo-datastream arroyo-datastream -COPY arroyo-node arroyo-node -COPY arroyo-openapi arroyo-openapi -COPY arroyo-operator arroyo-operator -COPY arroyo-df arroyo-df -COPY arroyo-sql-macro arroyo-sql-macro -COPY arroyo-sql-testing arroyo-sql-testing -COPY arroyo-controller arroyo-controller -COPY arroyo-console arroyo-console -COPY arroyo-openapi arroyo-openap -COPY arroyo-storage arroyo-storage -COPY docker/refinery.toml refinery.toml -COPY connector-schemas connector-schemas -COPY arroyo-bin arroyo-bin -COPY arroyo arroyo - -COPY copy-artifacts copy-artifacts -COPY integ integ - -# set git hash -ENV VERGEN_GIT_SHA=$GIT_SHA -RUN service postgresql start && \ - sudo -u postgres psql -c "CREATE USER arroyo WITH PASSWORD 'arroyo' SUPERUSER;" && \ - sudo -u postgres createdb arroyo && \ - refinery migrate -c refinery.toml -p arroyo-api/migrations && \ - CARGO_PROFILE_RELEASE_DEBUG=false cargo build --release --bin arroyo-api --bin arroyo-controller --all-features && \ - bash -c "cd arroyo-console && source ~/.bashrc && /root/.local/share/pnpm/pnpm install && /root/.local/share/pnpm/pnpm build" && \ - cp "$(which refinery)" /refinery - -FROM debian:bookworm-slim AS runner -WORKDIR /app -RUN apt-get update && \ - apt-get -y install libssl-dev openssl supervisor libsasl2-dev -COPY --from=builder /app/target/release/arroyo-api /app/target/release/arroyo-controller ./ -COPY --from=builder /app/arroyo-console/dist ./dist -COPY --from=builder /app/arroyo-api/migrations /migrations -COPY --from=builder /refinery /usr/local/bin/refinery - -COPY docker/cluster/services/supervisord.conf /supervisord.conf -COPY docker/cluster/services/entrypoint.sh /entrypoint.sh - -ENV PRODUCTION=true \ - ASSET_DIR="/app/dist" -RUN chmod +x /entrypoint.sh - -ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/docker/cluster/services/entrypoint.sh b/docker/cluster/services/entrypoint.sh deleted file mode 100644 index c981ff2f8..000000000 --- a/docker/cluster/services/entrypoint.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Run the migrations -export DATABASE_URL="postgres://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}" - -refinery migrate -e DATABASE_URL -p /migrations - -# start the services - -# if the arg is api or controller, run the respective service, otherwise run supervisord - -if [ "$1" = "api" ]; then - ASSET_DIR=/app/dist /app/arroyo-api -elif [ "$1" = "controller" ]; then - /app/arroyo-controller -elif [ "$1" = "" ]; then - /usr/bin/supervisord -c /supervisord.conf -fi diff --git a/docker/cluster/services/supervisord.conf b/docker/cluster/services/supervisord.conf deleted file mode 100644 index 27bfc2e31..000000000 --- a/docker/cluster/services/supervisord.conf +++ /dev/null @@ -1,19 +0,0 @@ -[supervisord] -nodaemon=true - -[program:api] -command=/app/arroyo-api -autostart=true -autorestart=true -stderr_logfile=/var/log/supervisor/api.err.log -stdout_logfile=/dev/fd/1 -stdout_logfile_maxbytes=0 -environment=ASSET_DIR="/app/dist" - -[program:controller] -command=/app/arroyo-controller -autostart=true -autorestart=true -stderr_logfile=/var/log/supervisor/controller.err.log -stdout_logfile=/dev/fd/1 -stdout_logfile_maxbytes=0 diff --git a/docker/cluster/worker/Dockerfile b/docker/cluster/worker/Dockerfile deleted file mode 100644 index 5eaae9abe..000000000 --- a/docker/cluster/worker/Dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -FROM rust:1-slim-bookworm as builder -ARG GIT_SHA - -RUN apt-get update && \ - apt-get -y install libssl-dev libsasl2-dev openssl pkg-config - -WORKDIR /build_dir - -COPY arroyo-types arroyo-types -COPY arroyo-worker arroyo-worker -COPY arroyo-rpc arroyo-rpc -COPY arroyo-macro arroyo-macro -COPY arroyo-formats arroyo-formats -COPY arroyo-server-common arroyo-server-common -COPY arroyo-state arroyo-state -COPY arroyo-metrics arroyo-metrics -COPY arroyo-compiler-service arroyo-compiler-service -COPY arroyo-connectors arroyo-connectors -COPY arroyo-api arroyo-api -COPY arroyo-datastream arroyo-datastream -COPY arroyo-node arroyo-node -COPY arroyo-openapi arroyo-openapi -COPY arroyo-operator arroyo-operator -COPY arroyo-df arroyo-df -COPY arroyo-sql-macro arroyo-sql-macro -COPY arroyo-sql-testing arroyo-sql-testing -COPY arroyo-controller arroyo-controller -COPY arroyo-console arroyo-console -COPY arroyo-storage arroyo-storage -COPY copy-artifacts copy-artifacts -COPY integ integ -COPY docker/refinery.toml refinery.toml -COPY connector-schemas connector-schemas -COPY arroyo-openapi arroyo-openapi -COPY arroyo-bin arroyo-bin -COPY arroyo arroyo -COPY Cargo.toml Cargo.toml -COPY Cargo.lock Cargo.lock - -# set git sha -ENV VERGEN_GIT_SHA=$GIT_SHA -RUN cargo build --release --package copy-artifacts - -FROM debian:bookworm-slim as runner -WORKDIR /app -RUN apt-get update && \ - apt-get -y install openssl ca-certificates libsasl2-dev - -COPY --from=builder /build_dir/target/release/copy-artifacts /usr/local/bin -COPY docker/cluster/worker/entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -ENV PRODUCTION=true - -ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/docker/cluster/worker/entrypoint.sh b/docker/cluster/worker/entrypoint.sh deleted file mode 100644 index bd9a5dc05..000000000 --- a/docker/cluster/worker/entrypoint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -e - -copy-artifacts "${WORKER_BIN}" "${WASM_BIN}" / - -chmod +x /pipeline -/pipeline diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 000000000..cdc65a6ca --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -e + +# Start Postgres + +echo "Starting Postgres" +sudo -u postgres /usr/lib/postgresql/15/bin/postgres -c config_file=/etc/postgresql/15/main/postgresql.conf \ + > /var/log/postgres.log 2>&1 & + +# Wait for postgres to become available + +echo "Waiting for Postgres to become available" +until /usr/lib/postgresql/15/bin/pg_isready -q -h localhost -p 5432 -U postgres; do + sleep 0.2 + echo "." +done + +echo "Postgres is ready" + +# Run the migrations + +echo "Running migrations" + +sudo -u postgres /usr/lib/postgresql/15/bin/createdb arroyo +sudo -u postgres psql -c "CREATE USER arroyo WITH PASSWORD 'arroyo' SUPERUSER;" +/app/arroyo-bin migrate + +# start the services + +echo "Starting Arroyo" +exec /usr/bin/supervisord -n -c /supervisord.conf \ No newline at end of file diff --git a/docker/install_deps.sh b/docker/install_deps.sh new file mode 100644 index 000000000..037eeb444 --- /dev/null +++ b/docker/install_deps.sh @@ -0,0 +1,45 @@ +#!/bin/env bash + +set -e + +# Install arch specific build dependencies +echo env +if [ "$(uname -m)" = "x86_64" ]; then + export PROTO_ARCH="x86_64"; + export MOLD_ARCH="x86_64"; + export PROM_ARCH="amd64"; +elif [ "$(uname -m)" = "aarch64" ]; then + export PROTO_ARCH="aarch_64"; + export MOLD_ARCH="aarch64"; + export PROM_ARCH="arm64"; +else + echo "Unsupported architecture: $(uname -m)" + exit 1; +fi + +if [ "$1" = "build" ]; then + # Install mold + curl -OL https://github.com/rui314/mold/releases/download/v1.11.0/mold-1.11.0-${MOLD_ARCH}-linux.tar.gz + tar xvfz mold*.tar.gz + mv mold*-linux/bin/* /usr/bin + mv mold*-linux/libexec/* /usr/libexec + rm -rf mold* + + # Install protoc + curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protoc-21.8-linux-${PROTO_ARCH}.zip + unzip protoc*.zip + mv bin/protoc /usr/local/bin + rm -rf protoc*.zip +elif [ "$1" = "run" ]; then + # Install prometheus and pushgateway + curl -OL https://github.com/prometheus/prometheus/releases/download/v2.49.1/prometheus-2.49.1.linux-${PROM_ARCH}.tar.gz + tar xvfz prometheus*.tar.gz + mv prometheus*/prometheus /usr/local/bin + curl -OL https://github.com/prometheus/pushgateway/releases/download/v1.5.1/pushgateway-1.5.1.linux-${PROM_ARCH}.tar.gz + tar xvfz pushgateway*.tar.gz + mv pushgateway*/pushgateway /usr/local/bin + rm -rf prometheus* pushgateway* +else + echo "Invalid argument: $1" + exit 1; +fi diff --git a/docker/single/prometheus.yml b/docker/prometheus.yml similarity index 100% rename from docker/single/prometheus.yml rename to docker/prometheus.yml diff --git a/docker/single/Dockerfile b/docker/single/Dockerfile deleted file mode 100644 index cec14fd62..000000000 --- a/docker/single/Dockerfile +++ /dev/null @@ -1,115 +0,0 @@ -FROM ubuntu:22.04 -ENV TZ=Etc/UTC -ENV DEBIAN_FRONTEND=noninteractive -WORKDIR /opt/arroyo/src -ARG MOLD_ARCH -ARG PROTO_ARCH -ARG PROM_ARCH -ARG GIT_SHA - -RUN apt-get update -RUN apt-get -y install git curl pkg-config unzip build-essential libssl-dev openssl libsasl2-dev \ - cmake clang wget postgresql postgresql-client supervisor python3 python-is-python3 sudo bash gpg - -RUN wget https://github.com/rui314/mold/releases/download/v1.11.0/mold-1.11.0-${MOLD_ARCH}-linux.tar.gz && \ - tar xvfz mold*.tar.gz && \ - mv mold*-linux/bin/* /usr/bin && \ - mv mold*-linux/libexec/* /usr/libexec && \ - rm -rf mold* - -RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protoc-21.8-linux-${PROTO_ARCH}.zip && \ - unzip protoc*.zip && \ - mv bin/protoc /usr/local/bin - -# Install prometheus and pushgateway -RUN wget https://github.com/prometheus/prometheus/releases/download/v2.43.0/prometheus-2.43.0.linux-${PROM_ARCH}.tar.gz && \ - tar xvfz prometheus*.tar.gz && \ - mv prometheus*/prometheus /usr/local/bin && \ - wget https://github.com/prometheus/pushgateway/releases/download/v1.5.1/pushgateway-1.5.1.linux-${PROM_ARCH}.tar.gz && \ - tar xvfz pushgateway*.tar.gz && \ - mv pushgateway*/pushgateway /usr/local/bin && \ - rm -rf prometheus* pushgateway* -COPY docker/single/prometheus.yml /etc/prometheus/prometheus.yml - -# Install Rust -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y - -ENV CARGO_HOME=/root/.cargo -ENV PATH=$CARGO_HOME/bin:$PATH - -COPY docker/config.toml $CARGO_HOME/config - -RUN cargo install wasm-pack - -# Set up DB -RUN cargo install refinery_cli - -# Install node & pnpm -RUN mkdir -p /etc/apt/keyrings && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | \ - sudo tee /etc/apt/sources.list.d/nodesource.list && \ - apt-get update && apt-get install -y nodejs && \ - curl -fsSL https://get.pnpm.io/install.sh | env PNPM_VERSION=8.10.4 SHELL=$(which bash) bash - - -COPY arroyo-types arroyo-types -COPY arroyo-worker arroyo-worker -COPY arroyo-rpc arroyo-rpc -COPY arroyo-macro arroyo-macro -COPY arroyo-formats arroyo-formats -COPY arroyo-server-common arroyo-server-common -COPY arroyo-state arroyo-state -COPY arroyo-metrics arroyo-metrics -COPY arroyo-compiler-service arroyo-compiler-service -COPY arroyo-connectors arroyo-connectors -COPY arroyo-api arroyo-api -COPY arroyo-datastream arroyo-datastream -COPY arroyo-node arroyo-node -COPY arroyo-openapi arroyo-openapi -COPY arroyo-operator arroyo-operator -COPY arroyo-df arroyo-df -COPY arroyo-sql-macro arroyo-sql-macro -COPY arroyo-sql-testing arroyo-sql-testing -COPY arroyo-controller arroyo-controller -COPY arroyo-console arroyo-console -COPY arroyo-storage arroyo-storage -COPY copy-artifacts copy-artifacts -COPY integ integ -COPY docker/refinery.toml refinery.toml -COPY connector-schemas connector-schemas -COPY arroyo-openapi arroyo-openapi -COPY arroyo-bin arroyo-bin -COPY arroyo arroyo - -COPY Cargo.toml Cargo.toml -COPY Cargo.lock Cargo.lock - -RUN service postgresql start && \ - sudo -u postgres psql -c "CREATE USER arroyo WITH PASSWORD 'arroyo' SUPERUSER;" && \ - sudo -u postgres createdb arroyo && \ - refinery migrate -c refinery.toml -p arroyo-api/migrations - -# set git hash -ENV VERGEN_GIT_SHA=$GIT_SHA - -# Build Arroyo -RUN service postgresql start && \ - cargo build --release --all-features -p arroyo-compiler-service -p arroyo-controller -p arroyo-api && \ - mv target/release/arroyo-* /usr/bin && \ - rm -rf target/release && \ - service postgresql stop - -RUN bash -c "cd arroyo-console && source ~/.bashrc && /root/.local/share/pnpm/pnpm install && /root/.local/share/pnpm/pnpm build" - -RUN rm -rf .git - -WORKDIR /opt/arroyo/build - -COPY docker/build_base pipeline/ -RUN cd pipeline && cargo build --release && cargo build --release -RUN cd pipeline/wasm-fns && wasm-pack build -COPY docker/single/supervisord.conf /opt/arroyo/src/docker/single/supervisord.conf -COPY docker/single/entrypoint.sh /entrypoint.sh -EXPOSE 8000 9000 9190 9191 -RUN mkdir -p /tmp/arroyo/build -ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/single/entrypoint.sh b/docker/single/entrypoint.sh deleted file mode 100755 index 5a5f0507c..000000000 --- a/docker/single/entrypoint.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Start Postgres - -echo "Starting Postgres" -sudo -u postgres /usr/lib/postgresql/14/bin/postgres -c config_file=/etc/postgresql/14/main/postgresql.conf \ - > /var/log/postgres.log 2>&1 & - -# Wait for postgres to become available - -echo "Waiting for Postgres to become available" -until /usr/lib/postgresql/14/bin/pg_isready -q -h localhost -p 5432 -U postgres; do - sleep 0.2 - echo "." -done - -echo "Postgres is ready" - -# Run the migrations - -echo "Running migrations" - -sudo -u postgres /usr/lib/postgresql/14/bin/dropdb arroyo -sudo -u postgres /usr/lib/postgresql/14/bin/createdb arroyo -refinery migrate -c /opt/arroyo/src/refinery.toml -p /opt/arroyo/src/arroyo-api/migrations - -# start the services - -echo "Starting Arroyo services" - -exec /usr/bin/supervisord -n -c /opt/arroyo/src/docker/single/supervisord.conf diff --git a/docker/single/supervisord.conf b/docker/single/supervisord.conf deleted file mode 100644 index 7240c3fbd..000000000 --- a/docker/single/supervisord.conf +++ /dev/null @@ -1,39 +0,0 @@ -[supervisord] -nodaemon=true - -[program:prometheus] -command=/usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml -autostart=true -autorestart=true - -[program:pushgateway] -command=/usr/local/bin/pushgateway -autostart=true -autorestart=true - -[program:api] -command=/usr/bin/arroyo-api -autostart=true -autorestart=true -stderr_logfile=/var/log/supervisor/api.err.log -stdout_logfile=/dev/fd/1 -stdout_logfile_maxbytes=0 -environment=ASSET_DIR="/opt/arroyo/src/arroyo-console/dist" - -[program:controller] -command=/usr/bin/arroyo-controller -autostart=true -autorestart=true -stderr_logfile=/var/log/supervisor/controller.err.log -stdout_logfile=/dev/fd/1 -stdout_logfile_maxbytes=0 -environment=REMOTE_COMPILER_ENDPOINT="http://localhost:9000" - -[program:compiler] -command=/usr/bin/arroyo-compiler-service start -autostart=true -autorestart=true -stderr_logfile=/var/log/supervisor/compiler.err.log -stdout_logfile=/dev/fd/1 -stdout_logfile_maxbytes=0 -environment=BUILD_DIR="/opt/arroyo/build/pipeline",ARTIFACT_URL="file:///tmp/arroyo/build" diff --git a/docker/supervisord.conf b/docker/supervisord.conf new file mode 100644 index 000000000..4d0eb4cbf --- /dev/null +++ b/docker/supervisord.conf @@ -0,0 +1,21 @@ +[supervisord] +nodaemon=true + +[program:prometheus] +command=/usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml +autostart=true +autorestart=true + +[program:pushgateway] +command=/usr/local/bin/pushgateway +autostart=true +autorestart=true + +[program:arroyo] +command=/app/arroyo-bin cluster +autostart=true +autorestart=true +stderr_logfile=/var/log/supervisor/arroyo.err.log +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +environment=ASSET_DIR="/app/dist" diff --git a/docs/images/arroyo_logo.png b/images/arroyo_logo.png similarity index 100% rename from docs/images/arroyo_logo.png rename to images/arroyo_logo.png diff --git a/docs/images/header_image.png b/images/header_image.png similarity index 100% rename from docs/images/header_image.png rename to images/header_image.png diff --git a/docs/images/running_job.png b/images/running_job.png similarity index 100% rename from docs/images/running_job.png rename to images/running_job.png diff --git a/integ/openapitools.json b/integ/openapitools.json deleted file mode 100644 index cd53ff4ca..000000000 --- a/integ/openapitools.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json", - "spaces": 2, - "generator-cli": { - "version": "6.6.0" - } -} diff --git a/justfile b/justfile deleted file mode 100644 index 7e51618ef..000000000 --- a/justfile +++ /dev/null @@ -1,55 +0,0 @@ -default: check - -check: - cargo check --all --all-features - -docker-single-amd64: - docker build --build-arg MOLD_ARCH=x86_64 \ - --build-arg PROTO_ARCH=x86_64 \ - --build-arg PROM_ARCH=amd64 \ - --file docker/single/Dockerfile . -t ghcr.io/arroyosystems/arroyo-single:amd64 - -docker-single-arm64: - docker build \ - --build-arg MOLD_ARCH=aarch64 \ - --build-arg PROTO_ARCH=aarch_64 \ - --build-arg PROM_ARCH=arm64 \ - --file docker/single/Dockerfile . -t ghcr.io/arroyosystems/arroyo-single:arm64 - -docker-services-amd64: - docker build \ - --build-arg PROTO_ARCH=x86_64 \ - --file docker/cluster/services/Dockerfile . -t ghcr.io/arroyosystems/arroyo-services:amd64 - -docker-services-arm64: - docker build \ - --build-arg PROTO_ARCH=aarch_64 \ - --file docker/cluster/services/Dockerfile . -t ghcr.io/arroyosystems/arroyo-services:arm64 - -docker-worker-amd64: - docker build \ - --file docker/cluster/worker/Dockerfile . -t ghcr.io/arroyosystems/arroyo-worker:amd64 - -docker-worker-arm64: - docker build \ - --file docker/cluster/worker/Dockerfile . -t ghcr.io/arroyosystems/arroyo-worker:arm64 - -docker-compiler-amd64: - docker build \ - --build-arg MOLD_ARCH=x86_64 \ - --build-arg PROTO_ARCH=x86_64 \ - --file docker/cluster/compiler/Dockerfile . -t ghcr.io/arroyosystems/arroyo-compiler:amd64 - -docker-compiler-arm64: - docker build \ - --build-arg MOLD_ARCH=aarch64 \ - --build-arg PROTO_ARCH=aarch_64 \ - --file docker/cluster/compiler/Dockerfile . -t ghcr.io/arroyosystems/arroyo-compiler:arm64 - - -docker-amd64: docker-single-amd64 docker-services-amd64 docker-compiler-amd64 - -docker-arm64: docker-single-arm64 docker-services-arm64 docker-compiler-arm64 - -docker-cluster-amd64: docker-services-amd64 docker-compiler-amd64 -docker-cluster-arm64: docker-services-arm64 docker-compiler-arm64 diff --git a/k8s/arroyo/templates/NOTES.txt b/k8s/arroyo/templates/NOTES.txt index 0465db76d..5c2b2658e 100644 --- a/k8s/arroyo/templates/NOTES.txt +++ b/k8s/arroyo/templates/NOTES.txt @@ -4,13 +4,13 @@ This release is {{ .Release.Name }} Once the release is fully deployed, you can navigate to the web ui by running - $ open "http://$(kubectl get service/{{ .Release.Name }}-api -o jsonpath='{.spec.clusterIP}')" + $ open "http://$(kubectl get service/{{ .Release.Name }} -o jsonpath='{.spec.clusterIP}')" (note this might take a few minutes if you are also deploying Postgres) If that doesn't work (for example on MacOS or a remote Kubernetes cluster), you can also try - $ kubectl port-forward service/{{ .Release.Name }}-api 8000:80 + $ kubectl port-forward service/{{ .Release.Name }} 8000:80 And opening http://localhost:8000 in your browser. diff --git a/k8s/arroyo/templates/api.yaml b/k8s/arroyo/templates/api.yaml deleted file mode 100644 index 86396040a..000000000 --- a/k8s/arroyo/templates/api.yaml +++ /dev/null @@ -1,98 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "arroyo.fullname" . }}-api - labels: - {{- include "arroyo.labels" . | nindent 4 }} - app: {{ include "arroyo.fullname" . }}-api -spec: - replicas: {{ .Values.api.replicas }} - selector: - matchLabels: - app: {{ include "arroyo.fullname" . }}-api - template: - metadata: - labels: - {{- include "arroyo.labels" . | nindent 8 }} - app: {{ include "arroyo.fullname" . }}-api - annotations: - {{- if .Values.prometheus.setAnnotations }} - prometheus.io/scrape: "true" - prometheus.io/path: /metrics - prometheus.io/port: "8001" - {{- end }} - {{- with .Values.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "arroyo.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: arroyo-api - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag }}" - imagePullPolicy: {{ .Values.api.image.pullPolicy }} - args: ["api"] - env: - {{- include "arroyo.databaseEnvVars" . | nindent 8 }} - - name: CONTROLLER_ADDR - value: "http://{{ include "arroyo.fullname" . }}-controller:9190" - - name: PROM_ENDPOINT - value: "{{ include "arroyo.prometheusEndpoint" .}}" - - name: API_METRICS_RATE - value: "{{ .Values.prometheus.queryRate }}" - {{- include "arroyo.existingConfigMap" . | nindent 8 }} - ports: - - containerPort: 8000 - name: http - - containerPort: 8001 - name: admin - livenessProbe: - httpGet: - path: /status - port: admin - initialDelaySeconds: 5 - readinessProbe: - httpGet: - path: /status - port: admin - initialDelaySeconds: 5 - {{- if .Values.api.resources }} - resources: {{- toYaml .Values.api.resources | nindent 12 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "arroyo.fullname" . }}-api -spec: - selector: - app: {{ include "arroyo.fullname" . }}-api - ports: - - name: http - protocol: TCP - port: {{ .Values.api.service.httpPort }} - targetPort: 8000 - - name: admin - protocol: TCP - port: {{ .Values.api.service.adminPort }} - targetPort: 8001 diff --git a/k8s/arroyo/templates/compiler.yaml b/k8s/arroyo/templates/compiler.yaml deleted file mode 100644 index db158d5fc..000000000 --- a/k8s/arroyo/templates/compiler.yaml +++ /dev/null @@ -1,97 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "arroyo.fullname" . }}-compiler - labels: - {{- include "arroyo.labels" . | nindent 4 }} - app: {{ include "arroyo.fullname" . }}-compiler -spec: - replicas: {{ .Values.compiler.replicas }} - selector: - matchLabels: - app: {{ include "arroyo.fullname" . }}-compiler - template: - metadata: - labels: - {{- include "arroyo.labels" . | nindent 8 }} - app: {{ include "arroyo.fullname" . }}-compiler - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - volumes: - {{- if .Values.volumes }} - {{- include "tplvalues.render" (dict "value" .Values.volumes "context" $) | nindent 8 }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "arroyo.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: arroyo-compiler - securityContext: - {{- toYaml .Values.securityContext | nindent 10 }} - image: "{{ .Values.compiler.image.repository }}:{{ .Values.compiler.image.tag }}" - imagePullPolicy: {{ .Values.compiler.image.pullPolicy }} - env: - {{- include "arroyo.storageEnvVars" . | nindent 10 }} - {{ if .Values.artifactUrl }} - - name: ARTIFACT_URL - value: {{ .Values.artifactUrl }} - {{- end }} - {{- include "arroyo.existingConfigMap" . | nindent 8 }} - ports: - - containerPort: 9000 - name: grpc - - containerPort: 9001 - name: admin - livenessProbe: - httpGet: - path: /status - port: admin - initialDelaySeconds: 5 - readinessProbe: - httpGet: - path: /status - port: admin - initialDelaySeconds: 5 - {{- if .Values.compiler.resources }} - resources: {{- toYaml .Values.compiler.resources | nindent 10 }} - {{- end }} - volumeMounts: - {{- if .Values.volumeMounts }} - {{- include "tplvalues.render" (dict "value" .Values.volumeMounts "context" $) | nindent 10 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "arroyo.fullname" . }}-compiler -spec: - selector: - app: {{ include "arroyo.fullname" . }}-compiler - ports: - - name: grpc - protocol: TCP - port: {{ .Values.compiler.service.grpcPort }} - targetPort: 9000 - - name: http - protocol: TCP - port: {{ .Values.compiler.service.adminPort }} - targetPort: 9001 diff --git a/k8s/arroyo/templates/controller.yaml b/k8s/arroyo/templates/controller.yaml index d2c64ec57..70558bb2c 100644 --- a/k8s/arroyo/templates/controller.yaml +++ b/k8s/arroyo/templates/controller.yaml @@ -21,7 +21,7 @@ spec: {{- if .Values.prometheus.setAnnotations }} prometheus.io/scrape: "true" prometheus.io/path: /metrics - prometheus.io/port: "9191" + prometheus.io/port: "8001" {{- end }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} @@ -36,13 +36,20 @@ spec: imagePullSecrets: {{- toYaml . | nindent 10 }} {{- end }} + initContainers: + - name: migrate-database + image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}" + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} + command: [ "/app/arroyo-bin", "migrate", "--wait", "300" ] + env: + {{- include "arroyo.databaseEnvVars" . | nindent 8 }} containers: - name: arroyo-controller securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}" imagePullPolicy: {{ .Values.controller.image.pullPolicy }} - args: ["controller"] + command: ["/app/arroyo-bin", "cluster"] env: {{- include "arroyo.storageEnvVars" . | nindent 8 }} @@ -53,9 +60,17 @@ spec: {{- include "arroyo.databaseEnvVars" . | nindent 8 }} - name: CONTROLLER_ADDR - value: "http://{{ include "arroyo.fullname" . }}-controller:9190" - - name: REMOTE_COMPILER_ENDPOINT - value: "http://{{ include "arroyo.fullname" . }}-compiler:9000" + value: "http://{{ include "arroyo.fullname" . }}:{{ .Values.controller.service.grpcPort }}" + - name: GRPC_PORT + value: {{ .Values.controller.service.grpcPort | quote }} + - name: ADMIN_PORT + value: {{ .Values.controller.service.adminPort | quote }} + - name: HTTP_PORT + value: {{ .Values.controller.service.httpPort | quote }} + - name: PROM_ENDPOINT + value: "{{ include "arroyo.prometheusEndpoint" .}}" + - name: API_METRICS_RATE + value: "{{ .Values.prometheus.queryRate }}" - name: SCHEDULER value: "kubernetes" - name: K8S_NAMESPACE @@ -102,10 +117,12 @@ spec: {{- include "arroyo.existingConfigMap" . | nindent 8 }} ports: - - containerPort: 9190 + - containerPort: {{ .Values.controller.service.grpcPort }} name: grpc - - containerPort: 9191 + - containerPort: {{ .Values.controller.service.adminPort }} name: admin + - containerPort: {{ .Values.controller.service.httpPort }} + name: http livenessProbe: httpGet: path: /status @@ -116,9 +133,6 @@ spec: path: /status port: admin initialDelaySeconds: 5 - {{- if .Values.compiler.resources }} - resources: {{- toYaml .Values.compiler.resources | nindent 10 }} - {{- end }} volumeMounts: {{- if .Values.volumeMounts }} {{- include "tplvalues.render" (dict "value" .Values.volumeMounts "context" $) | nindent 10 }} @@ -140,7 +154,7 @@ spec: apiVersion: v1 kind: Service metadata: - name: {{ include "arroyo.fullname" . }}-controller + name: {{ include "arroyo.fullname" . }} spec: selector: app: {{ include "arroyo.fullname" . }}-controller @@ -149,7 +163,11 @@ spec: protocol: TCP port: {{ .Values.controller.service.grpcPort }} targetPort: 9190 - - name: http + - name: admin protocol: TCP port: {{ .Values.controller.service.adminPort }} targetPort: 9191 + - name: http + protocol: TCP + port: {{ .Values.controller.service.httpPort }} + targetPort: 80 diff --git a/k8s/arroyo/templates/tests/test-console-connection.yaml b/k8s/arroyo/templates/tests/test-console-connection.yaml index 0e68818ee..d492f37ee 100644 --- a/k8s/arroyo/templates/tests/test-console-connection.yaml +++ b/k8s/arroyo/templates/tests/test-console-connection.yaml @@ -11,5 +11,5 @@ spec: - name: wget image: busybox command: ['wget'] - args: ['{{ include "arroyo.fullname" . }}-api:{{ .Values.api.service.httpPort }}'] + args: ['{{ include "arroyo.fullname" . }}:{{ .Values.controller.service.httpPort }}'] restartPolicy: Never diff --git a/k8s/arroyo/templates/tests/test-grpc-connection.yaml b/k8s/arroyo/templates/tests/test-grpc-connection.yaml index 63c8ff240..c9b785632 100644 --- a/k8s/arroyo/templates/tests/test-grpc-connection.yaml +++ b/k8s/arroyo/templates/tests/test-grpc-connection.yaml @@ -1,20 +1,5 @@ apiVersion: v1 kind: Pod -metadata: - name: "{{ include "arroyo.fullname" . }}-compiler-test-grpc-connection" - labels: - {{- include "arroyo.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: grpcurl - image: fullstorydev/grpcurl:v1.8.7-alpine - args: ['-plaintext', '{{ include "arroyo.fullname" . }}-compiler:{{ .Values.compiler.service.grpcPort }}','list'] - restartPolicy: Never ---- -apiVersion: v1 -kind: Pod metadata: name: "{{ include "arroyo.fullname" . }}-controller-test-grpc-connection" labels: @@ -25,5 +10,5 @@ spec: containers: - name: grpcurl image: fullstorydev/grpcurl:v1.8.7-alpine - args: ['-plaintext', '{{ include "arroyo.fullname" . }}-controller:{{ .Values.controller.service.grpcPort }}','list'] + args: ['-plaintext', '{{ include "arroyo.fullname" . }}:{{ .Values.controller.service.grpcPort }}','list'] restartPolicy: Never diff --git a/k8s/arroyo/values.yaml b/k8s/arroyo/values.yaml index 8937d53ac..e1e345c63 100644 --- a/k8s/arroyo/values.yaml +++ b/k8s/arroyo/values.yaml @@ -4,24 +4,6 @@ nameOverride: "" fullnameOverride: "" imagePullSecrets: [] -# service-specific configuration -api: - replicas: 1 - resources: - limits: {} - requests: - memory: 256Mi - cpu: 200m - image: - repository: ghcr.io/arroyosystems/arroyo-services - pullPolicy: IfNotPresent - tag: "tip" - service: - type: ClusterIP - httpPort: 80 - adminPort: 8001 - - controller: resources: limits: {} @@ -29,27 +11,13 @@ controller: memory: 256Mi cpu: 500m image: - repository: ghcr.io/arroyosystems/arroyo-services + repository: ghcr.io/arroyosystems/arroyo pullPolicy: IfNotPresent - tag: "tip" + tag: "dev" service: grpcPort: 9190 - adminPort: 9191 - -compiler: - replicas: 1 - resources: - limits: {} - requests: - memory: 2048Mi - cpu: 1000m - image: - repository: ghcr.io/arroyosystems/arroyo-compiler - pullPolicy: IfNotPresent - tag: "tip" - service: - grpcPort: 9000 - adminPort: 9001 + adminPort: 8001 + httpPort: 80 worker: resources: @@ -59,9 +27,9 @@ worker: cpu: 400m slots: 4 image: - repository: ghcr.io/arroyosystems/arroyo-worker + repository: ghcr.io/arroyosystems/arroyo pullPolicy: IfNotPresent - tag: "tip" + tag: "dev" postgresql: # set to true to deploy a postgres instance in-cluster @@ -151,9 +119,7 @@ securityContext: {} -apiResources: {} controllerResources: {} -compilerResources: {} nodeSelector: {} diff --git a/openapitools.json b/openapitools.json deleted file mode 100644 index cd53ff4ca..000000000 --- a/openapitools.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json", - "spaces": 2, - "generator-cli": { - "version": "6.6.0" - } -} diff --git a/arroyo-console/.eslintignore b/webui/.eslintignore similarity index 100% rename from arroyo-console/.eslintignore rename to webui/.eslintignore diff --git a/arroyo-console/.eslintrc.yml b/webui/.eslintrc.yml similarity index 100% rename from arroyo-console/.eslintrc.yml rename to webui/.eslintrc.yml diff --git a/arroyo-console/.gitignore b/webui/.gitignore similarity index 100% rename from arroyo-console/.gitignore rename to webui/.gitignore diff --git a/arroyo-console/.prettierignore b/webui/.prettierignore similarity index 100% rename from arroyo-console/.prettierignore rename to webui/.prettierignore diff --git a/arroyo-console/.prettierrc b/webui/.prettierrc similarity index 100% rename from arroyo-console/.prettierrc rename to webui/.prettierrc diff --git a/arroyo-console/index.html b/webui/index.html similarity index 100% rename from arroyo-console/index.html rename to webui/index.html diff --git a/arroyo-console/package.json b/webui/package.json similarity index 96% rename from arroyo-console/package.json rename to webui/package.json index b18888eb6..4a519ad21 100644 --- a/arroyo-console/package.json +++ b/webui/package.json @@ -9,7 +9,7 @@ "preview": "vite preview", "format": "npx prettier --write src/ && npx eslint --fix --ext .js,.jsx,.ts,.tsx src", "check": "npx prettier --check src/ && npx eslint --ext .js,.jsx,.ts,.tsx src", - "openapi": "cargo build --package arroyo-openapi && npx openapi-typescript $(pwd)/../arroyo-openapi/api-spec.json --output $(pwd)/src/gen/api-types.ts" + "openapi": "cargo build --package arroyo-openapi && npx openapi-typescript $(pwd)/../crates/arroyo-openapi/api-spec.json --output $(pwd)/src/gen/api-types.ts" }, "dependencies": { "@babel/core": "^7.22.5", diff --git a/arroyo-console/pnpm-lock.yaml b/webui/pnpm-lock.yaml similarity index 100% rename from arroyo-console/pnpm-lock.yaml rename to webui/pnpm-lock.yaml diff --git a/arroyo-console/post b/webui/post similarity index 100% rename from arroyo-console/post rename to webui/post diff --git a/arroyo-console/pre b/webui/pre similarity index 100% rename from arroyo-console/pre rename to webui/pre diff --git a/arroyo-console/public/apple-touch-icon.png b/webui/public/apple-touch-icon.png similarity index 100% rename from arroyo-console/public/apple-touch-icon.png rename to webui/public/apple-touch-icon.png diff --git a/arroyo-console/public/icon.png b/webui/public/icon.png similarity index 100% rename from arroyo-console/public/icon.png rename to webui/public/icon.png diff --git a/arroyo-console/public/icon.svg b/webui/public/icon.svg similarity index 100% rename from arroyo-console/public/icon.svg rename to webui/public/icon.svg diff --git a/arroyo-console/public/logo.svg b/webui/public/logo.svg similarity index 100% rename from arroyo-console/public/logo.svg rename to webui/public/logo.svg diff --git a/arroyo-console/src/App.css b/webui/src/App.css similarity index 100% rename from arroyo-console/src/App.css rename to webui/src/App.css diff --git a/arroyo-console/src/App.tsx b/webui/src/App.tsx similarity index 100% rename from arroyo-console/src/App.tsx rename to webui/src/App.tsx diff --git a/arroyo-console/src/components/CheckpointDetails.tsx b/webui/src/components/CheckpointDetails.tsx similarity index 100% rename from arroyo-console/src/components/CheckpointDetails.tsx rename to webui/src/components/CheckpointDetails.tsx diff --git a/arroyo-console/src/components/Checkpoints.tsx b/webui/src/components/Checkpoints.tsx similarity index 100% rename from arroyo-console/src/components/Checkpoints.tsx rename to webui/src/components/Checkpoints.tsx diff --git a/arroyo-console/src/components/CreatePipelineTourModal.tsx b/webui/src/components/CreatePipelineTourModal.tsx similarity index 100% rename from arroyo-console/src/components/CreatePipelineTourModal.tsx rename to webui/src/components/CreatePipelineTourModal.tsx diff --git a/arroyo-console/src/components/ExampleQueries.tsx b/webui/src/components/ExampleQueries.tsx similarity index 100% rename from arroyo-console/src/components/ExampleQueries.tsx rename to webui/src/components/ExampleQueries.tsx diff --git a/arroyo-console/src/components/ExampleQueryCard.tsx b/webui/src/components/ExampleQueryCard.tsx similarity index 100% rename from arroyo-console/src/components/ExampleQueryCard.tsx rename to webui/src/components/ExampleQueryCard.tsx diff --git a/arroyo-console/src/components/Indicator.tsx b/webui/src/components/Indicator.tsx similarity index 100% rename from arroyo-console/src/components/Indicator.tsx rename to webui/src/components/Indicator.tsx diff --git a/arroyo-console/src/components/JobCard.tsx b/webui/src/components/JobCard.tsx similarity index 100% rename from arroyo-console/src/components/JobCard.tsx rename to webui/src/components/JobCard.tsx diff --git a/arroyo-console/src/components/Loading.tsx b/webui/src/components/Loading.tsx similarity index 100% rename from arroyo-console/src/components/Loading.tsx rename to webui/src/components/Loading.tsx diff --git a/arroyo-console/src/components/OperatorDetail.tsx b/webui/src/components/OperatorDetail.tsx similarity index 100% rename from arroyo-console/src/components/OperatorDetail.tsx rename to webui/src/components/OperatorDetail.tsx diff --git a/arroyo-console/src/components/OperatorErrors.tsx b/webui/src/components/OperatorErrors.tsx similarity index 100% rename from arroyo-console/src/components/OperatorErrors.tsx rename to webui/src/components/OperatorErrors.tsx diff --git a/arroyo-console/src/components/PaginatedContent.tsx b/webui/src/components/PaginatedContent.tsx similarity index 100% rename from arroyo-console/src/components/PaginatedContent.tsx rename to webui/src/components/PaginatedContent.tsx diff --git a/arroyo-console/src/components/PipelineNotFound.tsx b/webui/src/components/PipelineNotFound.tsx similarity index 100% rename from arroyo-console/src/components/PipelineNotFound.tsx rename to webui/src/components/PipelineNotFound.tsx diff --git a/arroyo-console/src/components/PipelineRow.tsx b/webui/src/components/PipelineRow.tsx similarity index 100% rename from arroyo-console/src/components/PipelineRow.tsx rename to webui/src/components/PipelineRow.tsx diff --git a/arroyo-console/src/components/PipelinesTable.tsx b/webui/src/components/PipelinesTable.tsx similarity index 100% rename from arroyo-console/src/components/PipelinesTable.tsx rename to webui/src/components/PipelinesTable.tsx diff --git a/arroyo-console/src/components/StartPipelineModal.tsx b/webui/src/components/StartPipelineModal.tsx similarity index 100% rename from arroyo-console/src/components/StartPipelineModal.tsx rename to webui/src/components/StartPipelineModal.tsx diff --git a/arroyo-console/src/components/TimeSeriesGraph.tsx b/webui/src/components/TimeSeriesGraph.tsx similarity index 100% rename from arroyo-console/src/components/TimeSeriesGraph.tsx rename to webui/src/components/TimeSeriesGraph.tsx diff --git a/arroyo-console/src/components/TourCompleteModal.tsx b/webui/src/components/TourCompleteModal.tsx similarity index 100% rename from arroyo-console/src/components/TourCompleteModal.tsx rename to webui/src/components/TourCompleteModal.tsx diff --git a/arroyo-console/src/components/UdfNodeViewer.tsx b/webui/src/components/UdfNodeViewer.tsx similarity index 100% rename from arroyo-console/src/components/UdfNodeViewer.tsx rename to webui/src/components/UdfNodeViewer.tsx diff --git a/arroyo-console/src/components/WelcomeModal.tsx b/webui/src/components/WelcomeModal.tsx similarity index 100% rename from arroyo-console/src/components/WelcomeModal.tsx rename to webui/src/components/WelcomeModal.tsx diff --git a/arroyo-console/src/gen/api-types.ts b/webui/src/gen/api-types.ts similarity index 100% rename from arroyo-console/src/gen/api-types.ts rename to webui/src/gen/api-types.ts diff --git a/arroyo-console/src/index.css b/webui/src/index.css similarity index 100% rename from arroyo-console/src/index.css rename to webui/src/index.css diff --git a/arroyo-console/src/lib/CloudComponents.tsx b/webui/src/lib/CloudComponents.tsx similarity index 100% rename from arroyo-console/src/lib/CloudComponents.tsx rename to webui/src/lib/CloudComponents.tsx diff --git a/arroyo-console/src/lib/RadioGroup.tsx b/webui/src/lib/RadioGroup.tsx similarity index 100% rename from arroyo-console/src/lib/RadioGroup.tsx rename to webui/src/lib/RadioGroup.tsx diff --git a/arroyo-console/src/lib/data_fetching.ts b/webui/src/lib/data_fetching.ts similarity index 100% rename from arroyo-console/src/lib/data_fetching.ts rename to webui/src/lib/data_fetching.ts diff --git a/arroyo-console/src/lib/example_queries.ts b/webui/src/lib/example_queries.ts similarity index 100% rename from arroyo-console/src/lib/example_queries.ts rename to webui/src/lib/example_queries.ts diff --git a/arroyo-console/src/lib/types.ts b/webui/src/lib/types.ts similarity index 100% rename from arroyo-console/src/lib/types.ts rename to webui/src/lib/types.ts diff --git a/arroyo-console/src/lib/util.ts b/webui/src/lib/util.ts similarity index 100% rename from arroyo-console/src/lib/util.ts rename to webui/src/lib/util.ts diff --git a/arroyo-console/src/main.tsx b/webui/src/main.tsx similarity index 100% rename from arroyo-console/src/main.tsx rename to webui/src/main.tsx diff --git a/arroyo-console/src/router.tsx b/webui/src/router.tsx similarity index 100% rename from arroyo-console/src/router.tsx rename to webui/src/router.tsx diff --git a/arroyo-console/src/routes/connections/ChooseConnector.tsx b/webui/src/routes/connections/ChooseConnector.tsx similarity index 100% rename from arroyo-console/src/routes/connections/ChooseConnector.tsx rename to webui/src/routes/connections/ChooseConnector.tsx diff --git a/arroyo-console/src/routes/connections/ConfigureConnection.tsx b/webui/src/routes/connections/ConfigureConnection.tsx similarity index 100% rename from arroyo-console/src/routes/connections/ConfigureConnection.tsx rename to webui/src/routes/connections/ConfigureConnection.tsx diff --git a/arroyo-console/src/routes/connections/ConfigureProfile.tsx b/webui/src/routes/connections/ConfigureProfile.tsx similarity index 100% rename from arroyo-console/src/routes/connections/ConfigureProfile.tsx rename to webui/src/routes/connections/ConfigureProfile.tsx diff --git a/arroyo-console/src/routes/connections/ConfluentSchemaEditor.tsx b/webui/src/routes/connections/ConfluentSchemaEditor.tsx similarity index 100% rename from arroyo-console/src/routes/connections/ConfluentSchemaEditor.tsx rename to webui/src/routes/connections/ConfluentSchemaEditor.tsx diff --git a/arroyo-console/src/routes/connections/ConnectionTester.tsx b/webui/src/routes/connections/ConnectionTester.tsx similarity index 100% rename from arroyo-console/src/routes/connections/ConnectionTester.tsx rename to webui/src/routes/connections/ConnectionTester.tsx diff --git a/arroyo-console/src/routes/connections/Connections.tsx b/webui/src/routes/connections/Connections.tsx similarity index 100% rename from arroyo-console/src/routes/connections/Connections.tsx rename to webui/src/routes/connections/Connections.tsx diff --git a/arroyo-console/src/routes/connections/CreateConnection.tsx b/webui/src/routes/connections/CreateConnection.tsx similarity index 100% rename from arroyo-console/src/routes/connections/CreateConnection.tsx rename to webui/src/routes/connections/CreateConnection.tsx diff --git a/arroyo-console/src/routes/connections/CreateProfile.tsx b/webui/src/routes/connections/CreateProfile.tsx similarity index 100% rename from arroyo-console/src/routes/connections/CreateProfile.tsx rename to webui/src/routes/connections/CreateProfile.tsx diff --git a/arroyo-console/src/routes/connections/DefineSchema.tsx b/webui/src/routes/connections/DefineSchema.tsx similarity index 100% rename from arroyo-console/src/routes/connections/DefineSchema.tsx rename to webui/src/routes/connections/DefineSchema.tsx diff --git a/arroyo-console/src/routes/connections/JsonForm.tsx b/webui/src/routes/connections/JsonForm.tsx similarity index 100% rename from arroyo-console/src/routes/connections/JsonForm.tsx rename to webui/src/routes/connections/JsonForm.tsx diff --git a/arroyo-console/src/routes/connections/SchemaEditor.tsx b/webui/src/routes/connections/SchemaEditor.tsx similarity index 100% rename from arroyo-console/src/routes/connections/SchemaEditor.tsx rename to webui/src/routes/connections/SchemaEditor.tsx diff --git a/arroyo-console/src/routes/home/Home.tsx b/webui/src/routes/home/Home.tsx similarity index 100% rename from arroyo-console/src/routes/home/Home.tsx rename to webui/src/routes/home/Home.tsx diff --git a/arroyo-console/src/routes/not_found/ApiUnavailable.tsx b/webui/src/routes/not_found/ApiUnavailable.tsx similarity index 100% rename from arroyo-console/src/routes/not_found/ApiUnavailable.tsx rename to webui/src/routes/not_found/ApiUnavailable.tsx diff --git a/arroyo-console/src/routes/not_found/PageNotFound.tsx b/webui/src/routes/not_found/PageNotFound.tsx similarity index 100% rename from arroyo-console/src/routes/not_found/PageNotFound.tsx rename to webui/src/routes/not_found/PageNotFound.tsx diff --git a/arroyo-console/src/routes/pipelines/Catalog.tsx b/webui/src/routes/pipelines/Catalog.tsx similarity index 100% rename from arroyo-console/src/routes/pipelines/Catalog.tsx rename to webui/src/routes/pipelines/Catalog.tsx diff --git a/arroyo-console/src/routes/pipelines/CatalogTab.tsx b/webui/src/routes/pipelines/CatalogTab.tsx similarity index 100% rename from arroyo-console/src/routes/pipelines/CatalogTab.tsx rename to webui/src/routes/pipelines/CatalogTab.tsx diff --git a/arroyo-console/src/routes/pipelines/CodeEditor.tsx b/webui/src/routes/pipelines/CodeEditor.tsx similarity index 100% rename from arroyo-console/src/routes/pipelines/CodeEditor.tsx rename to webui/src/routes/pipelines/CodeEditor.tsx diff --git a/arroyo-console/src/routes/pipelines/CreatePipeline.tsx b/webui/src/routes/pipelines/CreatePipeline.tsx similarity index 100% rename from arroyo-console/src/routes/pipelines/CreatePipeline.tsx rename to webui/src/routes/pipelines/CreatePipeline.tsx diff --git a/arroyo-console/src/routes/pipelines/PipelineConfigModal.tsx b/webui/src/routes/pipelines/PipelineConfigModal.tsx similarity index 100% rename from arroyo-console/src/routes/pipelines/PipelineConfigModal.tsx rename to webui/src/routes/pipelines/PipelineConfigModal.tsx diff --git a/arroyo-console/src/routes/pipelines/PipelineDetails.tsx b/webui/src/routes/pipelines/PipelineDetails.tsx similarity index 100% rename from arroyo-console/src/routes/pipelines/PipelineDetails.tsx rename to webui/src/routes/pipelines/PipelineDetails.tsx diff --git a/arroyo-console/src/routes/pipelines/PipelineEditorTabs.tsx b/webui/src/routes/pipelines/PipelineEditorTabs.tsx similarity index 100% rename from arroyo-console/src/routes/pipelines/PipelineEditorTabs.tsx rename to webui/src/routes/pipelines/PipelineEditorTabs.tsx diff --git a/arroyo-console/src/routes/pipelines/PipelineGraph.tsx b/webui/src/routes/pipelines/PipelineGraph.tsx similarity index 100% rename from arroyo-console/src/routes/pipelines/PipelineGraph.tsx rename to webui/src/routes/pipelines/PipelineGraph.tsx diff --git a/arroyo-console/src/routes/pipelines/PipelineOutputs.tsx b/webui/src/routes/pipelines/PipelineOutputs.tsx similarity index 100% rename from arroyo-console/src/routes/pipelines/PipelineOutputs.tsx rename to webui/src/routes/pipelines/PipelineOutputs.tsx diff --git a/arroyo-console/src/routes/pipelines/PipelinesIndex.tsx b/webui/src/routes/pipelines/PipelinesIndex.tsx similarity index 100% rename from arroyo-console/src/routes/pipelines/PipelinesIndex.tsx rename to webui/src/routes/pipelines/PipelinesIndex.tsx diff --git a/arroyo-console/src/routes/pipelines/ResourcePanel.tsx b/webui/src/routes/pipelines/ResourcePanel.tsx similarity index 100% rename from arroyo-console/src/routes/pipelines/ResourcePanel.tsx rename to webui/src/routes/pipelines/ResourcePanel.tsx diff --git a/arroyo-console/src/routes/pipelines/pipelines.css b/webui/src/routes/pipelines/pipelines.css similarity index 100% rename from arroyo-console/src/routes/pipelines/pipelines.css rename to webui/src/routes/pipelines/pipelines.css diff --git a/arroyo-console/src/routes/udfs/DeleteUdfModal.tsx b/webui/src/routes/udfs/DeleteUdfModal.tsx similarity index 100% rename from arroyo-console/src/routes/udfs/DeleteUdfModal.tsx rename to webui/src/routes/udfs/DeleteUdfModal.tsx diff --git a/arroyo-console/src/routes/udfs/GlobalizeModal.tsx b/webui/src/routes/udfs/GlobalizeModal.tsx similarity index 100% rename from arroyo-console/src/routes/udfs/GlobalizeModal.tsx rename to webui/src/routes/udfs/GlobalizeModal.tsx diff --git a/arroyo-console/src/routes/udfs/UdfEditTab.tsx b/webui/src/routes/udfs/UdfEditTab.tsx similarity index 100% rename from arroyo-console/src/routes/udfs/UdfEditTab.tsx rename to webui/src/routes/udfs/UdfEditTab.tsx diff --git a/arroyo-console/src/routes/udfs/UdfEditor.tsx b/webui/src/routes/udfs/UdfEditor.tsx similarity index 100% rename from arroyo-console/src/routes/udfs/UdfEditor.tsx rename to webui/src/routes/udfs/UdfEditor.tsx diff --git a/arroyo-console/src/routes/udfs/UdfLabel.tsx b/webui/src/routes/udfs/UdfLabel.tsx similarity index 100% rename from arroyo-console/src/routes/udfs/UdfLabel.tsx rename to webui/src/routes/udfs/UdfLabel.tsx diff --git a/arroyo-console/src/routes/udfs/UdfNodePopover.tsx b/webui/src/routes/udfs/UdfNodePopover.tsx similarity index 100% rename from arroyo-console/src/routes/udfs/UdfNodePopover.tsx rename to webui/src/routes/udfs/UdfNodePopover.tsx diff --git a/arroyo-console/src/routes/udfs/UdfsResourceTab.tsx b/webui/src/routes/udfs/UdfsResourceTab.tsx similarity index 100% rename from arroyo-console/src/routes/udfs/UdfsResourceTab.tsx rename to webui/src/routes/udfs/UdfsResourceTab.tsx diff --git a/arroyo-console/src/theming.ts b/webui/src/theming.ts similarity index 100% rename from arroyo-console/src/theming.ts rename to webui/src/theming.ts diff --git a/arroyo-console/src/tour.ts b/webui/src/tour.ts similarity index 100% rename from arroyo-console/src/tour.ts rename to webui/src/tour.ts diff --git a/arroyo-console/src/udf_state.ts b/webui/src/udf_state.ts similarity index 100% rename from arroyo-console/src/udf_state.ts rename to webui/src/udf_state.ts diff --git a/arroyo-console/src/vite-env.d.ts b/webui/src/vite-env.d.ts similarity index 100% rename from arroyo-console/src/vite-env.d.ts rename to webui/src/vite-env.d.ts diff --git a/arroyo-console/tsconfig.json b/webui/tsconfig.json similarity index 100% rename from arroyo-console/tsconfig.json rename to webui/tsconfig.json diff --git a/arroyo-console/tsconfig.node.json b/webui/tsconfig.node.json similarity index 100% rename from arroyo-console/tsconfig.node.json rename to webui/tsconfig.node.json diff --git a/arroyo-console/vite.config.ts b/webui/vite.config.ts similarity index 100% rename from arroyo-console/vite.config.ts rename to webui/vite.config.ts