Eventstream destruction deadlock

.builder/actions/sdk-ci-test-setup.py

@@ -0,0 +1,20 @@
+import Builder
+
+class SdkCiTestSetup(Builder.Action):
+
+    def run(self, env):
+
+        actions = []
+        java_sdk_dir = None
+
+        try:
+            java_sdk_dir = Builder.SetupEventStreamEchoServer().run(env)
+            Builder.SetupCrossCICrtEnvironment().run(env)
+        except Exception as ex:
+            print(f'Failure while setting up tests: {ex}')
+            actions.append("exit 1")
+        finally:
+            if java_sdk_dir:
+                env.shell.rm(java_sdk_dir)
+
+        return Builder.Script(actions, name='sdk-ci-test-setup')

.builder/actions/setup-eventstream-server.py

@@ -0,0 +1,125 @@
+import atexit
+import Builder
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+class SetupEventstreamServer(Builder.Action):
+
+    def _build_and_run_eventstream_echo_server(self, env):
+        java_sdk_dir = None
+
+        try:
+            env.shell.exec(["mvn", "--version"], check=True)
+
+            # maven is installed, so this is a configuration we can start an event stream echo server
+            java_sdk_dir = env.shell.mktemp()
+
+            env.shell.exec(["git", "clone", "https://github.com/aws/aws-iot-device-sdk-java-v2"], working_dir=java_sdk_dir, check=True)
+
+            sdk_dir = os.path.join(java_sdk_dir, "aws-iot-device-sdk-java-v2", "sdk")
+            env.shell.pushd(sdk_dir)
+
+            try:
+                # The EchoTest server is in test-only code
+                env.shell.exec(["mvn", "test-compile"], check=True)
+                env.shell.exec(["mvn", "dependency:build-classpath", "-Dmdep.outputFile=classpath.txt"], check=True)
+
+                time.sleep(1)
+                with open('classpath.txt', 'r') as file:
+                    classpath = file.read()
+
+                test_class_path = os.path.join(sdk_dir, "target", "test-classes")
+                target_class_path = os.path.join(sdk_dir, "target", "classes")
+                directory_separator = os.pathsep
+
+                echo_server_probe_command = [
+                    "java",
+                    "-classpath",
+                    f"{test_class_path}{directory_separator}{target_class_path}{directory_separator}{classpath}",
+                    "software.amazon.awssdk.eventstreamrpc.echotest.EchoTestServiceRunner"]
+
+                probe_command_flat = " ".join(echo_server_probe_command)
+                print(f'Echo probe command: {probe_command_flat}')
+
+                """
+                Try to run the echo server in the foreground without required arguments.  This always fails, but
+                the output can tell us whether or not the Java CRT is available on the platform (we have SDK CI
+                that runs on platforms that the Java CRT does not support).
+
+                If the CRT supports the platform, we fail with an out-of-index exception (referencing non-existent
+                command line arguments)
+
+                If the CRT does not support the platform, we fail with 
+                'java.io.IOException: Unable to open library in jar for AWS CRT'
+                """
+                probe_output = ""
+                probe = subprocess.Popen(
+                    echo_server_probe_command,
+                    stderr=subprocess.PIPE)
+                with probe:
+
+                    # Convert all output to strings, which makes it much easier to both print
+                    # and process, since all known uses of parsing output want strings anyway
+                    line = probe.stderr.readline()
+                    while (line):
+                        # ignore weird characters coming back from the shell (colors, etc)
+                        if not isinstance(line, str):
+                            line = line.decode('ascii', 'ignore')
+                        # We're reading in binary mode, so no automatic newline translation
+                        if sys.platform == 'win32':
+                            line = line.replace('\r\n', '\n')
+                        probe_output += line
+                        line = probe.stderr.readline()
+                    probe.wait()
+
+                print("Probe stderr:\n\n")
+                print(probe_output)
+
+                if "java.io.IOException" in probe_output:
+                    print("Skipping eventstream server unsupported platform")
+                    raise Exception("Java CRT not supported by this platform")
+
+                echo_server_command = [
+                    "java",
+                    "-classpath",
+                    f"{test_class_path}{directory_separator}{target_class_path}{directory_separator}{classpath}",
+                    "software.amazon.awssdk.eventstreamrpc.echotest.EchoTestServiceRunner",
+                    "127.0.0.1",
+                    "8033"]
+
+                print(f'Echo server command: {echo_server_command}')
+
+                # bypass builder's exec wrapper since it doesn't allow for background execution
+                proc = subprocess.Popen(echo_server_command)
+
+                @atexit.register
+                def _terminate_echo_server():
+                    proc.terminate()
+                    proc.wait()
+                    shutil.rmtree(java_sdk_dir)
+
+                env.shell.setenv("AWS_TEST_EVENT_STREAM_ECHO_SERVER_HOST", "127.0.0.1", quiet=False)
+                env.shell.setenv("AWS_TEST_EVENT_STREAM_ECHO_SERVER_PORT", "8033", quiet=False)
+            finally:
+                env.shell.popd()
+
+        except Exception as ex:
+            print('Failed to set up event stream server.  Eventstream CI tests will not be run.')
+            print(ex)
+
+        return
+
+    def run(self, env):
+
+        actions = []
+
+        try:
+            self._build_and_run_eventstream_echo_server(env)
+            Builder.SetupCrossCICrtEnvironment().run(env)
+        except:
+            pass
+
+        return Builder.Script(actions, name='setup-eventstream-server')

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
       - 'docs'
 
 env:
-  BUILDER_VERSION: v0.9.56
+  BUILDER_VERSION: v0.9.79
   BUILDER_SOURCE: releases
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   PACKAGE_NAME: aws-iot-device-sdk-cpp-v2
@@ -57,7 +57,7 @@ jobs:
       id-token: write # This is required for requesting the JWT
     steps:
       - name: configure AWS credentials (containers)
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
           aws-region: ${{ env.AWS_DEFAULT_REGION }}
@@ -90,7 +90,7 @@ jobs:
           aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
           ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }}
   linux-compiler-compat:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         compiler:
@@ -153,6 +153,11 @@ jobs:
     permissions:
       id-token: write # This is required for requesting the JWT
     steps:
+      - name: configure AWS credentials (containers)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
         run: |
           md ${{ env.CI_FOLDER }}
@@ -202,6 +207,11 @@ jobs:
     permissions:
       id-token: write # This is required for requesting the JWT
     steps:
+      - name: configure AWS credentials (containers)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
         run: |
           md ${{ env.CI_FOLDER }}
@@ -247,6 +257,11 @@ jobs:
     permissions:
       id-token: write # This is required for requesting the JWT
     steps:
+      - name: configure AWS credentials (containers)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
         run: |
           md ${{ env.CI_FOLDER }}
@@ -292,30 +307,40 @@ jobs:
     permissions:
       id-token: write # This is required for requesting the JWT
     steps:
-    - name: Build ${{ env.PACKAGE_NAME }} + consumers
-      run: |
-        md ${{ env.CI_FOLDER }}
-        cd ${{ env.CI_FOLDER }}
-        python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder.pyz')"
-        python builder.pyz build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DUSE_CPU_EXTENSIONS=OFF
-    - name: Running samples in CI setup
-      run: |
-        python -m pip install boto3
-    - name: configure AWS credentials (CyclePubSub)
-      uses: aws-actions/configure-aws-credentials@v2
-      with:
-        role-to-assume: ${{ env.CI_CYCLEPUBSUB_ROLE }}
-        aws-region: ${{ env.AWS_DEFAULT_REGION }}
-    - name: Run and check AppVerifier
-      run: |
-        cd ${{ env.CI_FOLDER }}
-        echo "Starting to run AppVerifier with cycle pub-sub sample"
-        python ${{ env.CI_UTILS_FOLDER }}/appverifier_launch_sample.py --sample_file ".\aws-iot-device-sdk-cpp-v2\build\samples\pub_sub\cycle_pub_sub\RelWithDebInfo\cycle-pub-sub.exe" --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/CyclePubSub/cert' --sample_secret_private_key 'ci/CyclePubSub/key'
+      - name: configure AWS credentials (containers)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: Build ${{ env.PACKAGE_NAME }} + consumers
+        run: |
+          md ${{ env.CI_FOLDER }}
+          cd ${{ env.CI_FOLDER }}
+          python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder.pyz')"
+          python builder.pyz build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DUSE_CPU_EXTENSIONS=OFF
+      - name: Running samples in CI setup
+        run: |
+          python -m pip install boto3
+      - name: configure AWS credentials (CyclePubSub)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_CYCLEPUBSUB_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: Run and check AppVerifier
+        run: |
+          cd ${{ env.CI_FOLDER }}
+          echo "Starting to run AppVerifier with cycle pub-sub sample"
+          python ${{ env.CI_UTILS_FOLDER }}/appverifier_launch_sample.py --sample_file ".\aws-iot-device-sdk-cpp-v2\build\samples\pub_sub\cycle_pub_sub\RelWithDebInfo\cycle-pub-sub.exe" --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/CyclePubSub/cert' --sample_secret_private_key 'ci/CyclePubSub/key'
   windows-shared-lib:
     runs-on: windows-latest
     permissions:
       id-token: write # This is required for requesting the JWT
     steps:
+      - name: configure AWS credentials (containers)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
         run: |
           md ${{ env.CI_FOLDER }}
@@ -334,6 +359,11 @@ jobs:
       id-token: write # This is required for requesting the JWT
       security-events: write # This is required for pkcs12 sample to sign the key
     steps:
+      - name: configure AWS credentials (containers)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
         run: |
           python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')"
@@ -398,6 +428,11 @@ jobs:
     permissions:
       id-token: write # This is required for requesting the JWT
     steps:
+      - name: configure AWS credentials (containers)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
         run: |
           python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
@@ -458,6 +493,11 @@ jobs:
           sudo apt install cmake
           gcc --version
           cmake --version
+      - name: configure AWS credentials (containers)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
         run: |
           echo "Downloading source"
@@ -694,6 +734,11 @@ jobs:
     permissions:
       id-token: write # This is required for requesting the JWT
     steps:
+      - name: configure AWS credentials (containers)
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
         run: |
           echo "Downloading source"

builder.json

@@ -1,5 +1,8 @@
 {
   "name": "aws-iot-device-sdk-cpp-v2",
+  "packages": [
+    "maven"
+  ],
   "!cmake_args": [
     "-DPERFORM_HEADER_CHECK=OFF",
     "-DS2N_NO_PQ_ASM=ON"
@@ -23,6 +26,10 @@
     "build",
     "build-samples"
   ],
+  "test_steps": [
+    "setup-eventstream-server",
+    "test"
+  ],
   "variants" : {
     "skip_sample": {
         "!build_steps": [