MQTT5 Support (#340)

Add MQTT5 support to aws-iot-device-sdk-java-v2

Author: TwistedTwigleg

.github/workflows/ci.yml

@@ -24,6 +24,7 @@ env:
   CI_JOBS_ROLE: ${{ secrets.AWS_CI_JOBS_ROLE }}
   CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }}
   CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }}
+  CI_MQTT5_ROLE: ${{ secrets.AWS_CI_MQTT5_ROLE }}
 
 jobs:
   linux-compat:
@@ -51,7 +52,7 @@ jobs:
         ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }}
     # NOTE: we cannot run samples or DeviceAdvisor here due to container restrictions
 
-  
+
   raspberry:
     runs-on: ubuntu-20.04 # latest
     strategy:
@@ -74,7 +75,7 @@ jobs:
       run: |
         aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
         ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }}
- 
+
 
   windows:
     runs-on: windows-latest
@@ -100,6 +101,17 @@ jobs:
       run: |
         python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
         python builder.pyz build -p ${{ env.PACKAGE_NAME }} --spec=downstream
+    - name: configure AWS credentials (MQTT5)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.CI_MQTT5_ROLE }}
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+    - name: MQTT5 tests
+      shell: bash
+      run: |
+        source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1
+        mvn test -Dtest=Mqtt5BuilderTest -DfailIfNoTests=false
+        source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt cleanup
     - name: Running samples in CI setup
       run: |
         python -m pip install boto3
@@ -115,6 +127,14 @@ jobs:
     - name: run Windows Certificate Connect sample
       run: |
         python ./utils/run_sample_ci.py --language Java --sample_file 'samples/WindowsCertConnect' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --sample_run_certutil true --sample_main_class 'windowscertconnect.WindowsCertConnect'
+    - name: configure AWS credentials (MQTT5)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.CI_MQTT5_ROLE }}
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+    - name: run MQTT5 PubSub sample
+      run: |
+        python ./utils/run_sample_ci.py --language Java --sample_file 'samples/Mqtt5/PubSub' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/mqtt5/us/mqtt5_thing/cert' --sample_secret_private_key 'ci/mqtt5/us/mqtt5_thing/key' --sample_main_class 'mqtt5.pubsub.PubSub'
     - name: configure AWS credentials (Device Advisor)
       uses: aws-actions/configure-aws-credentials@v1
       with:
@@ -149,6 +169,16 @@ jobs:
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
         chmod a+x builder
         ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream
+    - name: configure AWS credentials (MQTT5)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.CI_MQTT5_ROLE }}
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+    - name: MQTT5 tests
+      run: |
+        source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1
+        mvn test -Dtest=Mqtt5BuilderTest -DfailIfNoTests=false
+        source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt cleanup
     - name: Running samples in CI setup
       run: |
         python3 -m pip install boto3
@@ -161,6 +191,14 @@ jobs:
     - name: run PubSub sample
       run: |
         python3 ./utils/run_sample_ci.py --language Java --sample_file 'samples/BasicPubSub' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --sample_main_class 'pubsub.PubSub'
+    - name: configure AWS credentials (MQTT5)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.CI_MQTT5_ROLE }}
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+    - name: run MQTT5 PubSub sample
+      run: |
+        python3 ./utils/run_sample_ci.py --language Java --sample_file 'samples/Mqtt5/PubSub' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/mqtt5/us/mqtt5_thing/cert' --sample_secret_private_key 'ci/mqtt5/us/mqtt5_thing/key' --sample_main_class 'mqtt5.pubsub.PubSub'
     - name: configure AWS credentials (Device Advisor)
       uses: aws-actions/configure-aws-credentials@v1
       with:
@@ -195,6 +233,16 @@ jobs:
           java -version
           mvn -B test -Daws.crt.debugnative=true
           mvn install -Dmaven.test.skip
+      - name: configure AWS credentials (MQTT5)
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ env.CI_MQTT5_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: MQTT5 tests
+        run: |
+          source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt us-east-1
+          mvn test -Dtest=Mqtt5BuilderTest -DfailIfNoTests=false
+          source utils/mqtt5_test_setup.sh s3://iot-sdk-ci-bucket-us-east1/IotUsProdMqtt5EnvironmentVariables.txt cleanup
       - name: Running samples in CI setup
         run: |
           python3 -m pip install boto3
@@ -206,6 +254,14 @@ jobs:
       - name: run PubSub sample
         run: |
           python3 ./utils/run_sample_ci.py --language Java --sample_file 'samples/BasicPubSub' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --sample_main_class 'pubsub.PubSub'
+      - name: configure AWS credentials (MQTT5)
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ env.CI_MQTT5_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: run MQTT5 PubSub sample
+        run: |
+          python3 ./utils/run_sample_ci.py --language Java --sample_file 'samples/Mqtt5/PubSub' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/mqtt5/us/mqtt5_thing/cert' --sample_secret_private_key 'ci/mqtt5/us/mqtt5_thing/key' --sample_main_class 'mqtt5.pubsub.PubSub'
       - name: configure AWS credentials (Device Advisor)
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -331,3 +387,11 @@ jobs:
           Sample_UUID=$(python3  -c "import uuid; print (uuid.uuid4())")
           python3 ./utils/run_sample_ci.py --language Java --sample_file 'samples/Identity' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/FleetProvisioning/cert' --sample_secret_private_key 'ci/FleetProvisioning/key' --sample_arguments "--template_name CI_FleetProvisioning_Template --template_parameters '{SerialNumber:${Sample_UUID}}'" --sample_main_class 'identity.FleetProvisioningSample'
           python3 utils/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1"
+      - name: configure AWS credentials (MQTT5)
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ env.CI_MQTT5_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: run MQTT5 PubSub sample
+        run: |
+          python3 ./utils/run_sample_ci.py --language Java --sample_file 'samples/Mqtt5/PubSub' --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/mqtt5/us/mqtt5_thing/cert' --sample_secret_private_key 'ci/mqtt5/us/mqtt5_thing/key' --sample_main_class 'mqtt5.pubsub.PubSub'

README.md

@@ -80,8 +80,8 @@ mvn clean install
 mkdir sdk-workspace
 cd sdk-workspace
 # Clone the CRT repository
-#     (Use the latest version of the CRT here instead of "v0.18.0")
-git clone --branch v0.19.11 --recurse-submodules https://github.com/awslabs/aws-crt-java.git
+#     (Use the latest version of the CRT here instead of "v0.20.0")
+git clone --branch v0.20.0 --recurse-submodules https://github.com/awslabs/aws-crt-java.git
 cd aws-crt-java
 # Compile and install the CRT
 mvn install -Dmaven.test.skip=true
@@ -102,8 +102,8 @@ NOTE: The shadow sample does not currently complete on android due to its depend
 mkdir sdk-workspace
 cd sdk-workspace
 # Clone the CRT repository
-#     (Use the latest version of the CRT here instead of "v0.18.0")
-git clone --branch v0.19.11 --recurse-submodules https://github.com/awslabs/aws-crt-java.git
+#     (Use the latest version of the CRT here instead of "v0.20.0")
+git clone --branch v0.20.0 --recurse-submodules https://github.com/awslabs/aws-crt-java.git
 # Compile and install the CRT for Android
 cd aws-crt-java/android
 ./gradlew connectedCheck # optional, will run the unit tests on any connected devices/emulators
@@ -127,11 +127,11 @@ repositories {
 }
 
 dependencies {
-    implementation 'software.amazon.awssdk.crt:android:0.19.11'
+    implementation 'software.amazon.awssdk.crt:android:0.20.0'
 }
 ```
 
-Replace `0.18.0` in `software.amazon.awssdk.crt:android:0.18.0` with the latest version of the CRT.
+Replace `0.20.0` in `software.amazon.awssdk.crt:android:0.20.0` with the latest version of the CRT.
 Look up the latest CRT version here: https://github.com/awslabs/aws-crt-java/releases
 
 #### Caution
@@ -149,6 +149,7 @@ Please make sure to check out our resources too before opening an issue:
 
 * Our [FAQ](./documents/FAQ.md)
 * Our [Developer Guide](https://docs.aws.amazon.com/iot/latest/developerguide/what-is-aws-iot.html) ([source](https://github.com/awsdocs/aws-iot-docs))
+* [MQTT5 User Guide](./documents/MQTT5_Userguide.md)
 * Check for similar [Issues](https://github.com/aws/aws-iot-device-sdk-java-v2/issues)
 * [AWS IoT Core Documentation](https://docs.aws.amazon.com/iot/)
 * [Dev Blog](https://aws.amazon.com/blogs/?awsf.blog-master-iot=category-internet-of-things%23amazon-freertos%7Ccategory-internet-of-things%23aws-greengrass%7Ccategory-internet-of-things%23aws-iot-analytics%7Ccategory-internet-of-things%23aws-iot-button%7Ccategory-internet-of-things%23aws-iot-device-defender%7Ccategory-internet-of-things%23aws-iot-device-management%7Ccategory-internet-of-things%23aws-iot-platform)

android/iotdevicesdk/build.gradle

@@ -91,7 +91,7 @@ repositories {
 }
 
 dependencies {
-    api 'software.amazon.awssdk.crt:aws-crt-android:0.19.11'
+    api 'software.amazon.awssdk.crt:aws-crt-android:0.20.0'
     implementation 'org.slf4j:slf4j-api:1.7.30'
     implementation 'com.google.code.gson:gson:2.9.0'
     implementation 'androidx.appcompat:appcompat:1.1.0'

builder.json

@@ -6,8 +6,7 @@
     "build_steps": [
         "mvn -B compile"
     ],
-    "test_steps": [
-    ],
+    "test_steps": [],
     "imports": [
         "JDK8"
     ],

codebuild/samples/connect-auth-linux.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+env
+
+pushd $CODEBUILD_SRC_DIR/samples/CustomAuthorizerConnect
+
+ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+
+AUTH_NAME=$(aws secretsmanager get-secret-value --secret-id "unit-test/authorizer-name" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+AUTH_PASSWORD=$(aws secretsmanager get-secret-value --secret-id "unit-test/authorizer-password" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+
+mvn compile
+
+echo "Mqtt Connect with Custom Authorizer test"
+mvn exec:java -Dexec.mainClass="customauthorizerconnect.CustomAuthorizerConnect" -Daws.crt.ci="True" -Dexec.arguments="--endpoint,$ENDPOINT,--custom_auth_authorizer_name,$AUTH_NAME,--custom_auth_password,$AUTH_PASSWORD"
+
+popd

codebuild/samples/customkeyops-linux.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+env
+
+pushd $CODEBUILD_SRC_DIR/samples/CustomKeyOpsPubSub
+
+ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+
+mvn compile
+
+echo "Custom Key Ops test"
+mvn exec:java -Dexec.mainClass="customkeyopspubsub.CustomKeyOpsPubSub" -Daws.crt.ci="True" -Dexec.arguments="--endpoint,$ENDPOINT,--key,/tmp/privatekey_p8.pem,--cert,/tmp/certificate.pem"
+
+popd

codebuild/samples/linux-smoke-tests.yml

@@ -24,6 +24,7 @@ phases:
       - $CODEBUILD_SRC_DIR/codebuild/samples/pkcs11-connect-linux.sh
       - $CODEBUILD_SRC_DIR/codebuild/samples/pubsub-linux.sh
       - $CODEBUILD_SRC_DIR/codebuild/samples/shadow-linux.sh
+      - $CODEBUILD_SRC_DIR/codebuild/samples/pubsub-mqtt5-linux.sh
   post_build:
     commands:
       - echo Build completed on `date`

codebuild/samples/pubsub-mqtt5-linux.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+env
+
+pushd $CODEBUILD_SRC_DIR/samples/Mqtt5/PubSub
+
+ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+
+mvn compile
+
+echo "MQTT5 PubSub test"
+mvn exec:java -Dexec.mainClass="mqtt5.pubsub.PubSub" -Daws.crt.ci="True" -Dexec.arguments="--endpoint,$ENDPOINT,--key,/tmp/privatekey.pem,--cert,/tmp/certificate.pem"
+
+popd

deviceadvisor/script/DATestRun.py

@@ -131,7 +131,6 @@ def sleep_with_backoff(base, max):
         create_cert_response = client.create_keys_and_certificate(
             setAsActive=True
         )
-
         # write certificate to file
         f = open(certificate_path, "w")
         f.write(create_cert_response['certificatePem'])
@@ -255,7 +254,6 @@ def sleep_with_backoff(base, max):
                 suiteDefinitionId=DATestConfig['test_suite_ids'][test_name],
                 suiteRunId=test_start_response['suiteRunId']
             )
-
             # If the status is PENDING or the responds does not loaded, the test suite is still loading
             if (test_result_responds['status'] == 'PENDING' or
             len(test_result_responds['testResult']['groups']) == 0 or # test group has not been loaded

deviceadvisor/tests/DATestUtils/DATestUtils.java

@@ -34,22 +34,18 @@ public static Boolean init(TestType type)
         thing_name = System.getenv(ENV_THING_NAME);
         shadowProperty = System.getenv(ENV_SHADOW_PROPERTY);
         shadowValue = System.getenv(ENV_SHADOW_VALUE_SET);
-
         if (endpoint.isEmpty() || certificatePath.isEmpty() || keyPath.isEmpty())
         {
             return false;
         }
-
         if (topic.isEmpty() && type == TestType.SUB_PUB)
         {
             return false;
         }
-
         if ((thing_name.isEmpty() || shadowProperty.isEmpty() || shadowValue.isEmpty()) && type == TestType.SHADOW)
         {
             return false;
         }
-
         return true;
 
     }