Cleaning up stale resources, adjusted logging (#2117)

* Global log level

* Add progress bar and run the deletion script

* Undo git_shallow for non-tagged versions

* Install dependencies for AWS SDK CPP

* Use ubuntu 22 container for the codecov

* Downgrade to Ubuntu 20

* Add CD and change to bash

* Install curl

* Increase history for codecov

* Seperate the steps for generating the report and uploading it

Author: sirknightj

.github/workflows/ci.yml

@@ -15,7 +15,57 @@ env:
   DEBIAN_FRONTEND: noninteractive
 
 jobs:
+  check-aws-credentials:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: us-west-2
+
+  cleanup-old-resources:
+    needs: check-aws-credentials
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          repository: 'aws-samples/amazon-kinesis-video-streams-demos'
+          path: demos-repo
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        working-directory: ./demos-repo/python/cleanup
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Setup credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: us-west-2
+
+      - name: Run cleanup script
+        working-directory: ./demos-repo/python/cleanup
+        run: |
+          python kvs_cleanup.py
+
   mac-tests:
+    needs: cleanup-old-resources
     strategy:
       matrix:
         os:
@@ -142,7 +192,6 @@ jobs:
     env:
       CC: gcc
       CXX: g++
-      AWS_KVS_LOG_LEVEL: 2
     permissions:
       id-token: write
       contents: read
@@ -161,6 +210,7 @@ jobs:
           make -j
 
   mbedtls-ubuntu-gcc-4-4:
+    needs: cleanup-old-resources
     permissions:
       id-token: write
       contents: read
@@ -210,6 +260,7 @@ jobs:
           timeout --signal=SIGABRT 60m ./tst/webrtc_client_test
 
   ubuntu:
+    needs: cleanup-old-resources
     strategy:
       matrix:
         container:
@@ -290,20 +341,14 @@ jobs:
           ./tst/webrtc_client_test
 
   windows-msvc-openssl:
+    needs: cleanup-old-resources
     runs-on: windows-2022
-    env:
-      AWS_KVS_LOG_LEVEL: 1
     permissions:
       id-token: write
       contents: read
     steps:
       - name: Clone repository
         uses: actions/checkout@v4
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
       - name: Move cloned repo
         shell: powershell
         run: |
@@ -341,6 +386,11 @@ jobs:
           cd C:\webrtc
           git config --system core.longpaths true
           .github\build_windows_openssl.bat
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}
       - name: Run tests
         shell: powershell
         run: |

.github/workflows/codecov.yml

@@ -11,11 +11,13 @@ on:
 
 env:
   AWS_KVS_LOG_LEVEL: 7
+  DEBIAN_FRONTEND: noninteractive
 
 jobs:
   linux-gcc-codecov:
-    if: ${{ github.repo == 'awslabs/amazon-kinesis-video-streams-webrtc-sdk-c' }}
+    if: ${{ github.repository == 'awslabs/amazon-kinesis-video-streams-webrtc-sdk-c' }}
     runs-on: ubuntu-latest
+    container: public.ecr.aws/ubuntu/ubuntu:20.04_stable
     timeout-minutes: 60
 
     env:
@@ -28,12 +30,20 @@ jobs:
     steps:
       - name: Clone repository
         uses: actions/checkout@v4
-
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
         with:
-          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
-          aws-region: ${{ secrets.AWS_REGION }}
+          fetch-depth: 0  # fetch all history, for Codecov
+
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get -y install git cmake build-essential pkg-config
+          apt-get -y install zlib1g-dev libcurl4-openssl-dev
+          apt-get -y install curl
+
+      - name: Check versions
+        run: |
+          gcc --version
+          gcov --version
 
       - name: Build repository
         run: |
@@ -42,12 +52,24 @@ jobs:
           cmake .. -DCODE_COVERAGE=ON -DBUILD_TEST=ON
           make -j
 
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
       - name: Run tests
         working-directory: ./build
         run: |
           ./tst/webrtc_client_test
 
-      - name: Code coverage
+      - name: Generate report
+        working-directory: ./build
+        shell: bash
         run: |
           for test_file in $(find CMakeFiles/kvsWebrtcClient.dir CMakeFiles/kvsWebrtcSignalingClient.dir -name '*.gcno'); do gcov $test_file; done
+
+      - name: Upload report
+        shell: bash
+        run: |
           bash <(curl -s https://codecov.io/bash)

CMake/Dependencies/libawscpp-CMakeLists.txt

@@ -10,5 +10,7 @@ ExternalProject_Add(libawscpp-download
                      -DBUILD_ONLY=kinesisvideo|kinesis-video-webrtc-storage
                      -DCMAKE_INSTALL_PREFIX=${OPEN_SRC_INSTALL_PREFIX}
     BUILD_ALWAYS      TRUE
+    GIT_PROGRESS      TRUE
+    GIT_SHALLOW       TRUE
     TEST_COMMAND      ""
 )

CMake/Dependencies/libgtest-CMakeLists.txt

@@ -7,6 +7,8 @@ include(ExternalProject)
 ExternalProject_Add(libgtest-download
     GIT_REPOSITORY    https://github.com/google/googletest.git
     GIT_TAG           release-1.12.1
+    GIT_PROGRESS      TRUE
+    GIT_SHALLOW       TRUE
     PREFIX            ${CMAKE_CURRENT_BINARY_DIR}/build
     CMAKE_ARGS
         -DCMAKE_INSTALL_PREFIX=${OPEN_SRC_INSTALL_PREFIX}

CMake/Dependencies/libkvsCommonLws-CMakeLists.txt

@@ -7,6 +7,8 @@ include(ExternalProject)
 ExternalProject_Add(libkvsCommonLws-download
     GIT_REPOSITORY    https://github.com/awslabs/amazon-kinesis-video-streams-producer-c.git
     GIT_TAG           v1.5.4
+    GIT_PROGRESS      TRUE
+    GIT_SHALLOW       TRUE
     PREFIX            ${CMAKE_CURRENT_BINARY_DIR}/build
     LIST_SEPARATOR    |
     CMAKE_ARGS        

CMake/Dependencies/libmbedtls-CMakeLists.txt

@@ -22,6 +22,8 @@ ExternalProject_Add(
   project_libmbedtls
   GIT_REPOSITORY  https://github.com/ARMmbed/mbedtls.git
   GIT_TAG         v2.28.8
+  GIT_PROGRESS    TRUE
+  GIT_SHALLOW     TRUE
   PREFIX          ${CMAKE_CURRENT_BINARY_DIR}/build
   CMAKE_ARGS
     -DCMAKE_INSTALL_PREFIX=${OPEN_SRC_INSTALL_PREFIX}

CMake/Dependencies/libopenssl-CMakeLists.txt

@@ -40,6 +40,8 @@ include(ExternalProject)
 ExternalProject_Add(project_libopenssl
     GIT_REPOSITORY    https://github.com/openssl/openssl.git
     GIT_TAG           OpenSSL_1_1_1t
+    GIT_PROGRESS      TRUE
+    GIT_SHALLOW       TRUE
     PREFIX            ${CMAKE_CURRENT_BINARY_DIR}/build
     CONFIGURE_COMMAND ${CONFIGURE_COMMAND}
     BUILD_COMMAND     ${MAKE_EXE}

CMake/Dependencies/libsrtp-CMakeLists.txt

@@ -59,6 +59,7 @@ ExternalProject_Add(project_libsrtp
     GIT_REPOSITORY    https://github.com/cisco/libsrtp.git
     GIT_TAG           bd0f27ec0e299ad101a396dde3f7c90d48efc8fc
     PREFIX            ${CMAKE_CURRENT_BINARY_DIR}/build
+    GIT_PROGRESS      TRUE
     CMAKE_ARGS        -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
                       -DCMAKE_INSTALL_PREFIX:STRING=${OPEN_SRC_INSTALL_PREFIX}
                       -DENABLE_OPENSSL=${LIBSRTP_ENABLE_OPENSSL}

CMake/Dependencies/libusrsctp-CMakeLists.txt

@@ -7,6 +7,7 @@ include(ExternalProject)
 ExternalProject_Add(project_libusrsctp
     GIT_REPOSITORY    https://github.com/sctplab/usrsctp.git
     GIT_TAG           1ade45cbadfd19298d2c47dc538962d4425ad2dd
+    GIT_PROGRESS      TRUE
     PREFIX            ${CMAKE_CURRENT_BINARY_DIR}/build
     CMAKE_ARGS        -DCMAKE_INSTALL_PREFIX=${OPEN_SRC_INSTALL_PREFIX}
                       -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}

CMake/Dependencies/libwebsockets-CMakeLists.txt

@@ -31,6 +31,8 @@ endif()
 ExternalProject_Add(project_libwebsockets
     GIT_REPOSITORY    https://github.com/warmcat/libwebsockets.git
     GIT_TAG           v4.3.3
+    GIT_PROGRESS      TRUE
+    GIT_SHALLOW       TRUE
     PATCH_COMMAND     ${PATCH_COMMAND}
     PREFIX            ${CMAKE_CURRENT_BINARY_DIR}/build
     LIST_SEPARATOR    |

tst/main.cpp

@@ -75,6 +75,16 @@ int main(int argc, char** argv)
         if (trial >= MAX_TRIALS) {
             ::testing::GTEST_FLAG(break_on_failure) = breakOnFailure;
         }
+
+        // Increase logging verbosity on retry
+        if (trial >= 1) {
+#ifdef _WIN32
+            _putenv_s("AWS_KVS_LOG_LEVEL", "1");
+#else
+            setenv("AWS_KVS_LOG_LEVEL", "1", 1);
+#endif
+        }
+
         rc = RUN_ALL_TESTS();
 
         // If there were some tests failed, set googletest filter flag to those failed tests.