From d81d9e1baf7ce5290d43670ebd2a4eeff7d70d79 Mon Sep 17 00:00:00 2001 From: hershd23 Date: Mon, 14 Feb 2022 22:55:37 +0530 Subject: [PATCH 1/5] Signed-off-by: hershd23 Changes made to configure the self-hosted-runner modified: .github/workflows/configurable-benchmark-test.yaml --- .github/workflows/configurable-benchmark-test.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/configurable-benchmark-test.yaml b/.github/workflows/configurable-benchmark-test.yaml index d831ffe..e36453b 100644 --- a/.github/workflows/configurable-benchmark-test.yaml +++ b/.github/workflows/configurable-benchmark-test.yaml @@ -31,15 +31,16 @@ on: jobs: manual-test: name: Configurable Benchmark Test - runs-on: ubuntu-latest + runs-on: self-hosted if: ${{ github.event_name == 'workflow_dispatch' }} steps: - name: Setup Kubernetes - uses: manusa/actions-setup-minikube@v2.4.1 + uses: manusa/actions-setup-minikube@v2.4.3 with: minikube version: 'v1.23.2' - kubernetes version: 'v1.22.2' + kubernetes version: 'v1.23.2' driver: docker + start args: "--v=5" - name: Checkout Code uses: actions/checkout@v2 @@ -51,7 +52,7 @@ jobs: shell: bash - name: Run Benchmark Tests - uses: layer5io/meshery-smp-action@master + uses: hershd23/meshery-smp-action@self-hosted-runner with: provider_token: ${{ secrets.MESHERY_TOKEN }} platform: docker From 33423db0e1211e845444ce262010caad70ae7497 Mon Sep 17 00:00:00 2001 From: hershd23 Date: Tue, 15 Feb 2022 23:01:04 +0530 Subject: [PATCH 2/5] Pointing back to action on layer5io/meshery-smp-action@self-hosted Signed-off-by: hershd23 --- .github/workflows/configurable-benchmark-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/configurable-benchmark-test.yaml b/.github/workflows/configurable-benchmark-test.yaml index e36453b..e59af1b 100644 --- a/.github/workflows/configurable-benchmark-test.yaml +++ b/.github/workflows/configurable-benchmark-test.yaml @@ -52,7 +52,7 @@ jobs: shell: bash - name: Run Benchmark Tests - uses: hershd23/meshery-smp-action@self-hosted-runner + uses: layer5io/meshery-smp-action@self-hosted with: provider_token: ${{ secrets.MESHERY_TOKEN }} platform: docker From 60e3848db7d100d4f91688870bb68228a1bd1482 Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Fri, 25 Feb 2022 17:10:45 +0800 Subject: [PATCH 3/5] Automating initialization of on-demand self-hosted CNCF CIL runner The commit automates below steps: 1. Create a CNCF CIL machine and register it as a self-hosted runner 2. Run SMP benchmarks on the self-hosted runner 3. Stop and remove the CNCF CIL machine and self-hosted runner Signed-off-by: Huang Xin --- ...nfigurable-benchmark-test-self-hosted.yaml | 133 ++++++++++++++++++ .../configurable-benchmark-test.yaml | 4 +- .../scripts/self-hosted-credentails.sh | 14 ++ .github/workflows/scripts/start-cil-runner.sh | 49 +++++++ .github/workflows/scripts/stop-cil-runner.sh | 17 +++ meshery.sh | 2 +- 6 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/configurable-benchmark-test-self-hosted.yaml create mode 100644 .github/workflows/scripts/self-hosted-credentails.sh create mode 100644 .github/workflows/scripts/start-cil-runner.sh create mode 100644 .github/workflows/scripts/stop-cil-runner.sh diff --git a/.github/workflows/configurable-benchmark-test-self-hosted.yaml b/.github/workflows/configurable-benchmark-test-self-hosted.yaml new file mode 100644 index 0000000..45ea655 --- /dev/null +++ b/.github/workflows/configurable-benchmark-test-self-hosted.yaml @@ -0,0 +1,133 @@ +name: Configurable Benchmark Test on Self-hosted Runner +on: + workflow_dispatch: + inputs: + profile_name: + description: "performance profile to use" + required: false + profile_filename: + description: "test configuration file" + required: false + service_mesh: + type: choice + required: false + description: "service mesh being tested" + options: + - istio + - linkerd + load_generator: + type: choice + required: false + description: "load generator to run tests with" + options: + - fortio + - wrk2 + - nighthawk + +jobs: + start-runner: + name: Start self-hosted CNCF CIL runner + runs-on: ubuntu-latest + if: ${{ github.event_name == 'workflow_dispatch' }} + outputs: + hostname: ${{ steps.start-cil-runner.outputs.hostname }} + label: ${{ steps.start-cil-runner.outputs.label }} + device_id: ${{ steps.start-cil-runner.outputs.device_id }} + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Configure CNCF CIL credentials + run: | + chmod +x .github/workflows/scripts/self-hosted-credentails.sh + .github/workflows/scripts/self-hosted-credentails.sh ${{ secrets.CNCF_CIL_TOKEN }} + shell: bash + + - name: Create registration token for CNCF CIL runner + id: getRegToken + run: | + reg_token=$(curl -s -X POST -H "Accept: application/vnd.github.v3+json" \ + -H 'Authorization: token ${{ secrets.PAT }}' \ + https://api.github.com/repos/${{github.repository}}/actions/runners/registration-token | jq -r .token) + echo REG_TOKEN=$reg_token >> $GITHUB_ENV + echo REPOSITORY=${{github.repository}} >> $GITHUB_ENV + shell: bash + + - name: Start CNCF CIL runner + id: start-cil-runner + run: | + chmod +x .github/workflows/scripts/start-cil-runner.sh + .github/workflows/scripts/start-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ github.event.inputs.service_mesh }}-${{ github.event.inputs.load_generator }} + shell: bash + + run-benchmarks: + name: Run the configurable benchmarks on the runner + needs: + - start-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + steps: + - name: Install dependencies + run: | + echo "Current user: $(whoami)" + echo "Installing kubectl..." + curl -LO https://dl.k8s.io/release/v1.23.2/bin/linux/amd64/kubectl + sudo install -o smp -g smp -m 0755 kubectl /usr/local/bin/kubectl + echo "Installing docker..." + sudo apt update -y + sudo apt install -y jq unzip apt-transport-https ca-certificates software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" + sudo apt-cache policy docker-ce + sudo apt install -y docker-ce + sudo systemctl status docker + + - name: Setup Kubernetes + uses: manusa/actions-setup-minikube@v2.4.3 + with: + minikube version: 'v1.23.2' + kubernetes version: 'v1.23.2' + driver: docker + + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Service Mesh and Deploy Application + run: | + chmod +x .github/workflows/scripts/${{ github.event.inputs.service_mesh }}_deploy.sh + .github/workflows/scripts/${{ github.event.inputs.service_mesh }}_deploy.sh + shell: bash + + - name: Run Benchmark Tests + uses: layer5io/meshery-smp-action@self-hosted + with: + provider_token: ${{ secrets.MESHERY_TOKEN }} + platform: docker + profile_name: ${{ github.event.inputs.profile_name }} + profile_filename: ${{ github.event.inputs.profile_filename }} + endpoint_url: ${{env.ENDPOINT_URL}} + service_mesh: ${{env.SERVICE_MESH}} + load_generator: ${{ github.event.inputs.load_generator }} + test_name: '${{ github.event.inputs.service_mesh }}-${{ github.event.inputs.load_generator }}-${{ github.event.inputs.profile_filename }}${{ github.event.inputs.profile_name }}' + + stop-runner: + name: Stop self-hosted runner + needs: + - start-runner # required to get output from the start-runner job + - run-benchmarks # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Stop CNCF CIL runner + run: | + chmod +x .github/workflows/scripts/stop-cil-runner.sh + .github/workflows/scripts/stop-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ needs.start-runner.outputs.device_id }} ${{ needs.start-runner.outputs.hostname }} + shell: bash + + - name: Remove CNCF CIL runner from github repository + run: | + runner_id=$(curl -s -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners | jq '.runners[] | select(.name == "${{ needs.start-runner.outputs.hostname }}") | {id}' | jq .id) + curl -X DELETE -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners/$runner_id + shell: bash diff --git a/.github/workflows/configurable-benchmark-test.yaml b/.github/workflows/configurable-benchmark-test.yaml index e59af1b..9a3bc63 100644 --- a/.github/workflows/configurable-benchmark-test.yaml +++ b/.github/workflows/configurable-benchmark-test.yaml @@ -31,7 +31,7 @@ on: jobs: manual-test: name: Configurable Benchmark Test - runs-on: self-hosted + runs-on: ubuntu-latest if: ${{ github.event_name == 'workflow_dispatch' }} steps: - name: Setup Kubernetes @@ -55,7 +55,7 @@ jobs: uses: layer5io/meshery-smp-action@self-hosted with: provider_token: ${{ secrets.MESHERY_TOKEN }} - platform: docker + platform: kubernetes profile_name: ${{ github.event.inputs.profile_name }} profile_filename: ${{ github.event.inputs.profile_filename }} endpoint_url: ${{env.ENDPOINT_URL}} diff --git a/.github/workflows/scripts/self-hosted-credentails.sh b/.github/workflows/scripts/self-hosted-credentails.sh new file mode 100644 index 0000000..4a841b7 --- /dev/null +++ b/.github/workflows/scripts/self-hosted-credentails.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# This script is used to configure and verify the token of self-hosted runner +# Token must be set as a github repo secret named "CNCF_CIL_TOKEN" + +token=$1 + +# https://metal.equinix.com/developers/api/authentication/#authentication +result=$(curl -I -s -w %{http_code} -o /dev/null -H "X-Auth-Token: $token" https://api.equinix.com/metal/v1) +if [[ $result != "200" ]]; then + echo "ERROR: Failed to authenticate the CNCF CIL token" + exit 1 +fi +echo "Authenticate CNCF CIL token sucessfully!" \ No newline at end of file diff --git a/.github/workflows/scripts/start-cil-runner.sh b/.github/workflows/scripts/start-cil-runner.sh new file mode 100644 index 0000000..7b1db8d --- /dev/null +++ b/.github/workflows/scripts/start-cil-runner.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# This script is used to start a CNCF CIL runner + +token=$1 +hostname=$2 + +# Generate random number from datastamp as the hostname of runner +label=$(date +%N) + +hostname="$hostname-$label" +echo "Creating CNCF CIL machine: $hostname..." + +# Use user_data_scripts to register the CNCF CIL runner as a self-hosted runner +user_data_scripts="#cloud-config\nusers:\n - default\n - name: smp\n groups: sudo, docker\n sudo: ALL=(ALL) NOPASSWD:ALL\n lock_passwd: true\nruncmd:\n - [runuser, -l, smp, -c, \'mkdir actions-runner && cd actions-runner\']\n - [runuser, -l, smp, -c, \'curl -o actions-runner-linux-x64-2.287.1.tar.gz -L https://github.com/actions/runner/releases/download/v2.287.1/actions-runner-linux-x64-2.287.1.tar.gz\']\n - [runuser, -l, smp, -c, \'tar xzf ./actions-runner-linux-x64-2.287.1.tar.gz\']\n - [runuser, -l, smp, -c, \'export RUNNER_ALLOW_RUNASROOT=1\']\n - [runuser, -l, smp, -c, \'./config.sh --url https://github.com/$REPOSITORY --token $REG_TOKEN --labels $hostname >> github-action-registeration.log\']\n - [runuser, -l, smp, -c, \'./run.sh >> github-action-registeration.log\']" + +# TODO: the options "operating_system", "facility", "plan" are hardcoded now, we should make them configurable +# https://metal.equinix.com/developers/api/devices/#devices-createdevice +device_id=$(curl -X POST -H "X-Auth-Token: $token" -s -H "Content-Type: application/json" \ +-d '{"operating_system": "ubuntu_20_04", "facility": "da11", "plan": "c3.small.x86", "hostname": "'"${hostname}"'", "userdata": "'"${user_data_scripts}"'"}' \ +https://api.equinix.com/metal/v1/projects/96a9d336-541b-42f7-9827-d845010da550/devices | jq -r .id) +if [[ -z $device_id ]]; then + echo "ERROR: Failed to create CNCF CIL machine: $hostname..." + exit 1 +fi + +# Wait 10 minutes until the machine is running +echo "Waiting for $hostname to run..." +n=0 +while [[ $n -le 10 ]] +do + if [[ $n -eq 10 ]]; then + echo "Waiting too long for $hostname to start, exiting..." + exit 1 + fi + sleep 1m + state=$(curl -s -H "X-Auth-Token: $token" https://api.equinix.com/metal/v1/devices/$device_id | jq -r .state) + if [[ $state == "active" ]]; then + echo "$hostname successfully created!" + break + fi + echo "Still waiting..." + let n++ +done + +# Set the outputs +echo "::set-output name=hostname::$hostname" +echo "::set-output name=label::$hostname" +echo "::set-output name=device_id::$device_id" diff --git a/.github/workflows/scripts/stop-cil-runner.sh b/.github/workflows/scripts/stop-cil-runner.sh new file mode 100644 index 0000000..9033db9 --- /dev/null +++ b/.github/workflows/scripts/stop-cil-runner.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# This script is used to start a CNCF CIL runner + +token=$1 +device_id=$2 +hostname=$3 + +echo "Removing CNCF CIL machine: $hostname..." + +# https://metal.equinix.com/developers/api/devices/#devices-deletedevice +remove_cil_result=$(curl -X DELETE -I -s -w %{http_code} -o /dev/null -H "X-Auth-Token: $token" https://api.equinix.com/metal/v1/devices/$device_id) + +if [[ $remove_cil_result != "204" ]]; then + echo "ERROR: Failed to remove CNCF CIL machine: $hostname." + exit 1 +fi \ No newline at end of file diff --git a/meshery.sh b/meshery.sh index 9701a05..35359b2 100755 --- a/meshery.sh +++ b/meshery.sh @@ -35,7 +35,7 @@ main() { kubectl config view --minify --flatten > ~/minified_config mv ~/minified_config ~/.kube/config - curl -L https://git.io/meshery | PLATFORM=$PLATFORM bash - + curl -L https://git.io/meshery | sudo PLATFORM=$PLATFORM bash - & sleep 60 } From c22122172fa9e9137e64254c7423f240d93ee6c3 Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Sat, 26 Mar 2022 00:18:46 +0800 Subject: [PATCH 4/5] Run scheduled benchmark test on self-hosted runner Signed-off-by: Huang Xin --- ...nfigurable-benchmark-test-self-hosted.yaml | 4 +- .../scheduled-benchmarks-self-hosted.yaml | 277 ++++++++++++++++++ .github/workflows/scripts/start-cil-runner.sh | 5 +- .github/workflows/scripts/stop-cil-runner.sh | 15 +- 4 files changed, 292 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/scheduled-benchmarks-self-hosted.yaml mode change 100644 => 100755 .github/workflows/scripts/start-cil-runner.sh diff --git a/.github/workflows/configurable-benchmark-test-self-hosted.yaml b/.github/workflows/configurable-benchmark-test-self-hosted.yaml index 45ea655..bc7985f 100644 --- a/.github/workflows/configurable-benchmark-test-self-hosted.yaml +++ b/.github/workflows/configurable-benchmark-test-self-hosted.yaml @@ -123,11 +123,11 @@ jobs: - name: Stop CNCF CIL runner run: | chmod +x .github/workflows/scripts/stop-cil-runner.sh - .github/workflows/scripts/stop-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ needs.start-runner.outputs.device_id }} ${{ needs.start-runner.outputs.hostname }} + .github/workflows/scripts/stop-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ needs.start-runner.outputs.hostname }} ${{ needs.start-runner.outputs.device_id }} shell: bash - name: Remove CNCF CIL runner from github repository run: | - runner_id=$(curl -s -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners | jq '.runners[] | select(.name == "${{ needs.start-runner.outputs.hostname }}") | {id}' | jq .id) + runner_id=$(curl -s -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners | jq '.runners[] | select(.name == "${{ needs.start-runner.outputs.hostname }}") | {id}' | jq -r .id) curl -X DELETE -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners/$runner_id shell: bash diff --git a/.github/workflows/scheduled-benchmarks-self-hosted.yaml b/.github/workflows/scheduled-benchmarks-self-hosted.yaml new file mode 100644 index 0000000..1235def --- /dev/null +++ b/.github/workflows/scheduled-benchmarks-self-hosted.yaml @@ -0,0 +1,277 @@ +# This workflow runs performance benchmarks with Meshery based on the configuration provided +# This workflow is scheduled to run daily but can also be triggered manually + +name: Scheduled Benchmark Tests on Self-hosted Runner +on: + # for triggering manually, provide a test configuration file name or a performance profile name + workflow_dispatch: + inputs: + profile_name: + description: "performance profile to use" + required: false + profile_filename: + description: "test configuration file" + required: false + # scheduled to run at everyday at 13:00 + schedule: + - cron: '0 13 * * *' + +jobs: + # Manual Benchmark Test + start-runners-manual: + name: Start self-hosted CNCF CIL runners for manual test + runs-on: ubuntu-latest + if: ${{ github.event_name == 'workflow_dispatch' }} + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd'] + load-generator: ['fortio', 'wrk2'] + outputs: + github_run_id: ${{ env.GITHUB_RUN_ID }} # use this github_run_id as a suffix for CIL machines and runners + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Configure CNCF CIL credentials + run: | + chmod +x .github/workflows/scripts/self-hosted-credentails.sh + .github/workflows/scripts/self-hosted-credentails.sh ${{ secrets.CNCF_CIL_TOKEN }} + shell: bash + + - name: Create registration token for CNCF CIL runner + id: getRegToken + run: | + reg_token=$(curl -s -X POST -H "Accept: application/vnd.github.v3+json" \ + -H 'Authorization: token ${{ secrets.PAT }}' \ + https://api.github.com/repos/${{github.repository}}/actions/runners/registration-token | jq -r .token) + echo REG_TOKEN=$reg_token >> $GITHUB_ENV + echo REPOSITORY=${{github.repository}} >> $GITHUB_ENV + shell: bash + + # The hostname will be like istio-fortio-1997512481 + - name: Start CNCF CIL runner + id: start-cil-runner + run: | + echo GITHUB_RUN_ID=${{ github.run_id }} >> $GITHUB_ENV + chmod +x .github/workflows/scripts/start-cil-runner.sh + .github/workflows/scripts/start-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ matrix.service-mesh }}-${{ matrix.load-generator }} + shell: bash + + manual-test: + name: Manual Benchmark Test + needs: + - start-runners-manual + runs-on: ${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ github.run_id }} + if: ${{ github.event_name == 'workflow_dispatch' }} + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd'] + load-generator: ['fortio', 'wrk2'] + steps: + - name: Install dependencies + run: | + echo "Current user: $(whoami)" + echo "Installing kubectl..." + curl -LO https://dl.k8s.io/release/v1.23.2/bin/linux/amd64/kubectl + sudo install -o smp -g smp -m 0755 kubectl /usr/local/bin/kubectl + echo "Installing docker..." + sudo apt update -y + sudo apt install -y jq unzip apt-transport-https ca-certificates software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" + sudo apt-cache policy docker-ce + sudo apt install -y docker-ce + sudo systemctl status docker + + - name: Setup Kubernetes + uses: manusa/actions-setup-minikube@v2.4.3 + with: + minikube version: 'v1.23.2' + kubernetes version: 'v1.23.2' + driver: docker + + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Service Mesh and Deploy Application + run: | + chmod +x .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + shell: bash + + - name: Get Date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H.%M.%S')" + + - name: Run Benchmark Tests + uses: layer5io/meshery-smp-action@self-hosted + with: + provider_token: ${{ secrets.MESHERY_TOKEN }} + platform: docker + profile_name: ${{ github.event.inputs.profile_name }} + profile_filename: ${{ github.event.inputs.profile_filename }} + endpoint_url: ${{env.ENDPOINT_URL}} + service_mesh: ${{env.SERVICE_MESH}} + load_generator: ${{ matrix.load-generator }} + test_name: '${{ steps.date.outputs.date }}' + + stop-runner-manual: + name: Stop self-hosted runner + needs: + - start-runners-manual # required to get output from the start-runner job + - manual-test # required to wait when the main job is done + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd'] + load-generator: ['fortio', 'wrk2'] + if: ${{ always() && github.event_name == 'workflow_dispatch' }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Stop CNCF CIL runner + run: | + chmod +x .github/workflows/scripts/stop-cil-runner.sh + .github/workflows/scripts/stop-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ needs.start-runners-manual.outputs.github_run_id }} + shell: bash + + - name: Remove CNCF CIL runner from github repository + run: | + runner_id=$(curl -s -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners | jq '.runners[] | select(.name == "${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ needs.start-runners-manual.outputs.github_run_id }}") | {id}' | jq -r .id) + curl -X DELETE -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners/$runner_id + shell: bash + + # Scheduled Benchmark Test + start-runners-scheduled: + name: Start self-hosted CNCF CIL runners for scheduled test + runs-on: ubuntu-latest + if: ${{ github.event_name == 'schedule' }} + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd', 'osm'] + load-generator: ['fortio', 'wrk2'] + test-configuration: ['load-test','soak-test'] + outputs: + github_run_id: ${{ env.GITHUB_RUN_ID }} # use this github_run_id as a suffix for CIL machines and runners + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Configure CNCF CIL credentials + run: | + chmod +x .github/workflows/scripts/self-hosted-credentails.sh + .github/workflows/scripts/self-hosted-credentails.sh ${{ secrets.CNCF_CIL_TOKEN }} + shell: bash + + - name: Create registration token for CNCF CIL runner + id: getRegToken + run: | + reg_token=$(curl -s -X POST -H "Accept: application/vnd.github.v3+json" \ + -H 'Authorization: token ${{ secrets.PAT }}' \ + https://api.github.com/repos/${{github.repository}}/actions/runners/registration-token | jq -r .token) + echo REG_TOKEN=$reg_token >> $GITHUB_ENV + echo REPOSITORY=${{github.repository}} >> $GITHUB_ENV + shell: bash + + # The hostname will be like istio-fortio-load-test-1997512481 + - name: Start CNCF CIL runner + id: start-cil-runner + run: | + echo GITHUB_RUN_ID=${{ github.run_id }} >> $GITHUB_ENV + chmod +x .github/workflows/scripts/start-cil-runner.sh + .github/workflows/scripts/start-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ matrix.test-configuration }} + shell: bash + + scheduled-test: + name: Scheduled Benchmark Test + needs: + - start-runners-scheduled + runs-on: ${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ matrix.test-configuration }}-${{ github.run_id }} + if: ${{ github.event_name == 'schedule' }} + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd', 'osm'] + load-generator: ['fortio', 'wrk2'] + test-configuration: ['load-test','soak-test'] + steps: + - name: Install dependencies + run: | + echo "Current user: $(whoami)" + echo "Installing kubectl..." + curl -LO https://dl.k8s.io/release/v1.23.2/bin/linux/amd64/kubectl + sudo install -o smp -g smp -m 0755 kubectl /usr/local/bin/kubectl + echo "Installing docker..." + sudo apt update -y + sudo apt install -y jq unzip apt-transport-https ca-certificates software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" + sudo apt-cache policy docker-ce + sudo apt install -y docker-ce + sudo systemctl status docker + + - name: Setup Kubernetes + uses: manusa/actions-setup-minikube@v2.4.3 + with: + minikube version: 'v1.23.2' + kubernetes version: 'v1.23.2' + driver: docker + + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Service Mesh and Deploy Application + run: | + chmod +x .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + shell: bash + + - name: Get Date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H.%M.%S')" + + - name: Run Benchmark Tests + uses: layer5io/meshery-smp-action@self-hosted + with: + provider_token: ${{ secrets.MESHERY_TOKEN }} + platform: docker + profile_filename: ${{ matrix.test-configuration }}.yaml + endpoint_url: ${{env.ENDPOINT_URL}} + service_mesh: ${{env.SERVICE_MESH}} + load_generator: ${{ matrix.load-generator }} + profile_name: '${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ matrix.test-configuration }}' + test_name: '${{ steps.date.outputs.date }}' + + stop-runner-scheduled: + name: Stop self-hosted runner + needs: + - start-runners-scheduled # required to get output from the start-runner job + - scheduled-test # required to wait when the main job is done + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd', 'osm'] + load-generator: ['fortio', 'wrk2'] + test-configuration: ['load-test','soak-test'] + if: ${{ always() && github.event_name == 'schedule' }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Stop CNCF CIL runner + run: | + chmod +x .github/workflows/scripts/stop-cil-runner.sh + .github/workflows/scripts/stop-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ matrix.test-configuration }}-${{ needs.start-runners-scheduled.outputs.github_run_id }} + shell: bash + + - name: Remove CNCF CIL runner from github repository + run: | + runner_id=$(curl -s -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners | jq '.runners[] | select(.name == "${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ matrix.test-configuration }}-${{ needs.start-runners-scheduled.outputs.github_run_id }}") | {id}' | jq -r .id) + curl -X DELETE -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners/$runner_id + shell: bash \ No newline at end of file diff --git a/.github/workflows/scripts/start-cil-runner.sh b/.github/workflows/scripts/start-cil-runner.sh old mode 100644 new mode 100755 index 7b1db8d..2b0613e --- a/.github/workflows/scripts/start-cil-runner.sh +++ b/.github/workflows/scripts/start-cil-runner.sh @@ -5,8 +5,9 @@ token=$1 hostname=$2 -# Generate random number from datastamp as the hostname of runner -label=$(date +%N) +# Use github.run_id as the lable of runner for scheduled benchmark test +# github.run_id is a unique number for each workflow run within a repository. +label=$GITHUB_RUN_ID hostname="$hostname-$label" echo "Creating CNCF CIL machine: $hostname..." diff --git a/.github/workflows/scripts/stop-cil-runner.sh b/.github/workflows/scripts/stop-cil-runner.sh index 9033db9..7b1d8b6 100644 --- a/.github/workflows/scripts/stop-cil-runner.sh +++ b/.github/workflows/scripts/stop-cil-runner.sh @@ -1,17 +1,22 @@ #!/usr/bin/env bash -# This script is used to start a CNCF CIL runner +# This script is used to stop a CNCF CIL runner token=$1 -device_id=$2 -hostname=$3 +hostname=$2 +device_id=$3 -echo "Removing CNCF CIL machine: $hostname..." +if [[ -z $device_id ]]; then + # If it's a scheduled benchmark test, we cannot get the orrespondence between hostname and + # device_id from previous job, so we need to use hostname to retrive device_id + device_id=$(curl -H "X-Auth-Token: $token " https://api.equinix.com/metal/v1/projects/96a9d336-541b-42f7-9827-d845010da550/devices | jq '.devices[] | select(.hostname == '\"$hostname\"') | {id}' | jq -r .id) +fi +echo "Removing CNCF CIL machine: $hostname, device id: $device_id..." # https://metal.equinix.com/developers/api/devices/#devices-deletedevice remove_cil_result=$(curl -X DELETE -I -s -w %{http_code} -o /dev/null -H "X-Auth-Token: $token" https://api.equinix.com/metal/v1/devices/$device_id) if [[ $remove_cil_result != "204" ]]; then - echo "ERROR: Failed to remove CNCF CIL machine: $hostname." + echo "ERROR: Failed to remove CNCF CIL machine: $hostname, device id: $device_id." exit 1 fi \ No newline at end of file From 724977d6d7be0574adbbb9bcf8c4df4e9b0b4df9 Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Fri, 15 Apr 2022 10:02:51 +0800 Subject: [PATCH 5/5] Cherry-pick self-hosted to master Signed-off-by: Huang Xin --- .../workflows/configurable-benchmark-test-self-hosted.yaml | 2 +- .github/workflows/configurable-benchmark-test.yaml | 4 ++-- .github/workflows/scheduled-benchmarks-self-hosted.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/configurable-benchmark-test-self-hosted.yaml b/.github/workflows/configurable-benchmark-test-self-hosted.yaml index bc7985f..271cdf9 100644 --- a/.github/workflows/configurable-benchmark-test-self-hosted.yaml +++ b/.github/workflows/configurable-benchmark-test-self-hosted.yaml @@ -98,7 +98,7 @@ jobs: shell: bash - name: Run Benchmark Tests - uses: layer5io/meshery-smp-action@self-hosted + uses: layer5io/meshery-smp-action@master with: provider_token: ${{ secrets.MESHERY_TOKEN }} platform: docker diff --git a/.github/workflows/configurable-benchmark-test.yaml b/.github/workflows/configurable-benchmark-test.yaml index 9a3bc63..7e1940f 100644 --- a/.github/workflows/configurable-benchmark-test.yaml +++ b/.github/workflows/configurable-benchmark-test.yaml @@ -52,10 +52,10 @@ jobs: shell: bash - name: Run Benchmark Tests - uses: layer5io/meshery-smp-action@self-hosted + uses: layer5io/meshery-smp-action@master with: provider_token: ${{ secrets.MESHERY_TOKEN }} - platform: kubernetes + platform: docker profile_name: ${{ github.event.inputs.profile_name }} profile_filename: ${{ github.event.inputs.profile_filename }} endpoint_url: ${{env.ENDPOINT_URL}} diff --git a/.github/workflows/scheduled-benchmarks-self-hosted.yaml b/.github/workflows/scheduled-benchmarks-self-hosted.yaml index 1235def..eefdf67 100644 --- a/.github/workflows/scheduled-benchmarks-self-hosted.yaml +++ b/.github/workflows/scheduled-benchmarks-self-hosted.yaml @@ -106,7 +106,7 @@ jobs: run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H.%M.%S')" - name: Run Benchmark Tests - uses: layer5io/meshery-smp-action@self-hosted + uses: layer5io/meshery-smp-action@master with: provider_token: ${{ secrets.MESHERY_TOKEN }} platform: docker