diff --git a/.github/workflows/configurable-benchmark-test-self-hosted.yaml b/.github/workflows/configurable-benchmark-test-self-hosted.yaml new file mode 100644 index 0000000..271cdf9 --- /dev/null +++ b/.github/workflows/configurable-benchmark-test-self-hosted.yaml @@ -0,0 +1,133 @@ +name: Configurable Benchmark Test on Self-hosted Runner +on: + workflow_dispatch: + inputs: + profile_name: + description: "performance profile to use" + required: false + profile_filename: + description: "test configuration file" + required: false + service_mesh: + type: choice + required: false + description: "service mesh being tested" + options: + - istio + - linkerd + load_generator: + type: choice + required: false + description: "load generator to run tests with" + options: + - fortio + - wrk2 + - nighthawk + +jobs: + start-runner: + name: Start self-hosted CNCF CIL runner + runs-on: ubuntu-latest + if: ${{ github.event_name == 'workflow_dispatch' }} + outputs: + hostname: ${{ steps.start-cil-runner.outputs.hostname }} + label: ${{ steps.start-cil-runner.outputs.label }} + device_id: ${{ steps.start-cil-runner.outputs.device_id }} + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Configure CNCF CIL credentials + run: | + chmod +x .github/workflows/scripts/self-hosted-credentails.sh + .github/workflows/scripts/self-hosted-credentails.sh ${{ secrets.CNCF_CIL_TOKEN }} + shell: bash + + - name: Create registration token for CNCF CIL runner + id: getRegToken + run: | + reg_token=$(curl -s -X POST -H "Accept: application/vnd.github.v3+json" \ + -H 'Authorization: token ${{ secrets.PAT }}' \ + https://api.github.com/repos/${{github.repository}}/actions/runners/registration-token | jq -r .token) + echo REG_TOKEN=$reg_token >> $GITHUB_ENV + echo REPOSITORY=${{github.repository}} >> $GITHUB_ENV + shell: bash + + - name: Start CNCF CIL runner + id: start-cil-runner + run: | + chmod +x .github/workflows/scripts/start-cil-runner.sh + .github/workflows/scripts/start-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ github.event.inputs.service_mesh }}-${{ github.event.inputs.load_generator }} + shell: bash + + run-benchmarks: + name: Run the configurable benchmarks on the runner + needs: + - start-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner + steps: + - name: Install dependencies + run: | + echo "Current user: $(whoami)" + echo "Installing kubectl..." + curl -LO https://dl.k8s.io/release/v1.23.2/bin/linux/amd64/kubectl + sudo install -o smp -g smp -m 0755 kubectl /usr/local/bin/kubectl + echo "Installing docker..." + sudo apt update -y + sudo apt install -y jq unzip apt-transport-https ca-certificates software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" + sudo apt-cache policy docker-ce + sudo apt install -y docker-ce + sudo systemctl status docker + + - name: Setup Kubernetes + uses: manusa/actions-setup-minikube@v2.4.3 + with: + minikube version: 'v1.23.2' + kubernetes version: 'v1.23.2' + driver: docker + + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Service Mesh and Deploy Application + run: | + chmod +x .github/workflows/scripts/${{ github.event.inputs.service_mesh }}_deploy.sh + .github/workflows/scripts/${{ github.event.inputs.service_mesh }}_deploy.sh + shell: bash + + - name: Run Benchmark Tests + uses: layer5io/meshery-smp-action@master + with: + provider_token: ${{ secrets.MESHERY_TOKEN }} + platform: docker + profile_name: ${{ github.event.inputs.profile_name }} + profile_filename: ${{ github.event.inputs.profile_filename }} + endpoint_url: ${{env.ENDPOINT_URL}} + service_mesh: ${{env.SERVICE_MESH}} + load_generator: ${{ github.event.inputs.load_generator }} + test_name: '${{ github.event.inputs.service_mesh }}-${{ github.event.inputs.load_generator }}-${{ github.event.inputs.profile_filename }}${{ github.event.inputs.profile_name }}' + + stop-runner: + name: Stop self-hosted runner + needs: + - start-runner # required to get output from the start-runner job + - run-benchmarks # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Stop CNCF CIL runner + run: | + chmod +x .github/workflows/scripts/stop-cil-runner.sh + .github/workflows/scripts/stop-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ needs.start-runner.outputs.hostname }} ${{ needs.start-runner.outputs.device_id }} + shell: bash + + - name: Remove CNCF CIL runner from github repository + run: | + runner_id=$(curl -s -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners | jq '.runners[] | select(.name == "${{ needs.start-runner.outputs.hostname }}") | {id}' | jq -r .id) + curl -X DELETE -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners/$runner_id + shell: bash diff --git a/.github/workflows/configurable-benchmark-test.yaml b/.github/workflows/configurable-benchmark-test.yaml index d831ffe..7e1940f 100644 --- a/.github/workflows/configurable-benchmark-test.yaml +++ b/.github/workflows/configurable-benchmark-test.yaml @@ -35,11 +35,12 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' }} steps: - name: Setup Kubernetes - uses: manusa/actions-setup-minikube@v2.4.1 + uses: manusa/actions-setup-minikube@v2.4.3 with: minikube version: 'v1.23.2' - kubernetes version: 'v1.22.2' + kubernetes version: 'v1.23.2' driver: docker + start args: "--v=5" - name: Checkout Code uses: actions/checkout@v2 diff --git a/.github/workflows/scheduled-benchmarks-self-hosted.yaml b/.github/workflows/scheduled-benchmarks-self-hosted.yaml new file mode 100644 index 0000000..eefdf67 --- /dev/null +++ b/.github/workflows/scheduled-benchmarks-self-hosted.yaml @@ -0,0 +1,277 @@ +# This workflow runs performance benchmarks with Meshery based on the configuration provided +# This workflow is scheduled to run daily but can also be triggered manually + +name: Scheduled Benchmark Tests on Self-hosted Runner +on: + # for triggering manually, provide a test configuration file name or a performance profile name + workflow_dispatch: + inputs: + profile_name: + description: "performance profile to use" + required: false + profile_filename: + description: "test configuration file" + required: false + # scheduled to run at everyday at 13:00 + schedule: + - cron: '0 13 * * *' + +jobs: + # Manual Benchmark Test + start-runners-manual: + name: Start self-hosted CNCF CIL runners for manual test + runs-on: ubuntu-latest + if: ${{ github.event_name == 'workflow_dispatch' }} + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd'] + load-generator: ['fortio', 'wrk2'] + outputs: + github_run_id: ${{ env.GITHUB_RUN_ID }} # use this github_run_id as a suffix for CIL machines and runners + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Configure CNCF CIL credentials + run: | + chmod +x .github/workflows/scripts/self-hosted-credentails.sh + .github/workflows/scripts/self-hosted-credentails.sh ${{ secrets.CNCF_CIL_TOKEN }} + shell: bash + + - name: Create registration token for CNCF CIL runner + id: getRegToken + run: | + reg_token=$(curl -s -X POST -H "Accept: application/vnd.github.v3+json" \ + -H 'Authorization: token ${{ secrets.PAT }}' \ + https://api.github.com/repos/${{github.repository}}/actions/runners/registration-token | jq -r .token) + echo REG_TOKEN=$reg_token >> $GITHUB_ENV + echo REPOSITORY=${{github.repository}} >> $GITHUB_ENV + shell: bash + + # The hostname will be like istio-fortio-1997512481 + - name: Start CNCF CIL runner + id: start-cil-runner + run: | + echo GITHUB_RUN_ID=${{ github.run_id }} >> $GITHUB_ENV + chmod +x .github/workflows/scripts/start-cil-runner.sh + .github/workflows/scripts/start-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ matrix.service-mesh }}-${{ matrix.load-generator }} + shell: bash + + manual-test: + name: Manual Benchmark Test + needs: + - start-runners-manual + runs-on: ${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ github.run_id }} + if: ${{ github.event_name == 'workflow_dispatch' }} + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd'] + load-generator: ['fortio', 'wrk2'] + steps: + - name: Install dependencies + run: | + echo "Current user: $(whoami)" + echo "Installing kubectl..." + curl -LO https://dl.k8s.io/release/v1.23.2/bin/linux/amd64/kubectl + sudo install -o smp -g smp -m 0755 kubectl /usr/local/bin/kubectl + echo "Installing docker..." + sudo apt update -y + sudo apt install -y jq unzip apt-transport-https ca-certificates software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" + sudo apt-cache policy docker-ce + sudo apt install -y docker-ce + sudo systemctl status docker + + - name: Setup Kubernetes + uses: manusa/actions-setup-minikube@v2.4.3 + with: + minikube version: 'v1.23.2' + kubernetes version: 'v1.23.2' + driver: docker + + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Service Mesh and Deploy Application + run: | + chmod +x .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + shell: bash + + - name: Get Date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H.%M.%S')" + + - name: Run Benchmark Tests + uses: layer5io/meshery-smp-action@master + with: + provider_token: ${{ secrets.MESHERY_TOKEN }} + platform: docker + profile_name: ${{ github.event.inputs.profile_name }} + profile_filename: ${{ github.event.inputs.profile_filename }} + endpoint_url: ${{env.ENDPOINT_URL}} + service_mesh: ${{env.SERVICE_MESH}} + load_generator: ${{ matrix.load-generator }} + test_name: '${{ steps.date.outputs.date }}' + + stop-runner-manual: + name: Stop self-hosted runner + needs: + - start-runners-manual # required to get output from the start-runner job + - manual-test # required to wait when the main job is done + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd'] + load-generator: ['fortio', 'wrk2'] + if: ${{ always() && github.event_name == 'workflow_dispatch' }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Stop CNCF CIL runner + run: | + chmod +x .github/workflows/scripts/stop-cil-runner.sh + .github/workflows/scripts/stop-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ needs.start-runners-manual.outputs.github_run_id }} + shell: bash + + - name: Remove CNCF CIL runner from github repository + run: | + runner_id=$(curl -s -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners | jq '.runners[] | select(.name == "${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ needs.start-runners-manual.outputs.github_run_id }}") | {id}' | jq -r .id) + curl -X DELETE -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners/$runner_id + shell: bash + + # Scheduled Benchmark Test + start-runners-scheduled: + name: Start self-hosted CNCF CIL runners for scheduled test + runs-on: ubuntu-latest + if: ${{ github.event_name == 'schedule' }} + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd', 'osm'] + load-generator: ['fortio', 'wrk2'] + test-configuration: ['load-test','soak-test'] + outputs: + github_run_id: ${{ env.GITHUB_RUN_ID }} # use this github_run_id as a suffix for CIL machines and runners + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Configure CNCF CIL credentials + run: | + chmod +x .github/workflows/scripts/self-hosted-credentails.sh + .github/workflows/scripts/self-hosted-credentails.sh ${{ secrets.CNCF_CIL_TOKEN }} + shell: bash + + - name: Create registration token for CNCF CIL runner + id: getRegToken + run: | + reg_token=$(curl -s -X POST -H "Accept: application/vnd.github.v3+json" \ + -H 'Authorization: token ${{ secrets.PAT }}' \ + https://api.github.com/repos/${{github.repository}}/actions/runners/registration-token | jq -r .token) + echo REG_TOKEN=$reg_token >> $GITHUB_ENV + echo REPOSITORY=${{github.repository}} >> $GITHUB_ENV + shell: bash + + # The hostname will be like istio-fortio-load-test-1997512481 + - name: Start CNCF CIL runner + id: start-cil-runner + run: | + echo GITHUB_RUN_ID=${{ github.run_id }} >> $GITHUB_ENV + chmod +x .github/workflows/scripts/start-cil-runner.sh + .github/workflows/scripts/start-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ matrix.test-configuration }} + shell: bash + + scheduled-test: + name: Scheduled Benchmark Test + needs: + - start-runners-scheduled + runs-on: ${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ matrix.test-configuration }}-${{ github.run_id }} + if: ${{ github.event_name == 'schedule' }} + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd', 'osm'] + load-generator: ['fortio', 'wrk2'] + test-configuration: ['load-test','soak-test'] + steps: + - name: Install dependencies + run: | + echo "Current user: $(whoami)" + echo "Installing kubectl..." + curl -LO https://dl.k8s.io/release/v1.23.2/bin/linux/amd64/kubectl + sudo install -o smp -g smp -m 0755 kubectl /usr/local/bin/kubectl + echo "Installing docker..." + sudo apt update -y + sudo apt install -y jq unzip apt-transport-https ca-certificates software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" + sudo apt-cache policy docker-ce + sudo apt install -y docker-ce + sudo systemctl status docker + + - name: Setup Kubernetes + uses: manusa/actions-setup-minikube@v2.4.3 + with: + minikube version: 'v1.23.2' + kubernetes version: 'v1.23.2' + driver: docker + + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Service Mesh and Deploy Application + run: | + chmod +x .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + .github/workflows/scripts/${{ matrix.service-mesh }}_deploy.sh + shell: bash + + - name: Get Date + id: date + run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H.%M.%S')" + + - name: Run Benchmark Tests + uses: layer5io/meshery-smp-action@self-hosted + with: + provider_token: ${{ secrets.MESHERY_TOKEN }} + platform: docker + profile_filename: ${{ matrix.test-configuration }}.yaml + endpoint_url: ${{env.ENDPOINT_URL}} + service_mesh: ${{env.SERVICE_MESH}} + load_generator: ${{ matrix.load-generator }} + profile_name: '${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ matrix.test-configuration }}' + test_name: '${{ steps.date.outputs.date }}' + + stop-runner-scheduled: + name: Stop self-hosted runner + needs: + - start-runners-scheduled # required to get output from the start-runner job + - scheduled-test # required to wait when the main job is done + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + service-mesh: ['istio', 'linkerd', 'osm'] + load-generator: ['fortio', 'wrk2'] + test-configuration: ['load-test','soak-test'] + if: ${{ always() && github.event_name == 'schedule' }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Stop CNCF CIL runner + run: | + chmod +x .github/workflows/scripts/stop-cil-runner.sh + .github/workflows/scripts/stop-cil-runner.sh ${{ secrets.cncf_cil_token }} ${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ matrix.test-configuration }}-${{ needs.start-runners-scheduled.outputs.github_run_id }} + shell: bash + + - name: Remove CNCF CIL runner from github repository + run: | + runner_id=$(curl -s -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners | jq '.runners[] | select(.name == "${{ matrix.service-mesh }}-${{ matrix.load-generator }}-${{ matrix.test-configuration }}-${{ needs.start-runners-scheduled.outputs.github_run_id }}") | {id}' | jq -r .id) + curl -X DELETE -H 'Authorization: token ${{ secrets.PAT }}' -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/${{github.repository}}/actions/runners/$runner_id + shell: bash \ No newline at end of file diff --git a/.github/workflows/scripts/self-hosted-credentails.sh b/.github/workflows/scripts/self-hosted-credentails.sh new file mode 100644 index 0000000..4a841b7 --- /dev/null +++ b/.github/workflows/scripts/self-hosted-credentails.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# This script is used to configure and verify the token of self-hosted runner +# Token must be set as a github repo secret named "CNCF_CIL_TOKEN" + +token=$1 + +# https://metal.equinix.com/developers/api/authentication/#authentication +result=$(curl -I -s -w %{http_code} -o /dev/null -H "X-Auth-Token: $token" https://api.equinix.com/metal/v1) +if [[ $result != "200" ]]; then + echo "ERROR: Failed to authenticate the CNCF CIL token" + exit 1 +fi +echo "Authenticate CNCF CIL token sucessfully!" \ No newline at end of file diff --git a/.github/workflows/scripts/start-cil-runner.sh b/.github/workflows/scripts/start-cil-runner.sh new file mode 100755 index 0000000..2b0613e --- /dev/null +++ b/.github/workflows/scripts/start-cil-runner.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# This script is used to start a CNCF CIL runner + +token=$1 +hostname=$2 + +# Use github.run_id as the lable of runner for scheduled benchmark test +# github.run_id is a unique number for each workflow run within a repository. +label=$GITHUB_RUN_ID + +hostname="$hostname-$label" +echo "Creating CNCF CIL machine: $hostname..." + +# Use user_data_scripts to register the CNCF CIL runner as a self-hosted runner +user_data_scripts="#cloud-config\nusers:\n - default\n - name: smp\n groups: sudo, docker\n sudo: ALL=(ALL) NOPASSWD:ALL\n lock_passwd: true\nruncmd:\n - [runuser, -l, smp, -c, \'mkdir actions-runner && cd actions-runner\']\n - [runuser, -l, smp, -c, \'curl -o actions-runner-linux-x64-2.287.1.tar.gz -L https://github.com/actions/runner/releases/download/v2.287.1/actions-runner-linux-x64-2.287.1.tar.gz\']\n - [runuser, -l, smp, -c, \'tar xzf ./actions-runner-linux-x64-2.287.1.tar.gz\']\n - [runuser, -l, smp, -c, \'export RUNNER_ALLOW_RUNASROOT=1\']\n - [runuser, -l, smp, -c, \'./config.sh --url https://github.com/$REPOSITORY --token $REG_TOKEN --labels $hostname >> github-action-registeration.log\']\n - [runuser, -l, smp, -c, \'./run.sh >> github-action-registeration.log\']" + +# TODO: the options "operating_system", "facility", "plan" are hardcoded now, we should make them configurable +# https://metal.equinix.com/developers/api/devices/#devices-createdevice +device_id=$(curl -X POST -H "X-Auth-Token: $token" -s -H "Content-Type: application/json" \ +-d '{"operating_system": "ubuntu_20_04", "facility": "da11", "plan": "c3.small.x86", "hostname": "'"${hostname}"'", "userdata": "'"${user_data_scripts}"'"}' \ +https://api.equinix.com/metal/v1/projects/96a9d336-541b-42f7-9827-d845010da550/devices | jq -r .id) +if [[ -z $device_id ]]; then + echo "ERROR: Failed to create CNCF CIL machine: $hostname..." + exit 1 +fi + +# Wait 10 minutes until the machine is running +echo "Waiting for $hostname to run..." +n=0 +while [[ $n -le 10 ]] +do + if [[ $n -eq 10 ]]; then + echo "Waiting too long for $hostname to start, exiting..." + exit 1 + fi + sleep 1m + state=$(curl -s -H "X-Auth-Token: $token" https://api.equinix.com/metal/v1/devices/$device_id | jq -r .state) + if [[ $state == "active" ]]; then + echo "$hostname successfully created!" + break + fi + echo "Still waiting..." + let n++ +done + +# Set the outputs +echo "::set-output name=hostname::$hostname" +echo "::set-output name=label::$hostname" +echo "::set-output name=device_id::$device_id" diff --git a/.github/workflows/scripts/stop-cil-runner.sh b/.github/workflows/scripts/stop-cil-runner.sh new file mode 100644 index 0000000..7b1d8b6 --- /dev/null +++ b/.github/workflows/scripts/stop-cil-runner.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# This script is used to stop a CNCF CIL runner + +token=$1 +hostname=$2 +device_id=$3 + +if [[ -z $device_id ]]; then + # If it's a scheduled benchmark test, we cannot get the orrespondence between hostname and + # device_id from previous job, so we need to use hostname to retrive device_id + device_id=$(curl -H "X-Auth-Token: $token " https://api.equinix.com/metal/v1/projects/96a9d336-541b-42f7-9827-d845010da550/devices | jq '.devices[] | select(.hostname == '\"$hostname\"') | {id}' | jq -r .id) +fi + +echo "Removing CNCF CIL machine: $hostname, device id: $device_id..." +# https://metal.equinix.com/developers/api/devices/#devices-deletedevice +remove_cil_result=$(curl -X DELETE -I -s -w %{http_code} -o /dev/null -H "X-Auth-Token: $token" https://api.equinix.com/metal/v1/devices/$device_id) + +if [[ $remove_cil_result != "204" ]]; then + echo "ERROR: Failed to remove CNCF CIL machine: $hostname, device id: $device_id." + exit 1 +fi \ No newline at end of file diff --git a/meshery.sh b/meshery.sh index 9701a05..35359b2 100755 --- a/meshery.sh +++ b/meshery.sh @@ -35,7 +35,7 @@ main() { kubectl config view --minify --flatten > ~/minified_config mv ~/minified_config ~/.kube/config - curl -L https://git.io/meshery | PLATFORM=$PLATFORM bash - + curl -L https://git.io/meshery | sudo PLATFORM=$PLATFORM bash - & sleep 60 }