diff --git a/.github/action.yml b/.github/action.yml index 921ae8bf..99ea473c 100644 --- a/.github/action.yml +++ b/.github/action.yml @@ -45,6 +45,10 @@ inputs: required: false default: false type: boolean +outputs: + container-group: + description: "Container Group" + value: ${{ steps.container-output.outputs.group }} runs: using: composite steps: @@ -69,6 +73,26 @@ runs: COMPOSE_PROJECT_NAME=${{ env.project-number }} \ ${{ inputs.env_overrides }} docker compose -p ${{ env.project-number }} up --build --force-recreate --always-recreate-deps --no-start working-directory: ${{ inputs.group_dir }} + - name: Print Containers + id: container-output + shell: bash + run: | + mkdir matrix + images=$(REGISTRY=${{ inputs.registry }} \ + REPO=${{ inputs.repo }} \ + COMPOSE_PROJECT_NAME=${{ env.project-number }} \ + ${{ inputs.env_overrides }} docker compose -p ${{ env.project-number }} images --format json) + for image in $(echo $images | jq -r --arg registry "$REGISTRY" '.[] | select(.Repository | contains($registry)) | .Tag'); do + echo "$image" > matrix/$image.txt + done + echo "group=${{ inputs.group_dir }}" | tr '/' '_' >> $GITHUB_OUTPUT + working-directory: ${{ inputs.group_dir }} + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.project-number }}-${{ steps.container-output.outputs.group }} + path: ${{ inputs.group_dir }}/matrix/* + retention-days: 1 + overwrite: true - name: Push Containers shell: bash if: ${{ !fromJson(inputs.no-push) }} diff --git a/.github/scan/action.yml b/.github/scan/action.yml new file mode 100644 index 00000000..fde7f94c --- /dev/null +++ b/.github/scan/action.yml @@ -0,0 +1,35 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Aqua Security Trivy' +description: 'Scans container images for vulnerabilities with Trivy without building the image. For use behind firewalls.' +author: 'tyler.titsworth@intel.com' +inputs: + image-ref: + description: 'image reference(for backward compatibility)' + required: true + output: + description: 'writes results to a file with the specified file name' + required: true +runs: + using: 'docker' + image: "docker://ghcr.io/aquasecurity/trivy" + entrypoint: trivy + args: + - '--timeout=30m' + - image + - '--format=sarif' + - '--no-progress' + - '--output=${{ inputs.output }}' + - ${{ inputs.image-ref }} diff --git a/.github/workflows/chart-ci.yaml b/.github/workflows/chart-ci.yaml index 5b00a4f8..6218dd28 100644 --- a/.github/workflows/chart-ci.yaml +++ b/.github/workflows/chart-ci.yaml @@ -15,9 +15,7 @@ name: Helm Chart CI on: merge_group: null - pull_request_target: - types: [opened, edited, reopened, synchronize] - branches: [main] + pull_request: paths: - 'workflows/charts/**' permissions: read-all @@ -33,15 +31,6 @@ jobs: with: egress-policy: audit - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'pull_request_target' }} - with: - fetch-depth: 0 - ref: "refs/pull/${{ github.event.number }}/merge" - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'merge_group' }} - with: - fetch-depth: 0 - ref: ${{ github.event.merge_group.head_ref }} - uses: intel/ai-containers/workflows/charts@main with: kubeconfig_path: ${{ secrets.KUBECONFIG_PATH }} diff --git a/.github/workflows/container-ci.yaml b/.github/workflows/container-ci.yaml index d973b220..eee64f56 100644 --- a/.github/workflows/container-ci.yaml +++ b/.github/workflows/container-ci.yaml @@ -25,11 +25,6 @@ on: description: 'Enter Bash Env Variable Overrides in `KEY=VAL KEY2=VAL2` format:' required: false type: string - ref: - description: 'Enter Git Ref:' - required: true - default: 'main' - type: string runner_label: description: 'Enter Validation Runner Label:' default: test-runner @@ -61,13 +56,10 @@ on: no_start: required: false type: boolean - ref: - required: true - type: string jobs: - #################################################################################################### - # Compose Build - #################################################################################################### +#################################################################################################### +# Compose Build +#################################################################################################### setup-build: outputs: matrix: ${{ steps.build-matrix.outputs.matrix }} @@ -78,8 +70,6 @@ jobs: with: egress-policy: audit - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - ref: ${{ inputs.ref }} - name: Set Matrix id: build-matrix run: echo "matrix=$(jq -c . < ${{ inputs.group_dir }}/.actions.json)" >> $GITHUB_OUTPUT @@ -93,10 +83,10 @@ jobs: strategy: matrix: ${{ fromJson(needs.setup-build.outputs.matrix) }} fail-fast: false + outputs: + group: ${{ steps.build-group.outputs.container-group }} steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - ref: ${{ inputs.ref }} if: ${{ !inputs.no_build }} - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 with: @@ -115,9 +105,63 @@ jobs: repo: ${{ secrets.REPO }} no-push: ${{ inputs.no_push }} no-start: ${{ inputs.no_start }} - #################################################################################################### - # Generic Test Runner - #################################################################################################### +#################################################################################################### +# Trivy Scan +#################################################################################################### + setup-scan: + needs: [build-containers] + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.scan-matrix.outputs.matrix }} + steps: + - name: Harden Runner + uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + with: + egress-policy: audit + - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 + with: + path: matrix + - name: Set Matrix + id: scan-matrix + run: echo "matrix=$(cat matrix/*-${{ needs.build-containers.outputs.group }}/*.txt | jq -R '.' | jq -sc '. | unique')" >> $GITHUB_OUTPUT + scan-containers: + needs: [setup-scan] + if: ${{ !inputs.no_build }} + runs-on: k8-runners + permissions: + actions: read + packages: read + pull-requests: write + security-events: write + strategy: + matrix: + container: ${{ fromJSON(needs.setup-scan.outputs.matrix) }} + fail-fast: false + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ${{ secrets.REGISTRY }} + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} + - name: Pull Image + run: docker pull ${{ secrets.REGISTRY }}/${{ secrets.REPO }}:${{ matrix.container }} + - name: Scan Container + uses: intel/ai-containers/.github/scan@5c68c9c8ff7f634fff3abcc17c46970e0b9b0dde # main + with: + image-ref: ${{ secrets.REGISTRY }}/${{ secrets.REPO }}:${{ matrix.container }} + output: ${{ matrix.container }}-scan.sarif + - name: Cleanup + if: always() + run: docker rmi -f ${{ secrets.REGISTRY }}/${{ secrets.REPO }}:${{ matrix.container }} + - uses: github/codeql-action/upload-sarif@2e230e8fe0ad3a14a340ad0815ddb96d599d2aff # v3.25.8 + with: + sarif_file: '${{ matrix.container }}-scan.sarif' + category: '${{ matrix.container }}' + continue-on-error: true +#################################################################################################### +# Generic Test Runner +#################################################################################################### setup-test: needs: [build-containers] runs-on: ubuntu-latest @@ -129,8 +173,6 @@ jobs: with: egress-policy: audit - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - ref: ${{ inputs.ref }} - name: Get Recipes id: test-matrix run: echo "matrix=$(find ${{ inputs.group_dir }} -type f -name 'tests.yaml' -exec dirname {} \; | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT @@ -148,8 +190,6 @@ jobs: with: egress-policy: audit - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - ref: ${{ inputs.ref }} - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 with: registry: ${{ secrets.REGISTRY }} @@ -159,9 +199,8 @@ jobs: uses: intel/ai-containers/test-runner@main with: cache_registry: ${{ secrets.CACHE_REGISTRY }} - perf_repo: ${{ secrets.PERF_REPO }} recipe_dir: ${{ inputs.group_dir }} registry: ${{ secrets.REGISTRY }} repo: ${{ secrets.REPO }} test_dir: ${{ matrix.tests }} - token: ${{ secrets.PERF_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/dependency-review.yaml b/.github/workflows/dependency-review.yaml index 9621e35c..f1bd3e8f 100644 --- a/.github/workflows/dependency-review.yaml +++ b/.github/workflows/dependency-review.yaml @@ -22,9 +22,7 @@ # Source repository: https://github.com/actions/dependency-review-action name: 'Dependency Review' on: - pull_request_target: - types: [opened, edited, reopened, synchronize] - branches: [main] + pull_request: null permissions: contents: read concurrency: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index ce23cdb9..b89294f2 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -15,9 +15,7 @@ name: Docs on: merge_group: null - pull_request_target: - types: [opened, edited, reopened, synchronize] - branches: [main] + pull_request: null push: branches: - main @@ -38,17 +36,6 @@ jobs: with: egress-policy: audit - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'push' }} - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'pull_request_target' }} - with: - fetch-depth: 0 - ref: "refs/pull/${{ github.event.number }}/merge" - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'merge_group' }} - with: - fetch-depth: 0 - ref: ${{ github.event.merge_group.head_ref }} - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: 3.8 diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index 87cdb83a..1eaa02f5 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -14,9 +14,7 @@ name: Integration Tests on: - pull_request_target: - types: [opened, edited, reopened, synchronize] - branches: [main] + pull_request: null permissions: read-all concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -32,15 +30,12 @@ jobs: with: egress-policy: audit - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - fetch-depth: 0 - ref: "refs/pull/${{ github.event.number }}/merge" - name: Output Modified Group Directories id: group-list run: | # Get diff array filtered by specific filetypes DIFF=$(git diff --diff-filter=d \ - --name-only ${{ github.event.merge_group.base_sha || github.event.pull_request.base.sha }}...${{ github.event.merge_group.head_sha || github.event.pull_request.head.sha }} \ + --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \ -- '*/*Dockerfile' '*.py' '*.yaml' '*.yml' '*.sh' '*/*requirements.txt' '*.json' | \ jq -R '.' | jq -sc '.' \ ) @@ -71,7 +66,6 @@ jobs: uses: intel/ai-containers/.github/workflows/container-ci.yaml@main with: group_dir: ${{ matrix.group }} - ref: "refs/pull/${{ github.event.number }}/merge" secrets: inherit status-check: needs: [group-diff, pipeline-ci] diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index fba5c4ad..cbaec176 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -30,14 +30,8 @@ jobs: with: egress-policy: audit - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'pull_request' }} with: fetch-depth: 0 - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'merge_group' }} - with: - fetch-depth: 0 - ref: ${{ github.event.merge_group.head_ref }} - uses: super-linter/super-linter/slim@88ea3923a7e1f89dd485d079f6eb5f5e8f937589 # v6.6.0 env: # To report GitHub Actions status checks diff --git a/.github/workflows/merge_check.yaml b/.github/workflows/merge_check.yaml index ca0569af..7065e80d 100644 --- a/.github/workflows/merge_check.yaml +++ b/.github/workflows/merge_check.yaml @@ -26,7 +26,7 @@ jobs: checkName: helm-ci ref: ${{ github.event.merge_group.head_ref }} timeoutSeconds: 1800 - token: ${{ secrets.ACTION_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} - uses: fountainhead/action-wait-for-check@5a908a24814494009c4bb27c242ea38c93c593be if: ${{ steps.helm-ci.outputs.conclusion == 'success' }} id: integration-test @@ -34,7 +34,7 @@ jobs: checkName: integration-test ref: ${{ github.event.merge_group.head_ref }} timeoutSeconds: 1800 - token: ${{ secrets.ACTION_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} - uses: fountainhead/action-wait-for-check@5a908a24814494009c4bb27c242ea38c93c593be if: ${{ steps.integration-test.outputs.conclusion == 'success' }} id: coverage @@ -42,6 +42,6 @@ jobs: checkName: coverage ref: ${{ github.event.merge_group.head_ref }} timeoutSeconds: 1800 - token: ${{ secrets.ACTION_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} - if: ${{ steps.coverage.outputs.conclusion != 'success' }} run: exit 1 diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index 78158e62..95e71d5d 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -46,7 +46,7 @@ jobs: with: results_file: results.sarif results_format: sarif - repo_token: ${{ secrets.ACTION_TOKEN }} + repo_token: ${{ secrets.GITHUB_TOKEN }} publish_results: true - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: diff --git a/.github/workflows/test-runner-ci.yaml b/.github/workflows/test-runner-ci.yaml index 0736a8c2..5feee138 100644 --- a/.github/workflows/test-runner-ci.yaml +++ b/.github/workflows/test-runner-ci.yaml @@ -15,9 +15,7 @@ name: Test Runner CI on: merge_group: null - pull_request_target: - types: [opened, edited, reopened, synchronize] - branches: [main] + pull_request: paths: - 'test-runner/**' permissions: read-all @@ -37,20 +35,6 @@ jobs: with: egress-policy: audit - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'pull_request_target' }} - with: - fetch-depth: 0 - ref: "refs/pull/${{ github.event.number }}/merge" - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'merge_group' }} - with: - fetch-depth: 0 - ref: ${{ github.event.merge_group.head_ref }} - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - with: - path: models-perf - repository: ${{ secrets.PERF_REPO }} - token: ${{ secrets.PERF_TOKEN }} - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 with: driver: docker @@ -69,7 +53,7 @@ jobs: env: CACHE_REGISTRY: ${{ secrets.CACHE_REGISTRY }} FORCE_COLOR: 1 - GITHUB_TOKEN: ${{ secrets.ACTION_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PERF_REPO: ${{ secrets.PERF_REPO }} REGISTRY: ${{ secrets.REGISTRY }} REPO: ${{ secrets.REPO }} @@ -86,15 +70,6 @@ jobs: with: egress-policy: audit - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'pull_request_target' }} - with: - fetch-depth: 0 - ref: "refs/pull/${{ github.event.number }}/merge" - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'merge_group' }} - with: - fetch-depth: 0 - ref: ${{ github.event.merge_group.head_ref }} - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: "3.10" @@ -111,7 +86,7 @@ jobs: uses: orgoro/coverage@4e3bc9302f715595336a40f7dd4a3b15c687bdb4 # v3.1 with: coverageFile: coverage.xml - token: ${{ secrets.ACTION_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} treshholdAll: 80 integration-test: runs-on: k8-runners @@ -121,15 +96,6 @@ jobs: with: egress-policy: audit - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'pull_request_target' }} - with: - fetch-depth: 0 - ref: "refs/pull/${{ github.event.number }}/merge" - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - if: ${{ github.event_name == 'merge_group' }} - with: - fetch-depth: 0 - ref: ${{ github.event.merge_group.head_ref }} - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 with: driver: docker @@ -145,9 +111,8 @@ jobs: uses: intel/ai-containers/test-runner@main with: cache_registry: ${{ secrets.CACHE_REGISTRY }} - perf_repo: ${{ secrets.PERF_REPO }} recipe_dir: test-runner registry: ${{ secrets.REGISTRY }} repo: ${{ secrets.REPO }} test_dir: test-runner - token: ${{ secrets.PERF_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/test-runner/action.yml b/test-runner/action.yml index bc14d039..61732e7f 100644 --- a/test-runner/action.yml +++ b/test-runner/action.yml @@ -20,10 +20,6 @@ inputs: description: 'Container Cache Registry URL' required: false type: string - perf_repo: - description: 'Performance Test Repo' - required: false - type: string recipe_dir: description: 'Path to Recipe Directory' required: true @@ -52,12 +48,6 @@ runs: ref: main repository: intel/ai-containers token: ${{ inputs.token }} - - uses: actions/checkout@v4 - with: - path: models-perf - ref: main - repository: ${{ inputs.perf_repo }} - token: ${{ inputs.token }} - name: Install Requirements shell: bash run: | diff --git a/test-runner/dev-requirements.txt b/test-runner/dev-requirements.txt index a40f4530..c5a881cf 100644 --- a/test-runner/dev-requirements.txt +++ b/test-runner/dev-requirements.txt @@ -1,9 +1,7 @@ black>=24.4.1 coverage>=7.5.0 expandvars>=0.12.0 -gitpython>=3.1.43 hypothesis>=6.100.1 -Pint>=0.21.1 pydantic==2.7.4 pylint>=3.1.0 pytest>=8.1.1 diff --git a/test-runner/requirements.txt b/test-runner/requirements.txt index 60fd33ff..f1757e50 100644 --- a/test-runner/requirements.txt +++ b/test-runner/requirements.txt @@ -1,6 +1,4 @@ expandvars>=0.12.0 -gitpython>=3.1.43 -Pint>=0.21.1 pydantic==2.7.4 python_on_whales>=0.70.1 pyyaml>=6.0.1 diff --git a/test-runner/tests.yaml b/test-runner/tests.yaml index 1e37c9a5..f5361a9a 100644 --- a/test-runner/tests.yaml +++ b/test-runner/tests.yaml @@ -46,8 +46,3 @@ test7: cmd: "echo 'world: hello'" mask: - world -test8: - cmd: "echo 'test: 123 throughput'" - mask: - - test - performance: perf/test.yaml:test diff --git a/test-runner/tests/utest.py b/test-runner/tests/utest.py index af8c75f0..e7dd72fb 100644 --- a/test-runner/tests/utest.py +++ b/test-runner/tests/utest.py @@ -21,7 +21,7 @@ from hypothesis import given from hypothesis.strategies import dictionaries, text from test_runner import get_test_list, parse_args, set_log_filename -from utils.test import PerfException, Test +from utils.test import Test @pytest.fixture @@ -143,11 +143,6 @@ def test_get_test_list(test_args_input, test_json_input): "mask": ["hello"], }, "test7": {"cmd": "echo 'world: hello'", "mask": ["world"]}, - "test8": { - "cmd": "echo 'test: 123 throughput'", - "mask": ["test"], - "performance": "perf/test.yaml:test", - }, } test_fn, disable_masking = get_test_list(test_args_input, test_json_input) @@ -164,44 +159,6 @@ def test_masking(test_class_input): assert ": ***" in test.run() -def test_perf_thresholds(): - "test performance thresholds." - test_cases = [ - { - "cmd": "echo 'test: 123 throughput'", - "performance": "perf/test.yaml:test", - "expected_output": "test: 123 throughput", - "should_raise_exception": False, - }, - { - "cmd": "echo 'test: 121 throughput'", - "performance": "perf/test.yaml:test", - "should_raise_exception": True, - }, - { - "cmd": "echo 'test: 123 millithroughput'", - "performance": "perf/test.yaml:test", - "should_raise_exception": True, - }, - { - "cmd": "echo 'test: 125 throughput'", - "performance": "perf/test.yaml:not-test", - "should_raise_exception": True, - }, - ] - - for test_case in test_cases: - test = Test(name="test", **test_case) - if test_case["should_raise_exception"]: - try: - with pytest.raises(Exception, match="Failed") as exc_info: - test.run() - except: - assert isinstance(exc_info.value, PerfException) - else: - assert test_case["expected_output"] in test.run() - - @given(name=text(), arguments=dictionaries(text(), text())) def test_fuzz_container_run(name, arguments): "Fuzz container_run()." diff --git a/test-runner/utils/test.py b/test-runner/utils/test.py index 72f41efb..92fc37dd 100644 --- a/test-runner/utils/test.py +++ b/test-runner/utils/test.py @@ -21,14 +21,9 @@ from subprocess import PIPE, Popen from typing import Dict, List, Optional -import pint from expandvars import expandvars -from git import Repo from pydantic import BaseModel from python_on_whales import DockerException, docker -from yaml import YAMLError, full_load - -units = pint.UnitRegistry() class PerfException(Exception): @@ -67,28 +62,12 @@ class Test(BaseModel): groups_add: Optional[List[str]] = ["109", "44"] hostname: Optional[str] = None ipc: Optional[str] = None - performance: Optional[str] = None privileged: Optional[bool] = False pull: Optional[str] = "missing" user: Optional[str] = None shm_size: Optional[str] = None workdir: Optional[str] = None - def __init__(self, **data): - super().__init__(**data) - if self.performance: - perf_repo = os.environ.get("PERF_REPO") - if perf_repo: - if not os.path.exists("models-perf"): - Repo.clone_from( - f"https://github.com/{perf_repo}", "models-perf", progress=None - ) - else: - logging.error( - "Performance mode enabled, but PERF_REPO environment variable not set" - ) - units.load_definitions("./models-perf/definitions.txt") - def get_path(self, name): """Given a filename, find that file from the users current working directory @@ -205,54 +184,6 @@ def notebook_run(self, img: str): load=True, ) - def check_perf(self, content): - """ - Check the performance of the test against the thresholds. - - Args: - content (str): test output log - - Raises: - PerfException: if the performance does not meet the target performance - """ - with open( - f"models-perf/{self.performance.split(':')[0]}", "r", encoding="utf-8" - ) as file: - try: - thresholds = full_load(file) - except YAMLError as yaml_exc: - raise YAMLError(yaml_exc) - model_thresholds = [ - threshold - for threshold in thresholds - if self.performance.split(":")[1] == threshold["test_id"] - ] - for threshold in model_thresholds: - perf = re.search( - rf"{threshold['key']}[:]?\s+(.\d+[\s]?.*)", - content, - re.IGNORECASE, - ) - if perf: - if threshold["lower_is_better"]: - if units.Quantity(perf.group(1)) > units.Quantity( - f"{threshold['boundary']} {threshold['unit']}" - ): - if not self.mask: - logging.info("%s: %s", threshold["key"], perf.group(1)) - raise PerfException( - f"Performance Threshold {threshold['name']} did not meet the target performance." - ) - else: - if units.Quantity(perf.group(1)) < units.Quantity( - f"{threshold['boundary']} {threshold['unit']}" - ): - if not self.mask: - logging.info("%s: %s", threshold["key"], perf.group(1)) - raise PerfException( - f"Performance Threshold {threshold['name']} did not meet the target performance." - ) - def container_run(self): """Runs the docker container. @@ -317,8 +248,6 @@ def container_run(self): log = "" for _, stream_content in output_generator: # All process logs will have the stream_type of stderr despite it being stdout - if self.performance: - self.check_perf(stream_content.decode("utf-8")) for item in self.mask: stream_content = re.sub( rf"({item}[:]?\s+)(.*)", @@ -355,8 +284,6 @@ def run(self): ) try: stdout, stderr = p.communicate() - if self.performance: - self.check_perf(stdout.decode("utf-8")) for item in self.mask: stdout = re.sub( rf"({item}[:]?\s+)(.*)", r"\1***", stdout.decode("utf-8")