diff --git a/.github/workflows/setup-python.yml b/.github/workflows/setup-python.yml new file mode 100644 index 00000000..7fd52e0a --- /dev/null +++ b/.github/workflows/setup-python.yml @@ -0,0 +1,27 @@ +name: Setup Python Testing Environment + +on: + workflow_call: + inputs: + python-version: + required: true + type: string + os: + required: false + type: string + default: ubuntu-latest + +jobs: + setup-python: + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@ff7abcd0c3c05ccf6adc123a8cd1fd4fb30fb493 + + - name: Set up Python + uses: actions/setup-python@9322b3ca74000aeb2c01eb777b646334015ddd72 + with: + python-version: ${{ inputs.python-version }} + + - name: Install tox + run: pip install tox diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4a895c54..ae4b5b8b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,8 +2,26 @@ name: Python CI on: [push, workflow_dispatch] jobs: - build: - runs-on: ${{ matrix.os }} + unit-tests: + uses: ./.github/workflows/setup-python.yml + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.9, 3.13] + include: + - os: ubuntu-22.04 + python-version: 3.7 + fail-fast: false + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + steps: + - name: Run Unit Tests + run: tox -e py -- tests/unit + + integration-tests: + needs: unit-tests + uses: ./.github/workflows/setup-python.yml strategy: fail-fast: false matrix: @@ -11,25 +29,44 @@ jobs: python-version: [3.9] splunk-version: [9.4, latest] include: - # Oldest possible configuration - # Last Ubuntu version with Python 3.7 binaries available - os: ubuntu-22.04 python-version: 3.7 - splunk-version: 9.1 - # Latest possible configuration + splunk-version: "9.1" - os: ubuntu-latest python-version: 3.13 splunk-version: latest steps: - - name: Checkout code - uses: actions/checkout@ff7abcd0c3c05ccf6adc123a8cd1fd4fb30fb493 - name: Run docker compose run: SPLUNK_VERSION=${{ matrix.splunk-version }} docker compose up -d - - name: Setup Python - uses: actions/setup-python@9322b3ca74000aeb2c01eb777b646334015ddd72 - with: - python-version: ${{ matrix.python-version }} - - name: Install tox - run: pip install tox - - name: Test Execution - run: tox -e py + + - name: Wait for Splunk setup completion + run: make wait_up + + - name: Run Integration Tests + run: tox -e py -- tests/integration + + system-tests: + needs: unit-tests + uses: ./.github/workflows/setup-python.yml + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: [3.9] + splunk-version: [9.4, latest] + include: + - os: ubuntu-22.04 + python-version: 3.7 + splunk-version: "9.1" + - os: ubuntu-latest + python-version: 3.13 + splunk-version: latest + steps: + - name: Run docker compose (System) + run: SPLUNK_VERSION=${{matrix.splunk-version}} docker compose up -d + + - name: Wait for Splunk setup completion + run: make wait_up + + - name: Run System Tests + run: tox -e py -- tests/system diff --git a/Makefile b/Makefile index 58d53228..61ee1a68 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ docs: .PHONY: test test: @echo "$(ATTN_COLOR)==> test $(NO_COLOR)" - @tox -e py37,py39 + @tox .PHONY: test_specific test_specific: diff --git a/tests/README.md b/tests/README.md index da02228c..df5eeda9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,50 +1,90 @@ -# Splunk Test Suite +# Python SDK tests -The test suite uses Python's standard library and the built-in **unittest** -library. The Splunk Enterprise SDK for Python has been tested with Python v3.7 -and v3.9. +The tests use both Python's standard **unittest** library and **pytest**, and have been tested with Python 3.7, 3.9, and 3.13. The test suite can be executed across all supported Python versions using **tox**. -To run the unit tests, open a command prompt in the **/splunk-sdk-python** -directory and enter: +## Test Types - python setup.py test +The SDK test suite is divided into three main types: -You can also run individual test files, which are located in -**/splunk-sdk-python/tests**. Each distinct area of the SDK is tested in a -single file. For example, roles are tested -in `test_role.py`. To run this test, open a command prompt in -the **/splunk-sdk-python/tests** subdirectory and enter: +1. [Unit Tests](./unit/) - python test_role.py + - Fast and isolated, do not require a running Splunk instance -NOTE: Before running the test suite, make sure the instance of Splunk you -are testing against doesn't have new events being dumped continuously -into it. Several of the tests rely on a stable event count. It's best -to test against a clean install of Splunk, but if you can't, you -should at least disable the *NIX and Windows apps. Do not run the test -suite against a production instance of Splunk! It will run just fine -with the free Splunk license. +2. [Integration Tests](./integration/) + - **Require a running Splunk instance** + - Test SDK being used to communicate with a real Splunk instance via API -## Code Coverage +3. [System Tests](./system/) + - **Require a running Splunk instance** + - Test SDK being used inside Splunk apps (SDK bundeled with apps inside Splunk instance) + +## Setting up Splunk in Docker + +Integration and system require a running test Splunk instance, which can be set up with Docker. Make sure Docker is installed and then start Splunk with: + +```bash + make up SPLUNK_VERSION=latest # runs docker compose up -d + make wait_up # wait until the Splunk is ready +``` + +If running on Mac, add this line to docker-compose: + +```yaml +architecture: linux/amd64 +``` + +> **NOTE**: Before running the test suite, make sure the instance of Splunk you +> are testing against doesn't have new events being dumped continuously +> into it. Several of the tests rely on a stable event count. It's best +> to test against a clean install of Splunk, but if you can't, you +> should at least disable the \*NIX and Windows apps. **Do not run the test +> suite against a production instance of Splunk!** It will run just fine +> with the free Splunk license. -Coverage.py is an excellent tool for measuring code coverage of Python programs. +## Running tests with tox -To install it, use easy_install: +**tox** allows running tests across multiple Python versions and environments. +The configurations are defined in the `tox.ini` file. - easy_install coverage +- Run all tests (unit, integration, and system) on all Python versions: -Or use pip: + ```bash + tox + ``` - pip install coverage +- Run all tests (unit, integration, and system) on a specific Python versions: -To generate a report of the code coverage of the unit test suite, open a command -prompt in the **/splunk-sdk-python** directory and enter: + ```bash + tox -e py39 # example for Python 3.9 + ``` + +- Run a specific type of tests on Python version currently active in your shell: + + ```bash + tox -e py -- tests/unit # example for unit tests + ``` + +- Run a specific type of tests on a single Python version: + + ```bash + tox -e py37 -- tests/unit # example for Python 3.7 unit tests + ``` + +- Run a specific test file and Python version: + ``` + tox -e py39 -- tests/unit/test_utils.py + ``` +- Run a specific test method from a specific file and Python version: + ``` + tox -e py313 -- tests/system/test_csc_apps.py::TestEventingApp::test_metadata + ``` + +## Code Coverage - python setup.py coverage +Code coverage is provided via `pytest-cov`, which uses `Coverage.py` under the hood. +Coverage statistics are displayed at the end of each tox or pytest run. -This command runs the entire test suite and writes an HTML coverage report to -the **/splunk-sdk-python/coverage_report** directory. +## Test reports -For more information about Coverage.py, see the author's website -([http://nedbatchelder.com/code/coverage/](http://nedbatchelder.com/code/coverage/)). \ No newline at end of file +Test reports are generated in JUnit XML format. For each tox environment, the reports are saved in: `test-reports/junit-{test-env}.xml` diff --git a/tox.ini b/tox.ini index e69c2e6a..31010685 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = clean,docs,py37,py39,313 +envlist = clean,py{37,39,313} skipsdist = {env:TOXBUILD:false} [testenv:pep8] @@ -32,7 +32,7 @@ deps = pytest distdir = build commands = - {env:TOXBUILD:python -m pytest --junitxml=test-reports/junit-{envname}.xml --cov --cov-config=.coveragerc} {posargs} + {env:TOXBUILD:python -m pytest tests/ --junitxml=test-reports/junit-{envname}.xml --cov --cov-config=.coveragerc} {posargs} [testenv:clean] deps = coverage