DVPL-12468 separate tox targets

.github/workflows/test.yml

@@ -3,29 +3,47 @@ name: Python CI
 on: [push, workflow_dispatch]
 
 jobs:
-  build:
+  unit-tests:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os:
-          - ubuntu-latest
-        python: [3.9, 3.13]
-        splunk-version:
-          - "8.1"
-          - "8.2"
-          - "latest"
+        os: [ubuntu-latest]
+        python-version: [3.9, 3.13]
         include:
           - os: ubuntu-22.04
-            python: 3.7
-            splunk-version: "8.1"
-          - os: ubuntu-22.04
-            python: 3.7
-            splunk-version: "8.2"
-          - os: ubuntu-22.04
-            python: 3.7
-            splunk-version: "latest"
+            python-version: 3.7
+      fail-fast: false
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@ff7abcd0c3c05ccf6adc123a8cd1fd4fb30fb493
+
+      - name: Setup Python
+        uses: actions/setup-python@9322b3ca74000aeb2c01eb777b646334015ddd72
+        with:
+          python-version: ${{ matrix.python-version }}
 
+      - name: Install tox
+        run: pip install tox
+
+      - name: Run Unit Tests
+        run: tox -e unit
+
+  integration-tests:
+    needs: unit-tests
+    runs-on: ${{ matrix.os }}
+    strategy:
       fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [3.9]
+        splunk-version: [9.4, latest]
+        include:
+          - os: ubuntu-22.04
+            python-version: 3.7
+            splunk-version: "9.1"
+          - os: ubuntu-latest
+            python-version: 3.13
+            splunk-version: latest
 
     steps:
       - name: Checkout code
@@ -37,13 +55,49 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@9322b3ca74000aeb2c01eb777b646334015ddd72
         with:
-          python-version: ${{ matrix.python }}
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install tox
+        run: pip install tox
+
+      - name: Run Integration Tests
+        run: tox -e integration
+
+  system-tests:
+    needs: [unit-tests, integration-tests]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [3.9]
+        splunk-version: [9.4, latest]
+        include:
+          - os: ubuntu-22.04
+            python-version: 3.7
+            splunk-version: "9.1"
+          - os: ubuntu-latest
+            python-version: 3.13
+            splunk-version: latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@ff7abcd0c3c05ccf6adc123a8cd1fd4fb30fb493
+
+      - name: Run docker compose (System)
+        run: SPLUNK_VERSION=${{matrix.splunk-version}} docker compose up -d
+
+      - name: Setup Python
+        uses: actions/setup-python@9322b3ca74000aeb2c01eb777b646334015ddd72
+        with:
+          python-version: ${{ matrix.python-version }}
 
       - name: Install tox
         run: pip install tox
 
-      - name: Test Execution
-        run: tox -e py
+      - name: Run System Tests
+        run: tox -e system
+
   fossa-scan:
     uses: splunk/oss-scanning-public/.github/workflows/oss-scan.yml@main
     secrets: inherit

Makefile

@@ -34,7 +34,22 @@ docs:
 .PHONY: test
 test:
 	@echo "$(ATTN_COLOR)==> test $(NO_COLOR)"
-	@tox -e py37,py39
+	@tox
+
+.PHONY: test-unit
+test-unit:
+	@echo "$(ATTN_COLOR)==> test $(NO_COLOR)"
+	@tox -e unit
+
+.PHONY: test-integration
+test-integration:
+	@echo "$(ATTN_COLOR)==> test $(NO_COLOR)"
+	@tox -e integration
+
+.PHONY: test-system
+test-system:
+	@echo "$(ATTN_COLOR)==> test $(NO_COLOR)"
+	@tox -e system
 
 .PHONY: test_specific
 test_specific:

tests/README.md

@@ -1,50 +1,88 @@
-# Splunk Test Suite
+# Python SDK tests
 
-The test suite uses Python's standard library and the built-in **unittest** 
-library. The Splunk Enterprise SDK for Python has been tested with Python v3.7 
-and v3.9.
+The tests use both Python's standard **unittest** library and **pytest**, and have been tested with Python 3.7, 3.9, and 3.13. The test suite can be executed across all supported Python versions using **tox**.
 
-To run the unit tests, open a command prompt in the **/splunk-sdk-python** 
-directory and enter:
+## Test Types
 
-    python setup.py test
+The SDK test suite is divided into three main types:
 
-You can also run individual test files, which are located in 
-**/splunk-sdk-python/tests**. Each distinct area of the SDK is tested in a 
-single file. For example, roles are tested
-in `test_role.py`. To run this test, open a command prompt in
-the **/splunk-sdk-python/tests** subdirectory and enter:
+1. [Unit Tests](./unit/)
 
-    python test_role.py
+   - Fast and isolated, do not require a running Splunk instance
 
-NOTE: Before running the test suite, make sure the instance of Splunk you
-are testing against doesn't have new events being dumped continuously
-into it. Several of the tests rely on a stable event count. It's best
-to test against a clean install of Splunk, but if you can't, you
-should at least disable the *NIX and Windows apps. Do not run the test
-suite against a production instance of Splunk! It will run just fine
-with the free Splunk license.
+2. [Integration Tests](./integration/)
 
+   - **Require a running Splunk instance**
+   - Test SDK being used to communicate with a real Splunk instance via API
 
-## Code Coverage
+3. [System Tests](./system/)
+   - **Require a running Splunk instance**
+   - Test SDK being used inside Splunk apps (SDK bundeled with apps inside Splunk instance)
+
+## Setting up Splunk in Docker
+
+Integration and system require a running test Splunk instance, which can be set up with Docker. Make sure Docker is installed and then start Splunk with:
+
+```bash
+ make up SPLUNK_VERSION=latest # runs docker compose up
+ make wait_up # wait until the Splunk is ready
+```
+
+If running on Mac, add this line to docker-compose:
+
+```yaml
+architecture: linux/amd64
+```
 
-Coverage.py is an excellent tool for measuring code coverage of Python programs.
+> **NOTE**: Before running the test suite, make sure the instance of Splunk you
+> are testing against doesn't have new events being dumped continuously
+> into it. Several of the tests rely on a stable event count. It's best
+> to test against a clean install of Splunk, but if you can't, you
+> should at least disable the \*NIX and Windows apps. **Do not run the test
+> suite against a production instance of Splunk!** It will run just fine
+> with the free Splunk license.
 
-To install it, use easy_install:
+## Running tests with tox
 
-    easy_install coverage
+**tox** allows running tests across multiple Python versions and environments.  
+The configurations are defined in the `tox.ini` file.
 
-Or use pip:
+- Run all tests (unit, integration, and system) on all Python versions:
+  ```bash
+  tox
+  ```
+- Run all unit tests on all Python versions:
 
-    pip install coverage
+  ```
+  tox -f unit
+  ```
 
-To generate a report of the code coverage of the unit test suite, open a command
-prompt in the **/splunk-sdk-python** directory and enter:
+- Run all unit tests on Python version currently active in your shell:
+
+  ```
+  tox -e unit
+  ```
+
+- Run a specific type of test on a single Python version:
+
+  ```bash
+  tox -e py37-unit  # example for Python 3.7 unit tests
+  ```
+
+- Run a specific test file and Python version:
+  ```
+  tox -e py39 -- tests/unit/test_utils.py
+  ```
+- Run a specific test method from a specific file and Python version:
+  ```
+  tox -e py313 -- tests/system/test_csc_apps.py::TestEventingApp::test_metadata
+  ```
+
+## Code Coverage
 
-    python setup.py coverage
+Code coverage is provided via `pytest-cov`, which uses `Coverage.py` under the hood.
+Coverage statistics are displayed at the end of each tox or pytest run.
 
-This command runs the entire test suite and writes an HTML coverage report to 
-the **/splunk-sdk-python/coverage_report** directory.
+## Test reports
 
-For more information about Coverage.py, see the author's website 
-([http://nedbatchelder.com/code/coverage/](http://nedbatchelder.com/code/coverage/)).
+Test reports are generated in JUnit XML format. For each tox environment, the reports are saved in: `test-reports/junit-{test-env}.xml`

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = clean,docs,py37,py39,313
+envlist = clean,py{37,39,313}-{unit,integration,system}
 skipsdist = {env:TOXBUILD:false}
 
 [testenv:pep8]
@@ -32,7 +32,10 @@ deps = pytest
 
 distdir = build
 commands =
-    {env:TOXBUILD:python -m pytest --junitxml=test-reports/junit-{envname}.xml --cov --cov-config=.coveragerc} {posargs}
+    unit: {env:TOXBUILD:python -m pytest tests/unit/ --junitxml=test-reports/junit-{envname}.xml --cov --cov-config=.coveragerc} {posargs}
+    integration: {env:TOXBUILD:python -m pytest tests/integration/ --junitxml=test-reports/junit-{envname}.xml --cov --cov-config=.coveragerc} {posargs}
+    system: {env:TOXBUILD:python -m pytest tests/system/ --junitxml=test-reports/junit-{envname}.xml --cov --cov-config=.coveragerc} {posargs}
+    all: {env:TOXBUILD:python -m pytest tests/ --junitxml=test-reports/junit-{envname}.xml --cov --cov-config=.coveragerc} {posargs}
 
 [testenv:clean]
 deps = coverage