remove fixture

Author: euripedesrocha

.github/actions/target-pytest/action.yml

@@ -26,7 +26,7 @@ runs:
         - name: Run apps
           shell: bash
           run: |
-              pytest ${{inputs.app_path}} --ignore-glob '*/managed_components/*' --ignore=.github --target=${{ inputs.target }} --embedded-services esp,idf --build-dir build_esp32_default -m ethernet -v
+              pytest ${{inputs.app_path}} --target=${{ inputs.target }}
         # - name: Upload test results
         #   uses: actions/upload-artifact@v4
         #   if: always()

.github/workflows/publish-connect.yml

@@ -10,17 +10,18 @@ env:
     target: esp32
     idf_version: latest
     app_name: publish_connect_test
-    app_path: $IDF_PATH/tools/test_apps/protocols/mqtt/publish_connect_test
+    app_path: test/publish_connect_test
 
 jobs:
     build:
+        if: false
         name: Build Test App
         runs-on: ubuntu-22.04
         env:
-            EXAMPLE_MQTT_BROKER_SSL: "mqtt.eclipseprojects.io:1883"
-            EXAMPLE_MQTT_BROKER_TCP: "mqtt.eclipseprojects.io:8883"
-            EXAMPLE_MQTT_BROKER_WS: "mqtt.eclipseprojects.io"
-            EXAMPLE_MQTT_BROKER_WSS: "mqtt.eclipseprojects.io"
+            EXAMPLE_MQTT_BROKER_TCP: "mqtt.eclipseprojects.io:1883"
+            EXAMPLE_MQTT_BROKER_SSL: "mqtt.eclipseprojects.io:8883"
+            EXAMPLE_MQTT_BROKER_WS: "mqtt.eclipseprojects.io:80"
+            EXAMPLE_MQTT_BROKER_WSS: "mqtt.eclipseprojects.io:443"
         outputs:
             build_path: ${{steps.build.outputs.artifacts_path}}
         container: espressif/idf:latest
@@ -52,10 +53,39 @@ jobs:
                       ${{steps.build.outputs.artifacts_path}}/build_*/*.bin
                       ${{steps.build.outputs.artifacts_path}}/build_*/flasher_args.json
                       ${{steps.build.outputs.artifacts_path}}/build_*/config/sdkconfig.json
-                      ${{steps.build.outputs.artifacts_path}}/*.py
                   if-no-files-found: error
+            - name: Output env
+              shell: bash
+              run: |
+                  echo ${EXAMPLE_MQTT_BROKER_TCP}
+                  echo ${EXAMPLE_MQTT_BROKER_SSL}
+                  echo ${EXAMPLE_MQTT_BROKER_WS}
+                  echo ${EXAMPLE_MQTT_BROKER_WSS}
+
+    get_ip:
+        # if: github.repository == 'espressif/esp-mqtt'
+        name: Get Ip
+        runs-on: [self-hosted, linux, docker, ESP32-ETHERNET-KIT]
+        steps:
+            # - name: Add node for local tests
+            #   if: ${{ env.ACT }}
+            #   shell: bash
+            #   run: |
+            #       curl -fsSL https://deb.nodesource.com/setup_18.x | bash -
+            #       apt-get install -y nodejs
+            - name: Install iproute2
+              shell: bash
+              run: apt-get update && apt-get install -y iproute2
+            - name: Get runner IP
+              id: get_ip
+              shell: bash
+              run: |
+                  IP=$(ip route get 1| awk '{print $7; exit}')
+                  echo $IP
+                  echo "RUNNER_IP=$IP" >> $GITHUB_ENV
 
     run-on-target:
+        if: false
         # if: github.repository == 'espressif/esp-mqtt'
         name: Publish and Connect target test
         needs: build
@@ -84,10 +114,23 @@ jobs:
         #     TEST_RESULT_NAME: test_results_${{ matrix.runner.target }}_${{ matrix.runner.marker }}_${{ matrix.idf_ver }}
         #     TEST_RESULT_FILE: test_results_${{ matrix.runner.target }}_${{ matrix.runner.marker }}_${{ matrix.idf_ver }}.xml
         # runs-on: [self-hosted, linux, docker, "${{ matrix.runner.runs-on }}"]
-        container:
-            image: python:3.11-bookworm
-            options: --privileged # Privileged mode has access to serial ports
-        runs-on: [self-hosted, linux, docker, ESP32-ETHERNET-KIT]
+        # container:
+        #     image: python:3.11-bookworm
+        #     options: --privileged # Privileged mode has access to serial ports
+        # runs-on: [self-hosted, linux, docker, ESP32-ETHERNET-KIT]
+        runs-on: ubuntu-22.04
+        services:
+            mqtt-test-broker:
+                image: eclipse-mosquitto:latest
+                ports:
+                    - 42351:42351
+                    - 42352:42352
+                    - 42353:42353
+                    - 42354:42354
+                volumes:
+                    - ${{ github.workspace }}/ci/broker/config:/mosquitto/config
+                    - ${{ github.workspace }}/ci/broker/certs:/mosquitto/certfiles
+                    - ${{ github.workspace }}/ci/broker/.mosquitto.log:/var/log/mosquitto.log
 
         strategy:
             matrix:
@@ -107,22 +150,76 @@ jobs:
               run: |
                   curl -fsSL https://deb.nodesource.com/setup_18.x | bash -
                   apt-get install -y nodejs
+            - name: Install Python packages
+              shell: bash
+              env:
+                  PIP_EXTRA_INDEX_URL: "https://dl.espressif.com/pypi/"
+              run: pip install --prefer-binary paho-mqtt
+            - name: Install dependencies
+              shell: bash
+              run: apt-get update && apt-get install -y iproute2
             - uses: actions/checkout@v4
             - name: Download artifacts
               uses: actions/download-artifact@v4
               id: artifacts
               with:
                   name: mqtt_bin_${{env.target}}_${{ env.idf_version }}_${{ env.app_name }}
-                  path: test
-            - name: Check artifacts
+                  path: test/publish_connect_test
+            - name: Get runner IP
+              id: get_ip
               shell: bash
-              run: ls test/
-            # - name: Checkout IDF ${{inputs.idf_version}}
-            #   uses: actions/checkout@v4
-            #   with:
-            #       repository: espressif/esp-idf
-            #       path: idf
-            #       ref: master
+              run: |
+                  IP=$(ip route get 1| awk '{print $7; exit}')
+                  echo "RUNNER_IP=$IP" >> $GITHUB_ENV
+            - name: Run app in Docker
+              shell: bash
+              env:
+                  RUNNER_IP: ${{ env.RUNNER_IP }}
+              run: |
+                  docker run --rm \
+                    -v "${{ github.workspace }}:/workspace" \
+                    -w /workspace \
+                    -e RUNNER_IP="${RUNNER_IP}" \
+                    python:3.11-bookworm \
+                    bash -c "
+                      pip install --prefer-binary cryptography pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code &&
+                      pytest test/publish_connect_test --target=esp32
+                    "
+            - name: Cleanup workspace
+              if: always()
+              shell: bash
+              run: |
+                  rm -rf test/publish_connect_test
+            - name: Add node for local tests
+              if: ${{ env.ACT }}
+              shell: bash
+              run: |
+                  curl -fsSL https://deb.nodesource.com/setup_18.x | bash -
+                  apt-get install -y nodejs
+            - name: Install Python packages
+              shell: bash
+              env:
+                  PIP_EXTRA_INDEX_URL: "https://dl.espressif.com/pypi/"
+              run: pip install --prefer-binary paho-mqtt
+            - name: Install dependencies
+              shell: bash
+              run: apt-get update && apt-get install -y iproute2
+            - uses: actions/checkout@v4
+            - name: Download artifacts
+              uses: actions/download-artifact@v4
+              id: artifacts
+              with:
+                  name: mqtt_bin_${{env.target}}_${{ env.idf_version }}_${{ env.app_name }}
+                  path: test/publish_connect_test
+            # - name: Run mosquitto
+            #   shell: bash
+            #   run: mosquitto -c ci/broker/mosquitto.conf &
+            - name: Get runner IP
+              id: get_ip
+              shell: bash
+              run: |
+                  IP=$(ip route get 1| awk '{print $7; exit}')
+                  echo "RUNNER_IP=$IP" >> $GITHUB_ENV
             # - name: Install Python packages
             #   shell: bash
             #   env:
@@ -139,4 +236,6 @@ jobs:
               with:
                   target: esp32
                   app_name: publish_connnect_test
-                  app_path: test
+                  app_path: test/publish_connect_test
+              env:
+                  RUNNER_IP: ${{ env.RUNNER_IP }}

ci/broker/README.md

@@ -0,0 +1,21 @@
+### Prerequisites
+* installed docker-compose (https://docs.docker.com/compose/install/)
+
+### Rebuild and run one-line command
+```
+docker-compose up -d
+```
+
+### Rebuild and run with output to console, for some debug
+```
+docker-compose up
+```
+
+### (Example) Successful state
+```
+CONTAINER ID        IMAGE                      COMMAND                  CREATED             STATUS              PORTS                      NAMES
+407b9ccb85b9        eclipse-mosquitto:latest   "/docker-entrypoin..."   14 minutes ago      Up 4 seconds        0.0.0.0:42351->42351/tcp   mqtt-tcp-server
+5204c0cb11ef        eclipse-mosquitto:latest   "/docker-entrypoin..."   14 minutes ago      Up 4 seconds        0.0.0.0:42352->42352/tcp   mqtt-ssl-server
+cc8aa247c32e        eclipse-mosquitto:latest   "/docker-entrypoin..."   14 minutes ago      Up 4 seconds        0.0.0.0:42353->42353/tcp   mqtt-websockets-tcp-server
+41e189081c47        eclipse-mosquitto:latest   "/docker-entrypoin..."   14 minutes ago      Up 4 seconds        0.0.0.0:42354->42354/tcp   mqtt-websockets-ssl-server
+```

ci/broker/config/mosquitto.conf

@@ -0,0 +1,27 @@
+# General configuration
+log_type all
+connection_messages true
+allow_anonymous true
+
+# TCP Listener
+listener 42351
+protocol mqtt
+
+# WebSocket Listener
+listener 42353
+protocol websockets
+
+# SSL Listener
+listener 42352
+protocol mqtt
+cafile /mosquitto/certfiles/server.crt
+certfile /mosquitto/certfiles/server.crt
+keyfile /mosquitto/certfiles/server.key
+
+# WebSocket over SSL
+listener 42354
+protocol websockets
+cafile /mosquitto/certfiles/server.crt
+certfile /mosquitto/certfiles/server.crt
+keyfile /mosquitto/certfiles/server.key
+

ci/broker/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3"
+services:
+  mqtt-test-broker:
+    image: eclipse-mosquitto:latest
+    container_name: mqtt-test-broker
+    ports:
+      - 42351:42351
+      - 42352:42352
+      - 42353:42353
+      - 42354:42354
+    volumes:
+      - ./config/:/mosquitto/config
+      - ./certs/:/mosquitto/certfiles
+      - .mosquitto.log:/var/log/mosquitto.log

conftest.py

@@ -1,23 +1,81 @@
 import pytest
+import typing as t
+import os
+import logging as logger
 from pytest_embedded.plugin import multi_dut_argument
 from pytest_embedded.plugin import multi_dut_fixture
 from _pytest.fixtures import FixtureRequest
 
-def pytest_configure(config):
-    for plugin in config.pluginmanager.list_name_plugin():
-        print("Loaded plugins", plugin)
-
 @pytest.fixture
 @multi_dut_argument
 def config(request: FixtureRequest) -> str:
     return getattr(request, 'param', None) or 'default'  # type: ignore
 
 @pytest.fixture
 @multi_dut_fixture
-def target(request: FixtureRequest, dut_total: int, dut_index: int) -> str:
-    plugin = request.config.stash[pytest.StashKey['IdfPytestEmbedded']]
+def build_dir(
+    request: FixtureRequest,
+    app_path: str,
+    target: t.Optional[str],
+    config: t.Optional[str],
+) -> str:
+    """Find a valid build directory based on priority rules.
+
+    Checks local build directories in the following order:
+
+    1. build_<target>_<config>
+    2. build_<target>
+    3. build_<config>
+    4. build
+
+    :param request: Pytest fixture request
+    :param app_path: Path to the application
+    :param target: Target being used
+    :param config: Configuration being used
+
+    :returns: Valid build directory name
+
+    :raises ValueError: If no valid build directory is found
+    """
+    check_dirs = []
+    build_dir_arg = request.config.getoption('build_dir')
+
+    if build_dir_arg:
+        check_dirs.append(build_dir_arg)
+    if target is not None and config is not None:
+        check_dirs.append(f'build_{target}_{config}')
+    if target is not None:
+        check_dirs.append(f'build_{target}')
+    if config is not None:
+        check_dirs.append(f'build_{config}')
+    check_dirs.append('build')
+
+    for check_dir in check_dirs:
+        binary_path = os.path.join(app_path, check_dir)
+        if os.path.isdir(binary_path):
+            logger.info(f'Found valid binary path: {binary_path}')
+            return check_dir
+
+        logger.warning('Checking binary path: %s... missing... trying another location', binary_path)
+
+    raise ValueError(
+        'No valid build directory found. '
+        f'Please build the binary via "idf.py -B {check_dirs[0]} build" and run pytest again'
+    )
+
+@pytest.fixture
+def log_performance(record_property: t.Callable[[str, object], None]) -> t.Callable[[str, str], None]:
+    """
+    log performance item with pre-defined format to the console
+    and record it under the ``properties`` tag in the junit report if available.
+    """
 
-    if dut_total == 1:
-        return plugin.target[0]  # type: ignore
+    def real_func(item: str, value: str) -> None:
+        """
+        :param item: performance item name
+        :param value: performance value
+        """
+        logger.info('[Performance][%s]: %s', item, value)
+        record_property(item, value)
 
-    return plugin.target[dut_index]  # type: ignore
+    return real_func

pytest.ini

@@ -9,5 +9,5 @@ addopts =
   --skip-check-coredump y
   --logfile-extension ".txt"
   --check-duplicates y
-  --ignore-glob managed_components/*
-  --ignore pytest-embedded
+  --ignore-glob **/managed_components/*
+  --ignore=.github

test/publish_connect_test/CMakeLists.txt

@@ -7,7 +7,7 @@ idf_build_set_property(MINIMAL_BUILD ON)
 
 project(mqtt_publish_connect_test)
 
-target_add_binary_data(mqtt_publish_connect_test.elf "main/mqtt_eclipseprojects_io.pem" TEXT)
+target_add_binary_data(mqtt_publish_connect_test.elf "../../ci/broker/certs/server.crt" TEXT)
 target_add_binary_data(mqtt_publish_connect_test.elf "ca.crt" TEXT)
 target_add_binary_data(mqtt_publish_connect_test.elf "ca.der" TEXT)
 target_add_binary_data(mqtt_publish_connect_test.elf "client_pwd.key" TEXT)

test/publish_connect_test/common.py

@@ -0,0 +1,20 @@
+import socket
+import logging
+from typing import Any
+from pytest_embedded import Dut
+
+def get_host_ip4_by_dest_ip(dest_ip: str = '') -> str:
+    if not dest_ip:
+        dest_ip = '8.8.8.8'
+    s1 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    s1.connect((dest_ip, 80))
+    host_ip = s1.getsockname()[0]
+    s1.close()
+    assert isinstance(host_ip, str)
+    print(f'Using host ip: {host_ip}')
+    return host_ip
+
+def get_runner_ip(dut: Dut) -> Any:
+    dut_ip = dut.expect(r'IPv4 address: (\d+\.\d+\.\d+\.\d+)[^\d]', timeout=30).group(1).decode()
+    logging.info('Got IP={}'.format(dut_ip))
+    return get_host_ip4_by_dest_ip(dut_ip)

test/publish_connect_test/main/CMakeLists.txt

@@ -1,4 +1,4 @@
 idf_component_register(SRCS "publish_test.c" "connect_test.c" "publish_connect_test.c"
                     INCLUDE_DIRS "."
                     REQUIRES mqtt nvs_flash console esp_netif)
-target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")
+# target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")