integration tests upgrade (#1401)

* fix .env with newer docker-compose
* fix basic auth integration test
* protoc test fix with newer Python and protobuf versions

Author: emasab

tests/README.md

@@ -60,7 +60,7 @@ The easiest way to arrange for this is:
 
 And also:
 
-    source ./tests/docker/.env
+    source ./tests/docker/.env.sh
 
 which sets environment variables referenced by `./tests/integration/testconf.json`.
 

tests/docker/.env renamed to tests/docker/.env.sh

@@ -5,7 +5,7 @@ set -eu
 PY_DOCKER_BIN="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 export PASS="abcdefgh"
 
-source ${PY_DOCKER_BIN}/../.env
+source ${PY_DOCKER_BIN}/../.env.sh
 
 mkdir -p ${TLS}
 

tests/docker/bin/certify.sh

@@ -3,7 +3,7 @@
 set -eu
 
 PY_DOCKER_BIN="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source ${PY_DOCKER_BIN}/../.env
+source ${PY_DOCKER_BIN}/../.env.sh
 
 echo "Destroying cluster.."
 docker-compose -f $PY_DOCKER_COMPOSE_FILE down -v --remove-orphans

tests/docker/bin/cluster_down.sh

@@ -3,7 +3,7 @@
 set -eu
 
 PY_DOCKER_BIN="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-source ${PY_DOCKER_BIN}/../.env
+source ${PY_DOCKER_BIN}/../.env.sh
 
 # Wait for http service listener to come up and start serving
 # $1 http service name
@@ -28,7 +28,7 @@ await_http() {
 }
 
 echo "Configuring Environment..."
-source ${PY_DOCKER_SOURCE}/.env
+source ${PY_DOCKER_SOURCE}/.env.sh
 
 echo "Generating SSL certs..."
 ${PY_DOCKER_BIN}/certify.sh

tests/docker/bin/cluster_up.sh

@@ -52,9 +52,9 @@ services:
     volumes:
       - ./conf:/conf
     environment:
-      SCHEMA_REGISTRY_HOST_NAME: schema-registry2
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry-basic-auth
       SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas2
-      SCHEMA_REGISTRY_SCHEMA_REGISTRY_ZK_NAMESPACE: schema_registry2
+      SCHEMA_REGISTRY_SCHEMA_REGISTRY_GROUP_ID: schema-registry-basic-auth
       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092
       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8083, https://0.0.0.0:8084
       SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: https

tests/docker/docker-compose.yaml

@@ -28,6 +28,7 @@
 import json
 import gc
 import struct
+import re
 
 try:
     # Memory tracker
@@ -795,13 +796,17 @@ def verify_avro_basic_auth(mode_conf):
     if mode_conf is None:
         abort_on_missing_configuration('avro-basic-auth')
 
-    url = {
-        'schema.registry.basic.auth.credentials.source': 'URL'
+    url = mode_conf.get('schema.registry.url')
+    credentials = mode_conf.get('schema.registry.basic.auth.user.info')
+
+    url_conf = {
+        'schema.registry.basic.auth.credentials.source': 'URL',
+        'schema.registry.url': str(re.sub("(^https?://)", f"\\1{credentials}@", url))
     }
 
     user_info = {
         'schema.registry.basic.auth.credentials.source': 'USER_INFO',
-        'schema.registry.basic.auth.user.info': mode_conf.get('schema.registry.basic.auth.user.info')
+        'schema.registry.basic.auth.user.info': credentials
     }
 
     sasl_inherit = {
@@ -813,7 +818,7 @@ def verify_avro_basic_auth(mode_conf):
     base_conf = {
         'bootstrap.servers': bootstrap_servers,
         'error_cb': error_cb,
-        'schema.registry.url': schema_registry_url
+        'schema.registry.url': url
     }
 
     consumer_conf = dict({'group.id': generate_group_id(),
@@ -829,7 +834,7 @@ def verify_avro_basic_auth(mode_conf):
     run_avro_loop(dict(base_conf, **sasl_inherit), dict(consumer_conf, **sasl_inherit))
 
     print('-' * 10, 'Verifying basic auth source URL', '-' * 10)
-    run_avro_loop(dict(base_conf, **url), dict(consumer_conf, **url))
+    run_avro_loop(dict(base_conf, **url_conf), dict(consumer_conf, **url_conf))
 
 
 def run_avro_loop(producer_conf, consumer_conf):

tests/integration/integration_test.py

@@ -1,7 +1,7 @@
 syntax = "proto3";
 
-import "NestedTestProto.proto";
-import "PublicTestProto.proto";
+import "tests/integration/schema_registry/data/proto/NestedTestProto.proto";
+import "tests/integration/schema_registry/data/proto/PublicTestProto.proto";
 
 package tests.integration.serialization.data;