From ca7033d16b60a1a205032843ed0d5d2098e84c54 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 5 Jan 2023 22:28:21 +0000 Subject: [PATCH 1/8] Initial work towards typing support --- .gitignore | 2 + pyproject.toml | 6 + setup.py | 9 +- src/confluent_kafka/__init__.py | 10 +- src/confluent_kafka/admin/__init__.py | 111 +++--- src/confluent_kafka/admin/_acl.py | 63 ++-- src/confluent_kafka/admin/_config.py | 49 +-- src/confluent_kafka/admin/_resource.py | 6 +- src/confluent_kafka/avro/__init__.py | 29 +- .../avro/cached_schema_registry_client.py | 114 +++--- src/confluent_kafka/avro/error.py | 9 +- src/confluent_kafka/avro/load.py | 61 ++-- .../avro/serializer/__init__.py | 6 +- .../avro/serializer/message_serializer.py | 26 +- src/confluent_kafka/cimpl.pyi | 339 ++++++++++++++++++ src/confluent_kafka/deserializing_consumer.py | 9 +- src/confluent_kafka/error.py | 25 +- .../kafkatest/verifiable_client.py | 21 +- .../kafkatest/verifiable_consumer.py | 51 +-- .../kafkatest/verifiable_producer.py | 18 +- .../schema_registry/__init__.py | 11 +- src/confluent_kafka/schema_registry/avro.py | 35 +- src/confluent_kafka/schema_registry/error.py | 6 +- .../schema_registry/json_schema.py | 24 +- .../schema_registry/protobuf.py | 61 ++-- .../schema_registry/schema_registry_client.py | 111 +++--- src/confluent_kafka/serialization/__init__.py | 27 +- src/confluent_kafka/serializing_producer.py | 10 +- tests/avro/mock_registry.py | 2 +- 29 files changed, 845 insertions(+), 406 deletions(-) create mode 100644 pyproject.toml create mode 100644 src/confluent_kafka/cimpl.pyi diff --git a/.gitignore b/.gitignore index bfccfbd0b..1b0cf3843 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ tmp-KafkaCluster .venv venv_test venv_examples +.vscode/ +.dmypy.json \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..130522ef0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[tool.mypy] +show_error_codes=true +disallow_untyped_defs=true +disallow_untyped_calls=true +warn_redundant_casts=true +strict_optional=true \ No newline at end of file diff --git a/setup.py b/setup.py index 7d0d9f0a6..0219ad06d 100755 --- a/setup.py +++ b/setup.py @@ -12,13 +12,20 @@ INSTALL_REQUIRES = [ 'futures;python_version<"3.2"', 'enum34;python_version<"3.4"', + 'six' ] TEST_REQUIRES = [ 'pytest==4.6.4;python_version<"3.0"', 'pytest;python_version>="3.0"', 'pytest-timeout', - 'flake8' + 'flake8', + # Cap the version to avoid issues with newer editions. Should be periodically updated! + 'mypy<=0.991', + 'types-protobuf', + 'types-jsonschema', + 'types-requests', + 'types-six' ] DOC_REQUIRES = ['sphinx', 'sphinx-rtd-theme'] diff --git a/src/confluent_kafka/__init__.py b/src/confluent_kafka/__init__.py index e8e5cc30c..e6e343981 100644 --- a/src/confluent_kafka/__init__.py +++ b/src/confluent_kafka/__init__.py @@ -60,19 +60,19 @@ class ThrottleEvent(object): :ivar float throttle_time: The amount of time (in seconds) the broker throttled (delayed) the request """ - def __init__(self, broker_name, - broker_id, - throttle_time): + def __init__(self, broker_name: str, + broker_id: int, + throttle_time: float): self.broker_name = broker_name self.broker_id = broker_id self.throttle_time = throttle_time - def __str__(self): + def __str__(self) -> str: return "{}/{} throttled for {} ms".format(self.broker_name, self.broker_id, int(self.throttle_time * 1000)) -def _resolve_plugins(plugins): +def _resolve_plugins(plugins: str) -> str: """ Resolve embedded plugins from the wheel's library directory. For internal module use only. diff --git a/src/confluent_kafka/admin/__init__.py b/src/confluent_kafka/admin/__init__.py index ef36a7f1b..11f65062d 100644 --- a/src/confluent_kafka/admin/__init__.py +++ b/src/confluent_kafka/admin/__init__.py @@ -15,7 +15,10 @@ """ Kafka admin client: create, view, alter, and delete topics and resources. """ -import concurrent.futures +from concurrent.futures import Future +from typing import Callable, Dict, List, Optional, Sequence, Tuple, Type, TypeVar + +from pytest import Config # Unused imports are keeped to be accessible using this public module from ._config import (ConfigSource, # noqa: F401 @@ -45,7 +48,7 @@ RESOURCE_BROKER) -class AdminClient (_AdminClientImpl): +class AdminClient(_AdminClientImpl): """ AdminClient provides admin operations for Kafka brokers, topics, groups, and other resource types supported by the broker. @@ -67,7 +70,7 @@ class AdminClient (_AdminClientImpl): Requires broker version v0.11.0.0 or later. """ - def __init__(self, conf): + def __init__(self, conf: Dict): """ Create a new AdminClient using the provided configuration dictionary. @@ -80,13 +83,14 @@ def __init__(self, conf): super(AdminClient, self).__init__(conf) @staticmethod - def _make_topics_result(f, futmap): + def _make_topics_result(f: Future, futmap: Dict[str, Future]) -> None: """ Map per-topic results to per-topic futures in futmap. The result value of each (successful) future is None. """ try: result = f.result() + assert isinstance(result, Dict) for topic, error in result.items(): fut = futmap.get(topic, None) if fut is None: @@ -104,13 +108,14 @@ def _make_topics_result(f, futmap): fut.set_exception(e) @staticmethod - def _make_resource_result(f, futmap): + def _make_resource_result(f: Future, futmap: Dict[ConfigResource, Future]) -> None: """ Map per-resource results to per-resource futures in futmap. The result value of each (successful) future is a ConfigResource. """ try: result = f.result() + assert isinstance(result, Dict) for resource, configs in result.items(): fut = futmap.get(resource, None) if fut is None: @@ -128,8 +133,10 @@ def _make_resource_result(f, futmap): for resource, fut in futmap.items(): fut.set_exception(e) + _acl_type = TypeVar("_acl_type", bound=AclBinding) + @staticmethod - def _make_acls_result(f, futmap): + def _make_acls_result(f: Future, futmap: Dict[_acl_type, Future]) -> None: """ Map create ACL binding results to corresponding futures in futmap. For create_acls the result value of each (successful) future is None. @@ -155,14 +162,16 @@ def _make_acls_result(f, futmap): fut.set_exception(e) @staticmethod - def _create_future(): - f = concurrent.futures.Future() + def _create_future() -> Future: + f: Future = Future() if not f.set_running_or_notify_cancel(): raise RuntimeError("Future was cancelled prematurely") return f + _futures_map_key = TypeVar("_futures_map_key") + @staticmethod - def _make_futures(futmap_keys, class_check, make_result_fn): + def _make_futures(futmap_keys: List[_futures_map_key], class_check: Optional[Type], make_result_fn: Callable[[Future, Dict[_futures_map_key, Future]], None]) -> Tuple[Future, Dict[_futures_map_key, Future]]: """ Create futures and a futuremap for the keys in futmap_keys, and create a request-level future to be bassed to the C API. @@ -182,10 +191,10 @@ def _make_futures(futmap_keys, class_check, make_result_fn): return f, futmap @staticmethod - def _has_duplicates(items): + def _has_duplicates(items: Sequence) -> bool: return len(set(items)) != len(items) - def create_topics(self, new_topics, **kwargs): + def create_topics(self, new_topics: List[NewTopic], **kwargs: object) -> Dict[str, Future]: # type: ignore[override] """ Create one or more new topics. @@ -219,7 +228,7 @@ def create_topics(self, new_topics, **kwargs): return futmap - def delete_topics(self, topics, **kwargs): + def delete_topics(self, topics: List[str], **kwargs: object) -> Dict[str, Future]: # type: ignore[override] """ Delete one or more topics. @@ -249,15 +258,15 @@ def delete_topics(self, topics, **kwargs): return futmap - def list_topics(self, *args, **kwargs): + def list_topics(self, *args: object, **kwargs: object) -> object: return super(AdminClient, self).list_topics(*args, **kwargs) - def list_groups(self, *args, **kwargs): + def list_groups(self, *args: object, **kwargs: object) -> object: return super(AdminClient, self).list_groups(*args, **kwargs) - def create_partitions(self, new_partitions, **kwargs): + def create_partitions(self, new_partitions: List[NewPartitions], **kwargs: object) -> Dict[str, Future]: # type: ignore[override] """ Create additional partitions for the given topics. @@ -290,7 +299,7 @@ def create_partitions(self, new_partitions, **kwargs): return futmap - def describe_configs(self, resources, **kwargs): + def describe_configs(self, resources: List[ConfigResource], **kwargs: object) -> Dict[ConfigResource, Future]: # type: ignore[override] """ Get the configuration of the specified resources. @@ -322,7 +331,7 @@ def describe_configs(self, resources, **kwargs): return futmap - def alter_configs(self, resources, **kwargs): + def alter_configs(self, resources: List[ConfigResource], **kwargs: object) -> Dict[ConfigResource, Future]: # type: ignore[override] """ Update configuration properties for the specified resources. Updates are not transactional so they may succeed for a subset @@ -365,7 +374,7 @@ def alter_configs(self, resources, **kwargs): return futmap - def create_acls(self, acls, **kwargs): + def create_acls(self, acls: List[AclBinding], **kwargs: object) -> Dict[AclBinding, Future]: # type: ignore[override] """ Create one or more ACL bindings. @@ -394,7 +403,7 @@ def create_acls(self, acls, **kwargs): return futmap - def describe_acls(self, acl_binding_filter, **kwargs): + def describe_acls(self, acl_binding_filter: List[AclBindingFilter], **kwargs: object) -> Future: # type: ignore[override] """ Match ACL bindings by filter. @@ -429,7 +438,7 @@ def describe_acls(self, acl_binding_filter, **kwargs): return f - def delete_acls(self, acl_binding_filters, **kwargs): + def delete_acls(self, acl_binding_filters: List[AclBindingFilter], **kwargs: object) -> Dict[AclBindingFilter, Future]: # type: ignore[override] """ Delete ACL bindings matching one or more ACL binding filters. @@ -477,24 +486,24 @@ class ClusterMetadata (object): This class is typically not user instantiated. """ - def __init__(self): + def __init__(self) -> None: self.cluster_id = None """Cluster id string, if supported by the broker, else None.""" - self.controller_id = -1 + self.controller_id: int = -1 """Current controller broker id, or -1.""" - self.brokers = {} + self.brokers: Dict[object, object] = {} """Map of brokers indexed by the broker id (int). Value is a BrokerMetadata object.""" - self.topics = {} + self.topics: Dict[object, object] = {} """Map of topics indexed by the topic name. Value is a TopicMetadata object.""" - self.orig_broker_id = -1 + self.orig_broker_id: int = -1 """The broker this metadata originated from.""" self.orig_broker_name = None """The broker name/address this metadata originated from.""" - def __repr__(self): + def __repr__(self) -> str: return "ClusterMetadata({})".format(self.cluster_id) - def __str__(self): + def __str__(self) -> str: return str(self.cluster_id) @@ -505,7 +514,7 @@ class BrokerMetadata (object): This class is typically not user instantiated. """ - def __init__(self): + def __init__(self) -> None: self.id = -1 """Broker id""" self.host = None @@ -513,10 +522,10 @@ def __init__(self): self.port = -1 """Broker port""" - def __repr__(self): + def __repr__(self) -> str: return "BrokerMetadata({}, {}:{})".format(self.id, self.host, self.port) - def __str__(self): + def __str__(self) -> str: return "{}:{}/{}".format(self.host, self.port, self.id) @@ -530,22 +539,22 @@ class TopicMetadata (object): # Sphinx issue where it tries to reference the same instance variable # on other classes which raises a warning/error. - def __init__(self): - self.topic = None + def __init__(self) -> None: + self.topic: Optional[str] = None """Topic name""" - self.partitions = {} + self.partitions: Dict[object, object] = {} """Map of partitions indexed by partition id. Value is a PartitionMetadata object.""" - self.error = None + self.error: Optional[KafkaError] = None """Topic error, or None. Value is a KafkaError object.""" - def __repr__(self): + def __repr__(self) -> str: if self.error is not None: return "TopicMetadata({}, {} partitions, {})".format(self.topic, len(self.partitions), self.error) else: return "TopicMetadata({}, {} partitions)".format(self.topic, len(self.partitions)) - def __str__(self): - return self.topic + def __str__(self) -> str: + return str(self.topic) class PartitionMetadata (object): @@ -560,25 +569,25 @@ class PartitionMetadata (object): of a broker id in the brokers dict. """ - def __init__(self): - self.id = -1 + def __init__(self) -> None: + self.id: int = -1 """Partition id.""" - self.leader = -1 + self.leader: int = -1 """Current leader broker for this partition, or -1.""" - self.replicas = [] + self.replicas: List[object] = [] """List of replica broker ids for this partition.""" - self.isrs = [] + self.isrs: List[object] = [] """List of in-sync-replica broker ids for this partition.""" self.error = None """Partition error, or None. Value is a KafkaError object.""" - def __repr__(self): + def __repr__(self) -> str: if self.error is not None: return "PartitionMetadata({}, {})".format(self.id, self.error) else: return "PartitionMetadata({})".format(self.id) - def __str__(self): + def __str__(self) -> str: return "{}".format(self.id) @@ -591,7 +600,7 @@ class GroupMember(object): This class is typically not user instantiated. """ # noqa: E501 - def __init__(self,): + def __init__(self) -> None: self.id = None """Member id (generated by broker).""" self.client_id = None @@ -610,10 +619,10 @@ class GroupMetadata(object): This class is typically not user instantiated. """ - def __init__(self): + def __init__(self) -> None: self.broker = None """Originating broker metadata.""" - self.id = None + self.id: Optional[str] = None """Group name.""" self.error = None """Broker-originated error, or None. Value is a KafkaError object.""" @@ -623,14 +632,14 @@ def __init__(self): """Group protocol type.""" self.protocol = None """Group protocol.""" - self.members = [] + self.members: List[object] = [] """Group members.""" - def __repr__(self): + def __repr__(self) -> str: if self.error is not None: return "GroupMetadata({}, {})".format(self.id, self.error) else: return "GroupMetadata({})".format(self.id) - def __str__(self): - return self.id + def __str__(self) -> str: + return str(self.id) diff --git a/src/confluent_kafka/admin/_acl.py b/src/confluent_kafka/admin/_acl.py index 853ad2158..461c3fc2e 100644 --- a/src/confluent_kafka/admin/_acl.py +++ b/src/confluent_kafka/admin/_acl.py @@ -14,13 +14,13 @@ from enum import Enum import functools +from typing import Dict, List, Tuple, Type, TypeVar, cast from .. import cimpl as _cimpl from ._resource import ResourceType, ResourcePatternType -try: - string_type = basestring -except NameError: - string_type = str +import six + +string_type = six.string_types[0] class AclOperation(Enum): @@ -41,9 +41,10 @@ class AclOperation(Enum): ALTER_CONFIGS = _cimpl.ACL_OPERATION_ALTER_CONFIGS #: ALTER_CONFIGS operation IDEMPOTENT_WRITE = _cimpl.ACL_OPERATION_IDEMPOTENT_WRITE #: IDEMPOTENT_WRITE operation - def __lt__(self, other): + def __lt__(self, other: object) -> bool: if self.__class__ != other.__class__: return NotImplemented + assert isinstance(other, self.__class__) return self.value < other.value @@ -56,9 +57,10 @@ class AclPermissionType(Enum): DENY = _cimpl.ACL_PERMISSION_TYPE_DENY #: Disallows access ALLOW = _cimpl.ACL_PERMISSION_TYPE_ALLOW #: Grants access - def __lt__(self, other): + def __lt__(self, other: object) -> bool: if self.__class__ != other.__class__: return NotImplemented + assert isinstance(other, self.__class__) return self.value < other.value @@ -88,9 +90,9 @@ class AclBinding(object): The permission type for the specified operation. """ - def __init__(self, restype, name, - resource_pattern_type, principal, host, - operation, permission_type): + def __init__(self, restype: ResourceType, name: str, + resource_pattern_type: ResourcePatternType, principal: str, host: str, + operation: AclOperation, permission_type: AclPermissionType): self.restype = restype self.name = name self.resource_pattern_type = resource_pattern_type @@ -105,18 +107,19 @@ def __init__(self, restype, name, self.operation_int = int(self.operation.value) self.permission_type_int = int(self.permission_type.value) - def _check_not_none(self, vars_to_check): + def _check_not_none(self, vars_to_check: List[str]) -> None: for param in vars_to_check: if getattr(self, param) is None: raise ValueError("Expected %s to be not None" % (param,)) - def _check_is_string(self, vars_to_check): + def _check_is_string(self, vars_to_check: List[str]) -> None: for param in vars_to_check: param_value = getattr(self, param) if param_value is not None and not isinstance(param_value, string_type): raise TypeError("Expected %s to be a string" % (param,)) - def _convert_to_enum(self, val, enum_clazz): + # FIXME: Should really return the enum_clazz, but not sure how! + def _convert_to_enum(self, val: object, enum_clazz: Type[Enum]) -> object: if type(val) == str: # Allow it to be specified as case-insensitive string, for convenience. try: @@ -133,26 +136,26 @@ def _convert_to_enum(self, val, enum_clazz): return val - def _convert_enums(self): - self.restype = self._convert_to_enum(self.restype, ResourceType) - self.resource_pattern_type = self._convert_to_enum(self.resource_pattern_type, ResourcePatternType) - self.operation = self._convert_to_enum(self.operation, AclOperation) - self.permission_type = self._convert_to_enum(self.permission_type, AclPermissionType) + def _convert_enums(self) -> None: + self.restype = cast(ResourceType, self._convert_to_enum(self.restype, ResourceType)) + self.resource_pattern_type = cast(ResourcePatternType, self._convert_to_enum(self.resource_pattern_type, ResourcePatternType)) + self.operation = cast(AclOperation, self._convert_to_enum(self.operation, AclOperation)) + self.permission_type = cast(AclPermissionType, self._convert_to_enum(self.permission_type, AclPermissionType)) - def _check_forbidden_enums(self, forbidden_enums): + def _check_forbidden_enums(self, forbidden_enums: Dict[str, List]) -> None: for k, v in forbidden_enums.items(): enum_value = getattr(self, k) if enum_value in v: raise ValueError("Cannot use enum %s, value %s in this class" % (k, enum_value.name)) - def _not_none_args(self): + def _not_none_args(self) -> List[str]: return ["restype", "name", "resource_pattern_type", "principal", "host", "operation", "permission_type"] - def _string_args(self): + def _string_args(self) -> List[str]: return ["name", "principal", "host"] - def _forbidden_enums(self): + def _forbidden_enums(self) -> Dict[str, List[Enum]]: return { "restype": [ResourceType.ANY], "resource_pattern_type": [ResourcePatternType.ANY, @@ -161,7 +164,7 @@ def _forbidden_enums(self): "permission_type": [AclPermissionType.ANY] } - def _convert_args(self): + def _convert_args(self) -> None: not_none_args = self._not_none_args() string_args = self._string_args() forbidden_enums = self._forbidden_enums() @@ -170,26 +173,28 @@ def _convert_args(self): self._convert_enums() self._check_forbidden_enums(forbidden_enums) - def __repr__(self): + def __repr__(self) -> str: type_name = type(self).__name__ return "%s(%s,%s,%s,%s,%s,%s,%s)" % ((type_name,) + self._to_tuple()) - def _to_tuple(self): + def _to_tuple(self) -> Tuple: return (self.restype, self.name, self.resource_pattern_type, self.principal, self.host, self.operation, self.permission_type) - def __hash__(self): + def __hash__(self) -> int: return hash(self._to_tuple()) - def __lt__(self, other): + def __lt__(self, other: object) -> bool: if self.__class__ != other.__class__: return NotImplemented + assert isinstance(other, self.__class__) return self._to_tuple() < other._to_tuple() - def __eq__(self, other): + def __eq__(self, other: object) -> bool: if self.__class__ != other.__class__: return NotImplemented + assert isinstance(other, self.__class__) return self._to_tuple() == other._to_tuple() @@ -218,11 +223,11 @@ class AclBindingFilter(AclBinding): The permission type to match or :attr:`AclPermissionType.ANY` to match any value. """ - def _not_none_args(self): + def _not_none_args(self) -> List[str]: return ["restype", "resource_pattern_type", "operation", "permission_type"] - def _forbidden_enums(self): + def _forbidden_enums(self) -> Dict[str, List[Enum]]: return { "restype": [ResourceType.UNKNOWN], "resource_pattern_type": [ResourcePatternType.UNKNOWN], diff --git a/src/confluent_kafka/admin/_config.py b/src/confluent_kafka/admin/_config.py index 678ffa88b..60eeee25d 100644 --- a/src/confluent_kafka/admin/_config.py +++ b/src/confluent_kafka/admin/_config.py @@ -14,6 +14,7 @@ from enum import Enum import functools +from typing import Dict, List, Optional, Union from .. import cimpl as _cimpl from ._resource import ResourceType @@ -40,13 +41,13 @@ class ConfigEntry(object): This class is typically not user instantiated. """ - def __init__(self, name, value, - source=ConfigSource.UNKNOWN_CONFIG, - is_read_only=False, - is_default=False, - is_sensitive=False, - is_synonym=False, - synonyms=[]): + def __init__(self, name: str, value: str, + source: ConfigSource=ConfigSource.UNKNOWN_CONFIG, + is_read_only: bool=False, + is_default: bool=False, + is_sensitive: bool=False, + is_synonym: bool=False, + synonyms: List[str]=[]): """ This class is typically not user instantiated. """ @@ -72,10 +73,10 @@ def __init__(self, name, value, self.synonyms = synonyms """A list of synonyms (ConfigEntry) and alternate sources for this configuration property.""" - def __repr__(self): + def __repr__(self) -> str: return "ConfigEntry(%s=\"%s\")" % (self.name, self.value) - def __str__(self): + def __str__(self) -> str: return "%s=\"%s\"" % (self.name, self.value) @@ -98,8 +99,8 @@ class ConfigResource(object): Type = ResourceType - def __init__(self, restype, name, - set_config=None, described_configs=None, error=None): + def __init__(self, restype: Union[str, int, ResourceType], name: str, + set_config: Optional[Dict[str, str]]=None, described_configs: Optional[object]=None, error: Optional[object]=None): """ :param ConfigResource.Type restype: Resource type. :param str name: The resource name, which depends on restype. @@ -113,18 +114,20 @@ def __init__(self, restype, name, if name is None: raise ValueError("Expected resource name to be a string") - if type(restype) == str: + if isinstance(restype, str): # Allow resource type to be specified as case-insensitive string, for convenience. try: - restype = ConfigResource.Type[restype.upper()] + self.restype = ConfigResource.Type[restype.upper()] except KeyError: raise ValueError("Unknown resource type \"%s\": should be a ConfigResource.Type" % restype) - elif type(restype) == int: + elif isinstance(restype, int): # The C-code passes restype as an int, convert to Type. - restype = ConfigResource.Type(restype) + self.restype = ConfigResource.Type(restype) + + else: + self.restype = restype - self.restype = restype self.restype_int = int(self.restype.value) # for the C code self.name = name @@ -136,31 +139,33 @@ def __init__(self, restype, name, self.configs = described_configs self.error = error - def __repr__(self): + def __repr__(self) -> str: if self.error is not None: return "ConfigResource(%s,%s,%r)" % (self.restype, self.name, self.error) else: return "ConfigResource(%s,%s)" % (self.restype, self.name) - def __hash__(self): + def __hash__(self) -> int: return hash((self.restype, self.name)) - def __lt__(self, other): + def __lt__(self, other: object) -> bool: + assert isinstance(other, ConfigResource) if self.restype < other.restype: return True return self.name.__lt__(other.name) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: + assert isinstance(other, ConfigResource) return self.restype == other.restype and self.name == other.name - def __len__(self): + def __len__(self) -> int: """ :rtype: int :returns: number of configuration entries/operations """ return len(self.set_config_dict) - def set_config(self, name, value, overwrite=True): + def set_config(self, name: str, value: str, overwrite: bool=True) -> None: """ Set/overwrite a configuration value. diff --git a/src/confluent_kafka/admin/_resource.py b/src/confluent_kafka/admin/_resource.py index b786f3a9a..5920fd2a4 100644 --- a/src/confluent_kafka/admin/_resource.py +++ b/src/confluent_kafka/admin/_resource.py @@ -26,9 +26,10 @@ class ResourceType(Enum): GROUP = _cimpl.RESOURCE_GROUP #: Group resource. Resource name is group.id. BROKER = _cimpl.RESOURCE_BROKER #: Broker resource. Resource name is broker id. - def __lt__(self, other): + def __lt__(self, other: object) -> bool: if self.__class__ != other.__class__: return NotImplemented + assert isinstance(other, self.__class__) return self.value < other.value @@ -42,7 +43,8 @@ class ResourcePatternType(Enum): LITERAL = _cimpl.RESOURCE_PATTERN_LITERAL #: Literal: A literal resource name PREFIXED = _cimpl.RESOURCE_PATTERN_PREFIXED #: Prefixed: A prefixed resource name - def __lt__(self, other): + def __lt__(self, other: object) -> bool: if self.__class__ != other.__class__: return NotImplemented + assert isinstance(other, self.__class__) return self.value < other.value diff --git a/src/confluent_kafka/avro/__init__.py b/src/confluent_kafka/avro/__init__.py index 4906f9d96..6d9cb214f 100644 --- a/src/confluent_kafka/avro/__init__.py +++ b/src/confluent_kafka/avro/__init__.py @@ -19,16 +19,18 @@ Avro schema registry module: Deals with encoding and decoding of messages with avro schemas """ +from typing import Dict, Optional, cast import warnings from confluent_kafka import Producer, Consumer from confluent_kafka.avro.error import ClientError -from confluent_kafka.avro.load import load, loads # noqa +from confluent_kafka.avro.load import load, loads, schema # noqa from confluent_kafka.avro.cached_schema_registry_client import CachedSchemaRegistryClient from confluent_kafka.avro.serializer import (SerializerError, # noqa KeySerializerError, ValueSerializerError) from confluent_kafka.avro.serializer.message_serializer import MessageSerializer +from confluent_kafka.cimpl import Message class AvroProducer(Producer): @@ -48,8 +50,8 @@ class AvroProducer(Producer): :param str default_value_schema: Optional default avro schema for value """ - def __init__(self, config, default_key_schema=None, - default_value_schema=None, schema_registry=None, **kwargs): + def __init__(self, config: Dict, default_key_schema: Optional[schema.Schema]=None, + default_value_schema: Optional[schema.Schema]=None, schema_registry: Optional[CachedSchemaRegistryClient]=None, **kwargs: object): warnings.warn( "AvroProducer has been deprecated. Use AvroSerializer instead.", category=DeprecationWarning, stacklevel=2) @@ -77,7 +79,7 @@ def __init__(self, config, default_key_schema=None, self._key_schema = default_key_schema self._value_schema = default_value_schema - def produce(self, **kwargs): + def produce(self, topic: str, **kwargs: object) -> None: """ Asynchronously sends message to Kafka by encoding with specified or default avro schema. @@ -93,12 +95,9 @@ def produce(self, **kwargs): :raises BufferError: If producer queue is full. :raises KafkaException: For other produce failures. """ - # get schemas from kwargs if defined - key_schema = kwargs.pop('key_schema', self._key_schema) - value_schema = kwargs.pop('value_schema', self._value_schema) - topic = kwargs.pop('topic', None) - if not topic: - raise ClientError("Topic name not specified.") + # get schemas from kwargs if defined + key_schema = cast(Optional[schema.Schema], kwargs.pop('key_schema', self._key_schema)) + value_schema = cast(Optional[schema.Schema], kwargs.pop('value_schema', self._value_schema)) value = kwargs.pop('value', None) key = kwargs.pop('key', None) @@ -108,13 +107,13 @@ def produce(self, **kwargs): else: raise ValueSerializerError("Avro schema required for values") - if key is not None: + if key is not None: if key_schema: key = self._serializer.encode_record_with_schema(topic, key_schema, key, True) else: raise KeySerializerError("Avro schema required for key") - super(AvroProducer, self).produce(topic, value, key, **kwargs) + super(AvroProducer, self).produce(topic, value=value, key=key, **kwargs) class AvroConsumer(Consumer): @@ -135,7 +134,7 @@ class AvroConsumer(Consumer): :raises ValueError: For invalid configurations """ - def __init__(self, config, schema_registry=None, reader_key_schema=None, reader_value_schema=None, **kwargs): + def __init__(self, config: Dict, schema_registry: Optional[CachedSchemaRegistryClient]=None, reader_key_schema: Optional[schema.Schema]=None, reader_value_schema: Optional[schema.Schema]=None, **kwargs: object): warnings.warn( "AvroConsumer has been deprecated. Use AvroDeserializer instead.", category=DeprecationWarning, stacklevel=2) @@ -160,7 +159,7 @@ def __init__(self, config, schema_registry=None, reader_key_schema=None, reader_ super(AvroConsumer, self).__init__(ap_conf, **kwargs) self._serializer = MessageSerializer(schema_registry, reader_key_schema, reader_value_schema) - def poll(self, timeout=None): + def poll(self, timeout: float=-1) -> Optional[Message]: """ This is an overriden method from confluent_kafka.Consumer class. This handles message deserialization using avro schema @@ -169,8 +168,6 @@ def poll(self, timeout=None): :returns: message object with deserialized key and value as dict objects :rtype: Message """ - if timeout is None: - timeout = -1 message = super(AvroConsumer, self).poll(timeout) if message is None: return None diff --git a/src/confluent_kafka/avro/cached_schema_registry_client.py b/src/confluent_kafka/avro/cached_schema_registry_client.py index 924c5f1d3..22868025b 100644 --- a/src/confluent_kafka/avro/cached_schema_registry_client.py +++ b/src/confluent_kafka/avro/cached_schema_registry_client.py @@ -20,19 +20,21 @@ # derived from https://github.com/verisign/python-confluent-schemaregistry.git # import logging +from turtle import pos +from typing import Dict, Optional, Sized, Tuple, TypeVar, Union, cast import warnings from collections import defaultdict from requests import Session, utils +from confluent_kafka.schema_registry.schema_registry_client import Schema + from .error import ClientError -from . import loads +from . import loads, schema + +import six -# Python 2 considers int an instance of str -try: - string_type = basestring # noqa -except NameError: - string_type = str +string_type = six.string_types[0] VALID_LEVELS = ['NONE', 'FULL', 'FORWARD', 'BACKWARD'] VALID_METHODS = ['GET', 'POST', 'PUT', 'DELETE'] @@ -63,10 +65,11 @@ class CachedSchemaRegistryClient(object): :param str key_location: Path to client's private key used for authentication. """ - def __init__(self, url, max_schemas_per_subject=1000, ca_location=None, cert_location=None, key_location=None): - # In order to maintain compatibility the url(conf in future versions) param has been preserved for now. - conf = url - if not isinstance(url, dict): + def __init__(self, url: Union[str, Dict[str, object]], max_schemas_per_subject: int=1000, ca_location: Optional[str]=None, cert_location: Optional[str]=None, key_location: Optional[str]=None): + # In order to maintain compatibility the url(conf in future versions) param has been preserved for now. + if isinstance(url, dict): + conf = url + else: conf = { 'url': url, 'ssl.ca.location': ca_location, @@ -84,7 +87,7 @@ def __init__(self, url, max_schemas_per_subject=1000, ca_location=None, cert_loc """Construct a Schema Registry client""" # Ensure URL valid scheme is included; http[s] - url = conf.pop('url', '') + url = cast(str, conf.pop('url', '')) if not isinstance(url, string_type): raise TypeError("URL must be of type str") @@ -94,17 +97,24 @@ def __init__(self, url, max_schemas_per_subject=1000, ca_location=None, cert_loc self.url = url.rstrip('/') # subj => { schema => id } - self.subject_to_schema_ids = defaultdict(dict) + self.subject_to_schema_ids: Dict[str, Dict[schema.Schema, int]] = defaultdict(dict) # id => avro_schema - self.id_to_schema = defaultdict(dict) + self.id_to_schema: Dict[int, schema.Schema] = {} # subj => { schema => version } - self.subject_to_schema_versions = defaultdict(dict) + self.subject_to_schema_versions: Dict[str, Dict[schema.Schema, int]] = defaultdict(dict) s = Session() - ca_path = conf.pop('ssl.ca.location', None) + ca_path = cast(Optional[str], conf.pop('ssl.ca.location', None)) if ca_path is not None: s.verify = ca_path - s.cert = self._configure_client_tls(conf) + _conf_cert = self._configure_client_tls(conf) + + # Logic in _configure_client_tls promises both of the output variables are of the same type + if _conf_cert == [None, None]: + s.cert = None + else: + s.cert = cast(Tuple[str, str], _conf_cert) + s.auth = self._configure_basic_auth(self.url, conf) self.url = utils.urldefragauth(self.url) @@ -115,46 +125,49 @@ def __init__(self, url, max_schemas_per_subject=1000, ca_location=None, cert_loc if len(conf) > 0: raise ValueError("Unrecognized configuration properties: {}".format(conf.keys())) - def __del__(self): + def __del__(self) -> None: self.close() - def __enter__(self): + def __enter__(self) -> "CachedSchemaRegistryClient": return self - def __exit__(self, *args): + def __exit__(self, *args: object) -> None: self.close() - def close(self): + def close(self) -> None: # Constructor exceptions may occur prior to _session being set. if hasattr(self, '_session'): self._session.close() @staticmethod - def _configure_basic_auth(url, conf): + def _configure_basic_auth(url: str, conf: Dict) -> Tuple[str, str]: auth_provider = conf.pop('basic.auth.credentials.source', 'URL').upper() if auth_provider not in VALID_AUTH_PROVIDERS: raise ValueError("schema.registry.basic.auth.credentials.source must be one of {}" .format(VALID_AUTH_PROVIDERS)) + auth: Tuple[str, str] if auth_provider == 'SASL_INHERIT': if conf.pop('sasl.mechanism', '').upper() == 'GSSAPI': raise ValueError("SASL_INHERIT does not support SASL mechanism GSSAPI") - auth = (conf.pop('sasl.username', ''), conf.pop('sasl.password', '')) + auth = (cast(str, conf.pop('sasl.username', '')), cast(str, conf.pop('sasl.password', ''))) elif auth_provider == 'USER_INFO': - auth = tuple(conf.pop('basic.auth.user.info', '').split(':')) + possible_auth = tuple(cast(str, conf.pop('basic.auth.user.info', ':')).split(':')) + assert len(possible_auth) == 2, possible_auth + auth = cast(Tuple[str, str], possible_auth) else: auth = utils.get_auth_from_url(url) return auth @staticmethod - def _configure_client_tls(conf): - cert = conf.pop('ssl.certificate.location', None), conf.pop('ssl.key.location', None) + def _configure_client_tls(conf: Dict) -> Tuple[Optional[str], Optional[str]]: + cert = cast(Optional[str], conf.pop('ssl.certificate.location', None)), cast(Optional[str], conf.pop('ssl.key.location', None)) # Both values can be None or no values can be None if bool(cert[0]) != bool(cert[1]): raise ValueError( "Both schema.registry.ssl.certificate.location and schema.registry.ssl.key.location must be set") return cert - def _send_request(self, url, method='GET', body=None, headers={}): + def _send_request(self, url: str, method: str='GET', body: Optional[Sized]=None, headers: Dict={}) -> Tuple[object, int]: if method not in VALID_METHODS: raise ClientError("Method {} is invalid; valid methods include {}".format(method, VALID_METHODS)) @@ -171,12 +184,16 @@ def _send_request(self, url, method='GET', body=None, headers={}): except ValueError: return response.content, response.status_code + CacheKey = TypeVar("CacheKey") + SubCacheKey = TypeVar("SubCacheKey") + SubCacheValue = TypeVar("SubCacheValue") + @staticmethod - def _add_to_cache(cache, subject, schema, value): + def _add_to_cache(cache: Dict[CacheKey, Dict[SubCacheKey, SubCacheValue]], subject: CacheKey, schema: SubCacheKey, value: SubCacheValue) -> None: sub_cache = cache[subject] sub_cache[schema] = value - def _cache_schema(self, schema, schema_id, subject=None, version=None): + def _cache_schema(self, schema: schema.Schema, schema_id: int, subject: Optional[str]=None, version: Optional[int]=None) -> None: # don't overwrite anything if schema_id in self.id_to_schema: schema = self.id_to_schema[schema_id] @@ -190,7 +207,7 @@ def _cache_schema(self, schema, schema_id, subject=None, version=None): self._add_to_cache(self.subject_to_schema_versions, subject, schema, version) - def register(self, subject, avro_schema): + def register(self, subject: str, avro_schema: schema.Schema) -> int: """ POST /subjects/(string: subject)/versions Register a schema with the registry under the given subject @@ -207,7 +224,7 @@ def register(self, subject, avro_schema): """ schemas_to_id = self.subject_to_schema_ids[subject] - schema_id = schemas_to_id.get(avro_schema, None) + schema_id: Optional[int] = schemas_to_id.get(avro_schema, None) if schema_id is not None: return schema_id # send it up @@ -229,12 +246,12 @@ def register(self, subject, avro_schema): raise ClientError("Unable to register schema. Error code:" + str(code) + " message:" + str(result)) # result is a dict - schema_id = result['id'] + schema_id = cast(Dict, result)['id'] # cache it self._cache_schema(avro_schema, schema_id, subject) return schema_id - def check_registration(self, subject, avro_schema): + def check_registration(self, subject: str, avro_schema: schema.Schema) -> int: """ POST /subjects/(string: subject) Check if a schema has already been registered under the specified subject. @@ -267,12 +284,12 @@ def check_registration(self, subject, avro_schema): elif not 200 <= code <= 299: raise ClientError("Unable to check schema registration. Error code:" + str(code)) # result is a dict - schema_id = result['id'] + schema_id = cast(Dict, result)['id'] # cache it self._cache_schema(avro_schema, schema_id, subject) return schema_id - def delete_subject(self, subject): + def delete_subject(self, subject: str) -> int: """ DELETE /subjects/(string: subject) Deletes the specified subject and its associated compatibility level if registered. @@ -287,9 +304,10 @@ def delete_subject(self, subject): result, code = self._send_request(url, method="DELETE") if not (code >= 200 and code <= 299): raise ClientError('Unable to delete subject: {}'.format(result)) + assert isinstance(result, int) return result - def get_by_id(self, schema_id): + def get_by_id(self, schema_id: int) -> Optional[schema.Schema]: """ GET /schemas/ids/{int: id} Retrieve a parsed avro schema by id or None if not found @@ -311,7 +329,9 @@ def get_by_id(self, schema_id): return None else: # need to parse the schema - schema_str = result.get("schema") + assert isinstance(result, Dict) + schema_str = result["schema"] + assert isinstance(schema_str, str) try: result = loads(schema_str) # cache it @@ -321,7 +341,7 @@ def get_by_id(self, schema_id): # bad schema - should not happen raise ClientError("Received bad schema (id %s) from registry: %s" % (schema_id, e)) - def get_latest_schema(self, subject): + def get_latest_schema(self, subject: str) -> Tuple[Optional[int], Optional[schema.Schema], Optional[int]]: """ GET /subjects/(string: subject)/versions/latest @@ -338,7 +358,7 @@ def get_latest_schema(self, subject): """ return self.get_by_version(subject, 'latest') - def get_by_version(self, subject, version): + def get_by_version(self, subject: str, version: object) -> Tuple[Optional[int], Optional[schema.Schema], Optional[int]]: """ GET /subjects/(string: subject)/versions/(versionId: version) @@ -365,8 +385,11 @@ def get_by_version(self, subject, version): return (None, None, None) elif not (code >= 200 and code <= 299): return (None, None, None) + assert isinstance(result, Dict) schema_id = result['id'] + assert isinstance(schema_id, int) version = result['version'] + assert isinstance(version, int) if schema_id in self.id_to_schema: schema = self.id_to_schema[schema_id] else: @@ -379,7 +402,7 @@ def get_by_version(self, subject, version): self._cache_schema(schema, schema_id, subject, version) return (schema_id, schema, version) - def get_version(self, subject, avro_schema): + def get_version(self, subject: str, avro_schema: schema.Schema) -> Optional[int]: """ POST /subjects/(string: subject) @@ -406,12 +429,13 @@ def get_version(self, subject, avro_schema): elif not (code >= 200 and code <= 299): log.error("Unable to get version of a schema:" + str(code)) return None + assert isinstance(result, Dict) schema_id = result['id'] version = result['version'] self._cache_schema(avro_schema, schema_id, subject, version) return version - def test_compatibility(self, subject, avro_schema, version='latest'): + def test_compatibility(self, subject: str, avro_schema: schema.Schema, version: str='latest') -> Optional[bool]: """ POST /compatibility/subjects/(string: subject)/versions/(versionId: version) @@ -435,6 +459,7 @@ def test_compatibility(self, subject, avro_schema, version='latest'): log.error(("Invalid subject or schema:" + str(code))) return False elif code >= 200 and code <= 299: + assert isinstance(result, Dict) return result.get('is_compatible') else: log.error("Unable to check the compatibility: " + str(code)) @@ -443,7 +468,7 @@ def test_compatibility(self, subject, avro_schema, version='latest'): log.error("_send_request() failed: %s", e) return False - def update_compatibility(self, level, subject=None): + def update_compatibility(self, level: str, subject: Optional[str]=None) -> str: """ PUT /config/(string: subject) @@ -461,11 +486,12 @@ def update_compatibility(self, level, subject=None): body = {"compatibility": level} result, code = self._send_request(url, method='PUT', body=body) if code >= 200 and code <= 299: + assert isinstance(result, Dict) return result['compatibility'] else: raise ClientError("Unable to update level: %s. Error code: %d" % (str(level), code)) - def get_compatibility(self, subject=None): + def get_compatibility(self, subject: Optional[str]=None) -> bool: """ GET /config Get the current compatibility level for a subject. Result will be one of: @@ -484,6 +510,7 @@ def get_compatibility(self, subject=None): if not is_successful_request: raise ClientError('Unable to fetch compatibility level. Error code: %d' % code) + assert isinstance(result, Dict) compatibility = result.get('compatibilityLevel', None) if compatibility not in VALID_LEVELS: if compatibility is None: @@ -492,4 +519,5 @@ def get_compatibility(self, subject=None): error_msg_suffix = str(compatibility) raise ClientError('Invalid compatibility level received: %s' % error_msg_suffix) - return compatibility + # Can't be None, as that's not in VALID_LEVELS + return cast(bool, compatibility) diff --git a/src/confluent_kafka/avro/error.py b/src/confluent_kafka/avro/error.py index b879c4b7a..2f8516ad8 100644 --- a/src/confluent_kafka/avro/error.py +++ b/src/confluent_kafka/avro/error.py @@ -16,16 +16,19 @@ # +from typing import Optional + + class ClientError(Exception): """ Error thrown by Schema Registry clients """ - def __init__(self, message, http_code=None): + def __init__(self, message: str, http_code: Optional[int]=None): self.message = message self.http_code = http_code super(ClientError, self).__init__(self.__str__()) - def __repr__(self): + def __repr__(self) -> str: return "ClientError(error={error})".format(error=self.message) - def __str__(self): + def __str__(self) -> str: return self.message diff --git a/src/confluent_kafka/avro/load.py b/src/confluent_kafka/avro/load.py index 9db8660e1..c22b2795b 100644 --- a/src/confluent_kafka/avro/load.py +++ b/src/confluent_kafka/avro/load.py @@ -16,30 +16,12 @@ # +from typing import TYPE_CHECKING from confluent_kafka.avro.error import ClientError - -def loads(schema_str): - """ Parse a schema given a schema string """ - try: - return schema.parse(schema_str) - except SchemaParseException as e: - raise ClientError("Schema parse failed: %s" % (str(e))) - - -def load(fp): - """ Parse a schema from a file path """ - with open(fp) as f: - return loads(f.read()) - - -# avro.schema.RecordSchema and avro.schema.PrimitiveSchema classes are not hashable. Hence defining them explicitly as -# a quick fix -def _hash_func(self): - return hash(str(self)) - - try: + # FIXME: Needs https://github.com/apache/avro/pull/1952 + # pip install git+https://github.com/apache/avro#subdirectory=lang/py from avro import schema try: @@ -47,11 +29,38 @@ def _hash_func(self): from avro.errors import SchemaParseException except ImportError: # avro < 1.11.0 - from avro.schema import SchemaParseException + from avro.schema import SchemaParseException # type:ignore[attr-defined, no-redef] + + # avro.schema.RecordSchema and avro.schema.PrimitiveSchema classes are not hashable. Hence defining them explicitly as + # a quick fix + def _hash_func(self: object) -> int: + return hash(str(self)) + + schema.RecordSchema.__hash__ = _hash_func # type:ignore[assignment] + schema.PrimitiveSchema.__hash__ = _hash_func # type:ignore[assignment] + schema.UnionSchema.__hash__ = _hash_func # type:ignore[assignment] + + def loads(schema_str: str) -> schema.Schema: + """ Parse a schema given a schema string """ + try: + return schema.parse(schema_str) + except SchemaParseException as e: + raise ClientError("Schema parse failed: %s" % (str(e))) + + + def load(fp: str) -> schema.Schema: + """ Parse a schema from a file path """ + with open(fp) as f: + return loads(f.read()) - schema.RecordSchema.__hash__ = _hash_func - schema.PrimitiveSchema.__hash__ = _hash_func - schema.UnionSchema.__hash__ = _hash_func except ImportError: - schema = None + if TYPE_CHECKING: + # Workaround hack so the type checking for Schema objects still works + class Schema: + pass + class TopLevelSchema: + Schema = Schema + schema = TopLevelSchema # type:ignore[assignment] + else: + schema = None diff --git a/src/confluent_kafka/avro/serializer/__init__.py b/src/confluent_kafka/avro/serializer/__init__.py index 845f58a84..65579719a 100644 --- a/src/confluent_kafka/avro/serializer/__init__.py +++ b/src/confluent_kafka/avro/serializer/__init__.py @@ -19,16 +19,16 @@ class SerializerError(Exception): """Generic error from serializer package""" - def __init__(self, message): + def __init__(self, message: str): self.message = message - def __repr__(self): + def __repr__(self) -> str: return '{klass}(error={error})'.format( klass=self.__class__.__name__, error=self.message ) - def __str__(self): + def __str__(self) -> str: return self.message diff --git a/src/confluent_kafka/avro/serializer/message_serializer.py b/src/confluent_kafka/avro/serializer/message_serializer.py index d92763e2c..9af7f2896 100644 --- a/src/confluent_kafka/avro/serializer/message_serializer.py +++ b/src/confluent_kafka/avro/serializer/message_serializer.py @@ -25,11 +25,14 @@ import struct import sys import traceback +from typing import Any, Callable, Dict, Optional, Union import avro import avro.io +from confluent_kafka.avro import schema from confluent_kafka.avro import ClientError +from confluent_kafka.avro.cached_schema_registry_client import CachedSchemaRegistryClient from confluent_kafka.avro.serializer import (SerializerError, KeySerializerError, ValueSerializerError) @@ -53,12 +56,11 @@ class ContextStringIO(io.BytesIO): Wrapper to allow use of StringIO via 'with' constructs. """ - def __enter__(self): + def __enter__(self) -> "ContextStringIO": return self - def __exit__(self, *args): + def __exit__(self, *args: Any) -> None: self.close() - return False class MessageSerializer(object): @@ -70,15 +72,15 @@ class MessageSerializer(object): All decode_* methods expect a buffer received from kafka. """ - def __init__(self, registry_client, reader_key_schema=None, reader_value_schema=None): + def __init__(self, registry_client: CachedSchemaRegistryClient, reader_key_schema: Optional[schema.Schema]=None, reader_value_schema: Optional[schema.Schema]=None): self.registry_client = registry_client - self.id_to_decoder_func = {} - self.id_to_writers = {} + self.id_to_decoder_func: Dict[int, Callable] = {} + self.id_to_writers: Dict[int, Callable] = {} self.reader_key_schema = reader_key_schema self.reader_value_schema = reader_value_schema # Encoder support - def _get_encoder_func(self, writer_schema): + def _get_encoder_func(self, writer_schema: Optional[schema.Schema]) -> Callable: if HAS_FAST: schema = json.loads(str(writer_schema)) parsed_schema = parse_schema(schema) @@ -86,7 +88,7 @@ def _get_encoder_func(self, writer_schema): writer = avro.io.DatumWriter(writer_schema) return lambda record, fp: writer.write(record, avro.io.BinaryEncoder(fp)) - def encode_record_with_schema(self, topic, schema, record, is_key=False): + def encode_record_with_schema(self, topic: str, schema: schema.Schema, record: object, is_key: bool=False) -> bytes: """ Given a parsed avro schema, encode a record for the given topic. The record is expected to be a dictionary. @@ -119,7 +121,7 @@ def encode_record_with_schema(self, topic, schema, record, is_key=False): return self.encode_record_with_schema_id(schema_id, record, is_key=is_key) - def encode_record_with_schema_id(self, schema_id, record, is_key=False): + def encode_record_with_schema_id(self, schema_id: int, record: object, is_key: bool=False) -> bytes: """ Encode a record with a given schema id. The record must be a python dictionary. @@ -156,7 +158,7 @@ def encode_record_with_schema_id(self, schema_id, record, is_key=False): return outf.getvalue() # Decoder support - def _get_decoder_func(self, schema_id, payload, is_key=False): + def _get_decoder_func(self, schema_id: int, payload: ContextStringIO, is_key: bool=False) -> Callable: if schema_id in self.id_to_decoder_func: return self.id_to_decoder_func[schema_id] @@ -206,14 +208,14 @@ def _get_decoder_func(self, schema_id, payload, is_key=False): # def __init__(self, writer_schema=None, reader_schema=None) avro_reader = avro.io.DatumReader(writer_schema_obj, reader_schema_obj) - def decoder(p): + def decoder(p: io.BytesIO) -> object: bin_decoder = avro.io.BinaryDecoder(p) return avro_reader.read(bin_decoder) self.id_to_decoder_func[schema_id] = decoder return self.id_to_decoder_func[schema_id] - def decode_message(self, message, is_key=False): + def decode_message(self, message: Optional[bytes], is_key: bool=False) -> Optional[Dict]: """ Decode a message from kafka that has been encoded for use with the schema registry. diff --git a/src/confluent_kafka/cimpl.pyi b/src/confluent_kafka/cimpl.pyi new file mode 100644 index 000000000..f11abd9c7 --- /dev/null +++ b/src/confluent_kafka/cimpl.pyi @@ -0,0 +1,339 @@ +from concurrent.futures import Future +from typing import Any, Callable, ClassVar, Dict, List, Optional, Tuple + +from typing import overload +ACL_OPERATION_ALL: int +ACL_OPERATION_ALTER: int +ACL_OPERATION_ALTER_CONFIGS: int +ACL_OPERATION_ANY: int +ACL_OPERATION_CLUSTER_ACTION: int +ACL_OPERATION_CREATE: int +ACL_OPERATION_DELETE: int +ACL_OPERATION_DESCRIBE: int +ACL_OPERATION_DESCRIBE_CONFIGS: int +ACL_OPERATION_IDEMPOTENT_WRITE: int +ACL_OPERATION_READ: int +ACL_OPERATION_UNKNOWN: int +ACL_OPERATION_WRITE: int +ACL_PERMISSION_TYPE_ALLOW: int +ACL_PERMISSION_TYPE_ANY: int +ACL_PERMISSION_TYPE_DENY: int +ACL_PERMISSION_TYPE_UNKNOWN: int +CONFIG_SOURCE_DEFAULT_CONFIG: int +CONFIG_SOURCE_DYNAMIC_BROKER_CONFIG: int +CONFIG_SOURCE_DYNAMIC_DEFAULT_BROKER_CONFIG: int +CONFIG_SOURCE_DYNAMIC_TOPIC_CONFIG: int +CONFIG_SOURCE_STATIC_BROKER_CONFIG: int +CONFIG_SOURCE_UNKNOWN_CONFIG: int +OFFSET_BEGINNING: int +OFFSET_END: int +OFFSET_INVALID: int +OFFSET_STORED: int +RESOURCE_ANY: int +RESOURCE_BROKER: int +RESOURCE_GROUP: int +RESOURCE_PATTERN_ANY: int +RESOURCE_PATTERN_LITERAL: int +RESOURCE_PATTERN_MATCH: int +RESOURCE_PATTERN_PREFIXED: int +RESOURCE_PATTERN_UNKNOWN: int +RESOURCE_TOPIC: int +RESOURCE_UNKNOWN: int +TIMESTAMP_CREATE_TIME: int +TIMESTAMP_LOG_APPEND_TIME: int +TIMESTAMP_NOT_AVAILABLE: int + +class TopicPartition: + error: Any + metadata: Any + offset: Any + partition: int + topic: str + def __init__(self, *args: object, **kwargs: object) -> None: ... + def __eq__(self, other: object) -> Any: ... + def __ge__(self, other: object) -> Any: ... + def __gt__(self, other: object) -> Any: ... + def __hash__(self) -> Any: ... + def __le__(self, other: object) -> Any: ... + def __lt__(self, other: object) -> Any: ... + def __ne__(self, other: object) -> Any: ... + +class Consumer: + def __init__(self, *args: object, **kwargs: object) -> None: ... + def assign(self, partitions: object) -> Any: ... + def assignment(self, *args: object, **kwargs: object) -> Any: ... + def close(self, *args: object, **kwargs: object) -> Any: ... + def commit(self, message: Optional[object] = None, offsets: Optional[object] = None, asynchronous: Optional[bool] = None) -> Any: ... + def committed(self, *args: object, **kwargs: object) -> Any: ... + def consume(self, num_messages: int, timeout: float) -> Any: ... + def consumer_group_metadata(self) -> Any: ... + def get_watermark_offsets(self, *args: object, **kwargs: object) -> Any: ... + def incremental_assign(self, partitions: object) -> Any: ... + def incremental_unassign(self, partitions: object) -> Any: ... + def list_topics(self, *args: object, **kwargs: object) -> Any: ... + def memberid(self) -> Any: ... + def offsets_for_times(self, *args: object, **kwargs: object) -> Any: ... + def pause(self, partitions: object) -> Any: ... + def poll(self, timeout: float) -> Optional[Message]: ... + def position(self, partitions: object) -> Any: ... + def resume(self, partitions: object) -> Any: ... + @overload + def seek(self, partition: object) -> Any: ... + @overload + def seek(self) -> Any: ... + def store_offsets(self, *args: object, **kwargs: object) -> Any: ... + def subscribe(self, topic: str, on_assign: Optional[Callable[[Consumer, List[TopicPartition]], None]] = None, on_revoke:Optional[Callable] = None) -> Any: ... + def unassign(self, *args: object, **kwargs: object) -> Any: ... + def unsubscribe(self, *args: object, **kwargs: object) -> Any: ... + +class KafkaError: + BROKER_NOT_AVAILABLE: ClassVar[int] = ... + CLUSTER_AUTHORIZATION_FAILED: ClassVar[int] = ... + CONCURRENT_TRANSACTIONS: ClassVar[int] = ... + COORDINATOR_LOAD_IN_PROGRESS: ClassVar[int] = ... + COORDINATOR_NOT_AVAILABLE: ClassVar[int] = ... + DELEGATION_TOKEN_AUTHORIZATION_FAILED: ClassVar[int] = ... + DELEGATION_TOKEN_AUTH_DISABLED: ClassVar[int] = ... + DELEGATION_TOKEN_EXPIRED: ClassVar[int] = ... + DELEGATION_TOKEN_NOT_FOUND: ClassVar[int] = ... + DELEGATION_TOKEN_OWNER_MISMATCH: ClassVar[int] = ... + DELEGATION_TOKEN_REQUEST_NOT_ALLOWED: ClassVar[int] = ... + DUPLICATE_RESOURCE: ClassVar[int] = ... + DUPLICATE_SEQUENCE_NUMBER: ClassVar[int] = ... + ELECTION_NOT_NEEDED: ClassVar[int] = ... + ELIGIBLE_LEADERS_NOT_AVAILABLE: ClassVar[int] = ... + FEATURE_UPDATE_FAILED: ClassVar[int] = ... + FENCED_INSTANCE_ID: ClassVar[int] = ... + FENCED_LEADER_EPOCH: ClassVar[int] = ... + FETCH_SESSION_ID_NOT_FOUND: ClassVar[int] = ... + GROUP_AUTHORIZATION_FAILED: ClassVar[int] = ... + GROUP_ID_NOT_FOUND: ClassVar[int] = ... + GROUP_MAX_SIZE_REACHED: ClassVar[int] = ... + GROUP_SUBSCRIBED_TO_TOPIC: ClassVar[int] = ... + ILLEGAL_GENERATION: ClassVar[int] = ... + ILLEGAL_SASL_STATE: ClassVar[int] = ... + INCONSISTENT_GROUP_PROTOCOL: ClassVar[int] = ... + INCONSISTENT_VOTER_SET: ClassVar[int] = ... + INVALID_COMMIT_OFFSET_SIZE: ClassVar[int] = ... + INVALID_CONFIG: ClassVar[int] = ... + INVALID_FETCH_SESSION_EPOCH: ClassVar[int] = ... + INVALID_GROUP_ID: ClassVar[int] = ... + INVALID_MSG: ClassVar[int] = ... + INVALID_MSG_SIZE: ClassVar[int] = ... + INVALID_PARTITIONS: ClassVar[int] = ... + INVALID_PRINCIPAL_TYPE: ClassVar[int] = ... + INVALID_PRODUCER_EPOCH: ClassVar[int] = ... + INVALID_PRODUCER_ID_MAPPING: ClassVar[int] = ... + INVALID_RECORD: ClassVar[int] = ... + INVALID_REPLICATION_FACTOR: ClassVar[int] = ... + INVALID_REPLICA_ASSIGNMENT: ClassVar[int] = ... + INVALID_REQUEST: ClassVar[int] = ... + INVALID_REQUIRED_ACKS: ClassVar[int] = ... + INVALID_SESSION_TIMEOUT: ClassVar[int] = ... + INVALID_TIMESTAMP: ClassVar[int] = ... + INVALID_TRANSACTION_TIMEOUT: ClassVar[int] = ... + INVALID_TXN_STATE: ClassVar[int] = ... + INVALID_UPDATE_VERSION: ClassVar[int] = ... + KAFKA_STORAGE_ERROR: ClassVar[int] = ... + LEADER_NOT_AVAILABLE: ClassVar[int] = ... + LISTENER_NOT_FOUND: ClassVar[int] = ... + LOG_DIR_NOT_FOUND: ClassVar[int] = ... + MEMBER_ID_REQUIRED: ClassVar[int] = ... + MSG_SIZE_TOO_LARGE: ClassVar[int] = ... + NETWORK_EXCEPTION: ClassVar[int] = ... + NON_EMPTY_GROUP: ClassVar[int] = ... + NOT_CONTROLLER: ClassVar[int] = ... + NOT_COORDINATOR: ClassVar[int] = ... + NOT_ENOUGH_REPLICAS: ClassVar[int] = ... + NOT_ENOUGH_REPLICAS_AFTER_APPEND: ClassVar[int] = ... + NOT_LEADER_FOR_PARTITION: ClassVar[int] = ... + NO_ERROR: ClassVar[int] = ... + NO_REASSIGNMENT_IN_PROGRESS: ClassVar[int] = ... + OFFSET_METADATA_TOO_LARGE: ClassVar[int] = ... + OFFSET_NOT_AVAILABLE: ClassVar[int] = ... + OFFSET_OUT_OF_RANGE: ClassVar[int] = ... + OPERATION_NOT_ATTEMPTED: ClassVar[int] = ... + OUT_OF_ORDER_SEQUENCE_NUMBER: ClassVar[int] = ... + POLICY_VIOLATION: ClassVar[int] = ... + PREFERRED_LEADER_NOT_AVAILABLE: ClassVar[int] = ... + PRINCIPAL_DESERIALIZATION_FAILURE: ClassVar[int] = ... + PRODUCER_FENCED: ClassVar[int] = ... + REASSIGNMENT_IN_PROGRESS: ClassVar[int] = ... + REBALANCE_IN_PROGRESS: ClassVar[int] = ... + RECORD_LIST_TOO_LARGE: ClassVar[int] = ... + REPLICA_NOT_AVAILABLE: ClassVar[int] = ... + REQUEST_TIMED_OUT: ClassVar[int] = ... + RESOURCE_NOT_FOUND: ClassVar[int] = ... + SASL_AUTHENTICATION_FAILED: ClassVar[int] = ... + SECURITY_DISABLED: ClassVar[int] = ... + STALE_BROKER_EPOCH: ClassVar[int] = ... + STALE_CTRL_EPOCH: ClassVar[int] = ... + THROTTLING_QUOTA_EXCEEDED: ClassVar[int] = ... + TOPIC_ALREADY_EXISTS: ClassVar[int] = ... + TOPIC_AUTHORIZATION_FAILED: ClassVar[int] = ... + TOPIC_DELETION_DISABLED: ClassVar[int] = ... + TOPIC_EXCEPTION: ClassVar[int] = ... + TRANSACTIONAL_ID_AUTHORIZATION_FAILED: ClassVar[int] = ... + TRANSACTION_COORDINATOR_FENCED: ClassVar[int] = ... + UNACCEPTABLE_CREDENTIAL: ClassVar[int] = ... + UNKNOWN: ClassVar[int] = ... + UNKNOWN_LEADER_EPOCH: ClassVar[int] = ... + UNKNOWN_MEMBER_ID: ClassVar[int] = ... + UNKNOWN_PRODUCER_ID: ClassVar[int] = ... + UNKNOWN_TOPIC_OR_PART: ClassVar[int] = ... + UNSTABLE_OFFSET_COMMIT: ClassVar[int] = ... + UNSUPPORTED_COMPRESSION_TYPE: ClassVar[int] = ... + UNSUPPORTED_FOR_MESSAGE_FORMAT: ClassVar[int] = ... + UNSUPPORTED_SASL_MECHANISM: ClassVar[int] = ... + UNSUPPORTED_VERSION: ClassVar[int] = ... + _ALL_BROKERS_DOWN: ClassVar[int] = ... + _APPLICATION: ClassVar[int] = ... + _ASSIGNMENT_LOST: ClassVar[int] = ... + _ASSIGN_PARTITIONS: ClassVar[int] = ... + _AUTHENTICATION: ClassVar[int] = ... + _AUTO_OFFSET_RESET: ClassVar[int] = ... + _BAD_COMPRESSION: ClassVar[int] = ... + _BAD_MSG: ClassVar[int] = ... + _CONFLICT: ClassVar[int] = ... + _CRIT_SYS_RESOURCE: ClassVar[int] = ... + _DESTROY: ClassVar[int] = ... + _EXISTING_SUBSCRIPTION: ClassVar[int] = ... + _FAIL: ClassVar[int] = ... + _FATAL: ClassVar[int] = ... + _FENCED: ClassVar[int] = ... + _FS: ClassVar[int] = ... + _GAPLESS_GUARANTEE: ClassVar[int] = ... + _INCONSISTENT: ClassVar[int] = ... + _INTR: ClassVar[int] = ... + _INVALID_ARG: ClassVar[int] = ... + _INVALID_TYPE: ClassVar[int] = ... + _IN_PROGRESS: ClassVar[int] = ... + _ISR_INSUFF: ClassVar[int] = ... + _KEY_DESERIALIZATION: ClassVar[int] = ... + _KEY_SERIALIZATION: ClassVar[int] = ... + _MAX_POLL_EXCEEDED: ClassVar[int] = ... + _MSG_TIMED_OUT: ClassVar[int] = ... + _NODE_UPDATE: ClassVar[int] = ... + _NOENT: ClassVar[int] = ... + _NOOP: ClassVar[int] = ... + _NOT_CONFIGURED: ClassVar[int] = ... + _NOT_IMPLEMENTED: ClassVar[int] = ... + _NO_OFFSET: ClassVar[int] = ... + _OUTDATED: ClassVar[int] = ... + _PARTIAL: ClassVar[int] = ... + _PARTITION_EOF: ClassVar[int] = ... + _PREV_IN_PROGRESS: ClassVar[int] = ... + _PURGE_INFLIGHT: ClassVar[int] = ... + _PURGE_QUEUE: ClassVar[int] = ... + _QUEUE_FULL: ClassVar[int] = ... + _READ_ONLY: ClassVar[int] = ... + _RESOLVE: ClassVar[int] = ... + _RETRY: ClassVar[int] = ... + _REVOKE_PARTITIONS: ClassVar[int] = ... + _SSL: ClassVar[int] = ... + _STATE: ClassVar[int] = ... + _TIMED_OUT: ClassVar[int] = ... + _TIMED_OUT_QUEUE: ClassVar[int] = ... + _TRANSPORT: ClassVar[int] = ... + _UNDERFLOW: ClassVar[int] = ... + _UNKNOWN_BROKER: ClassVar[int] = ... + _UNKNOWN_GROUP: ClassVar[int] = ... + _UNKNOWN_PARTITION: ClassVar[int] = ... + _UNKNOWN_PROTOCOL: ClassVar[int] = ... + _UNKNOWN_TOPIC: ClassVar[int] = ... + _UNSUPPORTED_FEATURE: ClassVar[int] = ... + _VALUE_DESERIALIZATION: ClassVar[int] = ... + _VALUE_SERIALIZATION: ClassVar[int] = ... + _WAIT_CACHE: ClassVar[int] = ... + _WAIT_COORD: ClassVar[int] = ... + def __init__(self, *args: object, **kwargs: object) -> None: ... + def code(self, *args: object, **kwargs: object) -> Any: ... + def fatal(self, *args: object, **kwargs: object) -> Any: ... + def name(self, *args: object, **kwargs: object) -> Any: ... + def retriable(self, *args: object, **kwargs: object) -> Any: ... + def str(self, *args: object, **kwargs: object) -> Any: ... + def txn_requires_abort(self, *args: object, **kwargs: object) -> Any: ... + def __eq__(self, other: object) -> Any: ... + def __ge__(self, other: object) -> Any: ... + def __gt__(self, other: object) -> Any: ... + def __hash__(self) -> Any: ... + def __le__(self, other: object) -> Any: ... + def __lt__(self, other: object) -> Any: ... + def __ne__(self, other: object) -> Any: ... + +class KafkaException(Exception): ... + +class Message: + def error(self) -> Any: ... + def headers(self, *args: object, **kwargs: object) -> Any: ... + def key(self, *args: object, **kwargs: object) -> Any: ... + def latency(self, *args: object, **kwargs: object) -> Any: ... + def offset(self, *args: object, **kwargs: object) -> Any: ... + def partition(self, *args: object, **kwargs: object) -> Any: ... + def set_headers(self, *args: object, **kwargs: object) -> Any: ... + def set_key(self, *args: object, **kwargs: object) -> Any: ... + def set_value(self, *args: object, **kwargs: object) -> Any: ... + def timestamp(self, *args: object, **kwargs: object) -> Any: ... + def topic(self, *args: object, **kwargs: object) -> Any: ... + def value(self) -> Any: ... + def __len__(self) -> Any: ... + +class NewPartitions: + new_total_count: Any + replica_assignment: Any + topic: Any + def __init__(self, *args: object, **kwargs: object) -> None: ... + def __eq__(self, other: object) -> Any: ... + def __ge__(self, other: object) -> Any: ... + def __gt__(self, other: object) -> Any: ... + def __hash__(self) -> Any: ... + def __le__(self, other: object) -> Any: ... + def __lt__(self, other: object) -> Any: ... + def __ne__(self, other: object) -> Any: ... + +class NewTopic: + config: Any + num_partitions: Any + replica_assignment: Any + replication_factor: Any + topic: str + def __init__(self, *args: object, **kwargs: object) -> None: ... + def __eq__(self, other: object) -> Any: ... + def __ge__(self, other: object) -> Any: ... + def __gt__(self, other: object) -> Any: ... + def __hash__(self) -> Any: ... + def __le__(self, other: object) -> Any: ... + def __lt__(self, other: object) -> Any: ... + def __ne__(self, other: object) -> Any: ... + +class Producer: + def __init__(self, *args: object, **kwargs: object) -> None: ... + def abort_transaction(self, *args: object, **kwargs: object) -> Any: ... + def begin_transaction(self) -> Any: ... + def commit_transaction(self, *args: object, **kwargs: object) -> Any: ... + def flush(self, *args: object, **kwargs: object) -> Any: ... + def init_transactions(self, *args: object, **kwargs: object) -> Any: ... + def list_topics(self, *args: object, **kwargs: object) -> Any: ... + def poll(self, *args: object, **kwargs: object) -> Any: ... + def produce(self, topic: str, **kwargs: object) -> None: ... + def purge(self, *args: object, **kwargs: object) -> Any: ... + def send_offsets_to_transaction(self, *args: object, **kwargs: object) -> Any: ... + def __len__(self) -> Any: ... + +class _AdminClientImpl: + def __init__(self, *args: object, **kwargs: object) -> None: ... + def alter_configs(self, resources: List, f: Future, **kwargs: object) -> Any: ... + def create_acls(self, acls: List, f: Future, **kwargs: object) -> Any: ... + def create_partitions(self, new_partitions: List[NewPartitions], f: Future) -> Any: ... + def create_topics(self, new_topics: List[NewTopic], f: Future) -> Dict[str, Future]: ... + def delete_acls(self, acl_binding_filters: List, f: Future) -> Any: ... + def delete_topics(self, topics: List[str], f: Future, **kwargs: object) -> Any: ... + def describe_acls(self, acl_binding_filter: List, f: Future, **kwargs: object) -> Any: ... + def describe_configs(self, resources: List, f: Future, **kwargs: object) -> Any: ... + def list_groups(self, *args: object, **kwargs: object) -> Any: ... + def list_topics(self, *args: object, **kwargs: object) -> Any: ... + def poll(self) -> Any: ... + def __len__(self) -> Any: ... + +def libversion(*args: object, **kwargs: object) -> Any: ... +def version(*args: object, **kwargs: object) -> Any: ... diff --git a/src/confluent_kafka/deserializing_consumer.py b/src/confluent_kafka/deserializing_consumer.py index 39f80943d..ad5416297 100644 --- a/src/confluent_kafka/deserializing_consumer.py +++ b/src/confluent_kafka/deserializing_consumer.py @@ -16,7 +16,8 @@ # limitations under the License. # -from confluent_kafka.cimpl import Consumer as _ConsumerImpl +from typing import Dict, Optional +from confluent_kafka.cimpl import Consumer as _ConsumerImpl, Message from .error import (ConsumeError, KeyDeserializationError, ValueDeserializationError) @@ -70,14 +71,14 @@ class DeserializingConsumer(_ConsumerImpl): ValueError: if configuration validation fails """ # noqa: E501 - def __init__(self, conf): + def __init__(self, conf: Dict): conf_copy = conf.copy() self._key_deserializer = conf_copy.pop('key.deserializer', None) self._value_deserializer = conf_copy.pop('value.deserializer', None) super(DeserializingConsumer, self).__init__(conf_copy) - def poll(self, timeout=-1): + def poll(self, timeout: float=-1) -> Optional[Message]: """ Consume messages and calls callbacks. @@ -123,7 +124,7 @@ def poll(self, timeout=-1): msg.set_value(value) return msg - def consume(self, num_messages=1, timeout=-1): + def consume(self, num_messages: int =1, timeout: float=-1) -> None: """ :py:func:`Consumer.consume` not implemented, use :py:func:`DeserializingConsumer.poll` instead diff --git a/src/confluent_kafka/error.py b/src/confluent_kafka/error.py index 07c733c23..8aae4de4b 100644 --- a/src/confluent_kafka/error.py +++ b/src/confluent_kafka/error.py @@ -15,7 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from confluent_kafka.cimpl import KafkaException, KafkaError +from typing import Any, Optional, cast +from confluent_kafka.cimpl import KafkaException, KafkaError, Message from confluent_kafka.serialization import SerializationError @@ -32,18 +33,18 @@ class _KafkaClientError(KafkaException): by the broker. """ - def __init__(self, kafka_error, exception=None, kafka_message=None): + def __init__(self, kafka_error: KafkaError, exception: Optional[Exception]=None, kafka_message: Optional[Message]=None): super(_KafkaClientError, self).__init__(kafka_error) self.exception = exception self.kafka_message = kafka_message @property - def code(self): - return self.args[0].code() + def code(self) -> Any: + return cast(KafkaError, self.args[0]).code() @property - def name(self): - return self.args[0].name() + def name(self) -> Any: + return cast(KafkaError, self.args[0]).name() class ConsumeError(_KafkaClientError): @@ -64,7 +65,7 @@ class ConsumeError(_KafkaClientError): """ - def __init__(self, kafka_error, exception=None, kafka_message=None): + def __init__(self, kafka_error: KafkaError, exception: Optional[Exception]=None, kafka_message: Optional[Message]=None): super(ConsumeError, self).__init__(kafka_error, exception, kafka_message) @@ -81,7 +82,7 @@ class KeyDeserializationError(ConsumeError, SerializationError): """ - def __init__(self, exception=None, kafka_message=None): + def __init__(self, exception: Optional[Exception]=None, kafka_message: Optional[Message]=None): super(KeyDeserializationError, self).__init__( KafkaError(KafkaError._KEY_DESERIALIZATION, str(exception)), exception=exception, kafka_message=kafka_message) @@ -100,7 +101,7 @@ class ValueDeserializationError(ConsumeError, SerializationError): """ - def __init__(self, exception=None, kafka_message=None): + def __init__(self, exception: Optional[Exception]=None, kafka_message: Optional[Message]=None): super(ValueDeserializationError, self).__init__( KafkaError(KafkaError._VALUE_DESERIALIZATION, str(exception)), exception=exception, kafka_message=kafka_message) @@ -116,7 +117,7 @@ class ProduceError(_KafkaClientError): exception(Exception, optional): The original exception. """ - def __init__(self, kafka_error, exception=None): + def __init__(self, kafka_error: KafkaError, exception: Optional[Exception]=None): super(ProduceError, self).__init__(kafka_error, exception, None) @@ -128,7 +129,7 @@ class KeySerializationError(ProduceError, SerializationError): exception (Exception): The exception that occurred during serialization. """ - def __init__(self, exception=None): + def __init__(self, exception: Optional[Exception]=None): super(KeySerializationError, self).__init__( KafkaError(KafkaError._KEY_SERIALIZATION, str(exception)), exception=exception) @@ -142,7 +143,7 @@ class ValueSerializationError(ProduceError, SerializationError): exception (Exception): The exception that occurred during serialization. """ - def __init__(self, exception=None): + def __init__(self, exception: Optional[Exception]=None): super(ValueSerializationError, self).__init__( KafkaError(KafkaError._VALUE_SERIALIZATION, str(exception)), exception=exception) diff --git a/src/confluent_kafka/kafkatest/verifiable_client.py b/src/confluent_kafka/kafkatest/verifiable_client.py index 714783e57..989560734 100644 --- a/src/confluent_kafka/kafkatest/verifiable_client.py +++ b/src/confluent_kafka/kafkatest/verifiable_client.py @@ -21,6 +21,7 @@ import socket import sys import time +from typing import Dict class VerifiableClient(object): @@ -28,7 +29,7 @@ class VerifiableClient(object): Generic base class for a kafkatest verifiable client. Implements the common kafkatest protocol and semantics. """ - def __init__(self, conf): + def __init__(self, conf: Dict): """ """ super(VerifiableClient, self).__init__() @@ -38,26 +39,26 @@ def __init__(self, conf): signal.signal(signal.SIGTERM, self.sig_term) self.dbg('Pid is %d' % os.getpid()) - def sig_term(self, sig, frame): + def sig_term(self, sig: int, frame: object) -> None: self.dbg('SIGTERM') self.run = False @staticmethod - def _timestamp(): + def _timestamp() -> str: return time.strftime('%H:%M:%S', time.localtime()) - def dbg(self, s): + def dbg(self, s: str) -> None: """ Debugging printout """ sys.stderr.write('%% %s DEBUG: %s\n' % (self._timestamp(), s)) - def err(self, s, term=False): + def err(self, s: str, term: bool =False) -> None: """ Error printout, if term=True the process will terminate immediately. """ sys.stderr.write('%% %s ERROR: %s\n' % (self._timestamp(), s)) if term: sys.stderr.write('%% FATAL ERROR ^\n') sys.exit(1) - def send(self, d): + def send(self, d: Dict) -> None: """ Send dict as JSON to stdout for consumtion by kafkatest handler """ d['_time'] = str(datetime.datetime.now()) self.dbg('SEND: %s' % json.dumps(d)) @@ -65,9 +66,9 @@ def send(self, d): sys.stdout.flush() @staticmethod - def set_config(conf, args): + def set_config(conf: Dict, args: Dict) -> None: """ Set client config properties using args dict. """ - for n, v in args.iteritems(): + for n, v in args.items(): if v is None: continue @@ -94,9 +95,9 @@ def set_config(conf, args): conf[n] = v @staticmethod - def read_config_file(path): + def read_config_file(path: str) -> Dict[str, str]: """Read (java client) config file and return dict with properties""" - conf = {} + conf: Dict[str, str] = {} with open(path, 'r') as f: for line in f: diff --git a/src/confluent_kafka/kafkatest/verifiable_consumer.py b/src/confluent_kafka/kafkatest/verifiable_consumer.py index 2e3bfbabe..2ac79d219 100755 --- a/src/confluent_kafka/kafkatest/verifiable_consumer.py +++ b/src/confluent_kafka/kafkatest/verifiable_consumer.py @@ -18,8 +18,11 @@ import argparse import os import time +from typing import Any, Dict, List, Optional, cast +from typing_extensions import Literal, TypedDict from confluent_kafka import Consumer, KafkaError, KafkaException -from verifiable_client import VerifiableClient +from confluent_kafka.cimpl import Message, TopicPartition +from .verifiable_client import VerifiableClient class VerifiableConsumer(VerifiableClient): @@ -27,7 +30,7 @@ class VerifiableConsumer(VerifiableClient): confluent-kafka-python backed VerifiableConsumer class for use with Kafka's kafkatests client tests. """ - def __init__(self, conf): + def __init__(self, conf: Dict): """ conf is a config dict passed to confluent_kafka.Consumer() """ @@ -40,16 +43,17 @@ def __init__(self, conf): self.use_auto_commit = False self.use_async_commit = False self.max_msgs = -1 - self.assignment = [] - self.assignment_dict = dict() + self.assignment: List[AssignedPartition] = [] + self.assignment_dict: Dict[str, AssignedPartition] = dict() + self.verbose: bool = False - def find_assignment(self, topic, partition): + def find_assignment(self, topic: str, partition: int) -> Optional["AssignedPartition"]: """ Find and return existing assignment based on topic and partition, or None on miss. """ skey = '%s %d' % (topic, partition) return self.assignment_dict.get(skey) - def send_records_consumed(self, immediate=False): + def send_records_consumed(self, immediate: bool=False) -> None: """ Send records_consumed, every 100 messages, on timeout, or if immediate is set. """ if self.consumed_msgs <= self.consumed_msgs_last_reported + (0 if immediate else 100): @@ -58,7 +62,8 @@ def send_records_consumed(self, immediate=False): if len(self.assignment) == 0: return - d = {'name': 'records_consumed', + SendDict = TypedDict("SendDict", {"name": Literal['records_consumed'], 'count': int, 'partitions': List[Dict] }) + d: SendDict = {'name': 'records_consumed', 'count': self.consumed_msgs - self.consumed_msgs_last_reported, 'partitions': []} @@ -70,16 +75,16 @@ def send_records_consumed(self, immediate=False): d['partitions'].append(a.to_dict()) a.min_offset = -1 - self.send(d) + self.send(cast(Dict, d)) self.consumed_msgs_last_reported = self.consumed_msgs - def send_assignment(self, evtype, partitions): + def send_assignment(self, evtype: str, partitions: List[TopicPartition]) -> None: """ Send assignment update, evtype is either 'assigned' or 'revoked' """ d = {'name': 'partitions_' + evtype, 'partitions': [{'topic': x.topic, 'partition': x.partition} for x in partitions]} self.send(d) - def on_assign(self, consumer, partitions): + def on_assign(self, consumer: Consumer, partitions: List[TopicPartition]) -> None: """ Rebalance on_assign callback """ old_assignment = self.assignment self.assignment = [AssignedPartition(p.topic, p.partition) for p in partitions] @@ -87,12 +92,13 @@ def on_assign(self, consumer, partitions): # minOffset even after a rebalance loop. for a in old_assignment: b = self.find_assignment(a.topic, a.partition) + assert b is not None b.min_offset = a.min_offset self.assignment_dict = {a.skey: a for a in self.assignment} self.send_assignment('assigned', partitions) - def on_revoke(self, consumer, partitions): + def on_revoke(self, consumer: Consumer, partitions: List[TopicPartition]) -> None: """ Rebalance on_revoke callback """ # Send final consumed records prior to rebalancing to make sure # latest consumed is in par with what is going to be committed. @@ -102,7 +108,7 @@ def on_revoke(self, consumer, partitions): self.assignment_dict = dict() self.send_assignment('revoked', partitions) - def on_commit(self, err, partitions): + def on_commit(self, err: Optional[KafkaError], partitions: List[TopicPartition]) -> None: """ Offsets Committed callback """ if err is not None and err.code() == KafkaError._NO_OFFSET: self.dbg('on_commit(): no offsets to commit') @@ -111,7 +117,7 @@ def on_commit(self, err, partitions): # Report consumed messages to make sure consumed position >= committed position self.send_records_consumed(immediate=True) - d = {'name': 'offsets_committed', + d: Dict[str, Any] = {'name': 'offsets_committed', 'offsets': []} if err is not None: @@ -133,7 +139,7 @@ def on_commit(self, err, partitions): self.send(d) - def do_commit(self, immediate=False, asynchronous=None): + def do_commit(self, immediate: bool=False, asynchronous: Optional[bool]=None) -> None: """ Commit every 1000 messages or whenever there is a consume timeout or immediate. """ if (self.use_auto_commit @@ -185,7 +191,7 @@ def do_commit(self, immediate=False, asynchronous=None): self.consumed_msgs_at_last_commit = self.consumed_msgs - def msg_consume(self, msg): + def msg_consume(self, msg: Message) -> None: """ Handle consumed message (or error event) """ if msg.error(): self.err('Consume failed: %s' % msg.error(), term=False) @@ -208,11 +214,12 @@ def msg_consume(self, msg): self.err('Received message on unassigned partition %s [%d] @ %d' % (msg.topic(), msg.partition(), msg.offset()), term=True) - a.consumed_msgs += 1 - if a.min_offset == -1: - a.min_offset = msg.offset() - if a.max_offset < msg.offset(): - a.max_offset = msg.offset() + else: + a.consumed_msgs += 1 + if a.min_offset == -1: + a.min_offset = msg.offset() + if a.max_offset < msg.offset(): + a.max_offset = msg.offset() self.consumed_msgs += 1 @@ -223,7 +230,7 @@ def msg_consume(self, msg): class AssignedPartition(object): """ Local state container for assigned partition. """ - def __init__(self, topic, partition): + def __init__(self, topic: str, partition: int): super(AssignedPartition, self).__init__() self.topic = topic self.partition = partition @@ -232,7 +239,7 @@ def __init__(self, topic, partition): self.min_offset = -1 self.max_offset = 0 - def to_dict(self): + def to_dict(self) -> Dict[str, Any]: """ Return a dict of this partition's state """ return {'topic': self.topic, 'partition': self.partition, 'minOffset': self.min_offset, 'maxOffset': self.max_offset} diff --git a/src/confluent_kafka/kafkatest/verifiable_producer.py b/src/confluent_kafka/kafkatest/verifiable_producer.py index fbf66a7e0..8decf1c71 100755 --- a/src/confluent_kafka/kafkatest/verifiable_producer.py +++ b/src/confluent_kafka/kafkatest/verifiable_producer.py @@ -17,8 +17,10 @@ import argparse import time +from typing import Dict, Optional from confluent_kafka import Producer, KafkaException -from verifiable_client import VerifiableClient +from confluent_kafka import KafkaError, Message +from .verifiable_client import VerifiableClient class VerifiableProducer(VerifiableClient): @@ -26,7 +28,7 @@ class VerifiableProducer(VerifiableClient): confluent-kafka-python backed VerifiableProducer class for use with Kafka's kafkatests client tests. """ - def __init__(self, conf): + def __init__(self, conf: Dict): """ conf is a config dict passed to confluent_kafka.Producer() """ @@ -34,10 +36,11 @@ def __init__(self, conf): self.conf['on_delivery'] = self.dr_cb self.producer = Producer(**self.conf) self.num_acked = 0 - self.num_sent = 0 - self.num_err = 0 + self.num_sent: int = 0 + self.num_err: int = 0 + self.max_msgs: int = 0 - def dr_cb(self, err, msg): + def dr_cb(self, err: KafkaError, msg: Message) -> None: """ Per-message Delivery report callback. Called from poll() """ if err: self.num_err += 1 @@ -95,7 +98,7 @@ def dr_cb(self, err, msg): value_fmt = '%d' repeating_keys = args['repeating_keys'] - key_counter = 0 + key_counter: int = 0 if throughput > 0: delay = 1.0/throughput @@ -111,6 +114,7 @@ def dr_cb(self, err, msg): t_end = time.time() + delay while vp.run: + key: Optional[str] if repeating_keys != 0: key = '%d' % key_counter key_counter = (key_counter + 1) % repeating_keys @@ -118,7 +122,7 @@ def dr_cb(self, err, msg): key = None try: - vp.producer.produce(topic, value=(value_fmt % i), key=key, + vp.producer.produce(topic=topic, value=(value_fmt % i), key=key, timestamp=args.get('create_time', 0)) vp.num_sent += 1 except KafkaException as e: diff --git a/src/confluent_kafka/schema_registry/__init__.py b/src/confluent_kafka/schema_registry/__init__.py index e9a5a17d4..7fa9d0add 100644 --- a/src/confluent_kafka/schema_registry/__init__.py +++ b/src/confluent_kafka/schema_registry/__init__.py @@ -15,6 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from confluent_kafka.serialization import SerializationContext from .schema_registry_client import (RegisteredSchema, Schema, SchemaRegistryClient, @@ -33,7 +34,7 @@ "record_subject_name_strategy"] -def topic_subject_name_strategy(ctx, record_name): +def topic_subject_name_strategy(ctx: SerializationContext, record_name: str) -> str: """ Constructs a subject name in the form of {topic}-key|value. @@ -44,10 +45,10 @@ def topic_subject_name_strategy(ctx, record_name): record_name (str): Record name. """ - return ctx.topic + "-" + ctx.field + return ctx.topic + "-" + str(ctx.field) -def topic_record_subject_name_strategy(ctx, record_name): +def topic_record_subject_name_strategy(ctx: SerializationContext, record_name: str) -> str: """ Constructs a subject name in the form of {topic}-{record_name}. @@ -61,7 +62,7 @@ def topic_record_subject_name_strategy(ctx, record_name): return ctx.topic + "-" + record_name -def record_subject_name_strategy(ctx, record_name): +def record_subject_name_strategy(ctx: SerializationContext, record_name: str) -> str: """ Constructs a subject name in the form of {record_name}. @@ -75,7 +76,7 @@ def record_subject_name_strategy(ctx, record_name): return record_name -def reference_subject_name_strategy(ctx, schema_ref): +def reference_subject_name_strategy(ctx: SerializationContext, schema_ref: SchemaReference) -> str: """ Constructs a subject reference name in the form of {reference name}. diff --git a/src/confluent_kafka/schema_registry/avro.py b/src/confluent_kafka/schema_registry/avro.py index 298373614..41a83b2e4 100644 --- a/src/confluent_kafka/schema_registry/avro.py +++ b/src/confluent_kafka/schema_registry/avro.py @@ -18,15 +18,19 @@ from io import BytesIO from json import loads from struct import pack, unpack +from typing import Any, Callable, Dict, Optional, Set, Tuple, cast +from typing_extensions import Literal from fastavro import (parse_schema, schemaless_reader, schemaless_writer) +from confluent_kafka.schema_registry.schema_registry_client import SchemaRegistryClient + from . import (_MAGIC_BYTE, Schema, topic_subject_name_strategy) -from confluent_kafka.serialization import (Deserializer, +from confluent_kafka.serialization import (Deserializer, SerializationContext, SerializationError, Serializer) @@ -36,15 +40,14 @@ class _ContextStringIO(BytesIO): Wrapper to allow use of StringIO via 'with' constructs. """ - def __enter__(self): + def __enter__(self) -> "_ContextStringIO": return self - def __exit__(self, *args): + def __exit__(self, *args: Any) -> None: self.close() - return False -def _schema_loads(schema_str): +def _schema_loads(schema_str: str) -> Schema: """ Instantiate a Schema instance from a declaration string. @@ -162,11 +165,11 @@ class AvroSerializer(Serializer): 'use.latest.version': False, 'subject.name.strategy': topic_subject_name_strategy} - def __init__(self, schema_registry_client, schema_str, - to_dict=None, conf=None): + def __init__(self, schema_registry_client: SchemaRegistryClient, schema_str: str, + to_dict: Optional[Callable[[object, SerializationContext], Dict]]=None, conf: Optional[Dict]=None): self._registry = schema_registry_client - self._schema_id = None - self._known_subjects = set() + self._schema_id: Optional[int] = None + self._known_subjects: Set[str] = set() if to_dict is not None and not callable(to_dict): raise ValueError("to_dict must be callable with the signature " @@ -222,7 +225,7 @@ def __init__(self, schema_registry_client, schema_str, self._schema_name = schema_name self._parsed_schema = parsed_schema - def __call__(self, obj, ctx): + def __call__(self, obj: Any, ctx: SerializationContext) -> Optional[bytes]: """ Serializes an object to Avro binary format, prepending it with Confluent Schema Registry framing. @@ -245,6 +248,7 @@ def __call__(self, obj, ctx): if obj is None: return None + assert callable(self._subject_name_func) subject = self._subject_name_func(ctx, self._schema_name) if subject not in self._known_subjects: @@ -254,6 +258,7 @@ def __call__(self, obj, ctx): else: # Check to ensure this schema has been registered under subject_name. + assert isinstance(self._normalize_schemas, bool) if self._auto_register: # The schema name will always be the same. We can't however register # a schema without a subject so we set the schema_id here to handle @@ -317,9 +322,9 @@ class AvroDeserializer(Deserializer): __slots__ = ['_reader_schema', '_registry', '_from_dict', '_writer_schemas', '_return_record_name'] - def __init__(self, schema_registry_client, schema_str=None, from_dict=None, return_record_name=False): + def __init__(self, schema_registry_client: SchemaRegistryClient, schema_str: Optional[str]=None, from_dict:Optional[Callable]=None, return_record_name: bool=False): self._registry = schema_registry_client - self._writer_schemas = {} + self._writer_schemas: Dict[int, Schema] = {} self._reader_schema = parse_schema(loads(schema_str)) if schema_str else None @@ -332,7 +337,7 @@ def __init__(self, schema_registry_client, schema_str=None, from_dict=None, retu if not isinstance(self._return_record_name, bool): raise ValueError("return_record_name must be a boolean value") - def __call__(self, data, ctx): + def __call__(self, data: Optional[bytes], ctx: Optional[SerializationContext]=None) -> Any: """ Deserialize Avro binary encoded data with Confluent Schema Registry framing to a dict, or object instance according to from_dict, if specified. @@ -361,9 +366,9 @@ def __call__(self, data, ctx): "Schema Registry serializer".format(len(data))) with _ContextStringIO(data) as payload: - magic, schema_id = unpack('>bI', payload.read(5)) + magic, schema_id = cast(Tuple[bytes, int], unpack('>bI', payload.read(5))) if magic != _MAGIC_BYTE: - raise SerializationError("Unexpected magic byte {}. This message " + raise SerializationError("Unexpected magic byte {!r}. This message " "was not produced with a Confluent " "Schema Registry serializer".format(magic)) diff --git a/src/confluent_kafka/schema_registry/error.py b/src/confluent_kafka/schema_registry/error.py index 77feaeee6..181dd0b9e 100644 --- a/src/confluent_kafka/schema_registry/error.py +++ b/src/confluent_kafka/schema_registry/error.py @@ -41,15 +41,15 @@ class SchemaRegistryError(Exception): """ # noqa: E501 UNKNOWN = -1 - def __init__(self, http_status_code, error_code, error_message): + def __init__(self, http_status_code: int, error_code: int, error_message: str): self.http_status_code = http_status_code self.error_code = error_code self.error_message = error_message - def __repr__(self): + def __repr__(self) -> str: return str(self) - def __str__(self): + def __str__(self) -> str: return "{} (HTTP status code {}, SR code {})".format(self.error_message, self.http_status_code, self.error_code) diff --git a/src/confluent_kafka/schema_registry/json_schema.py b/src/confluent_kafka/schema_registry/json_schema.py index 6fb8706ac..f3c68fce5 100644 --- a/src/confluent_kafka/schema_registry/json_schema.py +++ b/src/confluent_kafka/schema_registry/json_schema.py @@ -19,13 +19,15 @@ import json import struct +from typing import Any, Callable, Dict, Optional, Set, cast from jsonschema import validate, ValidationError from confluent_kafka.schema_registry import (_MAGIC_BYTE, Schema, topic_subject_name_strategy) -from confluent_kafka.serialization import (SerializationError, +from .schema_registry_client import SchemaRegistryClient +from confluent_kafka.serialization import (SerializationContext, SerializationError, Deserializer, Serializer) @@ -35,12 +37,11 @@ class _ContextStringIO(BytesIO): Wrapper to allow use of StringIO via 'with' constructs. """ - def __enter__(self): + def __enter__(self) -> "_ContextStringIO": return self - def __exit__(self, *args): + def __exit__(self, *args: Any) -> None: self.close() - return False class JSONSerializer(Serializer): @@ -141,10 +142,10 @@ class JSONSerializer(Serializer): 'use.latest.version': False, 'subject.name.strategy': topic_subject_name_strategy} - def __init__(self, schema_str, schema_registry_client, to_dict=None, conf=None): + def __init__(self, schema_str: str, schema_registry_client: SchemaRegistryClient, to_dict: Optional[Callable[[object, SerializationContext], Dict]]=None, conf: Optional[Dict]=None): self._registry = schema_registry_client - self._schema_id = None - self._known_subjects = set() + self._schema_id: Optional[int] = None + self._known_subjects: Set[str] = set() if to_dict is not None and not callable(to_dict): raise ValueError("to_dict must be callable with the signature " @@ -170,7 +171,7 @@ def __init__(self, schema_str, schema_registry_client, to_dict=None, conf=None): if self._use_latest_version and self._auto_register: raise ValueError("cannot enable both use.latest.version and auto.register.schemas") - self._subject_name_func = conf_copy.pop('subject.name.strategy') + self._subject_name_func: Callable = cast(Callable, conf_copy.pop('subject.name.strategy')) if not callable(self._subject_name_func): raise ValueError("subject.name.strategy must be callable") @@ -187,7 +188,7 @@ def __init__(self, schema_str, schema_registry_client, to_dict=None, conf=None): self._parsed_schema = schema_dict self._schema = Schema(schema_str, schema_type="JSON") - def __call__(self, obj, ctx): + def __call__(self, obj: Any, ctx: SerializationContext) -> Optional[bytes]: """ Serializes an object to JSON, prepending it with Confluent Schema Registry framing. @@ -218,6 +219,7 @@ def __call__(self, obj, ctx): else: # Check to ensure this schema has been registered under subject_name. + assert isinstance(self._normalize_schemas, bool) if self._auto_register: # The schema name will always be the same. We can't however register # a schema without a subject so we set the schema_id here to handle @@ -266,7 +268,7 @@ class JSONDeserializer(Deserializer): __slots__ = ['_parsed_schema', '_from_dict'] - def __init__(self, schema_str, from_dict=None): + def __init__(self, schema_str: str, from_dict: Optional[Callable]=None): self._parsed_schema = json.loads(schema_str) if from_dict is not None and not callable(from_dict): @@ -275,7 +277,7 @@ def __init__(self, schema_str, from_dict=None): self._from_dict = from_dict - def __call__(self, data, ctx): + def __call__(self, data: Optional[bytes], ctx: Optional[SerializationContext]=None) -> Any: """ Deserialize a JSON encoded record with Confluent Schema Registry framing to a dict, or object instance according to from_dict if from_dict is specified. diff --git a/src/confluent_kafka/schema_registry/protobuf.py b/src/confluent_kafka/schema_registry/protobuf.py index b1de06799..d1ad234e1 100644 --- a/src/confluent_kafka/schema_registry/protobuf.py +++ b/src/confluent_kafka/schema_registry/protobuf.py @@ -16,28 +16,33 @@ # limitations under the License. import io +from subprocess import call import sys import base64 import struct +from typing import Any, Callable, Deque, Dict, List, Optional, Set, cast import warnings from collections import deque +import six -from google.protobuf.message import DecodeError +from google.protobuf.message import DecodeError, Message from google.protobuf.message_factory import MessageFactory +from google.protobuf.descriptor import Descriptor, FileDescriptor +from google.protobuf.reflection import GeneratedProtocolMessageType from . import (_MAGIC_BYTE, reference_subject_name_strategy, topic_subject_name_strategy,) from .schema_registry_client import (Schema, - SchemaReference) -from confluent_kafka.serialization import SerializationError + SchemaReference, SchemaRegistryClient) +from confluent_kafka.serialization import SerializationContext, SerializationError # Convert an int to bytes (inverse of ord()) # Python3.chr() -> Unicode # Python2.chr() -> str(alias for bytes) -if sys.version > '3': - def _bytes(v): +if six.PY3: + def _bytes(v: int) -> bytes: """ Convert int to bytes @@ -46,7 +51,7 @@ def _bytes(v): """ return bytes((v,)) else: - def _bytes(v): + def _bytes(v: int) -> str: """ Convert int to bytes @@ -61,15 +66,14 @@ class _ContextStringIO(io.BytesIO): Wrapper to allow use of StringIO via 'with' constructs. """ - def __enter__(self): + def __enter__(self) -> "_ContextStringIO": return self - def __exit__(self, *args): + def __exit__(self, *args: Any) -> None: self.close() - return False -def _create_index_array(msg_desc): +def _create_index_array(msg_desc: Descriptor) -> List[int]: """ Creates an index array specifying the location of msg_desc in the referenced FileDescriptor. @@ -84,14 +88,14 @@ def _create_index_array(msg_desc): ValueError: If the message descriptor is malformed. """ - msg_idx = deque() + msg_idx: Deque[int] = deque() # Walk the nested MessageDescriptor tree up to the root. current = msg_desc found = False while current.containing_type is not None: previous = current - current = previous.containing_type + current = cast(Descriptor, previous.containing_type) # find child's position for idx, node in enumerate(current.nested_types): if node == previous: @@ -114,7 +118,7 @@ def _create_index_array(msg_desc): return list(msg_idx) -def _schema_to_str(file_descriptor): +def _schema_to_str(file_descriptor: FileDescriptor) -> str: """ Base64 encode a FileDescriptor @@ -245,7 +249,7 @@ class ProtobufSerializer(object): 'use.deprecated.format': False, } - def __init__(self, msg_type, schema_registry_client, conf=None): + def __init__(self, msg_type: GeneratedProtocolMessageType, schema_registry_client: SchemaRegistryClient, conf: Optional[Dict]=None): if conf is None or 'use.deprecated.format' not in conf: raise RuntimeError( @@ -303,17 +307,17 @@ def __init__(self, msg_type, schema_registry_client, conf=None): .format(", ".join(conf_copy.keys()))) self._registry = schema_registry_client - self._schema_id = None - self._known_subjects = set() + self._schema_id: Optional[int] = None + self._known_subjects: Set[str] = set() self._msg_class = msg_type - descriptor = msg_type.DESCRIPTOR + descriptor = msg_type.DESCRIPTOR # type:ignore[attr-defined] self._index_array = _create_index_array(descriptor) self._schema = Schema(_schema_to_str(descriptor.file), schema_type='PROTOBUF') @staticmethod - def _write_varint(buf, val, zigzag=True): + def _write_varint(buf: io.BytesIO, val: int, zigzag: bool=True) -> None: """ Writes val to buf, either using zigzag or uvarint encoding. @@ -332,7 +336,7 @@ def _write_varint(buf, val, zigzag=True): buf.write(_bytes(val)) @staticmethod - def _encode_varints(buf, ints, zigzag=True): + def _encode_varints(buf: io.BytesIO, ints: List[int], zigzag: bool=True) -> None: """ Encodes each int as a uvarint onto buf @@ -353,7 +357,7 @@ def _encode_varints(buf, ints, zigzag=True): for value in ints: ProtobufSerializer._write_varint(buf, value, zigzag=zigzag) - def _resolve_dependencies(self, ctx, file_desc): + def _resolve_dependencies(self, ctx: SerializationContext, file_desc: FileDescriptor) -> List[SchemaReference]: """ Resolves and optionally registers schema references recursively. @@ -363,11 +367,12 @@ def _resolve_dependencies(self, ctx, file_desc): file_desc (FileDescriptor): file descriptor to traverse. """ - schema_refs = [] + schema_refs: List[SchemaReference] = [] for dep in file_desc.dependencies: if self._skip_known_types and dep.name.startswith("google/protobuf/"): continue dep_refs = self._resolve_dependencies(ctx, dep) + assert callable(self._ref_reference_subject_func) subject = self._ref_reference_subject_func(ctx, dep) schema = Schema(_schema_to_str(dep), references=dep_refs, @@ -382,7 +387,7 @@ def _resolve_dependencies(self, ctx, file_desc): reference.version)) return schema_refs - def __call__(self, message, ctx): + def __call__(self, message: Message, ctx: SerializationContext) -> Optional[bytes]: """ Serializes an instance of a class derived from Protobuf Message, and prepends it with Confluent Schema Registry framing. @@ -408,6 +413,7 @@ def __call__(self, message, ctx): raise ValueError("message must be of type {} not {}" .format(self._msg_class, type(message))) + assert callable(self._subject_name_func) subject = self._subject_name_func(ctx, message.DESCRIPTOR.full_name) @@ -420,6 +426,7 @@ def __call__(self, message, ctx): self._schema.references = self._resolve_dependencies( ctx, message.DESCRIPTOR.file) + assert isinstance(self._normalize_schemas, bool) if self._auto_register: self._schema_id = self._registry.register_schema(subject, self._schema, @@ -479,7 +486,7 @@ class ProtobufDeserializer(object): 'use.deprecated.format': False, } - def __init__(self, message_type, conf=None): + def __init__(self, message_type: Any, conf: Optional[Dict]=None): # Require use.deprecated.format to be explicitly configured # during a transitionary period since old/new format are @@ -513,7 +520,7 @@ def __init__(self, message_type, conf=None): self._msg_class = MessageFactory().GetPrototype(descriptor) @staticmethod - def _decode_varint(buf, zigzag=True): + def _decode_varint(buf: io.BytesIO, zigzag: bool=True) -> int: """ Decodes a single varint from a buffer. @@ -548,7 +555,7 @@ def _decode_varint(buf, zigzag=True): raise EOFError("Unexpected EOF while reading index") @staticmethod - def _read_byte(buf): + def _read_byte(buf: io.BytesIO) -> int: """ Read one byte from buf as an int. @@ -565,7 +572,7 @@ def _read_byte(buf): return ord(i) @staticmethod - def _read_index_array(buf, zigzag=True): + def _read_index_array(buf: io.BytesIO, zigzag: bool=True) -> List[int]: """ Read an index array from buf that specifies the message descriptor of interest in the file descriptor. @@ -591,7 +598,7 @@ def _read_index_array(buf, zigzag=True): return msg_index - def __call__(self, data, ctx): + def __call__(self, data: bytes, ctx: Optional[SerializationContext]) -> Any: """ Deserialize a serialized protobuf message with Confluent Schema Registry framing. diff --git a/src/confluent_kafka/schema_registry/schema_registry_client.py b/src/confluent_kafka/schema_registry/schema_registry_client.py index 0a02e650b..bde0af3a8 100644 --- a/src/confluent_kafka/schema_registry/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/schema_registry_client.py @@ -17,34 +17,22 @@ # import json import logging -import urllib +from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast from collections import defaultdict from threading import Lock +from typing_extensions import TypedDict from requests import (Session, utils) from .error import SchemaRegistryError +import six +import six.moves.urllib.parse as urllib +string_type = six.string_types[0] -# TODO: consider adding `six` dependency or employing a compat file -# Python 2.7 is officially EOL so compatibility issue will be come more the norm. -# We need a better way to handle these issues. -# Six is one possibility but the compat file pattern used by requests -# is also quite nice. -# -# six: https://pypi.org/project/six/ -# compat file : https://github.com/psf/requests/blob/master/requests/compat.py -try: - string_type = basestring # noqa - - def _urlencode(value): - return urllib.quote(value, safe='') -except NameError: - string_type = str - - def _urlencode(value): - return urllib.parse.quote(value, safe='') +def _urlencode(value: str) -> str: + return urllib.quote(value, safe='') log = logging.getLogger(__name__) VALID_AUTH_PROVIDERS = ['URL', 'USER_INFO'] @@ -60,7 +48,7 @@ class _RestClient(object): conf (dict): Dictionary containing _RestClient configuration """ - def __init__(self, conf): + def __init__(self, conf: Dict): self.session = Session() # copy dict to avoid mutating the original @@ -105,7 +93,7 @@ def __init__(self, conf): " remove basic.auth.user.info from the" " configuration") - userinfo = tuple(conf_copy.pop('basic.auth.user.info', '').split(':')) + userinfo = cast(Tuple[str, str], tuple(conf_copy.pop('basic.auth.user.info', '').split(':'))) if len(userinfo) != 2: raise ValueError("basic.auth.user.info must be in the form" @@ -118,22 +106,22 @@ def __init__(self, conf): raise ValueError("Unrecognized properties: {}" .format(", ".join(conf_copy.keys()))) - def _close(self): + def _close(self) -> None: self.session.close() - def get(self, url, query=None): + def get(self, url: str, query: Optional[Dict]=None) -> Any: return self.send_request(url, method='GET', query=query) - def post(self, url, body, **kwargs): + def post(self, url: str, body: Any) -> Any: return self.send_request(url, method='POST', body=body) - def delete(self, url): + def delete(self, url: str) -> Any: return self.send_request(url, method='DELETE') - def put(self, url, body=None): + def put(self, url: str, body: Any=None) -> Any: return self.send_request(url, method='PUT', body=body) - def send_request(self, url, method, body=None, query=None): + def send_request(self, url: str, method: str, body: Any=None, query: Optional[Dict]=None) -> Any: """ Sends HTTP request to the SchemaRegistry. @@ -148,7 +136,7 @@ def send_request(self, url, method, body=None, query=None): method (str): HTTP method - body (str): Request content + body (object): Request content query (dict): Query params to attach to the URL @@ -191,13 +179,13 @@ class _SchemaCache(object): known subject membership. """ - def __init__(self): + def __init__(self) -> None: self.lock = Lock() - self.schema_id_index = {} - self.schema_index = {} - self.subject_schemas = defaultdict(set) + self.schema_id_index: Dict[int, Schema] = {} + self.schema_index: Dict[Schema, int] = {} + self.subject_schemas: Dict[str, Set[Schema]] = defaultdict(set) - def set(self, schema_id, schema, subject_name=None): + def set(self, schema_id: int, schema: "Schema", subject_name: Optional[str]=None) -> None: """ Add a Schema identified by schema_id to the cache. @@ -218,7 +206,7 @@ def set(self, schema_id, schema, subject_name=None): if subject_name is not None: self.subject_schemas[subject_name].add(schema) - def get_schema(self, schema_id): + def get_schema(self, schema_id: int) -> Optional["Schema"]: """ Get the schema instance associated with schema_id from the cache. @@ -231,7 +219,7 @@ def get_schema(self, schema_id): return self.schema_id_index.get(schema_id, None) - def get_schema_id_by_subject(self, subject, schema): + def get_schema_id_by_subject(self, subject: str, schema: "Schema") -> Optional[int]: """ Get the schema_id associated with this schema registered under subject. @@ -248,6 +236,8 @@ def get_schema_id_by_subject(self, subject, schema): if schema in self.subject_schemas[subject]: return self.schema_index.get(schema, None) + return None + class SchemaRegistryClient(object): """ @@ -289,18 +279,18 @@ class SchemaRegistryClient(object): `Confluent Schema Registry documentation `_ """ # noqa: E501 - def __init__(self, conf): + def __init__(self, conf: Dict): self._rest_client = _RestClient(conf) self._cache = _SchemaCache() - def __enter__(self): + def __enter__(self) -> "SchemaRegistryClient": return self - def __exit__(self, *args): + def __exit__(self, *args: Any) -> None: if self._rest_client is not None: self._rest_client._close() - def register_schema(self, subject_name, schema, normalize_schemas=False): + def register_schema(self, subject_name: str, schema: "Schema", normalize_schemas: bool=False) -> int: """ Registers a schema under ``subject_name``. @@ -324,7 +314,7 @@ def register_schema(self, subject_name, schema, normalize_schemas=False): if schema_id is not None: return schema_id - request = {'schema': schema.schema_str} + request: Dict[str, Any] = {'schema': schema.schema_str} # CP 5.5 adds new fields (for JSON and Protobuf). if len(schema.references) > 0 or schema.schema_type != 'AVRO': @@ -338,12 +328,12 @@ def register_schema(self, subject_name, schema, normalize_schemas=False): 'subjects/{}/versions?normalize={}'.format(_urlencode(subject_name), normalize_schemas), body=request) - schema_id = response['id'] + schema_id = cast(int, response['id']) self._cache.set(schema_id, schema, subject_name) return schema_id - def get_schema(self, schema_id): + def get_schema(self, schema_id: int) -> "Schema": """ Fetches the schema associated with ``schema_id`` from the Schema Registry. The result is cached so subsequent attempts will not @@ -381,7 +371,7 @@ def get_schema(self, schema_id): return schema - def lookup_schema(self, subject_name, schema, normalize_schemas=False): + def lookup_schema(self, subject_name: str, schema: "Schema", normalize_schemas: bool=False) -> "RegisteredSchema": """ Returns ``schema`` registration information for ``subject``. @@ -400,7 +390,7 @@ def lookup_schema(self, subject_name, schema, normalize_schemas=False): `POST Subject API Reference `_ """ # noqa: E501 - request = {'schema': schema.schema_str} + request: Dict[str, Any] = {'schema': schema.schema_str} # CP 5.5 adds new fields (for JSON and Protobuf). if len(schema.references) > 0 or schema.schema_type != 'AVRO': @@ -423,7 +413,7 @@ def lookup_schema(self, subject_name, schema, normalize_schemas=False): subject=response['subject'], version=response['version']) - def get_subjects(self): + def get_subjects(self) -> List[str]: """ List all subjects registered with the Schema Registry @@ -439,7 +429,7 @@ def get_subjects(self): return self._rest_client.get('subjects') - def delete_subject(self, subject_name, permanent=False): + def delete_subject(self, subject_name: str, permanent: bool=False) -> List[int]: """ Deletes the specified subject and its associated compatibility level if registered. It is recommended to use this API only when a topic needs @@ -468,7 +458,7 @@ def delete_subject(self, subject_name, permanent=False): return list - def get_latest_version(self, subject_name): + def get_latest_version(self, subject_name: str) -> "RegisteredSchema": """ Retrieves latest registered version for subject @@ -497,7 +487,7 @@ def get_latest_version(self, subject_name): subject=response['subject'], version=response['version']) - def get_version(self, subject_name, version): + def get_version(self, subject_name: str, version: int) -> "RegisteredSchema": """ Retrieves a specific schema registered under ``subject_name``. @@ -528,7 +518,7 @@ def get_version(self, subject_name, version): subject=response['subject'], version=response['version']) - def get_versions(self, subject_name): + def get_versions(self, subject_name: str) -> List[int]: """ Get a list of all versions registered with this subject. @@ -547,7 +537,7 @@ def get_versions(self, subject_name): return self._rest_client.get('subjects/{}/versions'.format(_urlencode(subject_name))) - def delete_version(self, subject_name, version): + def delete_version(self, subject_name: str, version: int) -> int: """ Deletes a specific version registered to ``subject_name``. @@ -571,7 +561,7 @@ def delete_version(self, subject_name, version): version)) return response - def set_compatibility(self, subject_name=None, level=None): + def set_compatibility(self, subject_name: Optional[str]=None, level: Optional[str]=None) -> str: """ Update global or subject level compatibility level. @@ -603,7 +593,7 @@ def set_compatibility(self, subject_name=None, level=None): .format(_urlencode(subject_name)), body={'compatibility': level.upper()}) - def get_compatibility(self, subject_name=None): + def get_compatibility(self, subject_name: Optional[str]=None) -> str: """ Get the current compatibility level. @@ -629,7 +619,7 @@ def get_compatibility(self, subject_name=None): result = self._rest_client.get(url) return result['compatibilityLevel'] - def test_compatibility(self, subject_name, schema, version="latest"): + def test_compatibility(self, subject_name: str, schema: "Schema", version: Union[int, str]="latest") -> bool: """Test the compatibility of a candidate schema for a given subject and version Args: @@ -649,7 +639,9 @@ def test_compatibility(self, subject_name, schema, version="latest"): `POST Test Compatibility API Reference `_ """ # noqa: E501 - request = {"schema": schema.schema_str} + Reference = TypedDict("Reference", {'name': str, 'subject': str, 'version': int}) + Request = TypedDict('Request', {'schema': str, 'schemaType': str, 'references': List[Reference]}, total=False) + request: Request = {"schema": schema.schema_str} if schema.schema_type != "AVRO": request['schemaType'] = schema.schema_type @@ -680,7 +672,7 @@ class Schema(object): __slots__ = ['schema_str', 'references', 'schema_type', '_hash'] - def __init__(self, schema_str, schema_type, references=[]): + def __init__(self, schema_str: str, schema_type: str, references: List["SchemaReference"] =[]): super(Schema, self).__init__() self.schema_str = schema_str @@ -688,11 +680,12 @@ def __init__(self, schema_str, schema_type, references=[]): self.references = references self._hash = hash(schema_str) - def __eq__(self, other): + def __eq__(self, other: object) -> bool: + assert isinstance(other, Schema) return all([self.schema_str == other.schema_str, self.schema_type == other.schema_type]) - def __hash__(self): + def __hash__(self) -> int: return self._hash @@ -713,7 +706,7 @@ class RegisteredSchema(object): version (int): Version of this subject this schema is registered to """ - def __init__(self, schema_id, schema, subject, version): + def __init__(self, schema_id: int, schema: Schema, subject: str, version: int): self.schema_id = schema_id self.schema = schema self.subject = subject @@ -736,7 +729,7 @@ class SchemaReference(object): version (int): This Schema's version """ - def __init__(self, name, subject, version): + def __init__(self, name: str, subject: str, version: int): super(SchemaReference, self).__init__() self.name = name self.subject = subject diff --git a/src/confluent_kafka/serialization/__init__.py b/src/confluent_kafka/serialization/__init__.py index fd08f47ed..f2c0fd88a 100644 --- a/src/confluent_kafka/serialization/__init__.py +++ b/src/confluent_kafka/serialization/__init__.py @@ -16,6 +16,7 @@ # limitations under the License. # import struct as _struct +from typing import Any, List, Optional, Tuple from confluent_kafka.error import KafkaException __all__ = ['Deserializer', @@ -60,7 +61,7 @@ class SerializationContext(object): headers (list): List of message header tuples. Defaults to None. """ - def __init__(self, topic, field, headers=None): + def __init__(self, topic: str, field: str, headers: Optional[List[Tuple[str, str]]]=None): self.topic = topic self.field = field self.headers = headers @@ -107,9 +108,9 @@ class Serializer(object): - unicode(encoding) """ - __slots__ = [] + __slots__: List[str] = [] - def __call__(self, obj, ctx=None): + def __call__(self, obj: Any, ctx: SerializationContext) -> Optional[bytes]: """ Converts obj to bytes. @@ -164,9 +165,9 @@ class Deserializer(object): - unicode(encoding) """ - __slots__ = [] + __slots__: List[str] = [] - def __call__(self, value, ctx=None): + def __call__(self, value: bytes, ctx: Optional[SerializationContext]=None) -> Any: """ Convert bytes to object @@ -194,7 +195,7 @@ class DoubleSerializer(Serializer): `DoubleSerializer Javadoc `_ """ # noqa: E501 - def __call__(self, obj, ctx=None): + def __call__(self, obj: Any, ctx: Optional[SerializationContext]=None) -> Any: """ Args: obj (object): object to be serialized @@ -229,7 +230,7 @@ class DoubleDeserializer(Deserializer): `DoubleDeserializer Javadoc `_ """ # noqa: E501 - def __call__(self, value, ctx=None): + def __call__(self, value: bytes, ctx: Optional[SerializationContext]=None) -> Optional[float]: """ Deserializes float from IEEE 764 binary64 bytes. @@ -263,7 +264,7 @@ class IntegerSerializer(Serializer): `IntegerSerializer Javadoc `_ """ # noqa: E501 - def __call__(self, obj, ctx=None): + def __call__(self, obj: Any, ctx: Optional[SerializationContext]=None) -> Optional[bytes]: """ Serializes int as int32 bytes. @@ -300,7 +301,7 @@ class IntegerDeserializer(Deserializer): `IntegerDeserializer Javadoc `_ """ # noqa: E501 - def __call__(self, value, ctx=None): + def __call__(self, value: bytes, ctx: Optional[SerializationContext]=None) -> Optional[int]: """ Deserializes int from int32 bytes. @@ -342,10 +343,10 @@ class StringSerializer(Serializer): `StringSerializer Javadoc `_ """ # noqa: E501 - def __init__(self, codec='utf_8'): + def __init__(self, codec: str='utf_8'): self.codec = codec - def __call__(self, obj, ctx=None): + def __call__(self, obj: Any, ctx: Optional[SerializationContext]=None) -> Optional[bytes]: """ Serializes a str(py2:unicode) to bytes. @@ -388,10 +389,10 @@ class StringDeserializer(Deserializer): `StringDeserializer Javadoc `_ """ # noqa: E501 - def __init__(self, codec='utf_8'): + def __init__(self, codec: str='utf_8'): self.codec = codec - def __call__(self, value, ctx=None): + def __call__(self, value: bytes, ctx: Optional[SerializationContext]=None) -> Optional[str]: """ Serializes unicode to bytes per the configured codec. Defaults to ``utf_8``. diff --git a/src/confluent_kafka/serializing_producer.py b/src/confluent_kafka/serializing_producer.py index 3b3ff82b0..a34da0cc5 100644 --- a/src/confluent_kafka/serializing_producer.py +++ b/src/confluent_kafka/serializing_producer.py @@ -16,7 +16,9 @@ # limitations under the License. # +from typing import Callable, Dict, List, Optional, Tuple from confluent_kafka.cimpl import Producer as _ProducerImpl + from .serialization import (MessageField, SerializationContext) from .error import (KeySerializationError, @@ -66,7 +68,7 @@ class SerializingProducer(_ProducerImpl): conf (producer): SerializingProducer configuration. """ # noqa E501 - def __init__(self, conf): + def __init__(self, conf: Dict): conf_copy = conf.copy() self._key_serializer = conf_copy.pop('key.serializer', None) @@ -74,8 +76,8 @@ def __init__(self, conf): super(SerializingProducer, self).__init__(conf_copy) - def produce(self, topic, key=None, value=None, partition=-1, - on_delivery=None, timestamp=0, headers=None): + def produce(self, topic: str, key: Optional[object]=None, value: Optional[object]=None, partition: int=-1, # type:ignore[override] + on_delivery: Optional[Callable]=None, timestamp: float=0, headers: Optional[List[Tuple[str, str]]]=None) -> None: # type:ignore[override] """ Produce a message. @@ -139,7 +141,7 @@ def produce(self, topic, key=None, value=None, partition=-1, except Exception as se: raise ValueSerializationError(se) - super(SerializingProducer, self).produce(topic, value, key, + super(SerializingProducer, self).produce(topic, value=value, key=key, headers=headers, partition=partition, timestamp=timestamp, diff --git a/tests/avro/mock_registry.py b/tests/avro/mock_registry.py index b7e0ab9a2..dd1e87fc8 100644 --- a/tests/avro/mock_registry.py +++ b/tests/avro/mock_registry.py @@ -51,7 +51,7 @@ def log_message(self, format, *args): class MockServer(HTTPSERVER.HTTPServer, object): - def __init__(self, *args, **kwargs): + def __init__(self, *args: object, **kwargs: object): super(MockServer, self).__init__(*args, **kwargs) self.counts = {} self.registry = MockSchemaRegistryClient() From ffc7f1d306868a01631e529976dfecbc3878a655 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 5 Jan 2023 22:35:59 +0000 Subject: [PATCH 2/8] Add mypy check to Travis --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a2829f57d..e1317c9d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,8 +65,10 @@ install: script: - flake8 # Build package - - pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf] + - pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf] - ldd staging/libs/* || otool -L staging/libs/* || true + # Run type checks + - mypy src/confluent_kafka # Run tests - if [[ $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs python -m pytest --timeout 600 --ignore=tmp-build || travis_terminate 1; fi # Build docs From bac9502e8636e93eb660701bb1e7aca186819314 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 5 Jan 2023 22:37:04 +0000 Subject: [PATCH 3/8] Add py.typed --- setup.py | 1 + src/confluent_kafka/py.typed | 0 2 files changed, 1 insertion(+) create mode 100644 src/confluent_kafka/py.typed diff --git a/setup.py b/setup.py index 0219ad06d..e0b55bf1c 100755 --- a/setup.py +++ b/setup.py @@ -88,6 +88,7 @@ def get_install_requirements(path): author_email='support@confluent.io', url='https://github.com/confluentinc/confluent-kafka-python', ext_modules=[module], + package_data={"confluent_kafka": ["py.typed"]}, packages=find_packages('src'), package_dir={'': 'src'}, data_files=[('', [os.path.join(work_dir, 'LICENSE.txt')])], diff --git a/src/confluent_kafka/py.typed b/src/confluent_kafka/py.typed new file mode 100644 index 000000000..e69de29bb From be54965368baec0357b9a58ba8edf5f1d3bae4bc Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 5 Jan 2023 23:10:53 +0000 Subject: [PATCH 4/8] Add mypy to test requirements --- tests/requirements.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/requirements.txt b/tests/requirements.txt index 120daf4ac..49e42cd23 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -8,3 +8,10 @@ fastavro avro>=1.11.1,<2 jsonschema protobuf + +# Cap the version to avoid issues with newer editions. Should be periodically updated! +mypy<=0.991 +types-protobuf +types-jsonschema +types-requests +types-six \ No newline at end of file From 59f4069b0c2aaac38d8151b484f30c5318d2a915 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Fri, 6 Jan 2023 14:18:49 +0000 Subject: [PATCH 5/8] Rename config to avoid issues with pyproject.toml changing pip install methods --- pyproject.toml => mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename pyproject.toml => mypy.ini (91%) diff --git a/pyproject.toml b/mypy.ini similarity index 91% rename from pyproject.toml rename to mypy.ini index 130522ef0..94b52b05f 100644 --- a/pyproject.toml +++ b/mypy.ini @@ -1,4 +1,4 @@ -[tool.mypy] +[mypy] show_error_codes=true disallow_untyped_defs=true disallow_untyped_calls=true From c9044eb0338990005a5cde56772a9e13991147fb Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Sun, 16 Jul 2023 18:31:34 +0100 Subject: [PATCH 6/8] Upgrade Avro requirement to 1.11.2 for py.typed --- setup.py | 2 +- src/confluent_kafka/avro/requirements.txt | 2 +- tests/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e0b55bf1c..b39cbe1a9 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ AVRO_REQUIRES = ['fastavro>=0.23.0,<1.0;python_version<"3.0"', 'fastavro>=1.0;python_version>"3.0"', - 'avro>=1.11.1,<2', + 'avro>=1.11.2,<2', ] + SCHEMA_REGISTRY_REQUIRES JSON_REQUIRES = ['pyrsistent==0.16.1;python_version<"3.0"', diff --git a/src/confluent_kafka/avro/requirements.txt b/src/confluent_kafka/avro/requirements.txt index e34a65dd8..3380a200c 100644 --- a/src/confluent_kafka/avro/requirements.txt +++ b/src/confluent_kafka/avro/requirements.txt @@ -1,3 +1,3 @@ fastavro>=0.23.0 requests -avro>=1.11.1,<2 +avro>=1.11.2,<2 diff --git a/tests/requirements.txt b/tests/requirements.txt index 49e42cd23..af97ea976 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,7 +5,7 @@ pytest-timeout requests-mock trivup>=0.8.3 fastavro -avro>=1.11.1,<2 +avro>=1.11.2,<2 jsonschema protobuf From 05ba9879e27e44c2301254f30007f86d24aafb86 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Sun, 16 Jul 2023 21:21:14 +0100 Subject: [PATCH 7/8] Merge master --- .clang-format | 136 + .formatignore | 14 + .semaphore/project.yml | 6 + .semaphore/project_public.yml | 20 + .semaphore/semaphore.yml | 182 +- .travis.yml | 94 - CHANGELOG.md | 91 +- Makefile | 16 + docs/conf.py | 108 +- docs/index.rst | 75 +- examples/adminapi.py | 288 +- examples/docker/Dockerfile.alpine | 2 +- examples/fips/.gitignore | 4 + examples/fips/README.md | 90 + examples/fips/docker/README.md | 5 + examples/fips/docker/docker-compose.yml | 51 + examples/fips/docker/secrets/broker_jaas.conf | 17 + .../docker/secrets/generate_certificates.sh | 33 + .../fips/docker/secrets/zookeeper_jaas.conf | 5 + examples/fips/fips_consumer.py | 84 + examples/fips/fips_producer.py | 83 + setup.py | 2 +- sonar-project.properties | 7 + src/confluent_kafka/__init__.py | 4 +- src/confluent_kafka/_model/__init__.py | 91 + src/confluent_kafka/_util/__init__.py | 16 + src/confluent_kafka/_util/conversion_util.py | 38 + src/confluent_kafka/_util/validation_util.py | 56 + src/confluent_kafka/admin/__init__.py | 691 ++++- src/confluent_kafka/admin/_acl.py | 47 +- src/confluent_kafka/admin/_config.py | 49 +- src/confluent_kafka/admin/_group.py | 128 + src/confluent_kafka/admin/_metadata.py | 179 ++ src/confluent_kafka/admin/_scram.py | 116 + src/confluent_kafka/avro/__init__.py | 4 +- .../avro/cached_schema_registry_client.py | 36 + src/confluent_kafka/schema_registry/avro.py | 60 +- .../schema_registry/json_schema.py | 79 +- .../schema_registry/schema_registry_client.py | 38 +- src/confluent_kafka/serialization/__init__.py | 2 +- src/confluent_kafka/src/Admin.c | 2563 +++++++++++++++-- src/confluent_kafka/src/AdminTypes.c | 50 +- src/confluent_kafka/src/Consumer.c | 62 +- src/confluent_kafka/src/Metadata.c | 8 + src/confluent_kafka/src/Producer.c | 24 +- src/confluent_kafka/src/confluent_kafka.c | 204 +- src/confluent_kafka/src/confluent_kafka.h | 23 +- tests/avro/test_cached_client.py | 2 +- tests/docker/.env.sh | 2 + tests/docker/bin/certify.sh | 1 + tests/docker/docker-compose.yaml | 4 +- .../admin/test_basic_operations.py | 177 +- .../admin/test_incremental_alter_configs.py | 128 + tests/integration/integration_test.py | 24 +- .../schema_registry/data/customer.json | 22 + .../schema_registry/data/order.json | 24 + .../schema_registry/data/order_details.json | 22 + .../schema_registry/test_avro_serializers.py | 163 +- .../schema_registry/test_json_serializers.py | 253 ++ .../schema_registry/test_proto_serializers.py | 2 +- tests/integration/testconf.json | 15 +- tests/schema_registry/test_avro_serializer.py | 22 +- tests/schema_registry/test_json.py | 51 + tests/soak/README.md | 20 +- tests/soak/build.sh | 43 + tests/soak/ccloud.config.example | 14 + tests/soak/run.sh | 31 + tests/soak/soakclient.py | 3 +- tests/test_Admin.py | 544 +++- tests/test_Producer.py | 10 + tests/test_misc.py | 18 + tools/RELEASE.md | 33 +- tools/bootstrap-librdkafka.sh | 2 +- tools/build-manylinux.sh | 46 +- tools/mingw-w64/msys2-dependencies.sh | 19 + tools/mingw-w64/semaphore_commands.sh | 11 + tools/mingw-w64/setup-msys2.ps1 | 31 + tools/smoketest.sh | 3 + tools/source-package-verification.sh | 26 + tools/style-format.sh | 132 + tools/wheels/build-wheels.bat | 10 +- tools/wheels/build-wheels.sh | 10 +- tools/wheels/install-librdkafka.sh | 6 +- ...l-macos-python-required-by-cibuildwheel.py | 72 + tools/windows-install-librdkafka.bat | 1 - 85 files changed, 7146 insertions(+), 832 deletions(-) create mode 100644 .clang-format create mode 100644 .formatignore create mode 100644 .semaphore/project_public.yml delete mode 100644 .travis.yml create mode 100644 examples/fips/.gitignore create mode 100644 examples/fips/README.md create mode 100644 examples/fips/docker/README.md create mode 100644 examples/fips/docker/docker-compose.yml create mode 100644 examples/fips/docker/secrets/broker_jaas.conf create mode 100644 examples/fips/docker/secrets/generate_certificates.sh create mode 100644 examples/fips/docker/secrets/zookeeper_jaas.conf create mode 100755 examples/fips/fips_consumer.py create mode 100755 examples/fips/fips_producer.py create mode 100644 sonar-project.properties create mode 100644 src/confluent_kafka/_model/__init__.py create mode 100644 src/confluent_kafka/_util/__init__.py create mode 100644 src/confluent_kafka/_util/conversion_util.py create mode 100644 src/confluent_kafka/_util/validation_util.py create mode 100644 src/confluent_kafka/admin/_group.py create mode 100644 src/confluent_kafka/admin/_metadata.py create mode 100644 src/confluent_kafka/admin/_scram.py create mode 100644 tests/integration/admin/test_incremental_alter_configs.py create mode 100644 tests/integration/schema_registry/data/customer.json create mode 100644 tests/integration/schema_registry/data/order.json create mode 100644 tests/integration/schema_registry/data/order_details.json create mode 100644 tests/schema_registry/test_json.py create mode 100755 tests/soak/build.sh create mode 100644 tests/soak/ccloud.config.example create mode 100755 tests/soak/run.sh create mode 100644 tools/mingw-w64/msys2-dependencies.sh create mode 100644 tools/mingw-w64/semaphore_commands.sh create mode 100644 tools/mingw-w64/setup-msys2.ps1 create mode 100644 tools/source-package-verification.sh create mode 100755 tools/style-format.sh create mode 100644 tools/wheels/install-macos-python-required-by-cibuildwheel.py diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000..17ba2603d --- /dev/null +++ b/.clang-format @@ -0,0 +1,136 @@ +--- +Language: Cpp +AccessModifierOffset: -2 +AlignAfterOpenBracket: Align +AlignConsecutiveMacros: true +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: true +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: MultiLine +BinPackArguments: true +BinPackParameters: false +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: AfterColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + - Regex: '.*' + Priority: 1 + SortPriority: 0 +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentCaseLabels: false +IndentGotoLabels: true +IndentPPDirectives: None +IndentWidth: 8 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 3 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Right +ReflowComments: true +SortIncludes: false +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: true +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: false +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +Standard: Latest +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 +UseCRLF: false +UseTab: Never +... + diff --git a/.formatignore b/.formatignore new file mode 100644 index 000000000..0e733ee7f --- /dev/null +++ b/.formatignore @@ -0,0 +1,14 @@ +# Files to not check/fix coding style for. +# These files are imported from other sources and we want to maintain +# them in the original form to make future updates easier. +docs/conf.py +examples/protobuf/user_pb2.py +tests/integration/schema_registry/data/proto/DependencyTestProto_pb2.py +tests/integration/schema_registry/data/proto/NestedTestProto_pb2.py +tests/integration/schema_registry/data/proto/PublicTestProto_pb2.py +tests/integration/schema_registry/data/proto/SInt32Value_pb2.py +tests/integration/schema_registry/data/proto/SInt64Value_pb2.py +tests/integration/schema_registry/data/proto/TestProto_pb2.py +tests/integration/schema_registry/data/proto/common_proto_pb2.py +tests/integration/schema_registry/data/proto/exampleProtoCriteo_pb2.py +tests/integration/schema_registry/data/proto/metadata_proto_pb2.py diff --git a/.semaphore/project.yml b/.semaphore/project.yml index 9922ecb67..0c1c2f3f9 100644 --- a/.semaphore/project.yml +++ b/.semaphore/project.yml @@ -1,3 +1,8 @@ +# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common +# template and configurations in service.yml. +# Modifications in this file will be overwritten by generated content in the nightly run. +# For more information, please refer to the page: +# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI apiVersion: v1alpha kind: Project metadata: @@ -22,6 +27,7 @@ spec: - master - main - /^v\d+\.\d+\.x$/ + - /^gh-readonly-queue.*/ custom_permissions: true debug_permissions: - empty diff --git a/.semaphore/project_public.yml b/.semaphore/project_public.yml new file mode 100644 index 000000000..4ad19dda6 --- /dev/null +++ b/.semaphore/project_public.yml @@ -0,0 +1,20 @@ +# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common +# template and configurations in service.yml. +# Modifications in this file will be overwritten by generated content in the nightly run. +# For more information, please refer to the page: +# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI +apiVersion: v1alpha +kind: Project +metadata: + name: confluent-kafka-python + description: "" +spec: + visibility: private + repository: + url: git@github.com:confluentinc/confluent-kafka-python.git + pipeline_file: .semaphore/semaphore.yml + integration_type: github_app + status: + pipeline_files: + - path: .semaphore/semaphore.yml + level: pipeline diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 3a715f7e2..65f786d39 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -6,13 +6,11 @@ agent: global_job_config: env_vars: - name: LIBRDKAFKA_VERSION - value: v1.9.2 + value: v2.2.0 prologue: commands: - - export HOME=$WORKSPACE - - mkdir $WORKSPACE/confluent-kafka-python - - cd $WORKSPACE/confluent-kafka-python - checkout + - mkdir artifacts blocks: - name: "Wheels: OSX x64" run: @@ -32,7 +30,7 @@ blocks: commands: - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse - - artifact push workflow wheelhouse-macOS-${ARCH}.tgz + - artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/ - name: "Wheels: OSX arm64" run: when: "tag =~ '.*'" @@ -53,14 +51,142 @@ blocks: commands: - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse - - artifact push workflow wheelhouse-macOS-${ARCH}.tgz - - - name: Source package verification with Python 3 (OSX x64) +docs + - artifact push workflow wheelhouse-macOS-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/ + - name: "Wheels: Linux arm64" + run: + when: "tag =~ '.*'" + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu20-04-arm64-1 + env_vars: + - name: OS_NAME + value: linux + - name: ARCH + value: arm64 + jobs: + - name: Build + commands: + - ./tools/build-manylinux.sh "${LIBRDKAFKA_VERSION#v}" + - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse + - artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/ + - name: "Wheels: Linux x64" + run: + when: "tag =~ '.*'" + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu20-04-amd64-3 + env_vars: + - name: OS_NAME + value: linux + - name: ARCH + value: x64 + jobs: + - name: Build + commands: + - ./tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse + - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse + - artifact push workflow wheelhouse-linux-${ARCH}.tgz --destination artifacts/wheels-${OS_NAME}-${ARCH}.tgz/ + - name: "Wheels: Windows" + run: + when: "tag =~ '.*'" + dependencies: [] + task: + agent: + machine: + type: s1-prod-windows + env_vars: + - name: OS_NAME + value: windows + - name: ARCH + value: x64 + prologue: + commands: + - cache restore msys2-x64 + - ".\\tools\\mingw-w64\\setup-msys2.ps1" + - $env:PATH = 'C:\msys64\usr\bin;' + $env:PATH + - bash -lc './tools/mingw-w64/msys2-dependencies.sh' + - cache delete msys2-x64 + - cache store msys2-x64 c:/msys64 + jobs: + - name: Build + env_vars: + - name: CHERE_INVOKING + value: 'yes' + - name: MSYSTEM + value: UCRT64 + commands: + - bash tools/mingw-w64/semaphore_commands.sh + - bash tools/wheels/install-librdkafka.sh $env:LIBRDKAFKA_VERSION.TrimStart("v") dest + - tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse + - tar -czf wheelhouse-windows-${Env:ARCH}.tgz wheelhouse + - artifact push workflow wheelhouse-windows-${Env:ARCH}.tgz --destination artifacts/wheels-${Env:OS_NAME}-${Env:ARCH}.tgz/ + - name: "Source package verification and Integration tests with Python 3 (Linux x64)" + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu20-04-amd64-2 + env_vars: + - name: OS_NAME + value: linux + - name: ARCH + value: x64 + jobs: + - name: Build + commands: + - sem-version python 3.8 + # use a virtualenv + - python3 -m venv _venv && source _venv/bin/activate + - chmod u+r+x tools/source-package-verification.sh + - tools/source-package-verification.sh + - name: "Source package verification with Python 3 (Linux arm64)" + dependencies: [] + task: + agent: + machine: + type: s1-prod-ubuntu20-04-arm64-1 + env_vars: + - name: OS_NAME + value: linux + - name: ARCH + value: arm64 + jobs: + - name: Build + commands: + - sem-version python 3.8 + # use a virtualenv + - python3 -m venv _venv && source _venv/bin/activate + - chmod u+r+x tools/source-package-verification.sh + - tools/source-package-verification.sh + - name: "Source package verification with Python 3 (OSX x64) +docs" dependencies: [] task: agent: machine: type: s1-prod-macos + env_vars: + - name: OS_NAME + value: osx + - name: ARCH + value: x64 + jobs: + - name: Build + commands: + - sem-version python 3.8 + # use a virtualenv + - python3 -m venv _venv && source _venv/bin/activate + - chmod u+r+x tools/source-package-verification.sh + - tools/source-package-verification.sh + - name: "Source package verification with Python 3 (OSX arm64) +docs" + dependencies: [] + task: + agent: + machine: + type: s1-prod-macos-arm64 env_vars: - name: OS_NAME value: osx @@ -69,17 +195,33 @@ blocks: jobs: - name: Build commands: + - sem-version python 3.8 # use a virtualenv - python3 -m venv _venv && source _venv/bin/activate - - pip install -r docs/requirements.txt - - pip install -U protobuf - # install librdkafka - - lib_dir=dest/runtimes/$OS_NAME-$ARCH/native - - tools/wheels/install-librdkafka.sh "${LIBRDKAFKA_VERSION#v}" dest - - export CFLAGS="$CFLAGS -I${PWD}/dest/build/native/include" - - export LDFLAGS="$LDFLAGS -L${PWD}/${lib_dir}" - - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/$lib_dir" - - export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/$lib_dir" - # install confluent-kafka - - python setup.py build && python setup.py install - - make docs + - chmod u+r+x tools/source-package-verification.sh + - tools/source-package-verification.sh + - name: "Packaging" + run: + when: "tag =~ '.*'" + dependencies: + - "Wheels: OSX x64" + - "Wheels: OSX arm64" + - "Wheels: Linux arm64" + - "Wheels: Linux x64" + - "Wheels: Windows" + task: + agent: + machine: + type: s1-prod-ubuntu20-04-amd64-3 + jobs: + - name: "Packaging all artifacts" + commands: + - artifact pull workflow artifacts + - cd artifacts + - ls *.tgz |xargs -n1 tar -xvf + - tar cvf confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz wheelhouse/ + - ls -la + - sha256sum confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz + - cd .. + - artifact push project artifacts/confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz --destination confluent-kafka-python-wheels-${SEMAPHORE_GIT_TAG_NAME}-${SEMAPHORE_WORKFLOW_ID}.tgz + - echo Thank you diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e1317c9d3..000000000 --- a/.travis.yml +++ /dev/null @@ -1,94 +0,0 @@ -env: - global: - - LIBRDKAFKA_VERSION=v1.9.2 - - LIBRDKAFKA_SRC_VERSION=master - -jobs: - include: - - name: "Source package verification with Python 3.8 (Linux)" - os: linux - language: python - dist: xenial - python: "3.8" - env: LD_LIBRARY_PATH="$PWD/tmp-build/lib" - services: docker - - - name: "Wheels: Windows x64" - if: tag is present - os: windows - language: shell - env: BUILD_WHEELS=1 - before_install: - - choco install python --version 3.8.0 - - export PATH="/c/Python38:/c/Python38/Scripts:$PATH" - # make sure it's on PATH as 'python3' - - ln -s /c/Python38/python.exe /c/Python38/python3.exe - install: - - bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest - script: - - tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse - - - name: "Wheels: Windows x86" - if: tag is present - os: windows - language: shell - env: BUILD_WHEELS=1 - before_install: - - choco install python --version 3.8.0 - - export PATH="/c/Python38:/c/Python38/Scripts:$PATH" - # make sure it's on PATH as 'python3' - - ln -s /c/Python38/python.exe /c/Python38/python3.exe - install: - - bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest - script: - - tools/wheels/build-wheels.bat x86 win32 dest wheelhouse - - - name: "Wheels: Linux x64" - if: tag is present - language: python - python: "3.8" - services: docker - env: BUILD_WHEELS=1 - script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse - -install: - # Install interceptors - - tools/install-interceptors.sh - - if [[ $BUILD_WHEELS != 1 ]]; then pip install -r tests/requirements.txt ; fi - - if [[ $MK_DOCS == y ]]; then pip install -r docs/requirements.txt; fi - # Install librdkafka and confluent_kafka[avro] if not building wheels - - if [[ $BUILD_WHEELS != 1 ]]; then pip install -U protobuf && tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_SRC_VERSION} tmp-build ; fi - - - -# Note: Will not be run for wheel builds. -script: - - flake8 - # Build package - - pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf] - - ldd staging/libs/* || otool -L staging/libs/* || true - # Run type checks - - mypy src/confluent_kafka - # Run tests - - if [[ $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs python -m pytest --timeout 600 --ignore=tmp-build || travis_terminate 1; fi - # Build docs - - if [[ $MK_DOCS == y ]]; then make docs; fi - # Checksum artifacts to build log - - sha256sum wheelhouse/* || true - -deploy: - provider: s3 - edge: true - access_key_id: - secure: "HGz8fJ0DLM0T7BddjuSBpPw+mRAmp1vP1xbl+gfhPsPTmXfjeAczj1YekfH6yrro7I3JwwNfXtUIofgRSeVW3Hl6rCI+ODnF82TFdjo/7yA3owUaVc4rx1qau6oMBCYWFCoHrQo+qHJ+bMpG1ppZvRR+pgaY+YBCTkki4p3q0MKYppnZmC89pvJkOBZatNUxzhZLcyMykShN+cdp3z7o2oED9/IJ8m30g0mkt/jIUZFdTiK/mQ2NxcZn667AAsSnyOud1pdXeg+UMvjR6U2thESw7DJpKyWJqv2eU/8SWGQyzvom+W44ZWdgjtqHo1ObTcKR2PHB3yYx6kixo5Wp9PUP/c0jAj78k+BURs/XXKj5fk/mM9FDlJpFv4FUDbhl1tTLdKNEltRYfjPfNApb963QEx0VyXWSxnBEVdOoi3SSdlx80EnYxPX3OfMKvIeYFTIURow7E1YMHl4kvBuYxRAUPjiuNn43jfqkgRhkdY7YaofJvu2hDw/togD1uUemP49QoCP9aC0TJ3aF/qfpcpoc8g8K6sF0cu5uOGyU2LGtYK0uYVZzkMKTX6QBP8I8SJU0Zw2Xprtwl7/qtZri47N83f5/WrWMNIVLOKdbLxPC2piT6N3M3aBt5l07dkEDmnzen585mQrNNbuXg/Din3D1XpGPWUxAZjw/LfcOR+A=" - secret_access_key: - secure: "YBAWciQ0WIhFJw/V72iERmukozz1hxd3HVbrhedHiNotYa53kQMd9qz9p6c4aJUF/y6xPrctCRVK1fUYlKVFJzCyPp5TOdqHcv56q9X9ip4Mt+oVwzbtdeHxGX+AYkr6sSn6xNlE6uNFaTVPzQ1EAL9WfhVkcuYDdponr8yc/HnCOIjafa7LZmtefnEpwhU1sAlyJFVeJkQfagZlmrz7cMiDpsvI10GXydJGWmwLoUTEKQiUf1ZKSlVd92bAWmALyCXmUZ0aS7SisO54580AsOR1TdWNIKZJ21n7PYY08GtoJtb9593CLgK0Z3FtTo2GZQbd2jehWA7ag8ku4e8rYNb78dSalKcN86kFH7GR6UJL+pL1c8O4LIJMzKALFcqpHYlxmyhPIzCCfVlCIWxwk/qTloa7dNBODb0vqAJJEbZ3q950Ig93gyU1kxiE8dwHkrQzlJCyufNtPY51Kv8MGa7n+/tI9z4LK4VVz+w4CrCzzl8T7SpWV4w+BKMKkymR4hL4JrpLnB8egpRqIL/V3E61qA1x9ik5Ck2AP+wEshiou3oJZ7BU0cGmp0ttVidwZ96sp2wxStulouQDwuT1A6jkvcCt5s3pM3eYIrucYk/mtBwHoz8DyYZH1Ds1JmTffCe4ZeEfFzIC5RIzpbj90t03aTd4oYVgYETPPyfdqhU=" - bucket: librdkafka-ci-packages - region: us-west-1 - local-dir: wheelhouse - upload_dir: confluent-kafka-python/p-confluent-kafka-python__bld-travis__plat-${TRAVIS_OS_NAME}__tag-${TRAVIS_TAG}__sha-${TRAVIS_COMMIT}__bid-${TRAVIS_BUILD_ID}__ - cleanup: false - on: - repo: confluentinc/confluent-kafka-python - tags: true - condition: "$BUILD_WHEELS == 1" diff --git a/CHANGELOG.md b/CHANGELOG.md index 136dd4110..585044bae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,98 @@ # Confluent's Python client for Apache Kafka +## v2.2.0 -## Next Version +v2.2.0 is a feature release with the following features, fixes and enhancements: + - [KIP-339](https://cwiki.apache.org/confluence/display/KAFKA/KIP-339%3A+Create+a+new+IncrementalAlterConfigs+API) + IncrementalAlterConfigs API (#1517). + - [KIP-554](https://cwiki.apache.org/confluence/display/KAFKA/KIP-554%3A+Add+Broker-side+SCRAM+Config+API): + User SASL/SCRAM credentials alteration and description (#1575). + - Added documentation with an example of FIPS compliant communication with Kafka cluster. + - Fixed wrong error code parameter name in KafkaError. + +confluent-kafka-python is based on librdkafka v2.2.0, see the +[librdkafka release notes](https://github.com/confluentinc/librdkafka/releases/tag/v2.2.0) +for a complete list of changes, enhancements, fixes and upgrade considerations. + + +## v2.1.1 + +v2.1.1 is a maintenance release with the following fixes and enhancements: + +### Fixes + +- Added a new ConsumerGroupState UNKNOWN. The typo state UNKOWN is deprecated and will be removed in the next major version. +- Fix some Admin API documentation stating -1 for infinite timeout incorrectly. + Request timeout can't be infinite. + +confluent-kafka-python is based on librdkafka v2.1.1, see the +[librdkafka release notes](https://github.com/edenhill/librdkafka/releases/tag/v2.1.1) +for a complete list of changes, enhancements, fixes and upgrade considerations. + + +## v2.1.0 + +v2.1.0 is a feature release with the following features, fixes and enhancements: + +- Added `set_sasl_credentials`. This new method (on the Producer, Consumer, and AdminClient) allows modifying the stored + SASL PLAIN/SCRAM credentials that will be used for subsequent (new) connections to a broker (#1511). +- Wheels for Linux / arm64 (#1496). +- Added support for Default num_partitions in CreateTopics Admin API. +- Added support for password protected private key in CachedSchemaRegistryClient. +- Add reference support in Schema Registry client. (@RickTalken, #1304) +- Migrated travis jobs to Semaphore CI (#1503) +- Added support for schema references. (#1514 and @slominskir #1088) +- [KIP-320](https://cwiki.apache.org/confluence/display/KAFKA/KIP-320%3A+Allow+fetchers+to+detect+and+handle+log+truncation): + add offset leader epoch methods to the TopicPartition and Message classes (#1540). + +confluent-kafka-python is based on librdkafka v2.1.0, see the +[librdkafka release notes](https://github.com/edenhill/librdkafka/releases/tag/v2.1.0) +for a complete list of changes, enhancements, fixes and upgrade considerations. + + +## v2.0.2 + +v2.0.2 is a feature release with the following features, fixes and enhancements: + + - Added Python 3.11 wheels. + - [KIP-222](https://cwiki.apache.org/confluence/display/KAFKA/KIP-222+-+Add+Consumer+Group+operations+to+Admin+API) + Add Consumer Group operations to Admin API. + - [KIP-518](https://cwiki.apache.org/confluence/display/KAFKA/KIP-518%3A+Allow+listing+consumer+groups+per+state) + Allow listing consumer groups per state. + - [KIP-396](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=97551484) + Partially implemented: support for AlterConsumerGroupOffsets. + - As result of the above KIPs, added (#1449) + - `list_consumer_groups` Admin operation. Supports listing by state. + - `describe_consumer_groups` Admin operation. Supports multiple groups. + - `delete_consumer_groups` Admin operation. Supports multiple groups. + - `list_consumer_group_offsets` Admin operation. Currently, only supports 1 group with multiple partitions. Supports require_stable option. + - `alter_consumer_group_offsets` Admin operation. Currently, only supports 1 group with multiple offsets. + - Added `normalize.schemas` configuration property to Schema Registry client (@rayokota, #1406) - Added metadata to `TopicPartition` type and `commit()` (#1410). - - Added `consumer.memberid()` for getting member id assigned to + - Added `consumer.memberid()` for getting member id assigned to the consumer in a consumer group (#1154). - - Added Python 3.11 wheels + - Implemented `nb_bool` method for the Producer, so that the default (which uses len) + will not be used. This avoids situations where producers with no enqueued items would + evaluate to False (@vladz-sternum, #1445). + - Deprecated `AvroProducer` and `AvroConsumer`. Use `AvroSerializer` and `AvroDeserializer` instead. + - Deprecated `list_groups`. Use `list_consumer_groups` and `describe_consumer_groups` instead. + - Improved Consumer Example to show atleast once semantics. + - Improved Serialization and Deserialization Examples. + - Documentation Improvements. + +## Upgrade considerations + +OpenSSL 3.0.x upgrade in librdkafka requires a major version bump, as some + legacy ciphers need to be explicitly configured to continue working, + but it is highly recommended NOT to use them. The rest of the API remains + backward compatible. + +confluent-kafka-python is based on librdkafka 2.0.2, see the +[librdkafka v2.0.0 release notes](https://github.com/edenhill/librdkafka/releases/tag/v2.0.0) +and later ones for a complete list of changes, enhancements, fixes and upgrade considerations. +**Note: There were no v2.0.0 and v2.0.1 releases.** ## v1.9.2 diff --git a/Makefile b/Makefile index 877356983..83fdd30c3 100644 --- a/Makefile +++ b/Makefile @@ -12,3 +12,19 @@ clean: docs: $(MAKE) -C docs html + +style-check: + @(tools/style-format.sh \ + $$(git ls-tree -r --name-only HEAD | egrep '\.(c|h|py)$$') ) + +style-check-changed: + @(tools/style-format.sh \ + $$( (git diff --name-only ; git diff --name-only --staged) | egrep '\.(c|h|py)$$')) + +style-fix: + @(tools/style-format.sh --fix \ + $$(git ls-tree -r --name-only HEAD | egrep '\.(c|h|py)$$')) + +style-fix-changed: + @(tools/style-format.sh --fix \ + $$( (git diff --name-only ; git diff --name-only --staged) | egrep '\.(c|h|py)$$')) diff --git a/docs/conf.py b/docs/conf.py index 613fba03a..1ef076576 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ # All configuration values have a default; values that are commented out # serve to show the default. +import sphinx_rtd_theme import sys import os from glob import glob @@ -27,21 +28,21 @@ ###################################################################### # General information about the project. project = u'confluent-kafka' -copyright = u'2016-2021, Confluent Inc.' +copyright = u'2016-2023, Confluent Inc.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.9.2' +version = '2.2.0' # The full version, including alpha/beta/rc tags. release = version ###################################################################### # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -67,13 +68,13 @@ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -81,32 +82,31 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # -- Options for HTML output ---------------------------------------------- -import sphinx_rtd_theme # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. @@ -118,74 +118,74 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -#html_static_path = ['_static'] +# html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'confluent-kafkadoc' @@ -194,43 +194,43 @@ # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'confluent-kafka.tex', u'confluent-kafka Documentation', - u'Magnus Edenhill', 'manual'), + ('index', 'confluent-kafka.tex', u'confluent-kafka Documentation', + u'Magnus Edenhill', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- @@ -243,7 +243,7 @@ ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -252,21 +252,21 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'confluent-kafka', u'confluent-kafka Documentation', - u'Magnus Edenhill', 'confluent-kafka', 'One line description of project.', - 'Miscellaneous'), + ('index', 'confluent-kafka', u'confluent-kafka Documentation', + u'Magnus Edenhill', 'confluent-kafka', 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False autodoc_member_order = 'bysource' diff --git a/docs/index.rst b/docs/index.rst index df5a62080..0515858fd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -45,10 +45,16 @@ Supporting classes - :ref:`ConfigResource ` - :ref:`ResourceType ` - :ref:`ResourcePatternType ` + - :ref:`AlterConfigOpType ` - :ref:`AclOperation ` - :ref:`AclPermissionType ` - :ref:`AclBinding ` - :ref:`AclBindingFilter ` + - :ref:`ScramCredentialInfo ` + - :ref:`UserScramCredentialsDescription ` + - :ref:`UserScramCredentialAlteration ` + - :ref:`UserScramCredentialUpsertion ` + - :ref:`UserScramCredentialDeletion ` Experimental These classes are experimental and are likely to be removed, or subject to incompatible @@ -142,6 +148,15 @@ ResourcePatternType .. autoclass:: confluent_kafka.admin.ResourcePatternType :members: +.. _pythonclient_alter_config_op_type: + +***************** +AlterConfigOpType +***************** + +.. autoclass:: confluent_kafka.admin.AlterConfigOpType + :members: + .. _pythonclient_acl_operation: ************** @@ -178,6 +193,60 @@ AclBindingFilter .. autoclass:: confluent_kafka.admin.AclBindingFilter :members: +.. _pythonclient_scram_mechanism: + +************** +ScramMechanism +************** + +.. autoclass:: confluent_kafka.admin.ScramMechanism + :members: + +.. _pythonclient_scram_credential_info: + +******************* +ScramCredentialInfo +******************* + +.. autoclass:: confluent_kafka.admin.ScramCredentialInfo + :members: + +.. _pythonclient_user_scram_credentials_description: + +******************************* +UserScramCredentialsDescription +******************************* + +.. autoclass:: confluent_kafka.admin.UserScramCredentialsDescription + :members: + +.. _pythonclient_user_scram_credential_alteration: + +***************************** +UserScramCredentialAlteration +***************************** + +.. autoclass:: confluent_kafka.admin.UserScramCredentialAlteration + :members: + +.. _pythonclient_user_scram_credential_upsertion: + +**************************** +UserScramCredentialUpsertion +**************************** + +.. autoclass:: confluent_kafka.admin.UserScramCredentialUpsertion + :members: + +.. _pythonclient_user_scram_credential_deletion: + +*************************** +UserScramCredentialDeletion +*************************** + +.. autoclass:: confluent_kafka.admin.UserScramCredentialDeletion + :members: + .. _pythonclient_consumer: ******** @@ -713,14 +782,14 @@ providing a dict of configuration properties to the instance constructor, e.g. .. code-block:: python conf = {'bootstrap.servers': 'mybroker.com', - 'group.id': 'mygroup', + 'group.id': 'mygroup', 'session.timeout.ms': 6000, 'on_commit': my_commit_callback, 'auto.offset.reset': 'earliest'} consumer = confluent_kafka.Consumer(conf) -The Python client provides the following configuration properties in +The Python client provides the following configuration properties in addition to the properties dictated by the underlying librdkafka C library: * ``default.topic.config``: value is a dict of client topic-level configuration @@ -778,4 +847,4 @@ addition to the properties dictated by the underlying librdkafka C library: In the Python client, the ``logger`` configuration property is used for log handler, not ``log_cb``. For the full range of configuration properties, please consult librdkafka's documentation: -https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md \ No newline at end of file +https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md diff --git a/examples/adminapi.py b/examples/adminapi.py index 5ea343188..c999bab3d 100755 --- a/examples/adminapi.py +++ b/examples/adminapi.py @@ -17,10 +17,14 @@ # Example use of AdminClient operations. -from confluent_kafka.admin import (AdminClient, NewTopic, NewPartitions, ConfigResource, ConfigSource, - AclBinding, AclBindingFilter, ResourceType, ResourcePatternType, - AclOperation, AclPermissionType) -from confluent_kafka import KafkaException +from confluent_kafka import (KafkaException, ConsumerGroupTopicPartitions, + TopicPartition, ConsumerGroupState) +from confluent_kafka.admin import (AdminClient, NewTopic, NewPartitions, ConfigResource, + ConfigEntry, ConfigSource, AclBinding, + AclBindingFilter, ResourceType, ResourcePatternType, + AclOperation, AclPermissionType, AlterConfigOpType, + ScramMechanism, ScramCredentialInfo, + UserScramCredentialUpsertion, UserScramCredentialDeletion) import sys import threading import logging @@ -259,6 +263,41 @@ def example_delete_acls(a, args): raise +def example_incremental_alter_configs(a, args): + """ Incrementally alter configs, keeping non-specified + configuration properties with their previous values. + + Input Format : ResourceType1 ResourceName1 Key=Operation:Value;Key2=Operation2:Value2;Key3=DELETE + ResourceType2 ResourceName2 ... + + Example: TOPIC T1 compression.type=SET:lz4;cleanup.policy=ADD:compact; + retention.ms=DELETE TOPIC T2 compression.type=SET:gzip ... + """ + resources = [] + for restype, resname, configs in zip(args[0::3], args[1::3], args[2::3]): + incremental_configs = [] + for name, operation_and_value in [conf.split('=') for conf in configs.split(';')]: + if operation_and_value == "DELETE": + operation, value = operation_and_value, None + else: + operation, value = operation_and_value.split(':') + operation = AlterConfigOpType[operation] + incremental_configs.append(ConfigEntry(name, value, + incremental_operation=operation)) + resources.append(ConfigResource(restype, resname, + incremental_configs=incremental_configs)) + + fs = a.incremental_alter_configs(resources) + + # Wait for operation to finish. + for res, f in fs.items(): + try: + f.result() # empty, but raises exception on failure + print("{} configuration successfully altered".format(res)) + except Exception: + raise + + def example_alter_configs(a, args): """ Alter configs atomically, replacing non-specified configuration properties with their default values. @@ -419,17 +458,229 @@ def example_list(a, args): print(" {} consumer groups".format(len(groups))) for g in groups: if g.error is not None: - errstr = ": {}".format(t.error) + errstr = ": {}".format(g.error) else: errstr = "" print(" \"{}\" with {} member(s), protocol: {}, protocol_type: {}{}".format( - g, len(g.members), g.protocol, g.protocol_type, errstr)) + g, len(g.members), g.protocol, g.protocol_type, errstr)) for m in g.members: print("id {} client_id: {} client_host: {}".format(m.id, m.client_id, m.client_host)) +def example_list_consumer_groups(a, args): + """ + List Consumer Groups + """ + states = {ConsumerGroupState[state] for state in args} + future = a.list_consumer_groups(request_timeout=10, states=states) + try: + list_consumer_groups_result = future.result() + print("{} consumer groups".format(len(list_consumer_groups_result.valid))) + for valid in list_consumer_groups_result.valid: + print(" id: {} is_simple: {} state: {}".format( + valid.group_id, valid.is_simple_consumer_group, valid.state)) + print("{} errors".format(len(list_consumer_groups_result.errors))) + for error in list_consumer_groups_result.errors: + print(" error: {}".format(error)) + except Exception: + raise + + +def example_describe_consumer_groups(a, args): + """ + Describe Consumer Groups + """ + + futureMap = a.describe_consumer_groups(args, request_timeout=10) + + for group_id, future in futureMap.items(): + try: + g = future.result() + print("Group Id: {}".format(g.group_id)) + print(" Is Simple : {}".format(g.is_simple_consumer_group)) + print(" State : {}".format(g.state)) + print(" Partition Assignor : {}".format(g.partition_assignor)) + print(" Coordinator : ({}) {}:{}".format(g.coordinator.id, g.coordinator.host, g.coordinator.port)) + print(" Members: ") + for member in g.members: + print(" Id : {}".format(member.member_id)) + print(" Host : {}".format(member.host)) + print(" Client Id : {}".format(member.client_id)) + print(" Group Instance Id : {}".format(member.group_instance_id)) + if member.assignment: + print(" Assignments :") + for toppar in member.assignment.topic_partitions: + print(" {} [{}]".format(toppar.topic, toppar.partition)) + except KafkaException as e: + print("Error while describing group id '{}': {}".format(group_id, e)) + except Exception: + raise + + +def example_delete_consumer_groups(a, args): + """ + Delete Consumer Groups + """ + groups = a.delete_consumer_groups(args, request_timeout=10) + for group_id, future in groups.items(): + try: + future.result() # The result itself is None + print("Deleted group with id '" + group_id + "' successfully") + except KafkaException as e: + print("Error deleting group id '{}': {}".format(group_id, e)) + except Exception: + raise + + +def example_list_consumer_group_offsets(a, args): + """ + List consumer group offsets + """ + + topic_partitions = [] + for topic, partition in zip(args[1::2], args[2::2]): + topic_partitions.append(TopicPartition(topic, int(partition))) + if len(topic_partitions) == 0: + topic_partitions = None + groups = [ConsumerGroupTopicPartitions(args[0], topic_partitions)] + + futureMap = a.list_consumer_group_offsets(groups) + + for group_id, future in futureMap.items(): + try: + response_offset_info = future.result() + print("Group: " + response_offset_info.group_id) + for topic_partition in response_offset_info.topic_partitions: + if topic_partition.error: + print(" Error: " + topic_partition.error.str() + " occurred with " + + topic_partition.topic + " [" + str(topic_partition.partition) + "]") + else: + print(" " + topic_partition.topic + + " [" + str(topic_partition.partition) + "]: " + str(topic_partition.offset)) + + except KafkaException as e: + print("Failed to list {}: {}".format(group_id, e)) + except Exception: + raise + + +def example_alter_consumer_group_offsets(a, args): + """ + Alter consumer group offsets + """ + + topic_partitions = [] + for topic, partition, offset in zip(args[1::3], args[2::3], args[3::3]): + topic_partitions.append(TopicPartition(topic, int(partition), int(offset))) + if len(topic_partitions) == 0: + topic_partitions = None + groups = [ConsumerGroupTopicPartitions(args[0], topic_partitions)] + + futureMap = a.alter_consumer_group_offsets(groups) + + for group_id, future in futureMap.items(): + try: + response_offset_info = future.result() + print("Group: " + response_offset_info.group_id) + for topic_partition in response_offset_info.topic_partitions: + if topic_partition.error: + print(" Error: " + topic_partition.error.str() + " occurred with " + + topic_partition.topic + " [" + str(topic_partition.partition) + "]") + else: + print(" " + topic_partition.topic + + " [" + str(topic_partition.partition) + "]: " + str(topic_partition.offset)) + + except KafkaException as e: + print("Failed to alter {}: {}".format(group_id, e)) + except Exception: + raise + + +def example_describe_user_scram_credentials(a, args): + """ + Describe User Scram Credentials + """ + futmap = a.describe_user_scram_credentials(args) + + for username, fut in futmap.items(): + print("Username: {}".format(username)) + try: + response = fut.result() + for scram_credential_info in response.scram_credential_infos: + print(f" Mechanism: {scram_credential_info.mechanism} " + + f"Iterations: {scram_credential_info.iterations}") + except KafkaException as e: + print(" Error: {}".format(e)) + except Exception as e: + print(f" Unexpected exception: {e}") + + +def example_alter_user_scram_credentials(a, args): + """ + AlterUserScramCredentials + """ + alterations_args = [] + alterations = [] + i = 0 + op_cnt = 0 + + while i < len(args): + op = args[i] + if op == "UPSERT": + if i + 5 >= len(args): + raise ValueError( + f"Invalid number of arguments for alteration {op_cnt}, expected 5, got {len(args) - i - 1}") + user = args[i + 1] + mechanism = ScramMechanism[args[i + 2]] + iterations = int(args[i + 3]) + password = bytes(args[i + 4], 'utf8') + # if salt is an empty string, + # set it to None to generate it randomly. + salt = args[i + 5] + if not salt: + salt = None + else: + salt = bytes(salt, 'utf8') + alterations_args.append([op, user, mechanism, iterations, + iterations, password, salt]) + i += 6 + elif op == "DELETE": + if i + 2 >= len(args): + raise ValueError( + f"Invalid number of arguments for alteration {op_cnt}, expected 2, got {len(args) - i - 1}") + user = args[i + 1] + mechanism = ScramMechanism[args[i + 2]] + alterations_args.append([op, user, mechanism]) + i += 3 + else: + raise ValueError(f"Invalid alteration {op}, must be UPSERT or DELETE") + op_cnt += 1 + + for alteration_arg in alterations_args: + op = alteration_arg[0] + if op == "UPSERT": + [_, user, mechanism, iterations, + iterations, password, salt] = alteration_arg + scram_credential_info = ScramCredentialInfo(mechanism, iterations) + upsertion = UserScramCredentialUpsertion(user, scram_credential_info, + password, salt) + alterations.append(upsertion) + elif op == "DELETE": + [_, user, mechanism] = alteration_arg + deletion = UserScramCredentialDeletion(user, mechanism) + alterations.append(deletion) + + futmap = a.alter_user_scram_credentials(alterations) + for username, fut in futmap.items(): + try: + fut.result() + print("{}: Success".format(username)) + except KafkaException as e: + print("{}: Error: {}".format(username, e)) + + if __name__ == '__main__': if len(sys.argv) < 3: sys.stderr.write('Usage: %s \n\n' % sys.argv[0]) @@ -440,6 +691,9 @@ def example_list(a, args): sys.stderr.write(' describe_configs ..\n') sys.stderr.write(' alter_configs ' + ' ..\n') + sys.stderr.write(' incremental_alter_configs ' + + ' ' + + ' ..\n') sys.stderr.write(' delta_alter_configs ' + ' ..\n') sys.stderr.write(' create_acls ' + @@ -449,6 +703,18 @@ def example_list(a, args): sys.stderr.write(' delete_acls ' + ' ..\n') sys.stderr.write(' list []\n') + sys.stderr.write(' list_consumer_groups [ ..]\n') + sys.stderr.write(' describe_consumer_groups ..\n') + sys.stderr.write(' delete_consumer_groups ..\n') + sys.stderr.write(' list_consumer_group_offsets [ ..]\n') + sys.stderr.write( + ' alter_consumer_group_offsets ' + + ' ..\n') + sys.stderr.write(' describe_user_scram_credentials [ ..]\n') + sys.stderr.write(' alter_user_scram_credentials UPSERT ' + + ' ' + + '[UPSERT ' + + ' DELETE ..]\n') sys.exit(1) broker = sys.argv[1] @@ -463,11 +729,19 @@ def example_list(a, args): 'create_partitions': example_create_partitions, 'describe_configs': example_describe_configs, 'alter_configs': example_alter_configs, + 'incremental_alter_configs': example_incremental_alter_configs, 'delta_alter_configs': example_delta_alter_configs, 'create_acls': example_create_acls, 'describe_acls': example_describe_acls, 'delete_acls': example_delete_acls, - 'list': example_list} + 'list': example_list, + 'list_consumer_groups': example_list_consumer_groups, + 'describe_consumer_groups': example_describe_consumer_groups, + 'delete_consumer_groups': example_delete_consumer_groups, + 'list_consumer_group_offsets': example_list_consumer_group_offsets, + 'alter_consumer_group_offsets': example_alter_consumer_group_offsets, + 'describe_user_scram_credentials': example_describe_user_scram_credentials, + 'alter_user_scram_credentials': example_alter_user_scram_credentials} if operation not in opsmap: sys.stderr.write('Unknown operation: %s\n' % operation) diff --git a/examples/docker/Dockerfile.alpine b/examples/docker/Dockerfile.alpine index 8e42153f9..13361a0ce 100644 --- a/examples/docker/Dockerfile.alpine +++ b/examples/docker/Dockerfile.alpine @@ -30,7 +30,7 @@ FROM alpine:3.12 COPY . /usr/src/confluent-kafka-python -ENV LIBRDKAFKA_VERSION v1.9.2 +ENV LIBRDKAFKA_VERSION v2.2.0 ENV KAFKACAT_VERSION master diff --git a/examples/fips/.gitignore b/examples/fips/.gitignore new file mode 100644 index 000000000..68d33ef04 --- /dev/null +++ b/examples/fips/.gitignore @@ -0,0 +1,4 @@ +docker/secrets/* +!docker/secrets/broker_jaas.conf +!docker/secrets/zookeeper_jaas.conf +!docker/secrets/generate_certificates.sh diff --git a/examples/fips/README.md b/examples/fips/README.md new file mode 100644 index 000000000..00e0732c2 --- /dev/null +++ b/examples/fips/README.md @@ -0,0 +1,90 @@ +# FIPS Compliance + +We tested FIPS compliance for the client using OpenSSL 3.0. To use the client in FIPS-compliant mode, use OpenSSL 3.0. Older versions of OpenSSL have not been verified (although they may work). + +## Communication between client and Kafka cluster + +### Installing client using OpenSSL and librdkafka bundled in wheels + +If you install this client through prebuilt wheels using `pip install confluent_kafka`, OpenSSL 3.0 is already statically linked with the librdkafka shared library. To enable this client to communicate with the Kafka cluster using the OpenSSL FIPS provider and FIPS-approved algorithms, you must enable the FIPS provider. You can find steps to enable the FIPS provider in section [Enabling FIPS provider](#enabling-fips-provider). + +You should follow the same above steps if you install this client from the source using `pip install confluent_kafka --no-binary :all:` with prebuilt librdkafka in which OpenSSL is statically linked + + +### Installing client using system OpenSSL with librdkafka shared library + +When you build the librdkafka from source, librdkafka dynamically links to the OpenSSL present in the system (if static linking is not used explicitly while building). If the installed OpenSSL is already working in FIPS mode, then you can directly jump to the section [Client configuration to enable FIPS provider](#client-configuration-to-enable-fips-provider) and enable `fips` provider. If you don't have OpenSSL working in FIPS mode, use the steps mentioned in the section [Enabling FIPS provider](#enabling-fips-provider) to make OpenSSL in your system FIPS compliant and then enable the `fips` provider. Once you have OpenSSL working in FIPS mode and the `fips` provider enabled, librdkafka and python client will use FIPS approved algorithms for the communication between client and Kafka Cluster using the producer, consumer or admin client. + +### Enabling FIPS provider + +To enable the FIPS provider, you must have the FIPS module available on your system, plug the module into OpenSSL, and then configure OpenSSL to use the module. +You can plug the FIPS provider into OpenSSL two ways: 1) put the module in the default module folder of OpenSSL or 2) point to the module with the environment variable, `OPENSSL_MODULES`. For example: `OPENSSL_MODULES="/path/to/fips/module/lib/folder/` + +You configure OpenSSL to use the FIPS provider using the FIPS configuration in OpenSSL config. You can 1) modify the default configuration file to include FIPS related config or 2) create a new configuration file and point to it using the environment variable,`OPENSSL_CONF`. For example `OPENSSL_CONF="/path/to/fips/enabled/openssl/config/openssl.cnf` For an example of OpenSSL config, see below. + +**NOTE:** You need to specify both `OPENSSL_MODULES` and `OPENSSL_CONF` environment variable when installing the client from pre-built wheels or when OpenSSL is statically linked to librdkafka. + +#### Steps to build FIPS provider module + +You can find steps to generate the FIPS provider module in the [README-FIPS doc](https://github.com/openssl/openssl/blob/openssl-3.0.8/README-FIPS.md) + +In short, you need to perform the following steps: + +1) Clone OpenSSL from [OpenSSL Github Repo](https://github.com/openssl/openssl) +2) Checkout the correct version. (v3.0.8 is the current FIPS compliant version for OpenSSL 3.0 at the time of writing this doc.) +3) Run `./Configure enable-fips` +4) Run `make install_fips` + +After last step, two files are generated. +* FIPS module (`fips.dylib` in Mac, `fips.so` in Linux and `fips.dll` in Windows) +* FIPS config (`fipsmodule.cnf`) file will be generated. Which needs to be used with OpenSSL. + +#### Referencing FIPS provider in OpenSSL + +As mentioned earlier, you can dynamically plug the FIPS module built above into OpenSSL by putting the FIPS module into the default OpenSSL module folder (only when installing from source). For default locations of OpenSSL on various operating systems, see the SSL section of the [Introduction to librdkafka - the Apache Kafka C/C++ client library](https://github.com/confluentinc/librdkafka/blob/master/INTRODUCTION.md#ssl). It should look something like `...lib/ossl-modules/`. + +You can also point to this module with the environment variable `OPENSSL_MODULES`. For example, `OPENSSL_MODULES="/path/to/fips/module/lib/folder/` + +#### Linking FIPS provider with OpenSSL + +To enable FIPS in OpenSSL, you must include `fipsmodule.cnf` in the file, `openssl.cnf`. The `fipsmodule.cnf` file includes `fips_sect` which OpenSSL requires to enable FIPS. See the example below. + +Some of the algorithms might have different implementation in FIPS or other providers. If you load two different providers like default and fips, any implementation could be used. To make sure you fetch only FIPS compliant version of the algorithm, use `fips=yes` default property in config file. + +OpenSSL config should look something like after applying the above changes. + +``` +config_diagnostics = 1 +openssl_conf = openssl_init + +.include /usr/local/ssl/fipsmodule.cnf + +[openssl_init] +providers = provider_sect +alg_section = algorithm_sect + +[provider_sect] +fips = fips_sect + +[algorithm_sect] +default_properties = fips=yes +. +. +. +``` + +### Client configuration to enable FIPS provider + +OpenSSL requires some non-crypto algorithms as well. These algorithms are not included in the FIPS provider and you need to use the `base` provider in conjunction with the `fips` provider. Base provider comes with OpenSSL by default. You must enable `base` provider in the client configuration. + +To make client (consumer, producer or admin client) FIPS compliant, you must enable only `fips` and `base` provider in the client using the `ssl.providers` configuration property i.e `'ssl.providers': 'fips,base'`. + + +## Communication between client and Schema Registry + +This part is not tested for FIPS compliance right now. + +## References +* [Generating FIPS module and config file](https://github.com/openssl/openssl/blob/openssl-3.0.8/README-FIPS.md) +* [How to use FIPS Module](https://www.openssl.org/docs/man3.0/man7/fips_module.html) +* [librdkafka SSL Information](https://github.com/confluentinc/librdkafka/blob/master/INTRODUCTION.md#ssl) diff --git a/examples/fips/docker/README.md b/examples/fips/docker/README.md new file mode 100644 index 000000000..bcaadf215 --- /dev/null +++ b/examples/fips/docker/README.md @@ -0,0 +1,5 @@ +Use `generate_certificates.sh` in secrets folder to generate the certificates. +Up the server using `docker-compose up`. +Use example producer and consumer to test the FIPS compliance. Note that you might need to point to FIPS module and FIPS enabled OpenSSL 3.0 config using environment variables like ` OPENSSL_CONF="/path/to/fips/enabled/openssl/config/openssl.cnf" OPENSSL_MODULES="/path/to/fips/module/lib/folder/" ./examples/fips/fips_producer.py localhost:9092 test-topic` + +Uncomment `KAFKA_SSL_CIPHER.SUITES: TLS_CHACHA20_POLY1305_SHA256` in `docker-compose.yml` to enable non FIPS compliant algorithm. Use this to verify that only FIPS compliant algorithms are used. \ No newline at end of file diff --git a/examples/fips/docker/docker-compose.yml b/examples/fips/docker/docker-compose.yml new file mode 100644 index 000000000..550d5aa27 --- /dev/null +++ b/examples/fips/docker/docker-compose.yml @@ -0,0 +1,51 @@ +version: "3.9" +services: + + zookeeper: + hostname: zookeeper + container_name: zookeeper + restart: always + image: confluentinc/cp-zookeeper:7.4.0 + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + KAFKA_OPTS: -Djava.security.auth.login.config=/etc/kafka/secrets/zookeeper_jaas.conf + -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider + -DrequireClientAuthScheme=sasl + -Dzookeeper.allowSaslFailedClients=false + volumes: + - ./secrets:/etc/kafka/secrets + + broker: + image: confluentinc/cp-kafka:7.4.0 + hostname: broker + container_name: broker + restart: always + ports: + - 29092:29092 + - 9092:9092 + volumes: + - ./secrets:/etc/kafka/secrets + environment: + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_INTER_BROKER_LISTENER_NAME: SASL_SSL + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SASL_SSL:SASL_SSL,SASL_SSL_HOST:SASL_SSL + KAFKA_ADVERTISED_LISTENERS: SASL_SSL://localhost:9092,SASL_SSL_HOST://broker:29092 + KAFKA_LISTENERS: SASL_SSL://:9092,SASL_SSL_HOST://:29092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + CONFLUENT_METRICS_REPORTER_SECURITY_PROTOCOL: SASL_SSL + CONFLUENT_METRICS_REPORTER_SASL_JAAS_CONFIG: "org.apache.kafka.common.security.plain.PlainLoginModule required \ + username=\"client\" \ + password=\"client-secret\";" + KAFKA_SASL_ENABLED_MECHANISMS: PLAIN + KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN + KAFKA_SSL_KEYSTORE_FILENAME: server.keystore.jks + KAFKA_SSL_KEYSTORE_CREDENTIALS: creds + KAFKA_SSL_KEY_CREDENTIALS: creds + KAFKA_SSL_TRUSTSTORE_FILENAME: server.truststore.jks + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: creds + # KAFKA_SSL_CIPHER.SUITES: TLS_CHACHA20_POLY1305_SHA256 # FIPS non compliant algo. + # enables 2-way authentication + KAFKA_SSL_CLIENT_AUTH: "required" + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" + KAFKA_OPTS: -Djava.security.auth.login.config=/etc/kafka/secrets/broker_jaas.conf + KAFKA_SSL_PRINCIPAL_MAPPING_RULES: RULE:^CN=(.*?),OU=TEST.*$$/$$1/,DEFAULT diff --git a/examples/fips/docker/secrets/broker_jaas.conf b/examples/fips/docker/secrets/broker_jaas.conf new file mode 100644 index 000000000..1b6960d92 --- /dev/null +++ b/examples/fips/docker/secrets/broker_jaas.conf @@ -0,0 +1,17 @@ +KafkaServer { + org.apache.kafka.common.security.plain.PlainLoginModule required + username="broker" + password="broker-secret" + user_broker="broker-secret" + user_client="client-secret" + user_schema-registry="schema-registry-secret" + user_restproxy="restproxy-secret" + user_clientrestproxy="clientrestproxy-secret" + user_badclient="badclient-secret"; +}; + +Client { + org.apache.zookeeper.server.auth.DigestLoginModule required + username="kafka" + password="kafkasecret"; +}; diff --git a/examples/fips/docker/secrets/generate_certificates.sh b/examples/fips/docker/secrets/generate_certificates.sh new file mode 100644 index 000000000..5e32c661a --- /dev/null +++ b/examples/fips/docker/secrets/generate_certificates.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -e +server_hostname=${server_hostname:-localhost} +client_hostname=${client_hostname:-localhost} +cert_password=${cert_password:-111111} +openssl=${openssl:-openssl} +keytool=${keytool:-keytool} +echo openssl version +$openssl version +echo creates server keystore +$keytool -keystore server.keystore.jks -storepass $cert_password -alias ${server_hostname} -validity 365 -genkey -keyalg RSA -dname "cn=$server_hostname" +echo creates root CA +$openssl req -nodes -new -x509 -keyout ca-root.key -out ca-root.crt -days 365 -subj "/C=US/ST=CA/L=MV/O=CFLT/CN=CFLT" +echo creates CSR +$keytool -keystore server.keystore.jks -alias ${server_hostname} -certreq -file ${server_hostname}_server.csr -storepass $cert_password +echo sign CSR +$openssl x509 -req -CA ca-root.crt -CAkey ca-root.key -in ${server_hostname}_server.csr -out ${server_hostname}_server.crt -days 365 -CAcreateserial +echo import root CA +$keytool -keystore server.keystore.jks -alias CARoot -import -noprompt -file ca-root.crt -storepass $cert_password +echo import server certificate +$keytool -keystore server.keystore.jks -alias ${server_hostname} -import -file ${server_hostname}_server.crt -storepass $cert_password +echo create client CSR +$openssl req -newkey rsa:2048 -nodes -keyout ${client_hostname}_client.key -out ${client_hostname}_client.csr -subj "/C=US/ST=CA/L=MV/O=CFLT/CN=CFLT" -passin pass:$cert_password +echo sign client CSR +$openssl x509 -req -CA ca-root.crt -CAkey ca-root.key -in ${client_hostname}_client.csr -out ${client_hostname}_client.crt -days 365 -CAcreateserial +echo create client keystore +$openssl pkcs12 -export -in ${client_hostname}_client.crt -inkey ${client_hostname}_client.key -name ${client_hostname} -out client.keystore.p12 -passin pass:$cert_password \ +-passout pass:$cert_password +echo create truststore +$keytool -noprompt -keystore server.truststore.jks -alias CARoot -import -file ca-root.crt -storepass $cert_password +echo create creds file +echo "$cert_password" > ./creds +echo verify with: openssl pkcs12 -info -nodes -in client.keystore.p12 diff --git a/examples/fips/docker/secrets/zookeeper_jaas.conf b/examples/fips/docker/secrets/zookeeper_jaas.conf new file mode 100644 index 000000000..2485b3324 --- /dev/null +++ b/examples/fips/docker/secrets/zookeeper_jaas.conf @@ -0,0 +1,5 @@ +Server { + org.apache.zookeeper.server.auth.DigestLoginModule required + user_super="adminsecret" + user_kafka="kafkasecret"; +}; diff --git a/examples/fips/fips_consumer.py b/examples/fips/fips_consumer.py new file mode 100755 index 000000000..b6f9ad6a5 --- /dev/null +++ b/examples/fips/fips_consumer.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# +# Copyright 2023 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Example FIPS Compliant Consumer +# +from confluent_kafka import Consumer, KafkaException +import sys + + +def print_usage_and_exit(program_name): + sys.stderr.write('Usage: %s [options..] ..\n' % program_name) + sys.exit(1) + + +if __name__ == '__main__': + if len(sys.argv) < 4: + print_usage_and_exit(sys.argv[0]) + + broker = sys.argv[1] + group = sys.argv[2] + topics = sys.argv[3:] + conf = {'bootstrap.servers': broker, + 'group.id': group, + 'auto.offset.reset': 'earliest', + 'security.protocol': 'SASL_SSL', + 'sasl.mechanism': 'PLAIN', + 'sasl.username': 'broker', + 'sasl.password': 'broker-secret', + # pkc12 keystores are not FIPS compliant and hence you will need to use + # path to key and certificate separately in FIPS mode + # 'ssl.keystore.location': './docker/secrets/client.keystore.p12', + # 'ssl.keystore.password': '111111', + 'ssl.key.location': './docker/secrets/localhost_client.key', + 'ssl.key.password': '111111', + 'ssl.certificate.location': './docker/secrets/localhost_client.crt', + 'ssl.ca.location': './docker/secrets/ca-root.crt', + 'ssl.providers': 'fips,base' + } + + # Create Consumer instance + c = Consumer(conf) + + def print_assignment(consumer, partitions): + print('Assignment:', partitions) + + # Subscribe to topics + c.subscribe(topics, on_assign=print_assignment) + + # Read messages from Kafka, print to stdout + try: + while True: + msg = c.poll(timeout=1.0) + if msg is None: + continue + if msg.error(): + raise KafkaException(msg.error()) + else: + # Proper message + sys.stderr.write('%% %s [%d] at offset %d with key %s:\n' % + (msg.topic(), msg.partition(), msg.offset(), + str(msg.key()))) + print(msg.value()) + + except KeyboardInterrupt: + sys.stderr.write('%% Aborted by user\n') + + finally: + # Close down consumer to commit final offsets. + c.close() diff --git a/examples/fips/fips_producer.py b/examples/fips/fips_producer.py new file mode 100755 index 000000000..0aa46609b --- /dev/null +++ b/examples/fips/fips_producer.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# +# Copyright 2023 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Example Kafka FIPS Compliant Producer. +# Reads lines from stdin and sends to Kafka. +# + +from confluent_kafka import Producer +import sys + +if __name__ == '__main__': + if len(sys.argv) != 3: + sys.stderr.write('Usage: %s \n' % sys.argv[0]) + sys.exit(1) + + broker = sys.argv[1] + topic = sys.argv[2] + + # Producer configuration + # See https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md + conf = {'bootstrap.servers': broker, + 'security.protocol': 'SASL_SSL', + 'sasl.mechanism': 'PLAIN', + 'sasl.username': 'broker', + 'sasl.password': 'broker-secret', + # pkc12 keystores are not FIPS compliant and hence you will need to use + # path to key and certificate separately in FIPS mode + # 'ssl.keystore.location': './docker/secrets/client.keystore.p12', + # 'ssl.keystore.password': '111111', + 'ssl.key.location': './docker/secrets/localhost_client.key', + 'ssl.key.password': '111111', + 'ssl.certificate.location': './docker/secrets/localhost_client.crt', + 'ssl.ca.location': './docker/secrets/ca-root.crt', + 'ssl.providers': 'fips,base' + } + + # Create Producer instance + p = Producer(**conf) + + # Optional per-message delivery callback (triggered by poll() or flush()) + # when a message has been successfully delivered or permanently + # failed delivery (after retries). + def delivery_callback(err, msg): + if err: + sys.stderr.write('%% Message failed delivery: %s\n' % err) + else: + sys.stderr.write('%% Message delivered to %s [%d] @ %d\n' % + (msg.topic(), msg.partition(), msg.offset())) + + # Read lines from stdin, produce each line to Kafka + for line in sys.stdin: + try: + # Produce line (without newline) + p.produce(topic, line.rstrip(), callback=delivery_callback) + + except BufferError: + sys.stderr.write('%% Local producer queue is full (%d messages awaiting delivery): try again\n' % + len(p)) + + # Serve delivery callback queue. + # NOTE: Since produce() is an asynchronous API this poll() call + # will most likely not serve the delivery callback for the + # last produce()d message. + p.poll(0) + + # Wait until all messages have been delivered + sys.stderr.write('%% Waiting for %d deliveries\n' % len(p)) + p.flush() diff --git a/setup.py b/setup.py index b39cbe1a9..fd63523a8 100755 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ def get_install_requirements(path): setup(name='confluent-kafka', # Make sure to bump CFL_VERSION* in confluent_kafka/src/confluent_kafka.h # and version in docs/conf.py. - version='1.9.2', + version='2.2.0', description='Confluent\'s Python client for Apache Kafka', author='Confluent Inc', author_email='support@confluent.io', diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 000000000..a4c89dd38 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,7 @@ +### service-bot sonarqube plugin managed file +sonar.coverage.exclusions=**/test/**/*,**/tests/**/*,**/mock/**/*,**/mocks/**/*,**/*mock*,**/*test* +sonar.exclusions=**/*.pb.*,**/mk-include/**/* +sonar.language=python +sonar.projectKey=confluent-kafka-python +sonar.python.coverage.reportPaths=**/coverage.xml +sonar.sources=. diff --git a/src/confluent_kafka/__init__.py b/src/confluent_kafka/__init__.py index e6e343981..6b808bfc6 100644 --- a/src/confluent_kafka/__init__.py +++ b/src/confluent_kafka/__init__.py @@ -19,6 +19,7 @@ from .deserializing_consumer import DeserializingConsumer from .serializing_producer import SerializingProducer from .error import KafkaException, KafkaError +from ._model import Node, ConsumerGroupTopicPartitions, ConsumerGroupState from .cimpl import (Producer, Consumer, @@ -40,7 +41,8 @@ 'OFFSET_BEGINNING', 'OFFSET_END', 'OFFSET_INVALID', 'OFFSET_STORED', 'Producer', 'DeserializingConsumer', 'SerializingProducer', 'TIMESTAMP_CREATE_TIME', 'TIMESTAMP_LOG_APPEND_TIME', - 'TIMESTAMP_NOT_AVAILABLE', 'TopicPartition'] + 'TIMESTAMP_NOT_AVAILABLE', 'TopicPartition', 'Node', + 'ConsumerGroupTopicPartitions', 'ConsumerGroupState'] __version__ = version()[0] diff --git a/src/confluent_kafka/_model/__init__.py b/src/confluent_kafka/_model/__init__.py new file mode 100644 index 000000000..2bab6a1bd --- /dev/null +++ b/src/confluent_kafka/_model/__init__.py @@ -0,0 +1,91 @@ +# Copyright 2022 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from .. import cimpl + + +class Node: + """ + Represents node information. + Used by :class:`ConsumerGroupDescription` + + Parameters + ---------- + id: int + The node id of this node. + id_string: + String representation of the node id. + host: + The host name for this node. + port: int + The port for this node. + rack: str + The rack for this node. + """ + def __init__(self, id, host, port, rack=None): + self.id = id + self.id_string = str(id) + self.host = host + self.port = port + self.rack = rack + + +class ConsumerGroupTopicPartitions: + """ + Represents consumer group and its topic partition information. + Used by :meth:`AdminClient.list_consumer_group_offsets` and + :meth:`AdminClient.alter_consumer_group_offsets`. + + Parameters + ---------- + group_id: str + Id of the consumer group. + topic_partitions: list(TopicPartition) + List of topic partitions information. + """ + def __init__(self, group_id, topic_partitions=None): + self.group_id = group_id + self.topic_partitions = topic_partitions + + +class ConsumerGroupState(Enum): + """ + Enumerates the different types of Consumer Group State. + + Note that the state UNKOWN (typo one) is deprecated and will be removed in + future major release. Use UNKNOWN instead. + + Values + ------ + UNKNOWN : State is not known or not set. + UNKOWN : State is not known or not set. Typo. + PREPARING_REBALANCING : Preparing rebalance for the consumer group. + COMPLETING_REBALANCING : Consumer Group is completing rebalancing. + STABLE : Consumer Group is stable. + DEAD : Consumer Group is Dead. + EMPTY : Consumer Group is Empty. + """ + UNKNOWN = cimpl.CONSUMER_GROUP_STATE_UNKNOWN + UNKOWN = UNKNOWN + PREPARING_REBALANCING = cimpl.CONSUMER_GROUP_STATE_PREPARING_REBALANCE + COMPLETING_REBALANCING = cimpl.CONSUMER_GROUP_STATE_COMPLETING_REBALANCE + STABLE = cimpl.CONSUMER_GROUP_STATE_STABLE + DEAD = cimpl.CONSUMER_GROUP_STATE_DEAD + EMPTY = cimpl.CONSUMER_GROUP_STATE_EMPTY + + def __lt__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return self.value < other.value diff --git a/src/confluent_kafka/_util/__init__.py b/src/confluent_kafka/_util/__init__.py new file mode 100644 index 000000000..315277f42 --- /dev/null +++ b/src/confluent_kafka/_util/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2022 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .validation_util import ValidationUtil # noqa: F401 +from .conversion_util import ConversionUtil # noqa: F401 diff --git a/src/confluent_kafka/_util/conversion_util.py b/src/confluent_kafka/_util/conversion_util.py new file mode 100644 index 000000000..82c9b7018 --- /dev/null +++ b/src/confluent_kafka/_util/conversion_util.py @@ -0,0 +1,38 @@ +# Copyright 2022 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum + + +class ConversionUtil: + @staticmethod + def convert_to_enum(val, enum_clazz): + if type(enum_clazz) is not type(Enum): + raise TypeError("'enum_clazz' must be of type Enum") + + if type(val) == str: + # Allow it to be specified as case-insensitive string, for convenience. + try: + val = enum_clazz[val.upper()] + except KeyError: + raise ValueError("Unknown value \"%s\": should be a %s" % (val, enum_clazz.__name__)) + + elif type(val) == int: + # The C-code passes restype as an int, convert to enum. + val = enum_clazz(val) + + elif type(val) != enum_clazz: + raise TypeError("Unknown value \"%s\": should be a %s" % (val, enum_clazz.__name__)) + + return val diff --git a/src/confluent_kafka/_util/validation_util.py b/src/confluent_kafka/_util/validation_util.py new file mode 100644 index 000000000..ffe5785f2 --- /dev/null +++ b/src/confluent_kafka/_util/validation_util.py @@ -0,0 +1,56 @@ +# Copyright 2022 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..cimpl import KafkaError + +try: + string_type = basestring +except NameError: + string_type = str + + +class ValidationUtil: + @staticmethod + def check_multiple_not_none(obj, vars_to_check): + for param in vars_to_check: + ValidationUtil.check_not_none(obj, param) + + @staticmethod + def check_not_none(obj, param): + if getattr(obj, param) is None: + raise ValueError("Expected %s to be not None" % (param,)) + + @staticmethod + def check_multiple_is_string(obj, vars_to_check): + for param in vars_to_check: + ValidationUtil.check_is_string(obj, param) + + @staticmethod + def check_is_string(obj, param): + param_value = getattr(obj, param) + if param_value is not None and not isinstance(param_value, string_type): + raise TypeError("Expected %s to be a string" % (param,)) + + @staticmethod + def check_kafka_errors(errors): + if not isinstance(errors, list): + raise TypeError("errors should be None or a list") + for error in errors: + if not isinstance(error, KafkaError): + raise TypeError("Expected list of KafkaError") + + @staticmethod + def check_kafka_error(error): + if not isinstance(error, KafkaError): + raise TypeError("Expected error to be a KafkaError") diff --git a/src/confluent_kafka/admin/__init__.py b/src/confluent_kafka/admin/__init__.py index 11f65062d..c608ee906 100644 --- a/src/confluent_kafka/admin/__init__.py +++ b/src/confluent_kafka/admin/__init__.py @@ -15,26 +15,44 @@ """ Kafka admin client: create, view, alter, and delete topics and resources. """ -from concurrent.futures import Future -from typing import Callable, Dict, List, Optional, Sequence, Tuple, Type, TypeVar - -from pytest import Config +import warnings +import concurrent.futures # Unused imports are keeped to be accessible using this public module from ._config import (ConfigSource, # noqa: F401 ConfigEntry, - ConfigResource) + ConfigResource, + AlterConfigOpType) from ._resource import (ResourceType, # noqa: F401 ResourcePatternType) from ._acl import (AclOperation, # noqa: F401 AclPermissionType, AclBinding, AclBindingFilter) +from ._metadata import (BrokerMetadata, # noqa: F401 + ClusterMetadata, + GroupMember, + GroupMetadata, + PartitionMetadata, + TopicMetadata) +from ._group import (ConsumerGroupListing, # noqa: F401 + ListConsumerGroupsResult, + ConsumerGroupDescription, + MemberAssignment, + MemberDescription) +from ._scram import (UserScramCredentialAlteration, # noqa: F401 + UserScramCredentialUpsertion, + UserScramCredentialDeletion, + ScramCredentialInfo, + ScramMechanism, + UserScramCredentialsDescription) + from ..cimpl import (KafkaException, # noqa: F401 KafkaError, _AdminClientImpl, NewTopic, NewPartitions, + TopicPartition as _TopicPartition, CONFIG_SOURCE_UNKNOWN_CONFIG, CONFIG_SOURCE_DYNAMIC_TOPIC_CONFIG, CONFIG_SOURCE_DYNAMIC_BROKER_CONFIG, @@ -45,7 +63,20 @@ RESOURCE_ANY, RESOURCE_TOPIC, RESOURCE_GROUP, - RESOURCE_BROKER) + RESOURCE_BROKER, + OFFSET_INVALID) + +from confluent_kafka import ConsumerGroupTopicPartitions \ + as _ConsumerGroupTopicPartitions + +from confluent_kafka import ConsumerGroupState \ + as _ConsumerGroupState + + +try: + string_type = basestring +except NameError: + string_type = str class AdminClient(_AdminClientImpl): @@ -136,7 +167,62 @@ def _make_resource_result(f: Future, futmap: Dict[ConfigResource, Future]) -> No _acl_type = TypeVar("_acl_type", bound=AclBinding) @staticmethod - def _make_acls_result(f: Future, futmap: Dict[_acl_type, Future]) -> None: + def _make_list_consumer_groups_result(f, futmap): + pass + + @staticmethod + def _make_consumer_groups_result(f, futmap): + """ + Map per-group results to per-group futures in futmap. + """ + try: + + results = f.result() + futmap_values = list(futmap.values()) + len_results = len(results) + len_futures = len(futmap_values) + if len_results != len_futures: + raise RuntimeError( + "Results length {} is different from future-map length {}".format(len_results, len_futures)) + for i, result in enumerate(results): + fut = futmap_values[i] + if isinstance(result, KafkaError): + fut.set_exception(KafkaException(result)) + else: + fut.set_result(result) + except Exception as e: + # Request-level exception, raise the same for all groups + for _, fut in futmap.items(): + fut.set_exception(e) + + @staticmethod + def _make_consumer_group_offsets_result(f, futmap): + """ + Map per-group results to per-group futures in futmap. + The result value of each (successful) future is ConsumerGroupTopicPartitions. + """ + try: + + results = f.result() + futmap_values = list(futmap.values()) + len_results = len(results) + len_futures = len(futmap_values) + if len_results != len_futures: + raise RuntimeError( + "Results length {} is different from future-map length {}".format(len_results, len_futures)) + for i, result in enumerate(results): + fut = futmap_values[i] + if isinstance(result, KafkaError): + fut.set_exception(KafkaException(result)) + else: + fut.set_result(result) + except Exception as e: + # Request-level exception, raise the same for all groups + for _, fut in futmap.items(): + fut.set_exception(e) + + @staticmethod + def _make_acls_result(f, futmap): """ Map create ACL binding results to corresponding futures in futmap. For create_acls the result value of each (successful) future is None. @@ -162,8 +248,30 @@ def _make_acls_result(f: Future, futmap: Dict[_acl_type, Future]) -> None: fut.set_exception(e) @staticmethod - def _create_future() -> Future: - f: Future = Future() + def _make_user_scram_credentials_result(f, futmap): + try: + results = f.result() + len_results = len(results) + len_futures = len(futmap) + if len(results) != len_futures: + raise RuntimeError( + f"Results length {len_results} is different from future-map length {len_futures}") + for username, value in results.items(): + fut = futmap.get(username, None) + if fut is None: + raise RuntimeError( + f"username {username} not found in future-map: {futmap}") + if isinstance(value, KafkaError): + fut.set_exception(KafkaException(value)) + else: + fut.set_result(value) + except Exception as e: + for _, fut in futmap.items(): + fut.set_exception(e) + + @staticmethod + def _create_future(): + f = concurrent.futures.Future() if not f.set_running_or_notify_cancel(): raise RuntimeError("Future was cancelled prematurely") return f @@ -175,6 +283,8 @@ def _make_futures(futmap_keys: List[_futures_map_key], class_check: Optional[Typ """ Create futures and a futuremap for the keys in futmap_keys, and create a request-level future to be bassed to the C API. + + FIXME: use _make_futures_v2 with TypeError in next major release. """ futmap = {} for key in futmap_keys: @@ -191,10 +301,161 @@ def _make_futures(futmap_keys: List[_futures_map_key], class_check: Optional[Typ return f, futmap @staticmethod - def _has_duplicates(items: Sequence) -> bool: + def _make_futures_v2(futmap_keys, class_check, make_result_fn): + """ + Create futures and a futuremap for the keys in futmap_keys, + and create a request-level future to be bassed to the C API. + """ + futmap = {} + for key in futmap_keys: + if class_check is not None and not isinstance(key, class_check): + raise TypeError("Expected list of {}".format(repr(class_check))) + futmap[key] = AdminClient._create_future() + + # Create an internal future for the entire request, + # this future will trigger _make_..._result() and set result/exception + # per topic,future in futmap. + f = AdminClient._create_future() + f.add_done_callback(lambda f: make_result_fn(f, futmap)) + + return f, futmap + + @staticmethod + def _has_duplicates(items): return len(set(items)) != len(items) - def create_topics(self, new_topics: List[NewTopic], **kwargs: object) -> Dict[str, Future]: # type: ignore[override] + @staticmethod + def _check_list_consumer_group_offsets_request(request): + if request is None: + raise TypeError("request cannot be None") + if not isinstance(request, list): + raise TypeError("request must be a list") + if len(request) != 1: + raise ValueError("Currently we support listing offsets for a single consumer group only") + for req in request: + if not isinstance(req, _ConsumerGroupTopicPartitions): + raise TypeError("Expected list of 'ConsumerGroupTopicPartitions'") + + if req.group_id is None: + raise TypeError("'group_id' cannot be None") + if not isinstance(req.group_id, string_type): + raise TypeError("'group_id' must be a string") + if not req.group_id: + raise ValueError("'group_id' cannot be empty") + + if req.topic_partitions is not None: + if not isinstance(req.topic_partitions, list): + raise TypeError("'topic_partitions' must be a list or None") + if len(req.topic_partitions) == 0: + raise ValueError("'topic_partitions' cannot be empty") + for topic_partition in req.topic_partitions: + if topic_partition is None: + raise ValueError("Element of 'topic_partitions' cannot be None") + if not isinstance(topic_partition, _TopicPartition): + raise TypeError("Element of 'topic_partitions' must be of type TopicPartition") + if topic_partition.topic is None: + raise TypeError("Element of 'topic_partitions' must not have 'topic' attribute as None") + if not topic_partition.topic: + raise ValueError("Element of 'topic_partitions' must not have 'topic' attribute as Empty") + if topic_partition.partition < 0: + raise ValueError("Element of 'topic_partitions' must not have negative 'partition' value") + if topic_partition.offset != OFFSET_INVALID: + print(topic_partition.offset) + raise ValueError("Element of 'topic_partitions' must not have 'offset' value") + + @staticmethod + def _check_alter_consumer_group_offsets_request(request): + if request is None: + raise TypeError("request cannot be None") + if not isinstance(request, list): + raise TypeError("request must be a list") + if len(request) != 1: + raise ValueError("Currently we support altering offsets for a single consumer group only") + for req in request: + if not isinstance(req, _ConsumerGroupTopicPartitions): + raise TypeError("Expected list of 'ConsumerGroupTopicPartitions'") + if req.group_id is None: + raise TypeError("'group_id' cannot be None") + if not isinstance(req.group_id, string_type): + raise TypeError("'group_id' must be a string") + if not req.group_id: + raise ValueError("'group_id' cannot be empty") + if req.topic_partitions is None: + raise ValueError("'topic_partitions' cannot be null") + if not isinstance(req.topic_partitions, list): + raise TypeError("'topic_partitions' must be a list") + if len(req.topic_partitions) == 0: + raise ValueError("'topic_partitions' cannot be empty") + for topic_partition in req.topic_partitions: + if topic_partition is None: + raise ValueError("Element of 'topic_partitions' cannot be None") + if not isinstance(topic_partition, _TopicPartition): + raise TypeError("Element of 'topic_partitions' must be of type TopicPartition") + if topic_partition.topic is None: + raise TypeError("Element of 'topic_partitions' must not have 'topic' attribute as None") + if not topic_partition.topic: + raise ValueError("Element of 'topic_partitions' must not have 'topic' attribute as Empty") + if topic_partition.partition < 0: + raise ValueError( + "Element of 'topic_partitions' must not have negative value for 'partition' field") + if topic_partition.offset < 0: + raise ValueError( + "Element of 'topic_partitions' must not have negative value for 'offset' field") + + @staticmethod + def _check_describe_user_scram_credentials_request(users): + if not isinstance(users, list): + raise TypeError("Expected input to be list of String") + for user in users: + if not isinstance(user, string_type): + raise TypeError("Each value should be a string") + if not user: + raise ValueError("'user' cannot be empty") + + @staticmethod + def _check_alter_user_scram_credentials_request(alterations): + if not isinstance(alterations, list): + raise TypeError("Expected input to be list") + if len(alterations) == 0: + raise ValueError("Expected at least one alteration") + for alteration in alterations: + if not isinstance(alteration, UserScramCredentialAlteration): + raise TypeError("Expected each element of list to be subclass of UserScramCredentialAlteration") + if alteration.user is None: + raise TypeError("'user' cannot be None") + if not isinstance(alteration.user, string_type): + raise TypeError("'user' must be a string") + if not alteration.user: + raise ValueError("'user' cannot be empty") + + if isinstance(alteration, UserScramCredentialUpsertion): + if alteration.password is None: + raise TypeError("'password' cannot be None") + if not isinstance(alteration.password, bytes): + raise TypeError("'password' must be bytes") + if not alteration.password: + raise ValueError("'password' cannot be empty") + + if alteration.salt is not None and not alteration.salt: + raise ValueError("'salt' can be None but cannot be empty") + if alteration.salt and not isinstance(alteration.salt, bytes): + raise TypeError("'salt' must be bytes") + + if not isinstance(alteration.scram_credential_info, ScramCredentialInfo): + raise TypeError("Expected credential_info to be ScramCredentialInfo Type") + if alteration.scram_credential_info.iterations < 1: + raise ValueError("Iterations should be positive") + if not isinstance(alteration.scram_credential_info.mechanism, ScramMechanism): + raise TypeError("Expected the mechanism to be ScramMechanism Type") + elif isinstance(alteration, UserScramCredentialDeletion): + if not isinstance(alteration.mechanism, ScramMechanism): + raise TypeError("Expected the mechanism to be ScramMechanism Type") + else: + raise TypeError("Expected each element of list 'alterations' " + + "to be either a UserScramCredentialUpsertion or a " + + "UserScramCredentialDeletion") + + def create_topics(self, new_topics, **kwargs): """ Create one or more new topics. @@ -333,6 +594,8 @@ def describe_configs(self, resources: List[ConfigResource], **kwargs: object) -> def alter_configs(self, resources: List[ConfigResource], **kwargs: object) -> Dict[ConfigResource, Future]: # type: ignore[override] """ + .. deprecated:: 2.2.0 + Update configuration properties for the specified resources. Updates are not transactional so they may succeed for a subset of the provided resources while the others fail. @@ -358,14 +621,17 @@ def alter_configs(self, resources: List[ConfigResource], **kwargs: object) -> Di without altering the configuration. Default: False :returns: A dict of futures for each resource, keyed by the ConfigResource. - The future result() method returns None. + The future result() method returns None or throws a KafkaException. :rtype: dict() :raises KafkaException: Operation failed locally or on broker. - :raises TypeException: Invalid input. - :raises ValueException: Invalid input. + :raises TypeError: Invalid type. + :raises ValueError: Invalid value. """ + warnings.warn( + "alter_configs has been deprecated. Use incremental_alter_configs instead.", + category=DeprecationWarning, stacklevel=2) f, futmap = AdminClient._make_futures(resources, ConfigResource, AdminClient._make_resource_result) @@ -374,7 +640,40 @@ def alter_configs(self, resources: List[ConfigResource], **kwargs: object) -> Di return futmap - def create_acls(self, acls: List[AclBinding], **kwargs: object) -> Dict[AclBinding, Future]: # type: ignore[override] + def incremental_alter_configs(self, resources, **kwargs): + """ + Update configuration properties for the specified resources. + Updates are incremental, i.e only the values mentioned are changed + and rest remain as is. + + :param list(ConfigResource) resources: Resources to update configuration of. + :param float request_timeout: The overall request timeout in seconds, + including broker lookup, request transmission, operation time + on broker, and response. Default: `socket.timeout.ms*1000.0`. + :param bool validate_only: If true, the request is validated only, + without altering the configuration. Default: False + :param int broker: Broker id to send the request to. When + altering broker configurations, it's ignored because + the request needs to go to that broker only. + Default: controller broker. + + :returns: A dict of futures for each resource, keyed by the ConfigResource. + The future result() method returns None or throws a KafkaException. + + :rtype: dict() + + :raises KafkaException: Operation failed locally or on broker. + :raises TypeError: Invalid type. + :raises ValueError: Invalid value. + """ + f, futmap = AdminClient._make_futures_v2(resources, ConfigResource, + AdminClient._make_resource_result) + + super(AdminClient, self).incremental_alter_configs(resources, f, **kwargs) + + return futmap + + def create_acls(self, acls, **kwargs): """ Create one or more ACL bindings. @@ -477,169 +776,245 @@ def delete_acls(self, acl_binding_filters: List[AclBindingFilter], **kwargs: obj return futmap + def list_consumer_groups(self, **kwargs): + """ + List consumer groups. -class ClusterMetadata (object): - """ - Provides information about the Kafka cluster, brokers, and topics. - Returned by list_topics(). + :param float request_timeout: The overall request timeout in seconds, + including broker lookup, request transmission, operation time + on broker, and response. Default: `socket.timeout.ms*1000.0` + :param set(ConsumerGroupState) states: only list consumer groups which are currently in + these states. - This class is typically not user instantiated. - """ + :returns: a future. Result method of the future returns :class:`ListConsumerGroupsResult`. - def __init__(self) -> None: - self.cluster_id = None - """Cluster id string, if supported by the broker, else None.""" - self.controller_id: int = -1 - """Current controller broker id, or -1.""" - self.brokers: Dict[object, object] = {} - """Map of brokers indexed by the broker id (int). Value is a BrokerMetadata object.""" - self.topics: Dict[object, object] = {} - """Map of topics indexed by the topic name. Value is a TopicMetadata object.""" - self.orig_broker_id: int = -1 - """The broker this metadata originated from.""" - self.orig_broker_name = None - """The broker name/address this metadata originated from.""" + :rtype: future - def __repr__(self) -> str: - return "ClusterMetadata({})".format(self.cluster_id) + :raises KafkaException: Operation failed locally or on broker. + :raises TypeException: Invalid input. + :raises ValueException: Invalid input. + """ + if "states" in kwargs: + states = kwargs["states"] + if states is not None: + if not isinstance(states, set): + raise TypeError("'states' must be a set") + for state in states: + if not isinstance(state, _ConsumerGroupState): + raise TypeError("All elements of states must be of type ConsumerGroupState") + kwargs["states_int"] = [state.value for state in states] + kwargs.pop("states") - def __str__(self) -> str: - return str(self.cluster_id) + f, _ = AdminClient._make_futures([], None, AdminClient._make_list_consumer_groups_result) + super(AdminClient, self).list_consumer_groups(f, **kwargs) -class BrokerMetadata (object): - """ - Provides information about a Kafka broker. + return f - This class is typically not user instantiated. - """ + def describe_consumer_groups(self, group_ids, **kwargs): + """ + Describe consumer groups. - def __init__(self) -> None: - self.id = -1 - """Broker id""" - self.host = None - """Broker hostname""" - self.port = -1 - """Broker port""" + :param list(str) group_ids: List of group_ids which need to be described. + :param float request_timeout: The overall request timeout in seconds, + including broker lookup, request transmission, operation time + on broker, and response. Default: `socket.timeout.ms*1000.0` - def __repr__(self) -> str: - return "BrokerMetadata({}, {}:{})".format(self.id, self.host, self.port) + :returns: A dict of futures for each group, keyed by the group_id. + The future result() method returns :class:`ConsumerGroupDescription`. - def __str__(self) -> str: - return "{}:{}/{}".format(self.host, self.port, self.id) + :rtype: dict[str, future] + :raises KafkaException: Operation failed locally or on broker. + :raises TypeException: Invalid input. + :raises ValueException: Invalid input. + """ -class TopicMetadata (object): - """ - Provides information about a Kafka topic. + if not isinstance(group_ids, list): + raise TypeError("Expected input to be list of group ids to be described") - This class is typically not user instantiated. - """ - # The dash in "-topic" and "-error" is needed to circumvent a - # Sphinx issue where it tries to reference the same instance variable - # on other classes which raises a warning/error. + if len(group_ids) == 0: + raise ValueError("Expected at least one group to be described") - def __init__(self) -> None: - self.topic: Optional[str] = None - """Topic name""" - self.partitions: Dict[object, object] = {} - """Map of partitions indexed by partition id. Value is a PartitionMetadata object.""" - self.error: Optional[KafkaError] = None - """Topic error, or None. Value is a KafkaError object.""" + f, futmap = AdminClient._make_futures(group_ids, None, + AdminClient._make_consumer_groups_result) - def __repr__(self) -> str: - if self.error is not None: - return "TopicMetadata({}, {} partitions, {})".format(self.topic, len(self.partitions), self.error) - else: - return "TopicMetadata({}, {} partitions)".format(self.topic, len(self.partitions)) + super(AdminClient, self).describe_consumer_groups(group_ids, f, **kwargs) - def __str__(self) -> str: - return str(self.topic) + return futmap + def delete_consumer_groups(self, group_ids, **kwargs): + """ + Delete the given consumer groups. -class PartitionMetadata (object): - """ - Provides information about a Kafka partition. + :param list(str) group_ids: List of group_ids which need to be deleted. + :param float request_timeout: The overall request timeout in seconds, + including broker lookup, request transmission, operation time + on broker, and response. Default: `socket.timeout.ms*1000.0` - This class is typically not user instantiated. + :returns: A dict of futures for each group, keyed by the group_id. + The future result() method returns None. - :warning: Depending on cluster state the broker ids referenced in - leader, replicas and ISRs may temporarily not be reported - in ClusterMetadata.brokers. Always check the availability - of a broker id in the brokers dict. - """ + :rtype: dict[str, future] - def __init__(self) -> None: - self.id: int = -1 - """Partition id.""" - self.leader: int = -1 - """Current leader broker for this partition, or -1.""" - self.replicas: List[object] = [] - """List of replica broker ids for this partition.""" - self.isrs: List[object] = [] - """List of in-sync-replica broker ids for this partition.""" - self.error = None - """Partition error, or None. Value is a KafkaError object.""" - - def __repr__(self) -> str: - if self.error is not None: - return "PartitionMetadata({}, {})".format(self.id, self.error) - else: - return "PartitionMetadata({})".format(self.id) - - def __str__(self) -> str: - return "{}".format(self.id) - - -class GroupMember(object): - """Provides information about a group member. - - For more information on the metadata format, refer to: - `A Guide To The Kafka Protocol `_. - - This class is typically not user instantiated. - """ # noqa: E501 - - def __init__(self) -> None: - self.id = None - """Member id (generated by broker).""" - self.client_id = None - """Client id.""" - self.client_host = None - """Client hostname.""" - self.metadata = None - """Member metadata(binary), format depends on protocol type.""" - self.assignment = None - """Member assignment(binary), format depends on protocol type.""" - - -class GroupMetadata(object): - """GroupMetadata provides information about a Kafka consumer group - - This class is typically not user instantiated. - """ + :raises KafkaException: Operation failed locally or on broker. + :raises TypeException: Invalid input. + :raises ValueException: Invalid input. + """ + if not isinstance(group_ids, list): + raise TypeError("Expected input to be list of group ids to be deleted") + + if len(group_ids) == 0: + raise ValueError("Expected at least one group to be deleted") + + f, futmap = AdminClient._make_futures(group_ids, string_type, AdminClient._make_consumer_groups_result) + + super(AdminClient, self).delete_consumer_groups(group_ids, f, **kwargs) + + return futmap + + def list_consumer_group_offsets(self, list_consumer_group_offsets_request, **kwargs): + """ + List offset information for the consumer group and (optional) topic partition provided in the request. + + :note: Currently, the API supports only a single group. + + :param list(ConsumerGroupTopicPartitions) list_consumer_group_offsets_request: List of + :class:`ConsumerGroupTopicPartitions` which consist of group name and topic + partition information for which offset detail is expected. If only group name is + provided, then offset information of all the topic and partition associated with + that group is returned. + :param bool require_stable: If True, fetches stable offsets. Default: False + :param float request_timeout: The overall request timeout in seconds, + including broker lookup, request transmission, operation time + on broker, and response. Default: `socket.timeout.ms*1000.0` + + :returns: A dict of futures for each group, keyed by the group id. + The future result() method returns :class:`ConsumerGroupTopicPartitions`. + + :rtype: dict[str, future] + + :raises KafkaException: Operation failed locally or on broker. + :raises TypeException: Invalid input. + :raises ValueException: Invalid input. + """ - def __init__(self) -> None: - self.broker = None - """Originating broker metadata.""" - self.id: Optional[str] = None - """Group name.""" - self.error = None - """Broker-originated error, or None. Value is a KafkaError object.""" - self.state = None - """Group state.""" - self.protocol_type = None - """Group protocol type.""" - self.protocol = None - """Group protocol.""" - self.members: List[object] = [] - """Group members.""" - - def __repr__(self) -> str: - if self.error is not None: - return "GroupMetadata({}, {})".format(self.id, self.error) - else: - return "GroupMetadata({})".format(self.id) - - def __str__(self) -> str: - return str(self.id) + AdminClient._check_list_consumer_group_offsets_request(list_consumer_group_offsets_request) + + f, futmap = AdminClient._make_futures([request.group_id for request in list_consumer_group_offsets_request], + string_type, + AdminClient._make_consumer_group_offsets_result) + + super(AdminClient, self).list_consumer_group_offsets(list_consumer_group_offsets_request, f, **kwargs) + + return futmap + + def alter_consumer_group_offsets(self, alter_consumer_group_offsets_request, **kwargs): + """ + Alter offset for the consumer group and topic partition provided in the request. + + :note: Currently, the API supports only a single group. + + :param list(ConsumerGroupTopicPartitions) alter_consumer_group_offsets_request: List of + :class:`ConsumerGroupTopicPartitions` which consist of group name and topic + partition; and corresponding offset to be updated. + :param float request_timeout: The overall request timeout in seconds, + including broker lookup, request transmission, operation time + on broker, and response. Default: `socket.timeout.ms*1000.0` + + :returns: A dict of futures for each group, keyed by the group id. + The future result() method returns :class:`ConsumerGroupTopicPartitions`. + + :rtype: dict[ConsumerGroupTopicPartitions, future] + + :raises KafkaException: Operation failed locally or on broker. + :raises TypeException: Invalid input. + :raises ValueException: Invalid input. + """ + + AdminClient._check_alter_consumer_group_offsets_request(alter_consumer_group_offsets_request) + + f, futmap = AdminClient._make_futures([request.group_id for request in alter_consumer_group_offsets_request], + string_type, + AdminClient._make_consumer_group_offsets_result) + + super(AdminClient, self).alter_consumer_group_offsets(alter_consumer_group_offsets_request, f, **kwargs) + + return futmap + + def set_sasl_credentials(self, username, password): + """ + Sets the SASL credentials used for this client. + These credentials will overwrite the old ones, and will be used the + next time the client needs to authenticate. + This method will not disconnect existing broker connections that + have been established with the old credentials. + This method is applicable only to SASL PLAIN and SCRAM mechanisms. + + :param str username: The username to set. + :param str password: The password to set. + + :rtype: None + + :raises KafkaException: Operation failed locally or on broker. + :raises TypeException: Invalid input. + """ + super(AdminClient, self).set_sasl_credentials(username, password) + + def describe_user_scram_credentials(self, users, **kwargs): + """ + Describe user SASL/SCRAM credentials. + + :param list(str) users: List of user names to describe. + Duplicate users aren't allowed. + :param float request_timeout: The overall request timeout in seconds, + including broker lookup, request transmission, operation time + on broker, and response. Default: `socket.timeout.ms*1000.0` + + :returns: A dict of futures keyed by user name. + The future result() method returns the + :class:`UserScramCredentialsDescription` or + raises KafkaException + + :rtype: dict[str, future] + + :raises TypeError: Invalid input type. + :raises ValueError: Invalid input value. + """ + AdminClient._check_describe_user_scram_credentials_request(users) + + f, futmap = AdminClient._make_futures_v2(users, None, + AdminClient._make_user_scram_credentials_result) + + super(AdminClient, self).describe_user_scram_credentials(users, f, **kwargs) + + return futmap + + def alter_user_scram_credentials(self, alterations, **kwargs): + """ + Alter user SASL/SCRAM credentials. + + :param list(UserScramCredentialAlteration) alterations: List of + :class:`UserScramCredentialAlteration` to apply. + The pair (user, mechanism) must be unique among alterations. + :param float request_timeout: The overall request timeout in seconds, + including broker lookup, request transmission, operation time + on broker, and response. Default: `socket.timeout.ms*1000.0` + + :returns: A dict of futures keyed by user name. + The future result() method returns None or + raises KafkaException + + :rtype: dict[str, future] + + :raises TypeError: Invalid input type. + :raises ValueError: Invalid input value. + """ + AdminClient._check_alter_user_scram_credentials_request(alterations) + + f, futmap = AdminClient._make_futures_v2(set([alteration.user for alteration in alterations]), None, + AdminClient._make_user_scram_credentials_result) + + super(AdminClient, self).alter_user_scram_credentials(alterations, f, **kwargs) + + return futmap diff --git a/src/confluent_kafka/admin/_acl.py b/src/confluent_kafka/admin/_acl.py index 461c3fc2e..6d8011ebb 100644 --- a/src/confluent_kafka/admin/_acl.py +++ b/src/confluent_kafka/admin/_acl.py @@ -17,6 +17,7 @@ from typing import Dict, List, Tuple, Type, TypeVar, cast from .. import cimpl as _cimpl from ._resource import ResourceType, ResourcePatternType +from .._util import ValidationUtil, ConversionUtil import six @@ -107,40 +108,14 @@ def __init__(self, restype: ResourceType, name: str, self.operation_int = int(self.operation.value) self.permission_type_int = int(self.permission_type.value) - def _check_not_none(self, vars_to_check: List[str]) -> None: - for param in vars_to_check: - if getattr(self, param) is None: - raise ValueError("Expected %s to be not None" % (param,)) - - def _check_is_string(self, vars_to_check: List[str]) -> None: - for param in vars_to_check: - param_value = getattr(self, param) - if param_value is not None and not isinstance(param_value, string_type): - raise TypeError("Expected %s to be a string" % (param,)) - - # FIXME: Should really return the enum_clazz, but not sure how! - def _convert_to_enum(self, val: object, enum_clazz: Type[Enum]) -> object: - if type(val) == str: - # Allow it to be specified as case-insensitive string, for convenience. - try: - val = enum_clazz[val.upper()] - except KeyError: - raise ValueError("Unknown value \"%s\": should be a %s" % (val, enum_clazz.__name__)) - - elif type(val) == int: - # The C-code passes restype as an int, convert to enum. - val = enum_clazz(val) - - elif type(val) != enum_clazz: - raise TypeError("Unknown value \"%s\": should be a %s" % (val, enum_clazz.__name__)) - - return val - - def _convert_enums(self) -> None: - self.restype = cast(ResourceType, self._convert_to_enum(self.restype, ResourceType)) - self.resource_pattern_type = cast(ResourcePatternType, self._convert_to_enum(self.resource_pattern_type, ResourcePatternType)) - self.operation = cast(AclOperation, self._convert_to_enum(self.operation, AclOperation)) - self.permission_type = cast(AclPermissionType, self._convert_to_enum(self.permission_type, AclPermissionType)) + def _convert_enums(self): + self.restype = ConversionUtil.convert_to_enum(self.restype, ResourceType) + self.resource_pattern_type = ConversionUtil.convert_to_enum( + self.resource_pattern_type, ResourcePatternType) + self.operation = ConversionUtil.convert_to_enum( + self.operation, AclOperation) + self.permission_type = ConversionUtil.convert_to_enum( + self.permission_type, AclPermissionType) def _check_forbidden_enums(self, forbidden_enums: Dict[str, List]) -> None: for k, v in forbidden_enums.items(): @@ -168,8 +143,8 @@ def _convert_args(self) -> None: not_none_args = self._not_none_args() string_args = self._string_args() forbidden_enums = self._forbidden_enums() - self._check_not_none(not_none_args) - self._check_is_string(string_args) + ValidationUtil.check_multiple_not_none(self, not_none_args) + ValidationUtil.check_multiple_is_string(self, string_args) self._convert_enums() self._check_forbidden_enums(forbidden_enums) diff --git a/src/confluent_kafka/admin/_config.py b/src/confluent_kafka/admin/_config.py index 60eeee25d..93793b46e 100644 --- a/src/confluent_kafka/admin/_config.py +++ b/src/confluent_kafka/admin/_config.py @@ -19,6 +19,32 @@ from ._resource import ResourceType +class AlterConfigOpType(Enum): + """ + Set of incremental operations that can be used with + incremental alter configs. + """ + + #: Set the value of the configuration entry. + SET = _cimpl.ALTER_CONFIG_OP_TYPE_SET + + #: Revert the configuration entry + #: to the default value (possibly null). + DELETE = _cimpl.ALTER_CONFIG_OP_TYPE_DELETE + + #: (For list-type configuration entries only.) + #: Add the specified values + #: to the current list of values + #: of the configuration entry. + APPEND = _cimpl.ALTER_CONFIG_OP_TYPE_APPEND + + #: (For list-type configuration entries only.) + #: Removes the specified values + #: from the current list of values + #: of the configuration entry. + SUBTRACT = _cimpl.ALTER_CONFIG_OP_TYPE_SUBTRACT + + class ConfigSource(Enum): """ Enumerates the different sources of configuration properties. @@ -47,7 +73,8 @@ def __init__(self, name: str, value: str, is_default: bool=False, is_sensitive: bool=False, is_synonym: bool=False, - synonyms: List[str]=[]): + synonyms: List[str]=[], + incremental_operation=None): """ This class is typically not user instantiated. """ @@ -56,7 +83,9 @@ def __init__(self, name: str, value: str, self.name = name """Configuration property name.""" self.value = value - """Configuration value (or None if not set or is_sensitive==True).""" + """Configuration value (or None if not set or is_sensitive==True. + Ignored when altering configurations incrementally + if incremental_operation is DELETE).""" self.source = source """Configuration source.""" self.is_read_only = bool(is_read_only) @@ -72,6 +101,8 @@ def __init__(self, name: str, value: str, """Indicates whether the configuration property is a synonym for the parent configuration entry.""" self.synonyms = synonyms """A list of synonyms (ConfigEntry) and alternate sources for this configuration property.""" + self.incremental_operation = incremental_operation + """The incremental operation (AlterConfigOpType) to use in incremental_alter_configs.""" def __repr__(self) -> str: return "ConfigEntry(%s=\"%s\")" % (self.name, self.value) @@ -100,12 +131,13 @@ class ConfigResource(object): Type = ResourceType def __init__(self, restype: Union[str, int, ResourceType], name: str, - set_config: Optional[Dict[str, str]]=None, described_configs: Optional[object]=None, error: Optional[object]=None): + set_config: Optional[Dict[str, str]]=None, described_configs: Optional[object]=None, error: Optional[object]=None, incremental_configs=None): """ :param ConfigResource.Type restype: Resource type. :param str name: The resource name, which depends on restype. For RESOURCE_BROKER, the resource name is the broker id. :param dict set_config: The configuration to set/overwrite. Dictionary of str, str. + :param list(ConfigEntry) incremental_configs: The configuration entries to alter incrementally. :param dict described_configs: For internal use only. :param KafkaError error: For internal use only. """ @@ -136,6 +168,8 @@ def __init__(self, restype: Union[str, int, ResourceType], name: str, else: self.set_config_dict = dict() + self.incremental_configs = list(incremental_configs or []) + self.configs = described_configs self.error = error @@ -182,3 +216,12 @@ def set_config(self, name: str, value: str, overwrite: bool=True) -> None: if not overwrite and name in self.set_config_dict: return self.set_config_dict[name] = value + + def add_incremental_config(self, config_entry): + """ + Add a ConfigEntry for incremental alter configs, using the + configured incremental_operation. + + :param ConfigEntry config_entry: config entry to incrementally alter. + """ + self.incremental_configs.append(config_entry) diff --git a/src/confluent_kafka/admin/_group.py b/src/confluent_kafka/admin/_group.py new file mode 100644 index 000000000..1c8d5e6fe --- /dev/null +++ b/src/confluent_kafka/admin/_group.py @@ -0,0 +1,128 @@ +# Copyright 2022 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .._util import ConversionUtil +from .._model import ConsumerGroupState + + +class ConsumerGroupListing: + """ + Represents consumer group listing information for a group used in list consumer group operation. + Used by :class:`ListConsumerGroupsResult`. + + Parameters + ---------- + group_id : str + The consumer group id. + is_simple_consumer_group : bool + Whether a consumer group is simple or not. + state : ConsumerGroupState + Current state of the consumer group. + """ + def __init__(self, group_id, is_simple_consumer_group, state=None): + self.group_id = group_id + self.is_simple_consumer_group = is_simple_consumer_group + if state is not None: + self.state = ConversionUtil.convert_to_enum(state, ConsumerGroupState) + + +class ListConsumerGroupsResult: + """ + Represents result of List Consumer Group operation. + Used by :meth:`AdminClient.list_consumer_groups`. + + Parameters + ---------- + valid : list(ConsumerGroupListing) + List of successful consumer group listing responses. + errors : list(KafkaException) + List of errors encountered during the operation, if any. + """ + def __init__(self, valid=None, errors=None): + self.valid = valid + self.errors = errors + + +class MemberAssignment: + """ + Represents member assignment information. + Used by :class:`MemberDescription`. + + Parameters + ---------- + topic_partitions : list(TopicPartition) + The topic partitions assigned to a group member. + """ + def __init__(self, topic_partitions=[]): + self.topic_partitions = topic_partitions + if self.topic_partitions is None: + self.topic_partitions = [] + + +class MemberDescription: + """ + Represents member information. + Used by :class:`ConsumerGroupDescription`. + + Parameters + ---------- + member_id : str + The consumer id of the group member. + client_id : str + The client id of the group member. + host: str + The host where the group member is running. + assignment: MemberAssignment + The assignment of the group member + group_instance_id : str + The instance id of the group member. + """ + def __init__(self, member_id, client_id, host, assignment, group_instance_id=None): + self.member_id = member_id + self.client_id = client_id + self.host = host + self.assignment = assignment + self.group_instance_id = group_instance_id + + +class ConsumerGroupDescription: + """ + Represents consumer group description information for a group used in describe consumer group operation. + Used by :meth:`AdminClient.describe_consumer_groups`. + + Parameters + ---------- + group_id : str + The consumer group id. + is_simple_consumer_group : bool + Whether a consumer group is simple or not. + members: list(MemberDescription) + Description of the memebers of the consumer group. + partition_assignor: str + Partition assignor. + state : ConsumerGroupState + Current state of the consumer group. + coordinator: Node + Consumer group coordinator. + """ + def __init__(self, group_id, is_simple_consumer_group, members, partition_assignor, state, + coordinator): + self.group_id = group_id + self.is_simple_consumer_group = is_simple_consumer_group + self.members = members + self.partition_assignor = partition_assignor + if state is not None: + self.state = ConversionUtil.convert_to_enum(state, ConsumerGroupState) + self.coordinator = coordinator diff --git a/src/confluent_kafka/admin/_metadata.py b/src/confluent_kafka/admin/_metadata.py new file mode 100644 index 000000000..201e4534b --- /dev/null +++ b/src/confluent_kafka/admin/_metadata.py @@ -0,0 +1,179 @@ +# Copyright 2022 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class ClusterMetadata (object): + """ + Provides information about the Kafka cluster, brokers, and topics. + Returned by list_topics(). + + This class is typically not user instantiated. + """ + + def __init__(self): + self.cluster_id = None + """Cluster id string, if supported by the broker, else None.""" + self.controller_id = -1 + """Current controller broker id, or -1.""" + self.brokers = {} + """Map of brokers indexed by the broker id (int). Value is a BrokerMetadata object.""" + self.topics = {} + """Map of topics indexed by the topic name. Value is a TopicMetadata object.""" + self.orig_broker_id = -1 + """The broker this metadata originated from.""" + self.orig_broker_name = None + """The broker name/address this metadata originated from.""" + + def __repr__(self): + return "ClusterMetadata({})".format(self.cluster_id) + + def __str__(self): + return str(self.cluster_id) + + +class BrokerMetadata (object): + """ + Provides information about a Kafka broker. + + This class is typically not user instantiated. + """ + + def __init__(self): + self.id = -1 + """Broker id""" + self.host = None + """Broker hostname""" + self.port = -1 + """Broker port""" + + def __repr__(self): + return "BrokerMetadata({}, {}:{})".format(self.id, self.host, self.port) + + def __str__(self): + return "{}:{}/{}".format(self.host, self.port, self.id) + + +class TopicMetadata (object): + """ + Provides information about a Kafka topic. + + This class is typically not user instantiated. + """ + # The dash in "-topic" and "-error" is needed to circumvent a + # Sphinx issue where it tries to reference the same instance variable + # on other classes which raises a warning/error. + + def __init__(self): + self.topic = None + """Topic name""" + self.partitions = {} + """Map of partitions indexed by partition id. Value is a PartitionMetadata object.""" + self.error = None + """Topic error, or None. Value is a KafkaError object.""" + + def __repr__(self): + if self.error is not None: + return "TopicMetadata({}, {} partitions, {})".format(self.topic, len(self.partitions), self.error) + else: + return "TopicMetadata({}, {} partitions)".format(self.topic, len(self.partitions)) + + def __str__(self): + return self.topic + + +class PartitionMetadata (object): + """ + Provides information about a Kafka partition. + + This class is typically not user instantiated. + + :warning: Depending on cluster state the broker ids referenced in + leader, replicas and ISRs may temporarily not be reported + in ClusterMetadata.brokers. Always check the availability + of a broker id in the brokers dict. + """ + + def __init__(self): + self.id = -1 + """Partition id.""" + self.leader = -1 + """Current leader broker for this partition, or -1.""" + self.replicas = [] + """List of replica broker ids for this partition.""" + self.isrs = [] + """List of in-sync-replica broker ids for this partition.""" + self.error = None + """Partition error, or None. Value is a KafkaError object.""" + + def __repr__(self): + if self.error is not None: + return "PartitionMetadata({}, {})".format(self.id, self.error) + else: + return "PartitionMetadata({})".format(self.id) + + def __str__(self): + return "{}".format(self.id) + + +class GroupMember(object): + """Provides information about a group member. + + For more information on the metadata format, refer to: + `A Guide To The Kafka Protocol `_. + + This class is typically not user instantiated. + """ # noqa: E501 + + def __init__(self,): + self.id = None + """Member id (generated by broker).""" + self.client_id = None + """Client id.""" + self.client_host = None + """Client hostname.""" + self.metadata = None + """Member metadata(binary), format depends on protocol type.""" + self.assignment = None + """Member assignment(binary), format depends on protocol type.""" + + +class GroupMetadata(object): + """GroupMetadata provides information about a Kafka consumer group + + This class is typically not user instantiated. + """ + + def __init__(self): + self.broker = None + """Originating broker metadata.""" + self.id = None + """Group name.""" + self.error = None + """Broker-originated error, or None. Value is a KafkaError object.""" + self.state = None + """Group state.""" + self.protocol_type = None + """Group protocol type.""" + self.protocol = None + """Group protocol.""" + self.members = [] + """Group members.""" + + def __repr__(self): + if self.error is not None: + return "GroupMetadata({}, {})".format(self.id, self.error) + else: + return "GroupMetadata({})".format(self.id) + + def __str__(self): + return self.id diff --git a/src/confluent_kafka/admin/_scram.py b/src/confluent_kafka/admin/_scram.py new file mode 100644 index 000000000..c20f55bbc --- /dev/null +++ b/src/confluent_kafka/admin/_scram.py @@ -0,0 +1,116 @@ +# Copyright 2023 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import cimpl + +from enum import Enum + + +class ScramMechanism(Enum): + """ + Enumerates SASL/SCRAM mechanisms. + """ + UNKNOWN = cimpl.SCRAM_MECHANISM_UNKNOWN #: Unknown SASL/SCRAM mechanism + SCRAM_SHA_256 = cimpl.SCRAM_MECHANISM_SHA_256 #: SCRAM-SHA-256 mechanism + SCRAM_SHA_512 = cimpl.SCRAM_MECHANISM_SHA_512 #: SCRAM-SHA-512 mechanism + + def __lt__(self, other): + if self.__class__ != other.__class__: + return NotImplemented + return self.value < other.value + + +class ScramCredentialInfo: + """ + Contains mechanism and iterations for a + SASL/SCRAM credential associated with a user. + + Parameters + ---------- + mechanism: ScramMechanism + SASL/SCRAM mechanism. + iterations: int + Positive number of iterations used when creating the credential. + """ + def __init__(self, mechanism, iterations): + self.mechanism = mechanism + self.iterations = iterations + + +class UserScramCredentialsDescription: + """ + Represent all SASL/SCRAM credentials + associated with a user that can be retrieved. + + Parameters + ---------- + user: str + The user name. + scram_credential_infos: list(ScramCredentialInfo) + SASL/SCRAM credential representations for the user. + """ + def __init__(self, user, scram_credential_infos): + self.user = user + self.scram_credential_infos = scram_credential_infos + + +class UserScramCredentialAlteration: + """ + Base class for SCRAM credential alterations. + + Parameters + ---------- + user: str + The user name. + """ + def __init__(self, user: str): + self.user = user + + +class UserScramCredentialUpsertion(UserScramCredentialAlteration): + """ + A request to update/insert a SASL/SCRAM credential for a user. + + Parameters + ---------- + user: str + The user name. + scram_credential_info: ScramCredentialInfo + The mechanism and iterations. + password: bytes + Password to HMAC before storage. + salt: bytes + Salt to use. Will be generated randomly if None. (optional) + """ + def __init__(self, user, scram_credential_info, password, salt=None): + super(UserScramCredentialUpsertion, self).__init__(user) + self.scram_credential_info = scram_credential_info + self.password = password + self.salt = salt + + +class UserScramCredentialDeletion(UserScramCredentialAlteration): + """ + A request to delete a SASL/SCRAM credential for a user. + + Parameters + ---------- + user: str + The user name. + mechanism: ScramMechanism + SASL/SCRAM mechanism. + """ + def __init__(self, user, mechanism): + super(UserScramCredentialDeletion, self).__init__(user) + self.mechanism = mechanism diff --git a/src/confluent_kafka/avro/__init__.py b/src/confluent_kafka/avro/__init__.py index 6d9cb214f..aaa970965 100644 --- a/src/confluent_kafka/avro/__init__.py +++ b/src/confluent_kafka/avro/__init__.py @@ -35,7 +35,7 @@ class AvroProducer(Producer): """ - .. deprecated:: 1.9.3 + .. deprecated:: 2.0.2 This class will be removed in a future version of the library. @@ -118,7 +118,7 @@ def produce(self, topic: str, **kwargs: object) -> None: class AvroConsumer(Consumer): """ - .. deprecated:: 1.9.3 + .. deprecated:: 2.0.2 This class will be removed in a future version of the library. diff --git a/src/confluent_kafka/avro/cached_schema_registry_client.py b/src/confluent_kafka/avro/cached_schema_registry_client.py index 22868025b..a2d571b97 100644 --- a/src/confluent_kafka/avro/cached_schema_registry_client.py +++ b/src/confluent_kafka/avro/cached_schema_registry_client.py @@ -23,6 +23,8 @@ from turtle import pos from typing import Dict, Optional, Sized, Tuple, TypeVar, Union, cast import warnings +import urllib3 +import json from collections import defaultdict from requests import Session, utils @@ -56,6 +58,7 @@ class CachedSchemaRegistryClient(object): Use CachedSchemaRegistryClient(dict: config) instead. Existing params ca_location, cert_location and key_location will be replaced with their librdkafka equivalents: `ssl.ca.location`, `ssl.certificate.location` and `ssl.key.location` respectively. + The support for password protected private key is via the Config only using 'ssl.key.password' field. Errors communicating to the server will result in a ClientError being raised. @@ -119,6 +122,9 @@ def __init__(self, url: Union[str, Dict[str, object]], max_schemas_per_subject: self.url = utils.urldefragauth(self.url) self._session = s + key_password = conf.pop('ssl.key.password', None) + self._is_key_password_provided = not key_password + self._https_session = self._make_https_session(s.cert[0], s.cert[1], ca_path, s.auth, key_password) self.auto_register_schemas = conf.pop("auto.register.schemas", True) @@ -138,6 +144,29 @@ def close(self) -> None: # Constructor exceptions may occur prior to _session being set. if hasattr(self, '_session'): self._session.close() + if hasattr(self, '_https_session'): + self._https_session.clear() + + @staticmethod + def _make_https_session(cert_location, key_location, ca_certs_path, auth, key_password): + https_session = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=ca_certs_path, + cert_file=cert_location, key_file=key_location, key_password=key_password) + https_session.auth = auth + return https_session + + def _send_https_session_request(self, url, method, headers, body): + request_headers = {'Accept': ACCEPT_HDR} + auth = self._https_session.auth + if body: + body = json.dumps(body).encode('UTF-8') + request_headers["Content-Length"] = str(len(body)) + request_headers["Content-Type"] = "application/vnd.schemaregistry.v1+json" + if auth[0] != '' and auth[1] != '': + request_headers.update(urllib3.make_headers(basic_auth=auth[0] + ":" + + auth[1])) + request_headers.update(headers) + response = self._https_session.request(method, url, headers=request_headers, body=body) + return response @staticmethod def _configure_basic_auth(url: str, conf: Dict) -> Tuple[str, str]: @@ -171,6 +200,13 @@ def _send_request(self, url: str, method: str='GET', body: Optional[Sized]=None, if method not in VALID_METHODS: raise ClientError("Method {} is invalid; valid methods include {}".format(method, VALID_METHODS)) + if url.startswith('https') and self._is_key_password_provided: + response = self._send_https_session_request(url, method, headers, body) + try: + return json.loads(response.data), response.status + except ValueError: + return response.content, response.status + _headers = {'Accept': ACCEPT_HDR} if body: _headers["Content-Length"] = str(len(body)) diff --git a/src/confluent_kafka/schema_registry/avro.py b/src/confluent_kafka/schema_registry/avro.py index 41a83b2e4..9031c0ffa 100644 --- a/src/confluent_kafka/schema_registry/avro.py +++ b/src/confluent_kafka/schema_registry/avro.py @@ -70,6 +70,24 @@ def _schema_loads(schema_str: str) -> Schema: return Schema(schema_str, schema_type='AVRO') +def _resolve_named_schema(schema, schema_registry_client, named_schemas=None): + """ + Resolves named schemas referenced by the provided schema recursively. + :param schema: Schema to resolve named schemas for. + :param schema_registry_client: SchemaRegistryClient to use for retrieval. + :param named_schemas: Dict of named schemas resolved recursively. + :return: named_schemas dict. + """ + if named_schemas is None: + named_schemas = {} + if schema.references is not None: + for ref in schema.references: + referenced_schema = schema_registry_client.get_version(ref.subject, ref.version) + _resolve_named_schema(referenced_schema.schema, schema_registry_client, named_schemas) + parse_schema(loads(referenced_schema.schema.schema_str), named_schemas=named_schemas) + return named_schemas + + class AvroSerializer(Serializer): """ Serializer that outputs Avro binary encoded data with Confluent Schema Registry framing. @@ -149,7 +167,7 @@ class AvroSerializer(Serializer): Args: schema_registry_client (SchemaRegistryClient): Schema Registry client instance. - schema_str (str): Avro `Schema Declaration. `_ + schema_str (str or Schema): Avro `Schema Declaration. `_ Accepts either a string or a `Schema`(Schema) instance. Note that string definitions cannot reference other schemas. For referencing other schemas, use a Schema instance. to_dict (callable, optional): Callable(object, SerializationContext) -> dict. Converts object to a dict. @@ -158,7 +176,7 @@ class AvroSerializer(Serializer): __slots__ = ['_hash', '_auto_register', '_normalize_schemas', '_use_latest_version', '_known_subjects', '_parsed_schema', '_registry', '_schema', '_schema_id', '_schema_name', - '_subject_name_func', '_to_dict'] + '_subject_name_func', '_to_dict', '_named_schemas'] _default_conf = {'auto.register.schemas': True, 'normalize.schemas': False, @@ -203,9 +221,9 @@ def __init__(self, schema_registry_client: SchemaRegistryClient, schema_str: str raise ValueError("Unrecognized properties: {}" .format(", ".join(conf_copy.keys()))) - schema = _schema_loads(schema_str) schema_dict = loads(schema.schema_str) - parsed_schema = parse_schema(schema_dict) + self._named_schemas = _resolve_named_schema(schema, schema_registry_client) + parsed_schema = parse_schema(schema_dict, named_schemas=self._named_schemas) if isinstance(parsed_schema, list): # if parsed_schema is a list, we have an Avro union and there @@ -304,8 +322,9 @@ class AvroDeserializer(Deserializer): schema_registry_client (SchemaRegistryClient): Confluent Schema Registry client instance. - schema_str (str, optional): The reader schema. - If not provided, the writer schema will be used as the reader schema. + schema_str (str, Schema, optional): Avro reader schema declaration Accepts either a string or a `Schema`( + Schema) instance. If not provided, the writer schema will be used as the reader schema. Note that string + definitions cannot reference other schemas. For referencing other schemas, use a Schema instance. from_dict (callable, optional): Callable(dict, SerializationContext) -> object. Converts a dict to an instance of some object. @@ -320,13 +339,31 @@ class AvroDeserializer(Deserializer): `Apache Avro Schema Resolution `_ """ - __slots__ = ['_reader_schema', '_registry', '_from_dict', '_writer_schemas', '_return_record_name'] + __slots__ = ['_reader_schema', '_registry', '_from_dict', '_writer_schemas', '_return_record_name', '_schema', + '_named_schemas'] def __init__(self, schema_registry_client: SchemaRegistryClient, schema_str: Optional[str]=None, from_dict:Optional[Callable]=None, return_record_name: bool=False): + schema = None + if schema_str is not None: + if isinstance(schema_str, str): + schema = _schema_loads(schema_str) + elif isinstance(schema_str, Schema): + schema = schema_str + else: + raise TypeError('You must pass either schema string or schema object') + + self._schema = schema self._registry = schema_registry_client self._writer_schemas: Dict[int, Schema] = {} - self._reader_schema = parse_schema(loads(schema_str)) if schema_str else None + if schema: + schema_dict = loads(self._schema.schema_str) + self._named_schemas = _resolve_named_schema(self._schema, schema_registry_client) + self._reader_schema = parse_schema(schema_dict, + named_schemas=self._named_schemas) + else: + self._named_schemas = None + self._reader_schema = None if from_dict is not None and not callable(from_dict): raise ValueError("from_dict must be callable with the signature " @@ -375,10 +412,11 @@ def __call__(self, data: Optional[bytes], ctx: Optional[SerializationContext]=No writer_schema = self._writer_schemas.get(schema_id, None) if writer_schema is None: - schema = self._registry.get_schema(schema_id) - prepared_schema = _schema_loads(schema.schema_str) + registered_schema = self._registry.get_schema(schema_id) + self._named_schemas = _resolve_named_schema(registered_schema, self._registry) + prepared_schema = _schema_loads(registered_schema.schema_str) writer_schema = parse_schema(loads( - prepared_schema.schema_str)) + prepared_schema.schema_str), named_schemas=self._named_schemas) self._writer_schemas[schema_id] = writer_schema obj_dict = schemaless_reader(payload, diff --git a/src/confluent_kafka/schema_registry/json_schema.py b/src/confluent_kafka/schema_registry/json_schema.py index f3c68fce5..4700041f4 100644 --- a/src/confluent_kafka/schema_registry/json_schema.py +++ b/src/confluent_kafka/schema_registry/json_schema.py @@ -21,7 +21,7 @@ import struct from typing import Any, Callable, Dict, Optional, Set, cast -from jsonschema import validate, ValidationError +from jsonschema import validate, ValidationError, RefResolver from confluent_kafka.schema_registry import (_MAGIC_BYTE, Schema, @@ -44,6 +44,25 @@ def __exit__(self, *args: Any) -> None: self.close() +def _resolve_named_schema(schema, schema_registry_client, named_schemas=None): + """ + Resolves named schemas referenced by the provided schema recursively. + :param schema: Schema to resolve named schemas for. + :param schema_registry_client: SchemaRegistryClient to use for retrieval. + :param named_schemas: Dict of named schemas resolved recursively. + :return: named_schemas dict. + """ + if named_schemas is None: + named_schemas = {} + if schema.references is not None: + for ref in schema.references: + referenced_schema = schema_registry_client.get_version(ref.subject, ref.version) + _resolve_named_schema(referenced_schema.schema, schema_registry_client, named_schemas) + referenced_schema_dict = json.loads(referenced_schema.schema.schema_str) + named_schemas[ref.name] = referenced_schema_dict + return named_schemas + + class JSONSerializer(Serializer): """ Serializer that outputs JSON encoded data with Confluent Schema Registry framing. @@ -123,7 +142,7 @@ class JSONSerializer(Serializer): callable with JSONSerializer. Args: - schema_str (str): `JSON Schema definition. `_ + schema_str (str, Schema): `JSON Schema definition. `_ Accepts schema as either a string or a `Schema`(Schema) instance. Note that string definitions cannot reference other schemas. For referencing other schemas, use a Schema instance. schema_registry_client (SchemaRegistryClient): Schema Registry client instance. @@ -135,7 +154,7 @@ class JSONSerializer(Serializer): """ # noqa: E501 __slots__ = ['_hash', '_auto_register', '_normalize_schemas', '_use_latest_version', '_known_subjects', '_parsed_schema', '_registry', '_schema', '_schema_id', - '_schema_name', '_subject_name_func', '_to_dict'] + '_schema_name', '_subject_name_func', '_to_dict', '_are_references_provided'] _default_conf = {'auto.register.schemas': True, 'normalize.schemas': False, @@ -143,6 +162,15 @@ class JSONSerializer(Serializer): 'subject.name.strategy': topic_subject_name_strategy} def __init__(self, schema_str: str, schema_registry_client: SchemaRegistryClient, to_dict: Optional[Callable[[object, SerializationContext], Dict]]=None, conf: Optional[Dict]=None): + self._are_references_provided = False + if isinstance(schema_str, str): + self._schema = Schema(schema_str, schema_type="JSON") + elif isinstance(schema_str, Schema): + self._schema = schema_str + self._are_references_provided = bool(schema_str.references) + else: + raise TypeError('You must pass either str or Schema') + self._registry = schema_registry_client self._schema_id: Optional[int] = None self._known_subjects: Set[str] = set() @@ -179,14 +207,13 @@ def __init__(self, schema_str: str, schema_registry_client: SchemaRegistryClient raise ValueError("Unrecognized properties: {}" .format(", ".join(conf_copy.keys()))) - schema_dict = json.loads(schema_str) + schema_dict = json.loads(self._schema.schema_str) schema_name = schema_dict.get('title', None) if schema_name is None: raise ValueError("Missing required JSON schema annotation title") self._schema_name = schema_name self._parsed_schema = schema_dict - self._schema = Schema(schema_str, schema_type="JSON") def __call__(self, obj: Any, ctx: SerializationContext) -> Optional[bytes]: """ @@ -240,7 +267,14 @@ def __call__(self, obj: Any, ctx: SerializationContext) -> Optional[bytes]: value = obj try: - validate(instance=value, schema=self._parsed_schema) + if self._are_references_provided: + named_schemas = _resolve_named_schema(self._schema, self._registry) + validate(instance=value, schema=self._parsed_schema, + resolver=RefResolver(self._parsed_schema.get('$id'), + self._parsed_schema, + store=named_schemas)) + else: + validate(instance=value, schema=self._parsed_schema) except ValidationError as ve: raise SerializationError(ve.message) @@ -260,16 +294,32 @@ class JSONDeserializer(Deserializer): framing. Args: - schema_str (str): `JSON schema definition `_ use for validating records. + schema_str (str, Schema): `JSON schema definition `_ Accepts schema as either a string or a `Schema`(Schema) instance. Note that string definitions cannot reference other schemas. For referencing other schemas, use a Schema instance. from_dict (callable, optional): Callable(dict, SerializationContext) -> object. Converts a dict to a Python object instance. + + schema_registry_client (SchemaRegistryClient, optional): Schema Registry client instance. Needed if ``schema_str`` is a schema referencing other schemas. """ # noqa: E501 - __slots__ = ['_parsed_schema', '_from_dict'] + __slots__ = ['_parsed_schema', '_from_dict', '_registry', '_are_references_provided', '_schema'] + + def __init__(self, schema_str: str, from_dict: Optional[Callable]=None, schema_registry_client=None): + self._are_references_provided = False + if isinstance(schema_str, str): + schema = Schema(schema_str, schema_type="JSON") + elif isinstance(schema_str, Schema): + schema = schema_str + self._are_references_provided = bool(schema_str.references) + if self._are_references_provided and schema_registry_client is None: + raise ValueError( + """schema_registry_client must be provided if "schema_str" is a Schema instance with references""") + else: + raise TypeError('You must pass either str or Schema') - def __init__(self, schema_str: str, from_dict: Optional[Callable]=None): - self._parsed_schema = json.loads(schema_str) + self._parsed_schema = json.loads(schema.schema_str) + self._schema = schema + self._registry = schema_registry_client if from_dict is not None and not callable(from_dict): raise ValueError("from_dict must be callable with the signature" @@ -315,7 +365,14 @@ def __call__(self, data: Optional[bytes], ctx: Optional[SerializationContext]=No obj_dict = json.loads(payload.read()) try: - validate(instance=obj_dict, schema=self._parsed_schema) + if self._are_references_provided: + named_schemas = _resolve_named_schema(self._schema, self._registry) + validate(instance=obj_dict, + schema=self._parsed_schema, resolver=RefResolver(self._parsed_schema.get('$id'), + self._parsed_schema, + store=named_schemas)) + else: + validate(instance=obj_dict, schema=self._parsed_schema) except ValidationError as ve: raise SerializationError(ve.message) diff --git a/src/confluent_kafka/schema_registry/schema_registry_client.py b/src/confluent_kafka/schema_registry/schema_registry_client.py index bde0af3a8..1ececfea0 100644 --- a/src/confluent_kafka/schema_registry/schema_registry_client.py +++ b/src/confluent_kafka/schema_registry/schema_registry_client.py @@ -360,12 +360,10 @@ def get_schema(self, schema_id: int) -> "Schema": schema = Schema(schema_str=response['schema'], schema_type=response.get('schemaType', 'AVRO')) - refs = [] - for ref in response.get('references', []): - refs.append(SchemaReference(name=ref['name'], - subject=ref['subject'], - version=ref['version'])) - schema.references = refs + schema.references = [ + SchemaReference(name=ref['name'], subject=ref['subject'], version=ref['version']) + for ref in response.get('references', []) + ] self._cache.set(schema_id, schema) @@ -409,7 +407,12 @@ def lookup_schema(self, subject_name: str, schema: "Schema", normalize_schemas: return RegisteredSchema(schema_id=response['id'], schema=Schema(response['schema'], schema_type, - response.get('references', [])), + [ + SchemaReference(name=ref['name'], + subject=ref['subject'], + version=ref['version']) + for ref in response.get('references', []) + ]), subject=response['subject'], version=response['version']) @@ -483,7 +486,12 @@ def get_latest_version(self, subject_name: str) -> "RegisteredSchema": return RegisteredSchema(schema_id=response['id'], schema=Schema(response['schema'], schema_type, - response.get('references', [])), + [ + SchemaReference(name=ref['name'], + subject=ref['subject'], + version=ref['version']) + for ref in response.get('references', []) + ]), subject=response['subject'], version=response['version']) @@ -514,7 +522,12 @@ def get_version(self, subject_name: str, version: int) -> "RegisteredSchema": return RegisteredSchema(schema_id=response['id'], schema=Schema(response['schema'], schema_type, - response.get('references', [])), + [ + SchemaReference(name=ref['name'], + subject=ref['subject'], + version=ref['version']) + for ref in response.get('references', []) + ]), subject=response['subject'], version=response['version']) @@ -665,12 +678,11 @@ class Schema(object): Args: schema_str (str): String representation of the schema. - references ([SchemaReference]): SchemaReferences used in this schema. - schema_type (str): The schema type: AVRO, PROTOBUF or JSON. - """ - __slots__ = ['schema_str', 'references', 'schema_type', '_hash'] + references ([SchemaReference]): SchemaReferences used in this schema. + """ + __slots__ = ['schema_str', 'schema_type', 'references', '_hash'] def __init__(self, schema_str: str, schema_type: str, references: List["SchemaReference"] =[]): super(Schema, self).__init__() diff --git a/src/confluent_kafka/serialization/__init__.py b/src/confluent_kafka/serialization/__init__.py index f2c0fd88a..daead5abe 100644 --- a/src/confluent_kafka/serialization/__init__.py +++ b/src/confluent_kafka/serialization/__init__.py @@ -195,7 +195,7 @@ class DoubleSerializer(Serializer): `DoubleSerializer Javadoc `_ """ # noqa: E501 - def __call__(self, obj: Any, ctx: Optional[SerializationContext]=None) -> Any: + def __call__(self, obj: Optional[float], ctx: Optional[SerializationContext]=None) -> Any: """ Args: obj (object): object to be serialized diff --git a/src/confluent_kafka/src/Admin.c b/src/confluent_kafka/src/Admin.c index 80db66e82..fa7f5dd0e 100644 --- a/src/confluent_kafka/src/Admin.c +++ b/src/confluent_kafka/src/Admin.c @@ -69,23 +69,34 @@ static int Admin_traverse (Handle *self, */ #define Admin_options_def_int (-12345) #define Admin_options_def_float ((float)Admin_options_def_int) +#define Admin_options_def_ptr (NULL) +#define Admin_options_def_cnt (0) struct Admin_options { - int validate_only; /* needs special bool parsing */ - float request_timeout; /* parser: f */ - float operation_timeout; /* parser: f */ - int broker; /* parser: i */ + int validate_only; /* needs special bool parsing */ + float request_timeout; /* parser: f */ + float operation_timeout; /* parser: f */ + int broker; /* parser: i */ + int require_stable_offsets; /* needs special bool parsing */ + rd_kafka_consumer_group_state_t* states; + int states_cnt; }; /**@brief "unset" value initializers for Admin_options * Make sure this is kept up to date with Admin_options above. */ -#define Admin_options_INITIALIZER { \ - Admin_options_def_int, Admin_options_def_float, \ - Admin_options_def_float, Admin_options_def_int, \ - } +#define Admin_options_INITIALIZER { \ + Admin_options_def_int, \ + Admin_options_def_float, \ + Admin_options_def_float, \ + Admin_options_def_int, \ + Admin_options_def_int, \ + Admin_options_def_ptr, \ + Admin_options_def_cnt, \ + } #define Admin_options_is_set_int(v) ((v) != Admin_options_def_int) #define Admin_options_is_set_float(v) Admin_options_is_set_int((int)(v)) +#define Admin_options_is_set_ptr(v) ((v) != NULL) /** @@ -104,6 +115,7 @@ Admin_options_to_c (Handle *self, rd_kafka_admin_op_t for_api, PyObject *future) { rd_kafka_AdminOptions_t *c_options; rd_kafka_resp_err_t err; + rd_kafka_error_t *err_obj = NULL; char errstr[512]; c_options = rd_kafka_AdminOptions_new(self->rk, for_api); @@ -141,11 +153,28 @@ Admin_options_to_c (Handle *self, rd_kafka_admin_op_t for_api, errstr, sizeof(errstr)))) goto err; + if (Admin_options_is_set_int(options->require_stable_offsets) && + (err_obj = rd_kafka_AdminOptions_set_require_stable_offsets( + c_options, options->require_stable_offsets))) { + strcpy(errstr, rd_kafka_error_string(err_obj)); + goto err; + } + + if (Admin_options_is_set_ptr(options->states) && + (err_obj = rd_kafka_AdminOptions_set_match_consumer_group_states( + c_options, options->states, options->states_cnt))) { + strcpy(errstr, rd_kafka_error_string(err_obj)); + goto err; + } + return c_options; err: if (c_options) rd_kafka_AdminOptions_destroy(c_options); PyErr_Format(PyExc_ValueError, "%s", errstr); + if(err_obj) { + rd_kafka_error_destroy(err_obj); + } return NULL; } @@ -300,6 +329,97 @@ static int Admin_set_replica_assignment (const char *forApi, void *c_obj, return 1; } + +static int +Admin_incremental_config_to_c(PyObject *incremental_configs, + rd_kafka_ConfigResource_t *c_obj, + PyObject *ConfigEntry_type){ + int config_entry_count = 0; + Py_ssize_t i = 0; + char *name = NULL; + char *value = NULL; + PyObject *incremental_operation = NULL; + + if (!PyList_Check(incremental_configs)) { + PyErr_Format(PyExc_TypeError, + "expected list of ConfigEntry " + "in incremental_configs field"); + goto err; + } + + if ((config_entry_count = (int)PyList_Size(incremental_configs)) < 1) { + PyErr_Format(PyExc_ValueError, + "expected non-empty list of ConfigEntry " + "to alter incrementally " + "in incremental_configs field"); + goto err; + } + + for (i = 0; i < config_entry_count; i++) { + PyObject *config_entry; + int incremental_operation_value, r; + rd_kafka_error_t *error; + + config_entry = PyList_GET_ITEM(incremental_configs, i); + + r = PyObject_IsInstance(config_entry, ConfigEntry_type); + if (r == -1) + goto err; /* Exception raised by IsInstance() */ + else if (r == 0) { + PyErr_Format(PyExc_TypeError, + "expected ConfigEntry type " + "in incremental_configs field, " + "index %zd", i); + goto err; + } + + if (!cfl_PyObject_GetAttr(config_entry, "incremental_operation", + &incremental_operation, NULL, 1, 0)) + goto err; + + if (!cfl_PyObject_GetInt(incremental_operation, "value", + &incremental_operation_value, -1, 1)) + goto err; + + if (!cfl_PyObject_GetString(config_entry, "name", &name, NULL, 1, 0)) + goto err; + + if (incremental_operation_value != RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE && + !cfl_PyObject_GetString(config_entry, "value", &value, NULL, 1, 0)) + goto err; + + error = rd_kafka_ConfigResource_add_incremental_config( + c_obj, + name, + (rd_kafka_AlterConfigOpType_t) incremental_operation_value, + value); + if (error) { + PyErr_Format(PyExc_ValueError, + "setting config entry \"%s\", " + "index %zd, failed: %s", + name, i, rd_kafka_error_string(error)); + rd_kafka_error_destroy(error); + goto err; + } + + Py_DECREF(incremental_operation); + free(name); + if (value) + free(value); + name = NULL; + value = NULL; + incremental_operation = NULL; + } + return 1; +err: + Py_XDECREF(incremental_operation); + if (name) + free(name); + if (value) + free(value); + return 0; +} + /** * @brief Translate a dict to ConfigResource set_config() calls, * or to NewTopic_add_config() calls. @@ -389,6 +509,7 @@ static PyObject *Admin_create_topics (Handle *self, PyObject *args, rd_kafka_AdminOptions_t *c_options = NULL; int tcnt; int i; + int topic_partition_count; rd_kafka_NewTopic_t **c_objs; rd_kafka_queue_t *rkqu; CallState cs; @@ -463,10 +584,16 @@ static PyObject *Admin_create_topics (Handle *self, PyObject *args, goto err; } + if (newt->num_partitions == -1) { + topic_partition_count = PyList_Size(newt->replica_assignment); + } else { + topic_partition_count = newt->num_partitions; + } if (!Admin_set_replica_assignment( "CreateTopics", (void *)c_objs[i], newt->replica_assignment, - newt->num_partitions, newt->num_partitions, + topic_partition_count, + topic_partition_count, "num_partitions")) { i++; goto err; @@ -765,7 +892,7 @@ static PyObject *Admin_describe_configs (Handle *self, PyObject *args, rd_kafka_queue_t *rkqu; CallState cs; - /* topics is a list of NewPartitions_t objects. */ + /* resources is a list of ConfigResource objects. */ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|fi", kws, &resources, &future, &options.request_timeout, @@ -875,7 +1002,166 @@ static PyObject *Admin_describe_configs (Handle *self, PyObject *args, return NULL; } +static PyObject *Admin_incremental_alter_configs(Handle *self,PyObject *args,PyObject *kwargs) { + PyObject *resources, *future; + PyObject *validate_only_obj = NULL; + static char *kws[] = { "resources", + "future", + /* options */ + "validate_only", + "request_timeout", + "broker", + NULL }; + struct Admin_options options = Admin_options_INITIALIZER; + rd_kafka_AdminOptions_t *c_options = NULL; + PyObject *ConfigResource_type, *ConfigEntry_type; + int cnt, i; + rd_kafka_ConfigResource_t **c_objs; + rd_kafka_queue_t *rkqu; + CallState cs; + + /* resources is a list of ConfigResource objects. */ + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|Ofi", kws, + &resources, &future, + &validate_only_obj, + &options.request_timeout, + &options.broker)) + return NULL; + + if (!PyList_Check(resources) || + (cnt = (int)PyList_Size(resources)) < 1) { + PyErr_SetString(PyExc_ValueError, + "Expected non-empty list of ConfigResource " + "objects"); + return NULL; + } + + if (validate_only_obj && + !cfl_PyBool_get(validate_only_obj, "validate_only", + &options.validate_only)) + return NULL; + c_options = Admin_options_to_c(self, RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS, + &options, future); + if (!c_options) + return NULL; /* Exception raised by options_to_c() */ + + /* Look up the ConfigResource class so we can check if the provided + * topics are of correct type. + * Since this is not in the fast path we treat ourselves + * to the luxury of looking up this for each call. */ + ConfigResource_type = cfl_PyObject_lookup("confluent_kafka.admin", + "ConfigResource"); + if (!ConfigResource_type) { + rd_kafka_AdminOptions_destroy(c_options); + return NULL; /* Exception raised by find() */ + } + + ConfigEntry_type = cfl_PyObject_lookup("confluent_kafka.admin", + "ConfigEntry"); + if (!ConfigEntry_type) { + Py_DECREF(ConfigResource_type); + rd_kafka_AdminOptions_destroy(c_options); + return NULL; /* Exception raised by find() */ + } + + /* options_to_c() sets future as the opaque, which is used in the + * event_cb to set the results on the future as the admin operation + * is finished, so we need to keep our own refcount. */ + Py_INCREF(future); + + /* + * Parse the list of ConfigResources and convert to + * corresponding C types. + */ + c_objs = malloc(sizeof(*c_objs) * cnt); + + for (i = 0 ; i < cnt ; i++) { + PyObject *res = PyList_GET_ITEM(resources, i); + int r; + int restype; + char *resname; + PyObject *incremental_configs; + + r = PyObject_IsInstance(res, ConfigResource_type); + if (r == -1) + goto err; /* Exception raised by IsInstance() */ + else if (r == 0) { + PyErr_SetString(PyExc_ValueError, + "Expected list of " + "ConfigResource objects"); + goto err; + } + + if (!cfl_PyObject_GetInt(res, "restype_int", &restype, 0, 0)) + goto err; + + if (!cfl_PyObject_GetString(res, "name", &resname, NULL, 0, 0)) + goto err; + + c_objs[i] = rd_kafka_ConfigResource_new( + (rd_kafka_ResourceType_t)restype, resname); + if (!c_objs[i]) { + PyErr_Format(PyExc_ValueError, + "Invalid ConfigResource(%d,%s)", + restype, resname); + free(resname); + goto err; + } + free(resname); + /* + * Translate and apply config entries in the various dicts. + */ + if (!cfl_PyObject_GetAttr(res, "incremental_configs", + &incremental_configs, + &PyList_Type, 1, 0)) { + i++; + goto err; + } + if (!Admin_incremental_config_to_c(incremental_configs, + c_objs[i], + ConfigEntry_type)) { + Py_DECREF(incremental_configs); + i++; + goto err; + } + Py_DECREF(incremental_configs); + } + + + /* Use librdkafka's background thread queue to automatically dispatch + * Admin_background_event_cb() when the admin operation is finished. */ + rkqu = rd_kafka_queue_get_background(self->rk); + + /* + * Call AlterConfigs + * + * We need to set up a CallState and release GIL here since + * the event_cb may be triggered immediately. + */ + CallState_begin(self, &cs); + rd_kafka_IncrementalAlterConfigs(self->rk, c_objs, cnt, c_options, rkqu); + CallState_end(self, &cs); + + rd_kafka_ConfigResource_destroy_array(c_objs, cnt); + rd_kafka_AdminOptions_destroy(c_options); + free(c_objs); + rd_kafka_queue_destroy(rkqu); /* drop reference from get_background */ + + Py_DECREF(ConfigResource_type); /* from lookup() */ + Py_DECREF(ConfigEntry_type); /* from lookup() */ + + Py_RETURN_NONE; + + err: + rd_kafka_ConfigResource_destroy_array(c_objs, i); + rd_kafka_AdminOptions_destroy(c_options); + free(c_objs); + Py_DECREF(ConfigResource_type); /* from lookup() */ + Py_DECREF(ConfigEntry_type); /* from lookup() */ + Py_DECREF(future); /* from options_to_c() */ + return NULL; +} /** @@ -900,7 +1186,7 @@ static PyObject *Admin_alter_configs (Handle *self, PyObject *args, rd_kafka_queue_t *rkqu; CallState cs; - /* topics is a list of NewPartitions_t objects. */ + /* resources is a list of ConfigResource objects. */ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|Ofi", kws, &resources, &future, &validate_only_obj, @@ -1390,53 +1676,1030 @@ static const char Admin_delete_acls_doc[] = PyDoc_STR( " This method should not be used directly, use confluent_kafka.AdminClient.delete_acls()\n" ); - /** - * @brief Call rd_kafka_poll() and keep track of crashing callbacks. - * @returns -1 if callback crashed (or poll() failed), else the number - * of events served. + * @brief List consumer groups */ -static int Admin_poll0 (Handle *self, int tmout) { - int r; +PyObject *Admin_list_consumer_groups (Handle *self, PyObject *args, PyObject *kwargs) { + PyObject *future, *states_int = NULL; + struct Admin_options options = Admin_options_INITIALIZER; + rd_kafka_AdminOptions_t *c_options = NULL; CallState cs; + rd_kafka_queue_t *rkqu; + rd_kafka_consumer_group_state_t *c_states = NULL; + int states_cnt = 0; + int i = 0; - CallState_begin(self, &cs); - - r = rd_kafka_poll(self->rk, tmout); + static char *kws[] = {"future", + /* options */ + "states_int", + "request_timeout", + NULL}; - if (!CallState_end(self, &cs)) { - return -1; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|Of", kws, + &future, + &states_int, + &options.request_timeout)) { + goto err; } - return r; -} - + if(states_int != NULL && states_int != Py_None) { + if(!PyList_Check(states_int)) { + PyErr_SetString(PyExc_ValueError, + "states must of type list"); + goto err; + } -static PyObject *Admin_poll (Handle *self, PyObject *args, - PyObject *kwargs) { - double tmout; - int r; - static char *kws[] = { "timeout", NULL }; + states_cnt = (int)PyList_Size(states_int); + + if(states_cnt > 0) { + c_states = (rd_kafka_consumer_group_state_t *) + malloc(states_cnt*sizeof(rd_kafka_consumer_group_state_t)); + for(i = 0 ; i < states_cnt ; i++) { + PyObject *state = PyList_GET_ITEM(states_int, i); + if(!cfl_PyInt_Check(state)) { + PyErr_SetString(PyExc_ValueError, + "Element of states must be a valid state"); + goto err; + } + c_states[i] = (rd_kafka_consumer_group_state_t) cfl_PyInt_AsInt(state); + } + options.states = c_states; + options.states_cnt = states_cnt; + } + } - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "d", kws, &tmout)) - return NULL; + c_options = Admin_options_to_c(self, RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPS, + &options, future); + if (!c_options) { + goto err; /* Exception raised by options_to_c() */ + } - r = Admin_poll0(self, (int)(tmout * 1000)); - if (r == -1) - return NULL; + /* options_to_c() sets future as the opaque, which is used in the + * background_event_cb to set the results on the future as the + * admin operation is finished, so we need to keep our own refcount. */ + Py_INCREF(future); - return cfl_PyInt_FromInt(r); -} + /* Use librdkafka's background thread queue to automatically dispatch + * Admin_background_event_cb() when the admin operation is finished. */ + rkqu = rd_kafka_queue_get_background(self->rk); + /* + * Call ListConsumerGroupOffsets + * + * We need to set up a CallState and release GIL here since + * the event_cb may be triggered immediately. + */ + CallState_begin(self, &cs); + rd_kafka_ListConsumerGroups(self->rk, c_options, rkqu); + CallState_end(self, &cs); + if(c_states) { + free(c_states); + } + rd_kafka_queue_destroy(rkqu); /* drop reference from get_background */ + rd_kafka_AdminOptions_destroy(c_options); -static PyMethodDef Admin_methods[] = { - { "create_topics", (PyCFunction)Admin_create_topics, - METH_VARARGS|METH_KEYWORDS, - ".. py:function:: create_topics(topics, future, [validate_only, request_timeout, operation_timeout])\n" - "\n" - " Create new topics.\n" - "\n" + Py_RETURN_NONE; +err: + if(c_states) { + free(c_states); + } + if (c_options) { + rd_kafka_AdminOptions_destroy(c_options); + Py_DECREF(future); + } + return NULL; +} +const char Admin_list_consumer_groups_doc[] = PyDoc_STR( + ".. py:function:: list_consumer_groups(future, [states_int], [request_timeout])\n" + "\n" + " List all the consumer groups.\n" + "\n" + " This method should not be used directly, use confluent_kafka.AdminClient.list_consumer_groups()\n"); + + +/** + * @brief DescribeUserScramCredentials +*/ +static PyObject *Admin_describe_user_scram_credentials(Handle *self, PyObject *args, + PyObject *kwargs){ + PyObject *users, *future; + static char *kws[] = { "users", + "future", + /* options */ + "request_timeout", + NULL }; + struct Admin_options options = Admin_options_INITIALIZER; + rd_kafka_AdminOptions_t *c_options = NULL; + int user_cnt, i; + const char **c_users = NULL; + rd_kafka_queue_t *rkqu; + CallState cs; + + /* users is a list of strings. */ + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|f", kws, + &users, &future, + &options.request_timeout)) + return NULL; + + if (!PyList_Check(users)) { + PyErr_SetString(PyExc_ValueError, + "Expected non-empty list of string " + "objects in 'users' parameter"); + return NULL; + } + + c_options = Admin_options_to_c(self, RD_KAFKA_ADMIN_OP_DESCRIBEUSERSCRAMCREDENTIALS, + &options, future); + if (!c_options) + return NULL; /* Exception raised by options_to_c() */ + /* options_to_c() sets future as the opaque, which is used in the + * event_cb to set the results on the future as the admin operation + * is finished, so we need to keep our own refcount. */ + Py_INCREF(future); + + user_cnt = (int)PyList_Size(users); + + + c_users = malloc(sizeof(char *) * user_cnt); + + for (i = 0 ; i < user_cnt ; i++) { + PyObject *user = PyList_GET_ITEM(users, i); + PyObject *u_user; + PyObject *uo_user = NULL; + + if (user == Py_None) { + PyErr_Format(PyExc_TypeError, + "User %d in 'users' parameters must not " + "be None", i); + goto err; + } + + if (!(u_user = cfl_PyObject_Unistr(user))) { + PyErr_Format(PyExc_ValueError, + "User %d in 'users' parameters must " + " be convertible to str", i); + goto err; + } + + c_users[i] = cfl_PyUnistr_AsUTF8(u_user, &uo_user); + Py_XDECREF(u_user); + Py_XDECREF(uo_user); + } + /* Use librdkafka's background thread queue to automatically dispatch + * Admin_background_event_cb() when the admin operation is finished. */ + rkqu = rd_kafka_queue_get_background(self->rk); + + /* + * Call DescribeUserScramCredentials + * + * We need to set up a CallState and release GIL here since + * the event_cb may be triggered immediately. + */ + CallState_begin(self, &cs); + rd_kafka_DescribeUserScramCredentials(self->rk, c_users, user_cnt, c_options, rkqu); + CallState_end(self, &cs); + + if(c_users) + free(c_users); + rd_kafka_queue_destroy(rkqu); /* drop reference from get_background */ + rd_kafka_AdminOptions_destroy(c_options); + Py_RETURN_NONE; +err: + if(c_users) + free(c_users); + if (c_options) { + rd_kafka_AdminOptions_destroy(c_options); + Py_DECREF(future); + } + return NULL; +} + +const char describe_user_scram_credentials_doc[] = PyDoc_STR( + ".. py:function:: describe_user_scram_credentials(users, future, [request_timeout])\n" + "\n" + " Describe all the credentials for a user.\n" + " \n" + " This method should not be used directly, use confluent_kafka.AdminClient.describe_user_scram_credentials()\n"); + +static PyObject *Admin_alter_user_scram_credentials(Handle *self, PyObject *args, + PyObject *kwargs){ + PyObject *alterations, *future; + static char *kws[] = { "alterations", + "future", + /* options */ + "request_timeout", + NULL }; + struct Admin_options options = Admin_options_INITIALIZER; + rd_kafka_AdminOptions_t *c_options = NULL; + int c_alteration_cnt = 0, i; + rd_kafka_UserScramCredentialAlteration_t **c_alterations = NULL; + PyObject *UserScramCredentialAlteration_type = NULL; + PyObject *UserScramCredentialUpsertion_type = NULL; + PyObject *UserScramCredentialDeletion_type = NULL; + PyObject *ScramCredentialInfo_type = NULL; + PyObject *ScramMechanism_type = NULL; + rd_kafka_queue_t *rkqu; + CallState cs; + + PyObject *user = NULL; + const char *c_user; + PyObject *u_user = NULL; + PyObject *uo_user = NULL; + + PyObject *salt = NULL; + const unsigned char *c_salt = NULL; + Py_ssize_t c_salt_size = 0; + + PyObject *password = NULL; + const unsigned char *c_password; + Py_ssize_t c_password_size; + + PyObject *scram_credential_info = NULL; + PyObject *mechanism = NULL; + int32_t iterations; + int c_mechanism; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|f", kws, + &alterations, &future, + &options.request_timeout)) + return NULL; + + if (!PyList_Check(alterations)) { + PyErr_SetString(PyExc_TypeError, + "Expected non-empty list of Alteration " + "objects"); + return NULL; + } + UserScramCredentialAlteration_type = cfl_PyObject_lookup("confluent_kafka.admin", + "UserScramCredentialAlteration"); + + + if (!UserScramCredentialAlteration_type) { + PyErr_SetString(PyExc_ImportError, + "Not able to load UserScramCredentialAlteration type"); + goto err; + } + + UserScramCredentialUpsertion_type = cfl_PyObject_lookup("confluent_kafka.admin", + "UserScramCredentialUpsertion"); + if (!UserScramCredentialUpsertion_type) { + PyErr_SetString(PyExc_ImportError, + "Not able to load UserScramCredentialUpsertion type"); + goto err; + } + UserScramCredentialDeletion_type = cfl_PyObject_lookup("confluent_kafka.admin", + "UserScramCredentialDeletion"); + if (!UserScramCredentialDeletion_type) { + PyErr_SetString(PyExc_ImportError, + "Not able to load UserScramCredentialDeletion type"); + goto err; + } + + ScramCredentialInfo_type = cfl_PyObject_lookup("confluent_kafka.admin", + "ScramCredentialInfo"); + if (!ScramCredentialInfo_type) { + PyErr_SetString(PyExc_ImportError, + "Not able to load ScramCredentialInfo type"); + goto err; + } + + ScramMechanism_type = cfl_PyObject_lookup("confluent_kafka.admin", + "ScramMechanism"); + if (!ScramMechanism_type) { + PyErr_SetString(PyExc_ImportError, + "Not able to load ScramMechanism type"); + goto err; + } + + c_options = Admin_options_to_c(self, RD_KAFKA_ADMIN_OP_ALTERUSERSCRAMCREDENTIALS, + &options, future); + if (!c_options) + goto err; /* Exception raised by options_to_c() */ + + /* options_to_c() sets future as the opaque, which is used in the + * event_cb to set the results on the future as the admin operation + * is finished, so we need to keep our own refcount. */ + Py_INCREF(future); + + c_alteration_cnt = (int)PyList_Size(alterations); + c_alterations = malloc(sizeof(rd_kafka_UserScramCredentialAlteration_t *) * c_alteration_cnt); + + for (i = 0 ; i < c_alteration_cnt ; i++) { + PyObject *alteration = PyList_GET_ITEM(alterations, i); + if(!PyObject_IsInstance(alteration, UserScramCredentialAlteration_type)) { + PyErr_Format(PyExc_TypeError, + "Alteration %d: should be a UserScramCredentialAlteration" + ", got %s", i, + ((PyTypeObject *)PyObject_Type(alteration))-> + tp_name); + goto err; + } + + cfl_PyObject_GetAttr(alteration, "user", &user,NULL,1,1); + if (user == Py_None || + !(u_user = cfl_PyObject_Unistr(user))) { + PyErr_Format(PyExc_TypeError, + "Alteration %d: user field should be a string, got %s", + i, + ((PyTypeObject *)PyObject_Type(user))-> + tp_name); + goto err; + } + + Py_DECREF(user); + c_user = cfl_PyUnistr_AsUTF8(u_user, &uo_user); + + if(PyObject_IsInstance(alteration,UserScramCredentialUpsertion_type)){ + /* Upsertion Type*/ + cfl_PyObject_GetAttr(alteration,"scram_credential_info",&scram_credential_info,NULL,0,0); + if (!PyObject_IsInstance(scram_credential_info, ScramCredentialInfo_type)) { + PyErr_Format(PyExc_TypeError, + "Alteration %d: field \"scram_credential_info\" " + "should be a ScramCredentialInfo" + ", got %s", i, + ((PyTypeObject *)PyObject_Type(scram_credential_info))-> + tp_name); + goto err; + } + + cfl_PyObject_GetInt(scram_credential_info,"iterations",&iterations,0,1); + cfl_PyObject_GetAttr(scram_credential_info,"mechanism", &mechanism, NULL, 0, 0); + if (!PyObject_IsInstance(mechanism, ScramMechanism_type)) { + PyErr_Format(PyExc_TypeError, + "Alteration %d: field \"scram_credential_info." + "mechanism\" should be a ScramMechanism" + ", got %s", i, + ((PyTypeObject *)PyObject_Type(mechanism))-> + tp_name); + goto err; + } + cfl_PyObject_GetInt(mechanism,"value", &c_mechanism,0,1); + + cfl_PyObject_GetAttr(alteration,"password",&password,NULL,0,0); + if (Py_TYPE(password) != &PyBytes_Type) { + PyErr_Format(PyExc_TypeError, + "Alteration %d: password field should be bytes, got %s", + i, + ((PyTypeObject *)PyObject_Type(password))-> + tp_name); + goto err; + } + PyBytes_AsStringAndSize(password, (char **) &c_password, &c_password_size); + + cfl_PyObject_GetAttr(alteration,"salt",&salt,NULL,0,0); + if (salt != Py_None && Py_TYPE(salt) != &PyBytes_Type) { + PyErr_Format(PyExc_TypeError, + "Alteration %d: salt field should be bytes, got %s", + i, + ((PyTypeObject *)PyObject_Type(salt))-> + tp_name); + goto err; + } + if (salt != Py_None) { + PyBytes_AsStringAndSize(salt, (char **) &c_salt, &c_salt_size); + } + + c_alterations[i] = rd_kafka_UserScramCredentialUpsertion_new(c_user, + (rd_kafka_ScramMechanism_t) c_mechanism, iterations, + c_password, c_password_size, + c_salt, c_salt_size); + + Py_DECREF(salt); + Py_DECREF(password); + Py_DECREF(scram_credential_info); + Py_DECREF(mechanism); + salt = NULL, + password = NULL, + scram_credential_info = NULL; + mechanism = NULL; + + } else if(PyObject_IsInstance(alteration,UserScramCredentialDeletion_type)){ + /* Deletion Type*/ + cfl_PyObject_GetAttr(alteration,"mechanism",&mechanism,NULL,0,0); + if (!PyObject_IsInstance(mechanism, ScramMechanism_type)) { + PyErr_Format(PyExc_TypeError, + "Alteration %d: field \"mechanism\" " + "should be a ScramMechanism" + ", got %s", i, + ((PyTypeObject *)PyObject_Type(mechanism))-> + tp_name); + goto err; + } + cfl_PyObject_GetInt(mechanism,"value",&c_mechanism,0,1); + + c_alterations[i] = rd_kafka_UserScramCredentialDeletion_new(c_user,(rd_kafka_ScramMechanism_t)c_mechanism); + Py_DECREF(mechanism); + mechanism = NULL; + } + + Py_DECREF(u_user); + Py_XDECREF(uo_user); + } + /* Use librdkafka's background thread queue to automatically dispatch + * Admin_background_event_cb() when the admin operation is finished. */ + rkqu = rd_kafka_queue_get_background(self->rk); + + /* + * Call AlterConfigs + * + * We need to set up a CallState and release GIL here since + * the event_cb may be triggered immediately. + */ + CallState_begin(self, &cs); + rd_kafka_AlterUserScramCredentials(self->rk, c_alterations, c_alteration_cnt, c_options, rkqu); + CallState_end(self, &cs); + + if(c_alterations) { + rd_kafka_UserScramCredentialAlteration_destroy_array(c_alterations, c_alteration_cnt); + free(c_alterations); + } + rd_kafka_queue_destroy(rkqu); /* drop reference from get_background */ + rd_kafka_AdminOptions_destroy(c_options); + Py_DECREF(UserScramCredentialAlteration_type); /* from lookup() */ + Py_DECREF(UserScramCredentialUpsertion_type); /* from lookup() */ + Py_DECREF(UserScramCredentialDeletion_type); /* from lookup() */ + Py_DECREF(ScramCredentialInfo_type); /* from lookup() */ + Py_DECREF(ScramMechanism_type); /* from lookup() */ + Py_RETURN_NONE; +err: + + Py_XDECREF(u_user); + Py_XDECREF(uo_user); + Py_XDECREF(salt); + Py_XDECREF(password); + Py_XDECREF(scram_credential_info); + Py_XDECREF(mechanism); + + Py_XDECREF(UserScramCredentialAlteration_type); /* from lookup() */ + Py_XDECREF(UserScramCredentialUpsertion_type); /* from lookup() */ + Py_XDECREF(UserScramCredentialDeletion_type); /* from lookup() */ + Py_XDECREF(ScramCredentialInfo_type); /* from lookup() */ + Py_XDECREF(ScramMechanism_type); /* from lookup() */ + + if(c_alterations) { + rd_kafka_UserScramCredentialAlteration_destroy_array(c_alterations, i); + free(c_alterations); + } + if (c_options) { + rd_kafka_AdminOptions_destroy(c_options); + Py_DECREF(future); + } + return NULL; +} + + +const char alter_user_scram_credentials_doc[] = PyDoc_STR( + ".. py:function:: alter_user_scram_credentials(alterations, future, [request_timeout])\n" + "\n" + " Alters the credentials for a user.\n" + " Supported : Upsertion , Deletion.\n" + " \n" + " This method should not be used directly, use confluent_kafka.AdminClient.alter_user_scram_credentials()\n"); + +/** + * @brief Describe consumer groups + */ +PyObject *Admin_describe_consumer_groups (Handle *self, PyObject *args, PyObject *kwargs) { + PyObject *future, *group_ids; + struct Admin_options options = Admin_options_INITIALIZER; + const char **c_groups = NULL; + rd_kafka_AdminOptions_t *c_options = NULL; + CallState cs; + rd_kafka_queue_t *rkqu; + int groups_cnt = 0; + int i = 0; + + static char *kws[] = {"future", + "group_ids", + /* options */ + "request_timeout", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|f", kws, + &group_ids, + &future, + &options.request_timeout)) { + goto err; + } + + if (!PyList_Check(group_ids) || (groups_cnt = (int)PyList_Size(group_ids)) < 1) { + PyErr_SetString(PyExc_ValueError, + "Expected non-empty list of group_ids"); + goto err; + } + + c_groups = malloc(sizeof(char *) * groups_cnt); + + for (i = 0 ; i < groups_cnt ; i++) { + PyObject *group = PyList_GET_ITEM(group_ids, i); + PyObject *ugroup; + PyObject *uogroup = NULL; + + if (group == Py_None || + !(ugroup = cfl_PyObject_Unistr(group))) { + PyErr_Format(PyExc_ValueError, + "Expected list of group strings, " + "not %s", + ((PyTypeObject *)PyObject_Type(group))-> + tp_name); + goto err; + } + + c_groups[i] = cfl_PyUnistr_AsUTF8(ugroup, &uogroup); + + Py_XDECREF(ugroup); + Py_XDECREF(uogroup); + } + + c_options = Admin_options_to_c(self, RD_KAFKA_ADMIN_OP_DESCRIBECONSUMERGROUPS, + &options, future); + if (!c_options) { + goto err; /* Exception raised by options_to_c() */ + } + + /* options_to_c() sets future as the opaque, which is used in the + * background_event_cb to set the results on the future as the + * admin operation is finished, so we need to keep our own refcount. */ + Py_INCREF(future); + + /* Use librdkafka's background thread queue to automatically dispatch + * Admin_background_event_cb() when the admin operation is finished. */ + rkqu = rd_kafka_queue_get_background(self->rk); + + /* + * Call ListConsumerGroupOffsets + * + * We need to set up a CallState and release GIL here since + * the event_cb may be triggered immediately. + */ + CallState_begin(self, &cs); + rd_kafka_DescribeConsumerGroups(self->rk, c_groups, groups_cnt, c_options, rkqu); + CallState_end(self, &cs); + + if(c_groups) { + free(c_groups); + } + rd_kafka_queue_destroy(rkqu); /* drop reference from get_background */ + rd_kafka_AdminOptions_destroy(c_options); + + Py_RETURN_NONE; +err: + if(c_groups) { + free(c_groups); + } + if (c_options) { + rd_kafka_AdminOptions_destroy(c_options); + Py_DECREF(future); + } + return NULL; +} + + +const char Admin_describe_consumer_groups_doc[] = PyDoc_STR( + ".. py:function:: describe_consumer_groups(future, group_ids, [request_timeout])\n" + "\n" + " Describes the provided consumer groups.\n" + "\n" + " This method should not be used directly, use confluent_kafka.AdminClient.describe_consumer_groups()\n"); + + +/** + * @brief Delete consumer groups offsets + */ +PyObject *Admin_delete_consumer_groups (Handle *self, PyObject *args, PyObject *kwargs) { + PyObject *group_ids, *future; + PyObject *group_id; + int group_ids_cnt; + struct Admin_options options = Admin_options_INITIALIZER; + rd_kafka_AdminOptions_t *c_options = NULL; + rd_kafka_DeleteGroup_t **c_delete_group_ids = NULL; + CallState cs; + rd_kafka_queue_t *rkqu; + int i; + + static char *kws[] = {"group_ids", + "future", + /* options */ + "request_timeout", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|f", kws, + &group_ids, + &future, + &options.request_timeout)) { + goto err; + } + + c_options = Admin_options_to_c(self, RD_KAFKA_ADMIN_OP_DELETEGROUPS, + &options, future); + if (!c_options) { + goto err; /* Exception raised by options_to_c() */ + } + + /* options_to_c() sets future as the opaque, which is used in the + * background_event_cb to set the results on the future as the + * admin operation is finished, so we need to keep our own refcount. */ + Py_INCREF(future); + + if (!PyList_Check(group_ids)) { + PyErr_SetString(PyExc_ValueError, "Expected 'group_ids' to be a list"); + goto err; + } + + group_ids_cnt = (int)PyList_Size(group_ids); + + c_delete_group_ids = malloc(sizeof(rd_kafka_DeleteGroup_t *) * group_ids_cnt); + for(i = 0 ; i < group_ids_cnt ; i++) { + group_id = PyList_GET_ITEM(group_ids, i); + + PyObject *ks, *ks8; + const char *group_id_string; + if (!(ks = cfl_PyObject_Unistr(group_id))) { + PyErr_SetString(PyExc_TypeError, + "Expected element of 'group_ids' " + "to be unicode string"); + goto err; + } + + group_id_string = cfl_PyUnistr_AsUTF8(ks, &ks8); + + Py_DECREF(ks); + Py_XDECREF(ks8); + + c_delete_group_ids[i] = rd_kafka_DeleteGroup_new(group_id_string); + } + + /* Use librdkafka's background thread queue to automatically dispatch + * Admin_background_event_cb() when the admin operation is finished. */ + rkqu = rd_kafka_queue_get_background(self->rk); + + /* + * Call DeleteGroups + * + * We need to set up a CallState and release GIL here since + * the event_cb may be triggered immediately. + */ + CallState_begin(self, &cs); + rd_kafka_DeleteGroups(self->rk, c_delete_group_ids, group_ids_cnt, c_options, rkqu); + CallState_end(self, &cs); + + rd_kafka_queue_destroy(rkqu); /* drop reference from get_background */ + rd_kafka_DeleteGroup_destroy_array(c_delete_group_ids, group_ids_cnt); + free(c_delete_group_ids); + rd_kafka_AdminOptions_destroy(c_options); + + Py_RETURN_NONE; +err: + if (c_delete_group_ids) { + rd_kafka_DeleteGroup_destroy_array(c_delete_group_ids, i); + free(c_delete_group_ids); + } + if (c_options) { + rd_kafka_AdminOptions_destroy(c_options); + Py_DECREF(future); + } + return NULL; +} + + +const char Admin_delete_consumer_groups_doc[] = PyDoc_STR( + ".. py:function:: delete_consumer_groups(request, future, [request_timeout])\n" + "\n" + " Deletes consumer groups provided in the request.\n" + "\n" + " This method should not be used directly, use confluent_kafka.AdminClient.delete_consumer_groups()\n"); + + +/** + * @brief List consumer groups offsets + */ +PyObject *Admin_list_consumer_group_offsets (Handle *self, PyObject *args, PyObject *kwargs) { + PyObject *request, *future, *require_stable_obj = NULL; + int requests_cnt; + struct Admin_options options = Admin_options_INITIALIZER; + PyObject *ConsumerGroupTopicPartitions_type = NULL; + rd_kafka_AdminOptions_t *c_options = NULL; + rd_kafka_ListConsumerGroupOffsets_t **c_obj = NULL; + rd_kafka_topic_partition_list_t *c_topic_partitions = NULL; + CallState cs; + rd_kafka_queue_t *rkqu; + PyObject *topic_partitions = NULL; + char *group_id = NULL; + + static char *kws[] = {"request", + "future", + /* options */ + "require_stable", + "request_timeout", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|Of", kws, + &request, + &future, + &require_stable_obj, + &options.request_timeout)) { + goto err; + } + + if (require_stable_obj && + !cfl_PyBool_get(require_stable_obj, "require_stable", + &options.require_stable_offsets)) + return NULL; + + c_options = Admin_options_to_c(self, RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPOFFSETS, + &options, future); + if (!c_options) { + goto err; /* Exception raised by options_to_c() */ + } + + /* options_to_c() sets future as the opaque, which is used in the + * background_event_cb to set the results on the future as the + * admin operation is finished, so we need to keep our own refcount. */ + Py_INCREF(future); + + if (PyList_Check(request) && + (requests_cnt = (int)PyList_Size(request)) != 1) { + PyErr_SetString(PyExc_ValueError, + "Currently we support listing only 1 consumer groups offset information"); + goto err; + } + + PyObject *single_request = PyList_GET_ITEM(request, 0); + + /* Look up the ConsumerGroupTopicPartition class so we can check if the provided + * topics are of correct type. + * Since this is not in the fast path we treat ourselves + * to the luxury of looking up this for each call. */ + ConsumerGroupTopicPartitions_type = cfl_PyObject_lookup("confluent_kafka", + "ConsumerGroupTopicPartitions"); + if (!ConsumerGroupTopicPartitions_type) { + PyErr_SetString(PyExc_ImportError, + "Not able to load ConsumerGroupTopicPartitions type"); + goto err; + } + + if(!PyObject_IsInstance(single_request, ConsumerGroupTopicPartitions_type)) { + PyErr_SetString(PyExc_ImportError, + "Each request should be of ConsumerGroupTopicPartitions type"); + goto err; + } + + cfl_PyObject_GetString(single_request, "group_id", &group_id, NULL, 1, 0); + + if(group_id == NULL) { + PyErr_SetString(PyExc_ValueError, + "Group name is mandatory for list consumer offset operation"); + goto err; + } + + cfl_PyObject_GetAttr(single_request, "topic_partitions", &topic_partitions, &PyList_Type, 0, 1); + + if(topic_partitions != Py_None) { + c_topic_partitions = py_to_c_parts(topic_partitions); + } + + c_obj = malloc(sizeof(rd_kafka_ListConsumerGroupOffsets_t *) * requests_cnt); + c_obj[0] = rd_kafka_ListConsumerGroupOffsets_new(group_id, c_topic_partitions); + + /* Use librdkafka's background thread queue to automatically dispatch + * Admin_background_event_cb() when the admin operation is finished. */ + rkqu = rd_kafka_queue_get_background(self->rk); + + /* + * Call ListConsumerGroupOffsets + * + * We need to set up a CallState and release GIL here since + * the event_cb may be triggered immediately. + */ + CallState_begin(self, &cs); + rd_kafka_ListConsumerGroupOffsets(self->rk, c_obj, requests_cnt, c_options, rkqu); + CallState_end(self, &cs); + + if (c_topic_partitions) { + rd_kafka_topic_partition_list_destroy(c_topic_partitions); + } + rd_kafka_queue_destroy(rkqu); /* drop reference from get_background */ + rd_kafka_ListConsumerGroupOffsets_destroy_array(c_obj, requests_cnt); + free(c_obj); + free(group_id); + Py_DECREF(ConsumerGroupTopicPartitions_type); /* from lookup() */ + Py_XDECREF(topic_partitions); + rd_kafka_AdminOptions_destroy(c_options); + + Py_RETURN_NONE; +err: + if (c_topic_partitions) { + rd_kafka_topic_partition_list_destroy(c_topic_partitions); + } + if (c_obj) { + rd_kafka_ListConsumerGroupOffsets_destroy_array(c_obj, requests_cnt); + free(c_obj); + } + if (c_options) { + rd_kafka_AdminOptions_destroy(c_options); + Py_DECREF(future); + } + if(group_id) { + free(group_id); + } + Py_XDECREF(topic_partitions); + Py_XDECREF(ConsumerGroupTopicPartitions_type); + return NULL; +} + + +const char Admin_list_consumer_group_offsets_doc[] = PyDoc_STR( + ".. py:function:: list_consumer_group_offsets(request, future, [require_stable], [request_timeout])\n" + "\n" + " List offset information for the consumer group and (optional) topic partition provided in the request.\n" + "\n" + " This method should not be used directly, use confluent_kafka.AdminClient.list_consumer_group_offsets()\n"); + + +/** + * @brief Alter consumer groups offsets + */ +PyObject *Admin_alter_consumer_group_offsets (Handle *self, PyObject *args, PyObject *kwargs) { + PyObject *request, *future; + int requests_cnt; + struct Admin_options options = Admin_options_INITIALIZER; + PyObject *ConsumerGroupTopicPartitions_type = NULL; + rd_kafka_AdminOptions_t *c_options = NULL; + rd_kafka_AlterConsumerGroupOffsets_t **c_obj = NULL; + rd_kafka_topic_partition_list_t *c_topic_partitions = NULL; + CallState cs; + rd_kafka_queue_t *rkqu; + PyObject *topic_partitions = NULL; + char *group_id = NULL; + + static char *kws[] = {"request", + "future", + /* options */ + "request_timeout", + NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|f", kws, + &request, + &future, + &options.request_timeout)) { + goto err; + } + + c_options = Admin_options_to_c(self, RD_KAFKA_ADMIN_OP_ALTERCONSUMERGROUPOFFSETS, + &options, future); + if (!c_options) { + goto err; /* Exception raised by options_to_c() */ + } + + /* options_to_c() sets future as the opaque, which is used in the + * background_event_cb to set the results on the future as the + * admin operation is finished, so we need to keep our own refcount. */ + Py_INCREF(future); + + if (PyList_Check(request) && + (requests_cnt = (int)PyList_Size(request)) != 1) { + PyErr_SetString(PyExc_ValueError, + "Currently we support alter consumer groups offset request for 1 group only"); + goto err; + } + + PyObject *single_request = PyList_GET_ITEM(request, 0); + + /* Look up the ConsumerGroupTopicPartition class so we can check if the provided + * topics are of correct type. + * Since this is not in the fast path we treat ourselves + * to the luxury of looking up this for each call. */ + ConsumerGroupTopicPartitions_type = cfl_PyObject_lookup("confluent_kafka", + "ConsumerGroupTopicPartitions"); + if (!ConsumerGroupTopicPartitions_type) { + PyErr_SetString(PyExc_ImportError, + "Not able to load ConsumerGroupTopicPartitions type"); + goto err; + } + + if(!PyObject_IsInstance(single_request, ConsumerGroupTopicPartitions_type)) { + PyErr_SetString(PyExc_ImportError, + "Each request should be of ConsumerGroupTopicPartitions type"); + goto err; + } + + cfl_PyObject_GetString(single_request, "group_id", &group_id, NULL, 1, 0); + + if(group_id == NULL) { + PyErr_SetString(PyExc_ValueError, + "Group name is mandatory for alter consumer offset operation"); + goto err; + } + + cfl_PyObject_GetAttr(single_request, "topic_partitions", &topic_partitions, &PyList_Type, 0, 1); + + if(topic_partitions != Py_None) { + c_topic_partitions = py_to_c_parts(topic_partitions); + } + + c_obj = malloc(sizeof(rd_kafka_AlterConsumerGroupOffsets_t *) * requests_cnt); + c_obj[0] = rd_kafka_AlterConsumerGroupOffsets_new(group_id, c_topic_partitions); + + /* Use librdkafka's background thread queue to automatically dispatch + * Admin_background_event_cb() when the admin operation is finished. */ + rkqu = rd_kafka_queue_get_background(self->rk); + + /* + * Call AlterConsumerGroupOffsets + * + * We need to set up a CallState and release GIL here since + * the event_cb may be triggered immediately. + */ + CallState_begin(self, &cs); + rd_kafka_AlterConsumerGroupOffsets(self->rk, c_obj, requests_cnt, c_options, rkqu); + CallState_end(self, &cs); + + rd_kafka_queue_destroy(rkqu); /* drop reference from get_background */ + rd_kafka_AlterConsumerGroupOffsets_destroy_array(c_obj, requests_cnt); + free(c_obj); + free(group_id); + Py_DECREF(ConsumerGroupTopicPartitions_type); /* from lookup() */ + Py_XDECREF(topic_partitions); + rd_kafka_AdminOptions_destroy(c_options); + rd_kafka_topic_partition_list_destroy(c_topic_partitions); + + Py_RETURN_NONE; +err: + if (c_obj) { + rd_kafka_AlterConsumerGroupOffsets_destroy_array(c_obj, requests_cnt); + free(c_obj); + } + if (c_options) { + rd_kafka_AdminOptions_destroy(c_options); + Py_DECREF(future); + } + if(c_topic_partitions) { + rd_kafka_topic_partition_list_destroy(c_topic_partitions); + } + if(group_id) { + free(group_id); + } + Py_XDECREF(topic_partitions); + Py_XDECREF(ConsumerGroupTopicPartitions_type); + return NULL; +} + + +const char Admin_alter_consumer_group_offsets_doc[] = PyDoc_STR( + ".. py:function:: alter_consumer_group_offsets(request, future, [request_timeout])\n" + "\n" + " Alter offset for the consumer group and topic partition provided in the request.\n" + "\n" + " This method should not be used directly, use confluent_kafka.AdminClient.alter_consumer_group_offsets()\n"); + + +/** + * @brief Call rd_kafka_poll() and keep track of crashing callbacks. + * @returns -1 if callback crashed (or poll() failed), else the number + * of events served. + */ +static int Admin_poll0 (Handle *self, int tmout) { + int r; + CallState cs; + + CallState_begin(self, &cs); + + r = rd_kafka_poll(self->rk, tmout); + + if (!CallState_end(self, &cs)) { + return -1; + } + + return r; +} + + +static PyObject *Admin_poll (Handle *self, PyObject *args, + PyObject *kwargs) { + double tmout; + int r; + static char *kws[] = { "timeout", NULL }; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "d", kws, &tmout)) + return NULL; + + r = Admin_poll0(self, (int)(tmout * 1000)); + if (r == -1) + return NULL; + + return cfl_PyInt_FromInt(r); +} + + + +static PyMethodDef Admin_methods[] = { + { "create_topics", (PyCFunction)Admin_create_topics, + METH_VARARGS|METH_KEYWORDS, + ".. py:function:: create_topics(topics, future, [validate_only, request_timeout, operation_timeout])\n" + "\n" + " Create new topics.\n" + "\n" " This method should not be used directly, use confluent_kafka.AdminClient.create_topics()\n" }, @@ -1461,6 +2724,13 @@ static PyMethodDef Admin_methods[] = { " This method should not be used directly, use confluent_kafka.AdminClient.describe_configs()\n" }, + {"incremental_alter_configs", (PyCFunction)Admin_incremental_alter_configs, + METH_VARARGS|METH_KEYWORDS, + ".. py:function:: incremental_alter_configs(resources, future, [request_timeout, validate_only, broker])\n" + "\n" + " This method should not be used directly, use confluent_kafka.AdminClient.incremental_alter_configs()\n" + }, + { "alter_configs", (PyCFunction)Admin_alter_configs, METH_VARARGS|METH_KEYWORDS, ".. py:function:: alter_configs(resources, future, [request_timeout, broker])\n" @@ -1468,7 +2738,6 @@ static PyMethodDef Admin_methods[] = { " This method should not be used directly, use confluent_kafka.AdminClient.alter_configs()\n" }, - { "poll", (PyCFunction)Admin_poll, METH_VARARGS|METH_KEYWORDS, ".. py:function:: poll([timeout])\n" "\n" @@ -1491,6 +2760,26 @@ static PyMethodDef Admin_methods[] = { list_groups_doc }, + { "list_consumer_groups", (PyCFunction)Admin_list_consumer_groups, METH_VARARGS|METH_KEYWORDS, + Admin_list_consumer_groups_doc + }, + + { "describe_consumer_groups", (PyCFunction)Admin_describe_consumer_groups, METH_VARARGS|METH_KEYWORDS, + Admin_describe_consumer_groups_doc + }, + + { "delete_consumer_groups", (PyCFunction)Admin_delete_consumer_groups, METH_VARARGS|METH_KEYWORDS, + Admin_delete_consumer_groups_doc + }, + + { "list_consumer_group_offsets", (PyCFunction)Admin_list_consumer_group_offsets, METH_VARARGS|METH_KEYWORDS, + Admin_list_consumer_group_offsets_doc + }, + + { "alter_consumer_group_offsets", (PyCFunction)Admin_alter_consumer_group_offsets, METH_VARARGS|METH_KEYWORDS, + Admin_alter_consumer_group_offsets_doc + }, + { "create_acls", (PyCFunction)Admin_create_acls, METH_VARARGS|METH_KEYWORDS, Admin_create_acls_doc }, @@ -1499,341 +2788,923 @@ static PyMethodDef Admin_methods[] = { Admin_describe_acls_doc }, - { "delete_acls", (PyCFunction)Admin_delete_acls, METH_VARARGS|METH_KEYWORDS, - Admin_delete_acls_doc - }, + { "delete_acls", (PyCFunction)Admin_delete_acls, METH_VARARGS|METH_KEYWORDS, + Admin_delete_acls_doc + }, + + { "set_sasl_credentials", (PyCFunction)set_sasl_credentials, METH_VARARGS|METH_KEYWORDS, + set_sasl_credentials_doc + }, + + { "alter_user_scram_credentials", (PyCFunction)Admin_alter_user_scram_credentials, METH_VARARGS|METH_KEYWORDS, + alter_user_scram_credentials_doc + }, + + { "describe_user_scram_credentials", (PyCFunction)Admin_describe_user_scram_credentials, METH_VARARGS|METH_KEYWORDS, + describe_user_scram_credentials_doc + }, + + { NULL } +}; + + +static Py_ssize_t Admin__len__ (Handle *self) { + return rd_kafka_outq_len(self->rk); +} + + +static PySequenceMethods Admin_seq_methods = { + (lenfunc)Admin__len__ /* sq_length */ +}; + + +/** + * @brief Convert C topic_result_t array to topic-indexed dict. + */ +static PyObject * +Admin_c_topic_result_to_py (const rd_kafka_topic_result_t **c_result, + size_t cnt) { + PyObject *result; + size_t i; + + result = PyDict_New(); + + for (i = 0 ; i < cnt ; i++) { + PyObject *error; + + error = KafkaError_new_or_None( + rd_kafka_topic_result_error(c_result[i]), + rd_kafka_topic_result_error_string(c_result[i])); + + PyDict_SetItemString( + result, + rd_kafka_topic_result_name(c_result[i]), + error); + + Py_DECREF(error); + } + + return result; +} + + + +/** + * @brief Convert C ConfigEntry array to dict of py ConfigEntry objects. + */ +static PyObject * +Admin_c_ConfigEntries_to_py (PyObject *ConfigEntry_type, + const rd_kafka_ConfigEntry_t **c_configs, + size_t config_cnt) { + PyObject *dict; + size_t ci; + + dict = PyDict_New(); + + for (ci = 0 ; ci < config_cnt ; ci++) { + PyObject *kwargs, *args; + const rd_kafka_ConfigEntry_t *ent = c_configs[ci]; + const rd_kafka_ConfigEntry_t **c_synonyms; + PyObject *entry, *synonyms; + size_t synonym_cnt; + const char *val; + + kwargs = PyDict_New(); + + cfl_PyDict_SetString(kwargs, "name", + rd_kafka_ConfigEntry_name(ent)); + val = rd_kafka_ConfigEntry_value(ent); + if (val) + cfl_PyDict_SetString(kwargs, "value", val); + else + PyDict_SetItemString(kwargs, "value", Py_None); + cfl_PyDict_SetInt(kwargs, "source", + (int)rd_kafka_ConfigEntry_source(ent)); + cfl_PyDict_SetInt(kwargs, "is_read_only", + rd_kafka_ConfigEntry_is_read_only(ent)); + cfl_PyDict_SetInt(kwargs, "is_default", + rd_kafka_ConfigEntry_is_default(ent)); + cfl_PyDict_SetInt(kwargs, "is_sensitive", + rd_kafka_ConfigEntry_is_sensitive(ent)); + cfl_PyDict_SetInt(kwargs, "is_synonym", + rd_kafka_ConfigEntry_is_synonym(ent)); + + c_synonyms = rd_kafka_ConfigEntry_synonyms(ent, + &synonym_cnt); + synonyms = Admin_c_ConfigEntries_to_py(ConfigEntry_type, + c_synonyms, + synonym_cnt); + if (!synonyms) { + Py_DECREF(kwargs); + Py_DECREF(dict); + return NULL; + } + PyDict_SetItemString(kwargs, "synonyms", synonyms); + Py_DECREF(synonyms); + + args = PyTuple_New(0); + entry = PyObject_Call(ConfigEntry_type, args, kwargs); + Py_DECREF(args); + Py_DECREF(kwargs); + if (!entry) { + Py_DECREF(dict); + return NULL; + } + + PyDict_SetItemString(dict, rd_kafka_ConfigEntry_name(ent), + entry); + Py_DECREF(entry); + } + + + return dict; +} + + +/** + * @brief Convert C ConfigResource array to dict indexed by ConfigResource + * with the value of dict(ConfigEntry). + * + * @param ret_configs If true, return configs rather than None. + */ +static PyObject * +Admin_c_ConfigResource_result_to_py (const rd_kafka_ConfigResource_t **c_resources, + size_t cnt, + int ret_configs) { + PyObject *result; + PyObject *ConfigResource_type; + PyObject *ConfigEntry_type; + size_t ri; + + ConfigResource_type = cfl_PyObject_lookup("confluent_kafka.admin", + "ConfigResource"); + if (!ConfigResource_type) + return NULL; + + ConfigEntry_type = cfl_PyObject_lookup("confluent_kafka.admin", + "ConfigEntry"); + if (!ConfigEntry_type) { + Py_DECREF(ConfigResource_type); + return NULL; + } + + result = PyDict_New(); + + for (ri = 0 ; ri < cnt ; ri++) { + const rd_kafka_ConfigResource_t *c_res = c_resources[ri]; + const rd_kafka_ConfigEntry_t **c_configs; + PyObject *kwargs, *wrap; + PyObject *key; + PyObject *configs, *error; + size_t config_cnt; + + c_configs = rd_kafka_ConfigResource_configs(c_res, &config_cnt); + configs = Admin_c_ConfigEntries_to_py(ConfigEntry_type, + c_configs, config_cnt); + if (!configs) + goto err; + + error = KafkaError_new_or_None( + rd_kafka_ConfigResource_error(c_res), + rd_kafka_ConfigResource_error_string(c_res)); + + kwargs = PyDict_New(); + cfl_PyDict_SetInt(kwargs, "restype", + (int)rd_kafka_ConfigResource_type(c_res)); + cfl_PyDict_SetString(kwargs, "name", + rd_kafka_ConfigResource_name(c_res)); + PyDict_SetItemString(kwargs, "described_configs", configs); + PyDict_SetItemString(kwargs, "error", error); + Py_DECREF(error); + + /* Instantiate ConfigResource */ + wrap = PyTuple_New(0); + key = PyObject_Call(ConfigResource_type, wrap, kwargs); + Py_DECREF(wrap); + Py_DECREF(kwargs); + if (!key) { + Py_DECREF(configs); + goto err; + } + + /* Set result to dict[ConfigResource(..)] = configs | None + * depending on ret_configs */ + if (ret_configs) + PyDict_SetItem(result, key, configs); + else + PyDict_SetItem(result, key, Py_None); + + Py_DECREF(configs); + Py_DECREF(key); + } + return result; + + err: + Py_DECREF(ConfigResource_type); + Py_DECREF(ConfigEntry_type); + Py_DECREF(result); + return NULL; +} + +/** + * @brief Convert C AclBinding to py + */ +static PyObject * +Admin_c_AclBinding_to_py (const rd_kafka_AclBinding_t *c_acl_binding) { + + PyObject *args, *kwargs, *AclBinding_type, *acl_binding; + + AclBinding_type = cfl_PyObject_lookup("confluent_kafka.admin", + "AclBinding"); + if (!AclBinding_type) { + return NULL; + } + + kwargs = PyDict_New(); + + cfl_PyDict_SetInt(kwargs, "restype", + rd_kafka_AclBinding_restype(c_acl_binding)); + cfl_PyDict_SetString(kwargs, "name", + rd_kafka_AclBinding_name(c_acl_binding)); + cfl_PyDict_SetInt(kwargs, "resource_pattern_type", + rd_kafka_AclBinding_resource_pattern_type(c_acl_binding)); + cfl_PyDict_SetString(kwargs, "principal", + rd_kafka_AclBinding_principal(c_acl_binding)); + cfl_PyDict_SetString(kwargs, "host", + rd_kafka_AclBinding_host(c_acl_binding)); + cfl_PyDict_SetInt(kwargs, "operation", + rd_kafka_AclBinding_operation(c_acl_binding)); + cfl_PyDict_SetInt(kwargs, "permission_type", + rd_kafka_AclBinding_permission_type(c_acl_binding)); + + args = PyTuple_New(0); + acl_binding = PyObject_Call(AclBinding_type, args, kwargs); - { NULL } -}; + Py_DECREF(args); + Py_DECREF(kwargs); + Py_DECREF(AclBinding_type); + return acl_binding; +} +/** + * @brief Convert C AclBinding array to py list. + */ +static PyObject * +Admin_c_AclBindings_to_py (const rd_kafka_AclBinding_t **c_acls, + size_t c_acls_cnt) { + size_t i; + PyObject *result; + PyObject *acl_binding; -static Py_ssize_t Admin__len__ (Handle *self) { - return rd_kafka_outq_len(self->rk); -} + result = PyList_New(c_acls_cnt); + for (i = 0 ; i < c_acls_cnt ; i++) { + acl_binding = Admin_c_AclBinding_to_py(c_acls[i]); + if (!acl_binding) { + Py_DECREF(result); + return NULL; + } + PyList_SET_ITEM(result, i, acl_binding); + } -static PySequenceMethods Admin_seq_methods = { - (lenfunc)Admin__len__ /* sq_length */ -}; + return result; +} /** - * @brief Convert C topic_result_t array to topic-indexed dict. + * @brief Convert C acl_result_t array to py list. */ static PyObject * -Admin_c_topic_result_to_py (const rd_kafka_topic_result_t **c_result, +Admin_c_acl_result_to_py (const rd_kafka_acl_result_t **c_result, size_t cnt) { PyObject *result; size_t i; - result = PyDict_New(); + result = PyList_New(cnt); for (i = 0 ; i < cnt ; i++) { PyObject *error; + const rd_kafka_error_t *c_error = rd_kafka_acl_result_error(c_result[i]); error = KafkaError_new_or_None( - rd_kafka_topic_result_error(c_result[i]), - rd_kafka_topic_result_error_string(c_result[i])); - - PyDict_SetItemString( - result, - rd_kafka_topic_result_name(c_result[i]), - error); + rd_kafka_error_code(c_error), + rd_kafka_error_string(c_error)); - Py_DECREF(error); + PyList_SET_ITEM(result, i, error); } return result; } +/** + * @brief Convert C DeleteAcls result response array to py list. + */ +static PyObject * +Admin_c_DeleteAcls_result_responses_to_py (const rd_kafka_DeleteAcls_result_response_t **c_result_responses, + size_t cnt) { + const rd_kafka_AclBinding_t **c_matching_acls; + size_t c_matching_acls_cnt; + PyObject *result; + PyObject *acl_bindings; + size_t i; + + result = PyList_New(cnt); + + for (i = 0 ; i < cnt ; i++) { + PyObject *error; + const rd_kafka_error_t *c_error = rd_kafka_DeleteAcls_result_response_error(c_result_responses[i]); + + if (c_error) { + error = KafkaError_new_or_None( + rd_kafka_error_code(c_error), + rd_kafka_error_string(c_error)); + PyList_SET_ITEM(result, i, error); + } else { + c_matching_acls = rd_kafka_DeleteAcls_result_response_matching_acls( + c_result_responses[i], + &c_matching_acls_cnt); + acl_bindings = Admin_c_AclBindings_to_py(c_matching_acls,c_matching_acls_cnt); + if (!acl_bindings) { + Py_DECREF(result); + return NULL; + } + PyList_SET_ITEM(result, i, acl_bindings); + } + } + + return result; +} /** - * @brief Convert C ConfigEntry array to dict of py ConfigEntry objects. + * @brief + * */ -static PyObject * -Admin_c_ConfigEntries_to_py (PyObject *ConfigEntry_type, - const rd_kafka_ConfigEntry_t **c_configs, - size_t config_cnt) { - PyObject *dict; - size_t ci; +static PyObject *Admin_c_ListConsumerGroupsResults_to_py( + const rd_kafka_ConsumerGroupListing_t **c_valid_responses, + size_t valid_cnt, + const rd_kafka_error_t **c_errors_responses, + size_t errors_cnt) { - dict = PyDict_New(); + PyObject *result = NULL; + PyObject *ListConsumerGroupsResult_type = NULL; + PyObject *ConsumerGroupListing_type = NULL; + PyObject *args = NULL; + PyObject *kwargs = NULL; + PyObject *valid_result = NULL; + PyObject *valid_results = NULL; + PyObject *error_result = NULL; + PyObject *error_results = NULL; + PyObject *py_is_simple_consumer_group = NULL; + size_t i = 0; + valid_results = PyList_New(valid_cnt); + error_results = PyList_New(errors_cnt); + if(valid_cnt > 0) { + ConsumerGroupListing_type = cfl_PyObject_lookup("confluent_kafka.admin", + "ConsumerGroupListing"); + if (!ConsumerGroupListing_type) { + goto err; + } + for(i = 0; i < valid_cnt; i++) { - for (ci = 0 ; ci < config_cnt ; ci++) { - PyObject *kwargs, *args; - const rd_kafka_ConfigEntry_t *ent = c_configs[ci]; - const rd_kafka_ConfigEntry_t **c_synonyms; - PyObject *entry, *synonyms; - size_t synonym_cnt; - const char *val; + kwargs = PyDict_New(); - kwargs = PyDict_New(); + cfl_PyDict_SetString(kwargs, + "group_id", + rd_kafka_ConsumerGroupListing_group_id(c_valid_responses[i])); - cfl_PyDict_SetString(kwargs, "name", - rd_kafka_ConfigEntry_name(ent)); - val = rd_kafka_ConfigEntry_value(ent); - if (val) - cfl_PyDict_SetString(kwargs, "value", val); - else - PyDict_SetItemString(kwargs, "value", Py_None); - cfl_PyDict_SetInt(kwargs, "source", - (int)rd_kafka_ConfigEntry_source(ent)); - cfl_PyDict_SetInt(kwargs, "is_read_only", - rd_kafka_ConfigEntry_is_read_only(ent)); - cfl_PyDict_SetInt(kwargs, "is_default", - rd_kafka_ConfigEntry_is_default(ent)); - cfl_PyDict_SetInt(kwargs, "is_sensitive", - rd_kafka_ConfigEntry_is_sensitive(ent)); - cfl_PyDict_SetInt(kwargs, "is_synonym", - rd_kafka_ConfigEntry_is_synonym(ent)); - c_synonyms = rd_kafka_ConfigEntry_synonyms(ent, - &synonym_cnt); - synonyms = Admin_c_ConfigEntries_to_py(ConfigEntry_type, - c_synonyms, - synonym_cnt); - if (!synonyms) { + py_is_simple_consumer_group = PyBool_FromLong( + rd_kafka_ConsumerGroupListing_is_simple_consumer_group(c_valid_responses[i])); + if(PyDict_SetItemString(kwargs, + "is_simple_consumer_group", + py_is_simple_consumer_group) == -1) { + PyErr_Format(PyExc_RuntimeError, + "Not able to set 'is_simple_consumer_group' in ConsumerGroupLising"); + Py_DECREF(py_is_simple_consumer_group); + goto err; + } + Py_DECREF(py_is_simple_consumer_group); + + cfl_PyDict_SetInt(kwargs, "state", rd_kafka_ConsumerGroupListing_state(c_valid_responses[i])); + + args = PyTuple_New(0); + + valid_result = PyObject_Call(ConsumerGroupListing_type, args, kwargs); + PyList_SET_ITEM(valid_results, i, valid_result); + + Py_DECREF(args); Py_DECREF(kwargs); - Py_DECREF(dict); - return NULL; } - PyDict_SetItemString(kwargs, "synonyms", synonyms); - Py_DECREF(synonyms); + Py_DECREF(ConsumerGroupListing_type); + } + + if(errors_cnt > 0) { + for(i = 0; i < errors_cnt; i++) { + + error_result = KafkaError_new_or_None( + rd_kafka_error_code(c_errors_responses[i]), + rd_kafka_error_string(c_errors_responses[i])); + PyList_SET_ITEM(error_results, i, error_result); - args = PyTuple_New(0); - entry = PyObject_Call(ConfigEntry_type, args, kwargs); - Py_DECREF(args); - Py_DECREF(kwargs); - if (!entry) { - Py_DECREF(dict); - return NULL; } + } - PyDict_SetItemString(dict, rd_kafka_ConfigEntry_name(ent), - entry); - Py_DECREF(entry); + ListConsumerGroupsResult_type = cfl_PyObject_lookup("confluent_kafka.admin", + "ListConsumerGroupsResult"); + if (!ListConsumerGroupsResult_type) { + return NULL; } + kwargs = PyDict_New(); + PyDict_SetItemString(kwargs, "valid", valid_results); + PyDict_SetItemString(kwargs, "errors", error_results); + args = PyTuple_New(0); + result = PyObject_Call(ListConsumerGroupsResult_type, args, kwargs); + + Py_DECREF(args); + Py_DECREF(kwargs); + Py_DECREF(valid_results); + Py_DECREF(error_results); + Py_DECREF(ListConsumerGroupsResult_type); + return result; +err: + Py_XDECREF(ListConsumerGroupsResult_type); + Py_XDECREF(ConsumerGroupListing_type); + Py_XDECREF(result); + Py_XDECREF(args); + Py_XDECREF(kwargs); - return dict; + return NULL; } +static PyObject *Admin_c_MemberAssignment_to_py(const rd_kafka_MemberAssignment_t *c_assignment) { + PyObject *MemberAssignment_type = NULL; + PyObject *assignment = NULL; + PyObject *args = NULL; + PyObject *kwargs = NULL; + PyObject *topic_partitions = NULL; + const rd_kafka_topic_partition_list_t *c_topic_partitions = NULL; + + MemberAssignment_type = cfl_PyObject_lookup("confluent_kafka.admin", + "MemberAssignment"); + if (!MemberAssignment_type) { + goto err; + } + c_topic_partitions = rd_kafka_MemberAssignment_partitions(c_assignment); -/** - * @brief Convert C ConfigResource array to dict indexed by ConfigResource - * with the value of dict(ConfigEntry). - * - * @param ret_configs If true, return configs rather than None. - */ -static PyObject * -Admin_c_ConfigResource_result_to_py (const rd_kafka_ConfigResource_t **c_resources, - size_t cnt, - int ret_configs) { - PyObject *result; - PyObject *ConfigResource_type; - PyObject *ConfigEntry_type; - size_t ri; + topic_partitions = c_parts_to_py(c_topic_partitions); - ConfigResource_type = cfl_PyObject_lookup("confluent_kafka.admin", - "ConfigResource"); - if (!ConfigResource_type) - return NULL; + kwargs = PyDict_New(); - ConfigEntry_type = cfl_PyObject_lookup("confluent_kafka.admin", - "ConfigEntry"); - if (!ConfigEntry_type) { - Py_DECREF(ConfigResource_type); - return NULL; + PyDict_SetItemString(kwargs, "topic_partitions", topic_partitions); + + args = PyTuple_New(0); + + assignment = PyObject_Call(MemberAssignment_type, args, kwargs); + + Py_DECREF(MemberAssignment_type); + Py_DECREF(args); + Py_DECREF(kwargs); + Py_DECREF(topic_partitions); + return assignment; + +err: + Py_XDECREF(MemberAssignment_type); + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_XDECREF(topic_partitions); + Py_XDECREF(assignment); + return NULL; + +} + +static PyObject *Admin_c_MemberDescription_to_py(const rd_kafka_MemberDescription_t *c_member) { + PyObject *member = NULL; + PyObject *MemberDescription_type = NULL; + PyObject *args = NULL; + PyObject *kwargs = NULL; + PyObject *assignment = NULL; + const rd_kafka_MemberAssignment_t *c_assignment; + + MemberDescription_type = cfl_PyObject_lookup("confluent_kafka.admin", + "MemberDescription"); + if (!MemberDescription_type) { + goto err; } - result = PyDict_New(); + kwargs = PyDict_New(); - for (ri = 0 ; ri < cnt ; ri++) { - const rd_kafka_ConfigResource_t *c_res = c_resources[ri]; - const rd_kafka_ConfigEntry_t **c_configs; - PyObject *kwargs, *wrap; - PyObject *key; - PyObject *configs, *error; - size_t config_cnt; + cfl_PyDict_SetString(kwargs, + "member_id", + rd_kafka_MemberDescription_consumer_id(c_member)); - c_configs = rd_kafka_ConfigResource_configs(c_res, &config_cnt); - configs = Admin_c_ConfigEntries_to_py(ConfigEntry_type, - c_configs, config_cnt); - if (!configs) - goto err; + cfl_PyDict_SetString(kwargs, + "client_id", + rd_kafka_MemberDescription_client_id(c_member)); + + cfl_PyDict_SetString(kwargs, + "host", + rd_kafka_MemberDescription_host(c_member)); + + const char * c_group_instance_id = rd_kafka_MemberDescription_group_instance_id(c_member); + if(c_group_instance_id) { + cfl_PyDict_SetString(kwargs, "group_instance_id", c_group_instance_id); + } + + c_assignment = rd_kafka_MemberDescription_assignment(c_member); + assignment = Admin_c_MemberAssignment_to_py(c_assignment); + if (!assignment) { + goto err; + } + + PyDict_SetItemString(kwargs, "assignment", assignment); + + args = PyTuple_New(0); + + member = PyObject_Call(MemberDescription_type, args, kwargs); + + Py_DECREF(args); + Py_DECREF(kwargs); + Py_DECREF(MemberDescription_type); + Py_DECREF(assignment); + return member; + +err: + + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_XDECREF(MemberDescription_type); + Py_XDECREF(assignment); + Py_XDECREF(member); + return NULL; +} + +static PyObject *Admin_c_MemberDescriptions_to_py_from_ConsumerGroupDescription( + const rd_kafka_ConsumerGroupDescription_t *c_consumer_group_description) { + PyObject *member_description = NULL; + PyObject *members = NULL; + size_t c_members_cnt; + const rd_kafka_MemberDescription_t *c_member; + size_t i = 0; + + c_members_cnt = rd_kafka_ConsumerGroupDescription_member_count(c_consumer_group_description); + members = PyList_New(c_members_cnt); + if(c_members_cnt > 0) { + for(i = 0; i < c_members_cnt; i++) { + + c_member = rd_kafka_ConsumerGroupDescription_member(c_consumer_group_description, i); + member_description = Admin_c_MemberDescription_to_py(c_member); + if(!member_description) { + goto err; + } + PyList_SET_ITEM(members, i, member_description); + } + } + return members; +err: + Py_XDECREF(members); + return NULL; +} + + +static PyObject *Admin_c_ConsumerGroupDescription_to_py( + const rd_kafka_ConsumerGroupDescription_t *c_consumer_group_description) { + PyObject *consumer_group_description = NULL; + PyObject *ConsumerGroupDescription_type = NULL; + PyObject *args = NULL; + PyObject *kwargs = NULL; + PyObject *py_is_simple_consumer_group = NULL; + PyObject *coordinator = NULL; + PyObject *members = NULL; + const rd_kafka_Node_t *c_coordinator = NULL; + + ConsumerGroupDescription_type = cfl_PyObject_lookup("confluent_kafka.admin", + "ConsumerGroupDescription"); + if (!ConsumerGroupDescription_type) { + PyErr_Format(PyExc_TypeError, "Not able to load ConsumerGroupDescrition type"); + goto err; + } - error = KafkaError_new_or_None( - rd_kafka_ConfigResource_error(c_res), - rd_kafka_ConfigResource_error_string(c_res)); + kwargs = PyDict_New(); - kwargs = PyDict_New(); - cfl_PyDict_SetInt(kwargs, "restype", - (int)rd_kafka_ConfigResource_type(c_res)); - cfl_PyDict_SetString(kwargs, "name", - rd_kafka_ConfigResource_name(c_res)); - PyDict_SetItemString(kwargs, "described_configs", configs); - PyDict_SetItemString(kwargs, "error", error); - Py_DECREF(error); + cfl_PyDict_SetString(kwargs, + "group_id", + rd_kafka_ConsumerGroupDescription_group_id(c_consumer_group_description)); - /* Instantiate ConfigResource */ - wrap = PyTuple_New(0); - key = PyObject_Call(ConfigResource_type, wrap, kwargs); - Py_DECREF(wrap); - Py_DECREF(kwargs); - if (!key) { - Py_DECREF(configs); - goto err; - } + cfl_PyDict_SetString(kwargs, + "partition_assignor", + rd_kafka_ConsumerGroupDescription_partition_assignor(c_consumer_group_description)); - /* Set result to dict[ConfigResource(..)] = configs | None - * depending on ret_configs */ - if (ret_configs) - PyDict_SetItem(result, key, configs); - else - PyDict_SetItem(result, key, Py_None); + members = Admin_c_MemberDescriptions_to_py_from_ConsumerGroupDescription(c_consumer_group_description); + if(!members) { + goto err; + } + PyDict_SetItemString(kwargs, "members", members); - Py_DECREF(configs); - Py_DECREF(key); + c_coordinator = rd_kafka_ConsumerGroupDescription_coordinator(c_consumer_group_description); + coordinator = c_Node_to_py(c_coordinator); + if(!coordinator) { + goto err; } - return result; + PyDict_SetItemString(kwargs, "coordinator", coordinator); - err: - Py_DECREF(ConfigResource_type); - Py_DECREF(ConfigEntry_type); - Py_DECREF(result); - return NULL; -} + py_is_simple_consumer_group = PyBool_FromLong( + rd_kafka_ConsumerGroupDescription_is_simple_consumer_group(c_consumer_group_description)); + if(PyDict_SetItemString(kwargs, "is_simple_consumer_group", py_is_simple_consumer_group) == -1) { + goto err; + } -/** - * @brief Convert C AclBinding to py - */ -static PyObject * -Admin_c_AclBinding_to_py (const rd_kafka_AclBinding_t *c_acl_binding) { + cfl_PyDict_SetInt(kwargs, "state", rd_kafka_ConsumerGroupDescription_state(c_consumer_group_description)); - PyObject *args, *kwargs, *AclBinding_type, *acl_binding; + args = PyTuple_New(0); - AclBinding_type = cfl_PyObject_lookup("confluent_kafka.admin", - "AclBinding"); - if (!AclBinding_type) { - return NULL; + consumer_group_description = PyObject_Call(ConsumerGroupDescription_type, args, kwargs); + + Py_DECREF(py_is_simple_consumer_group); + Py_DECREF(args); + Py_DECREF(kwargs); + Py_DECREF(ConsumerGroupDescription_type); + Py_DECREF(coordinator); + Py_DECREF(members); + return consumer_group_description; + +err: + Py_XDECREF(py_is_simple_consumer_group); + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_XDECREF(coordinator); + Py_XDECREF(ConsumerGroupDescription_type); + Py_XDECREF(members); + return NULL; + +} + +static PyObject *Admin_c_DescribeConsumerGroupsResults_to_py( + const rd_kafka_ConsumerGroupDescription_t **c_result_responses, + size_t cnt) { + PyObject *consumer_group_description = NULL; + PyObject *results = NULL; + size_t i = 0; + results = PyList_New(cnt); + if(cnt > 0) { + for(i = 0; i < cnt; i++) { + PyObject *error; + const rd_kafka_error_t *c_error = + rd_kafka_ConsumerGroupDescription_error(c_result_responses[i]); + + if (c_error) { + error = KafkaError_new_or_None( + rd_kafka_error_code(c_error), + rd_kafka_error_string(c_error)); + PyList_SET_ITEM(results, i, error); + } else { + consumer_group_description = + Admin_c_ConsumerGroupDescription_to_py(c_result_responses[i]); + + if(!consumer_group_description) { + goto err; + } + + PyList_SET_ITEM(results, i, consumer_group_description); + } + } } + return results; +err: + Py_XDECREF(results); + return NULL; +} +static PyObject *Admin_c_ScramMechanism_to_py(rd_kafka_ScramMechanism_t mechanism){ + PyObject *result = NULL; + PyObject *args = NULL, *kwargs = NULL; + PyObject *ScramMechanism_type; kwargs = PyDict_New(); + cfl_PyDict_SetInt(kwargs, "value",(int) mechanism); + args = PyTuple_New(0); + ScramMechanism_type = cfl_PyObject_lookup("confluent_kafka.admin", + "ScramMechanism"); + result = PyObject_Call(ScramMechanism_type, args, kwargs); + Py_DECREF(args); + Py_DECREF(kwargs); + Py_DECREF(ScramMechanism_type); + return result; +} - cfl_PyDict_SetInt(kwargs, "restype", - rd_kafka_AclBinding_restype(c_acl_binding)); - cfl_PyDict_SetString(kwargs, "name", - rd_kafka_AclBinding_name(c_acl_binding)); - cfl_PyDict_SetInt(kwargs, "resource_pattern_type", - rd_kafka_AclBinding_resource_pattern_type(c_acl_binding)); - cfl_PyDict_SetString(kwargs, "principal", - rd_kafka_AclBinding_principal(c_acl_binding)); - cfl_PyDict_SetString(kwargs, "host", - rd_kafka_AclBinding_host(c_acl_binding)); - cfl_PyDict_SetInt(kwargs, "operation", - rd_kafka_AclBinding_operation(c_acl_binding)); - cfl_PyDict_SetInt(kwargs, "permission_type", - rd_kafka_AclBinding_permission_type(c_acl_binding)); +static PyObject *Admin_c_ScramCredentialInfo_to_py(const rd_kafka_ScramCredentialInfo_t *scram_credential_info){ + PyObject *result = NULL; + PyObject *args = NULL; + PyObject *kwargs = NULL; + PyObject *ScramCredentialInfo_type = NULL; + PyObject *scram_mechanism = NULL; + rd_kafka_ScramMechanism_t c_mechanism; + int32_t iterations; + + kwargs = PyDict_New(); + c_mechanism = rd_kafka_ScramCredentialInfo_mechanism(scram_credential_info); + scram_mechanism = Admin_c_ScramMechanism_to_py(c_mechanism); + PyDict_SetItemString(kwargs,"mechanism", scram_mechanism); + Py_DECREF(scram_mechanism); + iterations = rd_kafka_ScramCredentialInfo_iterations(scram_credential_info); + cfl_PyDict_SetInt(kwargs,"iterations", iterations); args = PyTuple_New(0); - acl_binding = PyObject_Call(AclBinding_type, args, kwargs); + ScramCredentialInfo_type = cfl_PyObject_lookup("confluent_kafka.admin", + "ScramCredentialInfo"); + result = PyObject_Call(ScramCredentialInfo_type, args, kwargs); + Py_DECREF(args); + Py_DECREF(kwargs); + Py_DECREF(ScramCredentialInfo_type); + return result; +} +static PyObject *Admin_c_UserScramCredentialsDescription_to_py(const rd_kafka_UserScramCredentialsDescription_t *description){ + PyObject *result = NULL; + PyObject *args = NULL; + PyObject *kwargs = NULL; + PyObject *scram_credential_infos = NULL; + PyObject *UserScramCredentialsDescription_type = NULL; + int scram_credential_info_cnt; + int i; + kwargs = PyDict_New(); + cfl_PyDict_SetString(kwargs, "user", rd_kafka_UserScramCredentialsDescription_user(description)); + + scram_credential_info_cnt = rd_kafka_UserScramCredentialsDescription_scramcredentialinfo_count(description); + scram_credential_infos = PyList_New(scram_credential_info_cnt); + for(i=0; i < scram_credential_info_cnt; i++){ + const rd_kafka_ScramCredentialInfo_t *c_scram_credential_info = + rd_kafka_UserScramCredentialsDescription_scramcredentialinfo(description,i); + PyList_SET_ITEM(scram_credential_infos, i, + Admin_c_ScramCredentialInfo_to_py(c_scram_credential_info)); + } + + PyDict_SetItemString(kwargs,"scram_credential_infos", scram_credential_infos); + args = PyTuple_New(0); + UserScramCredentialsDescription_type = cfl_PyObject_lookup("confluent_kafka.admin", + "UserScramCredentialsDescription"); + result = PyObject_Call(UserScramCredentialsDescription_type, args, kwargs); Py_DECREF(args); Py_DECREF(kwargs); - Py_DECREF(AclBinding_type); - return acl_binding; + Py_DECREF(scram_credential_infos); + Py_DECREF(UserScramCredentialsDescription_type); + return result; } -/** - * @brief Convert C AclBinding array to py list. - */ -static PyObject * -Admin_c_AclBindings_to_py (const rd_kafka_AclBinding_t **c_acls, - size_t c_acls_cnt) { +static PyObject *Admin_c_UserScramCredentialsDescriptions_to_py( + const rd_kafka_UserScramCredentialsDescription_t **c_descriptions, + size_t c_description_cnt) { + PyObject *result = NULL; size_t i; - PyObject *result; - PyObject *acl_binding; - - result = PyList_New(c_acls_cnt); - for (i = 0 ; i < c_acls_cnt ; i++) { - acl_binding = Admin_c_AclBinding_to_py(c_acls[i]); - if (!acl_binding) { - Py_DECREF(result); - return NULL; + result = PyDict_New(); + for(i=0; i < c_description_cnt; i++){ + const char *c_username; + const rd_kafka_error_t *c_error; + rd_kafka_resp_err_t err; + PyObject *error, *user_scram_credentials_description; + const rd_kafka_UserScramCredentialsDescription_t *c_description = c_descriptions[i]; + c_username = rd_kafka_UserScramCredentialsDescription_user(c_description); + c_error = rd_kafka_UserScramCredentialsDescription_error(c_description); + err = rd_kafka_error_code(c_error); + if (err) { + error = KafkaError_new_or_None(err, + rd_kafka_error_string(c_error)); + PyDict_SetItemString(result, c_username, error); + Py_DECREF(error); + } else { + user_scram_credentials_description = + Admin_c_UserScramCredentialsDescription_to_py(c_description); + PyDict_SetItemString(result, c_username, user_scram_credentials_description); + Py_DECREF(user_scram_credentials_description); } - PyList_SET_ITEM(result, i, acl_binding); } - return result; } +static PyObject *Admin_c_AlterUserScramCredentialsResultResponses_to_py( + const rd_kafka_AlterUserScramCredentials_result_response_t **c_responses, + size_t c_response_cnt) { + PyObject *result = NULL; + PyObject* error = NULL; + size_t i; + result = PyDict_New(); + for(i=0; inum_partitions = -1; self->replication_factor = -1; self->replica_assignment = NULL; self->config = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "si|iOO", kws, + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|iiOO", kws, &topic, &self->num_partitions, &self->replication_factor, &self->replica_assignment, @@ -86,6 +87,7 @@ static int NewTopic_init (PyObject *self0, PyObject *args, return -1; + if (self->config) { if (!PyDict_Check(self->config)) { PyErr_SetString(PyExc_TypeError, @@ -125,7 +127,8 @@ static PyMemberDef NewTopic_members[] = { { "topic", T_STRING, offsetof(NewTopic, topic), READONLY, ":py:attribute:topic - Topic name (string)" }, { "num_partitions", T_INT, offsetof(NewTopic, num_partitions), 0, - ":py:attribute: Number of partitions (int)" }, + ":py:attribute: Number of partitions (int).\n" + "Or -1 if a replica_assignment is specified" }, { "replication_factor", T_INT, offsetof(NewTopic, replication_factor), 0, " :py:attribute: Replication factor (int).\n" @@ -147,6 +150,11 @@ static PyMemberDef NewTopic_members[] = { static PyObject *NewTopic_str0 (NewTopic *self) { + if (self->num_partitions == -1) { + return cfl_PyUnistr( + _FromFormat("NewTopic(topic=%s)", + self->topic)); + } return cfl_PyUnistr( _FromFormat("NewTopic(topic=%s,num_partitions=%d)", self->topic, self->num_partitions)); @@ -202,7 +210,12 @@ NewTopic_richcompare (NewTopic *self, PyObject *o2, int op) { static long NewTopic_hash (NewTopic *self) { PyObject *topic = cfl_PyUnistr(_FromString(self->topic)); - long r = PyObject_Hash(topic) ^ self->num_partitions; + long r; + if (self->num_partitions == -1) { + r = PyObject_Hash(topic); + } else { + r = PyObject_Hash(topic) ^ self->num_partitions; + } Py_DECREF(topic); return r; } @@ -233,12 +246,12 @@ PyTypeObject NewTopicType = { "NewTopic specifies per-topic settings for passing to " "AdminClient.create_topics().\n" "\n" - ".. py:function:: NewTopic(topic, num_partitions, [replication_factor], [replica_assignment], [config])\n" + ".. py:function:: NewTopic(topic, [num_partitions], [replication_factor], [replica_assignment], [config])\n" "\n" " Instantiate a NewTopic object.\n" "\n" " :param string topic: Topic name\n" - " :param int num_partitions: Number of partitions to create\n" + " :param int num_partitions: Number of partitions to create, or -1 if replica_assignment is used.\n" " :param int replication_factor: Replication factor of partitions, or -1 if replica_assignment is used.\n" " :param list replica_assignment: List of lists with the replication assignment for each new partition.\n" " :param dict config: Dict (str:str) of topic configuration. See http://kafka.apache.org/documentation.html#topicconfigs\n" @@ -547,6 +560,30 @@ static void AdminTypes_AddObjectsAclPermissionType (PyObject *m) { PyModule_AddIntConstant(m, "ACL_PERMISSION_TYPE_ALLOW", RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW); } +static void AdminTypes_AddObjectsConsumerGroupStates (PyObject *m) { + /* rd_kafka_consumer_group_state_t */ + PyModule_AddIntConstant(m, "CONSUMER_GROUP_STATE_UNKNOWN", RD_KAFKA_CONSUMER_GROUP_STATE_UNKNOWN); + PyModule_AddIntConstant(m, "CONSUMER_GROUP_STATE_PREPARING_REBALANCE", RD_KAFKA_CONSUMER_GROUP_STATE_PREPARING_REBALANCE); + PyModule_AddIntConstant(m, "CONSUMER_GROUP_STATE_COMPLETING_REBALANCE", RD_KAFKA_CONSUMER_GROUP_STATE_COMPLETING_REBALANCE); + PyModule_AddIntConstant(m, "CONSUMER_GROUP_STATE_STABLE", RD_KAFKA_CONSUMER_GROUP_STATE_STABLE); + PyModule_AddIntConstant(m, "CONSUMER_GROUP_STATE_DEAD", RD_KAFKA_CONSUMER_GROUP_STATE_DEAD); + PyModule_AddIntConstant(m, "CONSUMER_GROUP_STATE_EMPTY", RD_KAFKA_CONSUMER_GROUP_STATE_EMPTY); +} + +static void AdminTypes_AddObjectsAlterConfigOpType (PyObject *m) { + /* rd_kafka_consumer_group_state_t */ + PyModule_AddIntConstant(m, "ALTER_CONFIG_OP_TYPE_SET", RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET); + PyModule_AddIntConstant(m, "ALTER_CONFIG_OP_TYPE_DELETE", RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE); + PyModule_AddIntConstant(m, "ALTER_CONFIG_OP_TYPE_APPEND", RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND); + PyModule_AddIntConstant(m, "ALTER_CONFIG_OP_TYPE_SUBTRACT", RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT); +} + +static void AdminTypes_AddObjectsScramMechanismType (PyObject *m) { + PyModule_AddIntConstant(m, "SCRAM_MECHANISM_UNKNOWN", RD_KAFKA_SCRAM_MECHANISM_UNKNOWN); + PyModule_AddIntConstant(m, "SCRAM_MECHANISM_SHA_256", RD_KAFKA_SCRAM_MECHANISM_SHA_256); + PyModule_AddIntConstant(m, "SCRAM_MECHANISM_SHA_512", RD_KAFKA_SCRAM_MECHANISM_SHA_512); +} + /** * @brief Add Admin types to module */ @@ -561,4 +598,7 @@ void AdminTypes_AddObjects (PyObject *m) { AdminTypes_AddObjectsResourcePatternType(m); AdminTypes_AddObjectsAclOperation(m); AdminTypes_AddObjectsAclPermissionType(m); + AdminTypes_AddObjectsConsumerGroupStates(m); + AdminTypes_AddObjectsAlterConfigOpType(m); + AdminTypes_AddObjectsScramMechanismType(m); } diff --git a/src/confluent_kafka/src/Consumer.c b/src/confluent_kafka/src/Consumer.c index 66f5b7540..de574bebb 100644 --- a/src/confluent_kafka/src/Consumer.c +++ b/src/confluent_kafka/src/Consumer.c @@ -486,6 +486,7 @@ static PyObject *Consumer_commit (Handle *self, PyObject *args, } else if (msg) { Message *m; PyObject *uo8; + rd_kafka_topic_partition_t *rktpar; if (PyObject_Type((PyObject *)msg) != (PyObject *)&MessageType) { @@ -497,9 +498,12 @@ static PyObject *Consumer_commit (Handle *self, PyObject *args, m = (Message *)msg; c_offsets = rd_kafka_topic_partition_list_new(1); - rd_kafka_topic_partition_list_add( - c_offsets, cfl_PyUnistr_AsUTF8(m->topic, &uo8), - m->partition)->offset =m->offset + 1; + rktpar = rd_kafka_topic_partition_list_add( + c_offsets, cfl_PyUnistr_AsUTF8(m->topic, &uo8), + m->partition); + rktpar->offset =m->offset + 1; + rd_kafka_topic_partition_set_leader_epoch(rktpar, + m->leader_epoch); Py_XDECREF(uo8); } else { @@ -612,6 +616,7 @@ static PyObject *Consumer_store_offsets (Handle *self, PyObject *args, } else { Message *m; PyObject *uo8; + rd_kafka_topic_partition_t *rktpar; if (PyObject_Type((PyObject *)msg) != (PyObject *)&MessageType) { @@ -623,9 +628,12 @@ static PyObject *Consumer_store_offsets (Handle *self, PyObject *args, m = (Message *)msg; c_offsets = rd_kafka_topic_partition_list_new(1); - rd_kafka_topic_partition_list_add( + rktpar = rd_kafka_topic_partition_list_add( c_offsets, cfl_PyUnistr_AsUTF8(m->topic, &uo8), - m->partition)->offset = m->offset + 1; + m->partition); + rktpar->offset = m->offset + 1; + rd_kafka_topic_partition_set_leader_epoch(rktpar, + m->leader_epoch); Py_XDECREF(uo8); } @@ -783,9 +791,11 @@ static PyObject *Consumer_resume (Handle *self, PyObject *args, static PyObject *Consumer_seek (Handle *self, PyObject *args, PyObject *kwargs) { TopicPartition *tp; - rd_kafka_resp_err_t err; + rd_kafka_resp_err_t err = RD_KAFKA_RESP_ERR_NO_ERROR; static char *kws[] = { "partition", NULL }; - rd_kafka_topic_t *rkt; + rd_kafka_topic_partition_list_t *seek_partitions; + rd_kafka_topic_partition_t *rktpar; + rd_kafka_error_t *error; if (!self->rk) { PyErr_SetString(PyExc_RuntimeError, "Consumer closed"); @@ -803,21 +813,26 @@ static PyObject *Consumer_seek (Handle *self, PyObject *args, PyObject *kwargs) return NULL; } - rkt = rd_kafka_topic_new(self->rk, tp->topic, NULL); - if (!rkt) { - cfl_PyErr_Format(rd_kafka_last_error(), - "Failed to get topic object for " - "topic \"%s\": %s", - tp->topic, - rd_kafka_err2str(rd_kafka_last_error())); - return NULL; - } + seek_partitions = rd_kafka_topic_partition_list_new(1); + rktpar = rd_kafka_topic_partition_list_add(seek_partitions, + tp->topic, tp->partition); + rktpar->offset = tp->offset; + rd_kafka_topic_partition_set_leader_epoch(rktpar, tp->leader_epoch); Py_BEGIN_ALLOW_THREADS; - err = rd_kafka_seek(rkt, tp->partition, tp->offset, -1); + error = rd_kafka_seek_partitions(self->rk, seek_partitions, -1); Py_END_ALLOW_THREADS; - rd_kafka_topic_destroy(rkt); + if (error) { + err = rd_kafka_error_code(error); + rd_kafka_error_destroy(error); + } + + if (!err && seek_partitions->elems[0].err) { + err = seek_partitions->elems[0].err; + } + + rd_kafka_topic_partition_list_destroy(seek_partitions); if (err) { cfl_PyErr_Format(err, @@ -970,8 +985,8 @@ static PyObject *Consumer_poll (Handle *self, PyObject *args, msgobj = Message_new0(self, rkm); #ifdef RD_KAFKA_V_HEADERS - // Have to detach headers outside Message_new0 because it declares the - // rk message as a const + /** Have to detach headers outside Message_new0 because it declares the + * rk message as a const */ rd_kafka_message_detach_headers(rkm, &((Message *)msgobj)->c_headers); #endif rd_kafka_message_destroy(rkm); @@ -1062,8 +1077,8 @@ static PyObject *Consumer_consume (Handle *self, PyObject *args, for (i = 0; i < n; i++) { PyObject *msgobj = Message_new0(self, rkmessages[i]); #ifdef RD_KAFKA_V_HEADERS - // Have to detach headers outside Message_new0 because it declares the - // rk message as a const + /** Have to detach headers outside Message_new0 because it declares the + * rk message as a const */ rd_kafka_message_detach_headers(rkmessages[i], &((Message *)msgobj)->c_headers); #endif PyList_SET_ITEM(msglist, i, msgobj); @@ -1477,6 +1492,9 @@ static PyMethodDef Consumer_methods[] = { "send_offsets_to_transaction() API.\n" "\n" }, + { "set_sasl_credentials", (PyCFunction)set_sasl_credentials, METH_VARARGS|METH_KEYWORDS, + set_sasl_credentials_doc + }, { NULL } diff --git a/src/confluent_kafka/src/Metadata.c b/src/confluent_kafka/src/Metadata.c index e35461d3e..31e1db933 100644 --- a/src/confluent_kafka/src/Metadata.c +++ b/src/confluent_kafka/src/Metadata.c @@ -595,6 +595,11 @@ list_groups (Handle *self, PyObject *args, PyObject *kwargs) { double tmout = -1.0f; static char *kws[] = {"group", "timeout", NULL}; + PyErr_WarnEx(PyExc_DeprecationWarning, + "list_groups() is deprecated, use list_consumer_groups() " + "and describe_consumer_groups() instead.", + 2); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|zd", kws, &group, &tmout)) return NULL; @@ -625,6 +630,9 @@ list_groups (Handle *self, PyObject *args, PyObject *kwargs) { } const char list_groups_doc[] = PyDoc_STR( + ".. deprecated:: 2.0.2" + " Use :func:`list_consumer_groups` and `describe_consumer_groups` instead." + "\n" ".. py:function:: list_groups([group=None], [timeout=-1])\n" "\n" " Request Group Metadata from cluster.\n" diff --git a/src/confluent_kafka/src/Producer.c b/src/confluent_kafka/src/Producer.c index 76834a1ea..b6a51f510 100644 --- a/src/confluent_kafka/src/Producer.c +++ b/src/confluent_kafka/src/Producer.c @@ -811,6 +811,9 @@ static PyMethodDef Producer_methods[] = { " Treat any other error as a fatal error.\n" "\n" }, + { "set_sasl_credentials", (PyCFunction)set_sasl_credentials, METH_VARARGS|METH_KEYWORDS, + set_sasl_credentials_doc + }, { NULL } }; @@ -824,6 +827,23 @@ static PySequenceMethods Producer_seq_methods = { (lenfunc)Producer__len__ /* sq_length */ }; +static int Producer__bool__ (Handle *self) { + return 1; +} + +static PyNumberMethods Producer_num_methods = { + 0, // nb_add + 0, // nb_subtract + 0, // nb_multiply + 0, // nb_remainder + 0, // nb_divmod + 0, // nb_power + 0, // nb_negative + 0, // nb_positive + 0, // nb_absolute + (inquiry)Producer__bool__ // nb_bool +}; + static int Producer_init (PyObject *selfobj, PyObject *args, PyObject *kwargs) { Handle *self = (Handle *)selfobj; @@ -879,8 +899,8 @@ PyTypeObject ProducerType = { 0, /*tp_setattr*/ 0, /*tp_compare*/ 0, /*tp_repr*/ - 0, /*tp_as_number*/ - &Producer_seq_methods, /*tp_as_sequence*/ + &Producer_num_methods, /*tp_as_number*/ + &Producer_seq_methods, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ diff --git a/src/confluent_kafka/src/confluent_kafka.c b/src/confluent_kafka/src/confluent_kafka.c index bd57f2877..36f55c854 100644 --- a/src/confluent_kafka/src/confluent_kafka.c +++ b/src/confluent_kafka/src/confluent_kafka.c @@ -310,7 +310,7 @@ static PyTypeObject KafkaErrorType = { " - Exceptions\n" "\n" "Args:\n" - " error_code (KafkaError): Error code indicating the type of error.\n" + " error (KafkaError): Error code indicating the type of error.\n" "\n" " reason (str): Alternative message to describe the error.\n" "\n" @@ -476,6 +476,13 @@ static PyObject *Message_offset (Message *self, PyObject *ignore) { Py_RETURN_NONE; } +static PyObject *Message_leader_epoch (Message *self, PyObject *ignore) { + if (self->leader_epoch >= 0) + return cfl_PyInt_FromInt(self->leader_epoch); + else + Py_RETURN_NONE; +} + static PyObject *Message_timestamp (Message *self, PyObject *ignore) { return Py_BuildValue("iL", @@ -571,6 +578,11 @@ static PyMethodDef Message_methods[] = { " :rtype: int or None\n" "\n" }, + { "leader_epoch", (PyCFunction)Message_leader_epoch, METH_NOARGS, + " :returns: message offset leader epoch or None if not available.\n" + " :rtype: int or None\n" + "\n" + }, { "timestamp", (PyCFunction)Message_timestamp, METH_NOARGS, "Retrieve timestamp type and timestamp from message.\n" "The timestamp type is one of:\n\n" @@ -743,7 +755,7 @@ PyTypeObject MessageType = { 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ - Message_methods, /* tp_methods */ + Message_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ @@ -784,6 +796,7 @@ PyObject *Message_new0 (const Handle *handle, const rd_kafka_message_t *rkm) { self->partition = rkm->partition; self->offset = rkm->offset; + self->leader_epoch = rd_kafka_message_leader_epoch(rkm); self->timestamp = rd_kafka_message_timestamp(rkm, &self->tstype); @@ -825,12 +838,17 @@ static int TopicPartition_clear (TopicPartition *self) { static void TopicPartition_setup (TopicPartition *self, const char *topic, int partition, long long offset, + int32_t leader_epoch, const char *metadata, rd_kafka_resp_err_t err) { self->topic = strdup(topic); self->partition = partition; self->offset = offset; + if (leader_epoch < 0) + leader_epoch = -1; + self->leader_epoch = leader_epoch; + if (metadata != NULL) { self->metadata = strdup(metadata); } else { @@ -854,6 +872,7 @@ static int TopicPartition_init (PyObject *self, PyObject *args, PyObject *kwargs) { const char *topic; int partition = RD_KAFKA_PARTITION_UA; + int32_t leader_epoch = -1; long long offset = RD_KAFKA_OFFSET_INVALID; const char *metadata = NULL; @@ -861,16 +880,19 @@ static int TopicPartition_init (PyObject *self, PyObject *args, "partition", "offset", "metadata", + "leader_epoch", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|iLs", kws, + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|iLsi", kws, &topic, &partition, &offset, - &metadata)) { + &metadata, + &leader_epoch)) { return -1; } TopicPartition_setup((TopicPartition *)self, - topic, partition, offset, metadata, 0); + topic, partition, offset, + leader_epoch, metadata, 0); return 0; } @@ -890,6 +912,13 @@ static int TopicPartition_traverse (TopicPartition *self, return 0; } +static PyObject *TopicPartition_get_leader_epoch (TopicPartition *tp, void *closure) { + if (tp->leader_epoch >= 0) { + return cfl_PyInt_FromInt(tp->leader_epoch); + } + Py_RETURN_NONE; +} + static PyMemberDef TopicPartition_members[] = { { "topic", T_STRING, offsetof(TopicPartition, topic), READONLY, @@ -913,6 +942,20 @@ static PyMemberDef TopicPartition_members[] = { { NULL } }; +static PyGetSetDef TopicPartition_getters_and_setters[] = { + { + /* name */ + "leader_epoch", + (getter) TopicPartition_get_leader_epoch, + NULL, + /* doc */ + ":attribute leader_epoch: Offset leader epoch (int), or None", + /* closure */ + NULL + }, + { NULL } +}; + static PyObject *TopicPartition_str0 (TopicPartition *self) { PyObject *errstr = NULL; @@ -920,8 +963,15 @@ static PyObject *TopicPartition_str0 (TopicPartition *self) { const char *c_errstr = NULL; PyObject *ret; char offset_str[40]; + char leader_epoch_str[12]; snprintf(offset_str, sizeof(offset_str), "%"CFL_PRId64"", self->offset); + if (self->leader_epoch >= 0) + snprintf(leader_epoch_str, sizeof(leader_epoch_str), + "%"CFL_PRId32"", self->leader_epoch); + else + snprintf(leader_epoch_str, sizeof(leader_epoch_str), + "None"); if (self->error != Py_None) { errstr = cfl_PyObject_Unistr(self->error); @@ -930,9 +980,10 @@ static PyObject *TopicPartition_str0 (TopicPartition *self) { ret = cfl_PyUnistr( _FromFormat("TopicPartition{topic=%s,partition=%"CFL_PRId32 - ",offset=%s,error=%s}", + ",offset=%s,leader_epoch=%s,error=%s}", self->topic, self->partition, offset_str, + leader_epoch_str, c_errstr ? c_errstr : "None")); Py_XDECREF(errstr8); Py_XDECREF(errstr); @@ -1024,40 +1075,44 @@ PyTypeObject TopicPartitionType = { "It is typically used to provide a list of topics or partitions for " "various operations, such as :py:func:`Consumer.assign()`.\n" "\n" - ".. py:function:: TopicPartition(topic, [partition], [offset])\n" + ".. py:function:: TopicPartition(topic, [partition], [offset]," + " [metadata], [leader_epoch])\n" "\n" " Instantiate a TopicPartition object.\n" "\n" " :param string topic: Topic name\n" " :param int partition: Partition id\n" " :param int offset: Initial partition offset\n" + " :param string metadata: Offset metadata\n" + " :param int leader_epoch: Offset leader epoch\n" " :rtype: TopicPartition\n" "\n" "\n", /*tp_doc*/ (traverseproc)TopicPartition_traverse, /* tp_traverse */ (inquiry)TopicPartition_clear, /* tp_clear */ (richcmpfunc)TopicPartition_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - TopicPartition_members,/* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - TopicPartition_init, /* tp_init */ - 0, /* tp_alloc */ - TopicPartition_new /* tp_new */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + TopicPartition_members, /* tp_members */ + TopicPartition_getters_and_setters, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + TopicPartition_init, /* tp_init */ + 0, /* tp_alloc */ + TopicPartition_new /* tp_new */ }; /** * @brief Internal factory to create a TopicPartition object. */ static PyObject *TopicPartition_new0 (const char *topic, int partition, - long long offset, const char *metadata, + long long offset, int32_t leader_epoch, + const char *metadata, rd_kafka_resp_err_t err) { TopicPartition *self; @@ -1065,7 +1120,8 @@ static PyObject *TopicPartition_new0 (const char *topic, int partition, &TopicPartitionType, NULL, NULL); TopicPartition_setup(self, topic, partition, - offset, metadata, err); + offset, leader_epoch, + metadata, err); return (PyObject *)self; } @@ -1090,6 +1146,7 @@ PyObject *c_parts_to_py (const rd_kafka_topic_partition_list_t *c_parts) { TopicPartition_new0( rktpar->topic, rktpar->partition, rktpar->offset, + rd_kafka_topic_partition_get_leader_epoch(rktpar), rktpar->metadata, rktpar->err)); } @@ -1133,6 +1190,8 @@ rd_kafka_topic_partition_list_t *py_to_c_parts (PyObject *plist) { tp->topic, tp->partition); rktpar->offset = tp->offset; + rd_kafka_topic_partition_set_leader_epoch(rktpar, + tp->leader_epoch); if (tp->metadata != NULL) { rktpar->metadata_size = strlen(tp->metadata) + 1; rktpar->metadata = strdup(tp->metadata); @@ -1318,7 +1377,7 @@ PyObject *c_headers_to_py (rd_kafka_headers_t *headers) { while (!rd_kafka_header_get_all(headers, idx++, &header_key, &header_value, &header_value_size)) { - // Create one (key, value) tuple for each header + /* Create one (key, value) tuple for each header */ PyObject *header_tuple = PyTuple_New(2); PyTuple_SetItem(header_tuple, 0, cfl_PyUnistr(_FromString(header_key)) @@ -1388,6 +1447,40 @@ rd_kafka_consumer_group_metadata_t *py_to_c_cgmd (PyObject *obj) { return cgmd; } +PyObject *c_Node_to_py(const rd_kafka_Node_t *c_node) { + PyObject *node = NULL; + PyObject *Node_type = NULL; + PyObject *args = NULL; + PyObject *kwargs = NULL; + + Node_type = cfl_PyObject_lookup("confluent_kafka", + "Node"); + if (!Node_type) { + goto err; + } + + kwargs = PyDict_New(); + + cfl_PyDict_SetInt(kwargs, "id", rd_kafka_Node_id(c_node)); + cfl_PyDict_SetInt(kwargs, "port", rd_kafka_Node_port(c_node)); + cfl_PyDict_SetString(kwargs, "host", rd_kafka_Node_host(c_node)); + + args = PyTuple_New(0); + + node = PyObject_Call(Node_type, args, kwargs); + + Py_DECREF(Node_type); + Py_DECREF(args); + Py_DECREF(kwargs); + return node; + +err: + Py_XDECREF(Node_type); + Py_XDECREF(args); + Py_XDECREF(kwargs); + return NULL; +} + /**************************************************************************** * @@ -1660,7 +1753,8 @@ static void oauth_cb (rd_kafka_t *rk, const char *oauthbearer_config, sizeof(err_msg)); Py_DECREF(result); if (rd_extensions) { - for(int i = 0; i < rd_extensions_size; i++) { + int i; + for(i = 0; i < rd_extensions_size; i++) { free(rd_extensions[i]); } free(rd_extensions); @@ -2347,6 +2441,11 @@ void cfl_PyDict_SetInt (PyObject *dict, const char *name, int val) { Py_DECREF(vo); } +void cfl_PyDict_SetLong (PyObject *dict, const char *name, long val) { + PyObject *vo = cfl_PyLong_FromLong(val); + PyDict_SetItemString(dict, name, vo); + Py_DECREF(vo); +} int cfl_PyObject_SetString (PyObject *o, const char *name, const char *val) { PyObject *vo = cfl_PyUnistr(_FromString(val)); @@ -2539,6 +2638,55 @@ PyObject *cfl_int32_array_to_py_list (const int32_t *arr, size_t cnt) { } +/**************************************************************************** + * + * + * Methods common across all types of clients. + * + * + * + * + ****************************************************************************/ + +const char set_sasl_credentials_doc[] = PyDoc_STR( + ".. py:function:: set_sasl_credentials(username, password)\n" + "\n" + " Sets the SASL credentials used for this client.\n" + " These credentials will overwrite the old ones, and will be used the next time the client needs to authenticate.\n" + " This method will not disconnect existing broker connections that have been established with the old credentials.\n" + " This method is applicable only to SASL PLAIN and SCRAM mechanisms.\n"); + + +PyObject *set_sasl_credentials(Handle *self, PyObject *args, PyObject *kwargs) { + const char *username = NULL; + const char *password = NULL; + rd_kafka_error_t* error; + CallState cs; + static char *kws[] = {"username", "password", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ss", kws, + &username, &password)) { + return NULL; + } + + CallState_begin(self, &cs); + error = rd_kafka_sasl_set_credentials(self->rk, username, password); + + if (!CallState_end(self, &cs)) { + if (error) /* Ignore error in favour of callstate exception */ + rd_kafka_error_destroy(error); + return NULL; + } + + if (error) { + cfl_PyErr_from_error_destroy(error); + return NULL; + } + + Py_RETURN_NONE; +} + + /**************************************************************************** * * @@ -2651,7 +2799,7 @@ static char *KafkaError_add_errs (PyObject *dict, const char *origdoc) { _PRINT("\n"); - return doc; // FIXME: leak + return doc; /* FIXME: leak */ } @@ -2669,7 +2817,11 @@ static struct PyModuleDef cimpl_moduledef = { static PyObject *_init_cimpl (void) { PyObject *m; +/* PyEval_InitThreads became deprecated in Python 3.9 and will be removed in Python 3.11. + * Prior to Python 3.7, this call was required to initialize the GIL. */ +#if PY_VERSION_HEX < 0x03090000 PyEval_InitThreads(); +#endif if (PyType_Ready(&KafkaErrorType) < 0) return NULL; diff --git a/src/confluent_kafka/src/confluent_kafka.h b/src/confluent_kafka/src/confluent_kafka.h index 45aba2f9e..7942a3663 100644 --- a/src/confluent_kafka/src/confluent_kafka.h +++ b/src/confluent_kafka/src/confluent_kafka.h @@ -42,8 +42,8 @@ * 0xMMmmRRPP * MM=major, mm=minor, RR=revision, PP=patchlevel (not used) */ -#define CFL_VERSION 0x01090200 -#define CFL_VERSION_STR "1.9.2" +#define CFL_VERSION 0x02020000 +#define CFL_VERSION_STR "2.2.0" /** * Minimum required librdkafka version. This is checked both during @@ -51,19 +51,19 @@ * Make sure to keep the MIN_RD_KAFKA_VERSION, MIN_VER_ERRSTR and #error * defines and strings in sync. */ -#define MIN_RD_KAFKA_VERSION 0x010900ff +#define MIN_RD_KAFKA_VERSION 0x020200ff #ifdef __APPLE__ -#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v1.9.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`" +#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v2.2.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`" #else -#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v1.9.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html" +#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v2.2.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html" #endif #if RD_KAFKA_VERSION < MIN_RD_KAFKA_VERSION #ifdef __APPLE__ -#error "confluent-kafka-python requires librdkafka v1.9.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`" +#error "confluent-kafka-python requires librdkafka v2.2.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`" #else -#error "confluent-kafka-python requires librdkafka v1.9.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html" +#error "confluent-kafka-python requires librdkafka v2.2.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html" #endif #endif @@ -319,11 +319,15 @@ void CallState_crash (CallState *cs); #define cfl_PyInt_FromInt(v) PyInt_FromLong(v) #endif +#define cfl_PyLong_Check(o) PyLong_Check(o) +#define cfl_PyLong_AsLong(o) (int)PyLong_AsLong(o) +#define cfl_PyLong_FromLong(v) PyLong_FromLong(v) PyObject *cfl_PyObject_lookup (const char *modulename, const char *typename); void cfl_PyDict_SetString (PyObject *dict, const char *name, const char *val); void cfl_PyDict_SetInt (PyObject *dict, const char *name, int val); +void cfl_PyDict_SetLong (PyObject *dict, const char *name, long val); int cfl_PyObject_SetString (PyObject *o, const char *name, const char *val); int cfl_PyObject_SetInt (PyObject *o, const char *name, int val); int cfl_PyObject_GetAttr (PyObject *object, const char *attr_name, @@ -352,6 +356,7 @@ typedef struct { char *topic; int partition; int64_t offset; + int32_t leader_epoch; char *metadata; PyObject *error; } TopicPartition; @@ -378,13 +383,16 @@ rd_kafka_conf_t *common_conf_setup (rd_kafka_type_t ktype, PyObject *args, PyObject *kwargs); PyObject *c_parts_to_py (const rd_kafka_topic_partition_list_t *c_parts); +PyObject *c_Node_to_py(const rd_kafka_Node_t *c_node); rd_kafka_topic_partition_list_t *py_to_c_parts (PyObject *plist); PyObject *list_topics (Handle *self, PyObject *args, PyObject *kwargs); PyObject *list_groups (Handle *self, PyObject *args, PyObject *kwargs); +PyObject *set_sasl_credentials(Handle *self, PyObject *args, PyObject *kwargs); extern const char list_topics_doc[]; extern const char list_groups_doc[]; +extern const char set_sasl_credentials_doc[]; #ifdef RD_KAFKA_V_HEADERS @@ -421,6 +429,7 @@ typedef struct { PyObject *error; int32_t partition; int64_t offset; + int32_t leader_epoch; int64_t timestamp; rd_kafka_timestamp_type_t tstype; int64_t latency; /**< Producer: time it took to produce message */ diff --git a/tests/avro/test_cached_client.py b/tests/avro/test_cached_client.py index 8128da3e3..16b0f5df9 100644 --- a/tests/avro/test_cached_client.py +++ b/tests/avro/test_cached_client.py @@ -193,7 +193,7 @@ def test_invalid_type_url_dict(self): with self.assertRaises(TypeError): self.client = CachedSchemaRegistryClient({ "url": 1 - }) + }) def test_invalid_url(self): with self.assertRaises(ValueError): diff --git a/tests/docker/.env.sh b/tests/docker/.env.sh index 14bca4182..0847124bb 100644 --- a/tests/docker/.env.sh +++ b/tests/docker/.env.sh @@ -13,3 +13,5 @@ export MY_SCHEMA_REGISTRY_SSL_URL_ENV=https://$(hostname -f):8082 export MY_SCHEMA_REGISTRY_SSL_CA_LOCATION_ENV=$TLS/ca-cert export MY_SCHEMA_REGISTRY_SSL_CERTIFICATE_LOCATION_ENV=$TLS/client.pem export MY_SCHEMA_REGISTRY_SSL_KEY_LOCATION_ENV=$TLS/client.key +export MY_SCHEMA_REGISTRY_SSL_KEY_WITH_PASSWORD_LOCATION_ENV=$TLS/client_with_password.key +export MY_SCHEMA_REGISTRY_SSL_KEY_PASSWORD="abcdefgh" \ No newline at end of file diff --git a/tests/docker/bin/certify.sh b/tests/docker/bin/certify.sh index dd139195a..d753bb9d8 100755 --- a/tests/docker/bin/certify.sh +++ b/tests/docker/bin/certify.sh @@ -24,5 +24,6 @@ echo "Creating client cert..." ${PY_DOCKER_BIN}/gen-ssl-certs.sh client ${TLS}/ca-cert ${TLS}/ ${HOST} ${HOST} echo "Creating key ..." +cp ${TLS}/client.key ${TLS}/client_with_password.key openssl rsa -in ${TLS}/client.key -out ${TLS}/client.key -passin pass:${PASS} diff --git a/tests/docker/docker-compose.yaml b/tests/docker/docker-compose.yaml index ab3383d5c..663eba360 100644 --- a/tests/docker/docker-compose.yaml +++ b/tests/docker/docker-compose.yaml @@ -21,7 +21,7 @@ services: KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer KAFKA_SUPER_USERS: "User:ANONYMOUS" schema-registry: - image: confluentinc/cp-schema-registry:7.1.0 + image: confluentinc/cp-schema-registry depends_on: - zookeeper - kafka @@ -42,7 +42,7 @@ services: SCHEMA_REGISTRY_SSL_TRUSTSTORE_PASSWORD: abcdefgh SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181 schema-registry-basic-auth: - image: confluentinc/cp-schema-registry:7.1.0 + image: confluentinc/cp-schema-registry depends_on: - zookeeper - kafka diff --git a/tests/integration/admin/test_basic_operations.py b/tests/integration/admin/test_basic_operations.py index 3ee522513..f928eb4d0 100644 --- a/tests/integration/admin/test_basic_operations.py +++ b/tests/integration/admin/test_basic_operations.py @@ -16,10 +16,15 @@ import confluent_kafka import struct import time +import pytest +from confluent_kafka import ConsumerGroupTopicPartitions, TopicPartition, ConsumerGroupState from confluent_kafka.admin import (NewPartitions, ConfigResource, AclBinding, AclBindingFilter, ResourceType, - ResourcePatternType, AclOperation, AclPermissionType) -from confluent_kafka.error import ConsumeError + ResourcePatternType, AclOperation, AclPermissionType, + UserScramCredentialsDescription, UserScramCredentialUpsertion, + UserScramCredentialDeletion, ScramCredentialInfo, + ScramMechanism) +from confluent_kafka.error import ConsumeError, KafkaException, KafkaError topic_prefix = "test-topic" @@ -139,6 +144,135 @@ def verify_topic_metadata(client, exp_topics, *args, **kwargs): time.sleep(1) +def verify_consumer_group_offsets_operations(client, our_topic, group_id): + + # List Consumer Group Offsets check with just group name + request = ConsumerGroupTopicPartitions(group_id) + fs = client.list_consumer_group_offsets([request]) + f = fs[group_id] + res = f.result() + assert isinstance(res, ConsumerGroupTopicPartitions) + assert res.group_id == group_id + assert len(res.topic_partitions) == 2 + is_any_message_consumed = False + for topic_partition in res.topic_partitions: + assert topic_partition.topic == our_topic + if topic_partition.offset > 0: + is_any_message_consumed = True + assert is_any_message_consumed + + # Alter Consumer Group Offsets check + alter_group_topic_partitions = list(map(lambda topic_partition: TopicPartition(topic_partition.topic, + topic_partition.partition, + 0), + res.topic_partitions)) + alter_group_topic_partition_request = ConsumerGroupTopicPartitions(group_id, + alter_group_topic_partitions) + afs = client.alter_consumer_group_offsets([alter_group_topic_partition_request]) + af = afs[group_id] + ares = af.result() + assert isinstance(ares, ConsumerGroupTopicPartitions) + assert ares.group_id == group_id + assert len(ares.topic_partitions) == 2 + for topic_partition in ares.topic_partitions: + assert topic_partition.topic == our_topic + assert topic_partition.offset == 0 + + # List Consumer Group Offsets check with group name and partitions + list_group_topic_partitions = list(map(lambda topic_partition: TopicPartition(topic_partition.topic, + topic_partition.partition), + ares.topic_partitions)) + list_group_topic_partition_request = ConsumerGroupTopicPartitions(group_id, + list_group_topic_partitions) + lfs = client.list_consumer_group_offsets([list_group_topic_partition_request]) + lf = lfs[group_id] + lres = lf.result() + + assert isinstance(lres, ConsumerGroupTopicPartitions) + assert lres.group_id == group_id + assert len(lres.topic_partitions) == 2 + for topic_partition in lres.topic_partitions: + assert topic_partition.topic == our_topic + assert topic_partition.offset == 0 + + +def verify_admin_scram(admin_client): + newuser = "non-existent" + newmechanism = ScramMechanism.SCRAM_SHA_256 + newiterations = 10000 + + futmap = admin_client.describe_user_scram_credentials([newuser]) + assert isinstance(futmap, dict) + assert len(futmap) == 1 + assert newuser in futmap + fut = futmap[newuser] + with pytest.raises(KafkaException) as ex: + result = fut.result() + assert ex.value.args[0] == KafkaError.RESOURCE_NOT_FOUND + + futmap = admin_client.alter_user_scram_credentials([UserScramCredentialUpsertion(newuser, + ScramCredentialInfo(newmechanism, newiterations), + b"password", b"salt")]) + fut = futmap[newuser] + result = fut.result() + assert result is None + + futmap = admin_client.alter_user_scram_credentials([UserScramCredentialUpsertion( + newuser, + ScramCredentialInfo( + ScramMechanism.SCRAM_SHA_256, 10000), + b"password", b"salt"), + UserScramCredentialUpsertion( + newuser, + ScramCredentialInfo( + ScramMechanism.SCRAM_SHA_512, 10000), + b"password") + ]) + fut = futmap[newuser] + result = fut.result() + assert result is None + + futmap = admin_client.alter_user_scram_credentials([UserScramCredentialUpsertion( + newuser, + ScramCredentialInfo( + ScramMechanism.SCRAM_SHA_256, 10000), + b"password", b"salt"), + UserScramCredentialDeletion( + newuser, + ScramMechanism.SCRAM_SHA_512) + ]) + fut = futmap[newuser] + result = fut.result() + assert result is None + + futmap = admin_client.describe_user_scram_credentials([newuser]) + assert isinstance(futmap, dict) + assert len(futmap) == 1 + assert newuser in futmap + description = futmap[newuser].result() + assert isinstance(description, UserScramCredentialsDescription) + for scram_credential_info in description.scram_credential_infos: + assert ((scram_credential_info.mechanism == newmechanism) and + (scram_credential_info.iterations == newiterations)) + + futmap = admin_client.alter_user_scram_credentials([UserScramCredentialDeletion(newuser, newmechanism)]) + assert isinstance(futmap, dict) + assert len(futmap) == 1 + assert newuser in futmap + fut = futmap[newuser] + result = fut.result() + assert result is None + + futmap = admin_client.describe_user_scram_credentials([newuser]) + assert isinstance(futmap, dict) + assert len(futmap) == 1 + assert newuser in futmap + fut = futmap[newuser] + with pytest.raises(KafkaException) as ex: + result = fut.result() + assert ex.value.args[0] == KafkaError.RESOURCE_NOT_FOUND + + def test_basic_operations(kafka_cluster): num_partitions = 2 topic_config = {"compression.type": "gzip"} @@ -190,6 +324,7 @@ def test_basic_operations(kafka_cluster): p = kafka_cluster.producer() p.produce(our_topic, 'Hello Python!', headers=produce_headers) p.produce(our_topic, key='Just a key and headers', headers=produce_headers) + p.flush() def consume_messages(group_id, num_messages=None): # Consume messages @@ -226,11 +361,15 @@ def consume_messages(group_id, num_messages=None): else: print('Consumer error: %s: ignoring' % str(e)) break + c.close() group1 = 'test-group-1' group2 = 'test-group-2' + acls_topic = our_topic + "-acls" + acls_group = "test-group-acls" consume_messages(group1, 2) consume_messages(group2, 2) + # list_groups without group argument groups = set(group.id for group in admin_client.list_groups(timeout=10)) assert group1 in groups, "Consumer group {} not found".format(group1) @@ -241,6 +380,26 @@ def consume_messages(group_id, num_messages=None): groups = set(group.id for group in admin_client.list_groups(group2)) assert group2 in groups, "Consumer group {} not found".format(group2) + # List Consumer Groups new API test + future = admin_client.list_consumer_groups(request_timeout=10) + result = future.result() + group_ids = [group.group_id for group in result.valid] + assert group1 in group_ids, "Consumer group {} not found".format(group1) + assert group2 in group_ids, "Consumer group {} not found".format(group2) + + future = admin_client.list_consumer_groups(request_timeout=10, states={ConsumerGroupState.STABLE}) + result = future.result() + assert isinstance(result.valid, list) + assert not result.valid + + # Describe Consumer Groups API test + futureMap = admin_client.describe_consumer_groups([group1, group2], request_timeout=10) + for group_id, future in futureMap.items(): + g = future.result() + assert group_id == g.group_id + assert g.is_simple_consumer_group is False + assert g.state == ConsumerGroupState.EMPTY + def verify_config(expconfig, configs): """ Verify that the config key,values in expconfig are found @@ -284,8 +443,13 @@ def verify_config(expconfig, configs): # Verify config matches our expectations verify_config(topic_config, configs) - # Verify ACL operations - verify_admin_acls(admin_client, our_topic, group1) + # Verify Consumer Offset Operations + verify_consumer_group_offsets_operations(admin_client, our_topic, group1) + + # Delete groups + fs = admin_client.delete_consumer_groups([group1, group2], request_timeout=10) + fs[group1].result() # will raise exception on failure + fs[group2].result() # will raise exception on failure # # Delete the topic @@ -293,3 +457,8 @@ def verify_config(expconfig, configs): fs = admin_client.delete_topics([our_topic]) fs[our_topic].result() # will raise exception on failure print("Topic {} marked for deletion".format(our_topic)) + + # Verify ACL operations + verify_admin_acls(admin_client, acls_topic, acls_group) + # Verify user SCRAM credentials API + verify_admin_scram(admin_client) diff --git a/tests/integration/admin/test_incremental_alter_configs.py b/tests/integration/admin/test_incremental_alter_configs.py new file mode 100644 index 000000000..4f279d49c --- /dev/null +++ b/tests/integration/admin/test_incremental_alter_configs.py @@ -0,0 +1,128 @@ +from confluent_kafka.admin import ConfigResource,\ + ConfigEntry, ResourceType,\ + AlterConfigOpType + + +def assert_expected_config_entries(fs, num_fs, expected): + """ + Verify that the list of non-default entries corresponds + to the expected one for each config resource. + """ + assert len(fs.items()) == num_fs + for res, f in fs.items(): + configs = f.result() + entries = sorted([str(entry) for entry in configs.values() + if not entry.is_default]) + assert entries == expected[res] + + +def assert_operation_succeeded(fs, num_fs): + """ + Verify that the operation succeeded + for each resource. + """ + assert len(fs.items()) == num_fs + for _, f in fs.items(): + assert f.result() is None # empty, but raises exception on failure + + +def test_incremental_alter_configs(kafka_cluster): + """ + Incrementally change the configuration entries of two topics + and verify that the configuration description corresponds. + """ + + topic_prefix = "test-topic" + topic_prefix2 = "test-topic2" + num_partitions = 2 + topic_config = {"compression.type": "gzip"} + + our_topic = kafka_cluster.create_topic(topic_prefix, + { + "num_partitions": num_partitions, + "config": topic_config, + "replication_factor": 1, + }) + our_topic2 = kafka_cluster.create_topic(topic_prefix2, + { + "num_partitions": num_partitions, + "config": topic_config, + "replication_factor": 1, + }) + + admin_client = kafka_cluster.admin() + + res1 = ConfigResource( + ResourceType.TOPIC, + our_topic, + incremental_configs=[ + ConfigEntry("cleanup.policy", "compact", + incremental_operation=AlterConfigOpType.APPEND), + ConfigEntry("retention.ms", "10000", + incremental_operation=AlterConfigOpType.SET) + ] + ) + res2 = ConfigResource( + ResourceType.TOPIC, + our_topic2, + incremental_configs=[ + ConfigEntry("cleanup.policy", "delete", + incremental_operation=AlterConfigOpType.SUBTRACT), + ConfigEntry("retention.ms", "5000", + incremental_operation=AlterConfigOpType.SET) + ] + ) + expected = { + res1: ['cleanup.policy="delete,compact"', + 'compression.type="gzip"', + 'retention.ms="10000"'], + res2: ['cleanup.policy=""', + 'compression.type="gzip"', + 'retention.ms="5000"'] + } + + # + # Incrementally alter some configuration values + # + fs = admin_client.incremental_alter_configs([res1, res2]) + + assert_operation_succeeded(fs, 2) + + # + # Get current topic config + # + fs = admin_client.describe_configs([res1, res2]) + + # Assert expected config entries. + assert_expected_config_entries(fs, 2, expected) + + # + # Delete an entry and change a second one. + # + res2 = ConfigResource( + ResourceType.TOPIC, + our_topic2, + incremental_configs=[ + ConfigEntry("compression.type", None, + incremental_operation=AlterConfigOpType.DELETE), + ConfigEntry("retention.ms", "10000", + incremental_operation=AlterConfigOpType.SET) + ] + ) + expected[res2] = ['cleanup.policy=""', + 'retention.ms="10000"'] + + # + # Incrementally alter some configuration values + # + fs = admin_client.incremental_alter_configs([res2]) + + assert_operation_succeeded(fs, 1) + + # + # Get current topic config + # + fs = admin_client.describe_configs([res2]) + + # Assert expected config entries. + assert_expected_config_entries(fs, 1, expected) diff --git a/tests/integration/integration_test.py b/tests/integration/integration_test.py index a109c2299..e4ae6deca 100755 --- a/tests/integration/integration_test.py +++ b/tests/integration/integration_test.py @@ -569,7 +569,8 @@ def verify_consumer_seek(c, seek_to_msg): tp = confluent_kafka.TopicPartition(seek_to_msg.topic(), seek_to_msg.partition(), - seek_to_msg.offset()) + seek_to_msg.offset(), + leader_epoch=seek_to_msg.leader_epoch()) print('seek: Seeking to %s' % tp) c.seek(tp) @@ -583,9 +584,14 @@ def verify_consumer_seek(c, seek_to_msg): if msg.topic() != seek_to_msg.topic() or msg.partition() != seek_to_msg.partition(): continue - print('seek: message at offset %d' % msg.offset()) - assert msg.offset() == seek_to_msg.offset(), \ - 'expected message at offset %d, not %d' % (seek_to_msg.offset(), msg.offset()) + print('seek: message at offset %d (epoch %d)' % + (msg.offset(), msg.leader_epoch())) + assert msg.offset() == seek_to_msg.offset() and \ + msg.leader_epoch() == seek_to_msg.leader_epoch(), \ + ('expected message at offset %d (epoch %d), ' % (seek_to_msg.offset(), + seek_to_msg.leader_epoch())) + \ + ('not %d (epoch %d)' % (msg.offset(), + msg.leader_epoch())) break @@ -1254,6 +1260,16 @@ def print_usage(exitcode, reason=None): if 'avro-https' in modes: print('=' * 30, 'Verifying AVRO with HTTPS', '=' * 30) verify_avro_https(testconf.get('avro-https', None)) + key_with_password_conf = testconf.get("avro-https-key-with-password", None) + print('=' * 30, 'Verifying AVRO with HTTPS Flow with Password', + 'Protected Private Key of Cached-Schema-Registry-Client', '=' * 30) + verify_avro_https(key_with_password_conf) + print('Verifying Error with Wrong Password of Password Protected Private Key of Cached-Schema-Registry-Client') + try: + key_with_password_conf['schema.registry.ssl.key.password'] += '->wrongpassword' + verify_avro_https(key_with_password_conf) + except Exception: + print("Wrong Password Gives Error -> Successful") if 'avro-basic-auth' in modes: print("=" * 30, 'Verifying AVRO with Basic Auth', '=' * 30) diff --git a/tests/integration/schema_registry/data/customer.json b/tests/integration/schema_registry/data/customer.json new file mode 100644 index 000000000..7b9887fa2 --- /dev/null +++ b/tests/integration/schema_registry/data/customer.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://example.com/customer.schema.json", + "title": "Customer", + "description": "Customer data", + "type": "object", + "properties": { + "name": { + "description": "Customer name", + "type": "string" + }, + "id": { + "description": "Customer id", + "type": "integer" + }, + "email": { + "description": "Customer email", + "type": "string" + } + }, + "required": [ "name", "id"] +} diff --git a/tests/integration/schema_registry/data/order.json b/tests/integration/schema_registry/data/order.json new file mode 100644 index 000000000..5ba94c932 --- /dev/null +++ b/tests/integration/schema_registry/data/order.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://example.com/referencedproduct.schema.json", + "title": "Order", + "description": "Order", + "type": "object", + "properties": { + "order_details": { + "description": "Order Details", + "$ref": "http://example.com/order_details.schema.json" + }, + "order_date": { + "description": "Order Date", + "type": "string", + "format": "date-time" + }, + "product": { + "description": "Product", + "$ref": "http://example.com/product.schema.json" + } + }, + "required": [ + "order_details", "product"] +} diff --git a/tests/integration/schema_registry/data/order_details.json b/tests/integration/schema_registry/data/order_details.json new file mode 100644 index 000000000..5fa933d71 --- /dev/null +++ b/tests/integration/schema_registry/data/order_details.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://example.com/order_details.schema.json", + "title": "Order Details", + "description": "Order Details", + "type": "object", + "properties": { + "id": { + "description": "Order Id", + "type": "integer" + }, + "customer": { + "description": "Customer", + "$ref": "http://example.com/customer.schema.json" + }, + "payment_id": { + "description": "Payment Id", + "type": "string" + } + }, + "required": [ "id", "customer"] +} diff --git a/tests/integration/schema_registry/test_avro_serializers.py b/tests/integration/schema_registry/test_avro_serializers.py index c7aacec91..882af40db 100644 --- a/tests/integration/schema_registry/test_avro_serializers.py +++ b/tests/integration/schema_registry/test_avro_serializers.py @@ -15,7 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import pytest from confluent_kafka import TopicPartition @@ -23,6 +22,7 @@ SerializationContext) from confluent_kafka.schema_registry.avro import (AvroSerializer, AvroDeserializer) +from confluent_kafka.schema_registry import Schema, SchemaReference class User(object): @@ -51,6 +51,145 @@ def __eq__(self, other): self.favorite_color == other.favorite_color]) +class AwardProperties(object): + schema_str = """ + { + "namespace": "confluent.io.examples.serialization.avro", + "name": "AwardProperties", + "type": "record", + "fields": [ + {"name": "year", "type": "int"}, + {"name": "points", "type": "int"} + ] + } + """ + + def __init__(self, points, year): + self.points = points + self.year = year + + def __eq__(self, other): + return all([ + self.points == other.points, + self.year == other.year + ]) + + +class Award(object): + schema_str = """ + { + "namespace": "confluent.io.examples.serialization.avro", + "name": "Award", + "type": "record", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "properties", "type": "AwardProperties"} + ] + } + """ + + def __init__(self, name, properties): + self.name = name + self.properties = properties + + def __eq__(self, other): + return all([ + self.name == other.name, + self.properties == other.properties + ]) + + +class AwardedUser(object): + schema_str = """ + { + "namespace": "confluent.io.examples.serialization.avro", + "name": "AwardedUser", + "type": "record", + "fields": [ + {"name": "award", "type": "Award"}, + {"name": "user", "type": "User"} + ] + } + """ + + def __init__(self, award, user): + self.award = award + self.user = user + + def __eq__(self, other): + return all([ + self.award == other.award, + self.user == other.user + ]) + + +def _register_avro_schemas_and_build_awarded_user_schema(kafka_cluster): + sr = kafka_cluster.schema_registry() + + user = User('Bowie', 47, 'purple') + award_properties = AwardProperties(10, 2023) + award = Award("Best In Show", award_properties) + awarded_user = AwardedUser(award, user) + + user_schema_ref = SchemaReference("confluent.io.examples.serialization.avro.User", "user", 1) + award_properties_schema_ref = SchemaReference("confluent.io.examples.serialization.avro.AwardProperties", + "award_properties", 1) + award_schema_ref = SchemaReference("confluent.io.examples.serialization.avro.Award", "award", 1) + + sr.register_schema("user", Schema(User.schema_str, 'AVRO')) + sr.register_schema("award_properties", Schema(AwardProperties.schema_str, 'AVRO')) + sr.register_schema("award", Schema(Award.schema_str, 'AVRO', [award_properties_schema_ref])) + + references = [user_schema_ref, award_schema_ref] + schema = Schema(AwardedUser.schema_str, 'AVRO', references) + return awarded_user, schema + + +def _references_test_common(kafka_cluster, awarded_user, serializer_schema, deserializer_schema): + """ + Common (both reader and writer) avro schema reference test. + Args: + kafka_cluster (KafkaClusterFixture): cluster fixture + """ + topic = kafka_cluster.create_topic("reference-avro") + sr = kafka_cluster.schema_registry() + + value_serializer = AvroSerializer(sr, serializer_schema, + lambda user, ctx: + dict(award=dict(name=user.award.name, + properties=dict(year=user.award.properties.year, + points=user.award.properties.points)), + user=dict(name=user.user.name, + favorite_number=user.user.favorite_number, + favorite_color=user.user.favorite_color))) + + value_deserializer = \ + AvroDeserializer(sr, deserializer_schema, + lambda user, ctx: + AwardedUser(award=Award(name=user.get('award').get('name'), + properties=AwardProperties( + year=user.get('award').get('properties').get( + 'year'), + points=user.get('award').get('properties').get( + 'points'))), + user=User(name=user.get('user').get('name'), + favorite_number=user.get('user').get('favorite_number'), + favorite_color=user.get('user').get('favorite_color')))) + + producer = kafka_cluster.producer(value_serializer=value_serializer) + + producer.produce(topic, value=awarded_user, partition=0) + producer.flush() + + consumer = kafka_cluster.consumer(value_deserializer=value_deserializer) + consumer.assign([TopicPartition(topic, 0)]) + + msg = consumer.poll() + awarded_user2 = msg.value() + + assert awarded_user2 == awarded_user + + @pytest.mark.parametrize("avsc, data, record_type", [('basic_schema.avsc', {'name': 'abc'}, "record"), ('primitive_string.avsc', u'Jämtland', "string"), @@ -185,3 +324,25 @@ def test_avro_record_serialization_custom(kafka_cluster): user2 = msg.value() assert user2 == user + + +def test_avro_reference(kafka_cluster): + """ + Tests Avro schema reference with both serializer and deserializer schemas provided. + Args: + kafka_cluster (KafkaClusterFixture): cluster fixture + """ + awarded_user, schema = _register_avro_schemas_and_build_awarded_user_schema(kafka_cluster) + + _references_test_common(kafka_cluster, awarded_user, schema, schema) + + +def test_avro_reference_deserializer_none(kafka_cluster): + """ + Tests Avro schema reference with serializer schema provided and deserializer schema set to None. + Args: + kafka_cluster (KafkaClusterFixture): cluster fixture + """ + awarded_user, schema = _register_avro_schemas_and_build_awarded_user_schema(kafka_cluster) + + _references_test_common(kafka_cluster, awarded_user, schema, None) diff --git a/tests/integration/schema_registry/test_json_serializers.py b/tests/integration/schema_registry/test_json_serializers.py index f28bd7af3..3a60598d4 100644 --- a/tests/integration/schema_registry/test_json_serializers.py +++ b/tests/integration/schema_registry/test_json_serializers.py @@ -19,6 +19,7 @@ from confluent_kafka import TopicPartition from confluent_kafka.error import ConsumeError, ValueSerializationError +from confluent_kafka.schema_registry import SchemaReference, Schema from confluent_kafka.schema_registry.json_schema import (JSONSerializer, JSONDeserializer) @@ -32,6 +33,64 @@ def __init__(self, product_id, name, price, tags, dimensions, location): self.dimensions = dimensions self.location = location + def __eq__(self, other): + return all([ + self.product_id == other.product_id, + self.name == other.name, + self.price == other.price, + self.tags == other.tags, + self.dimensions == other.dimensions, + self.location == other.location + ]) + + +class _TestCustomer(object): + def __init__(self, name, id): + self.name = name + self.id = id + + def __eq__(self, other): + return all([ + self.name == other.name, + self.id == other.id + ]) + + +class _TestOrderDetails(object): + def __init__(self, id, customer): + self.id = id + self.customer = customer + + def __eq__(self, other): + return all([ + self.id == other.id, + self.customer == other.customer + ]) + + +class _TestOrder(object): + def __init__(self, order_details, product): + self.order_details = order_details + self.product = product + + def __eq__(self, other): + return all([ + self.order_details == other.order_details, + self.product == other.product + ]) + + +class _TestReferencedProduct(object): + def __init__(self, name, product): + self.name = name + self.product = product + + def __eq__(self, other): + return all([ + self.name == other.name, + self.product == other.product + ]) + def _testProduct_to_dict(product_obj, ctx): """ @@ -55,6 +114,60 @@ def _testProduct_to_dict(product_obj, ctx): "warehouseLocation": product_obj.location} +def _testCustomer_to_dict(customer_obj, ctx): + """ + Returns testCustomer instance in dict format. + + Args: + customer_obj (_TestCustomer): testCustomer instance. + + ctx (SerializationContext): Metadata pertaining to the serialization + operation. + + Returns: + dict: customer_obj as a dictionary. + + """ + return {"name": customer_obj.name, + "id": customer_obj.id} + + +def _testOrderDetails_to_dict(orderdetails_obj, ctx): + """ + Returns testOrderDetails instance in dict format. + + Args: + orderdetails_obj (_TestOrderDetails): testOrderDetails instance. + + ctx (SerializationContext): Metadata pertaining to the serialization + operation. + + Returns: + dict: orderdetails_obj as a dictionary. + + """ + return {"id": orderdetails_obj.id, + "customer": _testCustomer_to_dict(orderdetails_obj.customer, ctx)} + + +def _testOrder_to_dict(order_obj, ctx): + """ + Returns testOrder instance in dict format. + + Args: + order_obj (_TestOrder): testOrder instance. + + ctx (SerializationContext): Metadata pertaining to the serialization + operation. + + Returns: + dict: order_obj as a dictionary. + + """ + return {"order_details": _testOrderDetails_to_dict(order_obj.order_details, ctx), + "product": _testProduct_to_dict(order_obj.product, ctx)} + + def _testProduct_from_dict(product_dict, ctx): """ Returns testProduct instance from its dict format. @@ -77,6 +190,60 @@ def _testProduct_from_dict(product_dict, ctx): product_dict['warehouseLocation']) +def _testCustomer_from_dict(customer_dict, ctx): + """ + Returns testCustomer instance from its dict format. + + Args: + customer_dict (dict): testCustomer in dict format. + + ctx (SerializationContext): Metadata pertaining to the serialization + operation. + + Returns: + _TestCustomer: customer_obj instance. + + """ + return _TestCustomer(customer_dict['name'], + customer_dict['id']) + + +def _testOrderDetails_from_dict(orderdetails_dict, ctx): + """ + Returns testOrderDetails instance from its dict format. + + Args: + orderdetails_dict (dict): testOrderDetails in dict format. + + ctx (SerializationContext): Metadata pertaining to the serialization + operation. + + Returns: + _TestOrderDetails: orderdetails_obj instance. + + """ + return _TestOrderDetails(orderdetails_dict['id'], + _testCustomer_from_dict(orderdetails_dict['customer'], ctx)) + + +def _testOrder_from_dict(order_dict, ctx): + """ + Returns testOrder instance from its dict format. + + Args: + order_dict (dict): testOrder in dict format. + + ctx (SerializationContext): Metadata pertaining to the serialization + operation. + + Returns: + _TestOrder: order_obj instance. + + """ + return _TestOrder(_testOrderDetails_from_dict(order_dict['order_details'], ctx), + _testProduct_from_dict(order_dict['product'], ctx)) + + def test_json_record_serialization(kafka_cluster, load_file): """ Tests basic JsonSerializer and JsonDeserializer basic functionality. @@ -253,3 +420,89 @@ def test_json_record_deserialization_mismatch(kafka_cluster, load_file): ConsumeError, match="'productId' is a required property"): consumer.poll() + + +def _register_referenced_schemas(sr, load_file): + sr.register_schema("product", Schema(load_file("product.json"), 'JSON')) + sr.register_schema("customer", Schema(load_file("customer.json"), 'JSON')) + sr.register_schema("order_details", Schema(load_file("order_details.json"), 'JSON', [ + SchemaReference("http://example.com/customer.schema.json", "customer", 1)])) + + order_schema = Schema(load_file("order.json"), 'JSON', + [SchemaReference("http://example.com/order_details.schema.json", "order_details", 1), + SchemaReference("http://example.com/product.schema.json", "product", 1)]) + return order_schema + + +def test_json_reference(kafka_cluster, load_file): + topic = kafka_cluster.create_topic("serialization-json") + sr = kafka_cluster.schema_registry() + + product = {"productId": 1, + "productName": "An ice sculpture", + "price": 12.50, + "tags": ["cold", "ice"], + "dimensions": { + "length": 7.0, + "width": 12.0, + "height": 9.5 + }, + "warehouseLocation": { + "latitude": -78.75, + "longitude": 20.4 + }} + customer = {"name": "John Doe", "id": 1} + order_details = {"id": 1, "customer": customer} + order = {"order_details": order_details, "product": product} + + schema = _register_referenced_schemas(sr, load_file) + + value_serializer = JSONSerializer(schema, sr) + value_deserializer = JSONDeserializer(schema, schema_registry_client=sr) + + producer = kafka_cluster.producer(value_serializer=value_serializer) + producer.produce(topic, value=order, partition=0) + producer.flush() + + consumer = kafka_cluster.consumer(value_deserializer=value_deserializer) + consumer.assign([TopicPartition(topic, 0)]) + + msg = consumer.poll() + actual = msg.value() + + assert all([actual[k] == v for k, v in order.items()]) + + +def test_json_reference_custom(kafka_cluster, load_file): + topic = kafka_cluster.create_topic("serialization-json") + sr = kafka_cluster.schema_registry() + + product = _TestProduct(product_id=1, + name="The ice sculpture", + price=12.50, + tags=["cold", "ice"], + dimensions={"length": 7.0, + "width": 12.0, + "height": 9.5}, + location={"latitude": -78.75, + "longitude": 20.4}) + customer = _TestCustomer(name="John Doe", id=1) + order_details = _TestOrderDetails(id=1, customer=customer) + order = _TestOrder(order_details=order_details, product=product) + + schema = _register_referenced_schemas(sr, load_file) + + value_serializer = JSONSerializer(schema, sr, to_dict=_testOrder_to_dict) + value_deserializer = JSONDeserializer(schema, schema_registry_client=sr, from_dict=_testOrder_from_dict) + + producer = kafka_cluster.producer(value_serializer=value_serializer) + producer.produce(topic, value=order, partition=0) + producer.flush() + + consumer = kafka_cluster.consumer(value_deserializer=value_deserializer) + consumer.assign([TopicPartition(topic, 0)]) + + msg = consumer.poll() + actual = msg.value() + + assert actual == order diff --git a/tests/integration/schema_registry/test_proto_serializers.py b/tests/integration/schema_registry/test_proto_serializers.py index 93f822f22..621beac43 100644 --- a/tests/integration/schema_registry/test_proto_serializers.py +++ b/tests/integration/schema_registry/test_proto_serializers.py @@ -38,7 +38,7 @@ 'test_float': 12.0}), (NestedTestProto_pb2.NestedMessage, {'user_id': NestedTestProto_pb2.UserId( - kafka_user_id='oneof_str'), + kafka_user_id='oneof_str'), 'is_active': True, 'experiments_active': ['x', 'y', '1'], 'status': NestedTestProto_pb2.INACTIVE, diff --git a/tests/integration/testconf.json b/tests/integration/testconf.json index 1bc02bc74..15b9ca375 100644 --- a/tests/integration/testconf.json +++ b/tests/integration/testconf.json @@ -3,15 +3,22 @@ "bootstrap.servers": "$MY_BOOTSTRAP_SERVER_ENV", "schema.registry.url": "$MY_SCHEMA_REGISTRY_URL_ENV", "avro-https": { - "schema.registry.url": "$MY_SCHEMA_REGISTRY_SSL_URL_ENV", - "schema.registry.ssl.ca.location": "$MY_SCHEMA_REGISTRY_SSL_CA_LOCATION_ENV", - "schema.registry.ssl.certificate.location": "$MY_SCHEMA_REGISTRY_SSL_CERTIFICATE_LOCATION_ENV", - "schema.registry.ssl.key.location": "$MY_SCHEMA_REGISTRY_SSL_KEY_LOCATION_ENV" + "schema.registry.url": "$MY_SCHEMA_REGISTRY_SSL_URL_ENV", + "schema.registry.ssl.ca.location": "$MY_SCHEMA_REGISTRY_SSL_CA_LOCATION_ENV", + "schema.registry.ssl.certificate.location": "$MY_SCHEMA_REGISTRY_SSL_CERTIFICATE_LOCATION_ENV", + "schema.registry.ssl.key.location": "$MY_SCHEMA_REGISTRY_SSL_KEY_LOCATION_ENV" }, "avro-basic-auth": { "schema.registry.url": "http://localhost:8083", "schema.registry.basic.auth.user.info": "ckp_tester:test_secret", "sasl.username": "ckp_tester", "sasl.password": "test_secret" + }, + "avro-https-key-with-password": { + "schema.registry.url": "$MY_SCHEMA_REGISTRY_SSL_URL_ENV", + "schema.registry.ssl.ca.location": "$MY_SCHEMA_REGISTRY_SSL_CA_LOCATION_ENV", + "schema.registry.ssl.certificate.location": "$MY_SCHEMA_REGISTRY_SSL_CERTIFICATE_LOCATION_ENV", + "schema.registry.ssl.key.location": "$MY_SCHEMA_REGISTRY_SSL_KEY_WITH_PASSWORD_LOCATION_ENV", + "schema.registry.ssl.key.password": "$MY_SCHEMA_REGISTRY_SSL_KEY_PASSWORD" } } diff --git a/tests/schema_registry/test_avro_serializer.py b/tests/schema_registry/test_avro_serializer.py index 1a4042229..b9edc69ae 100644 --- a/tests/schema_registry/test_avro_serializer.py +++ b/tests/schema_registry/test_avro_serializer.py @@ -20,7 +20,7 @@ from confluent_kafka.schema_registry import (record_subject_name_strategy, SchemaRegistryClient, topic_record_subject_name_strategy) -from confluent_kafka.schema_registry.avro import AvroSerializer +from confluent_kafka.schema_registry.avro import AvroSerializer, AvroDeserializer from confluent_kafka.serialization import (MessageField, SerializationContext) @@ -221,3 +221,23 @@ def test_avro_serializer_schema_loads_union(load_avsc): assert isinstance(schema, list) assert schema[0]["name"] == "RecordOne" assert schema[1]["name"] == "RecordTwo" + + +def test_avro_serializer_invalid_schema_type(): + """ + Ensures invalid schema types are rejected + """ + conf = {'url': TEST_URL} + test_client = SchemaRegistryClient(conf) + with pytest.raises(TypeError, match="You must pass either schema string or schema object"): + AvroSerializer(test_client, 1) + + +def test_avro_deserializer_invalid_schema_type(): + """ + Ensures invalid schema types are rejected + """ + conf = {'url': TEST_URL} + test_client = SchemaRegistryClient(conf) + with pytest.raises(TypeError, match="You must pass either schema string or schema object"): + AvroDeserializer(test_client, 1) diff --git a/tests/schema_registry/test_json.py b/tests/schema_registry/test_json.py new file mode 100644 index 000000000..877170585 --- /dev/null +++ b/tests/schema_registry/test_json.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2023 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +from confluent_kafka.schema_registry import SchemaReference, Schema +from confluent_kafka.schema_registry.json_schema import JSONDeserializer, JSONSerializer + + +def test_json_deserializer_referenced_schema_no_schema_registry_client(load_avsc): + """ + Ensures that the deserializer raises a ValueError if a referenced schema is provided but no schema registry + client is provided. + """ + schema = Schema(load_avsc("order_details.json"), 'JSON', + [SchemaReference("http://example.com/customer.schema.json", "customer", 1)]) + with pytest.raises( + ValueError, + match="""schema_registry_client must be provided if "schema_str" is a Schema instance with references"""): + JSONDeserializer(schema, schema_registry_client=None) + + +def test_json_deserializer_invalid_schema_type(): + """ + Ensures that the deserializer raises a ValueError if an invalid schema type is provided. + """ + with pytest.raises(TypeError, match="You must pass either str or Schema"): + JSONDeserializer(1) + + +def test_json_serializer_invalid_schema_type(): + """ + Ensures that the serializer raises a ValueError if an invalid schema type is provided. + """ + with pytest.raises(TypeError, match="You must pass either str or Schema"): + JSONSerializer(1, schema_registry_client=None) diff --git a/tests/soak/README.md b/tests/soak/README.md index 9eae249fb..a26a2fc45 100644 --- a/tests/soak/README.md +++ b/tests/soak/README.md @@ -10,4 +10,22 @@ DataDog reporting supported by setting datadog.api_key a and datadog.app_key in the soak client configuration file. -Use ubuntu-bootstrap.sh in this directory set up the environment (e.g., on ec2). +There are some convenience script to get you started. + +On the host (ec2) where you aim to run the soaktest, do: + +$ git clone https://github.com/confluentinc/librdkafka +$ git clone https://github.com/confluentinc/confluent-kafka-python + +# Build librdkafka and python +$ ~/confluent-kafka-python/tests/soak/build.sh + +# Set up config: +$ cp ~/confluent-kafka-python/tests/soak/ccloud.config.example ~/confluent-kafka-python/ccloud.config + +# Start a screen session +$ screen bash + +# Within the screen session, run the soak client +(screen)$ ~/run.sh +(screen)$ Ctrl-A d # to detach diff --git a/tests/soak/build.sh b/tests/soak/build.sh new file mode 100755 index 000000000..795dcfb69 --- /dev/null +++ b/tests/soak/build.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# + +librdkafka_version=$1 +cflpy_version=$2 + +if [[ -z $cflpy_version ]]; then + echo "Usage: $0 " + exit 1 +fi + +set -eu + + + +echo "Building and installing librdkafka $librdkafka_version" +pushd librdkafka +sudo make uninstall +git fetch --tags +git checkout $librdkafka_version +./configure --reconfigure +make clean +make -j +sudo make install +popd + + +echo "Building confluent-kafka-python $cflpy_version" +set +u +source venv/bin/activate +set -u +pushd confluent-kafka-python +git fetch --tags +git checkout $cflpy_version +python3 setup.py clean -a +python3 setup.py build +python3 -m pip install . +popd + +echo "" +echo "==============================================================================" +(cd / ; python3 -c 'import confluent_kafka as c; print("python", c.version(), "librdkafka", c.libversion())') + diff --git a/tests/soak/ccloud.config.example b/tests/soak/ccloud.config.example new file mode 100644 index 000000000..328642a22 --- /dev/null +++ b/tests/soak/ccloud.config.example @@ -0,0 +1,14 @@ +bootstrap.servers= +sasl.mechanisms=PLAIN +security.protocol=SASL_SSL +sasl.username= +sasl.password= +enable.idempotence=true +debug=eos,generic,broker,security,consumer +linger.ms=2 +compression.type=lz4 +# DataDog options/config +datadog.api_key= +datadog.app_key= + + diff --git a/tests/soak/run.sh b/tests/soak/run.sh new file mode 100755 index 000000000..bac5fa822 --- /dev/null +++ b/tests/soak/run.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# + +set -e +source venv/bin/activate + +librdkafka_version=$(python3 -c 'from confluent_kafka import libversion; print(libversion()[0])') + +if [[ -z $librdkafka_version ]]; then + echo "No librdkafka version found.." + exit 1 +fi + +if [[ -z $STY ]]; then + echo "This script should be run from inside a screen session" + exit 1 +fi + +set -u +topic="pysoak-$librdkafka_version" +logfile="${topic}.log.bz2" + +echo "Starting soak client using topic $topic with logs written to $logfile" +set +x +time confluent-kafka-python/tests/soak/soakclient.py -t $topic -r 80 -f confluent-kafka-python/ccloud.config 2>&1 \ + | tee /dev/stderr | bzip2 > $logfile +ret=$? +echo "Python client exited with status $ret" +exit $ret + + diff --git a/tests/soak/soakclient.py b/tests/soak/soakclient.py index 8fdfedf26..e7e914cee 100755 --- a/tests/soak/soakclient.py +++ b/tests/soak/soakclient.py @@ -45,6 +45,7 @@ class SoakRecord (object): """ A private record type, with JSON serializer and deserializer """ + def __init__(self, msgid, name=None): self.msgid = msgid if name is None: @@ -222,7 +223,7 @@ def consumer_run(self): try: # Deserialize message - record = SoakRecord.deserialize(msg.value()) # noqa unused variable + record = SoakRecord.deserialize(msg.value()) # noqa unused variable except ValueError as ex: self.logger.info("consumer: Failed to deserialize message in " "{} [{}] at offset {} (headers {}): {}".format( diff --git a/tests/test_Admin.py b/tests/test_Admin.py index f50685bd9..fe8ac6719 100644 --- a/tests/test_Admin.py +++ b/tests/test_Admin.py @@ -2,9 +2,13 @@ import pytest from confluent_kafka.admin import AdminClient, NewTopic, NewPartitions, \ - ConfigResource, AclBinding, AclBindingFilter, ResourceType, ResourcePatternType, \ - AclOperation, AclPermissionType -from confluent_kafka import KafkaException, KafkaError, libversion + ConfigResource, ConfigEntry, AclBinding, AclBindingFilter, ResourceType, \ + ResourcePatternType, AclOperation, AclPermissionType, AlterConfigOpType, \ + ScramCredentialInfo, ScramMechanism, \ + UserScramCredentialAlteration, UserScramCredentialDeletion, \ + UserScramCredentialUpsertion +from confluent_kafka import KafkaException, KafkaError, libversion, \ + TopicPartition, ConsumerGroupTopicPartitions, ConsumerGroupState import concurrent.futures @@ -121,6 +125,11 @@ def test_create_topics_api(): with pytest.raises(Exception): a.create_topics([None, NewTopic("mytopic", 1, 2)]) + try: + a.create_topics([NewTopic("mytopic")]) + except Exception as err: + assert False, f"When none of the partitions, \ + replication and assignment is present, the request should not fail, but it does with error {err}" fs = a.create_topics([NewTopic("mytopic", 3, 2)]) with pytest.raises(KafkaException): for f in concurrent.futures.as_completed(iter(fs.values())): @@ -314,6 +323,142 @@ def test_alter_configs_api(): f.result(timeout=1) +def verify_incremental_alter_configs_api_call(a, + restype, resname, + incremental_configs, + error, + constructor_param=True): + if constructor_param: + resources = [ConfigResource(restype, resname, + incremental_configs=incremental_configs)] + else: + resources = [ConfigResource(restype, resname)] + for config_entry in incremental_configs: + resources[0].add_incremental_config(config_entry) + + if error: + with pytest.raises(error): + fs = a.incremental_alter_configs(resources) + for f in concurrent.futures.as_completed(iter(fs.values())): + f.result(timeout=1) + else: + fs = a.incremental_alter_configs(resources) + for f in concurrent.futures.as_completed(iter(fs.values())): + f.result(timeout=1) + + +def test_incremental_alter_configs_api(): + a = AdminClient({"socket.timeout.ms": 10}) + + with pytest.raises(TypeError): + a.incremental_alter_configs(None) + + with pytest.raises(TypeError): + a.incremental_alter_configs("something") + + with pytest.raises(ValueError): + a.incremental_alter_configs([]) + + for use_constructor in [True, False]: + # incremental_operation not of type AlterConfigOpType + verify_incremental_alter_configs_api_call(a, ResourceType.BROKER, "1", + [ + ConfigEntry("advertised.listeners", + "host1", + incremental_operation="NEW_OPERATION") + ], + TypeError, + use_constructor) + # None name + verify_incremental_alter_configs_api_call(a, ResourceType.BROKER, "1", + [ + ConfigEntry(None, + "host1", + incremental_operation=AlterConfigOpType.APPEND) + ], + TypeError, + use_constructor) + + # name type + verify_incremental_alter_configs_api_call(a, ResourceType.BROKER, "1", + [ + ConfigEntry(5, + "host1", + incremental_operation=AlterConfigOpType.APPEND) + ], + TypeError, + use_constructor) + + # Empty list + verify_incremental_alter_configs_api_call(a, ResourceType.BROKER, "1", + [], + ValueError, + use_constructor) + + # String instead of ConfigEntry list, treated as an iterable + verify_incremental_alter_configs_api_call(a, ResourceType.BROKER, "1", + "something", + TypeError, + use_constructor) + + # Duplicate ConfigEntry found + verify_incremental_alter_configs_api_call(a, ResourceType.BROKER, "1", + [ + ConfigEntry( + name="advertised.listeners", + value="host1:9092", + incremental_operation=AlterConfigOpType.APPEND + ), + ConfigEntry( + name="advertised.listeners", + value=None, + incremental_operation=AlterConfigOpType.DELETE + ) + ], + KafkaException, + use_constructor) + + # Request timeout + verify_incremental_alter_configs_api_call(a, ResourceType.BROKER, "1", + [ + ConfigEntry( + name="advertised.listeners", + value="host1:9092", + incremental_operation=AlterConfigOpType.APPEND + ), + ConfigEntry( + name="background.threads", + value=None, + incremental_operation=AlterConfigOpType.DELETE + ) + ], + KafkaException, + use_constructor) + + # Positive test that times out + resources = [ConfigResource(ResourceType.BROKER, "1"), + ConfigResource(ResourceType.TOPIC, "test2")] + + resources[0].add_incremental_config( + ConfigEntry("advertised.listeners", "host:9092", + incremental_operation=AlterConfigOpType.SUBTRACT)) + resources[0].add_incremental_config( + ConfigEntry("background.threads", None, + incremental_operation=AlterConfigOpType.DELETE)) + resources[1].add_incremental_config( + ConfigEntry("cleanup.policy", "compact", + incremental_operation=AlterConfigOpType.APPEND)) + resources[1].add_incremental_config( + ConfigEntry("retention.ms", "10000", + incremental_operation=AlterConfigOpType.SET)) + + fs = a.incremental_alter_configs(resources) + + with pytest.raises(KafkaException): + for f in concurrent.futures.as_completed(iter(fs.values())): + f.result(timeout=1) + + def test_create_acls_api(): """ create_acls() tests, these wont really do anything since there is no broker configured. """ @@ -377,9 +522,9 @@ def test_delete_acls_api(): a = AdminClient({"socket.timeout.ms": 10}) - acl_binding_filter1 = AclBindingFilter(ResourceType.ANY, None, ResourcePatternType.ANY, + acl_binding_filter1 = AclBindingFilter(ResourceType.ANY, None, ResourcePatternType.ANY, None, None, AclOperation.ANY, AclPermissionType.ANY) - acl_binding_filter2 = AclBindingFilter(ResourceType.ANY, "topic2", ResourcePatternType.MATCH, + acl_binding_filter2 = AclBindingFilter(ResourceType.ANY, "topic2", ResourcePatternType.MATCH, None, "*", AclOperation.WRITE, AclPermissionType.ALLOW) fs = a.delete_acls([acl_binding_filter1]) @@ -424,7 +569,7 @@ def test_describe_acls_api(): a = AdminClient({"socket.timeout.ms": 10}) - acl_binding_filter1 = AclBindingFilter(ResourceType.ANY, None, ResourcePatternType.ANY, + acl_binding_filter1 = AclBindingFilter(ResourceType.ANY, None, ResourcePatternType.ANY, None, None, AclOperation.ANY, AclPermissionType.ANY) acl_binding1 = AclBinding(ResourceType.TOPIC, "topic1", ResourcePatternType.LITERAL, "User:u1", "*", AclOperation.WRITE, AclPermissionType.ALLOW) @@ -455,3 +600,390 @@ def test_describe_acls_api(): with pytest.raises(TypeError): a.describe_acls(acl_binding_filter1, unknown_operation="it is") + + +def test_list_consumer_groups_api(): + a = AdminClient({"socket.timeout.ms": 10}) + + a.list_consumer_groups() + + a.list_consumer_groups(states={ConsumerGroupState.EMPTY, ConsumerGroupState.STABLE}) + + with pytest.raises(TypeError): + a.list_consumer_groups(states="EMPTY") + + with pytest.raises(TypeError): + a.list_consumer_groups(states=["EMPTY"]) + + with pytest.raises(TypeError): + a.list_consumer_groups(states=[ConsumerGroupState.EMPTY, ConsumerGroupState.STABLE]) + + +def test_describe_consumer_groups_api(): + a = AdminClient({"socket.timeout.ms": 10}) + + group_ids = ["test-group-1", "test-group-2"] + + a.describe_consumer_groups(group_ids) + + with pytest.raises(TypeError): + a.describe_consumer_groups("test-group-1") + + with pytest.raises(ValueError): + a.describe_consumer_groups([]) + + +def test_delete_consumer_groups_api(): + a = AdminClient({"socket.timeout.ms": 10}) + + group_ids = ["test-group-1", "test-group-2"] + + a.delete_consumer_groups(group_ids) + + with pytest.raises(TypeError): + a.delete_consumer_groups("test-group-1") + + with pytest.raises(ValueError): + a.delete_consumer_groups([]) + + +def test_list_consumer_group_offsets_api(): + + a = AdminClient({"socket.timeout.ms": 10}) + + only_group_id_request = ConsumerGroupTopicPartitions("test-group1") + request_with_group_and_topic_partition = ConsumerGroupTopicPartitions( + "test-group2", [TopicPartition("test-topic1", 1)]) + same_name_request = ConsumerGroupTopicPartitions("test-group2", [TopicPartition("test-topic1", 3)]) + + a.list_consumer_group_offsets([only_group_id_request]) + + with pytest.raises(TypeError): + a.list_consumer_group_offsets(None) + + with pytest.raises(TypeError): + a.list_consumer_group_offsets(1) + + with pytest.raises(TypeError): + a.list_consumer_group_offsets("") + + with pytest.raises(ValueError): + a.list_consumer_group_offsets([]) + + with pytest.raises(ValueError): + a.list_consumer_group_offsets([only_group_id_request, + request_with_group_and_topic_partition]) + + with pytest.raises(ValueError): + a.list_consumer_group_offsets([request_with_group_and_topic_partition, + same_name_request]) + + fs = a.list_consumer_group_offsets([only_group_id_request]) + with pytest.raises(KafkaException): + for f in fs.values(): + f.result(timeout=10) + + fs = a.list_consumer_group_offsets([only_group_id_request], + request_timeout=0.5) + for f in concurrent.futures.as_completed(iter(fs.values())): + e = f.exception(timeout=1) + assert isinstance(e, KafkaException) + assert e.args[0].code() == KafkaError._TIMED_OUT + + with pytest.raises(ValueError): + a.list_consumer_group_offsets([only_group_id_request], + request_timeout=-5) + + with pytest.raises(TypeError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions()]) + + with pytest.raises(TypeError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions(1)]) + + with pytest.raises(TypeError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions(None)]) + + with pytest.raises(TypeError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions([])]) + + with pytest.raises(ValueError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions("")]) + + with pytest.raises(TypeError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", "test-topic")]) + + with pytest.raises(ValueError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", [])]) + + with pytest.raises(ValueError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", [None])]) + + with pytest.raises(TypeError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", ["test"])]) + + with pytest.raises(TypeError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", [TopicPartition(None)])]) + + with pytest.raises(ValueError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", [TopicPartition("")])]) + + with pytest.raises(ValueError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions( + "test-group1", [TopicPartition("test-topic", -1)])]) + + with pytest.raises(ValueError): + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions( + "test-group1", [TopicPartition("test-topic", 1, 1)])]) + + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1")]) + a.list_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group2", [TopicPartition("test-topic1", 1)])]) + + +def test_alter_consumer_group_offsets_api(): + + a = AdminClient({"socket.timeout.ms": 10}) + + request_with_group_and_topic_partition_offset1 = ConsumerGroupTopicPartitions( + "test-group1", [TopicPartition("test-topic1", 1, 5)]) + same_name_request = ConsumerGroupTopicPartitions("test-group1", [TopicPartition("test-topic2", 4, 3)]) + request_with_group_and_topic_partition_offset2 = ConsumerGroupTopicPartitions( + "test-group2", [TopicPartition("test-topic2", 1, 5)]) + + a.alter_consumer_group_offsets([request_with_group_and_topic_partition_offset1]) + + with pytest.raises(TypeError): + a.alter_consumer_group_offsets(None) + + with pytest.raises(TypeError): + a.alter_consumer_group_offsets(1) + + with pytest.raises(TypeError): + a.alter_consumer_group_offsets("") + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([]) + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([request_with_group_and_topic_partition_offset1, + request_with_group_and_topic_partition_offset2]) + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([request_with_group_and_topic_partition_offset1, + same_name_request]) + + fs = a.alter_consumer_group_offsets([request_with_group_and_topic_partition_offset1]) + with pytest.raises(KafkaException): + for f in fs.values(): + f.result(timeout=10) + + fs = a.alter_consumer_group_offsets([request_with_group_and_topic_partition_offset1], + request_timeout=0.5) + for f in concurrent.futures.as_completed(iter(fs.values())): + e = f.exception(timeout=1) + assert isinstance(e, KafkaException) + assert e.args[0].code() == KafkaError._TIMED_OUT + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([request_with_group_and_topic_partition_offset1], + request_timeout=-5) + + with pytest.raises(TypeError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions()]) + + with pytest.raises(TypeError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions(1)]) + + with pytest.raises(TypeError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions(None)]) + + with pytest.raises(TypeError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions([])]) + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions("")]) + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1")]) + + with pytest.raises(TypeError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", "test-topic")]) + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", [])]) + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", [None])]) + + with pytest.raises(TypeError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", ["test"])]) + + with pytest.raises(TypeError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", [TopicPartition(None)])]) + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", [TopicPartition("")])]) + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions("test-group1", [TopicPartition("test-topic")])]) + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions( + "test-group1", [TopicPartition("test-topic", -1)])]) + + with pytest.raises(ValueError): + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions( + "test-group1", [TopicPartition("test-topic", 1, -1001)])]) + + a.alter_consumer_group_offsets([ConsumerGroupTopicPartitions( + "test-group2", [TopicPartition("test-topic1", 1, 23)])]) + + +def test_describe_user_scram_credentials_api(): + # Describe User Scram API + a = AdminClient({"socket.timeout.ms": 10}) + + with pytest.raises(TypeError): + a.describe_user_scram_credentials(10) + with pytest.raises(TypeError): + a.describe_user_scram_credentials(None) + with pytest.raises(TypeError): + a.describe_user_scram_credentials([None]) + with pytest.raises(ValueError): + a.describe_user_scram_credentials([""]) + with pytest.raises(KafkaException) as ex: + futmap = a.describe_user_scram_credentials(["sam", "sam"]) + futmap["sam"].result(timeout=3) + assert "Duplicate users" in str(ex.value) + + fs = a.describe_user_scram_credentials(["user1", "user2"]) + for f in concurrent.futures.as_completed(iter(fs.values())): + e = f.exception(timeout=1) + assert isinstance(e, KafkaException) + assert e.args[0].code() == KafkaError._TIMED_OUT + + +def test_alter_user_scram_credentials_api(): + # Alter User Scram API + a = AdminClient({"socket.timeout.ms": 10}) + + scram_credential_info = ScramCredentialInfo(ScramMechanism.SCRAM_SHA_512, 10000) + upsertion = UserScramCredentialUpsertion("sam", scram_credential_info, b"password", b"salt") + upsertion_without_salt = UserScramCredentialUpsertion("sam", scram_credential_info, b"password") + upsertion_with_none_salt = UserScramCredentialUpsertion("sam", scram_credential_info, b"password", None) + deletion = UserScramCredentialDeletion("sam", ScramMechanism.SCRAM_SHA_512) + alterations = [upsertion, upsertion_without_salt, upsertion_with_none_salt, deletion] + + fs = a.alter_user_scram_credentials(alterations) + for f in concurrent.futures.as_completed(iter(fs.values())): + e = f.exception(timeout=1) + assert isinstance(e, KafkaException) + assert e.args[0].code() == KafkaError._TIMED_OUT + + # Request type tests + with pytest.raises(TypeError): + a.alter_user_scram_credentials(None) + with pytest.raises(TypeError): + a.alter_user_scram_credentials(234) + with pytest.raises(TypeError): + a.alter_user_scram_credentials("test") + + # Individual request tests + with pytest.raises(ValueError): + a.alter_user_scram_credentials([]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([None]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials(["test"]) + + # User tests + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialAlteration(None)]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialAlteration(123)]) + with pytest.raises(ValueError): + a.alter_user_scram_credentials([UserScramCredentialAlteration("")]) + + # Upsertion request user test + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion(None, + scram_credential_info, + b"password", + b"salt")]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion(123, + scram_credential_info, + b"password", + b"salt")]) + with pytest.raises(ValueError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("", + scram_credential_info, + b"password", + b"salt")]) + + # Upsertion password user test + with pytest.raises(ValueError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", scram_credential_info, b"", b"salt")]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", scram_credential_info, None, b"salt")]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", + scram_credential_info, + "password", + b"salt")]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", scram_credential_info, 123, b"salt")]) + + # Upsertion salt user test + with pytest.raises(ValueError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", scram_credential_info, b"password", b"")]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", + scram_credential_info, + b"password", + "salt")]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", scram_credential_info, b"password", 123)]) + + # Upsertion scram_credential_info tests + sci_incorrect_mechanism_type = ScramCredentialInfo("string type", 10000) + sci_incorrect_iteration_type = ScramCredentialInfo(ScramMechanism.SCRAM_SHA_512, "string type") + sci_negative_iteration = ScramCredentialInfo(ScramMechanism.SCRAM_SHA_512, -1) + sci_zero_iteration = ScramCredentialInfo(ScramMechanism.SCRAM_SHA_512, 0) + + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", None, b"password", b"salt")]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", "string type", b"password", b"salt")]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", + sci_incorrect_mechanism_type, + b"password", + b"salt")]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", + sci_incorrect_iteration_type, + b"password", + b"salt")]) + with pytest.raises(ValueError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", + sci_negative_iteration, + b"password", + b"salt")]) + with pytest.raises(ValueError): + a.alter_user_scram_credentials([UserScramCredentialUpsertion("sam", sci_zero_iteration, b"password", b"salt")]) + + # Deletion user tests + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialDeletion(None, ScramMechanism.SCRAM_SHA_256)]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialDeletion(123, ScramMechanism.SCRAM_SHA_256)]) + with pytest.raises(ValueError): + a.alter_user_scram_credentials([UserScramCredentialDeletion("", ScramMechanism.SCRAM_SHA_256)]) + + # Deletion mechanism tests + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialDeletion("sam", None)]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialDeletion("sam", "string type")]) + with pytest.raises(TypeError): + a.alter_user_scram_credentials([UserScramCredentialDeletion("sam", 123)]) diff --git a/tests/test_Producer.py b/tests/test_Producer.py index 41eac1f0c..4c45c5683 100644 --- a/tests/test_Producer.py +++ b/tests/test_Producer.py @@ -271,3 +271,13 @@ def on_delivery(err, msg): p.purge() p.flush(0.002) assert cb_detector["on_delivery_called"] + + +def test_producer_bool_value(): + """ + Make sure producer has a truth-y bool value + See https://github.com/confluentinc/confluent-kafka-python/issues/1427 + """ + + p = Producer({}) + assert bool(p) diff --git a/tests/test_misc.py b/tests/test_misc.py index ae016a3a9..aca7b5a4f 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -262,3 +262,21 @@ def on_delivery(err, msg): if "CI" in os.environ: pytest.xfail("Timeout exceeded") pytest.fail("Timeout exceeded") + + +def test_set_sasl_credentials_api(): + clients = [ + AdminClient({}), + confluent_kafka.Consumer({"group.id": "dummy"}), + confluent_kafka.Producer({})] + + for c in clients: + c.set_sasl_credentials('username', 'password') + + c.set_sasl_credentials('override', 'override') + + with pytest.raises(TypeError): + c.set_sasl_credentials(None, 'password') + + with pytest.raises(TypeError): + c.set_sasl_credentials('username', None) diff --git a/tools/RELEASE.md b/tools/RELEASE.md index 2a03b4873..72bc622d2 100644 --- a/tools/RELEASE.md +++ b/tools/RELEASE.md @@ -8,10 +8,10 @@ confluent-kafka-python uses semver versioning and loosely follows librdkafka's version, e.g. v0.11.4 for the final release and v0.11.4rc3 for the 3rd v0.11.4 release candidate. -With the addition of prebuilt binary wheels we make use of travis-ci.org -to build OSX, Linux and Winodws binaries which are uploaded to Confluent's -private S3 bucket. These artifacts are downloaded by the `tools/download-s3.py` script -and then uploaded manually to PyPi. +With the addition of prebuilt binary wheels we make use of Semaphore CI +to build OSX, Linux and Windows binaries which are uploaded to build's +artifact directory. These artifacts are downloaded and then uploaded manually +to PyPi. **Note**: Python package versions use a lowercase `rcN` suffix to indicate release candidates while librdkafka uses `-RCN`. The Python format @@ -116,7 +116,7 @@ tag (e.g., v0.11.4-RC5). Change to the latest librdkafka version in the following files: - * `.travis.yml` + * `.semaphore/semaphore.yml` * `examples/docker/Dockerfile.alpine` Change to the latest version of the confluent-librdkafka-plugins in (this step @@ -126,7 +126,7 @@ is usually not necessary): Commit these changes as necessary: - $ git commit -m "librdkafka version v0.11.4-RC5" .travis.yml examples/docker/Dockerfile.alpine + $ git commit -m "librdkafka version v0.11.4-RC5" .semaphore/semaphore.yml examples/docker/Dockerfile.alpine $ git commit -m "confluent-librdkafka-plugins version v0.11.0" tools/install-interceptors.sh @@ -183,10 +183,10 @@ be removed after the build passes. **TEST ITERATION**: # Repeat with new tags until all build issues are solved. - $ git tag v0.11.4rc1-test2 + $ git tag v0.11.4rc1-dev2 # Delete any previous test tag you've created. - $ git tag tag -d v0.11.4rc1-test1 + $ git tag tag -d v0.11.4rc1-dev1 **CANDIDATE ITERATION**: @@ -221,8 +221,8 @@ Remove `--dry-run` when you're happy with the results. ### 5.3. Wait for CI builds to complete -Monitor travis-ci builds by looking at the *tag* build at -[travis-ci](https://travis-ci.org/confluentinc/confluent-kafka-python) +Monitor Semaphore CI builds by looking at the *tag* build at +[Semaphore CI](https://confluentinc.semaphoreci.com/projects/confluent-kafka-python) CI jobs are flaky and may fail temporarily. If you see a temporary build error, e.g., a timeout, restart the specific job. @@ -231,18 +231,13 @@ If there are permanent errors, fix them and then go back to 5.1. to create and push a new test tag. Don't forget to delete your previous test tag. -### 5.4. Download build artifacts from S3 - -*Note*: You will need set up your AWS credentials in `~/.aws/credentials` to - gain access to the S3 bucket. +### 5.4. Download build artifacts When all CI builds are successful it is time to download the resulting -artifacts from S3 using: - - $ tools/download-s3.py v0.11.4rc1 # replace with your tagged version - -The artifacts will be downloaded to `dl-/`. +artifacts from build's Artifact directory located in another tab in the build: +**Note:** The artifacts should be extracted in the folder `tools\dl-` for +subsequent steps to work properly. ### 5.5. Verify packages diff --git a/tools/bootstrap-librdkafka.sh b/tools/bootstrap-librdkafka.sh index bbfc6b48e..a8c0e01f3 100755 --- a/tools/bootstrap-librdkafka.sh +++ b/tools/bootstrap-librdkafka.sh @@ -29,7 +29,7 @@ mkdir -p "$BUILDDIR/librdkafka" pushd "$BUILDDIR/librdkafka" test -f configure || -curl -q -L "https://github.com/edenhill/librdkafka/archive/${VERSION}.tar.gz" | \ +curl -q -L "https://github.com/confluentinc/librdkafka/archive/refs/tags/${VERSION}.tar.gz" | \ tar -xz --strip-components=1 -f - ./configure --clean diff --git a/tools/build-manylinux.sh b/tools/build-manylinux.sh index 92e9dab75..56c414bc7 100755 --- a/tools/build-manylinux.sh +++ b/tools/build-manylinux.sh @@ -15,6 +15,7 @@ # docker run -t -v $(pwd):/io quay.io/pypa/manylinux2010_x86_64:latest /io/tools/build-manylinux.sh LIBRDKAFKA_VERSION=$1 +PYTHON_VERSIONS=("cp36" "cp37" "cp38" "cp39" "cp310" "cp311" ) if [[ -z "$LIBRDKAFKA_VERSION" ]]; then echo "Usage: $0 " @@ -33,7 +34,13 @@ if [[ ! -f /.dockerenv ]]; then exit 1 fi - docker run -t -v $(pwd):/io quay.io/pypa/manylinux2010_x86_64:latest /io/tools/build-manylinux.sh "$LIBRDKAFKA_VERSION" + if [[ $ARCH == arm64* ]]; then + docker_image=quay.io/pypa/manylinux_2_28_aarch64:latest + else + docker_image=quay.io/pypa/manylinux_2_28_x86_64:latest + fi + + docker run -t -v $(pwd):/io $docker_image /io/tools/build-manylinux.sh "v${LIBRDKAFKA_VERSION}" exit $? fi @@ -44,17 +51,22 @@ fi # echo "# Installing basic system dependencies" -yum install -y zlib-devel gcc-c++ +yum install -y zlib-devel gcc-c++ python3 curl-devel perl-IPC-Cmd perl-Pod-Html echo "# Building librdkafka ${LIBRDKAFKA_VERSION}" $(dirname $0)/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} /usr # Compile wheels echo "# Compile" -for PYBIN in /opt/python/*/bin; do - echo "## Compiling $PYBIN" - CFLAGS="-Werror -Wno-strict-aliasing -Wno-parentheses" \ - "${PYBIN}/pip" wheel /io/ -w unrepaired-wheelhouse/ +for PYBIN in /opt/python/cp*/bin; do + for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do + if [[ $PYBIN == *"$PYTHON_VERSION"* ]]; then + echo "## Compiling $PYBIN" + CFLAGS="-Werror -Wno-strict-aliasing -Wno-parentheses" \ + "${PYBIN}/pip" wheel /io/ -w unrepaired-wheelhouse/ + break + fi + done done # Bundle external shared libraries into the wheels @@ -73,13 +85,17 @@ done # Install packages and test echo "# Installing wheels" -for PYBIN in /opt/python/*/bin/; do - echo "## Installing $PYBIN" - "${PYBIN}/pip" install confluent_kafka -f /io/wheelhouse - "${PYBIN}/python" -c 'import confluent_kafka; print(confluent_kafka.libversion())' - echo "## Uninstalling $PYBIN" - "${PYBIN}/pip" uninstall -y confluent_kafka +for PYBIN in /opt/python/cp*/bin/; do + for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do + if [[ $PYBIN == *"$PYTHON_VERSION"* ]]; then + echo "## Installing $PYBIN" + "${PYBIN}/pip" install confluent_kafka -f /io/wheelhouse + "${PYBIN}/python" -c 'import confluent_kafka; print(confluent_kafka.libversion())' + "${PYBIN}/pip" install -r /io/tests/requirements.txt + "${PYBIN}/pytest" /io/tests/test_Producer.py + echo "## Uninstalling $PYBIN" + "${PYBIN}/pip" uninstall -y confluent_kafka + break + fi + done done - - - diff --git a/tools/mingw-w64/msys2-dependencies.sh b/tools/mingw-w64/msys2-dependencies.sh new file mode 100644 index 000000000..7baedc724 --- /dev/null +++ b/tools/mingw-w64/msys2-dependencies.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +export msys2='cmd //C RefreshEnv.cmd ' +export msys2+='& set MSYS=winsymlinks:nativestrict ' +export msys2+='& C:\\msys64\\msys2_shell.cmd -defterm -no-start' +export mingw64="$msys2 -mingw64 -full-path -here -c "\"\$@"\" --" +export msys2+=" -msys2 -c "\"\$@"\" --" + +# Have to update pacman first or choco upgrade will failure due to migration +# to zstd instead of xz compression +$msys2 pacman -Sy --noconfirm pacman + +## Install more MSYS2 packages from https://packages.msys2.org/base here +$msys2 pacman --sync --noconfirm --needed mingw-w64-x86_64-gcc + +## Install unzip +$msys2 pacman --sync --noconfirm --needed unzip diff --git a/tools/mingw-w64/semaphore_commands.sh b/tools/mingw-w64/semaphore_commands.sh new file mode 100644 index 000000000..1c93901fb --- /dev/null +++ b/tools/mingw-w64/semaphore_commands.sh @@ -0,0 +1,11 @@ +#!/bin/bash +$msys2 pacman -S python --version 3.8.0 + +set -e + +export PATH="$PATH;C:\Python38;C:\Python38\Scripts" +export MAKE=mingw32-make # so that Autotools can find it + +cmd /c mklink /D C:\Python38\python3.exe C:\Python38\python.exe + +python -m pip install cibuildwheel==2.12.0 diff --git a/tools/mingw-w64/setup-msys2.ps1 b/tools/mingw-w64/setup-msys2.ps1 new file mode 100644 index 000000000..cf7285041 --- /dev/null +++ b/tools/mingw-w64/setup-msys2.ps1 @@ -0,0 +1,31 @@ +# Install (if necessary) and set up msys2. + + +$url="https://github.com/msys2/msys2-installer/releases/download/2022-10-28/msys2-base-x86_64-20221028.sfx.exe" +$sha256="e365b79b4b30b6f4baf34bd93f3d2a41c0a92801c7a96d79cddbfca1090a0554" + + +if (!(Test-Path -Path "c:\msys64\usr\bin\bash.exe")) { + echo "Downloading and installing msys2 to c:\msys64" + + (New-Object System.Net.WebClient).DownloadFile($url, './msys2-installer.exe') + + # Verify checksum + (Get-FileHash -Algorithm "SHA256" .\msys2-installer.exe).hash -eq $sha256 + + # Install msys2 + .\msys2-installer.exe -y -oc:\ + + Remove-Item msys2-installer.exe + + # Set up msys2 the first time + echo "Setting up msys" + c:\msys64\usr\bin\bash -lc ' ' + +} else { + echo "Using previously installed msys2" +} + +# Update packages +echo "Updating msys2 packages" +c:\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu --overwrite '*'" diff --git a/tools/smoketest.sh b/tools/smoketest.sh index acfb4ac9a..ea1afa840 100755 --- a/tools/smoketest.sh +++ b/tools/smoketest.sh @@ -93,6 +93,9 @@ for py in 3.8 ; do echo "$0: Running unit tests" pytest + echo "$0: Running type checks" + mypy src/confluent_kafka + fails="" echo "$0: Verifying OpenSSL" diff --git a/tools/source-package-verification.sh b/tools/source-package-verification.sh new file mode 100644 index 000000000..caa6ee294 --- /dev/null +++ b/tools/source-package-verification.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# +# Source Package Verification +# +set -e + +pip install -r docs/requirements.txt +pip install -U protobuf +pip install -r tests/requirements.txt + +lib_dir=dest/runtimes/$OS_NAME-$ARCH/native +tools/wheels/install-librdkafka.sh "${LIBRDKAFKA_VERSION#v}" dest +export CFLAGS="$CFLAGS -I${PWD}/dest/build/native/include" +export LDFLAGS="$LDFLAGS -L${PWD}/${lib_dir}" +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/$lib_dir" +export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/$lib_dir" + +python setup.py build && python setup.py install +if [[ $OS_NAME == linux && $ARCH == x64 ]]; then + flake8 --exclude ./_venv,*_pb2.py + make docs + python -m pytest --timeout 600 --ignore=dest +else + python -m pytest --timeout 600 --ignore=dest --ignore=tests/integration +fi diff --git a/tools/style-format.sh b/tools/style-format.sh new file mode 100755 index 000000000..a686cc6f0 --- /dev/null +++ b/tools/style-format.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# +# Check or apply/fix the project coding style to all files passed as arguments. +# Uses clang-format for C and flake8 for Python. +# +# Requires clang-format version 10 (apt install clang-format-10). +# + + +CLANG_FORMAT=${CLANG_FORMAT:-clang-format} + +set -e + +ret=0 + +if [[ -z $1 ]]; then + echo "Usage: $0 [--fix] srcfile1.c srcfile2.h srcfile3.c ..." + echo "" + exit 0 +fi + +if [[ $1 == "--fix" ]]; then + fix=1 + shift +else + fix=0 +fi + +clang_format_version=$(${CLANG_FORMAT} --version | sed -Ee 's/.*version ([[:digit:]]+)\.[[:digit:]]+\.[[:digit:]]+.*/\1/') +if [[ $clang_format_version != "10" ]] ; then + echo "$0: clang-format version 10, '$clang_format_version' detected" + exit 1 +fi + +# Get list of files from .formatignore to ignore formatting for. +ignore_files=( $(grep '^[^#]..' .formatignore) ) + +function ignore { + local file=$1 + + local f + for f in "${ignore_files[@]}" ; do + [[ $file == $f ]] && return 0 + done + + return 1 +} + +extra_info="" + +for f in $*; do + + if ignore $f ; then + echo "$f is ignored by .formatignore" 1>&2 + continue + fi + + lang="c" + if [[ $f == *.py ]]; then + lang="py" + style="pep8" + stylename="pep8" + else + style="file" # Use .clang-format + stylename="C" + fi + + check=0 + + if [[ $fix == 1 ]]; then + # Convert tabs to 8 spaces first. + if grep -ql $'\t' "$f"; then + sed -i -e 's/\t/ /g' "$f" + echo "$f: tabs converted to spaces" + fi + + if [[ $lang == c ]]; then + # Run clang-format to reformat the file + ${CLANG_FORMAT} --style="$style" "$f" > _styletmp + + else + # Run autopep8 to reformat the file. + python3 -m autopep8 -a "$f" > _styletmp + # autopep8 can't fix all errors, so we also perform a flake8 check. + check=1 + fi + + if ! cmp -s "$f" _styletmp; then + echo "$f: style fixed ($stylename)" + # Use cp to preserve target file mode/attrs. + cp _styletmp "$f" + rm _styletmp + fi + fi + + if [[ $fix == 0 || $check == 1 ]]; then + # Check for tabs + if grep -q $'\t' "$f" ; then + echo "$f: contains tabs: convert to 8 spaces instead" + ret=1 + fi + + # Check style + if [[ $lang == c ]]; then + if ! ${CLANG_FORMAT} --style="$style" --Werror --dry-run "$f" ; then + echo "$f: had style errors ($stylename): see clang-format output above" + ret=1 + fi + elif [[ $lang == py ]]; then + if ! python3 -m flake8 "$f"; then + echo "$f: had style errors ($stylename): see flake8 output above" + if [[ $fix == 1 ]]; then + # autopep8 couldn't fix all errors. Let the user know. + extra_info="Error: autopep8 could not fix all errors, fix the flake8 errors manually and run again." + fi + ret=1 + fi + fi + fi + +done + +rm -f _styletmp + +if [[ $ret != 0 ]]; then + echo "" + echo "You can run the following command to automatically fix the style:" + echo " $ make style-fix" + [[ -n $extra_info ]] && echo "$extra_info" +fi + +exit $ret diff --git a/tools/wheels/build-wheels.bat b/tools/wheels/build-wheels.bat index ec217f9e5..8c5ef3e76 100644 --- a/tools/wheels/build-wheels.bat +++ b/tools/wheels/build-wheels.bat @@ -14,7 +14,7 @@ if [%WHEELHOUSE%]==[] goto usage echo on set CIBW_BUILD=cp36-%BW_ARCH% cp37-%BW_ARCH% cp38-%BW_ARCH% cp39-%BW_ARCH% cp310-%BW_ARCH% cp311-%BW_ARCH% -set CIBW_BEFORE_BUILD=python -m pip install delvewheel==0.0.6 +set CIBW_BEFORE_BUILD=python -m pip install delvewheel==1.1.4 set CIBW_TEST_REQUIRES=-r tests/requirements.txt set CIBW_TEST_COMMAND=pytest {project}\tests\test_Producer.py rem set CIBW_BUILD_VERBOSITY=3 @@ -25,16 +25,12 @@ set CIBW_REPAIR_WHEEL_COMMAND=python -m delvewheel repair --add-path %DLL_DIR% - set PATH=%PATH%;c:\Program Files\Git\bin\ -python -m pip install cibuildwheel==2.11.2 || goto :error - -python -m cibuildwheel --output-dir %WHEELHOUSE% --platform windows || goto :error - -dir %WHEELHOUSE% +python3 -m cibuildwheel --output-dir %WHEELHOUSE% --platform windows || goto :error goto :eof :usage -@echo "Usage: %0 x86|x64 win32|win_amd64 wheelhouse-dir" +@echo "Usage: %0 x86|x64 win32|win_amd64 librdkafka-dir wheelhouse-dir" exit /B 1 :error diff --git a/tools/wheels/build-wheels.sh b/tools/wheels/build-wheels.sh index ac4ccb13e..162b67fca 100755 --- a/tools/wheels/build-wheels.sh +++ b/tools/wheels/build-wheels.sh @@ -16,6 +16,7 @@ export CIBW_TEST_COMMAND="pytest {project}/tests/test_Producer.py" librdkafka_version=$1 wheeldir=$2 +cibuildwheel_version="2.12.0" if [[ -z $wheeldir ]]; then echo "Usage: $0 " @@ -49,18 +50,21 @@ case $OSTYPE in ;; esac - $this_dir/install-librdkafka.sh $librdkafka_version dest -install_pkgs=cibuildwheel==2.11.2 +install_pkgs=cibuildwheel==$cibuildwheel_version -python3 -m pip install ${PIP_INSTALL_OPTS} $install_pkgs || +python -m pip install ${PIP_INSTALL_OPTS} $install_pkgs || pip3 install ${PIP_INSTALL_OPTS} $install_pkgs if [[ -z $TRAVIS ]]; then cibw_args="--platform $os" fi +if [[ $os == "macos" ]]; then + python3 $this_dir/install-macos-python-required-by-cibuildwheel.py $cibuildwheel_version +fi + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/$lib_dir python3 -m cibuildwheel --output-dir $wheeldir $cibw_args ls $wheeldir diff --git a/tools/wheels/install-librdkafka.sh b/tools/wheels/install-librdkafka.sh index cee2ac115..03c73356e 100755 --- a/tools/wheels/install-librdkafka.sh +++ b/tools/wheels/install-librdkafka.sh @@ -29,7 +29,11 @@ if [[ $OSTYPE == linux* ]]; then # Linux # Copy the librdkafka build with least dependencies to librdkafka.so.1 - cp -v runtimes/linux-$ARCH/native/{centos6-librdkafka.so,librdkafka.so.1} + if [[ $ARCH == arm64* ]]; then + cp -v runtimes/linux-$ARCH/native/{librdkafka.so,librdkafka.so.1} + else + cp -v runtimes/linux-$ARCH/native/{centos6-librdkafka.so,librdkafka.so.1} + fi ldd runtimes/linux-$ARCH/native/librdkafka.so.1 elif [[ $OSTYPE == darwin* ]]; then diff --git a/tools/wheels/install-macos-python-required-by-cibuildwheel.py b/tools/wheels/install-macos-python-required-by-cibuildwheel.py new file mode 100644 index 000000000..76745caf8 --- /dev/null +++ b/tools/wheels/install-macos-python-required-by-cibuildwheel.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# +# +# Get python versions required for cibuildwheel from their config and +# install them. This implementation is based on cibuildwheel 2.12.0 +# version. Might need tweak if something changes in cibuildwheel. +# +# This was added as there is a permission issue when cibuildwheel +# tries to install these versions on its own. +# + +import platform +import sys +import os +import tomli +import urllib.request +import re +import shutil + + +cibuildwheel_version = sys.argv[1] +config_url = "https://raw.githubusercontent.com/pypa/cibuildwheel/" + \ + f"v{cibuildwheel_version}/cibuildwheel/resources/build-platforms.toml" +print(f"Config URL is '{config_url}'") + +response = urllib.request.urlopen(config_url).read() + +content = response.decode('utf-8') +d = tomli.loads(content) +macos_config = d['macos']['python_configurations'] + +machine_arc = platform.machine() +print(f"Machine Architecture is '{machine_arc}'") +machine_arc_regex_string = f".*{machine_arc}" +machine_arc_regex = re.compile(machine_arc_regex_string) + +skip_versions = os.environ['CIBW_SKIP'] +print(f"Versions to skip are '{skip_versions}'") +skip_versions_list = skip_versions.split() +skip_versions_regex_string = ("|".join(skip_versions_list)).replace("*", ".*") +skip_versions_regex = re.compile(skip_versions_regex_string) + +py_versions_info = [] + +for py_version_config in macos_config: + identifier = py_version_config['identifier'] + if not skip_versions_regex.match(identifier) and machine_arc_regex.match(identifier): + pkg_url = py_version_config['url'] + py_versions_info.append((identifier, pkg_url)) + +tmp_download_dir = "tmp_download_dir" +tmp_pkg_file_name = "Package.pkg" +this_file_path = os.getcwd() +print(f"CWD is: '{this_file_path}'") +tmp_download_dir_full_path = os.path.join(os.getcwd(), tmp_download_dir) +tmp_pkg_file_full_path = os.path.join(tmp_download_dir_full_path, tmp_pkg_file_name) +if os.path.exists(tmp_download_dir_full_path): + shutil.rmtree(tmp_download_dir_full_path) +os.mkdir(tmp_download_dir) +os.chdir(tmp_download_dir) +install_command = f"sudo installer -pkg {tmp_pkg_file_name} -target /" + +for py_version_info in py_versions_info: + identifier = py_version_info[0] + pkg_url = py_version_info[1] + print(f"Installing '{identifier}' from '{pkg_url}'") + os.system(f"curl {pkg_url} --output {tmp_pkg_file_name}") + os.system(install_command) + os.remove(tmp_pkg_file_full_path) + +os.chdir(this_file_path) +shutil.rmtree(tmp_download_dir_full_path) diff --git a/tools/windows-install-librdkafka.bat b/tools/windows-install-librdkafka.bat index 74057528c..83221332e 100644 --- a/tools/windows-install-librdkafka.bat +++ b/tools/windows-install-librdkafka.bat @@ -16,4 +16,3 @@ curl -s https://raw.githubusercontent.com/chemeris/msinttypes/master/stdint.h -o for %%V in (27, 35, 36, 37) do ( call tools\windows-copy-librdkafka.bat %librdkafka_version% c:\Python%%~V || exit /b 1 ) - From cb7b4989e193e73b7b21ef44e9b2a298b6bb7cbb Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Sun, 16 Jul 2023 23:25:22 +0100 Subject: [PATCH 8/8] Update all the typing bits --- src/confluent_kafka/_model/__init__.py | 9 +- src/confluent_kafka/_util/conversion_util.py | 3 +- src/confluent_kafka/_util/validation_util.py | 21 ++- src/confluent_kafka/admin/__init__.py | 151 ++++++------------ src/confluent_kafka/admin/_acl.py | 2 +- src/confluent_kafka/admin/_config.py | 8 +- src/confluent_kafka/admin/_group.py | 17 +- src/confluent_kafka/admin/_metadata.py | 57 +++---- src/confluent_kafka/admin/_scram.py | 13 +- src/confluent_kafka/avro/__init__.py | 2 +- .../avro/cached_schema_registry_client.py | 37 ++--- src/confluent_kafka/cimpl.pyi | 22 +++ src/confluent_kafka/schema_registry/avro.py | 10 +- .../schema_registry/json_schema.py | 27 +--- 14 files changed, 166 insertions(+), 213 deletions(-) diff --git a/src/confluent_kafka/_model/__init__.py b/src/confluent_kafka/_model/__init__.py index 2bab6a1bd..dd24c0eb9 100644 --- a/src/confluent_kafka/_model/__init__.py +++ b/src/confluent_kafka/_model/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. from enum import Enum +from typing import List, Optional from .. import cimpl @@ -34,7 +35,7 @@ class Node: rack: str The rack for this node. """ - def __init__(self, id, host, port, rack=None): + def __init__(self, id: int, host: str, port: int, rack: Optional[str]=None): self.id = id self.id_string = str(id) self.host = host @@ -55,7 +56,7 @@ class ConsumerGroupTopicPartitions: topic_partitions: list(TopicPartition) List of topic partitions information. """ - def __init__(self, group_id, topic_partitions=None): + def __init__(self, group_id: str, topic_partitions: Optional[List[cimpl.TopicPartition]]=None): self.group_id = group_id self.topic_partitions = topic_partitions @@ -85,7 +86,7 @@ class ConsumerGroupState(Enum): DEAD = cimpl.CONSUMER_GROUP_STATE_DEAD EMPTY = cimpl.CONSUMER_GROUP_STATE_EMPTY - def __lt__(self, other): - if self.__class__ != other.__class__: + def __lt__(self, other: object) -> bool: + if not isinstance(other, self.__class__): return NotImplemented return self.value < other.value diff --git a/src/confluent_kafka/_util/conversion_util.py b/src/confluent_kafka/_util/conversion_util.py index 82c9b7018..60dcb5741 100644 --- a/src/confluent_kafka/_util/conversion_util.py +++ b/src/confluent_kafka/_util/conversion_util.py @@ -13,11 +13,12 @@ # limitations under the License. from enum import Enum +from typing import Any, Type class ConversionUtil: @staticmethod - def convert_to_enum(val, enum_clazz): + def convert_to_enum(val: object, enum_clazz: Type) -> Any: if type(enum_clazz) is not type(Enum): raise TypeError("'enum_clazz' must be of type Enum") diff --git a/src/confluent_kafka/_util/validation_util.py b/src/confluent_kafka/_util/validation_util.py index ffe5785f2..cd4011f5d 100644 --- a/src/confluent_kafka/_util/validation_util.py +++ b/src/confluent_kafka/_util/validation_util.py @@ -12,38 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List, Optional from ..cimpl import KafkaError -try: - string_type = basestring -except NameError: - string_type = str - +import six class ValidationUtil: @staticmethod - def check_multiple_not_none(obj, vars_to_check): + def check_multiple_not_none(obj: object, vars_to_check: List[str]) -> None: for param in vars_to_check: ValidationUtil.check_not_none(obj, param) @staticmethod - def check_not_none(obj, param): + def check_not_none(obj: object, param: str) -> None: if getattr(obj, param) is None: raise ValueError("Expected %s to be not None" % (param,)) @staticmethod - def check_multiple_is_string(obj, vars_to_check): + def check_multiple_is_string(obj: object, vars_to_check: List[str]) -> None: for param in vars_to_check: ValidationUtil.check_is_string(obj, param) @staticmethod - def check_is_string(obj, param): + def check_is_string(obj: object, param: str) -> None: param_value = getattr(obj, param) - if param_value is not None and not isinstance(param_value, string_type): + if param_value is not None and not isinstance(param_value, six.string_types): raise TypeError("Expected %s to be a string" % (param,)) @staticmethod - def check_kafka_errors(errors): + def check_kafka_errors(errors: Optional[List]) -> None: if not isinstance(errors, list): raise TypeError("errors should be None or a list") for error in errors: @@ -51,6 +48,6 @@ def check_kafka_errors(errors): raise TypeError("Expected list of KafkaError") @staticmethod - def check_kafka_error(error): + def check_kafka_error(error: object) -> None: if not isinstance(error, KafkaError): raise TypeError("Expected error to be a KafkaError") diff --git a/src/confluent_kafka/admin/__init__.py b/src/confluent_kafka/admin/__init__.py index 913455170..5962abcfa 100644 --- a/src/confluent_kafka/admin/__init__.py +++ b/src/confluent_kafka/admin/__init__.py @@ -15,9 +15,13 @@ """ Kafka admin client: create, view, alter, and delete topics and resources. """ +from typing import Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple, Type, TypeVar, Union, cast +from typing_extensions import TypeAlias import warnings import concurrent.futures +from concurrent.futures import Future + # Unused imports are keeped to be accessible using this public module from ._config import (ConfigSource, # noqa: F401 ConfigEntry, @@ -73,10 +77,7 @@ as _ConsumerGroupState -try: - string_type = basestring -except NameError: - string_type = str +import six class AdminClient(_AdminClientImpl): @@ -164,69 +165,12 @@ def _make_resource_result(f: Future, futmap: Dict[ConfigResource, Future]) -> No for resource, fut in futmap.items(): fut.set_exception(e) - _acl_type = TypeVar("_acl_type", bound=AclBinding) - - @staticmethod - def _make_list_consumer_groups_result(f, futmap): - pass - - @staticmethod - def _make_consumer_groups_result(f, futmap): - """ - Map per-group results to per-group futures in futmap. - """ - try: - - results = f.result() - futmap_values = list(futmap.values()) - len_results = len(results) - len_futures = len(futmap_values) - if len_results != len_futures: - raise RuntimeError( - "Results length {} is different from future-map length {}".format(len_results, len_futures)) - for i, result in enumerate(results): - fut = futmap_values[i] - if isinstance(result, KafkaError): - fut.set_exception(KafkaException(result)) - else: - fut.set_result(result) - except Exception as e: - # Request-level exception, raise the same for all groups - for _, fut in futmap.items(): - fut.set_exception(e) - - @staticmethod - def _make_consumer_group_offsets_result(f, futmap): - """ - Map per-group results to per-group futures in futmap. - The result value of each (successful) future is ConsumerGroupTopicPartitions. - """ - try: - - results = f.result() - futmap_values = list(futmap.values()) - len_results = len(results) - len_futures = len(futmap_values) - if len_results != len_futures: - raise RuntimeError( - "Results length {} is different from future-map length {}".format(len_results, len_futures)) - for i, result in enumerate(results): - fut = futmap_values[i] - if isinstance(result, KafkaError): - fut.set_exception(KafkaException(result)) - else: - fut.set_result(result) - except Exception as e: - # Request-level exception, raise the same for all groups - for _, fut in futmap.items(): - fut.set_exception(e) - @staticmethod - def _make_list_consumer_groups_result(f, futmap): + def _make_list_consumer_groups_result(f: Future, futmap: Dict[str, Future]) -> None: pass @staticmethod - def _make_consumer_groups_result(f, futmap): + def _make_consumer_groups_result(f: Future, futmap: Dict[str, Future]) -> None: """ Map per-group results to per-group futures in futmap. """ @@ -251,7 +195,7 @@ def _make_consumer_groups_result(f, futmap): fut.set_exception(e) @staticmethod - def _make_consumer_group_offsets_result(f, futmap): + def _make_consumer_group_offsets_result(f: Future, futmap: Dict[str, Future]) -> None: """ Map per-group results to per-group futures in futmap. The result value of each (successful) future is ConsumerGroupTopicPartitions. @@ -277,7 +221,7 @@ def _make_consumer_group_offsets_result(f, futmap): fut.set_exception(e) @staticmethod - def _make_acls_result(f, futmap): + def _make_acls_result(f: Future, futmap: Dict[AclBinding, Future]) -> None: """ Map create ACL binding results to corresponding futures in futmap. For create_acls the result value of each (successful) future is None. @@ -303,7 +247,7 @@ def _make_acls_result(f, futmap): fut.set_exception(e) @staticmethod - def _make_user_scram_credentials_result(f, futmap): + def _make_user_scram_credentials_result(f: Future, futmap: Dict[str, Future]) -> None: try: results = f.result() len_results = len(results) @@ -325,8 +269,8 @@ def _make_user_scram_credentials_result(f, futmap): fut.set_exception(e) @staticmethod - def _create_future(): - f = concurrent.futures.Future() + def _create_future() -> Future: + f: Future = concurrent.futures.Future() if not f.set_running_or_notify_cancel(): raise RuntimeError("Future was cancelled prematurely") return f @@ -356,7 +300,7 @@ def _make_futures(futmap_keys: List[_futures_map_key], class_check: Optional[Typ return f, futmap @staticmethod - def _make_futures_v2(futmap_keys, class_check, make_result_fn): + def _make_futures_v2(futmap_keys: Iterable[_futures_map_key], class_check: Optional[Type], make_result_fn: Callable[[Future, Dict[_futures_map_key, Future]], None]) -> Tuple[Future, Dict[_futures_map_key, Future]]: """ Create futures and a futuremap for the keys in futmap_keys, and create a request-level future to be bassed to the C API. @@ -376,11 +320,11 @@ def _make_futures_v2(futmap_keys, class_check, make_result_fn): return f, futmap @staticmethod - def _has_duplicates(items): + def _has_duplicates(items: Sequence[object]) -> bool: return len(set(items)) != len(items) @staticmethod - def _check_list_consumer_group_offsets_request(request): + def _check_list_consumer_group_offsets_request(request: List[_ConsumerGroupTopicPartitions]) -> None: if request is None: raise TypeError("request cannot be None") if not isinstance(request, list): @@ -393,7 +337,7 @@ def _check_list_consumer_group_offsets_request(request): if req.group_id is None: raise TypeError("'group_id' cannot be None") - if not isinstance(req.group_id, string_type): + if not isinstance(req.group_id, six.string_types): raise TypeError("'group_id' must be a string") if not req.group_id: raise ValueError("'group_id' cannot be empty") @@ -419,7 +363,7 @@ def _check_list_consumer_group_offsets_request(request): raise ValueError("Element of 'topic_partitions' must not have 'offset' value") @staticmethod - def _check_alter_consumer_group_offsets_request(request): + def _check_alter_consumer_group_offsets_request(request: List[_ConsumerGroupTopicPartitions]) -> None: if request is None: raise TypeError("request cannot be None") if not isinstance(request, list): @@ -431,7 +375,7 @@ def _check_alter_consumer_group_offsets_request(request): raise TypeError("Expected list of 'ConsumerGroupTopicPartitions'") if req.group_id is None: raise TypeError("'group_id' cannot be None") - if not isinstance(req.group_id, string_type): + if not isinstance(req.group_id, six.string_types): raise TypeError("'group_id' must be a string") if not req.group_id: raise ValueError("'group_id' cannot be empty") @@ -458,17 +402,17 @@ def _check_alter_consumer_group_offsets_request(request): "Element of 'topic_partitions' must not have negative value for 'offset' field") @staticmethod - def _check_describe_user_scram_credentials_request(users): + def _check_describe_user_scram_credentials_request(users: List) -> None: if not isinstance(users, list): raise TypeError("Expected input to be list of String") for user in users: - if not isinstance(user, string_type): + if not isinstance(user, six.string_types): raise TypeError("Each value should be a string") if not user: raise ValueError("'user' cannot be empty") @staticmethod - def _check_alter_user_scram_credentials_request(alterations): + def _check_alter_user_scram_credentials_request(alterations: List[UserScramCredentialAlteration]) -> None: if not isinstance(alterations, list): raise TypeError("Expected input to be list") if len(alterations) == 0: @@ -478,7 +422,7 @@ def _check_alter_user_scram_credentials_request(alterations): raise TypeError("Expected each element of list to be subclass of UserScramCredentialAlteration") if alteration.user is None: raise TypeError("'user' cannot be None") - if not isinstance(alteration.user, string_type): + if not isinstance(alteration.user, six.string_types): raise TypeError("'user' must be a string") if not alteration.user: raise ValueError("'user' cannot be empty") @@ -510,7 +454,7 @@ def _check_alter_user_scram_credentials_request(alterations): "to be either a UserScramCredentialUpsertion or a " + "UserScramCredentialDeletion") - def create_topics(self, new_topics, **kwargs): + def create_topics(self, new_topics: List[NewTopic], **kwargs: object) -> Dict[str, Future]: # type: ignore[override] """ Create one or more new topics. @@ -695,7 +639,7 @@ def alter_configs(self, resources: List[ConfigResource], **kwargs: object) -> Di return futmap - def incremental_alter_configs(self, resources, **kwargs): + def incremental_alter_configs(self, resources: List[ConfigResource], **kwargs: object) -> Dict[ConfigResource, Future]: # type: ignore[override] """ Update configuration properties for the specified resources. Updates are incremental, i.e only the values mentioned are changed @@ -728,7 +672,7 @@ def incremental_alter_configs(self, resources, **kwargs): return futmap - def create_acls(self, acls, **kwargs): + def create_acls(self, acls: List[AclBinding], **kwargs: object) -> Dict[AclBinding, Future]: # type: ignore[override] """ Create one or more ACL bindings. @@ -825,13 +769,13 @@ def delete_acls(self, acl_binding_filters: List[AclBindingFilter], **kwargs: obj raise ValueError("duplicate ACL binding filters not allowed") f, futmap = AdminClient._make_futures(acl_binding_filters, AclBindingFilter, - AdminClient._make_acls_result) + cast(Callable[[Future, Dict[AclBindingFilter, Future]], None], AdminClient._make_acls_result)) super(AdminClient, self).delete_acls(acl_binding_filters, f, **kwargs) return futmap - def list_consumer_groups(self, **kwargs): + def list_consumer_groups(self, states:Optional[Set[_ConsumerGroupState]] = None, **kwargs: object) -> Future: # type: ignore[override] """ List consumer groups. @@ -849,24 +793,21 @@ def list_consumer_groups(self, **kwargs): :raises TypeException: Invalid input. :raises ValueException: Invalid input. """ - if "states" in kwargs: - states = kwargs["states"] - if states is not None: - if not isinstance(states, set): - raise TypeError("'states' must be a set") - for state in states: - if not isinstance(state, _ConsumerGroupState): - raise TypeError("All elements of states must be of type ConsumerGroupState") - kwargs["states_int"] = [state.value for state in states] - kwargs.pop("states") - - f, _ = AdminClient._make_futures([], None, AdminClient._make_list_consumer_groups_result) + if states is not None: + if not isinstance(states, set): + raise TypeError("'states' must be a set") + for state in states: + if not isinstance(state, _ConsumerGroupState): + raise TypeError("All elements of states must be of type ConsumerGroupState") + kwargs["states_int"] = [state.value for state in states] + + f, _ = AdminClient._make_futures(cast(List[str], []), None, AdminClient._make_list_consumer_groups_result) super(AdminClient, self).list_consumer_groups(f, **kwargs) return f - def describe_consumer_groups(self, group_ids, **kwargs): + def describe_consumer_groups(self, group_ids: List[str], **kwargs: object) -> Dict[str, Future]: # type: ignore[override] """ Describe consumer groups. @@ -898,7 +839,7 @@ def describe_consumer_groups(self, group_ids, **kwargs): return futmap - def delete_consumer_groups(self, group_ids, **kwargs): + def delete_consumer_groups(self, group_ids: List[str], **kwargs: object) -> Dict[str, Future]: # type: ignore[override] """ Delete the given consumer groups. @@ -922,13 +863,13 @@ def delete_consumer_groups(self, group_ids, **kwargs): if len(group_ids) == 0: raise ValueError("Expected at least one group to be deleted") - f, futmap = AdminClient._make_futures(group_ids, string_type, AdminClient._make_consumer_groups_result) + f, futmap = AdminClient._make_futures(group_ids, str, AdminClient._make_consumer_groups_result) super(AdminClient, self).delete_consumer_groups(group_ids, f, **kwargs) return futmap - def list_consumer_group_offsets(self, list_consumer_group_offsets_request, **kwargs): + def list_consumer_group_offsets(self, list_consumer_group_offsets_request: List[_ConsumerGroupTopicPartitions], **kwargs: object) -> Dict[str, Future]: # type: ignore[override] """ List offset information for the consumer group and (optional) topic partition provided in the request. @@ -957,14 +898,14 @@ def list_consumer_group_offsets(self, list_consumer_group_offsets_request, **kwa AdminClient._check_list_consumer_group_offsets_request(list_consumer_group_offsets_request) f, futmap = AdminClient._make_futures([request.group_id for request in list_consumer_group_offsets_request], - string_type, + str, AdminClient._make_consumer_group_offsets_result) super(AdminClient, self).list_consumer_group_offsets(list_consumer_group_offsets_request, f, **kwargs) return futmap - def alter_consumer_group_offsets(self, alter_consumer_group_offsets_request, **kwargs): + def alter_consumer_group_offsets(self, alter_consumer_group_offsets_request: List[_ConsumerGroupTopicPartitions], **kwargs: object) -> Dict[str, Future]: # type: ignore[override] """ Alter offset for the consumer group and topic partition provided in the request. @@ -990,14 +931,14 @@ def alter_consumer_group_offsets(self, alter_consumer_group_offsets_request, **k AdminClient._check_alter_consumer_group_offsets_request(alter_consumer_group_offsets_request) f, futmap = AdminClient._make_futures([request.group_id for request in alter_consumer_group_offsets_request], - string_type, + str, AdminClient._make_consumer_group_offsets_result) super(AdminClient, self).alter_consumer_group_offsets(alter_consumer_group_offsets_request, f, **kwargs) return futmap - def set_sasl_credentials(self, username, password): + def set_sasl_credentials(self, username: str, password: str) -> None: """ Sets the SASL credentials used for this client. These credentials will overwrite the old ones, and will be used the @@ -1016,7 +957,7 @@ def set_sasl_credentials(self, username, password): """ super(AdminClient, self).set_sasl_credentials(username, password) - def describe_user_scram_credentials(self, users, **kwargs): + def describe_user_scram_credentials(self, users: List[str], **kwargs: object) -> Dict[str, Future]: # type: ignore[override] """ Describe user SASL/SCRAM credentials. @@ -1045,7 +986,7 @@ def describe_user_scram_credentials(self, users, **kwargs): return futmap - def alter_user_scram_credentials(self, alterations, **kwargs): + def alter_user_scram_credentials(self, alterations: List[UserScramCredentialAlteration], **kwargs: object) -> Dict: # type: ignore[override] """ Alter user SASL/SCRAM credentials. diff --git a/src/confluent_kafka/admin/_acl.py b/src/confluent_kafka/admin/_acl.py index 6d8011ebb..d37118fe3 100644 --- a/src/confluent_kafka/admin/_acl.py +++ b/src/confluent_kafka/admin/_acl.py @@ -108,7 +108,7 @@ def __init__(self, restype: ResourceType, name: str, self.operation_int = int(self.operation.value) self.permission_type_int = int(self.permission_type.value) - def _convert_enums(self): + def _convert_enums(self) -> None: self.restype = ConversionUtil.convert_to_enum(self.restype, ResourceType) self.resource_pattern_type = ConversionUtil.convert_to_enum( self.resource_pattern_type, ResourcePatternType) diff --git a/src/confluent_kafka/admin/_config.py b/src/confluent_kafka/admin/_config.py index 93793b46e..3d1586ab7 100644 --- a/src/confluent_kafka/admin/_config.py +++ b/src/confluent_kafka/admin/_config.py @@ -74,7 +74,7 @@ def __init__(self, name: str, value: str, is_sensitive: bool=False, is_synonym: bool=False, synonyms: List[str]=[], - incremental_operation=None): + incremental_operation: Optional[AlterConfigOpType]=None): """ This class is typically not user instantiated. """ @@ -131,7 +131,7 @@ class ConfigResource(object): Type = ResourceType def __init__(self, restype: Union[str, int, ResourceType], name: str, - set_config: Optional[Dict[str, str]]=None, described_configs: Optional[object]=None, error: Optional[object]=None, incremental_configs=None): + set_config: Optional[Dict[str, str]]=None, described_configs: Optional[object]=None, error: Optional[object]=None, incremental_configs: Optional[List[ConfigEntry]]=None): """ :param ConfigResource.Type restype: Resource type. :param str name: The resource name, which depends on restype. @@ -168,7 +168,7 @@ def __init__(self, restype: Union[str, int, ResourceType], name: str, else: self.set_config_dict = dict() - self.incremental_configs = list(incremental_configs or []) + self.incremental_configs = incremental_configs or [] self.configs = described_configs self.error = error @@ -217,7 +217,7 @@ def set_config(self, name: str, value: str, overwrite: bool=True) -> None: return self.set_config_dict[name] = value - def add_incremental_config(self, config_entry): + def add_incremental_config(self, config_entry: ConfigEntry) -> None: """ Add a ConfigEntry for incremental alter configs, using the configured incremental_operation. diff --git a/src/confluent_kafka/admin/_group.py b/src/confluent_kafka/admin/_group.py index 1c8d5e6fe..5ac117775 100644 --- a/src/confluent_kafka/admin/_group.py +++ b/src/confluent_kafka/admin/_group.py @@ -13,8 +13,11 @@ # limitations under the License. +from typing import List, Optional + +from confluent_kafka.cimpl import KafkaException, TopicPartition from .._util import ConversionUtil -from .._model import ConsumerGroupState +from .._model import ConsumerGroupState, Node class ConsumerGroupListing: @@ -31,7 +34,7 @@ class ConsumerGroupListing: state : ConsumerGroupState Current state of the consumer group. """ - def __init__(self, group_id, is_simple_consumer_group, state=None): + def __init__(self, group_id: str, is_simple_consumer_group: bool, state: Optional[ConsumerGroupState]=None): self.group_id = group_id self.is_simple_consumer_group = is_simple_consumer_group if state is not None: @@ -50,7 +53,7 @@ class ListConsumerGroupsResult: errors : list(KafkaException) List of errors encountered during the operation, if any. """ - def __init__(self, valid=None, errors=None): + def __init__(self, valid: Optional[List[ConsumerGroupListing]]=None, errors: Optional[List[KafkaException]]=None): self.valid = valid self.errors = errors @@ -65,7 +68,7 @@ class MemberAssignment: topic_partitions : list(TopicPartition) The topic partitions assigned to a group member. """ - def __init__(self, topic_partitions=[]): + def __init__(self, topic_partitions: Optional[List[TopicPartition]]=None): self.topic_partitions = topic_partitions if self.topic_partitions is None: self.topic_partitions = [] @@ -89,7 +92,7 @@ class MemberDescription: group_instance_id : str The instance id of the group member. """ - def __init__(self, member_id, client_id, host, assignment, group_instance_id=None): + def __init__(self, member_id: str, client_id: str, host: str, assignment: MemberAssignment, group_instance_id: Optional[str]=None): self.member_id = member_id self.client_id = client_id self.host = host @@ -117,8 +120,8 @@ class ConsumerGroupDescription: coordinator: Node Consumer group coordinator. """ - def __init__(self, group_id, is_simple_consumer_group, members, partition_assignor, state, - coordinator): + def __init__(self, group_id:str , is_simple_consumer_group: bool, members: List[MemberDescription], partition_assignor: str, state: ConsumerGroupState, + coordinator: Node): self.group_id = group_id self.is_simple_consumer_group = is_simple_consumer_group self.members = members diff --git a/src/confluent_kafka/admin/_metadata.py b/src/confluent_kafka/admin/_metadata.py index 201e4534b..0d68c0862 100644 --- a/src/confluent_kafka/admin/_metadata.py +++ b/src/confluent_kafka/admin/_metadata.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Dict, List, Optional + + class ClusterMetadata (object): """ Provides information about the Kafka cluster, brokers, and topics. @@ -20,24 +23,24 @@ class ClusterMetadata (object): This class is typically not user instantiated. """ - def __init__(self): - self.cluster_id = None + def __init__(self) -> None: + self.cluster_id: Optional[str] = None """Cluster id string, if supported by the broker, else None.""" self.controller_id = -1 """Current controller broker id, or -1.""" - self.brokers = {} + self.brokers: Dict[int, BrokerMetadata] = {} """Map of brokers indexed by the broker id (int). Value is a BrokerMetadata object.""" - self.topics = {} + self.topics: Dict[str, TopicMetadata] = {} """Map of topics indexed by the topic name. Value is a TopicMetadata object.""" self.orig_broker_id = -1 """The broker this metadata originated from.""" self.orig_broker_name = None """The broker name/address this metadata originated from.""" - def __repr__(self): + def __repr__(self) -> str: return "ClusterMetadata({})".format(self.cluster_id) - def __str__(self): + def __str__(self) -> str: return str(self.cluster_id) @@ -48,7 +51,7 @@ class BrokerMetadata (object): This class is typically not user instantiated. """ - def __init__(self): + def __init__(self) -> None: self.id = -1 """Broker id""" self.host = None @@ -56,10 +59,10 @@ def __init__(self): self.port = -1 """Broker port""" - def __repr__(self): + def __repr__(self) -> str: return "BrokerMetadata({}, {}:{})".format(self.id, self.host, self.port) - def __str__(self): + def __str__(self) -> str: return "{}:{}/{}".format(self.host, self.port, self.id) @@ -73,22 +76,22 @@ class TopicMetadata (object): # Sphinx issue where it tries to reference the same instance variable # on other classes which raises a warning/error. - def __init__(self): - self.topic = None + def __init__(self) -> None: + self.topic: Optional[str] = None """Topic name""" - self.partitions = {} + self.partitions: Dict[int, PartitionMetadata] = {} """Map of partitions indexed by partition id. Value is a PartitionMetadata object.""" self.error = None """Topic error, or None. Value is a KafkaError object.""" - def __repr__(self): + def __repr__(self) -> str: if self.error is not None: return "TopicMetadata({}, {} partitions, {})".format(self.topic, len(self.partitions), self.error) else: return "TopicMetadata({}, {} partitions)".format(self.topic, len(self.partitions)) - def __str__(self): - return self.topic + def __str__(self) -> str: + return str(self.topic) class PartitionMetadata (object): @@ -103,25 +106,25 @@ class PartitionMetadata (object): of a broker id in the brokers dict. """ - def __init__(self): + def __init__(self) -> None: self.id = -1 """Partition id.""" self.leader = -1 """Current leader broker for this partition, or -1.""" - self.replicas = [] + self.replicas: List[int] = [] """List of replica broker ids for this partition.""" - self.isrs = [] + self.isrs: List[int] = [] """List of in-sync-replica broker ids for this partition.""" self.error = None """Partition error, or None. Value is a KafkaError object.""" - def __repr__(self): + def __repr__(self) -> str: if self.error is not None: return "PartitionMetadata({}, {})".format(self.id, self.error) else: return "PartitionMetadata({})".format(self.id) - def __str__(self): + def __str__(self) -> str: return "{}".format(self.id) @@ -134,7 +137,7 @@ class GroupMember(object): This class is typically not user instantiated. """ # noqa: E501 - def __init__(self,): + def __init__(self) -> None: self.id = None """Member id (generated by broker).""" self.client_id = None @@ -153,10 +156,10 @@ class GroupMetadata(object): This class is typically not user instantiated. """ - def __init__(self): + def __init__(self) -> None: self.broker = None """Originating broker metadata.""" - self.id = None + self.id: Optional[str] = None """Group name.""" self.error = None """Broker-originated error, or None. Value is a KafkaError object.""" @@ -166,14 +169,14 @@ def __init__(self): """Group protocol type.""" self.protocol = None """Group protocol.""" - self.members = [] + self.members: List = [] """Group members.""" - def __repr__(self): + def __repr__(self) -> str: if self.error is not None: return "GroupMetadata({}, {})".format(self.id, self.error) else: return "GroupMetadata({})".format(self.id) - def __str__(self): - return self.id + def __str__(self) -> str: + return str(self.id) diff --git a/src/confluent_kafka/admin/_scram.py b/src/confluent_kafka/admin/_scram.py index c20f55bbc..a6a34cf74 100644 --- a/src/confluent_kafka/admin/_scram.py +++ b/src/confluent_kafka/admin/_scram.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List, Optional from .. import cimpl from enum import Enum @@ -25,8 +26,8 @@ class ScramMechanism(Enum): SCRAM_SHA_256 = cimpl.SCRAM_MECHANISM_SHA_256 #: SCRAM-SHA-256 mechanism SCRAM_SHA_512 = cimpl.SCRAM_MECHANISM_SHA_512 #: SCRAM-SHA-512 mechanism - def __lt__(self, other): - if self.__class__ != other.__class__: + def __lt__(self, other: object) -> bool: + if not isinstance(other, self.__class__): return NotImplemented return self.value < other.value @@ -43,7 +44,7 @@ class ScramCredentialInfo: iterations: int Positive number of iterations used when creating the credential. """ - def __init__(self, mechanism, iterations): + def __init__(self, mechanism: ScramMechanism, iterations: int): self.mechanism = mechanism self.iterations = iterations @@ -60,7 +61,7 @@ class UserScramCredentialsDescription: scram_credential_infos: list(ScramCredentialInfo) SASL/SCRAM credential representations for the user. """ - def __init__(self, user, scram_credential_infos): + def __init__(self, user: str, scram_credential_infos: List[ScramCredentialInfo]): self.user = user self.scram_credential_infos = scram_credential_infos @@ -93,7 +94,7 @@ class UserScramCredentialUpsertion(UserScramCredentialAlteration): salt: bytes Salt to use. Will be generated randomly if None. (optional) """ - def __init__(self, user, scram_credential_info, password, salt=None): + def __init__(self, user: str, scram_credential_info: ScramCredentialInfo, password: bytes, salt: Optional[bytes]=None): super(UserScramCredentialUpsertion, self).__init__(user) self.scram_credential_info = scram_credential_info self.password = password @@ -111,6 +112,6 @@ class UserScramCredentialDeletion(UserScramCredentialAlteration): mechanism: ScramMechanism SASL/SCRAM mechanism. """ - def __init__(self, user, mechanism): + def __init__(self, user: str, mechanism: ScramMechanism): super(UserScramCredentialDeletion, self).__init__(user) self.mechanism = mechanism diff --git a/src/confluent_kafka/avro/__init__.py b/src/confluent_kafka/avro/__init__.py index aaa970965..f668da7de 100644 --- a/src/confluent_kafka/avro/__init__.py +++ b/src/confluent_kafka/avro/__init__.py @@ -107,7 +107,7 @@ def produce(self, topic: str, **kwargs: object) -> None: else: raise ValueSerializerError("Avro schema required for values") - if key is not None: + if key is not None: if key_schema: key = self._serializer.encode_record_with_schema(topic, key_schema, key, True) else: diff --git a/src/confluent_kafka/avro/cached_schema_registry_client.py b/src/confluent_kafka/avro/cached_schema_registry_client.py index a2d571b97..0733f53ef 100644 --- a/src/confluent_kafka/avro/cached_schema_registry_client.py +++ b/src/confluent_kafka/avro/cached_schema_registry_client.py @@ -21,13 +21,13 @@ # import logging from turtle import pos -from typing import Dict, Optional, Sized, Tuple, TypeVar, Union, cast +from typing import Any, Dict, Optional, Sized, Tuple, TypeVar, Union, cast import warnings import urllib3 import json from collections import defaultdict -from requests import Session, utils +from requests import Session, utils, Response from confluent_kafka.schema_registry.schema_registry_client import Schema @@ -116,7 +116,7 @@ def __init__(self, url: Union[str, Dict[str, object]], max_schemas_per_subject: if _conf_cert == [None, None]: s.cert = None else: - s.cert = cast(Tuple[str, str], _conf_cert) + s.cert = _conf_cert s.auth = self._configure_basic_auth(self.url, conf) self.url = utils.urldefragauth(self.url) @@ -124,7 +124,7 @@ def __init__(self, url: Union[str, Dict[str, object]], max_schemas_per_subject: self._session = s key_password = conf.pop('ssl.key.password', None) self._is_key_password_provided = not key_password - self._https_session = self._make_https_session(s.cert[0], s.cert[1], ca_path, s.auth, key_password) + self._https_session = self._make_https_session(s.cert[0] if s.cert is not None else None, s.cert[1] if s.cert is not None else None, ca_path, s.auth, key_password) self.auto_register_schemas = conf.pop("auto.register.schemas", True) @@ -145,27 +145,26 @@ def close(self) -> None: if hasattr(self, '_session'): self._session.close() if hasattr(self, '_https_session'): - self._https_session.clear() + self._https_session.clear() # type: ignore[no-untyped-call] @staticmethod - def _make_https_session(cert_location, key_location, ca_certs_path, auth, key_password): + def _make_https_session(cert_location: Optional[str], key_location: Optional[str], ca_certs_path: Optional[str], auth: Tuple[str, str], key_password: object) -> urllib3.PoolManager: https_session = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=ca_certs_path, cert_file=cert_location, key_file=key_location, key_password=key_password) - https_session.auth = auth + https_session.auth = auth # type: ignore[attr-defined] return https_session - def _send_https_session_request(self, url, method, headers, body): + def _send_https_session_request(self, url: str, method: str, headers: Dict, body: Any) -> urllib3.response.HTTPResponse: request_headers = {'Accept': ACCEPT_HDR} - auth = self._https_session.auth + auth = self._https_session.auth # type: ignore[attr-defined] if body: body = json.dumps(body).encode('UTF-8') request_headers["Content-Length"] = str(len(body)) request_headers["Content-Type"] = "application/vnd.schemaregistry.v1+json" if auth[0] != '' and auth[1] != '': - request_headers.update(urllib3.make_headers(basic_auth=auth[0] + ":" + - auth[1])) + request_headers.update(urllib3.make_headers(basic_auth=auth[0] + ":" + auth[1])) # type:ignore[no-untyped-call] request_headers.update(headers) - response = self._https_session.request(method, url, headers=request_headers, body=body) + response = self._https_session.request(method, url, headers=request_headers, body=body) # type:ignore[no-untyped-call] return response @staticmethod @@ -187,25 +186,27 @@ def _configure_basic_auth(url: str, conf: Dict) -> Tuple[str, str]: auth = utils.get_auth_from_url(url) return auth + _client_tls_ret = TypeVar("_client_tls_ret", str, None) + @staticmethod - def _configure_client_tls(conf: Dict) -> Tuple[Optional[str], Optional[str]]: + def _configure_client_tls(conf: Dict) -> Tuple[_client_tls_ret, _client_tls_ret]: cert = cast(Optional[str], conf.pop('ssl.certificate.location', None)), cast(Optional[str], conf.pop('ssl.key.location', None)) # Both values can be None or no values can be None - if bool(cert[0]) != bool(cert[1]): + if (cert[0] is None) != (cert[1] is None): raise ValueError( "Both schema.registry.ssl.certificate.location and schema.registry.ssl.key.location must be set") - return cert + return cert # type: ignore[return-value] def _send_request(self, url: str, method: str='GET', body: Optional[Sized]=None, headers: Dict={}) -> Tuple[object, int]: if method not in VALID_METHODS: raise ClientError("Method {} is invalid; valid methods include {}".format(method, VALID_METHODS)) if url.startswith('https') and self._is_key_password_provided: - response = self._send_https_session_request(url, method, headers, body) + http_response = self._send_https_session_request(url, method, headers, body) try: - return json.loads(response.data), response.status + return json.loads(http_response.data), http_response.status except ValueError: - return response.content, response.status + return http_response.data, http_response.status _headers = {'Accept': ACCEPT_HDR} if body: diff --git a/src/confluent_kafka/cimpl.pyi b/src/confluent_kafka/cimpl.pyi index f11abd9c7..285018cfc 100644 --- a/src/confluent_kafka/cimpl.pyi +++ b/src/confluent_kafka/cimpl.pyi @@ -19,12 +19,22 @@ ACL_PERMISSION_TYPE_ALLOW: int ACL_PERMISSION_TYPE_ANY: int ACL_PERMISSION_TYPE_DENY: int ACL_PERMISSION_TYPE_UNKNOWN: int +ALTER_CONFIG_OP_TYPE_APPEND: int +ALTER_CONFIG_OP_TYPE_DELETE: int +ALTER_CONFIG_OP_TYPE_SET: int +ALTER_CONFIG_OP_TYPE_SUBTRACT: int CONFIG_SOURCE_DEFAULT_CONFIG: int CONFIG_SOURCE_DYNAMIC_BROKER_CONFIG: int CONFIG_SOURCE_DYNAMIC_DEFAULT_BROKER_CONFIG: int CONFIG_SOURCE_DYNAMIC_TOPIC_CONFIG: int CONFIG_SOURCE_STATIC_BROKER_CONFIG: int CONFIG_SOURCE_UNKNOWN_CONFIG: int +CONSUMER_GROUP_STATE_COMPLETING_REBALANCE: int +CONSUMER_GROUP_STATE_DEAD: int +CONSUMER_GROUP_STATE_EMPTY: int +CONSUMER_GROUP_STATE_PREPARING_REBALANCE: int +CONSUMER_GROUP_STATE_STABLE: int +CONSUMER_GROUP_STATE_UNKNOWN: int OFFSET_BEGINNING: int OFFSET_END: int OFFSET_INVALID: int @@ -39,6 +49,9 @@ RESOURCE_PATTERN_PREFIXED: int RESOURCE_PATTERN_UNKNOWN: int RESOURCE_TOPIC: int RESOURCE_UNKNOWN: int +SCRAM_MECHANISM_SHA_256: int +SCRAM_MECHANISM_SHA_512: int +SCRAM_MECHANISM_UNKNOWN: int TIMESTAMP_CREATE_TIME: int TIMESTAMP_LOG_APPEND_TIME: int TIMESTAMP_NOT_AVAILABLE: int @@ -333,6 +346,15 @@ class _AdminClientImpl: def list_groups(self, *args: object, **kwargs: object) -> Any: ... def list_topics(self, *args: object, **kwargs: object) -> Any: ... def poll(self) -> Any: ... + def alter_user_scram_credentials(self, alterations: List, f: Future) -> Any: ... + def describe_consumer_groups(self, group_ids: List[str], f: Future, **kwargs: object) -> Any: ... + def list_consumer_groups(self, f: Future, **kwargs: object) -> Any: ... + def list_consumer_group_offsets(self, offsets: List, f: Future, **kwargs: object) -> Any: ... + def incremental_alter_configs(self, resources: List, f: Future, **kwargs: object) -> Any: ... + def delete_consumer_groups(self, group_ids: List, f: Future, **kwargs: object) -> Any: ... + def set_sasl_credentials(self, username: str, password: str) -> Any: ... + def describe_user_scram_credentials(self, users: List[str], f: Future, **kwargs: object) -> Any: ... + def alter_consumer_group_offsets(self, request: List, f: Future) -> Any: ... def __len__(self) -> Any: ... def libversion(*args: object, **kwargs: object) -> Any: ... diff --git a/src/confluent_kafka/schema_registry/avro.py b/src/confluent_kafka/schema_registry/avro.py index 4e12be50a..b234ddb63 100644 --- a/src/confluent_kafka/schema_registry/avro.py +++ b/src/confluent_kafka/schema_registry/avro.py @@ -18,7 +18,7 @@ from io import BytesIO from json import loads from struct import pack, unpack -from typing import Any, Callable, Dict, Optional, Set, Tuple, cast +from typing import Any, Callable, Dict, Optional, Set, Tuple, Union, cast from typing_extensions import Literal from fastavro import (parse_schema, @@ -70,7 +70,7 @@ def _schema_loads(schema_str: str) -> Schema: return Schema(schema_str, schema_type='AVRO') -def _resolve_named_schema(schema, schema_registry_client, named_schemas=None): +def _resolve_named_schema(schema: Schema, schema_registry_client: SchemaRegistryClient, named_schemas: Optional[Dict]=None) -> Dict: """ Resolves named schemas referenced by the provided schema recursively. :param schema: Schema to resolve named schemas for. @@ -183,7 +183,7 @@ class AvroSerializer(Serializer): 'use.latest.version': False, 'subject.name.strategy': topic_subject_name_strategy} - def __init__(self, schema_registry_client: SchemaRegistryClient, schema_str: str, + def __init__(self, schema_registry_client: SchemaRegistryClient, schema_str: Union[str, Schema], to_dict: Optional[Callable[[object, SerializationContext], Dict]]=None, conf: Optional[Dict]=None): if isinstance(schema_str, str): schema = _schema_loads(schema_str) @@ -362,8 +362,10 @@ def __init__(self, schema_registry_client: SchemaRegistryClient, schema_str: Opt self._schema = schema self._registry = schema_registry_client self._writer_schemas: Dict[int, Schema] = {} + self._named_schemas: Optional[Dict] + self._reader_schema: Optional[Dict] - if schema: + if self._schema: schema_dict = loads(self._schema.schema_str) self._named_schemas = _resolve_named_schema(self._schema, schema_registry_client) self._reader_schema = parse_schema(schema_dict, diff --git a/src/confluent_kafka/schema_registry/json_schema.py b/src/confluent_kafka/schema_registry/json_schema.py index d291924dc..8d9baeb19 100644 --- a/src/confluent_kafka/schema_registry/json_schema.py +++ b/src/confluent_kafka/schema_registry/json_schema.py @@ -19,7 +19,7 @@ import json import struct -from typing import Any, Callable, Dict, Optional, Set, cast +from typing import Any, Callable, Dict, Optional, Set, Union, cast from jsonschema import validate, ValidationError, RefResolver @@ -44,26 +44,7 @@ def __exit__(self, *args: Any) -> None: self.close() -def _resolve_named_schema(schema, schema_registry_client, named_schemas=None): - """ - Resolves named schemas referenced by the provided schema recursively. - :param schema: Schema to resolve named schemas for. - :param schema_registry_client: SchemaRegistryClient to use for retrieval. - :param named_schemas: Dict of named schemas resolved recursively. - :return: named_schemas dict. - """ - if named_schemas is None: - named_schemas = {} - if schema.references is not None: - for ref in schema.references: - referenced_schema = schema_registry_client.get_version(ref.subject, ref.version) - _resolve_named_schema(referenced_schema.schema, schema_registry_client, named_schemas) - referenced_schema_dict = json.loads(referenced_schema.schema.schema_str) - named_schemas[ref.name] = referenced_schema_dict - return named_schemas - - -def _resolve_named_schema(schema, schema_registry_client, named_schemas=None): +def _resolve_named_schema(schema: Schema, schema_registry_client: SchemaRegistryClient, named_schemas: Optional[Dict]=None) -> Dict: """ Resolves named schemas referenced by the provided schema recursively. :param schema: Schema to resolve named schemas for. @@ -323,7 +304,7 @@ class JSONDeserializer(Deserializer): __slots__ = ['_parsed_schema', '_from_dict', '_registry', '_are_references_provided', '_schema'] - def __init__(self, schema_str: str, from_dict: Optional[Callable]=None, schema_registry_client=None): + def __init__(self, schema_str: Union[Schema, str], from_dict: Optional[Callable]=None, schema_registry_client: Optional[SchemaRegistryClient]=None): self._are_references_provided = False if isinstance(schema_str, str): schema = Schema(schema_str, schema_type="JSON") @@ -385,7 +366,7 @@ def __call__(self, data: Optional[bytes], ctx: Optional[SerializationContext]=No try: if self._are_references_provided: - named_schemas = _resolve_named_schema(self._schema, self._registry) + named_schemas = _resolve_named_schema(self._schema, cast(SchemaRegistryClient, self._registry)) validate(instance=obj_dict, schema=self._parsed_schema, resolver=RefResolver(self._parsed_schema.get('$id'), self._parsed_schema,