From e4a148a9a5b8b33209b711e5ce1b7f363539f3e8 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 1 Jul 2026 12:18:50 -0400 Subject: [PATCH 01/20] Generated code --- .../azure/apiview-properties.json | 4 + .../aio/operations/_container_operations.py | 261 ++++++++++++ .../operations/_container_operations.py | 373 ++++++++++++++++++ .../azure/storage/blob/_serialize.py | 1 + 4 files changed, 639 insertions(+) diff --git a/sdk/storage/azure-storage-blob/azure/apiview-properties.json b/sdk/storage/azure-storage-blob/azure/apiview-properties.json index 64ad80365c09..c692e7ef730e 100644 --- a/sdk/storage/azure-storage-blob/azure/apiview-properties.json +++ b/sdk/storage/azure-storage-blob/azure/apiview-properties.json @@ -130,8 +130,12 @@ "azure.storage.blob.aio.operations.ContainerOperations.change_lease": null, "azure.storage.blob.operations.ContainerOperations.list_blob_flat_segment": null, "azure.storage.blob.aio.operations.ContainerOperations.list_blob_flat_segment": null, + "azure.storage.blob.operations.ContainerOperations.list_blob_flat_segment_apache_arrow": null, + "azure.storage.blob.aio.operations.ContainerOperations.list_blob_flat_segment_apache_arrow": null, "azure.storage.blob.operations.ContainerOperations.list_blob_hierarchy_segment": null, "azure.storage.blob.aio.operations.ContainerOperations.list_blob_hierarchy_segment": null, + "azure.storage.blob.operations.ContainerOperations.list_blob_hierarchy_segment_apache_arrow": null, + "azure.storage.blob.aio.operations.ContainerOperations.list_blob_hierarchy_segment_apache_arrow": null, "azure.storage.blob.operations.ContainerOperations.get_account_info": null, "azure.storage.blob.aio.operations.ContainerOperations.get_account_info": null, "azure.storage.blob.operations.BlobOperations.download": null, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_container_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_container_operations.py index 09bb123a20af..a8e9cabfb742 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_container_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_container_operations.py @@ -37,7 +37,9 @@ build_get_access_policy_request, build_get_account_info_request, build_get_properties_request, + build_list_blob_flat_segment_apache_arrow_request, build_list_blob_flat_segment_request, + build_list_blob_hierarchy_segment_apache_arrow_request, build_list_blob_hierarchy_segment_request, build_release_lease_request, build_rename_request, @@ -1666,6 +1668,132 @@ async def list_blob_flat_segment( return deserialized # type: ignore + @distributed_trace_async + async def list_blob_flat_segment_apache_arrow( + self, + prefix: Optional[str] = None, + marker: Optional[str] = None, + maxresults: Optional[int] = None, + include: Optional[list[Union[str, _models.ListBlobsIncludeItem]]] = None, + timeout: Optional[int] = None, + start_from: Optional[str] = None, + end_before: Optional[str] = None, + request_id_parameter: Optional[str] = None, + **kwargs: Any + ) -> AsyncIterator[bytes]: + """The List Blobs operation returns a list of the blobs under the specified container. This + operation is for Apache Arrow use case so response is returned as raw to be deserialized by the + client. + + :param prefix: Filters the results to return only containers whose name begins with the + specified prefix. Default value is None. + :type prefix: str + :param marker: A string value that identifies the portion of the list of containers to be + returned with the next listing operation. The operation returns the NextMarker value within the + response body if the listing operation did not return all containers remaining to be listed + with the current page. The NextMarker value can be used as the value for the marker parameter + in a subsequent call to request the next page of list items. The marker value is opaque to the + client. Default value is None. + :type marker: str + :param maxresults: Specifies the maximum number of containers to return. If the request does + not specify maxresults, or specifies a value greater than 5000, the server will return up to + 5000 items. Note that if the listing operation crosses a partition boundary, then the service + will return a continuation token for retrieving the remainder of the results. For this reason, + it is possible that the service will return fewer results than specified by maxresults, or than + the default of 5000. Default value is None. + :type maxresults: int + :param include: Include this parameter to specify one or more datasets to include in the + response. Default value is None. + :type include: list[str or ~azure.storage.blob.models.ListBlobsIncludeItem] + :param timeout: The timeout parameter is expressed in seconds. For more information, see + :code:`Setting + Timeouts for Blob Service Operations.`. Default value is None. + :type timeout: int + :param start_from: Specifies the relative path to list paths from. For non-recursive list, only + one entity level is supported; For recursive list, multiple entity levels are supported. + (Inclusive). Default value is None. + :type start_from: str + :param end_before: Specifies the relative path to end before list paths. (Exclusive). Default + value is None. + :type end_before: str + :param request_id_parameter: Provides a client-generated, opaque value with a 1 KB character + limit that is recorded in the analytics logs when storage analytics logging is enabled. Default + value is None. + :type request_id_parameter: str + :return: AsyncIterator[bytes] or the result of cls(response) + :rtype: AsyncIterator[bytes] + :raises ~azure.core.exceptions.HttpResponseError: + """ + error_map: MutableMapping = { + 401: ClientAuthenticationError, + 404: ResourceNotFoundError, + 409: ResourceExistsError, + 304: ResourceNotModifiedError, + } + error_map.update(kwargs.pop("error_map", {}) or {}) + + _headers = kwargs.pop("headers", {}) or {} + _params = case_insensitive_dict(kwargs.pop("params", {}) or {}) + + restype: Literal["container"] = kwargs.pop("restype", _params.pop("restype", "container")) + comp: Literal["list"] = kwargs.pop("comp", _params.pop("comp", "list")) + cls: ClsType[AsyncIterator[bytes]] = kwargs.pop("cls", None) + + _request = build_list_blob_flat_segment_apache_arrow_request( + url=self._config.url, + version=self._config.version, + prefix=prefix, + marker=marker, + maxresults=maxresults, + include=include, + timeout=timeout, + start_from=start_from, + end_before=end_before, + request_id_parameter=request_id_parameter, + restype=restype, + comp=comp, + headers=_headers, + params=_params, + ) + _request.url = self._client.format_url(_request.url) + + _decompress = kwargs.pop("decompress", True) + _stream = True + pipeline_response: PipelineResponse = await self._client._pipeline.run( # pylint: disable=protected-access + _request, stream=_stream, **kwargs + ) + + response = pipeline_response.http_response + + if response.status_code not in [200]: + try: + await response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass + map_error(status_code=response.status_code, response=response, error_map=error_map) + error = self._deserialize.failsafe_deserialize( + _models.StorageError, + pipeline_response, + ) + raise HttpResponseError(response=response, model=error) + + response_headers = {} + response_headers["Content-Type"] = self._deserialize("str", response.headers.get("Content-Type")) + response_headers["x-ms-client-request-id"] = self._deserialize( + "str", response.headers.get("x-ms-client-request-id") + ) + response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) + response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) + response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) + + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) + + if cls: + return cls(pipeline_response, deserialized, response_headers) # type: ignore + + return deserialized # type: ignore + @distributed_trace_async async def list_blob_hierarchy_segment( self, @@ -1787,6 +1915,139 @@ async def list_blob_hierarchy_segment( return deserialized # type: ignore + @distributed_trace_async + async def list_blob_hierarchy_segment_apache_arrow( + self, + delimiter: str, + prefix: Optional[str] = None, + marker: Optional[str] = None, + maxresults: Optional[int] = None, + include: Optional[list[Union[str, _models.ListBlobsIncludeItem]]] = None, + timeout: Optional[int] = None, + start_from: Optional[str] = None, + end_before: Optional[str] = None, + request_id_parameter: Optional[str] = None, + **kwargs: Any + ) -> AsyncIterator[bytes]: + """[Update] The List Blobs operation returns a list of the blobs under the specified container. + This operation is for Apache Arrow use case so response is returned as raw to be deserialized + by the client. + + :param delimiter: When the request includes this parameter, the operation returns a BlobPrefix + element in the response body that acts as a placeholder for all blobs whose names begin with + the same substring up to the appearance of the delimiter character. The delimiter may be a + single character or a string. Required. + :type delimiter: str + :param prefix: Filters the results to return only containers whose name begins with the + specified prefix. Default value is None. + :type prefix: str + :param marker: A string value that identifies the portion of the list of containers to be + returned with the next listing operation. The operation returns the NextMarker value within the + response body if the listing operation did not return all containers remaining to be listed + with the current page. The NextMarker value can be used as the value for the marker parameter + in a subsequent call to request the next page of list items. The marker value is opaque to the + client. Default value is None. + :type marker: str + :param maxresults: Specifies the maximum number of containers to return. If the request does + not specify maxresults, or specifies a value greater than 5000, the server will return up to + 5000 items. Note that if the listing operation crosses a partition boundary, then the service + will return a continuation token for retrieving the remainder of the results. For this reason, + it is possible that the service will return fewer results than specified by maxresults, or than + the default of 5000. Default value is None. + :type maxresults: int + :param include: Include this parameter to specify one or more datasets to include in the + response. Default value is None. + :type include: list[str or ~azure.storage.blob.models.ListBlobsIncludeItem] + :param timeout: The timeout parameter is expressed in seconds. For more information, see + :code:`Setting + Timeouts for Blob Service Operations.`. Default value is None. + :type timeout: int + :param start_from: Specifies the relative path to list paths from. For non-recursive list, only + one entity level is supported; For recursive list, multiple entity levels are supported. + (Inclusive). Default value is None. + :type start_from: str + :param end_before: Specifies the relative path to end before list paths. (Exclusive). Default + value is None. + :type end_before: str + :param request_id_parameter: Provides a client-generated, opaque value with a 1 KB character + limit that is recorded in the analytics logs when storage analytics logging is enabled. Default + value is None. + :type request_id_parameter: str + :return: AsyncIterator[bytes] or the result of cls(response) + :rtype: AsyncIterator[bytes] + :raises ~azure.core.exceptions.HttpResponseError: + """ + error_map: MutableMapping = { + 401: ClientAuthenticationError, + 404: ResourceNotFoundError, + 409: ResourceExistsError, + 304: ResourceNotModifiedError, + } + error_map.update(kwargs.pop("error_map", {}) or {}) + + _headers = kwargs.pop("headers", {}) or {} + _params = case_insensitive_dict(kwargs.pop("params", {}) or {}) + + restype: Literal["container"] = kwargs.pop("restype", _params.pop("restype", "container")) + comp: Literal["list"] = kwargs.pop("comp", _params.pop("comp", "list")) + cls: ClsType[AsyncIterator[bytes]] = kwargs.pop("cls", None) + + _request = build_list_blob_hierarchy_segment_apache_arrow_request( + url=self._config.url, + delimiter=delimiter, + version=self._config.version, + prefix=prefix, + marker=marker, + maxresults=maxresults, + include=include, + timeout=timeout, + start_from=start_from, + end_before=end_before, + request_id_parameter=request_id_parameter, + restype=restype, + comp=comp, + headers=_headers, + params=_params, + ) + _request.url = self._client.format_url(_request.url) + + _decompress = kwargs.pop("decompress", True) + _stream = True + pipeline_response: PipelineResponse = await self._client._pipeline.run( # pylint: disable=protected-access + _request, stream=_stream, **kwargs + ) + + response = pipeline_response.http_response + + if response.status_code not in [200]: + try: + await response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass + map_error(status_code=response.status_code, response=response, error_map=error_map) + error = self._deserialize.failsafe_deserialize( + _models.StorageError, + pipeline_response, + ) + raise HttpResponseError(response=response, model=error) + + response_headers = {} + response_headers["Content-Type"] = self._deserialize("str", response.headers.get("Content-Type")) + response_headers["x-ms-client-request-id"] = self._deserialize( + "str", response.headers.get("x-ms-client-request-id") + ) + response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) + response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) + response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) + + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) + + if cls: + return cls(pipeline_response, deserialized, response_headers) # type: ignore + + return deserialized # type: ignore + @distributed_trace_async async def get_account_info( self, timeout: Optional[int] = None, request_id_parameter: Optional[str] = None, **kwargs: Any diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_container_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_container_operations.py index ec2deb0de1c0..ea0758fdb7aa 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_container_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_container_operations.py @@ -800,6 +800,62 @@ def build_list_blob_flat_segment_request( return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs) +def build_list_blob_flat_segment_apache_arrow_request( # pylint: disable=name-too-long + url: str, + *, + version: str, + prefix: Optional[str] = None, + marker: Optional[str] = None, + maxresults: Optional[int] = None, + include: Optional[list[Union[str, _models.ListBlobsIncludeItem]]] = None, + timeout: Optional[int] = None, + start_from: Optional[str] = None, + end_before: Optional[str] = None, + request_id_parameter: Optional[str] = None, + **kwargs: Any +) -> HttpRequest: + _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {}) + _params = case_insensitive_dict(kwargs.pop("params", {}) or {}) + + restype: Literal["container"] = kwargs.pop("restype", _params.pop("restype", "container")) + comp: Literal["list"] = kwargs.pop("comp", _params.pop("comp", "list")) + accept = _headers.pop("Accept", "application/vnd.apache.arrow.stream,application/xml") + + # Construct URL + _url = kwargs.pop("template_url", "{url}") + path_format_arguments = { + "url": _SERIALIZER.url("url", url, "str", skip_quote=True), + } + + _url: str = _url.format(**path_format_arguments) # type: ignore + + # Construct parameters + _params["restype"] = _SERIALIZER.query("restype", restype, "str") + _params["comp"] = _SERIALIZER.query("comp", comp, "str") + if prefix is not None: + _params["prefix"] = _SERIALIZER.query("prefix", prefix, "str") + if marker is not None: + _params["marker"] = _SERIALIZER.query("marker", marker, "str") + if maxresults is not None: + _params["maxresults"] = _SERIALIZER.query("maxresults", maxresults, "int", minimum=1) + if include is not None: + _params["include"] = _SERIALIZER.query("include", include, "[str]", div=",") + if timeout is not None: + _params["timeout"] = _SERIALIZER.query("timeout", timeout, "int", minimum=0) + if start_from is not None: + _params["startFrom"] = _SERIALIZER.query("start_from", start_from, "str") + if end_before is not None: + _params["endBefore"] = _SERIALIZER.query("end_before", end_before, "str") + + # Construct headers + _headers["Accept"] = _SERIALIZER.header("accept", accept, "str") + _headers["x-ms-version"] = _SERIALIZER.header("version", version, "str") + if request_id_parameter is not None: + _headers["x-ms-client-request-id"] = _SERIALIZER.header("request_id_parameter", request_id_parameter, "str") + + return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs) + + def build_list_blob_hierarchy_segment_request( # pylint: disable=name-too-long url: str, *, @@ -855,6 +911,64 @@ def build_list_blob_hierarchy_segment_request( # pylint: disable=name-too-long return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs) +def build_list_blob_hierarchy_segment_apache_arrow_request( # pylint: disable=name-too-long + url: str, + *, + delimiter: str, + version: str, + prefix: Optional[str] = None, + marker: Optional[str] = None, + maxresults: Optional[int] = None, + include: Optional[list[Union[str, _models.ListBlobsIncludeItem]]] = None, + timeout: Optional[int] = None, + start_from: Optional[str] = None, + end_before: Optional[str] = None, + request_id_parameter: Optional[str] = None, + **kwargs: Any +) -> HttpRequest: + _headers = case_insensitive_dict(kwargs.pop("headers", {}) or {}) + _params = case_insensitive_dict(kwargs.pop("params", {}) or {}) + + restype: Literal["container"] = kwargs.pop("restype", _params.pop("restype", "container")) + comp: Literal["list"] = kwargs.pop("comp", _params.pop("comp", "list")) + accept = _headers.pop("Accept", "application/vnd.apache.arrow.stream,application/xml") + + # Construct URL + _url = kwargs.pop("template_url", "{url}") + path_format_arguments = { + "url": _SERIALIZER.url("url", url, "str", skip_quote=True), + } + + _url: str = _url.format(**path_format_arguments) # type: ignore + + # Construct parameters + _params["restype"] = _SERIALIZER.query("restype", restype, "str") + _params["comp"] = _SERIALIZER.query("comp", comp, "str") + if prefix is not None: + _params["prefix"] = _SERIALIZER.query("prefix", prefix, "str") + _params["delimiter"] = _SERIALIZER.query("delimiter", delimiter, "str") + if marker is not None: + _params["marker"] = _SERIALIZER.query("marker", marker, "str") + if maxresults is not None: + _params["maxresults"] = _SERIALIZER.query("maxresults", maxresults, "int", minimum=1) + if include is not None: + _params["include"] = _SERIALIZER.query("include", include, "[str]", div=",") + if timeout is not None: + _params["timeout"] = _SERIALIZER.query("timeout", timeout, "int", minimum=0) + if start_from is not None: + _params["startFrom"] = _SERIALIZER.query("start_from", start_from, "str") + if end_before is not None: + _params["endBefore"] = _SERIALIZER.query("end_before", end_before, "str") + + # Construct headers + _headers["Accept"] = _SERIALIZER.header("accept", accept, "str") + _headers["x-ms-version"] = _SERIALIZER.header("version", version, "str") + if request_id_parameter is not None: + _headers["x-ms-client-request-id"] = _SERIALIZER.header("request_id_parameter", request_id_parameter, "str") + + return HttpRequest(method="GET", url=_url, params=_params, headers=_headers, **kwargs) + + def build_get_account_info_request( url: str, *, version: str, timeout: Optional[int] = None, request_id_parameter: Optional[str] = None, **kwargs: Any ) -> HttpRequest: @@ -2501,6 +2615,132 @@ def list_blob_flat_segment( return deserialized # type: ignore + @distributed_trace + def list_blob_flat_segment_apache_arrow( + self, + prefix: Optional[str] = None, + marker: Optional[str] = None, + maxresults: Optional[int] = None, + include: Optional[list[Union[str, _models.ListBlobsIncludeItem]]] = None, + timeout: Optional[int] = None, + start_from: Optional[str] = None, + end_before: Optional[str] = None, + request_id_parameter: Optional[str] = None, + **kwargs: Any + ) -> Iterator[bytes]: + """The List Blobs operation returns a list of the blobs under the specified container. This + operation is for Apache Arrow use case so response is returned as raw to be deserialized by the + client. + + :param prefix: Filters the results to return only containers whose name begins with the + specified prefix. Default value is None. + :type prefix: str + :param marker: A string value that identifies the portion of the list of containers to be + returned with the next listing operation. The operation returns the NextMarker value within the + response body if the listing operation did not return all containers remaining to be listed + with the current page. The NextMarker value can be used as the value for the marker parameter + in a subsequent call to request the next page of list items. The marker value is opaque to the + client. Default value is None. + :type marker: str + :param maxresults: Specifies the maximum number of containers to return. If the request does + not specify maxresults, or specifies a value greater than 5000, the server will return up to + 5000 items. Note that if the listing operation crosses a partition boundary, then the service + will return a continuation token for retrieving the remainder of the results. For this reason, + it is possible that the service will return fewer results than specified by maxresults, or than + the default of 5000. Default value is None. + :type maxresults: int + :param include: Include this parameter to specify one or more datasets to include in the + response. Default value is None. + :type include: list[str or ~azure.storage.blob.models.ListBlobsIncludeItem] + :param timeout: The timeout parameter is expressed in seconds. For more information, see + :code:`Setting + Timeouts for Blob Service Operations.`. Default value is None. + :type timeout: int + :param start_from: Specifies the relative path to list paths from. For non-recursive list, only + one entity level is supported; For recursive list, multiple entity levels are supported. + (Inclusive). Default value is None. + :type start_from: str + :param end_before: Specifies the relative path to end before list paths. (Exclusive). Default + value is None. + :type end_before: str + :param request_id_parameter: Provides a client-generated, opaque value with a 1 KB character + limit that is recorded in the analytics logs when storage analytics logging is enabled. Default + value is None. + :type request_id_parameter: str + :return: Iterator[bytes] or the result of cls(response) + :rtype: Iterator[bytes] + :raises ~azure.core.exceptions.HttpResponseError: + """ + error_map: MutableMapping = { + 401: ClientAuthenticationError, + 404: ResourceNotFoundError, + 409: ResourceExistsError, + 304: ResourceNotModifiedError, + } + error_map.update(kwargs.pop("error_map", {}) or {}) + + _headers = kwargs.pop("headers", {}) or {} + _params = case_insensitive_dict(kwargs.pop("params", {}) or {}) + + restype: Literal["container"] = kwargs.pop("restype", _params.pop("restype", "container")) + comp: Literal["list"] = kwargs.pop("comp", _params.pop("comp", "list")) + cls: ClsType[Iterator[bytes]] = kwargs.pop("cls", None) + + _request = build_list_blob_flat_segment_apache_arrow_request( + url=self._config.url, + version=self._config.version, + prefix=prefix, + marker=marker, + maxresults=maxresults, + include=include, + timeout=timeout, + start_from=start_from, + end_before=end_before, + request_id_parameter=request_id_parameter, + restype=restype, + comp=comp, + headers=_headers, + params=_params, + ) + _request.url = self._client.format_url(_request.url) + + _decompress = kwargs.pop("decompress", True) + _stream = True + pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access + _request, stream=_stream, **kwargs + ) + + response = pipeline_response.http_response + + if response.status_code not in [200]: + try: + response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass + map_error(status_code=response.status_code, response=response, error_map=error_map) + error = self._deserialize.failsafe_deserialize( + _models.StorageError, + pipeline_response, + ) + raise HttpResponseError(response=response, model=error) + + response_headers = {} + response_headers["Content-Type"] = self._deserialize("str", response.headers.get("Content-Type")) + response_headers["x-ms-client-request-id"] = self._deserialize( + "str", response.headers.get("x-ms-client-request-id") + ) + response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) + response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) + response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) + + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) + + if cls: + return cls(pipeline_response, deserialized, response_headers) # type: ignore + + return deserialized # type: ignore + @distributed_trace def list_blob_hierarchy_segment( self, @@ -2622,6 +2862,139 @@ def list_blob_hierarchy_segment( return deserialized # type: ignore + @distributed_trace + def list_blob_hierarchy_segment_apache_arrow( + self, + delimiter: str, + prefix: Optional[str] = None, + marker: Optional[str] = None, + maxresults: Optional[int] = None, + include: Optional[list[Union[str, _models.ListBlobsIncludeItem]]] = None, + timeout: Optional[int] = None, + start_from: Optional[str] = None, + end_before: Optional[str] = None, + request_id_parameter: Optional[str] = None, + **kwargs: Any + ) -> Iterator[bytes]: + """[Update] The List Blobs operation returns a list of the blobs under the specified container. + This operation is for Apache Arrow use case so response is returned as raw to be deserialized + by the client. + + :param delimiter: When the request includes this parameter, the operation returns a BlobPrefix + element in the response body that acts as a placeholder for all blobs whose names begin with + the same substring up to the appearance of the delimiter character. The delimiter may be a + single character or a string. Required. + :type delimiter: str + :param prefix: Filters the results to return only containers whose name begins with the + specified prefix. Default value is None. + :type prefix: str + :param marker: A string value that identifies the portion of the list of containers to be + returned with the next listing operation. The operation returns the NextMarker value within the + response body if the listing operation did not return all containers remaining to be listed + with the current page. The NextMarker value can be used as the value for the marker parameter + in a subsequent call to request the next page of list items. The marker value is opaque to the + client. Default value is None. + :type marker: str + :param maxresults: Specifies the maximum number of containers to return. If the request does + not specify maxresults, or specifies a value greater than 5000, the server will return up to + 5000 items. Note that if the listing operation crosses a partition boundary, then the service + will return a continuation token for retrieving the remainder of the results. For this reason, + it is possible that the service will return fewer results than specified by maxresults, or than + the default of 5000. Default value is None. + :type maxresults: int + :param include: Include this parameter to specify one or more datasets to include in the + response. Default value is None. + :type include: list[str or ~azure.storage.blob.models.ListBlobsIncludeItem] + :param timeout: The timeout parameter is expressed in seconds. For more information, see + :code:`Setting + Timeouts for Blob Service Operations.`. Default value is None. + :type timeout: int + :param start_from: Specifies the relative path to list paths from. For non-recursive list, only + one entity level is supported; For recursive list, multiple entity levels are supported. + (Inclusive). Default value is None. + :type start_from: str + :param end_before: Specifies the relative path to end before list paths. (Exclusive). Default + value is None. + :type end_before: str + :param request_id_parameter: Provides a client-generated, opaque value with a 1 KB character + limit that is recorded in the analytics logs when storage analytics logging is enabled. Default + value is None. + :type request_id_parameter: str + :return: Iterator[bytes] or the result of cls(response) + :rtype: Iterator[bytes] + :raises ~azure.core.exceptions.HttpResponseError: + """ + error_map: MutableMapping = { + 401: ClientAuthenticationError, + 404: ResourceNotFoundError, + 409: ResourceExistsError, + 304: ResourceNotModifiedError, + } + error_map.update(kwargs.pop("error_map", {}) or {}) + + _headers = kwargs.pop("headers", {}) or {} + _params = case_insensitive_dict(kwargs.pop("params", {}) or {}) + + restype: Literal["container"] = kwargs.pop("restype", _params.pop("restype", "container")) + comp: Literal["list"] = kwargs.pop("comp", _params.pop("comp", "list")) + cls: ClsType[Iterator[bytes]] = kwargs.pop("cls", None) + + _request = build_list_blob_hierarchy_segment_apache_arrow_request( + url=self._config.url, + delimiter=delimiter, + version=self._config.version, + prefix=prefix, + marker=marker, + maxresults=maxresults, + include=include, + timeout=timeout, + start_from=start_from, + end_before=end_before, + request_id_parameter=request_id_parameter, + restype=restype, + comp=comp, + headers=_headers, + params=_params, + ) + _request.url = self._client.format_url(_request.url) + + _decompress = kwargs.pop("decompress", True) + _stream = True + pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access + _request, stream=_stream, **kwargs + ) + + response = pipeline_response.http_response + + if response.status_code not in [200]: + try: + response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass + map_error(status_code=response.status_code, response=response, error_map=error_map) + error = self._deserialize.failsafe_deserialize( + _models.StorageError, + pipeline_response, + ) + raise HttpResponseError(response=response, model=error) + + response_headers = {} + response_headers["Content-Type"] = self._deserialize("str", response.headers.get("Content-Type")) + response_headers["x-ms-client-request-id"] = self._deserialize( + "str", response.headers.get("x-ms-client-request-id") + ) + response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) + response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) + response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) + + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) + + if cls: + return cls(pipeline_response, deserialized, response_headers) # type: ignore + + return deserialized # type: ignore + @distributed_trace def get_account_info( # pylint: disable=inconsistent-return-statements self, timeout: Optional[int] = None, request_id_parameter: Optional[str] = None, **kwargs: Any diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py index 8538f9344212..0e2f54d1ed51 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py @@ -66,6 +66,7 @@ "2026-04-06", "2026-06-06", "2026-10-06", + "2026-12-06", ] From 52e9ba27a96e5d7b76133c5c8c0a0fcd41d47545 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 1 Jul 2026 13:02:28 -0400 Subject: [PATCH 02/20] Arrow support for list blobs --- sdk/storage/azure-storage-blob/CHANGELOG.md | 4 + .../azure/storage/blob/_container_client.py | 73 ++- .../azure/storage/blob/_container_client.pyi | 2 + .../azure/storage/blob/_list_blobs_helper.py | 200 ++++++- .../blob/aio/_container_client_async.py | 78 ++- .../storage/blob/aio/_list_blobs_helper.py | 67 +++ .../azure-storage-blob/dev_requirements.txt | 2 + sdk/storage/azure-storage-blob/setup.py | 8 +- .../azure-storage-blob/tests/test_arrow.py | 481 +++++++++++++++++ .../tests/test_arrow_async.py | 486 ++++++++++++++++++ 10 files changed, 1394 insertions(+), 7 deletions(-) create mode 100644 sdk/storage/azure-storage-blob/tests/test_arrow.py create mode 100644 sdk/storage/azure-storage-blob/tests/test_arrow_async.py diff --git a/sdk/storage/azure-storage-blob/CHANGELOG.md b/sdk/storage/azure-storage-blob/CHANGELOG.md index bab7f3e87b36..c85315d6476f 100644 --- a/sdk/storage/azure-storage-blob/CHANGELOG.md +++ b/sdk/storage/azure-storage-blob/CHANGELOG.md @@ -4,6 +4,10 @@ ### Features Added - Added support for service version 2026-10-06. +- Added support for parsing List Blobs responses in Apache Arrow format. Pass the +`use_arrow=True` keyword to `ContainerClient`'s `list_blobs` and `walk_blobs` APIs +to opt in. This requires the `nanoarrow` package and also enables the new +`start_from` and `end_before` keywords for range-based listing. - Added access tier information to the response of `BlobClient`'s `download_blob` API. The `blob_tier`, `blob_tier_inferred`, `blob_tier_change_time`, and `smart_access_tier` properties are now populated on the downloaded blob's `properties`. diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py index 90c151a9ddc7..80d7f42b22ca 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py @@ -30,6 +30,8 @@ from ._generated.models import SignedIdentifier from ._lease import BlobLeaseClient from ._list_blobs_helper import ( + ArrowBlobPropertiesPaged, + ArrowBlobPrefixPaged, BlobNamesPaged, BlobPrefix, BlobPropertiesPaged, @@ -803,9 +805,19 @@ def list_blobs( :keyword int results_per_page: Controls the maximum number of Blobs that will be included in each page of results if using `ItemPaged.by_page()`. + :keyword response_format: + The format used to return and parse the List Blobs response. Possible values are + "auto", "xml", and "arrow". When set to "arrow", the response is returned and parsed in + Apache Arrow format (this requires the `nanoarrow` package). When set to "xml", the + standard XML response is parsed. "auto" currently behaves the same as "xml". This value + defaults to "xml". + :paramtype response_format: Literal["auto", "xml", "arrow"] :keyword str start_from: Specifies the full path (inclusive) to list paths from. Only one entity level is supported. + :keyword str end_before: + Specifies the relative path (exclusive) to end before list paths. + This may be used if response_format is set to "arrow". :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. @@ -834,15 +846,35 @@ def list_blobs( results_per_page = kwargs.pop("results_per_page", None) timeout = kwargs.pop("timeout", None) + response_format = kwargs.pop("response_format", "xml") + use_arrow = response_format == "arrow" + if use_arrow: + try: + import nanoarrow # pylint: disable=import-outside-toplevel,unused-import + except ImportError as e: + raise ImportError( + "The 'nanoarrow' package is required to use Apache Arrow deserialization. " + "Install it with: pip install nanoarrow" + ) from e + command = functools.partial( - self._client.container.list_blob_flat_segment, include=include, timeout=timeout, **kwargs + ( + self._client.container.list_blob_flat_segment_apache_arrow + if use_arrow + else self._client.container.list_blob_flat_segment + ), + include=include, + timeout=timeout, + **kwargs, ) return ItemPaged( command, prefix=name_starts_with, results_per_page=results_per_page, container=self.container_name, - page_iterator_class=BlobPropertiesPaged, + page_iterator_class=ArrowBlobPropertiesPaged if use_arrow else BlobPropertiesPaged, + # pylint: disable-next=protected-access + **({"deserializer": self._client.container._deserialize} if use_arrow else {}), ) @distributed_trace @@ -922,8 +954,18 @@ def walk_blobs( element in the response body that acts as a placeholder for all blobs whose names begin with the same substring up to the appearance of the delimiter character. The delimiter may be a single character or a string. + :keyword response_format: + The format used to return and parse the List Blobs response. Possible values are + "auto", "xml", and "arrow". When set to "arrow", the response is returned and parsed in + Apache Arrow format (this requires the `nanoarrow` package). When set to "xml", the + standard XML response is parsed. "auto" currently behaves the same as "xml". This value + defaults to "xml". + :paramtype response_format: Literal["auto", "xml", "arrow"] :keyword str start_from: Specifies the full path (inclusive) to list paths from. + :keyword str end_before: + Specifies the relative path (exclusive) to end before list paths. + This may be used if response_format is set to "arrow". :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. @@ -943,6 +985,33 @@ def walk_blobs( results_per_page = kwargs.pop("results_per_page", None) timeout = kwargs.pop("timeout", None) + response_format = kwargs.pop("response_format", "xml") + use_arrow = response_format == "arrow" + if use_arrow: + try: + import nanoarrow # pylint: disable=import-outside-toplevel,unused-import + except ImportError as e: + raise ImportError( + "The 'nanoarrow' package is required to use Apache Arrow deserialization. " + "Install it with: pip install nanoarrow" + ) from e + command = functools.partial( + self._client.container.list_blob_hierarchy_segment_apache_arrow, + delimiter=delimiter, + include=include, + timeout=timeout, + **kwargs, + ) + return ItemPaged( + command, + prefix=name_starts_with, + results_per_page=results_per_page, + container=self.container_name, + delimiter=delimiter, + deserializer=self._client.container._deserialize, # pylint: disable=protected-access + page_iterator_class=ArrowBlobPrefixPaged, + ) + command = functools.partial( self._client.container.list_blob_hierarchy_segment, delimiter=delimiter, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.pyi b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.pyi index fe3e0142beef..081c783c226c 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.pyi +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.pyi @@ -202,7 +202,9 @@ class ContainerClient(StorageAccountHostsMixin, StorageEncryptionMixin): include: Optional[Union[str, List[str]]] = None, *, results_per_page: Optional[int] = None, + response_format: Literal["auto", "xml", "arrow"] = "xml", start_from: Optional[str] = None, + end_before: Optional[str] = None, timeout: Optional[int] = None, **kwargs: Any, ) -> ItemPaged[BlobProperties]: ... diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py index b57e2f172da6..9b274a96d097 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py @@ -19,10 +19,155 @@ ) from ._generated.models import BlobItemInternal, BlobPrefix as GenBlobPrefix, FilterBlobItem from ._generated._utils.serialization import Deserializer -from ._models import BlobProperties, FilteredBlob +from ._models import ( + BlobProperties, + BlobType, + ContentSettings, + CopyProperties, + FilteredBlob, + ImmutabilityPolicy, + LeaseProperties, +) from ._shared.models import DictMixin from ._shared.response_handlers import process_storage_error, return_context_and_deserialized, return_raw_deserialized +_ARROW_CONTENT_TYPE = "application/vnd.apache.arrow.stream,application/xml" + + +def _parse_arrow_response( # pylint: disable=too-many-locals + raw_bytes: bytes, container: Optional[str] +) -> Tuple[Optional[str], List[BlobProperties]]: + """ + Parse an Apache Arrow IPC stream response into a list of BlobProperties. + + :param bytes raw_bytes: The raw Arrow IPC bytes. + :param Optional[str] container: The container name to stamp on each item. + :returns: A tuple of next marker and a list of BlobProperties. + :rtype: Tuple[Optional[str], List[~azure.storage.blob.BlobProperties]] + """ + import io # pylint: disable=import-outside-toplevel + import nanoarrow # pylint: disable=import-outside-toplevel + from nanoarrow.ipc import InputStream # pylint: disable=import-outside-toplevel + + # Declarative mapping: Arrow column name -> (BlobProperties attr, default value). + # Only scalar fields that map 1-to-1 are listed here; composite sub-objects are + # handled separately below. + _SCALAR_FIELDS: List[Tuple[str, str, Any]] = [ + ("Name", "name", None), + ("Snapshot", "snapshot", None), + ("VersionId", "version_id", None), + ("IsCurrentVersion", "is_current_version", None), + ("Etag", "etag", None), + ("Deleted", "deleted", False), + ("LastModified", "last_modified", None), + ("CreationTime", "creation_time", None), + ("ContentLength", "size", None), + ("ServerEncrypted", "server_encrypted", False), + ("EncryptionScope", "encryption_scope", None), + ("DeletedTime", "deleted_time", None), + ("RemainingRetentionDays", "remaining_retention_days", None), + ("AccessTier", "blob_tier", None), + ("SmartAccessTier", "smart_access_tier", None), + ("RehydratePriority", "rehydrate_priority", None), + ("AccessTierChangeTime", "blob_tier_change_time", None), + ("AccessTierInferred", "blob_tier_inferred", None), + ("ArchiveStatus", "archive_status", None), + ("BlobSequenceNumber", "page_blob_sequence_number", None), + ("IsSealed", "is_append_blob_sealed", None), + ("LastAccessedOn", "last_accessed_on", None), + ("TagCount", "tag_count", None), + ("HasVersionsOnly", "has_versions_only", None), + ("LegalHold", "has_legal_hold", None), + ("CustomerProvidedKeySha256", "encryption_key_sha256", None), + ] + + # Sub-object field mappings: Arrow column name -> constructor kwarg name. + _CONTENT_SETTINGS_FIELDS = { + "ContentType": "content_type", + "ContentEncoding": "content_encoding", + "ContentLanguage": "content_language", + "ContentMD5": "content_md5", + "ContentDisposition": "content_disposition", + "CacheControl": "cache_control", + } + _LEASE_FIELDS = { + "LeaseStatus": "x-ms-lease-status", + "LeaseState": "x-ms-lease-state", + "LeaseDuration": "x-ms-lease-duration", + } + _COPY_FIELDS = { + "CopyId": "x-ms-copy-id", + "CopySource": "x-ms-copy-source", + "CopyStatus": "x-ms-copy-status", + "CopyProgress": "x-ms-copy-progress", + "CopyCompletionTime": "x-ms-copy-completion-time", + "CopyStatusDescription": "x-ms-copy-status-description", + "IncrementalCopy": "x-ms-incremental-copy", + "DestinationSnapshot": "x-ms-copy-destination-snapshot", + } + + next_marker: Optional[str] = None + blob_items: List[BlobProperties] = [] + + with InputStream.from_readable(io.BytesIO(raw_bytes)) as stream: + reader = nanoarrow.ArrayStream(stream) + schema = reader.schema + + # The continuation token is embedded in the Arrow schema metadata. + metadata = schema.metadata + if metadata is not None: + meta_map = dict(metadata.items()) + raw_marker = meta_map.get(b"NextMarker") or meta_map.get(b"nextMarker") + if raw_marker: + next_marker = raw_marker.decode("utf-8") if isinstance(raw_marker, bytes) else raw_marker + next_marker = next_marker or None + + field_names = [field.name for field in schema.fields] + + for batch in reader: + num_rows = len(batch) + if num_rows == 0: + continue + # Pre-resolve columns once per batch; missing columns stay as None. + cols = {field_names[i]: batch.child(i).to_pylist() for i in range(batch.n_children)} + + for row in range(num_rows): + + def _get( + col_name: str, default: Any = None, _r: int = row + ) -> Any: # pylint: disable=cell-var-from-loop + col = cols.get(col_name) # pylint: disable=cell-var-from-loop + if col is None: + return default + val = col[_r] + return val if val is not None else default + + blob = BlobProperties() + blob.container = container # type: ignore[assignment] + blob.metadata = {} + + # Apply all scalar 1-to-1 fields from the mapping table. + for arrow_col, blob_attr, default in _SCALAR_FIELDS: + setattr(blob, blob_attr, _get(arrow_col, default)) + + # BlobType needs an enum conversion. + blob_type_val = _get("BlobType") + blob.blob_type = BlobType(blob_type_val) if blob_type_val else None # type: ignore[assignment] + + # Composite sub-objects built from their own column sub-sets. + blob.content_settings = ContentSettings( + **{kwarg: _get(col) for col, kwarg in _CONTENT_SETTINGS_FIELDS.items()} + ) + blob.lease = LeaseProperties(**{kwarg: _get(col) for col, kwarg in _LEASE_FIELDS.items()}) + blob.copy = CopyProperties(**{kwarg: _get(col) for col, kwarg in _COPY_FIELDS.items()}) + blob.immutability_policy = ImmutabilityPolicy( + expiry_time=_get("ImmutabilityPolicyUntilDate"), + policy_mode=_get("ImmutabilityPolicyMode"), + ) + blob_items.append(blob) + + return next_marker, blob_items + class IgnoreListBlobsDeserializer(Deserializer): def __call__(self, target_obj, response_data, content_type=None): # pylint: disable=inconsistent-return-statements @@ -323,3 +468,56 @@ def _build_item(item): blob = FilteredBlob(name=item.name, container_name=item.container_name, tags=tags) return blob return item + + +class ArrowBlobPropertiesPaged(BlobPropertiesPaged): + """A PageIterator that deserializes Apache Arrow IPC responses from list-blobs operations.""" + + def __init__(self, *args: Any, deserializer: Any = None, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._deserializer = deserializer + self._arrow_response: Optional[Tuple[Optional[str], List[BlobProperties]]] = None + + def _get_next_cb(self, continuation_token: Optional[str]) -> Any: + try: + return self._command( + prefix=self.prefix, + marker=continuation_token or None, + maxresults=self.results_per_page, + cls=self._arrow_cls, + use_location=self.location_mode, + ) + except HttpResponseError as error: + process_storage_error(error) + + def _arrow_cls(self, pipeline_response: Any, deserialized: Any, response_headers: Any) -> Any: + content_type = response_headers.get("Content-Type", "") + location_mode = getattr(pipeline_response.http_response, "location_mode", None) + # The response is Arrow only when the service returns the Arrow stream media type. + if _ARROW_CONTENT_TYPE.split(",", 1)[0] in content_type: + raw_bytes = b"".join(deserialized) + next_marker, blob_items = _parse_arrow_response(raw_bytes, self.container) + self._arrow_response = (next_marker, blob_items) + return location_mode, raw_bytes + if hasattr(pipeline_response.http_response, "read"): + pipeline_response.http_response.read() + xml_response = self._deserializer("ListBlobsFlatSegmentResponse", pipeline_response.http_response) + self._arrow_response = None + return location_mode, xml_response + + def _extract_data_cb(self, get_next_return: Any) -> Tuple[Optional[str], List[BlobProperties]]: + if self._arrow_response is not None: + self.location_mode, _ = cast(Tuple[Optional[str], Any], get_next_return) + next_marker, self.current_page = self._arrow_response + self._arrow_response = None + return next_marker or None, self.current_page or [] + return super()._extract_data_cb(get_next_return) + + +class ArrowBlobPrefixPaged(ArrowBlobPropertiesPaged): + """Arrow-backed PageIterator for walk_blobs.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.name = self.prefix + self.delimiter: Optional[str] = kwargs.get("delimiter") diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py index 97927c5367cf..fc4a0e079807 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py @@ -36,7 +36,13 @@ from ._blob_client_async import BlobClient from ._download_async import StorageStreamDownloader from ._lease_async import BlobLeaseClient -from ._list_blobs_helper import BlobNamesPaged, BlobPropertiesPaged, BlobPrefix +from ._list_blobs_helper import ( + ArrowBlobPropertiesPaged, + ArrowBlobPrefixPaged, + BlobNamesPaged, + BlobPropertiesPaged, + BlobPrefix, +) from ._models import FilteredBlobPaged from .._container_client_helpers import ( _format_url, @@ -820,9 +826,19 @@ def list_blobs( :keyword int results_per_page: Controls the maximum number of Blobs that will be included in each page of results if using `AsyncItemPaged.by_page()`. + :keyword response_format: + The format used to return and parse the List Blobs response. Possible values are + "auto", "xml", and "arrow". When set to "arrow", the response is returned and parsed in + Apache Arrow format (this requires the `nanoarrow` package). When set to "xml", the + standard XML response is parsed. "auto" currently behaves the same as "xml". This value + defaults to "xml". + :paramtype response_format: Literal["auto", "xml", "arrow"] :keyword str start_from: Specifies the full path (inclusive) to list paths from. Only one entity level is supported. + :keyword str end_before: + Specifies the relative path (exclusive) to end before list paths. + This may be used if response_format is set to "arrow". :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. @@ -851,15 +867,34 @@ def list_blobs( results_per_page = kwargs.pop("results_per_page", None) timeout = kwargs.pop("timeout", None) + response_format = kwargs.pop("response_format", "xml") + use_arrow = response_format == "arrow" + if use_arrow: + try: + import nanoarrow # pylint: disable=import-outside-toplevel,unused-import + except ImportError as e: + raise ImportError( + "The 'nanoarrow' package is required to use Apache Arrow deserialization. " + "Install it with: pip install nanoarrow" + ) from e command = functools.partial( - self._client.container.list_blob_flat_segment, include=include, timeout=timeout, **kwargs + ( + self._client.container.list_blob_flat_segment_apache_arrow + if use_arrow + else self._client.container.list_blob_flat_segment + ), + include=include, + timeout=timeout, + **kwargs, ) return AsyncItemPaged( command, prefix=name_starts_with, results_per_page=results_per_page, container=self.container_name, - page_iterator_class=BlobPropertiesPaged, + page_iterator_class=ArrowBlobPropertiesPaged if use_arrow else BlobPropertiesPaged, + # pylint: disable-next=protected-access + **({"deserializer": self._client.container._deserialize} if use_arrow else {}), ) @distributed_trace @@ -939,8 +974,18 @@ def walk_blobs( element in the response body that acts as a placeholder for all blobs whose names begin with the same substring up to the appearance of the delimiter character. The delimiter may be a single character or a string. + :keyword response_format: + The format used to return and parse the List Blobs response. Possible values are + "auto", "xml", and "arrow". When set to "arrow", the response is returned and parsed in + Apache Arrow format (this requires the `nanoarrow` package). When set to "xml", the + standard XML response is parsed. "auto" currently behaves the same as "xml". This value + defaults to "xml". + :paramtype response_format: Literal["auto", "xml", "arrow"] :keyword str start_from: Specifies the full path (inclusive) to list paths from. + :keyword str end_before: + Specifies the relative path (exclusive) to end before list paths. + This may be used if response_format is set to "arrow". :keyword int timeout: Sets the server-side timeout for the operation in seconds. For more details see https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations. @@ -961,6 +1006,33 @@ def walk_blobs( results_per_page = kwargs.pop("results_per_page", None) timeout = kwargs.pop("timeout", None) + response_format = kwargs.pop("response_format", "xml") + use_arrow = response_format == "arrow" + if use_arrow: + try: + import nanoarrow # pylint: disable=import-outside-toplevel,unused-import + except ImportError as e: + raise ImportError( + "The 'nanoarrow' package is required to use Apache Arrow deserialization. " + "Install it with: pip install nanoarrow" + ) from e + command = functools.partial( + self._client.container.list_blob_hierarchy_segment_apache_arrow, + delimiter=delimiter, + include=include, + timeout=timeout, + **kwargs, + ) + return AsyncItemPaged( + command, + prefix=name_starts_with, + results_per_page=results_per_page, + container=self.container_name, + delimiter=delimiter, + deserializer=self._client.container._deserialize, # pylint: disable=protected-access + page_iterator_class=ArrowBlobPrefixPaged, + ) + command = functools.partial( self._client.container.list_blob_hierarchy_segment, delimiter=delimiter, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py index 7fa0d5365625..abecde2b10d7 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +import inspect from typing import Callable, List, Optional from urllib.parse import unquote @@ -12,6 +13,7 @@ from .._deserialize import get_blob_properties_from_generated_code, load_many_xml_nodes, load_xml_int, load_xml_string from .._generated.models import BlobItemInternal, BlobPrefix as GenBlobPrefix +from .._list_blobs_helper import _ARROW_CONTENT_TYPE, _parse_arrow_response from .._models import BlobProperties from .._shared.models import DictMixin from .._shared.response_handlers import process_storage_error, return_context_and_deserialized, return_raw_deserialized @@ -239,3 +241,68 @@ def _build_item(self, item): location_mode=self.location_mode, ) return item + + +class ArrowBlobPropertiesPaged(BlobPropertiesPaged): + """An async PageIterator that deserializes Apache Arrow IPC responses from list-blobs operations.""" + + def __init__(self, *args, deserializer=None, **kwargs): + super().__init__(*args, **kwargs) + self._deserializer = deserializer + self._arrow_response = None + + async def _arrow_cls(self, pipeline_response, deserialized, response_headers): + content_type = response_headers.get("Content-Type", "") + location_mode = getattr(pipeline_response.http_response, "location_mode", None) + # The response is Arrow only when the service returns the Arrow stream media type. + if _ARROW_CONTENT_TYPE.split(",", 1)[0] in content_type: + chunks = [] + if hasattr(deserialized, "__aiter__"): + async for chunk in deserialized: + chunks.append(chunk) + else: + for chunk in deserialized: + chunks.append(chunk) + raw_bytes = b"".join(chunks) + next_marker, blob_items = _parse_arrow_response(raw_bytes, self.container) + self._arrow_response = (next_marker, blob_items) + return location_mode, raw_bytes + if hasattr(pipeline_response.http_response, "read"): + read_result = pipeline_response.http_response.read() + if inspect.isawaitable(read_result): + await read_result + xml_response = self._deserializer("ListBlobsFlatSegmentResponse", pipeline_response.http_response) + self._arrow_response = None + return location_mode, xml_response + + async def _get_next_cb(self, continuation_token): + try: + result = await self._command( + prefix=self.prefix, + marker=continuation_token or None, + maxresults=self.results_per_page, + cls=self._arrow_cls, + use_location=self.location_mode, + ) + if inspect.isawaitable(result): + return await result + return result + except HttpResponseError as error: + process_storage_error(error) + + async def _extract_data_cb(self, get_next_return): + if self._arrow_response is not None: + self.location_mode, _ = get_next_return + next_marker, self.current_page = self._arrow_response + self._arrow_response = None + return next_marker or None, self.current_page or [] + return await super()._extract_data_cb(get_next_return) + + +class ArrowBlobPrefixPaged(ArrowBlobPropertiesPaged): + """Arrow-backed AsyncPageIterator for walk_blobs.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.name = self.prefix + self.delimiter = kwargs.get("delimiter") diff --git a/sdk/storage/azure-storage-blob/dev_requirements.txt b/sdk/storage/azure-storage-blob/dev_requirements.txt index b8770ca0db3a..80ae73f833ae 100644 --- a/sdk/storage/azure-storage-blob/dev_requirements.txt +++ b/sdk/storage/azure-storage-blob/dev_requirements.txt @@ -4,3 +4,5 @@ ../azure-storage-extensions azure-mgmt-storage==20.1.0 aiohttp>=3.13.5 +nanoarrow>=0.8.0 +pyarrow>=23.0.1 diff --git a/sdk/storage/azure-storage-blob/setup.py b/sdk/storage/azure-storage-blob/setup.py index 6cc641bc3312..a05a2cc4a955 100644 --- a/sdk/storage/azure-storage-blob/setup.py +++ b/sdk/storage/azure-storage-blob/setup.py @@ -78,7 +78,13 @@ ] ), python_requires=">=3.9", - install_requires=["azure-core>=1.37.0", "cryptography>=2.1.4", "typing-extensions>=4.6.0", "isodate>=0.6.1"], + install_requires=[ + "azure-core>=1.37.0", + "cryptography>=2.1.4", + "typing-extensions>=4.6.0", + "isodate>=0.6.1", + "nanoarrow>=0.8.0", + ], extras_require={ "aio": [ "azure-core[aio]>=1.37.0", diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow.py b/sdk/storage/azure-storage-blob/tests/test_arrow.py new file mode 100644 index 000000000000..8ec188639043 --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/test_arrow.py @@ -0,0 +1,481 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pyarrow as pa + +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +from azure.core.credentials import AzureNamedKeyCredential +from azure.core.exceptions import ResourceExistsError +from azure.storage.blob import BlobProperties, BlobServiceClient, ContainerClient +from azure.storage.blob._generated.models import ( + BlobFlatListSegment, + BlobItemInternal, + BlobName, + BlobPropertiesInternal, + ListBlobsFlatSegmentResponse, +) + +from devtools_testutils import recorded_by_proxy +from devtools_testutils.storage import StorageRecordedTestCase +from settings.testcase import BlobPreparer + +# ------------------------------------------------------------------------------ +TEST_DATA = b"abc123" +# ------------------------------------------------------------------------------ + +class TestStorageApacheArrow(StorageRecordedTestCase): + def _setup(self, storage_account_name, storage_account_key): + self.bsc = BlobServiceClient( + self.account_url(storage_account_name, "blob"), + credential=storage_account_key.secret + ) + self.container_name = self.get_resource_name("utcontainerarrow") + if self.is_live: + try: + self.bsc.create_container(self.container_name) + except ResourceExistsError: + pass + + def create_blobs(self, blob_names: list[str]): + for blob_name in blob_names: + blob_client = self.bsc.get_blob_client(self.container_name, blob_name) + blob_client.upload_blob(TEST_DATA, overwrite=True) + + def verify_blobs(self, blobs_list: list[BlobProperties], blob_names: list[str]): + assert len(blobs_list) == len(blob_names) + all_names = {blob.name for blob in blobs_list} + for blob_name in blob_names: + assert blob_name in all_names + + @BlobPreparer() + @recorded_by_proxy + def test_arrow_list_no_blobs(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + container = self.bsc.get_container_client(self.container_name) + blobs_list = list(container.list_blobs(response_format="arrow")) + + self.verify_blobs(blobs_list, []) + + @BlobPreparer() + @recorded_by_proxy + def test_arrow_list_multiple_blobs(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + blob_names = ["blob1", "blob2"] + self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blobs_list = list(container.list_blobs(response_format="arrow")) + self.verify_blobs(blobs_list, blob_names) + + @BlobPreparer() + @recorded_by_proxy + def test_arrow_list_blobs_paging(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + blob_names = ["blob1", "blob2", "blob3", "blob4"] + self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blob_pages = container.list_blobs(response_format="arrow", results_per_page=2).by_page() + first_blobs_list = list(next(blob_pages)) + self.verify_blobs(first_blobs_list, blob_names[:2]) + second_blobs_list = list(next(blob_pages)) + self.verify_blobs(second_blobs_list, blob_names[2:]) + + @BlobPreparer() + @recorded_by_proxy + def test_arrow_list_nested_blobs_paging(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blob_pages = container.list_blobs(response_format="arrow", results_per_page=3).by_page() + first_blobs_list = list(next(blob_pages)) + self.verify_blobs(first_blobs_list, blob_names[:3]) + second_blobs_list = list(next(blob_pages)) + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = list(next(blob_pages)) + self.verify_blobs(third_blobs_list, blob_names[6:]) + + def test_arrow_mock_xml_response(self): + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + account_url = "https://account.blob.core.windows.net/" + container_name = "mycontainer" + + def _make_xml_response(names: list[str], next_marker: str | None = None) -> ListBlobsFlatSegmentResponse: + items = [ + BlobItemInternal( + name=BlobName(content=name, encoded=False), + snapshot="", + properties=BlobPropertiesInternal( + etag="", + last_modified=datetime(2024, 1, 1, tzinfo=timezone.utc), + blob_type="BlockBlob", + ), + deleted=False, + ) + for name in names + ] + return ListBlobsFlatSegmentResponse( + service_endpoint=account_url, + container_name=container_name, + segment=BlobFlatListSegment(blob_items=items), + next_marker=next_marker, + ) + + # Three pages of 3 blobs each, chained via next_marker. + xml_pages = [ + _make_xml_response(blob_names[0:3], next_marker="marker1"), + _make_xml_response(blob_names[3:6], next_marker="marker2"), + _make_xml_response(blob_names[6:9], next_marker=None), + ] + page_index = 0 + + container_client = ContainerClient( + account_url=account_url, + container_name=container_name, + credential=AzureNamedKeyCredential("account", "A" * 64), + ) + + mock_http_response = MagicMock() + mock_http_response.location_mode = None + mock_pipeline_response = MagicMock() + mock_pipeline_response.http_response = mock_http_response + + def fake_list_blob_flat_segment_apache_arrow(**kwargs): + cls = kwargs.get("cls") + return cls(mock_pipeline_response, None, {"Content-Type": "application/xml"}) + + def deserializer_side_effect(target, _http_response): + nonlocal page_index + response = xml_pages[page_index] + page_index += 1 + return response + + with patch.object( + container_client._client.container, # pylint: disable=protected-access + "list_blob_flat_segment_apache_arrow", + side_effect=fake_list_blob_flat_segment_apache_arrow, + ): + blob_pages = container_client.list_blobs(response_format="arrow", results_per_page=3).by_page() + mock_deserializer = MagicMock(side_effect=deserializer_side_effect) + blob_pages._deserializer = mock_deserializer + first_blobs_list = list(next(blob_pages)) + self.verify_blobs(first_blobs_list, blob_names[0:3]) + second_blobs_list = list(next(blob_pages)) + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = list(next(blob_pages)) + self.verify_blobs(third_blobs_list, blob_names[6:9]) + + # Confirm the XML deserializer was called once per page. + assert mock_deserializer.call_count == 3 + + def test_arrow_mock_expected_response(self): + def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: + schema_meta = {b"NextMarker": (next_marker or "").encode()} if next_marker else {} + schema = pa.schema([pa.field("Name", pa.string())], metadata=schema_meta) + batch = pa.record_batch([pa.array(names, type=pa.string())], schema=schema) + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, schema) as writer: + writer.write_batch(batch) + return sink.getvalue().to_pybytes() + + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + # Three pages of 3 blobs each, using the marker to chain pages. + pages = [ + _make_arrow_page(blob_names[0:3], next_marker="marker1"), + _make_arrow_page(blob_names[3:6], next_marker="marker2"), + _make_arrow_page(blob_names[6:9], next_marker=None), + ] + page_index = 0 + + container_client = ContainerClient( + account_url="https://account.blob.core.windows.net", + container_name="mycontainer", + credential=AzureNamedKeyCredential("account", "A" * 64), + ) + + mock_http_response = MagicMock() + mock_http_response.location_mode = None + mock_pipeline_response = MagicMock() + mock_pipeline_response.http_response = mock_http_response + + def fake_list_blob_flat_segment_apache_arrow(**kwargs): + nonlocal page_index + cls = kwargs.get("cls") + raw = iter([pages[page_index]]) + page_index += 1 + return cls(mock_pipeline_response, raw, {"Content-Type": "application/vnd.apache.arrow.stream"}) + + with patch.object( + container_client._client.container, # pylint: disable=protected-access + "list_blob_flat_segment_apache_arrow", + side_effect=fake_list_blob_flat_segment_apache_arrow, + ): + blob_pages = container_client.list_blobs(response_format="arrow", results_per_page=3).by_page() + first_blobs_list = list(next(blob_pages)) + self.verify_blobs(first_blobs_list, blob_names[0:3]) + second_blobs_list = list(next(blob_pages)) + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = list(next(blob_pages)) + self.verify_blobs(third_blobs_list, blob_names[6:9]) + + @BlobPreparer() + @recorded_by_proxy + def test_arrow_walk_no_blobs(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + container = self.bsc.get_container_client(self.container_name) + blobs_list = list(container.walk_blobs(response_format="arrow")) + + self.verify_blobs(blobs_list, []) + + @BlobPreparer() + @recorded_by_proxy + def test_arrow_walk_multiple_blobs(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + blob_names = ["blob1", "blob2"] + self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blobs_list = list(container.walk_blobs(response_format="arrow")) + self.verify_blobs(blobs_list, blob_names) + + @BlobPreparer() + @recorded_by_proxy + def test_arrow_walk_blobs_paging(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + blob_names = ["blob1", "blob2", "blob3", "blob4"] + self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blob_pages = container.walk_blobs(response_format="arrow", results_per_page=2).by_page() + first_blobs_list = list(next(blob_pages)) + self.verify_blobs(first_blobs_list, blob_names[:2]) + second_blobs_list = list(next(blob_pages)) + self.verify_blobs(second_blobs_list, blob_names[2:]) + + @BlobPreparer() + @recorded_by_proxy + def test_arrow_walk_nested_blobs_paging(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blob_pages = container.walk_blobs(response_format="arrow", results_per_page=3).by_page() + first_blobs_list = list(next(blob_pages)) + self.verify_blobs(first_blobs_list, blob_names[:3]) + second_blobs_list = list(next(blob_pages)) + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = list(next(blob_pages)) + self.verify_blobs(third_blobs_list, blob_names[6:]) + + def test_arrow_mock_walk_xml_response(self): + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + account_url = "https://account.blob.core.windows.net/" + container_name = "mycontainer" + + def _make_xml_response(names: list[str], next_marker: str | None = None) -> ListBlobsFlatSegmentResponse: + items = [ + BlobItemInternal( + name=BlobName(content=name, encoded=False), + snapshot="", + properties=BlobPropertiesInternal( + etag="", + last_modified=datetime(2024, 1, 1, tzinfo=timezone.utc), + blob_type="BlockBlob", + ), + deleted=False, + ) + for name in names + ] + return ListBlobsFlatSegmentResponse( + service_endpoint=account_url, + container_name=container_name, + segment=BlobFlatListSegment(blob_items=items), + next_marker=next_marker, + ) + + # Three pages of 3 blobs each, chained via next_marker. + xml_pages = [ + _make_xml_response(blob_names[0:3], next_marker="marker1"), + _make_xml_response(blob_names[3:6], next_marker="marker2"), + _make_xml_response(blob_names[6:9], next_marker=None), + ] + page_index = 0 + + container_client = ContainerClient( + account_url=account_url, + container_name=container_name, + credential=AzureNamedKeyCredential("account", "A" * 64), + ) + + mock_http_response = MagicMock() + mock_http_response.location_mode = None + mock_pipeline_response = MagicMock() + mock_pipeline_response.http_response = mock_http_response + + def fake_list_blob_hierarchy_segment_apache_arrow(**kwargs): + cls = kwargs.get("cls") + return cls(mock_pipeline_response, None, {"Content-Type": "application/xml"}) + + def deserializer_side_effect(target, _http_response): + nonlocal page_index + response = xml_pages[page_index] + page_index += 1 + return response + + with patch.object( + container_client._client.container, # pylint: disable=protected-access + "list_blob_hierarchy_segment_apache_arrow", + side_effect=fake_list_blob_hierarchy_segment_apache_arrow, + ): + blob_pages = container_client.walk_blobs(response_format="arrow", results_per_page=3).by_page() + mock_deserializer = MagicMock(side_effect=deserializer_side_effect) + blob_pages._deserializer = mock_deserializer + first_blobs_list = list(next(blob_pages)) + self.verify_blobs(first_blobs_list, blob_names[0:3]) + second_blobs_list = list(next(blob_pages)) + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = list(next(blob_pages)) + self.verify_blobs(third_blobs_list, blob_names[6:9]) + + # Confirm the XML deserializer was called once per page. + assert mock_deserializer.call_count == 3 + + def test_arrow_mock_walk_expected_response(self): + def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: + schema_meta = {b"NextMarker": (next_marker or "").encode()} if next_marker else {} + schema = pa.schema([pa.field("Name", pa.string())], metadata=schema_meta) + batch = pa.record_batch([pa.array(names, type=pa.string())], schema=schema) + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, schema) as writer: + writer.write_batch(batch) + return sink.getvalue().to_pybytes() + + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + pages = [ + _make_arrow_page(blob_names[0:3], next_marker="marker1"), + _make_arrow_page(blob_names[3:6], next_marker="marker2"), + _make_arrow_page(blob_names[6:9], next_marker=None), + ] + page_index = 0 + + container_client = ContainerClient( + account_url="https://account.blob.core.windows.net", + container_name="mycontainer", + credential=AzureNamedKeyCredential("account", "A" * 64), + ) + + mock_http_response = MagicMock() + mock_http_response.location_mode = None + mock_pipeline_response = MagicMock() + mock_pipeline_response.http_response = mock_http_response + + def fake_list_blob_hierarchy_segment_apache_arrow(**kwargs): + nonlocal page_index + cls = kwargs.get("cls") + raw = iter([pages[page_index]]) + page_index += 1 + return cls(mock_pipeline_response, raw, {"Content-Type": "application/vnd.apache.arrow.stream"}) + + with patch.object( + container_client._client.container, # pylint: disable=protected-access + "list_blob_hierarchy_segment_apache_arrow", + side_effect=fake_list_blob_hierarchy_segment_apache_arrow, + ): + blob_pages = container_client.walk_blobs(response_format="arrow", results_per_page=3).by_page() + first_blobs_list = list(next(blob_pages)) + self.verify_blobs(first_blobs_list, blob_names[0:3]) + second_blobs_list = list(next(blob_pages)) + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = list(next(blob_pages)) + self.verify_blobs(third_blobs_list, blob_names[6:9]) diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py new file mode 100644 index 000000000000..fd6d8740721c --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py @@ -0,0 +1,486 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pyarrow as pa +import pytest + +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +from azure.core.credentials import AzureNamedKeyCredential +from azure.core.exceptions import ResourceExistsError +from azure.storage.blob import BlobProperties +from azure.storage.blob._generated.models import ( + BlobFlatListSegment, + BlobItemInternal, + BlobName, + BlobPropertiesInternal, + ListBlobsFlatSegmentResponse, +) +from azure.storage.blob.aio import BlobServiceClient, ContainerClient + +from devtools_testutils.aio import recorded_by_proxy_async +from devtools_testutils.storage.aio import AsyncStorageRecordedTestCase +from settings.testcase import BlobPreparer + +# ------------------------------------------------------------------------------ +TEST_DATA = b"abc123" +# ------------------------------------------------------------------------------ + +class TestStorageApacheArrowAsync(AsyncStorageRecordedTestCase): + async def _setup(self, storage_account_name, storage_account_key): + self.bsc = BlobServiceClient( + self.account_url(storage_account_name, "blob"), + credential=storage_account_key.secret + ) + self.container_name = self.get_resource_name("utcontainerarrow") + if self.is_live: + try: + await self.bsc.create_container(self.container_name) + except ResourceExistsError: + pass + + async def create_blobs(self, blob_names: list[str]): + for blob_name in blob_names: + blob_client = self.bsc.get_blob_client(self.container_name, blob_name) + await blob_client.upload_blob(TEST_DATA, overwrite=True) + + def verify_blobs(self, blobs_list: list[BlobProperties], blob_names: list[str]): + assert len(blobs_list) == len(blob_names) + all_names = {blob.name for blob in blobs_list} + for blob_name in blob_names: + assert blob_name in all_names + + @BlobPreparer() + @recorded_by_proxy_async + async def test_arrow_list_no_blobs(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + container = self.bsc.get_container_client(self.container_name) + blobs_list = [blob async for blob in container.list_blobs(response_format="arrow")] + + self.verify_blobs(blobs_list, []) + + @BlobPreparer() + @recorded_by_proxy_async + async def test_arrow_list_multiple_blobs(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + blob_names = ["blob1", "blob2"] + await self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blobs_list = [blob async for blob in container.list_blobs(response_format="arrow")] + self.verify_blobs(blobs_list, blob_names) + + @BlobPreparer() + @recorded_by_proxy_async + async def test_arrow_list_blobs_paging(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + blob_names = ["blob1", "blob2", "blob3", "blob4"] + await self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blob_pages = container.list_blobs(response_format="arrow", results_per_page=2).by_page() + first_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(first_blobs_list, blob_names[:2]) + second_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(second_blobs_list, blob_names[2:]) + + @BlobPreparer() + @recorded_by_proxy_async + async def test_arrow_list_nested_blobs_paging(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + await self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blob_pages = container.list_blobs(response_format="arrow", results_per_page=3).by_page() + first_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(first_blobs_list, blob_names[:3]) + second_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(third_blobs_list, blob_names[6:]) + + @pytest.mark.asyncio + async def test_arrow_mock_xml_response(self): + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + account_url = "https://account.blob.core.windows.net/" + container_name = "mycontainer" + + def _make_xml_response(names: list[str], next_marker: str | None = None) -> ListBlobsFlatSegmentResponse: + items = [ + BlobItemInternal( + name=BlobName(content=name, encoded=False), + snapshot="", + properties=BlobPropertiesInternal( + etag="", + last_modified=datetime(2024, 1, 1, tzinfo=timezone.utc), + blob_type="BlockBlob", + ), + deleted=False, + ) + for name in names + ] + return ListBlobsFlatSegmentResponse( + service_endpoint=account_url, + container_name=container_name, + segment=BlobFlatListSegment(blob_items=items), + next_marker=next_marker, + ) + + # Three pages of 3 blobs each, chained via next_marker. + xml_pages = [ + _make_xml_response(blob_names[0:3], next_marker="marker1"), + _make_xml_response(blob_names[3:6], next_marker="marker2"), + _make_xml_response(blob_names[6:9], next_marker=None), + ] + page_index = 0 + + container_client = ContainerClient( + account_url=account_url, + container_name=container_name, + credential=AzureNamedKeyCredential("account", "A" * 64), + ) + + mock_http_response = MagicMock() + mock_http_response.location_mode = None + mock_pipeline_response = MagicMock() + mock_pipeline_response.http_response = mock_http_response + + async def fake_list_blob_flat_segment_apache_arrow(**kwargs): + cls = kwargs.get("cls") + return cls(mock_pipeline_response, None, {"Content-Type": "application/xml"}) + + def deserializer_side_effect(target, _http_response): + nonlocal page_index + response = xml_pages[page_index] + page_index += 1 + return response + + with patch.object( + container_client._client.container, # pylint: disable=protected-access + "list_blob_flat_segment_apache_arrow", + side_effect=fake_list_blob_flat_segment_apache_arrow, + ): + blob_pages = container_client.list_blobs(response_format="arrow", results_per_page=3).by_page() + mock_deserializer = MagicMock(side_effect=deserializer_side_effect) + blob_pages._deserializer = mock_deserializer + first_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(first_blobs_list, blob_names[0:3]) + second_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(third_blobs_list, blob_names[6:9]) + + # Confirm the XML deserializer was called once per page. + assert mock_deserializer.call_count == 3 + + @pytest.mark.asyncio + async def test_arrow_mock_expected_response(self): + def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: + schema_meta = {b"NextMarker": (next_marker or "").encode()} if next_marker else {} + schema = pa.schema([pa.field("Name", pa.string())], metadata=schema_meta) + batch = pa.record_batch([pa.array(names, type=pa.string())], schema=schema) + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, schema) as writer: + writer.write_batch(batch) + return sink.getvalue().to_pybytes() + + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + pages = [ + _make_arrow_page(blob_names[0:3], next_marker="marker1"), + _make_arrow_page(blob_names[3:6], next_marker="marker2"), + _make_arrow_page(blob_names[6:9], next_marker=None), + ] + page_index = 0 + + container_client = ContainerClient( + account_url="https://account.blob.core.windows.net", + container_name="mycontainer", + credential=AzureNamedKeyCredential("account", "A" * 64), + ) + + mock_http_response = MagicMock() + mock_http_response.location_mode = None + mock_pipeline_response = MagicMock() + mock_pipeline_response.http_response = mock_http_response + + async def fake_list_blob_flat_segment_apache_arrow(**kwargs): + nonlocal page_index + cls = kwargs.get("cls") + raw = iter([pages[page_index]]) + page_index += 1 + return cls(mock_pipeline_response, raw, {"Content-Type": "application/vnd.apache.arrow.stream"}) + + with patch.object( + container_client._client.container, # pylint: disable=protected-access + "list_blob_flat_segment_apache_arrow", + side_effect=fake_list_blob_flat_segment_apache_arrow, + ): + blob_pages = container_client.list_blobs(response_format="arrow", results_per_page=3).by_page() + first_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(first_blobs_list, blob_names[0:3]) + second_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(third_blobs_list, blob_names[6:9]) + + @BlobPreparer() + @recorded_by_proxy_async + async def test_arrow_walk_no_blobs(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + container = self.bsc.get_container_client(self.container_name) + blobs_list = [blob async for blob in container.walk_blobs(response_format="arrow")] + + self.verify_blobs(blobs_list, []) + + @BlobPreparer() + @recorded_by_proxy_async + async def test_arrow_walk_multiple_blobs(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + blob_names = ["blob1", "blob2"] + await self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blobs_list = [blob async for blob in container.walk_blobs(response_format="arrow")] + self.verify_blobs(blobs_list, blob_names) + + @BlobPreparer() + @recorded_by_proxy_async + async def test_arrow_walk_blobs_paging(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + blob_names = ["blob1", "blob2", "blob3", "blob4"] + await self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blob_pages = container.walk_blobs(response_format="arrow", results_per_page=2).by_page() + first_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(first_blobs_list, blob_names[:2]) + second_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(second_blobs_list, blob_names[2:]) + + @BlobPreparer() + @recorded_by_proxy_async + async def test_arrow_walk_nested_blobs_paging(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + await self.create_blobs(blob_names) + + container = self.bsc.get_container_client(self.container_name) + blob_pages = container.walk_blobs(response_format="arrow", results_per_page=3).by_page() + first_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(first_blobs_list, blob_names[:3]) + second_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(third_blobs_list, blob_names[6:]) + + @pytest.mark.asyncio + async def test_arrow_mock_walk_xml_response(self): + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + account_url = "https://account.blob.core.windows.net/" + container_name = "mycontainer" + + def _make_xml_response(names: list[str], next_marker: str | None = None) -> ListBlobsFlatSegmentResponse: + items = [ + BlobItemInternal( + name=BlobName(content=name, encoded=False), + snapshot="", + properties=BlobPropertiesInternal( + etag="", + last_modified=datetime(2024, 1, 1, tzinfo=timezone.utc), + blob_type="BlockBlob", + ), + deleted=False, + ) + for name in names + ] + return ListBlobsFlatSegmentResponse( + service_endpoint=account_url, + container_name=container_name, + segment=BlobFlatListSegment(blob_items=items), + next_marker=next_marker, + ) + + # Three pages of 3 blobs each, chained via next_marker. + xml_pages = [ + _make_xml_response(blob_names[0:3], next_marker="marker1"), + _make_xml_response(blob_names[3:6], next_marker="marker2"), + _make_xml_response(blob_names[6:9], next_marker=None), + ] + page_index = 0 + + container_client = ContainerClient( + account_url=account_url, + container_name=container_name, + credential=AzureNamedKeyCredential("account", "A" * 64), + ) + + mock_http_response = MagicMock() + mock_http_response.location_mode = None + mock_pipeline_response = MagicMock() + mock_pipeline_response.http_response = mock_http_response + + async def fake_list_blob_hierarchy_segment_apache_arrow(**kwargs): + cls = kwargs.get("cls") + return cls(mock_pipeline_response, None, {"Content-Type": "application/xml"}) + + def deserializer_side_effect(target, _http_response): + nonlocal page_index + response = xml_pages[page_index] + page_index += 1 + return response + + with patch.object( + container_client._client.container, # pylint: disable=protected-access + "list_blob_hierarchy_segment_apache_arrow", + side_effect=fake_list_blob_hierarchy_segment_apache_arrow, + ): + blob_pages = container_client.walk_blobs(response_format="arrow", results_per_page=3).by_page() + mock_deserializer = MagicMock(side_effect=deserializer_side_effect) + blob_pages._deserializer = mock_deserializer + first_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(first_blobs_list, blob_names[0:3]) + second_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(third_blobs_list, blob_names[6:9]) + + # Confirm the XML deserializer was called once per page. + assert mock_deserializer.call_count == 3 + + @pytest.mark.asyncio + async def test_arrow_mock_walk_expected_response(self): + def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: + schema_meta = {b"NextMarker": (next_marker or "").encode()} if next_marker else {} + schema = pa.schema([pa.field("Name", pa.string())], metadata=schema_meta) + batch = pa.record_batch([pa.array(names, type=pa.string())], schema=schema) + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, schema) as writer: + writer.write_batch(batch) + return sink.getvalue().to_pybytes() + + blob_names = [ + "a/b/blob1", + "a/b/blob2", + "a/c/blob3", + "d/blob4", + "d/e/f/blob5", + "flat_blob1", + "flat_blob2", + "flat_blob3", + "flat_blob4", + ] + pages = [ + _make_arrow_page(blob_names[0:3], next_marker="marker1"), + _make_arrow_page(blob_names[3:6], next_marker="marker2"), + _make_arrow_page(blob_names[6:9], next_marker=None), + ] + page_index = 0 + + container_client = ContainerClient( + account_url="https://account.blob.core.windows.net", + container_name="mycontainer", + credential=AzureNamedKeyCredential("account", "A" * 64), + ) + + mock_http_response = MagicMock() + mock_http_response.location_mode = None + mock_pipeline_response = MagicMock() + mock_pipeline_response.http_response = mock_http_response + + async def fake_list_blob_hierarchy_segment_apache_arrow(**kwargs): + nonlocal page_index + cls = kwargs.get("cls") + raw = iter([pages[page_index]]) + page_index += 1 + return cls(mock_pipeline_response, raw, {"Content-Type": "application/vnd.apache.arrow.stream"}) + + with patch.object( + container_client._client.container, # pylint: disable=protected-access + "list_blob_hierarchy_segment_apache_arrow", + side_effect=fake_list_blob_hierarchy_segment_apache_arrow, + ): + blob_pages = container_client.walk_blobs(response_format="arrow", results_per_page=3).by_page() + first_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(first_blobs_list, blob_names[0:3]) + second_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(second_blobs_list, blob_names[3:6]) + third_blobs_list = [blob async for blob in await blob_pages.__anext__()] + self.verify_blobs(third_blobs_list, blob_names[6:9]) From 4193a2e2054d758ce65b835672326ae1a418e354 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 1 Jul 2026 13:22:36 -0400 Subject: [PATCH 03/20] We don't need pyarrow --- sdk/storage/azure-storage-blob/dev_requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/dev_requirements.txt b/sdk/storage/azure-storage-blob/dev_requirements.txt index 80ae73f833ae..a5edfde94aa1 100644 --- a/sdk/storage/azure-storage-blob/dev_requirements.txt +++ b/sdk/storage/azure-storage-blob/dev_requirements.txt @@ -5,4 +5,3 @@ azure-mgmt-storage==20.1.0 aiohttp>=3.13.5 nanoarrow>=0.8.0 -pyarrow>=23.0.1 From b86894e78cebc1664f8b7aee6d3cdd43c8e7f03c Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 1 Jul 2026 13:24:36 -0400 Subject: [PATCH 04/20] updateed changelogs --- sdk/storage/azure-storage-blob/CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sdk/storage/azure-storage-blob/CHANGELOG.md b/sdk/storage/azure-storage-blob/CHANGELOG.md index c85315d6476f..3d0278a04d5c 100644 --- a/sdk/storage/azure-storage-blob/CHANGELOG.md +++ b/sdk/storage/azure-storage-blob/CHANGELOG.md @@ -5,9 +5,10 @@ ### Features Added - Added support for service version 2026-10-06. - Added support for parsing List Blobs responses in Apache Arrow format. Pass the -`use_arrow=True` keyword to `ContainerClient`'s `list_blobs` and `walk_blobs` APIs -to opt in. This requires the `nanoarrow` package and also enables the new -`start_from` and `end_before` keywords for range-based listing. +`response_format="arrow"` keyword to `ContainerClient`'s `list_blobs` and `walk_blobs` APIs +to opt in (`response_format` defaults to `"xml"`, and `"auto"` currently also means `"xml"`). +This requires the `nanoarrow` package and also enables the new `start_from` and `end_before` +keywords for range-based listing. - Added access tier information to the response of `BlobClient`'s `download_blob` API. The `blob_tier`, `blob_tier_inferred`, `blob_tier_change_time`, and `smart_access_tier` properties are now populated on the downloaded blob's `properties`. From b6851bd36def7a0bd3bb223b3b2c6798f5cff912 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 2 Jul 2026 09:43:11 -0400 Subject: [PATCH 05/20] nanoarrow tests --- .../azure-storage-blob/tests/test_arrow.py | 41 +++++++++++-------- .../tests/test_arrow_async.py | 41 +++++++++++-------- 2 files changed, 48 insertions(+), 34 deletions(-) diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow.py b/sdk/storage/azure-storage-blob/tests/test_arrow.py index 8ec188639043..537a36278a59 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow.py @@ -3,7 +3,10 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -import pyarrow as pa +import io + +import nanoarrow as na +from nanoarrow import ipc from datetime import datetime, timezone from unittest.mock import MagicMock, patch @@ -27,11 +30,11 @@ TEST_DATA = b"abc123" # ------------------------------------------------------------------------------ + class TestStorageApacheArrow(StorageRecordedTestCase): def _setup(self, storage_account_name, storage_account_key): self.bsc = BlobServiceClient( - self.account_url(storage_account_name, "blob"), - credential=storage_account_key.secret + self.account_url(storage_account_name, "blob"), credential=storage_account_key.secret ) self.container_name = self.get_resource_name("utcontainerarrow") if self.is_live: @@ -208,13 +211,15 @@ def deserializer_side_effect(target, _http_response): def test_arrow_mock_expected_response(self): def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: - schema_meta = {b"NextMarker": (next_marker or "").encode()} if next_marker else {} - schema = pa.schema([pa.field("Name", pa.string())], metadata=schema_meta) - batch = pa.record_batch([pa.array(names, type=pa.string())], schema=schema) - sink = pa.BufferOutputStream() - with pa.ipc.new_stream(sink, schema) as writer: - writer.write_batch(batch) - return sink.getvalue().to_pybytes() + metadata = {"NextMarker": next_marker} if next_marker else None + schema = na.Schema(na.struct({"Name": na.string()}), metadata=metadata) + name_array = na.c_array(names, na.string()) + batch = na.c_array_from_buffers(schema, length=len(names), buffers=[None], children=[name_array]) + out = io.BytesIO() + writer = ipc.StreamWriter.from_writable(out) + writer.write_stream(batch) + writer.close() + return out.getvalue() blob_names = [ "a/b/blob1", @@ -423,13 +428,15 @@ def deserializer_side_effect(target, _http_response): def test_arrow_mock_walk_expected_response(self): def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: - schema_meta = {b"NextMarker": (next_marker or "").encode()} if next_marker else {} - schema = pa.schema([pa.field("Name", pa.string())], metadata=schema_meta) - batch = pa.record_batch([pa.array(names, type=pa.string())], schema=schema) - sink = pa.BufferOutputStream() - with pa.ipc.new_stream(sink, schema) as writer: - writer.write_batch(batch) - return sink.getvalue().to_pybytes() + metadata = {"NextMarker": next_marker} if next_marker else None + schema = na.Schema(na.struct({"Name": na.string()}), metadata=metadata) + name_array = na.c_array(names, na.string()) + batch = na.c_array_from_buffers(schema, length=len(names), buffers=[None], children=[name_array]) + out = io.BytesIO() + writer = ipc.StreamWriter.from_writable(out) + writer.write_stream(batch) + writer.close() + return out.getvalue() blob_names = [ "a/b/blob1", diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py index fd6d8740721c..e3a9fd9e312d 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py @@ -3,8 +3,11 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -import pyarrow as pa +import io + +import nanoarrow as na import pytest +from nanoarrow import ipc from datetime import datetime, timezone from unittest.mock import MagicMock, patch @@ -29,11 +32,11 @@ TEST_DATA = b"abc123" # ------------------------------------------------------------------------------ + class TestStorageApacheArrowAsync(AsyncStorageRecordedTestCase): async def _setup(self, storage_account_name, storage_account_key): self.bsc = BlobServiceClient( - self.account_url(storage_account_name, "blob"), - credential=storage_account_key.secret + self.account_url(storage_account_name, "blob"), credential=storage_account_key.secret ) self.container_name = self.get_resource_name("utcontainerarrow") if self.is_live: @@ -212,13 +215,15 @@ def deserializer_side_effect(target, _http_response): @pytest.mark.asyncio async def test_arrow_mock_expected_response(self): def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: - schema_meta = {b"NextMarker": (next_marker or "").encode()} if next_marker else {} - schema = pa.schema([pa.field("Name", pa.string())], metadata=schema_meta) - batch = pa.record_batch([pa.array(names, type=pa.string())], schema=schema) - sink = pa.BufferOutputStream() - with pa.ipc.new_stream(sink, schema) as writer: - writer.write_batch(batch) - return sink.getvalue().to_pybytes() + metadata = {"NextMarker": next_marker} if next_marker else None + schema = na.Schema(na.struct({"Name": na.string()}), metadata=metadata) + name_array = na.c_array(names, na.string()) + batch = na.c_array_from_buffers(schema, length=len(names), buffers=[None], children=[name_array]) + out = io.BytesIO() + writer = ipc.StreamWriter.from_writable(out) + writer.write_stream(batch) + writer.close() + return out.getvalue() blob_names = [ "a/b/blob1", @@ -428,13 +433,15 @@ def deserializer_side_effect(target, _http_response): @pytest.mark.asyncio async def test_arrow_mock_walk_expected_response(self): def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: - schema_meta = {b"NextMarker": (next_marker or "").encode()} if next_marker else {} - schema = pa.schema([pa.field("Name", pa.string())], metadata=schema_meta) - batch = pa.record_batch([pa.array(names, type=pa.string())], schema=schema) - sink = pa.BufferOutputStream() - with pa.ipc.new_stream(sink, schema) as writer: - writer.write_batch(batch) - return sink.getvalue().to_pybytes() + metadata = {"NextMarker": next_marker} if next_marker else None + schema = na.Schema(na.struct({"Name": na.string()}), metadata=metadata) + name_array = na.c_array(names, na.string()) + batch = na.c_array_from_buffers(schema, length=len(names), buffers=[None], children=[name_array]) + out = io.BytesIO() + writer = ipc.StreamWriter.from_writable(out) + writer.write_stream(batch) + writer.close() + return out.getvalue() blob_names = [ "a/b/blob1", From 328720d5b60830c0f4327b313fa8569a25007698 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 2 Jul 2026 09:46:19 -0400 Subject: [PATCH 06/20] api.md --- sdk/storage/azure-storage-file-share/api.md | 108 ++++++++++++++++++ .../azure-storage-file-share/api.metadata.yml | 2 +- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-file-share/api.md b/sdk/storage/azure-storage-file-share/api.md index dec031a05dee..960008b75c15 100644 --- a/sdk/storage/azure-storage-file-share/api.md +++ b/sdk/storage/azure-storage-file-share/api.md @@ -455,6 +455,62 @@ namespace azure.storage.fileshare def values(self): ... + class azure.storage.fileshare.FileRange(DictMixin): + cleared: bool + end: int + start: int + + def __contains__(self, key): ... + + def __delitem__(self, key): ... + + def __eq__(self, other): ... + + def __getitem__(self, key): ... + + def __init__( + self, + start: int, + end: int, + *, + cleared: bool = False + ) -> None: ... + + def __len__(self): ... + + def __ne__(self, other): ... + + def __repr__(self): ... + + def __setitem__( + self, + key, + item + ): ... + + def __str__(self): ... + + def get( + self, + key, + default = None + ): ... + + def has_key(self, k): ... + + def items(self): ... + + def keys(self): ... + + def update( + self, + *args, + **kwargs + ): ... + + def values(self): ... + + class azure.storage.fileshare.FileSasPermissions: create: bool = False delete: bool = False @@ -1510,6 +1566,32 @@ namespace azure.storage.fileshare **kwargs: Any ) -> ItemPaged[Handle]: ... + @distributed_trace + def list_ranges( + self, + *, + lease: Union[ShareLeaseClient, str] = ..., + length: Optional[int] = ..., + offset: Optional[int] = ..., + results_per_page: Optional[int] = ..., + timeout: Optional[int] = ..., + **kwargs: Any + ) -> ItemPaged[FileRange]: ... + + @distributed_trace + def list_ranges_diff( + self, + previous_sharesnapshot: Union[str, Dict[str, Any]], + *, + include_renames: Optional[bool] = ..., + lease: Union[ShareLeaseClient, str] = ..., + length: Optional[int] = ..., + offset: Optional[int] = ..., + results_per_page: Optional[int] = ..., + timeout: Optional[int] = ..., + **kwargs: Any + ) -> ItemPaged[FileRange]: ... + @distributed_trace def rename_file( self, @@ -3067,6 +3149,32 @@ namespace azure.storage.fileshare.aio **kwargs: Any ) -> AsyncItemPaged[Handle]: ... + @distributed_trace + def list_ranges( + self, + *, + lease: Union[ShareLeaseClient, str] = ..., + length: Optional[int] = ..., + offset: Optional[int] = ..., + results_per_page: Optional[int] = ..., + timeout: Optional[int] = ..., + **kwargs: Any + ) -> AsyncItemPaged[FileRange]: ... + + @distributed_trace + def list_ranges_diff( + self, + previous_sharesnapshot: Union[str, Dict[str, Any]], + *, + include_renames: Optional[bool] = ..., + lease: Union[ShareLeaseClient, str] = ..., + length: Optional[int] = ..., + offset: Optional[int] = ..., + results_per_page: Optional[int] = ..., + timeout: Optional[int] = ..., + **kwargs: Any + ) -> AsyncItemPaged[FileRange]: ... + @distributed_trace_async async def rename_file( self, diff --git a/sdk/storage/azure-storage-file-share/api.metadata.yml b/sdk/storage/azure-storage-file-share/api.metadata.yml index 971c843d375e..b33bc80ad668 100644 --- a/sdk/storage/azure-storage-file-share/api.metadata.yml +++ b/sdk/storage/azure-storage-file-share/api.metadata.yml @@ -1,3 +1,3 @@ -apiMdSha256: 3c2f71c139d0f1e3430e1f6802e9b27f9940c011b1da95748721350c53a0c6f8 +apiMdSha256: bff208fab40e4aaf5c2b4056336d622ebc1861f60d37ba1fe16969b86e8b7c46 parserVersion: 0.3.28 pythonVersion: 3.13.14 From 147ef4a601651709bb1ff91d2c20ba7c750153c3 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 2 Jul 2026 09:47:14 -0400 Subject: [PATCH 07/20] Revert "api.md" This reverts commit 328720d5b60830c0f4327b313fa8569a25007698. --- sdk/storage/azure-storage-file-share/api.md | 108 ------------------ .../azure-storage-file-share/api.metadata.yml | 2 +- 2 files changed, 1 insertion(+), 109 deletions(-) diff --git a/sdk/storage/azure-storage-file-share/api.md b/sdk/storage/azure-storage-file-share/api.md index 960008b75c15..dec031a05dee 100644 --- a/sdk/storage/azure-storage-file-share/api.md +++ b/sdk/storage/azure-storage-file-share/api.md @@ -455,62 +455,6 @@ namespace azure.storage.fileshare def values(self): ... - class azure.storage.fileshare.FileRange(DictMixin): - cleared: bool - end: int - start: int - - def __contains__(self, key): ... - - def __delitem__(self, key): ... - - def __eq__(self, other): ... - - def __getitem__(self, key): ... - - def __init__( - self, - start: int, - end: int, - *, - cleared: bool = False - ) -> None: ... - - def __len__(self): ... - - def __ne__(self, other): ... - - def __repr__(self): ... - - def __setitem__( - self, - key, - item - ): ... - - def __str__(self): ... - - def get( - self, - key, - default = None - ): ... - - def has_key(self, k): ... - - def items(self): ... - - def keys(self): ... - - def update( - self, - *args, - **kwargs - ): ... - - def values(self): ... - - class azure.storage.fileshare.FileSasPermissions: create: bool = False delete: bool = False @@ -1566,32 +1510,6 @@ namespace azure.storage.fileshare **kwargs: Any ) -> ItemPaged[Handle]: ... - @distributed_trace - def list_ranges( - self, - *, - lease: Union[ShareLeaseClient, str] = ..., - length: Optional[int] = ..., - offset: Optional[int] = ..., - results_per_page: Optional[int] = ..., - timeout: Optional[int] = ..., - **kwargs: Any - ) -> ItemPaged[FileRange]: ... - - @distributed_trace - def list_ranges_diff( - self, - previous_sharesnapshot: Union[str, Dict[str, Any]], - *, - include_renames: Optional[bool] = ..., - lease: Union[ShareLeaseClient, str] = ..., - length: Optional[int] = ..., - offset: Optional[int] = ..., - results_per_page: Optional[int] = ..., - timeout: Optional[int] = ..., - **kwargs: Any - ) -> ItemPaged[FileRange]: ... - @distributed_trace def rename_file( self, @@ -3149,32 +3067,6 @@ namespace azure.storage.fileshare.aio **kwargs: Any ) -> AsyncItemPaged[Handle]: ... - @distributed_trace - def list_ranges( - self, - *, - lease: Union[ShareLeaseClient, str] = ..., - length: Optional[int] = ..., - offset: Optional[int] = ..., - results_per_page: Optional[int] = ..., - timeout: Optional[int] = ..., - **kwargs: Any - ) -> AsyncItemPaged[FileRange]: ... - - @distributed_trace - def list_ranges_diff( - self, - previous_sharesnapshot: Union[str, Dict[str, Any]], - *, - include_renames: Optional[bool] = ..., - lease: Union[ShareLeaseClient, str] = ..., - length: Optional[int] = ..., - offset: Optional[int] = ..., - results_per_page: Optional[int] = ..., - timeout: Optional[int] = ..., - **kwargs: Any - ) -> AsyncItemPaged[FileRange]: ... - @distributed_trace_async async def rename_file( self, diff --git a/sdk/storage/azure-storage-file-share/api.metadata.yml b/sdk/storage/azure-storage-file-share/api.metadata.yml index b33bc80ad668..971c843d375e 100644 --- a/sdk/storage/azure-storage-file-share/api.metadata.yml +++ b/sdk/storage/azure-storage-file-share/api.metadata.yml @@ -1,3 +1,3 @@ -apiMdSha256: bff208fab40e4aaf5c2b4056336d622ebc1861f60d37ba1fe16969b86e8b7c46 +apiMdSha256: 3c2f71c139d0f1e3430e1f6802e9b27f9940c011b1da95748721350c53a0c6f8 parserVersion: 0.3.28 pythonVersion: 3.13.14 From 3795f2be647811f2d431489f17fad515e5c925f1 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 2 Jul 2026 09:48:27 -0400 Subject: [PATCH 08/20] api.md --- sdk/storage/azure-storage-blob/api.md | 8 ++++++++ sdk/storage/azure-storage-blob/api.metadata.yml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/api.md b/sdk/storage/azure-storage-blob/api.md index 02fa1476ca01..4f7ebb7bca9d 100644 --- a/sdk/storage/azure-storage-blob/api.md +++ b/sdk/storage/azure-storage-blob/api.md @@ -1792,6 +1792,8 @@ namespace azure.storage.blob name_starts_with: Optional[str] = None, include: Optional[Union[str, List[str]]] = None, *, + end_before: Optional[str] = ..., + response_format: Literal[auto, xml, arrow] = ..., results_per_page: Optional[int] = ..., start_from: Optional[str] = ..., timeout: Optional[int] = ..., @@ -1883,6 +1885,8 @@ namespace azure.storage.blob include: Optional[Union[List[str], str]] = None, delimiter: str = "/", *, + end_before: Optional[str] = ..., + response_format: Literal[auto, xml, arrow] = ..., start_from: Optional[str] = ..., timeout: Optional[int] = ..., **kwargs: Any @@ -4438,6 +4442,8 @@ namespace azure.storage.blob.aio name_starts_with: Optional[str] = None, include: Optional[Union[str, List[str]]] = None, *, + end_before: Optional[str] = ..., + response_format: Literal[auto, xml, arrow] = ..., results_per_page: Optional[int] = ..., start_from: Optional[str] = ..., timeout: Optional[int] = ..., @@ -4529,6 +4535,8 @@ namespace azure.storage.blob.aio include: Optional[Union[List[str], str]] = None, delimiter: str = "/", *, + end_before: Optional[str] = ..., + response_format: Literal[auto, xml, arrow] = ..., start_from: Optional[str] = ..., timeout: Optional[int] = ..., **kwargs: Any diff --git a/sdk/storage/azure-storage-blob/api.metadata.yml b/sdk/storage/azure-storage-blob/api.metadata.yml index 17ac8c442fcd..8b9785334df4 100644 --- a/sdk/storage/azure-storage-blob/api.metadata.yml +++ b/sdk/storage/azure-storage-blob/api.metadata.yml @@ -1,3 +1,3 @@ -apiMdSha256: 38a660f5d55527fe389aaf9017736875f51f1f564acd4ea8aaaa547552b820b2 +apiMdSha256: f8f6db831d73daab6cbc18c90924b1c164055d92775d0accc82a1e3ae0a364ff parserVersion: 0.3.28 pythonVersion: 3.13.14 From b293df34d4204b0dc3051fa834e8444d8d0c3ac0 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 2 Jul 2026 13:01:18 -0400 Subject: [PATCH 09/20] Dependency comments --- sdk/storage/azure-storage-blob/CHANGELOG.md | 4 ++-- .../azure/storage/blob/_container_client.py | 12 ++++++------ .../storage/blob/aio/_container_client_async.py | 12 ++++++------ sdk/storage/azure-storage-blob/setup.py | 4 +++- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/sdk/storage/azure-storage-blob/CHANGELOG.md b/sdk/storage/azure-storage-blob/CHANGELOG.md index 3d0278a04d5c..96a200dddf93 100644 --- a/sdk/storage/azure-storage-blob/CHANGELOG.md +++ b/sdk/storage/azure-storage-blob/CHANGELOG.md @@ -7,8 +7,8 @@ - Added support for parsing List Blobs responses in Apache Arrow format. Pass the `response_format="arrow"` keyword to `ContainerClient`'s `list_blobs` and `walk_blobs` APIs to opt in (`response_format` defaults to `"xml"`, and `"auto"` currently also means `"xml"`). -This requires the `nanoarrow` package and also enables the new `start_from` and `end_before` -keywords for range-based listing. +This requires the optional `[arrow]` extra to be installed, and also enables the new +`start_from` and `end_before` keywords for range-based listing. - Added access tier information to the response of `BlobClient`'s `download_blob` API. The `blob_tier`, `blob_tier_inferred`, `blob_tier_change_time`, and `smart_access_tier` properties are now populated on the downloaded blob's `properties`. diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py index 80d7f42b22ca..c62a42826d6d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py @@ -852,9 +852,9 @@ def list_blobs( try: import nanoarrow # pylint: disable=import-outside-toplevel,unused-import except ImportError as e: - raise ImportError( - "The 'nanoarrow' package is required to use Apache Arrow deserialization. " - "Install it with: pip install nanoarrow" + raise ValueError( + "The use of Apache Arrow deserialization requires the extra [arrow] to be installed. " + "Please install this extra and try again." ) from e command = functools.partial( @@ -991,9 +991,9 @@ def walk_blobs( try: import nanoarrow # pylint: disable=import-outside-toplevel,unused-import except ImportError as e: - raise ImportError( - "The 'nanoarrow' package is required to use Apache Arrow deserialization. " - "Install it with: pip install nanoarrow" + raise ValueError( + "The use of Apache Arrow deserialization requires the extra [arrow] to be installed. " + "Please install this extra and try again." ) from e command = functools.partial( self._client.container.list_blob_hierarchy_segment_apache_arrow, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py index fc4a0e079807..f8767069bbf7 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py @@ -873,9 +873,9 @@ def list_blobs( try: import nanoarrow # pylint: disable=import-outside-toplevel,unused-import except ImportError as e: - raise ImportError( - "The 'nanoarrow' package is required to use Apache Arrow deserialization. " - "Install it with: pip install nanoarrow" + raise ValueError( + "The use of Apache Arrow deserialization requires the extra [arrow] to be installed. " + "Please install this extra and try again." ) from e command = functools.partial( ( @@ -1012,9 +1012,9 @@ def walk_blobs( try: import nanoarrow # pylint: disable=import-outside-toplevel,unused-import except ImportError as e: - raise ImportError( - "The 'nanoarrow' package is required to use Apache Arrow deserialization. " - "Install it with: pip install nanoarrow" + raise ValueError( + "The use of Apache Arrow deserialization requires the extra [arrow] to be installed. " + "Please install this extra and try again." ) from e command = functools.partial( self._client.container.list_blob_hierarchy_segment_apache_arrow, diff --git a/sdk/storage/azure-storage-blob/setup.py b/sdk/storage/azure-storage-blob/setup.py index a05a2cc4a955..c2bf4197858f 100644 --- a/sdk/storage/azure-storage-blob/setup.py +++ b/sdk/storage/azure-storage-blob/setup.py @@ -83,12 +83,14 @@ "cryptography>=2.1.4", "typing-extensions>=4.6.0", "isodate>=0.6.1", - "nanoarrow>=0.8.0", ], extras_require={ "aio": [ "azure-core[aio]>=1.37.0", ], + "arrow": [ + "nanoarrow>=0.8.0", + ], "ext-checksums": [ "azure-storage-extensions>=0.1.0,<1.0.0", ], From 61ee4c8ae09b36c61261fe72a3811c2995f40651 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 2 Jul 2026 13:29:02 -0400 Subject: [PATCH 10/20] test recordings --- sdk/storage/azure-storage-blob/assets.json | 2 +- sdk/storage/azure-storage-blob/tests/test_arrow.py | 6 ++---- sdk/storage/azure-storage-blob/tests/test_arrow_async.py | 6 ++---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/sdk/storage/azure-storage-blob/assets.json b/sdk/storage/azure-storage-blob/assets.json index c7253bb07847..add35578f3d2 100644 --- a/sdk/storage/azure-storage-blob/assets.json +++ b/sdk/storage/azure-storage-blob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/storage/azure-storage-blob", - "Tag": "python/storage/azure-storage-blob_cd3cfd448d" + "Tag": "python/storage/azure-storage-blob_8fceda8e9d" } diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow.py b/sdk/storage/azure-storage-blob/tests/test_arrow.py index 537a36278a59..efb3bd020d0a 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow.py @@ -337,11 +337,9 @@ def test_arrow_walk_nested_blobs_paging(self, **kwargs): container = self.bsc.get_container_client(self.container_name) blob_pages = container.walk_blobs(response_format="arrow", results_per_page=3).by_page() first_blobs_list = list(next(blob_pages)) - self.verify_blobs(first_blobs_list, blob_names[:3]) + self.verify_blobs(first_blobs_list, ["a/", "d/", "flat_blob1"]) second_blobs_list = list(next(blob_pages)) - self.verify_blobs(second_blobs_list, blob_names[3:6]) - third_blobs_list = list(next(blob_pages)) - self.verify_blobs(third_blobs_list, blob_names[6:]) + self.verify_blobs(second_blobs_list, ["flat_blob2", "flat_blob3", "flat_blob4"]) def test_arrow_mock_walk_xml_response(self): blob_names = [ diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py index e3a9fd9e312d..fbc64ca080ea 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py @@ -340,11 +340,9 @@ async def test_arrow_walk_nested_blobs_paging(self, **kwargs): container = self.bsc.get_container_client(self.container_name) blob_pages = container.walk_blobs(response_format="arrow", results_per_page=3).by_page() first_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(first_blobs_list, blob_names[:3]) + self.verify_blobs(first_blobs_list, ["a/", "d/", "flat_blob1"]) second_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(second_blobs_list, blob_names[3:6]) - third_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(third_blobs_list, blob_names[6:]) + self.verify_blobs(second_blobs_list, ["flat_blob2", "flat_blob3", "flat_blob4"]) @pytest.mark.asyncio async def test_arrow_mock_walk_xml_response(self): From f9183730cbd461d560a1397c88aec586451c38e6 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 8 Jul 2026 14:29:08 -0400 Subject: [PATCH 11/20] Feedback part 1 --- .../azure/storage/blob/_container_client.py | 30 ++++++++++--------- .../azure/storage/blob/_container_client.pyi | 4 ++- .../azure/storage/blob/_list_blobs_helper.py | 12 ++++---- .../blob/aio/_container_client_async.py | 30 ++++++++++--------- .../blob/aio/_container_client_async.pyi | 4 +++ .../storage/blob/aio/_list_blobs_helper.py | 2 +- sdk/storage/azure-storage-blob/setup.py | 3 -- 7 files changed, 46 insertions(+), 39 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py index c62a42826d6d..9d594c048d8c 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py @@ -806,11 +806,12 @@ def list_blobs( Controls the maximum number of Blobs that will be included in each page of results if using `ItemPaged.by_page()`. :keyword response_format: - The format used to return and parse the List Blobs response. Possible values are - "auto", "xml", and "arrow". When set to "arrow", the response is returned and parsed in - Apache Arrow format (this requires the `nanoarrow` package). When set to "xml", the - standard XML response is parsed. "auto" currently behaves the same as "xml". This value - defaults to "xml". + The format used to return and parse the List Blobs response. + Possible values are "auto", "xml", and "arrow". + Choose "auto" to let the SDK choose the best algorithm, currently "xml". The default value is "auto". + + .. note:: + The use of "arrow" requires `nanoarrow` to be installed. :paramtype response_format: Literal["auto", "xml", "arrow"] :keyword str start_from: Specifies the full path (inclusive) to list paths from. @@ -853,8 +854,8 @@ def list_blobs( import nanoarrow # pylint: disable=import-outside-toplevel,unused-import except ImportError as e: raise ValueError( - "The use of Apache Arrow deserialization requires the extra [arrow] to be installed. " - "Please install this extra and try again." + "The use of Apache Arrow deserialization requires nanoarrow to be installed. " + "Please install nanoarrow and try again." ) from e command = functools.partial( @@ -955,11 +956,12 @@ def walk_blobs( names begin with the same substring up to the appearance of the delimiter character. The delimiter may be a single character or a string. :keyword response_format: - The format used to return and parse the List Blobs response. Possible values are - "auto", "xml", and "arrow". When set to "arrow", the response is returned and parsed in - Apache Arrow format (this requires the `nanoarrow` package). When set to "xml", the - standard XML response is parsed. "auto" currently behaves the same as "xml". This value - defaults to "xml". + The format used to return and parse the List Blobs response. + Possible values are "auto", "xml", and "arrow". + Choose "auto" to let the SDK choose the best algorithm, currently "xml". The default value is "auto". + + .. note:: + The use of "arrow" requires `nanoarrow` to be installed. :paramtype response_format: Literal["auto", "xml", "arrow"] :keyword str start_from: Specifies the full path (inclusive) to list paths from. @@ -992,8 +994,8 @@ def walk_blobs( import nanoarrow # pylint: disable=import-outside-toplevel,unused-import except ImportError as e: raise ValueError( - "The use of Apache Arrow deserialization requires the extra [arrow] to be installed. " - "Please install this extra and try again." + "The use of Apache Arrow deserialization requires nanoarrow to be installed. " + "Please install nanoarrow and try again." ) from e command = functools.partial( self._client.container.list_blob_hierarchy_segment_apache_arrow, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.pyi b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.pyi index 081c783c226c..b8c190c876f8 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.pyi +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.pyi @@ -202,7 +202,7 @@ class ContainerClient(StorageAccountHostsMixin, StorageEncryptionMixin): include: Optional[Union[str, List[str]]] = None, *, results_per_page: Optional[int] = None, - response_format: Literal["auto", "xml", "arrow"] = "xml", + response_format: Literal["auto", "xml", "arrow"] = "auto", start_from: Optional[str] = None, end_before: Optional[str] = None, timeout: Optional[int] = None, @@ -225,7 +225,9 @@ class ContainerClient(StorageAccountHostsMixin, StorageEncryptionMixin): include: Optional[Union[List[str], str]] = None, delimiter: str = "/", *, + response_format: Literal["auto", "xml", "arrow"] = "auto", start_from: Optional[str] = None, + end_before: Optional[str] = None, timeout: Optional[int] = None, **kwargs: Any, ) -> ItemPaged[Union[BlobProperties, BlobPrefix]]: ... diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py index 9b274a96d097..0d89e15c081d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py @@ -4,6 +4,7 @@ # license information. # -------------------------------------------------------------------------- +from io import BytesIO from typing import Any, Callable, cast, List, Optional, Tuple, Union from urllib.parse import unquote @@ -31,7 +32,7 @@ from ._shared.models import DictMixin from ._shared.response_handlers import process_storage_error, return_context_and_deserialized, return_raw_deserialized -_ARROW_CONTENT_TYPE = "application/vnd.apache.arrow.stream,application/xml" +_ARROW_CONTENT_TYPE = "application/vnd.apache.arrow.stream" def _parse_arrow_response( # pylint: disable=too-many-locals @@ -45,8 +46,7 @@ def _parse_arrow_response( # pylint: disable=too-many-locals :returns: A tuple of next marker and a list of BlobProperties. :rtype: Tuple[Optional[str], List[~azure.storage.blob.BlobProperties]] """ - import io # pylint: disable=import-outside-toplevel - import nanoarrow # pylint: disable=import-outside-toplevel + from nanoarrow import ArrayStream # pylint: disable=import-outside-toplevel from nanoarrow.ipc import InputStream # pylint: disable=import-outside-toplevel # Declarative mapping: Arrow column name -> (BlobProperties attr, default value). @@ -109,8 +109,8 @@ def _parse_arrow_response( # pylint: disable=too-many-locals next_marker: Optional[str] = None blob_items: List[BlobProperties] = [] - with InputStream.from_readable(io.BytesIO(raw_bytes)) as stream: - reader = nanoarrow.ArrayStream(stream) + with InputStream.from_readable(BytesIO(raw_bytes)) as stream: + reader = ArrayStream(stream) schema = reader.schema # The continuation token is embedded in the Arrow schema metadata. @@ -494,7 +494,7 @@ def _arrow_cls(self, pipeline_response: Any, deserialized: Any, response_headers content_type = response_headers.get("Content-Type", "") location_mode = getattr(pipeline_response.http_response, "location_mode", None) # The response is Arrow only when the service returns the Arrow stream media type. - if _ARROW_CONTENT_TYPE.split(",", 1)[0] in content_type: + if _ARROW_CONTENT_TYPE in content_type: raw_bytes = b"".join(deserialized) next_marker, blob_items = _parse_arrow_response(raw_bytes, self.container) self._arrow_response = (next_marker, blob_items) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py index f8767069bbf7..d0dab61d8daf 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py @@ -827,11 +827,12 @@ def list_blobs( Controls the maximum number of Blobs that will be included in each page of results if using `AsyncItemPaged.by_page()`. :keyword response_format: - The format used to return and parse the List Blobs response. Possible values are - "auto", "xml", and "arrow". When set to "arrow", the response is returned and parsed in - Apache Arrow format (this requires the `nanoarrow` package). When set to "xml", the - standard XML response is parsed. "auto" currently behaves the same as "xml". This value - defaults to "xml". + The format used to return and parse the List Blobs response. + Possible values are "auto", "xml", and "arrow". + Choose "auto" to let the SDK choose the best algorithm, currently "xml". The default value is "auto". + + .. note:: + The use of "arrow" requires `nanoarrow` to be installed. :paramtype response_format: Literal["auto", "xml", "arrow"] :keyword str start_from: Specifies the full path (inclusive) to list paths from. @@ -874,8 +875,8 @@ def list_blobs( import nanoarrow # pylint: disable=import-outside-toplevel,unused-import except ImportError as e: raise ValueError( - "The use of Apache Arrow deserialization requires the extra [arrow] to be installed. " - "Please install this extra and try again." + "The use of Apache Arrow deserialization requires nanoarrow to be installed. " + "Please install nanoarrow and try again." ) from e command = functools.partial( ( @@ -975,11 +976,12 @@ def walk_blobs( names begin with the same substring up to the appearance of the delimiter character. The delimiter may be a single character or a string. :keyword response_format: - The format used to return and parse the List Blobs response. Possible values are - "auto", "xml", and "arrow". When set to "arrow", the response is returned and parsed in - Apache Arrow format (this requires the `nanoarrow` package). When set to "xml", the - standard XML response is parsed. "auto" currently behaves the same as "xml". This value - defaults to "xml". + The format used to return and parse the List Blobs response. + Possible values are "auto", "xml", and "arrow". + Choose "auto" to let the SDK choose the best algorithm, currently "xml". The default value is "auto". + + .. note:: + The use of "arrow" requires `nanoarrow` to be installed. :paramtype response_format: Literal["auto", "xml", "arrow"] :keyword str start_from: Specifies the full path (inclusive) to list paths from. @@ -1013,8 +1015,8 @@ def walk_blobs( import nanoarrow # pylint: disable=import-outside-toplevel,unused-import except ImportError as e: raise ValueError( - "The use of Apache Arrow deserialization requires the extra [arrow] to be installed. " - "Please install this extra and try again." + "The use of Apache Arrow deserialization requires nanoarrow to be installed. " + "Please install nanoarrow and try again." ) from e command = functools.partial( self._client.container.list_blob_hierarchy_segment_apache_arrow, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.pyi b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.pyi index 07cca4ad5f39..4b9d58011f9e 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.pyi +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.pyi @@ -210,7 +210,9 @@ class ContainerClient( # type: ignore[misc] include: Optional[Union[str, List[str]]] = None, *, results_per_page: Optional[int] = None, + response_format: Literal["auto", "xml", "arrow"] = "auto", start_from: Optional[str] = None, + end_before: Optional[str] = None, timeout: Optional[int] = None, **kwargs: Any ) -> AsyncItemPaged[BlobProperties]: ... @@ -231,7 +233,9 @@ class ContainerClient( # type: ignore[misc] include: Optional[Union[List[str], str]] = None, delimiter: str = "/", *, + response_format: Literal["auto", "xml", "arrow"] = "auto", start_from: Optional[str] = None, + end_before: Optional[str] = None, timeout: Optional[int] = None, **kwargs: Any ) -> AsyncItemPaged[Union[BlobProperties, BlobPrefix]]: ... diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py index abecde2b10d7..8b561567eca8 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py @@ -255,7 +255,7 @@ async def _arrow_cls(self, pipeline_response, deserialized, response_headers): content_type = response_headers.get("Content-Type", "") location_mode = getattr(pipeline_response.http_response, "location_mode", None) # The response is Arrow only when the service returns the Arrow stream media type. - if _ARROW_CONTENT_TYPE.split(",", 1)[0] in content_type: + if _ARROW_CONTENT_TYPE in content_type: chunks = [] if hasattr(deserialized, "__aiter__"): async for chunk in deserialized: diff --git a/sdk/storage/azure-storage-blob/setup.py b/sdk/storage/azure-storage-blob/setup.py index c2bf4197858f..2e832f1bc411 100644 --- a/sdk/storage/azure-storage-blob/setup.py +++ b/sdk/storage/azure-storage-blob/setup.py @@ -88,9 +88,6 @@ "aio": [ "azure-core[aio]>=1.37.0", ], - "arrow": [ - "nanoarrow>=0.8.0", - ], "ext-checksums": [ "azure-storage-extensions>=0.1.0,<1.0.0", ], From 9edd951e37c47012a6cd3115d0d98a78edfdcae1 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 8 Jul 2026 18:27:48 -0400 Subject: [PATCH 12/20] Changelogs --- sdk/storage/azure-storage-blob/CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-blob/CHANGELOG.md b/sdk/storage/azure-storage-blob/CHANGELOG.md index 96a200dddf93..52ddca1dc6dc 100644 --- a/sdk/storage/azure-storage-blob/CHANGELOG.md +++ b/sdk/storage/azure-storage-blob/CHANGELOG.md @@ -4,11 +4,11 @@ ### Features Added - Added support for service version 2026-10-06. -- Added support for parsing List Blobs responses in Apache Arrow format. Pass the +- Added support for parsing List Blobs responses in Apache Arrow format by passing `response_format="arrow"` keyword to `ContainerClient`'s `list_blobs` and `walk_blobs` APIs -to opt in (`response_format` defaults to `"xml"`, and `"auto"` currently also means `"xml"`). -This requires the optional `[arrow]` extra to be installed, and also enables the new -`start_from` and `end_before` keywords for range-based listing. +(`response_format` defaults to `"auto"`, and `"auto"` currently means `"xml"`). +The use of `"response_format=arrow"` requires `nanoarrow` to be installed. +This change also enables the new `end_before` keyword for range-based listing. - Added access tier information to the response of `BlobClient`'s `download_blob` API. The `blob_tier`, `blob_tier_inferred`, `blob_tier_change_time`, and `smart_access_tier` properties are now populated on the downloaded blob's `properties`. From c6b83aa832c5b8a8435c4592c017f90ce0428488 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 8 Jul 2026 18:33:51 -0400 Subject: [PATCH 13/20] PR Feedback --- .../azure/storage/blob/_list_blobs_helper.py | 5 +++-- .../azure/storage/blob/aio/_list_blobs_helper.py | 15 ++++----------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py index 0d89e15c081d..f57be6dca4b3 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py @@ -36,12 +36,13 @@ def _parse_arrow_response( # pylint: disable=too-many-locals - raw_bytes: bytes, container: Optional[str] + raw_bytes: Union[bytes, bytearray], container: Optional[str] ) -> Tuple[Optional[str], List[BlobProperties]]: """ Parse an Apache Arrow IPC stream response into a list of BlobProperties. - :param bytes raw_bytes: The raw Arrow IPC bytes. + :param raw_bytes: The raw Arrow IPC bytes. + :type raw_bytes: bytes or bytearray :param Optional[str] container: The container name to stamp on each item. :returns: A tuple of next marker and a list of BlobProperties. :rtype: Tuple[Optional[str], List[~azure.storage.blob.BlobProperties]] diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py index 8b561567eca8..881fadaab0e4 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py @@ -256,14 +256,9 @@ async def _arrow_cls(self, pipeline_response, deserialized, response_headers): location_mode = getattr(pipeline_response.http_response, "location_mode", None) # The response is Arrow only when the service returns the Arrow stream media type. if _ARROW_CONTENT_TYPE in content_type: - chunks = [] - if hasattr(deserialized, "__aiter__"): - async for chunk in deserialized: - chunks.append(chunk) - else: - for chunk in deserialized: - chunks.append(chunk) - raw_bytes = b"".join(chunks) + raw_bytes = bytearray() + async for chunk in deserialized: + raw_bytes += chunk next_marker, blob_items = _parse_arrow_response(raw_bytes, self.container) self._arrow_response = (next_marker, blob_items) return location_mode, raw_bytes @@ -284,9 +279,7 @@ async def _get_next_cb(self, continuation_token): cls=self._arrow_cls, use_location=self.location_mode, ) - if inspect.isawaitable(result): - return await result - return result + return await result except HttpResponseError as error: process_storage_error(error) From 8a92b72cf328f67163e3ade42b31d0ab555a463c Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 8 Jul 2026 20:23:08 -0400 Subject: [PATCH 14/20] mocked tests fallback to xml --- .../azure/storage/blob/_list_blobs_helper.py | 55 ++- .../storage/blob/aio/_list_blobs_helper.py | 54 ++- .../azure-storage-blob/tests/test_arrow.py | 441 +++++++---------- .../tests/test_arrow_async.py | 442 +++++++----------- 4 files changed, 440 insertions(+), 552 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py index f57be6dca4b3..9851b564dda1 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py @@ -474,6 +474,9 @@ def _build_item(item): class ArrowBlobPropertiesPaged(BlobPropertiesPaged): """A PageIterator that deserializes Apache Arrow IPC responses from list-blobs operations.""" + # The response type used to deserialize an XML fallback response. + _xml_response_type = "ListBlobsFlatSegmentResponse" + def __init__(self, *args: Any, deserializer: Any = None, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self._deserializer = deserializer @@ -502,7 +505,7 @@ def _arrow_cls(self, pipeline_response: Any, deserialized: Any, response_headers return location_mode, raw_bytes if hasattr(pipeline_response.http_response, "read"): pipeline_response.http_response.read() - xml_response = self._deserializer("ListBlobsFlatSegmentResponse", pipeline_response.http_response) + xml_response = self._deserializer(self._xml_response_type, pipeline_response.http_response) self._arrow_response = None return location_mode, xml_response @@ -518,7 +521,57 @@ def _extract_data_cb(self, get_next_return: Any) -> Tuple[Optional[str], List[Bl class ArrowBlobPrefixPaged(ArrowBlobPropertiesPaged): """Arrow-backed PageIterator for walk_blobs.""" + _xml_response_type = "ListBlobsHierarchySegmentResponse" + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.name = self.prefix self.delimiter: Optional[str] = kwargs.get("delimiter") + + def _extract_data_cb(self, get_next_return): + # Arrow responses only carry blob items; defer to the base class for those. + if self._arrow_response is not None: + return super()._extract_data_cb(get_next_return) + # XML fallback: reuse the base to populate the response, then preserve the + # hierarchy's virtual directories (BlobPrefix) alongside the blobs. + next_marker, _ = super()._extract_data_cb(get_next_return) + self.current_page = self._response.segment.blob_prefixes + self._response.segment.blob_items + self.current_page = [self._build_item(item) for item in self.current_page] + self.delimiter = self._response.delimiter + return next_marker, self.current_page + + def _build_item(self, item): + item = super()._build_item(item) + if isinstance(item, GenBlobPrefix): + if item.name.encoded: + name = unquote(item.name.content) + else: + name = item.name.content + return ArrowBlobPrefix( + self._command, + container=self.container, + prefix=name, + results_per_page=self.results_per_page, + location_mode=self.location_mode, + delimiter=self.delimiter, + deserializer=self._deserializer, + ) + return item + + +class ArrowBlobPrefix(BlobPrefix): + """An Arrow-backed virtual blob directory returned from walk_blobs. + + Drilling into this prefix re-lists using the Apache Arrow operation, so nested pages + are parsed the same way (Arrow when available, XML otherwise).""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super(BlobPrefix, self).__init__( # pylint: disable=bad-super-call + *args, page_iterator_class=ArrowBlobPrefixPaged, **kwargs + ) + self.name = kwargs.get("prefix") # type: ignore [assignment] + self.prefix = kwargs.get("prefix") # type: ignore [assignment] + self.results_per_page = kwargs.get("results_per_page") + self.container = kwargs.get("container") # type: ignore [assignment] + self.delimiter = kwargs.get("delimiter") # type: ignore [assignment] + self.location_mode = kwargs.get("location_mode") # type: ignore [assignment] diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py index 881fadaab0e4..ddf7e32703e2 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_list_blobs_helper.py @@ -246,6 +246,8 @@ def _build_item(self, item): class ArrowBlobPropertiesPaged(BlobPropertiesPaged): """An async PageIterator that deserializes Apache Arrow IPC responses from list-blobs operations.""" + _xml_response_type = "ListBlobsFlatSegmentResponse" + def __init__(self, *args, deserializer=None, **kwargs): super().__init__(*args, **kwargs) self._deserializer = deserializer @@ -266,7 +268,7 @@ async def _arrow_cls(self, pipeline_response, deserialized, response_headers): read_result = pipeline_response.http_response.read() if inspect.isawaitable(read_result): await read_result - xml_response = self._deserializer("ListBlobsFlatSegmentResponse", pipeline_response.http_response) + xml_response = self._deserializer(self._xml_response_type, pipeline_response.http_response) self._arrow_response = None return location_mode, xml_response @@ -295,7 +297,57 @@ async def _extract_data_cb(self, get_next_return): class ArrowBlobPrefixPaged(ArrowBlobPropertiesPaged): """Arrow-backed AsyncPageIterator for walk_blobs.""" + _xml_response_type = "ListBlobsHierarchySegmentResponse" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.name = self.prefix self.delimiter = kwargs.get("delimiter") + + async def _extract_data_cb(self, get_next_return): + # Arrow responses only carry blob items; defer to the base class for those. + if self._arrow_response is not None: + return await super()._extract_data_cb(get_next_return) + # XML fallback: reuse the base to populate the response, then preserve the + # hierarchy's virtual directories (BlobPrefix) alongside the blobs. + next_marker, _ = await super()._extract_data_cb(get_next_return) + self.current_page = self._response.segment.blob_prefixes + self._response.segment.blob_items + self.current_page = [self._build_item(item) for item in self.current_page] + self.delimiter = self._response.delimiter + return next_marker, self.current_page + + def _build_item(self, item): + item = super()._build_item(item) + if isinstance(item, GenBlobPrefix): + if item.name.encoded: + name = unquote(item.name.content) + else: + name = item.name.content + return ArrowBlobPrefix( + self._command, + container=self.container, + prefix=name, + results_per_page=self.results_per_page, + location_mode=self.location_mode, + delimiter=self.delimiter, + deserializer=self._deserializer, + ) + return item + + +class ArrowBlobPrefix(BlobPrefix): + """An Arrow-backed virtual blob directory returned from walk_blobs. + + Drilling into this prefix re-lists using the Apache Arrow operation, so nested pages + are parsed the same way (Arrow when available, XML otherwise).""" + + def __init__(self, *args, **kwargs): + super(BlobPrefix, self).__init__( # pylint: disable=bad-super-call + *args, page_iterator_class=ArrowBlobPrefixPaged, **kwargs + ) + self.name = kwargs.get("prefix") + self.prefix = kwargs.get("prefix") + self.results_per_page = kwargs.get("results_per_page") + self.container = kwargs.get("container") + self.delimiter = kwargs.get("delimiter") + self.location_mode = kwargs.get("location_mode") diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow.py b/sdk/storage/azure-storage-blob/tests/test_arrow.py index efb3bd020d0a..074d25b27746 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow.py @@ -3,24 +3,13 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -import io - -import nanoarrow as na -from nanoarrow import ipc - from datetime import datetime, timezone -from unittest.mock import MagicMock, patch +from types import SimpleNamespace +from unittest.mock import patch from azure.core.credentials import AzureNamedKeyCredential from azure.core.exceptions import ResourceExistsError -from azure.storage.blob import BlobProperties, BlobServiceClient, ContainerClient -from azure.storage.blob._generated.models import ( - BlobFlatListSegment, - BlobItemInternal, - BlobName, - BlobPropertiesInternal, - ListBlobsFlatSegmentResponse, -) +from azure.storage.blob import BlobProperties, BlobServiceClient, BlobType, ContainerClient from devtools_testutils import recorded_by_proxy from devtools_testutils.storage import StorageRecordedTestCase @@ -31,6 +20,105 @@ # ------------------------------------------------------------------------------ +def _rich_blob_xml(name: str) -> str: + """A single element with as many properties populated as possible.""" + return ( + "" + f"{name}" + "" + "Mon, 01 Jan 2024 00:00:00 GMT" + "Tue, 02 Jan 2024 03:04:05 GMT" + "0x8DABCDEF1234567" + "1234" + "text/plain" + "gzip" + "en-US" + "ASNFZ4mrze8BI0VniavN7w==" + "no-cache" + "inline" + "BlockBlob" + "Hot" + "true" + "unlocked" + "available" + "true" + "1" + "" + "bluelarge" + "envprod" + "" + ) + + +def _simple_blob_xml(name: str) -> str: + """A minimal but valid element.""" + return ( + "" + f"{name}" + "" + "Tue, 02 Jan 2024 03:04:05 GMT" + "0x8DABCDEF1234567" + "BlockBlob" + "" + "" + ) + + +def _enumeration_results_xml(blobs_xml: str, next_marker: str = "") -> bytes: + """Wrap one or more elements in a valid List Blobs response body.""" + marker = f"{next_marker}" if next_marker else "" + return ( + '' + '' + "10" + f"{blobs_xml}{marker}" + "" + ).encode("utf-8") + + +def _blob_prefix_xml(name: str) -> str: + """A (virtual directory) element for a hierarchy listing.""" + return f"{name}" + + +def _hierarchy_results_xml(blobs_xml: str, next_marker: str = "") -> bytes: + """Wrap / elements in a valid hierarchical List Blobs response body.""" + marker = f"{next_marker}" if next_marker else "" + return ( + '' + '' + "10/" + f"{blobs_xml}{marker}" + "" + ).encode("utf-8") + + +class _MockXmlHttpResponse: + """A minimal HTTP response carrying a raw XML body for the deserializer to parse.""" + + def __init__(self, xml_body: bytes) -> None: + self._body = xml_body + self.headers = {"content-type": "application/xml"} + self.location_mode = None + + def body(self) -> bytes: + return self._body + + def text(self, encoding=None) -> str: # pylint: disable=unused-argument + return self._body.decode("utf-8") + + def read(self) -> bytes: + return self._body + + +def _mock_container_client() -> ContainerClient: + return ContainerClient( + account_url="https://account.blob.core.windows.net", + container_name="mycontainer", + credential=AzureNamedKeyCredential("account", "A" * 64), + ) + + class TestStorageApacheArrow(StorageRecordedTestCase): def _setup(self, storage_account_name, storage_account_key): self.bsc = BlobServiceClient( @@ -54,6 +142,29 @@ def verify_blobs(self, blobs_list: list[BlobProperties], blob_names: list[str]): for blob_name in blob_names: assert blob_name in all_names + def verify_all_fields(self, blob: BlobProperties): + # Verifies the properties produced by _rich_blob_xml were fully deserialized. + assert blob.name == "dir/blob1" + assert blob.blob_type == BlobType.BLOCKBLOB + assert blob.etag == "0x8DABCDEF1234567" + assert blob.size == 1234 + assert blob.server_encrypted is True + assert blob.tag_count == 1 + assert blob.creation_time == datetime(2024, 1, 1, tzinfo=timezone.utc) + assert blob.last_modified == datetime(2024, 1, 2, 3, 4, 5, tzinfo=timezone.utc) + assert blob.blob_tier == "Hot" + assert blob.blob_tier_inferred is True + assert blob.content_settings.content_type == "text/plain" + assert blob.content_settings.content_encoding == "gzip" + assert blob.content_settings.content_language == "en-US" + assert blob.content_settings.cache_control == "no-cache" + assert blob.content_settings.content_disposition == "inline" + assert blob.content_settings.content_md5 is not None + assert blob.lease.status == "unlocked" + assert blob.lease.state == "available" + assert blob.metadata == {"color": "blue", "size": "large"} + assert blob.tags == {"env": "prod"} + @BlobPreparer() @recorded_by_proxy def test_arrow_list_no_blobs(self, **kwargs): @@ -126,150 +237,39 @@ def test_arrow_list_nested_blobs_paging(self, **kwargs): third_blobs_list = list(next(blob_pages)) self.verify_blobs(third_blobs_list, blob_names[6:]) - def test_arrow_mock_xml_response(self): - blob_names = [ - "a/b/blob1", - "a/b/blob2", - "a/c/blob3", - "d/blob4", - "d/e/f/blob5", - "flat_blob1", - "flat_blob2", - "flat_blob3", - "flat_blob4", - ] - account_url = "https://account.blob.core.windows.net/" - container_name = "mycontainer" - - def _make_xml_response(names: list[str], next_marker: str | None = None) -> ListBlobsFlatSegmentResponse: - items = [ - BlobItemInternal( - name=BlobName(content=name, encoded=False), - snapshot="", - properties=BlobPropertiesInternal( - etag="", - last_modified=datetime(2024, 1, 1, tzinfo=timezone.utc), - blob_type="BlockBlob", - ), - deleted=False, - ) - for name in names - ] - return ListBlobsFlatSegmentResponse( - service_endpoint=account_url, - container_name=container_name, - segment=BlobFlatListSegment(blob_items=items), - next_marker=next_marker, - ) - - # Three pages of 3 blobs each, chained via next_marker. - xml_pages = [ - _make_xml_response(blob_names[0:3], next_marker="marker1"), - _make_xml_response(blob_names[3:6], next_marker="marker2"), - _make_xml_response(blob_names[6:9], next_marker=None), - ] + def test_arrow_list_xml_fallback(self): + page1 = _enumeration_results_xml(_rich_blob_xml("dir/blob1") + _simple_blob_xml("blob2"), next_marker="marker1") + page2 = _enumeration_results_xml(_simple_blob_xml("blob3")) + pages = [page1, page2] page_index = 0 - container_client = ContainerClient( - account_url=account_url, - container_name=container_name, - credential=AzureNamedKeyCredential("account", "A" * 64), - ) - - mock_http_response = MagicMock() - mock_http_response.location_mode = None - mock_pipeline_response = MagicMock() - mock_pipeline_response.http_response = mock_http_response + container_client = _mock_container_client() def fake_list_blob_flat_segment_apache_arrow(**kwargs): - cls = kwargs.get("cls") - return cls(mock_pipeline_response, None, {"Content-Type": "application/xml"}) - - def deserializer_side_effect(target, _http_response): nonlocal page_index - response = xml_pages[page_index] + cls = kwargs["cls"] + pipeline_response = SimpleNamespace(http_response=_MockXmlHttpResponse(pages[page_index])) page_index += 1 - return response + return cls(pipeline_response, iter([]), {"Content-Type": "application/xml"}) with patch.object( container_client._client.container, # pylint: disable=protected-access "list_blob_flat_segment_apache_arrow", side_effect=fake_list_blob_flat_segment_apache_arrow, ): - blob_pages = container_client.list_blobs(response_format="arrow", results_per_page=3).by_page() - mock_deserializer = MagicMock(side_effect=deserializer_side_effect) - blob_pages._deserializer = mock_deserializer - first_blobs_list = list(next(blob_pages)) - self.verify_blobs(first_blobs_list, blob_names[0:3]) - second_blobs_list = list(next(blob_pages)) - self.verify_blobs(second_blobs_list, blob_names[3:6]) - third_blobs_list = list(next(blob_pages)) - self.verify_blobs(third_blobs_list, blob_names[6:9]) - - # Confirm the XML deserializer was called once per page. - assert mock_deserializer.call_count == 3 - - def test_arrow_mock_expected_response(self): - def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: - metadata = {"NextMarker": next_marker} if next_marker else None - schema = na.Schema(na.struct({"Name": na.string()}), metadata=metadata) - name_array = na.c_array(names, na.string()) - batch = na.c_array_from_buffers(schema, length=len(names), buffers=[None], children=[name_array]) - out = io.BytesIO() - writer = ipc.StreamWriter.from_writable(out) - writer.write_stream(batch) - writer.close() - return out.getvalue() + blob_pages = container_client.list_blobs( + response_format="arrow", results_per_page=2, include=["metadata", "tags"] + ).by_page() + first_page = list(next(blob_pages)) + second_page = list(next(blob_pages)) - blob_names = [ - "a/b/blob1", - "a/b/blob2", - "a/c/blob3", - "d/blob4", - "d/e/f/blob5", - "flat_blob1", - "flat_blob2", - "flat_blob3", - "flat_blob4", - ] - # Three pages of 3 blobs each, using the marker to chain pages. - pages = [ - _make_arrow_page(blob_names[0:3], next_marker="marker1"), - _make_arrow_page(blob_names[3:6], next_marker="marker2"), - _make_arrow_page(blob_names[6:9], next_marker=None), - ] - page_index = 0 + # Paging is driven by the NextMarker parsed out of the XML. + assert page_index == 2 + assert [blob.name for blob in first_page] == ["dir/blob1", "blob2"] + assert [blob.name for blob in second_page] == ["blob3"] - container_client = ContainerClient( - account_url="https://account.blob.core.windows.net", - container_name="mycontainer", - credential=AzureNamedKeyCredential("account", "A" * 64), - ) - - mock_http_response = MagicMock() - mock_http_response.location_mode = None - mock_pipeline_response = MagicMock() - mock_pipeline_response.http_response = mock_http_response - - def fake_list_blob_flat_segment_apache_arrow(**kwargs): - nonlocal page_index - cls = kwargs.get("cls") - raw = iter([pages[page_index]]) - page_index += 1 - return cls(mock_pipeline_response, raw, {"Content-Type": "application/vnd.apache.arrow.stream"}) - - with patch.object( - container_client._client.container, # pylint: disable=protected-access - "list_blob_flat_segment_apache_arrow", - side_effect=fake_list_blob_flat_segment_apache_arrow, - ): - blob_pages = container_client.list_blobs(response_format="arrow", results_per_page=3).by_page() - first_blobs_list = list(next(blob_pages)) - self.verify_blobs(first_blobs_list, blob_names[0:3]) - second_blobs_list = list(next(blob_pages)) - self.verify_blobs(second_blobs_list, blob_names[3:6]) - third_blobs_list = list(next(blob_pages)) - self.verify_blobs(third_blobs_list, blob_names[6:9]) + # The first blob has every property populated so we can assert the XML was fully deserialized. + self.verify_all_fields(first_page[0]) @BlobPreparer() @recorded_by_proxy @@ -341,146 +341,39 @@ def test_arrow_walk_nested_blobs_paging(self, **kwargs): second_blobs_list = list(next(blob_pages)) self.verify_blobs(second_blobs_list, ["flat_blob2", "flat_blob3", "flat_blob4"]) - def test_arrow_mock_walk_xml_response(self): - blob_names = [ - "a/b/blob1", - "a/b/blob2", - "a/c/blob3", - "d/blob4", - "d/e/f/blob5", - "flat_blob1", - "flat_blob2", - "flat_blob3", - "flat_blob4", - ] - account_url = "https://account.blob.core.windows.net/" - container_name = "mycontainer" - - def _make_xml_response(names: list[str], next_marker: str | None = None) -> ListBlobsFlatSegmentResponse: - items = [ - BlobItemInternal( - name=BlobName(content=name, encoded=False), - snapshot="", - properties=BlobPropertiesInternal( - etag="", - last_modified=datetime(2024, 1, 1, tzinfo=timezone.utc), - blob_type="BlockBlob", - ), - deleted=False, - ) - for name in names - ] - return ListBlobsFlatSegmentResponse( - service_endpoint=account_url, - container_name=container_name, - segment=BlobFlatListSegment(blob_items=items), - next_marker=next_marker, - ) - - # Three pages of 3 blobs each, chained via next_marker. - xml_pages = [ - _make_xml_response(blob_names[0:3], next_marker="marker1"), - _make_xml_response(blob_names[3:6], next_marker="marker2"), - _make_xml_response(blob_names[6:9], next_marker=None), - ] - page_index = 0 - - container_client = ContainerClient( - account_url=account_url, - container_name=container_name, - credential=AzureNamedKeyCredential("account", "A" * 64), + def test_arrow_walk_xml_fallback(self): + page1 = _hierarchy_results_xml( + _blob_prefix_xml("a/") + _blob_prefix_xml("d/") + _rich_blob_xml("dir/blob1"), + next_marker="marker1", ) - - mock_http_response = MagicMock() - mock_http_response.location_mode = None - mock_pipeline_response = MagicMock() - mock_pipeline_response.http_response = mock_http_response - - def fake_list_blob_hierarchy_segment_apache_arrow(**kwargs): - cls = kwargs.get("cls") - return cls(mock_pipeline_response, None, {"Content-Type": "application/xml"}) - - def deserializer_side_effect(target, _http_response): - nonlocal page_index - response = xml_pages[page_index] - page_index += 1 - return response - - with patch.object( - container_client._client.container, # pylint: disable=protected-access - "list_blob_hierarchy_segment_apache_arrow", - side_effect=fake_list_blob_hierarchy_segment_apache_arrow, - ): - blob_pages = container_client.walk_blobs(response_format="arrow", results_per_page=3).by_page() - mock_deserializer = MagicMock(side_effect=deserializer_side_effect) - blob_pages._deserializer = mock_deserializer - first_blobs_list = list(next(blob_pages)) - self.verify_blobs(first_blobs_list, blob_names[0:3]) - second_blobs_list = list(next(blob_pages)) - self.verify_blobs(second_blobs_list, blob_names[3:6]) - third_blobs_list = list(next(blob_pages)) - self.verify_blobs(third_blobs_list, blob_names[6:9]) - - # Confirm the XML deserializer was called once per page. - assert mock_deserializer.call_count == 3 - - def test_arrow_mock_walk_expected_response(self): - def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: - metadata = {"NextMarker": next_marker} if next_marker else None - schema = na.Schema(na.struct({"Name": na.string()}), metadata=metadata) - name_array = na.c_array(names, na.string()) - batch = na.c_array_from_buffers(schema, length=len(names), buffers=[None], children=[name_array]) - out = io.BytesIO() - writer = ipc.StreamWriter.from_writable(out) - writer.write_stream(batch) - writer.close() - return out.getvalue() - - blob_names = [ - "a/b/blob1", - "a/b/blob2", - "a/c/blob3", - "d/blob4", - "d/e/f/blob5", - "flat_blob1", - "flat_blob2", - "flat_blob3", - "flat_blob4", - ] - pages = [ - _make_arrow_page(blob_names[0:3], next_marker="marker1"), - _make_arrow_page(blob_names[3:6], next_marker="marker2"), - _make_arrow_page(blob_names[6:9], next_marker=None), - ] + page2 = _hierarchy_results_xml(_simple_blob_xml("blob3")) + pages = [page1, page2] page_index = 0 - container_client = ContainerClient( - account_url="https://account.blob.core.windows.net", - container_name="mycontainer", - credential=AzureNamedKeyCredential("account", "A" * 64), - ) - - mock_http_response = MagicMock() - mock_http_response.location_mode = None - mock_pipeline_response = MagicMock() - mock_pipeline_response.http_response = mock_http_response + container_client = _mock_container_client() def fake_list_blob_hierarchy_segment_apache_arrow(**kwargs): nonlocal page_index - cls = kwargs.get("cls") - raw = iter([pages[page_index]]) + cls = kwargs["cls"] + pipeline_response = SimpleNamespace(http_response=_MockXmlHttpResponse(pages[page_index])) page_index += 1 - return cls(mock_pipeline_response, raw, {"Content-Type": "application/vnd.apache.arrow.stream"}) + return cls(pipeline_response, iter([]), {"Content-Type": "application/xml"}) with patch.object( container_client._client.container, # pylint: disable=protected-access "list_blob_hierarchy_segment_apache_arrow", side_effect=fake_list_blob_hierarchy_segment_apache_arrow, ): - blob_pages = container_client.walk_blobs(response_format="arrow", results_per_page=3).by_page() - first_blobs_list = list(next(blob_pages)) - self.verify_blobs(first_blobs_list, blob_names[0:3]) - second_blobs_list = list(next(blob_pages)) - self.verify_blobs(second_blobs_list, blob_names[3:6]) - third_blobs_list = list(next(blob_pages)) - self.verify_blobs(third_blobs_list, blob_names[6:9]) + blob_pages = container_client.walk_blobs( + response_format="arrow", results_per_page=2, include=["metadata", "tags"] + ).by_page() + first_page = list(next(blob_pages)) + second_page = list(next(blob_pages)) + + assert page_index == 2 + assert [item.name for item in first_page] == ["a/", "d/", "dir/blob1"] + assert not isinstance(first_page[0], BlobProperties) + assert not isinstance(first_page[1], BlobProperties) + assert isinstance(first_page[2], BlobProperties) + self.verify_all_fields(first_page[2]) + assert [item.name for item in second_page] == ["blob3"] diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py index fbc64ca080ea..1a28ca3664da 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py @@ -3,25 +3,15 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -import io +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import patch -import nanoarrow as na import pytest -from nanoarrow import ipc - -from datetime import datetime, timezone -from unittest.mock import MagicMock, patch from azure.core.credentials import AzureNamedKeyCredential from azure.core.exceptions import ResourceExistsError -from azure.storage.blob import BlobProperties -from azure.storage.blob._generated.models import ( - BlobFlatListSegment, - BlobItemInternal, - BlobName, - BlobPropertiesInternal, - ListBlobsFlatSegmentResponse, -) +from azure.storage.blob import BlobProperties, BlobType from azure.storage.blob.aio import BlobServiceClient, ContainerClient from devtools_testutils.aio import recorded_by_proxy_async @@ -33,6 +23,105 @@ # ------------------------------------------------------------------------------ +def _rich_blob_xml(name: str) -> str: + """A single element with as many properties populated as possible.""" + return ( + "" + f"{name}" + "" + "Mon, 01 Jan 2024 00:00:00 GMT" + "Tue, 02 Jan 2024 03:04:05 GMT" + "0x8DABCDEF1234567" + "1234" + "text/plain" + "gzip" + "en-US" + "ASNFZ4mrze8BI0VniavN7w==" + "no-cache" + "inline" + "BlockBlob" + "Hot" + "true" + "unlocked" + "available" + "true" + "1" + "" + "bluelarge" + "envprod" + "" + ) + + +def _simple_blob_xml(name: str) -> str: + """A minimal but valid element.""" + return ( + "" + f"{name}" + "" + "Tue, 02 Jan 2024 03:04:05 GMT" + "0x8DABCDEF1234567" + "BlockBlob" + "" + "" + ) + + +def _enumeration_results_xml(blobs_xml: str, next_marker: str = "") -> bytes: + """Wrap one or more elements in a valid List Blobs response body.""" + marker = f"{next_marker}" if next_marker else "" + return ( + '' + '' + "10" + f"{blobs_xml}{marker}" + "" + ).encode("utf-8") + + +def _blob_prefix_xml(name: str) -> str: + """A (virtual directory) element for a hierarchy listing.""" + return f"{name}" + + +def _hierarchy_results_xml(blobs_xml: str, next_marker: str = "") -> bytes: + """Wrap / elements in a valid hierarchical List Blobs response body.""" + marker = f"{next_marker}" if next_marker else "" + return ( + '' + '' + "10/" + f"{blobs_xml}{marker}" + "" + ).encode("utf-8") + + +class _MockXmlHttpResponse: + """A minimal HTTP response carrying a raw XML body for the deserializer to parse.""" + + def __init__(self, xml_body: bytes) -> None: + self._body = xml_body + self.headers = {"content-type": "application/xml"} + self.location_mode = None + + def body(self) -> bytes: + return self._body + + def text(self, encoding=None) -> str: # pylint: disable=unused-argument + return self._body.decode("utf-8") + + def read(self) -> bytes: + return self._body + + +def _mock_container_client() -> ContainerClient: + return ContainerClient( + account_url="https://account.blob.core.windows.net", + container_name="mycontainer", + credential=AzureNamedKeyCredential("account", "A" * 64), + ) + + class TestStorageApacheArrowAsync(AsyncStorageRecordedTestCase): async def _setup(self, storage_account_name, storage_account_key): self.bsc = BlobServiceClient( @@ -56,6 +145,29 @@ def verify_blobs(self, blobs_list: list[BlobProperties], blob_names: list[str]): for blob_name in blob_names: assert blob_name in all_names + def verify_all_fields(self, blob: BlobProperties): + # Verifies the properties produced by _rich_blob_xml were fully deserialized. + assert blob.name == "dir/blob1" + assert blob.blob_type == BlobType.BLOCKBLOB + assert blob.etag == "0x8DABCDEF1234567" + assert blob.size == 1234 + assert blob.server_encrypted is True + assert blob.tag_count == 1 + assert blob.creation_time == datetime(2024, 1, 1, tzinfo=timezone.utc) + assert blob.last_modified == datetime(2024, 1, 2, 3, 4, 5, tzinfo=timezone.utc) + assert blob.blob_tier == "Hot" + assert blob.blob_tier_inferred is True + assert blob.content_settings.content_type == "text/plain" + assert blob.content_settings.content_encoding == "gzip" + assert blob.content_settings.content_language == "en-US" + assert blob.content_settings.cache_control == "no-cache" + assert blob.content_settings.content_disposition == "inline" + assert blob.content_settings.content_md5 is not None + assert blob.lease.status == "unlocked" + assert blob.lease.state == "available" + assert blob.metadata == {"color": "blue", "size": "large"} + assert blob.tags == {"env": "prod"} + @BlobPreparer() @recorded_by_proxy_async async def test_arrow_list_no_blobs(self, **kwargs): @@ -129,150 +241,36 @@ async def test_arrow_list_nested_blobs_paging(self, **kwargs): self.verify_blobs(third_blobs_list, blob_names[6:]) @pytest.mark.asyncio - async def test_arrow_mock_xml_response(self): - blob_names = [ - "a/b/blob1", - "a/b/blob2", - "a/c/blob3", - "d/blob4", - "d/e/f/blob5", - "flat_blob1", - "flat_blob2", - "flat_blob3", - "flat_blob4", - ] - account_url = "https://account.blob.core.windows.net/" - container_name = "mycontainer" - - def _make_xml_response(names: list[str], next_marker: str | None = None) -> ListBlobsFlatSegmentResponse: - items = [ - BlobItemInternal( - name=BlobName(content=name, encoded=False), - snapshot="", - properties=BlobPropertiesInternal( - etag="", - last_modified=datetime(2024, 1, 1, tzinfo=timezone.utc), - blob_type="BlockBlob", - ), - deleted=False, - ) - for name in names - ] - return ListBlobsFlatSegmentResponse( - service_endpoint=account_url, - container_name=container_name, - segment=BlobFlatListSegment(blob_items=items), - next_marker=next_marker, - ) - - # Three pages of 3 blobs each, chained via next_marker. - xml_pages = [ - _make_xml_response(blob_names[0:3], next_marker="marker1"), - _make_xml_response(blob_names[3:6], next_marker="marker2"), - _make_xml_response(blob_names[6:9], next_marker=None), - ] + async def test_arrow_list_xml_fallback(self): + page1 = _enumeration_results_xml(_rich_blob_xml("dir/blob1") + _simple_blob_xml("blob2"), next_marker="marker1") + page2 = _enumeration_results_xml(_simple_blob_xml("blob3")) + pages = [page1, page2] page_index = 0 - container_client = ContainerClient( - account_url=account_url, - container_name=container_name, - credential=AzureNamedKeyCredential("account", "A" * 64), - ) - - mock_http_response = MagicMock() - mock_http_response.location_mode = None - mock_pipeline_response = MagicMock() - mock_pipeline_response.http_response = mock_http_response + container_client = _mock_container_client() async def fake_list_blob_flat_segment_apache_arrow(**kwargs): - cls = kwargs.get("cls") - return cls(mock_pipeline_response, None, {"Content-Type": "application/xml"}) - - def deserializer_side_effect(target, _http_response): nonlocal page_index - response = xml_pages[page_index] + cls = kwargs["cls"] + pipeline_response = SimpleNamespace(http_response=_MockXmlHttpResponse(pages[page_index])) page_index += 1 - return response + return cls(pipeline_response, iter([]), {"Content-Type": "application/xml"}) with patch.object( container_client._client.container, # pylint: disable=protected-access "list_blob_flat_segment_apache_arrow", side_effect=fake_list_blob_flat_segment_apache_arrow, ): - blob_pages = container_client.list_blobs(response_format="arrow", results_per_page=3).by_page() - mock_deserializer = MagicMock(side_effect=deserializer_side_effect) - blob_pages._deserializer = mock_deserializer - first_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(first_blobs_list, blob_names[0:3]) - second_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(second_blobs_list, blob_names[3:6]) - third_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(third_blobs_list, blob_names[6:9]) - - # Confirm the XML deserializer was called once per page. - assert mock_deserializer.call_count == 3 - - @pytest.mark.asyncio - async def test_arrow_mock_expected_response(self): - def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: - metadata = {"NextMarker": next_marker} if next_marker else None - schema = na.Schema(na.struct({"Name": na.string()}), metadata=metadata) - name_array = na.c_array(names, na.string()) - batch = na.c_array_from_buffers(schema, length=len(names), buffers=[None], children=[name_array]) - out = io.BytesIO() - writer = ipc.StreamWriter.from_writable(out) - writer.write_stream(batch) - writer.close() - return out.getvalue() + blob_pages = container_client.list_blobs( + response_format="arrow", results_per_page=2, include=["metadata", "tags"] + ).by_page() + first_page = [blob async for blob in await blob_pages.__anext__()] + second_page = [blob async for blob in await blob_pages.__anext__()] - blob_names = [ - "a/b/blob1", - "a/b/blob2", - "a/c/blob3", - "d/blob4", - "d/e/f/blob5", - "flat_blob1", - "flat_blob2", - "flat_blob3", - "flat_blob4", - ] - pages = [ - _make_arrow_page(blob_names[0:3], next_marker="marker1"), - _make_arrow_page(blob_names[3:6], next_marker="marker2"), - _make_arrow_page(blob_names[6:9], next_marker=None), - ] - page_index = 0 - - container_client = ContainerClient( - account_url="https://account.blob.core.windows.net", - container_name="mycontainer", - credential=AzureNamedKeyCredential("account", "A" * 64), - ) - - mock_http_response = MagicMock() - mock_http_response.location_mode = None - mock_pipeline_response = MagicMock() - mock_pipeline_response.http_response = mock_http_response - - async def fake_list_blob_flat_segment_apache_arrow(**kwargs): - nonlocal page_index - cls = kwargs.get("cls") - raw = iter([pages[page_index]]) - page_index += 1 - return cls(mock_pipeline_response, raw, {"Content-Type": "application/vnd.apache.arrow.stream"}) - - with patch.object( - container_client._client.container, # pylint: disable=protected-access - "list_blob_flat_segment_apache_arrow", - side_effect=fake_list_blob_flat_segment_apache_arrow, - ): - blob_pages = container_client.list_blobs(response_format="arrow", results_per_page=3).by_page() - first_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(first_blobs_list, blob_names[0:3]) - second_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(second_blobs_list, blob_names[3:6]) - third_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(third_blobs_list, blob_names[6:9]) + assert page_index == 2 + assert [blob.name for blob in first_page] == ["dir/blob1", "blob2"] + assert [blob.name for blob in second_page] == ["blob3"] + self.verify_all_fields(first_page[0]) @BlobPreparer() @recorded_by_proxy_async @@ -345,147 +343,39 @@ async def test_arrow_walk_nested_blobs_paging(self, **kwargs): self.verify_blobs(second_blobs_list, ["flat_blob2", "flat_blob3", "flat_blob4"]) @pytest.mark.asyncio - async def test_arrow_mock_walk_xml_response(self): - blob_names = [ - "a/b/blob1", - "a/b/blob2", - "a/c/blob3", - "d/blob4", - "d/e/f/blob5", - "flat_blob1", - "flat_blob2", - "flat_blob3", - "flat_blob4", - ] - account_url = "https://account.blob.core.windows.net/" - container_name = "mycontainer" - - def _make_xml_response(names: list[str], next_marker: str | None = None) -> ListBlobsFlatSegmentResponse: - items = [ - BlobItemInternal( - name=BlobName(content=name, encoded=False), - snapshot="", - properties=BlobPropertiesInternal( - etag="", - last_modified=datetime(2024, 1, 1, tzinfo=timezone.utc), - blob_type="BlockBlob", - ), - deleted=False, - ) - for name in names - ] - return ListBlobsFlatSegmentResponse( - service_endpoint=account_url, - container_name=container_name, - segment=BlobFlatListSegment(blob_items=items), - next_marker=next_marker, - ) - - # Three pages of 3 blobs each, chained via next_marker. - xml_pages = [ - _make_xml_response(blob_names[0:3], next_marker="marker1"), - _make_xml_response(blob_names[3:6], next_marker="marker2"), - _make_xml_response(blob_names[6:9], next_marker=None), - ] - page_index = 0 - - container_client = ContainerClient( - account_url=account_url, - container_name=container_name, - credential=AzureNamedKeyCredential("account", "A" * 64), + async def test_arrow_walk_xml_fallback(self): + page1 = _hierarchy_results_xml( + _blob_prefix_xml("a/") + _blob_prefix_xml("d/") + _rich_blob_xml("dir/blob1"), + next_marker="marker1", ) - - mock_http_response = MagicMock() - mock_http_response.location_mode = None - mock_pipeline_response = MagicMock() - mock_pipeline_response.http_response = mock_http_response - - async def fake_list_blob_hierarchy_segment_apache_arrow(**kwargs): - cls = kwargs.get("cls") - return cls(mock_pipeline_response, None, {"Content-Type": "application/xml"}) - - def deserializer_side_effect(target, _http_response): - nonlocal page_index - response = xml_pages[page_index] - page_index += 1 - return response - - with patch.object( - container_client._client.container, # pylint: disable=protected-access - "list_blob_hierarchy_segment_apache_arrow", - side_effect=fake_list_blob_hierarchy_segment_apache_arrow, - ): - blob_pages = container_client.walk_blobs(response_format="arrow", results_per_page=3).by_page() - mock_deserializer = MagicMock(side_effect=deserializer_side_effect) - blob_pages._deserializer = mock_deserializer - first_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(first_blobs_list, blob_names[0:3]) - second_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(second_blobs_list, blob_names[3:6]) - third_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(third_blobs_list, blob_names[6:9]) - - # Confirm the XML deserializer was called once per page. - assert mock_deserializer.call_count == 3 - - @pytest.mark.asyncio - async def test_arrow_mock_walk_expected_response(self): - def _make_arrow_page(names: list[str], next_marker: str | None = None) -> bytes: - metadata = {"NextMarker": next_marker} if next_marker else None - schema = na.Schema(na.struct({"Name": na.string()}), metadata=metadata) - name_array = na.c_array(names, na.string()) - batch = na.c_array_from_buffers(schema, length=len(names), buffers=[None], children=[name_array]) - out = io.BytesIO() - writer = ipc.StreamWriter.from_writable(out) - writer.write_stream(batch) - writer.close() - return out.getvalue() - - blob_names = [ - "a/b/blob1", - "a/b/blob2", - "a/c/blob3", - "d/blob4", - "d/e/f/blob5", - "flat_blob1", - "flat_blob2", - "flat_blob3", - "flat_blob4", - ] - pages = [ - _make_arrow_page(blob_names[0:3], next_marker="marker1"), - _make_arrow_page(blob_names[3:6], next_marker="marker2"), - _make_arrow_page(blob_names[6:9], next_marker=None), - ] + page2 = _hierarchy_results_xml(_simple_blob_xml("blob3")) + pages = [page1, page2] page_index = 0 - container_client = ContainerClient( - account_url="https://account.blob.core.windows.net", - container_name="mycontainer", - credential=AzureNamedKeyCredential("account", "A" * 64), - ) - - mock_http_response = MagicMock() - mock_http_response.location_mode = None - mock_pipeline_response = MagicMock() - mock_pipeline_response.http_response = mock_http_response + container_client = _mock_container_client() async def fake_list_blob_hierarchy_segment_apache_arrow(**kwargs): nonlocal page_index - cls = kwargs.get("cls") - raw = iter([pages[page_index]]) + cls = kwargs["cls"] + pipeline_response = SimpleNamespace(http_response=_MockXmlHttpResponse(pages[page_index])) page_index += 1 - return cls(mock_pipeline_response, raw, {"Content-Type": "application/vnd.apache.arrow.stream"}) + return cls(pipeline_response, iter([]), {"Content-Type": "application/xml"}) with patch.object( container_client._client.container, # pylint: disable=protected-access "list_blob_hierarchy_segment_apache_arrow", side_effect=fake_list_blob_hierarchy_segment_apache_arrow, ): - blob_pages = container_client.walk_blobs(response_format="arrow", results_per_page=3).by_page() - first_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(first_blobs_list, blob_names[0:3]) - second_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(second_blobs_list, blob_names[3:6]) - third_blobs_list = [blob async for blob in await blob_pages.__anext__()] - self.verify_blobs(third_blobs_list, blob_names[6:9]) + blob_pages = container_client.walk_blobs( + response_format="arrow", results_per_page=2, include=["metadata", "tags"] + ).by_page() + first_page = [blob async for blob in await blob_pages.__anext__()] + second_page = [blob async for blob in await blob_pages.__anext__()] + + assert page_index == 2 + assert [item.name for item in first_page] == ["a/", "d/", "dir/blob1"] + assert not isinstance(first_page[0], BlobProperties) + assert not isinstance(first_page[1], BlobProperties) + assert isinstance(first_page[2], BlobProperties) + self.verify_all_fields(first_page[2]) + assert [item.name for item in second_page] == ["blob3"] From 24cadfb4a7037b3f84fc10cdc0fa2ee3adf3e35e Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 9 Jul 2026 12:18:45 -0400 Subject: [PATCH 15/20] Moved to nanoarrow=0.7.0 --- sdk/storage/azure-storage-blob/dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/dev_requirements.txt b/sdk/storage/azure-storage-blob/dev_requirements.txt index a5edfde94aa1..98bc1487cca0 100644 --- a/sdk/storage/azure-storage-blob/dev_requirements.txt +++ b/sdk/storage/azure-storage-blob/dev_requirements.txt @@ -4,4 +4,4 @@ ../azure-storage-extensions azure-mgmt-storage==20.1.0 aiohttp>=3.13.5 -nanoarrow>=0.8.0 +nanoarrow==0.7.0 From 6dca9fed6a7bb8ae85bf88a5336636ef6e4ea7ad Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 9 Jul 2026 19:29:21 -0400 Subject: [PATCH 16/20] Removed 104 sv --- sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py index 0e2f54d1ed51..8538f9344212 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_serialize.py @@ -66,7 +66,6 @@ "2026-04-06", "2026-06-06", "2026-10-06", - "2026-12-06", ] From 223d9b423676f2a31e859f17f20a4cc1c4051dcc Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 9 Jul 2026 21:26:49 -0400 Subject: [PATCH 17/20] feedback --- .../azure/storage/blob/_list_blobs_helper.py | 23 ++++++++++--------- .../azure-storage-blob/tests/test_arrow.py | 17 ++++++++++++++ .../tests/test_arrow_async.py | 18 +++++++++++++++ 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py index 9851b564dda1..ad793c155d06 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py @@ -51,6 +51,7 @@ def _parse_arrow_response( # pylint: disable=too-many-locals from nanoarrow.ipc import InputStream # pylint: disable=import-outside-toplevel # Declarative mapping: Arrow column name -> (BlobProperties attr, default value). + # Arrow column names mirror the List Blobs XML element names (e.g. "Last-Modified"). # Only scalar fields that map 1-to-1 are listed here; composite sub-objects are # handled separately below. _SCALAR_FIELDS: List[Tuple[str, str, Any]] = [ @@ -60,9 +61,9 @@ def _parse_arrow_response( # pylint: disable=too-many-locals ("IsCurrentVersion", "is_current_version", None), ("Etag", "etag", None), ("Deleted", "deleted", False), - ("LastModified", "last_modified", None), - ("CreationTime", "creation_time", None), - ("ContentLength", "size", None), + ("Last-Modified", "last_modified", None), + ("Creation-Time", "creation_time", None), + ("Content-Length", "size", None), ("ServerEncrypted", "server_encrypted", False), ("EncryptionScope", "encryption_scope", None), ("DeletedTime", "deleted_time", None), @@ -74,8 +75,8 @@ def _parse_arrow_response( # pylint: disable=too-many-locals ("AccessTierInferred", "blob_tier_inferred", None), ("ArchiveStatus", "archive_status", None), ("BlobSequenceNumber", "page_blob_sequence_number", None), - ("IsSealed", "is_append_blob_sealed", None), - ("LastAccessedOn", "last_accessed_on", None), + ("Sealed", "is_append_blob_sealed", None), + ("LastAccessTime", "last_accessed_on", None), ("TagCount", "tag_count", None), ("HasVersionsOnly", "has_versions_only", None), ("LegalHold", "has_legal_hold", None), @@ -84,12 +85,12 @@ def _parse_arrow_response( # pylint: disable=too-many-locals # Sub-object field mappings: Arrow column name -> constructor kwarg name. _CONTENT_SETTINGS_FIELDS = { - "ContentType": "content_type", - "ContentEncoding": "content_encoding", - "ContentLanguage": "content_language", - "ContentMD5": "content_md5", - "ContentDisposition": "content_disposition", - "CacheControl": "cache_control", + "Content-Type": "content_type", + "Content-Encoding": "content_encoding", + "Content-Language": "content_language", + "Content-MD5": "content_md5", + "Content-Disposition": "content_disposition", + "Cache-Control": "cache_control", } _LEASE_FIELDS = { "LeaseStatus": "x-ms-lease-status", diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow.py b/sdk/storage/azure-storage-blob/tests/test_arrow.py index 074d25b27746..f4ebf193d3fd 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow.py @@ -141,6 +141,23 @@ def verify_blobs(self, blobs_list: list[BlobProperties], blob_names: list[str]): all_names = {blob.name for blob in blobs_list} for blob_name in blob_names: assert blob_name in all_names + for blob in blobs_list: + # BlobPrefix (virtual directory) entries from walk_blobs carry only a name. + if not isinstance(blob, BlobProperties): + continue + assert blob.blob_type == BlobType.BLOCKBLOB + assert blob.size == len(TEST_DATA) + assert blob.etag is not None + assert blob.last_modified is not None + assert blob.creation_time is not None + assert blob.last_accessed_on is not None + assert blob.server_encrypted is True + assert blob.blob_tier is not None + assert blob.blob_tier_inferred is not None + assert blob.lease.state == "available" + assert blob.lease.status == "unlocked" + assert blob.content_settings.content_type == "application/octet-stream" + assert blob.content_settings.content_md5 is not None def verify_all_fields(self, blob: BlobProperties): # Verifies the properties produced by _rich_blob_xml were fully deserialized. diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py index 1a28ca3664da..3ef953603b98 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py @@ -144,6 +144,24 @@ def verify_blobs(self, blobs_list: list[BlobProperties], blob_names: list[str]): all_names = {blob.name for blob in blobs_list} for blob_name in blob_names: assert blob_name in all_names + for blob in blobs_list: + # BlobPrefix (virtual directory) entries from walk_blobs carry only a name. + if not isinstance(blob, BlobProperties): + continue + # Validate that many fields (not just the name) survive Arrow deserialization. + assert blob.blob_type == BlobType.BLOCKBLOB + assert blob.size == len(TEST_DATA) + assert blob.etag is not None + assert blob.last_modified is not None + assert blob.creation_time is not None + assert blob.last_accessed_on is not None + assert blob.server_encrypted is True + assert blob.blob_tier is not None + assert blob.blob_tier_inferred is not None + assert blob.lease.state == "available" + assert blob.lease.status == "unlocked" + assert blob.content_settings.content_type == "application/octet-stream" + assert blob.content_settings.content_md5 is not None def verify_all_fields(self, blob: BlobProperties): # Verifies the properties produced by _rich_blob_xml were fully deserialized. From 9ac199ed3bae779dea3a975fc5bdfd01ee48f8b5 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 9 Jul 2026 21:29:05 -0400 Subject: [PATCH 18/20] Test failures --- sdk/storage/azure-storage-blob/tests/test_arrow.py | 4 ++-- sdk/storage/azure-storage-blob/tests/test_arrow_async.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow.py b/sdk/storage/azure-storage-blob/tests/test_arrow.py index f4ebf193d3fd..d1d222918f20 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow.py @@ -142,8 +142,8 @@ def verify_blobs(self, blobs_list: list[BlobProperties], blob_names: list[str]): for blob_name in blob_names: assert blob_name in all_names for blob in blobs_list: - # BlobPrefix (virtual directory) entries from walk_blobs carry only a name. - if not isinstance(blob, BlobProperties): + # Skip virtual-directory entries from walk_blobs; they carry only a name. + if not isinstance(blob, BlobProperties) or blob.name.endswith("/"): continue assert blob.blob_type == BlobType.BLOCKBLOB assert blob.size == len(TEST_DATA) diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py index 3ef953603b98..f28a60fd3421 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py @@ -145,10 +145,9 @@ def verify_blobs(self, blobs_list: list[BlobProperties], blob_names: list[str]): for blob_name in blob_names: assert blob_name in all_names for blob in blobs_list: - # BlobPrefix (virtual directory) entries from walk_blobs carry only a name. - if not isinstance(blob, BlobProperties): + # Skip virtual-directory entries from walk_blobs; they carry only a name. + if not isinstance(blob, BlobProperties) or blob.name.endswith("/"): continue - # Validate that many fields (not just the name) survive Arrow deserialization. assert blob.blob_type == BlobType.BLOCKBLOB assert blob.size == len(TEST_DATA) assert blob.etag is not None From 17a5119c8e71511b6c630a6edee3581d11055e4d Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 9 Jul 2026 22:16:37 -0400 Subject: [PATCH 19/20] Dependency --- sdk/storage/azure-storage-blob/dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/dev_requirements.txt b/sdk/storage/azure-storage-blob/dev_requirements.txt index 98bc1487cca0..674fd493412d 100644 --- a/sdk/storage/azure-storage-blob/dev_requirements.txt +++ b/sdk/storage/azure-storage-blob/dev_requirements.txt @@ -4,4 +4,4 @@ ../azure-storage-extensions azure-mgmt-storage==20.1.0 aiohttp>=3.13.5 -nanoarrow==0.7.0 +nanoarrow>=0.7.0 From fbd546c0b85794ff7462ecff8345116c46473974 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Fri, 10 Jul 2026 14:30:06 -0400 Subject: [PATCH 20/20] Feedback --- sdk/storage/azure-storage-blob/assets.json | 2 +- .../azure/storage/blob/_list_blobs_helper.py | 26 ++++++++++++++++--- .../azure-storage-blob/tests/test_arrow.py | 20 ++++++++++++++ .../tests/test_arrow_async.py | 22 ++++++++++++++++ 4 files changed, 66 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-blob/assets.json b/sdk/storage/azure-storage-blob/assets.json index add35578f3d2..ef20b4b146cc 100644 --- a/sdk/storage/azure-storage-blob/assets.json +++ b/sdk/storage/azure-storage-blob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/storage/azure-storage-blob", - "Tag": "python/storage/azure-storage-blob_8fceda8e9d" + "Tag": "python/storage/azure-storage-blob_42c2d90038" } diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py index ad793c155d06..076fb0af66d6 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_list_blobs_helper.py @@ -35,7 +35,7 @@ _ARROW_CONTENT_TYPE = "application/vnd.apache.arrow.stream" -def _parse_arrow_response( # pylint: disable=too-many-locals +def _parse_arrow_response( # pylint: disable=too-many-locals,too-many-statements raw_bytes: Union[bytes, bytearray], container: Optional[str] ) -> Tuple[Optional[str], List[BlobProperties]]: """ @@ -130,8 +130,19 @@ def _parse_arrow_response( # pylint: disable=too-many-locals num_rows = len(batch) if num_rows == 0: continue - # Pre-resolve columns once per batch; missing columns stay as None. - cols = {field_names[i]: batch.child(i).to_pylist() for i in range(batch.n_children)} + cols = {} + for i in range(batch.n_children): + child = batch.child(i) + try: + cols[field_names[i]] = child.to_pylist() + except KeyError: + offsets = list(child.buffer(1)) + entries = child.child(0) + keys = entries.child(0).to_pylist() + values = entries.child(1).to_pylist() + cols[field_names[i]] = [ + {keys[j]: values[j] for j in range(offsets[r], offsets[r + 1])} for r in range(num_rows) + ] for row in range(num_rows): @@ -166,6 +177,15 @@ def _get( expiry_time=_get("ImmutabilityPolicyUntilDate"), policy_mode=_get("ImmutabilityPolicyMode"), ) + + # Metadata and tags are returned as their own map-typed columns when the + # caller opts in via ``include``. They are absent otherwise. + metadata_val = _get("Metadata") + if isinstance(metadata_val, dict): + blob.metadata = metadata_val # type: ignore[assignment] + tags_val = _get("Tags", _get("BlobTags")) + if isinstance(tags_val, dict): + blob.tags = tags_val blob_items.append(blob) return next_marker, blob_items diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow.py b/sdk/storage/azure-storage-blob/tests/test_arrow.py index d1d222918f20..719b78f1bbd4 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow.py @@ -208,6 +208,26 @@ def test_arrow_list_multiple_blobs(self, **kwargs): blobs_list = list(container.list_blobs(response_format="arrow")) self.verify_blobs(blobs_list, blob_names) + @BlobPreparer() + @recorded_by_proxy + def test_arrow_list_blobs_with_metadata_and_tags(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + metadata = {"color": "blue", "size": "large"} + tags = {"tag1": "firsttag", "tag2": "secondtag"} + blob_client = self.bsc.get_blob_client(self.container_name, "blob1") + blob_client.upload_blob(TEST_DATA, overwrite=True, metadata=metadata, tags=tags) + + container = self.bsc.get_container_client(self.container_name) + blobs_list = list(container.list_blobs(response_format="arrow", include=["metadata", "tags"])) + + self.verify_blobs(blobs_list, ["blob1"]) + blob = blobs_list[0] + assert blob.metadata == metadata + assert blob.tags == tags + @BlobPreparer() @recorded_by_proxy def test_arrow_list_blobs_paging(self, **kwargs): diff --git a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py index f28a60fd3421..5f8ba0d17820 100644 --- a/sdk/storage/azure-storage-blob/tests/test_arrow_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_arrow_async.py @@ -211,6 +211,28 @@ async def test_arrow_list_multiple_blobs(self, **kwargs): blobs_list = [blob async for blob in container.list_blobs(response_format="arrow")] self.verify_blobs(blobs_list, blob_names) + @BlobPreparer() + @recorded_by_proxy_async + async def test_arrow_list_blobs_with_metadata_and_tags(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + metadata = {"color": "blue", "size": "large"} + tags = {"tag1": "firsttag", "tag2": "secondtag"} + blob_client = self.bsc.get_blob_client(self.container_name, "blob1") + await blob_client.upload_blob(TEST_DATA, overwrite=True, metadata=metadata, tags=tags) + + container = self.bsc.get_container_client(self.container_name) + blobs_list = [ + blob async for blob in container.list_blobs(response_format="arrow", include=["metadata", "tags"]) + ] + + self.verify_blobs(blobs_list, ["blob1"]) + blob = blobs_list[0] + assert blob.metadata == metadata + assert blob.tags == tags + @BlobPreparer() @recorded_by_proxy_async async def test_arrow_list_blobs_paging(self, **kwargs):