Skip to content

Commit 23fc381

Browse files
authoredJun 20, 2022
Fix some issues with automatic gzip decompression (#861)
This plugs a buffer stream in between the decompressor stream and the "user" stream. This make sure that (i) the correct number of bytes is read from the http stream and thus decoded (fixes #859) and (ii) that we can read the http stream in chunks instead of byte-by-byte (the previous code even warns about this usage). Fixes #859.
1 parent 80b6816 commit 23fc381

File tree

2 files changed

+39
-8
lines changed

2 files changed

+39
-8
lines changed
 

‎Project.toml

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name = "HTTP"
22
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
33
authors = ["Jacob Quinn", "contributors: https://github.com/JuliaWeb/HTTP.jl/graphs/contributors"]
4-
version = "1.0.1"
4+
version = "1.0.2"
55

66
[deps]
77
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -13,6 +13,7 @@ LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
1313
MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d"
1414
NetworkOptions = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
1515
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
16+
SimpleBufferStream = "777ac1f9-54b0-4bf8-805c-2214025038e7"
1617
Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
1718
URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
1819
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
@@ -22,6 +23,7 @@ CodecZlib = "0.7"
2223
IniFile = "0.5"
2324
LoggingExtras = "0.4.9"
2425
MbedTLS = "0.6.8, 0.7, 1"
26+
SimpleBufferStream = "1.1"
2527
URIs = "1.3"
2628
julia = "1.6"
2729

‎src/clientlayers/StreamRequest.jl

+36-7
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ module StreamRequest
22

33
using ..IOExtras, ..Messages, ..Streams, ..ConnectionPool, ..Strings, ..RedirectRequest, ..Exceptions
44
using LoggingExtras, CodecZlib, URIs
5+
using SimpleBufferStream: BufferStream
56

67
export streamlayer
78

@@ -103,13 +104,41 @@ writechunk(stream, body::Union{Dict, NamedTuple}) = writebodystream(stream, body
103104
writechunk(stream, body) = write(stream, body)
104105

105106
function readbody(stream::Stream, res::Response, decompress)
106-
readstream = decompress && header(res, "Content-Encoding") == "gzip" ? GzipDecompressorStream(stream) : stream
107-
if isbytes(res.body)
108-
res.body = read(readstream)
109-
elseif !isredirect(stream) && !retryable(stream)
110-
# if the request/response pair are going to be redirected or retried,
111-
# we want to avoid "contaminating" our response body stream
112-
write(res.body, readstream)
107+
# Bail early if we are not going to read anything.
108+
# If the request/response pair are going to be redirected or retried,
109+
# we want to avoid "contaminating" our response body stream.
110+
willread = isbytes(res.body) || (!isredirect(stream) && !retryable(stream))
111+
willread || return
112+
113+
if decompress && header(res, "Content-Encoding") == "gzip"
114+
# Plug in a buffer stream in between so that we can (i) read the http stream in
115+
# chunks instead of byte-by-byte and (ii) make sure to stop reading the http stream
116+
# at eof.
117+
buf = BufferStream()
118+
gzstream = GzipDecompressorStream(buf)
119+
tsk = @async begin
120+
try
121+
write(gzstream, stream)
122+
finally
123+
# Close here to (i) deallocate resources in zlib and (ii) make sure that
124+
# read(buf)/write(..., buf) below don't block forever. Note that this will
125+
# close the stream wrapped by the decompressor (buf) but *not* the http
126+
# stream, which should be left open.
127+
close(gzstream)
128+
end
129+
end
130+
if isbytes(res.body)
131+
res.body = read(buf)
132+
else
133+
write(res.body, buf)
134+
end
135+
wait(tsk)
136+
else
137+
if isbytes(res.body)
138+
res.body = read(stream)
139+
else
140+
write(res.body, stream)
141+
end
113142
end
114143
end
115144

2 commit comments

Comments
 (2)

quinnj commented on Jun 20, 2022

@quinnj
Member

JuliaRegistrator commented on Jun 20, 2022

@JuliaRegistrator

Registration pull request created: JuliaRegistries/General/62724

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v1.0.2 -m "<description of version>" 23fc3818e8330c2b508dfd2fbce554d99c84f18d
git push origin v1.0.2
Please sign in to comment.