mirror of
https://github.com/NixOS/nixpkgs.git
synced 2025-01-29 16:24:10 +00:00
1ffbe2530e
Fixes errors observed with compression using curl 8.7.1 by backporting the upstream fix. See linked issues for more information.
169 lines
6.8 KiB
Diff
169 lines
6.8 KiB
Diff
From b30d694a027eb771c02a3db0dee0ca03ccab7377 Mon Sep 17 00:00:00 2001
|
|
From: Stefan Eissing <stefan@eissing.org>
|
|
Date: Thu, 28 Mar 2024 11:08:15 +0100
|
|
Subject: [PATCH] content_encoding: brotli and others, pass through 0-length
|
|
writes
|
|
|
|
- curl's transfer handling may write 0-length chunks at the end of the
|
|
download with an EOS flag. (HTTP/2 does this commonly)
|
|
|
|
- content encoders need to pass-through such a write and not count this
|
|
as error in case they are finished decoding
|
|
|
|
Fixes #13209
|
|
Fixes #13212
|
|
Closes #13219
|
|
---
|
|
lib/content_encoding.c | 10 +++++-----
|
|
tests/http/test_02_download.py | 13 +++++++++++++
|
|
tests/http/testenv/env.py | 7 ++++++-
|
|
tests/http/testenv/httpd.py | 20 ++++++++++++++++++++
|
|
4 files changed, 44 insertions(+), 6 deletions(-)
|
|
|
|
diff --git a/lib/content_encoding.c b/lib/content_encoding.c
|
|
index c1abf24e8c027c..8e926dd2ecd5ad 100644
|
|
--- a/lib/content_encoding.c
|
|
+++ b/lib/content_encoding.c
|
|
@@ -300,7 +300,7 @@ static CURLcode deflate_do_write(struct Curl_easy *data,
|
|
struct zlib_writer *zp = (struct zlib_writer *) writer;
|
|
z_stream *z = &zp->z; /* zlib state structure */
|
|
|
|
- if(!(type & CLIENTWRITE_BODY))
|
|
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
|
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
|
|
|
/* Set the compressed input when this function is called */
|
|
@@ -457,7 +457,7 @@ static CURLcode gzip_do_write(struct Curl_easy *data,
|
|
struct zlib_writer *zp = (struct zlib_writer *) writer;
|
|
z_stream *z = &zp->z; /* zlib state structure */
|
|
|
|
- if(!(type & CLIENTWRITE_BODY))
|
|
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
|
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
|
|
|
if(zp->zlib_init == ZLIB_INIT_GZIP) {
|
|
@@ -669,7 +669,7 @@ static CURLcode brotli_do_write(struct Curl_easy *data,
|
|
CURLcode result = CURLE_OK;
|
|
BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
|
|
|
|
- if(!(type & CLIENTWRITE_BODY))
|
|
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
|
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
|
|
|
if(!bp->br)
|
|
@@ -762,7 +762,7 @@ static CURLcode zstd_do_write(struct Curl_easy *data,
|
|
ZSTD_outBuffer out;
|
|
size_t errorCode;
|
|
|
|
- if(!(type & CLIENTWRITE_BODY))
|
|
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
|
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
|
|
|
if(!zp->decomp) {
|
|
@@ -916,7 +916,7 @@ static CURLcode error_do_write(struct Curl_easy *data,
|
|
(void) buf;
|
|
(void) nbytes;
|
|
|
|
- if(!(type & CLIENTWRITE_BODY))
|
|
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
|
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
|
|
|
failf(data, "Unrecognized content encoding type. "
|
|
diff --git a/tests/http/test_02_download.py b/tests/http/test_02_download.py
|
|
index 4db9c9d36e9ed5..395fc862f2f839 100644
|
|
--- a/tests/http/test_02_download.py
|
|
+++ b/tests/http/test_02_download.py
|
|
@@ -394,6 +394,19 @@ def test_02_27_paused_no_cl(self, env: Env, httpd, nghttpx, repeat):
|
|
r = client.run(args=[url])
|
|
r.check_exit_code(0)
|
|
|
|
+ @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3'])
|
|
+ def test_02_28_get_compressed(self, env: Env, httpd, nghttpx, repeat, proto):
|
|
+ if proto == 'h3' and not env.have_h3():
|
|
+ pytest.skip("h3 not supported")
|
|
+ count = 1
|
|
+ urln = f'https://{env.authority_for(env.domain1brotli, proto)}/data-100k?[0-{count-1}]'
|
|
+ curl = CurlClient(env=env)
|
|
+ r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
|
|
+ '--compressed'
|
|
+ ])
|
|
+ r.check_exit_code(code=0)
|
|
+ r.check_response(count=count, http_status=200)
|
|
+
|
|
def check_downloads(self, client, srcfile: str, count: int,
|
|
complete: bool = True):
|
|
for i in range(count):
|
|
diff --git a/tests/http/testenv/env.py b/tests/http/testenv/env.py
|
|
index a207059dcd57c5..13c5d6bd46ee57 100644
|
|
--- a/tests/http/testenv/env.py
|
|
+++ b/tests/http/testenv/env.py
|
|
@@ -129,10 +129,11 @@ def __init__(self):
|
|
self.htdocs_dir = os.path.join(self.gen_dir, 'htdocs')
|
|
self.tld = 'http.curl.se'
|
|
self.domain1 = f"one.{self.tld}"
|
|
+ self.domain1brotli = f"brotli.one.{self.tld}"
|
|
self.domain2 = f"two.{self.tld}"
|
|
self.proxy_domain = f"proxy.{self.tld}"
|
|
self.cert_specs = [
|
|
- CertificateSpec(domains=[self.domain1, 'localhost'], key_type='rsa2048'),
|
|
+ CertificateSpec(domains=[self.domain1, self.domain1brotli, 'localhost'], key_type='rsa2048'),
|
|
CertificateSpec(domains=[self.domain2], key_type='rsa2048'),
|
|
CertificateSpec(domains=[self.proxy_domain, '127.0.0.1'], key_type='rsa2048'),
|
|
CertificateSpec(name="clientsX", sub_specs=[
|
|
@@ -376,6 +377,10 @@ def htdocs_dir(self) -> str:
|
|
def domain1(self) -> str:
|
|
return self.CONFIG.domain1
|
|
|
|
+ @property
|
|
+ def domain1brotli(self) -> str:
|
|
+ return self.CONFIG.domain1brotli
|
|
+
|
|
@property
|
|
def domain2(self) -> str:
|
|
return self.CONFIG.domain2
|
|
diff --git a/tests/http/testenv/httpd.py b/tests/http/testenv/httpd.py
|
|
index c04c22699a62c4..b8615875a9a558 100644
|
|
--- a/tests/http/testenv/httpd.py
|
|
+++ b/tests/http/testenv/httpd.py
|
|
@@ -50,6 +50,7 @@ class Httpd:
|
|
'alias', 'env', 'filter', 'headers', 'mime', 'setenvif',
|
|
'socache_shmcb',
|
|
'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect',
|
|
+ 'brotli',
|
|
'mpm_event',
|
|
]
|
|
COMMON_MODULES_DIRS = [
|
|
@@ -203,6 +204,7 @@ def _mkpath(self, path):
|
|
|
|
def _write_config(self):
|
|
domain1 = self.env.domain1
|
|
+ domain1brotli = self.env.domain1brotli
|
|
creds1 = self.env.get_credentials(domain1)
|
|
domain2 = self.env.domain2
|
|
creds2 = self.env.get_credentials(domain2)
|
|
@@ -285,6 +287,24 @@ def _write_config(self):
|
|
f'</VirtualHost>',
|
|
f'',
|
|
])
|
|
+ # Alternate to domain1 with BROTLI compression
|
|
+ conf.extend([ # https host for domain1, h1 + h2
|
|
+ f'<VirtualHost *:{self.env.https_port}>',
|
|
+ f' ServerName {domain1brotli}',
|
|
+ f' Protocols h2 http/1.1',
|
|
+ f' SSLEngine on',
|
|
+ f' SSLCertificateFile {creds1.cert_file}',
|
|
+ f' SSLCertificateKeyFile {creds1.pkey_file}',
|
|
+ f' DocumentRoot "{self._docs_dir}"',
|
|
+ f' SetOutputFilter BROTLI_COMPRESS',
|
|
+ ])
|
|
+ conf.extend(self._curltest_conf(domain1))
|
|
+ if domain1 in self._extra_configs:
|
|
+ conf.extend(self._extra_configs[domain1])
|
|
+ conf.extend([
|
|
+ f'</VirtualHost>',
|
|
+ f'',
|
|
+ ])
|
|
conf.extend([ # https host for domain2, no h2
|
|
f'<VirtualHost *:{self.env.https_port}>',
|
|
f' ServerName {domain2}',
|