Skip to content

Commit 127aa55

Browse files
authored
compression: add brotli compressor and decompressor (envoyproxy#12998)
Commit Message: compression: add brotli compressor and decompressor Additional Description: Add new brotli compression extensions in addition to gzip. Risk Level: Low, no existing functionality is touched Testing: uni tests, manual tests with curl. Docs Changes: updated docs for compression and decompression HTTP filters to refer the new available encoder/decoder. Release Notes: updated current.rst Fixes envoyproxy#4429 The PR adds a new dependency on https://github.com/google/brotli. Here's the current criteria answers: | Criteria | Answer | |---------|---------| | Cloud Native Computing Foundation (CNCF) approved license | MIT | | Dependencies must not substantially increase the binary size unless they are optional | brotli's binary size built with `-c opt` is 752K | | No duplication of existing dependencies | no other dep provides Brotli | | Hosted on a git repository and the archive fetch must directly reference this repository. | https://github.com/google/brotli | | CVE history appears reasonable, no pathological CVE arcs | so far 4 CVEs related to brotli have been registered | | Code review (ideally PRs) before merge | PRs are reviewed before merge | | Security vulnerability process exists, with contact details and reporting/disclosure process | no policy exists, submitted google/brotli#878 | | > 1 contributor responsible for a non-trivial number of commits | 75 contributors | | Tests run in CI | CI set up with AppVeyor and Github actions | | High test coverage (also static/dynamic analysis, fuzzing) | Fuzzers are run in CI | | Envoy can obtain advanced notification of vulnerabilities or of security releases | brotli is registered in CPE | | Do other significant projects have shared fate by using this dependency? | Google Chrome is using the library | | Releases (with release notes) | https://github.com/google/brotli/releases | | Commits/releases in last 90 days | last commit 9 days ago | Signed-off-by: Dmitry Rozhkov <[email protected]>
1 parent d1dd30f commit 127aa55

File tree

38 files changed

+1397
-5
lines changed

38 files changed

+1397
-5
lines changed

CODEOWNERS

+1
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@ extensions/filters/common/original_src @snowp @klarose
147147
# Compression
148148
/*/extensions/compression/common @junr03 @rojkov
149149
/*/extensions/compression/gzip @junr03 @rojkov
150+
/*/extensions/compression/brotli @junr03 @rojkov
150151
/*/extensions/filters/http/decompressor @rojkov @dio
151152
# Watchdog Extensions
152153
/*/extensions/watchdog/profile_action @kbaichoo @antoniovicente

api/BUILD

+2
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,8 @@ proto_library(
165165
"//envoy/extensions/common/matching/v3:pkg",
166166
"//envoy/extensions/common/ratelimit/v3:pkg",
167167
"//envoy/extensions/common/tap/v3:pkg",
168+
"//envoy/extensions/compression/brotli/compressor/v3:pkg",
169+
"//envoy/extensions/compression/brotli/decompressor/v3:pkg",
168170
"//envoy/extensions/compression/gzip/compressor/v3:pkg",
169171
"//envoy/extensions/compression/gzip/decompressor/v3:pkg",
170172
"//envoy/extensions/filters/common/dependency/v3:pkg",
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
2+
3+
load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
4+
5+
licenses(["notice"]) # Apache 2
6+
7+
api_proto_package(
8+
deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
9+
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
syntax = "proto3";
2+
3+
package envoy.extensions.compression.brotli.compressor.v3;
4+
5+
import "google/protobuf/wrappers.proto";
6+
7+
import "udpa/annotations/status.proto";
8+
import "validate/validate.proto";
9+
10+
option java_package = "io.envoyproxy.envoy.extensions.compression.brotli.compressor.v3";
11+
option java_outer_classname = "BrotliProto";
12+
option java_multiple_files = true;
13+
option (udpa.annotations.file_status).package_version_status = ACTIVE;
14+
15+
// [#protodoc-title: Brotli Compressor]
16+
// [#extension: envoy.compression.brotli.compressor]
17+
18+
// [#next-free-field: 7]
19+
message Brotli {
20+
enum EncoderMode {
21+
DEFAULT = 0;
22+
GENERIC = 1;
23+
TEXT = 2;
24+
FONT = 3;
25+
}
26+
27+
// Value from 0 to 11 that controls the main compression speed-density lever.
28+
// The higher quality, the slower compression. The default value is 3.
29+
google.protobuf.UInt32Value quality = 1 [(validate.rules).uint32 = {lte: 11}];
30+
31+
// A value used to tune encoder for specific input. For more information about modes,
32+
// please refer to brotli manual: https://brotli.org/encode.html#aa6f
33+
// This field will be set to "DEFAULT" if not specified.
34+
EncoderMode encoder_mode = 2 [(validate.rules).enum = {defined_only: true}];
35+
36+
// Value from 10 to 24 that represents the base two logarithmic of the compressor's window size.
37+
// Larger window results in better compression at the expense of memory usage. The default is 18.
38+
// For more details about this parameter, please refer to brotli manual:
39+
// https://brotli.org/encode.html#a9a8
40+
google.protobuf.UInt32Value window_bits = 3 [(validate.rules).uint32 = {lte: 24 gte: 10}];
41+
42+
// Value from 16 to 24 that represents the base two logarithmic of the compressor's input block
43+
// size. Larger input block results in better compression at the expense of memory usage. The
44+
// default is 24. For more details about this parameter, please refer to brotli manual:
45+
// https://brotli.org/encode.html#a9a8
46+
google.protobuf.UInt32Value input_block_bits = 4 [(validate.rules).uint32 = {lte: 24 gte: 16}];
47+
48+
// Value for compressor's next output buffer. If not set, defaults to 4096.
49+
google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}];
50+
51+
// If true, disables "literal context modeling" format feature.
52+
// This flag is a "decoding-speed vs compression ratio" trade-off.
53+
bool disable_literal_context_modeling = 6;
54+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
2+
3+
load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
4+
5+
licenses(["notice"]) # Apache 2
6+
7+
api_proto_package(
8+
deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
9+
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
syntax = "proto3";
2+
3+
package envoy.extensions.compression.brotli.decompressor.v3;
4+
5+
import "google/protobuf/wrappers.proto";
6+
7+
import "udpa/annotations/status.proto";
8+
import "validate/validate.proto";
9+
10+
option java_package = "io.envoyproxy.envoy.extensions.compression.brotli.decompressor.v3";
11+
option java_outer_classname = "BrotliProto";
12+
option java_multiple_files = true;
13+
option (udpa.annotations.file_status).package_version_status = ACTIVE;
14+
15+
// [#protodoc-title: Brotli Decompressor]
16+
// [#extension: envoy.compression.brotli.decompressor]
17+
18+
message Brotli {
19+
// If true, disables "canny" ring buffer allocation strategy.
20+
// Ring buffer is allocated according to window size, despite the real size of the content.
21+
bool disable_ring_buffer_reallocation = 1;
22+
23+
// Value for decompressor's next output buffer. If not set, defaults to 4096.
24+
google.protobuf.UInt32Value chunk_size = 2 [(validate.rules).uint32 = {lte: 65536 gte: 4096}];
25+
}

api/versioning/BUILD

+2
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ proto_library(
4848
"//envoy/extensions/common/matching/v3:pkg",
4949
"//envoy/extensions/common/ratelimit/v3:pkg",
5050
"//envoy/extensions/common/tap/v3:pkg",
51+
"//envoy/extensions/compression/brotli/compressor/v3:pkg",
52+
"//envoy/extensions/compression/brotli/decompressor/v3:pkg",
5153
"//envoy/extensions/compression/gzip/compressor/v3:pkg",
5254
"//envoy/extensions/compression/gzip/decompressor/v3:pkg",
5355
"//envoy/extensions/filters/common/dependency/v3:pkg",

bazel/repositories.bzl

+14
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@ def envoy_dependencies(skip_targets = []):
158158
_io_opentracing_cpp()
159159
_net_zlib()
160160
_com_github_zlib_ng_zlib_ng()
161+
_org_brotli()
161162
_upb()
162163
_proxy_wasm_cpp_sdk()
163164
_proxy_wasm_cpp_host()
@@ -352,6 +353,19 @@ def _com_github_zlib_ng_zlib_ng():
352353
patches = ["@envoy//bazel/foreign_cc:zlib_ng.patch"],
353354
)
354355

356+
def _org_brotli():
357+
external_http_archive(
358+
name = "org_brotli",
359+
)
360+
native.bind(
361+
name = "brotlienc",
362+
actual = "@org_brotli//:brotlienc",
363+
)
364+
native.bind(
365+
name = "brotlidec",
366+
actual = "@org_brotli//:brotlidec",
367+
)
368+
355369
def _com_google_cel_cpp():
356370
external_http_archive("com_google_cel_cpp")
357371
external_http_archive("rules_antlr")

bazel/repository_locations.bzl

+18
Original file line numberDiff line numberDiff line change
@@ -406,6 +406,24 @@ REPOSITORY_LOCATIONS_SPEC = dict(
406406
release_date = "2019-04-14",
407407
cpe = "cpe:2.3:a:gnu:zlib:*",
408408
),
409+
org_brotli = dict(
410+
project_name = "brotli",
411+
project_desc = "brotli compression library",
412+
project_url = "https://brotli.org",
413+
# Use the dev branch of brotli to resolve compilation issues.
414+
# TODO(rojkov): Remove when brotli > 1.0.9 is released.
415+
version = "0cd2e3926e95e7e2930f57ae3f4885508d462a25",
416+
sha256 = "93810780e60304b51f2c9645fe313a6e4640711063ed0b860cfa60999dd256c5",
417+
strip_prefix = "brotli-{version}",
418+
urls = ["https://github.com/google/brotli/archive/{version}.tar.gz"],
419+
use_category = ["dataplane_ext"],
420+
extensions = [
421+
"envoy.compression.brotli.compressor",
422+
"envoy.compression.brotli.decompressor",
423+
],
424+
release_date = "2020-09-08",
425+
cpe = "cpe:2.3:a:google:brotli:*",
426+
),
409427
com_github_zlib_ng_zlib_ng = dict(
410428
project_name = "zlib-ng",
411429
project_desc = "zlib fork (higher performance)",

docs/root/api-v3/config/compression/compression.rst

+1
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ Compression
66
:maxdepth: 2
77

88
../../extensions/compression/gzip/*/v3/*
9+
../../extensions/compression/brotli/*/v3/*

docs/root/configuration/http/http_filters/compressor_filter.rst

+3-2
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@ determine whether or not the content should be compressed. The content is
2323
compressed and then sent to the client with the appropriate headers, if
2424
response and request allow.
2525

26-
Currently the filter supports :ref:`gzip compression <envoy_v3_api_msg_extensions.compression.gzip.compressor.v3.Gzip>`
27-
only. Other compression libraries can be supported as extensions.
26+
Currently the filter supports :ref:`gzip <envoy_v3_api_msg_extensions.compression.gzip.compressor.v3.Gzip>`
27+
and :ref:`brotli <envoy_v3_api_msg_extensions.compression.brotli.compressor.v3.Brotli>`
28+
compression only. Other compression libraries can be supported as extensions.
2829

2930
An example configuration of the filter may look like the following:
3031

docs/root/configuration/http/http_filters/decompressor_filter.rst

+3-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,9 @@ determine whether or not the content should be decompressed. The content is
1616
decompressed and passed on to the rest of the filter chain. Note that decompression happens
1717
independently for request and responses based on the rules described below.
1818

19-
Currently the filter supports :ref:`gzip compression <envoy_v3_api_msg_extensions.compression.gzip.decompressor.v3.Gzip>`
20-
only. Other compression libraries can be supported as extensions.
19+
Currently the filter supports :ref:`gzip <envoy_v3_api_msg_extensions.compression.gzip.decompressor.v3.Gzip>`
20+
and :ref:`brotli <envoy_v3_api_msg_extensions.compression.brotli.decompressor.v3.Brotli>`
21+
compression only. Other compression libraries can be supported as extensions.
2122

2223
An example configuration of the filter may look like the following:
2324

docs/root/intro/arch_overview/other_features/compression/libraries.rst

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ Compression Libraries
66
Underlying implementation
77
-------------------------
88

9-
Currently Envoy uses `zlib <http://zlib.net>`_ as a compression library.
9+
Currently Envoy uses `zlib <http://zlib.net>`_ and `brotli <https://brotli.org>`_ as compression
10+
libraries.
1011

1112
.. note::
1213

docs/root/version_history/current.rst

+1
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ New Features
7373
------------
7474
* access log: added the :ref:`formatters <envoy_v3_api_field_config.core.v3.SubstitutionFormatString.formatters>` extension point for custom formatters (command operators).
7575
* access log: support command operator: %REQUEST_HEADERS_BYTES%, %RESPONSE_HEADERS_BYTES%, and %RESPONSE_TRAILERS_BYTES%.
76+
* compression: add brotli :ref:`compressor <envoy_v3_api_msg_extensions.compression.brotli.compressor.v3.Brotli>` and :ref:`decompressor <envoy_v3_api_msg_extensions.compression.brotli.decompressor.v3.Brotli>`.
7677
* config: add `envoy.features.fail_on_any_deprecated_feature` runtime key, which matches the behaviour of compile-time flag `ENVOY_DISABLE_DEPRECATED_FEATURES`, i.e. use of deprecated fields will cause a crash.
7778
* dispatcher: supports a stack of `Envoy::ScopeTrackedObject` instead of a single tracked object. This will allow Envoy to dump more debug information on crash.
7879
* grpc_json_transcoder: added option :ref:`strict_http_request_validation <envoy_v3_api_field_extensions.filters.http.grpc_json_transcoder.v3.GrpcJsonTranscoder.strict_http_request_validation>` to reject invalid requests early.

generated_api_shadow/envoy/extensions/compression/brotli/compressor/v3/BUILD

+9
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

generated_api_shadow/envoy/extensions/compression/brotli/compressor/v3/brotli.proto

+54
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

generated_api_shadow/envoy/extensions/compression/brotli/decompressor/v3/BUILD

+9
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

generated_api_shadow/envoy/extensions/compression/brotli/decompressor/v3/brotli.proto

+25
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

source/common/http/headers.h

+1
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ class CustomHeaderValues {
9696
} CacheControlValues;
9797

9898
struct {
99+
const std::string Brotli{"br"};
99100
const std::string Gzip{"gzip"};
100101
} ContentEncodingValues;
101102

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
load(
2+
"//bazel:envoy_build_system.bzl",
3+
"envoy_cc_library",
4+
"envoy_extension_package",
5+
)
6+
7+
licenses(["notice"]) # Apache 2
8+
9+
envoy_extension_package()
10+
11+
envoy_cc_library(
12+
name = "brotli_base_lib",
13+
srcs = ["base.cc"],
14+
hdrs = ["base.h"],
15+
deps = [
16+
"//source/common/buffer:buffer_lib",
17+
],
18+
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#include "extensions/compression/brotli/common/base.h"
2+
3+
namespace Envoy {
4+
namespace Extensions {
5+
namespace Compression {
6+
namespace Brotli {
7+
namespace Common {
8+
9+
BrotliContext::BrotliContext(const uint32_t chunk_size)
10+
: chunk_size_{chunk_size}, chunk_ptr_{std::make_unique<uint8_t[]>(chunk_size)}, next_in_{},
11+
next_out_{chunk_ptr_.get()}, avail_in_{0}, avail_out_{chunk_size} {}
12+
13+
void BrotliContext::updateOutput(Buffer::Instance& output_buffer) {
14+
if (avail_out_ == 0) {
15+
output_buffer.add(static_cast<void*>(chunk_ptr_.get()), chunk_size_);
16+
resetOut();
17+
}
18+
}
19+
20+
void BrotliContext::finalizeOutput(Buffer::Instance& output_buffer) {
21+
const size_t n_output = chunk_size_ - avail_out_;
22+
if (n_output > 0) {
23+
output_buffer.add(static_cast<void*>(chunk_ptr_.get()), n_output);
24+
}
25+
}
26+
27+
void BrotliContext::resetOut() {
28+
avail_out_ = chunk_size_;
29+
next_out_ = chunk_ptr_.get();
30+
}
31+
32+
} // namespace Common
33+
} // namespace Brotli
34+
} // namespace Compression
35+
} // namespace Extensions
36+
} // namespace Envoy

0 commit comments

Comments
 (0)