Skip to content

Commit 18ed548

Browse files
pavolloffaypavolloffay
authored
Use bounded size buffers (hypertrace#181)
* Use bounded size buffers Signed-off-by: Pavol Loffay <[email protected]> * Use config Signed-off-by: Pavol Loffay <[email protected]>
1 parent 56d5f1f commit 18ed548

File tree

15 files changed

+199
-9
lines changed

15 files changed

+199
-9
lines changed

‎instrumentation/apache-httpclient-4.0/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/apachehttpclient/v4_0/ApacheClientInstrumentationModule.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.apache.http.HttpMessage;
5252
import org.apache.http.HttpResponse;
5353
import org.hypertrace.agent.config.Config.AgentConfig;
54+
import org.hypertrace.agent.core.BoundedByteArrayOutputStreamFactory;
5455
import org.hypertrace.agent.core.ContentEncodingUtils;
5556
import org.hypertrace.agent.core.ContentLengthUtils;
5657
import org.hypertrace.agent.core.ContentTypeUtils;
@@ -238,7 +239,7 @@ public static void exit(@Advice.This HttpEntity thizz, @Advice.Return InputStrea
238239
SpanAndBuffer spanAndBuffer =
239240
new SpanAndBuffer(
240241
clientSpan,
241-
new ByteArrayOutputStream((int) contentSize),
242+
BoundedByteArrayOutputStreamFactory.create((int) contentSize),
242243
HypertraceSemanticAttributes.HTTP_RESPONSE_BODY,
243244
charset);
244245
GlobalObjectRegistry.inputStreamToSpanAndBufferMap.put(inputStream, spanAndBuffer);
@@ -257,7 +258,8 @@ public static void enter(
257258
if (contentSize <= 0 || contentSize == Long.MAX_VALUE) {
258259
contentSize = ContentLengthUtils.DEFAULT;
259260
}
260-
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream((int) contentSize);
261+
ByteArrayOutputStream byteArrayOutputStream =
262+
BoundedByteArrayOutputStreamFactory.create((int) contentSize);
261263

262264
GlobalObjectRegistry.outputStreamToBufferMap.put(outputStream, byteArrayOutputStream);
263265
}

‎instrumentation/apache-httpclient-4.0/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/apachehttpclient/v4_0/ApacheHttpClientUtils.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.http.HttpEntityEnclosingRequest;
3030
import org.apache.http.HttpMessage;
3131
import org.hypertrace.agent.config.Config.AgentConfig;
32+
import org.hypertrace.agent.core.BoundedByteArrayOutputStreamFactory;
3233
import org.hypertrace.agent.core.ContentEncodingUtils;
3334
import org.hypertrace.agent.core.HypertraceConfig;
3435
import org.hypertrace.agent.core.HypertraceSemanticAttributes;
@@ -81,7 +82,7 @@ public static void traceEntity(Span span, String bodyAttributeKey, HttpEntity en
8182

8283
if (entity.isRepeatable()) {
8384
try {
84-
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
85+
ByteArrayOutputStream byteArrayOutputStream = BoundedByteArrayOutputStreamFactory.create();
8586
entity.writeTo(byteArrayOutputStream);
8687
String encoding =
8788
entity.getContentEncoding() != null ? entity.getContentEncoding().getValue() : "";

‎instrumentation/apache-httpclient-4.0/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/apachehttpclient/v4_0/readall/ApacheClientReadAllInstrumentationModule.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.apache.http.Header;
4949
import org.apache.http.HttpEntity;
5050
import org.apache.http.HttpResponse;
51+
import org.hypertrace.agent.core.BoundedByteArrayOutputStreamFactory;
5152
import org.hypertrace.agent.core.ContentTypeUtils;
5253
import org.hypertrace.agent.core.GlobalObjectRegistry;
5354
import org.hypertrace.agent.core.HypertraceSemanticAttributes;
@@ -195,7 +196,8 @@ public static void exit(@Return Object response) {
195196
}
196197

197198
BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream);
198-
ByteArrayOutputStream buffer = new ByteArrayOutputStream((int) contentSize);
199+
ByteArrayOutputStream buffer =
200+
BoundedByteArrayOutputStreamFactory.create((int) contentSize);
199201
byte ch;
200202
while ((ch = (byte) bufferedInputStream.read()) != -1) {
201203
buffer.write(ch);

‎instrumentation/jaxrs-client-2.0/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/jaxrs/v2_0/JaxrsClientEntityInterceptor.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import javax.ws.rs.ext.WriterInterceptor;
3232
import javax.ws.rs.ext.WriterInterceptorContext;
3333
import org.hypertrace.agent.config.Config.AgentConfig;
34+
import org.hypertrace.agent.core.BoundedByteArrayOutputStreamFactory;
3435
import org.hypertrace.agent.core.ContentEncodingUtils;
3536
import org.hypertrace.agent.core.ContentLengthUtils;
3637
import org.hypertrace.agent.core.ContentTypeUtils;
@@ -78,7 +79,7 @@ public Object aroundReadFrom(ReaderInterceptorContext responseContext)
7879
String contentLengthStr = responseContext.getHeaders().getFirst(HttpHeaders.CONTENT_LENGTH);
7980
int contentLength = ContentLengthUtils.parseLength(contentLengthStr);
8081

81-
ByteArrayOutputStream buffer = new ByteArrayOutputStream(contentLength);
82+
ByteArrayOutputStream buffer = BoundedByteArrayOutputStreamFactory.create(contentLength);
8283
GlobalObjectRegistry.inputStreamToSpanAndBufferMap.put(
8384
entityStream,
8485
new SpanAndBuffer(
@@ -118,7 +119,7 @@ public void aroundWriteTo(WriterInterceptorContext requestContext)
118119
}
119120

120121
// TODO length is not known
121-
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
122+
ByteArrayOutputStream buffer = BoundedByteArrayOutputStreamFactory.create();
122123
OutputStream entityStream = requestContext.getOutputStream();
123124
try {
124125
GlobalObjectRegistry.outputStreamToBufferMap.put(entityStream, buffer);

‎instrumentation/servlet/servlet-common/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/servlet/common/ByteBufferData.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,13 @@
1919
import java.nio.charset.Charset;
2020
import java.nio.charset.StandardCharsets;
2121
import java.util.Arrays;
22+
import org.hypertrace.agent.core.HypertraceConfig;
2223

2324
public class ByteBufferData {
2425

2526
private static final int MIN_BUFFER_SIZE = 128;
26-
private static final int MAX_BUFFER_SIZE = 1048576; // 1MB
27+
private static final int MAX_BUFFER_SIZE =
28+
HypertraceConfig.get().getDataCapture().getBodyMaxSizeBytes().getValue();
2729
private static final int GROW_FACTOR = 4;
2830
private static final Charset ISO_8859_1 = StandardCharsets.ISO_8859_1;
2931
private Charset charset;

‎instrumentation/servlet/servlet-common/src/main/java/io/opentelemetry/javaagent/instrumentation/hypertrace/servlet/common/CharBufferData.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,13 @@
1717
package io.opentelemetry.javaagent.instrumentation.hypertrace.servlet.common;
1818

1919
import java.util.Arrays;
20+
import org.hypertrace.agent.core.HypertraceConfig;
2021

2122
public class CharBufferData {
2223

2324
private static final int MIN_BUFFER_SIZE = 128;
24-
private static final int MAX_BUFFER_SIZE = 1048576; // 1MB
25+
private static final int MAX_BUFFER_SIZE =
26+
HypertraceConfig.get().getDataCapture().getBodyMaxSizeBytes().getValue();
2527
private static final int GROW_FACTOR = 4;
2628
private char[] buffer;
2729
private int bufferLen;

‎javaagent-core/src/main/java/org/hypertrace/agent/core/BoundedByteArrayOutputStream.java

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Copyright The Hypertrace Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.hypertrace.agent.core;
18+
19+
import java.io.ByteArrayOutputStream;
20+
import java.io.IOException;
21+
22+
/**
23+
* {@link ByteArrayOutputStream} with a bounded capacity. Write methods are no-op if the size
24+
* reaches the maximum capacity.
25+
*/
26+
public class BoundedByteArrayOutputStream extends ByteArrayOutputStream {
27+
28+
private final int maxCapacity;
29+
30+
BoundedByteArrayOutputStream(int maxCapacity) {
31+
this.maxCapacity = maxCapacity;
32+
}
33+
34+
BoundedByteArrayOutputStream(int maxCapacity, int size) {
35+
super(size);
36+
this.maxCapacity = maxCapacity;
37+
}
38+
39+
@Override
40+
public synchronized void write(int b) {
41+
if (size() == maxCapacity) {
42+
return;
43+
}
44+
super.write(b);
45+
}
46+
47+
@Override
48+
public void write(byte[] b) throws IOException {
49+
// ByteArrayOutputStream calls write(b, off, len)
50+
super.write(b);
51+
}
52+
53+
@Override
54+
public synchronized void write(byte[] b, int off, int len) {
55+
int size = size();
56+
if (size + len > maxCapacity) {
57+
super.write(b, off, maxCapacity - size);
58+
return;
59+
}
60+
super.write(b, off, len);
61+
}
62+
}

‎javaagent-core/src/main/java/org/hypertrace/agent/core/BoundedByteArrayOutputStreamFactory.java

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Copyright The Hypertrace Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.hypertrace.agent.core;
18+
19+
public class BoundedByteArrayOutputStreamFactory {
20+
21+
private static final int DEFAULT_SIZE = 128;
22+
23+
public static BoundedByteArrayOutputStream create() {
24+
return new BoundedByteArrayOutputStream(DEFAULT_SIZE);
25+
}
26+
27+
public static BoundedByteArrayOutputStream create(int initialSize) {
28+
if (initialSize > DEFAULT_SIZE) {
29+
initialSize = DEFAULT_SIZE;
30+
}
31+
return new BoundedByteArrayOutputStream(DEFAULT_SIZE, initialSize);
32+
}
33+
}

‎javaagent-core/src/main/java/org/hypertrace/agent/core/EnvironmentConfig.java

+6
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ private EnvironmentConfig() {}
4747
static final String OPA_POLL_PERIOD = OPA_PREFIX + "poll.period.seconds";
4848

4949
private static final String CAPTURE_PREFIX = HT_PREFIX + "data.capture.";
50+
public static final String CAPTURE_BODY_MAX_SIZE_BYTES = CAPTURE_PREFIX + "body.max.size.bytes";
5051
public static final String CAPTURE_HTTP_HEADERS_PREFIX = CAPTURE_PREFIX + "http.headers.";
5152
public static final String CAPTURE_HTTP_BODY_PREFIX = CAPTURE_PREFIX + "http.body.";
5253
public static final String CAPTURE_RPC_METADATA_PREFIX = CAPTURE_PREFIX + "rpc.metadata.";
@@ -106,6 +107,11 @@ private static Opa.Builder applyOpa(Opa.Builder builder) {
106107
}
107108

108109
private static DataCapture.Builder setDefaultsToDataCapture(DataCapture.Builder builder) {
110+
String bodyMaxSizeBytes = getProperty(CAPTURE_BODY_MAX_SIZE_BYTES);
111+
if (bodyMaxSizeBytes != null) {
112+
builder.setBodyMaxSizeBytes(
113+
Int32Value.newBuilder().setValue(Integer.valueOf(bodyMaxSizeBytes)).build());
114+
}
109115
builder.setHttpHeaders(
110116
applyMessageDefaults(builder.getHttpHeaders().toBuilder(), CAPTURE_HTTP_HEADERS_PREFIX));
111117
builder.setHttpBody(

‎javaagent-core/src/main/java/org/hypertrace/agent/core/HypertraceConfig.java

+6
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ private HypertraceConfig() {}
5757
static final String DEFAULT_REPORTING_ENDPOINT = "http://localhost:9411/api/v2/spans";
5858
static final String DEFAULT_OPA_ENDPOINT = "http://opa.traceableai:8181/";
5959
static final int DEFAULT_OPA_POLL_PERIOD_SECONDS = 30;
60+
// 128 KiB
61+
static final int DEFAULT_BODY_MAX_SIZE_BYTES = 128 * 1024;
6062

6163
public static AgentConfig get() {
6264
if (agentConfig == null) {
@@ -199,6 +201,10 @@ private static DataCapture.Builder setDefaultsToDataCapture(DataCapture.Builder
199201
builder.setHttpBody(applyMessageDefaults(builder.getHttpBody().toBuilder()));
200202
builder.setRpcMetadata(applyMessageDefaults(builder.getRpcMetadata().toBuilder()));
201203
builder.setRpcBody(applyMessageDefaults(builder.getRpcBody().toBuilder()));
204+
if (!builder.hasBodyMaxSizeBytes()) {
205+
builder.setBodyMaxSizeBytes(
206+
Int32Value.newBuilder().setValue(DEFAULT_BODY_MAX_SIZE_BYTES).build());
207+
}
202208
return builder;
203209
}
204210

‎javaagent-core/src/main/proto

‎javaagent-core/src/test/java/org/hypertrace/agent/core/BoundedByteArrayOutputStreamTest.java

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright The Hypertrace Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.hypertrace.agent.core;
18+
19+
import java.io.IOException;
20+
import org.junit.jupiter.api.Assertions;
21+
import org.junit.jupiter.api.Test;
22+
23+
public class BoundedByteArrayOutputStreamTest {
24+
25+
private static final String ONE_TO_TEN = "0123456789";
26+
27+
@Test
28+
public void writeArrTheSameSizeAsBuffer() throws IOException {
29+
BoundedByteArrayOutputStream boundedBuffer = new BoundedByteArrayOutputStream(10);
30+
31+
boundedBuffer.write(ONE_TO_TEN.getBytes());
32+
Assertions.assertEquals(10, boundedBuffer.size());
33+
boundedBuffer.write("01234".getBytes());
34+
Assertions.assertEquals(10, boundedBuffer.size());
35+
Assertions.assertEquals(ONE_TO_TEN, boundedBuffer.toString());
36+
}
37+
38+
@Test
39+
public void writeArrSmallerSizeAsBuffer() throws IOException {
40+
BoundedByteArrayOutputStream boundedBuffer = new BoundedByteArrayOutputStream(15);
41+
42+
boundedBuffer.write(ONE_TO_TEN.getBytes());
43+
Assertions.assertEquals(10, boundedBuffer.size());
44+
boundedBuffer.write("0123456".getBytes());
45+
Assertions.assertEquals(15, boundedBuffer.size());
46+
Assertions.assertEquals(ONE_TO_TEN + "01234", boundedBuffer.toString());
47+
}
48+
49+
@Test
50+
public void writeByteSmallerSizeAsBuffer() throws IOException {
51+
BoundedByteArrayOutputStream boundedBuffer = new BoundedByteArrayOutputStream(5);
52+
53+
boundedBuffer.write('0');
54+
boundedBuffer.write('1');
55+
boundedBuffer.write('2');
56+
boundedBuffer.write('3');
57+
boundedBuffer.write('4');
58+
Assertions.assertEquals(5, boundedBuffer.size());
59+
boundedBuffer.write('5');
60+
Assertions.assertEquals(5, boundedBuffer.size());
61+
boundedBuffer.write("01234".getBytes());
62+
Assertions.assertEquals(5, boundedBuffer.size());
63+
Assertions.assertEquals("01234", boundedBuffer.toString());
64+
}
65+
}

‎javaagent-core/src/test/java/org/hypertrace/agent/core/EnvironmentConfigTest.java

+3
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ class EnvironmentConfigTest {
3333
@ClearSystemProperty(key = EnvironmentConfig.OPA_POLL_PERIOD)
3434
@ClearSystemProperty(key = EnvironmentConfig.PROPAGATION_FORMATS)
3535
@ClearSystemProperty(key = EnvironmentConfig.CAPTURE_HTTP_BODY_PREFIX + "request")
36+
@ClearSystemProperty(key = EnvironmentConfig.CAPTURE_BODY_MAX_SIZE_BYTES)
3637
public void systemProperties() {
3738
// when tests are run in parallel the env vars/sys props set it junit-pioneer are visible to
3839
// parallel tests
@@ -42,6 +43,7 @@ public void systemProperties() {
4243
System.setProperty(EnvironmentConfig.OPA_ENDPOINT, "http://azkaban:9090");
4344
System.setProperty(EnvironmentConfig.OPA_POLL_PERIOD, "10");
4445
System.setProperty(EnvironmentConfig.PROPAGATION_FORMATS, "B3,TRACECONTEXT");
46+
System.setProperty(EnvironmentConfig.CAPTURE_BODY_MAX_SIZE_BYTES, "512");
4547

4648
AgentConfig.Builder configBuilder = AgentConfig.newBuilder();
4749
configBuilder.setServiceName(StringValue.newBuilder().setValue("foo"));
@@ -56,6 +58,7 @@ public void systemProperties() {
5658
"http://azkaban:9090", agentConfig.getReporting().getOpa().getEndpoint().getValue());
5759
Assertions.assertEquals(
5860
10, agentConfig.getReporting().getOpa().getPollPeriodSeconds().getValue());
61+
Assertions.assertEquals(512, agentConfig.getDataCapture().getBodyMaxSizeBytes().getValue());
5962
Assertions.assertEquals(true, agentConfig.getReporting().getSecure().getValue());
6063
Assertions.assertEquals(
6164
true, agentConfig.getDataCapture().getHttpBody().getRequest().getValue());

‎javaagent-core/src/test/java/org/hypertrace/agent/core/HypertraceConfigTest.java

+4
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ public void defaultValues() throws IOException {
4848
Assertions.assertEquals(
4949
HypertraceConfig.DEFAULT_OPA_POLL_PERIOD_SECONDS,
5050
agentConfig.getReporting().getOpa().getPollPeriodSeconds().getValue());
51+
Assertions.assertEquals(
52+
HypertraceConfig.DEFAULT_BODY_MAX_SIZE_BYTES,
53+
agentConfig.getDataCapture().getBodyMaxSizeBytes().getValue());
5154
Assertions.assertEquals(
5255
true, agentConfig.getDataCapture().getHttpHeaders().getRequest().getValue());
5356
Assertions.assertEquals(
@@ -99,6 +102,7 @@ private void assertConfig(AgentConfig agentConfig) {
99102
"http://opa.localhost:8181/", agentConfig.getReporting().getOpa().getEndpoint().getValue());
100103
Assertions.assertEquals(
101104
12, agentConfig.getReporting().getOpa().getPollPeriodSeconds().getValue());
105+
Assertions.assertEquals(16, agentConfig.getDataCapture().getBodyMaxSizeBytes().getValue());
102106
Assertions.assertEquals(
103107
true, agentConfig.getDataCapture().getHttpHeaders().getRequest().getValue());
104108
Assertions.assertEquals(

‎javaagent-core/src/test/resources/config.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ reporting:
88
endpoint: http://opa.localhost:8181/
99
pollPeriodSeconds: 12
1010
dataCapture:
11+
bodyMaxSizeBytes: 16
1112
httpHeaders:
1213
request: true
1314
response: false

0 commit comments

Comments
 (0)