Skip to content

Commit 6d4e9d1

Browse files
martijnvgmartijnvg
authored
Remove track_live_docs_in_memory_bytes feature flag (elastic#134900)
1 parent ba60ed7 commit 6d4e9d1

File tree

5 files changed

+106
-87
lines changed

5 files changed

+106
-87
lines changed

‎server/src/main/java/org/elasticsearch/index/engine/Engine.java‎

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -301,26 +301,20 @@ protected static ShardFieldStats shardFieldStats(List<LeafReaderContext> leaves,
301301
} else {
302302
usages = -1;
303303
}
304-
boolean trackPostingsMemoryEnabled = isStateless;
305-
boolean trackLiveDocsMemoryEnabled = ShardFieldStats.TRACK_LIVE_DOCS_IN_MEMORY_BYTES.isEnabled();
306-
if (trackLiveDocsMemoryEnabled || trackPostingsMemoryEnabled) {
304+
if (isStateless) {
307305
SegmentReader segmentReader = Lucene.tryUnwrapSegmentReader(leaf.reader());
308306
if (segmentReader != null) {
309-
if (trackPostingsMemoryEnabled) {
310-
String postingBytes = segmentReader.getSegmentInfo().info.getAttribute(
311-
TrackingPostingsInMemoryBytesCodec.IN_MEMORY_POSTINGS_BYTES_KEY
312-
);
313-
if (postingBytes != null) {
314-
totalPostingBytes += Long.parseLong(postingBytes);
315-
}
307+
String postingBytes = segmentReader.getSegmentInfo().info.getAttribute(
308+
TrackingPostingsInMemoryBytesCodec.IN_MEMORY_POSTINGS_BYTES_KEY
309+
);
310+
if (postingBytes != null) {
311+
totalPostingBytes += Long.parseLong(postingBytes);
316312
}
317-
if (trackLiveDocsMemoryEnabled) {
318-
var liveDocs = segmentReader.getLiveDocs();
319-
if (liveDocs != null) {
320-
assert validateLiveDocsClass(liveDocs);
321-
long liveDocsBytes = getLiveDocsBytes(liveDocs);
322-
totalLiveDocsBytes += liveDocsBytes;
323-
}
313+
var liveDocs = segmentReader.getLiveDocs();
314+
if (liveDocs != null) {
315+
assert validateLiveDocsClass(liveDocs);
316+
long liveDocsBytes = getLiveDocsBytes(liveDocs);
317+
totalLiveDocsBytes += liveDocsBytes;
324318
}
325319
}
326320
}

‎server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111

1212
import org.apache.lucene.util.FixedBitSet;
1313
import org.apache.lucene.util.RamUsageEstimator;
14-
import org.elasticsearch.common.util.FeatureFlag;
1514

1615
/**
1716
* A per shard stats including the number of segments and total fields across those segments.
@@ -26,7 +25,6 @@
2625
*/
2726
public record ShardFieldStats(int numSegments, int totalFields, long fieldUsages, long postingsInMemoryBytes, long liveDocsBytes) {
2827

29-
public static final FeatureFlag TRACK_LIVE_DOCS_IN_MEMORY_BYTES = new FeatureFlag("track_live_docs_in_memory_bytes");
3028
public static final long FIXED_BITSET_BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FixedBitSet.class);
3129

3230
}

‎server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java‎

Lines changed: 0 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import org.apache.lucene.store.IOContext;
2828
import org.apache.lucene.util.BytesRef;
2929
import org.apache.lucene.util.Constants;
30-
import org.apache.lucene.util.FixedBitSet;
3130
import org.elasticsearch.ElasticsearchException;
3231
import org.elasticsearch.ExceptionsHelper;
3332
import org.elasticsearch.action.ActionListener;
@@ -78,7 +77,6 @@
7877
import org.elasticsearch.index.IndexModule;
7978
import org.elasticsearch.index.IndexSettings;
8079
import org.elasticsearch.index.IndexVersion;
81-
import org.elasticsearch.index.MergePolicyConfig;
8280
import org.elasticsearch.index.codec.CodecService;
8381
import org.elasticsearch.index.engine.CommitStats;
8482
import org.elasticsearch.index.engine.DocIdSeqNoAndSource;
@@ -1981,71 +1979,6 @@ public void testShardFieldStats() throws IOException {
19811979
closeShards(shard);
19821980
}
19831981

1984-
public void testShardFieldStatsWithDeletes() throws IOException {
1985-
Settings settings = Settings.builder()
1986-
.put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
1987-
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE)
1988-
.build();
1989-
IndexShard shard = newShard(true, settings);
1990-
assertNull(shard.getShardFieldStats());
1991-
recoverShardFromStore(shard);
1992-
boolean liveDocsTrackingEnabled = ShardFieldStats.TRACK_LIVE_DOCS_IN_MEMORY_BYTES.isEnabled();
1993-
1994-
// index some documents
1995-
int numDocs = 10;
1996-
for (int i = 0; i < numDocs; i++) {
1997-
indexDoc(shard, "_doc", "first_" + i, """
1998-
{
1999-
"f1": "foo",
2000-
"f2": "bar"
2001-
}
2002-
""");
2003-
}
2004-
shard.refresh("test");
2005-
var stats = shard.getShardFieldStats();
2006-
assertThat(stats.numSegments(), equalTo(1));
2007-
assertThat(stats.liveDocsBytes(), equalTo(0L));
2008-
2009-
// delete a doc
2010-
deleteDoc(shard, "first_0");
2011-
2012-
// Refresh and fetch new stats:
2013-
shard.refresh("test");
2014-
stats = shard.getShardFieldStats();
2015-
// More segments because delete operation is stored in the new segment for replication purposes.
2016-
assertThat(stats.numSegments(), equalTo(2));
2017-
long expectedLiveDocsSize = 0;
2018-
if (liveDocsTrackingEnabled) {
2019-
// Delete op is stored in new segment, but marked as deleted. All segements have live docs:
2020-
expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed();
2021-
// Second segment the delete operation that is marked as deleted:
2022-
expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
2023-
}
2024-
assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize));
2025-
2026-
// delete another doc:
2027-
deleteDoc(shard, "first_1");
2028-
shard.getMinRetainedSeqNo();
2029-
2030-
// Refresh and fetch new stats:
2031-
shard.refresh("test");
2032-
stats = shard.getShardFieldStats();
2033-
// More segments because delete operation is stored in the new segment for replication purposes.
2034-
assertThat(stats.numSegments(), equalTo(3));
2035-
expectedLiveDocsSize = 0;
2036-
if (liveDocsTrackingEnabled) {
2037-
// Delete op is stored in new segment, but marked as deleted. All segements have live docs:
2038-
// First segment with deletes
2039-
expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed();
2040-
// Second and third segments the delete operation that is marked as deleted:
2041-
expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
2042-
expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
2043-
}
2044-
assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize));
2045-
2046-
closeShards(shard);
2047-
}
2048-
20491982
public void testIndexingOperationsListeners() throws IOException {
20501983
IndexShard shard = newStartedShard(true);
20511984
indexDoc(shard, "_doc", "0", "{\"foo\" : \"bar\"}");

‎server/src/test/java/org/elasticsearch/index/shard/LiveDocsEstimationTests.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.shard;
11+
12+
import org.apache.lucene.util.FixedBitSet;
13+
import org.elasticsearch.cluster.node.DiscoveryNode;
14+
import org.elasticsearch.common.settings.Settings;
15+
import org.elasticsearch.core.TimeValue;
16+
import org.elasticsearch.index.IndexSettings;
17+
import org.elasticsearch.index.MergePolicyConfig;
18+
19+
import java.io.IOException;
20+
21+
import static org.hamcrest.Matchers.equalTo;
22+
23+
public class LiveDocsEstimationTests extends IndexShardTestCase {
24+
25+
@Override
26+
protected Settings nodeSettings() {
27+
return Settings.builder().put(DiscoveryNode.STATELESS_ENABLED_SETTING_NAME, true).build();
28+
}
29+
30+
public void testShardFieldStatsWithDeletes() throws IOException {
31+
Settings settings = Settings.builder()
32+
.put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
33+
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE)
34+
.build();
35+
IndexShard shard = newShard(true, settings);
36+
assertNull(shard.getShardFieldStats());
37+
recoverShardFromStore(shard);
38+
39+
// index some documents
40+
int numDocs = 10;
41+
for (int i = 0; i < numDocs; i++) {
42+
indexDoc(shard, "_doc", "first_" + i, """
43+
{
44+
"f1": "foo",
45+
"f2": "bar"
46+
}
47+
""");
48+
}
49+
shard.refresh("test");
50+
var stats = shard.getShardFieldStats();
51+
assertThat(stats.numSegments(), equalTo(1));
52+
assertThat(stats.liveDocsBytes(), equalTo(0L));
53+
54+
// delete a doc
55+
deleteDoc(shard, "first_0");
56+
57+
// Refresh and fetch new stats:
58+
shard.refresh("test");
59+
stats = shard.getShardFieldStats();
60+
// More segments because delete operation is stored in the new segment for replication purposes.
61+
assertThat(stats.numSegments(), equalTo(2));
62+
long expectedLiveDocsSize = 0;
63+
// Delete op is stored in new segment, but marked as deleted. All segements have live docs:
64+
expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed();
65+
// Second segment the delete operation that is marked as deleted:
66+
expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
67+
assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize));
68+
69+
// delete another doc:
70+
deleteDoc(shard, "first_1");
71+
shard.getMinRetainedSeqNo();
72+
73+
// Refresh and fetch new stats:
74+
shard.refresh("test");
75+
stats = shard.getShardFieldStats();
76+
// More segments because delete operation is stored in the new segment for replication purposes.
77+
assertThat(stats.numSegments(), equalTo(3));
78+
expectedLiveDocsSize = 0;
79+
// Delete op is stored in new segment, but marked as deleted. All segements have live docs:
80+
// First segment with deletes
81+
expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed();
82+
// Second and third segments the delete operation that is marked as deleted:
83+
expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
84+
expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed();
85+
assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize));
86+
87+
closeShards(shard);
88+
}
89+
90+
}

‎test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -616,7 +616,7 @@ protected IndexShard newShard(
616616
List<SearchOperationListener> soListener,
617617
IndexingOperationListener... listeners
618618
) throws IOException {
619-
final Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build();
619+
final Settings nodeSettings = Settings.builder().put(nodeSettings()).put("node.name", routing.currentNodeId()).build();
620620
final IndexSettings indexSettings = new IndexSettings(indexMetadata, nodeSettings);
621621
final IndexShard indexShard;
622622
if (storeProvider == null) {
@@ -1336,4 +1336,8 @@ public static Engine.Warmer createTestWarmer(IndexSettings indexSettings) {
13361336
public static long recoverLocallyUpToGlobalCheckpoint(IndexShard indexShard) {
13371337
return safeAwait(indexShard::recoverLocallyUpToGlobalCheckpoint);
13381338
}
1339+
1340+
protected Settings nodeSettings() {
1341+
return Settings.EMPTY;
1342+
}
13391343
}

0 commit comments

Comments
 (0)