Skip to content

Commit b1ca0da

Browse files
davidkyledavidkyle
authored
Merge branch 'main' into fix-dot-tests
2 parents 80aba35 + 56e6660 commit b1ca0da

File tree

567 files changed

+14395
-8143
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

567 files changed

+14395
-8143
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
import org.apache.lucene.util.BytesRef;
2727
import org.elasticsearch.cluster.metadata.DataStream;
2828
import org.elasticsearch.common.logging.LogConfigurator;
29-
import org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
29+
import org.elasticsearch.index.codec.Elasticsearch92Lucene103Codec;
3030
import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
3131
import org.openjdk.jmh.annotations.Benchmark;
3232
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -63,7 +63,6 @@
6363
public class TSDBDocValuesMergeBenchmark {
6464

6565
static {
66-
// For Elasticsearch900Lucene101Codec:
6766
LogConfigurator.loadLog4jPlugins();
6867
LogConfigurator.configureESLogging();
6968
LogConfigurator.setNodeName("test");
@@ -259,8 +258,7 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
259258
config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
260259
config.setMergePolicy(new LogByteSizeMergePolicy());
261260
var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
262-
config.setCodec(new Elasticsearch900Lucene101Codec() {
263-
261+
config.setCodec(new Elasticsearch92Lucene103Codec() {
264262
@Override
265263
public DocValuesFormat getDocValuesFormatForField(String field) {
266264
return docValuesFormat;

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,9 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
115115
include ':myserver'
116116
include ':myplugin'
117117
"""
118+
propertiesFile << """
119+
org.elasticsearch.transports.upstreamRef=main
120+
"""
118121
versionPropertiesFile.text = versionPropertiesFile.text.replace("9.1.0", "9.2.0")
119122

120123
file("myserver/build.gradle") << """

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,9 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
9090
assertUpperBound("9.2", "new_tv,8124000")
9191
}
9292

93+
/*
94+
temporarily muted, see https://github.com/elastic/elasticsearch/pull/135226
95+
9396
def "invalid changes to a upper bounds should be reverted"() {
9497
given:
9598
transportVersionUpperBound("9.2", "modification", "9000000")
@@ -144,7 +147,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
144147
assertReferableDefinitionDoesNotExist("test_tv")
145148
assertUpperBound("9.2", "existing_92,8123000")
146149
assertUpperBound("9.1", "existing_92,8012001")
147-
}
150+
}*/
148151

149152
def "a reference can be renamed"() {
150153
given:
@@ -242,8 +245,11 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
242245
def "unreferenced definitions are removed"() {
243246
given:
244247
referableTransportVersion("test_tv", "8124000,8012002")
248+
/*
249+
TODO: reset of upper bounds
245250
transportVersionUpperBound("9.2", "test_tv", "8124000")
246251
transportVersionUpperBound("9.1", "test_tv", "8012002")
252+
*/
247253

248254
when:
249255
def result = runGenerateAndValidateTask().build()
@@ -406,6 +412,8 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
406412
assertUpperBound("9.2", "new_tv,8124000")
407413
}
408414

415+
/*
416+
TODO: reset of upper bounds
409417
def "deleted upper bounds files are restored"() {
410418
given:
411419
file("myserver/src/main/resources/transport/upper_bounds/9.2.csv").delete()
@@ -416,7 +424,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
416424
then:
417425
assertGenerateAndValidateSuccess(result)
418426
assertUpperBound("9.2", "existing_92,8123000")
419-
}
427+
}*/
420428

421429
def "upper bounds files must exist for backport branches"() {
422430
when:

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,14 @@
4444
import org.gradle.api.file.FileCollection;
4545
import org.gradle.api.file.FileTree;
4646
import org.gradle.api.internal.artifacts.dependencies.ProjectDependencyInternal;
47+
import org.gradle.api.plugins.JvmToolchainsPlugin;
4748
import org.gradle.api.provider.ProviderFactory;
4849
import org.gradle.api.tasks.ClasspathNormalizer;
4950
import org.gradle.api.tasks.PathSensitivity;
5051
import org.gradle.api.tasks.util.PatternFilterable;
52+
import org.gradle.jvm.toolchain.JavaLanguageVersion;
53+
import org.gradle.jvm.toolchain.JavaToolchainService;
54+
import org.gradle.jvm.toolchain.JvmVendorSpec;
5155

5256
import java.util.Collection;
5357
import java.util.Iterator;
@@ -59,6 +63,7 @@
5963
import javax.inject.Inject;
6064

6165
import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
66+
import static org.elasticsearch.gradle.util.OsUtils.jdkIsIncompatibleWithOS;
6267

6368
/**
6469
* Base plugin used for wiring up build tasks to REST testing tasks using new JUnit rule-based test clusters framework.
@@ -94,6 +99,7 @@ public RestTestBasePlugin(ProviderFactory providerFactory) {
9499
public void apply(Project project) {
95100
project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
96101
project.getPluginManager().apply(InternalDistributionDownloadPlugin.class);
102+
project.getPluginManager().apply(JvmToolchainsPlugin.class);
97103
var bwcVersions = loadBuildParams(project).get().getBwcVersions();
98104

99105
// Register integ-test and default distributions
@@ -236,6 +242,17 @@ public Void call(Object... args) {
236242
String versionString = version.toString();
237243
ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
238244

245+
if (jdkIsIncompatibleWithOS(Version.fromString(versionString))) {
246+
var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
247+
var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
248+
spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
249+
spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
250+
});
251+
task.environment(
252+
"ES_FALLBACK_JAVA_HOME",
253+
fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
254+
);
255+
}
239256
task.dependsOn(bwcDistro);
240257
registerDistributionInputs(task, bwcDistro);
241258

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,8 @@ public void run() throws IOException {
103103

104104
getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
105105
if (targetDefinitionName.isEmpty()) {
106-
resetAllUpperBounds(resources);
106+
// TODO: resetting upper bounds needs to be done locally, otherwise it pulls in some (incomplete) changes from upstream main
107+
// resetAllUpperBounds(resources);
107108
} else {
108109
List<TransportVersionId> ids = updateUpperBounds(resources, upstreamUpperBounds, targetUpperBoundNames, targetDefinitionName);
109110
// (Re)write the definition file.

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.gradle.api.Project;
2020
import org.gradle.api.file.Directory;
2121
import org.gradle.api.plugins.JavaPlugin;
22+
import org.gradle.api.provider.Provider;
2223
import org.gradle.api.tasks.Copy;
2324
import org.gradle.language.base.plugins.LifecycleBasePlugin;
2425

@@ -49,6 +50,10 @@ public void apply(Project project) {
4950
Directory transportResources = project.getLayout().getProjectDirectory().dir("src/main/resources/" + resourceRoot);
5051
spec.getParameters().getTransportResourcesDirectory().set(transportResources);
5152
spec.getParameters().getRootDirectory().set(project.getLayout().getSettingsDirectory().getAsFile());
53+
Provider<String> upstreamRef = project.getProviders().gradleProperty("org.elasticsearch.transport.upstreamRef");
54+
if (upstreamRef.isPresent()) {
55+
spec.getParameters().getUpstreamRefOverride().set(upstreamRef.get());
56+
}
5257
});
5358

5459
var depsHandler = project.getDependencies();
@@ -116,10 +121,7 @@ public void apply(Project project) {
116121
}
117122

118123
private static String getResourceRoot(Project project) {
119-
var resourceRoot = project.findProperty("org.elasticsearch.transport.resourceRoot");
120-
if (resourceRoot == null) {
121-
resourceRoot = "transport";
122-
}
123-
return resourceRoot.toString();
124+
Provider<String> resourceRootProperty = project.getProviders().gradleProperty("org.elasticsearch.transport.resourceRoot");
125+
return resourceRootProperty.isPresent() ? resourceRootProperty.get() : "transport";
124126
}
125127
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesService.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,10 @@
1212
import org.gradle.api.file.DirectoryProperty;
1313
import org.gradle.api.logging.Logger;
1414
import org.gradle.api.logging.Logging;
15+
import org.gradle.api.provider.Property;
1516
import org.gradle.api.services.BuildService;
1617
import org.gradle.api.services.BuildServiceParameters;
18+
import org.gradle.api.tasks.Optional;
1719
import org.gradle.process.ExecOperations;
1820
import org.gradle.process.ExecResult;
1921

@@ -59,6 +61,9 @@ public interface Parameters extends BuildServiceParameters {
5961
DirectoryProperty getTransportResourcesDirectory();
6062

6163
DirectoryProperty getRootDirectory();
64+
65+
@Optional
66+
Property<String> getUpstreamRefOverride();
6267
}
6368

6469
@Inject
@@ -79,6 +84,9 @@ public interface Parameters extends BuildServiceParameters {
7984
public TransportVersionResourcesService(Parameters params) {
8085
this.transportResourcesDir = params.getTransportResourcesDirectory().get().getAsFile().toPath();
8186
this.rootDir = params.getRootDirectory().get().getAsFile().toPath();
87+
if (params.getUpstreamRefOverride().isPresent()) {
88+
upstreamRefName.set(params.getUpstreamRefOverride().get());
89+
}
8290
}
8391

8492
/**

‎build-tools-internal/version.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
elasticsearch = 9.2.0
2-
lucene = 10.2.2
2+
lucene = 10.3.0
33

44
bundled_jdk_vendor = openjdk
55
bundled_jdk = 25+36@bd75d5f9689641da8e1daabeccb5528b

‎build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy‎

Lines changed: 63 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,14 @@
99

1010
package org.elasticsearch.gradle
1111

12-
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
13-
import org.gradle.testkit.runner.GradleRunner
1412
import spock.lang.IgnoreIf
1513
import spock.lang.Unroll
14+
import spock.util.environment.RestoreSystemProperties
15+
16+
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
17+
import org.gradle.testkit.runner.GradleRunner
1618

17-
import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withChangedClasspathMockedDistributionDownload
18-
import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withChangedConfigMockedDistributionDownload
19-
import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withMockedDistributionDownload
19+
import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.*
2020

2121
/**
2222
* We do not have coverage for the test cluster startup on windows yet.
@@ -108,8 +108,8 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
108108
def runningClosure = { GradleRunner r -> r.build() }
109109
withMockedDistributionDownload(runner, runningClosure)
110110
def result = inputProperty == "distributionClasspath" ?
111-
withChangedClasspathMockedDistributionDownload(runner, runningClosure) :
112-
withChangedConfigMockedDistributionDownload(runner, runningClosure)
111+
withChangedClasspathMockedDistributionDownload(runner, runningClosure) :
112+
withChangedConfigMockedDistributionDownload(runner, runningClosure)
113113

114114
then:
115115
result.output.contains("Task ':myTask' is not up-to-date because:\n Input property 'clusters.myCluster\$0.nodes.\$0.$inputProperty'")
@@ -166,18 +166,24 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
166166
}
167167

168168
then:
169-
result.output.contains("Task ':myTask' is not up-to-date because:\n" +
170-
" Input property 'clusters.myCluster\$0.$propertyName'")
169+
result.output.contains(
170+
"Task ':myTask' is not up-to-date because:\n" +
171+
" Input property 'clusters.myCluster\$0.$propertyName'"
172+
)
171173
result.output.contains("elasticsearch-keystore script executed!")
172174
assertEsOutputContains("myCluster", "Starting Elasticsearch process")
173175
assertEsOutputContains("myCluster", "Stopping node")
174176

175177
where:
176178
pluginType | propertyName | fileChange
177-
'module' | "installedFiles" | { def testClazz -> testClazz.file("test-module/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
178-
'plugin' | "installedFiles" | { def testClazz -> testClazz.file("test-plugin/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
179-
'module' | "installedClasspath" | { def testClazz -> testClazz.file("test-module/src/main/java/SomeClass.java") << "class SomeClass {}" }
180-
'plugin' | "installedClasspath" | { def testClazz -> testClazz.file("test-plugin/src/main/java/SomeClass.java") << "class SomeClass {}" }
179+
'module' | "installedFiles" |
180+
{ def testClazz -> testClazz.file("test-module/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
181+
'plugin' | "installedFiles" |
182+
{ def testClazz -> testClazz.file("test-plugin/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
183+
'module' | "installedClasspath" |
184+
{ def testClazz -> testClazz.file("test-module/src/main/java/SomeClass.java") << "class SomeClass {}" }
185+
'plugin' | "installedClasspath" |
186+
{ def testClazz -> testClazz.file("test-plugin/src/main/java/SomeClass.java") << "class SomeClass {}" }
181187
}
182188

183189
def "can declare test cluster in lazy evaluated task configuration block"() {
@@ -232,9 +238,51 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
232238
assertCustomDistro('myCluster')
233239
}
234240

241+
@RestoreSystemProperties
242+
def "override jdk usage via ES_JAVA_HOME for known jdk os incompatibilities"() {
243+
given:
244+
245+
settingsFile.text = """
246+
plugins {
247+
id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0'
248+
}
249+
""" + settingsFile.text
250+
251+
buildFile << """
252+
testClusters {
253+
myCluster {
254+
testDistribution = 'default'
255+
version = '8.10.4'
256+
}
257+
}
258+
259+
// Force linux platform to trigger jdk override
260+
elasticsearch_distributions.forEach { d ->
261+
d.platform = org.elasticsearch.gradle.ElasticsearchDistribution.Platform.LINUX
262+
}
263+
264+
tasks.register('myTask', SomeClusterAwareTask) {
265+
useCluster testClusters.myCluster
266+
}
267+
"""
268+
when:
269+
def result = withMockedDistributionDownload(
270+
"8.10.4",
271+
ElasticsearchDistribution.Platform.LINUX,
272+
gradleRunner("myTask", '-Dos.name=Linux', '-Dos.version=6.14.0-1015-gcp', '-i')
273+
) {
274+
build()
275+
}
276+
277+
then:
278+
result.output.lines().anyMatch { line -> line.startsWith("Running") && line.split().find { it.startsWith("ES_JAVA_HOME=") }.contains("eclipse_adoptium-17") }
279+
}
280+
235281
boolean assertEsOutputContains(String testCluster, String expectedOutput) {
236-
assert new File(testProjectDir.root,
237-
"build/testclusters/${testCluster}-0/logs/es.out").text.contains(expectedOutput)
282+
assert new File(
283+
testProjectDir.root,
284+
"build/testclusters/${testCluster}-0/logs/es.out"
285+
).text.contains(expectedOutput)
238286
true
239287
}
240288

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.gradle.api.tasks.TaskProvider;
3838
import org.gradle.api.tasks.bundling.AbstractArchiveTask;
3939
import org.gradle.api.tasks.bundling.Zip;
40+
import org.gradle.jvm.toolchain.JavaLauncher;
4041
import org.gradle.process.ExecOperations;
4142

4243
import java.io.File;
@@ -84,6 +85,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
8485
private int nodeIndex = 0;
8586

8687
private final ConfigurableFileCollection pluginAndModuleConfiguration;
88+
private final Provider<JavaLauncher> jdk17FallbackLauncher;
8789

8890
private boolean shared = false;
8991

@@ -101,7 +103,8 @@ public ElasticsearchCluster(
101103
FileOperations fileOperations,
102104
File workingDirBase,
103105
Provider<File> runtimeJava,
104-
Function<Version, Boolean> isReleasedVersion
106+
Function<Version, Boolean> isReleasedVersion,
107+
Provider<JavaLauncher> jdk17FallbackLauncher
105108
) {
106109
this.path = path;
107110
this.clusterName = clusterName;
@@ -117,6 +120,7 @@ public ElasticsearchCluster(
117120
this.isReleasedVersion = isReleasedVersion;
118121
this.nodes = project.container(ElasticsearchNode.class);
119122
this.pluginAndModuleConfiguration = project.getObjects().fileCollection();
123+
this.jdk17FallbackLauncher = jdk17FallbackLauncher;
120124
this.nodes.add(
121125
new ElasticsearchNode(
122126
safeName(clusterName),
@@ -131,7 +135,8 @@ public ElasticsearchCluster(
131135
fileOperations,
132136
workingDirBase,
133137
runtimeJava,
134-
isReleasedVersion
138+
isReleasedVersion,
139+
jdk17FallbackLauncher
135140
)
136141
);
137142

@@ -189,7 +194,8 @@ public void setNumberOfNodes(int numberOfNodes) {
189194
fileOperations,
190195
workingDirBase,
191196
runtimeJava,
192-
isReleasedVersion
197+
isReleasedVersion,
198+
jdk17FallbackLauncher
193199
)
194200
);
195201
}

0 commit comments

Comments
 (0)