diff --git a/Dockerfile b/Dockerfile index 280baedce41bb..73b5a353f967e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # -# base: Stage which installs necessary runtime dependencies (OS packages, java,...) +# base: Stage which installs necessary runtime dependencies (OS packages, etc.) # FROM python:3.11.10-slim-bookworm@sha256:50ec89bdac0a845ec1751f91cb6187a3d8adb2b919d6e82d17acf48d1a9743fc AS base ARG TARGETARCH @@ -89,7 +89,6 @@ ADD bin/hosts /etc/hosts # expose default environment # Set edge bind host so localstack can be reached by other containers # set library path and default LocalStack hostname -ENV LD_LIBRARY_PATH=$JAVA_HOME/lib:$JAVA_HOME/lib/server ENV USER=localstack ENV PYTHONUNBUFFERED=1 @@ -155,18 +154,12 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LOCALSTACK_CORE=${LOCALSTACK_BUILD_VERSIO RUN --mount=type=cache,target=/root/.cache \ --mount=type=cache,target=/var/lib/localstack/cache \ source .venv/bin/activate && \ - python -m localstack.cli.lpm install java --version 11 && \ python -m localstack.cli.lpm install \ lambda-runtime \ dynamodb-local && \ chown -R localstack:localstack /usr/lib/localstack && \ chmod -R 777 /usr/lib/localstack -# Set up Java -ENV JAVA_HOME /usr/lib/localstack/java/11 -RUN ln -s $JAVA_HOME/bin/java /usr/bin/java -ENV PATH="${PATH}:${JAVA_HOME}/bin" - # link the python package installer virtual environments into the localstack venv RUN echo /var/lib/localstack/lib/python-packages/lib/python3.11/site-packages > localstack-var-python-packages-venv.pth && \ mv localstack-var-python-packages-venv.pth .venv/lib/python*/site-packages/ diff --git a/localstack-core/localstack/services/dynamodb/packages.py b/localstack-core/localstack/services/dynamodb/packages.py index 52e76e761ae41..5e2fccc99539e 100644 --- a/localstack-core/localstack/services/dynamodb/packages.py +++ b/localstack-core/localstack/services/dynamodb/packages.py @@ -4,6 +4,7 @@ from localstack import config from localstack.constants import ARTIFACTS_REPO, MAVEN_REPO_URL from localstack.packages import InstallTarget, Package, PackageInstaller +from localstack.packages.java import java_package from localstack.utils.archives import ( download_and_extract_with_retry, update_jar_manifest, @@ -41,6 +42,11 @@ class DynamoDBLocalPackageInstaller(PackageInstaller): def __init__(self): super().__init__("dynamodb-local", "latest") + self.java_version = "11" + + def _prepare_installation(self, target: InstallTarget) -> None: + java_package.get_installer(self.java_version).install(target) + def _install(self, target: InstallTarget): # download and extract archive tmp_archive = os.path.join(config.dirs.cache, "localstack.ddb.zip") diff --git a/localstack-core/localstack/services/dynamodb/server.py b/localstack-core/localstack/services/dynamodb/server.py index d250b47c195b6..41b802b71dfc3 100644 --- a/localstack-core/localstack/services/dynamodb/server.py +++ b/localstack-core/localstack/services/dynamodb/server.py @@ -7,6 +7,7 @@ from localstack.aws.forwarder import AwsRequestProxy from localstack.config import is_env_true from localstack.constants import DEFAULT_AWS_ACCOUNT_ID +from localstack.packages.java import java_package from localstack.services.dynamodb.packages import dynamodblocal_package from localstack.utils.common import TMP_THREADS, ShellCommandThread, get_free_tcp_port, mkdir from localstack.utils.functions import run_safe @@ -150,9 +151,20 @@ def _create_shell_command(self) -> list[str]: return cmd + parameters def do_start_thread(self) -> FuncThread: - dynamodblocal_package.install() + dynamodblocal_installer = dynamodblocal_package.get_installer() + dynamodblocal_installer.install() + + java_home = java_package.get_installer(dynamodblocal_installer.java_version).get_java_home() + + path = f"{java_home}/bin:{os.getenv('PATH')}" cmd = self._create_shell_command() + env_vars = { + "DDB_LOCAL_TELEMETRY": "0", + "JAVA_HOME": java_home, + "PATH": path, + } + LOG.debug("Starting DynamoDB Local: %s", cmd) t = ShellCommandThread( cmd, @@ -160,7 +172,7 @@ def do_start_thread(self) -> FuncThread: log_listener=_log_listener, auto_restart=True, name="dynamodb-local", - env_vars={"DDB_LOCAL_TELEMETRY": "0"}, + env_vars=env_vars, ) TMP_THREADS.append(t) t.start() diff --git a/localstack-core/localstack/services/events/event_ruler.py b/localstack-core/localstack/services/events/event_ruler.py index e48712687bbce..0d8ef889ba20a 100644 --- a/localstack-core/localstack/services/events/event_ruler.py +++ b/localstack-core/localstack/services/events/event_ruler.py @@ -2,7 +2,9 @@ import os from functools import cache from pathlib import Path +from typing import Tuple +from localstack.packages.java import java_package from localstack.services.events.models import InvalidEventPatternException from localstack.services.events.packages import event_ruler_package from localstack.utils.objects import singleton_factory @@ -25,16 +27,26 @@ def start_jvm() -> None: jpype_config.destroy_jvm = False if not jpype.isJVMStarted(): - event_ruler_libs_path = get_event_ruler_libs_path() + jvm_lib, event_ruler_libs_path = get_jpype_lib_paths() event_ruler_libs_pattern = event_ruler_libs_path.joinpath("*") - jpype.startJVM(classpath=[event_ruler_libs_pattern]) + + jpype.startJVM(str(jvm_lib), classpath=[event_ruler_libs_pattern]) @cache -def get_event_ruler_libs_path() -> Path: +def get_jpype_lib_paths() -> Tuple[Path, Path]: + """ + Downloads Event Ruler, its dependencies and returns a tuple of: + - Path to libjvm.so to be used by JPype as jvmpath + - Path to Event Ruler libraries to be used by JPype as classpath + """ installer = event_ruler_package.get_installer() installer.install() - return Path(installer.get_installed_dir()) + + java_home = java_package.get_installer(installer.java_version).get_java_home() + jvm_lib = Path(java_home) / "lib" / "server" / "libjvm.so" + + return jvm_lib, Path(installer.get_installed_dir()) def matches_rule(event: str, rule: str) -> bool: diff --git a/localstack-core/localstack/services/events/packages.py b/localstack-core/localstack/services/events/packages.py index 5686b6844d454..cbc1979f43180 100644 --- a/localstack-core/localstack/services/events/packages.py +++ b/localstack-core/localstack/services/events/packages.py @@ -1,5 +1,6 @@ -from localstack.packages import Package, PackageInstaller +from localstack.packages import InstallTarget, Package, PackageInstaller from localstack.packages.core import MavenPackageInstaller +from localstack.packages.java import java_package # https://central.sonatype.com/artifact/software.amazon.event.ruler/event-ruler EVENT_RULER_VERSION = "1.7.3" @@ -15,12 +16,22 @@ def get_versions(self) -> list[str]: return [EVENT_RULER_VERSION] def _get_installer(self, version: str) -> PackageInstaller: - return MavenPackageInstaller( + return EventRulerPackageInstaller() + + +class EventRulerPackageInstaller(MavenPackageInstaller): + def __init__(self): + super().__init__( f"pkg:maven/software.amazon.event.ruler/event-ruler@{EVENT_RULER_VERSION}", f"pkg:maven/com.fasterxml.jackson.core/jackson-annotations@{JACKSON_VERSION}", f"pkg:maven/com.fasterxml.jackson.core/jackson-core@{JACKSON_VERSION}", f"pkg:maven/com.fasterxml.jackson.core/jackson-databind@{JACKSON_VERSION}", ) + self.java_version = "11" + + def _prepare_installation(self, target: InstallTarget) -> None: + java_package.get_installer(self.java_version).install(target) + event_ruler_package = EventRulerPackage() diff --git a/localstack-core/localstack/services/opensearch/cluster.py b/localstack-core/localstack/services/opensearch/cluster.py index c50e676c887b2..b312fbcbdaad6 100644 --- a/localstack-core/localstack/services/opensearch/cluster.py +++ b/localstack-core/localstack/services/opensearch/cluster.py @@ -16,6 +16,7 @@ ) from localstack.http.client import SimpleRequestsClient from localstack.http.proxy import ProxyHandler +from localstack.packages.java import java_package from localstack.services.edge import ROUTER from localstack.services.opensearch import versions from localstack.services.opensearch.packages import elasticsearch_package, opensearch_package @@ -465,7 +466,10 @@ def _create_run_command( return cmd def _create_env_vars(self, directories: Directories) -> Dict: + elasticsearch_installer = elasticsearch_package.get_installer(self.version) + java_home = java_package.get_installer(elasticsearch_installer.java_version).get_java_home() env_vars = { + "JAVA_HOME": java_home, "OPENSEARCH_JAVA_OPTS": os.environ.get("OPENSEARCH_JAVA_OPTS", "-Xms200m -Xmx600m"), "OPENSEARCH_TMPDIR": directories.tmp, } @@ -688,7 +692,10 @@ def _base_settings(self, dirs) -> CommandSettings: return settings def _create_env_vars(self, directories: Directories) -> Dict: + opensearch_installer = opensearch_package.get_installer(self.version) + java_home = java_package.get_installer(opensearch_installer.java_version).get_java_home() return { + "JAVA_HOME": java_home, "ES_JAVA_OPTS": os.environ.get("ES_JAVA_OPTS", "-Xms200m -Xmx600m"), "ES_TMPDIR": directories.tmp, } diff --git a/localstack-core/localstack/services/opensearch/packages.py b/localstack-core/localstack/services/opensearch/packages.py index 6dfaa6ae6b781..f9d1ce163ce09 100644 --- a/localstack-core/localstack/services/opensearch/packages.py +++ b/localstack-core/localstack/services/opensearch/packages.py @@ -18,6 +18,7 @@ OPENSEARCH_PLUGIN_LIST, ) from localstack.packages import InstallTarget, Package, PackageInstaller +from localstack.packages.java import java_package from localstack.services.opensearch import versions from localstack.utils.archives import download_and_extract_with_retry from localstack.utils.files import chmod_r, load_file, mkdir, rm_rf, save_file @@ -49,6 +50,15 @@ class OpensearchPackageInstaller(PackageInstaller): def __init__(self, version: str): super().__init__("opensearch", version) + # JRE version to use + # OpenSearch has the widest compatibility with Java 11 + # See: https://opensearch.org/docs/latest/install-and-configure/install-opensearch/index/#java-compatibility + self.java_version = "11" + + def _prepare_installation(self, target: InstallTarget) -> None: + # OpenSearch ships with a bundled JRE, but we still use LocalStack's JRE for predictability + java_package.get_installer(self.java_version).install(target) + def _install(self, target: InstallTarget): # locally import to avoid having a dependency on ASF when starting the CLI from localstack.aws.api.opensearch import EngineType @@ -216,6 +226,15 @@ class ElasticsearchPackageInstaller(PackageInstaller): def __init__(self, version: str): super().__init__("elasticsearch", version) + # JRE version to use + # ElasticSearch has the widest compatibility with Java 8 + # See: https://www.elastic.co/support/matrix + self.java_version = "8" + + def _prepare_installation(self, target: InstallTarget) -> None: + # ElasticSearch ships with a bundled JRE, but we still use LocalStack's JRE for predictability + java_package.get_installer(self.java_version).install(target) + def _install(self, target: InstallTarget): # locally import to avoid having a dependency on ASF when starting the CLI from localstack.aws.api.opensearch import EngineType diff --git a/localstack-core/localstack/services/stepfunctions/legacy/stepfunctions_starter.py b/localstack-core/localstack/services/stepfunctions/legacy/stepfunctions_starter.py index 9a456bf50905d..7273ba6be0b30 100644 --- a/localstack-core/localstack/services/stepfunctions/legacy/stepfunctions_starter.py +++ b/localstack-core/localstack/services/stepfunctions/legacy/stepfunctions_starter.py @@ -1,9 +1,11 @@ import logging +import os import threading from typing import Any, Dict from localstack import config -from localstack.services.stepfunctions.packages import stepfunctions_local_package +from localstack.packages.java import java_package +from localstack.services.stepfunctions.packages import SFN_JAVA_VERSION, stepfunctions_local_package from localstack.utils.aws import aws_stack from localstack.utils.net import get_free_tcp_port, port_can_be_bound from localstack.utils.run import ShellCommandThread @@ -42,11 +44,17 @@ def do_start_thread(self) -> FuncThread: return t def generate_env_vars(self) -> Dict[str, Any]: + java_home = java_package.get_installer(SFN_JAVA_VERSION).get_java_home() + + path = f"{java_home}/bin:{os.getenv('PATH')}" + return { "EDGE_PORT": config.GATEWAY_LISTEN[0].port, "EDGE_PORT_HTTP": config.GATEWAY_LISTEN[0].port, "DATA_DIR": config.dirs.data, + "JAVA_HOME": java_home, "PORT": self._port, + "PATH": path, } def generate_shell_command(self) -> str: diff --git a/localstack-core/localstack/services/stepfunctions/packages.py b/localstack-core/localstack/services/stepfunctions/packages.py index 81f0699b90696..4565fef16607b 100644 --- a/localstack-core/localstack/services/stepfunctions/packages.py +++ b/localstack-core/localstack/services/stepfunctions/packages.py @@ -9,6 +9,7 @@ from localstack.constants import ARTIFACTS_REPO, MAVEN_REPO_URL from localstack.packages import InstallTarget, Package, PackageInstaller from localstack.packages.core import ExecutableInstaller +from localstack.packages.java import java_package from localstack.utils.archives import add_file_to_jar, untar, update_jar_manifest from localstack.utils.files import file_exists_not_empty, mkdir, new_tmp_file, rm_rf from localstack.utils.http import download @@ -18,6 +19,8 @@ URL_ASPECTJWEAVER = f"{MAVEN_REPO_URL}/org/aspectj/aspectjweaver/1.9.7/aspectjweaver-1.9.7.jar" JAR_URLS = [URL_ASPECTJRT, URL_ASPECTJWEAVER] +SFN_JAVA_VERSION = "11" + SFN_PATCH_URL_PREFIX = ( f"{ARTIFACTS_REPO}/raw/ac84739adc87ff4b5553478f6849134bcd259672/stepfunctions-local-patch" ) @@ -77,6 +80,9 @@ class StepFunctionsLocalPackageInstaller(ExecutableInstaller): def _get_install_marker_path(self, install_dir: str) -> str: return os.path.join(install_dir, "StepFunctionsLocal.jar") + def _prepare_installation(self, target: InstallTarget) -> None: + java_package.get_installer(SFN_JAVA_VERSION).install(target) + def _install(self, target: InstallTarget) -> None: """ The StepFunctionsLocal JAR files are downloaded using the artifacts in DockerHub (because AWS only provides an diff --git a/localstack-core/localstack/utils/kinesis/kinesis_connector.py b/localstack-core/localstack/utils/kinesis/kinesis_connector.py index 334a1f0573475..7852b4171c3de 100644 --- a/localstack-core/localstack/utils/kinesis/kinesis_connector.py +++ b/localstack-core/localstack/utils/kinesis/kinesis_connector.py @@ -13,6 +13,7 @@ from localstack import config from localstack.constants import LOCALSTACK_ROOT_FOLDER, LOCALSTACK_VENV_FOLDER +from localstack.packages.java import java_package from localstack.utils.aws import arns from localstack.utils.files import TMP_FILES, chmod_r, save_file from localstack.utils.kinesis import kclipy_helper @@ -26,6 +27,7 @@ # define Java class names MULTI_LANG_DAEMON_CLASS = "software.amazon.kinesis.multilang.MultiLangDaemon" +KCL_JAVA_VERSION = "11" # set up local logger LOG = logging.getLogger(__name__) @@ -38,6 +40,7 @@ CHECKPOINT_SLEEP_SECS = 5 CHECKPOINT_FREQ_SECS = 60 + ListenerFunction = Callable[[list], Any] @@ -240,12 +243,21 @@ def _start_kcl_client_process( ): # make sure to convert stream ARN to stream name stream_name = arns.kinesis_stream_name(stream_name) + + # install Java + java_installer = java_package.get_installer(KCL_JAVA_VERSION) + java_installer.install() + java_home = java_installer.get_java_home() + path = f"{java_home}/bin:{os.getenv('PATH')}" + # disable CBOR protocol, enforce use of plain JSON # TODO evaluate why? env_vars = { "AWS_CBOR_DISABLE": "true", "AWS_ACCESS_KEY_ID": account_id, "AWS_SECRET_ACCESS_KEY": account_id, + "JAVA_HOME": java_home, + "PATH": path, } events_file = os.path.join(tempfile.gettempdir(), f"kclipy.{short_uid()}.fifo")