Reduce binary distribution size: unbundle optional Hadoop/Kafka deps + de-duplicate shared jars

bin/storm-autocreds-fetch

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Fetch the storm-autocreds plugin and its (Hadoop/HBase) runtime dependencies
+# into the daemon classpath, so Nimbus/Supervisor can populate and renew HDFS
+# and HBase delegation tokens on a secure (Kerberos) cluster.
+#
+# These jars are intentionally NOT bundled in the binary distribution to keep it
+# small; only secure-Hadoop deployments need them. See
+# external/storm-autocreds/README.md for details.
+
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage: storm-autocreds-fetch [options] [-- <extra maven args>]
+
+Resolves org.apache.storm:storm-autocreds and its runtime dependencies from a
+Maven repository (Maven Central by default) and copies them into the Storm
+daemon classpath directory (extlib-daemon).
+
+Options:
+  --version <ver>   Storm version to fetch (default: read from $STORM_HOME/RELEASE)
+  --dest <dir>      Target directory (default: $STORM_HOME/extlib-daemon)
+  -h, --help        Show this help
+
+Any arguments after "--" are passed through to Maven, e.g. to use an internal
+mirror or an offline local repository:
+  storm-autocreds-fetch -- -s /path/settings.xml
+  storm-autocreds-fetch -- -Dmaven.repo.local=/path/to/offline-repo -o
+EOF
+}
+
+# Resolve symlinks so STORM_HOME is correct even when invoked via a link.
+PRG="${0}"
+while [ -h "${PRG}" ]; do
+  ls=$(ls -ld "${PRG}")
+  link=$(expr "${ls}" : '.*-> \(.*\)$')
+  if expr "${link}" : '/.*' > /dev/null; then
+    PRG="${link}"
+  else
+    PRG="$(dirname "${PRG}")/${link}"
+  fi
+done
+STORM_BIN_DIR=$(dirname "${PRG}")
+STORM_HOME=$(cd "${STORM_BIN_DIR}/.." && pwd)
+
+VERSION=""
+DEST=""
+MVN_ARGS=()
+while [ $# -gt 0 ]; do
+  case "${1}" in
+    --version) VERSION="${2}"; shift 2 ;;
+    --dest)    DEST="${2}"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    --)        shift; MVN_ARGS=("$@"); break ;;
+    *)         echo "Unknown option: ${1}" >&2; usage; exit 1 ;;
+  esac
+done
+
+if [ -z "${VERSION}" ]; then
+  if [ -f "${STORM_HOME}/RELEASE" ]; then
+    VERSION=$(tr -d '[:space:]' < "${STORM_HOME}/RELEASE")
+  fi
+fi
+if [ -z "${VERSION}" ]; then
+  echo "Error: could not determine Storm version. Pass --version <ver>." >&2
+  exit 1
+fi
+
+if [ -z "${DEST}" ]; then
+  DEST="${STORM_HOME}/extlib-daemon"
+fi
+
+MVN="${MAVEN_HOME:+${MAVEN_HOME}/bin/}mvn"
+if ! command -v "${MVN}" > /dev/null 2>&1; then
+  echo "Error: '${MVN}' not found on PATH. Install Apache Maven or set MAVEN_HOME." >&2
+  exit 1
+fi
+
+mkdir -p "${DEST}"
+
+# Use a throwaway POM that depends on storm-autocreds; copy-dependencies then
+# pulls the exact runtime closure (honoring the exclusions declared in the
+# published storm-autocreds POM). storm-client is 'provided' there and is
+# correctly skipped, since it already ships in lib/.
+TMP_DIR=$(mktemp -d)
+trap 'rm -rf "${TMP_DIR}"' EXIT
+cat > "${TMP_DIR}/pom.xml" <<EOF
+<project xmlns="http://maven.apache.org/POM/4.0.0">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.storm.tools</groupId>
+  <artifactId>storm-autocreds-fetch</artifactId>
+  <version>${VERSION}</version>
+  <packaging>pom</packaging>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.storm</groupId>
+      <artifactId>storm-autocreds</artifactId>
+      <version>${VERSION}</version>
+    </dependency>
+  </dependencies>
+</project>
+EOF
+
+echo "Fetching org.apache.storm:storm-autocreds:${VERSION} (runtime closure) into:"
+echo "  ${DEST}"
+"${MVN}" -q -f "${TMP_DIR}/pom.xml" \
+  org.apache.maven.plugins:maven-dependency-plugin:copy-dependencies \
+  -DincludeScope=runtime \
+  -DoutputDirectory="${DEST}" \
+  ${MVN_ARGS[@]+"${MVN_ARGS[@]}"}
+
+echo "Done. ${DEST} now contains:"
+ls -1 "${DEST}" | sed 's/^/  /'
+echo
+echo "Restart the Storm daemons (Nimbus, Supervisor) to pick up the new classpath,"
+echo "then configure the autocreds plugins in storm.yaml. See"
+echo "external/storm-autocreds/README.md for the required settings."

bin/storm-kafka-monitor

@@ -49,4 +49,11 @@ if [ -z "$JAVA_HOME" ]; then
 else
   JAVA="$JAVA_HOME/bin/java"
 fi
-exec $JAVA $STORM_JAAS_CONF_PARAM $STORM_JAR_JVM_OPTS -cp "$STORM_BASE_DIR/lib-tools/storm-kafka-monitor/*" org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@"
+# The storm-kafka-monitor jars are not bundled in the distribution; they are fetched on demand.
+KAFKA_MONITOR_LIB="$STORM_BASE_DIR/lib-tools/storm-kafka-monitor"
+if ! ls "$KAFKA_MONITOR_LIB"/*.jar >/dev/null 2>&1; then
+  echo "storm-kafka-monitor is not installed (no jars in $KAFKA_MONITOR_LIB)." >&2
+  echo "Run '$STORM_BIN_DIR/storm-kafka-monitor-fetch' to download it, then retry." >&2
+  exit 1
+fi
+exec $JAVA $STORM_JAAS_CONF_PARAM $STORM_JAR_JVM_OPTS -cp "$KAFKA_MONITOR_LIB/*" org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@"

bin/storm-kafka-monitor-fetch

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Fetch the storm-kafka-monitor tool and its (Kafka client) runtime dependencies
+# into lib-tools/storm-kafka-monitor, enabling the "Kafka spout lag" display in
+# the Storm UI and the bin/storm-kafka-monitor command.
+#
+# These jars are intentionally NOT bundled in the binary distribution to keep it
+# small; they are only needed when running Kafka spouts and wanting lag info. The
+# UI degrades gracefully when they are absent. See
+# external/storm-kafka-monitor/README.md for details.
+
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage: storm-kafka-monitor-fetch [options] [-- <extra maven args>]
+
+Resolves org.apache.storm:storm-kafka-monitor and its runtime dependencies from
+a Maven repository (Maven Central by default) and copies them into
+lib-tools/storm-kafka-monitor.
+
+Options:
+  --version <ver>   Storm version to fetch (default: read from $STORM_HOME/RELEASE)
+  --dest <dir>      Target directory (default: $STORM_HOME/lib-tools/storm-kafka-monitor)
+  -h, --help        Show this help
+
+Any arguments after "--" are passed through to Maven, e.g. to use an internal
+mirror or an offline local repository:
+  storm-kafka-monitor-fetch -- -s /path/settings.xml
+  storm-kafka-monitor-fetch -- -Dmaven.repo.local=/path/to/offline-repo -o
+EOF
+}
+
+# Resolve symlinks so STORM_HOME is correct even when invoked via a link.
+PRG="${0}"
+while [ -h "${PRG}" ]; do
+  ls=$(ls -ld "${PRG}")
+  link=$(expr "${ls}" : '.*-> \(.*\)$')
+  if expr "${link}" : '/.*' > /dev/null; then
+    PRG="${link}"
+  else
+    PRG="$(dirname "${PRG}")/${link}"
+  fi
+done
+STORM_BIN_DIR=$(dirname "${PRG}")
+STORM_HOME=$(cd "${STORM_BIN_DIR}/.." && pwd)
+
+VERSION=""
+DEST=""
+MVN_ARGS=()
+while [ $# -gt 0 ]; do
+  case "${1}" in
+    --version) VERSION="${2}"; shift 2 ;;
+    --dest)    DEST="${2}"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    --)        shift; MVN_ARGS=("$@"); break ;;
+    *)         echo "Unknown option: ${1}" >&2; usage; exit 1 ;;
+  esac
+done
+
+if [ -z "${VERSION}" ]; then
+  if [ -f "${STORM_HOME}/RELEASE" ]; then
+    VERSION=$(tr -d '[:space:]' < "${STORM_HOME}/RELEASE")
+  fi
+fi
+if [ -z "${VERSION}" ]; then
+  echo "Error: could not determine Storm version. Pass --version <ver>." >&2
+  exit 1
+fi
+
+if [ -z "${DEST}" ]; then
+  DEST="${STORM_HOME}/lib-tools/storm-kafka-monitor"
+fi
+
+MVN="${MAVEN_HOME:+${MAVEN_HOME}/bin/}mvn"
+if ! command -v "${MVN}" > /dev/null 2>&1; then
+  echo "Error: '${MVN}' not found on PATH. Install Apache Maven or set MAVEN_HOME." >&2
+  exit 1
+fi
+
+mkdir -p "${DEST}"
+
+# Use a throwaway POM that depends on storm-kafka-monitor; copy-dependencies then
+# pulls the exact runtime closure. The artifact itself is a direct dependency and
+# is therefore copied too.
+TMP_DIR=$(mktemp -d)
+trap 'rm -rf "${TMP_DIR}"' EXIT
+cat > "${TMP_DIR}/pom.xml" <<EOF
+<project xmlns="http://maven.apache.org/POM/4.0.0">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.storm.tools</groupId>
+  <artifactId>storm-kafka-monitor-fetch</artifactId>
+  <version>${VERSION}</version>
+  <packaging>pom</packaging>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.storm</groupId>
+      <artifactId>storm-kafka-monitor</artifactId>
+      <version>${VERSION}</version>
+    </dependency>
+  </dependencies>
+</project>
+EOF
+
+echo "Fetching org.apache.storm:storm-kafka-monitor:${VERSION} (runtime closure) into:"
+echo "  ${DEST}"
+"${MVN}" -q -f "${TMP_DIR}/pom.xml" \
+  org.apache.maven.plugins:maven-dependency-plugin:copy-dependencies \
+  -DincludeScope=runtime \
+  -DoutputDirectory="${DEST}" \
+  ${MVN_ARGS[@]+"${MVN_ARGS[@]}"}
+
+echo "Done. ${DEST} now contains:"
+ls -1 "${DEST}" | sed 's/^/  /'
+echo
+echo "Restart the Storm UI to enable Kafka spout lag display, or run"
+echo "bin/storm-kafka-monitor directly. See external/storm-kafka-monitor/README.md."

bin/storm.py

@@ -107,6 +107,7 @@ def confvalue(name, storm_config_opts, extrapaths, overriding_conf_file=None, da
 
 def get_classpath(extrajars, daemon=True, client=False):
     ret = get_wildcard_dir(STORM_DIR)
+    ret.extend(get_wildcard_dir(STORM_COMMON_LIB_DIR))
     if client:
         ret.extend(get_wildcard_dir(STORM_WORKER_LIB_DIR))
     else:
@@ -125,9 +126,9 @@ def get_classpath(extrajars, daemon=True, client=False):
 def init_storm_env(within_unittest=False):
 
     global NORMAL_CLASS_PATH, STORM_DIR, USER_CONF_DIR, STORM_CONF_DIR, STORM_WORKER_LIB_DIR, STORM_LIB_DIR,\
-        STORM_TOOLS_LIB_DIR, STORM_WEBAPP_LIB_DIR, STORM_BIN_DIR, STORM_LOG4J2_CONF_DIR, STORM_SUPERVISOR_LOG_FILE,\
-        CLUSTER_CONF_DIR, JAR_JVM_OPTS, JAVA_HOME, JAVA_CMD, CONF_FILE, STORM_EXT_CLASSPATH, \
-        STORM_EXT_CLASSPATH_DAEMON, LOCAL_TTL_DEFAULT
+        STORM_COMMON_LIB_DIR, STORM_TOOLS_LIB_DIR, STORM_WEBAPP_LIB_DIR, STORM_BIN_DIR, STORM_LOG4J2_CONF_DIR,\
+        STORM_SUPERVISOR_LOG_FILE, CLUSTER_CONF_DIR, JAR_JVM_OPTS, JAVA_HOME, JAVA_CMD, CONF_FILE, \
+        STORM_EXT_CLASSPATH, STORM_EXT_CLASSPATH_DAEMON, LOCAL_TTL_DEFAULT
 
     NORMAL_CLASS_PATH = cygpath if sys.platform == 'cygwin' else identity
     STORM_DIR = os.sep.join(os.path.realpath( __file__ ).split(os.sep)[:-2])
@@ -141,6 +142,10 @@ def init_storm_env(within_unittest=False):
 
     STORM_WORKER_LIB_DIR = os.path.join(STORM_DIR, "lib-worker")
     STORM_LIB_DIR = os.path.join(STORM_DIR, "lib")
+    # Jars shared by the daemon (lib) and worker (lib-worker) classpaths are de-duplicated into
+    # lib-common to keep the distribution small. It is added to both classpaths; absent in older
+    # layouts, in which case it contributes nothing.
+    STORM_COMMON_LIB_DIR = os.path.join(STORM_DIR, "lib-common")
 
     STORM_TOOLS_LIB_DIR = os.path.join(STORM_DIR, "lib-tools")
     STORM_WEBAPP_LIB_DIR = os.path.join(STORM_DIR, "lib-webapp")

bin/test_storm.py

@@ -66,6 +66,19 @@ def test_get_classpath(self):
         expected = ":".join(extrajars)
         self.assertEqual(s[-len(expected):], expected)
 
+    def test_get_classpath_includes_lib_common(self):
+        extrajars = []
+        # When lib-common exists, it is included on both the daemon and the client classpaths.
+        storm.STORM_COMMON_LIB_DIR = storm.STORM_BIN_DIR
+        expected = os.path.join(storm.STORM_BIN_DIR, "*")
+        for client in (True, False):
+            cp = storm.get_classpath(extrajars, daemon=True, client=client)
+            self.assertIn(expected, cp.split(os.pathsep))
+        # When it does not exist, it contributes nothing (backward compatible with older layouts).
+        storm.STORM_COMMON_LIB_DIR = os.path.join(storm.STORM_DIR, "no-such-lib-common")
+        cp = storm.get_classpath(extrajars, daemon=True, client=False)
+        self.assertNotIn(os.path.join(storm.STORM_COMMON_LIB_DIR, "*"), cp.split(os.pathsep))
+
     def test_resolve_dependencies(self):
         artifacts = "org.apache.commons.commons-api"
         artifact_repositories = "maven-central"