[ISSUE #4700] Remove logging backends from runtime deps (#4719)

* [ISSUE #4700] Remove logging backends from runtime deps

This PR removes all the logging backends from the Maven artifacts. As
explained in #4700 libraries should only depend on logging APIs, not
logging implementations the same way web applications depend on the
Servlet API, not its implementations.

The logging backends are added to the `dist` folder to be included in
the TAR binary distribution.

* Fix licenses

* Fix Logback exclusions

* Fix license check

* Fix `printProjects` according to the review

* Add logging backend to `eventmesh-starter`

* Remove task description

* Fix task dependencies of task `installPlugin`

* Fix `installPlugin` task

* Add comment about exclusions

* Minimize changes to current configuration

This commit minimizes the changes to EventMesh dependencies.

Since a global exclusion is not an effective way to stop propagating
logging backends as **transitive** dependencies we:

* explictly remove logging backends from third-party dependencies that
  include them: RocketMQ, Pulsar, Spring Boot and Zookeeper,
* restore Log4j Core as dependency of `eventmesh-common`,
* exclude Log4j Core as dependency of `eventmesh-sdk-java`.

* Add comments to remove exclusions after upgrade

* Make `installPlugin` independent from `dist`

* Make `copy` tasks easier to understand

* Add `eventmesh-common` to EventMesh OpenConnect deps

* Refactor RocketMQ deps

* Delete `output.dirs`

* Fix typo

* Remove last `outputs.dir`

* Remove dependencies from `installPlugin`

* Add `eventmesh-common` to OpenConnect artifacts

Author: ppkarwasz

.gitignore

@@ -27,7 +27,7 @@ node_modules
 h2/db.mv.db
 
 # license check tmp file
-all-dependencies.txt
+/tools/dependency-check/all-dependencies.txt
 self-modules.txt
 third-party-dependencies.txt
 
@@ -50,4 +50,4 @@ bld/
 **/org/apache/eventmesh/connector/jdbc/antlr4/autogeneration/*
 
 #rust
-Cargo.lock
+Cargo.lock

build.gradle

@@ -139,7 +139,58 @@ allprojects {
     }
 }
 
-task tar(type: Tar) {
+tasks.register('dist') {
+    subprojects.forEach { subProject ->
+        dependsOn("${subProject.path}:jar")
+    }
+    def includedProjects =
+            ["eventmesh-common",
+             "eventmesh-meta:eventmesh-meta-api",
+             "eventmesh-metrics-plugin:eventmesh-metrics-api",
+             "eventmesh-protocol-plugin:eventmesh-protocol-api",
+             "eventmesh-retry:eventmesh-retry-api",
+             "eventmesh-runtime",
+             "eventmesh-security-plugin:eventmesh-security-api",
+             "eventmesh-spi",
+             "eventmesh-starter",
+             "eventmesh-storage-plugin:eventmesh-storage-api",
+             "eventmesh-trace-plugin:eventmesh-trace-api",
+             "eventmesh-webhook:eventmesh-webhook-api",
+             "eventmesh-webhook:eventmesh-webhook-admin",
+             "eventmesh-webhook:eventmesh-webhook-receive"]
+    doLast {
+        includedProjects.each {
+            def subProject = findProject(it)
+            logger.lifecycle('Install module: module: {}', subProject.name)
+            copy {
+                from subProject.jar.archivePath
+                into rootProject.file('dist/apps')
+            }
+            copy {
+                from subProject.file('bin')
+                into rootProject.file('dist/bin')
+            }
+            copy {
+                from subProject.file('conf')
+                from subProject.sourceSets.main.resources.srcDirs
+                into rootProject.file('dist/conf')
+                duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+                exclude 'META-INF'
+            }
+            copy {
+                from subProject.configurations.runtimeClasspath
+                into rootProject.file('dist/lib')
+                exclude 'eventmesh-*'
+            }
+        }
+        copy {
+            from 'tools/third-party-licenses'
+            into rootProject.file('dist')
+        }
+    }
+}
+
+tasks.register('tar', Tar) {
     archiveBaseName.set(project.name)
     archiveVersion.set(project.version.toString())
     archiveExtension.set('tar.gz')
@@ -150,7 +201,7 @@ task tar(type: Tar) {
     }
 }
 
-task zip(type: Zip) {
+tasks.register('zip', Zip) {
     archiveBaseName.set(project.name)
     archiveVersion.set(project.version.toString())
     archiveExtension.set('zip')
@@ -160,50 +211,39 @@ task zip(type: Zip) {
     }
 }
 
-task installPlugin() {
-    if (!new File("${rootDir}/dist").exists()) {
-        return
+tasks.register('installPlugin') {
+    var pluginProjects = subprojects.findAll {
+        it.file('gradle.properties').exists()
+                && it.properties.containsKey('pluginType')
+                && it.properties.containsKey('pluginName')
+    }
+    doLast {
+        String[] libJars = java.util.Optional.ofNullable(file('dist/lib').list()).orElse(new String[0])
+        pluginProjects.forEach(subProject -> {
+            var pluginType = subProject.properties.get('pluginType')
+            var pluginName = subProject.properties.get('pluginName')
+            logger.lifecycle('Install plugin: pluginType: {}, pluginInstanceName: {}, module: {}', pluginType,
+                    pluginName, subProject.name)
+            copy {
+                from subProject.jar.archivePath
+                into rootProject.file("dist/plugin/${pluginType}/${pluginName}")
+            }
+            copy {
+                from subProject.configurations.runtimeClasspath
+                into rootProject.file("dist/plugin/${pluginType}/${pluginName}")
+                exclude(libJars)
+            }
+            copy {
+                from subProject.file('conf')
+                from subProject.sourceSets.main.resources.srcDirs
+                into rootProject.file("dist/conf")
+                exclude 'META-INF'
+            }
+        })
     }
-    String[] libJars = java.util.Optional.ofNullable(new File("${rootDir}/dist/lib").list()).orElseGet(() -> new String[0])
-    getAllprojects().forEach(subProject -> {
-        var file = new File("${subProject.projectDir}/gradle.properties")
-        if (!file.exists()) {
-            return
-        }
-        var properties = new Properties()
-        properties.load(new FileInputStream(file))
-        var pluginType = properties.getProperty("pluginType")
-        var pluginName = properties.getProperty("pluginName")
-        if (pluginType == null || pluginName == null) {
-            return
-        }
-        var pluginFile = new File("${rootDir}/dist/plugin/${pluginType}/${pluginName}")
-        if (pluginFile.exists()) {
-            return
-        }
-        pluginFile.mkdirs()
-        println String.format(
-                "install plugin, pluginType: %s, pluginInstanceName: %s, module: %s", pluginType, pluginName, subProject.getName()
-        )
-
-        copy {
-            into "${rootDir}/dist/plugin/${pluginType}/${pluginName}"
-            from "${subProject.getProjectDir()}/dist/apps"
-        }
-        copy {
-            into "${rootDir}/dist/plugin/${pluginType}/${pluginName}"
-            from "${subProject.getProjectDir()}/dist/lib/"
-            exclude(libJars)
-        }
-        copy {
-            into "${rootDir}/dist/conf"
-            from "${subProject.getProjectDir()}/dist/conf"
-            exclude 'META-INF'
-        }
-    })
 }
 
-task printProjects() {
+tasks.register('printProjects') {
     getAllprojects().forEach(subProject -> {
         if ("EventMesh".equals(subProject.getName())) {
             return
@@ -303,77 +343,6 @@ subprojects {
         }
     }
 
-    task dist(dependsOn: ['jar']) {
-        doFirst {
-            new File("${projectDir}/dist/bin").mkdirs()
-            new File("${projectDir}/dist/apps").mkdirs()
-            new File("${projectDir}/dist/conf").mkdirs()
-            new File("${projectDir}/dist/lib").mkdirs()
-            new File("${projectDir}/dist/licenses").mkdirs()
-        }
-        Set<String> rootProject = ["eventmesh-common",
-                                   "eventmesh-storage-api",
-                                   "eventmesh-metrics-api",
-                                   "eventmesh-meta-api",
-                                   "eventmesh-trace-api",
-                                   "eventmesh-retry-api",
-                                   "eventmesh-runtime",
-                                   "eventmesh-security-api",
-                                   "eventmesh-protocol-api",
-                                   "eventmesh-starter",
-                                   "eventmesh-spi",
-                                   "eventmesh-webhook-api",
-                                   "eventmesh-webhook-admin",
-                                   "eventmesh-webhook-receive"]
-        doLast {
-            copy {
-                into("${projectDir}/dist/apps")
-                from project.jar.getArchivePath()
-            }
-            copy {
-                into("${projectDir}/dist/lib")
-                from project.configurations.runtimeClasspath
-            }
-            copy {
-                into("${projectDir}/dist/bin")
-                from 'bin'
-            }
-            copy {
-                into("${projectDir}/dist/conf")
-                from 'conf', sourceSets.main.resources.srcDirs
-                setDuplicatesStrategy(DuplicatesStrategy.EXCLUDE)
-                exclude 'META-INF'
-            }
-            if (rootProject.contains(project.name)) {
-                new File("${rootDir}/dist/apps").mkdirs()
-                new File("${rootDir}/dist/lib").mkdirs()
-                new File("${rootDir}/dist/bin").mkdirs()
-                new File("${rootDir}/dist/conf").mkdirs()
-                copy {
-                    into("${rootDir}/dist/apps")
-                    from "${projectDir}/dist/apps"
-                }
-                copy {
-                    into "${rootDir}/dist/lib"
-                    from "${projectDir}/dist/lib"
-                    exclude "eventmesh-*"
-                }
-                copy {
-                    into "${rootDir}/dist/bin"
-                    from "${projectDir}/dist/bin"
-                }
-                copy {
-                    into "${rootDir}/dist/conf"
-                    from "${projectDir}/dist/conf"
-                }
-            }
-            copy {
-                into "${rootDir}/dist"
-                from "${rootDir}/tools/third-party-licenses"
-            }
-        }
-    }
-
     javadoc {
         source = sourceSets.main.java
         destinationDir = reporting.file("javadoc")
@@ -491,7 +460,6 @@ subprojects {
             dependency "org.apache.logging.log4j:log4j-api:${log4jVersion}"
             dependency "org.apache.logging.log4j:log4j-core:${log4jVersion}"
             dependency "org.apache.logging.log4j:log4j-slf4j2-impl:${log4jVersion}"
-            dependency "org.apache.logging.log4j:log4j-slf4j-impl:${log4jVersion}" // used with SLF4J 1.7.x or older for third-party dependencies
 
             dependency "com.lmax:disruptor:3.4.2"
 

eventmesh-common/build.gradle

@@ -32,9 +32,8 @@ dependencies {
 
     api "com.alibaba.fastjson2:fastjson2"
 
-    implementation "org.apache.logging.log4j:log4j-api"
-    implementation "org.apache.logging.log4j:log4j-core"
-    implementation "org.apache.logging.log4j:log4j-slf4j2-impl"
+    runtimeOnly "org.apache.logging.log4j:log4j-core"
+    runtimeOnly "org.apache.logging.log4j:log4j-slf4j2-impl"
 
     implementation 'com.github.seancfoley:ipaddress'
 

eventmesh-connectors/eventmesh-connector-pravega/build.gradle

@@ -15,11 +15,6 @@
  * limitations under the License.
  */
 
-configurations {
-    implementation.exclude group: 'ch.qos.logback', module: 'logback-classic'
-    implementation.exclude group: 'log4j', module: 'log4j'
-}
-
 dependencies {
     api project(":eventmesh-openconnect:eventmesh-openconnect-java")
     implementation project(":eventmesh-common")

eventmesh-connectors/eventmesh-connector-pulsar/build.gradle

@@ -15,12 +15,27 @@
  * limitations under the License.
  */
 
-List pulsar = [
-        "org.apache.pulsar:pulsar-client:$pulsar_version"
-]
 dependencies {
     implementation project(":eventmesh-openconnect:eventmesh-openconnect-java")
-    implementation pulsar
+
+    /*
+     * TODO: This is a shaded artifact that contains 20 MiB of external libraries. It could probably be replaced by:
+     *
+     *     implementation "org.apache.pulsar:pulsar-client-api:$pulsar_version"
+     *     runtimeOnly "org.apache.pulsar:pulsar-client-original:$pulsar_version"
+     *
+     * The exclusions can be removed after an upgrade of the transitive:
+     *
+     *     "org.apache.bookkeeper:bookkeeper"
+     *
+     * dependency to 4.15.4 or higher (used by Pulsar 2.11.2 or higher).
+     */
+    implementation("org.apache.pulsar:pulsar-client:$pulsar_version") {
+        // Remove logging backend implementations
+        exclude group: 'org.apache.logging.log4j', module: 'log4j-core'
+        exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j-impl'
+    }
+
     compileOnly 'org.projectlombok:lombok'
     annotationProcessor 'org.projectlombok:lombok'
 }

eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle

@@ -16,11 +16,6 @@
  */
 
 
-configurations {
-    implementation.exclude group: 'ch.qos.logback', module: 'logback-classic'
-    implementation.exclude group: 'log4j', module: 'log4j'
-}
-
 dependencies {
     api project(":eventmesh-openconnect:eventmesh-openconnect-java")
     implementation project(":eventmesh-common")

eventmesh-connectors/eventmesh-connector-rocketmq/build.gradle

@@ -34,7 +34,16 @@ List rocketmq = [
 dependencies {
     api project(":eventmesh-openconnect:eventmesh-openconnect-java")
     implementation project(":eventmesh-common")
-    implementation rocketmq
+    /*
+     * The exclusions can be removed after this issue is fixed:
+     * https://github.com/apache/rocketmq/issues/5347
+     */
+    rocketmq.each {
+        implementation(it) {
+            exclude group: 'ch.qos.logback', module: 'logback-classic'
+        }
+    }
+
     compileOnly 'org.projectlombok:lombok'
     annotationProcessor 'org.projectlombok:lombok'
     testImplementation "org.mockito:mockito-core"

eventmesh-connectors/eventmesh-connector-slack/build.gradle

@@ -15,12 +15,6 @@
  * limitations under the License.
  */
 
-configurations {
-    implementation.exclude group: 'ch.qos.logback', module: 'logback-classic'
-    implementation.exclude group: 'log4j', module: 'log4j'
-    testImplementation.exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j'
-}
-
 dependencies {
     implementation project(":eventmesh-common")
     implementation project(":eventmesh-sdks:eventmesh-sdk-java")

eventmesh-connectors/eventmesh-connector-spring/build.gradle

@@ -15,20 +15,23 @@
  * limitations under the License.
  */
 
-configurations {
-    implementation.exclude group: 'ch.qos.logback', module: 'logback-classic'
-    implementation.exclude group: 'log4j', module: 'log4j'
-    implementation.exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j'
-    testImplementation.exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j'
-}
-
 dependencies {
     api project(":eventmesh-openconnect:eventmesh-openconnect-java")
     implementation project(":eventmesh-common")
     implementation project(":eventmesh-sdks:eventmesh-sdk-java")
-    implementation "org.springframework.boot:spring-boot-starter:$spring_boot_version"
-    implementation "org.springframework.boot:spring-boot-starter-validation:$spring_boot_version"
+
+    /*
+     * TODO: Are these dependencies necessary? The source code only requires these two dependencies
+     * that do not propagate logging backends:
+     *
+     *      api "org.springframework:spring-context:$spring_version"
+     *      implementation "org.springframework.boot:spring-boot-autoconfigure:$spring_boot_version"
+     */
+    implementation("org.springframework.boot:spring-boot-starter-validation:$spring_boot_version") {
+        exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging'
+    }
     implementation "org.springframework:spring-messaging:$spring_version"
+
     compileOnly 'org.projectlombok:lombok'
     annotationProcessor 'org.projectlombok:lombok'
 

eventmesh-connectors/eventmesh-connector-wechat/build.gradle

@@ -15,12 +15,6 @@
  * limitations under the License.
  */
 
-configurations {
-    implementation.exclude group: 'ch.qos.logback', module: 'logback-classic'
-    implementation.exclude group: 'log4j', module: 'log4j'
-    testImplementation.exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j'
-}
-
 dependencies {
     implementation project(":eventmesh-common")
     implementation project(":eventmesh-sdks:eventmesh-sdk-java")