From 1fdb5b2000a2f94e761367e32e901b9156f1d310 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 18 Apr 2024 08:27:11 +0200 Subject: [PATCH 1/4] [ISSUE #4700] Remove logging backends from runtime deps (#4719) * [ISSUE #4700] Remove logging backends from runtime deps This PR removes all the logging backends from the Maven artifacts. As explained in #4700 libraries should only depend on logging APIs, not logging implementations the same way web applications depend on the Servlet API, not its implementations. The logging backends are added to the `dist` folder to be included in the TAR binary distribution. * Fix licenses * Fix Logback exclusions * Fix license check * Fix `printProjects` according to the review * Add logging backend to `eventmesh-starter` * Remove task description * Fix task dependencies of task `installPlugin` * Fix `installPlugin` task * Add comment about exclusions * Minimize changes to current configuration This commit minimizes the changes to EventMesh dependencies. Since a global exclusion is not an effective way to stop propagating logging backends as **transitive** dependencies we: * explictly remove logging backends from third-party dependencies that include them: RocketMQ, Pulsar, Spring Boot and Zookeeper, * restore Log4j Core as dependency of `eventmesh-common`, * exclude Log4j Core as dependency of `eventmesh-sdk-java`. * Add comments to remove exclusions after upgrade * Make `installPlugin` independent from `dist` * Make `copy` tasks easier to understand * Add `eventmesh-common` to EventMesh OpenConnect deps * Refactor RocketMQ deps * Delete `output.dirs` * Fix typo * Remove last `outputs.dir` * Remove dependencies from `installPlugin` * Add `eventmesh-common` to OpenConnect artifacts --- .gitignore | 4 +- build.gradle | 198 ++++++++---------- eventmesh-common/build.gradle | 5 +- .../eventmesh-connector-pravega/build.gradle | 5 - .../eventmesh-connector-pulsar/build.gradle | 23 +- .../eventmesh-connector-rabbitmq/build.gradle | 5 - .../eventmesh-connector-rocketmq/build.gradle | 11 +- .../eventmesh-connector-slack/build.gradle | 6 - .../eventmesh-connector-spring/build.gradle | 21 +- .../eventmesh-connector-wechat/build.gradle | 6 - .../eventmesh-connector-wecom/build.gradle | 6 - eventmesh-examples/build.gradle | 7 +- .../eventmesh-meta-zookeeper/build.gradle | 10 +- .../eventmesh-openconnect-java/build.gradle | 4 +- .../build.gradle | 6 +- .../build.gradle | 4 +- .../eventmesh-retry-rocketmq/build.gradle | 10 +- .../eventmesh-sdk-java/build.gradle | 6 +- .../eventmesh-storage-kafka/build.gradle | 4 - .../eventmesh-storage-pulsar/build.gradle | 25 ++- .../eventmesh-storage-rabbitmq/build.gradle | 5 - .../eventmesh-storage-redis/build.gradle | 4 - .../eventmesh-storage-rocketmq/build.gradle | 17 +- tools/dependency-check/check-dependencies.sh | 8 +- tools/dependency-check/known-dependencies.txt | 5 - .../licenses/java/LICENSE-logback-core.txt | 14 -- .../licenses/java/LICENSE-slf4j-simple.txt | 21 -- 27 files changed, 189 insertions(+), 251 deletions(-) mode change 100644 => 100755 tools/dependency-check/check-dependencies.sh delete mode 100644 tools/third-party-licenses/licenses/java/LICENSE-logback-core.txt delete mode 100644 tools/third-party-licenses/licenses/java/LICENSE-slf4j-simple.txt diff --git a/.gitignore b/.gitignore index a22c82f795..2379b9f175 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,7 @@ node_modules h2/db.mv.db # license check tmp file -all-dependencies.txt +/tools/dependency-check/all-dependencies.txt self-modules.txt third-party-dependencies.txt @@ -50,4 +50,4 @@ bld/ **/org/apache/eventmesh/connector/jdbc/antlr4/autogeneration/* #rust -Cargo.lock \ No newline at end of file +Cargo.lock diff --git a/build.gradle b/build.gradle index 3cd8460b9a..fbaaa92640 100644 --- a/build.gradle +++ b/build.gradle @@ -139,7 +139,58 @@ allprojects { } } -task tar(type: Tar) { +tasks.register('dist') { + subprojects.forEach { subProject -> + dependsOn("${subProject.path}:jar") + } + def includedProjects = + ["eventmesh-common", + "eventmesh-meta:eventmesh-meta-api", + "eventmesh-metrics-plugin:eventmesh-metrics-api", + "eventmesh-protocol-plugin:eventmesh-protocol-api", + "eventmesh-retry:eventmesh-retry-api", + "eventmesh-runtime", + "eventmesh-security-plugin:eventmesh-security-api", + "eventmesh-spi", + "eventmesh-starter", + "eventmesh-storage-plugin:eventmesh-storage-api", + "eventmesh-trace-plugin:eventmesh-trace-api", + "eventmesh-webhook:eventmesh-webhook-api", + "eventmesh-webhook:eventmesh-webhook-admin", + "eventmesh-webhook:eventmesh-webhook-receive"] + doLast { + includedProjects.each { + def subProject = findProject(it) + logger.lifecycle('Install module: module: {}', subProject.name) + copy { + from subProject.jar.archivePath + into rootProject.file('dist/apps') + } + copy { + from subProject.file('bin') + into rootProject.file('dist/bin') + } + copy { + from subProject.file('conf') + from subProject.sourceSets.main.resources.srcDirs + into rootProject.file('dist/conf') + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + exclude 'META-INF' + } + copy { + from subProject.configurations.runtimeClasspath + into rootProject.file('dist/lib') + exclude 'eventmesh-*' + } + } + copy { + from 'tools/third-party-licenses' + into rootProject.file('dist') + } + } +} + +tasks.register('tar', Tar) { archiveBaseName.set(project.name) archiveVersion.set(project.version.toString()) archiveExtension.set('tar.gz') @@ -150,7 +201,7 @@ task tar(type: Tar) { } } -task zip(type: Zip) { +tasks.register('zip', Zip) { archiveBaseName.set(project.name) archiveVersion.set(project.version.toString()) archiveExtension.set('zip') @@ -160,50 +211,39 @@ task zip(type: Zip) { } } -task installPlugin() { - if (!new File("${rootDir}/dist").exists()) { - return +tasks.register('installPlugin') { + var pluginProjects = subprojects.findAll { + it.file('gradle.properties').exists() + && it.properties.containsKey('pluginType') + && it.properties.containsKey('pluginName') + } + doLast { + String[] libJars = java.util.Optional.ofNullable(file('dist/lib').list()).orElse(new String[0]) + pluginProjects.forEach(subProject -> { + var pluginType = subProject.properties.get('pluginType') + var pluginName = subProject.properties.get('pluginName') + logger.lifecycle('Install plugin: pluginType: {}, pluginInstanceName: {}, module: {}', pluginType, + pluginName, subProject.name) + copy { + from subProject.jar.archivePath + into rootProject.file("dist/plugin/${pluginType}/${pluginName}") + } + copy { + from subProject.configurations.runtimeClasspath + into rootProject.file("dist/plugin/${pluginType}/${pluginName}") + exclude(libJars) + } + copy { + from subProject.file('conf') + from subProject.sourceSets.main.resources.srcDirs + into rootProject.file("dist/conf") + exclude 'META-INF' + } + }) } - String[] libJars = java.util.Optional.ofNullable(new File("${rootDir}/dist/lib").list()).orElseGet(() -> new String[0]) - getAllprojects().forEach(subProject -> { - var file = new File("${subProject.projectDir}/gradle.properties") - if (!file.exists()) { - return - } - var properties = new Properties() - properties.load(new FileInputStream(file)) - var pluginType = properties.getProperty("pluginType") - var pluginName = properties.getProperty("pluginName") - if (pluginType == null || pluginName == null) { - return - } - var pluginFile = new File("${rootDir}/dist/plugin/${pluginType}/${pluginName}") - if (pluginFile.exists()) { - return - } - pluginFile.mkdirs() - println String.format( - "install plugin, pluginType: %s, pluginInstanceName: %s, module: %s", pluginType, pluginName, subProject.getName() - ) - - copy { - into "${rootDir}/dist/plugin/${pluginType}/${pluginName}" - from "${subProject.getProjectDir()}/dist/apps" - } - copy { - into "${rootDir}/dist/plugin/${pluginType}/${pluginName}" - from "${subProject.getProjectDir()}/dist/lib/" - exclude(libJars) - } - copy { - into "${rootDir}/dist/conf" - from "${subProject.getProjectDir()}/dist/conf" - exclude 'META-INF' - } - }) } -task printProjects() { +tasks.register('printProjects') { getAllprojects().forEach(subProject -> { if ("EventMesh".equals(subProject.getName())) { return @@ -303,77 +343,6 @@ subprojects { } } - task dist(dependsOn: ['jar']) { - doFirst { - new File("${projectDir}/dist/bin").mkdirs() - new File("${projectDir}/dist/apps").mkdirs() - new File("${projectDir}/dist/conf").mkdirs() - new File("${projectDir}/dist/lib").mkdirs() - new File("${projectDir}/dist/licenses").mkdirs() - } - Set rootProject = ["eventmesh-common", - "eventmesh-storage-api", - "eventmesh-metrics-api", - "eventmesh-meta-api", - "eventmesh-trace-api", - "eventmesh-retry-api", - "eventmesh-runtime", - "eventmesh-security-api", - "eventmesh-protocol-api", - "eventmesh-starter", - "eventmesh-spi", - "eventmesh-webhook-api", - "eventmesh-webhook-admin", - "eventmesh-webhook-receive"] - doLast { - copy { - into("${projectDir}/dist/apps") - from project.jar.getArchivePath() - } - copy { - into("${projectDir}/dist/lib") - from project.configurations.runtimeClasspath - } - copy { - into("${projectDir}/dist/bin") - from 'bin' - } - copy { - into("${projectDir}/dist/conf") - from 'conf', sourceSets.main.resources.srcDirs - setDuplicatesStrategy(DuplicatesStrategy.EXCLUDE) - exclude 'META-INF' - } - if (rootProject.contains(project.name)) { - new File("${rootDir}/dist/apps").mkdirs() - new File("${rootDir}/dist/lib").mkdirs() - new File("${rootDir}/dist/bin").mkdirs() - new File("${rootDir}/dist/conf").mkdirs() - copy { - into("${rootDir}/dist/apps") - from "${projectDir}/dist/apps" - } - copy { - into "${rootDir}/dist/lib" - from "${projectDir}/dist/lib" - exclude "eventmesh-*" - } - copy { - into "${rootDir}/dist/bin" - from "${projectDir}/dist/bin" - } - copy { - into "${rootDir}/dist/conf" - from "${projectDir}/dist/conf" - } - } - copy { - into "${rootDir}/dist" - from "${rootDir}/tools/third-party-licenses" - } - } - } - javadoc { source = sourceSets.main.java destinationDir = reporting.file("javadoc") @@ -491,7 +460,6 @@ subprojects { dependency "org.apache.logging.log4j:log4j-api:${log4jVersion}" dependency "org.apache.logging.log4j:log4j-core:${log4jVersion}" dependency "org.apache.logging.log4j:log4j-slf4j2-impl:${log4jVersion}" - dependency "org.apache.logging.log4j:log4j-slf4j-impl:${log4jVersion}" // used with SLF4J 1.7.x or older for third-party dependencies dependency "com.lmax:disruptor:3.4.2" diff --git a/eventmesh-common/build.gradle b/eventmesh-common/build.gradle index a6dc2cd269..23f9455e1a 100644 --- a/eventmesh-common/build.gradle +++ b/eventmesh-common/build.gradle @@ -32,9 +32,8 @@ dependencies { api "com.alibaba.fastjson2:fastjson2" - implementation "org.apache.logging.log4j:log4j-api" - implementation "org.apache.logging.log4j:log4j-core" - implementation "org.apache.logging.log4j:log4j-slf4j2-impl" + runtimeOnly "org.apache.logging.log4j:log4j-core" + runtimeOnly "org.apache.logging.log4j:log4j-slf4j2-impl" implementation 'com.github.seancfoley:ipaddress' diff --git a/eventmesh-connectors/eventmesh-connector-pravega/build.gradle b/eventmesh-connectors/eventmesh-connector-pravega/build.gradle index 0365334311..24876409f1 100644 --- a/eventmesh-connectors/eventmesh-connector-pravega/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-pravega/build.gradle @@ -15,11 +15,6 @@ * limitations under the License. */ -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' -} - dependencies { api project(":eventmesh-openconnect:eventmesh-openconnect-java") implementation project(":eventmesh-common") diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/build.gradle b/eventmesh-connectors/eventmesh-connector-pulsar/build.gradle index 4a532ec2b8..f087842ea8 100644 --- a/eventmesh-connectors/eventmesh-connector-pulsar/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-pulsar/build.gradle @@ -15,12 +15,27 @@ * limitations under the License. */ -List pulsar = [ - "org.apache.pulsar:pulsar-client:$pulsar_version" -] dependencies { implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation pulsar + + /* + * TODO: This is a shaded artifact that contains 20 MiB of external libraries. It could probably be replaced by: + * + * implementation "org.apache.pulsar:pulsar-client-api:$pulsar_version" + * runtimeOnly "org.apache.pulsar:pulsar-client-original:$pulsar_version" + * + * The exclusions can be removed after an upgrade of the transitive: + * + * "org.apache.bookkeeper:bookkeeper" + * + * dependency to 4.15.4 or higher (used by Pulsar 2.11.2 or higher). + */ + implementation("org.apache.pulsar:pulsar-client:$pulsar_version") { + // Remove logging backend implementations + exclude group: 'org.apache.logging.log4j', module: 'log4j-core' + exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j-impl' + } + compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' } \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle b/eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle index b4edbe1f8a..f96d81bf6d 100644 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle @@ -16,11 +16,6 @@ */ -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' -} - dependencies { api project(":eventmesh-openconnect:eventmesh-openconnect-java") implementation project(":eventmesh-common") diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/build.gradle b/eventmesh-connectors/eventmesh-connector-rocketmq/build.gradle index 769e9c6cf8..64c43e29d5 100644 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-rocketmq/build.gradle @@ -34,7 +34,16 @@ List rocketmq = [ dependencies { api project(":eventmesh-openconnect:eventmesh-openconnect-java") implementation project(":eventmesh-common") - implementation rocketmq + /* + * The exclusions can be removed after this issue is fixed: + * https://github.com/apache/rocketmq/issues/5347 + */ + rocketmq.each { + implementation(it) { + exclude group: 'ch.qos.logback', module: 'logback-classic' + } + } + compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' testImplementation "org.mockito:mockito-core" diff --git a/eventmesh-connectors/eventmesh-connector-slack/build.gradle b/eventmesh-connectors/eventmesh-connector-slack/build.gradle index ad66b78b96..851887e3c3 100644 --- a/eventmesh-connectors/eventmesh-connector-slack/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-slack/build.gradle @@ -15,12 +15,6 @@ * limitations under the License. */ -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' - testImplementation.exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j' -} - dependencies { implementation project(":eventmesh-common") implementation project(":eventmesh-sdks:eventmesh-sdk-java") diff --git a/eventmesh-connectors/eventmesh-connector-spring/build.gradle b/eventmesh-connectors/eventmesh-connector-spring/build.gradle index 48459b9004..e0680d2bf2 100644 --- a/eventmesh-connectors/eventmesh-connector-spring/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-spring/build.gradle @@ -15,20 +15,23 @@ * limitations under the License. */ -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' - implementation.exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j' - testImplementation.exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j' -} - dependencies { api project(":eventmesh-openconnect:eventmesh-openconnect-java") implementation project(":eventmesh-common") implementation project(":eventmesh-sdks:eventmesh-sdk-java") - implementation "org.springframework.boot:spring-boot-starter:$spring_boot_version" - implementation "org.springframework.boot:spring-boot-starter-validation:$spring_boot_version" + + /* + * TODO: Are these dependencies necessary? The source code only requires these two dependencies + * that do not propagate logging backends: + * + * api "org.springframework:spring-context:$spring_version" + * implementation "org.springframework.boot:spring-boot-autoconfigure:$spring_boot_version" + */ + implementation("org.springframework.boot:spring-boot-starter-validation:$spring_boot_version") { + exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' + } implementation "org.springframework:spring-messaging:$spring_version" + compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' diff --git a/eventmesh-connectors/eventmesh-connector-wechat/build.gradle b/eventmesh-connectors/eventmesh-connector-wechat/build.gradle index 9a73345d1d..afd4e115c0 100644 --- a/eventmesh-connectors/eventmesh-connector-wechat/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-wechat/build.gradle @@ -15,12 +15,6 @@ * limitations under the License. */ -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' - testImplementation.exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j' -} - dependencies { implementation project(":eventmesh-common") implementation project(":eventmesh-sdks:eventmesh-sdk-java") diff --git a/eventmesh-connectors/eventmesh-connector-wecom/build.gradle b/eventmesh-connectors/eventmesh-connector-wecom/build.gradle index 746a4f3722..e89567c6f9 100644 --- a/eventmesh-connectors/eventmesh-connector-wecom/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-wecom/build.gradle @@ -15,12 +15,6 @@ * limitations under the License. */ -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' - testImplementation.exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j' -} - dependencies { implementation project(":eventmesh-common") implementation project(":eventmesh-sdks:eventmesh-sdk-java") diff --git a/eventmesh-examples/build.gradle b/eventmesh-examples/build.gradle index 9ef70836bd..212b567799 100644 --- a/eventmesh-examples/build.gradle +++ b/eventmesh-examples/build.gradle @@ -17,15 +17,14 @@ def grpcVersion = '1.43.2' -configurations { - implementation.exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' -} - dependencies { implementation project(":eventmesh-sdks:eventmesh-sdk-java") implementation project(":eventmesh-common") implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") implementation project(":eventmesh-connectors:eventmesh-connector-spring") + implementation('org.springframework.boot:spring-boot-starter') { + exclude module: 'spring-boot-starter-logging' + } implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'io.netty:netty-all' implementation "io.cloudevents:cloudevents-core" diff --git a/eventmesh-meta/eventmesh-meta-zookeeper/build.gradle b/eventmesh-meta/eventmesh-meta-zookeeper/build.gradle index 1d8c871e2b..149f78e99d 100644 --- a/eventmesh-meta/eventmesh-meta-zookeeper/build.gradle +++ b/eventmesh-meta/eventmesh-meta-zookeeper/build.gradle @@ -20,7 +20,15 @@ dependencies { compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' - implementation 'org.apache.zookeeper:zookeeper' + /* + * The exclusion can be removed once ZOOKEEPER-4820 has been fixed. + * + * See https://github.com/apache/zookeeper/pull/2155 + */ + implementation('org.apache.zookeeper:zookeeper') { + exclude group: 'ch.qos.logback', module: 'logback-core' + exclude group: 'ch.qos.logback', module: 'logback-classic' + } implementation 'org.apache.curator:curator-client' implementation 'org.apache.curator:curator-framework' implementation 'org.apache.curator:curator-recipes' diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/build.gradle b/eventmesh-openconnect/eventmesh-openconnect-java/build.gradle index d47a81a4ff..b41f7fbfae 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/build.gradle +++ b/eventmesh-openconnect/eventmesh-openconnect-java/build.gradle @@ -17,9 +17,6 @@ dependencies { api "org.slf4j:slf4j-api" - implementation "org.apache.logging.log4j:log4j-api" - implementation "org.apache.logging.log4j:log4j-core" - implementation "org.apache.logging.log4j:log4j-slf4j2-impl" implementation "com.fasterxml.jackson.core:jackson-databind" implementation "com.fasterxml.jackson.core:jackson-core" @@ -29,6 +26,7 @@ dependencies { api project (":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api") implementation project (":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-nacos") implementation project(":eventmesh-sdks:eventmesh-sdk-java") + implementation project(":eventmesh-common") compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/build.gradle b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/build.gradle index 3f5f0a240b..537c74fafc 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/build.gradle +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/build.gradle @@ -17,17 +17,15 @@ dependencies { api "org.slf4j:slf4j-api" - implementation "org.apache.logging.log4j:log4j-api" - implementation "org.apache.logging.log4j:log4j-core" - implementation "org.apache.logging.log4j:log4j-slf4j2-impl" implementation "com.fasterxml.jackson.core:jackson-databind" implementation "com.fasterxml.jackson.core:jackson-core" implementation "com.fasterxml.jackson.core:jackson-annotations" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" + implementation project(":eventmesh-common") implementation project(":eventmesh-sdks:eventmesh-sdk-java") compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/build.gradle b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/build.gradle index c5ac62b00f..97c3b8c33c 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/build.gradle +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/build.gradle @@ -18,12 +18,10 @@ dependencies { api project(":eventmesh-spi") api "org.slf4j:slf4j-api" - implementation "org.apache.logging.log4j:log4j-api" - implementation "org.apache.logging.log4j:log4j-core" - implementation "org.apache.logging.log4j:log4j-slf4j2-impl" compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' + implementation project(":eventmesh-common") testCompileOnly 'org.projectlombok:lombok' testAnnotationProcessor 'org.projectlombok:lombok' diff --git a/eventmesh-retry/eventmesh-retry-rocketmq/build.gradle b/eventmesh-retry/eventmesh-retry-rocketmq/build.gradle index 3d33929b44..883081271c 100644 --- a/eventmesh-retry/eventmesh-retry-rocketmq/build.gradle +++ b/eventmesh-retry/eventmesh-retry-rocketmq/build.gradle @@ -33,7 +33,15 @@ List rocketmq = [ dependencies { implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") implementation project(":eventmesh-storage-plugin:eventmesh-storage-rocketmq") - implementation rocketmq + /* + * The exclusions can be removed after this issue is fixed: + * https://github.com/apache/rocketmq/issues/5347 + */ + rocketmq.each { + implementation(it) { + exclude group: 'ch.qos.logback', module: 'logback-classic' + } + } implementation project(":eventmesh-retry:eventmesh-retry-api") implementation project(":eventmesh-common") diff --git a/eventmesh-sdks/eventmesh-sdk-java/build.gradle b/eventmesh-sdks/eventmesh-sdk-java/build.gradle index db15554031..c83b77d26b 100644 --- a/eventmesh-sdks/eventmesh-sdk-java/build.gradle +++ b/eventmesh-sdks/eventmesh-sdk-java/build.gradle @@ -18,7 +18,11 @@ def grpcVersion = '1.43.2' dependencies { - api project(":eventmesh-common") + api(project(":eventmesh-common")) { + // Remove logging backend implementations to allow users to choose their own + exclude group: 'org.apache.logging.log4j', module: 'log4j-core' + exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j2-impl' + } implementation "com.fasterxml.jackson.core:jackson-databind" implementation "com.fasterxml.jackson.core:jackson-core" diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle index d776cc46a7..1a467ed8c3 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle +++ b/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle @@ -14,10 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' -} dependencies { implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") diff --git a/eventmesh-storage-plugin/eventmesh-storage-pulsar/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-pulsar/build.gradle index 77922e241a..d439016bea 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-pulsar/build.gradle +++ b/eventmesh-storage-plugin/eventmesh-storage-pulsar/build.gradle @@ -15,19 +15,30 @@ * limitations under the License. */ -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' -} - dependencies { implementation project(":eventmesh-common") implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - implementation 'org.apache.pulsar:pulsar-client:2.10.1' + + /* + * TODO: This is a shaded artifact that contains 20 MiB of external libraries. It could probably be replaced by: + * + * implementation "org.apache.pulsar:pulsar-client-api:$pulsar_version" + * runtimeOnly "org.apache.pulsar:pulsar-client-original:$pulsar_version" + * + * The exclusions can be removed after an upgrade of the transitive: + * + * "org.apache.bookkeeper:bookkeeper" + * + * dependency to 4.15.4 or higher (used by Pulsar 2.11.2 or higher). + */ + implementation('org.apache.pulsar:pulsar-client:2.10.1') { + // Remove logging backend implementations + exclude group: 'org.apache.logging.log4j', module: 'log4j-core' + exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j-impl' + } testImplementation project(":eventmesh-storage-plugin:eventmesh-storage-api") testImplementation project(":eventmesh-common") - testImplementation 'org.apache.pulsar:pulsar-client:2.10.1' implementation 'io.cloudevents:cloudevents-json-jackson' diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/build.gradle index 69b4f75bac..2e8ddac448 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/build.gradle +++ b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/build.gradle @@ -15,11 +15,6 @@ * limitations under the License. */ -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' -} - dependencies { implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") implementation project(":eventmesh-common") diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-redis/build.gradle index 5a103cb8e3..bec0767638 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/build.gradle +++ b/eventmesh-storage-plugin/eventmesh-storage-redis/build.gradle @@ -15,10 +15,6 @@ * limitations under the License. */ -configurations { - implementation.exclude group: 'org.slf4j', module: 'slf4j-simple' -} - dependencies { implementation project(":eventmesh-common") implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") diff --git a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/build.gradle index be30240097..fd14357fb9 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-rocketmq/build.gradle +++ b/eventmesh-storage-plugin/eventmesh-storage-rocketmq/build.gradle @@ -15,11 +15,6 @@ * limitations under the License. */ -configurations { - implementation.exclude group: 'ch.qos.logback', module: 'logback-classic' - implementation.exclude group: 'log4j', module: 'log4j' -} - List rocketmq = [ "org.apache.rocketmq:rocketmq-client:$rocketmq_version", "org.apache.rocketmq:rocketmq-broker:$rocketmq_version", @@ -39,7 +34,15 @@ List rocketmq = [ dependencies { implementation project(":eventmesh-common") implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") - implementation rocketmq + /* + * The exclusions can be removed after this issue is fixed: + * https://github.com/apache/rocketmq/issues/5347 + */ + rocketmq.each { + implementation(it) { + exclude group: 'ch.qos.logback', module: 'logback-classic' + } + } testImplementation project(":eventmesh-storage-plugin:eventmesh-storage-api") testImplementation project(":eventmesh-common") @@ -47,8 +50,6 @@ dependencies { testImplementation "org.mockito:mockito-core" testImplementation "org.mockito:mockito-junit-jupiter" - testImplementation rocketmq - compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' diff --git a/tools/dependency-check/check-dependencies.sh b/tools/dependency-check/check-dependencies.sh old mode 100644 new mode 100755 index 5353df817e..3842323c1f --- a/tools/dependency-check/check-dependencies.sh +++ b/tools/dependency-check/check-dependencies.sh @@ -1,4 +1,4 @@ -#!/usr/bin bash +#!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -34,14 +34,14 @@ self_modules_txt='tools/dependency-check/self-modules.txt' # store all third part dependencies third_party_dependencies_txt='tools/dependency-check/third-party-dependencies.txt' -mkdir $decompress_conf || true +mkdir -p $decompress_conf tar -zxf build/eventmesh*.tar.gz -C $decompress_conf ./gradlew printProjects | grep '.jar' > "$self_modules_txt" -find "$decompress_conf" -name "*.jar" -exec basename {} \; | uniq | sort > "$all_dependencies_txt" +find "$decompress_conf" -name "*.jar" -exec basename {} \; | sort | uniq > "$all_dependencies_txt" -grep -wvf "$self_modules_txt" "$all_dependencies_txt" | uniq | sort > "$third_party_dependencies_txt" +grep -wvf "$self_modules_txt" "$all_dependencies_txt" | sort | uniq > "$third_party_dependencies_txt" # If the check is success it will return 0 sort "$known_third_party_dependencies_txt" | diff - "$third_party_dependencies_txt" diff --git a/tools/dependency-check/known-dependencies.txt b/tools/dependency-check/known-dependencies.txt index b30ca5d5ed..f6627a693e 100644 --- a/tools/dependency-check/known-dependencies.txt +++ b/tools/dependency-check/known-dependencies.txt @@ -154,15 +154,11 @@ json-path-2.7.0.jar json-smart-2.4.7.jar json-utils-2.20.29.jar jsr305-3.0.2.jar -jul-to-slf4j-1.7.33.jar kafka-clients-3.0.0.jar listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar log4j-api-2.22.1.jar log4j-core-2.22.1.jar -log4j-slf4j-impl-2.22.1.jar log4j-slf4j2-impl-2.22.1.jar -logback-classic-1.2.10.jar -logback-core-1.2.10.jar lz4-java-1.7.1.jar lz4-java-1.8.0.jar metrics-annotation-4.1.0.jar @@ -325,7 +321,6 @@ spring-beans-5.3.20.jar spring-boot-2.5.9.jar spring-boot-autoconfigure-2.5.9.jar spring-boot-starter-2.5.9.jar -spring-boot-starter-logging-2.5.9.jar spring-boot-starter-validation-2.5.9.jar spring-context-5.3.15.jar spring-core-5.3.20.jar diff --git a/tools/third-party-licenses/licenses/java/LICENSE-logback-core.txt b/tools/third-party-licenses/licenses/java/LICENSE-logback-core.txt deleted file mode 100644 index 8953762a3c..0000000000 --- a/tools/third-party-licenses/licenses/java/LICENSE-logback-core.txt +++ /dev/null @@ -1,14 +0,0 @@ -Logback LICENSE ---------------- - -Logback: the reliable, generic, fast and flexible logging framework. -Copyright (C) 1999-2015, QOS.ch. All rights reserved. - -This program and the accompanying materials are dual-licensed under -either the terms of the Eclipse Public License v1.0 as published by -the Eclipse Foundation - - or (per the licensee's choosing) - -under the terms of the GNU Lesser General Public License version 2.1 -as published by the Free Software Foundation. \ No newline at end of file diff --git a/tools/third-party-licenses/licenses/java/LICENSE-slf4j-simple.txt b/tools/third-party-licenses/licenses/java/LICENSE-slf4j-simple.txt deleted file mode 100644 index 744377c437..0000000000 --- a/tools/third-party-licenses/licenses/java/LICENSE-slf4j-simple.txt +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2017 QOS.ch -All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From a899e549d879716fe264c8e7bc28dbf3bf411241 Mon Sep 17 00:00:00 2001 From: mxsm Date: Fri, 19 Apr 2024 10:00:21 +0800 Subject: [PATCH 2/4] [ISSUE #3430] Refactor eventmesh-metrics-prometheus module (#4840) * [ISSUE #3430]Refactor eventmesh-metrics-prometheus module * optimize code * optimize code --- build.gradle | 16 +- .../eventmesh/common/MetricsConstants.java | 65 ++ .../org/apache/eventmesh}/common/Pair.java | 31 +- .../eventmesh/common/enums/ProtocolType.java | 14 +- .../eventmesh-metrics-api/build.gradle | 3 + .../metrics/api/MetricsRegistry.java | 15 + .../metrics/api/model/AbstractMetric.java | 49 ++ .../model/AbstractObservableDoubleMetric.java | 49 ++ .../model/AbstractObservableLongMetric.java | 48 + .../api/model/AbstractObservableMetric.java | 64 ++ .../metrics/api/model/AbstractSyncMetric.java | 29 + .../api/model/DoubleCounterMetric.java | 57 ++ .../api/model/DoubleHistogramMetric.java | 57 ++ .../api/model/DoubleUpDownCounterMetric.java | 57 ++ .../metrics/api/model/GrpcSummaryMetrics.java | 105 --- .../metrics/api/model/HttpSummaryMetrics.java | 436 --------- .../metrics/api/model/InstrumentFurther.java | 45 + .../metrics/api/model/InstrumentType.java | 57 ++ .../metrics/api/model/LongCounterMetric.java | 57 ++ .../api/model/LongHistogramMetric.java | 57 ++ .../api/model/LongUpDownCounterMetric.java | 57 ++ .../eventmesh/metrics/api/model/Metric.java | 27 + .../metrics/api/model/NoopDoubleCounter.java | 64 ++ .../api/model/NoopDoubleHistogram.java | 64 ++ .../api/model/NoopDoubleUpDownCounter.java | 64 ++ .../metrics/api/model/NoopLongCounter.java | 64 ++ .../metrics/api/model/NoopLongHistogram.java | 64 ++ .../api/model/NoopLongUpDownCounter.java | 64 ++ .../model/ObservableDoubleCounterMetric.java | 41 + .../model/ObservableDoubleGaugeMetric.java | 37 + .../ObservableDoubleUpDownCounterMetric.java | 38 + .../model/ObservableLongCounterMetric.java | 40 + .../api/model/ObservableLongGaugeMetric.java | 41 + .../ObservableLongUpDownCounterMetric.java | 39 + .../metrics/api/model/ObservableMetric.java | 56 ++ .../metrics/api/model/SyncMetric.java} | 11 +- .../metrics/api/model/TcpSummaryMetrics.java | 140 --- .../api/model/GrpcSummaryMetricsTest.java | 42 - .../eventmesh-metrics-prometheus/build.gradle | 11 +- .../OpenTelemetryPrometheusManager.java | 51 ++ .../prometheus/PrometheusMetricsRegistry.java | 69 +- .../PrometheusMetricsRegistryManager.java | 278 ++++++ .../config/PrometheusConfiguration.java | 3 + .../metrics/PrometheusGrpcExporter.java | 60 -- .../metrics/PrometheusHttpExporter.java | 120 --- .../metrics/PrometheusTcpExporter.java | 69 -- .../utils/PrometheusExporterUtils.java | 76 -- .../admin/handler/v1/MetricsHandler.java | 96 +- .../admin/response/v1/GetMetricsResponse.java | 99 ++- .../runtime/boot/AbstractHTTPServer.java | 23 +- .../runtime/boot/AbstractTCPServer.java | 24 +- .../runtime/boot/EventMeshGrpcServer.java | 23 +- .../runtime/boot/EventMeshHTTPServer.java | 10 +- .../runtime/boot/EventMeshServer.java | 43 + .../runtime/boot/EventMeshTCPServer.java | 11 +- .../grpc/consumer/EventMeshConsumer.java | 2 +- .../BatchPublishCloudEventProcessor.java | 2 +- .../PublishCloudEventsProcessor.java | 4 +- .../grpc/processor/ReplyMessageProcessor.java | 2 +- .../processor/RequestCloudEventProcessor.java | 6 +- .../grpc/service/ConsumerService.java | 9 +- .../grpc/service/PublisherService.java | 9 +- .../processor/BatchSendMessageProcessor.java | 4 +- .../BatchSendMessageV2Processor.java | 4 +- .../http/processor/CreateTopicProcessor.java | 6 +- .../http/processor/DeleteTopicProcessor.java | 6 +- .../http/processor/HandlerService.java | 16 +- .../http/processor/HeartBeatProcessor.java | 10 +- .../processor/QuerySubscriptionProcessor.java | 6 +- .../http/processor/ReplyMessageProcessor.java | 4 +- .../processor/SendAsyncEventProcessor.java | 5 +- .../processor/SendAsyncMessageProcessor.java | 4 +- .../SendAsyncRemoteEventProcessor.java | 5 +- .../processor/SendSyncMessageProcessor.java | 6 +- .../http/processor/SubscribeProcessor.java | 11 +- .../http/processor/UnSubscribeProcessor.java | 4 +- .../http/push/AsyncHTTPPushRequest.java | 6 +- .../tcp/client/EventMeshTcp2Client.java | 7 +- .../tcp/client/group/ClientGroupWrapper.java | 24 +- .../protocol/tcp/client/session/Session.java | 6 +- .../client/session/push/SessionPusher.java | 8 +- .../client/session/send/SessionSender.java | 11 +- .../metrics/EventMeshMetricsManager.java | 128 +++ .../runtime/metrics/GeneralMetrics.java | 81 ++ .../runtime/metrics/MetricInstrumentUnit.java | 35 + .../runtime/metrics/MetricsManager.java | 52 ++ .../runtime/metrics/MetricsUtils.java | 110 +++ .../metrics/MonitorMetricConstants.java | 2 + .../grpc/EventMeshGrpcMetricsManager.java | 148 ++++ .../metrics/grpc/EventMeshGrpcMonitor.java | 96 -- .../runtime/metrics/grpc/GrpcMetrics.java | 263 ++++++ .../http/EventMeshHttpMetricsManager.java | 242 +++++ .../metrics/http/HTTPMetricsServer.java | 165 ---- .../runtime/metrics/http/HttpMetrics.java | 827 ++++++++++++++++++ .../tcp/EventMeshTcpMetricsManager.java | 130 +++ .../metrics/tcp/EventMeshTcpMonitor.java | 186 ---- .../runtime/metrics/tcp/TcpMetrics.java | 280 ++++++ .../metrics/tcp/TcpMetricsCalculator.java | 134 +++ .../apache/eventmesh/runtime/util/Utils.java | 3 +- tools/dependency-check/known-dependencies.txt | 50 +- 100 files changed, 4610 insertions(+), 1839 deletions(-) create mode 100644 eventmesh-common/src/main/java/org/apache/eventmesh/common/MetricsConstants.java rename {eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime => eventmesh-common/src/main/java/org/apache/eventmesh}/common/Pair.java (63%) rename eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/RetrySummaryMetrics.java => eventmesh-common/src/main/java/org/apache/eventmesh/common/enums/ProtocolType.java (79%) create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableDoubleMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableLongMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractSyncMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleCounterMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleHistogramMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleUpDownCounterMetric.java delete mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/GrpcSummaryMetrics.java delete mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/HttpSummaryMetrics.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentFurther.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentType.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongCounterMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongHistogramMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongUpDownCounterMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleCounter.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleHistogram.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleUpDownCounter.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongCounter.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongHistogram.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongUpDownCounter.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleCounterMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleGaugeMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleUpDownCounterMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongCounterMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongGaugeMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongUpDownCounterMetric.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableMetric.java rename eventmesh-metrics-plugin/{eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/utils/PrometheusExporterConstants.java => eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/SyncMetric.java} (74%) delete mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/TcpSummaryMetrics.java delete mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-api/src/test/java/org/apache/eventmesh/metrics/api/model/GrpcSummaryMetricsTest.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/OpenTelemetryPrometheusManager.java create mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistryManager.java delete mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusGrpcExporter.java delete mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusHttpExporter.java delete mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusTcpExporter.java delete mode 100644 eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/utils/PrometheusExporterUtils.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/EventMeshMetricsManager.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/GeneralMetrics.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricInstrumentUnit.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsManager.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsUtils.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/EventMeshGrpcMetricsManager.java delete mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/EventMeshGrpcMonitor.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/GrpcMetrics.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/EventMeshHttpMetricsManager.java delete mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HTTPMetricsServer.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HttpMetrics.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/EventMeshTcpMetricsManager.java delete mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/EventMeshTcpMonitor.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetrics.java create mode 100644 eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetricsCalculator.java diff --git a/build.gradle b/build.gradle index fbaaa92640..fb84e3f892 100644 --- a/build.gradle +++ b/build.gradle @@ -481,15 +481,13 @@ subprojects { dependency "io.dropwizard.metrics:metrics-annotation:4.1.0" dependency "io.dropwizard.metrics:metrics-json:4.1.0" - dependency 'io.opentelemetry:opentelemetry-api:1.3.0' - dependency 'io.opentelemetry:opentelemetry-sdk:1.3.0' - dependency 'io.opentelemetry:opentelemetry-sdk-metrics:1.3.0-alpha' - dependency 'io.opentelemetry:opentelemetry-exporter-prometheus:1.3.0-alpha' - dependency 'io.prometheus:simpleclient:0.8.1' - dependency 'io.prometheus:simpleclient_httpserver:0.8.1' - dependency 'io.opentelemetry:opentelemetry-exporter-zipkin:1.3.0' - dependency 'io.opentelemetry:opentelemetry-semconv:1.3.0-alpha' - dependency 'io.opentelemetry:opentelemetry-exporter-jaeger:1.4.0' + dependency 'io.opentelemetry:opentelemetry-api:1.36.0' + dependency 'io.opentelemetry:opentelemetry-sdk:1.36.0' + dependency 'io.opentelemetry:opentelemetry-sdk-metrics:1.36.0' + dependency 'io.opentelemetry:opentelemetry-exporter-prometheus:1.36.0-alpha' + dependency 'io.opentelemetry:opentelemetry-exporter-zipkin:1.36.0' + dependency 'io.opentelemetry:opentelemetry-semconv:1.30.1-alpha' + dependency 'io.opentelemetry:opentelemetry-exporter-jaeger:1.34.1' dependency "io.openmessaging:openmessaging-api:2.2.1-pubsub" diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/MetricsConstants.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/MetricsConstants.java new file mode 100644 index 0000000000..d1c74c90c3 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/MetricsConstants.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common; + + +public class MetricsConstants { + + private MetricsConstants() { + + } + + public static final String UNKOWN = "unkown"; + + public static final String LABEL_PROCESSOR = "processor"; + + public static final String CLIENT_ADDRESS = "client.address"; + + public static final String RPC_SYSTEM = "rpc.system"; + + public static final String RPC_SERVICE = "rpc.service"; + + //GRPC-https://opentelemetry.io/docs/reference/specification/metrics/semantic_conventions/rpc-metrics/ + public static final String GRPC_NET_PEER_PORT = "net.peer.port"; + + public static final String GRPC_NET_PEER_NAME = "net.peer.name"; + + public static final String GRPC_NET_SOCK_PEER_ADDR = "net.sock.peer.addr"; + + public static final String GRPC_NET_SOCK_PEER_PORT = "net.sock.peer.port"; + + // HTTP https://opentelemetry.io/docs/reference/specification/metrics/semantic_conventions/http-metrics/ + + public static final String HTTP_HTTP_SCHEME = "http.scheme"; + + public static final String HTTP_HTTP_FLAVOR = "http.flavor"; + + public static final String HTTP_NET_HOST_NAME = "net.host.name"; + + public static final String HTTP_NET_HOST_PORT = "net.host.port"; + + //TCP + public static final String TCP_NET_HOST_NAME = "net.host.name"; + + public static final String TCP_NET_HOST_PORT = "net.host.port"; + + + public static final String CLIENT_PROTOCOL_TYPE = "client.protocol.type"; + + +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/Pair.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/Pair.java similarity index 63% rename from eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/Pair.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/Pair.java index ddab176557..25b2a6f3c4 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/common/Pair.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/Pair.java @@ -15,31 +15,32 @@ * limitations under the License. */ -package org.apache.eventmesh.runtime.common; +package org.apache.eventmesh.common; -public class Pair { +public class Pair { - private T1 object1; - private T2 object2; + private Left left; - public Pair(T1 object1, T2 object2) { - this.object1 = object1; - this.object2 = object2; + private Right right; + + public Pair(Left left, Right right) { + this.left = left; + this.right = right; } - public T1 getObject1() { - return object1; + public Left getLeft() { + return left; } - public void setObject1(T1 object1) { - this.object1 = object1; + public void setLeft(Left left) { + this.left = left; } - public T2 getObject2() { - return object2; + public Right getRight() { + return right; } - public void setObject2(T2 object2) { - this.object2 = object2; + public void setRight(Right right) { + this.right = right; } } diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/RetrySummaryMetrics.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/enums/ProtocolType.java similarity index 79% rename from eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/RetrySummaryMetrics.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/enums/ProtocolType.java index 1962527cae..d7ac9aabbb 100644 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/RetrySummaryMetrics.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/enums/ProtocolType.java @@ -15,14 +15,10 @@ * limitations under the License. */ -package org.apache.eventmesh.metrics.api.model; +package org.apache.eventmesh.common.enums; -import lombok.AllArgsConstructor; -import lombok.Data; - -@Data -@AllArgsConstructor -public class RetrySummaryMetrics { - - private long pendingRetryTimeouts; +public enum ProtocolType { + TCP, + HTTP, + GRPC } diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/build.gradle b/eventmesh-metrics-plugin/eventmesh-metrics-api/build.gradle index 3f433bac4e..ddc5f43270 100644 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/build.gradle +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/build.gradle @@ -19,6 +19,7 @@ dependencies { api project(":eventmesh-spi") implementation project(":eventmesh-common") + implementation 'io.opentelemetry:opentelemetry-api' compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' @@ -27,4 +28,6 @@ dependencies { testAnnotationProcessor 'org.projectlombok:lombok' testImplementation "org.mockito:mockito-core" + testImplementation "org.mockito:mockito-inline" } + diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/MetricsRegistry.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/MetricsRegistry.java index 2c99af891d..b805ec518e 100644 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/MetricsRegistry.java +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/MetricsRegistry.java @@ -21,6 +21,10 @@ import org.apache.eventmesh.spi.EventMeshExtensionType; import org.apache.eventmesh.spi.EventMeshSPI; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.Collection; + /** * The top-level interface of metrics registry, used to register different metrics. It should have multiple sub implementation, e.g. JVM, Prometheus, * i.g. @@ -47,6 +51,17 @@ public interface MetricsRegistry { */ void register(Metric metric); + /** + * Register Metrics, if the metric is already exist, it will do nothing. + * + * @param metrics + */ + default void register(Collection metrics) { + if (CollectionUtils.isNotEmpty(metrics)) { + metrics.forEach(metric -> register(metric)); + } + } + /** * Remove a metric, if the metric is not exist, it will do nothing. * diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractMetric.java new file mode 100644 index 0000000000..59f1b09323 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractMetric.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +public abstract class AbstractMetric implements Metric { + + private InstrumentFurther further; + + private String metricName; + + public AbstractMetric(InstrumentFurther further, String metricName) { + this.further = further; + this.metricName = metricName; + } + + public AbstractMetric() { + + } + + @Override + public void setInstrumentFurther(InstrumentFurther further) { + this.further = further; + } + + @Override + public InstrumentFurther getInstrumentFurther() { + return this.further; + } + + @Override + public String getName() { + return metricName; + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableDoubleMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableDoubleMetric.java new file mode 100644 index 0000000000..b0165863ca --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableDoubleMetric.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + + +import java.util.function.Supplier; + +public abstract class AbstractObservableDoubleMetric extends AbstractObservableMetric { + + private Supplier supplier; + + public AbstractObservableDoubleMetric(InstrumentFurther further, String metricName) { + super(further, metricName); + } + + public AbstractObservableDoubleMetric(InstrumentFurther further, String metricName, Supplier supplier) { + super(further, metricName); + this.supplier = supplier; + } + + public AbstractObservableDoubleMetric() { + + } + + public void supplier(Supplier supplier) { + this.supplier = supplier; + } + + @Override + public Supplier supplier() { + return supplier; + } + +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableLongMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableLongMetric.java new file mode 100644 index 0000000000..bb8189329e --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableLongMetric.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + + +import java.util.function.Supplier; + +public abstract class AbstractObservableLongMetric extends AbstractObservableMetric { + + private Supplier supplier; + + public AbstractObservableLongMetric(InstrumentFurther further, String metricName) { + super(further, metricName); + } + + public AbstractObservableLongMetric(InstrumentFurther further, String metricName, Supplier supplier) { + super(further, metricName); + this.supplier = supplier; + } + + public AbstractObservableLongMetric() { + + } + + public void supplier(Supplier supplier) { + this.supplier = supplier; + } + + @Override + public Supplier supplier() { + return supplier; + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableMetric.java new file mode 100644 index 0000000000..7c96969fbc --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractObservableMetric.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public abstract class AbstractObservableMetric extends AbstractMetric implements ObservableMetric { + + private Map attributes = new HashMap<>(32); + + public AbstractObservableMetric(InstrumentFurther further, String metricName) { + super(further, metricName); + } + + public AbstractObservableMetric() { + + } + + @Override + public void put(String key, String value) { + this.attributes.put(key, value); + } + + @Override + public void putAll(Map attributes) { + if (Objects.isNull(attributes)) { + return; + } + this.attributes.putAll(attributes); + } + + @Override + public Map getAttributes() { + return this.attributes; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractSyncMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractSyncMetric.java new file mode 100644 index 0000000000..d1ecbee53f --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/AbstractSyncMetric.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +public abstract class AbstractSyncMetric extends AbstractMetric implements SyncMetric { + + public AbstractSyncMetric(InstrumentFurther further, String metricName) { + super(further, metricName); + } + + public AbstractSyncMetric() { + + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleCounterMetric.java new file mode 100644 index 0000000000..b8131184be --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleCounterMetric.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.metrics.DoubleCounter; + +public class DoubleCounterMetric extends AbstractSyncMetric { + + private DoubleCounter counter; + + public DoubleCounterMetric(InstrumentFurther further, String metricName, DoubleCounter counter) { + super(further, metricName); + this.counter = counter; + } + + public DoubleCounterMetric(InstrumentFurther further, String metricName) { + super(further, metricName); + } + + public DoubleCounterMetric(String metricName) { + super(null, metricName); + } + + public DoubleCounterMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.DOUBLE_COUNTER; + } + + @Override + public DoubleCounter getInstrument() { + return this.counter; + } + + @Override + public void setInstrument(DoubleCounter counter) { + this.counter = counter; + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleHistogramMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleHistogramMetric.java new file mode 100644 index 0000000000..0f1c32d569 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleHistogramMetric.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.metrics.DoubleHistogram; + +public class DoubleHistogramMetric extends AbstractSyncMetric { + + private DoubleHistogram histogram; + + public DoubleHistogramMetric(InstrumentFurther further, String metricName, DoubleHistogram histogram) { + super(further, metricName); + this.histogram = histogram; + } + + public DoubleHistogramMetric(InstrumentFurther further, String metricName) { + super(further, metricName); + } + + public DoubleHistogramMetric(String metricName) { + super(null, metricName); + } + + public DoubleHistogramMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.DOUBLE_HISTOGRAM; + } + + @Override + public DoubleHistogram getInstrument() { + return this.histogram; + } + + @Override + public void setInstrument(DoubleHistogram counter) { + this.histogram = counter; + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleUpDownCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleUpDownCounterMetric.java new file mode 100644 index 0000000000..f617615be8 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/DoubleUpDownCounterMetric.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.metrics.DoubleUpDownCounter; + +public class DoubleUpDownCounterMetric extends AbstractSyncMetric { + + private DoubleUpDownCounter counter; + + public DoubleUpDownCounterMetric(InstrumentFurther further, String metricName, DoubleUpDownCounter counter) { + super(further, metricName); + this.counter = counter; + } + + public DoubleUpDownCounterMetric(InstrumentFurther further, String metricName) { + super(further, metricName); + } + + public DoubleUpDownCounterMetric(String metricName) { + super(null, metricName); + } + + public DoubleUpDownCounterMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.DOUBLE_UP_DOWN_COUNTER; + } + + @Override + public DoubleUpDownCounter getInstrument() { + return this.counter; + } + + @Override + public void setInstrument(DoubleUpDownCounter counter) { + this.counter = counter; + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/GrpcSummaryMetrics.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/GrpcSummaryMetrics.java deleted file mode 100644 index d8299ebe9b..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/GrpcSummaryMetrics.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import java.util.concurrent.atomic.AtomicLong; - -public class GrpcSummaryMetrics implements Metric { - - private final AtomicLong client2EventMeshMsgNum; - private final AtomicLong eventMesh2MqMsgNum; - private final AtomicLong mq2EventMeshMsgNum; - private final AtomicLong eventMesh2ClientMsgNum; - - private long client2EventMeshTPS; - private long eventMesh2ClientTPS; - private long eventMesh2MqTPS; - private long mq2EventMeshTPS; - - private long retrySize; - private long subscribeTopicNum; - - public GrpcSummaryMetrics() { - this.client2EventMeshMsgNum = new AtomicLong(0); - this.eventMesh2MqMsgNum = new AtomicLong(0); - this.mq2EventMeshMsgNum = new AtomicLong(0); - this.eventMesh2ClientMsgNum = new AtomicLong(0); - } - - public void clearAllMessageCounter() { - client2EventMeshMsgNum.set(0L); - eventMesh2MqMsgNum.set(0L); - mq2EventMeshMsgNum.set(0L); - eventMesh2ClientMsgNum.set(0L); - } - - public void refreshTpsMetrics(long intervalMills) { - client2EventMeshTPS = 1000 * client2EventMeshMsgNum.get() / intervalMills; - eventMesh2ClientTPS = 1000 * eventMesh2ClientMsgNum.get() / intervalMills; - eventMesh2MqTPS = 1000 * eventMesh2MqMsgNum.get() / intervalMills; - mq2EventMeshTPS = 1000 * mq2EventMeshMsgNum.get() / intervalMills; - } - - public AtomicLong getClient2EventMeshMsgNum() { - return client2EventMeshMsgNum; - } - - public AtomicLong getEventMesh2MqMsgNum() { - return eventMesh2MqMsgNum; - } - - public AtomicLong getMq2EventMeshMsgNum() { - return mq2EventMeshMsgNum; - } - - public AtomicLong getEventMesh2ClientMsgNum() { - return eventMesh2ClientMsgNum; - } - - public long getClient2EventMeshTPS() { - return client2EventMeshTPS; - } - - public long getEventMesh2ClientTPS() { - return eventMesh2ClientTPS; - } - - public long getEventMesh2MqTPS() { - return eventMesh2MqTPS; - } - - public long getMq2EventMeshTPS() { - return mq2EventMeshTPS; - } - - public long getRetrySize() { - return retrySize; - } - - public void setRetrySize(long retrySize) { - this.retrySize = retrySize; - } - - public long getSubscribeTopicNum() { - return subscribeTopicNum; - } - - public void setSubscribeTopicNum(long subscribeTopicNum) { - this.subscribeTopicNum = subscribeTopicNum; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/HttpSummaryMetrics.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/HttpSummaryMetrics.java deleted file mode 100644 index e09ff63884..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/HttpSummaryMetrics.java +++ /dev/null @@ -1,436 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import java.util.Collections; -import java.util.LinkedList; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import lombok.extern.slf4j.Slf4j; - -// todo: split this class -@Slf4j -public class HttpSummaryMetrics implements Metric { - - private static final int STATIC_PERIOD = 30 * 1000; - - private float wholeCost = 0f; - - private final AtomicLong wholeRequestNum = new AtomicLong(0); - - // cumulative value - private final AtomicLong httpDiscard = new AtomicLong(0); - - private final AtomicLong maxCost = new AtomicLong(0); - - private final AtomicLong httpRequestPerSecond = new AtomicLong(0); - - private final LinkedList httpRequestTPSSnapshots = new LinkedList<>(); - - private float httpDecodeTimeCost = 0f; - - private final AtomicLong httpDecodeNum = new AtomicLong(0); - - private final AtomicLong sendBatchMsgNumPerSecond = new AtomicLong(0); - - private final AtomicLong sendBatchMsgNumSum = new AtomicLong(0); - - private final AtomicLong sendBatchMsgFailNumSum = new AtomicLong(0); - - // This is a cumulative value - private final AtomicLong sendBatchMsgDiscardNumSum = new AtomicLong(0); - - private final LinkedList sendBatchMsgTPSSnapshots = new LinkedList(); - - private final AtomicLong sendMsgNumSum = new AtomicLong(0); - - private final AtomicLong sendMsgFailNumSum = new AtomicLong(0); - - private final AtomicLong replyMsgNumSum = new AtomicLong(0); - - private final AtomicLong replyMsgFailNumSum = new AtomicLong(0); - - private final AtomicLong sendMsgNumPerSecond = new AtomicLong(0); - - private final LinkedList sendMsgTPSSnapshots = new LinkedList(); - - private float wholePushCost = 0f; - - private final AtomicLong wholePushRequestNum = new AtomicLong(0); - - private final AtomicLong maxHttpPushLatency = new AtomicLong(0); - - private final AtomicLong pushMsgNumPerSecond = new AtomicLong(0); - - private final LinkedList pushMsgTPSSnapshots = new LinkedList(); - - private final AtomicLong httpPushMsgNumSum = new AtomicLong(0); - - private final AtomicLong httpPushFailNumSum = new AtomicLong(0); - - private float batchSend2MQWholeCost = 0f; - - private final AtomicLong batchSend2MQNum = new AtomicLong(0); - - private float send2MQWholeCost = 0f; - - private final AtomicLong send2MQNum = new AtomicLong(0); - - private float reply2MQWholeCost = 0f; - - private final AtomicLong reply2MQNum = new AtomicLong(0); - - // execute metrics - private final ThreadPoolExecutor batchMsgExecutor; - - private final ThreadPoolExecutor sendMsgExecutor; - - private final ThreadPoolExecutor pushMsgExecutor; - - private final RetrySummaryMetrics retrySummaryMetrics; - - private Lock lock = new ReentrantLock(); - - public HttpSummaryMetrics(final ThreadPoolExecutor batchMsgExecutor, - final ThreadPoolExecutor sendMsgExecutor, - final ThreadPoolExecutor pushMsgExecutor, - final RetrySummaryMetrics retrySummaryMetrics) { - this.batchMsgExecutor = batchMsgExecutor; - this.sendMsgExecutor = sendMsgExecutor; - this.pushMsgExecutor = pushMsgExecutor; - this.retrySummaryMetrics = retrySummaryMetrics; - } - - public float avgHTTPCost() { - return (wholeRequestNum.longValue() == 0L) ? 0f : wholeCost / wholeRequestNum.longValue(); - } - - public long maxHTTPCost() { - return maxCost.longValue(); - } - - public long getHttpDiscard() { - return httpDiscard.longValue(); - } - - public void recordHTTPRequest() { - httpRequestPerSecond.incrementAndGet(); - } - - public void recordHTTPDiscard() { - httpDiscard.incrementAndGet(); - } - - public void snapshotHTTPTPS() { - try { - lock.lock(); - Integer tps = httpRequestPerSecond.intValue(); - httpRequestTPSSnapshots.add(tps); - httpRequestPerSecond.set(0); - if (httpRequestTPSSnapshots.size() > STATIC_PERIOD / 1000) { - httpRequestTPSSnapshots.removeFirst(); - } - } finally { - lock.unlock(); - } - } - - public float maxHTTPTPS() { - try { - lock.lock(); - float tps = Collections.max(httpRequestTPSSnapshots); - return tps; - } finally { - lock.unlock(); - } - } - - public float avgHTTPTPS() { - try { - lock.lock(); - float tps = avg(httpRequestTPSSnapshots); - return tps; - } finally { - lock.unlock(); - } - } - - public void recordHTTPReqResTimeCost(long cost) { - wholeRequestNum.incrementAndGet(); - wholeCost = wholeCost + cost; - if (cost > maxCost.longValue()) { - maxCost.set(cost); - } - } - - public void httpStatInfoClear() { - wholeRequestNum.set(0L); - wholeCost = 0f; - maxCost.set(0L); - httpDecodeNum.set(0L); - httpDecodeTimeCost = 0f; - } - - public void recordDecodeTimeCost(long cost) { - httpDecodeNum.incrementAndGet(); - httpDecodeTimeCost = httpDecodeTimeCost + cost; - } - - public float avgHTTPBodyDecodeCost() { - return (httpDecodeNum.longValue() == 0L) ? 0f : httpDecodeTimeCost / httpDecodeNum.longValue(); - } - - public void recordSendBatchMsgDiscard(long delta) { - sendBatchMsgDiscardNumSum.addAndGet(delta); - } - - public void snapshotSendBatchMsgTPS() { - Integer tps = sendBatchMsgNumPerSecond.intValue(); - sendBatchMsgTPSSnapshots.add(tps); - sendBatchMsgNumPerSecond.set(0); - if (sendBatchMsgTPSSnapshots.size() > STATIC_PERIOD / 1000) { - sendBatchMsgTPSSnapshots.removeFirst(); - } - } - - public float maxSendBatchMsgTPS() { - return Collections.max(sendBatchMsgTPSSnapshots); - } - - public float avgSendBatchMsgTPS() { - return avg(sendBatchMsgTPSSnapshots); - } - - public void recordSendBatchMsg(long delta) { - sendBatchMsgNumPerSecond.addAndGet(delta); - sendBatchMsgNumSum.addAndGet(delta); - } - - public void recordSendBatchMsgFailed(long delta) { - sendBatchMsgFailNumSum.getAndAdd(delta); - } - - public long getSendBatchMsgNumSum() { - return sendBatchMsgNumSum.longValue(); - } - - public long getSendBatchMsgFailNumSum() { - return sendBatchMsgFailNumSum.longValue(); - } - - public float getSendBatchMsgFailRate() { - return (sendBatchMsgNumSum.longValue() == 0L) ? 0f : sendBatchMsgFailNumSum.floatValue() / sendBatchMsgNumSum.longValue(); - } - - public void cleanSendBatchStat() { - sendBatchMsgNumSum.set(0L); - sendBatchMsgFailNumSum.set(0L); - } - - public long getSendBatchMsgDiscardNumSum() { - return sendBatchMsgDiscardNumSum.longValue(); - } - - public void snapshotSendMsgTPS() { - Integer tps = sendMsgNumPerSecond.intValue(); - sendMsgTPSSnapshots.add(tps); - sendMsgNumPerSecond.set(0); - if (sendMsgTPSSnapshots.size() > STATIC_PERIOD / 1000) { - sendMsgTPSSnapshots.removeFirst(); - } - } - - public float maxSendMsgTPS() { - return Collections.max(sendMsgTPSSnapshots); - } - - public float avgSendMsgTPS() { - return avg(sendMsgTPSSnapshots); - } - - public void recordSendMsg() { - sendMsgNumPerSecond.incrementAndGet(); - sendMsgNumSum.incrementAndGet(); - } - - public void recordReplyMsg() { - replyMsgNumSum.incrementAndGet(); - } - - public void recordReplyMsgFailed() { - replyMsgFailNumSum.incrementAndGet(); - } - - public long getReplyMsgNumSum() { - return replyMsgNumSum.longValue(); - } - - public long getReplyMsgFailNumSum() { - return replyMsgFailNumSum.longValue(); - } - - public long getSendMsgNumSum() { - return sendMsgNumSum.longValue(); - } - - public long getSendMsgFailNumSum() { - return sendMsgFailNumSum.longValue(); - } - - public float getSendMsgFailRate() { - return (sendMsgNumSum.longValue() == 0L) ? 0f : sendMsgFailNumSum.floatValue() / sendMsgNumSum.longValue(); - } - - public void recordSendMsgFailed() { - sendMsgFailNumSum.incrementAndGet(); - } - - public void cleanSendMsgStat() { - sendMsgNumSum.set(0L); - replyMsgNumSum.set(0L); - sendMsgFailNumSum.set(0L); - replyMsgFailNumSum.set(0L); - } - - public void snapshotPushMsgTPS() { - Integer tps = pushMsgNumPerSecond.intValue(); - pushMsgTPSSnapshots.add(tps); - pushMsgNumPerSecond.set(0); - if (pushMsgTPSSnapshots.size() > STATIC_PERIOD / 1000) { - pushMsgTPSSnapshots.removeFirst(); - } - } - - public void recordHTTPPushTimeCost(long cost) { - wholePushRequestNum.incrementAndGet(); - wholePushCost = wholePushCost + cost; - if (cost > maxHttpPushLatency.longValue()) { - maxHttpPushLatency.set(cost); - } - } - - public float avgHTTPPushLatency() { - return (wholePushRequestNum.longValue() == 0L) ? 0f : wholePushCost / wholePushRequestNum.longValue(); - } - - public float maxHTTPPushLatency() { - return maxHttpPushLatency.floatValue(); - } - - public float maxPushMsgTPS() { - return Collections.max(pushMsgTPSSnapshots); - } - - public float avgPushMsgTPS() { - return avg(pushMsgTPSSnapshots); - } - - public void recordPushMsg() { - pushMsgNumPerSecond.incrementAndGet(); - httpPushMsgNumSum.incrementAndGet(); - } - - public long getHttpPushMsgNumSum() { - return httpPushMsgNumSum.longValue(); - } - - public long getHttpPushFailNumSum() { - return httpPushFailNumSum.longValue(); - } - - public float getHttpPushMsgFailRate() { - return (httpPushMsgNumSum.longValue() == 0L) ? 0f : httpPushFailNumSum.floatValue() / httpPushMsgNumSum.longValue(); - } - - public void recordHttpPushMsgFailed() { - sendMsgFailNumSum.incrementAndGet(); - } - - public void cleanHttpPushMsgStat() { - httpPushFailNumSum.set(0L); - httpPushMsgNumSum.set(0L); - wholeRequestNum.set(0L); - wholeCost = 0f; - maxCost.set(0L); - } - - public void recordBatchSendMsgCost(long cost) { - batchSend2MQNum.incrementAndGet(); - batchSend2MQWholeCost = batchSend2MQWholeCost + cost; - } - - public float avgBatchSendMsgCost() { - return (batchSend2MQNum.intValue() == 0) ? 0f : batchSend2MQWholeCost / batchSend2MQNum.intValue(); - } - - public void recordSendMsgCost(long cost) { - send2MQNum.incrementAndGet(); - send2MQWholeCost = send2MQWholeCost + cost; - } - - public float avgSendMsgCost() { - return (send2MQNum.intValue() == 0) ? 0f : send2MQWholeCost / send2MQNum.intValue(); - } - - public void recordReplyMsgCost(long cost) { - reply2MQNum.incrementAndGet(); - reply2MQWholeCost = reply2MQWholeCost + cost; - } - - public float avgReplyMsgCost() { - return (reply2MQNum.intValue() == 0) ? 0f : reply2MQWholeCost / reply2MQNum.intValue(); - } - - public void send2MQStatInfoClear() { - batchSend2MQWholeCost = 0f; - batchSend2MQNum.set(0L); - send2MQWholeCost = 0f; - send2MQNum.set(0L); - reply2MQWholeCost = 0f; - reply2MQNum.set(0L); - } - - public int getBatchMsgQueueSize() { - return batchMsgExecutor.getQueue().size(); - } - - public int getSendMsgQueueSize() { - return sendMsgExecutor.getQueue().size(); - } - - public int getPushMsgQueueSize() { - return pushMsgExecutor.getQueue().size(); - } - - public long getHttpRetryQueueSize() { - return retrySummaryMetrics.getPendingRetryTimeouts(); - } - - private float avg(LinkedList linkedList) { - if (linkedList.isEmpty()) { - return 0.0f; - } - - int sum = linkedList.stream().reduce(Integer::sum).get(); - return (float) sum / linkedList.size(); - } - -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentFurther.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentFurther.java new file mode 100644 index 0000000000..c309ff6719 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentFurther.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import java.util.HashMap; +import java.util.Map; + +import lombok.Data; + +/** + * Represents an instrument with a name, description, unit, and additional properties. + */ +@Data +public class InstrumentFurther { + + public static final String INSTRUMENT_VIEW = "instrument_view"; + + private String name; + + private String description; + + private String unit; + + private Map ext = new HashMap<>(); + + public void putExt(String key, Object value) { + ext.put(key, value); + } + +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentType.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentType.java new file mode 100644 index 0000000000..b0511d37de --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/InstrumentType.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.metrics.DoubleCounter; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleUpDownCounter; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongHistogram; +import io.opentelemetry.api.metrics.LongUpDownCounter; +import io.opentelemetry.api.metrics.ObservableDoubleCounter; +import io.opentelemetry.api.metrics.ObservableDoubleGauge; +import io.opentelemetry.api.metrics.ObservableDoubleUpDownCounter; +import io.opentelemetry.api.metrics.ObservableLongCounter; +import io.opentelemetry.api.metrics.ObservableLongGauge; +import io.opentelemetry.api.metrics.ObservableLongUpDownCounter; + +public enum InstrumentType { + LONG_COUNTER(LongCounter.class), + DOUBLE_COUNTER(DoubleCounter.class), + LONG_UP_DOWN_COUNTER(LongUpDownCounter.class), + DOUBLE_UP_DOWN_COUNTER(DoubleUpDownCounter.class), + OBSERVABLE_LONG_COUNTER(ObservableLongCounter.class), + OBSERVABLE_LONG_UP_DOWN_COUNTER(ObservableLongUpDownCounter.class), + OBSERVABLE_DOUBLE_COUNTER(ObservableDoubleCounter.class), + OBSERVABLE_DOUBLE_UP_DOWN_COUNTER(ObservableDoubleUpDownCounter.class), + OBSERVABLE_LONG_GAUGE(ObservableLongGauge.class), + OBSERVABLE_DOUBLE_GAUGE(ObservableDoubleGauge.class), + LONG_HISTOGRAM(LongHistogram.class), + DOUBLE_HISTOGRAM(DoubleHistogram.class); + + + private Class type; + + InstrumentType(Class type) { + this.type = type; + } + + public Class getType() { + return this.type; + } +} \ No newline at end of file diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongCounterMetric.java new file mode 100644 index 0000000000..6c8a292c61 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongCounterMetric.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.metrics.LongCounter; + +public class LongCounterMetric extends AbstractSyncMetric { + + private LongCounter counter = new NoopLongCounter(); + + public LongCounterMetric(InstrumentFurther further, String metricName, LongCounter counter) { + super(further, metricName); + this.counter = counter; + } + + public LongCounterMetric(InstrumentFurther further, String metricName) { + super(further, metricName); + } + + public LongCounterMetric(String metricName) { + super(null, metricName); + } + + public LongCounterMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.LONG_COUNTER; + } + + @Override + public LongCounter getInstrument() { + return this.counter; + } + + @Override + public void setInstrument(LongCounter counter) { + this.counter = counter; + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongHistogramMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongHistogramMetric.java new file mode 100644 index 0000000000..af6deb6a31 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongHistogramMetric.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.metrics.LongHistogram; + +public class LongHistogramMetric extends AbstractSyncMetric { + + private LongHistogram histogram; + + public LongHistogramMetric(InstrumentFurther further, String metricName, LongHistogram histogram) { + super(further, metricName); + this.histogram = histogram; + } + + public LongHistogramMetric(InstrumentFurther further, String metricName) { + super(further, metricName); + } + + public LongHistogramMetric(String metricName) { + super(null, metricName); + } + + public LongHistogramMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.LONG_HISTOGRAM; + } + + @Override + public LongHistogram getInstrument() { + return this.histogram; + } + + @Override + public void setInstrument(LongHistogram counter) { + this.histogram = counter; + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongUpDownCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongUpDownCounterMetric.java new file mode 100644 index 0000000000..5cf9b7dd6e --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/LongUpDownCounterMetric.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.metrics.LongUpDownCounter; + +public class LongUpDownCounterMetric extends AbstractSyncMetric { + + private LongUpDownCounter counter; + + public LongUpDownCounterMetric(InstrumentFurther further, String metricName, LongUpDownCounter counter) { + super(further, metricName); + this.counter = counter; + } + + public LongUpDownCounterMetric(InstrumentFurther further, String metricName) { + super(further, metricName); + } + + public LongUpDownCounterMetric(String metricName) { + super(null, metricName); + } + + public LongUpDownCounterMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.LONG_UP_DOWN_COUNTER; + } + + @Override + public LongUpDownCounter getInstrument() { + return this.counter; + } + + @Override + public void setInstrument(LongUpDownCounter counter) { + this.counter = counter; + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/Metric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/Metric.java index 7abe3c81fb..0ea918bfbf 100644 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/Metric.java +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/Metric.java @@ -22,4 +22,31 @@ */ public interface Metric { + /** + * Sets the instrument further. + * + * @param further the instrument further to be set + */ + void setInstrumentFurther(InstrumentFurther further); + + /** + * Gets the instrument further. + * + * @return the instrument further + */ + InstrumentFurther getInstrumentFurther(); + + /** + * Gets the name of the metric. + * + * @return the name of the metric + */ + String getName(); + + /** + * Gets the type of the instrument. + * + * @return the type of the instrument + */ + InstrumentType getInstrumentType(); } diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleCounter.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleCounter.java new file mode 100644 index 0000000000..69cb054934 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleCounter.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleCounter; +import io.opentelemetry.context.Context; + +public class NoopDoubleCounter implements DoubleCounter { + + /** + * Records a value. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The increment amount. MUST be non-negative. + */ + @Override + public void add(double value) { + + } + + /** + * Records a value with a set of attributes. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The increment amount. MUST be non-negative. + * @param attributes A set of attributes to associate with the value. + */ + @Override + public void add(double value, Attributes attributes) { + + } + + /** + * Records a value with a set of attributes. + * + * @param value The increment amount. MUST be non-negative. + * @param attributes A set of attributes to associate with the value. + * @param context The explicit context to associate with this measurement. + */ + @Override + public void add(double value, Attributes attributes, Context context) { + + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleHistogram.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleHistogram.java new file mode 100644 index 0000000000..1dd2db3624 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleHistogram.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.context.Context; + +public class NoopDoubleHistogram implements DoubleHistogram { + + /** + * Records a value. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The amount of the measurement. MUST be non-negative. + */ + @Override + public void record(double value) { + + } + + /** + * Records a value with a set of attributes. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The amount of the measurement. MUST be non-negative. + * @param attributes A set of attributes to associate with the value. + */ + @Override + public void record(double value, Attributes attributes) { + + } + + /** + * Records a value with a set of attributes. + * + * @param value The amount of the measurement. MUST be non-negative. + * @param attributes A set of attributes to associate with the value. + * @param context The explicit context to associate with this measurement. + */ + @Override + public void record(double value, Attributes attributes, Context context) { + + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleUpDownCounter.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleUpDownCounter.java new file mode 100644 index 0000000000..ee18d0f21f --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopDoubleUpDownCounter.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleUpDownCounter; +import io.opentelemetry.context.Context; + +public class NoopDoubleUpDownCounter implements DoubleUpDownCounter { + + /** + * Records a value. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The increment amount. May be positive, negative or zero. + */ + @Override + public void add(double value) { + + } + + /** + * Records a value with a set of attributes. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The increment amount. May be positive, negative or zero. + * @param attributes A set of attributes to associate with the value. + */ + @Override + public void add(double value, Attributes attributes) { + + } + + /** + * Records a value with a set of attributes. + * + * @param value The increment amount. May be positive, negative or zero. + * @param attributes A set of attributes to associate with the value. + * @param context The explicit context to associate with this measurement. + */ + @Override + public void add(double value, Attributes attributes, Context context) { + + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongCounter.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongCounter.java new file mode 100644 index 0000000000..d22959a42e --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongCounter.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.context.Context; + +public class NoopLongCounter implements LongCounter { + + /** + * Records a value. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The increment amount. MUST be non-negative. + */ + @Override + public void add(long value) { + + } + + /** + * Records a value with a set of attributes. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The increment amount. MUST be non-negative. + * @param attributes A set of attributes to associate with the value. + */ + @Override + public void add(long value, Attributes attributes) { + + } + + /** + * Records a value with a set of attributes. + * + * @param value The increment amount. MUST be non-negative. + * @param attributes A set of attributes to associate with the value. + * @param context The explicit context to associate with this measurement. + */ + @Override + public void add(long value, Attributes attributes, Context context) { + + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongHistogram.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongHistogram.java new file mode 100644 index 0000000000..605a33b771 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongHistogram.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongHistogram; +import io.opentelemetry.context.Context; + +public class NoopLongHistogram implements LongHistogram { + + /** + * Records a value. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The amount of the measurement. MUST be non-negative. + */ + @Override + public void record(long value) { + + } + + /** + * Records a value with a set of attributes. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The amount of the measurement. MUST be non-negative. + * @param attributes A set of attributes to associate with the value. + */ + @Override + public void record(long value, Attributes attributes) { + + } + + /** + * Records a value with a set of attributes. + * + * @param value The amount of the measurement. MUST be non-negative. + * @param attributes A set of attributes to associate with the value. + * @param context The explicit context to associate with this measurement. + */ + @Override + public void record(long value, Attributes attributes, Context context) { + + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongUpDownCounter.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongUpDownCounter.java new file mode 100644 index 0000000000..b25b1e1fdc --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/NoopLongUpDownCounter.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongUpDownCounter; +import io.opentelemetry.context.Context; + +public class NoopLongUpDownCounter implements LongUpDownCounter { + + /** + * Records a value. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The increment amount. May be positive, negative or zero. + */ + @Override + public void add(long value) { + + } + + /** + * Record a value with a set of attributes. + * + *

Note: This may use {@code Context.current()} to pull the context associated with this + * measurement. + * + * @param value The increment amount. May be positive, negative or zero. + * @param attributes A set of attributes to associate with the value. + */ + @Override + public void add(long value, Attributes attributes) { + + } + + /** + * Records a value with a set of attributes. + * + * @param value The increment amount. May be positive, negative or zero. + * @param attributes A set of attributes to associate with the value. + * @param context The explicit context to associate with this measurement. + */ + @Override + public void add(long value, Attributes attributes, Context context) { + + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleCounterMetric.java new file mode 100644 index 0000000000..372fb151c1 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleCounterMetric.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import java.util.function.Supplier; + +public class ObservableDoubleCounterMetric extends AbstractObservableDoubleMetric { + + public ObservableDoubleCounterMetric(InstrumentFurther further, String metricName, Supplier supplier) { + super(further, metricName, supplier); + } + + public ObservableDoubleCounterMetric(InstrumentFurther further, String metricName) { + super(further, metricName); + } + + public ObservableDoubleCounterMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.OBSERVABLE_DOUBLE_COUNTER; + } + +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleGaugeMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleGaugeMetric.java new file mode 100644 index 0000000000..ff1345ccc8 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleGaugeMetric.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import java.util.function.Supplier; + +public class ObservableDoubleGaugeMetric extends AbstractObservableDoubleMetric { + + public ObservableDoubleGaugeMetric(InstrumentFurther further, String metricName, Supplier supplier) { + super(further, metricName, supplier); + } + + public ObservableDoubleGaugeMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.OBSERVABLE_DOUBLE_GAUGE; + } + +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleUpDownCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleUpDownCounterMetric.java new file mode 100644 index 0000000000..95dc8de120 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableDoubleUpDownCounterMetric.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import java.util.function.Supplier; + +public class ObservableDoubleUpDownCounterMetric extends AbstractObservableDoubleMetric { + + + public ObservableDoubleUpDownCounterMetric(InstrumentFurther further, String metricName, Supplier supplier) { + super(further, metricName, supplier); + } + + public ObservableDoubleUpDownCounterMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.OBSERVABLE_DOUBLE_UP_DOWN_COUNTER; + } + +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongCounterMetric.java new file mode 100644 index 0000000000..ca8e3227b8 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongCounterMetric.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import java.util.function.Supplier; + +public class ObservableLongCounterMetric extends AbstractObservableLongMetric { + + public ObservableLongCounterMetric(InstrumentFurther further, String metricName, Supplier supplier) { + super(further, metricName, supplier); + } + + public ObservableLongCounterMetric(String metricName) { + super(null, metricName); + } + + public ObservableLongCounterMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.OBSERVABLE_LONG_COUNTER; + } +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongGaugeMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongGaugeMetric.java new file mode 100644 index 0000000000..206398474d --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongGaugeMetric.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import java.util.function.Supplier; + +public class ObservableLongGaugeMetric extends AbstractObservableLongMetric { + + public ObservableLongGaugeMetric(InstrumentFurther further, String metricName, Supplier supplier) { + super(further, metricName, supplier); + } + + public ObservableLongGaugeMetric(String metricName) { + super(null, metricName); + } + + public ObservableLongGaugeMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.OBSERVABLE_LONG_GAUGE; + } + +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongUpDownCounterMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongUpDownCounterMetric.java new file mode 100644 index 0000000000..f00075ff27 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableLongUpDownCounterMetric.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + + +import java.util.function.Supplier; + +public class ObservableLongUpDownCounterMetric extends AbstractObservableLongMetric { + + public ObservableLongUpDownCounterMetric(InstrumentFurther further, String metricName, Supplier supplier) { + super(further, metricName, supplier); + } + + public ObservableLongUpDownCounterMetric() { + super(null, null); + } + + @Override + public InstrumentType getInstrumentType() { + return InstrumentType.OBSERVABLE_LONG_UP_DOWN_COUNTER; + } + + +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableMetric.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableMetric.java new file mode 100644 index 0000000000..6e53035627 --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/ObservableMetric.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.api.model; + +import java.util.Map; +import java.util.function.Supplier; + +/** + * Observable metric interface, all observable(asynchronization) metrics should implement this interface. + */ +public interface ObservableMetric extends Metric { + + /** + * Puts a key-value pair into the metric. + * + * @param key the key to put + * @param value the value to put + */ + void put(Key key, Value value); + + /** + * Puts all key-value pairs from a map into the metric. + * + * @param attributes the map containing key-value pairs + */ + void putAll(Map attributes); + + /** + * Retrieves all attributes of the metric. + * + * @return a map containing all attributes + */ + Map getAttributes(); + + /** + * Retrieves the supplier of the instrument associated with the metric. + * + * @return the supplier of the instrument + */ + Supplier supplier(); +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/utils/PrometheusExporterConstants.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/SyncMetric.java similarity index 74% rename from eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/utils/PrometheusExporterConstants.java rename to eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/SyncMetric.java index 9da69e2b7d..90948a07cd 100644 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/utils/PrometheusExporterConstants.java +++ b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/SyncMetric.java @@ -15,15 +15,14 @@ * limitations under the License. */ -package org.apache.eventmesh.metrics.prometheus.utils; +package org.apache.eventmesh.metrics.api.model; /** - * Constants in metrics-prometheus module + * synchronization metric, all synchronization metrics should implement this interface. */ -public class PrometheusExporterConstants { +public interface SyncMetric extends Metric { - public static final String METRICS_GRPC_PREFIX = "eventmesh.grpc."; - - public static final String METRICS_TCP_PREFIX = "eventmesh.tcp."; + Instrument getInstrument(); + void setInstrument(Instrument instrument); } diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/TcpSummaryMetrics.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/TcpSummaryMetrics.java deleted file mode 100644 index 1e54c8e758..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/main/java/org/apache/eventmesh/metrics/api/model/TcpSummaryMetrics.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import java.util.concurrent.atomic.AtomicInteger; - -import lombok.Data; - -@Data -public class TcpSummaryMetrics implements Metric { - - private AtomicInteger client2eventMeshMsgNum; - private AtomicInteger eventMesh2mqMsgNum; - private AtomicInteger mq2eventMeshMsgNum; - private AtomicInteger eventMesh2clientMsgNum; - - private int client2eventMeshTPS; - private int eventMesh2clientTPS; - private int eventMesh2mqTPS; - private int mq2eventMeshTPS; - private int subTopicNum; - - private int allConnections; - - private long retrySize; - - public TcpSummaryMetrics() { - this.client2eventMeshMsgNum = new AtomicInteger(0); - this.eventMesh2mqMsgNum = new AtomicInteger(0); - this.mq2eventMeshMsgNum = new AtomicInteger(0); - this.eventMesh2clientMsgNum = new AtomicInteger(0); - } - - public int client2eventMeshMsgNum() { - return client2eventMeshMsgNum.get(); - } - - public int eventMesh2mqMsgNum() { - return eventMesh2mqMsgNum.get(); - } - - public int mq2eventMeshMsgNum() { - return mq2eventMeshMsgNum.get(); - } - - public int eventMesh2clientMsgNum() { - return eventMesh2clientMsgNum.get(); - } - - public void resetClient2EventMeshMsgNum() { - this.client2eventMeshMsgNum = new AtomicInteger(0); - } - - public void resetEventMesh2mqMsgNum() { - this.eventMesh2mqMsgNum = new AtomicInteger(0); - } - - public void resetMq2eventMeshMsgNum() { - this.mq2eventMeshMsgNum = new AtomicInteger(0); - } - - public void resetEventMesh2ClientMsgNum() { - this.eventMesh2clientMsgNum = new AtomicInteger(0); - } - - public int getClient2eventMeshTPS() { - return client2eventMeshTPS; - } - - public void setClient2eventMeshTPS(int client2eventMeshTPS) { - this.client2eventMeshTPS = client2eventMeshTPS; - } - - public int getEventMesh2clientTPS() { - return eventMesh2clientTPS; - } - - public void setEventMesh2clientTPS(int eventMesh2clientTPS) { - this.eventMesh2clientTPS = eventMesh2clientTPS; - } - - public int getEventMesh2mqTPS() { - return eventMesh2mqTPS; - } - - public void setEventMesh2mqTPS(int eventMesh2mqTPS) { - this.eventMesh2mqTPS = eventMesh2mqTPS; - } - - public int getMq2eventMeshTPS() { - return mq2eventMeshTPS; - } - - public void setMq2eventMeshTPS(int mq2eventMeshTPS) { - this.mq2eventMeshTPS = mq2eventMeshTPS; - } - - public int getAllTPS() { - return client2eventMeshTPS + eventMesh2clientTPS; - } - - public int getSubTopicNum() { - return subTopicNum; - } - - public void setSubTopicNum(int subTopicNum) { - this.subTopicNum = subTopicNum; - } - - public int getAllConnections() { - return allConnections; - } - - public void setAllConnections(int allConnections) { - this.allConnections = allConnections; - } - - public void setRetrySize(long retrySize) { - this.retrySize = retrySize; - } - - public long getRetrySize() { - return retrySize; - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/test/java/org/apache/eventmesh/metrics/api/model/GrpcSummaryMetricsTest.java b/eventmesh-metrics-plugin/eventmesh-metrics-api/src/test/java/org/apache/eventmesh/metrics/api/model/GrpcSummaryMetricsTest.java deleted file mode 100644 index 73a788b6fa..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-api/src/test/java/org/apache/eventmesh/metrics/api/model/GrpcSummaryMetricsTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.api.model; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class GrpcSummaryMetricsTest { - - @Test - public void testTpsCalculate() { - GrpcSummaryMetrics grpcSummaryMetrics = new GrpcSummaryMetrics(); - grpcSummaryMetrics.getEventMesh2ClientMsgNum().addAndGet(128); - grpcSummaryMetrics.getEventMesh2MqMsgNum().addAndGet(256); - grpcSummaryMetrics.getMq2EventMeshMsgNum().addAndGet(512); - grpcSummaryMetrics.getClient2EventMeshMsgNum().addAndGet(1024); - - grpcSummaryMetrics.refreshTpsMetrics(500); - Assertions.assertEquals(256, grpcSummaryMetrics.getEventMesh2ClientTPS()); - Assertions.assertEquals(512, grpcSummaryMetrics.getEventMesh2MqTPS()); - Assertions.assertEquals(1024, grpcSummaryMetrics.getMq2EventMeshTPS()); - Assertions.assertEquals(2048, grpcSummaryMetrics.getClient2EventMeshTPS()); - - grpcSummaryMetrics.clearAllMessageCounter(); - Assertions.assertEquals(0, grpcSummaryMetrics.getEventMesh2ClientMsgNum().get()); - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/build.gradle b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/build.gradle index 512110cd39..48c3430294 100644 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/build.gradle +++ b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/build.gradle @@ -22,17 +22,20 @@ dependencies { implementation 'org.apache.commons:commons-lang3' implementation 'com.google.guava:guava' - // todo: Can we remove some dependency? - implementation 'io.opentelemetry:opentelemetry-api' implementation 'io.opentelemetry:opentelemetry-sdk' implementation 'io.opentelemetry:opentelemetry-sdk-metrics' implementation 'io.opentelemetry:opentelemetry-exporter-prometheus' - implementation 'io.prometheus:simpleclient' - implementation 'io.prometheus:simpleclient_httpserver' + implementation 'io.opentelemetry:opentelemetry-semconv' compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' + testImplementation "org.mockito:mockito-core" + testImplementation "org.assertj:assertj-core" + + testImplementation "org.mockito:mockito-core" + testImplementation "org.mockito:mockito-junit-jupiter" + testCompileOnly 'org.projectlombok:lombok' testAnnotationProcessor 'org.projectlombok:lombok' } diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/OpenTelemetryPrometheusManager.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/OpenTelemetryPrometheusManager.java new file mode 100644 index 0000000000..248ce41fde --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/OpenTelemetryPrometheusManager.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.prometheus; + +import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; + +import org.apache.commons.lang3.StringUtils; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.resources.Resource; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class OpenTelemetryPrometheusManager { + + public static OpenTelemetry initOpenTelemetry(String host, int prometheusExportPort) { + if (StringUtils.isBlank(host)) { + host = "0.0.0.0"; + } + Resource resource = Resource.getDefault().merge(Resource.builder().put(SERVICE_NAME, "EventMeshPrometheusExporter").build()); + SdkMeterProviderBuilder meterProviderBuilder = SdkMeterProvider.builder(); + //register view for instrument + PrometheusMetricsRegistryManager.getMetricsView().forEach(pair -> meterProviderBuilder.registerView(pair.getLeft(), pair.getRight())); + OpenTelemetrySdk openTelemetrySdk = OpenTelemetrySdk.builder().setMeterProvider(meterProviderBuilder.setResource(resource) + .registerMetricReader(PrometheusHttpServer.builder().setHost(host).setPort(prometheusExportPort).build()).build()) + .buildAndRegisterGlobal(); + return openTelemetrySdk; + + } + +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistry.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistry.java index 6fd0b9acdb..8516ffbd42 100644 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistry.java +++ b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistry.java @@ -19,20 +19,14 @@ import org.apache.eventmesh.common.config.Config; import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.metrics.api.model.GrpcSummaryMetrics; -import org.apache.eventmesh.metrics.api.model.HttpSummaryMetrics; import org.apache.eventmesh.metrics.api.model.Metric; -import org.apache.eventmesh.metrics.api.model.TcpSummaryMetrics; import org.apache.eventmesh.metrics.prometheus.config.PrometheusConfiguration; -import org.apache.eventmesh.metrics.prometheus.metrics.PrometheusGrpcExporter; -import org.apache.eventmesh.metrics.prometheus.metrics.PrometheusHttpExporter; -import org.apache.eventmesh.metrics.prometheus.metrics.PrometheusTcpExporter; -import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; -import io.opentelemetry.exporter.prometheus.PrometheusCollector; -import io.opentelemetry.sdk.metrics.SdkMeterProvider; -import io.prometheus.client.exporter.HTTPServer; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.OpenTelemetrySdk; import lombok.extern.slf4j.Slf4j; @@ -40,7 +34,9 @@ @Config(field = "prometheusConfiguration") public class PrometheusMetricsRegistry implements MetricsRegistry { - private volatile HTTPServer prometheusHttpServer; + private final AtomicBoolean started = new AtomicBoolean(false); + + private OpenTelemetry openTelemetry; /** * Unified configuration class corresponding to prometheus.properties @@ -49,48 +45,37 @@ public class PrometheusMetricsRegistry implements MetricsRegistry { @Override public void start() { - if (prometheusHttpServer == null) { - synchronized (PrometheusMetricsRegistry.class) { - if (prometheusHttpServer == null) { - SdkMeterProvider sdkMeterProvider = SdkMeterProvider.builder().buildAndRegisterGlobal(); - PrometheusCollector - .builder().setMetricProducer(sdkMeterProvider).buildAndRegister(); - int port = prometheusConfiguration.getEventMeshPrometheusPort(); - try { - // Use the daemon thread to start an HTTP server to serve the default Prometheus registry. - prometheusHttpServer = new HTTPServer(port, true); - } catch (IOException e) { - log.error("failed to start prometheus server, port: {} due to {}", port, e.getMessage()); - } - } - } + + if (!started.compareAndSet(false, true)) { + return; } + try { + this.prometheusConfiguration = Objects.requireNonNull(this.prometheusConfiguration, "prometheusConfiguration can't be null!"); + String eventMeshPrometheusExportHost = prometheusConfiguration.getEventMeshPrometheusExportHost(); + int eventMeshPrometheusPort = prometheusConfiguration.getEventMeshPrometheusPort(); + this.openTelemetry = OpenTelemetryPrometheusManager.initOpenTelemetry(eventMeshPrometheusExportHost, eventMeshPrometheusPort); + PrometheusMetricsRegistryManager.createMetric(this.openTelemetry); + } catch (Exception e) { + log.error("failed to start prometheus export, Host: {}:{} due to {}", prometheusConfiguration.getEventMeshPrometheusExportHost(), + prometheusConfiguration.getEventMeshPrometheusPort(), e.getMessage()); + } } @Override public void showdown() { - if (prometheusHttpServer != null) { - prometheusHttpServer.stop(); + if (this.openTelemetry instanceof OpenTelemetrySdk) { + + try (OpenTelemetrySdk ignored = (OpenTelemetrySdk) this.openTelemetry) { + //OpenTelemetrySdk will call close auto + } } + } @Override public void register(Metric metric) { - if (metric == null) { - throw new IllegalArgumentException("Metric cannot be null"); - } - if (metric instanceof HttpSummaryMetrics) { - PrometheusHttpExporter.export("apache-eventmesh", (HttpSummaryMetrics) metric); - } - - if (metric instanceof TcpSummaryMetrics) { - PrometheusTcpExporter.export("apache-eventmesh", (TcpSummaryMetrics) metric); - } - - if (metric instanceof GrpcSummaryMetrics) { - PrometheusGrpcExporter.export("apache-eventmesh", (GrpcSummaryMetrics) metric); - } + PrometheusMetricsRegistryManager.registerMetric(metric); } @Override diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistryManager.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistryManager.java new file mode 100644 index 0000000000..7d4d13c9ec --- /dev/null +++ b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/PrometheusMetricsRegistryManager.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.metrics.prometheus; + +import org.apache.eventmesh.common.Pair; +import org.apache.eventmesh.metrics.api.model.InstrumentFurther; +import org.apache.eventmesh.metrics.api.model.InstrumentType; +import org.apache.eventmesh.metrics.api.model.Metric; +import org.apache.eventmesh.metrics.api.model.ObservableMetric; +import org.apache.eventmesh.metrics.api.model.SyncMetric; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.DoubleCounter; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleUpDownCounter; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongHistogram; +import io.opentelemetry.api.metrics.LongUpDownCounter; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.View; + +import lombok.experimental.UtilityClass; + +/** + * Registering and handling Prometheus metrics. + */ +@UtilityClass +public class PrometheusMetricsRegistryManager { + + private static final Map meterCache = new ConcurrentHashMap<>(32); + + private static final Map> metricCache = new ConcurrentHashMap<>(16); + + /** + * Registers a metric in the metric cache. + * + * @param metric The metric to register. + */ + public static void registerMetric(final Metric metric) { + Set metrics = metricCache.computeIfAbsent(metric.getName(), (k) -> new HashSet<>()); + metrics.add(metric); + } + + /** + * Retrieves the metrics associated with a given metric name. + * + * @param metricName The name of the metric. + * @return The set of metrics associated with the metric name. + */ + protected static Set getMetrics(final String metricName) { + return metricCache.get(metricName); + } + + /** + * Creates and associates metrics with a Prometheus meter using the provided OpenTelemetry instance. + * + * @param openTelemetry The OpenTelemetry instance. + */ + public static void createMetric(final OpenTelemetry openTelemetry) { + + metricCache.values().stream().flatMap(metricSet -> metricSet.stream()).forEach(metric -> { + Meter meter = meterCache.computeIfAbsent(metric.getName(), (meterName) -> openTelemetry.getMeter(meterName)); + InstrumentFurther instrumentFurther = metric.getInstrumentFurther(); + if (instrumentFurther == null) { + instrumentFurther = new InstrumentFurther(); + } + + // Handle observable metrics + if (metric instanceof ObservableMetric) { + handleObservableMetric((ObservableMetric) metric, meter, instrumentFurther); + return; + } + + // Handle sync metrics + if (metric instanceof SyncMetric) { + handleMetric((SyncMetric) metric, meter, instrumentFurther); + } + }); + } + + /** + * Retrieves the list of metrics and their corresponding views. + * + * @return The list of metric views. + */ + public static List> getMetricsView() { + Collection> metricSetCollection = metricCache.values(); + return metricSetCollection.stream().map( + metricSet -> metricSet.stream().map(metric -> { + InstrumentFurther instrumentFurther = metric.getInstrumentFurther(); + if (!Objects.nonNull(instrumentFurther)) { + return null; + } + Map ext = instrumentFurther.getExt(); + if (!Objects.nonNull(ext)) { + return null; + } + return (Pair) ext.get(InstrumentFurther.INSTRUMENT_VIEW); + }).filter(pair -> pair != null).collect(Collectors.toList())).flatMap(viewSet -> viewSet.stream()).collect(Collectors.toList()); + } + + /** + * Handles the creation of a sync metric based on its instrument type. + * + * @param metric The sync metric. + * @param meter The Prometheus meter. + * @param instrumentFurther Additional instrument information. + */ + private static void handleMetric(final SyncMetric metric, final Meter meter, final InstrumentFurther instrumentFurther) { + InstrumentType instrumentType = metric.getInstrumentType(); + switch (instrumentType) { + case LONG_COUNTER: { + LongCounter counter = meter.counterBuilder(instrumentFurther.getName()) + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .build(); + metric.setInstrument(counter); + break; + } + case DOUBLE_COUNTER: { + DoubleCounter counter = meter.counterBuilder(instrumentFurther.getName()) + .ofDoubles() + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .build(); + metric.setInstrument(counter); + break; + } + case LONG_HISTOGRAM: { + LongHistogram histogram = meter.histogramBuilder(instrumentFurther.getName()) + .ofLongs() + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .build(); + metric.setInstrument(histogram); + break; + } + case DOUBLE_HISTOGRAM: { + DoubleHistogram histogram = meter.histogramBuilder(instrumentFurther.getName()) + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .build(); + metric.setInstrument(histogram); + break; + } + case LONG_UP_DOWN_COUNTER: { + LongUpDownCounter counter = meter.upDownCounterBuilder(instrumentFurther.getName()) + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .build(); + metric.setInstrument(counter); + break; + } + case DOUBLE_UP_DOWN_COUNTER: { + DoubleUpDownCounter counter = meter.upDownCounterBuilder(instrumentFurther.getName()) + .ofDoubles() + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .build(); + metric.setInstrument(counter); + break; + } + default: + throw new IllegalArgumentException(String.format("%s not Support", instrumentType.getType().getName())); + } + } + + /** + * Handles the observable metric based on the type of the instrument. + * + * @param observableMetric The observable metric that needs to be handled. + * @param meter The meter object used to build the instrument. + * @param instrumentFurther The instrument further which contains the name, description and unit of the instrument. + */ + private static void handleObservableMetric(final ObservableMetric observableMetric, final Meter meter, + final InstrumentFurther instrumentFurther) { + InstrumentType instrumentType = observableMetric.getInstrumentType(); + Attributes attributes = buildAttributes(observableMetric); + switch (instrumentType) { + case OBSERVABLE_LONG_GAUGE: { + meter.gaugeBuilder(instrumentFurther.getName()) + .ofLongs() + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .buildWithCallback( + measurement -> measurement.record((Long) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); + break; + } + case OBSERVABLE_DOUBLE_GAUGE: { + meter.gaugeBuilder(instrumentFurther.getName()) + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .buildWithCallback( + measurement -> measurement.record((Double) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); + break; + } + case OBSERVABLE_LONG_COUNTER: { + meter.counterBuilder(instrumentFurther.getName()) + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .buildWithCallback( + measurement -> measurement.record((Long) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); + + break; + } + case OBSERVABLE_DOUBLE_COUNTER: { + meter.counterBuilder(instrumentFurther.getName()) + .ofDoubles() + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .buildWithCallback( + measurement -> measurement.record((Double) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); + + break; + } + case OBSERVABLE_LONG_UP_DOWN_COUNTER: { + meter.upDownCounterBuilder(instrumentFurther.getName()) + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .buildWithCallback( + measurement -> measurement.record((Long) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); + break; + } + case OBSERVABLE_DOUBLE_UP_DOWN_COUNTER: { + meter.upDownCounterBuilder(instrumentFurther.getName()) + .ofDoubles() + .setDescription(instrumentFurther.getDescription()) + .setUnit(instrumentFurther.getUnit()) + .buildWithCallback( + measurement -> measurement.record((Double) Objects.requireNonNull(observableMetric.supplier()).get(), attributes)); + + break; + } + default: + } + } + + /** + * Builds the attributes for an ObservableMetric. + * + * @param observableMetric The ObservableMetric from which to retrieve the attributes. + * @return The built attributes. + */ + private static Attributes buildAttributes(ObservableMetric observableMetric) { + Map attributes = observableMetric.getAttributes(); + AttributesBuilder builder = Attributes.builder(); + attributes.forEach(builder::put); + return builder.build(); + } + +} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/config/PrometheusConfiguration.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/config/PrometheusConfiguration.java index e2d684f04b..0e3adb44ff 100644 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/config/PrometheusConfiguration.java +++ b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/config/PrometheusConfiguration.java @@ -28,4 +28,7 @@ public class PrometheusConfiguration { @ConfigField(field = "port") private int eventMeshPrometheusPort = 19090; + + @ConfigField(field = "export.host") + private String eventMeshPrometheusExportHost = "0.0.0.0"; } diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusGrpcExporter.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusGrpcExporter.java deleted file mode 100644 index ca61f9add5..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusGrpcExporter.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.prometheus.metrics; - -import static org.apache.eventmesh.common.Constants.GRPC; -import static org.apache.eventmesh.metrics.prometheus.utils.PrometheusExporterConstants.METRICS_GRPC_PREFIX; -import static org.apache.eventmesh.metrics.prometheus.utils.PrometheusExporterUtils.join; -import static org.apache.eventmesh.metrics.prometheus.utils.PrometheusExporterUtils.observeOfValue; - -import org.apache.eventmesh.metrics.api.model.GrpcSummaryMetrics; - -import java.util.Map; -import java.util.function.Function; - -import io.opentelemetry.api.metrics.GlobalMeterProvider; -import io.opentelemetry.api.metrics.Meter; - -import com.google.common.collect.ImmutableMap; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class PrometheusGrpcExporter { - - /** - * Map structure : [metric name, description of name] -> the method of get corresponding metric. - */ - private final Map> paramPairs = ImmutableMap - .>builder() - .put(join("sub.topic.num", "get sub topic num."), GrpcSummaryMetrics::getSubscribeTopicNum) - .put(join("retry.queue.size", "get size of retry queue."), GrpcSummaryMetrics::getRetrySize) - .put(join("server.tps", "get size of retry queue."), GrpcSummaryMetrics::getClient2EventMeshTPS) - .put(join("client.tps", "get tps of eventMesh to mq."), GrpcSummaryMetrics::getEventMesh2ClientTPS) - .put(join("mq.provider.tps", "get tps of eventMesh to mq."), GrpcSummaryMetrics::getEventMesh2MqTPS) - .put(join("mq.consumer.tps", "get tps of eventMesh to mq."), GrpcSummaryMetrics::getMq2EventMeshTPS) - .build(); - - public static void export(final String meterName, final GrpcSummaryMetrics summaryMetrics) { - final Meter meter = GlobalMeterProvider.getMeter(meterName); - - paramPairs.forEach( - (metricInfo, getMetric) -> observeOfValue(meter, METRICS_GRPC_PREFIX + metricInfo[0], metricInfo[1], - GRPC, summaryMetrics, getMetric, GrpcSummaryMetrics.class)); - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusHttpExporter.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusHttpExporter.java deleted file mode 100644 index a09cb074d9..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusHttpExporter.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.prometheus.metrics; - -import static org.apache.eventmesh.common.Constants.HTTP; -import static org.apache.eventmesh.metrics.prometheus.utils.PrometheusExporterUtils.join; -import static org.apache.eventmesh.metrics.prometheus.utils.PrometheusExporterUtils.observeOfValue; - -import org.apache.eventmesh.metrics.api.model.HttpSummaryMetrics; - -import java.util.Map; -import java.util.function.Function; - -import io.opentelemetry.api.metrics.GlobalMeterProvider; -import io.opentelemetry.api.metrics.Meter; - -import com.google.common.collect.ImmutableMap; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class PrometheusHttpExporter { - - /** - * Map structure : [metric name, description of name] -> the method of get corresponding metric. - */ - private final Map> paramPairs = ImmutableMap - .>builder() - // maxHTTPTPS - .put(join("eventmesh.http.request.tps.max", "max TPS of HTTP."), HttpSummaryMetrics::maxHTTPTPS) - // avgHTTPTPS - .put(join("eventmesh.http.request.tps.avg", "avg TPS of HTTP."), HttpSummaryMetrics::avgHTTPTPS) - // maxHTTPCost - .put(join("eventmesh.http.request.cost.max", "max cost of HTTP."), HttpSummaryMetrics::maxHTTPCost) - // avgHTTPCost - .put(join("eventmesh.http.request.cost.avg", "avg cost of HTTP."), HttpSummaryMetrics::avgHTTPCost) - // avgHTTPBodyDecodeCost - .put(join("eventmesh.http.body.decode.cost.avg", "avg body decode cost of HTTP."), HttpSummaryMetrics::avgHTTPBodyDecodeCost) - // httpDiscard - .put(join("eventmesh.http.request.discard.num", "http request discard num."), HttpSummaryMetrics::getHttpDiscard) - // maxBatchSendMsgTPS - .put(join("eventmesh.batch.send.message.tps.max", "max of batch send message tps."), HttpSummaryMetrics::maxSendBatchMsgTPS) - // avgBatchSendMsgTPS - .put(join("eventmesh.batch.send.message.tps.avg", "avg of batch send message tps."), HttpSummaryMetrics::avgSendBatchMsgTPS) - // sum - .put(join("eventmesh.batch.send.message.num", "sum of batch send message number."), HttpSummaryMetrics::getSendBatchMsgNumSum) - // sumFail - .put(join("eventmesh.batch.send.message.fail.num", "sum of batch send message fail message number."), - HttpSummaryMetrics::getSendBatchMsgFailNumSum) - // sumFailRate - .put(join("eventmesh.batch.send.message.fail.rate", "send batch message fail rate."), HttpSummaryMetrics::getSendBatchMsgFailRate) - // discard - .put(join("eventmesh.batch.send.message.discard.num", "sum of send batch message discard number."), - HttpSummaryMetrics::getSendBatchMsgDiscardNumSum) - // maxSendMsgTPS - .put(join("eventmesh.send.message.tps.max", "max of send message tps."), HttpSummaryMetrics::maxSendMsgTPS) - // avgSendMsgTPS - .put(join("eventmesh.send.message.tps.avg", "avg of send message tps."), HttpSummaryMetrics::avgSendMsgTPS) - // sum - .put(join("eventmesh.send.message.num", "sum of send message number."), HttpSummaryMetrics::getSendMsgNumSum) - // sumFail - .put(join("eventmesh.send.message.fail.num", "sum of send message fail number."), HttpSummaryMetrics::getSendMsgFailNumSum) - // sumFailRate - .put(join("eventmesh.send.message.fail.rate", "send message fail rate."), HttpSummaryMetrics::getSendMsgFailRate) - // replyMsg - .put(join("eventmesh.reply.message.num", "sum of reply message number."), HttpSummaryMetrics::getReplyMsgNumSum) - // replyFail - .put(join("eventmesh.reply.message.fail.num", "sum of reply message fail number."), HttpSummaryMetrics::getReplyMsgFailNumSum) - // maxPushMsgTPS - .put(join("eventmesh.push.message.tps.max", "max of push message tps."), HttpSummaryMetrics::maxPushMsgTPS) - // avgPushMsgTPS - .put(join("eventmesh.push.message.tps.avg", "avg of push message tps."), HttpSummaryMetrics::avgPushMsgTPS) - // sum - .put(join("eventmesh.http.push.message.num", "sum of http push message number."), HttpSummaryMetrics::getHttpPushMsgNumSum) - // sumFail - .put(join("eventmesh.http.push.message.fail.num", "sum of http push message fail number."), HttpSummaryMetrics::getHttpPushFailNumSum) - // sumFailRate - .put(join("eventmesh.http.push.message.fail.rate", "http push message fail rate."), HttpSummaryMetrics::getHttpPushMsgFailRate) - // maxClientLatency - .put(join("eventmesh.http.push.latency.max", "max of http push latency."), HttpSummaryMetrics::maxHTTPPushLatency) - // avgClientLatency - .put(join("eventmesh.http.push.latency.avg", "avg of http push latency."), HttpSummaryMetrics::avgHTTPPushLatency) - // batchMsgQ - .put(join("eventmesh.batch.message.queue.size", "size of batch message queue."), HttpSummaryMetrics::getBatchMsgQueueSize) - // sendMsgQ - .put(join("eventmesh.send.message.queue.size", "size of send message queue."), HttpSummaryMetrics::getSendMsgQueueSize) - // pushMsgQ - .put(join("eventmesh.push.message.queue.size", "size of push message queue."), HttpSummaryMetrics::getPushMsgQueueSize) - // httpRetryQ - .put(join("eventmesh.http.retry.queue.size", "size of http retry queue."), HttpSummaryMetrics::getHttpRetryQueueSize) - // batchAvgSend2MQCost - .put(join("eventmesh.batch.send.message.cost.avg", "avg of batch send message cost."), HttpSummaryMetrics::avgBatchSendMsgCost) - // avgSend2MQCost - .put(join("eventmesh.send.message.cost.avg", "avg of send message cost."), HttpSummaryMetrics::avgSendMsgCost) - // avgReply2MQCost - .put(join("eventmesh.reply.message.cost.avg", "avg of reply message cost."), HttpSummaryMetrics::avgReplyMsgCost) - .build(); - - public void export(String name, HttpSummaryMetrics summaryMetrics) { - Meter meter = GlobalMeterProvider.getMeter(name); - paramPairs.forEach((metricInfo, getMetric) -> observeOfValue(meter, metricInfo[0], metricInfo[1], - HTTP, summaryMetrics, getMetric, HttpSummaryMetrics.class)); - } - -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusTcpExporter.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusTcpExporter.java deleted file mode 100644 index ff9eef9a4f..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/metrics/PrometheusTcpExporter.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.prometheus.metrics; - -import static org.apache.eventmesh.common.Constants.TCP; -import static org.apache.eventmesh.metrics.prometheus.utils.PrometheusExporterConstants.METRICS_TCP_PREFIX; -import static org.apache.eventmesh.metrics.prometheus.utils.PrometheusExporterUtils.join; -import static org.apache.eventmesh.metrics.prometheus.utils.PrometheusExporterUtils.observeOfValue; - -import org.apache.eventmesh.metrics.api.model.TcpSummaryMetrics; - -import java.util.Map; -import java.util.function.Function; - -import io.opentelemetry.api.metrics.GlobalMeterProvider; -import io.opentelemetry.api.metrics.Meter; - -import com.google.common.collect.ImmutableMap; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class PrometheusTcpExporter { - - /** - * Map structure : [metric name, description of name] -> the method of get corresponding metric. - */ - private final Map> paramPairs = ImmutableMap - .>builder() - // retryQueueSize - .put(join("retry.queue.size", "get size of retry queue."), TcpSummaryMetrics::getRetrySize) - // client2eventMeshTPS - .put(join("server.tps", "get tps of client to eventMesh."), TcpSummaryMetrics::getClient2eventMeshTPS) - // eventMesh2mqTPS - .put(join("mq.provider.tps", "get tps of eventMesh to mq."), TcpSummaryMetrics::getEventMesh2mqTPS) - // mq2eventMeshTPS - .put(join("mq.consumer.tps", "get tps of mq to eventMesh."), TcpSummaryMetrics::getMq2eventMeshTPS) - // eventMesh2clientTPS - .put(join("client.tps", "get tps of eventMesh to client."), TcpSummaryMetrics::getEventMesh2clientTPS) - // allTPS - .put(join("all.tps", "get all TPS."), TcpSummaryMetrics::getAllTPS) - // EventMeshTcpConnectionHandler.connections - .put(join("connection.num", "EventMeshTcpConnectionHandler.connections."), TcpSummaryMetrics::getAllConnections) - // subTopicNum - .put(join("sub.topic.num", "get sub topic num."), TcpSummaryMetrics::getSubTopicNum) - .build(); - - public void export(final String meterName, final TcpSummaryMetrics summaryMetrics) { - final Meter meter = GlobalMeterProvider.getMeter(meterName); - paramPairs.forEach( - (metricInfo, getMetric) -> observeOfValue(meter, METRICS_TCP_PREFIX + metricInfo[0], metricInfo[1], - TCP, summaryMetrics, getMetric, TcpSummaryMetrics.class)); - } -} diff --git a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/utils/PrometheusExporterUtils.java b/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/utils/PrometheusExporterUtils.java deleted file mode 100644 index 1a772020c3..0000000000 --- a/eventmesh-metrics-plugin/eventmesh-metrics-prometheus/src/main/java/org/apache/eventmesh/metrics/prometheus/utils/PrometheusExporterUtils.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.metrics.prometheus.utils; - -import org.apache.eventmesh.metrics.api.model.Metric; - -import java.lang.reflect.Method; -import java.util.function.Function; - -import io.opentelemetry.api.metrics.Meter; -import io.opentelemetry.api.metrics.common.Labels; - -import lombok.SneakyThrows; - -/** - * Utils for metrics-prometheus module - */ -public class PrometheusExporterUtils { - - /** - * Build the OpenTelemetry's Meter - * - * @param meter - * @param metricName - * @param metricDesc - * @param protocol - * @param summaryMetrics - * @param getMetric - */ - @SneakyThrows - public static void observeOfValue(Meter meter, String metricName, String metricDesc, String protocol, - Metric summaryMetrics, Function getMetric, Class clazz) { - Method method = getMetric.getClass().getMethod("apply", Object.class); - Class metricType = (Class) method.getGenericReturnType(); - if (metricType == Long.class) { - meter.longValueObserverBuilder(metricName) - .setDescription(metricDesc) - .setUnit(protocol) - .setUpdater(result -> result.observe((long) getMetric.apply(clazz.cast(summaryMetrics)), Labels.empty())) - .build(); - } else if (metricType == Double.class) { - meter.doubleValueObserverBuilder(metricName) - .setDescription(metricDesc) - .setUnit(protocol) - .setUpdater(result -> result.observe((double) getMetric.apply(clazz.cast(summaryMetrics)), Labels.empty())) - .build(); - } - } - - /** - * create and init an array contains 2 String. - * - * @param metricName the metric name - * @param desc the description of metric - * @return - */ - public static String[] join(String metricName, String desc) { - return new String[]{metricName, desc}; - } - -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/MetricsHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/MetricsHandler.java index 2a63f8f581..a2be02aaf1 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/MetricsHandler.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/MetricsHandler.java @@ -18,13 +18,13 @@ package org.apache.eventmesh.runtime.admin.handler.v1; import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.metrics.api.model.HttpSummaryMetrics; -import org.apache.eventmesh.metrics.api.model.TcpSummaryMetrics; import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; import org.apache.eventmesh.runtime.admin.response.v1.GetMetricsResponse; import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; +import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; +import org.apache.eventmesh.runtime.metrics.tcp.TcpMetrics; import java.io.IOException; @@ -45,8 +45,8 @@ @EventMeshHttpHandler(path = "/metrics") public class MetricsHandler extends AbstractHttpHandler { - private final HttpSummaryMetrics httpSummaryMetrics; - private final TcpSummaryMetrics tcpSummaryMetrics; + private final HttpMetrics httpMetrics; + private final TcpMetrics tcpMetrics; /** * Constructs a new instance with the provided EventMesh server instance. @@ -57,54 +57,56 @@ public class MetricsHandler extends AbstractHttpHandler { public MetricsHandler(EventMeshHTTPServer eventMeshHTTPServer, EventMeshTCPServer eventMeshTcpServer) { super(); - this.httpSummaryMetrics = eventMeshHTTPServer.getMetrics().getSummaryMetrics(); - this.tcpSummaryMetrics = eventMeshTcpServer.getEventMeshTcpMonitor().getTcpSummaryMetrics(); + this.httpMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); + this.tcpMetrics = eventMeshTcpServer.getEventMeshTcpMetricsManager().getTcpMetrics(); } @Override protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) throws IOException { GetMetricsResponse getMetricsResponse = new GetMetricsResponse( - httpSummaryMetrics.maxHTTPTPS(), - httpSummaryMetrics.avgHTTPTPS(), - httpSummaryMetrics.maxHTTPCost(), - httpSummaryMetrics.avgHTTPCost(), - httpSummaryMetrics.avgHTTPBodyDecodeCost(), - httpSummaryMetrics.getHttpDiscard(), - httpSummaryMetrics.maxSendBatchMsgTPS(), - httpSummaryMetrics.avgSendBatchMsgTPS(), - httpSummaryMetrics.getSendBatchMsgNumSum(), - httpSummaryMetrics.getSendBatchMsgFailNumSum(), - httpSummaryMetrics.getSendBatchMsgFailRate(), - httpSummaryMetrics.getSendBatchMsgDiscardNumSum(), - httpSummaryMetrics.maxSendMsgTPS(), - httpSummaryMetrics.avgSendMsgTPS(), - httpSummaryMetrics.getSendMsgNumSum(), - httpSummaryMetrics.getSendMsgFailNumSum(), - httpSummaryMetrics.getSendMsgFailRate(), - httpSummaryMetrics.getReplyMsgNumSum(), - httpSummaryMetrics.getReplyMsgFailNumSum(), - httpSummaryMetrics.maxPushMsgTPS(), - httpSummaryMetrics.avgPushMsgTPS(), - httpSummaryMetrics.getHttpPushMsgNumSum(), - httpSummaryMetrics.getHttpPushFailNumSum(), - httpSummaryMetrics.getHttpPushMsgFailRate(), - httpSummaryMetrics.maxHTTPPushLatency(), - httpSummaryMetrics.avgHTTPPushLatency(), - httpSummaryMetrics.getBatchMsgQueueSize(), - httpSummaryMetrics.getSendMsgQueueSize(), - httpSummaryMetrics.getPushMsgQueueSize(), - httpSummaryMetrics.getHttpRetryQueueSize(), - httpSummaryMetrics.avgBatchSendMsgCost(), - httpSummaryMetrics.avgSendMsgCost(), - httpSummaryMetrics.avgReplyMsgCost(), - tcpSummaryMetrics.getRetrySize(), - tcpSummaryMetrics.getClient2eventMeshTPS(), - tcpSummaryMetrics.getEventMesh2mqTPS(), - tcpSummaryMetrics.getMq2eventMeshTPS(), - tcpSummaryMetrics.getEventMesh2clientTPS(), - tcpSummaryMetrics.getAllTPS(), - tcpSummaryMetrics.getAllConnections(), - tcpSummaryMetrics.getSubTopicNum()); + httpMetrics.maxHTTPTPS(), + httpMetrics.avgHTTPTPS(), + httpMetrics.maxHTTPCost(), + httpMetrics.avgHTTPCost(), + httpMetrics.avgHTTPBodyDecodeCost(), + httpMetrics.getHttpDiscard(), + httpMetrics.maxSendBatchMsgTPS(), + httpMetrics.avgSendBatchMsgTPS(), + httpMetrics.getSendBatchMsgNumSum(), + httpMetrics.getSendBatchMsgFailNumSum(), + httpMetrics.getSendBatchMsgFailRate(), + httpMetrics.getSendBatchMsgDiscardNumSum(), + httpMetrics.maxSendMsgTPS(), + httpMetrics.avgSendMsgTPS(), + httpMetrics.getSendMsgNumSum(), + httpMetrics.getSendMsgFailNumSum(), + httpMetrics.getSendMsgFailRate(), + httpMetrics.getReplyMsgNumSum(), + httpMetrics.getReplyMsgFailNumSum(), + httpMetrics.maxPushMsgTPS(), + httpMetrics.avgPushMsgTPS(), + httpMetrics.getHttpPushMsgNumSum(), + httpMetrics.getHttpPushFailNumSum(), + httpMetrics.getHttpPushMsgFailRate(), + httpMetrics.maxHTTPPushLatency(), + httpMetrics.avgHTTPPushLatency(), + httpMetrics.getBatchMsgQueueSize(), + httpMetrics.getSendMsgQueueSize(), + httpMetrics.getPushMsgQueueSize(), + httpMetrics.getHttpRetryQueueSize(), + httpMetrics.avgBatchSendMsgCost(), + httpMetrics.avgSendMsgCost(), + httpMetrics.avgReplyMsgCost(), + + tcpMetrics.getRetrySize(), + tcpMetrics.getClient2eventMeshTPS(), + tcpMetrics.getEventMesh2mqTPS(), + tcpMetrics.getMq2eventMeshTPS(), + tcpMetrics.getEventMesh2clientTPS(), + tcpMetrics.getAllTPS(), + tcpMetrics.getAllConnections(), + tcpMetrics.getSubTopicNum() + ); String result = JsonUtils.toJSONString(getMetricsResponse); writeJson(ctx, result); } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetMetricsResponse.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetMetricsResponse.java index df95eabdbb..16f1e232f3 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetMetricsResponse.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/response/v1/GetMetricsResponse.java @@ -20,101 +20,104 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Builder; import lombok.Data; @Data +@Builder public class GetMetricsResponse { // HTTP Metrics - private float maxHTTPTPS; - private float avgHTTPTPS; + private double maxHTTPTPS; + private double avgHTTPTPS; private long maxHTTPCost; - private float avgHTTPCost; - private float avgHTTPBodyDecodeCost; + private double avgHTTPCost; + private double avgHTTPBodyDecodeCost; private long httpDiscard; - private float maxBatchSendMsgTPS; - private float avgBatchSendMsgTPS; + private double maxBatchSendMsgTPS; + private double avgBatchSendMsgTPS; private long sendBatchMsgNumSum; private long sendBatchMsgFailNumSum; - private float sendBatchMsgFailRate; + private double sendBatchMsgFailRate; private long sendBatchMsgDiscardNumSum; - private float maxSendMsgTPS; - private float avgSendMsgTPS; + private double maxSendMsgTPS; + private double avgSendMsgTPS; private long sendMsgNumSum; private long sendMsgFailNumSum; - private float sendMsgFailRate; + private double sendMsgFailRate; private long replyMsgNumSum; private long replyMsgFailNumSum; - private float maxPushMsgTPS; - private float avgPushMsgTPS; + private double maxPushMsgTPS; + private double avgPushMsgTPS; private long pushHTTPMsgNumSum; private long pushHTTPMsgFailNumSum; - private float pushHTTPMsgFailRate; - private float maxHTTPPushLatency; - private float avgHTTPPushLatency; - private int batchMsgQueueSize; - private int sendMsgQueueSize; - private int pushMsgQueueSize; + private double pushHTTPMsgFailRate; + private double maxHTTPPushLatency; + private double avgHTTPPushLatency; + private long batchMsgQueueSize; + private long sendMsgQueueSize; + private long pushMsgQueueSize; private long retryHTTPQueueSize; - private float avgBatchSendMsgCost; - private float avgSendMsgCost; - private float avgReplyMsgCost; + private double avgBatchSendMsgCost; + private double avgSendMsgCost; + private double avgReplyMsgCost; // TCP Metrics - private long retryTCPQueueSize; + private int retryTCPQueueSize; private double client2eventMeshTCPTPS; private double eventMesh2mqTCPTPS; private double mq2eventMeshTCPTPS; private double eventMesh2clientTCPTPS; private double allTCPTPS; - private int allTCPConnections; - private int subTopicTCPNum; + private long allTCPConnections; + private long subTopicTCPNum; + @JsonCreator public GetMetricsResponse( // HTTP Metrics - @JsonProperty("maxHTTPTPS") float maxHTTPTPS, - @JsonProperty("avgHTTPTPS") float avgHTTPTPS, + @JsonProperty("maxHTTPTPS") double maxHTTPTPS, + @JsonProperty("avgHTTPTPS") double avgHTTPTPS, @JsonProperty("maxHTTPCost") long maxHTTPCost, - @JsonProperty("avgHTTPCost") float avgHTTPCost, - @JsonProperty("avgHTTPBodyDecodeCost") float avgHTTPBodyDecodeCost, + @JsonProperty("avgHTTPCost") double avgHTTPCost, + @JsonProperty("avgHTTPBodyDecodeCost") double avgHTTPBodyDecodeCost, @JsonProperty("httpDiscard") long httpDiscard, - @JsonProperty("maxBatchSendMsgTPS") float maxBatchSendMsgTPS, - @JsonProperty("avgBatchSendMsgTPS") float avgBatchSendMsgTPS, + @JsonProperty("maxBatchSendMsgTPS") double maxBatchSendMsgTPS, + @JsonProperty("avgBatchSendMsgTPS") double avgBatchSendMsgTPS, @JsonProperty("sendBatchMsgNumSum") long sendBatchMsgNumSum, @JsonProperty("sendBatchMsgFailNumSum") long sendBatchMsgFailNumSum, - @JsonProperty("sendBatchMsgFailRate") float sendBatchMsgFailRate, + @JsonProperty("sendBatchMsgFailRate") double sendBatchMsgFailRate, @JsonProperty("sendBatchMsgDiscardNumSum") long sendBatchMsgDiscardNumSum, - @JsonProperty("maxSendMsgTPS") float maxSendMsgTPS, - @JsonProperty("avgSendMsgTPS") float avgSendMsgTPS, + @JsonProperty("maxSendMsgTPS") double maxSendMsgTPS, + @JsonProperty("avgSendMsgTPS") double avgSendMsgTPS, @JsonProperty("sendMsgNumSum") long sendMsgNumSum, @JsonProperty("sendMsgFailNumSum") long sendMsgFailNumSum, - @JsonProperty("sendMsgFailRate") float sendMsgFailRate, + @JsonProperty("sendMsgFailRate") double sendMsgFailRate, @JsonProperty("replyMsgNumSum") long replyMsgNumSum, @JsonProperty("replyMsgFailNumSum") long replyMsgFailNumSum, - @JsonProperty("maxPushMsgTPS") float maxPushMsgTPS, - @JsonProperty("avgPushMsgTPS") float avgPushMsgTPS, + @JsonProperty("maxPushMsgTPS") double maxPushMsgTPS, + @JsonProperty("avgPushMsgTPS") double avgPushMsgTPS, @JsonProperty("pushHTTPMsgNumSum") long pushHTTPMsgNumSum, @JsonProperty("pushHTTPMsgFailNumSum") long pushHTTPMsgFailNumSum, - @JsonProperty("pushHTTPMsgFailRate") float pushHTTPMsgFailRate, - @JsonProperty("maxHTTPPushLatency") float maxHTTPPushLatency, - @JsonProperty("avgHTTPPushLatency") float avgHTTPPushLatency, - @JsonProperty("batchMsgQueueSize") int batchMsgQueueSize, - @JsonProperty("sendMsgQueueSize") int sendMsgQueueSize, - @JsonProperty("pushMsgQueueSize") int pushMsgQueueSize, + @JsonProperty("pushHTTPMsgFailRate") double pushHTTPMsgFailRate, + @JsonProperty("maxHTTPPushLatency") double maxHTTPPushLatency, + @JsonProperty("avgHTTPPushLatency") double avgHTTPPushLatency, + @JsonProperty("batchMsgQueueSize") long batchMsgQueueSize, + @JsonProperty("sendMsgQueueSize") long sendMsgQueueSize, + @JsonProperty("pushMsgQueueSize") long pushMsgQueueSize, @JsonProperty("retryHTTPQueueSize") long retryHTTPQueueSize, - @JsonProperty("avgBatchSendMsgCost") float avgBatchSendMsgCost, - @JsonProperty("avgSendMsgCost") float avgSendMsgCost, - @JsonProperty("avgReplyMsgCost") float avgReplyMsgCost, + @JsonProperty("avgBatchSendMsgCost") double avgBatchSendMsgCost, + @JsonProperty("avgSendMsgCost") double avgSendMsgCost, + @JsonProperty("avgReplyMsgCost") double avgReplyMsgCost, // TCP Metrics - @JsonProperty("retryTCPQueueSize") long retryTCPQueueSize, + @JsonProperty("retryTCPQueueSize") int retryTCPQueueSize, @JsonProperty("client2eventMeshTCPTPS") double client2eventMeshTCPTPS, @JsonProperty("eventMesh2mqTCPTPS") double eventMesh2mqTCPTPS, @JsonProperty("mq2eventMeshTCPTPS") double mq2eventMeshTCPTPS, @JsonProperty("eventMesh2clientTCPTPS") double eventMesh2clientTCPTPS, @JsonProperty("allTCPTPS") double allTCPTPS, - @JsonProperty("allTCPConnections") int allTCPConnections, - @JsonProperty("subTopicTCPNum") int subTopicTCPNum) { + @JsonProperty("allTCPConnections") long allTCPConnections, + @JsonProperty("subTopicTCPNum") long subTopicTCPNum) { super(); this.maxHTTPTPS = maxHTTPTPS; diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractHTTPServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractHTTPServer.java index 5065b73a01..97f05a8764 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractHTTPServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractHTTPServer.java @@ -32,7 +32,7 @@ import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; import org.apache.eventmesh.runtime.core.protocol.http.processor.HandlerService; import org.apache.eventmesh.runtime.core.protocol.http.processor.inf.HttpRequestProcessor; -import org.apache.eventmesh.runtime.metrics.http.HTTPMetricsServer; +import org.apache.eventmesh.runtime.metrics.http.EventMeshHttpMetricsManager; import org.apache.eventmesh.runtime.util.HttpRequestUtil; import org.apache.eventmesh.runtime.util.RemotingHelper; import org.apache.eventmesh.runtime.util.TraceUtils; @@ -96,7 +96,7 @@ public abstract class AbstractHTTPServer extends AbstractRemotingServer { private final transient EventMeshHTTPConfiguration eventMeshHttpConfiguration; - private HTTPMetricsServer metrics; + private EventMeshHttpMetricsManager eventMeshHttpMetricsManager; private static final DefaultHttpDataFactory DEFAULT_HTTP_DATA_FACTORY = new DefaultHttpDataFactory(false); @@ -253,7 +253,7 @@ public void sendResponse(final ChannelHandlerContext ctx, final DefaultFullHttpR */ private Map parseHttpRequestBody(final HttpRequest httpRequest) throws IOException { return HttpRequestUtil.parseHttpRequestBody(httpRequest, () -> System.currentTimeMillis(), - (startTime) -> metrics.getSummaryMetrics().recordDecodeTimeCost(System.currentTimeMillis() - startTime)); + (startTime) -> eventMeshHttpMetricsManager.getHttpMetrics().recordDecodeTimeCost(System.currentTimeMillis() - startTime)); } @Sharable @@ -292,7 +292,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) { TraceUtils.finishSpanWithException(span, headerMap, errorStatus.reasonPhrase(), null); return; } - metrics.getSummaryMetrics().recordHTTPRequest(); + eventMeshHttpMetricsManager.getHttpMetrics().recordHTTPRequest(); // process http final HttpCommand requestCommand = new HttpCommand(); @@ -403,8 +403,7 @@ private void processHttpCommandRequest(final ChannelHandlerContext ctx, final As } log.debug("{}", asyncContext.getResponse()); - metrics.getSummaryMetrics() - .recordHTTPReqResTimeCost(System.currentTimeMillis() - request.getReqTime()); + eventMeshHttpMetricsManager.getHttpMetrics().recordHTTPReqResTimeCost(System.currentTimeMillis() - request.getReqTime()); sendResponse(ctx, asyncContext.getResponse().httpResponse()); } catch (Exception e) { @@ -423,8 +422,8 @@ private void processHttpCommandRequest(final ChannelHandlerContext ctx, final As } catch (RejectedExecutionException re) { asyncContext.onComplete(request.createHttpCommandResponse(EventMeshRetCode.OVERLOAD)); - metrics.getSummaryMetrics().recordHTTPDiscard(); - metrics.getSummaryMetrics().recordHTTPReqResTimeCost(System.currentTimeMillis() - request.getReqTime()); + eventMeshHttpMetricsManager.getHttpMetrics().recordHTTPDiscard(); + eventMeshHttpMetricsManager.getHttpMetrics().recordHTTPReqResTimeCost(System.currentTimeMillis() - request.getReqTime()); try { sendResponse(ctx, asyncContext.getResponse().httpResponse()); @@ -533,12 +532,12 @@ public Boolean getUseTrace() { return useTrace; } - public HTTPMetricsServer getMetrics() { - return metrics; + public EventMeshHttpMetricsManager getEventMeshHttpMetricsManager() { + return eventMeshHttpMetricsManager; } - public void setMetrics(final HTTPMetricsServer metrics) { - this.metrics = metrics; + public void setEventMeshHttpMetricsManager(final EventMeshHttpMetricsManager eventMeshHttpMetricsManager) { + this.eventMeshHttpMetricsManager = eventMeshHttpMetricsManager; } public HTTPThreadPoolGroup getHttpThreadPoolGroup() { diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractTCPServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractTCPServer.java index ed50415594..c9a464bf8a 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractTCPServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractTCPServer.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.runtime.boot; +import org.apache.eventmesh.common.Pair; import org.apache.eventmesh.common.config.CommonConfiguration; import org.apache.eventmesh.common.protocol.tcp.Command; import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; @@ -26,7 +27,7 @@ import org.apache.eventmesh.common.protocol.tcp.UserAgent; import org.apache.eventmesh.common.protocol.tcp.codec.Codec; import org.apache.eventmesh.common.utils.AssertUtils; -import org.apache.eventmesh.runtime.common.Pair; +import org.apache.eventmesh.common.utils.IPUtils; import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; import org.apache.eventmesh.runtime.constants.EventMeshConstants; import org.apache.eventmesh.runtime.core.protocol.tcp.client.EventMeshTcp2Client; @@ -34,7 +35,7 @@ import org.apache.eventmesh.runtime.core.protocol.tcp.client.processor.TcpProcessor; import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.SessionState; -import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMonitor; +import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMetricsManager; import org.apache.eventmesh.runtime.util.EventMeshUtil; import org.apache.eventmesh.runtime.util.RemotingHelper; import org.apache.eventmesh.runtime.util.TraceUtils; @@ -84,7 +85,7 @@ public class AbstractTCPServer extends AbstractRemotingServer { private final EventMeshTCPConfiguration eventMeshTCPConfiguration; private ClientSessionGroupMapping clientSessionGroupMapping; - private EventMeshTcpMonitor eventMeshTcpMonitor; + protected EventMeshTcpMetricsManager eventMeshTcpMetricsManager; private transient GlobalTrafficShapingHandler globalTrafficShapingHandler; private TcpConnectionHandler tcpConnectionHandler; @@ -234,7 +235,7 @@ protected void channelRead0(ChannelHandlerContext ctx, Package pkg) throws Excep long startTime = System.currentTimeMillis(); validateMsg(pkg); - eventMeshTcpMonitor.getTcpSummaryMetrics().getClient2eventMeshMsgNum().incrementAndGet(); + eventMeshTcpMetricsManager.client2eventMeshMsgNumIncrement(IPUtils.parseChannelRemoteAddr(ctx.channel())); Command cmd = pkg.getHeader().getCmd(); try { @@ -290,8 +291,8 @@ private void processTcpCommandRequest(final Package pkg, final ChannelHandlerCon final long startTime, final Command cmd) { Pair pair = tcpRequestProcessorTable.get(cmd); - pair.getObject2().submit(() -> { - TcpProcessor processor = pair.getObject1(); + pair.getRight().submit(() -> { + TcpProcessor processor = pair.getLeft(); processor.process(pkg, ctx, startTime); @@ -388,7 +389,7 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { EventMeshTcp2Client.goodBye2Client(tcpThreadPoolGroup, session, errMsg, OPStatus.FAIL.getCode(), clientSessionGroupMapping); } else { - EventMeshTcp2Client.goodBye2Client(ctx, errMsg, clientSessionGroupMapping, eventMeshTcpMonitor); + EventMeshTcp2Client.goodBye2Client(ctx, errMsg, clientSessionGroupMapping, eventMeshTcpMetricsManager); } } @@ -459,12 +460,12 @@ public TcpConnectionHandler getTcpConnectionHandler() { return tcpConnectionHandler; } - public EventMeshTcpMonitor getEventMeshTcpMonitor() { - return eventMeshTcpMonitor; + public EventMeshTcpMetricsManager getEventMeshTcpMetricsManager() { + return eventMeshTcpMetricsManager; } - public void setEventMeshTcpMonitor(EventMeshTcpMonitor eventMeshTcpMonitor) { - this.eventMeshTcpMonitor = eventMeshTcpMonitor; + public void setEventMeshTcpMetricsManager(EventMeshTcpMetricsManager eventMeshTcpMetricsManager) { + this.eventMeshTcpMetricsManager = eventMeshTcpMetricsManager; } public TcpDispatcher getTcpDispatcher() { @@ -486,5 +487,4 @@ public ClientSessionGroupMapping getClientSessionGroupMapping() { public void setClientSessionGroupMapping(ClientSessionGroupMapping clientSessionGroupMapping) { this.clientSessionGroupMapping = clientSessionGroupMapping; } - } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcServer.java index 58832d1d39..17165012d8 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcServer.java @@ -36,7 +36,7 @@ import org.apache.eventmesh.runtime.core.protocol.grpc.service.HeartbeatService; import org.apache.eventmesh.runtime.core.protocol.grpc.service.PublisherService; import org.apache.eventmesh.runtime.meta.MetaStorage; -import org.apache.eventmesh.runtime.metrics.grpc.EventMeshGrpcMonitor; +import org.apache.eventmesh.runtime.metrics.grpc.EventMeshGrpcMetricsManager; import org.apache.commons.lang3.RandomUtils; import org.apache.http.impl.client.CloseableHttpClient; @@ -92,7 +92,7 @@ public class EventMeshGrpcServer extends AbstractRemotingServer { private final EventMeshServer eventMeshServer; - private EventMeshGrpcMonitor eventMeshGrpcMonitor; + private EventMeshGrpcMetricsManager eventMeshGrpcMetricsManager; public EventMeshGrpcServer(final EventMeshServer eventMeshServer, final EventMeshGrpcConfiguration eventMeshGrpcConfiguration) { this.eventMeshServer = eventMeshServer; @@ -147,7 +147,7 @@ public void start() throws Exception { this.register(); } - eventMeshGrpcMonitor.start(); + eventMeshGrpcMetricsManager.start(); log.info("---------------EventMeshGRPCServer running-------------------"); } @@ -167,7 +167,7 @@ public void shutdown() throws Exception { this.unRegister(); } - eventMeshGrpcMonitor.shutdown(); + eventMeshGrpcMetricsManager.shutdown(); log.info("---------------EventMeshGRPCServer stopped-------------------"); } @@ -237,8 +237,8 @@ public CloseableHttpClient getHttpClient() { return httpClientPool.get(RandomUtils.nextInt(size, 2 * size) % size); } - public EventMeshGrpcMonitor getMetricsMonitor() { - return eventMeshGrpcMonitor; + public EventMeshGrpcMetricsManager getEventMeshGrpcMetricsManager() { + return eventMeshGrpcMetricsManager; } private void initThreadPool() { @@ -281,14 +281,13 @@ private void initHttpClientPool() { } } + + private void initMetricsMonitor() throws Exception { final List metricsRegistries = Lists.newArrayList(); - Optional.ofNullable(eventMeshGrpcConfiguration.getEventMeshMetricsPluginType()) - .ifPresent( - metricsPlugins -> metricsPlugins.forEach( - pluginType -> metricsRegistries.add(MetricsPluginFactory.getMetricsRegistry(pluginType)))); - eventMeshGrpcMonitor = new EventMeshGrpcMonitor(this, metricsRegistries); - eventMeshGrpcMonitor.init(); + Optional.ofNullable(eventMeshGrpcConfiguration.getEventMeshMetricsPluginType()).ifPresent( + metricsPlugins -> metricsPlugins.forEach(pluginType -> metricsRegistries.add(MetricsPluginFactory.getMetricsRegistry(pluginType)))); + eventMeshGrpcMetricsManager = new EventMeshGrpcMetricsManager(this, metricsRegistries); } private void shutdownThreadPools() { diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHTTPServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHTTPServer.java index b43931370d..cc47cc9727 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHTTPServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHTTPServer.java @@ -55,7 +55,7 @@ import org.apache.eventmesh.runtime.core.protocol.http.retry.HttpRetryer; import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; import org.apache.eventmesh.runtime.meta.MetaStorage; -import org.apache.eventmesh.runtime.metrics.http.HTTPMetricsServer; +import org.apache.eventmesh.runtime.metrics.http.EventMeshHttpMetricsManager; import org.apache.eventmesh.webhook.receive.WebHookController; import org.apache.commons.lang3.StringUtils; @@ -121,7 +121,7 @@ public void init() throws Exception { httpRetryer = new HttpRetryer(this); - super.setMetrics(new HTTPMetricsServer(this, metricsRegistries)); + super.setEventMeshHttpMetricsManager(new EventMeshHttpMetricsManager(this, metricsRegistries)); subscriptionManager = new SubscriptionManager(eventMeshHttpConfiguration.isEventMeshServerMetaStorageEnable(), metaStorage); consumerManager = new ConsumerManager(this); @@ -135,7 +135,7 @@ public void init() throws Exception { transformerEngine = new TransformerEngine(metaStorage, producerManager, consumerManager); super.setHandlerService(new HandlerService()); - super.getHandlerService().setMetrics(this.getMetrics()); + super.getHandlerService().setMetrics(this.getEventMeshHttpMetricsManager()); // get the trace-plugin if (StringUtils.isNotEmpty(eventMeshHttpConfiguration.getEventMeshTracePluginType()) @@ -152,7 +152,7 @@ public void init() throws Exception { @Override public void start() throws Exception { super.start(); - this.getMetrics().start(); + this.getEventMeshHttpMetricsManager().start(); consumerManager.start(); producerManager.start(); @@ -173,7 +173,7 @@ public void shutdown() throws Exception { super.shutdown(); - this.getMetrics().shutdown(); + this.getEventMeshHttpMetricsManager().shutdown(); filterEngine.shutdown(); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshServer.java index 3b33a71b8f..b3250795d6 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshServer.java @@ -25,16 +25,23 @@ import org.apache.eventmesh.common.config.ConfigService; import org.apache.eventmesh.common.utils.AssertUtils; import org.apache.eventmesh.common.utils.ConfigurationContextUtil; +import org.apache.eventmesh.metrics.api.MetricsPluginFactory; +import org.apache.eventmesh.metrics.api.MetricsRegistry; import org.apache.eventmesh.runtime.acl.Acl; import org.apache.eventmesh.runtime.common.ServiceState; import org.apache.eventmesh.runtime.core.protocol.http.producer.ProducerTopicManager; import org.apache.eventmesh.runtime.meta.MetaStorage; +import org.apache.eventmesh.runtime.metrics.EventMeshMetricsManager; +import org.apache.eventmesh.runtime.metrics.MetricsManager; import org.apache.eventmesh.runtime.storage.StorageResource; import org.apache.eventmesh.runtime.trace.Trace; +import org.apache.commons.collections4.CollectionUtils; + import java.util.List; import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -71,6 +78,8 @@ public class EventMeshServer { private EventMeshHTTPServer eventMeshHTTPServer = null; + private EventMeshMetricsManager eventMeshMetricsManager; + public EventMeshServer() { // Initialize configuration @@ -105,6 +114,13 @@ public EventMeshServer() { if (BOOTSTRAP_LIST.isEmpty()) { BOOTSTRAP_LIST.add(new EventMeshTcpBootstrap(this)); } + + List metricsPluginTypes = configuration.getEventMeshMetricsPluginType(); + if (CollectionUtils.isNotEmpty(metricsPluginTypes)) { + List metricsRegistries = metricsPluginTypes.stream().map(metric -> MetricsPluginFactory.getMetricsRegistry(metric)) + .collect(Collectors.toList()); + eventMeshMetricsManager = new EventMeshMetricsManager(metricsRegistries); + } } public void init() throws Exception { @@ -133,6 +149,26 @@ public void init() throws Exception { } } + if (Objects.nonNull(eventMeshTCPServer)) { + MetricsManager metricsManager = eventMeshTCPServer.getEventMeshTcpMetricsManager(); + addMetricsManagerAndMetrics(metricsManager); + } + + if (Objects.nonNull(eventMeshGrpcServer)) { + MetricsManager metricsManager = eventMeshGrpcServer.getEventMeshGrpcMetricsManager(); + addMetricsManagerAndMetrics(metricsManager); + } + + if (Objects.nonNull(eventMeshHTTPServer)) { + MetricsManager metricsManager = eventMeshHTTPServer.getEventMeshHttpMetricsManager(); + addMetricsManagerAndMetrics(metricsManager); + } + + if (Objects.nonNull(eventMeshMetricsManager)) { + eventMeshMetricsManager.init(); + } + + if (Objects.nonNull(eventMeshTCPServer) && Objects.nonNull(eventMeshHTTPServer) && Objects.nonNull(eventMeshGrpcServer)) { adminBootstrap = new EventMeshAdminBootstrap(this); adminBootstrap.init(); @@ -145,6 +181,13 @@ public void init() throws Exception { log.info(SERVER_STATE_MSG, serviceState); } + private void addMetricsManagerAndMetrics(MetricsManager metricsManager) { + if (Objects.nonNull(metricsManager)) { + this.eventMeshMetricsManager.addMetricManager(metricsManager); + this.eventMeshMetricsManager.addMetrics(metricsManager.getMetrics()); + } + } + public void start() throws Exception { if (Objects.nonNull(configuration)) { if (configuration.isEventMeshServerSecurityEnable()) { diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTCPServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTCPServer.java index 7c81501d98..1c75e583f4 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTCPServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTCPServer.java @@ -44,7 +44,7 @@ import org.apache.eventmesh.runtime.core.protocol.tcp.client.rebalance.EventMeshRebalanceService; import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.retry.TcpRetryer; import org.apache.eventmesh.runtime.meta.MetaStorage; -import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMonitor; +import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMetricsManager; import org.apache.eventmesh.webhook.admin.AdminWebHookConfigOperationManager; import java.util.List; @@ -105,8 +105,7 @@ public void init() throws Exception { clientSessionGroupMapping.init(); super.setClientSessionGroupMapping(clientSessionGroupMapping); - super.setEventMeshTcpMonitor(new EventMeshTcpMonitor(this, metricsRegistries)); - super.getEventMeshTcpMonitor().init(); + super.setEventMeshTcpMetricsManager(new EventMeshTcpMetricsManager(this, metricsRegistries)); if (eventMeshTCPConfiguration.isEventMeshServerMetaStorageEnable()) { eventMeshRebalanceService = new EventMeshRebalanceService(this, new EventMeshRebalanceImpl(this)); @@ -124,7 +123,7 @@ public void init() throws Exception { @Override public void start() throws Exception { super.start(); - super.getEventMeshTcpMonitor().start(); + super.getEventMeshTcpMetricsManager().start(); clientSessionGroupMapping.start(); tcpRetryer.start(); @@ -141,7 +140,7 @@ public void start() throws Exception { public void shutdown() throws Exception { super.shutdown(); - super.getEventMeshTcpMonitor().shutdown(); + super.getEventMeshTcpMetricsManager().shutdown(); clientSessionGroupMapping.shutdown(); ThreadUtils.sleep(40, TimeUnit.SECONDS); @@ -285,4 +284,6 @@ public void setRateLimiter(RateLimiter rateLimiter) { public TcpRetryer getTcpRetryer() { return tcpRetryer; } + + } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/EventMeshConsumer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/EventMeshConsumer.java index 6f677ab4a4..6a70458b00 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/EventMeshConsumer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/consumer/EventMeshConsumer.java @@ -315,7 +315,7 @@ private EventListener createEventListener(final SubscriptionMode subscriptionMod log.debug("message|mq2eventMesh|topic={}|msg={}", topic, event); } else { log.info("message|mq2eventMesh|topic={}|bizSeqNo={}|uniqueId={}", topic, bizSeqNo, uniqueId); - eventMeshGrpcServer.getMetricsMonitor().recordReceiveMsgFromQueue(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromQueue(); } final EventMeshAsyncConsumeContext eventMeshAsyncConsumeContext = (EventMeshAsyncConsumeContext) context; diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/BatchPublishCloudEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/BatchPublishCloudEventProcessor.java index d912d82ace..a83083aec4 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/BatchPublishCloudEventProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/BatchPublishCloudEventProcessor.java @@ -66,7 +66,7 @@ public void handleCloudEvent(CloudEventBatch cloudEventBatch, EventEmitter emitte SendMessageContext sendMessageContext = new SendMessageContext(seqNum, cloudEvent, eventMeshProducer, eventMeshGrpcServer); - eventMeshGrpcServer.getMetricsMonitor().recordSendMsgToQueue(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordSendMsgToQueue(); long startTime = System.currentTimeMillis(); eventMeshProducer.send(sendMessageContext, new SendCallback() { @@ -71,7 +71,7 @@ public void onSuccess(SendResult sendResult) { long endTime = System.currentTimeMillis(); log.info("message|eventMesh2mq|REQ|ASYNC|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", endTime - startTime, topic, seqNum, uniqueId); - eventMeshGrpcServer.getMetricsMonitor().recordSendMsgToClient(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordSendMsgToClient(EventMeshCloudEventUtils.getIp(message)); } @Override diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/ReplyMessageProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/ReplyMessageProcessor.java index fe10ff5dac..e55e15d651 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/ReplyMessageProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/ReplyMessageProcessor.java @@ -109,7 +109,7 @@ public void process(CloudEvent message, EventEmitter emitter) throws SendMessageContext sendMessageContext = new SendMessageContext(seqNum, cloudEvent, eventMeshProducer, eventMeshGrpcServer); - eventMeshGrpcServer.getMetricsMonitor().recordSendMsgToQueue(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordSendMsgToQueue(); long startTime = System.currentTimeMillis(); eventMeshProducer.reply(sendMessageContext, new SendCallback() { diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/RequestCloudEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/RequestCloudEventProcessor.java index 1e934ff85c..1a6398b93d 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/RequestCloudEventProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/processor/RequestCloudEventProcessor.java @@ -60,14 +60,14 @@ public void handleCloudEvent(CloudEvent message, EventEmitter emitte SendMessageContext sendMessageContext = new SendMessageContext(seqNum, cloudEvent, eventMeshProducer, eventMeshGrpcServer); - eventMeshGrpcServer.getMetricsMonitor().recordSendMsgToQueue(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordSendMsgToQueue(); long startTime = System.currentTimeMillis(); eventMeshProducer.request(sendMessageContext, new RequestReplyCallback() { @Override public void onSuccess(io.cloudevents.CloudEvent event) { try { - eventMeshGrpcServer.getMetricsMonitor().recordReceiveMsgFromQueue(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromQueue(); EventMeshCloudEventWrapper wrapper = (EventMeshCloudEventWrapper) grpcCommandProtocolAdaptor.fromCloudEvent(event); emitter.onNext(wrapper.getMessage()); @@ -76,7 +76,7 @@ public void onSuccess(io.cloudevents.CloudEvent event) { long endTime = System.currentTimeMillis(); log.info("message|eventmesh2client|REPLY|RequestReply|send2MQCost={}ms|topic={}|bizSeqNo={}|uniqueId={}", endTime - startTime, topic, seqNum, uniqueId); - eventMeshGrpcServer.getMetricsMonitor().recordSendMsgToClient(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordSendMsgToClient(EventMeshCloudEventUtils.getIp(wrapper.getMessage())); } catch (Exception e) { ServiceUtils.sendStreamResponseCompleted(message, StatusCode.EVENTMESH_REQUEST_REPLY_MSG_ERR, EventMeshUtil.stackTrace(e, 2), emitter); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/ConsumerService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/ConsumerService.java index 8963cb95bc..2c2c501178 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/ConsumerService.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/ConsumerService.java @@ -58,10 +58,11 @@ public ConsumerService(EventMeshGrpcServer eventMeshGrpcServer, @Override public void subscribe(CloudEvent request, StreamObserver responseObserver) { + String clientIp = EventMeshCloudEventUtils.getIp(request); log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "subscribe", EventMeshConstants.PROTOCOL_GRPC, EventMeshCloudEventUtils.getIp(request), eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - eventMeshGrpcServer.getMetricsMonitor().recordReceiveMsgFromClient(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(clientIp); EventEmitter emitter = new EventEmitter<>(responseObserver); subscribeThreadPoolExecutor.submit(() -> { @@ -86,6 +87,7 @@ public StreamObserver subscribeStream(StreamObserver res public void onNext(CloudEvent subscription) { final String subMessageType = Optional.ofNullable(subscription.getAttributesMap().get(ProtocolKey.SUB_MESSAGE_TYPE)) .orElse(CloudEventAttributeValue.newBuilder().build()).getCeString(); + String clientId = EventMeshCloudEventUtils.getIp(subscription); if (StringUtils.equals(subMessageType, ProtocolKey.SUB_REPLY_MESSAGE)) { log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "reply-to-server", EventMeshConstants.PROTOCOL_GRPC, EventMeshCloudEventUtils.getIp(subscription), eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); @@ -94,7 +96,7 @@ public void onNext(CloudEvent subscription) { log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "subscribeStream", EventMeshConstants.PROTOCOL_GRPC, EventMeshCloudEventUtils.getIp(subscription), eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - eventMeshGrpcServer.getMetricsMonitor().recordReceiveMsgFromClient(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(clientId); handleSubscriptionStream(subscription, emitter); } } @@ -139,9 +141,10 @@ private void handleSubscribeReply(CloudEvent subscription, EventEmitter responseObserver) { + String clientIp = EventMeshCloudEventUtils.getIp(request); log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "unsubscribe", EventMeshConstants.PROTOCOL_GRPC, EventMeshCloudEventUtils.getIp(request), eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - eventMeshGrpcServer.getMetricsMonitor().recordReceiveMsgFromClient(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(clientIp); EventEmitter emitter = new EventEmitter<>(responseObserver); subscribeThreadPoolExecutor.submit(() -> { diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/PublisherService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/PublisherService.java index 1d67ab8669..3e62ae9215 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/PublisherService.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/grpc/service/PublisherService.java @@ -57,9 +57,10 @@ public PublisherService(EventMeshGrpcServer eventMeshGrpcServer, */ @Override public void publish(CloudEvent request, StreamObserver responseObserver) { + String clientId = EventMeshCloudEventUtils.getIp(request); log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "publish", EventMeshConstants.PROTOCOL_GRPC, EventMeshCloudEventUtils.getIp(request), eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - eventMeshGrpcServer.getMetricsMonitor().recordReceiveMsgFromClient(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(clientId); EventEmitter emitter = new EventEmitter<>(responseObserver); threadPoolExecutor.submit(() -> { @@ -84,10 +85,11 @@ public void publish(CloudEvent request, StreamObserver responseObser */ @Override public void requestReply(CloudEvent request, StreamObserver responseObserver) { + String clientIp = EventMeshCloudEventUtils.getIp(request); log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "RequestReply", EventMeshConstants.PROTOCOL_GRPC, EventMeshCloudEventUtils.getIp(request), eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - eventMeshGrpcServer.getMetricsMonitor().recordReceiveMsgFromClient(); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(clientIp); EventEmitter emitter = new EventEmitter<>(responseObserver); threadPoolExecutor.submit(() -> { @@ -112,10 +114,11 @@ public void requestReply(CloudEvent request, StreamObserver response */ @Override public void batchPublish(CloudEventBatch request, StreamObserver responseObserver) { + String clientIp = EventMeshCloudEventUtils.getIp(request.getEvents(0)); log.info("cmd={}|{}|client2eventMesh|from={}|to={}", "BatchPublish", EventMeshConstants.PROTOCOL_GRPC, null, eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshIp()); - eventMeshGrpcServer.getMetricsMonitor().recordReceiveMsgFromClient(request.getEventsCount()); + eventMeshGrpcServer.getEventMeshGrpcMetricsManager().recordReceiveMsgFromClient(request.getEventsCount(), clientIp); EventEmitter emitter = new EventEmitter<>(responseObserver); threadPoolExecutor.submit(() -> { diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageProcessor.java index 35fffd1cea..7b86661246 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageProcessor.java @@ -32,7 +32,6 @@ import org.apache.eventmesh.common.protocol.http.header.message.SendMessageBatchRequestHeader; import org.apache.eventmesh.common.protocol.http.header.message.SendMessageBatchResponseHeader; import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.metrics.api.model.HttpSummaryMetrics; import org.apache.eventmesh.protocol.api.ProtocolAdaptor; import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; import org.apache.eventmesh.runtime.acl.Acl; @@ -42,6 +41,7 @@ import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; +import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; import org.apache.eventmesh.runtime.util.RemotingHelper; import org.apache.commons.collections4.CollectionUtils; @@ -159,7 +159,7 @@ public void processRequest(ChannelHandlerContext ctx, AsyncContext } - HttpSummaryMetrics summaryMetrics = eventMeshHTTPServer.getMetrics().getSummaryMetrics(); + HttpMetrics summaryMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); if (!eventMeshHTTPServer.getBatchRateLimiter() .tryAcquire(eventSize, EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { summaryMetrics.recordSendBatchMsgDiscard(eventSize); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageV2Processor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageV2Processor.java index 323b2462e9..e36e51dd76 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageV2Processor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/BatchSendMessageV2Processor.java @@ -31,7 +31,6 @@ import org.apache.eventmesh.common.protocol.http.common.RequestCode; import org.apache.eventmesh.common.protocol.http.header.message.SendMessageBatchV2RequestHeader; import org.apache.eventmesh.common.protocol.http.header.message.SendMessageBatchV2ResponseHeader; -import org.apache.eventmesh.metrics.api.model.HttpSummaryMetrics; import org.apache.eventmesh.protocol.api.ProtocolAdaptor; import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; import org.apache.eventmesh.runtime.acl.Acl; @@ -41,6 +40,7 @@ import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; +import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; import org.apache.eventmesh.runtime.util.EventMeshUtil; import org.apache.eventmesh.runtime.util.RemotingHelper; @@ -161,7 +161,7 @@ public void processRequest(ChannelHandlerContext ctx, AsyncContext } } - HttpSummaryMetrics summaryMetrics = eventMeshHTTPServer.getMetrics().getSummaryMetrics(); + HttpMetrics summaryMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); if (!eventMeshHTTPServer.getBatchRateLimiter() .tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { summaryMetrics.recordSendBatchMsgDiscard(1); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/CreateTopicProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/CreateTopicProcessor.java index 64a5ddfd90..13892591e6 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/CreateTopicProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/CreateTopicProcessor.java @@ -120,7 +120,7 @@ public void handler(HandlerService.HandlerSpecific handlerSpecific, HttpRequest try { HTTP_LOGGER.debug("{}", httpEventWrapper); eventMeshHTTPServer.sendResponse(ctx, httpEventWrapper.httpResponse()); - eventMeshHTTPServer.getMetrics().getSummaryMetrics().recordHTTPReqResTimeCost( + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHTTPReqResTimeCost( System.currentTimeMillis() - requestWrapper.getReqTime()); } catch (Exception ex) { // ignore @@ -143,8 +143,8 @@ public void handler(HandlerService.HandlerSpecific handlerSpecific, HttpRequest long endTime = System.currentTimeMillis(); HTTP_LOGGER.warn( "create topic fail, eventMesh2client|cost={}ms|topic={}", endTime - startTime, topic, e); - eventMeshHTTPServer.getMetrics().getSummaryMetrics().recordSendMsgFailed(); - eventMeshHTTPServer.getMetrics().getSummaryMetrics().recordSendMsgCost(endTime - startTime); + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgFailed(); + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgCost(endTime - startTime); } } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/DeleteTopicProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/DeleteTopicProcessor.java index 0def2507f1..039754a815 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/DeleteTopicProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/DeleteTopicProcessor.java @@ -143,7 +143,7 @@ public void handler(HandlerService.HandlerSpecific handlerSpecific, HttpRequest try { HTTP_LOGGER.debug("{}", httpEventWrapper); eventMeshHTTPServer.sendResponse(ctx, httpEventWrapper.httpResponse()); - eventMeshHTTPServer.getMetrics().getSummaryMetrics().recordHTTPReqResTimeCost( + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHTTPReqResTimeCost( System.currentTimeMillis() - requestWrapper.getReqTime()); } catch (Exception ex) { // ignore @@ -166,8 +166,8 @@ public void handler(HandlerService.HandlerSpecific handlerSpecific, HttpRequest long endTime = System.currentTimeMillis(); HTTP_LOGGER.warn( "delete topic fail, eventMesh2client|cost={}ms|topic={}", endTime - startTime, topic, e); - eventMeshHTTPServer.getMetrics().getSummaryMetrics().recordSendMsgFailed(); - eventMeshHTTPServer.getMetrics().getSummaryMetrics().recordSendMsgCost(endTime - startTime); + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgFailed(); + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsgCost(endTime - startTime); } } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HandlerService.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HandlerService.java index b68ea2f648..6ecf745fa5 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HandlerService.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HandlerService.java @@ -27,7 +27,7 @@ import org.apache.eventmesh.runtime.common.EventMeshTrace; import org.apache.eventmesh.runtime.constants.EventMeshConstants; import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; -import org.apache.eventmesh.runtime.metrics.http.HTTPMetricsServer; +import org.apache.eventmesh.runtime.metrics.http.EventMeshHttpMetricsManager; import org.apache.eventmesh.runtime.util.HttpResponseUtils; import org.apache.eventmesh.runtime.util.RemotingHelper; @@ -76,7 +76,7 @@ public class HandlerService { private final Map httpProcessorMap = new ConcurrentHashMap<>(); @Setter - private HTTPMetricsServer metrics; + private EventMeshHttpMetricsManager metrics; @Setter private HTTPTrace httpTrace; @@ -241,7 +241,7 @@ private HttpEventWrapper parseHttpRequest(HttpRequest httpRequest) throws IOExce httpEventWrapper.setBody(requestBody); - metrics.getSummaryMetrics().recordDecodeTimeCost(System.currentTimeMillis() - bodyDecodeStart); + metrics.getHttpMetrics().recordDecodeTimeCost(System.currentTimeMillis() - bodyDecodeStart); return httpEventWrapper; } @@ -301,13 +301,13 @@ public void run() { } private void postHandlerWithTimeCostRecord(ConnectionType type) { - metrics.getSummaryMetrics().recordHTTPReqResTimeCost(System.currentTimeMillis() - requestTime); + metrics.getHttpMetrics().recordHTTPReqResTimeCost(System.currentTimeMillis() - requestTime); HTTP_LOGGER.debug("{}", response); postHandler(type); } private void postHandler(ConnectionType type) { - metrics.getSummaryMetrics().recordHTTPRequest(); + metrics.getHttpMetrics().recordHTTPRequest(); HTTP_LOGGER.debug("{}", request); if (Objects.isNull(response)) { this.response = HttpResponseUtils.createSuccess(); @@ -324,8 +324,8 @@ private void postHandler(ConnectionType type) { private void error() { log.error(this.exception.getMessage(), this.exception); this.traceOperation.exceptionTrace(this.exception, this.traceMap); - metrics.getSummaryMetrics().recordHTTPDiscard(); - metrics.getSummaryMetrics().recordHTTPReqResTimeCost(System.currentTimeMillis() - requestTime); + metrics.getHttpMetrics().recordHTTPDiscard(); + metrics.getHttpMetrics().recordHTTPReqResTimeCost(System.currentTimeMillis() - requestTime); HandlerService.this.sendResponse(ctx, this.request, this.response); } @@ -373,7 +373,7 @@ public void sendErrorResponse(EventMeshRetCode retCode, Map resp * @param count */ public void recordSendBatchMsgFailed(int count) { - metrics.getSummaryMetrics().recordSendBatchMsgFailed(1); + metrics.getHttpMetrics().recordSendBatchMsgFailed(1); } } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HeartBeatProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HeartBeatProcessor.java index be85d0a87f..add93edcab 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HeartBeatProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/HeartBeatProcessor.java @@ -25,7 +25,6 @@ import org.apache.eventmesh.common.protocol.http.header.client.HeartbeatRequestHeader; import org.apache.eventmesh.common.protocol.http.header.client.HeartbeatResponseHeader; import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.metrics.api.model.HttpSummaryMetrics; import org.apache.eventmesh.runtime.acl.Acl; import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; @@ -157,14 +156,13 @@ public void processRequest(final ChannelHandlerContext ctx, final AsyncContext handler = httpCommand -> { try { log.debug("{}", httpCommand); eventMeshHTTPServer.sendResponse(ctx, httpCommand.httpResponse()); - summaryMetrics.recordHTTPReqResTimeCost( - System.currentTimeMillis() - request.getReqTime()); + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHTTPReqResTimeCost( + System.currentTimeMillis() - asyncContext.getRequest().getReqTime()); } catch (Exception ex) { // ignore } @@ -178,8 +176,8 @@ public void processRequest(final ChannelHandlerContext ctx, final AsyncContext } // control flow rate limit - HttpSummaryMetrics summaryMetrics = eventMeshHTTPServer.getMetrics().getSummaryMetrics(); + HttpMetrics summaryMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); if (!eventMeshHTTPServer.getMsgRateLimiter() .tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { summaryMetrics.recordHTTPDiscard(); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncEventProcessor.java index 82cb65fc71..b30238a28c 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncEventProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncEventProcessor.java @@ -237,9 +237,8 @@ public void handler(final HandlerService.HandlerSpecific handlerSpecific, final return; } - final SendMessageContext sendMessageContext = new SendMessageContext(bizNo, event, eventMeshProducer, - eventMeshHTTPServer); - eventMeshHTTPServer.getMetrics().getSummaryMetrics().recordSendMsg(); + final SendMessageContext sendMessageContext = new SendMessageContext(bizNo, event, eventMeshProducer, eventMeshHTTPServer); + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsg(); final long startTime = System.currentTimeMillis(); boolean isFiltered = true; diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncMessageProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncMessageProcessor.java index 6c4e60fd82..f4dcc65a97 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncMessageProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncMessageProcessor.java @@ -31,7 +31,6 @@ import org.apache.eventmesh.common.protocol.http.header.message.SendMessageRequestHeader; import org.apache.eventmesh.common.protocol.http.header.message.SendMessageResponseHeader; import org.apache.eventmesh.common.utils.IPUtils; -import org.apache.eventmesh.metrics.api.model.HttpSummaryMetrics; import org.apache.eventmesh.protocol.api.ProtocolAdaptor; import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; import org.apache.eventmesh.runtime.acl.Acl; @@ -42,6 +41,7 @@ import org.apache.eventmesh.runtime.core.protocol.http.async.CompleteHandler; import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; +import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; import org.apache.eventmesh.runtime.util.EventMeshUtil; import org.apache.eventmesh.runtime.util.RemotingHelper; import org.apache.eventmesh.runtime.util.TraceUtils; @@ -165,7 +165,7 @@ public void processRequest(ChannelHandlerContext ctx, AsyncContext } } - final HttpSummaryMetrics summaryMetrics = eventMeshHTTPServer.getMetrics().getSummaryMetrics(); + final HttpMetrics summaryMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); // control flow rate limit if (!eventMeshHTTPServer.getMsgRateLimiter() .tryAcquire(EventMeshConstants.DEFAULT_FASTFAIL_TIMEOUT_IN_MILLISECONDS, TimeUnit.MILLISECONDS)) { diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncRemoteEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncRemoteEventProcessor.java index baa93d4cb3..fb4f21a82d 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncRemoteEventProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncRemoteEventProcessor.java @@ -264,9 +264,8 @@ public void handler(final HandlerService.HandlerSpecific handlerSpecific, final return; } - final SendMessageContext sendMessageContext = new SendMessageContext(bizNo, event, eventMeshProducer, - eventMeshHTTPServer); - eventMeshHTTPServer.getMetrics().getSummaryMetrics().recordSendMsg(); + final SendMessageContext sendMessageContext = new SendMessageContext(bizNo, event, eventMeshProducer, eventMeshHTTPServer); + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordSendMsg(); final long startTime = System.currentTimeMillis(); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendSyncMessageProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendSyncMessageProcessor.java index b687156a43..abce65d12f 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendSyncMessageProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendSyncMessageProcessor.java @@ -30,7 +30,6 @@ import org.apache.eventmesh.common.protocol.http.header.message.SendMessageResponseHeader; import org.apache.eventmesh.common.utils.IPUtils; import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.metrics.api.model.HttpSummaryMetrics; import org.apache.eventmesh.protocol.api.ProtocolAdaptor; import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; import org.apache.eventmesh.runtime.acl.Acl; @@ -41,6 +40,7 @@ import org.apache.eventmesh.runtime.core.protocol.http.async.CompleteHandler; import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; import org.apache.eventmesh.runtime.core.protocol.producer.SendMessageContext; +import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; import org.apache.eventmesh.runtime.util.EventMeshUtil; import org.apache.eventmesh.runtime.util.RemotingHelper; @@ -145,7 +145,7 @@ public void processRequest(final ChannelHandlerContext ctx, final AsyncContext unSubTopicList = unSubscribeRequestBody.getTopics(); - HttpSummaryMetrics summaryMetrics = eventMeshHTTPServer.getMetrics().getSummaryMetrics(); + final HttpMetrics summaryMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); final CompleteHandler handler = httpCommand -> { try { log.debug("{}", httpCommand); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AsyncHTTPPushRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AsyncHTTPPushRequest.java index 2d0a659e42..be95971536 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AsyncHTTPPushRequest.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AsyncHTTPPushRequest.java @@ -218,7 +218,7 @@ public void tryHTTPRequest() { WebhookUtil.setWebhookHeaders(builder, httpEntity.getContentType().getValue(), eventMeshHttpConfiguration.getEventMeshWebhookOrigin(), urlAuthType); - eventMeshHTTPServer.getMetrics().getSummaryMetrics().recordPushMsg(); + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordPushMsg(); this.lastPushTime = System.currentTimeMillis(); @@ -230,7 +230,7 @@ public void tryHTTPRequest() { eventMeshHTTPServer.getHttpClientPool().getClient().execute(builder, response -> { removeWaitingMap(AsyncHTTPPushRequest.this); long cost = System.currentTimeMillis() - lastPushTime; - eventMeshHTTPServer.getMetrics().getSummaryMetrics().recordHTTPPushTimeCost(cost); + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHTTPPushTimeCost(cost); if (processResponseStatus(response.getStatusLine().getStatusCode(), response)) { // this is successful response, process response payload @@ -265,7 +265,7 @@ public void tryHTTPRequest() { default: // do nothing } } else { - eventMeshHTTPServer.getMetrics().getSummaryMetrics().recordHttpPushMsgFailed(); + eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics().recordHttpPushMsgFailed(); MESSAGE_LOGGER.info("message|eventMesh2client|exception|url={}|topic={}|bizSeqNo={}|uniqueId={}|cost={}", currPushUrl, handleMsgContext.getTopic(), handleMsgContext.getBizSeqNo(), handleMsgContext.getUniqueId(), cost); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/EventMeshTcp2Client.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/EventMeshTcp2Client.java index fdde4aeb6c..6ef8e2bc11 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/EventMeshTcp2Client.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/EventMeshTcp2Client.java @@ -24,11 +24,12 @@ import org.apache.eventmesh.common.protocol.tcp.OPStatus; import org.apache.eventmesh.common.protocol.tcp.Package; import org.apache.eventmesh.common.protocol.tcp.RedirectInfo; +import org.apache.eventmesh.common.utils.IPUtils; import org.apache.eventmesh.runtime.boot.TCPThreadPoolGroup; import org.apache.eventmesh.runtime.core.protocol.tcp.client.group.ClientSessionGroupMapping; import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.SessionState; -import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMonitor; +import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMetricsManager; import org.apache.eventmesh.runtime.util.RemotingHelper; import org.apache.eventmesh.runtime.util.Utils; @@ -89,10 +90,10 @@ public static InetSocketAddress goodBye2Client(TCPThreadPoolGroup tcpThreadPoolG } public static void goodBye2Client(ChannelHandlerContext ctx, String errMsg, ClientSessionGroupMapping mapping, - EventMeshTcpMonitor eventMeshTcpMonitor) { + EventMeshTcpMetricsManager eventMeshTcpMetricsManager) { long startTime = System.currentTimeMillis(); Package pkg = new Package(new Header(SERVER_GOODBYE_REQUEST, OPStatus.FAIL.getCode(), errMsg, null)); - eventMeshTcpMonitor.getTcpSummaryMetrics().getEventMesh2clientMsgNum().incrementAndGet(); + eventMeshTcpMetricsManager.eventMesh2clientMsgNumIncrement(IPUtils.parseChannelRemoteAddr(ctx.channel())); log.info("goodBye2Client client[{}]", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); ctx.writeAndFlush(pkg).addListener((ChannelFutureListener) future -> { Utils.logSucceedMessageFlow(pkg, null, startTime, startTime); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/ClientGroupWrapper.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/ClientGroupWrapper.java index 8b4ce0a4e1..f2ab7fe686 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/ClientGroupWrapper.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/group/ClientGroupWrapper.java @@ -38,7 +38,7 @@ import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.push.DownStreamMsgContext; import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.retry.TcpRetryer; import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.send.UpStreamMsgContext; -import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMonitor; +import org.apache.eventmesh.runtime.metrics.tcp.EventMeshTcpMetricsManager; import org.apache.eventmesh.runtime.util.EventMeshUtil; import org.apache.eventmesh.runtime.util.HttpTinyClient; import org.apache.eventmesh.runtime.util.TraceUtils; @@ -84,7 +84,7 @@ public class ClientGroupWrapper { private TcpRetryer tcpRetryer; - private EventMeshTcpMonitor eventMeshTcpMonitor; + private EventMeshTcpMetricsManager eventMeshTcpMetricsManager; private DownstreamDispatchStrategy downstreamDispatchStrategy; @@ -130,8 +130,8 @@ public ClientGroupWrapper(String sysId, String group, this.eventMeshTCPServer = eventMeshTCPServer; this.eventMeshTCPConfiguration = eventMeshTCPServer.getEventMeshTCPConfiguration(); this.tcpRetryer = eventMeshTCPServer.getTcpRetryer(); - this.eventMeshTcpMonitor = - Preconditions.checkNotNull(eventMeshTCPServer.getEventMeshTcpMonitor()); + this.eventMeshTcpMetricsManager = + Preconditions.checkNotNull(eventMeshTCPServer.getEventMeshTcpMetricsManager()); this.downstreamDispatchStrategy = downstreamDispatchStrategy; this.persistentMsgConsumer = new MQConsumerWrapper(eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshStoragePluginType()); this.broadCastMsgConsumer = new MQConsumerWrapper(eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshStoragePluginType()); @@ -437,8 +437,7 @@ public synchronized void initClientGroupPersistentConsumer() throws Exception { EventMeshTraceConstants.TRACE_DOWNSTREAM_EVENTMESH_SERVER_SPAN, false); try { - eventMeshTcpMonitor.getTcpSummaryMetrics().getMq2eventMeshMsgNum() - .incrementAndGet(); + eventMeshTcpMetricsManager.mq2eventMeshMsgNumIncrement(); event = CloudEventBuilder.from(event) .withExtension(EventMeshConstants.REQ_MQ2EVENTMESH_TIMESTAMP, String.valueOf(System.currentTimeMillis())) @@ -545,8 +544,7 @@ public synchronized void initClientGroupBroadcastConsumer() throws Exception { EventMeshUtil.getCloudEventExtensionMap(protocolVersion, event), EventMeshTraceConstants.TRACE_DOWNSTREAM_EVENTMESH_SERVER_SPAN, false); try { - eventMeshTcpMonitor.getTcpSummaryMetrics().getMq2eventMeshMsgNum() - .incrementAndGet(); + eventMeshTcpMetricsManager.mq2eventMeshMsgNumIncrement(); event = CloudEventBuilder.from(event) .withExtension(EventMeshConstants.REQ_MQ2EVENTMESH_TIMESTAMP, String.valueOf(System.currentTimeMillis())) @@ -663,12 +661,12 @@ public void setTcpRetryer(TcpRetryer tcpRetryer) { this.tcpRetryer = tcpRetryer; } - public EventMeshTcpMonitor getEventMeshTcpMonitor() { - return eventMeshTcpMonitor; + public EventMeshTcpMetricsManager getEventMeshTcpMetricsManager() { + return eventMeshTcpMetricsManager; } - public void setEventMeshTcpMonitor(EventMeshTcpMonitor eventMeshTcpMonitor) { - this.eventMeshTcpMonitor = eventMeshTcpMonitor; + public void setEventMeshTcpMetricsManager(EventMeshTcpMetricsManager eventMeshTcpMetricsManager) { + this.eventMeshTcpMetricsManager = eventMeshTcpMetricsManager; } public DownstreamDispatchStrategy getDownstreamDispatchStrategy() { @@ -745,7 +743,7 @@ public void onException(OnExceptionContext context) { } }); - eventMeshTcpMonitor.getTcpSummaryMetrics().getEventMesh2mqMsgNum().incrementAndGet(); + eventMeshTcpMetricsManager.eventMesh2mqMsgNumIncrement(); } catch (Exception e) { log.warn("try send msg back to broker failed"); throw e; diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/Session.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/Session.java index e2e36690a6..1da3e9922e 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/Session.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/Session.java @@ -25,6 +25,7 @@ import org.apache.eventmesh.common.protocol.tcp.OPStatus; import org.apache.eventmesh.common.protocol.tcp.Package; import org.apache.eventmesh.common.protocol.tcp.UserAgent; +import org.apache.eventmesh.common.utils.IPUtils; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; import org.apache.eventmesh.runtime.constants.EventMeshConstants; @@ -188,10 +189,7 @@ public void operationComplete(ChannelFuture future) throws Exception { MESSAGE_LOGGER.error("write2Client fail, pkg[{}] session[{}]", pkg, this); } else { Objects.requireNonNull(clientGroupWrapper.get()) - .getEventMeshTcpMonitor() - .getTcpSummaryMetrics() - .getEventMesh2clientMsgNum() - .incrementAndGet(); + .getEventMeshTcpMetricsManager().eventMesh2clientMsgNumIncrement(IPUtils.parseChannelRemoteAddr(future.channel())); } } }); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/SessionPusher.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/SessionPusher.java index 0c000b6f43..22ebc57687 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/SessionPusher.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/push/SessionPusher.java @@ -25,6 +25,7 @@ import org.apache.eventmesh.common.protocol.tcp.Header; import org.apache.eventmesh.common.protocol.tcp.OPStatus; import org.apache.eventmesh.common.protocol.tcp.Package; +import org.apache.eventmesh.common.utils.IPUtils; import org.apache.eventmesh.protocol.api.ProtocolAdaptor; import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; import org.apache.eventmesh.runtime.constants.EventMeshConstants; @@ -109,11 +110,8 @@ public void push(final DownStreamMsgContext downStreamMsgContext) { } catch (Exception e) { pkg.setHeader(new Header(cmd, OPStatus.FAIL.getCode(), Arrays.toString(e.getStackTrace()), downStreamMsgContext.seq)); } finally { - Objects.requireNonNull(session.getClientGroupWrapper().get()) - .getEventMeshTcpMonitor() - .getTcpSummaryMetrics() - .getEventMesh2clientMsgNum() - .incrementAndGet(); + Objects.requireNonNull(session.getClientGroupWrapper().get()).getEventMeshTcpMetricsManager() + .eventMesh2clientMsgNumIncrement(IPUtils.parseChannelRemoteAddr(downStreamMsgContext.getSession().getContext().channel())); // TODO uploadTrace String protocolVersion = Objects.requireNonNull(downStreamMsgContext.event.getSpecVersion()).toString(); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/SessionSender.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/SessionSender.java index d89d0e1710..1b455514dc 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/SessionSender.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/tcp/client/session/send/SessionSender.java @@ -141,12 +141,7 @@ public EventMeshTcpSendResult send(Header header, CloudEvent event, SendCallback TraceUtils.finishSpan(span, event); } } - - Objects.requireNonNull(session.getClientGroupWrapper().get()) - .getEventMeshTcpMonitor() - .getTcpSummaryMetrics() - .getEventMesh2mqMsgNum() - .incrementAndGet(); + Objects.requireNonNull(session.getClientGroupWrapper().get()).getEventMeshTcpMetricsManager().eventMesh2mqMsgNumIncrement(); } else { log.warn("send too fast,session flow control,session:{}", session.getClient()); return new EventMeshTcpSendResult(header.getSeq(), EventMeshTcpSendStatus.SEND_TOO_FAST, @@ -178,9 +173,7 @@ public void onSuccess(CloudEvent event) { .withExtension(EventMeshConstants.RSP_RECEIVE_EVENTMESH_IP, session.getEventMeshTCPConfiguration().getEventMeshServerIp()) .build(); - Objects.requireNonNull(session.getClientGroupWrapper().get()) - .getEventMeshTcpMonitor().getTcpSummaryMetrics().getMq2eventMeshMsgNum() - .incrementAndGet(); + Objects.requireNonNull(session.getClientGroupWrapper().get()).getEventMeshTcpMetricsManager().mq2eventMeshMsgNumIncrement(); Command cmd; if (Command.REQUEST_TO_SERVER == header.getCmd()) { diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/EventMeshMetricsManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/EventMeshMetricsManager.java new file mode 100644 index 0000000000..a796ad81a5 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/EventMeshMetricsManager.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics; + +import org.apache.eventmesh.metrics.api.MetricsRegistry; +import org.apache.eventmesh.metrics.api.model.Metric; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * EventMeshMetricsManager class manages the metrics for EventMesh. + */ +public class EventMeshMetricsManager { + + private List metricsRegistries = new ArrayList<>(64); + private List metrics = new ArrayList<>(64); + private List metricsManagers = new ArrayList<>(32); + + /** + * Constructs an EventMeshMetricsManager with the provided metrics registries. + * + * @param metricsRegistries The list of metrics registries. + */ + public EventMeshMetricsManager(final List metricsRegistries) { + if (Objects.nonNull(metricsRegistries)) { + this.metricsRegistries.addAll(metricsRegistries); + } + } + + /** + * Constructs an EventMeshMetricsManager. + */ + public EventMeshMetricsManager() { + } + + /** + * Retrieves the list of metrics registries. + * + * @return The list of metrics registries. + */ + public List getMetricsRegistries() { + return metricsRegistries; + } + + /** + * Adds the provided metrics registries to the existing list. + * + * @param metricsRegistries The list of metrics registries to add. + */ + public void addMetricsRegistries(List metricsRegistries) { + if (Objects.nonNull(metricsRegistries)) { + this.metricsRegistries.addAll(metricsRegistries); + } + } + + /** + * Adds the provided metrics to the existing list. + * + * @param metrics The list of metrics to add. + */ + public void addMetrics(List metrics) { + if (Objects.nonNull(metrics)) { + this.metrics.addAll(metrics); + } + } + + /** + * Adds a metric manager to the list of metrics managers. + * + * @param metricsManager The metric manager to add. + */ + public void addMetricManager(final MetricsManager metricsManager) { + this.metricsManagers.add(metricsManager); + } + + /** + * Adds a metric to the existing list of metrics. + * + * @param metric The metric to add. + */ + public void addMetric(Metric metric) { + if (Objects.nonNull(metric)) { + this.metrics.add(metric); + } + } + + /** + * Initializes the EventMeshMetricsManager by registering the metrics with the metrics registries. + */ + public void init() { + MetricsUtils.registerMetrics(metricsRegistries); + // Register metrics + metricsRegistries.stream().forEach(metricsRegistry -> metricsRegistry.register(metrics)); + } + + /** + * Starts the metrics managers and metrics registries. + */ + public void start() { + metricsManagers.stream().forEach(MetricsManager::start); + metricsRegistries.stream().forEach(MetricsRegistry::start); + } + + /** + * Shuts down the metrics registries and metrics managers. + */ + public void shutdown() { + metricsRegistries.stream().forEach(MetricsRegistry::showdown); + metricsManagers.stream().forEach(MetricsManager::shutdown); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/GeneralMetrics.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/GeneralMetrics.java new file mode 100644 index 0000000000..0043a32e25 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/GeneralMetrics.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics; + +import org.apache.eventmesh.metrics.api.model.InstrumentFurther; +import org.apache.eventmesh.metrics.api.model.LongCounterMetric; +import org.apache.eventmesh.metrics.api.model.Metric; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +public class GeneralMetrics { + + private static final String METRIC_NAME = "general"; + + private static final String GENERAL_METRICS_NAME_PREFIX = "eventmesh.general."; + + protected static final Map metrics = new HashMap<>(32); + + //message number of client(TCP,HTTP,GRPC) send to EventMesh + protected static final LongCounterMetric client2eventMeshMsgNum; + + //message number of EventMesh send to MQ + protected static final LongCounterMetric eventMesh2mqMsgNum; + + //message number of MQ send to EventMesh + protected static final LongCounterMetric mq2eventMeshMsgNum; + + //message number of EventMesh send to client(TCP,HTTP,GRPC) + protected static final LongCounterMetric eventMesh2clientMsgNum; + + static { + InstrumentFurther furtherCl2Em = new InstrumentFurther(); + furtherCl2Em.setUnit(MetricInstrumentUnit.SINGLETON); + furtherCl2Em.setDescription("message number of client send to EventMesh"); + furtherCl2Em.setName(GENERAL_METRICS_NAME_PREFIX + "client.eventmesh.message.num"); + client2eventMeshMsgNum = new LongCounterMetric(furtherCl2Em, METRIC_NAME); + metrics.put("client2eventMeshMsgNum", client2eventMeshMsgNum); + + InstrumentFurther furtherEm2Mq = new InstrumentFurther(); + furtherEm2Mq.setUnit(MetricInstrumentUnit.SINGLETON); + furtherEm2Mq.setDescription("message number of EventMesh send to MQ"); + furtherEm2Mq.setName(GENERAL_METRICS_NAME_PREFIX + "eventmesh.mq.message.num"); + eventMesh2mqMsgNum = new LongCounterMetric(furtherEm2Mq, METRIC_NAME); + metrics.put("eventMesh2mqMsgNum", eventMesh2mqMsgNum); + + InstrumentFurther furtherMq2Em = new InstrumentFurther(); + furtherMq2Em.setUnit(MetricInstrumentUnit.SINGLETON); + furtherMq2Em.setDescription("message number of MQ send to EventMesh"); + furtherMq2Em.setName(GENERAL_METRICS_NAME_PREFIX + "mq.eventmesh.message.num"); + mq2eventMeshMsgNum = new LongCounterMetric(furtherMq2Em, METRIC_NAME); + metrics.put("mq2eventMeshMsgNum", mq2eventMeshMsgNum); + + InstrumentFurther furtherEm2Cl = new InstrumentFurther(); + furtherEm2Cl.setUnit(MetricInstrumentUnit.SINGLETON); + furtherEm2Cl.setDescription("message number of EventMesh send to client"); + furtherEm2Cl.setName(GENERAL_METRICS_NAME_PREFIX + "eventmesh.client.message.num"); + eventMesh2clientMsgNum = new LongCounterMetric(furtherEm2Cl, METRIC_NAME); + metrics.put("eventMesh2clientMsgNum", eventMesh2clientMsgNum); + } + + public static Collection getMetrics() { + return metrics.values(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricInstrumentUnit.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricInstrumentUnit.java new file mode 100644 index 0000000000..2186cbb447 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricInstrumentUnit.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics; + + +public class MetricInstrumentUnit { + + private MetricInstrumentUnit() { + + } + + public static final String SINGLETON = "1"; + + public static final String TPS = "tps"; + + public static final String MILLISECONDS = "ms"; + + public static final String PERCENT = "%"; + +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsManager.java new file mode 100644 index 0000000000..a1e333e482 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsManager.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics; + +import org.apache.eventmesh.metrics.api.model.Metric; + +import java.util.List; + +/** + * MetricsManager is an interface for managing metrics. + */ +public interface MetricsManager { + + /** + * Retrieves the list of metrics. + * + * @return The list of metrics. + */ + List getMetrics(); + + /** + * Retrieves the name of the MetricsManager. + * + * @return The name of the MetricsManager. + */ + String getName(); + + /** + * Starts the MetricsManager. + */ + void start(); + + /** + * Shuts down the MetricsManager. + */ + void shutdown(); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsUtils.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsUtils.java new file mode 100644 index 0000000000..fb0e5d378d --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MetricsUtils.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics; + + +import org.apache.eventmesh.metrics.api.MetricsRegistry; + +import org.apache.commons.collections4.MapUtils; + +import java.util.List; +import java.util.Map; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; + +import lombok.experimental.UtilityClass; + +/** + * Managing general metrics. + */ +@UtilityClass +public class MetricsUtils { + + private static Attributes EMPTY_ATTRIBUTES = Attributes.builder().build(); + + /** + * Registers the general metrics with the provided metrics registries. + * + * @param metricsRegistries The list of metrics registries. + */ + public static void registerMetrics(final List metricsRegistries) { + metricsRegistries.forEach(metricsRegistry -> metricsRegistry.register(GeneralMetrics.getMetrics())); + } + + /** + * Increments the client-to-EventMesh message count metric by 1, with the given attributes. + * + * @param attributes The attributes for the metric. + */ + public static void incrementClientToEventMeshMsgNum(final Map attributes) { + GeneralMetrics.client2eventMeshMsgNum.getInstrument().add(1, buildAttributes(attributes)); + } + + /** + * Increments the client-to-EventMesh message count metric by a specific count, with the given attributes. + * + * @param attributes The attributes for the metric. + * @param count The count to increment the metric by. + */ + public static void incrementClientToEventMeshMsgNum(final Map attributes, final int count) { + GeneralMetrics.client2eventMeshMsgNum.getInstrument().add(count, buildAttributes(attributes)); + } + + /** + * Increments the EventMesh-to-MQ message count metric by 1, with the given attributes. + * + * @param attributes The attributes for the metric. + */ + public static void incrementEventMeshToMQMsgNum(final Map attributes) { + GeneralMetrics.eventMesh2mqMsgNum.getInstrument().add(1, buildAttributes(attributes)); + } + + /** + * Increments the MQ-to-EventMesh message count metric by 1, with the given attributes. + * + * @param attributes The attributes for the metric. + */ + public static void incrementMQToEventMeshMsgNum(final Map attributes) { + GeneralMetrics.mq2eventMeshMsgNum.getInstrument().add(1, buildAttributes(attributes)); + } + + /** + * Increments the EventMesh-to-client message count metric by 1, with the given attributes. + * + * @param attributes The attributes for the metric. + */ + public static void incrementEventMeshToClientMsgNum(final Map attributes) { + GeneralMetrics.eventMesh2clientMsgNum.getInstrument().add(1, buildAttributes(attributes)); + } + + /** + * Builds the attributes from the provided map. + * + * @param attributes The map of attributes. + * @return The built attributes. + */ + public static Attributes buildAttributes(final Map attributes) { + if (MapUtils.isEmpty(attributes)) { + return EMPTY_ATTRIBUTES; + } + AttributesBuilder attributesBuilder = Attributes.builder(); + attributes.forEach(attributesBuilder::put); + return attributesBuilder.build(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MonitorMetricConstants.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MonitorMetricConstants.java index 0655421b79..3f2a3e8b20 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MonitorMetricConstants.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/MonitorMetricConstants.java @@ -33,8 +33,10 @@ public class MonitorMetricConstants { public static final String RETRY_QUEUE_SIZE = "retryQueueSize"; + public static final String QUEUE_SIZE = "queueSize"; public static final String POOL_SIZE = "poolSize"; public static final String ACTIVE_COUNT = "activeCount"; public static final String COMPLETED_TASK = "completedTask"; + } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/EventMeshGrpcMetricsManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/EventMeshGrpcMetricsManager.java new file mode 100644 index 0000000000..2710474818 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/EventMeshGrpcMetricsManager.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics.grpc; + +import org.apache.eventmesh.common.EventMeshThreadFactory; +import org.apache.eventmesh.common.MetricsConstants; +import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.common.enums.ProtocolType; +import org.apache.eventmesh.common.utils.IPUtils; +import org.apache.eventmesh.metrics.api.MetricsRegistry; +import org.apache.eventmesh.metrics.api.model.Metric; +import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; +import org.apache.eventmesh.runtime.metrics.MetricsManager; +import org.apache.eventmesh.runtime.metrics.MetricsUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Preconditions; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class EventMeshGrpcMetricsManager implements MetricsManager { + + private static final long DELAY_MILLS = 60 * 1000; + private static final long SCHEDULE_PERIOD_MILLS = 60 * 1000; + private static final int SCHEDULE_THREAD_SIZE = 1; + private static final String THREAD_NAME_PREFIX = "eventMesh-grpc-monitor-scheduler"; + + private final Map labelMap = new HashMap<>(); + + private final EventMeshGrpcServer eventMeshGrpcServer; + private final List metricsRegistries; + + private final ScheduledExecutorService scheduler; + private ScheduledFuture scheduleTask; + + private final GrpcMetrics grpcMetrics; + + public EventMeshGrpcMetricsManager(EventMeshGrpcServer eventMeshGrpcServer, List metricsRegistries) { + this.eventMeshGrpcServer = Preconditions.checkNotNull(eventMeshGrpcServer); + this.metricsRegistries = Preconditions.checkNotNull(metricsRegistries); + this.grpcMetrics = new GrpcMetrics(eventMeshGrpcServer, labelMap); + this.scheduler = ThreadPoolFactory.createScheduledExecutor(SCHEDULE_THREAD_SIZE, new EventMeshThreadFactory(THREAD_NAME_PREFIX, true)); + init(); + } + + private void init() { + String eventMeshServerIp = this.eventMeshGrpcServer.getEventMeshGrpcConfiguration().getEventMeshServerIp(); + int eventMeshTcpServerPort = this.eventMeshGrpcServer.getEventMeshGrpcConfiguration().getGrpcServerPort(); + labelMap.put(MetricsConstants.RPC_SYSTEM, "grpc"); + labelMap.put(MetricsConstants.RPC_SERVICE, this.eventMeshGrpcServer.getClass().getName()); + labelMap.put(MetricsConstants.GRPC_NET_PEER_NAME, Optional.ofNullable(eventMeshServerIp).orElse(IPUtils.getLocalAddress())); + labelMap.put(MetricsConstants.GRPC_NET_PEER_PORT, Integer.toString(eventMeshTcpServerPort)); + } + + @Override + public void start() { + // update tps metrics and clear counter + scheduleTask = scheduler.scheduleAtFixedRate(() -> { + grpcMetrics.refreshTpsMetrics(SCHEDULE_PERIOD_MILLS); + grpcMetrics.clearAllMessageCounter(); + grpcMetrics.setRetrySize(0); + grpcMetrics.setSubscribeTopicNum(eventMeshGrpcServer.getConsumerManager().getAllConsumerTopic().size()); + }, DELAY_MILLS, SCHEDULE_PERIOD_MILLS, TimeUnit.MILLISECONDS); + } + + public void recordReceiveMsgFromClient(final String clientAddress) { + grpcMetrics.getClient2EventMeshMsgNum().incrementAndGet(); + Map attributes = new HashMap<>(labelMap); + attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.GRPC.name()); + attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); + MetricsUtils.incrementClientToEventMeshMsgNum(attributes); + } + + public void recordReceiveMsgFromClient(final int count, String clientAddress) { + grpcMetrics.getClient2EventMeshMsgNum().addAndGet(count); + Map attributes = new HashMap<>(labelMap); + attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.GRPC.name()); + attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); + MetricsUtils.incrementClientToEventMeshMsgNum(attributes, count); + } + + public void recordSendMsgToQueue() { + grpcMetrics.getEventMesh2MqMsgNum().incrementAndGet(); + Map attributes = new HashMap<>(labelMap); + MetricsUtils.incrementEventMeshToMQMsgNum(attributes); + } + + public void recordReceiveMsgFromQueue() { + grpcMetrics.getMq2EventMeshMsgNum().incrementAndGet(); + Map attributes = new HashMap<>(labelMap); + MetricsUtils.incrementMQToEventMeshMsgNum(attributes); + } + + public void recordSendMsgToClient(final String clientAddress) { + grpcMetrics.getEventMesh2ClientMsgNum().incrementAndGet(); + Map attributes = new HashMap<>(labelMap); + attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.TCP.name()); + attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); + MetricsUtils.incrementEventMeshToClientMsgNum(attributes); + } + + public void recordGrpcPublishHandleCost(long costTime, String clientAddress) { + Map attributes = new HashMap<>(labelMap); + attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.TCP.name()); + attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); + grpcMetrics.recordGrpcPublishHandleCost(costTime, MetricsUtils.buildAttributes(attributes)); + } + + @Override + public void shutdown() { + scheduleTask.cancel(true); + scheduler.shutdown(); + } + + @Override + public List getMetrics() { + return new ArrayList<>(grpcMetrics.getMetrics()); + } + + @Override + public String getName() { + return this.getClass().getName(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/EventMeshGrpcMonitor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/EventMeshGrpcMonitor.java deleted file mode 100644 index fbd3be6163..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/EventMeshGrpcMonitor.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics.grpc; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.common.ThreadPoolFactory; -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.metrics.api.model.GrpcSummaryMetrics; -import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; - -import java.util.List; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import com.google.common.base.Preconditions; - -public class EventMeshGrpcMonitor { - - private static final int DELAY_MILLS = 60 * 1000; - private static final int SCHEDULE_PERIOD_MILLS = 60 * 1000; - private static final int SCHEDULE_THREAD_SIZE = 1; - private static final String THREAD_NAME_PREFIX = "eventMesh-grpc-monitor-scheduler"; - - private final EventMeshGrpcServer eventMeshGrpcServer; - private final List metricsRegistries; - - private final ScheduledExecutorService scheduler; - private ScheduledFuture scheduleTask; - private final GrpcSummaryMetrics grpcSummaryMetrics; - - public EventMeshGrpcMonitor(EventMeshGrpcServer eventMeshGrpcServer, List metricsRegistries) { - this.eventMeshGrpcServer = Preconditions.checkNotNull(eventMeshGrpcServer); - this.metricsRegistries = Preconditions.checkNotNull(metricsRegistries); - this.grpcSummaryMetrics = new GrpcSummaryMetrics(); - this.scheduler = ThreadPoolFactory.createScheduledExecutor(SCHEDULE_THREAD_SIZE, - new EventMeshThreadFactory(THREAD_NAME_PREFIX, true)); - } - - public void init() throws Exception { - metricsRegistries.forEach(MetricsRegistry::start); - } - - public void start() throws Exception { - metricsRegistries.forEach(metricsRegistry -> metricsRegistry.register(grpcSummaryMetrics)); - - // update tps metrics and clear counter - scheduleTask = scheduler.scheduleAtFixedRate(() -> { - grpcSummaryMetrics.refreshTpsMetrics(SCHEDULE_PERIOD_MILLS); - grpcSummaryMetrics.clearAllMessageCounter(); - grpcSummaryMetrics.setRetrySize(eventMeshGrpcServer.getGrpcRetryer().getPendingTimeouts()); - grpcSummaryMetrics.setSubscribeTopicNum(eventMeshGrpcServer.getConsumerManager().getAllConsumerTopic().size()); - }, DELAY_MILLS, SCHEDULE_PERIOD_MILLS, TimeUnit.MILLISECONDS); - } - - public void recordReceiveMsgFromClient() { - grpcSummaryMetrics.getClient2EventMeshMsgNum().incrementAndGet(); - } - - public void recordReceiveMsgFromClient(int count) { - grpcSummaryMetrics.getClient2EventMeshMsgNum().addAndGet(count); - } - - public void recordSendMsgToQueue() { - grpcSummaryMetrics.getEventMesh2MqMsgNum().incrementAndGet(); - } - - public void recordReceiveMsgFromQueue() { - grpcSummaryMetrics.getMq2EventMeshMsgNum().incrementAndGet(); - } - - public void recordSendMsgToClient() { - grpcSummaryMetrics.getEventMesh2ClientMsgNum().incrementAndGet(); - } - - public void shutdown() throws Exception { - scheduleTask.cancel(true); - metricsRegistries.forEach(MetricsRegistry::showdown); - scheduler.shutdown(); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/GrpcMetrics.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/GrpcMetrics.java new file mode 100644 index 0000000000..8bb8fa9edc --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/grpc/GrpcMetrics.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics.grpc; + +import org.apache.eventmesh.common.Pair; +import org.apache.eventmesh.metrics.api.model.DoubleHistogramMetric; +import org.apache.eventmesh.metrics.api.model.InstrumentFurther; +import org.apache.eventmesh.metrics.api.model.Metric; +import org.apache.eventmesh.metrics.api.model.NoopDoubleHistogram; +import org.apache.eventmesh.metrics.api.model.ObservableDoubleGaugeMetric; +import org.apache.eventmesh.metrics.api.model.ObservableLongGaugeMetric; +import org.apache.eventmesh.runtime.boot.EventMeshGrpcServer; +import org.apache.eventmesh.runtime.metrics.MetricInstrumentUnit; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.sdk.metrics.Aggregation; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.View; + +public class GrpcMetrics { + + private static final String GRPC_METRICS_NAME_PREFIX = "eventmesh.grpc."; + + private static final String METRIC_NAME = "GRPC"; + + private static double ms = 1; + + private static double s = 1000 * ms; + + private final EventMeshGrpcServer eventMeshGrpcServer; + + private final AtomicLong client2EventMeshMsgNum; + private final AtomicLong eventMesh2MqMsgNum; + private final AtomicLong mq2EventMeshMsgNum; + private final AtomicLong eventMesh2ClientMsgNum; + + private volatile double client2EventMeshTPS; + private volatile double eventMesh2ClientTPS; + private volatile double eventMesh2MqTPS; + private volatile double mq2EventMeshTPS; + + private volatile long retrySize; + private volatile long subscribeTopicNum; + + private final DoubleHistogram grpcPublishHandleCost = new NoopDoubleHistogram(); + + private ObservableDoubleGaugeMetric mq2eventMeshTPSGauge; + + private ObservableDoubleGaugeMetric client2eventMeshTPSGauge; + + private ObservableDoubleGaugeMetric eventMesh2clientTPSGauge; + + private ObservableDoubleGaugeMetric eventMesh2mqTPSGauge; + + private ObservableLongGaugeMetric subTopicGauge; + + private DoubleHistogramMetric grpcPublishHandleCostHistogram; + + private final Map metrics = new HashMap<>(32); + + private final Map labelMap; + + public GrpcMetrics(final EventMeshGrpcServer eventMeshGrpcServer, final Map labelMap) { + this.eventMeshGrpcServer = eventMeshGrpcServer; + this.labelMap = labelMap; + this.client2EventMeshMsgNum = new AtomicLong(0); + this.eventMesh2MqMsgNum = new AtomicLong(0); + this.mq2EventMeshMsgNum = new AtomicLong(0); + this.eventMesh2ClientMsgNum = new AtomicLong(0); + initMetric(); + } + + private void initMetric() { + final Map commonAttributes = new HashMap<>(labelMap); + + InstrumentFurther furtherTopic = new InstrumentFurther(); + furtherTopic.setUnit(MetricInstrumentUnit.SINGLETON); + furtherTopic.setDescription("Number of grpc client subscribe for topic"); + furtherTopic.setName(GRPC_METRICS_NAME_PREFIX + "sub.topic.num"); + subTopicGauge = new ObservableLongGaugeMetric(furtherTopic, METRIC_NAME, buildSubTopicSupplier()); + subTopicGauge.putAll(commonAttributes); + metrics.put("subTopicGauge", subTopicGauge); + + InstrumentFurther furtherCl2Em = new InstrumentFurther(); + furtherCl2Em.setUnit(MetricInstrumentUnit.TPS); + furtherCl2Em.setDescription("Tps of client to EventMesh."); + furtherCl2Em.setName(GRPC_METRICS_NAME_PREFIX + "client.eventmesh.tps"); + client2eventMeshTPSGauge = new ObservableDoubleGaugeMetric(furtherCl2Em, METRIC_NAME, () -> GrpcMetrics.this.client2EventMeshTPS); + client2eventMeshTPSGauge.putAll(commonAttributes); + metrics.put("client2eventMeshTPSGauge", client2eventMeshTPSGauge); + + InstrumentFurther furtherEm2Cl = new InstrumentFurther(); + furtherEm2Cl.setUnit(MetricInstrumentUnit.TPS); + furtherEm2Cl.setDescription("Tps of EventMesh to client."); + furtherEm2Cl.setName(GRPC_METRICS_NAME_PREFIX + "eventmesh.client.tps"); + eventMesh2clientTPSGauge = new ObservableDoubleGaugeMetric(furtherEm2Cl, METRIC_NAME, () -> GrpcMetrics.this.eventMesh2ClientTPS); + eventMesh2clientTPSGauge.putAll(commonAttributes); + metrics.put("eventMesh2clientTPSGauge", eventMesh2clientTPSGauge); + + InstrumentFurther furtherEm2Mq = new InstrumentFurther(); + furtherEm2Mq.setUnit(MetricInstrumentUnit.TPS); + furtherEm2Mq.setDescription("Tps of EventMesh to MQ."); + furtherEm2Mq.setName(GRPC_METRICS_NAME_PREFIX + "eventmesh.mq.tps"); + eventMesh2mqTPSGauge = new ObservableDoubleGaugeMetric(furtherEm2Mq, METRIC_NAME, () -> GrpcMetrics.this.eventMesh2MqTPS); + eventMesh2mqTPSGauge.putAll(commonAttributes); + metrics.put("eventMesh2mqTPSGauge", eventMesh2mqTPSGauge); + + InstrumentFurther furtherMq2Em = new InstrumentFurther(); + furtherMq2Em.setUnit(MetricInstrumentUnit.TPS); + furtherMq2Em.setDescription("Tps of MQ to EventMesh."); + furtherMq2Em.setName(GRPC_METRICS_NAME_PREFIX + "mq.eventmesh.tps"); + mq2eventMeshTPSGauge = new ObservableDoubleGaugeMetric(furtherMq2Em, METRIC_NAME, () -> GrpcMetrics.this.mq2EventMeshTPS); + mq2eventMeshTPSGauge.putAll(commonAttributes); + metrics.put("mq2eventMeshTPSGauge", mq2eventMeshTPSGauge); + + InstrumentFurther furtherGrpcPublishHandleCost = new InstrumentFurther(); + furtherGrpcPublishHandleCost.setUnit(MetricInstrumentUnit.MILLISECONDS); + furtherGrpcPublishHandleCost.setDescription("Grpc publish handle cost time"); + String grpcPublishHandleCostName = GRPC_METRICS_NAME_PREFIX + "publish.handle.cost"; + furtherGrpcPublishHandleCost.setName(grpcPublishHandleCostName); + Pair pair = buildGrpcPublishHandleCostMetricsView(grpcPublishHandleCostName); + furtherGrpcPublishHandleCost.putExt(InstrumentFurther.INSTRUMENT_VIEW, pair); + grpcPublishHandleCostHistogram = new DoubleHistogramMetric(furtherGrpcPublishHandleCost, METRIC_NAME); + metrics.put("grpcPublishHandleCost", grpcPublishHandleCostHistogram); + } + + private Pair buildGrpcPublishHandleCostMetricsView(String metricName) { + List latencyBuckets = Arrays.asList( + 1 * ms, 3 * ms, 5 * ms, + 10 * ms, 30 * ms, 50 * ms, + 100 * ms, 300 * ms, 500 * ms, + 1 * s, 3 * s, 5 * s, 10 * s); + View view = View.builder() + .setAggregation(Aggregation.explicitBucketHistogram(latencyBuckets)) + .build(); + InstrumentSelector selector = InstrumentSelector.builder() + .setType(InstrumentType.HISTOGRAM) + .setName(metricName) + .build(); + + return new Pair<>(selector, view); + } + + public void recordGrpcPublishHandleCost(long time, Attributes attributes) { + grpcPublishHandleCostHistogram.getInstrument().record(time, attributes); + } + + /** + * Count the number of GRPC clients subscribed to a topic. + * + * @return Supplier + */ + private Supplier buildSubTopicSupplier() { + return () -> (long) this.eventMeshGrpcServer.getConsumerManager().getAllConsumerTopic().size(); + } + + public void clearAllMessageCounter() { + client2EventMeshMsgNum.set(0L); + eventMesh2MqMsgNum.set(0L); + mq2EventMeshMsgNum.set(0L); + eventMesh2ClientMsgNum.set(0L); + } + + public void refreshTpsMetrics(long intervalMills) { + BigDecimal intervalMillisBD = BigDecimal.valueOf(intervalMills); + // Calculate TPS for client2EventMesh messages + client2EventMeshTPS = BigDecimal.valueOf(1000).multiply(BigDecimal.valueOf(client2EventMeshMsgNum.get())) + .divide(intervalMillisBD, 2, RoundingMode.HALF_UP).doubleValue(); + + // Calculate TPS for eventMesh2Client messages + eventMesh2ClientTPS = BigDecimal.valueOf(1000).multiply(BigDecimal.valueOf(eventMesh2ClientMsgNum.get())) + .divide(intervalMillisBD, 2, RoundingMode.HALF_UP).doubleValue(); + + // Calculate TPS for eventMesh2Mq messages + eventMesh2MqTPS = BigDecimal.valueOf(1000).multiply(BigDecimal.valueOf(eventMesh2MqMsgNum.get())) + .divide(intervalMillisBD, 2, RoundingMode.HALF_UP).doubleValue(); + + // Calculate TPS for mq2EventMesh messages + mq2EventMeshTPS = BigDecimal.valueOf(1000).multiply(BigDecimal.valueOf(mq2EventMeshMsgNum.get())) + .divide(intervalMillisBD, 2, RoundingMode.HALF_UP).doubleValue(); + + } + + public Collection getMetrics() { + return metrics.values(); + } + + public AtomicLong getClient2EventMeshMsgNum() { + return client2EventMeshMsgNum; + } + + public AtomicLong getEventMesh2MqMsgNum() { + return eventMesh2MqMsgNum; + } + + public AtomicLong getMq2EventMeshMsgNum() { + return mq2EventMeshMsgNum; + } + + public AtomicLong getEventMesh2ClientMsgNum() { + return eventMesh2ClientMsgNum; + } + + public double getClient2EventMeshTPS() { + return client2EventMeshTPS; + } + + public double getEventMesh2ClientTPS() { + return eventMesh2ClientTPS; + } + + public double getEventMesh2MqTPS() { + return eventMesh2MqTPS; + } + + public double getMq2EventMeshTPS() { + return mq2EventMeshTPS; + } + + public long getRetrySize() { + return retrySize; + } + + public void setRetrySize(long retrySize) { + this.retrySize = retrySize; + } + + public long getSubscribeTopicNum() { + return subscribeTopicNum; + } + + public void setSubscribeTopicNum(long subscribeTopicNum) { + this.subscribeTopicNum = subscribeTopicNum; + } + +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/EventMeshHttpMetricsManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/EventMeshHttpMetricsManager.java new file mode 100644 index 0000000000..1ee6e8de35 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/EventMeshHttpMetricsManager.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics.http; + +import org.apache.eventmesh.common.EventMeshThreadFactory; +import org.apache.eventmesh.common.MetricsConstants; +import org.apache.eventmesh.common.enums.ProtocolType; +import org.apache.eventmesh.common.utils.IPUtils; +import org.apache.eventmesh.metrics.api.MetricsRegistry; +import org.apache.eventmesh.metrics.api.model.Metric; +import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; +import org.apache.eventmesh.runtime.metrics.MetricsManager; +import org.apache.eventmesh.runtime.metrics.MetricsUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.DelayQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class EventMeshHttpMetricsManager implements MetricsManager { + + private Map labelMap = new HashMap<>(); + + private final EventMeshHTTPServer eventMeshHTTPServer; + + private final List metricsRegistries; + + private final HttpMetrics httpMetrics; + + public EventMeshHttpMetricsManager(final EventMeshHTTPServer eventMeshHTTPServer, + final List metricsRegistries) { + Objects.requireNonNull(eventMeshHTTPServer, "EventMeshHTTPServer can not be null"); + Objects.requireNonNull(metricsRegistries, "List can not be null"); + + this.eventMeshHTTPServer = eventMeshHTTPServer; + this.metricsRegistries = metricsRegistries; + init(); + this.httpMetrics = new HttpMetrics( + eventMeshHTTPServer.getHttpThreadPoolGroup().getBatchMsgExecutor(), + eventMeshHTTPServer.getHttpThreadPoolGroup().getSendMsgExecutor(), + eventMeshHTTPServer.getHttpThreadPoolGroup().getPushMsgExecutor(), + new DelayQueue<>(), + labelMap); + } + + private void init() { + boolean useTls = eventMeshHTTPServer.getEventMeshHttpConfiguration().isEventMeshServerUseTls(); + String eventMeshServerIp = this.eventMeshHTTPServer.getEventMeshHttpConfiguration().getEventMeshServerIp(); + int httpServerPort = this.eventMeshHTTPServer.getEventMeshHttpConfiguration().getHttpServerPort(); + this.labelMap.put(MetricsConstants.HTTP_HTTP_SCHEME, useTls ? "https" : "http"); + this.labelMap.put(MetricsConstants.HTTP_HTTP_FLAVOR, "1.1"); + this.labelMap.put(MetricsConstants.HTTP_NET_HOST_NAME, Optional.ofNullable(eventMeshServerIp).orElse(IPUtils.getLocalAddress())); + this.labelMap.put(MetricsConstants.HTTP_NET_HOST_PORT, Integer.toString(httpServerPort)); + this.labelMap.put(MetricsConstants.RPC_SYSTEM, "HTTP"); + this.labelMap.put(MetricsConstants.RPC_SERVICE, this.eventMeshHTTPServer.getClass().getName()); + if (log.isInfoEnabled()) { + log.info("HTTPMetricsServer initialized."); + } + } + + @Override + public void start() { + + metricsSchedule.scheduleAtFixedRate(() -> { + try { + httpMetrics.snapshotHTTPTPS(); + httpMetrics.snapshotSendBatchMsgTPS(); + httpMetrics.snapshotSendMsgTPS(); + httpMetrics.snapshotPushMsgTPS(); + } catch (Exception ex) { + log.error("eventMesh snapshot tps metrics err", ex); + } + }, 0, 1000, TimeUnit.MILLISECONDS); + + metricsSchedule.scheduleAtFixedRate(() -> { + try { + logPrintServerMetrics(httpMetrics, eventMeshHTTPServer); + } catch (Exception ex) { + log.error("eventMesh print metrics err", ex); + } + }, 1000, 30 * 1000, TimeUnit.MILLISECONDS); + + if (log.isInfoEnabled()) { + log.info("HTTPMetricsServer started."); + } + } + + public void recordReceiveMsgFromClient(final String clientAddress) { + + Map attributes = new HashMap<>(labelMap); + attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.GRPC.name()); + attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); + MetricsUtils.incrementClientToEventMeshMsgNum(attributes); + } + + public void recordReceiveMsgFromClient(final int count, String clientAddress) { + Map attributes = new HashMap<>(labelMap); + attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.GRPC.name()); + attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); + MetricsUtils.incrementClientToEventMeshMsgNum(attributes, count); + } + + public void recordSendMsgToQueue() { + Map attributes = new HashMap<>(labelMap); + MetricsUtils.incrementEventMeshToMQMsgNum(attributes); + } + + public void recordReceiveMsgFromQueue() { + Map attributes = new HashMap<>(labelMap); + MetricsUtils.incrementMQToEventMeshMsgNum(attributes); + } + + public void recordSendMsgToClient(final String clientAddress) { + Map attributes = new HashMap<>(labelMap); + attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.TCP.name()); + attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); + MetricsUtils.incrementEventMeshToClientMsgNum(attributes); + } + + + @Override + public void shutdown() { + metricsSchedule.shutdown(); + } + + private static ScheduledExecutorService metricsSchedule = Executors.newScheduledThreadPool(2, + new EventMeshThreadFactory("eventMesh-metrics", true)); + + // todo: move this into standalone metrics plugin + + private void logPrintServerMetrics(final HttpMetrics summaryMetrics, + final EventMeshHTTPServer eventMeshHTTPServer) { + if (log.isInfoEnabled()) { + log.info("===========================================HTTP SERVER METRICS=================================================="); + + log.info("maxHTTPTPS: {}, avgHTTPTPS: {}, maxHTTPCOST: {}, avgHTTPCOST: {}, avgHTTPBodyDecodeCost: {}, httpDiscard: {}", + summaryMetrics.maxHTTPTPS(), + summaryMetrics.avgHTTPTPS(), + summaryMetrics.maxHTTPCost(), + summaryMetrics.avgHTTPCost(), + summaryMetrics.avgHTTPBodyDecodeCost(), + summaryMetrics.getHttpDiscard()); + } + + summaryMetrics.httpStatInfoClear(); + + if (log.isInfoEnabled()) { + log.info("maxBatchSendMsgTPS: {}, avgBatchSendMsgTPS: {}, sum: {}. sumFail: {}, sumFailRate: {}, discard : {}", + summaryMetrics.maxSendBatchMsgTPS(), + summaryMetrics.avgSendBatchMsgTPS(), + summaryMetrics.getSendBatchMsgNumSum(), + summaryMetrics.getSendBatchMsgFailNumSum(), + summaryMetrics.getSendBatchMsgFailRate(), + summaryMetrics.getSendBatchMsgDiscardNumSum() + ); + } + + summaryMetrics.cleanSendBatchStat(); + + if (log.isInfoEnabled()) { + log.info("maxSendMsgTPS: {}, avgSendMsgTPS: {}, sum: {}, sumFail: {}, sumFailRate: {}, replyMsg: {}, replyFail: {}", + summaryMetrics.maxSendMsgTPS(), + summaryMetrics.avgSendMsgTPS(), + summaryMetrics.getSendMsgNumSum(), + summaryMetrics.getSendMsgFailNumSum(), + summaryMetrics.getSendMsgFailRate(), + summaryMetrics.getReplyMsgNumSum(), + summaryMetrics.getReplyMsgFailNumSum() + ); + } + + summaryMetrics.cleanSendMsgStat(); + + if (log.isInfoEnabled()) { + log.info( + "maxPushMsgTPS: {}, avgPushMsgTPS: {}, sum: {}, sumFail: {}, sumFailRate: {}, maxClientLatency: {}, avgClientLatency: {}", + summaryMetrics.maxPushMsgTPS(), + summaryMetrics.avgPushMsgTPS(), + summaryMetrics.getHttpPushMsgNumSum(), + summaryMetrics.getHttpPushFailNumSum(), + summaryMetrics.getHttpPushMsgFailRate(), + summaryMetrics.maxHTTPPushLatency(), + summaryMetrics.avgHTTPPushLatency() + ); + } + + summaryMetrics.cleanHttpPushMsgStat(); + + if (log.isInfoEnabled()) { + log.info("batchMsgQ: {}, sendMsgQ: {}, pushMsgQ: {}", + eventMeshHTTPServer.getHttpThreadPoolGroup().getBatchMsgExecutor().getQueue().size(), + eventMeshHTTPServer.getHttpThreadPoolGroup().getSendMsgExecutor().getQueue().size(), + eventMeshHTTPServer.getHttpThreadPoolGroup().getPushMsgExecutor().getQueue().size()); + } + + if (log.isInfoEnabled()) { + log.info("batchAvgSend2MQCost: {}, avgSend2MQCost: {}, avgReply2MQCost: {}", + summaryMetrics.avgBatchSendMsgCost(), + summaryMetrics.avgSendMsgCost(), + summaryMetrics.avgReplyMsgCost()); + } + summaryMetrics.send2MQStatInfoClear(); + } + + public HttpMetrics getHttpMetrics() { + return httpMetrics; + } + + @Override + public List getMetrics() { + return new ArrayList<>(httpMetrics.getMetrics()); + } + + @Override + public String getName() { + return this.getClass().getName(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HTTPMetricsServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HTTPMetricsServer.java deleted file mode 100644 index d0f1e01492..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HTTPMetricsServer.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics.http; - -import org.apache.eventmesh.common.EventMeshThreadFactory; -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.metrics.api.model.HttpSummaryMetrics; -import org.apache.eventmesh.metrics.api.model.RetrySummaryMetrics; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; - -import java.util.List; -import java.util.Objects; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HTTPMetricsServer { - - private final transient EventMeshHTTPServer eventMeshHTTPServer; - - private final transient List metricsRegistries; - - private final transient HttpSummaryMetrics summaryMetrics; - - public HTTPMetricsServer(final EventMeshHTTPServer eventMeshHTTPServer, - final List metricsRegistries) { - Objects.requireNonNull(eventMeshHTTPServer, "EventMeshHTTPServer can not be null"); - Objects.requireNonNull(metricsRegistries, "List can not be null"); - - this.eventMeshHTTPServer = eventMeshHTTPServer; - this.metricsRegistries = metricsRegistries; - this.summaryMetrics = new HttpSummaryMetrics( - eventMeshHTTPServer.getHttpThreadPoolGroup().getBatchMsgExecutor(), - eventMeshHTTPServer.getHttpThreadPoolGroup().getSendMsgExecutor(), - eventMeshHTTPServer.getHttpThreadPoolGroup().getPushMsgExecutor(), - new RetrySummaryMetrics(eventMeshHTTPServer.getHttpRetryer().getPendingTimeouts())); - - init(); - } - - private void init() { - metricsRegistries.forEach(MetricsRegistry::start); - log.info("HTTPMetricsServer initialized."); - } - - public void start() { - metricsRegistries.forEach(metricsRegistry -> { - metricsRegistry.register(summaryMetrics); - log.info("Register httpMetrics to {}", metricsRegistry.getClass().getName()); - }); - - metricsSchedule.scheduleAtFixedRate(() -> { - try { - summaryMetrics.snapshotHTTPTPS(); - summaryMetrics.snapshotSendBatchMsgTPS(); - summaryMetrics.snapshotSendMsgTPS(); - summaryMetrics.snapshotPushMsgTPS(); - } catch (Exception ex) { - log.error("eventMesh snapshot tps metrics err", ex); - } - }, 0, 1000, TimeUnit.MILLISECONDS); - - metricsSchedule.scheduleAtFixedRate(() -> { - try { - logPrintServerMetrics(summaryMetrics, eventMeshHTTPServer); - } catch (Exception ex) { - log.error("eventMesh print metrics err", ex); - } - }, 1000, 30 * 1000, TimeUnit.MILLISECONDS); - - log.info("HTTPMetricsServer started."); - } - - public void shutdown() { - metricsSchedule.shutdown(); - metricsRegistries.forEach(MetricsRegistry::showdown); - log.info("HTTPMetricsServer shutdown."); - } - - private static ScheduledExecutorService metricsSchedule = Executors.newScheduledThreadPool(2, - new EventMeshThreadFactory("eventMesh-metrics", true)); - - // todo: move this into standalone metrics plugin - - private void logPrintServerMetrics(final HttpSummaryMetrics summaryMetrics, - final EventMeshHTTPServer eventMeshHTTPServer) { - log.info("===========================================SERVER METRICS=================================================="); - log.info("maxHTTPTPS: {}, avgHTTPTPS: {}, maxHTTPCOST: {}, avgHTTPCOST: {}, avgHTTPBodyDecodeCost: {}, httpDiscard: {}", - summaryMetrics.maxHTTPTPS(), - summaryMetrics.avgHTTPTPS(), - summaryMetrics.maxHTTPCost(), - summaryMetrics.avgHTTPCost(), - summaryMetrics.avgHTTPBodyDecodeCost(), - summaryMetrics.getHttpDiscard()); - - summaryMetrics.httpStatInfoClear(); - - log.info("maxBatchSendMsgTPS: {}, avgBatchSendMsgTPS: {}, sum: {}. sumFail: {}, sumFailRate: {}, discard : {}", - summaryMetrics.maxSendBatchMsgTPS(), - summaryMetrics.avgSendBatchMsgTPS(), - summaryMetrics.getSendBatchMsgNumSum(), - summaryMetrics.getSendBatchMsgFailNumSum(), - summaryMetrics.getSendBatchMsgFailRate(), - summaryMetrics.getSendBatchMsgDiscardNumSum()); - - summaryMetrics.cleanSendBatchStat(); - - log.info("maxSendMsgTPS: {}, avgSendMsgTPS: {}, sum: {}, sumFail: {}, sumFailRate: {}, replyMsg: {}, replyFail: {}", - summaryMetrics.maxSendMsgTPS(), - summaryMetrics.avgSendMsgTPS(), - summaryMetrics.getSendMsgNumSum(), - summaryMetrics.getSendMsgFailNumSum(), - summaryMetrics.getSendMsgFailRate(), - summaryMetrics.getReplyMsgNumSum(), - summaryMetrics.getReplyMsgFailNumSum()); - - summaryMetrics.cleanSendMsgStat(); - - log.info("maxPushMsgTPS: {}, avgPushMsgTPS: {}, sum: {}, sumFail: {}, sumFailRate: {}, maxClientLatency: {}, avgClientLatency: {}", - summaryMetrics.maxPushMsgTPS(), - summaryMetrics.avgPushMsgTPS(), - summaryMetrics.getHttpPushMsgNumSum(), - summaryMetrics.getHttpPushFailNumSum(), - summaryMetrics.getHttpPushMsgFailRate(), - summaryMetrics.maxHTTPPushLatency(), - summaryMetrics.avgHTTPPushLatency()); - - summaryMetrics.cleanHttpPushMsgStat(); - - log.info("batchMsgQ: {}, sendMsgQ: {}, pushMsgQ: {}, httpRetryQ: {}", - eventMeshHTTPServer.getHttpThreadPoolGroup().getBatchMsgExecutor().getQueue().size(), - eventMeshHTTPServer.getHttpThreadPoolGroup().getSendMsgExecutor().getQueue().size(), - eventMeshHTTPServer.getHttpThreadPoolGroup().getPushMsgExecutor().getQueue().size(), - eventMeshHTTPServer.getHttpRetryer().getPendingTimeouts()); - - log.info("batchAvgSend2MQCost: {}, avgSend2MQCost: {}, avgReply2MQCost: {}", - summaryMetrics.avgBatchSendMsgCost(), - summaryMetrics.avgSendMsgCost(), - summaryMetrics.avgReplyMsgCost()); - - summaryMetrics.send2MQStatInfoClear(); - } - - public HttpSummaryMetrics getSummaryMetrics() { - return summaryMetrics; - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HttpMetrics.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HttpMetrics.java new file mode 100644 index 0000000000..5a0e230e4c --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/http/HttpMetrics.java @@ -0,0 +1,827 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics.http; + +import org.apache.eventmesh.metrics.api.model.InstrumentFurther; +import org.apache.eventmesh.metrics.api.model.LongCounterMetric; +import org.apache.eventmesh.metrics.api.model.Metric; +import org.apache.eventmesh.metrics.api.model.ObservableDoubleGaugeMetric; +import org.apache.eventmesh.metrics.api.model.ObservableLongGaugeMetric; +import org.apache.eventmesh.runtime.metrics.MetricInstrumentUnit; + +import org.apache.commons.collections4.MapUtils; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.DelayQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicLong; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; + +public class HttpMetrics { + + private static Attributes EMPTY = Attributes.builder().build(); + + private static final int STATIC_PERIOD = 30 * 1000; + + private static final String HTTP_METRICS_NAME_PREFIX = "eventmesh.http."; + + private static final String METRIC_NAME = "HTTP"; + + private float wholeCost = 0f; + + private final AtomicLong wholeRequestNum = new AtomicLong(0); + + //cumulative value + private final AtomicLong httpDiscard = new AtomicLong(0); + + private LongCounterMetric httpDiscardMetric; + + private final AtomicLong maxCost = new AtomicLong(0); + + private final AtomicLong httpRequestPerSecond = new AtomicLong(0); + + private final LinkedList httpRequestTPSSnapshots = new LinkedList<>(); + + private float httpDecodeTimeCost = 0f; + + private final AtomicLong httpDecodeNum = new AtomicLong(0); + + private final AtomicLong sendBatchMsgNumPerSecond = new AtomicLong(0); + + private final AtomicLong sendBatchMsgNumSum = new AtomicLong(0); + + private LongCounterMetric sendBatchMsgNumSumMetric; + + private final AtomicLong sendBatchMsgFailNumSum = new AtomicLong(0); + + private LongCounterMetric sendBatchMsgFailNumSumMetric; + + // This is a cumulative value + private final AtomicLong sendBatchMsgDiscardNumSum = new AtomicLong(0); + + private LongCounterMetric sendBatchMsgDiscardNumSumMetric; + + private final LinkedList sendBatchMsgTPSSnapshots = new LinkedList(); + + private final AtomicLong sendMsgNumSum = new AtomicLong(0); + + private LongCounterMetric sendMsgNumSumMetric; + + private final AtomicLong sendMsgFailNumSum = new AtomicLong(0); + + private LongCounterMetric sendMsgFailNumSumMetric; + + private final AtomicLong replyMsgNumSum = new AtomicLong(0); + + private LongCounterMetric replyMsgNumSumMetric; + + private final AtomicLong replyMsgFailNumSum = new AtomicLong(0); + + private LongCounterMetric replyMsgFailNumSumMetric; + + private final AtomicLong sendMsgNumPerSecond = new AtomicLong(0); + + private final LinkedList sendMsgTPSSnapshots = new LinkedList(); + + private float wholePushCost = 0f; + + private final AtomicLong wholePushRequestNum = new AtomicLong(0); + + private final AtomicLong maxHttpPushLatency = new AtomicLong(0); + + private final AtomicLong pushMsgNumPerSecond = new AtomicLong(0); + + private final LinkedList pushMsgTPSSnapshots = new LinkedList(); + + private final AtomicLong httpPushMsgNumSum = new AtomicLong(0); + + private LongCounterMetric httpPushMsgNumSumMetric; + + private final AtomicLong httpPushFailNumSum = new AtomicLong(0); + + private LongCounterMetric httpPushFailNumSumMetric; + + private float batchSend2MQWholeCost = 0f; + + private final AtomicLong batchSend2MQNum = new AtomicLong(0); + + private float send2MQWholeCost = 0f; + + private final AtomicLong send2MQNum = new AtomicLong(0); + + private float reply2MQWholeCost = 0f; + + private final AtomicLong reply2MQNum = new AtomicLong(0); + + // execute metrics + private final ThreadPoolExecutor batchMsgExecutor; + + private final ThreadPoolExecutor sendMsgExecutor; + + private final ThreadPoolExecutor pushMsgExecutor; + + private final DelayQueue httpFailedQueue; + + private final Map labelMap; + + private final Map metrics = new HashMap<>(32); + + private ObservableDoubleGaugeMetric avgHttpBodyDecodeCostMetric; + + private ObservableDoubleGaugeMetric maxHttpTpsMetric; + + private ObservableDoubleGaugeMetric avgHttpTpsMetric; + + private ObservableLongGaugeMetric maxHttpCostMetric; + + private ObservableDoubleGaugeMetric avgHttpCostMetric; + + private ObservableDoubleGaugeMetric maxBatchSendMsgTpsMetric; + + private ObservableDoubleGaugeMetric avgBatchSendMsgTpsMetric; + + private ObservableDoubleGaugeMetric sumBatchFailRateMetric; + + private ObservableDoubleGaugeMetric maxSendMsgTpsMetric; + + private ObservableDoubleGaugeMetric avgSendMsgTpsMetric; + + private ObservableDoubleGaugeMetric sumFailRateMetric; + + private ObservableDoubleGaugeMetric maxPushMsgTpsMetric; + + private ObservableDoubleGaugeMetric avgPushMsgTpsMetric; + + private ObservableDoubleGaugeMetric pushSumFailRateMetric; + + private ObservableDoubleGaugeMetric maxClientLatencyMetric; + + private ObservableDoubleGaugeMetric avgClientLatencyMetric; + + private ObservableLongGaugeMetric batchMsgQMetric; + + private ObservableLongGaugeMetric sendMsgQMetric; + + private ObservableLongGaugeMetric pushMsgQMetric; + + private ObservableLongGaugeMetric httpRetryQMetric; + + private ObservableDoubleGaugeMetric batchAvgSend2MQCostMetric; + + private ObservableDoubleGaugeMetric avgSend2MQCostMetric; + + private ObservableDoubleGaugeMetric avgReply2MQCostMetric; + + public HttpMetrics(final ThreadPoolExecutor batchMsgExecutor, + final ThreadPoolExecutor sendMsgExecutor, + final ThreadPoolExecutor pushMsgExecutor, + final DelayQueue httpFailedQueue, + final Map labelMap) { + this.batchMsgExecutor = batchMsgExecutor; + this.sendMsgExecutor = sendMsgExecutor; + this.pushMsgExecutor = pushMsgExecutor; + this.httpFailedQueue = httpFailedQueue; + this.labelMap = Optional.ofNullable(labelMap).orElse(new HashMap<>(0)); + initMetrics(); + } + + private void initMetrics() { + + InstrumentFurther furtherHttpDiscard = new InstrumentFurther(); + furtherHttpDiscard.setUnit(MetricInstrumentUnit.SINGLETON); + furtherHttpDiscard.setDescription("Http request discard num."); + furtherHttpDiscard.setName(HTTP_METRICS_NAME_PREFIX + "request.discard.num"); + httpDiscardMetric = new LongCounterMetric(furtherHttpDiscard, METRIC_NAME); + metrics.put("httpDiscardMetric", httpDiscardMetric); + + //sum of batch send message number + InstrumentFurther furtherSendBatchMsgNumSum = new InstrumentFurther(); + furtherSendBatchMsgNumSum.setUnit(MetricInstrumentUnit.SINGLETON); + furtherSendBatchMsgNumSum.setDescription("Sum of batch send message number."); + furtherSendBatchMsgNumSum.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.num"); + sendBatchMsgNumSumMetric = new LongCounterMetric(furtherSendBatchMsgNumSum, METRIC_NAME); + metrics.put("sendBatchMsgNumSumMetric", sendBatchMsgNumSumMetric); + + //sum of batch send message fail message number. + InstrumentFurther furtherSendBatchMsgFailNumSum = new InstrumentFurther(); + furtherSendBatchMsgFailNumSum.setUnit(MetricInstrumentUnit.SINGLETON); + furtherSendBatchMsgFailNumSum.setDescription("Sum of batch send message fail message number."); + furtherSendBatchMsgFailNumSum.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.fail.num"); + sendBatchMsgFailNumSumMetric = new LongCounterMetric(furtherSendBatchMsgFailNumSum, METRIC_NAME); + metrics.put("sendBatchMsgFailNumSumMetric", sendBatchMsgFailNumSumMetric); + + //sum of send batch message discard number. + InstrumentFurther furtherSendBatchMsgDiscardNumSum = new InstrumentFurther(); + furtherSendBatchMsgDiscardNumSum.setUnit(MetricInstrumentUnit.SINGLETON); + furtherSendBatchMsgDiscardNumSum.setDescription("Sum of batch send message fail message number."); + furtherSendBatchMsgDiscardNumSum.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.discard.num"); + sendBatchMsgDiscardNumSumMetric = new LongCounterMetric(furtherSendBatchMsgDiscardNumSum, METRIC_NAME); + metrics.put("sendBatchMsgDiscardNumSumMetric", sendBatchMsgDiscardNumSumMetric); + + //Sum of send message number. + InstrumentFurther furtherSendMsgNumSum = new InstrumentFurther(); + furtherSendMsgNumSum.setUnit(MetricInstrumentUnit.SINGLETON); + furtherSendMsgNumSum.setDescription("Sum of send message number."); + furtherSendMsgNumSum.setName(HTTP_METRICS_NAME_PREFIX + "send.message.num"); + sendMsgNumSumMetric = new LongCounterMetric(furtherSendMsgNumSum, METRIC_NAME); + metrics.put("sendMsgNumSumMetric", sendMsgNumSumMetric); + + //Sum of send message fail number. + InstrumentFurther furtherSendMsgFailNumSum = new InstrumentFurther(); + furtherSendMsgFailNumSum.setUnit(MetricInstrumentUnit.SINGLETON); + furtherSendMsgFailNumSum.setDescription("Sum of send message fail number."); + furtherSendMsgFailNumSum.setName(HTTP_METRICS_NAME_PREFIX + "send.message.fail.num"); + sendMsgFailNumSumMetric = new LongCounterMetric(furtherSendMsgFailNumSum, METRIC_NAME); + metrics.put("sendMsgFailNumSumMetric", sendMsgFailNumSumMetric); + + //Sum of reply message number. + InstrumentFurther furtherReplyMsgNumSum = new InstrumentFurther(); + furtherReplyMsgNumSum.setUnit(MetricInstrumentUnit.SINGLETON); + furtherReplyMsgNumSum.setDescription("Sum of reply message number."); + furtherReplyMsgNumSum.setName(HTTP_METRICS_NAME_PREFIX + "reply.message.num"); + replyMsgNumSumMetric = new LongCounterMetric(furtherReplyMsgNumSum, METRIC_NAME); + metrics.put("replyMsgNumSumMetric", replyMsgNumSumMetric); + + //Sum of reply message fail number. + InstrumentFurther furtherReplyMsgFailNumSum = new InstrumentFurther(); + furtherReplyMsgFailNumSum.setUnit(MetricInstrumentUnit.SINGLETON); + furtherReplyMsgFailNumSum.setDescription("Sum of reply message fail number."); + furtherReplyMsgFailNumSum.setName(HTTP_METRICS_NAME_PREFIX + "reply.message.fail.num"); + replyMsgFailNumSumMetric = new LongCounterMetric(furtherReplyMsgFailNumSum, METRIC_NAME); + metrics.put("replyMsgFailNumSumMetric", replyMsgFailNumSumMetric); + + //Sum of http push message number. + InstrumentFurther furtherHttpPushMsgNumSum = new InstrumentFurther(); + furtherHttpPushMsgNumSum.setUnit(MetricInstrumentUnit.SINGLETON); + furtherHttpPushMsgNumSum.setDescription("Sum of http push message number."); + furtherHttpPushMsgNumSum.setName(HTTP_METRICS_NAME_PREFIX + "push.message.num"); + httpPushMsgNumSumMetric = new LongCounterMetric(furtherHttpPushMsgNumSum, METRIC_NAME); + metrics.put("httpPushMsgNumSumMetric", httpPushMsgNumSumMetric); + + //Sum of http push message fail number. + InstrumentFurther furtherHttpPushFailNumSum = new InstrumentFurther(); + furtherHttpPushFailNumSum.setUnit(MetricInstrumentUnit.SINGLETON); + furtherHttpPushFailNumSum.setDescription("Sum of http push message fail number."); + furtherHttpPushFailNumSum.setName(HTTP_METRICS_NAME_PREFIX + "push.message.fail.num"); + httpPushFailNumSumMetric = new LongCounterMetric(furtherHttpPushFailNumSum, METRIC_NAME); + metrics.put("httpPushFailNumSumMetric", httpPushFailNumSumMetric); + + //avg body decode cost time of http + InstrumentFurther furtherHttpDecode = new InstrumentFurther(); + furtherHttpDecode.setUnit(MetricInstrumentUnit.MILLISECONDS); + furtherHttpDecode.setDescription("Avg body decode cost time of http"); + furtherHttpDecode.setName(HTTP_METRICS_NAME_PREFIX + "body.decode.cost.avg"); + avgHttpBodyDecodeCostMetric = new ObservableDoubleGaugeMetric(furtherHttpDecode, METRIC_NAME, () -> this.avgHTTPBodyDecodeCost()); + avgHttpBodyDecodeCostMetric.putAll(labelMap); + metrics.put("avgHttpBodyDecodeCostMetric", avgHttpBodyDecodeCostMetric); + + //Max TPS of HTTP. + InstrumentFurther furtherMaxHttpTps = new InstrumentFurther(); + furtherMaxHttpTps.setUnit(MetricInstrumentUnit.TPS); + furtherMaxHttpTps.setDescription("Max TPS of HTTP."); + furtherMaxHttpTps.setName(HTTP_METRICS_NAME_PREFIX + "request.tps.max"); + maxHttpTpsMetric = new ObservableDoubleGaugeMetric(furtherMaxHttpTps, METRIC_NAME, () -> this.maxHTTPTPS()); + maxHttpTpsMetric.putAll(labelMap); + metrics.put("maxHttpTpsMetric", maxHttpTpsMetric); + + //Avg TPS of HTTP. + InstrumentFurther furtherAvgHttpTps = new InstrumentFurther(); + furtherAvgHttpTps.setUnit(MetricInstrumentUnit.TPS); + furtherAvgHttpTps.setDescription("Avg TPS of HTTP."); + furtherAvgHttpTps.setName(HTTP_METRICS_NAME_PREFIX + "request.tps.avg"); + avgHttpTpsMetric = new ObservableDoubleGaugeMetric(furtherAvgHttpTps, METRIC_NAME, () -> this.avgHTTPTPS()); + avgHttpTpsMetric.putAll(labelMap); + metrics.put("avgHttpTpsMetric", avgHttpTpsMetric); + + //max cost of HTTP. + InstrumentFurther furtherMaxCostHttpTps = new InstrumentFurther(); + furtherMaxCostHttpTps.setUnit(MetricInstrumentUnit.MILLISECONDS); + furtherMaxCostHttpTps.setDescription("Max cost of HTTP."); + furtherMaxCostHttpTps.setName(HTTP_METRICS_NAME_PREFIX + "request.cost.max"); + maxHttpCostMetric = new ObservableLongGaugeMetric(furtherMaxCostHttpTps, METRIC_NAME, () -> this.maxHTTPCost()); + maxHttpCostMetric.putAll(labelMap); + metrics.put("maxHttpCostMetric", maxHttpCostMetric); + + //Avg cost of HTTP. + InstrumentFurther furtherAvgHttpCost = new InstrumentFurther(); + furtherAvgHttpCost.setUnit(MetricInstrumentUnit.TPS); + furtherAvgHttpCost.setDescription("Avg cost of HTTP."); + furtherAvgHttpCost.setName(HTTP_METRICS_NAME_PREFIX + "request.cost.avg"); + avgHttpCostMetric = new ObservableDoubleGaugeMetric(furtherAvgHttpCost, METRIC_NAME, () -> this.avgHTTPCost()); + avgHttpCostMetric.putAll(labelMap); + metrics.put("avgHttpCostMetric", avgHttpCostMetric); + + //Max of batch send message tps + InstrumentFurther furtherMaxBatchSendMsgTps = new InstrumentFurther(); + furtherMaxBatchSendMsgTps.setUnit(MetricInstrumentUnit.TPS); + furtherMaxBatchSendMsgTps.setDescription("Max of batch send message tps"); + furtherMaxBatchSendMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.tps.max"); + maxBatchSendMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherMaxBatchSendMsgTps, METRIC_NAME, () -> this.maxSendBatchMsgTPS()); + maxBatchSendMsgTpsMetric.putAll(labelMap); + metrics.put("maxBatchSendMsgTpsMetric", maxBatchSendMsgTpsMetric); + + //Avg of batch send message tps. + InstrumentFurther furtherAvgBatchSendMsgTps = new InstrumentFurther(); + furtherAvgBatchSendMsgTps.setUnit(MetricInstrumentUnit.TPS); + furtherAvgBatchSendMsgTps.setDescription("Avg of batch send message tps."); + furtherAvgBatchSendMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.tps.avg"); + avgBatchSendMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherAvgBatchSendMsgTps, METRIC_NAME, () -> this.avgSendBatchMsgTPS()); + avgBatchSendMsgTpsMetric.putAll(labelMap); + metrics.put("avgBatchSendMsgTpsMetric", avgBatchSendMsgTpsMetric); + + //Send batch message fail rate. + InstrumentFurther furtherSumBatchFailRate = new InstrumentFurther(); + furtherSumBatchFailRate.setUnit(MetricInstrumentUnit.PERCENT); + furtherSumBatchFailRate.setDescription("Send batch message fail rate."); + furtherSumBatchFailRate.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.fail.rate"); + sumBatchFailRateMetric = new ObservableDoubleGaugeMetric(furtherSumBatchFailRate, METRIC_NAME, () -> this.getSendBatchMsgFailRate()); + sumBatchFailRateMetric.putAll(labelMap); + metrics.put("sumBatchFailRateMetric", sumBatchFailRateMetric); + + //Max of send message tps + InstrumentFurther furtherMaxSendMsgTps = new InstrumentFurther(); + furtherMaxSendMsgTps.setUnit(MetricInstrumentUnit.TPS); + furtherMaxSendMsgTps.setDescription("Max of send message tps"); + furtherMaxSendMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "send.message.tps.max"); + maxSendMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherMaxSendMsgTps, METRIC_NAME, () -> this.maxSendMsgTPS()); + maxSendMsgTpsMetric.putAll(labelMap); + metrics.put("maxSendMsgTpsMetric", maxSendMsgTpsMetric); + + //Avg of send message tps + InstrumentFurther furtherAvgSendMsgTps = new InstrumentFurther(); + furtherAvgSendMsgTps.setUnit(MetricInstrumentUnit.TPS); + furtherAvgSendMsgTps.setDescription("Avg of send message tps"); + furtherAvgSendMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "send.message.tps.avg"); + avgSendMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherAvgSendMsgTps, METRIC_NAME, () -> this.avgSendMsgTPS()); + avgSendMsgTpsMetric.putAll(labelMap); + metrics.put("avgSendMsgTpsMetric", avgSendMsgTpsMetric); + + //Send message fail rate. + InstrumentFurther furtherSumFailRate = new InstrumentFurther(); + furtherSumFailRate.setUnit(MetricInstrumentUnit.PERCENT); + furtherSumFailRate.setDescription("Send message fail rate."); + furtherSumFailRate.setName(HTTP_METRICS_NAME_PREFIX + "send.message.fail.rate"); + sumFailRateMetric = new ObservableDoubleGaugeMetric(furtherSumFailRate, METRIC_NAME, () -> this.getSendBatchMsgFailRate()); + sumFailRateMetric.putAll(labelMap); + metrics.put("sumFailRateMetric", sumFailRateMetric); + + //Max of push message tps. + InstrumentFurther furtherMaxPushMsgTps = new InstrumentFurther(); + furtherMaxPushMsgTps.setUnit(MetricInstrumentUnit.TPS); + furtherMaxPushMsgTps.setDescription("Max of push message tps."); + furtherMaxPushMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "push.message.tps.max"); + maxPushMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherMaxPushMsgTps, METRIC_NAME, () -> this.maxPushMsgTPS()); + maxPushMsgTpsMetric.putAll(labelMap); + metrics.put("maxPushMsgTpsMetric", maxPushMsgTpsMetric); + + //Avg of push message tps. + InstrumentFurther furtherAvgPushMsgTps = new InstrumentFurther(); + furtherAvgPushMsgTps.setUnit(MetricInstrumentUnit.TPS); + furtherAvgPushMsgTps.setDescription("Avg of push message tps."); + furtherAvgPushMsgTps.setName(HTTP_METRICS_NAME_PREFIX + "push.message.tps.avg"); + avgPushMsgTpsMetric = new ObservableDoubleGaugeMetric(furtherAvgPushMsgTps, METRIC_NAME, () -> this.avgPushMsgTPS()); + avgPushMsgTpsMetric.putAll(labelMap); + metrics.put("avgPushMsgTpsMetric", avgPushMsgTpsMetric); + + //Http push message fail rate. + InstrumentFurther furtherPushSumFailRate = new InstrumentFurther(); + furtherPushSumFailRate.setUnit(MetricInstrumentUnit.PERCENT); + furtherPushSumFailRate.setDescription("Http push message fail rate."); + furtherPushSumFailRate.setName(HTTP_METRICS_NAME_PREFIX + "push.message.fail.rate"); + pushSumFailRateMetric = new ObservableDoubleGaugeMetric(furtherPushSumFailRate, METRIC_NAME, () -> this.getHttpPushMsgFailRate()); + pushSumFailRateMetric.putAll(labelMap); + metrics.put("pushSumFailRateMetric", pushSumFailRateMetric); + + //Max of http push latency. + InstrumentFurther furtherMaxClientLatency = new InstrumentFurther(); + furtherMaxClientLatency.setUnit(MetricInstrumentUnit.MILLISECONDS); + furtherMaxClientLatency.setDescription("Max of http push latency."); + furtherMaxClientLatency.setName(HTTP_METRICS_NAME_PREFIX + "push.latency.max"); + maxClientLatencyMetric = new ObservableDoubleGaugeMetric(furtherMaxClientLatency, METRIC_NAME, () -> this.maxHTTPPushLatency()); + maxClientLatencyMetric.putAll(labelMap); + metrics.put("maxClientLatencyMetric", maxClientLatencyMetric); + + //Avg of http push latency. + InstrumentFurther furtherAvgClientLatency = new InstrumentFurther(); + furtherAvgClientLatency.setUnit(MetricInstrumentUnit.MILLISECONDS); + furtherAvgClientLatency.setDescription("Avg of http push latency."); + furtherAvgClientLatency.setName(HTTP_METRICS_NAME_PREFIX + "push.latency.avg"); + avgClientLatencyMetric = new ObservableDoubleGaugeMetric(furtherAvgClientLatency, METRIC_NAME, () -> this.avgHTTPPushLatency()); + avgClientLatencyMetric.putAll(labelMap); + metrics.put("avgClientLatencyMetric", avgClientLatencyMetric); + + //Size of batch message queue. + InstrumentFurther furtherBatchMsgQ = new InstrumentFurther(); + furtherBatchMsgQ.setUnit(MetricInstrumentUnit.SINGLETON); + furtherBatchMsgQ.setDescription("Size of batch message queue."); + furtherBatchMsgQ.setName(HTTP_METRICS_NAME_PREFIX + "batch.message.queue.size"); + batchMsgQMetric = new ObservableLongGaugeMetric(furtherBatchMsgQ, METRIC_NAME, () -> this.getBatchMsgQueueSize()); + batchMsgQMetric.putAll(labelMap); + metrics.put("batchMsgQMetric", batchMsgQMetric); + + //Size of send message queue. + InstrumentFurther furtherSendMsgQ = new InstrumentFurther(); + furtherSendMsgQ.setUnit(MetricInstrumentUnit.SINGLETON); + furtherSendMsgQ.setDescription("Size of send message queue."); + furtherSendMsgQ.setName(HTTP_METRICS_NAME_PREFIX + "send.message.queue.size"); + sendMsgQMetric = new ObservableLongGaugeMetric(furtherSendMsgQ, METRIC_NAME, () -> this.getSendMsgQueueSize()); + sendMsgQMetric.putAll(labelMap); + metrics.put("sendMsgQMetric", sendMsgQMetric); + + //Size of push message queue. + InstrumentFurther furtherPushMsgQ = new InstrumentFurther(); + furtherPushMsgQ.setUnit(MetricInstrumentUnit.SINGLETON); + furtherPushMsgQ.setDescription("Size of push message queue."); + furtherPushMsgQ.setName(HTTP_METRICS_NAME_PREFIX + "push.message.queue.size"); + pushMsgQMetric = new ObservableLongGaugeMetric(furtherPushMsgQ, METRIC_NAME, () -> this.getPushMsgQueueSize()); + pushMsgQMetric.putAll(labelMap); + metrics.put("pushMsgQMetric", pushMsgQMetric); + + //Size of http retry queue. + InstrumentFurther furtherHttpRetryQ = new InstrumentFurther(); + furtherHttpRetryQ.setUnit(MetricInstrumentUnit.SINGLETON); + furtherHttpRetryQ.setDescription("Size of http retry queue."); + furtherHttpRetryQ.setName(HTTP_METRICS_NAME_PREFIX + "retry.queue.size"); + httpRetryQMetric = new ObservableLongGaugeMetric(furtherHttpRetryQ, METRIC_NAME, () -> this.getHttpRetryQueueSize()); + httpRetryQMetric.putAll(labelMap); + metrics.put("httpRetryQMetric", httpRetryQMetric); + + + //Avg of batch send message cost. + InstrumentFurther furtherBatchAvgSend2MQCost = new InstrumentFurther(); + furtherBatchAvgSend2MQCost.setUnit(MetricInstrumentUnit.MILLISECONDS); + furtherBatchAvgSend2MQCost.setDescription("Avg of batch send message cost."); + furtherBatchAvgSend2MQCost.setName(HTTP_METRICS_NAME_PREFIX + "batch.send.message.cost.avg"); + batchAvgSend2MQCostMetric = new ObservableDoubleGaugeMetric(furtherBatchAvgSend2MQCost, METRIC_NAME, () -> this.avgBatchSendMsgCost()); + batchAvgSend2MQCostMetric.putAll(labelMap); + metrics.put("avgClientLatencyMetric", batchAvgSend2MQCostMetric); + + //Avg of send message cost. + InstrumentFurther furtherAvgSend2MQCost = new InstrumentFurther(); + furtherAvgSend2MQCost.setUnit(MetricInstrumentUnit.TPS); + furtherAvgSend2MQCost.setDescription("Avg of send message cost."); + furtherAvgSend2MQCost.setName(HTTP_METRICS_NAME_PREFIX + "send.message.cost.avg"); + avgSend2MQCostMetric = new ObservableDoubleGaugeMetric(furtherAvgSend2MQCost, METRIC_NAME, () -> this.avgSendMsgCost()); + avgSend2MQCostMetric.putAll(labelMap); + metrics.put("avgSend2MQCostMetric", avgSend2MQCostMetric); + + //Avg of reply message cost. + InstrumentFurther furtherAvgReply2MQCost = new InstrumentFurther(); + furtherAvgReply2MQCost.setUnit(MetricInstrumentUnit.TPS); + furtherAvgReply2MQCost.setDescription("Avg of reply message cost."); + furtherAvgReply2MQCost.setName(HTTP_METRICS_NAME_PREFIX + "reply.message.cost.avg"); + avgReply2MQCostMetric = new ObservableDoubleGaugeMetric(furtherAvgReply2MQCost, METRIC_NAME, () -> this.avgReplyMsgCost()); + avgReply2MQCostMetric.putAll(labelMap); + metrics.put("avgReply2MQCostMetric", avgReply2MQCostMetric); + } + + public Collection getMetrics() { + return metrics.values(); + } + + public double avgHTTPCost() { + return (wholeRequestNum.longValue() == 0L) ? 0f : wholeCost / wholeRequestNum.longValue(); + } + + public long maxHTTPCost() { + return maxCost.longValue(); + } + + public long getHttpDiscard() { + return httpDiscard.longValue(); + } + + public void recordHTTPRequest() { + httpRequestPerSecond.incrementAndGet(); + } + + public void recordHTTPDiscard() { + httpDiscard.incrementAndGet(); + Map attributes = new HashMap<>(this.labelMap); + httpDiscardMetric.getInstrument().add(1, buildAttributes(attributes)); + } + + private static Attributes buildAttributes(final Map attributes) { + if (MapUtils.isEmpty(attributes)) { + return EMPTY; + } + AttributesBuilder attributesBuilder = Attributes.builder(); + attributes.forEach(attributesBuilder::put); + return attributesBuilder.build(); + } + + public void snapshotHTTPTPS() { + Integer tps = httpRequestPerSecond.intValue(); + httpRequestTPSSnapshots.add(tps); + httpRequestPerSecond.set(0); + if (httpRequestTPSSnapshots.size() > STATIC_PERIOD / 1000) { + httpRequestTPSSnapshots.removeFirst(); + } + } + + public double maxHTTPTPS() { + return Collections.max(httpRequestTPSSnapshots); + } + + public double avgHTTPTPS() { + return avg(httpRequestTPSSnapshots); + } + + public void recordHTTPReqResTimeCost(long cost) { + wholeRequestNum.incrementAndGet(); + wholeCost = wholeCost + cost; + if (cost > maxCost.longValue()) { + maxCost.set(cost); + } + } + + public void httpStatInfoClear() { + wholeRequestNum.set(0L); + wholeCost = 0f; + maxCost.set(0L); + httpDecodeNum.set(0L); + httpDecodeTimeCost = 0f; + } + + + public void recordDecodeTimeCost(long cost) { + httpDecodeNum.incrementAndGet(); + httpDecodeTimeCost = httpDecodeTimeCost + cost; + } + + public double avgHTTPBodyDecodeCost() { + return (httpDecodeNum.longValue() == 0L) ? 0d : (double) httpDecodeTimeCost / httpDecodeNum.longValue(); + } + + + public void recordSendBatchMsgDiscard(long delta) { + sendBatchMsgDiscardNumSum.addAndGet(delta); + sendBatchMsgDiscardNumSumMetric.getInstrument().add(delta, buildAttributes(labelMap)); + } + + public void snapshotSendBatchMsgTPS() { + Integer tps = sendBatchMsgNumPerSecond.intValue(); + sendBatchMsgTPSSnapshots.add(tps); + sendBatchMsgNumPerSecond.set(0); + if (sendBatchMsgTPSSnapshots.size() > STATIC_PERIOD / 1000) { + sendBatchMsgTPSSnapshots.removeFirst(); + } + } + + public double maxSendBatchMsgTPS() { + return Collections.max(sendBatchMsgTPSSnapshots); + } + + public double avgSendBatchMsgTPS() { + return avg(sendBatchMsgTPSSnapshots); + } + + public void recordSendBatchMsg(long delta) { + sendBatchMsgNumPerSecond.addAndGet(delta); + sendBatchMsgNumSum.addAndGet(delta); + sendBatchMsgNumSumMetric.getInstrument().add(delta, buildAttributes(labelMap)); + } + + public void recordSendBatchMsgFailed(long delta) { + sendBatchMsgFailNumSum.getAndAdd(delta); + sendBatchMsgFailNumSumMetric.getInstrument().add(delta, buildAttributes(labelMap)); + } + + public long getSendBatchMsgNumSum() { + return sendBatchMsgNumSum.longValue(); + } + + public long getSendBatchMsgFailNumSum() { + return sendBatchMsgFailNumSum.longValue(); + } + + public double getSendBatchMsgFailRate() { + return (sendBatchMsgNumSum.longValue() == 0L) ? 0f : sendBatchMsgFailNumSum.floatValue() / sendBatchMsgNumSum.longValue(); + } + + public void cleanSendBatchStat() { + sendBatchMsgNumSum.set(0L); + sendBatchMsgFailNumSum.set(0L); + } + + public long getSendBatchMsgDiscardNumSum() { + return sendBatchMsgDiscardNumSum.longValue(); + } + + + public void snapshotSendMsgTPS() { + Integer tps = sendMsgNumPerSecond.intValue(); + sendMsgTPSSnapshots.add(tps); + sendMsgNumPerSecond.set(0); + if (sendMsgTPSSnapshots.size() > STATIC_PERIOD / 1000) { + sendMsgTPSSnapshots.removeFirst(); + } + } + + public double maxSendMsgTPS() { + return Collections.max(sendMsgTPSSnapshots); + } + + public double avgSendMsgTPS() { + return avg(sendMsgTPSSnapshots); + } + + public void recordSendMsg() { + sendMsgNumPerSecond.incrementAndGet(); + sendMsgNumSum.incrementAndGet(); + sendMsgNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); + } + + public void recordReplyMsg() { + replyMsgNumSum.incrementAndGet(); + replyMsgNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); + } + + public void recordReplyMsgFailed() { + replyMsgFailNumSum.incrementAndGet(); + replyMsgFailNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); + } + + public long getReplyMsgNumSum() { + return replyMsgNumSum.longValue(); + } + + public long getReplyMsgFailNumSum() { + return replyMsgFailNumSum.longValue(); + } + + public long getSendMsgNumSum() { + return sendMsgNumSum.longValue(); + } + + public long getSendMsgFailNumSum() { + return sendMsgFailNumSum.longValue(); + } + + public float getSendMsgFailRate() { + return (sendMsgNumSum.longValue() == 0L) ? 0f : sendMsgFailNumSum.floatValue() / sendMsgNumSum.longValue(); + } + + public void recordSendMsgFailed() { + sendMsgFailNumSum.incrementAndGet(); + sendMsgFailNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); + } + + public void cleanSendMsgStat() { + sendMsgNumSum.set(0L); + replyMsgNumSum.set(0L); + sendMsgFailNumSum.set(0L); + replyMsgFailNumSum.set(0L); + } + + + public void snapshotPushMsgTPS() { + Integer tps = pushMsgNumPerSecond.intValue(); + pushMsgTPSSnapshots.add(tps); + pushMsgNumPerSecond.set(0); + if (pushMsgTPSSnapshots.size() > STATIC_PERIOD / 1000) { + pushMsgTPSSnapshots.removeFirst(); + } + } + + public void recordHTTPPushTimeCost(long cost) { + wholePushRequestNum.incrementAndGet(); + wholePushCost = wholePushCost + cost; + if (cost > maxHttpPushLatency.longValue()) { + maxHttpPushLatency.set(cost); + } + } + + public double avgHTTPPushLatency() { + return (wholePushRequestNum.longValue() == 0L) ? 0f : wholePushCost / wholePushRequestNum.longValue(); + } + + public double maxHTTPPushLatency() { + return maxHttpPushLatency.floatValue(); + } + + public double maxPushMsgTPS() { + return Collections.max(pushMsgTPSSnapshots); + } + + public double avgPushMsgTPS() { + return avg(pushMsgTPSSnapshots); + } + + public void recordPushMsg() { + pushMsgNumPerSecond.incrementAndGet(); + httpPushMsgNumSum.incrementAndGet(); + httpPushMsgNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); + } + + public long getHttpPushMsgNumSum() { + return httpPushMsgNumSum.longValue(); + } + + public long getHttpPushFailNumSum() { + return httpPushFailNumSum.longValue(); + } + + public double getHttpPushMsgFailRate() { + return (httpPushMsgNumSum.longValue() == 0L) ? 0f : httpPushFailNumSum.floatValue() / httpPushMsgNumSum.longValue(); + } + + public void recordHttpPushMsgFailed() { + sendMsgFailNumSum.incrementAndGet(); + sendMsgFailNumSumMetric.getInstrument().add(1, buildAttributes(labelMap)); + } + + public void cleanHttpPushMsgStat() { + httpPushFailNumSum.set(0L); + httpPushMsgNumSum.set(0L); + wholeRequestNum.set(0L); + wholeCost = 0f; + maxCost.set(0L); + } + + + public void recordBatchSendMsgCost(long cost) { + batchSend2MQNum.incrementAndGet(); + batchSend2MQWholeCost = batchSend2MQWholeCost + cost; + } + + public double avgBatchSendMsgCost() { + return (batchSend2MQNum.intValue() == 0) ? 0f : batchSend2MQWholeCost / batchSend2MQNum.intValue(); + } + + public void recordSendMsgCost(long cost) { + send2MQNum.incrementAndGet(); + send2MQWholeCost = send2MQWholeCost + cost; + } + + public double avgSendMsgCost() { + return (send2MQNum.intValue() == 0) ? 0f : send2MQWholeCost / send2MQNum.intValue(); + } + + public void recordReplyMsgCost(long cost) { + reply2MQNum.incrementAndGet(); + reply2MQWholeCost = reply2MQWholeCost + cost; + } + + public double avgReplyMsgCost() { + return (reply2MQNum.intValue() == 0) ? 0f : reply2MQWholeCost / reply2MQNum.intValue(); + } + + public void send2MQStatInfoClear() { + batchSend2MQWholeCost = 0f; + batchSend2MQNum.set(0L); + send2MQWholeCost = 0f; + send2MQNum.set(0L); + reply2MQWholeCost = 0f; + reply2MQNum.set(0L); + } + + + public long getBatchMsgQueueSize() { + return batchMsgExecutor.getQueue().size(); + } + + public long getSendMsgQueueSize() { + return sendMsgExecutor.getQueue().size(); + } + + public long getPushMsgQueueSize() { + return pushMsgExecutor.getQueue().size(); + } + + public long getHttpRetryQueueSize() { + return httpFailedQueue.size(); + } + + + private float avg(LinkedList linkedList) { + if (linkedList.isEmpty()) { + return 0.0f; + } + + int sum = linkedList.stream().reduce(Integer::sum).get(); + return (float) sum / linkedList.size(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/EventMeshTcpMetricsManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/EventMeshTcpMetricsManager.java new file mode 100644 index 0000000000..dab6df7118 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/EventMeshTcpMetricsManager.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics.tcp; + +import org.apache.eventmesh.common.MetricsConstants; +import org.apache.eventmesh.common.enums.ProtocolType; +import org.apache.eventmesh.common.utils.IPUtils; +import org.apache.eventmesh.metrics.api.MetricsRegistry; +import org.apache.eventmesh.metrics.api.model.Metric; +import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; +import org.apache.eventmesh.runtime.metrics.MetricsManager; +import org.apache.eventmesh.runtime.metrics.MetricsUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.google.common.base.Preconditions; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class EventMeshTcpMetricsManager implements MetricsManager { + + private final Map labelMap = new HashMap<>(); + + private final EventMeshTCPServer eventMeshTCPServer; + + private final TcpMetrics tcpMetrics; + + private final List metricsRegistries; + + private final TcpMetricsCalculator calculator; + + public EventMeshTcpMetricsManager(EventMeshTCPServer eventMeshTCPServer, List metricsRegistries) { + + this.eventMeshTCPServer = eventMeshTCPServer; + init(); + this.tcpMetrics = new TcpMetrics(eventMeshTCPServer, labelMap); + this.metricsRegistries = Preconditions.checkNotNull(metricsRegistries); + this.calculator = new TcpMetricsCalculator(eventMeshTCPServer, tcpMetrics); + + + } + + private void init() { + String eventMeshServerIp = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshServerIp(); + int eventMeshTcpServerPort = eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshTcpServerPort(); + labelMap.put(MetricsConstants.TCP_NET_HOST_NAME, Optional.ofNullable(eventMeshServerIp).orElse(IPUtils.getLocalAddress())); + labelMap.put(MetricsConstants.TCP_NET_HOST_PORT, Integer.toString(eventMeshTcpServerPort)); + labelMap.put(MetricsConstants.RPC_SYSTEM, "TCP"); + labelMap.put(MetricsConstants.RPC_SERVICE, this.eventMeshTCPServer.getClass().getName()); + log.info("EventMeshTcpMetricsManager initialized......"); + } + + public EventMeshTCPServer getEventMeshTCPServer() { + return eventMeshTCPServer; + } + + @Override + public void start() { + this.calculator.start(); + log.info("EventMeshTcpMetricsManager started......"); + } + + public void client2eventMeshMsgNumIncrement(final String clientAddress) { + tcpMetrics.getClient2eventMeshMsgNum().getAndIncrement(); + Map attributes = new HashMap<>(labelMap); + attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.TCP.name()); + attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); + MetricsUtils.incrementClientToEventMeshMsgNum(attributes); + } + + public void eventMesh2mqMsgNumIncrement() { + tcpMetrics.getEventMesh2mqMsgNum().getAndIncrement(); + Map attributes = new HashMap<>(labelMap); + MetricsUtils.incrementEventMeshToMQMsgNum(attributes); + } + + public void mq2eventMeshMsgNumIncrement() { + tcpMetrics.getMq2eventMeshMsgNum().getAndIncrement(); + Map attributes = new HashMap<>(labelMap); + MetricsUtils.incrementMQToEventMeshMsgNum(attributes); + } + + public void eventMesh2clientMsgNumIncrement(final String clientAddress) { + tcpMetrics.getEventMesh2clientMsgNum().getAndIncrement(); + Map attributes = new HashMap<>(labelMap); + attributes.put(MetricsConstants.CLIENT_PROTOCOL_TYPE, ProtocolType.TCP.name()); + attributes.put(MetricsConstants.CLIENT_ADDRESS, Optional.ofNullable(clientAddress).orElse(MetricsConstants.UNKOWN)); + MetricsUtils.incrementEventMeshToClientMsgNum(attributes); + } + + public TcpMetrics getTcpMetrics() { + return tcpMetrics; + } + + @Override + public void shutdown() { + this.calculator.shutdown(); + log.info("EventMeshTcpMetricsManager shutdown......"); + } + + @Override + public List getMetrics() { + return new ArrayList<>(tcpMetrics.getMetrics()); + } + + @Override + public String getName() { + return this.getClass().getName(); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/EventMeshTcpMonitor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/EventMeshTcpMonitor.java deleted file mode 100644 index f01869ac1f..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/EventMeshTcpMonitor.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.metrics.tcp; - -import org.apache.eventmesh.metrics.api.MetricsRegistry; -import org.apache.eventmesh.metrics.api.model.TcpSummaryMetrics; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.constants.EventMeshConstants; -import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; -import org.apache.eventmesh.runtime.metrics.MonitorMetricConstants; - -import java.net.InetSocketAddress; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Preconditions; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class EventMeshTcpMonitor { - - private final EventMeshTCPServer eventMeshTCPServer; - - public EventMeshTCPServer getEventMeshTCPServer() { - return eventMeshTCPServer; - } - - private static final Logger TCP_LOGGER = LoggerFactory.getLogger(EventMeshConstants.TCP_MONITOR); - - private static final Logger APP_LOGGER = LoggerFactory.getLogger(EventMeshConstants.APP_MONITOR); - - private static final int period = 60 * 1000; - - private static int PRINT_THREADPOOLSTATE_INTERVAL = 1; - - public ScheduledFuture monitorTpsTask; - - public ScheduledFuture monitorThreadPoolTask; - - private final TcpSummaryMetrics tcpSummaryMetrics; - - private final List metricsRegistries; - - public EventMeshTcpMonitor(EventMeshTCPServer eventMeshTCPServer, List metricsRegistries) { - this.eventMeshTCPServer = eventMeshTCPServer; - this.tcpSummaryMetrics = new TcpSummaryMetrics(); - this.metricsRegistries = Preconditions.checkNotNull(metricsRegistries); - } - - public void init() throws Exception { - metricsRegistries.forEach(MetricsRegistry::start); - log.info("EventMeshTcpMonitor initialized......"); - } - - public void start() throws Exception { - metricsRegistries.forEach(metricsRegistry -> { - metricsRegistry.register(tcpSummaryMetrics); - log.info("Register tcpMetrics to {}", metricsRegistry.getClass().getName()); - }); - - int delay = 60 * 1000; - monitorTpsTask = eventMeshTCPServer.getTcpThreadPoolGroup().getScheduler().scheduleAtFixedRate((() -> { - int msgNum = tcpSummaryMetrics.client2eventMeshMsgNum(); - tcpSummaryMetrics.resetClient2EventMeshMsgNum(); - tcpSummaryMetrics.setClient2eventMeshTPS((int) 1000.0d * msgNum / period); - - msgNum = tcpSummaryMetrics.eventMesh2clientMsgNum(); - tcpSummaryMetrics.resetEventMesh2ClientMsgNum(); - tcpSummaryMetrics.setEventMesh2clientTPS((int) 1000.0d * msgNum / period); - - msgNum = tcpSummaryMetrics.eventMesh2mqMsgNum(); - tcpSummaryMetrics.resetEventMesh2mqMsgNum(); - tcpSummaryMetrics.setEventMesh2mqTPS((int) 1000.0d * msgNum / period); - - msgNum = tcpSummaryMetrics.mq2eventMeshMsgNum(); - tcpSummaryMetrics.resetMq2eventMeshMsgNum(); - tcpSummaryMetrics.setMq2eventMeshTPS((int) 1000.0d * msgNum / period); - - // count topics subscribed by client in this eventMesh - ConcurrentHashMap sessionMap = - eventMeshTCPServer.getClientSessionGroupMapping().getSessionMap(); - Iterator sessionIterator = sessionMap.values().iterator(); - Set topicSet = new HashSet<>(); - while (sessionIterator.hasNext()) { - Session session = sessionIterator.next(); - AtomicLong deliveredMsgsCount = session.getPusher().getDeliveredMsgsCount(); - AtomicLong deliveredFailCount = session.getPusher().getDeliverFailMsgsCount(); - int unAckMsgsCount = session.getPusher().getTotalUnackMsgs(); - int sendTopics = session.getSessionContext().getSendTopics().size(); - int subscribeTopics = session.getSessionContext().getSubscribeTopics().size(); - - TCP_LOGGER.info("session|deliveredFailCount={}|deliveredMsgsCount={}|unAckMsgsCount={}|sendTopics={}|subscribeTopics={}|user={}", - deliveredFailCount.longValue(), deliveredMsgsCount.longValue(), - unAckMsgsCount, sendTopics, subscribeTopics, session.getClient()); - - topicSet.addAll(session.getSessionContext().getSubscribeTopics().keySet()); - } - tcpSummaryMetrics.setSubTopicNum(topicSet.size()); - tcpSummaryMetrics.setAllConnections(eventMeshTCPServer.getTcpConnectionHandler().getConnectionCount()); - printAppLogger(tcpSummaryMetrics); - - }), delay, period, TimeUnit.MILLISECONDS); - - monitorThreadPoolTask = eventMeshTCPServer.getTcpThreadPoolGroup().getScheduler().scheduleAtFixedRate(() -> { - APP_LOGGER.info("{TaskHandle:{},Send:{},Ack:{},Reply:{},Push:{},Scheduler:{},Rebalance:{}}", - eventMeshTCPServer.getTcpThreadPoolGroup().getTaskHandleExecutorService().getQueue().size(), - eventMeshTCPServer.getTcpThreadPoolGroup().getSendExecutorService().getQueue().size(), - eventMeshTCPServer.getTcpThreadPoolGroup().getAckExecutorService().getQueue().size(), - eventMeshTCPServer.getTcpThreadPoolGroup().getReplyExecutorService().getQueue().size(), - eventMeshTCPServer.getTcpThreadPoolGroup().getBroadcastMsgDownstreamExecutorService().getQueue().size(), - ((ThreadPoolExecutor) eventMeshTCPServer.getTcpThreadPoolGroup().getScheduler()).getQueue().size(), - eventMeshTCPServer.getEventMeshRebalanceService().getRebalanceThreadPoolQueueSize()); - - eventMeshTCPServer.getTcpRetryer().printState(); - - // monitor retry queue size - tcpSummaryMetrics.setRetrySize(eventMeshTCPServer.getTcpRetryer().getPendingTimeouts()); - APP_LOGGER.info( - MonitorMetricConstants.EVENTMESH_MONITOR_FORMAT_COMMON, - EventMeshConstants.PROTOCOL_TCP, - MonitorMetricConstants.RETRY_QUEUE_SIZE, - tcpSummaryMetrics.getRetrySize()); - }, 10, PRINT_THREADPOOLSTATE_INTERVAL, TimeUnit.SECONDS); - log.info("EventMeshTcpMonitor started......"); - } - - private void printAppLogger(TcpSummaryMetrics tcpSummaryMetrics) { - APP_LOGGER.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.CLIENT_2_EVENTMESH_TPS, - tcpSummaryMetrics.getClient2eventMeshTPS()); - - APP_LOGGER.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.EVENTMESH_2_MQ_TPS, - tcpSummaryMetrics.getEventMesh2mqTPS()); - - APP_LOGGER.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.MQ_2_EVENTMESH_TPS, - tcpSummaryMetrics.getMq2eventMeshTPS()); - - APP_LOGGER.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.EVENTMESH_2_CLIENT_TPS, - tcpSummaryMetrics.getEventMesh2clientTPS()); - - APP_LOGGER.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.ALL_TPS, - tcpSummaryMetrics.getAllTPS()); - - APP_LOGGER.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.CONNECTION, - tcpSummaryMetrics.getAllConnections()); - - APP_LOGGER.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.SUB_TOPIC_NUM, - tcpSummaryMetrics.getSubTopicNum()); - } - - public TcpSummaryMetrics getTcpSummaryMetrics() { - return tcpSummaryMetrics; - } - - public void shutdown() throws Exception { - monitorTpsTask.cancel(true); - monitorThreadPoolTask.cancel(true); - metricsRegistries.forEach(MetricsRegistry::showdown); - log.info("EventMeshTcpMonitor shutdown......"); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetrics.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetrics.java new file mode 100644 index 0000000000..14ef926b59 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetrics.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics.tcp; + +import org.apache.eventmesh.metrics.api.model.InstrumentFurther; +import org.apache.eventmesh.metrics.api.model.Metric; +import org.apache.eventmesh.metrics.api.model.ObservableDoubleGaugeMetric; +import org.apache.eventmesh.metrics.api.model.ObservableLongGaugeMetric; +import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; +import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; +import org.apache.eventmesh.runtime.metrics.MetricInstrumentUnit; + +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Getter +@Setter +@Slf4j +public class TcpMetrics { + + private static final String TPC_METRICS_NAME_PREFIX = "eventmesh.tcp."; + + private static final String METRIC_NAME = "TCP"; + + private final EventMeshTCPServer eventMeshTCPServer; + + private AtomicLong client2eventMeshMsgNum; + private AtomicLong eventMesh2mqMsgNum; + private AtomicLong mq2eventMeshMsgNum; + private AtomicLong eventMesh2clientMsgNum; + + private volatile double client2eventMeshTPS; + private volatile double eventMesh2clientTPS; + private volatile double eventMesh2mqTPS; + private volatile double mq2eventMeshTPS; + + private int subTopicNum; + + private volatile int allConnections; + + private volatile int retrySize; + //TCP connections + private ObservableLongGaugeMetric allConnectionsGauge; + + private ObservableLongGaugeMetric retrySizeGauge; + + private ObservableLongGaugeMetric subTopicGauge; + + private ObservableDoubleGaugeMetric mq2eventMeshTPSGauge; + + private ObservableDoubleGaugeMetric client2eventMeshTPSGauge; + + private ObservableDoubleGaugeMetric eventMesh2clientTPSGauge; + + private ObservableDoubleGaugeMetric eventMesh2mqTPSGauge; + + private final Map labelMap; + + private final Map metrics = new HashMap<>(32); + + public TcpMetrics(final EventMeshTCPServer eventMeshTCPServer, final Map labelMap) { + this.client2eventMeshMsgNum = new AtomicLong(0); + this.eventMesh2mqMsgNum = new AtomicLong(0); + this.mq2eventMeshMsgNum = new AtomicLong(0); + this.eventMesh2clientMsgNum = new AtomicLong(0); + this.eventMeshTCPServer = eventMeshTCPServer; + this.labelMap = labelMap; + initMetric(); + } + + private void initMetric() { + final Map commonAttributes = new HashMap<>(this.labelMap); + + InstrumentFurther furtherConn = new InstrumentFurther(); + furtherConn.setUnit(MetricInstrumentUnit.SINGLETON); + furtherConn.setDescription("Number of TCP client connects to EventMesh runtime"); + furtherConn.setName(TPC_METRICS_NAME_PREFIX + "connection.num"); + allConnectionsGauge = new ObservableLongGaugeMetric(furtherConn, METRIC_NAME, buildAllConnectionSupplier()); + allConnectionsGauge.putAll(commonAttributes); + metrics.put("allConnections", allConnectionsGauge); + + InstrumentFurther furtherTopic = new InstrumentFurther(); + furtherTopic.setUnit(MetricInstrumentUnit.SINGLETON); + furtherTopic.setDescription("Number of TCP client subscribe for topic"); + furtherTopic.setName(TPC_METRICS_NAME_PREFIX + "sub.topic.num"); + subTopicGauge = new ObservableLongGaugeMetric(furtherTopic, METRIC_NAME, buildSubTopicSupplier()); + subTopicGauge.putAll(commonAttributes); + metrics.put("subTopicGauge", subTopicGauge); + + InstrumentFurther furtherCl2Em = new InstrumentFurther(); + furtherCl2Em.setUnit(MetricInstrumentUnit.TPS); + furtherCl2Em.setDescription("Tps of client to EventMesh."); + furtherCl2Em.setName(TPC_METRICS_NAME_PREFIX + "client.eventmesh.tps"); + client2eventMeshTPSGauge = new ObservableDoubleGaugeMetric(furtherCl2Em, METRIC_NAME, () -> TcpMetrics.this.client2eventMeshTPS); + client2eventMeshTPSGauge.putAll(commonAttributes); + metrics.put("client2eventMeshTPSGauge", client2eventMeshTPSGauge); + + InstrumentFurther furtherEm2Cl = new InstrumentFurther(); + furtherEm2Cl.setUnit(MetricInstrumentUnit.TPS); + furtherEm2Cl.setDescription("Tps of EventMesh to client."); + furtherEm2Cl.setName(TPC_METRICS_NAME_PREFIX + "eventmesh.client.tps"); + eventMesh2clientTPSGauge = new ObservableDoubleGaugeMetric(furtherEm2Cl, METRIC_NAME, () -> TcpMetrics.this.eventMesh2clientTPS); + eventMesh2clientTPSGauge.putAll(commonAttributes); + metrics.put("eventMesh2clientTPSGauge", eventMesh2clientTPSGauge); + + InstrumentFurther furtherEm2Mq = new InstrumentFurther(); + furtherEm2Mq.setUnit(MetricInstrumentUnit.TPS); + furtherEm2Mq.setDescription("Tps of EventMesh to MQ."); + furtherEm2Mq.setName(TPC_METRICS_NAME_PREFIX + "eventmesh.mq.tps"); + eventMesh2mqTPSGauge = new ObservableDoubleGaugeMetric(furtherEm2Mq, METRIC_NAME, () -> TcpMetrics.this.eventMesh2mqTPS); + eventMesh2mqTPSGauge.putAll(commonAttributes); + metrics.put("eventMesh2mqTPSGauge", eventMesh2mqTPSGauge); + + InstrumentFurther furtherMq2Em = new InstrumentFurther(); + furtherMq2Em.setUnit(MetricInstrumentUnit.TPS); + furtherMq2Em.setDescription("Tps of MQ to EventMesh."); + furtherMq2Em.setName(TPC_METRICS_NAME_PREFIX + "mq.eventmesh.tps"); + mq2eventMeshTPSGauge = new ObservableDoubleGaugeMetric(furtherMq2Em, METRIC_NAME, () -> TcpMetrics.this.eventMesh2mqTPS); + mq2eventMeshTPSGauge.putAll(commonAttributes); + metrics.put("mq2eventMeshTPSGauge", mq2eventMeshTPSGauge); + } + + /** + * Count the number of TCP clients connected to EventMesh. + * + * @return Supplier + */ + private Supplier buildAllConnectionSupplier() { + return () -> (long) eventMeshTCPServer.getTcpConnectionHandler().getConnectionCount(); + } + + /** + * Count the number of TCP clients subscribed to a topic. + * + * @return Supplier + */ + private Supplier buildSubTopicSupplier() { + return () -> { + //count topics subscribed by client in this eventMesh + ConcurrentHashMap sessionMap = eventMeshTCPServer.getClientSessionGroupMapping().getSessionMap(); + Iterator sessionIterator = sessionMap.values().iterator(); + Set topicSet = new HashSet<>(); + while (sessionIterator.hasNext()) { + Session session = sessionIterator.next(); + AtomicLong deliveredMsgsCount = session.getPusher().getDeliveredMsgsCount(); + AtomicLong deliveredFailCount = session.getPusher().getDeliverFailMsgsCount(); + int unAckMsgsCount = session.getPusher().getTotalUnackMsgs(); + int sendTopics = session.getSessionContext().getSendTopics().size(); + int subscribeTopics = session.getSessionContext().getSubscribeTopics().size(); + + log.info("session|deliveredFailCount={}|deliveredMsgsCount={}|unAckMsgsCount={}|sendTopics={}|subscribeTopics={}|user={}", + deliveredFailCount.longValue(), deliveredMsgsCount.longValue(), + unAckMsgsCount, sendTopics, subscribeTopics, session.getClient()); + topicSet.addAll(session.getSessionContext().getSubscribeTopics().keySet()); + } + return (long) topicSet.size(); + }; + } + + public Collection getMetrics() { + return metrics.values(); + } + + public long client2eventMeshMsgNum() { + return client2eventMeshMsgNum.get(); + } + + public long eventMesh2mqMsgNum() { + return eventMesh2mqMsgNum.get(); + } + + public long mq2eventMeshMsgNum() { + return mq2eventMeshMsgNum.get(); + } + + public long eventMesh2clientMsgNum() { + return eventMesh2clientMsgNum.get(); + } + + public void resetClient2EventMeshMsgNum() { + this.client2eventMeshMsgNum = new AtomicLong(0); + } + + public void resetEventMesh2mqMsgNum() { + this.eventMesh2mqMsgNum = new AtomicLong(0); + } + + public void resetMq2eventMeshMsgNum() { + this.mq2eventMeshMsgNum = new AtomicLong(0); + } + + public void resetEventMesh2ClientMsgNum() { + this.eventMesh2clientMsgNum = new AtomicLong(0); + } + + + public void setClient2eventMeshTPS(double client2eventMeshTPS) { + this.client2eventMeshTPS = client2eventMeshTPS; + } + + public double getEventMesh2clientTPS() { + return eventMesh2clientTPS; + } + + public void setEventMesh2clientTPS(double eventMesh2clientTPS) { + this.eventMesh2clientTPS = eventMesh2clientTPS; + } + + public double getEventMesh2mqTPS() { + return eventMesh2mqTPS; + } + + public void setEventMesh2mqTPS(double eventMesh2mqTPS) { + this.eventMesh2mqTPS = eventMesh2mqTPS; + } + + public double getMq2eventMeshTPS() { + return mq2eventMeshTPS; + } + + public void setMq2eventMeshTPS(double mq2eventMeshTPS) { + this.mq2eventMeshTPS = mq2eventMeshTPS; + } + + public double getAllTPS() { + return client2eventMeshTPS + eventMesh2clientTPS; + } + + public int getSubTopicNum() { + return subTopicNum; + } + + public void setSubTopicNum(int subTopicNum) { + this.subTopicNum = subTopicNum; + } + + public int getAllConnections() { + return allConnections; + } + + public void setAllConnections(int allConnections) { + this.allConnections = allConnections; + } + + public void setRetrySize(int retrySize) { + this.retrySize = retrySize; + } + + public int getRetrySize() { + return retrySize; + } + +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetricsCalculator.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetricsCalculator.java new file mode 100644 index 0000000000..0281f59bc7 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/metrics/tcp/TcpMetricsCalculator.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.metrics.tcp; + +import org.apache.eventmesh.common.EventMeshThreadFactory; +import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; +import org.apache.eventmesh.runtime.constants.EventMeshConstants; +import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; +import org.apache.eventmesh.runtime.metrics.MonitorMetricConstants; + +import java.math.BigDecimal; +import java.net.InetSocketAddress; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +class TcpMetricsCalculator { + + private static final int period = 30 * 1000; + + private static int PRINT_THREADPOOLSTATE_INTERVAL = 1; + + private final EventMeshTCPServer eventMeshTCPServer; + + private final TcpMetrics tcpMetrics; + + private ScheduledFuture monitorTpsTask; + + private ScheduledExecutorService scheduler; + + public TcpMetricsCalculator(EventMeshTCPServer eventMeshTCPServer, TcpMetrics tcpMetrics) { + this.eventMeshTCPServer = eventMeshTCPServer; + this.tcpMetrics = tcpMetrics; + this.scheduler = ThreadPoolFactory.createScheduledExecutor(2, new EventMeshThreadFactory("EventMesh-TcpMetricsCalculator-scheduler", true)); + } + + public void start() { + int delay = 60 * 1000; + monitorTpsTask = this.scheduler.scheduleAtFixedRate((() -> { + long msgNum = tcpMetrics.client2eventMeshMsgNum(); + tcpMetrics.resetClient2EventMeshMsgNum(); + tcpMetrics.setClient2eventMeshTPS( + new BigDecimal(1000 * msgNum).divide(new BigDecimal(period), 2, BigDecimal.ROUND_HALF_UP).doubleValue()); + + msgNum = tcpMetrics.eventMesh2clientMsgNum(); + tcpMetrics.resetEventMesh2ClientMsgNum(); + tcpMetrics.setEventMesh2clientTPS( + new BigDecimal(1000 * msgNum).divide(new BigDecimal(period), 2, BigDecimal.ROUND_HALF_UP).doubleValue()); + + msgNum = tcpMetrics.eventMesh2mqMsgNum(); + tcpMetrics.resetEventMesh2mqMsgNum(); + tcpMetrics.setEventMesh2mqTPS(new BigDecimal(1000 * msgNum).divide(new BigDecimal(period), 2, BigDecimal.ROUND_HALF_UP).doubleValue()); + + msgNum = tcpMetrics.mq2eventMeshMsgNum(); + tcpMetrics.resetMq2eventMeshMsgNum(); + tcpMetrics.setMq2eventMeshTPS(new BigDecimal(1000 * msgNum).divide(new BigDecimal(period), 2, BigDecimal.ROUND_HALF_UP).doubleValue()); + + //count topics subscribed by client in this eventMesh + ConcurrentHashMap sessionMap = eventMeshTCPServer.getClientSessionGroupMapping().getSessionMap(); + Iterator sessionIterator = sessionMap.values().iterator(); + Set topicSet = new HashSet<>(); + while (sessionIterator.hasNext()) { + Session session = sessionIterator.next(); + AtomicLong deliveredMsgsCount = session.getPusher().getDeliveredMsgsCount(); + AtomicLong deliveredFailCount = session.getPusher().getDeliverFailMsgsCount(); + int unAckMsgsCount = session.getPusher().getTotalUnackMsgs(); + int sendTopics = session.getSessionContext().getSendTopics().size(); + int subscribeTopics = session.getSessionContext().getSubscribeTopics().size(); + + log.info("session|deliveredFailCount={}|deliveredMsgsCount={}|unAckMsgsCount={}|sendTopics={}|subscribeTopics={}|user={}", + deliveredFailCount.longValue(), deliveredMsgsCount.longValue(), + unAckMsgsCount, sendTopics, subscribeTopics, session.getClient()); + topicSet.addAll(session.getSessionContext().getSubscribeTopics().keySet()); + } + tcpMetrics.setSubTopicNum(topicSet.size()); + tcpMetrics.setAllConnections(eventMeshTCPServer.getTcpConnectionHandler().getConnectionCount()); + printAppLogger(tcpMetrics); + }), delay, period, TimeUnit.MILLISECONDS); + } + + private void printAppLogger(TcpMetrics tcpSummaryMetrics) { + + log.info("===========================================TCP SERVER METRICS=================================================="); + + log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.CLIENT_2_EVENTMESH_TPS, + tcpSummaryMetrics.getClient2eventMeshTPS()); + + log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.EVENTMESH_2_MQ_TPS, + tcpSummaryMetrics.getEventMesh2mqTPS()); + + log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.MQ_2_EVENTMESH_TPS, + tcpSummaryMetrics.getMq2eventMeshTPS()); + + log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.EVENTMESH_2_CLIENT_TPS, + tcpSummaryMetrics.getEventMesh2clientTPS()); + + log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.ALL_TPS, + tcpSummaryMetrics.getAllTPS()); + + log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.CONNECTION, + tcpSummaryMetrics.getAllConnectionsGauge()); + + log.info("protocol: {}, s: {}, t: {}", EventMeshConstants.PROTOCOL_TCP, MonitorMetricConstants.SUB_TOPIC_NUM, + tcpSummaryMetrics.getSubTopicNum()); + } + + public void shutdown() { + monitorTpsTask.cancel(true); + } +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/Utils.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/Utils.java index eb9bf9b06a..63f5d232cb 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/Utils.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/util/Utils.java @@ -20,6 +20,7 @@ import org.apache.eventmesh.common.protocol.tcp.EventMeshMessage; import org.apache.eventmesh.common.protocol.tcp.Package; import org.apache.eventmesh.common.protocol.tcp.UserAgent; +import org.apache.eventmesh.common.utils.IPUtils; import org.apache.eventmesh.runtime.constants.EventMeshConstants; import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.Session; import org.apache.eventmesh.runtime.core.protocol.tcp.client.session.SessionState; @@ -72,7 +73,7 @@ public static void writeAndFlush(final Package pkg, long startTime, long taskExe if (session != null) { Objects.requireNonNull(session.getClientGroupWrapper().get()) - .getEventMeshTcpMonitor().getTcpSummaryMetrics().getEventMesh2clientMsgNum().incrementAndGet(); + .getEventMeshTcpMetricsManager().eventMesh2clientMsgNumIncrement(IPUtils.parseChannelRemoteAddr(ctx.channel())); } } }); diff --git a/tools/dependency-check/known-dependencies.txt b/tools/dependency-check/known-dependencies.txt index f6627a693e..05c99f3d37 100644 --- a/tools/dependency-check/known-dependencies.txt +++ b/tools/dependency-check/known-dependencies.txt @@ -117,6 +117,7 @@ jackson-core-2.13.0.jar jackson-databind-2.13.0.jar jackson-dataformat-yaml-2.13.0.jar jackson-datatype-jsr310-2.13.0.jar +jackson-jr-objects-2.16.1.jar jakarta.annotation-api-1.3.5.jar jakarta.validation-api-2.0.2.jar jandex-2.4.2.Final.jar @@ -242,17 +243,24 @@ okhttp-3.14.9.jar okio-1.17.2.jar openapiutil-0.2.1.jar openmessaging-api-2.2.1-pubsub.jar -opentelemetry-api-1.3.0.jar -opentelemetry-api-metrics-1.3.0-alpha.jar -opentelemetry-context-1.3.0.jar -opentelemetry-exporter-jaeger-1.4.0.jar -opentelemetry-exporter-prometheus-1.3.0-alpha.jar -opentelemetry-exporter-zipkin-1.3.0.jar -opentelemetry-sdk-1.3.0.jar -opentelemetry-sdk-common-1.3.0.jar -opentelemetry-sdk-metrics-1.3.0-alpha.jar -opentelemetry-sdk-trace-1.3.0.jar -opentelemetry-semconv-1.3.0-alpha.jar +opentelemetry-api-1.36.0.jar +opentelemetry-api-events-1.36.0-alpha.jar +opentelemetry-context-1.36.0.jar +opentelemetry-exporter-common-1.34.1.jar +opentelemetry-exporter-common-1.36.0.jar +opentelemetry-exporter-jaeger-1.34.1.jar +opentelemetry-exporter-prometheus-1.36.0-alpha.jar +opentelemetry-exporter-sender-okhttp-1.34.1.jar +opentelemetry-exporter-zipkin-1.36.0.jar +opentelemetry-extension-incubator-1.36.0-alpha.jar +opentelemetry-sdk-1.36.0.jar +opentelemetry-sdk-common-1.36.0.jar +opentelemetry-sdk-extension-autoconfigure-spi-1.34.1.jar +opentelemetry-sdk-extension-autoconfigure-spi-1.36.0.jar +opentelemetry-sdk-logs-1.36.0.jar +opentelemetry-sdk-metrics-1.36.0.jar +opentelemetry-sdk-trace-1.36.0.jar +opentelemetry-semconv-1.30.1-alpha.jar org.abego.treelayout.core-1.0.3.jar org.jacoco.agent-0.8.4-runtime.jar perfmark-api-0.23.0.jar @@ -272,12 +280,17 @@ pravega-shared-controller-api-0.11.0.jar pravega-shared-protocol-0.11.0.jar pravega-shared-security-0.11.0.jar profiles-2.20.29.jar +prometheus-metrics-config-1.1.0.jar +prometheus-metrics-exporter-common-1.1.0.jar +prometheus-metrics-exporter-httpserver-1.1.0.jar +prometheus-metrics-exposition-formats-1.1.0.jar +prometheus-metrics-model-1.1.0.jar +prometheus-metrics-shaded-protobuf-1.1.0.jar proto-google-common-protos-2.0.1.jar protobuf-java-3.19.2.jar protobuf-java-3.21.5.jar protobuf-java-3.21.9.jar protobuf-java-util-3.15.0.jar -protobuf-java-util-3.17.2.jar protobuf-java-util-3.21.5.jar protobuf-java-util-3.5.1.jar protocol-core-2.20.29.jar @@ -307,9 +320,10 @@ rocketmq-tools-4.9.5.jar rxjava-3.0.12.jar s3-2.20.29.jar sdk-core-2.20.29.jar -simpleclient-0.8.1.jar -simpleclient_common-0.8.1.jar -simpleclient_httpserver-0.8.1.jar +simpleclient-0.12.0.jar +simpleclient_tracer_common-0.12.0.jar +simpleclient_tracer_otel-0.12.0.jar +simpleclient_tracer_otel_agent-0.12.0.jar slack-api-client-1.1.6.jar slack-api-model-1.1.6.jar slack-app-backend-1.1.6.jar @@ -346,9 +360,9 @@ vertx-web-client-4.0.0.jar vertx-web-common-4.4.6.jar xpp3-1.1.4c.jar xsdlib-2013.6.1.jar -zipkin-2.23.2.jar -zipkin-reporter-2.16.3.jar -zipkin-sender-okhttp3-2.16.3.jar +zipkin-2.27.1.jar +zipkin-reporter-3.3.0.jar +zipkin-sender-okhttp3-3.3.0.jar zookeeper-3.7.1.jar zookeeper-jute-3.7.1.jar zstd-jni-1.5.0-2.jar From fdb96d83e6960d594ae08c9459cb4c06bef5135b Mon Sep 17 00:00:00 2001 From: Jevin Jiang <40829263+jevinjiang@users.noreply.github.com> Date: Sun, 21 Apr 2024 08:27:14 +0800 Subject: [PATCH 3/4] [ISSUE #4047] Support chatGPT source connector (#4817) * [ISSUE #4047] Support chatGPT source connector * [ISSUE #4047] Add OpenAI configuration and adjust DTO * [ISSUE #4047] Join parse request support * [ISSUE #4047] impl Parse request * [ISSUE #4047] fix code style * [ISSUE #4047] fix code style * [ISSUE #4047] fix dependencies check failed * [ISSUE #4047] fix dependencies check * [ISSUE #4047] fix license check * [ISSUE #4047] fix review question * [ISSUE #4047] fix review question * [ISSUE #4047] add default value * [ISSUE #4047] fix test * [ISSUE #4047] default timeout value is zero , not timeout . * [ISSUE #4047] fix review * [ISSUE #4047] fix license check --------- Co-authored-by: JiangShuJu --- .../eventmesh-connector-chatgpt/build.gradle | 28 ++ .../gradle.properties | 18 ++ .../chatgpt/config/ChatGPTServerConfig.java | 32 +++ .../chatgpt/server/ChatGPTConnectServer.java | 40 +++ .../source/config/ChatGPTSourceConfig.java | 35 +++ .../config/ChatGPTSourceConnectorConfig.java | 37 +++ .../chatgpt/source/config/OpenaiConfig.java | 42 +++ .../source/config/OpenaiProxyConfig.java | 29 ++ .../connector/ChatGPTSourceConnector.java | 252 ++++++++++++++++++ .../chatgpt/source/dto/ChatGPTRequestDTO.java | 61 +++++ .../source/enums/ChatGPTRequestType.java | 25 ++ .../chatgpt/source/handlers/ChatHandler.java | 69 +++++ .../chatgpt/source/handlers/ParseHandler.java | 134 ++++++++++ .../source/managers/OpenaiManager.java | 131 +++++++++ .../src/main/resources/prompt | 44 +++ .../src/main/resources/server-config.yml | 19 ++ .../src/main/resources/source-config.yml | 51 ++++ .../connector/ChatGPTSourceConnectorTest.java | 166 ++++++++++++ .../src/test/resources/server-config.yml | 19 ++ .../src/test/resources/source-config.yml | 52 ++++ settings.gradle | 1 + tools/dependency-check/known-dependencies.txt | 12 + 22 files changed, 1297 insertions(+) create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/build.gradle create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/gradle.properties create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/config/ChatGPTServerConfig.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/server/ChatGPTConnectServer.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConfig.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConnectorConfig.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiConfig.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiProxyConfig.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnector.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/dto/ChatGPTRequestDTO.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/enums/ChatGPTRequestType.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ChatHandler.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ParseHandler.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/managers/OpenaiManager.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/prompt create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/server-config.yml create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/source-config.yml create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/test/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnectorTest.java create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/server-config.yml create mode 100644 eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/source-config.yml diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/build.gradle b/eventmesh-connectors/eventmesh-connector-chatgpt/build.gradle new file mode 100644 index 0000000000..b2c45b620f --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/build.gradle @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + api project(":eventmesh-openconnect:eventmesh-openconnect-java") + implementation project(":eventmesh-common") + implementation 'com.theokanning.openai-gpt3-java:service:0.18.2' + implementation 'io.cloudevents:cloudevents-http-vertx:2.3.0' + implementation 'io.vertx:vertx-web:4.4.6' + + testImplementation "org.apache.httpcomponents:httpclient" + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/gradle.properties b/eventmesh-connectors/eventmesh-connector-chatgpt/gradle.properties new file mode 100644 index 0000000000..715bad3de4 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/gradle.properties @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +pluginType=connector +pluginName=chatgpt \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/config/ChatGPTServerConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/config/ChatGPTServerConfig.java new file mode 100644 index 0000000000..7d162920d7 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/config/ChatGPTServerConfig.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.config; + +import org.apache.eventmesh.openconnect.api.config.Config; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class ChatGPTServerConfig extends Config { + + private boolean sourceEnable; + + private boolean sinkEnable; +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/server/ChatGPTConnectServer.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/server/ChatGPTConnectServer.java new file mode 100644 index 0000000000..ca104fe562 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/server/ChatGPTConnectServer.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.server; + +import org.apache.eventmesh.connector.chatgpt.config.ChatGPTServerConfig; +import org.apache.eventmesh.connector.chatgpt.source.connector.ChatGPTSourceConnector; +import org.apache.eventmesh.openconnect.Application; +import org.apache.eventmesh.openconnect.util.ConfigUtil; + +public class ChatGPTConnectServer { + + public static void main(String[] args) throws Exception { + ChatGPTServerConfig serverConfig = ConfigUtil.parse(ChatGPTServerConfig.class, "server-config.yml"); + + if (serverConfig.isSourceEnable()) { + Application chatGPTSourceApp = new Application(); + chatGPTSourceApp.run(ChatGPTSourceConnector.class); + } + + if (serverConfig.isSinkEnable()) { + // TODO support sink connector + } + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConfig.java new file mode 100644 index 0000000000..9596866910 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConfig.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.source.config; + +import org.apache.eventmesh.openconnect.api.config.SourceConfig; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class ChatGPTSourceConfig extends SourceConfig { + + public ChatGPTSourceConnectorConfig connectorConfig; + + public OpenaiProxyConfig openaiProxyConfig; + + public OpenaiConfig openaiConfig; + +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConnectorConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConnectorConfig.java new file mode 100644 index 0000000000..316fb5f241 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConnectorConfig.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.source.config; + +import lombok.Data; + +@Data +public class ChatGPTSourceConnectorConfig { + + private String connectorName = "chatgptSource"; + + private String path = "/chatgpt"; + + private int port = 3756; + + private int idleTimeout; + + private boolean proxyEnable = false; + + private String parsePromptFileName = "prompt"; + +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiConfig.java new file mode 100644 index 0000000000..51858a709a --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiConfig.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.source.config; + + +import java.util.List; +import java.util.Map; + +import lombok.Data; + +@Data +public class OpenaiConfig { + + private String token; + private String model = "gpt-3.5-turbo"; + private long timeout; + private Double temperature; + private Integer maxTokens; + private Boolean logprob; + private Double topLogprobs; + private Map logitBias; + private Double frequencyPenalty; + private Double presencePenalty; + private String user = "eventMesh"; + private List stop; + +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiProxyConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiProxyConfig.java new file mode 100644 index 0000000000..14dd69f350 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/OpenaiProxyConfig.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.source.config; + +import lombok.Data; + +@Data +public class OpenaiProxyConfig { + + private String host; + + private int port; + +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnector.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnector.java new file mode 100644 index 0000000000..a947bc135d --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnector.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.source.connector; + +import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.connector.chatgpt.source.config.ChatGPTSourceConfig; +import org.apache.eventmesh.connector.chatgpt.source.dto.ChatGPTRequestDTO; +import org.apache.eventmesh.connector.chatgpt.source.enums.ChatGPTRequestType; +import org.apache.eventmesh.connector.chatgpt.source.handlers.ChatHandler; +import org.apache.eventmesh.connector.chatgpt.source.handlers.ParseHandler; +import org.apache.eventmesh.connector.chatgpt.source.managers.OpenaiManager; +import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; +import org.apache.eventmesh.openconnect.api.source.Source; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.util.CloudEventUtil; + +import org.apache.commons.lang3.StringUtils; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import io.cloudevents.CloudEvent; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.http.HttpServer; +import io.vertx.core.http.HttpServerOptions; +import io.vertx.ext.web.RequestBody; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import io.vertx.ext.web.handler.BodyHandler; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ChatGPTSourceConnector implements Source { + + private static final int DEFAULT_BATCH_SIZE = 10; + + private ChatGPTSourceConfig sourceConfig; + private BlockingQueue queue; + private HttpServer server; + private final ExecutorService chatgptSourceExecutorService = + ThreadPoolFactory.createThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 2, Runtime.getRuntime().availableProcessors() * 2, + "ChatGPTSourceThread"); + + private OpenaiManager openaiManager; + private String parsePromptTemplateStr; + private ChatHandler chatHandler; + private ParseHandler parseHandler; + private static final int DEFAULT_TIMEOUT = 0; + + private static final String APPLICATION_JSON = "application/json"; + private static final String TEXT_PLAIN = "text/plain"; + + + @Override + public Class configClass() { + return ChatGPTSourceConfig.class; + } + + @Override + public void init(Config config) { + this.sourceConfig = (ChatGPTSourceConfig) config; + doInit(); + } + + @Override + public void init(ConnectorContext connectorContext) { + SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; + this.sourceConfig = (ChatGPTSourceConfig) sourceConnectorContext.getSourceConfig(); + doInit(); + } + + public void initParsePrompt() { + String parsePromptFileName = sourceConfig.getConnectorConfig().getParsePromptFileName(); + URL resource = Thread.currentThread().getContextClassLoader().getResource(parsePromptFileName); + if (resource == null) { + log.warn("cannot find prompt file {} in resources", parsePromptFileName); + return; + } + String filePath = resource.getPath(); + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + StringBuilder builder = new StringBuilder(); + String line; + while ((line = br.readLine()) != null) { + if (!line.startsWith("#") && StringUtils.isNotBlank(line)) { + builder.append(line).append("\n"); + } + } + this.parsePromptTemplateStr = builder.toString(); + } catch (IOException e) { + throw new IllegalStateException("Unable to read file", e); + } + } + + + @SuppressWarnings("checkstyle:WhitespaceAround") + private void doInit() { + initParsePrompt(); + this.openaiManager = new OpenaiManager(sourceConfig); + this.chatHandler = new ChatHandler(this.openaiManager); + if (StringUtils.isNotEmpty(parsePromptTemplateStr)) { + this.parseHandler = new ParseHandler(openaiManager, parsePromptTemplateStr); + } + this.queue = new LinkedBlockingQueue<>(1024); + final Vertx vertx = Vertx.vertx(); + final Router router = Router.router(vertx); + router.route().path(this.sourceConfig.connectorConfig.getPath()).method(HttpMethod.POST).handler(BodyHandler.create()).handler(ctx -> { + try { + RequestBody body = ctx.body(); + ChatGPTRequestDTO bodyObject = body.asPojo(ChatGPTRequestDTO.class); + validateRequestDTO(bodyObject); + handleRequest(bodyObject, ctx); + } catch (Exception e) { + handleError(e, ctx); + } + }); + if (sourceConfig.connectorConfig.getIdleTimeout() < 0) { + log.warn("idleTimeout must be >= 0, your config value is {}, idleTimeout will be reset {}", sourceConfig.connectorConfig.getIdleTimeout(), + DEFAULT_TIMEOUT); + sourceConfig.connectorConfig.setIdleTimeout(DEFAULT_TIMEOUT); + } + this.server = vertx.createHttpServer(new HttpServerOptions().setPort(this.sourceConfig.connectorConfig.getPort()) + .setIdleTimeout(this.sourceConfig.connectorConfig.getIdleTimeout())).requestHandler(router); + } + + + private void validateRequestDTO(ChatGPTRequestDTO bodyObject) { + if (StringUtils.isBlank(bodyObject.getText())) { + throw new IllegalArgumentException("Attributes 'text' cannot be null"); + } + } + + private void handleRequest(ChatGPTRequestDTO bodyObject, RoutingContext ctx) { + chatgptSourceExecutorService.execute(() -> { + try { + ChatGPTRequestType chatgptRequestType = ChatGPTRequestType.valueOf(bodyObject.getRequestType()); + CloudEvent cloudEvent = invokeHandler(chatgptRequestType, bodyObject); + queue.add(cloudEvent); + log.info("[ChatGPTSourceConnector] Succeed to convert payload into CloudEvent."); + ctx.response().setStatusCode(HttpResponseStatus.OK.code()).end(); + } catch (IllegalArgumentException e) { + log.error("[ChatGPTSourceConnector] the request type is illegal: {}", e.getMessage(), e); + ctx.response().setStatusCode(HttpResponseStatus.BAD_REQUEST.code()) + .setStatusMessage(String.format("request type '%s' is not supported", bodyObject.getRequestType())).end(); + } catch (Exception e) { + log.error("[ChatGPTSourceConnector] Error processing request: {}", e.getMessage(), e); + ctx.response().setStatusCode(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()).end(); + } + }); + } + + private CloudEvent invokeHandler(ChatGPTRequestType chatgptRequestType, ChatGPTRequestDTO bodyObject) { + switch (chatgptRequestType) { + case CHAT: + if (StringUtils.isBlank(bodyObject.getDataContentType())) { + bodyObject.setDataContentType(TEXT_PLAIN); + } + return chatHandler.invoke(bodyObject); + case PARSE: + if (StringUtils.isBlank(parsePromptTemplateStr)) { + throw new IllegalStateException( + "the request type of PARSE must be configured with the correct parsePromptFileName in source-config.yml"); + } + if (StringUtils.isBlank(bodyObject.getFields())) { + throw new IllegalStateException("Attributes 'fields' cannot be null in PARSE"); + } + if (StringUtils.isBlank(bodyObject.getDataContentType())) { + bodyObject.setDataContentType(APPLICATION_JSON); + } + return parseHandler.invoke(bodyObject); + default: + throw new IllegalStateException("the request type is illegal"); + } + } + + private void handleError(Exception e, RoutingContext ctx) { + log.error("[ChatGPTSourceConnector] Malformed request.", e); + ctx.response().setStatusCode(HttpResponseStatus.BAD_REQUEST.code()).end(); + } + + @Override + public void start() { + Throwable t = this.server.listen().cause(); + if (t != null) { + throw new EventMeshException("failed to start Vertx server", t); + } + } + + @Override + public void commit(ConnectRecord record) { + + } + + @Override + public String name() { + return this.sourceConfig.getConnectorConfig().getConnectorName(); + } + + @Override + public void stop() { + Throwable t = this.server.close().cause(); + if (t != null) { + throw new EventMeshException("failed to stop Vertx server", t); + } + } + + @Override + public List poll() { + List connectRecords = new ArrayList<>(DEFAULT_BATCH_SIZE); + for (int i = 0; i < DEFAULT_BATCH_SIZE; i++) { + try { + CloudEvent event = queue.poll(3, TimeUnit.SECONDS); + if (event == null) { + break; + } + connectRecords.add(CloudEventUtil.convertEventToRecord(event)); + } catch (InterruptedException e) { + break; + } + } + return connectRecords; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/dto/ChatGPTRequestDTO.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/dto/ChatGPTRequestDTO.java new file mode 100644 index 0000000000..a203a24e5a --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/dto/ChatGPTRequestDTO.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.source.dto; + +import org.apache.eventmesh.connector.chatgpt.source.enums.ChatGPTRequestType; + +import java.time.ZonedDateTime; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ChatGPTRequestDTO { + + private String requestType = ChatGPTRequestType.CHAT.name(); + + private String source = "/"; + + private String subject = "chatGPT"; + + @JsonProperty("datacontenttype") + private String dataContentType; + + private String type = "cloudevents"; + + private String text; + + private String fields; + + @JsonInclude + private String id = UUID.randomUUID().toString(); + + @JsonInclude + private String time = ZonedDateTime.now().toOffsetDateTime().toString(); + + public String getFields() { + return fields.replace(";", "\n"); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/enums/ChatGPTRequestType.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/enums/ChatGPTRequestType.java new file mode 100644 index 0000000000..9930525651 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/enums/ChatGPTRequestType.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.source.enums; + + +public enum ChatGPTRequestType { + + CHAT, PARSE; + +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ChatHandler.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ChatHandler.java new file mode 100644 index 0000000000..6d79a0559f --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ChatHandler.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.source.handlers; + + +import org.apache.eventmesh.connector.chatgpt.source.dto.ChatGPTRequestDTO; +import org.apache.eventmesh.connector.chatgpt.source.managers.OpenaiManager; + +import java.net.URI; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import io.cloudevents.CloudEvent; +import io.cloudevents.core.builder.CloudEventBuilder; + +import com.theokanning.openai.completion.chat.ChatCompletionRequest; +import com.theokanning.openai.completion.chat.ChatMessage; +import com.theokanning.openai.completion.chat.ChatMessageRole; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ChatHandler { + + private final OpenaiManager openaiManager; + + public ChatHandler(OpenaiManager openaiManager) { + this.openaiManager = openaiManager; + } + + public CloudEvent invoke(ChatGPTRequestDTO event) { + return genGptConnectRecord(event); + } + + private CloudEvent genGptConnectRecord(ChatGPTRequestDTO event) { + List chatMessages = new ArrayList<>(); + chatMessages.add(new ChatMessage(ChatMessageRole.USER.value(), event.getText())); + ChatCompletionRequest req = openaiManager.newChatCompletionRequest(chatMessages); + String chatResult = openaiManager.getResult(req); + + return CloudEventBuilder.v1() + .withId(UUID.randomUUID().toString()) + .withSource(URI.create(event.getSource())) + .withType(event.getType()) + .withTime(ZonedDateTime.now().toOffsetDateTime()) + .withData(chatResult.getBytes()) + .withSubject(event.getSubject()) + .withDataContentType(event.getDataContentType()) + .build(); + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ParseHandler.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ParseHandler.java new file mode 100644 index 0000000000..aad3d384cf --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/handlers/ParseHandler.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.source.handlers; + + +import org.apache.eventmesh.common.Constants; +import org.apache.eventmesh.connector.chatgpt.source.dto.ChatGPTRequestDTO; +import org.apache.eventmesh.connector.chatgpt.source.managers.OpenaiManager; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.text.StringSubstitutor; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import io.cloudevents.CloudEvent; +import io.cloudevents.jackson.JsonFormat; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.theokanning.openai.completion.chat.ChatCompletionRequest; +import com.theokanning.openai.completion.chat.ChatMessage; +import com.theokanning.openai.completion.chat.ChatMessageRole; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ParseHandler { + + private final OpenaiManager openaiManager; + + private final String promptTemplate; + + private static final JsonFormat jsonFormat = new JsonFormat(false, true); + + + public ParseHandler(OpenaiManager openaiManager, String promptTemplate) { + this.openaiManager = openaiManager; + this.promptTemplate = promptTemplate; + } + + @SuppressWarnings("checkstyle:WhitespaceAfter") + public CloudEvent invoke(ChatGPTRequestDTO event) { + Map map = convertToMap(event); + + StringSubstitutor substitute = new StringSubstitutor(map); + String finalPrompt = substitute.replace(promptTemplate); + List chatMessages = new ArrayList<>(); + chatMessages.add(new ChatMessage(ChatMessageRole.USER.value(), finalPrompt)); + ChatCompletionRequest req = openaiManager.newChatCompletionRequest(chatMessages); + String chatResult = openaiManager.getResult(req); + chatResult = StringUtils.removeFirst(chatResult, "```json"); + chatResult = StringUtils.removeEnd(chatResult, "```"); + CloudEvent cloudEvent; + try { + cloudEvent = jsonFormat.deserialize(chatResult.getBytes(Constants.DEFAULT_CHARSET)); + } catch (Exception e) { + throw new IllegalStateException("cloudEvent parse fail, please check your parse prompt file content", e); + } + return cloudEvent; + } + + public Map convertToMap(Object obj) { + Map map = new HashMap<>(); + Class clazz = obj.getClass(); + Field[] fields = clazz.getDeclaredFields(); + for (Field field : fields) { + if (field.isSynthetic()) { + continue; + } + if (Map.class.isAssignableFrom(field.getType()) || List.class.isAssignableFrom(field.getType())) { + continue; + } + try { + String key = field.getName(); + if (field.isAnnotationPresent(JsonProperty.class)) { + JsonProperty annotation = field.getAnnotation(JsonProperty.class); + key = annotation.value(); + } + Method getter = getGetter(field, clazz); + map.put(key, String.valueOf(getter.invoke(obj))); + } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { + throw new IllegalStateException("convert to Map is fail", e); + } + } + + return map; + } + + public Method getGetter(Field field, Class clazz) throws NoSuchMethodException { + boolean isBooleanField = boolean.class.isAssignableFrom(field.getType()) || Boolean.class.isAssignableFrom(field.getType()); + String handledFieldName = upperFirst(field.getName()); + String methodName; + if (isBooleanField) { + methodName = "is" + handledFieldName; + } else { + methodName = "get" + handledFieldName; + } + return clazz.getDeclaredMethod(methodName); + } + + public String upperFirst(String str) { + if (null == str) { + return null; + } + if (!str.isEmpty()) { + char firstChar = str.charAt(0); + if (Character.isLowerCase(firstChar)) { + return Character.toUpperCase(firstChar) + StringUtils.substring(str, 1); + } + } + return str; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/managers/OpenaiManager.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/managers/OpenaiManager.java new file mode 100644 index 0000000000..fda5216bbf --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/managers/OpenaiManager.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.source.managers; + +import static com.theokanning.openai.service.OpenAiService.defaultClient; +import static com.theokanning.openai.service.OpenAiService.defaultObjectMapper; +import static com.theokanning.openai.service.OpenAiService.defaultRetrofit; + +import org.apache.eventmesh.common.utils.AssertUtils; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.chatgpt.source.config.ChatGPTSourceConfig; +import org.apache.eventmesh.connector.chatgpt.source.config.OpenaiConfig; +import org.apache.eventmesh.connector.chatgpt.source.config.OpenaiProxyConfig; + +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.time.Duration; +import java.util.List; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.theokanning.openai.client.OpenAiApi; +import com.theokanning.openai.completion.chat.ChatCompletionRequest; +import com.theokanning.openai.completion.chat.ChatCompletionRequest.ChatCompletionRequestBuilder; +import com.theokanning.openai.completion.chat.ChatMessage; +import com.theokanning.openai.service.OpenAiService; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import okhttp3.OkHttpClient; +import retrofit2.Retrofit; + + +@Slf4j +public class OpenaiManager { + + @Getter + private OpenAiService openAiService; + + private String chatCompletionRequestTemplateStr; + + private static final int DEFAULT_TIMEOUT = 0; + + public OpenaiManager(ChatGPTSourceConfig sourceConfig) { + initOpenAi(sourceConfig); + } + + public String getResult(ChatCompletionRequest req) { + StringBuilder gptData = new StringBuilder(); + try { + openAiService.createChatCompletion(req).getChoices() + .forEach(chatCompletionChoice -> gptData.append(chatCompletionChoice.getMessage().getContent())); + } catch (Exception e) { + log.error("Failed to generate GPT connection record: {}", e.getMessage()); + } + return gptData.toString(); + } + + public ChatCompletionRequest newChatCompletionRequest(List chatMessages) { + ChatCompletionRequest request = JsonUtils.parseObject(chatCompletionRequestTemplateStr, ChatCompletionRequest.class); + request.setMessages(chatMessages); + return request; + } + + private void initOpenAi(ChatGPTSourceConfig sourceConfig) { + OpenaiConfig openaiConfig = sourceConfig.getOpenaiConfig(); + if (openaiConfig.getTimeout() < 0) { + log.warn("openaiTimeout must be >= 0, your config value is {}, openaiTimeout will be reset {}", openaiConfig.getTimeout(), + DEFAULT_TIMEOUT); + openaiConfig.setTimeout(DEFAULT_TIMEOUT); + } + boolean proxyEnable = sourceConfig.connectorConfig.isProxyEnable(); + if (proxyEnable) { + OpenaiProxyConfig chatgptProxyConfig = sourceConfig.openaiProxyConfig; + if (chatgptProxyConfig.getHost() == null) { + throw new IllegalStateException("chatgpt proxy config 'host' cannot be null"); + } + ObjectMapper mapper = defaultObjectMapper(); + Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(chatgptProxyConfig.getHost(), chatgptProxyConfig.getPort())); + OkHttpClient client = + defaultClient(openaiConfig.getToken(), Duration.ofSeconds(openaiConfig.getTimeout())).newBuilder().proxy(proxy).build(); + Retrofit retrofit = defaultRetrofit(client, mapper); + OpenAiApi api = retrofit.create(OpenAiApi.class); + this.openAiService = new OpenAiService(api); + } else { + this.openAiService = new OpenAiService(openaiConfig.getToken(), Duration.ofSeconds(openaiConfig.getTimeout())); + } + ChatCompletionRequestBuilder builder = ChatCompletionRequest.builder().model(openaiConfig.getModel()); + AssertUtils.notNull(openaiConfig.getModel(), "model cannot be null"); + builder = builder.model(openaiConfig.getModel()); + if (openaiConfig.getUser() != null) { + builder = builder.user(openaiConfig.getUser()); + } + if (openaiConfig.getPresencePenalty() != null) { + builder = builder.presencePenalty(openaiConfig.getPresencePenalty()); + } + if (openaiConfig.getFrequencyPenalty() != null) { + builder = builder.frequencyPenalty(openaiConfig.getFrequencyPenalty()); + } + if (openaiConfig.getMaxTokens() != null) { + builder = builder.maxTokens(openaiConfig.getMaxTokens()); + } + if (openaiConfig.getTemperature() != null) { + builder = builder.temperature(openaiConfig.getTemperature()); + } + if (openaiConfig.getLogitBias() != null && !openaiConfig.getLogitBias().isEmpty()) { + builder = builder.logitBias(openaiConfig.getLogitBias()); + } + if (openaiConfig.getStop() != null && !openaiConfig.getStop().isEmpty()) { + builder = builder.stop(openaiConfig.getStop()); + } + this.chatCompletionRequestTemplateStr = JsonUtils.toJSONString(builder.build()); + } + + +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/prompt b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/prompt new file mode 100644 index 0000000000..e10ecc331d --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/prompt @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +You are an AI assistant named CloudEventsConverter. avoid escape characters . +Your task is to construct a JSON object in CloudEvents format. Based on the field name and field description in the 'data' field of the CloudEvents formatted JSON object, convert the input text provided by the user into the content of the 'data' field, which must comply with the specifications of the content of the 'datacontenttype' field. +The role is : + - If the 'datacontenttype' field content is 'application/json', then the' data 'field content should be a JSON object, + - else If the 'datacontenttype' field content is not 'application/json' and is 'application/xml', then the' data 'field content should be a string in XML format and the outermost of XML format is , inside is the XML generated by you based on field info; + - else the 'datacontenttype' field content is not 'application/json' and 'application/xml', then the' data 'field content is string of the 'text' field content; +Except for the content of the data field, all other values should be set to and cannot be modified. Finally, return to me the JSON object in CloudEvents format that you constructed + +The following text is the field name and field description in the 'data' field of the CloudEvents-formatted JSON object, extract the following information: + +${fields} + + +text: ${text} + +The output should be a markdown code snippet formatted in the following schema, including the leading and trailing "```json" and "```": +```json +{ + "specversion": string, Set to "1.0" + "type": string, Set to ${type} + "source": string, Set to ${source} + "subject": string, Set to ${subject} + "id": string, Set to ${id} + "time": string, Set to ${time} + "datacontenttype": string, Set to ${datacontenttype} + "data": object or string +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/server-config.yml new file mode 100644 index 0000000000..0cd7b5b5ab --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/server-config.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +sourceEnable: true +sinkEnable: false diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/source-config.yml new file mode 100644 index 0000000000..b194e99ecb --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/resources/source-config.yml @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pubSubConfig: + meshAddress: 127.0.0.1:10000 + subject: TopicTest + idc: FT + env: PRD + group: chatgptSource + appId: 5032 + userName: chatgptSourceUser + passWord: chatgptPassWord +connectorConfig: + connectorName: chatgptSource + path: /chatgpt + port: 3756 + idleTimeout: 0 + proxyEnable: false + parsePromptFileName: prompt + +# https://platform.openai.com/docs/api-reference/chat/create +openaiConfig: + token: + model: gpt-3.5-turbo + timeout: 0 + temperature: 1 + maxTokens: + frequencyPenalty: 0 + presencePenalty: 0 + user: eventMesh + stop: [] + logitBias: {} + +openaiProxyConfig: + host: 127.0.0.1 + port: 7890 + diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnectorTest.java new file mode 100644 index 0000000000..8347fdcbb1 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnectorTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.chatgpt.source.connector; + +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.chatgpt.source.config.ChatGPTSourceConfig; +import org.apache.eventmesh.connector.chatgpt.source.config.ChatGPTSourceConnectorConfig; +import org.apache.eventmesh.connector.chatgpt.source.config.OpenaiConfig; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.util.ConfigUtil; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; + +import java.util.List; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +class ChatGPTSourceConnectorTest { + + private static final Logger LOGGER = LoggerFactory.getLogger("ChatGPTSourceConnectorTest"); + + private ChatGPTSourceConnector connector; + private ChatGPTSourceConnectorConfig config; + private CloseableHttpClient httpClient; + private String uri; + private final String expectedMessage = "Hello, can you tell me a story."; + + private final String expectedParseMessage = "User 13356288979 from Tianjin store placed an order with order number 11221122"; + + + public boolean checkOpenAi() throws Exception { + ChatGPTSourceConfig sourceConfig = (ChatGPTSourceConfig) ConfigUtil.parse(connector.configClass()); + OpenaiConfig openaiConfig = sourceConfig.getOpenaiConfig(); + if (StringUtils.isBlank(openaiConfig.getToken())) { + return false; + } + return true; + } + + @BeforeEach + void setUp() throws Exception { + connector = new ChatGPTSourceConnector(); + if (!checkOpenAi()) { + LOGGER.error("please set openai token in the config"); + return; + } + ChatGPTSourceConfig sourceConfig = (ChatGPTSourceConfig) ConfigUtil.parse(connector.configClass()); + config = sourceConfig.getConnectorConfig(); + connector.init(sourceConfig); + connector.start(); + + uri = new URIBuilder().setScheme("http").setHost("127.0.0.1").setPort(config.getPort()).setPath(config.getPath()).build().toString(); + + httpClient = HttpClients.createDefault(); + } + + @Test + void testPoll() throws Exception { + ChatGPTSourceConfig sourceConfig = (ChatGPTSourceConfig) ConfigUtil.parse(connector.configClass()); + OpenaiConfig openaiConfig = sourceConfig.getOpenaiConfig(); + if (StringUtils.isBlank(openaiConfig.getToken())) { + LOGGER.error("please set openai token in the config"); + return; + } + + final int batchSize = 10; + + for (int i = 0; i < batchSize; i++) { + HttpResponse resp = mockStructuredChatRequest(); + Assertions.assertEquals(resp.getStatusLine().getStatusCode(), HttpStatus.SC_OK); + } + + List res = connector.poll(); + Assertions.assertEquals(batchSize, res.size()); + + for (int i = 0; i < batchSize; i++) { + HttpResponse resp = mockStructuredParseRequest(); + Assertions.assertEquals(resp.getStatusLine().getStatusCode(), HttpStatus.SC_OK); + } + + List res1 = connector.poll(); + Assertions.assertEquals(batchSize, res1.size()); + } + + + HttpResponse mockStructuredChatRequest() throws Exception { + TestEvent event = new TestEvent(); + event.type = "com.example.someevent"; + event.source = "/mycontext"; + event.subject = "test"; + event.datacontenttype = "text/plain"; + event.text = expectedMessage; + event.requestType = "CHAT"; + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(JsonUtils.toJSONString(event))); + + return httpClient.execute(httpPost); + } + + + HttpResponse mockStructuredParseRequest() throws Exception { + TestEvent event = new TestEvent(); + event.type = "com.example.someevent"; + event.source = "/mycontext"; + event.subject = "test"; + event.datacontenttype = "application/json"; + event.text = expectedParseMessage; + event.requestType = "PARSE"; + event.fields = "orderNo:this is order number;address:this is a address;phone:this is phone number"; + HttpPost httpPost = new HttpPost(uri); + httpPost.setEntity(new StringEntity(JsonUtils.toJSONString(event))); + return httpClient.execute(httpPost); + } + + @AfterEach + void tearDown() throws Exception { + if (!checkOpenAi()) { + return; + } + if (connector != null) { + connector.stop(); + } + if (httpClient != null) { + httpClient.close(); + } + } + + class TestEvent { + + public String requestType; + public String type; + public String source; + public String subject; + public String datacontenttype; + public String text; + public String fields; + } +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/server-config.yml new file mode 100644 index 0000000000..0cd7b5b5ab --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/server-config.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +sourceEnable: true +sinkEnable: false diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/source-config.yml new file mode 100644 index 0000000000..47f25edbb2 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/test/resources/source-config.yml @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pubSubConfig: + meshAddress: 127.0.0.1:10000 + subject: TopicTest + idc: FT + env: PRD + group: chatgptSource + appId: 5032 + userName: chatgptSourceUser + passWord: chatgptPassWord +connectorConfig: + connectorName: chatgptSource + path: /chatgpt + port: 3756 + idleTimeout: 0 + proxyEnable: true + parsePromptFileName: prompt + +# https://platform.openai.com/docs/api-reference/chat/create +openaiConfig: + token: + model: gpt-3.5-turbo + timeout: 0 + temperature: 1 + maxTokens: + frequencyPenalty: 0 + presencePenalty: 0 + user: eventMesh + stop: [] + logitBias: { + + } + +openaiProxyConfig: + host: 127.0.0.1 + port: 7890 diff --git a/settings.gradle b/settings.gradle index 645e6fb365..6a8e27bf98 100644 --- a/settings.gradle +++ b/settings.gradle @@ -77,6 +77,7 @@ include 'eventmesh-connectors:eventmesh-connector-wecom' include 'eventmesh-connectors:eventmesh-connector-slack' include 'eventmesh-connectors:eventmesh-connector-wechat' include 'eventmesh-connectors:eventmesh-connector-http' +include 'eventmesh-connectors:eventmesh-connector-chatgpt' include 'eventmesh-storage-plugin:eventmesh-storage-api' include 'eventmesh-storage-plugin:eventmesh-storage-standalone' diff --git a/tools/dependency-check/known-dependencies.txt b/tools/dependency-check/known-dependencies.txt index 05c99f3d37..73feb5fd3b 100644 --- a/tools/dependency-check/known-dependencies.txt +++ b/tools/dependency-check/known-dependencies.txt @@ -1,6 +1,7 @@ FastInfoset-1.2.15.jar ST4-4.3.4.jar accessors-smart-2.4.7.jar +adapter-rxjava2-2.9.0.jar alibabacloud-gateway-spi-0.0.1.jar amqp-client-5.16.0.jar animal-sniffer-annotations-1.19.jar @@ -12,6 +13,7 @@ antlr4-4.13.0.jar antlr4-runtime-4.13.0.jar aopalliance-1.0.jar apache-client-2.20.29.jar +api-0.18.2.jar arns-2.20.29.jar asm-9.1.jar asm-9.2.jar @@ -43,7 +45,9 @@ byte-buddy-1.11.0.jar byte-buddy-1.12.18.jar cache-api-1.1.1.jar checker-qual-3.12.0.jar +classgraph-4.8.21.jar classmate-1.5.1.jar +client-0.18.2.jar cloudevents-api-2.4.2.jar cloudevents-core-2.4.2.jar cloudevents-http-vertx-2.3.0.jar @@ -63,6 +67,7 @@ commons-logging-1.2.jar commons-text-1.9.jar commons-validator-1.7.jar consul-api-1.4.5.jar +converter-jackson-2.9.0.jar credentials-java-0.2.4.jar crt-core-2.20.29.jar curator-client-5.4.0.jar @@ -155,6 +160,7 @@ json-path-2.7.0.jar json-smart-2.4.7.jar json-utils-2.20.29.jar jsr305-3.0.2.jar +jtokkit-0.5.1.jar kafka-clients-3.0.0.jar listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar log4j-api-2.22.1.jar @@ -162,6 +168,7 @@ log4j-core-2.22.1.jar log4j-slf4j2-impl-2.22.1.jar lz4-java-1.7.1.jar lz4-java-1.8.0.jar +mbknor-jackson-jsonschema_2.12-1.0.34.jar metrics-annotation-4.1.0.jar metrics-core-4.1.0.jar metrics-healthchecks-4.1.0.jar @@ -306,6 +313,7 @@ reactor-core-3.4.13.jar redisson-3.17.3.jar regions-2.20.29.jar relaxngDatatype-20020414.jar +retrofit-2.9.0.jar rocketmq-acl-4.9.5.jar rocketmq-broker-4.9.5.jar rocketmq-client-4.9.5.jar @@ -317,9 +325,12 @@ rocketmq-remoting-4.9.5.jar rocketmq-srvutil-4.9.5.jar rocketmq-store-4.9.5.jar rocketmq-tools-4.9.5.jar +rxjava-2.0.0.jar rxjava-3.0.12.jar s3-2.20.29.jar +scala-library-2.12.8.jar sdk-core-2.20.29.jar +service-0.18.2.jar simpleclient-0.12.0.jar simpleclient_tracer_common-0.12.0.jar simpleclient_tracer_otel-0.12.0.jar @@ -352,6 +363,7 @@ tomcat-embed-el-9.0.56.jar txw2-2.3.1.jar utils-2.20.29.jar validation-api-1.1.0.Final.jar +validation-api-2.0.1.Final.jar vertx-auth-common-4.4.6.jar vertx-bridge-common-4.4.6.jar vertx-core-4.4.6.jar From 2e4fc1bb9f8b11f6e306072d1f2c36564abb88c4 Mon Sep 17 00:00:00 2001 From: Pil0tXia Date: Mon, 22 Apr 2024 10:05:47 +0800 Subject: [PATCH 4/4] [ISSUE #4849] Enlarge stale bot operations-per-run to process more issues --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 33f64c5cca..81d4bc4c1e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -47,4 +47,4 @@ jobs: exempt-issue-labels: 'pinned,discussion,help wanted,WIP,weopen-star,GLCC,summer of code' exempt-pr-labels: 'help wanted,dependencies' exempt-all-milestones: true # Exempt all issues/PRs with milestones from stale - operations-per-run: 30 + operations-per-run: 300