From 63d67f74566d3453ef28af39a4b6366d900dec50 Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Thu, 25 Jun 2026 10:37:13 +0900 Subject: [PATCH 01/13] feat(kafka-logger): add TLS support for Kafka brokers --- apisix/plugins/kafka-logger.lua | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apisix/plugins/kafka-logger.lua b/apisix/plugins/kafka-logger.lua index 845fa684b8ba..c52996f698b6 100644 --- a/apisix/plugins/kafka-logger.lua +++ b/apisix/plugins/kafka-logger.lua @@ -89,6 +89,16 @@ local schema = { }, uniqueItems = true, }, + tls = { + type = "object", + description = "tls config for connecting to kafka brokers", + properties = { + verify = { + type = "boolean", + default = false, + }, + }, + }, kafka_topic = {type = "string"}, producer_type = { type = "string", @@ -279,6 +289,10 @@ function _M.log(conf, ctx) broker_config["flush_time"] = conf.producer_time_linger * 1000 broker_config["refresh_interval"] = conf.meta_refresh_interval * 1000 broker_config["api_version"] = conf.api_version + if conf.tls then + broker_config["ssl"] = true + broker_config["ssl_verify"] = conf.tls.verify + end local prod, err = core.lrucache.plugin_ctx(lrucache, ctx, nil, create_producer, broker_list, broker_config, conf.cluster_name) From e6188df40aecd89dbfee8444ebdb9e7e559dc04c Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Thu, 25 Jun 2026 10:37:46 +0900 Subject: [PATCH 02/13] feat(error-log-logger): add TLS support for Kafka brokers --- apisix/plugins/error-log-logger.lua | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apisix/plugins/error-log-logger.lua b/apisix/plugins/error-log-logger.lua index ee530e14ad91..e79831829faf 100644 --- a/apisix/plugins/error-log-logger.lua +++ b/apisix/plugins/error-log-logger.lua @@ -108,6 +108,16 @@ local metadata_schema = { }, uniqueItems = true, }, + tls = { + type = "object", + description = "tls config for connecting to kafka brokers", + properties = { + verify = { + type = "boolean", + default = false, + }, + }, + }, kafka_topic = {type = "string"}, producer_type = { type = "string", @@ -382,6 +392,10 @@ local function send_to_kafka(log_message) broker_config["producer_type"] = config.kafka.producer_type broker_config["required_acks"] = config.kafka.required_acks broker_config["refresh_interval"] = config.kafka.meta_refresh_interval * 1000 + if config.kafka.tls then + broker_config["ssl"] = true + broker_config["ssl_verify"] = config.kafka.tls.verify + end -- reuse producer via kafka_prod_lrucache to avoid unbalanced partitions of messages in kafka local prod, err = kafka_prod_lrucache(plugin_name, metadata.modifiedIndex, From f65e419312b1b963fa79aadd335f55a8529116ec Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Thu, 25 Jun 2026 11:44:28 +0900 Subject: [PATCH 03/13] test(kafka-logger): add TLS schema and integration tests --- t/plugin/kafka-logger-tls.t | 185 ++++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 t/plugin/kafka-logger-tls.t diff --git a/t/plugin/kafka-logger-tls.t b/t/plugin/kafka-logger-tls.t new file mode 100644 index 000000000000..df86c31355d3 --- /dev/null +++ b/t/plugin/kafka-logger-tls.t @@ -0,0 +1,185 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } +}); + +add_block_preprocessor(sub { + my ($block) = @_; + + my $extra_init_by_lua = <<_EOC_; + local bp_manager = require("apisix.utils.batch-processor-manager") + local core = require("apisix.core") + local function log_send_data(entry) + local data = type(entry) == "table" and core.json.encode(entry) or entry + core.log.info("send data to kafka: ", data) + end + local old_add = bp_manager.add_entry + bp_manager.add_entry = function(self, conf, entry, max_pending_entries) + local ok = old_add(self, conf, entry, max_pending_entries) + if ok then + log_send_data(entry) + end + return ok + end + local old_new = bp_manager.add_entry_to_new_processor + bp_manager.add_entry_to_new_processor = function(self, conf, entry, ctx, func, max_pending_entries) + local ok = old_new(self, conf, entry, ctx, func, max_pending_entries) + if ok then + log_send_data(entry) + end + return ok + end +_EOC_ + + if (!defined $block->extra_init_by_lua) { + $block->set_value("extra_init_by_lua", $extra_init_by_lua); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: tls schema validation - valid tls config +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.kafka-logger") + local ok, err = plugin.check_schema({ + brokers = {{host = "127.0.0.1", port = 9093}}, + kafka_topic = "test", + tls = { verify = false } + }) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- response_body +done + + + +=== TEST 2: tls schema validation - without tls (backward compatibility) +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.kafka-logger") + local ok, err = plugin.check_schema({ + brokers = {{host = "127.0.0.1", port = 9092}}, + kafka_topic = "test" + }) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- response_body +done + + + +=== TEST 3: tls schema validation - wrong type for verify +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.kafka-logger") + local ok, err = plugin.check_schema({ + brokers = {{host = "127.0.0.1", port = 9093}}, + kafka_topic = "test", + tls = { verify = "abc" } + }) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- response_body +property "tls" validation failed: property "verify" validation failed: wrong type: expected boolean, got string +done + + + +=== TEST 4: set route with tls config to SSL port +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins":{ + "kafka-logger":{ + "brokers":[ + { + "host":"127.0.0.1", + "port":9093 + }], + "kafka_topic":"test2", + "producer_type":"sync", + "key":"key1", + "timeout":1, + "batch_max_size":1, + "include_req_body": true, + "tls":{"verify":false} + } + }, + "upstream":{ + "nodes":{ + "127.0.0.1:1980":1 + }, + "type":"roundrobin" + }, + "uri":"/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 5: hit route, send data to kafka via TLS successfully +--- request +POST /hello?name=qwerty +abcdef +--- response_body +hello world +--- error_log eval +qr/send data to kafka: \{.*"body":"abcdef"/ +--- no_error_log +[error] +--- wait: 2 From bcaa0f68a67aa6efeb110b6d217eaba0db93adc5 Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Thu, 25 Jun 2026 11:46:34 +0900 Subject: [PATCH 04/13] test(error-log-logger): add TLS schema and integration tests for Kafka --- t/plugin/error-log-logger-kafka.t | 119 ++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/t/plugin/error-log-logger-kafka.t b/t/plugin/error-log-logger-kafka.t index 0ae6a52aca90..cb4ced00f888 100644 --- a/t/plugin/error-log-logger-kafka.t +++ b/t/plugin/error-log-logger-kafka.t @@ -270,3 +270,122 @@ true true --- no_error_log [alert] + + + +=== TEST 8: tls schema validation - valid tls config +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local plugin = require("apisix.plugins.error-log-logger") + local ok, err = plugin.check_schema( + { + kafka = { + brokers = { + { + host = "127.0.0.1", + port = 9093 + } + }, + kafka_topic = "test2", + tls = { verify = false } + } + }, + core.schema.TYPE_METADATA + ) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- response_body +done + + + +=== TEST 9: tls schema validation - wrong type for verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local plugin = require("apisix.plugins.error-log-logger") + local ok, err = plugin.check_schema( + { + kafka = { + brokers = { + { + host = "127.0.0.1", + port = 9093 + } + }, + kafka_topic = "test2", + tls = { verify = "abc" } + } + }, + core.schema.TYPE_METADATA + ) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- response_body +property "kafka" validation failed: property "tls" validation failed: property "verify" validation failed: wrong type: expected boolean, got string +done + + + +=== TEST 10: put plugin metadata with tls config and log an error level message +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/plugin_metadata/error-log-logger', + ngx.HTTP_PUT, + [[{ + "kafka": { + "brokers": [{ + "host": "127.0.0.1", + "port": 9093 + }], + "producer_type": "sync", + "kafka_topic": "test2", + "tls": {"verify": false} + }, + "level": "ERROR", + "inactive_timeout": 1 + }]] + ) + ngx.sleep(2) + core.log.error("this is a error message for tls test.") + } + } +--- error_log eval +[qr/this is a error message for tls test/, +qr/send data to kafka: .*tls test/] +--- wait: 3 + + + +=== TEST 11: delete metadata for the plugin, recover to the default +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/plugin_metadata/error-log-logger', + ngx.HTTP_DELETE) + + if code >= 300 then + ngx.status = code + end + + ngx.say(body) + } + } +--- response_body +passed From a1d30c3fc0b795efe357f453d2b447772e64462d Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Thu, 25 Jun 2026 12:05:05 +0900 Subject: [PATCH 05/13] docs: add TLS attributes and examples for kafka-logger and error-log-logger --- docs/en/latest/plugins/error-log-logger.md | 31 ++++++++++++++ docs/en/latest/plugins/kafka-logger.md | 49 ++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/docs/en/latest/plugins/error-log-logger.md b/docs/en/latest/plugins/error-log-logger.md index 2af9dac59d32..bd46666d6d6b 100644 --- a/docs/en/latest/plugins/error-log-logger.md +++ b/docs/en/latest/plugins/error-log-logger.md @@ -66,6 +66,8 @@ There are no attributes to configure this Plugin on Routes or Services. All conf | kafka.brokers[].sasl_config.mechanism | string | False | PLAIN | ["PLAIN"] | The mechanism of SASL configuration. | | kafka.brokers[].sasl_config.user | string | True | | | The user of SASL configuration. Required if `sasl_config` is present. | | kafka.brokers[].sasl_config.password | string | True | | | The password of SASL configuration. Required if `sasl_config` is present. | +| kafka.tls | object | False | | | TLS configuration for connecting to Kafka brokers. | +| kafka.tls.verify | boolean | False | false | | If true, verify the Kafka broker TLS certificate. | | kafka.kafka_topic | string | True | | | Target topic to push the logs for organization. | | kafka.producer_type | string | False | async | ["async", "sync"] | Message sending mode of the producer. | | kafka.required_acks | integer | False | 1 | [-1, 0, 1] | Number of acknowledgements the leader needs to receive for the producer to consider the request complete. See [Apache Kafka documentation](https://kafka.apache.org/documentation/#producerconfigs_acks) for more. | @@ -238,3 +240,32 @@ curl "http://127.0.0.1:9180/apisix/admin/plugin_metadata/error-log-logger" -X PU "inactive_timeout": 1 }' ``` + +### Send Logs to TLS-Enabled Kafka Brokers + +The following example demonstrates how to configure the `error-log-logger` Plugin to send error logs to TLS-enabled Kafka brokers. + +Configure the Plugin metadata with the Kafka broker details and TLS configuration: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/plugin_metadata/error-log-logger" -X PUT \ + -H "X-API-KEY: ${admin_key}" \ + -d '{ + "kafka": { + "brokers": [ + { + "host": "kafka.example.com", + "port": 9093 + } + ], + "kafka_topic": "apisix-error-logs", + "tls": { + "verify": true + } + }, + "level": "ERROR", + "inactive_timeout": 1 + }' +``` + +When using self-signed certificates, set `tls.verify` to `false` to skip certificate verification. diff --git a/docs/en/latest/plugins/kafka-logger.md b/docs/en/latest/plugins/kafka-logger.md index f57081183822..22ecb3c3fb59 100644 --- a/docs/en/latest/plugins/kafka-logger.md +++ b/docs/en/latest/plugins/kafka-logger.md @@ -49,6 +49,8 @@ It might take some time to receive the log data. It will be automatically sent a | brokers.sasl_config.mechanism | string | False | "PLAIN" | ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"] | The mechanism of SASL config. | | brokers.sasl_config.user | string | True | | | The user of `sasl_config`. Required if `sasl_config` is configured. | | brokers.sasl_config.password | string | True | | | The password of `sasl_config`. Required if `sasl_config` is configured. | +| tls | object | False | | | TLS configuration for connecting to Kafka brokers. | +| tls.verify | boolean | False | false | | If true, verify the Kafka broker TLS certificate. | | kafka_topic | string | True | | | Target topic to push the logs. | | producer_type | string | False | async | ["async", "sync"] | Message sending mode of the producer. | | required_acks | integer | False | 1 | [1, -1] | Number of acknowledgements the leader needs to receive for the producer to consider the request complete. This controls the durability of the sent records. The attribute follows the same configuration as the Kafka `acks` attribute. `required_acks` cannot be 0. See [Apache Kafka documentation](https://kafka.apache.org/documentation/#producerconfigs_acks) for more. | @@ -487,3 +489,50 @@ If you have customized the `log_format` in addition to setting `include_req_body ``` ::: + +### Log to TLS-Enabled Kafka Brokers + +The following example demonstrates how to connect to TLS-enabled Kafka brokers, such as AWS MSK. + +Create a Route with `kafka-logger` and configure the `tls` attribute to connect to the TLS-enabled Kafka broker: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ + -H "X-API-KEY: ${admin_key}" \ + -d '{ + "id": "kafka-logger-tls-route", + "uri": "/get", + "plugins": { + "kafka-logger": { + "brokers": [ + { + "host": "kafka.example.com", + "port": 9093 + } + ], + "kafka_topic": "test2", + "key": "key1", + "batch_max_size": 1, + "tls": { + "verify": true + } + } + }, + "upstream": { + "nodes": { + "httpbin.org:80": 1 + }, + "type": "roundrobin" + } + }' +``` + +When using self-signed certificates, set `tls.verify` to `false` to skip certificate verification: + +```json +{ + "tls": { + "verify": false + } +} +``` From c8eca19e67ff19a953b6e8070d5a94684f5a830f Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Mon, 6 Jul 2026 21:12:40 +0900 Subject: [PATCH 06/13] test(kafka-logger): remove TLS integration tests that require SSL Kafka not available in plugin CI --- t/plugin/error-log-logger-kafka.t | 51 --------------------------- t/plugin/kafka-logger-tls.t | 58 ------------------------------- 2 files changed, 109 deletions(-) diff --git a/t/plugin/error-log-logger-kafka.t b/t/plugin/error-log-logger-kafka.t index cb4ced00f888..587af37e3351 100644 --- a/t/plugin/error-log-logger-kafka.t +++ b/t/plugin/error-log-logger-kafka.t @@ -338,54 +338,3 @@ done -=== TEST 10: put plugin metadata with tls config and log an error level message ---- config - location /t { - content_by_lua_block { - local core = require("apisix.core") - local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/plugin_metadata/error-log-logger', - ngx.HTTP_PUT, - [[{ - "kafka": { - "brokers": [{ - "host": "127.0.0.1", - "port": 9093 - }], - "producer_type": "sync", - "kafka_topic": "test2", - "tls": {"verify": false} - }, - "level": "ERROR", - "inactive_timeout": 1 - }]] - ) - ngx.sleep(2) - core.log.error("this is a error message for tls test.") - } - } ---- error_log eval -[qr/this is a error message for tls test/, -qr/send data to kafka: .*tls test/] ---- wait: 3 - - - -=== TEST 11: delete metadata for the plugin, recover to the default ---- config - location /t { - content_by_lua_block { - local core = require("apisix.core") - local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/plugin_metadata/error-log-logger', - ngx.HTTP_DELETE) - - if code >= 300 then - ngx.status = code - end - - ngx.say(body) - } - } ---- response_body -passed diff --git a/t/plugin/kafka-logger-tls.t b/t/plugin/kafka-logger-tls.t index df86c31355d3..4711908b97da 100644 --- a/t/plugin/kafka-logger-tls.t +++ b/t/plugin/kafka-logger-tls.t @@ -125,61 +125,3 @@ done --- response_body property "tls" validation failed: property "verify" validation failed: wrong type: expected boolean, got string done - - - -=== TEST 4: set route with tls config to SSL port ---- config - location /t { - content_by_lua_block { - local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins":{ - "kafka-logger":{ - "brokers":[ - { - "host":"127.0.0.1", - "port":9093 - }], - "kafka_topic":"test2", - "producer_type":"sync", - "key":"key1", - "timeout":1, - "batch_max_size":1, - "include_req_body": true, - "tls":{"verify":false} - } - }, - "upstream":{ - "nodes":{ - "127.0.0.1:1980":1 - }, - "type":"roundrobin" - }, - "uri":"/hello" - }]] - ) - if code >= 300 then - ngx.status = code - end - ngx.say(body) - } - } ---- response_body -passed - - - -=== TEST 5: hit route, send data to kafka via TLS successfully ---- request -POST /hello?name=qwerty -abcdef ---- response_body -hello world ---- error_log eval -qr/send data to kafka: \{.*"body":"abcdef"/ ---- no_error_log -[error] ---- wait: 2 From 4136aa23285f1684250ec3cdaa802a7a709db258 Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Mon, 6 Jul 2026 21:40:15 +0900 Subject: [PATCH 07/13] style(error-log-logger): fix trailing blank lines in test file --- t/plugin/error-log-logger-kafka.t | 3 --- 1 file changed, 3 deletions(-) diff --git a/t/plugin/error-log-logger-kafka.t b/t/plugin/error-log-logger-kafka.t index 587af37e3351..c5819f90a4d3 100644 --- a/t/plugin/error-log-logger-kafka.t +++ b/t/plugin/error-log-logger-kafka.t @@ -335,6 +335,3 @@ done --- response_body property "kafka" validation failed: property "tls" validation failed: property "verify" validation failed: wrong type: expected boolean, got string done - - - From 3d1c5912b5e77f23852da8a00e8b8de2ed916ecb Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Mon, 13 Jul 2026 20:51:02 +0900 Subject: [PATCH 08/13] feat(kafka-logger): warn when TLS certificate verification is disabled Follow the check_tls_bool convention used by http-logger and clickhouse-logger so users are warned when tls.verify is set to false. --- apisix/plugins/kafka-logger.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apisix/plugins/kafka-logger.lua b/apisix/plugins/kafka-logger.lua index c52996f698b6..b9b254e2faaa 100644 --- a/apisix/plugins/kafka-logger.lua +++ b/apisix/plugins/kafka-logger.lua @@ -189,6 +189,8 @@ function _M.check_schema(conf, schema_type) if not ok then return nil, err end + + core.utils.check_tls_bool({"tls.verify"}, conf, plugin_name) return log_util.check_log_schema(conf) end From 5fbd9d7958711f110282b14ee64a38109abcf6f6 Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Mon, 13 Jul 2026 20:54:06 +0900 Subject: [PATCH 09/13] feat(error-log-logger): warn when TLS certificate verification is disabled Call check_tls_bool inside the metadata branch, since the kafka config only exists in metadata_schema and check_schema returns early for TYPE_METADATA before reaching the non-metadata checks. --- apisix/plugins/error-log-logger.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/apisix/plugins/error-log-logger.lua b/apisix/plugins/error-log-logger.lua index e79831829faf..a69341ceabd5 100644 --- a/apisix/plugins/error-log-logger.lua +++ b/apisix/plugins/error-log-logger.lua @@ -195,6 +195,7 @@ local _M = { function _M.check_schema(conf, schema_type) if schema_type == core.schema.TYPE_METADATA then + core.utils.check_tls_bool({"kafka.tls.verify"}, conf, plugin_name) return core.schema.check(metadata_schema, conf) end From 34369cf0c22109960276e06db290accd79ee06f3 Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Mon, 13 Jul 2026 21:21:20 +0900 Subject: [PATCH 10/13] test(kafka-logger): move TLS schema tests into kafka-logger2.t Drop the dedicated kafka-logger-tls.t file and its leftover batch-processor monkey-patch preprocessor, which was dead code since the integration tests were removed. The three schema-validation tests now live in kafka-logger2.t, consistent with the error-log-logger side of this PR. --- t/plugin/kafka-logger-tls.t | 127 ------------------------------------ t/plugin/kafka-logger2.t | 63 ++++++++++++++++++ 2 files changed, 63 insertions(+), 127 deletions(-) delete mode 100644 t/plugin/kafka-logger-tls.t diff --git a/t/plugin/kafka-logger-tls.t b/t/plugin/kafka-logger-tls.t deleted file mode 100644 index 4711908b97da..000000000000 --- a/t/plugin/kafka-logger-tls.t +++ /dev/null @@ -1,127 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -use t::APISIX 'no_plan'; - -repeat_each(1); -no_long_string(); -no_root_location(); - -add_block_preprocessor(sub { - my ($block) = @_; - - if (!$block->request) { - $block->set_value("request", "GET /t"); - } -}); - -add_block_preprocessor(sub { - my ($block) = @_; - - my $extra_init_by_lua = <<_EOC_; - local bp_manager = require("apisix.utils.batch-processor-manager") - local core = require("apisix.core") - local function log_send_data(entry) - local data = type(entry) == "table" and core.json.encode(entry) or entry - core.log.info("send data to kafka: ", data) - end - local old_add = bp_manager.add_entry - bp_manager.add_entry = function(self, conf, entry, max_pending_entries) - local ok = old_add(self, conf, entry, max_pending_entries) - if ok then - log_send_data(entry) - end - return ok - end - local old_new = bp_manager.add_entry_to_new_processor - bp_manager.add_entry_to_new_processor = function(self, conf, entry, ctx, func, max_pending_entries) - local ok = old_new(self, conf, entry, ctx, func, max_pending_entries) - if ok then - log_send_data(entry) - end - return ok - end -_EOC_ - - if (!defined $block->extra_init_by_lua) { - $block->set_value("extra_init_by_lua", $extra_init_by_lua); - } -}); - -run_tests; - -__DATA__ - -=== TEST 1: tls schema validation - valid tls config ---- config - location /t { - content_by_lua_block { - local plugin = require("apisix.plugins.kafka-logger") - local ok, err = plugin.check_schema({ - brokers = {{host = "127.0.0.1", port = 9093}}, - kafka_topic = "test", - tls = { verify = false } - }) - if not ok then - ngx.say(err) - end - ngx.say("done") - } - } ---- response_body -done - - - -=== TEST 2: tls schema validation - without tls (backward compatibility) ---- config - location /t { - content_by_lua_block { - local plugin = require("apisix.plugins.kafka-logger") - local ok, err = plugin.check_schema({ - brokers = {{host = "127.0.0.1", port = 9092}}, - kafka_topic = "test" - }) - if not ok then - ngx.say(err) - end - ngx.say("done") - } - } ---- response_body -done - - - -=== TEST 3: tls schema validation - wrong type for verify ---- config - location /t { - content_by_lua_block { - local plugin = require("apisix.plugins.kafka-logger") - local ok, err = plugin.check_schema({ - brokers = {{host = "127.0.0.1", port = 9093}}, - kafka_topic = "test", - tls = { verify = "abc" } - }) - if not ok then - ngx.say(err) - end - ngx.say("done") - } - } ---- response_body -property "tls" validation failed: property "verify" validation failed: wrong type: expected boolean, got string -done diff --git a/t/plugin/kafka-logger2.t b/t/plugin/kafka-logger2.t index c1fccb7ebb8c..5a23281d3592 100644 --- a/t/plugin/kafka-logger2.t +++ b/t/plugin/kafka-logger2.t @@ -1123,3 +1123,66 @@ hello world --- error_log eval qr/send data to kafka: \{.*"body":"abcdef"/ --- wait: 2 + + + +=== TEST 28: tls schema validation - valid tls config +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.kafka-logger") + local ok, err = plugin.check_schema({ + brokers = {{host = "127.0.0.1", port = 9093}}, + kafka_topic = "test", + tls = { verify = false } + }) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- response_body +done + + + +=== TEST 29: tls schema validation - without tls (backward compatibility) +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.kafka-logger") + local ok, err = plugin.check_schema({ + brokers = {{host = "127.0.0.1", port = 9092}}, + kafka_topic = "test" + }) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- response_body +done + + + +=== TEST 30: tls schema validation - wrong type for verify +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.kafka-logger") + local ok, err = plugin.check_schema({ + brokers = {{host = "127.0.0.1", port = 9093}}, + kafka_topic = "test", + tls = { verify = "abc" } + }) + if not ok then + ngx.say(err) + end + ngx.say("done") + } + } +--- response_body +property "tls" validation failed: property "verify" validation failed: wrong type: expected boolean, got string +done From a6649ac2b5931804599852845cb4161d7e8d61dc Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Mon, 13 Jul 2026 22:35:12 +0900 Subject: [PATCH 11/13] test(kafka-logger): add SSL Kafka listener to plugin CI environment Enable a TLS listener on 9093 for kafka-server1 so the TLS integration tests can connect. Generate a self-signed keystore via keytool and mount it as the broker keystore/truststore, mirroring the last test env. --- ci/init-plugin-test-service.sh | 3 +++ ci/pod/docker-compose.plugin.yml | 4 ++++ ci/pod/kafka/kafka-server/env/common.env | 7 ++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ci/init-plugin-test-service.sh b/ci/init-plugin-test-service.sh index 6ef72f1df65f..7c65a8af360f 100755 --- a/ci/init-plugin-test-service.sh +++ b/ci/init-plugin-test-service.sh @@ -94,6 +94,9 @@ after() { before() { # download keycloak cas provider sudo wget -q https://github.com/jacekkow/keycloak-protocol-cas/releases/download/18.0.2/keycloak-protocol-cas-18.0.2.jar -O /opt/keycloak-protocol-cas-18.0.2.jar + + # generating SSL certificates for Kafka + sudo keytool -genkeypair -keyalg RSA -dname "CN=127.0.0.1" -alias 127.0.0.1 -keystore ./ci/pod/kafka/kafka-server/selfsigned.jks -validity 365 -keysize 2048 -storepass changeit } case $1 in diff --git a/ci/pod/docker-compose.plugin.yml b/ci/pod/docker-compose.plugin.yml index e66877ca6c81..81b2da0df3dc 100644 --- a/ci/pod/docker-compose.plugin.yml +++ b/ci/pod/docker-compose.plugin.yml @@ -175,11 +175,15 @@ services: restart: unless-stopped ports: - "9092:9092" + - "9093:9093" depends_on: - zookeeper-server1 - zookeeper-server2 networks: kafka_net: + volumes: + - ./ci/pod/kafka/kafka-server/selfsigned.jks:/opt/bitnami/kafka/config/certs/kafka.keystore.jks:ro + - ./ci/pod/kafka/kafka-server/selfsigned.jks:/opt/bitnami/kafka/config/certs/kafka.truststore.jks:ro kafka-server2: image: bitnamilegacy/kafka:2.8.1 diff --git a/ci/pod/kafka/kafka-server/env/common.env b/ci/pod/kafka/kafka-server/env/common.env index 06200b9b0042..2414fd2c51d6 100644 --- a/ci/pod/kafka/kafka-server/env/common.env +++ b/ci/pod/kafka/kafka-server/env/common.env @@ -1,3 +1,8 @@ ALLOW_PLAINTEXT_LISTENER=yes KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true -KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 +KAFKA_CFG_LISTENERS=PLAINTEXT://0.0.0.0:9092,SSL://0.0.0.0:9093 +KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092,SSL://127.0.0.1:9093 +KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM= +KAFKA_CFG_SSL_KEYSTORE_LOCATION=/opt/bitnami/kafka/config/certs/kafka.keystore.jks +KAFKA_CFG_SSL_KEYSTORE_PASSWORD=changeit +KAFKA_CFG_SSL_KEY_PASSWORD=changeit From ff6a2dafea11111717c65f0e7d538dab1e62a7a7 Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Mon, 13 Jul 2026 22:35:12 +0900 Subject: [PATCH 12/13] test(kafka-logger): add TLS integration and security-warning tests - kafka-logger: produce over TLS (verify=false) and reject self-signed cert (verify=true) - error-log-logger: produce over TLS via plugin metadata - security-warning: assert check_tls_bool warns when tls.verify / kafka.tls.verify is disabled --- t/plugin/error-log-logger-kafka.t | 36 ++++++++++ t/plugin/kafka-logger2.t | 114 ++++++++++++++++++++++++++++++ t/plugin/security-warning.t | 66 +++++++++++++++++ 3 files changed, 216 insertions(+) diff --git a/t/plugin/error-log-logger-kafka.t b/t/plugin/error-log-logger-kafka.t index c5819f90a4d3..1ede14cd15b2 100644 --- a/t/plugin/error-log-logger-kafka.t +++ b/t/plugin/error-log-logger-kafka.t @@ -335,3 +335,39 @@ done --- response_body property "kafka" validation failed: property "tls" validation failed: property "verify" validation failed: wrong type: expected boolean, got string done + + + +=== TEST 10: put metadata with kafka tls and log an error message - send via TLS +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/plugin_metadata/error-log-logger', + ngx.HTTP_PUT, + [[{ + "kafka": { + "brokers": [{ + "host": "127.0.0.1", + "port": 9093 + }], + "kafka_topic": "test2", + "meta_refresh_interval": 1, + "tls": {"verify": false} + }, + "level": "ERROR", + "inactive_timeout": 1 + }]] + ) + ngx.sleep(2) + core.log.error("this is a error message for tls test.") + } + } +--- error_log eval +[qr/this is a error message for tls test/, +qr/sending a batch logs to kafka brokers: \[\{"host":"127.0.0.1","port":9093\}\]/, +qr/send data to kafka: .*this is a error message for tls test/] +--- no_error_log +failed to do SSL handshake +--- wait: 3 diff --git a/t/plugin/kafka-logger2.t b/t/plugin/kafka-logger2.t index 5a23281d3592..8ba42999b3d7 100644 --- a/t/plugin/kafka-logger2.t +++ b/t/plugin/kafka-logger2.t @@ -1186,3 +1186,117 @@ done --- response_body property "tls" validation failed: property "verify" validation failed: wrong type: expected boolean, got string done + + + +=== TEST 31: set route with tls config to SSL port +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins":{ + "kafka-logger":{ + "brokers":[ + { + "host":"127.0.0.1", + "port":9093 + }], + "kafka_topic":"test2", + "producer_type":"sync", + "key":"key1", + "timeout":1, + "batch_max_size":1, + "include_req_body": true, + "tls":{"verify":false} + } + }, + "upstream":{ + "nodes":{ + "127.0.0.1:1980":1 + }, + "type":"roundrobin" + }, + "uri":"/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 32: hit route, send data to kafka via TLS successfully +--- request +POST /hello?name=qwerty +abcdef +--- response_body +hello world +--- error_log eval +qr/send data to kafka: \{.*"body":"abcdef"/ +--- no_error_log +[error] +--- wait: 2 + + + +=== TEST 33: set route with tls verify enabled against self-signed broker +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins":{ + "kafka-logger":{ + "brokers":[ + { + "host":"127.0.0.1", + "port":9093 + }], + "kafka_topic":"test2", + "producer_type":"sync", + "key":"key1", + "timeout":1, + "batch_max_size":1, + "include_req_body": true, + "tls":{"verify":true} + } + }, + "upstream":{ + "nodes":{ + "127.0.0.1:1980":1 + }, + "type":"roundrobin" + }, + "uri":"/hello" + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 34: hit route, tls verify rejects the self-signed certificate +--- request +POST /hello?name=qwerty +abcdef +--- response_body +hello world +--- error_log +failed to do SSL handshake with 127.0.0.1:9093 +--- wait: 2 diff --git a/t/plugin/security-warning.t b/t/plugin/security-warning.t index 0dca62e7bbd4..4b200d7f03da 100644 --- a/t/plugin/security-warning.t +++ b/t/plugin/security-warning.t @@ -574,3 +574,69 @@ Using loki-logger endpoint_addrs with no TLS is a security risk done --- no_error_log Using loki-logger endpoint_addrs with no TLS is a security risk + + + +=== TEST 21: kafka-logger with tls verify disabled +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.kafka-logger") + + local ok, err = plugin.check_schema({ + brokers = {{host = "127.0.0.1", port = 9093}}, + kafka_topic = "test", + tls = { verify = false } + }) + ngx.say(ok and "done" or err) + } + } +--- response_body +done +--- error_log +Keeping tls.verify disabled in kafka-logger configuration is a security risk + + + +=== TEST 22: kafka-logger with tls verify enabled +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.kafka-logger") + + local ok, err = plugin.check_schema({ + brokers = {{host = "127.0.0.1", port = 9093}}, + kafka_topic = "test", + tls = { verify = true } + }) + ngx.say(ok and "done" or err) + } + } +--- response_body +done +--- no_error_log +Keeping tls.verify disabled in kafka-logger configuration is a security risk + + + +=== TEST 23: error-log-logger with kafka tls verify disabled (metadata) +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local plugin = require("apisix.plugins.error-log-logger") + + local ok, err = plugin.check_schema({ + kafka = { + brokers = {{host = "127.0.0.1", port = 9093}}, + kafka_topic = "test", + tls = { verify = false } + } + }, core.schema.TYPE_METADATA) + ngx.say(ok and "done" or err) + } + } +--- response_body +done +--- error_log +Keeping kafka.tls.verify disabled in error-log-logger configuration is a security risk From 83c7a92dfdd53391164f6ce0d20fb9353d4daea3 Mon Sep 17 00:00:00 2001 From: ecsimsw Date: Wed, 15 Jul 2026 20:54:59 +0900 Subject: [PATCH 13/13] fix(error-log-logger): run check_tls_bool after schema check check_tls_bool ran before core.schema.check filled schema defaults, so a metadata config with "tls": {} (verify omitted, defaults to false) left kafka.tls.verify as nil and the security warning was skipped. Move the call after the schema check, matching kafka-logger, and add security-warning tests for the verify-omitted case. --- apisix/plugins/error-log-logger.lua | 6 +++- t/plugin/security-warning.t | 45 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/apisix/plugins/error-log-logger.lua b/apisix/plugins/error-log-logger.lua index a69341ceabd5..0551c70013d3 100644 --- a/apisix/plugins/error-log-logger.lua +++ b/apisix/plugins/error-log-logger.lua @@ -195,8 +195,12 @@ local _M = { function _M.check_schema(conf, schema_type) if schema_type == core.schema.TYPE_METADATA then + local ok, err = core.schema.check(metadata_schema, conf) + if not ok then + return nil, err + end core.utils.check_tls_bool({"kafka.tls.verify"}, conf, plugin_name) - return core.schema.check(metadata_schema, conf) + return true end local check = {"skywalking.endpoint_addr", "clickhouse.endpoint_addr"} diff --git a/t/plugin/security-warning.t b/t/plugin/security-warning.t index 4b200d7f03da..ed22f1506130 100644 --- a/t/plugin/security-warning.t +++ b/t/plugin/security-warning.t @@ -640,3 +640,48 @@ Keeping tls.verify disabled in kafka-logger configuration is a security risk done --- error_log Keeping kafka.tls.verify disabled in error-log-logger configuration is a security risk + + + +=== TEST 24: error-log-logger with kafka tls but verify omitted (defaults to false) +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local plugin = require("apisix.plugins.error-log-logger") + + local ok, err = plugin.check_schema({ + kafka = { + brokers = {{host = "127.0.0.1", port = 9093}}, + kafka_topic = "test", + tls = {} + } + }, core.schema.TYPE_METADATA) + ngx.say(ok and "done" or err) + } + } +--- response_body +done +--- error_log +Keeping kafka.tls.verify disabled in error-log-logger configuration is a security risk + + + +=== TEST 25: kafka-logger with tls but verify omitted (defaults to false) +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.kafka-logger") + + local ok, err = plugin.check_schema({ + brokers = {{host = "127.0.0.1", port = 9093}}, + kafka_topic = "test", + tls = {} + }) + ngx.say(ok and "done" or err) + } + } +--- response_body +done +--- error_log +Keeping tls.verify disabled in kafka-logger configuration is a security risk