Add support for Kafka

core/add_project.sh

@@ -48,6 +48,12 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_kafka() {
+    [ ! -d "app/ctest-kafka" ] && git clone https://github.com/wykkevin2/kafka.git app/ctest-kafka
+    cd app/ctest-kafka
+    git fetch && git checkout ctest-injection
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -64,7 +70,8 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            kafka) setup_kafka ;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio and kafka." ;;
         esac
     fi
 }

core/ctest_const.py

@@ -12,18 +12,21 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+KAFKA = "kafka-core"
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_KAFKA_DIR = os.path.join(APP_DIR, "ctest-kafka")
 
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    KAFKA: CTEST_KAFKA_DIR,
 }
 
 
@@ -34,6 +37,7 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    KAFKA: "",  # Although we are testing the core module, the commands need to be run under the root.
 }
 
 
@@ -58,6 +62,7 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    # Kafka doesn't have surefire
 }
 
 # default or deprecate conf path
@@ -74,7 +79,8 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    KAFKA: os.path.join(DEFAULT_CONF_DIR, KAFKA + "-default.tsv")
 }
 
 
@@ -96,6 +102,9 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
+    ],
+    KAFKA: [
+        os.path.join(CTEST_KAFKA_DIR, "clients/src/main/resources/ctest.xml")
     ]
 }
 

core/generate_ctest/inject.py

@@ -21,7 +21,7 @@ def inject_config(param_value_pairs):
             for p, v in param_value_pairs.items():
                 file.write(p + "=" + v + "\n")
             file.close()
-    elif project in [HCOMMON, HDFS, HBASE]:
+    elif project in [HCOMMON, HDFS, HBASE, KAFKA]:
         conf = ET.Element("configuration")
         for p, v in param_value_pairs.items():
             prop = ET.SubElement(conf, "property")
@@ -46,7 +46,7 @@ def clean_conf_file(project):
             file = open(inject_path, "w")
             file.write("\n")
             file.close()
-    elif project in [HCOMMON, HDFS, HBASE]:
+    elif project in [HCOMMON, HDFS, HBASE, KAFKA]:
         conf = ET.Element("configuration")
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "wb")

core/generate_ctest/kafka-core-30.tsv

@@ -0,0 +1,30 @@
+metadata.log.max.record.bytes.between.snapshots	10485760	41943040
+controller.quorum.request.timeout.ms	1000	4000
+sasl.mechanism.controller.protocol	SASL	SKIP
+password.encoder.iterations	2048	8192
+password.encoder.key.length	64	256
+broker.heartbeat.interval.ms	1000	4000
+broker.session.timeout.ms	4500	18000
+initial.broker.registration.timeout.ms	30000	120000
+sasl.enabled.mechanisms	SASL	GSSAPI
+controlled.shutdown.enable	false	SKIP
+group.initial.rebalance.delay.ms	1500	6000
+offsets.topic.num.partitions	25	100
+default.replication.factor	0	2
+delete.topic.enable	false	SKIP
+zookeeper.ssl.ocsp.enable	true	SKIP
+zookeeper.ssl.crl.enable	true	SKIP
+log.message.format.version	.10.0	SKIP
+log.cleaner.dedupe.buffer.size	67108864	268435456
+offsets.topic.replication.factor	1	6
+zookeeper.ssl.client.enable	true	SKIP
+zookeeper.set.acl	true	SKIP
+zookeeper.max.in.flight.requests	1	20
+background.threads	1	20
+metrics.num.samples	1	4
+metrics.sample.window.ms	15000	60000
+replication.quota.window.size.seconds	0	2
+alter.log.dirs.replication.quota.window.num	5	22
+alter.log.dirs.replication.quota.window.size.seconds	0	2
+log.flush.scheduler.interval.ms	4611686018427387903	18446744073709551614
+log.cleaner.io.buffer.load.factor	0.45	1.8

core/generate_ctest/program_input.py

@@ -4,13 +4,15 @@
     # run mode
     "run_mode": "generate_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
-    "project": "hadoop-common", # string
+    "project": "kafka-core", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-kafka-core.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "sample-hadoop-common.tsv", # string
+    "param_value_tsv": "kafka-core-30.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
+    # Add a new boolean to handle gradle projects. If this is true, it will ignore the mvn related fields.
+    "is_gradle": True, # bool
     # whether to use mvn test or mvn surefire:test
     "use_surefire": True, # bool
     # additional maven options to pass to `mvn surefire:test -Dtest=...`

core/generate_ctest/run_test.py

@@ -14,6 +14,7 @@
 display_mode = p_input["display_mode"]
 project = p_input["project"]
 cmd_timeout = p_input["cmd_timeout"]
+is_gradle = p_input["is_gradle"]
 testing_dir = os.path.join(PROJECT_DIR[project], MODULE_SUBDIR[project])
 
 
@@ -25,6 +26,7 @@ def run_test_seperate(param, value, associated_tests):
     print(">>>>[ctest_core] chdir to {}".format(testing_dir))
     start_time = time.time()
     for test in associated_tests:
+        single_test_start_time = time.time()
         cmd = run_test_utils.maven_cmd(test)
         if display_mode:
             os.system(" ".join(cmd))
@@ -49,12 +51,18 @@ def run_test_seperate(param, value, associated_tests):
 
         print_output = run_test_utils.strip_ansi(stdout.decode("ascii", "ignore"))
         print(print_output)
-        clsname, testname = test.split("#")
-        times, errors = parse_surefire(clsname, [testname])
-        if testname in times:
-            tr.ran_tests_and_time.add(test + "\t" + times[testname])
-            if testname in errors:
+        if is_gradle:
+            clsname, testname = test.split(".")
+            tr.ran_tests_and_time.add(test + "\t" + str(time.time() - single_test_start_time))
+            if "BUILD SUCCESSFUL" not in print_output:
                 tr.failed_tests.add(test)
+        else:
+            clsname, testname = test.split("#")
+            times, errors = parse_surefire(clsname, [testname])
+            if testname in times:
+                tr.ran_tests_and_time.add(test + "\t" + times[testname])
+                if testname in errors:
+                    tr.failed_tests.add(test)
     duration = time.time() - start_time
     os.chdir(CUR_DIR)
     print(">>>>[ctest_core] chdir to {}".format(CUR_DIR))

core/generate_ctest/run_test_utils.py

@@ -4,6 +4,7 @@
 sys.path.append("..")
 from ctest_const import *
 
+is_gradle = p_input["is_gradle"]
 maven_args = p_input["maven_args"]
 use_surefire = p_input["use_surefire"]
 ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]')
@@ -18,7 +19,10 @@ def maven_cmd(test, add_time=False):
     # surefire:test reuses test build from last compilation
     # if you modified the test and want to rerun it, you must use `mvn test`
     test_mode = "surefire:test" if use_surefire else "test"
-    cmd = ["mvn", test_mode, "-Dtest={}".format(test)] + maven_args
+    if is_gradle:
+        cmd = ["./gradlew", "-Prerun-tests", "core:test", "--tests", test, "-i"]
+    else:
+        cmd = ["mvn", test_mode, "-Dtest={}".format(test)] + maven_args
     if add_time:
         cmd = ["time"] + cmd
     print(">>>>[ctest_core] command: " + " ".join(cmd))