Issue 49: Revisit memory settings (#50)

To make memory settings consistent across Kafka, Pravega and Pulsar, as well as with the new Confluent Kafka Driver.

Signed-off-by: Raúl Gracia <[email protected]>
Co-authored-by: jingerbread <[email protected]>

driver-kafka/deploy/deploy.yaml

@@ -160,7 +160,7 @@
       lineinfile:
          dest: /opt/benchmark/bin/benchmark-worker
          regexp: '^JVM_MEM='
-         line: 'JVM_MEM="-Xms6G -Xmx6G -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking"'
+         line: 'JVM_MEM="-Xms16G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking"'
     - name: Configure memory
       lineinfile:
          dest: /opt/benchmark/bin/benchmark

driver-pravega/deploy/deploy.yaml

@@ -461,7 +461,7 @@
       lineinfile:
          dest: /opt/benchmark/bin/benchmark-worker
          regexp: '^JVM_MEM='
-         line: 'JVM_MEM="-Xms6G -Xmx6G -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking"'
+         line: 'JVM_MEM="-Xms16G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking"'
     - name: Configure memory
       lineinfile:
          dest: /opt/benchmark/bin/benchmark

driver-pravega/deploy/templates/pravega-segmentstore.service

@@ -3,6 +3,7 @@ Description=Pravega Segment Store
 After=network.target
 
 [Service]
+Environment="PRAVEGA_SEGMENTSTORE_OPTS=-Xms6G -Xmx6G -XX:MaxDirectMemorySize=20g"
 ExecStart=/opt/pravega/bin/pravega-segmentstore
 WorkingDirectory=/opt/pravega
 RestartSec=15s

driver-pulsar/deploy/deploy.yaml

@@ -93,6 +93,9 @@
     - template:
         src: "templates/pulsar_env.sh"
         dest: "/opt/pulsar/conf/pulsar_env.sh"
+    - template:
+        src: "templates/bkenv.sh"
+        dest: "/opt/pulsar/conf/bkenv.sh"
     - name: Change locale to en_US.utf-8
       shell: |
         echo 'LANG=en_US.utf-8
@@ -237,7 +240,7 @@
       lineinfile:
          dest: /opt/benchmark/bin/benchmark-worker
          regexp: '^JVM_MEM='
-         line: 'JVM_MEM="-Xms6G -Xmx6G -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking"'
+         line: 'JVM_MEM="-Xms16G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking"'
     - name: Configure memory
       lineinfile:
          dest: /opt/benchmark/bin/benchmark

driver-pulsar/deploy/templates/bkenv.sh

@@ -0,0 +1,84 @@
+#!/bin/sh
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Set JAVA_HOME here to override the environment setting
+# JAVA_HOME=
+
+########################################
+# default settings for bookkeeper
+########################################
+
+# Configuration file of settings used in bookie server
+# BOOKIE_CONF=
+
+# Configuration file of settings used in zookeeper server
+# BOOKIE_ZK_CONF=
+
+# Extra options to be passed to the jvm
+# BOOKIE_EXTRA_OPTS=
+
+# Add extra paths to the bookkeeper classpath
+# BOOKIE_EXTRA_CLASSPATH=
+
+# Folder where the Bookie server PID file should be stored
+# BOOKIE_PID_DIR=
+
+# Wait time before forcefully kill the Bookie server instance, if the stop is not successful
+# BOOKIE_STOP_TIMEOUT=
+
+# Entry formatter class to format entries.
+# ENTRY_FORMATTER_CLASS=
+
+# this default config dir should match the 'localBookiesConfigDirectory' config value in the conf file of LocalBookKeeper
+# LOCALBOOKIES_CONFIG_DIR=/tmp/localbookies-config
+
+#################################
+# BookKeeper Logging Options
+#################################
+
+# Log4j configuration file
+# BOOKIE_LOG_CONF=
+
+# Logs location
+# BOOKIE_LOG_DIR=
+
+# Log file name
+# BOOKIE_LOG_FILE="bookkeeper.log"
+
+# Log level & appender
+# BOOKIE_ROOT_LOGGER="INFO,CONSOLE"
+
+#################################
+# BookKeeper JVM memory options
+#################################
+
+BOOKIE_MAX_HEAP_MEMORY=6g
+BOOKIE_MIN_HEAP_MEMORY=6g
+BOOKIE_MAX_DIRECT_MEMORY=6g
+# BOOKIE_MEM_OPTS=
+
+# JVM GC options
+# BOOKIE_GC_OPTS=
+
+# JVM GC logging options
+# BOOKIE_GC_LOGGING_OPTS=
+
+# JVM performance options
+# BOOKIE_PERF_OPTS="-XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch -XX:-UseBiasedLocking"