hot restart: provide a mechanism for obtaining a base-id dynamically (envoyproxy#11357)

Provides a --use-dynamic-base-id flag to select an unused base-id.
Primarily useful for testing, but generally available. Adds a
--base-id-path flag where Envoy writes the base id to a file.
Converts tests to use the dynamic base id selection rather than
trying to keep all base ids unique.

Signed-off-by: Stephan Zuercher <[email protected]>

Author: zuercher

api/envoy/admin/v3/server_info.proto

@@ -54,7 +54,7 @@ message ServerInfo {
   CommandLineOptions command_line_options = 6;
 }
 
-// [#next-free-field: 31]
+// [#next-free-field: 33]
 message CommandLineOptions {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.admin.v2alpha.CommandLineOptions";
@@ -82,6 +82,12 @@ message CommandLineOptions {
   // See :option:`--base-id` for details.
   uint64 base_id = 1;
 
+  // See :option:`--use-dynamic-base-id` for details.
+  bool use_dynamic_base_id = 31;
+
+  // See :option:`--base-id-path` for details.
+  string base_id_path = 32;
+
   // See :option:`--concurrency` for details.
   uint32 concurrency = 2;
 

api/envoy/admin/v4alpha/server_info.proto

@@ -54,7 +54,7 @@ message ServerInfo {
   CommandLineOptions command_line_options = 6;
 }
 
-// [#next-free-field: 31]
+// [#next-free-field: 33]
 message CommandLineOptions {
   option (udpa.annotations.versioning).previous_message_type = "envoy.admin.v3.CommandLineOptions";
 
@@ -81,6 +81,12 @@ message CommandLineOptions {
   // See :option:`--base-id` for details.
   uint64 base_id = 1;
 
+  // See :option:`--use-dynamic-base-id` for details.
+  bool use_dynamic_base_id = 31;
+
+  // See :option:`--base-id-path` for details.
+  string base_id_path = 32;
+
   // See :option:`--concurrency` for details.
   uint32 concurrency = 2;
 

docs/root/intro/arch_overview/operations/hot_restart.rst

@@ -26,3 +26,9 @@ hot restart functionality has the following general architecture:
   the processes takes place only using unix domain sockets.
 * An example restarter/parent process written in Python is included in the source distribution. This
   parent process is usable with standard process control utilities such as monit/runit/etc.
+
+Envoy's default command line options assume that only a single set of Envoy processes is running on
+a given host: an active Envoy server process and, potentially, a draining Envoy server process that
+will exit as described above. The :option:`--base-id` or :option:`--use-dynamic-base-id` options
+may be used to allow multiple, distinctly configured Envoys to run on the same host and hot restart
+independently.

docs/root/operations/cli.rst

@@ -66,10 +66,24 @@ following are the command line options that Envoy supports.
   set this option. However, if Envoy needs to be run multiple times on the same machine, each
   running Envoy will need a unique base ID so that the shared memory regions do not conflict.
 
+.. option:: --use-dynamic-base-id
+
+  *(optional)* Selects an unused base ID to use when allocating shared memory regions. Using
+  preselected values with :option:`--base-id` is preferred, however. If this option is enabled,
+  it supersedes the :option:`--base-id` value. This flag may not be used when the value of
+  :option:`--restart-epoch` is non-zero. Instead, for subsequent hot restarts, set
+  :option:`--base-id` option with the selected base ID. See :option:`--base-id-path`.
+
+.. option:: --base-id-path <path_string>
+
+  *(optional)* Writes the base ID to the given path. While this option is compatible with
+  :option:`--base-id`, its intended use is to provide access to the dynamic base ID selected by
+  :option:`--use-dynamic-base-id`.
+
 .. option:: --concurrency <integer>
 
   *(optional)* The number of :ref:`worker threads <arch_overview_threading>` to run. If not
-  specified defaults to the number of hardware threads on the machine. If set to zero, Envoy will 
+  specified defaults to the number of hardware threads on the machine. If set to zero, Envoy will
   still run one worker thread.
 
 .. option:: -l <string>, --log-level <string>
@@ -79,9 +93,9 @@ following are the command line options that Envoy supports.
 
 .. option:: --component-log-level <string>
 
-  *(optional)* The comma separated list of logging level per component. Non developers should generally 
-  never set this option. For example, if you want `upstream` component to run at `debug` level and 
-  `connection` component to run at `trace` level, you should pass ``upstream:debug,connection:trace`` to 
+  *(optional)* The comma separated list of logging level per component. Non developers should generally
+  never set this option. For example, if you want `upstream` component to run at `debug` level and
+  `connection` component to run at `trace` level, you should pass ``upstream:debug,connection:trace`` to
   this flag. See ``ALL_LOGGER_IDS`` in :repo:`/source/common/common/logger.h` for a list of components.
 
 .. option:: --cpuset-threads
@@ -239,11 +253,11 @@ following are the command line options that Envoy supports.
 
 .. option:: --drain-time-s <integer>
 
-  *(optional)* The time in seconds that Envoy will drain connections during 
+  *(optional)* The time in seconds that Envoy will drain connections during
   a :ref:`hot restart <arch_overview_hot_restart>` or when individual listeners are being
-  modified or removed via :ref:`LDS <arch_overview_dynamic_config_lds>`. 
-  Defaults to 600 seconds (10 minutes). Generally the drain time should be less than 
-  the parent shutdown time set via the :option:`--parent-shutdown-time-s` option. How the two 
+  modified or removed via :ref:`LDS <arch_overview_dynamic_config_lds>`.
+  Defaults to 600 seconds (10 minutes). Generally the drain time should be less than
+  the parent shutdown time set via the :option:`--parent-shutdown-time-s` option. How the two
   settings are configured depends on the specific deployment. In edge scenarios, it might be
   desirable to have a very long drain time. In service to service scenarios, it might be possible
   to make the drain and shutdown time much shorter (e.g., 60s/90s).

docs/root/operations/hot_restarter.rst

@@ -21,15 +21,22 @@ The restarter is invoked like so:
 
   ulimit -n {{ pillar.get('envoy_max_open_files', '102400') }}
   sysctl fs.inotify.max_user_watches={{ pillar.get('envoy_max_inotify_watches', '524288') }}
-  
+
   exec /usr/sbin/envoy -c /etc/envoy/envoy.cfg --restart-epoch $RESTART_EPOCH --service-cluster {{ grains['cluster_name'] }} --service-node {{ grains['service_node'] }} --service-zone {{ grains.get('ec2_availability-zone', 'unknown') }}
 
 Note on `inotify.max_user_watches`: If Envoy is being configured to watch many files for configuration in a directory
 on a Linux machine, increase this value as Linux enforces limits on the maximum number of files that can be watched.
-  
-The *RESTART_EPOCH* environment variable is set by the restarter on each restart and can be passed
+
+The *RESTART_EPOCH* environment variable is set by the restarter on each restart and must be passed
 to the :option:`--restart-epoch` option.
 
+.. warning::
+
+   Special care must be taken if you wish to use the :option:`--use-dynamic-base-id` option. That
+   flag may only be set when the *RESTART_EPOCH* is 0 and your *start_envoy.sh* must obtain the
+   chosen base ID (via :option:`--base-id-path`), store it, and use it as the :option:`--base-id`
+   value on subsequent invocations (when *RESTART_EPOCH* is greater than 0).
+
 The restarter handles the following signals:
 
 * **SIGTERM** or **SIGINT** (Ctrl-C): Will cleanly terminate all child processes and exit.

docs/root/version_history/current.rst

@@ -13,6 +13,7 @@ Minor Behavior Changes
 *Changes that may cause incompatibilities for some users, but should not for most*
 
 * access loggers: applied existing buffer limits to access logs, as well as :ref:`stats <config_access_log_stats>` for logged / dropped logs. This can be reverted temporarily by setting runtime feature `envoy.reloadable_features.disallow_unbounded_access_logs` to false.
+* hot restart: added the option :option:`--use-dynamic-base-id` to select an unused base ID at startup and the option :option:`--base-id-path` to write the base id to a file (for reuse with later hot restarts).
 * http: fixed several bugs with applying correct connection close behavior across the http connection manager, health checker, and connection pool. This behavior may be temporarily reverted by setting runtime feature `envoy.reloadable_features.fix_connection_close` to false.
 * http: fixed a bug where the upgrade header was not cleared on responses to non-upgrade requests.
   Can be reverted temporarily by setting runtime feature `envoy.reloadable_features.fix_upgrade_response` to false.
@@ -68,7 +69,7 @@ New Features
 * listener: added in place filter chain update flow for tcp listener update which doesn't close connections if the corresponding network filter chain is equivalent during the listener update.
   Can be disabled by setting runtime feature `envoy.reloadable_features.listener_in_place_filterchain_update` to false.
   Also added additional draining filter chain stat for :ref:`listener manager <config_listener_manager_stats>` to track the number of draining filter chains and the number of in place update attempts.
-* logger: added :ref:`--log-format-prefix-with-location <operations_cli>` command line option to prefix '%v' with file path and line number.
+* logger: added :option:`--log-format-prefix-with-location` command line option to prefix '%v' with file path and line number.
 * lrs: added new *envoy_api_field_service.load_stats.v2.LoadStatsResponse.send_all_clusters* field
   in LRS response, which allows management servers to avoid explicitly listing all clusters it is
   interested in; behavior is allowed based on new "envoy.lrs.supports_send_all_clusters" capability

generated_api_shadow/envoy/admin/v3/server_info.proto

@@ -79,6 +79,11 @@ class HotRestart {
    */
   virtual void shutdown() PURE;
 
+  /**
+   * Return the base id used to generate a domain socket name.
+   */
+  virtual uint32_t baseId() PURE;
+
   /**
    * Return the hot restart compatibility version so that operations code can decide whether to
    * perform a full or hot restart.
@@ -96,5 +101,14 @@ class HotRestart {
   virtual Thread::BasicLockable& accessLogLock() PURE;
 };
 
+/**
+ * HotRestartDomainSocketInUseException is thrown during HotRestart construction only when the
+ * underlying domain socket is in use.
+ */
+class HotRestartDomainSocketInUseException : public EnvoyException {
+public:
+  HotRestartDomainSocketInUseException(const std::string& what) : EnvoyException(what) {}
+};
+
 } // namespace Server
 } // namespace Envoy

generated_api_shadow/envoy/admin/v4alpha/server_info.proto

@@ -59,6 +59,17 @@ class Options {
    */
   virtual uint64_t baseId() const PURE;
 
+  /**
+   * @return bool choose an unused base ID dynamically. The chosen base id can be written to a
+   *         a file using the baseIdPath option.
+   */
+  virtual bool useDynamicBaseId() const PURE;
+
+  /**
+   * @return const std::string& the dynamic base id output file.
+   */
+  virtual const std::string& baseIdPath() const PURE;
+
   /**
    * @return the number of worker threads to run in the server.
    */

include/envoy/server/hot_restart.h

@@ -1,12 +1,14 @@
 #include "exe/main_common.h"
 
+#include <fstream>
 #include <iostream>
 #include <memory>
 #include <new>
 
 #include "envoy/config/listener/v3/listener.pb.h"
 
 #include "common/common/compiler_requirements.h"
+#include "common/common/logger.h"
 #include "common/common/perf_annotation.h"
 #include "common/network/utility.h"
 #include "common/stats/symbol_table_creator.h"
@@ -58,14 +60,7 @@ MainCommonBase::MainCommonBase(const OptionsImpl& options, Event::TimeSystem& ti
   switch (options_.mode()) {
   case Server::Mode::InitOnly:
   case Server::Mode::Serve: {
-#ifdef ENVOY_HOT_RESTART
-    if (!options.hotRestartDisabled()) {
-      restarter_ = std::make_unique<Server::HotRestartImpl>(options_);
-    }
-#endif
-    if (restarter_ == nullptr) {
-      restarter_ = std::make_unique<Server::HotRestartNopImpl>();
-    }
+    configureHotRestarter(*random_generator);
 
     tls_ = std::make_unique<ThreadLocal::InstanceImpl>();
     Thread::BasicLockable& log_lock = restarter_->logLock();
@@ -106,6 +101,60 @@ void MainCommonBase::configureComponentLogLevels() {
   }
 }
 
+void MainCommonBase::configureHotRestarter(Runtime::RandomGenerator& random_generator) {
+#ifdef ENVOY_HOT_RESTART
+  if (!options_.hotRestartDisabled()) {
+    uint32_t base_id = options_.baseId();
+
+    if (options_.useDynamicBaseId()) {
+      ASSERT(options_.restartEpoch() == 0, "cannot use dynamic base id during hot restart");
+
+      std::unique_ptr<Server::HotRestart> restarter;
+
+      // Try 100 times to get an unused base ID and then give up under the assumption
+      // that some other problem has occurred to prevent binding the domain socket.
+      for (int i = 0; i < 100 && restarter == nullptr; i++) {
+        // HotRestartImpl is going to multiply this value by 10, so leave head room.
+        base_id = static_cast<uint32_t>(random_generator.random()) & 0x0FFFFFFF;
+
+        try {
+          restarter = std::make_unique<Server::HotRestartImpl>(base_id, 0);
+        } catch (Server::HotRestartDomainSocketInUseException& ex) {
+          // No luck, try again.
+          ENVOY_LOG_MISC(debug, "dynamic base id: {}", ex.what());
+        }
+      }
+
+      if (restarter == nullptr) {
+        throw EnvoyException("unable to select a dynamic base id");
+      }
+
+      restarter_.swap(restarter);
+    } else {
+      restarter_ = std::make_unique<Server::HotRestartImpl>(base_id, options_.restartEpoch());
+    }
+
+    // Write the base-id to the requested path whether we selected it
+    // dynamically or not.
+    if (!options_.baseIdPath().empty()) {
+      std::ofstream base_id_out_file(options_.baseIdPath());
+      if (!base_id_out_file) {
+        ENVOY_LOG_MISC(critical, "cannot open base id output file {} for writing.",
+                       options_.baseIdPath());
+      } else {
+        base_id_out_file << base_id;
+      }
+    }
+  }
+#else
+  UNREFERENCED_PARAMETER(random_generator);
+#endif
+
+  if (restarter_ == nullptr) {
+    restarter_ = std::make_unique<Server::HotRestartNopImpl>();
+  }
+}
+
 bool MainCommonBase::run() {
   switch (options_.mode()) {
   case Server::Mode::Serve:

include/envoy/server/options.h

@@ -87,6 +87,7 @@ class MainCommonBase {
 
 private:
   void configureComponentLogLevels();
+  void configureHotRestarter(Runtime::RandomGenerator& random_generator);
 };
 
 // TODO(jmarantz): consider removing this class; I think it'd be more useful to