Merge pull request #11 from jagerman/multipart

Add support for large, multi-part config messages

Author: mpretty-cyro

.gitmodules

@@ -19,3 +19,6 @@
 [submodule "external/protobuf"]
 	path = external/protobuf
 	url = https://github.com/protocolbuffers/protobuf.git
+[submodule "external/oxen-logging"]
+	path = external/oxen-logging
+	url = https://github.com/oxen-io/oxen-logging.git

CMakeLists.txt

@@ -17,7 +17,7 @@ if(CCACHE_PROGRAM)
 endif()
 
 project(libsession-util
-    VERSION 1.3.1
+    VERSION 1.4.0
     DESCRIPTION "Session client utility library"
     LANGUAGES ${LANGS})
 

external/CMakeLists.txt

@@ -126,7 +126,6 @@ if(APPLE AND CMAKE_CXX_COMPILER_ID STREQUAL AppleClang AND NOT CMAKE_CXX_COMPILE
     oxen_logging_add_source_dir("${CMAKE_CURRENT_SOURCE_DIR}/oxen-libquic/external/oxen-logging/include/oxen")
 endif()
 
-
 if(CMAKE_C_COMPILER_LAUNCHER)
   set(deps_cc "${CMAKE_C_COMPILER_LAUNCHER} ${deps_cc}")
 endif()

external/oxen-logging

@@ -15,8 +15,20 @@
 
 namespace session::config {
 
-// FIXME: for multi-message we encode to longer and then split it up
-inline constexpr int MAX_MESSAGE_SIZE = 76800;  // 76.8kB = Storage server's limit
+// The maximum size of a single message in storage server.  For larger configs we have to split a
+// config message into a multipart message.
+inline constexpr int MAX_MESSAGE_SIZE = 76'800;  // 76.8kB = Storage server's limit
+
+// Maximum size of a post-compression outgoing multipart config message (before chopping into
+// multiple parts, and so not counting the small encryption and encoding overhead of each piece)
+// that we will allow.  This is not a strict network limit, but rather is simply to prevent bugs
+// that have some sort of accidental runaway size.  This limit does not apply to *incoming*
+// multipart messages so that if a future version changes the limit it will not break compatibility
+// with existing clients.
+//
+// Also note that we encode part index and length as 1 byte each, so this must be small enough to
+// not exceed 255 parts (but that would be insane anyway).
+inline constexpr int MAX_MULTIPART_SIZE = 5'000'000;
 
 // Application data data types:
 using scalar = std::variant<int64_t, std::string>;
@@ -337,11 +349,12 @@ class MutableConfigMessage : public ConfigMessage {
     /// pruning.
     bool prune();
 
-    /// Calculates the hash of the current message.  Can optionally be given the already-serialized
-    /// value, if available; if empty/omitted, `serialize()` will be called to compute it.
+    /// Calculates the hash of the current message.
     const hash_t& hash() override;
 
   protected:
+    /// Internal version of hash() that takes the already-serialized value, to avoid needing a call
+    /// to `serialize()` when such a call has already been done for other reasons.
     const hash_t& hash(std::span<const unsigned char> serialized);
     void increment_impl();
 };

include/session/config.hpp

@@ -123,10 +123,12 @@ LIBSESSION_EXPORT bool config_needs_push(const config_object* conf);
 typedef struct config_push_data {
     // The config seqno (to be provided later in `config_confirm_pushed`).
     seqno_t seqno;
-    // The config message to push (binary data, not null-terminated).
-    unsigned char* config;
-    // The length of `config`
-    size_t config_len;
+    // Array of config message(s) to push (each is binary data, not null-terminated).
+    unsigned char** config;
+    // Array of lengths of the messages in `config`
+    size_t* config_lens;
+    // Length of config and config_len arrays.
+    size_t n_configs;
     // Array of obsolete message hashes to delete; each element is a null-terminated C string
     char** obsolete;
     // length of `obsolete`
@@ -161,24 +163,28 @@ LIBSESSION_EXPORT config_push_data* config_push(config_object* conf);
 /// API: base/config_confirm_pushed
 ///
 /// Reports that data obtained from `config_push` has been successfully stored on the server with
-/// message hash `msg_hash`.  The seqno value is the one returned by the config_push call that
-/// yielded the config data.
+/// message hash(es) `msg_hashes` containing `hashes_len` hashes.  The seqno value is the one
+/// returned by the config_push call that yielded the config data.  The order of hashes in the
+/// msg_hashes array is unimportant.
 ///
 /// Declaration:
 /// ```cpp
 /// VOID config_confirm_pushed(
 ///     [in, out]   config_object*      conf,
-///     [out]       seqno_t             seqno,
-///     [out]       const char*         msg_hash
+///     [in]        seqno_t             seqno,
+///     [in]        const char* const*  msg_hashes
+///     [in]        size_t              hashes_len
 /// );
 /// ```
 ///
 /// Inputs:
 /// - `conf` -- [in] Pointer to config_object object
-/// - `seqno` -- [out] Value returned by config_push call
-/// - `msg_hash` -- [out] Value returned by config_push call
+/// - `seqno` -- [in] Value returned by config_push call
+/// - `msg_hashes` -- [in] array of message hashes (null terminated C strings) returned by the
+///   storage server when stored.
+/// - `hashes_len` -- [in] length of the `msg_hashes` array
 LIBSESSION_EXPORT void config_confirm_pushed(
-        config_object* conf, seqno_t seqno, const char* msg_hash);
+        config_object* conf, seqno_t seqno, const char* const* msg_hashes, size_t hashes_len);
 
 /// API: base/config_dump
 ///
@@ -229,17 +235,46 @@ LIBSESSION_EXPORT bool config_dump(config_object* conf, unsigned char** out, siz
 /// - `bool` -- True if config has changed since last call to `dump()`
 LIBSESSION_EXPORT bool config_needs_dump(const config_object* conf);
 
-/// API: base/config_current_hashes
+/// API: base/config_curr_hashes
 ///
-/// Obtains the current active hashes.  Note that this will be empty if the current hash is unknown
-/// or not yet determined (for example, because the current state is dirty or because the most
-/// recent push is still pending and we don't know the hash yet).
+/// Obtains the hashes of the current config state.  Note that this will be empty if the current
+/// hash is unknown or not yet determined (for example, because the current state is dirty or
+/// because the most recent push is still pending and we don't know the hash yet).
+///
+/// See also config_active_hashes(), which you often want to use instead of this.
+///
+/// The returned pointer belongs to the caller and must be freed via `free()` when done with it.
+///
+/// Declaration:
+/// ```cpp
+/// CONFIG_STRING_LIST* config_curr_hashes(
+///     [in]    const config_object*          conf
+/// );
+///
+/// ```
+///
+/// Inputs:
+/// - `conf` -- [in] Pointer to config_object object
+///
+/// Outputs:
+/// - `config_string_list*` -- pointer to the list of hashes; the pointer belongs to the caller
+LIBSESSION_EXPORT config_string_list* config_curr_hashes(const config_object* conf)
+        LIBSESSION_WARN_UNUSED;
+
+/// API: base/config_active_hashes
+///
+/// Obtains the hashes of currently active config messages.  This includes both the hashes of the
+/// current config (as returned by config_curr_hashes), but also any partially-arrived multipart
+/// config messages for which we are still waiting for remaining parts to complete the config.
+///
+/// The returned array of hashes have no particular ordering (and may differ from one call to the
+/// next even without any change to the values considered as a set).
 ///
 /// The returned pointer belongs to the caller and must be freed via `free()` when done with it.
 ///
 /// Declaration:
 /// ```cpp
-/// CONFIG_STRING_LIST* config_current_hashes(
+/// CONFIG_STRING_LIST* config_active_hashes(
 ///     [in]    const config_object*          conf
 /// );
 ///
@@ -250,7 +285,7 @@ LIBSESSION_EXPORT bool config_needs_dump(const config_object* conf);
 ///
 /// Outputs:
 /// - `config_string_list*` -- pointer to the list of hashes; the pointer belongs to the caller
-LIBSESSION_EXPORT config_string_list* config_current_hashes(const config_object* conf)
+LIBSESSION_EXPORT config_string_list* config_active_hashes(const config_object* conf)
         LIBSESSION_WARN_UNUSED;
 
 /// API: base/config_old_hashes