Implement rfc8785

Fix: #1038

Directly lift https://github.com/dryruby/json-canonicalization

At least for now, implementing most of the logic in Ruby make it
shareable between the 3 generator implementations, largely reducing
the maintenance cost.

I don't particularly expect canonicalization to be a hotspot,
but if it happens to be, I'd be open to specialize the implementation,
within reason, at least in the C generator.

Author: byroot

ext/json/ext/generator/generator.c

@@ -27,14 +27,16 @@ typedef struct JSON_Generator_StateStruct {
     bool ascii_only;
     bool script_safe;
     bool strict;
+    bool rfc8785;
     VALUE sort_keys;
 } JSON_Generator_State;
 
-static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8, default_sort_keys_proc;
+static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8, default_sort_keys_proc,
+             rfc8785_number_formater_proc, rfc8785_sort_keys_proc;
 
 static ID i_to_s, i_to_json, i_new, i_encode;
 static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, sym_allow_duplicate_key,
-             sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_strict, sym_as_json, sym_sort_keys;
+             sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_strict, sym_as_json, sym_sort_keys, sym_rfc8785;
 
 
 #define GET_STATE_TO(self, state) \
@@ -1052,7 +1054,10 @@ static inline long increase_depth(struct generate_json_data *data)
 
 static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
-    if (RB_UNLIKELY(data->state->sort_keys)) {
+    if (RB_UNLIKELY(data->state->rfc8785)) {
+        obj = rb_proc_call_with_block(rfc8785_sort_keys_proc, 1, &obj, Qnil);
+        Check_Type(obj, T_HASH);
+    } else if (RB_UNLIKELY(data->state->sort_keys)) {
         obj = rb_proc_call_with_block(data->state->sort_keys, 1, &obj, Qnil);
         Check_Type(obj, T_HASH);
     }
@@ -1180,6 +1185,13 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data
         return;
     }
 
+    if (RB_UNLIKELY(data->state->rfc8785)) {
+        VALUE str = rb_proc_call_with_block(rfc8785_number_formater_proc, 1, &obj, Qnil);
+        Check_Type(str, T_STRING);
+        fbuffer_append_str(buffer, str);
+        return;
+    }
+
     /* This implementation writes directly into the buffer. We reserve
      * the 32 characters that fpconv_dtoa states as its maximum.
      */
@@ -1757,6 +1769,28 @@ static VALUE cState_set_default_sort_keys_proc(VALUE self, VALUE proc)
     return default_sort_keys_proc = proc;
 }
 
+static VALUE cState_set_rfc8785_number_formater_proc(VALUE self, VALUE proc)
+{
+    if (!rb_obj_is_proc(proc)) {
+        rb_raise(rb_eTypeError, "rfc8785_number_formater_proc must be a Proc");
+    }
+    if (rfc8785_number_formater_proc) {
+        rb_raise(rb_eArgError, "rfc8785_number_formater_proc can only be set once");
+    }
+    return rfc8785_number_formater_proc = proc;
+}
+
+static VALUE cState_set_rfc8785_sort_keys_proc(VALUE self, VALUE proc)
+{
+    if (!rb_obj_is_proc(proc)) {
+        rb_raise(rb_eTypeError, "rfc8785_sort_keys_proc must be a Proc");
+    }
+    if (rfc8785_sort_keys_proc) {
+        rb_raise(rb_eArgError, "rfc8785_sort_keys_proc can only be set once");
+    }
+    return rfc8785_sort_keys_proc = proc;
+}
+
 static VALUE normalize_sort_keys(VALUE value)
 {
     if (rb_obj_is_proc(value)) {
@@ -1775,7 +1809,7 @@ static VALUE normalize_sort_keys(VALUE value)
  *
  * Get the value of sort_keys.
  */
-static VALUE cState_sort_keys_p(VALUE self)
+static VALUE cState_sort_keys(VALUE self)
 {
     GET_STATE(self);
     return state->sort_keys;
@@ -1795,7 +1829,33 @@ static VALUE cState_sort_keys_set(VALUE self, VALUE value)
     rb_check_frozen(self);
     GET_STATE(self);
     RB_OBJ_WRITE(self, &state->sort_keys, normalize_sort_keys(value));
-    return Qnil;
+    return state->sort_keys;
+}
+
+
+/*
+ * call-seq: rfc8785?
+ *
+ * Get whether RFC8785 (canonicalization) is enabled.
+ */
+static VALUE cState_rfc8785_p(VALUE self)
+{
+    GET_STATE(self);
+    return state->rfc8785 ? Qtrue : Qfalse;
+}
+
+/*
+ * call-seq: rfc8785=
+ *
+ * Sets whether RFC8785 (canonicalization) is enabled.
+ */
+
+static VALUE cState_rfc8785_set(VALUE self, VALUE value)
+{
+    rb_check_frozen(self);
+    GET_STATE(self);
+    state->rfc8785 = RTEST(value);
+    return state->rfc8785 ? Qtrue : Qfalse;
 }
 
 static VALUE cState_allow_duplicate_key_p(VALUE self)
@@ -1895,6 +1955,7 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg)
     else if (key == sym_script_safe)           { state->script_safe = RTEST(val); }
     else if (key == sym_strict)                { state->strict = RTEST(val); }
     else if (key == sym_allow_duplicate_key)   { state->allow_duplicate_key = RTEST(val); }
+    else if (key == sym_rfc8785)               { state->rfc8785 = RTEST(val); }
     else if (key == sym_as_json)               {
         VALUE proc = RTEST(val) ? as_json_config(val) : Qfalse;
         state->as_json_single_arg = proc && rb_proc_arity(proc) == 1;
@@ -1990,6 +2051,8 @@ void Init_generator(void)
     VALUE mGenerator = rb_define_module_under(mExt, "Generator");
 
     rb_global_variable(&default_sort_keys_proc);
+    rb_global_variable(&rfc8785_number_formater_proc);
+    rb_global_variable(&rfc8785_sort_keys_proc);
 
     rb_global_variable(&eGeneratorError);
     eGeneratorError = rb_path2class("JSON::GeneratorError");
@@ -2001,6 +2064,8 @@ void Init_generator(void)
     rb_define_alloc_func(cState, cState_s_allocate);
     rb_define_singleton_method(cState, "from_state", cState_from_state_s, 1);
     rb_define_singleton_method(cState, "default_sort_keys_proc=", cState_set_default_sort_keys_proc, 1);
+    rb_define_singleton_method(cState, "rfc8785_number_formater_proc=", cState_set_rfc8785_number_formater_proc, 1);
+    rb_define_singleton_method(cState, "rfc8785_sort_keys_proc=", cState_set_rfc8785_sort_keys_proc, 1);
 
     rb_define_method(cState, "initialize", cState_initialize, -1);
     rb_define_alias(cState, "initialize", "initialize"); // avoid method redefinition warnings
@@ -2038,8 +2103,10 @@ void Init_generator(void)
     rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1);
     rb_define_method(cState, "generate", cState_generate, -1);
     rb_define_method(cState, "_generate_no_fallback", cState_generate_no_fallback, -1);
-    rb_define_method(cState, "sort_keys", cState_sort_keys_p, 0);
+    rb_define_method(cState, "sort_keys", cState_sort_keys, 0);
     rb_define_method(cState, "sort_keys=", cState_sort_keys_set, 1);
+    rb_define_method(cState, "rfc8785?", cState_rfc8785_p, 0);
+    rb_define_method(cState, "rfc8785=", cState_rfc8785_set, 1);
 
     rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0);
 
@@ -2069,6 +2136,7 @@ void Init_generator(void)
     sym_as_json = ID2SYM(rb_intern("as_json"));
     sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key"));
     sym_sort_keys = ID2SYM(rb_intern("sort_keys"));
+    sym_rfc8785 = ID2SYM(rb_intern("rfc8785"));
 
     usascii_encindex = rb_usascii_encindex();
     utf8_encindex = rb_utf8_encindex();

java/src/json/ext/Generator.java

@@ -403,11 +403,10 @@ void generate(ThreadContext context, Session session, RubyFloat object, OutputSt
     }
 
     static void generateFloat(ThreadContext context, Session session, RubyFloat object, OutputStream buffer) throws IOException {
+        final GeneratorState state = session.getState(context);
         double value = object.getValue();
 
         if (Double.isInfinite(value) || Double.isNaN(value)) {
-            GeneratorState state = session.getState(context);
-
             if (!state.allowNaN()) {
                 if (state.strict() && state.getAsJSON() != null) {
                     IRubyObject castedValue = state.getAsJSON().call(context, object, context.getRuntime().getFalse());
@@ -421,7 +420,13 @@ static void generateFloat(ThreadContext context, Session session, RubyFloat obje
             }
         }
 
-        buffer.write(Double.toString(value).getBytes(UTF_8));
+        if (state.rfc8785()) {
+            RubyProc numberProc = (RubyProc)GeneratorState.rfc8785NumberFormaterProc;
+            RubyString numberString = (RubyString)Helpers.invoke(context, numberProc, "call", object);
+            buffer.write(numberString.toString().getBytes(UTF_8));
+        } else {
+            buffer.write(Double.toString(value).getBytes(UTF_8));
+        }
     }
 
     private static final byte[] EMPTY_ARRAY_BYTES = "[]".getBytes();
@@ -570,8 +575,12 @@ static void generateHash(ThreadContext context, Session session, RubyHash object
         }
 
         RubyProc sortKeysProc = state.getSortKeysProc();
+        if (state.rfc8785()) {
+            sortKeysProc = (RubyProc)GeneratorState.rfc8785SortKeysProc;
+        }
+
         if (sortKeysProc != null) {
-            object = (RubyHash) Helpers.invoke(context, sortKeysProc, "call", object);
+            object = (RubyHash)Helpers.invoke(context, sortKeysProc, "call", object);
         }
 
         final ByteList objectNl = state.getObjectNl();

java/src/json/ext/GeneratorState.java

@@ -37,6 +37,8 @@ public class GeneratorState extends RubyObject {
     private boolean allowDuplicateKey = false;
 
     private static IRubyObject defaultSortKeyProc;
+    public static IRubyObject rfc8785NumberFormaterProc;
+    public static IRubyObject rfc8785SortKeysProc;
 
     /**
      * The indenting unit string. Will be repeated several times for larger
@@ -112,6 +114,8 @@ public class GeneratorState extends RubyObject {
      */
     private IRubyObject sortKeys;
 
+    private boolean rfc8785 = false;
+
     /**
      * The current depth (inside a #to_json call)
      */
@@ -172,6 +176,18 @@ public static IRubyObject setDefaultSortKeyProc(IRubyObject klass, IRubyObject p
         return proc;
     }
 
+    @JRubyMethod(meta=true, name="rfc8785_number_formater_proc=")
+    public static IRubyObject setRfc8785NumberFormaterProc(IRubyObject klass, IRubyObject proc) {
+        rfc8785NumberFormaterProc = proc;
+        return proc;
+    }
+
+    @JRubyMethod(meta=true, name="rfc8785_sort_keys_proc=")
+    public static IRubyObject setRfc8785SortKeysProc(IRubyObject klass, IRubyObject proc) {
+        rfc8785SortKeysProc = proc;
+        return proc;
+    }
+
     /**
      * <code>State#initialize(opts = {})</code>
      * <p>
@@ -493,6 +509,22 @@ public IRubyObject sort_keys_set(ThreadContext context, IRubyObject sortKeys) {
         return sortKeys;
     }
 
+    @JRubyMethod(name="rfc8785?")
+    public IRubyObject rfc8785_p(ThreadContext context) {
+        return RubyBoolean.newBoolean(context, rfc8785);
+    }
+
+    @JRubyMethod(name="rfc8785=")
+    public IRubyObject rfc8785_set(ThreadContext context, IRubyObject rfc8785) {
+        checkFrozen();
+        this.rfc8785 = rfc8785.isTrue();
+        return rfc8785;
+    }
+
+    public boolean rfc8785() {
+        return this.rfc8785;
+    }
+
     public int getDepth() {
         return depth;
     }
@@ -568,6 +600,8 @@ public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) {
 
         if (opts.hasKey("sort_keys")) sortKeys = normalizeSortKeys(context, opts.get("sort_keys"));
 
+        rfc8785 = opts.getBool("rfc8785", rfc8785);
+
         opts.ensureEmpty();
 
         return this;

lib/json.rb

@@ -414,6 +414,8 @@
 #   When +true+, keys are sorted lexicographically. When a \Proc, it receives
 #   the entire \Hash and must return a \Hash with its pairs in the desired
 #   order, allowing for arbitrary sort orders.
+# - Option +rfc8785+ (boolean) controls whether the generated JSON will be canonicalized
+#   as defined in RFC8785.
 #
 # In this example, +obj+ is used first to generate the shortest
 # \JSON data (no whitespace), then again with all formatting options

lib/json/ext.rb

@@ -28,6 +28,18 @@ def parse
       end
     end
 
+    class << self
+      if defined?(::Ractor) && Ractor.respond_to?(:shareable_lambda)
+        def shareable_lambda(block) # :nodoc:
+          Ractor.shareable_lambda(&block)
+        end
+      else
+        def shareable_lambda(block) # :nodoc:
+          block
+        end
+      end
+    end
+
     require 'json/ext/parser'
     Ext::Parser::Config = Ext::ParserConfig
     JSON.parser = Ext::Parser
@@ -42,11 +54,50 @@ def parse
 
     # The default proc used when the +sort_keys+ generation option is +true+.
     # It returns a new hash with the entries sorted by their keys.
-    sort_keys_proc = ->(hash) { hash.sort.to_h }
-    if defined?(::Ractor) && Ractor.respond_to?(:shareable_lambda)
-      sort_keys_proc = Ractor.shareable_lambda(&sort_keys_proc)
-    end
-    generator::State.default_sort_keys_proc = sort_keys_proc
+    generator::State.default_sort_keys_proc = shareable_lambda(->(hash) {
+      hash.sort.to_h
+    })
+
+    # Directly lifted from Gregg Kellogg's json-canonicalization
+    generator::State.rfc8785_number_formater_proc = shareable_lambda(->(num) {
+      if num.zero?
+        "0"
+      else
+        if num < 0
+          num, sign = -num, '-'
+        end
+        native_rep = "%.15E" % num
+        decimal, exponential = native_rep.split('E')
+        exp_val = exponential.to_i
+        exponential = exp_val > 0 ? ('+' + exp_val.to_s) : exp_val.to_s
+
+        integral, fractional = decimal.split('.')
+        fractional = fractional.sub(/0+$/, '')  # Remove trailing zeros
+
+        if exp_val > 0 && exp_val < 21
+          while exp_val > 0
+            integral += fractional.to_s[0] || '0'
+            fractional = fractional.to_s[1..-1]
+            exp_val -= 1
+          end
+          exponential = nil
+        elsif exp_val == 0
+          exponential = nil
+        elsif exp_val < 0 && exp_val > -7
+          # Small numbers are shown as 0.etc with e-6 as lower limit
+          fractional, integral, exponential = integral + fractional.to_s, '0', nil
+          fractional = ("0" * (-exp_val - 1)) + fractional
+        end
+
+        fractional = nil if fractional.to_s.empty?
+        sign.to_s + integral + (fractional ? ".#{fractional}" : '') + (exponential ? "e#{exponential}" : '')
+      end
+    })
+
+    generator::State.rfc8785_sort_keys_proc = shareable_lambda(->(hash) {
+      hash.sort_by { |k| k.to_s.encode(Encoding::UTF_16) }.to_h
+    })
+
     JSON.generator = generator
   end
 

lib/json/ext/generator/state.rb

@@ -54,7 +54,8 @@ def to_h
             strict: strict?,
             depth: depth,
             buffer_initial_length: buffer_initial_length,
-            sort_keys: sort_keys
+            sort_keys: sort_keys,
+            rfc8785: rfc8785?,
           }
 
           allow_duplicate_key = allow_duplicate_key?