cipher: make output buffer String independent

OpenSSL::Cipher#update accepts a String as the second argument to be
used as the output buffer. The buffer must be directly writable, in
other words, it must not be frozen and not a shared string.

rb_str_resize() does not make the String independent if the String
already has the intended length. Use the rb_str_modify() family instead
to check it.

Fixes: https://bugs.ruby-lang.org/issues/20937

Author: rhenium

ext/openssl/ossl_cipher.c

@@ -409,7 +409,10 @@ ossl_cipher_update(int argc, VALUE *argv, VALUE self)
         str = rb_str_new(0, out_len);
     } else {
         StringValue(str);
-        rb_str_resize(str, out_len);
+        if ((long)rb_str_capacity(str) >= out_len)
+            rb_str_modify(str);
+        else
+            rb_str_modify_expand(str, out_len - RSTRING_LEN(str));
     }
 
     if (!ossl_cipher_update_long(ctx, (unsigned char *)RSTRING_PTR(str), &out_len, in, in_len))

test/openssl/test_cipher.rb

@@ -134,6 +134,30 @@ def test_ctr_if_exists
     assert_equal pt, cipher.update(ct) << cipher.final
   end
 
+  def test_update_with_buffer
+    cipher = OpenSSL::Cipher.new("aes-128-ecb").encrypt
+    cipher.random_key
+    expected = cipher.update("data") << cipher.final
+    assert_equal 16, expected.bytesize
+
+    # Buffer is supplied
+    cipher.reset
+    buf = String.new
+    assert_same buf, cipher.update("data", buf)
+    assert_equal expected, buf + cipher.final
+
+    # Buffer is frozen
+    cipher.reset
+    assert_raise(FrozenError) { cipher.update("data", String.new.freeze) }
+
+    # Buffer is a shared string [ruby-core:120141] [Bug #20937]
+    cipher.reset
+    buf = "x" * 1024
+    shared = buf[-("data".bytesize + 32)..-1]
+    assert_same shared, cipher.update("data", shared)
+    assert_equal expected, shared + cipher.final
+  end
+
   def test_ciphers
     ciphers = OpenSSL::Cipher.ciphers
     assert_kind_of Array, ciphers