Fix div and mult sign when result overflows or underflows

Author: tompng

ext/bigdecimal/bigdecimal.c

@@ -5996,11 +5996,11 @@ VpMult(Real *c, Real *a, Real *b)
     /* set LHSV c info */
 
     c->exponent = a->exponent;    /* set exponent */
+    VpSetSign(c, VpGetSign(a) * VpGetSign(b));    /* set sign  */
     if (!AddExponent(c, b->exponent)) {
         if (w) rbd_free_struct(c);
         return 0;
     }
-    VpSetSign(c, VpGetSign(a) * VpGetSign(b));    /* set sign  */
     carry = 0;
     nc = ind_c = MxIndAB;
     memset(c->frac, 0, (nc + 1) * sizeof(DECDIG));        /* Initialize c  */
@@ -6247,10 +6247,10 @@ VpDivd(Real *c, Real *r, Real *a, Real *b)
 out_side:
     c->Prec = word_c;
     c->exponent = a->exponent;
+    VpSetSign(c, VpGetSign(a) * VpGetSign(b));
     if (!AddExponent(c, 2)) return 0;
     if (!AddExponent(c, -(b->exponent))) return 0;
 
-    VpSetSign(c, VpGetSign(a) * VpGetSign(b));
     VpNmlz(c);            /* normalize c */
     r->Prec = word_r;
     r->exponent = a->exponent;

test/bigdecimal/test_bigdecimal.rb

@@ -460,6 +460,45 @@ def test_exception_overflow
     end
   end
 
+  def test_mult_div_overflow_underflow_sign
+    BigDecimal.mode(BigDecimal::EXCEPTION_OVERFLOW, false)
+    BigDecimal.mode(BigDecimal::EXCEPTION_UNDERFLOW, false)
+
+    large_x = BigDecimal("10")
+    100.times do
+      x2 = large_x * large_x
+      break if x2.infinite?
+      large_x = x2
+    end
+
+    small_x = BigDecimal("0.1")
+    100.times do
+      x2 = small_x * small_x
+      break if x2.zero?
+      small_x = x2
+    end
+
+    assert_positive_infinite(large_x * large_x)
+    assert_negative_infinite(large_x * (-large_x))
+    assert_negative_infinite((-large_x) * large_x)
+    assert_positive_infinite((-large_x) * (-large_x))
+
+    assert_positive_zero(small_x * small_x)
+    assert_negative_zero(small_x * (-small_x))
+    assert_negative_zero((-small_x) * small_x)
+    assert_positive_zero((-small_x) * (-small_x))
+
+    assert_positive_infinite(large_x.div(small_x, 10))
+    assert_negative_infinite(large_x.div(-small_x, 10))
+    assert_negative_infinite((-large_x).div(small_x, 10))
+    assert_positive_infinite((-large_x).div(-small_x, 10))
+
+    assert_positive_zero(small_x.div(large_x, 10))
+    assert_negative_zero(small_x.div(-large_x, 10))
+    assert_negative_zero((-small_x).div(large_x, 10))
+    assert_positive_zero((-small_x).div(-large_x, 10))
+  end
+
   def test_exception_zerodivide
     BigDecimal.mode(BigDecimal::EXCEPTION_OVERFLOW, false)
     _test_mode(BigDecimal::EXCEPTION_ZERODIVIDE) { 1 / BigDecimal("0") }