8300207: Add a pre-check for the number of canonical equivalent permutations in j.u.r.Pattern

src/java.base/share/classes/java/util/regex/Pattern.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1090,6 +1090,10 @@ public static Pattern compile(String regex) {
      *
      * @throws  PatternSyntaxException
      *          If the expression's syntax is invalid
+     *
+     * @implNote If {@link #CANON_EQ} is specified and the number of combining
+     * marks for any character is too large, an {@link java.lang.OutOfMemoryError}
+     * is thrown.
      */
     public static Pattern compile(String regex, int flags) {
         return new Pattern(regex, flags);
@@ -1123,6 +1127,13 @@ public String toString() {
      *         The character sequence to be matched
      *
      * @return  A new matcher for this pattern
+     *
+     * @implNote When a {@link Pattern} is deserialized, compilation is deferred
+     * until a direct or indirect invocation of this method. Thus, if a
+     * deserialized pattern has {@link #CANON_EQ} among its flags and the number
+     * of combining marks for any character is too large, an
+     * {@link java.lang.OutOfMemoryError} is thrown,
+     * as in {@link #compile(String, int)}.
      */
     public Matcher matcher(CharSequence input) {
         if (!compiled) {
@@ -1596,14 +1607,30 @@ private static String[] producePermutations(String input) {
             return result;
         }
 
-        int length = 1;
+        /*
+         * Since
+         *      12! =   479_001_600 < Integer.MAX_VALUE
+         *      13! = 6_227_020_800 > Integer.MAX_VALUE
+         * the computation of n! using int arithmetic will overflow iff
+         *      n < 0 or n > 12
+         *
+         * Here, nCodePoints! is computed in the next for-loop below.
+         * As nCodePoints >= 0, the computation overflows iff nCodePoints > 12.
+         * In that case, throw OOME to simulate length > Integer.MAX_VALUE.
+         */
         int nCodePoints = countCodePoints(input);
-        for(int x=1; x<nCodePoints; x++)
-            length = length * (x+1);
+        if (nCodePoints > 12) {
+            throw new OutOfMemoryError("Pattern too complex");
+        }
 
+        /* Compute length = nCodePoints! */
+        int length = 1;
+        for (int x = 2; x <= nCodePoints; ++x) {
+            length *= x;
+        }
         String[] temp = new String[length];
 
-        int combClass[] = new int[nCodePoints];
+        int[] combClass = new int[nCodePoints];
         for(int x=0, i=0; x<nCodePoints; x++) {
             int c = Character.codePointAt(input, i);
             combClass[x] = getClass(c);