Unicode: Update to UCD 16 (PCRE2Project#503)

Updates to Unicode files to Unicode 16, adjusts tests, and the
scripts used to parse UCD, to adapt to minor formatting differences
in UCD 16.

The `GenerateTest26.py` and `GenerateCommon.py` had a regexp to
extract properties from the `ScriptExtensions.txt` file. Previously,
all property lines had one space after space-separated list of scripts.
In UCD-16, this list is adjusted with right-padding, which throws off
the parser.

This commit adjusts the regexps to ignore padding spaces.

Author: Ayesh

maint/GenerateCommon.py

@@ -232,7 +232,7 @@ def reorder_scripts():
 
   extended_script_abbrevs = set()
   with open("Unicode.tables/ScriptExtensions.txt") as f:
-    names_re = re.compile(r'^[0-9A-F]{4,6}(?:\.\.[0-9A-F]{4,6})? +; ([A-Za-z_ ]+) #')
+    names_re = re.compile(r'^[0-9A-F]{4,6}(?:\.\.[0-9A-F]{4,6})? +; ([A-Za-z_ ]+[A-Za-z]) +#')
 
     for line in f:
       match_obj = names_re.match(line)

maint/GenerateTest26.py

@@ -55,7 +55,7 @@ def gen_script_tests():
   script_data = [None] * len(script_names)
   char_data = [None] * 0x110000
 
-  property_re = re.compile("^([0-9A-F]{4,6})(?:\\.\\.([0-9A-F]{4,6}))? +; ([A-Za-z_ ]+) #")
+  property_re = re.compile("^([0-9A-F]{4,6})(?:\\.\\.([0-9A-F]{4,6}))? +; ([A-Za-z_ ]+[A-Za-z]) +#")
   prev_name = ""
   script_idx = -1
 

maint/Unicode.tables/BidiMirroring.txt

@@ -1,7 +1,8 @@
-# BidiMirroring-15.0.0.txt
-# Date: 2022-05-03, 18:47:00 GMT [KW, RP]
-# © 2022 Unicode®, Inc.
-# For terms of use, see https://www.unicode.org/terms_of_use.html
+# BidiMirroring-16.0.0.txt
+# Date: 2024-01-30
+# © 2024 Unicode®, Inc.
+# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
+# For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Unicode Character Database
 # For documentation, see https://www.unicode.org/reports/tr44/
@@ -15,7 +16,7 @@
 # value, for which there is another Unicode character that typically has a glyph
 # that is the mirror image of the original character's glyph.
 #
-# The repertoire covered by the file is Unicode 15.0.0.
+# The repertoire covered by the file is Unicode 16.0.0.
 #
 # The file contains a list of lines with mappings from one code point
 # to another one for character-based mirroring.
@@ -44,7 +45,8 @@
 #
 # This file was originally created by Markus Scherer.
 # Extended for Unicode 3.2, 4.0, 4.1, 5.0, 5.1, 5.2, and 6.0 by Ken Whistler,
-# and for subsequent versions by Ken Whistler, Laurentiu Iancu, and Roozbeh Pournader.
+# and for subsequent versions by Ken Whistler, Laurentiu Iancu, Roozbeh Pournader,
+# and Robin Leroy.
 #
 # Historical and Compatibility Information:
 #
@@ -542,6 +544,7 @@ FF63; FF62 # [BEST FIT] HALFWIDTH RIGHT CORNER BRACKET
 # 225F; QUESTIONED EQUAL TO
 # 2260; NOT EQUAL TO
 # 2262; NOT IDENTICAL TO
+# 226D; NOT EQUIVALENT TO
 # 228C; MULTISET
 # 22A7; MODELS
 # 22AA; TRIPLE VERTICAL BAR RIGHT TURNSTILE

maint/Unicode.tables/CaseFolding.txt

@@ -1,8 +1,8 @@
-# CaseFolding-15.0.0.txt
-# Date: 2022-02-02, 23:35:35 GMT
-# © 2022 Unicode®, Inc.
+# CaseFolding-16.0.0.txt
+# Date: 2024-04-30, 21:48:11 GMT
+# © 2024 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
-# For terms of use, see https://www.unicode.org/terms_of_use.html
+# For terms of use and license, see https://www.unicode.org/terms_of_use.html
 #
 # Unicode Character Database
 #   For documentation, see https://www.unicode.org/reports/tr44/
@@ -603,6 +603,7 @@
 1C86; C; 044A; # CYRILLIC SMALL LETTER TALL HARD SIGN
 1C87; C; 0463; # CYRILLIC SMALL LETTER TALL YAT
 1C88; C; A64B; # CYRILLIC SMALL LETTER UNBLENDED UK
+1C89; C; 1C8A; # CYRILLIC CAPITAL LETTER TJE
 1C90; C; 10D0; # GEORGIAN MTAVRULI CAPITAL LETTER AN
 1C91; C; 10D1; # GEORGIAN MTAVRULI CAPITAL LETTER BAN
 1C92; C; 10D2; # GEORGIAN MTAVRULI CAPITAL LETTER GAN
@@ -929,6 +930,7 @@
 1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
 1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
 1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+1FD3; S; 0390; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
 1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI
 1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
 1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY
@@ -937,6 +939,7 @@
 1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA
 1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
 1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+1FE3; S; 03B0; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
 1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI
 1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI
 1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
@@ -1238,9 +1241,13 @@ A7C5; C; 0282; # LATIN CAPITAL LETTER S WITH HOOK
 A7C6; C; 1D8E; # LATIN CAPITAL LETTER Z WITH PALATAL HOOK
 A7C7; C; A7C8; # LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY
 A7C9; C; A7CA; # LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY
+A7CB; C; 0264; # LATIN CAPITAL LETTER RAMS HORN
+A7CC; C; A7CD; # LATIN CAPITAL LETTER S WITH DIAGONAL STROKE
 A7D0; C; A7D1; # LATIN CAPITAL LETTER CLOSED INSULAR G
 A7D6; C; A7D7; # LATIN CAPITAL LETTER MIDDLE SCOTS S
 A7D8; C; A7D9; # LATIN CAPITAL LETTER SIGMOID S
+A7DA; C; A7DB; # LATIN CAPITAL LETTER LAMBDA
+A7DC; C; 019B; # LATIN CAPITAL LETTER LAMBDA WITH STROKE
 A7F5; C; A7F6; # LATIN CAPITAL LETTER REVERSED HALF H
 AB70; C; 13A0; # CHEROKEE SMALL LETTER A
 AB71; C; 13A1; # CHEROKEE SMALL LETTER E
@@ -1328,6 +1335,7 @@ FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL
 FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI
 FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL
 FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T
+FB05; S; FB06; # LATIN SMALL LIGATURE LONG S T
 FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST
 FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW
 FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH
@@ -1522,6 +1530,28 @@ FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z
 10CB0; C; 10CF0; # OLD HUNGARIAN CAPITAL LETTER EZS
 10CB1; C; 10CF1; # OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN
 10CB2; C; 10CF2; # OLD HUNGARIAN CAPITAL LETTER US
+10D50; C; 10D70; # GARAY CAPITAL LETTER A
+10D51; C; 10D71; # GARAY CAPITAL LETTER CA
+10D52; C; 10D72; # GARAY CAPITAL LETTER MA
+10D53; C; 10D73; # GARAY CAPITAL LETTER KA
+10D54; C; 10D74; # GARAY CAPITAL LETTER BA
+10D55; C; 10D75; # GARAY CAPITAL LETTER JA
+10D56; C; 10D76; # GARAY CAPITAL LETTER SA
+10D57; C; 10D77; # GARAY CAPITAL LETTER WA
+10D58; C; 10D78; # GARAY CAPITAL LETTER LA
+10D59; C; 10D79; # GARAY CAPITAL LETTER GA
+10D5A; C; 10D7A; # GARAY CAPITAL LETTER DA
+10D5B; C; 10D7B; # GARAY CAPITAL LETTER XA
+10D5C; C; 10D7C; # GARAY CAPITAL LETTER YA
+10D5D; C; 10D7D; # GARAY CAPITAL LETTER TA
+10D5E; C; 10D7E; # GARAY CAPITAL LETTER RA
+10D5F; C; 10D7F; # GARAY CAPITAL LETTER NYA
+10D60; C; 10D80; # GARAY CAPITAL LETTER FA
+10D61; C; 10D81; # GARAY CAPITAL LETTER NA
+10D62; C; 10D82; # GARAY CAPITAL LETTER PA
+10D63; C; 10D83; # GARAY CAPITAL LETTER HA
+10D64; C; 10D84; # GARAY CAPITAL LETTER OLD KA
+10D65; C; 10D85; # GARAY CAPITAL LETTER OLD NA
 118A0; C; 118C0; # WARANG CITI CAPITAL LETTER NGAA
 118A1; C; 118C1; # WARANG CITI CAPITAL LETTER A
 118A2; C; 118C2; # WARANG CITI CAPITAL LETTER WI