Update UCD to Unicode 16.0.0

They added some new scripts.

There were a few changes to the line break algorithm,
most notably there is more rules that require more context than before.
While not major, there was some shuffling and additions to our
implementation to match the new rules.

IDNA test data now disallows the trailing dot/empty root label,
technically to be toggled off by an option that controls a few things,
but we don't have options. For test-data they changed the format a
little - "" is used to mean empty string, while a blank segment is
null/no string, update the parser to read this.

[ChangeLog][Third-Party Code] Updated the Unicode Character Database to
UCD revision 34/Unicode 16.

Fixes: QTBUG-132902
Task-number: QTBUG-132851
Pick-to: 6.9 6.8 6.5
Change-Id: I4569703659f6fd0f20943110a03301c1cf8cc1ed
Reviewed-by: Edward Welbourne <[email protected]>

Author: Morten242

src/corelib/text/qchar.cpp

@@ -161,6 +161,7 @@ QT_BEGIN_NAMESPACE
     \value [since 6.3] Unicode_14_0 Version 14.0
     \value [since 6.5] Unicode_15_0 Version 15.0
     \value [since 6.8] Unicode_15_1 Version 15.1
+    \value [since 6.9] Unicode_16_0 Version 16.0
     \value Unicode_Unassigned  The value is not assigned to any character
                                in version 8.0 of Unicode.
 
@@ -298,6 +299,7 @@ QT_BEGIN_NAMESPACE
     \value [since 5.5] Script_Elbasan
     \value [since 5.15] Script_Elymaic
     \value Script_Ethiopic
+    \value [since 6.9] Script_Garay
     \value Script_Georgian
     \value Script_Glagolitic
     \value Script_Gothic
@@ -306,6 +308,7 @@ QT_BEGIN_NAMESPACE
     \value Script_Gujarati
     \value [since 5.15] Script_GunjalaGondi
     \value Script_Gurmukhi
+    \value [since 6.9] Script_GurungKhema
     \value Script_Han
     \value Script_Hangul
     \value [since 5.15] Script_HanifiRohingya
@@ -327,6 +330,7 @@ QT_BEGIN_NAMESPACE
     \value Script_Khmer
     \value [since 5.5] Script_Khojki
     \value [since 5.5] Script_Khudawadi
+    \value [since 6.9] Script_KiratRai
     \value Script_Lao
     \value Script_Latin
     \value Script_Lepcha
@@ -364,6 +368,7 @@ QT_BEGIN_NAMESPACE
     \value [since 5.15] Script_NyiakengPuachueHmong
     \value Script_Ogham
     \value Script_OlChiki
+    \value [since 6.9] Script_OlOnal
     \value [since 5.6] Script_OldHungarian
     \value Script_OldItalic
     \value [since 5.5] Script_OldNorthArabian
@@ -395,6 +400,7 @@ QT_BEGIN_NAMESPACE
     \value Script_SoraSompeng
     \value [since 5.11] Script_Soyombo
     \value Script_Sundanese
+    \value [since 6.9] Script_Sunuwar
     \value Script_SylotiNagri
     \value Script_Syriac
     \value Script_Tagalog
@@ -412,7 +418,9 @@ QT_BEGIN_NAMESPACE
     \value Script_Tibetan
     \value Script_Tifinagh
     \value [since 5.5] Script_Tirhuta
+    \value [since 6.9] Script_Todhri
     \value [since 6.3] Script_Toto
+    \value [since 6.9] Script_TuluTigalari
     \value Script_Ugaritic
     \value Script_Vai
     \value [since 6.3] Script_Vithkuqi

src/corelib/text/qchar.h

@@ -392,6 +392,15 @@ class QT6_ONLY(Q_CORE_EXPORT) QChar {
         Script_Kawi,
         Script_NagMundari,
 
+        // Unicode 16.0 additions
+        Script_Garay,
+        Script_GurungKhema,
+        Script_KiratRai,
+        Script_OlOnal,
+        Script_Sunuwar,
+        Script_Todhri,
+        Script_TuluTigalari,
+
         ScriptCount
     };
 
@@ -486,6 +495,7 @@ class QT6_ONLY(Q_CORE_EXPORT) QChar {
         Unicode_14_0,
         Unicode_15_0,
         Unicode_15_1,
+        Unicode_16_0,
     };
 
     inline Category category() const noexcept { return QChar::category(ucs); }

src/corelib/text/qt_attribution.json

@@ -17,7 +17,7 @@
         "Version": [ "Don't use the Unicode standard version;",
                      "UCD has its own 'Revision' numbers",
                      "see the 'UAX #44, UCD' page (https://www.unicode.org/reports/tr44/)" ]},
-    "Version": "32",
+    "Version": "34",
     "License": "Unicode License Agreement - Data Files and Software (2016)",
     "LicenseId": "Unicode-3.0",
     "Copyright": "Copyright (C) 1991-2022 Unicode, Inc."

src/corelib/text/qunicodetables.cpp

@@ -1,7 +1,7 @@
 // Copyright (C) 2020 The Qt Company Ltd.
 // SPDX-License-Identifier: Unicode-3.0
 
-/* This file is autogenerated from the Unicode 15.1 database. Do not edit */
+/* This file is autogenerated from the Unicode 16.0 database. Do not edit */
 
 //
 //  W A R N I N G
@@ -23,7 +23,7 @@
 
 QT_BEGIN_NAMESPACE
 
-#define UNICODE_DATA_VERSION QChar::Unicode_15_1
+#define UNICODE_DATA_VERSION QChar::Unicode_16_0
 
 namespace QUnicodeTables {
 
@@ -143,17 +143,26 @@ enum SentenceBreakClass {
 };
 
 // see http://www.unicode.org/reports/tr14/tr14-30.html
-// we don't use the XX, AK, AP, AS and AI classes and map them to AL instead.
+// we don't use the XX and AI classes but map them to AL instead.
 // VI and VF classes are mapped to CM.
 enum LineBreakClass {
     LineBreak_OP, LineBreak_CL, LineBreak_CP,
-    LineBreak_QU, LineBreak_QU_Pi, LineBreak_QU_Pf, LineBreak_GL,
-    LineBreak_NS, LineBreak_EX, LineBreak_SY, LineBreak_IS, LineBreak_PR,
+    LineBreak_QU, LineBreak_QU_Pi, LineBreak_QU_Pf, LineBreak_QU_19,
+    LineBreak_GL, LineBreak_NS, LineBreak_EX, LineBreak_SY,
+    LineBreak_IS, LineBreak_PR,
     LineBreak_PO, LineBreak_NU, LineBreak_AL, LineBreak_HL, LineBreak_ID,
-    LineBreak_IN, LineBreak_HY, LineBreak_BA, LineBreak_BB, LineBreak_B2,
+    LineBreak_IN, LineBreak_HY, LineBreak_WS_HY,
+    LineBreak_BA, LineBreak_WS_BA,
+    LineBreak_HYBA,
+    LineBreak_BB, LineBreak_B2,
     LineBreak_ZW, LineBreak_CM, LineBreak_WJ, LineBreak_H2, LineBreak_H3,
     LineBreak_JL, LineBreak_JV, LineBreak_JT, LineBreak_RI, LineBreak_CB,
-    LineBreak_EB, LineBreak_EM, LineBreak_ZWJ,
+    LineBreak_EB, LineBreak_EM,
+
+    LineBreak_AK, LineBreak_AP, LineBreak_AS,
+    LineBreak_VI, LineBreak_VF,
+
+    LineBreak_ZWJ,
     LineBreak_SA, LineBreak_SG, LineBreak_SP,
     LineBreak_CR, LineBreak_LF, LineBreak_BK,