From 595725068183a07391e4f097792e84b4522771e9 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Mon, 19 Feb 2024 14:43:15 -0800
Subject: [PATCH v18 6/6] Change collation UCS_BASIC to use Unicode semantics.

The SQL standard defines UCS_BASIC to sort by code point order, but
the character behavior (e.g. for LOWER() and UPPER()) is defined in
terms of Unicode.

The pre-existing sort order of UCS_BASIC is correct, and is not
changed by this commit. The character behavior is changed to use
Unicode rather than C semantics. This affects case mapping (LOWER(),
UPPER(), and INITCAP()) as well as pattern matching (LIKE and
regexes).

Also add standard collation PG_C_UTF8 which uses the builtin locale
C.UTF-8.
---
 doc/src/sgml/charset.sgml                  | 30 +++++++++++---
 src/include/catalog/pg_collation.dat       |  5 ++-
 src/test/regress/expected/collate.utf8.out | 44 ++++++++++-----------
 src/test/regress/sql/collate.utf8.sql      | 46 ++++++++++------------
 4 files changed, 71 insertions(+), 54 deletions(-)

diff --git a/doc/src/sgml/charset.sgml b/doc/src/sgml/charset.sgml
index 2abd898115..01a3657c7f 100644
--- a/doc/src/sgml/charset.sgml
+++ b/doc/src/sgml/charset.sgml
@@ -865,11 +865,31 @@ SELECT * FROM test1 ORDER BY a || b COLLATE "fr_FR";
       <listitem>
        <para>
         This SQL standard collation sorts by Unicode code point.  Behavior is
-        efficient and stable across all versions.  Case mapping and pattern
-        matching behavior is based on simple ASCII semantics.  It is only
-        available for encoding <literal>UTF8</literal>.  (This collation has
-        the same behavior as the libc locale specification
-        <literal>C</literal> in <literal>UTF8</literal> encoding.)
+        efficient and stable within a <productname>Postgres</productname>
+        major version.  For the functions <function>lower</function>,
+        <function>initcap</function>, and <function>upper</function> it uses
+        Unicode full case mapping. For pattern matching (including regular
+        expressions), it uses the Standard variant of Unicode <ulink
+        url="https://www.unicode.org/reports/tr18/#Compatibility_Properties">Compatibility
+        Properties</ulink>.  It is only available for encoding
+        <literal>UTF8</literal>.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry>
+      <term><literal>PG_C_UTF8</literal></term>
+      <listitem>
+       <para>
+        This collation sorts by Unicode code point.  Behavior is efficient and
+        stable within a <productname>Postgres</productname> major version.
+        For the functions <function>lower</function>,
+        <function>initcap</function>, and <function>upper</function> it uses
+        Unicode simple case mapping.  For pattern matching (including regular
+        expressions), it uses the POSIX Compatible variant of Unicode <ulink
+        url="https://www.unicode.org/reports/tr18/#Compatibility_Properties">Compatibility
+        Properties</ulink>.  This collation is only available for encoding
+        <literal>UTF8</literal>.
        </para>
       </listitem>
      </varlistentry>
diff --git a/src/include/catalog/pg_collation.dat b/src/include/catalog/pg_collation.dat
index 938432e8a4..f12c84e9ae 100644
--- a/src/include/catalog/pg_collation.dat
+++ b/src/include/catalog/pg_collation.dat
@@ -25,10 +25,13 @@
   collcollate => 'POSIX', collctype => 'POSIX' },
 { oid => '962', descr => 'sorts by Unicode code point, C character semantics',
   collname => 'ucs_basic', collprovider => 'b', collencoding => '6',
-  colllocale => 'C' },
+  colllocale => 'UCS_BASIC' },
 { oid => '963',
   descr => 'sorts using the Unicode Collation Algorithm with default settings',
   collname => 'unicode', collprovider => 'i', collencoding => '-1',
   colllocale => 'und' },
+{ oid => '970', descr => 'sorts by Unicode code point; Unicode & POSIX character semantics',
+  collname => 'pg_c_utf8', collprovider => 'b', collencoding => '6',
+  colllocale => 'C.UTF8' },
 
 ]
diff --git a/src/test/regress/expected/collate.utf8.out b/src/test/regress/expected/collate.utf8.out
index 58085f300c..1f903ed561 100644
--- a/src/test/regress/expected/collate.utf8.out
+++ b/src/test/regress/expected/collate.utf8.out
@@ -11,9 +11,8 @@ SET client_encoding TO UTF8;
 --
 -- Test builtin UCS_BASIC locale.
 --
-CREATE COLLATION BUILTIN_UCS_BASIC ( provider = builtin, locale = 'UCS_BASIC' );
 CREATE TABLE builtin_test1 (
-  t TEXT COLLATE BUILTIN_UCS_BASIC
+  t TEXT COLLATE UCS_BASIC
 );
 INSERT INTO builtin_test1 VALUES
   ('abc DEF'),
@@ -41,130 +40,129 @@ SELECT
 
 DROP TABLE builtin_test1;
 -- test Final_Sigma
-SELECT lower('ΑΣ' COLLATE BUILTIN_UCS_BASIC); -- 0391 03A3
+SELECT lower('ΑΣ' COLLATE UCS_BASIC); -- 0391 03A3
  lower 
 -------
  ας
 (1 row)
 
-SELECT lower('ΑΣ0' COLLATE BUILTIN_UCS_BASIC); -- 0391 03A3 0030
+SELECT lower('ΑΣ0' COLLATE UCS_BASIC); -- 0391 03A3 0030
  lower 
 -------
  ας0
 (1 row)
 
-SELECT lower('ἈΣ̓' COLLATE BUILTIN_UCS_BASIC); -- 0391 0343 03A3 0343
+SELECT lower('ἈΣ̓' COLLATE UCS_BASIC); -- 0391 0343 03A3 0343
  lower 
 -------
  ἀς̓
 (1 row)
 
-SELECT lower('ᾼΣͅ' COLLATE BUILTIN_UCS_BASIC); -- 0391 0345 03A3 0345
+SELECT lower('ᾼΣͅ' COLLATE UCS_BASIC); -- 0391 0345 03A3 0345
  lower 
 -------
  ᾳςͅ
 (1 row)
 
 -- test !Final_Sigma
-SELECT lower('Σ' COLLATE BUILTIN_UCS_BASIC); -- 03A3
+SELECT lower('Σ' COLLATE UCS_BASIC); -- 03A3
  lower 
 -------
  σ
 (1 row)
 
-SELECT lower('0Σ' COLLATE BUILTIN_UCS_BASIC); -- 0030 03A3
+SELECT lower('0Σ' COLLATE UCS_BASIC); -- 0030 03A3
  lower 
 -------
  0σ
 (1 row)
 
-SELECT lower('ΑΣΑ' COLLATE BUILTIN_UCS_BASIC); -- 0391 03A3 0391
+SELECT lower('ΑΣΑ' COLLATE UCS_BASIC); -- 0391 03A3 0391
  lower 
 -------
  ασα
 (1 row)
 
-SELECT lower('ἈΣ̓Α' COLLATE BUILTIN_UCS_BASIC); -- 0391 0343 03A3 0343 0391
+SELECT lower('ἈΣ̓Α' COLLATE UCS_BASIC); -- 0391 0343 03A3 0343 0391
  lower 
 -------
  ἀσ̓α
 (1 row)
 
-SELECT lower('ᾼΣͅΑ' COLLATE BUILTIN_UCS_BASIC); -- 0391 0345 03A3 0345 0391
+SELECT lower('ᾼΣͅΑ' COLLATE UCS_BASIC); -- 0391 0345 03A3 0345 0391
  lower 
 -------
  ᾳσͅα
 (1 row)
 
 -- properties
-SELECT 'xyz' ~ '[[:alnum:]]' COLLATE BUILTIN_UCS_BASIC;
+SELECT 'xyz' ~ '[[:alnum:]]' COLLATE UCS_BASIC;
  ?column? 
 ----------
  t
 (1 row)
 
-SELECT 'xyz' !~ '[[:upper:]]' COLLATE BUILTIN_UCS_BASIC;
+SELECT 'xyz' !~ '[[:upper:]]' COLLATE UCS_BASIC;
  ?column? 
 ----------
  t
 (1 row)
 
-SELECT '@' !~ '[[:alnum:]]' COLLATE BUILTIN_UCS_BASIC;
+SELECT '@' !~ '[[:alnum:]]' COLLATE UCS_BASIC;
  ?column? 
 ----------
  t
 (1 row)
 
-SELECT '=' !~ '[[:punct:]]' COLLATE BUILTIN_UCS_BASIC; -- symbols are not punctuation
+SELECT '=' !~ '[[:punct:]]' COLLATE UCS_BASIC; -- symbols are not punctuation
  ?column? 
 ----------
  t
 (1 row)
 
-SELECT 'a8a' ~ '[[:digit:]]' COLLATE BUILTIN_UCS_BASIC;
+SELECT 'a8a' ~ '[[:digit:]]' COLLATE UCS_BASIC;
  ?column? 
 ----------
  t
 (1 row)
 
-SELECT '൧' ~ '\d' COLLATE BUILTIN_UCS_BASIC;
+SELECT '൧' ~ '\d' COLLATE UCS_BASIC;
  ?column? 
 ----------
  t
 (1 row)
 
 -- case mapping
-SELECT 'xYz' ~* 'XyZ' COLLATE BUILTIN_UCS_BASIC;
+SELECT 'xYz' ~* 'XyZ' COLLATE UCS_BASIC;
  ?column? 
 ----------
  t
 (1 row)
 
-SELECT 'xAb' ~* '[W-Y]' COLLATE BUILTIN_UCS_BASIC;
+SELECT 'xAb' ~* '[W-Y]' COLLATE UCS_BASIC;
  ?column? 
 ----------
  t
 (1 row)
 
-SELECT 'xAb' !~* '[c-d]' COLLATE BUILTIN_UCS_BASIC;
+SELECT 'xAb' !~* '[c-d]' COLLATE UCS_BASIC;
  ?column? 
 ----------
  t
 (1 row)
 
-SELECT 'Δ' ~* '[α-λ]' COLLATE BUILTIN_UCS_BASIC;
+SELECT 'Δ' ~* '[α-λ]' COLLATE UCS_BASIC;
  ?column? 
 ----------
  t
 (1 row)
 
-SELECT 'δ' ~* '[Γ-Λ]' COLLATE BUILTIN_UCS_BASIC; -- same as above with cases reversed
+SELECT 'δ' ~* '[Γ-Λ]' COLLATE UCS_BASIC; -- same as above with cases reversed
  ?column? 
 ----------
  t
 (1 row)
 
-DROP COLLATION BUILTIN_UCS_BASIC;
 --
 -- Test builtin C.UTF-8 locale.
 --
diff --git a/src/test/regress/sql/collate.utf8.sql b/src/test/regress/sql/collate.utf8.sql
index 747fc63bf4..26e3d0ef88 100644
--- a/src/test/regress/sql/collate.utf8.sql
+++ b/src/test/regress/sql/collate.utf8.sql
@@ -15,10 +15,8 @@ SET client_encoding TO UTF8;
 -- Test builtin UCS_BASIC locale.
 --
 
-CREATE COLLATION BUILTIN_UCS_BASIC ( provider = builtin, locale = 'UCS_BASIC' );
-
 CREATE TABLE builtin_test1 (
-  t TEXT COLLATE BUILTIN_UCS_BASIC
+  t TEXT COLLATE UCS_BASIC
 );
 INSERT INTO builtin_test1 VALUES
   ('abc DEF'),
@@ -39,36 +37,34 @@ SELECT
 DROP TABLE builtin_test1;
 
 -- test Final_Sigma
-SELECT lower('ΑΣ' COLLATE BUILTIN_UCS_BASIC); -- 0391 03A3
-SELECT lower('ΑΣ0' COLLATE BUILTIN_UCS_BASIC); -- 0391 03A3 0030
-SELECT lower('ἈΣ̓' COLLATE BUILTIN_UCS_BASIC); -- 0391 0343 03A3 0343
-SELECT lower('ᾼΣͅ' COLLATE BUILTIN_UCS_BASIC); -- 0391 0345 03A3 0345
+SELECT lower('ΑΣ' COLLATE UCS_BASIC); -- 0391 03A3
+SELECT lower('ΑΣ0' COLLATE UCS_BASIC); -- 0391 03A3 0030
+SELECT lower('ἈΣ̓' COLLATE UCS_BASIC); -- 0391 0343 03A3 0343
+SELECT lower('ᾼΣͅ' COLLATE UCS_BASIC); -- 0391 0345 03A3 0345
 
 -- test !Final_Sigma
-SELECT lower('Σ' COLLATE BUILTIN_UCS_BASIC); -- 03A3
-SELECT lower('0Σ' COLLATE BUILTIN_UCS_BASIC); -- 0030 03A3
-SELECT lower('ΑΣΑ' COLLATE BUILTIN_UCS_BASIC); -- 0391 03A3 0391
-SELECT lower('ἈΣ̓Α' COLLATE BUILTIN_UCS_BASIC); -- 0391 0343 03A3 0343 0391
-SELECT lower('ᾼΣͅΑ' COLLATE BUILTIN_UCS_BASIC); -- 0391 0345 03A3 0345 0391
+SELECT lower('Σ' COLLATE UCS_BASIC); -- 03A3
+SELECT lower('0Σ' COLLATE UCS_BASIC); -- 0030 03A3
+SELECT lower('ΑΣΑ' COLLATE UCS_BASIC); -- 0391 03A3 0391
+SELECT lower('ἈΣ̓Α' COLLATE UCS_BASIC); -- 0391 0343 03A3 0343 0391
+SELECT lower('ᾼΣͅΑ' COLLATE UCS_BASIC); -- 0391 0345 03A3 0345 0391
 
 -- properties
 
-SELECT 'xyz' ~ '[[:alnum:]]' COLLATE BUILTIN_UCS_BASIC;
-SELECT 'xyz' !~ '[[:upper:]]' COLLATE BUILTIN_UCS_BASIC;
-SELECT '@' !~ '[[:alnum:]]' COLLATE BUILTIN_UCS_BASIC;
-SELECT '=' !~ '[[:punct:]]' COLLATE BUILTIN_UCS_BASIC; -- symbols are not punctuation
-SELECT 'a8a' ~ '[[:digit:]]' COLLATE BUILTIN_UCS_BASIC;
-SELECT '൧' ~ '\d' COLLATE BUILTIN_UCS_BASIC;
+SELECT 'xyz' ~ '[[:alnum:]]' COLLATE UCS_BASIC;
+SELECT 'xyz' !~ '[[:upper:]]' COLLATE UCS_BASIC;
+SELECT '@' !~ '[[:alnum:]]' COLLATE UCS_BASIC;
+SELECT '=' !~ '[[:punct:]]' COLLATE UCS_BASIC; -- symbols are not punctuation
+SELECT 'a8a' ~ '[[:digit:]]' COLLATE UCS_BASIC;
+SELECT '൧' ~ '\d' COLLATE UCS_BASIC;
 
 -- case mapping
 
-SELECT 'xYz' ~* 'XyZ' COLLATE BUILTIN_UCS_BASIC;
-SELECT 'xAb' ~* '[W-Y]' COLLATE BUILTIN_UCS_BASIC;
-SELECT 'xAb' !~* '[c-d]' COLLATE BUILTIN_UCS_BASIC;
-SELECT 'Δ' ~* '[α-λ]' COLLATE BUILTIN_UCS_BASIC;
-SELECT 'δ' ~* '[Γ-Λ]' COLLATE BUILTIN_UCS_BASIC; -- same as above with cases reversed
-
-DROP COLLATION BUILTIN_UCS_BASIC;
+SELECT 'xYz' ~* 'XyZ' COLLATE UCS_BASIC;
+SELECT 'xAb' ~* '[W-Y]' COLLATE UCS_BASIC;
+SELECT 'xAb' !~* '[c-d]' COLLATE UCS_BASIC;
+SELECT 'Δ' ~* '[α-λ]' COLLATE UCS_BASIC;
+SELECT 'δ' ~* '[Γ-Λ]' COLLATE UCS_BASIC; -- same as above with cases reversed
 
 --
 -- Test builtin C.UTF-8 locale.
-- 
2.34.1

