initdb: derive encoding from locale for ICU; similar to libc.

Previously, the default encoding was derived from the locale when
using libc; while the default was always UTF-8 when using ICU. That
would throw an error when the locale was not compatible with UTF-8.

This commit causes initdb to derive the default encoding from the
locale for both providers. If --no-locale is specified (or if the
locale is C or POSIX), the default encoding will be UTF-8 for ICU
(because ICU does not support SQL_ASCII) and SQL_ASCII for libc.

Per buildfarm failure on system "hoverfly" related to commit
27b62377b4.

Discussion: https://postgr.es/m/d191d5841347301a8f1238f609471ddd957fc47e.camel%40j-davis.com

diff --git a/contrib/unaccent/meson.build b/contrib/unaccent/meson.build
index 613dd0be228ddac77db0f75d4acb97e2403b3e40..284d34ee291dbd704bf84669d945fa1139663598 100644 (file)
--- a/contrib/unaccent/meson.build
+++ b/contrib/unaccent/meson.build
@@ -37,6 +37,6 @@ tests += {
     'sql': [
       'unaccent',
     ],
-    'regress_args': ['--encoding=UTF8'],
+    'regress_args': ['--encoding=UTF8', '--no-locale'],
   },
 }

diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml
index 05a9c2cf588ce2b0e8fedca331926b7e030ef4ac..daebe75ee4091fe7b6eac61de082e79e94a86be0 100644 (file)
--- a/doc/src/sgml/ref/initdb.sgml
+++ b/doc/src/sgml/ref/initdb.sgml
@@ -213,13 +213,19 @@ PostgreSQL documentation
       --encoding=encoding
       
        
-        Selects the encoding of the template databases. This will also
-        be the default encoding of any database you create later,
-        unless you override it then.  The default is derived from the locale,
-        if the libc locale provider is used, or UTF8 if the
-        ICU locale provider is used.  The character sets supported by
-        the PostgreSQL server are described
-        in .
+        Selects the encoding of the template databases. This will also be the
+        default encoding of any database you create later, unless you override
+        it then. The character sets supported by the
+        PostgreSQL server are described in 
+        linkend="multibyte-charset-supported"/>.
+       
+       
+        By default, the template database encoding is derived from the
+        locale. If  is specified
+        (or equivalently, if the locale is C or
+        POSIX), then the default is UTF8
+        for the ICU provider and SQL_ASCII for the
+        libc provider.
        
       
      


diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c

index 497f2205f0c93e8dfa321779e8b2485412ec74bc..d48b7b6060c9e2679b86d4aa3c631d46e66a2f32 100644 (file)

--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -2350,18 +2350,19 @@ setup_locale_encoding(void)
               lc_time);
    }
 
-   if (!encoding && locale_provider == COLLPROVIDER_ICU)
-   {
-       encodingid = PG_UTF8;
-       printf(_("The default database encoding has been set to \"%s\".\n"),
-              pg_encoding_to_char(encodingid));
-   }
-   else if (!encoding)
+   if (!encoding)
    {
        int         ctype_enc;
 
        ctype_enc = pg_get_encoding_from_locale(lc_ctype, true);
 
+       /*
+        * If ctype_enc=SQL_ASCII, it's compatible with any encoding. ICU does
+        * not support SQL_ASCII, so select UTF-8 instead.
+        */
+       if (locale_provider == COLLPROVIDER_ICU && ctype_enc == PG_SQL_ASCII)
+           ctype_enc = PG_UTF8;
+
        if (ctype_enc == -1)
        {
            /* Couldn't recognize the locale's codeset */


diff --git a/src/bin/pg_upgrade/t/002_pg_upgrade.pl b/src/bin/pg_upgrade/t/002_pg_upgrade.pl

index 1b5df730e99d4cf03af5cb39441412df2e31deff..90669f3c6d629d5e1809771b0c8f630e5bca0bf7 100644 (file)

--- a/src/bin/pg_upgrade/t/002_pg_upgrade.pl
+++ b/src/bin/pg_upgrade/t/002_pg_upgrade.pl
@@ -108,7 +108,7 @@ if ($oldnode->pg_version >= 11)
 
 my $original_encoding = "6"; # UTF-8
 my $original_provider = "c";
-my $original_collate = "C";
+my $original_locale = "C";
 my $original_iculocale = "";
 my $provider_field = "'c' AS datlocprovider";
 my $iculocale_field = "NULL AS daticulocale";
@@ -123,7 +123,7 @@ if ($oldnode->pg_version >= 15 && $ENV{with_icu} eq 'yes')
 my @initdb_params = @custom_opts;
 
 push @initdb_params, ('--encoding', 'UTF-8');
-push @initdb_params, ('--lc-collate', $original_collate);
+push @initdb_params, ('--locale', $original_locale);
 if ($original_provider eq "i")
 {
    push @initdb_params, ('--locale-provider', 'icu');
@@ -136,16 +136,12 @@ $oldnode->start;
 
 my $result;
 $result = $oldnode->safe_psql(
-   'postgres', "SELECT encoding, $provider_field, datcollate, $iculocale_field
+   'postgres', "SELECT encoding, $provider_field, datcollate, datctype, $iculocale_field
                  FROM pg_database WHERE datname='template0'");
-is($result, "$original_encoding|$original_provider|$original_collate|$original_iculocale",
+is($result, "$original_encoding|$original_provider|$original_locale|$original_locale|$original_iculocale",
        "check locales in original cluster"
    );
 
-# check ctype, which was acquired from environment by initdb
-my $original_ctype = $oldnode->safe_psql(
-   'postgres', q{SELECT datctype FROM pg_database WHERE datname='template0'});
-
 # The default location of the source code is the root of this directory.
 my $srcdir = abs_path("../../..");
 
@@ -224,7 +220,6 @@ my $newnode = PostgreSQL::Test::Cluster->new('new_node');
 # cluster.
 push @initdb_params, ('--encoding', 'SQL_ASCII');
 push @initdb_params, ('--locale-provider', 'libc');
-push @initdb_params, ('--lc-ctype', 'C');
 
 $node_params{extra} = \@initdb_params;
 $newnode->init(%node_params);
@@ -401,7 +396,7 @@ if (-d $log_path)
 $result = $newnode->safe_psql(
    'postgres', "SELECT encoding, $provider_field, datcollate, datctype, $iculocale_field
                  FROM pg_database WHERE datname='template0'");
-is($result, "$original_encoding|$original_provider|$original_collate|$original_ctype|$original_iculocale",
+is($result, "$original_encoding|$original_provider|$original_locale|$original_locale|$original_iculocale",
        "check that locales in new cluster match original cluster"
    );
 


diff --git a/src/bin/scripts/t/020_createdb.pl b/src/bin/scripts/t/020_createdb.pl

index 8ec58cdd648513860f42447acf24183739500078..af3b1492e3832be7d4fdd3afd21c226d58c38966 100644 (file)

--- a/src/bin/scripts/t/020_createdb.pl
+++ b/src/bin/scripts/t/020_createdb.pl
@@ -41,7 +41,7 @@ if ($ENV{with_icu} eq 'yes')
        [
            'createdb',        '-T',
            'template0',       '-E', 'UTF8', '--locale-provider=icu',
-           '--icu-locale=en', 'foobar5'
+           '--locale=C',      '--icu-locale=en', 'foobar5'
        ],
        qr/statement: CREATE DATABASE foobar5 .* LOCALE_PROVIDER icu ICU_LOCALE 'en'/,
        'create database with ICU locale specified');


diff --git a/src/test/icu/t/010_database.pl b/src/test/icu/t/010_database.pl

index 45d77c319a30870261631ba75fc69d9397f774c1..715b1bffd666cce7f0a53af3f44d163b030a1baa 100644 (file)

--- a/src/test/icu/t/010_database.pl
+++ b/src/test/icu/t/010_database.pl
@@ -54,7 +54,7 @@ b),
 # Test error cases in CREATE DATABASE involving locale-related options
 
 my ($ret, $stdout, $stderr) = $node1->psql('postgres',
-   q{CREATE DATABASE dbicu LOCALE_PROVIDER icu TEMPLATE template0 ENCODING UTF8});
+   q{CREATE DATABASE dbicu LOCALE_PROVIDER icu LOCALE 'C' TEMPLATE template0 ENCODING UTF8});
 isnt($ret, 0,
    "ICU locale must be specified for ICU provider: exit code not 0");
 like(

diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 497f2205f0c93e8dfa321779e8b2485412ec74bc..d48b7b6060c9e2679b86d4aa3c631d46e66a2f32 100644 (file)
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -2350,18 +2350,19 @@ setup_locale_encoding(void)
               lc_time);
    }
 
-   if (!encoding && locale_provider == COLLPROVIDER_ICU)
-   {
-       encodingid = PG_UTF8;
-       printf(_("The default database encoding has been set to \"%s\".\n"),
-              pg_encoding_to_char(encodingid));
-   }
-   else if (!encoding)
+   if (!encoding)
    {
        int         ctype_enc;
 
        ctype_enc = pg_get_encoding_from_locale(lc_ctype, true);
 
+       /*
+        * If ctype_enc=SQL_ASCII, it's compatible with any encoding. ICU does
+        * not support SQL_ASCII, so select UTF-8 instead.
+        */
+       if (locale_provider == COLLPROVIDER_ICU && ctype_enc == PG_SQL_ASCII)
+           ctype_enc = PG_UTF8;
+
        if (ctype_enc == -1)
        {
            /* Couldn't recognize the locale's codeset */

diff --git a/src/bin/pg_upgrade/t/002_pg_upgrade.pl b/src/bin/pg_upgrade/t/002_pg_upgrade.pl
index 1b5df730e99d4cf03af5cb39441412df2e31deff..90669f3c6d629d5e1809771b0c8f630e5bca0bf7 100644 (file)
--- a/src/bin/pg_upgrade/t/002_pg_upgrade.pl
+++ b/src/bin/pg_upgrade/t/002_pg_upgrade.pl
@@ -108,7 +108,7 @@ if ($oldnode->pg_version >= 11)
 
 my $original_encoding = "6"; # UTF-8
 my $original_provider = "c";
-my $original_collate = "C";
+my $original_locale = "C";
 my $original_iculocale = "";
 my $provider_field = "'c' AS datlocprovider";
 my $iculocale_field = "NULL AS daticulocale";
@@ -123,7 +123,7 @@ if ($oldnode->pg_version >= 15 && $ENV{with_icu} eq 'yes')
 my @initdb_params = @custom_opts;
 
 push @initdb_params, ('--encoding', 'UTF-8');
-push @initdb_params, ('--lc-collate', $original_collate);
+push @initdb_params, ('--locale', $original_locale);
 if ($original_provider eq "i")
 {
    push @initdb_params, ('--locale-provider', 'icu');
@@ -136,16 +136,12 @@ $oldnode->start;
 
 my $result;
 $result = $oldnode->safe_psql(
-   'postgres', "SELECT encoding, $provider_field, datcollate, $iculocale_field
+   'postgres', "SELECT encoding, $provider_field, datcollate, datctype, $iculocale_field
                  FROM pg_database WHERE datname='template0'");
-is($result, "$original_encoding|$original_provider|$original_collate|$original_iculocale",
+is($result, "$original_encoding|$original_provider|$original_locale|$original_locale|$original_iculocale",
        "check locales in original cluster"
    );
 
-# check ctype, which was acquired from environment by initdb
-my $original_ctype = $oldnode->safe_psql(
-   'postgres', q{SELECT datctype FROM pg_database WHERE datname='template0'});
-
 # The default location of the source code is the root of this directory.
 my $srcdir = abs_path("../../..");
 
@@ -224,7 +220,6 @@ my $newnode = PostgreSQL::Test::Cluster->new('new_node');
 # cluster.
 push @initdb_params, ('--encoding', 'SQL_ASCII');
 push @initdb_params, ('--locale-provider', 'libc');
-push @initdb_params, ('--lc-ctype', 'C');
 
 $node_params{extra} = \@initdb_params;
 $newnode->init(%node_params);
@@ -401,7 +396,7 @@ if (-d $log_path)
 $result = $newnode->safe_psql(
    'postgres', "SELECT encoding, $provider_field, datcollate, datctype, $iculocale_field
                  FROM pg_database WHERE datname='template0'");
-is($result, "$original_encoding|$original_provider|$original_collate|$original_ctype|$original_iculocale",
+is($result, "$original_encoding|$original_provider|$original_locale|$original_locale|$original_iculocale",
        "check that locales in new cluster match original cluster"
    );
 

diff --git a/src/bin/scripts/t/020_createdb.pl b/src/bin/scripts/t/020_createdb.pl
index 8ec58cdd648513860f42447acf24183739500078..af3b1492e3832be7d4fdd3afd21c226d58c38966 100644 (file)
--- a/src/bin/scripts/t/020_createdb.pl
+++ b/src/bin/scripts/t/020_createdb.pl
@@ -41,7 +41,7 @@ if ($ENV{with_icu} eq 'yes')
        [
            'createdb',        '-T',
            'template0',       '-E', 'UTF8', '--locale-provider=icu',
-           '--icu-locale=en', 'foobar5'
+           '--locale=C',      '--icu-locale=en', 'foobar5'
        ],
        qr/statement: CREATE DATABASE foobar5 .* LOCALE_PROVIDER icu ICU_LOCALE 'en'/,
        'create database with ICU locale specified');

diff --git a/src/test/icu/t/010_database.pl b/src/test/icu/t/010_database.pl
index 45d77c319a30870261631ba75fc69d9397f774c1..715b1bffd666cce7f0a53af3f44d163b030a1baa 100644 (file)
--- a/src/test/icu/t/010_database.pl
+++ b/src/test/icu/t/010_database.pl
@@ -54,7 +54,7 @@ b),
 # Test error cases in CREATE DATABASE involving locale-related options
 
 my ($ret, $stdout, $stderr) = $node1->psql('postgres',
-   q{CREATE DATABASE dbicu LOCALE_PROVIDER icu TEMPLATE template0 ENCODING UTF8});
+   q{CREATE DATABASE dbicu LOCALE_PROVIDER icu LOCALE 'C' TEMPLATE template0 ENCODING UTF8});
 isnt($ret, 0,
    "ICU locale must be specified for ICU provider: exit code not 0");
 like(