Extend TAP tests of pg_dump to test for compression with gzip

The test logic is extended with two new concepts:
- Addition of a compression command called compress_cmd, executed
between restore_cmd and dump_cmd to control the contents of the dumps.
In the case of this commit, this is used to compress or decompress
elements of a dump to test new code paths.
- Addition of a new flag called compile_option, to check if a set of
tests can be executed depending on the ./configure options used in a
given build.

The tests introduced here are for gzip, but they are designed so as they
can easily be extended for new compression methods.

Author: Georgios Kokolatos, Rachel Heaton
Discussion: https://postgr.es/m/faUNEOpts9vunEaLnmxmG-DldLSg_ql137OC3JYDmgrOMHm1RvvWY2IdBkv_CRxm5spCCb_OmKNk2T03TMm0fBEWveFF9wA1WizPuAgB7Ss=@protonmail.com

diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 302f7e02d6806a338a2d45e2d917c1602e502a1d..2f524b09bf53a55037013d78148f8cbca4fa7eee 100644 (file)
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -16,6 +16,8 @@ subdir = src/bin/pg_dump
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+export GZIP_PROGRAM=$(GZIP)
+
 override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 

diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index af5d6fa5a334db19a0c91d2132a239f420fecaf2..75b754a4202d3c18c20b52fd29bb8a0183c85328 100644 (file)
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -20,12 +20,22 @@ my $tempdir       = PostgreSQL::Test::Utils::tempdir;
 # test_key indicates that a given run should simply use the same
 # set of like/unlike tests as another run, and which run that is.
 #
+# compile_option indicates if the commands run depend on a compilation
+# option, if any.  This can be used to control if tests should be
+# skipped when a build dependency is not satisfied.
+#
 # dump_cmd is the pg_dump command to run, which is an array of
 # the full command and arguments to run.  Note that this is run
 # using $node->command_ok(), so the port does not need to be
 # specified and is pulled from $PGPORT, which is set by the
 # PostgreSQL::Test::Cluster system.
 #
+# compress_cmd is the utility command for (de)compression, if any.
+# Note that this should generally be used on pg_dump's output
+# either to generate a text file to run the through the tests, or
+# to test pg_restore's ability to parse manually compressed files
+# that otherwise pg_dump does not compress on its own (e.g. *.toc).
+#
 # restore_cmd is the pg_restore command to run, if any.  Note
 # that this should generally be used when the pg_dump goes to
 # a non-text file and that the restore can then be used to
@@ -54,6 +64,58 @@ my %pgdump_runs = (
            "$tempdir/binary_upgrade.dump",
        ],
    },
+
+   # Do not use --no-sync to give test coverage for data sync.
+   compression_gzip_custom => {
+       test_key       => 'compression',
+       compile_option => 'gzip',
+       dump_cmd       => [
+           'pg_dump',      '--format=custom',
+           '--compress=1', "--file=$tempdir/compression_gzip_custom.dump",
+           'postgres',
+       ],
+       restore_cmd => [
+           'pg_restore',
+           "--file=$tempdir/compression_gzip_custom.sql",
+           "$tempdir/compression_gzip_custom.dump",
+       ],
+   },
+
+   # Do not use --no-sync to give test coverage for data sync.
+   compression_gzip_dir => {
+       test_key       => 'compression',
+       compile_option => 'gzip',
+       dump_cmd       => [
+           'pg_dump',                              '--jobs=2',
+           '--format=directory',                   '--compress=1',
+           "--file=$tempdir/compression_gzip_dir", 'postgres',
+       ],
+       # Give coverage for manually compressed blob.toc files during
+       # restore.
+       compress_cmd => {
+           program => $ENV{'GZIP_PROGRAM'},
+           args    => [ '-f', "$tempdir/compression_gzip_dir/blobs.toc", ],
+       },
+       restore_cmd => [
+           'pg_restore', '--jobs=2',
+           "--file=$tempdir/compression_gzip_dir.sql",
+           "$tempdir/compression_gzip_dir",
+       ],
+   },
+
+   compression_gzip_plain => {
+       test_key       => 'compression',
+       compile_option => 'gzip',
+       dump_cmd       => [
+           'pg_dump', '--format=plain', '-Z1',
+           "--file=$tempdir/compression_gzip_plain.sql.gz", 'postgres',
+       ],
+       # Decompress the generated file to run through the tests.
+       compress_cmd => {
+           program => $ENV{'GZIP_PROGRAM'},
+           args    => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
+       },
+   },
    clean => {
        dump_cmd => [
            'pg_dump',
@@ -424,6 +486,7 @@ my %full_runs = (
    binary_upgrade           => 1,
    clean                    => 1,
    clean_if_exists          => 1,
+   compression              => 1,
    createdb                 => 1,
    defaults                 => 1,
    exclude_dump_test_schema => 1,
@@ -3098,6 +3161,7 @@ my %tests = (
            binary_upgrade          => 1,
            clean                   => 1,
            clean_if_exists         => 1,
+           compression             => 1,
            createdb                => 1,
            defaults                => 1,
            exclude_test_table      => 1,
@@ -3171,6 +3235,7 @@ my %tests = (
            binary_upgrade           => 1,
            clean                    => 1,
            clean_if_exists          => 1,
+           compression              => 1,
            createdb                 => 1,
            defaults                 => 1,
            exclude_dump_test_schema => 1,
@@ -3833,8 +3898,9 @@ if ($collation_check_stderr !~ /ERROR: /)
    $collation_support = 1;
 }
 
-# Determine whether build supports LZ4.
-my $supports_lz4 = check_pg_config("#define USE_LZ4 1");
+# Determine whether build supports LZ4 and gzip.
+my $supports_lz4  = check_pg_config("#define USE_LZ4 1");
+my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
 
 # Create additional databases for mutations of schema public
 $node->psql('postgres', 'create database regress_pg_dump_test;');
@@ -3947,9 +4013,32 @@ foreach my $run (sort keys %pgdump_runs)
    my $test_key = $run;
    my $run_db   = 'postgres';
 
+   # Skip command-level tests for gzip if there is no support for it.
+   if (   defined($pgdump_runs{$run}->{compile_option})
+       && $pgdump_runs{$run}->{compile_option} eq 'gzip'
+       && !$supports_gzip)
+   {
+       note "$run: skipped due to no gzip support";
+       next;
+   }
+
    $node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
        "$run: pg_dump runs");
 
+   if ($pgdump_runs{$run}->{compress_cmd})
+   {
+       my ($compress_cmd) = $pgdump_runs{$run}->{compress_cmd};
+       my $compress_program = $compress_cmd->{program};
+
+       # Skip the rest of the test if the compression program is
+       # not defined.
+       next if (!defined($compress_program) || $compress_program eq '');
+
+       my @full_compress_cmd =
+         ($compress_cmd->{program}, @{ $compress_cmd->{args} });
+       command_ok(\@full_compress_cmd, "$run: compression commands");
+   }
+
    if ($pgdump_runs{$run}->{restore_cmd})
    {
        $node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },