Make sure the pg_dump tar archiver can handle members larger than 2 GB, but

does not create members larger than allowed by the tar format.  Also, fix
the generation of the tar header to conform to POSIX.

diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index fbfdcb3c2a46a0f8a8b48b9483eba3947e3b4822..5b5759a1d43a1b09a35d8da268901a50bd0995bc 100644 (file)
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -1,5 +1,5 @@
 
 
@@ -662,6 +662,15 @@ CREATE DATABASE foo WITH TEMPLATE = template0;
 
    
   
+
+  
+   Members of tar archives are limited to a size less than 8 GB.
+   (This is an inherent limitation of the tar file format.)  Therefore
+   this format cannot be used if the textual representation of a table
+   exceeds that size.  The total size of a tar archive and any of the
+   other output formats is not limited, except possibly by the
+   operating system.
+  
  
 
  

diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index eb6e9fca129da2a4f94136286686c31d6fcb1052..37feef179f007ea77aa59bc23381388c7dac7db4 100644 (file)
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -16,7 +16,7 @@
  *
  *
  * IDENTIFICATION
- *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.28 2002/09/04 20:31:34 momjian Exp $
+ *     $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.29 2002/09/06 21:58:36 petere Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -73,6 +73,17 @@ typedef struct
    ArchiveHandle *AH;
 } TAR_MEMBER;
 
+/*
+ * Maximum file size for a tar member: The limit inherent in the
+ * format is 2^33-1 bytes (nearly 8 GB).  But we don't want to exceed
+ * what we can represent by an off_t.
+ */
+#ifdef INT64_IS_BUSTED
+#define MAX_TAR_MEMBER_FILELEN INT_MAX
+#else
+#define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(off_t)*8 - 1)) - 1)
+#endif
+
 typedef struct
 {
    int         hasSeek;
@@ -1006,6 +1017,8 @@ _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
     */
    fseeko(tmp, 0, SEEK_END);
    th->fileLen = ftello(tmp);
+   if (th->fileLen > MAX_TAR_MEMBER_FILELEN)
+       die_horribly(AH, modulename, "archive member too large for tar format\n");
    fseeko(tmp, 0, SEEK_SET);
 
    _tarWriteHeader(th);
@@ -1219,6 +1232,23 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
    return 1;
 }
 
+
+/*
+ * Utility routine to print possibly larger than 32 bit integers in a
+ * portable fashion.  Filled with zeros.
+ */
+static void print_val(char *s, uint64 val, unsigned int base, size_t len)
+{
+        int i;
+        for (i = len; i > 0; i--)
+        {
+                int digit = val % base;
+                s[i - 1] = '0' + digit;
+                val = val / base;
+        }
+}
+
+
 static void
 _tarWriteHeader(TAR_MEMBER *th)
 {
@@ -1235,34 +1265,30 @@ _tarWriteHeader(TAR_MEMBER *th)
    sprintf(&h[100], "100600 ");
 
    /* User ID 8 */
-   sprintf(&h[108], " 04000 ");
+   sprintf(&h[108], "004000 ");
 
    /* Group 8 */
-   sprintf(&h[116], " 02000 ");
+   sprintf(&h[116], "002000 ");
 
-   /* File size 12 */
-   /* FIXME: This goes only up to 2^30. -- What about larger files? */
-   sprintf(&h[124], "%10o ", (unsigned int) th->fileLen);
+   /* File size 12 - 11 digits, 1 space, no NUL */
+   print_val(&h[124], th->fileLen, 8, 11);
+   sprintf(&h[135], " ");
 
    /* Mod Time 12 */
-   sprintf(&h[136], "%10o ", (int) time(NULL));
+   sprintf(&h[136], "%011o ", (int) time(NULL));
 
    /* Checksum 8 */
-   sprintf(&h[148], "%6o ", lastSum);
+   sprintf(&h[148], "%06o ", lastSum);
 
-   /* Type 1 */
-   /* sprintf(&h[156], "%c", LF_NORMAL); */
+   /* Type - regular file */
    sprintf(&h[156], "0");
 
    /* Link tag 100 (NULL) */
 
-   /* Magic 8 */
-   sprintf(&h[257], "ustar  ");
-
-   /*
-    * GNU Version... sprintf(&h[257], "ustar"); sprintf(&h[263], "00");
-    */
+   /* Magic 6 + Version 2 */
+   sprintf(&h[257], "ustar00");
 
+#if 0
    /* User 32 */
    sprintf(&h[265], "%.31s", "");      /* How do I get username reliably?
                                         * Do I need to? */
@@ -1272,15 +1298,15 @@ _tarWriteHeader(TAR_MEMBER *th)
                                         * I need to? */
 
    /* Maj Dev 8 */
-   /* sprintf(&h[329], "%6o ", 0); */
-
-   /* Min Dev */
-   /* sprintf(&h[337], "%6o ", 0); */
+   sprintf(&h[329], "%6o ", 0);
 
+   /* Min Dev 8 */
+   sprintf(&h[337], "%6o ", 0);
+#endif
 
    while ((sum = _tarChecksum(h)) != lastSum)
    {
-       sprintf(&h[148], "%6o ", sum);
+       sprintf(&h[148], "%06o ", sum);
        lastSum = sum;
    }