pg_upgrade: dramatically reduce memory consumption

Report from Jeff Janes

diff --git a/contrib/pg_upgrade/check.c b/contrib/pg_upgrade/check.c
index f04707f9c6f510349ad3375d47ebf18a8cf04ee8..acf8660044d42ae6685353167b27e9873bd25eee 100644 (file)
--- a/contrib/pg_upgrade/check.c
+++ b/contrib/pg_upgrade/check.c
@@ -707,20 +707,18 @@ create_script_for_old_cluster_deletion(char **deletion_script_file_name)
            fprintf(script, "\n");
            /* remove PG_VERSION? */
            if (GET_MAJOR_VERSION(old_cluster.major_version) <= 804)
-               fprintf(script, RM_CMD " %s%s%cPG_VERSION\n",
+               fprintf(script, RM_CMD " %s%cPG_VERSION\n",
                        fix_path_separator(os_info.old_tablespaces[tblnum]),
-                       fix_path_separator(old_cluster.tablespace_suffix),
                        PATH_SEPARATOR);
 
            for (dbnum = 0; dbnum < old_cluster.dbarr.ndbs; dbnum++)
-           {
-               fprintf(script, RMDIR_CMD " %s%s%c%d\n",
+               fprintf(script, RMDIR_CMD " %s%c%d\n",
                        fix_path_separator(os_info.old_tablespaces[tblnum]),
-                       fix_path_separator(old_cluster.tablespace_suffix),
                        PATH_SEPARATOR, old_cluster.dbarr.dbs[dbnum].db_oid);
-           }
        }
        else
+       {
+           char    *suffix_path = pg_strdup(old_cluster.tablespace_suffix);
 
            /*
             * Simply delete the tablespace directory, which might be ".old"
@@ -728,7 +726,9 @@ create_script_for_old_cluster_deletion(char **deletion_script_file_name)
             */
            fprintf(script, RMDIR_CMD " %s%s\n",
                    fix_path_separator(os_info.old_tablespaces[tblnum]),
-                   fix_path_separator(old_cluster.tablespace_suffix));
+                   fix_path_separator(suffix_path));
+           pfree(suffix_path);
+       }
    }
 
    fclose(script);

diff --git a/contrib/pg_upgrade/info.c b/contrib/pg_upgrade/info.c
index 3d0dd1a1015152e375f3f930e1a45e83bc1228ab..fd083de821f1c4bdef611a1fc3fd9f197a65cae6 100644 (file)
--- a/contrib/pg_upgrade/info.c
+++ b/contrib/pg_upgrade/info.c
@@ -108,20 +108,18 @@ create_rel_filename_map(const char *old_data, const char *new_data,
         * relation belongs to the default tablespace, hence relfiles should
         * exist in the data directories.
         */
-       strlcpy(map->old_tablespace, old_data, sizeof(map->old_tablespace));
-       strlcpy(map->new_tablespace, new_data, sizeof(map->new_tablespace));
-       strlcpy(map->old_tablespace_suffix, "/base", sizeof(map->old_tablespace_suffix));
-       strlcpy(map->new_tablespace_suffix, "/base", sizeof(map->new_tablespace_suffix));
+       map->old_tablespace = old_data;
+       map->new_tablespace = new_data;
+       map->old_tablespace_suffix = "/base";
+       map->new_tablespace_suffix = "/base";
    }
    else
    {
        /* relation belongs to a tablespace, so use the tablespace location */
-       strlcpy(map->old_tablespace, old_rel->tablespace, sizeof(map->old_tablespace));
-       strlcpy(map->new_tablespace, new_rel->tablespace, sizeof(map->new_tablespace));
-       strlcpy(map->old_tablespace_suffix, old_cluster.tablespace_suffix,
-               sizeof(map->old_tablespace_suffix));
-       strlcpy(map->new_tablespace_suffix, new_cluster.tablespace_suffix,
-               sizeof(map->new_tablespace_suffix));
+       map->old_tablespace = old_rel->tablespace;
+       map->new_tablespace = new_rel->tablespace;
+       map->old_tablespace_suffix = old_cluster.tablespace_suffix;
+       map->new_tablespace_suffix = new_cluster.tablespace_suffix;
    }
 
    map->old_db_oid = old_db->db_oid;
@@ -231,7 +229,7 @@ get_db_infos(ClusterInfo *cluster)
    {
        dbinfos[tupnum].db_oid = atooid(PQgetvalue(res, tupnum, i_oid));
        dbinfos[tupnum].db_name = pg_strdup(PQgetvalue(res, tupnum, i_datname));
-       snprintf(dbinfos[tupnum].db_tblspace, sizeof(dbinfos[tupnum].db_tblspace), "%s",
+       snprintf(dbinfos[tupnum].db_tablespace, sizeof(dbinfos[tupnum].db_tablespace), "%s",
                 PQgetvalue(res, tupnum, i_spclocation));
    }
    PQclear(res);
@@ -264,6 +262,7 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo)
    int         num_rels = 0;
    char       *nspname = NULL;
    char       *relname = NULL;
+   char       *tablespace = NULL;
    int         i_spclocation,
                i_nspname,
                i_relname,
@@ -271,6 +270,7 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo)
                i_relfilenode,
                i_reltablespace;
    char        query[QUERY_ALLOC];
+   char       *last_namespace = NULL, *last_tablespace = NULL;
 
    /*
     * pg_largeobject contains user data that does not appear in pg_dumpall
@@ -366,26 +366,53 @@ get_rel_infos(ClusterInfo *cluster, DbInfo *dbinfo)
    for (relnum = 0; relnum < ntups; relnum++)
    {
        RelInfo    *curr = &relinfos[num_rels++];
-       const char *tblspace;
 
        curr->reloid = atooid(PQgetvalue(res, relnum, i_oid));
 
        nspname = PQgetvalue(res, relnum, i_nspname);
-       curr->nspname = pg_strdup(nspname);
+       curr->nsp_alloc = false;
+
+       /*
+        * Many of the namespace and tablespace strings are identical,
+        * so we try to reuse the allocated string pointers where possible
+        * to reduce memory consumption.
+        */
+       /* Can we reuse the previous string allocation? */
+       if (last_namespace && strcmp(nspname, last_namespace) == 0)
+           curr->nspname = last_namespace;
+       else
+       {
+           last_namespace = curr->nspname = pg_strdup(nspname);
+           curr->nsp_alloc = true;
+       }
 
        relname = PQgetvalue(res, relnum, i_relname);
        curr->relname = pg_strdup(relname);
 
        curr->relfilenode = atooid(PQgetvalue(res, relnum, i_relfilenode));
+       curr->tblsp_alloc = false;
 
+       /* Is the tablespace oid non-zero? */
        if (atooid(PQgetvalue(res, relnum, i_reltablespace)) != 0)
-           /* Might be "", meaning the cluster default location. */
-           tblspace = PQgetvalue(res, relnum, i_spclocation);
+       {
+           /*
+            * The tablespace location might be "", meaning the cluster
+            * default location, i.e. pg_default or pg_global.
+            */
+           tablespace = PQgetvalue(res, relnum, i_spclocation);
+
+           /* Can we reuse the previous string allocation? */
+           if (last_tablespace && strcmp(tablespace, last_tablespace) == 0)
+               curr->tablespace = last_tablespace;
+           else
+           {
+               last_tablespace = curr->tablespace = pg_strdup(tablespace);
+               curr->tblsp_alloc = true;
+           }
+       }
        else
-           /* A zero reltablespace indicates the database tablespace. */
-           tblspace = dbinfo->db_tblspace;
-
-       strlcpy(curr->tablespace, tblspace, sizeof(curr->tablespace));
+           /* A zero reltablespace oid indicates the database tablespace. */
+           curr->tablespace = dbinfo->db_tablespace;
    }
    PQclear(res);
 
@@ -419,8 +446,11 @@ free_rel_infos(RelInfoArr *rel_arr)
 
    for (relnum = 0; relnum < rel_arr->nrels; relnum++)
    {
-       pg_free(rel_arr->rels[relnum].nspname);
+       if (rel_arr->rels[relnum].nsp_alloc)
+           pg_free(rel_arr->rels[relnum].nspname);
        pg_free(rel_arr->rels[relnum].relname);
+       if (rel_arr->rels[relnum].tblsp_alloc)
+           pg_free(rel_arr->rels[relnum].tablespace);
    }
    pg_free(rel_arr->rels);
    rel_arr->nrels = 0;

diff --git a/contrib/pg_upgrade/pg_upgrade.h b/contrib/pg_upgrade/pg_upgrade.h
index de487bcb21eadd90acdbb3ea6bbc427fc6431198..389ec936d04858d1dfdb62be9e741e0d3bffcdbf 100644 (file)
--- a/contrib/pg_upgrade/pg_upgrade.h
+++ b/contrib/pg_upgrade/pg_upgrade.h
@@ -127,7 +127,9 @@ typedef struct
    Oid         reloid;         /* relation oid */
    Oid         relfilenode;    /* relation relfile node */
    /* relation tablespace path, or "" for the cluster default */
-   char        tablespace[MAXPGPATH];
+   char       *tablespace;
+   bool        nsp_alloc;
+   bool        tblsp_alloc;
 } RelInfo;
 
 typedef struct
@@ -141,10 +143,10 @@ typedef struct
  */
 typedef struct
 {
-   char        old_tablespace[MAXPGPATH];
-   char        new_tablespace[MAXPGPATH];
-   char        old_tablespace_suffix[MAXPGPATH];
-   char        new_tablespace_suffix[MAXPGPATH];
+   const char      *old_tablespace;
+   const char      *new_tablespace;
+   const char      *old_tablespace_suffix;
+   const char      *new_tablespace_suffix;
    Oid         old_db_oid;
    Oid         new_db_oid;
 
@@ -166,7 +168,7 @@ typedef struct
 {
    Oid         db_oid;         /* oid of the database */
    char       *db_name;        /* database name */
-   char        db_tblspace[MAXPGPATH]; /* database default tablespace path */
+   char        db_tablespace[MAXPGPATH]; /* database default tablespace path */
    RelInfoArr  rel_arr;        /* array of all user relinfos */
 } DbInfo;
 
@@ -256,7 +258,7 @@ typedef struct
    Oid         pg_database_oid;    /* OID of pg_database relation */
    Oid         install_role_oid;       /* OID of connected role */
    Oid         role_count;     /* number of roles defined in the cluster */
-   char       *tablespace_suffix;      /* directory specification */
+   const char *tablespace_suffix;      /* directory specification */
 } ClusterInfo;