created it.
Simon Riggs
-
+
Performance Tips
prepared INSERT statement, and then use
EXECUTE as many times as required. This avoids
some of the overhead of repeatedly parsing and planning
- INSERT.
+ INSERT. Different interfaces provide this facility
+ in different ways; look for Prepared Statements in the interface
+ documentation.
INSERT, even if PREPARE is used and
multiple insertions are batched into a single transaction.
+
+ COPY is fastest when used within the same
+ transaction as an earlier CREATE TABLE or
+ TRUNCATE command. In those cases, no WAL
+ needs to be written because in case of an error, the files
+ containing the newly loaded data will be removed automatically.
+ CREATE TABLE AS SELECT is also optimized
+ to avoid writing WAL. COPY and
+ CREATE TABLE AS SELECT will write WAL
+ when is set and will not
+ therefore be optimized in that case.
+
+
+
+
Turn off <varname>archive_command</varname>
+
+ When loading large amounts of data you may want to unset the
+ before loading. It may be
+ faster to take a new base backup once the load has completed
+ than to allow a large archive to accumulate.
+
+
+ This is particularly important advice because certain commands
+ will perform more slowly when archive_command
+ is set, as a result of their needing to write large amounts of WAL.
+ This applies to the following commands:
+ CREATE TABLE AS SELECT,
+ CREATE INDEX and also COPY, when
+ it is executed in the same transaction as a prior
+ CREATE TABLE or TRUNCATE command.
+
+
+
+
Run <command>ANALYZE</command> Afterwards
the first several guidelines are handled automatically. What is left
for you to do is to set appropriate (i.e., larger than normal) values
for maintenance_work_mem and
- checkpoint_segments before loading the dump script,
- and then to run ANALYZE afterwards.
+ checkpoint_segments, as well as unsetting
+ archive_command before loading the dump script,
+ and then to run ANALYZE afterwards and resetting
+ archive_command if required. All of the
+ parameters can be reset once the load has completed without needing
+ to restart the server, as described in .
*
*
* IDENTIFICATION
- * $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.224 2007/01/09 22:00:59 momjian Exp $
+ * $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.225 2007/01/25 02:17:25 momjian Exp $
*
*
* INTERFACE ROUTINES
* heap_update - replace a tuple in a relation with another tuple
* heap_markpos - mark scan position
* heap_restrpos - restore position to marked location
+ * heap_sync - sync heap, for when no WAL has been written
*
* NOTES
* This file contains the heap_ routines which implement
#include "miscadmin.h"
#include "pgstat.h"
#include "storage/procarray.h"
+#include "storage/smgr.h"
#include "utils/inval.h"
#include "utils/lsyscache.h"
#include "utils/relcache.h"
* that all new tuples go into new pages not containing any tuples from other
* transactions, that the relation gets fsync'd before commit, and that the
* transaction emits at least one WAL record to ensure RecordTransactionCommit
- * will decide to WAL-log the commit.
+ * will decide to WAL-log the commit. (see heap_sync() comments also)
*
* use_fsm is passed directly to RelationGetBufferForTuple, which see for
* more info.
*/
if (HeapTupleHasExternal(tup) ||
(MAXALIGN(tup->t_len) > TOAST_TUPLE_THRESHOLD))
- heaptup = toast_insert_or_update(relation, tup, NULL);
+ heaptup = toast_insert_or_update(relation, tup, NULL, use_wal);
else
heaptup = tup;
return heap_insert(relation, tup, GetCurrentCommandId(), true, true);
}
+/*
+ * fast_heap_insert - insert a tuple with options to improve speed
+ *
+ * Currently, this routine allows specifying additional options for speed
+ * in certain cases, such as WAL-avoiding COPY command
+ */
+Oid
+fast_heap_insert(Relation relation, HeapTuple tup, bool use_wal)
+{
+ return heap_insert(relation, tup, GetCurrentCommandId(), use_wal, use_wal);
+}
+
/*
* heap_delete - delete a tuple
*
*
* Note: below this point, heaptup is the data we actually intend to
* store into the relation; newtup is the caller's original untoasted
- * data.
+ * data. (We always use WAL for toast table updates.)
*/
if (need_toast)
{
- heaptup = toast_insert_or_update(relation, newtup, &oldtup);
+ heaptup = toast_insert_or_update(relation, newtup, &oldtup, true);
newtupsize = MAXALIGN(heaptup->t_len);
}
else
else
appendStringInfo(buf, "UNKNOWN");
}
+
+/* ----------------
+ * heap_sync - sync a heap, for use when no WAL has been written
+ *
+ * ----------------
+ */
+void
+heap_sync(Relation rel)
+{
+ if (!rel->rd_istemp)
+ {
+ /*
+ * If we skipped using WAL, and it's not a temp relation,
+ * we must force the relation down to disk before it's
+ * safe to commit the transaction. This requires forcing
+ * out any dirty buffers and then doing a forced fsync.
+ */
+ FlushRelationBuffers(rel);
+ smgrimmedsync(rel->rd_smgr);
+ }
+}
*
*
* IDENTIFICATION
- * $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.68 2007/01/05 22:19:22 momjian Exp $
+ * $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.69 2007/01/25 02:17:26 momjian Exp $
*
*
* INTERFACE ROUTINES
#undef TOAST_DEBUG
static void toast_delete_datum(Relation rel, Datum value);
-static Datum toast_save_datum(Relation rel, Datum value);
+static Datum toast_save_datum(Relation rel, Datum value, bool use_wal);
static varattrib *toast_fetch_datum(varattrib *attr);
static varattrib *toast_fetch_datum_slice(varattrib *attr,
int32 sliceoffset, int32 length);
* ----------
*/
HeapTuple
-toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup)
+toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, bool use_wal)
{
HeapTuple result_tuple;
TupleDesc tupleDesc;
i = biggest_attno;
old_value = toast_values[i];
toast_action[i] = 'p';
- toast_values[i] = toast_save_datum(rel, toast_values[i]);
+ toast_values[i] = toast_save_datum(rel, toast_values[i], use_wal);
if (toast_free[i])
pfree(DatumGetPointer(old_value));
i = biggest_attno;
old_value = toast_values[i];
toast_action[i] = 'p';
- toast_values[i] = toast_save_datum(rel, toast_values[i]);
+ toast_values[i] = toast_save_datum(rel, toast_values[i], use_wal);
if (toast_free[i])
pfree(DatumGetPointer(old_value));
* ----------
*/
static Datum
-toast_save_datum(Relation rel, Datum value)
+toast_save_datum(Relation rel, Datum value, bool use_wal)
{
Relation toastrel;
Relation toastidx;
if (!HeapTupleIsValid(toasttup))
elog(ERROR, "failed to build TOAST tuple");
- simple_heap_insert(toastrel, toasttup);
+ fast_heap_insert(toastrel, toasttup, use_wal);
/*
* Create the index entry. We cheat a little here by not using
*
*
* IDENTIFICATION
- * $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.276 2007/01/09 02:14:11 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.277 2007/01/25 02:17:26 momjian Exp $
*
*
* INTERFACE ROUTINES
heap_close(pg_class, RowExclusiveLock);
+ /* Remember we did this in current transaction, to allow later optimisations */
+ relation->rd_newRelfilenodeSubid = GetCurrentSubTransactionId();
+
/* Make sure the relfilenode change is visible */
CommandCounterIncrement();
}
*
*
* IDENTIFICATION
- * $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.274 2007/01/05 22:19:25 momjian Exp $
+ * $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.275 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
ExprContext *econtext; /* used for ExecEvalExpr for default atts */
MemoryContext oldcontext = CurrentMemoryContext;
ErrorContextCallback errcontext;
+ bool use_wal = true; /* By default, we use WAL to log db changes */
Assert(cstate->rel);
nfields = file_has_oids ? (attr_count + 1) : attr_count;
field_strings = (char **) palloc(nfields * sizeof(char *));
+ /*
+ * Check for performance optimization by avoiding WAL writes
+ *
+ * If archive logging is not be enabled *and* either
+ * - table is created in same transaction as this COPY
+ * - table data is now being written to new relfilenode
+ * then we can safely avoid writing WAL. Why?
+ * The data files for the table plus toast table/index, plus any indexes
+ * will all be dropped at the end of the transaction if it fails, so we
+ * do not need to worry about inconsistent states.
+ * As mentioned in comments in utils/rel.h, the in-same-transaction test is
+ * not completely reliable, since rd_createSubId can be reset to zero in
+ * certain cases before the end of the creating transaction.
+ * We are doing this for performance only, so we only need to know:
+ * if rd_createSubid != InvalidSubTransactionId then it is *always* just
+ * created. If we have PITR enabled, then we *must* use_wal
+ */
+ if ((cstate->rel->rd_createSubid != InvalidSubTransactionId ||
+ cstate->rel->rd_newRelfilenodeSubid != InvalidSubTransactionId)
+ && !XLogArchivingActive())
+ use_wal = false;
+
/* Initialize state variables */
cstate->fe_eof = false;
cstate->eol_type = EOL_UNKNOWN;
ExecConstraints(resultRelInfo, slot, estate);
/* OK, store the tuple and create index entries for it */
- simple_heap_insert(cstate->rel, tuple);
+ fast_heap_insert(cstate->rel, tuple, use_wal);
if (resultRelInfo->ri_NumIndices > 0)
ExecInsertIndexTuples(slot, &(tuple->t_self), estate, false);
}
}
+ /*
+ * If we skipped writing WAL for heaps, then we need to sync
+ */
+ if (!use_wal)
+ {
+ /* main heap */
+ heap_sync(cstate->rel);
+
+ /* main heap indexes, if any */
+ /* we always use WAL for index inserts, so no need to sync */
+
+ /* toast heap, if any */
+ if (OidIsValid(cstate->rel->rd_rel->reltoastrelid))
+ {
+ Relation toastrel;
+
+ toastrel = heap_open(cstate->rel->rd_rel->reltoastrelid,
+ AccessShareLock);
+ heap_sync(toastrel);
+ heap_close(toastrel, AccessShareLock);
+ }
+
+ /* toast index, if toast heap */
+ /* we always use WAL for index inserts, so no need to sync */
+ }
+
/* Done, clean up */
error_context_stack = errcontext.previous;
*
*
* IDENTIFICATION
- * $PostgreSQL: pgsql/src/backend/executor/execMain.c,v 1.283 2007/01/05 22:19:27 momjian Exp $
+ * $PostgreSQL: pgsql/src/backend/executor/execMain.c,v 1.284 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
*/
if (!estate->es_into_relation_use_wal &&
!estate->es_into_relation_descriptor->rd_istemp)
- {
- FlushRelationBuffers(estate->es_into_relation_descriptor);
- /* FlushRelationBuffers will have opened rd_smgr */
- smgrimmedsync(estate->es_into_relation_descriptor->rd_smgr);
- }
+ heap_sync(estate->es_into_relation_descriptor);
/* close rel, but keep lock until commit */
heap_close(estate->es_into_relation_descriptor, NoLock);
*
*
* IDENTIFICATION
- * $PostgreSQL: pgsql/src/backend/utils/cache/relcache.c,v 1.254 2007/01/09 02:14:15 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/utils/cache/relcache.c,v 1.255 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
relation->rd_refcnt = 0;
relation->rd_isnailed = false;
relation->rd_createSubid = InvalidSubTransactionId;
+ relation->rd_newRelfilenodeSubid = InvalidSubTransactionId;
relation->rd_istemp = isTempNamespace(relation->rd_rel->relnamespace);
/*
*/
relation->rd_isnailed = true;
relation->rd_createSubid = InvalidSubTransactionId;
+ relation->rd_newRelfilenodeSubid = InvalidSubTransactionId;
relation->rd_istemp = false;
/*
Oid save_relid = RelationGetRelid(relation);
int old_refcnt = relation->rd_refcnt;
SubTransactionId old_createSubid = relation->rd_createSubid;
+ SubTransactionId old_newRelfilenodeSubid = relation->rd_newRelfilenodeSubid;
TupleDesc old_att = relation->rd_att;
RuleLock *old_rules = relation->rd_rules;
MemoryContext old_rulescxt = relation->rd_rulescxt;
}
relation->rd_refcnt = old_refcnt;
relation->rd_createSubid = old_createSubid;
+ relation->rd_newRelfilenodeSubid = old_newRelfilenodeSubid;
+
if (equalTupleDescs(old_att, relation->rd_att))
{
/* needn't flush typcache here */
{
bool rebuild;
- if (relation->rd_createSubid != InvalidSubTransactionId)
+ if (relation->rd_createSubid != InvalidSubTransactionId ||
+ relation->rd_newRelfilenodeSubid != InvalidSubTransactionId)
{
/*
* New relcache entries are always rebuilt, not flushed; else we'd
* so we do not touch new-in-transaction relations; they cannot be targets
* of cross-backend SI updates (and our own updates now go through a
* separate linked list that isn't limited by the SI message buffer size).
+ * We don't do anything special for newRelfilenode-in-transaction relations,
+ * though since we have a lock on the relation nobody else should be
+ * generating cache invalidation messages for it anyhow.
*
* We do this in two phases: the first pass deletes deletable items, and
* the second one rebuilds the rebuildable items. This is essential for
continue;
}
}
+ relation->rd_newRelfilenodeSubid = InvalidSubTransactionId;
/*
* Flush any temporary index list.
continue;
}
}
+ if (relation->rd_newRelfilenodeSubid == mySubid)
+ {
+ if (isCommit)
+ relation->rd_newRelfilenodeSubid = parentSubid;
+ else
+ relation->rd_newRelfilenodeSubid = InvalidSubTransactionId;
+ }
/*
* Flush any temporary index list.
/* it's being created in this transaction */
rel->rd_createSubid = GetCurrentSubTransactionId();
+ rel->rd_newRelfilenodeSubid = InvalidSubTransactionId;
/* must flag that we have rels created in this transaction */
need_eoxact_work = true;
rel->rd_indexlist = NIL;
rel->rd_oidindex = InvalidOid;
rel->rd_createSubid = InvalidSubTransactionId;
+ rel->rd_newRelfilenodeSubid = InvalidSubTransactionId;
rel->rd_amcache = NULL;
MemSet(&rel->pgstat_info, 0, sizeof(rel->pgstat_info));
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
- * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.119 2007/01/09 22:01:00 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.120 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
extern void simple_heap_update(Relation relation, ItemPointer otid,
HeapTuple tup);
+extern Oid fast_heap_insert(Relation relation, HeapTuple tup, bool use_wal);
+
+
extern void heap_markpos(HeapScanDesc scan);
extern void heap_restrpos(HeapScanDesc scan);
extern HeapTuple heap_addheader(int natts, bool withoid,
Size structlen, void *structure);
+extern void heap_sync(Relation relation);
+
#endif /* HEAPAM_H */
*
* Copyright (c) 2000-2007, PostgreSQL Global Development Group
*
- * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.29 2007/01/05 22:19:51 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.30 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
* ----------
*/
extern HeapTuple toast_insert_or_update(Relation rel,
- HeapTuple newtup, HeapTuple oldtup);
+ HeapTuple newtup, HeapTuple oldtup, bool use_wal);
/* ----------
* toast_delete -
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
- * $PostgreSQL: pgsql/src/include/utils/rel.h,v 1.95 2007/01/09 02:14:16 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/utils/rel.h,v 1.96 2007/01/25 02:17:26 momjian Exp $
*
*-------------------------------------------------------------------------
*/
char rd_indexvalid; /* state of rd_indexlist: 0 = not valid, 1 =
* valid, 2 = temporarily forced */
SubTransactionId rd_createSubid; /* rel was created in current xact */
+ SubTransactionId rd_newRelfilenodeSubid; /* rel had new relfilenode in current xact */
/*
* rd_createSubid is the ID of the highest subtransaction the rel has
projects / postgresql.git / commitdiff