Although we can't support out-of-line TOAST storage in indexes (yet),

compressed storage works perfectly well.  Might as well have a coherent
strategy for applying it, rather than the haphazard store-what-you-get
approach that was in the code before.  The strategy I've set up here is
to attempt compression of any compressible index value exceeding
BLCKSZ/16, or about 500 bytes by default.

diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c
index 16b5233eb3907b2b1ead4dde2175a00a4bf1d6b3..4b46c202dcd47064038267fcbb5f99fb47df8787 100644 (file)
--- a/src/backend/access/common/indextuple.c
+++ b/src/backend/access/common/indextuple.c
@@ -2,14 +2,14 @@
  *
  * indextuple.c
  *    This file contains index tuple accessor and mutator routines,
- *    as well as a few various tuple utilities.
+ *    as well as various tuple utilities.
  *
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  *
  * IDENTIFICATION
- *   $Header: /cvsroot/pgsql/src/backend/access/common/indextuple.c,v 1.50 2001/01/24 19:42:47 momjian Exp $
+ *   $Header: /cvsroot/pgsql/src/backend/access/common/indextuple.c,v 1.51 2001/02/15 20:57:01 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -57,25 +57,45 @@ index_formtuple(TupleDesc tupleDescriptor,
 #ifdef TOAST_INDEX_HACK
    for (i = 0; i < numberOfAttributes; i++)
    {
-       if (null[i] != ' ' || tupleDescriptor->attrs[i]->attlen >= 0)
+       Form_pg_attribute  att = tupleDescriptor->attrs[i];
+
+       untoasted_value[i] = value[i];
+       untoasted_free[i] = false;
+
+       /* Do nothing if value is NULL or not of varlena type */
+       if (null[i] != ' ' || att->attlen >= 0)
+           continue;
+
+       /*
+        * If value is stored EXTERNAL, must fetch it so we are not
+        * depending on outside storage.  This should be improved someday.
+        */
+       if (VARATT_IS_EXTERNAL(value[i]))
        {
-           untoasted_value[i] = value[i];
-           untoasted_free[i] = false;
+           untoasted_value[i] = PointerGetDatum(
+               heap_tuple_fetch_attr(
+                   (varattrib *) DatumGetPointer(value[i])));
+           untoasted_free[i] = true;
        }
-       else
+
+       /*
+        * If value is above size target, and is of a compressible datatype,
+        * try to compress it in-line.
+        */
+       if (VARATT_SIZE(untoasted_value[i]) > TOAST_INDEX_TARGET &&
+           !VARATT_IS_EXTENDED(untoasted_value[i]) &&
+           (att->attstorage == 'x' || att->attstorage == 'm'))
        {
-           if (VARATT_IS_EXTERNAL(value[i]))
+           Datum   cvalue = toast_compress_datum(untoasted_value[i]);
+
+           if (DatumGetPointer(cvalue) != NULL)
            {
-               untoasted_value[i] = PointerGetDatum(
-                       heap_tuple_fetch_attr(
-                       (varattrib *)DatumGetPointer(value[i])));
+               /* successful compression */
+               if (untoasted_free[i])
+                   pfree(DatumGetPointer(untoasted_value[i]));
+               untoasted_value[i] = cvalue;
                untoasted_free[i] = true;
            }
-           else
-           {
-               untoasted_value[i] = value[i];
-               untoasted_free[i] = false;
-           }
        }
    }
 #endif
@@ -137,10 +157,9 @@ index_formtuple(TupleDesc tupleDescriptor,
     * Here we make sure that the size will fit in the field reserved for
     * it in t_info.
     */
-
    if ((size & INDEX_SIZE_MASK) != size)
-       elog(ERROR, "index_formtuple: data takes %lu bytes: too big",
-           (unsigned long)size);
+       elog(ERROR, "index_formtuple: data takes %lu bytes, max is %d",
+            (unsigned long) size, INDEX_SIZE_MASK);
 
    infomask |= size;
 

diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c
index a1497e40f1f0b16bc42fe1c846ff749ece09083a..a3cf6ae7116a7e4dc11eb40752e1dfe74efc9c9f 100644 (file)
--- a/src/backend/access/heap/tuptoaster.c
+++ b/src/backend/access/heap/tuptoaster.c
@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *   $Header: /cvsroot/pgsql/src/backend/access/heap/tuptoaster.c,v 1.16 2001/02/09 17:30:03 tgl Exp $
+ *   $Header: /cvsroot/pgsql/src/backend/access/heap/tuptoaster.c,v 1.17 2001/02/15 20:57:01 tgl Exp $
  *
  *
  * INTERFACE ROUTINES
@@ -45,7 +45,6 @@ static void           toast_delete(Relation rel, HeapTuple oldtup);
 static void            toast_delete_datum(Relation rel, Datum value);
 static void            toast_insert_or_update(Relation rel, HeapTuple newtup,
                                HeapTuple oldtup);
-static Datum       toast_compress_datum(Datum value);
 static Datum       toast_save_datum(Relation rel, Oid mainoid, int16 attno, Datum value);
 static varattrib   *toast_fetch_datum(varattrib *attr);
 
@@ -721,7 +720,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup)
  * the tuple!
  * ----------
  */
-static Datum
+Datum
 toast_compress_datum(Datum value)
 {
    varattrib      *tmp;

diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h
index 23f31aec522572def9504b8c91623880d3d088bd..05c4b2c1e408eb353ec4b997a5d921dca5f9f34f 100644 (file)
--- a/src/include/access/tuptoaster.h
+++ b/src/include/access/tuptoaster.h
@@ -6,7 +6,7 @@
  *
  * Copyright (c) 2000, PostgreSQL Development Team
  *
- * $Id: tuptoaster.h,v 1.8 2000/08/04 04:16:10 tgl Exp $
+ * $Id: tuptoaster.h,v 1.9 2001/02/15 20:57:01 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -40,6 +40,12 @@
 
 #define TOAST_TUPLE_TARGET     (MaxTupleSize / 4)
 
+/*
+ * If an index value is larger than TOAST_INDEX_TARGET, we will try to
+ * compress it (we can't move it out-of-line, however).  Note that this
+ * number is per-datum, not per-tuple, for simplicity in index_formtuple().
+ */
+#define TOAST_INDEX_TARGET     (MaxTupleSize / 16)
 
 /*
  * When we store an oversize datum externally, we divide it into chunks
@@ -95,6 +101,14 @@ extern varattrib *heap_tuple_fetch_attr(varattrib * attr);
  */
 extern varattrib *heap_tuple_untoast_attr(varattrib * attr);
 
+/* ----------
+ * toast_compress_datum -
+ *
+ * Create a compressed version of a varlena datum, if possible
+ * ----------
+ */
+extern Datum toast_compress_datum(Datum value);
+
 #endif  /* TUPLE_TOASTER_ACTIVE */