Accept XML documents when xmloption = content, as required by SQL:2006+.

Previously we were using the SQL:2003 definition, which doesn't allow
this, but that creates a serious dump/restore gotcha: there is no
setting of xmloption that will allow all valid XML data.  Hence,
switch to the 2006 definition.

Since libxml doesn't accept  directives in the mode we
use for CONTENT parsing, the implementation is to detect 
in the input and switch to DOCUMENT parsing mode.  This should not
cost much, because  should be close to the front of the
input if it's there at all.  It's possible that this causes the
error messages for malformed input to be slightly different than
they were before, if said input includes ; but that does
not seem like a big problem.

In passing, buy back a few cycles in parsing of large XML documents
by not doing strlen() of the whole input in parse_xml_decl().

Back-patch because dump/restore failures are not nice.  This change
shouldn't break any cases that worked before, so it seems safe to
back-patch.

Chapman Flack (revised a bit by me)

Discussion: https://postgr.es/m/CAN-V+g-6JqUQEQZ55Q3toXEN6d5Ez5uvzL4VR+8KtvJKj31taw@mail.gmail.com

diff --git a/doc/src/sgml/datatype.sgml b/doc/src/sgml/datatype.sgml
index f519500d66f40361fc01d358a82e60c0e6d38401..6fdcc12661b08df76d925b0aab1e557395a75586 100644 (file)
--- a/doc/src/sgml/datatype.sgml
+++ b/doc/src/sgml/datatype.sgml
@@ -4197,9 +4197,11 @@ a0ee-bc99-9c0b-4ef8-bb6d-6bb9-bd38-0a11
    
     The xml type can store well-formed
     documents, as defined by the XML standard, as well
-    as content fragments, which are defined by the
-    production XMLDecl? content in the XML
-    standard.  Roughly, this means that content fragments can have
+    as content fragments, which are defined by reference
+    to the more permissive
+    document node
+    of the XQuery and XPath data model.
+    Roughly, this means that content fragments can have
     more than one top-level element or character node.  The expression
     xmlvalue IS DOCUMENT
     can be used to evaluate whether a particular xml
@@ -4274,16 +4276,6 @@ SET xmloption TO { DOCUMENT | CONTENT };
     data are allowed.
    
 
-   
-    
-     With the default XML option setting, you cannot directly cast
-     character strings to type xml if they contain a
-     document type declaration, because the definition of XML content
-     fragment does not accept them.  If you need to do that, either
-     use XMLPARSE or change the XML option.
-    
-   
-
    
 
    

diff --git a/src/backend/utils/adt/xml.c b/src/backend/utils/adt/xml.c
index 2bd118c0ef54028a4e31c07d1a59f9dc629a52b5..67e95e43268767ed55d5993831c470d70708ceb6 100644 (file)
--- a/src/backend/utils/adt/xml.c
+++ b/src/backend/utils/adt/xml.c
@@ -140,6 +140,7 @@ static int parse_xml_decl(const xmlChar *str, size_t *lenp,
               xmlChar **version, xmlChar **encoding, int *standalone);
 static bool print_xml_decl(StringInfo buf, const xmlChar *version,
               pg_enc encoding, int standalone);
+static bool xml_doctype_in_content(const xmlChar *str);
 static xmlDocPtr xml_parse(text *data, XmlOptionType xmloption_arg,
          bool preserve_whitespace, int encoding);
 static text *xml_xmlnodetoxmltype(xmlNodePtr cur, PgXmlErrorContext *xmlerrcxt);
@@ -1212,8 +1213,15 @@ parse_xml_decl(const xmlChar *str, size_t *lenp,
    if (xmlStrncmp(p, (xmlChar *) "
        goto finished;
 
-   /* if next char is name char, it's a PI like  */
-   utf8len = strlen((const char *) (p + 5));
+   /*
+    * If next char is a name char, it's a PI like 
+    * rather than an XMLDecl, so we have done what we came to do and found no
+    * XMLDecl.
+    *
+    * We need an input length value for xmlGetUTF8Char, but there's no need
+    * to count the whole document size, so use strnlen not strlen.
+    */
+   utf8len = strnlen((const char *) (p + 5), MAX_MULTIBYTE_CHAR_LEN);
    utf8char = xmlGetUTF8Char(p + 5, &utf8len);
    if (PG_XMLISNAMECHAR(utf8char))
        goto finished;
@@ -1384,6 +1392,88 @@ print_xml_decl(StringInfo buf, const xmlChar *version,
        return false;
 }
 
+/*
+ * Test whether an input that is to be parsed as CONTENT contains a DTD.
+ *
+ * The SQL/XML:2003 definition of CONTENT ("XMLDecl? content") is not
+ * satisfied by a document with a DTD, which is a bit of a wart, as it means
+ * the CONTENT type is not a proper superset of DOCUMENT.  SQL/XML:2006 and
+ * later fix that, by redefining content with reference to the "more
+ * permissive" Document Node of the XQuery/XPath Data Model, such that any
+ * DOCUMENT value is indeed also a CONTENT value.  That definition is more
+ * useful, as CONTENT becomes usable for parsing input of unknown form (think
+ * pg_restore).
+ *
+ * As used below in parse_xml when parsing for CONTENT, libxml does not give
+ * us the 2006+ behavior, but only the 2003; it will choke if the input has
+ * a DTD.  But we can provide the 2006+ definition of CONTENT easily enough,
+ * by detecting this case first and simply doing the parse as DOCUMENT.
+ *
+ * A DTD can be found arbitrarily far in, but that would be a contrived case;
+ * it will ordinarily start within a few dozen characters.  The only things
+ * that can precede it are an XMLDecl (here, the caller will have called
+ * parse_xml_decl already), whitespace, comments, and processing instructions.
+ * This function need only return true if it sees a valid sequence of such
+ * things leading to 
+ * cases, including malformed input; that will mean the input gets parsed as
+ * CONTENT as originally planned, with libxml reporting any errors.
+ *
+ * This is only to be called from xml_parse, when pg_xml_init has already
+ * been called.  The input is already in UTF8 encoding.
+ */
+static bool
+xml_doctype_in_content(const xmlChar *str)
+{
+   const xmlChar *p = str;
+
+   for (;;)
+   {
+       const xmlChar *e;
+
+       SKIP_XML_SPACE(p);
+       if (*p != '<')
+           return false;
+       p++;
+
+       if (*p == '!')
+       {
+           p++;
+
+           /* if we see 
+           if (xmlStrncmp(p, (xmlChar *) "DOCTYPE", 7) == 0)
+               return true;
+
+           /* otherwise, if it's not a comment, fail */
+           if (xmlStrncmp(p, (xmlChar *) "--", 2) != 0)
+               return false;
+           /* find end of comment: find -- and a > must follow */
+           p = xmlStrstr(p + 2, (xmlChar *) "--");
+           if (!p || p[2] != '>')
+               return false;
+           /* advance over comment, and keep scanning */
+           p += 3;
+           continue;
+       }
+
+       /* otherwise, if it's not a PI , fail */
+       if (*p != '?')
+           return false;
+       p++;
+
+       /* find end of PI (the string ?> is forbidden within a PI) */
+       e = xmlStrstr(p, (xmlChar *) "?>");
+       if (!e)
+           return false;
+
+       /* we don't check PIs carefully, but do reject "xml" target */
+       if (e - p >= 3 && xmlStrncasecmp(p, (xmlChar *) "xml", 3) == 0)
+           return false;
+
+       /* advance over PI, keep scanning */
+       p = e + 2;
+   }
+}
+
 
 /*
  * Convert a C string to XML internal representation
@@ -1419,6 +1509,12 @@ xml_parse(text *data, XmlOptionType xmloption_arg, bool preserve_whitespace,
    /* Use a TRY block to ensure we clean up correctly */
    PG_TRY();
    {
+       bool        parse_as_document = false;
+       int         res_code;
+       size_t      count = 0;
+       xmlChar    *version = NULL;
+       int         standalone = 0;
+
        xmlInitParser();
 
        ctxt = xmlNewParserCtxt();
@@ -1426,7 +1522,25 @@ xml_parse(text *data, XmlOptionType xmloption_arg, bool preserve_whitespace,
            xml_ereport(xmlerrcxt, ERROR, ERRCODE_OUT_OF_MEMORY,
                        "could not allocate parser context");
 
+       /* Decide whether to parse as document or content */
        if (xmloption_arg == XMLOPTION_DOCUMENT)
+           parse_as_document = true;
+       else
+       {
+           /* Parse and skip over the XML declaration, if any */
+           res_code = parse_xml_decl(utf8string,
+                                     &count, &version, NULL, &standalone);
+           if (res_code != 0)
+               xml_ereport_by_code(ERROR, ERRCODE_INVALID_XML_CONTENT,
+                                   "invalid XML content: invalid XML declaration",
+                                   res_code);
+
+           /* Is there a DOCTYPE element? */
+           if (xml_doctype_in_content(utf8string + count))
+               parse_as_document = true;
+       }
+
+       if (parse_as_document)
        {
            /*
             * Note, that here we try to apply DTD defaults
@@ -1441,23 +1555,18 @@ xml_parse(text *data, XmlOptionType xmloption_arg, bool preserve_whitespace,
                                 XML_PARSE_NOENT | XML_PARSE_DTDATTR
                                 | (preserve_whitespace ? 0 : XML_PARSE_NOBLANKS));
            if (doc == NULL || xmlerrcxt->err_occurred)
-               xml_ereport(xmlerrcxt, ERROR, ERRCODE_INVALID_XML_DOCUMENT,
-                           "invalid XML document");
+           {
+               /* Use original option to decide which error code to throw */
+               if (xmloption_arg == XMLOPTION_DOCUMENT)
+                   xml_ereport(xmlerrcxt, ERROR, ERRCODE_INVALID_XML_DOCUMENT,
+                               "invalid XML document");
+               else
+                   xml_ereport(xmlerrcxt, ERROR, ERRCODE_INVALID_XML_CONTENT,
+                               "invalid XML content");
+           }
        }
        else
        {
-           int         res_code;
-           size_t      count;
-           xmlChar    *version;
-           int         standalone;
-
-           res_code = parse_xml_decl(utf8string,
-                                     &count, &version, NULL, &standalone);
-           if (res_code != 0)
-               xml_ereport_by_code(ERROR, ERRCODE_INVALID_XML_CONTENT,
-                                   "invalid XML content: invalid XML declaration",
-                                   res_code);
-
            doc = xmlNewDoc(version);
            Assert(doc->encoding == NULL);
            doc->encoding = xmlStrdup((const xmlChar *) "UTF-8");


diff --git a/src/test/regress/expected/xml.out b/src/test/regress/expected/xml.out

index 6e1f885112ca0ac4b8e2fa3564727bea07cb21d4..2ed0d44b1e091edbbaf8b8f708d4303b31f3e96b 100644 (file)

--- a/src/test/regress/expected/xml.out
+++ b/src/test/regress/expected/xml.out
@@ -532,6 +532,13 @@ LINE 1: EXECUTE foo ('bad');
 DETAIL:  line 1: Start tag expected, '<' not found
 bad
 ^
+SELECT xml '';
+ERROR:  invalid XML document
+LINE 1: SELECT xml '';
+                   ^
+DETAIL:  line 1: Extra content at the end of the document
+
+                ^
 SET XML OPTION CONTENT;
 EXECUTE foo ('');
   xmlconcat   
@@ -545,6 +552,45 @@ EXECUTE foo ('good');
  good
 (1 row)
 
+SELECT xml '  ';
+                                xml                                 
+--------------------------------------------------------------------
+   
+(1 row)
+
+SELECT xml '  ';
+             xml              
+------------------------------
+   
+(1 row)
+
+SELECT xml '';
+       xml        
+------------------
+ 
+(1 row)
+
+SELECT xml ' oops ';
+ERROR:  invalid XML content
+LINE 1: SELECT xml ' oops ';
+                   ^
+DETAIL:  line 1: StartTag: invalid element name
+ oops 
+                 ^
+SELECT xml '  ';
+ERROR:  invalid XML content
+LINE 1: SELECT xml '  ';
+                   ^
+DETAIL:  line 1: StartTag: invalid element name
+  
+                    ^
+SELECT xml '';
+ERROR:  invalid XML content
+LINE 1: SELECT xml '';
+                   ^
+DETAIL:  line 1: Extra content at the end of the document
+
+                ^
 -- Test backwards parsing
 CREATE VIEW xmlview1 AS SELECT xmlcomment('test');
 CREATE VIEW xmlview2 AS SELECT xmlconcat('hello', 'you');


diff --git a/src/test/regress/expected/xml_1.out b/src/test/regress/expected/xml_1.out

index 0eba424346285a3ade2a7f816740812889953edb..9dd9b3f7ca5c1fbd4edcefade6502eaec3915d85 100644 (file)

--- a/src/test/regress/expected/xml_1.out
+++ b/src/test/regress/expected/xml_1.out
@@ -429,11 +429,53 @@ EXECUTE foo ('');
 ERROR:  prepared statement "foo" does not exist
 EXECUTE foo ('bad');
 ERROR:  prepared statement "foo" does not exist
+SELECT xml '';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '';
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
 SET XML OPTION CONTENT;
 EXECUTE foo ('');
 ERROR:  prepared statement "foo" does not exist
 EXECUTE foo ('good');
 ERROR:  prepared statement "foo" does not exist
+SELECT xml '  ';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml ' 
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
+SELECT xml '  ';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '  
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
+SELECT xml '';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '';
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
+SELECT xml ' oops ';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml ' oops ';
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
+SELECT xml '  ';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '  ';
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
+SELECT xml '';
+ERROR:  unsupported XML feature
+LINE 1: SELECT xml '';
+                   ^
+DETAIL:  This functionality requires the server to be built with libxml support.
+HINT:  You need to rebuild PostgreSQL using --with-libxml.
 -- Test backwards parsing
 CREATE VIEW xmlview1 AS SELECT xmlcomment('test');
 CREATE VIEW xmlview2 AS SELECT xmlconcat('hello', 'you');


diff --git a/src/test/regress/expected/xml_2.out b/src/test/regress/expected/xml_2.out

index 3ec56e4c54b960c590e2651c1484cf9adda35724..6fe363142bfc37f730b07ba7bf4a332b237c3aee 100644 (file)

--- a/src/test/regress/expected/xml_2.out
+++ b/src/test/regress/expected/xml_2.out
@@ -512,6 +512,13 @@ LINE 1: EXECUTE foo ('bad');
 DETAIL:  line 1: Start tag expected, '<' not found
 bad
 ^
+SELECT xml '';
+ERROR:  invalid XML document
+LINE 1: SELECT xml '';
+                   ^
+DETAIL:  line 1: Extra content at the end of the document
+
+                ^
 SET XML OPTION CONTENT;
 EXECUTE foo ('');
   xmlconcat   
@@ -525,6 +532,45 @@ EXECUTE foo ('good');
  good
 (1 row)
 
+SELECT xml '  ';
+                                xml                                 
+--------------------------------------------------------------------
+   
+(1 row)
+
+SELECT xml '  ';
+             xml              
+------------------------------
+   
+(1 row)
+
+SELECT xml '';
+       xml        
+------------------
+ 
+(1 row)
+
+SELECT xml ' oops ';
+ERROR:  invalid XML content
+LINE 1: SELECT xml ' oops ';
+                   ^
+DETAIL:  line 1: StartTag: invalid element name
+ oops 
+                 ^
+SELECT xml '  ';
+ERROR:  invalid XML content
+LINE 1: SELECT xml '  ';
+                   ^
+DETAIL:  line 1: StartTag: invalid element name
+  
+                    ^
+SELECT xml '';
+ERROR:  invalid XML content
+LINE 1: SELECT xml '';
+                   ^
+DETAIL:  line 1: Extra content at the end of the document
+
+                ^
 -- Test backwards parsing
 CREATE VIEW xmlview1 AS SELECT xmlcomment('test');
 CREATE VIEW xmlview2 AS SELECT xmlconcat('hello', 'you');


diff --git a/src/test/regress/sql/xml.sql b/src/test/regress/sql/xml.sql

index 3b91b56d5a9800bc4ab5467aded03127e9f2bf1a..4e844234c53fd78bf622719043f7efb22bba7986 100644 (file)

--- a/src/test/regress/sql/xml.sql
+++ b/src/test/regress/sql/xml.sql
@@ -149,10 +149,17 @@ PREPARE foo (xml) AS SELECT xmlconcat('', $1);
 SET XML OPTION DOCUMENT;
 EXECUTE foo ('');
 EXECUTE foo ('bad');
+SELECT xml '';
 
 SET XML OPTION CONTENT;
 EXECUTE foo ('');
 EXECUTE foo ('good');
+SELECT xml '  ';
+SELECT xml '  ';
+SELECT xml '';
+SELECT xml ' oops ';
+SELECT xml '  ';
+SELECT xml '';
 
 
 -- Test backwards parsing




This is the main PostgreSQL git repository.
RSS
Atom



window.onload = function () {
	var tz_cookie = { name: 'gitweb_tz', expires: 14, path: '/' };
	onloadTZSetup('local', tz_cookie, 'datetime');
};

diff --git a/src/test/regress/expected/xml.out b/src/test/regress/expected/xml.out
index 6e1f885112ca0ac4b8e2fa3564727bea07cb21d4..2ed0d44b1e091edbbaf8b8f708d4303b31f3e96b 100644 (file)
--- a/src/test/regress/expected/xml.out
+++ b/src/test/regress/expected/xml.out
@@ -532,6 +532,13 @@ LINE 1: EXECUTE foo ('bad');
 DETAIL:  line 1: Start tag expected, '<' not found
 bad
 ^
+SELECT xml '';

diff --git a/src/test/regress/expected/xml_1.out b/src/test/regress/expected/xml_1.out
index 0eba424346285a3ade2a7f816740812889953edb..9dd9b3f7ca5c1fbd4edcefade6502eaec3915d85 100644 (file)
--- a/src/test/regress/expected/xml_1.out
+++ b/src/test/regress/expected/xml_1.out
@@ -429,11 +429,53 @@ EXECUTE foo ('');
 ERROR:  prepared statement "foo" does not exist
 EXECUTE foo ('bad');
 ERROR:  prepared statement "foo" does not exist
+SELECT xml '';

diff --git a/src/test/regress/expected/xml_2.out b/src/test/regress/expected/xml_2.out
index 3ec56e4c54b960c590e2651c1484cf9adda35724..6fe363142bfc37f730b07ba7bf4a332b237c3aee 100644 (file)
--- a/src/test/regress/expected/xml_2.out
+++ b/src/test/regress/expected/xml_2.out
@@ -512,6 +512,13 @@ LINE 1: EXECUTE foo ('bad');
 DETAIL:  line 1: Start tag expected, '<' not found
 bad
 ^
+SELECT xml '';

diff --git a/src/test/regress/sql/xml.sql b/src/test/regress/sql/xml.sql
index 3b91b56d5a9800bc4ab5467aded03127e9f2bf1a..4e844234c53fd78bf622719043f7efb22bba7986 100644 (file)
--- a/src/test/regress/sql/xml.sql
+++ b/src/test/regress/sql/xml.sql
@@ -149,10 +149,17 @@ PREPARE foo (xml) AS SELECT xmlconcat('', $1);
 SET XML OPTION DOCUMENT;
 EXECUTE foo ('');
 EXECUTE foo ('bad');
+SELECT xml '';