bpo-30694: Upgrade Modules/expat/ to libexpat 2.2.1 (#2300)

New file: Modules/expat/siphash.h.

Misc/NEWS

@@ -368,6 +368,15 @@ Extension Modules
 Library
 -------
 
+- [Security] bpo-30694: Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes
+  of multiple security vulnerabilities including: CVE-2017-9233 (External
+  entity infinite loop DoS), CVE-2016-9063 (Integer overflow, re-fix),
+  CVE-2016-0718 (Fix regression bugs from 2.2.0's fix to CVE-2016-0718)
+  and CVE-2012-0876 (Counter hash flooding with SipHash).
+  Note: the CVE-2016-5300 (Use os-specific entropy sources like getrandom)
+  doesn't impact Python, since Python already gets entropy from the OS to set
+  the expat secret using ``XML_SetHashSalt()``.
+
 - bpo-29755: Fixed the lgettext() family of functions in the gettext module.
   They now always return bytes.
 

Modules/expat/COPYING

 Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper
-Copyright (c) 2001-2016 Expat maintainers
+Copyright (c) 2001-2017 Expat maintainers
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

Modules/expat/expat.h

@@ -95,7 +95,9 @@ enum XML_Error {
   /* Added in 2.0. */
   XML_ERROR_RESERVED_PREFIX_XML,
   XML_ERROR_RESERVED_PREFIX_XMLNS,
-  XML_ERROR_RESERVED_NAMESPACE_URI
+  XML_ERROR_RESERVED_NAMESPACE_URI,
+  /* Added in 2.2.1. */
+  XML_ERROR_INVALID_ARGUMENT
 };
 
 enum XML_Content_Type {
@@ -706,6 +708,7 @@ XML_UseParserAsHandlerArg(XML_Parser parser);
      be called, despite an external subset being parsed.
    Note: If XML_DTD is not defined when Expat is compiled, returns
      XML_ERROR_FEATURE_REQUIRES_XML_DTD.
+   Note: If parser == NULL, returns XML_ERROR_INVALID_ARGUMENT.
 */
 XMLPARSEAPI(enum XML_Error)
 XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD);
@@ -729,15 +732,16 @@ XML_GetBase(XML_Parser parser);
    to the XML_StartElementHandler that were specified in the start-tag
    rather than defaulted. Each attribute/value pair counts as 2; thus
    this correspondds to an index into the atts array passed to the
-   XML_StartElementHandler.
+   XML_StartElementHandler.  Returns -1 if parser == NULL.
 */
 XMLPARSEAPI(int)
 XML_GetSpecifiedAttributeCount(XML_Parser parser);
 
 /* Returns the index of the ID attribute passed in the last call to
-   XML_StartElementHandler, or -1 if there is no ID attribute.  Each
-   attribute/value pair counts as 2; thus this correspondds to an
-   index into the atts array passed to the XML_StartElementHandler.
+   XML_StartElementHandler, or -1 if there is no ID attribute or
+   parser == NULL.  Each attribute/value pair counts as 2; thus this
+   correspondds to an index into the atts array passed to the
+   XML_StartElementHandler.
 */
 XMLPARSEAPI(int)
 XML_GetIdAttributeIndex(XML_Parser parser);
@@ -901,6 +905,7 @@ enum XML_ParamEntityParsing {
    entities is requested; otherwise it will return non-zero.
    Note: If XML_SetParamEntityParsing is called after XML_Parse or
       XML_ParseBuffer, then it has no effect and will always return 0.
+   Note: If parser == NULL, the function will do nothing and return 0.
 */
 XMLPARSEAPI(int)
 XML_SetParamEntityParsing(XML_Parser parser,
@@ -910,6 +915,7 @@ XML_SetParamEntityParsing(XML_Parser parser,
    Helps in preventing DoS attacks based on predicting hash
    function behavior. This must be called before parsing is started.
    Returns 1 if successful, 0 when called after parsing has started.
+   Note: If parser == NULL, the function will do nothing and return 0.
 */
 XMLPARSEAPI(int)
 XML_SetHashSalt(XML_Parser parser,
@@ -936,6 +942,10 @@ XML_GetErrorCode(XML_Parser parser);
    the location is the location of the character at which the error
    was detected; otherwise the location is the location of the last
    parse event, as described above.
+
+   Note: XML_GetCurrentLineNumber and XML_GetCurrentColumnNumber
+   return 0 to indicate an error.
+   Note: XML_GetCurrentByteIndex returns -1 to indicate an error.
 */
 XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser);
 XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser);
@@ -1039,7 +1049,7 @@ XML_GetFeatureList(void);
 */
 #define XML_MAJOR_VERSION 2
 #define XML_MINOR_VERSION 2
-#define XML_MICRO_VERSION 0
+#define XML_MICRO_VERSION 1
 
 #ifdef __cplusplus
 }

Modules/expat/expat_external.h

@@ -97,7 +97,10 @@ extern "C" {
 #endif
 
 #ifdef XML_UNICODE_WCHAR_T
-#define XML_UNICODE
+# define XML_UNICODE
+# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2)
+#  error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc"
+# endif
 #endif
 
 #ifdef XML_UNICODE     /* Information is UTF-16 encoded. */

Modules/expat/winconfig.h

@@ -17,6 +17,12 @@
 #include <memory.h>
 #include <string.h>
 
+
+#if defined(HAVE_EXPAT_CONFIG_H)  /* e.g. MinGW */
+# include <expat_config.h>
+#else  /* !defined(HAVE_EXPAT_CONFIG_H) */
+
+
 #define XML_NS 1
 #define XML_DTD 1
 #define XML_CONTEXT_BYTES 1024
@@ -27,4 +33,8 @@
 /* Windows has memmove() available. */
 #define HAVE_MEMMOVE
 
+
+#endif /* !defined(HAVE_EXPAT_CONFIG_H) */
+
+
 #endif /* ndef WINCONFIG_H */

Modules/expat/xmlrole.c

@@ -4,19 +4,13 @@
 
 #include <stddef.h>
 
-#ifdef WIN32
+#ifdef _WIN32
 #include "winconfig.h"
-#elif defined(MACOS_CLASSIC)
-#include "macconfig.h"
-#elif defined(__amigaos__)
-#include "amigaconfig.h"
-#elif defined(__WATCOMC__)
-#include "watcomconfig.h"
 #else
 #ifdef HAVE_EXPAT_CONFIG_H
 #include <expat_config.h>
 #endif
-#endif /* ndef WIN32 */
+#endif /* ndef _WIN32 */
 
 #include "expat_external.h"
 #include "internal.h"

Modules/expat/xmltok.c

@@ -4,19 +4,13 @@
 
 #include <stddef.h>
 
-#ifdef WIN32
+#ifdef _WIN32
 #include "winconfig.h"
-#elif defined(MACOS_CLASSIC)
-#include "macconfig.h"
-#elif defined(__amigaos__)
-#include "amigaconfig.h"
-#elif defined(__WATCOMC__)
-#include "watcomconfig.h"
 #else
 #ifdef HAVE_EXPAT_CONFIG_H
 #include <expat_config.h>
 #endif
-#endif /* ndef WIN32 */
+#endif /* ndef _WIN32 */
 
 #include "expat_external.h"
 #include "internal.h"
@@ -369,24 +363,24 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
             const char **fromP, const char *fromLim,
             char **toP, const char *toLim)
 {
-  enum XML_Convert_Result res = XML_CONVERT_COMPLETED;
   char *to;
   const char *from;
-  if (fromLim - *fromP > toLim - *toP) {
-    /* Avoid copying partial characters. */
-    res = XML_CONVERT_OUTPUT_EXHAUSTED;
-    fromLim = *fromP + (toLim - *toP);
-    align_limit_to_full_utf8_characters(*fromP, &fromLim);
-  }
+  const char *fromLimInitial = fromLim;
+
+  /* Avoid copying partial characters. */
+  align_limit_to_full_utf8_characters(*fromP, &fromLim);
+
   for (to = *toP, from = *fromP; (from < fromLim) && (to < toLim); from++, to++)
     *to = *from;
   *fromP = from;
   *toP = to;
 
-  if ((to == toLim) && (from < fromLim))
+  if (fromLim < fromLimInitial)
+    return XML_CONVERT_INPUT_INCOMPLETE;
+  else if ((to == toLim) && (from < fromLim))
     return XML_CONVERT_OUTPUT_EXHAUSTED;
   else
-    return res;
+    return XML_CONVERT_COMPLETED;
 }
 
 static enum XML_Convert_Result PTRCALL
@@ -402,7 +396,7 @@ utf8_toUtf16(const ENCODING *enc,
     case BT_LEAD2:
       if (fromLim - from < 2) {
         res = XML_CONVERT_INPUT_INCOMPLETE;
-        break;
+        goto after;
       }
       *to++ = (unsigned short)(((from[0] & 0x1f) << 6) | (from[1] & 0x3f));
       from += 2;
@@ -410,7 +404,7 @@ utf8_toUtf16(const ENCODING *enc,
     case BT_LEAD3:
       if (fromLim - from < 3) {
         res = XML_CONVERT_INPUT_INCOMPLETE;
-        break;
+        goto after;
       }
       *to++ = (unsigned short)(((from[0] & 0xf) << 12)
                                | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f));
@@ -441,6 +435,8 @@ utf8_toUtf16(const ENCODING *enc,
       break;
     }
   }
+  if (from < fromLim)
+    res = XML_CONVERT_OUTPUT_EXHAUSTED;
 after:
   *fromP = from;
   *toP = to;