bpo-30947: Update libexpat from 2.2.1 to 2.2.3 (#3106)

* bpo-30947: Update libexpat from 2.2.1 to 2.2.3

* Add NEWS entry

* Add new loadlibrary.c

* expat_external.h: restore include "pyexpatns.h"

* PCbuild: add expat/loadlibrary.c

* Define XML_POOR_ENTROPY to compile expat

Misc/NEWS.d/next/Security/2017-08-16-16-35-59.bpo-30947.iNMmm4.rst

+Upgrade libexpat embedded copy from version 2.2.1 to 2.2.3 to get security
+fixes.

Modules/expat/expat.h

@@ -24,7 +24,6 @@ extern "C" {
 struct XML_ParserStruct;
 typedef struct XML_ParserStruct *XML_Parser;
 
-/* Should this be defined using stdbool.h when C99 is available? */
 typedef unsigned char XML_Bool;
 #define XML_TRUE   ((XML_Bool) 1)
 #define XML_FALSE  ((XML_Bool) 0)
@@ -1049,7 +1048,7 @@ XML_GetFeatureList(void);
 */
 #define XML_MAJOR_VERSION 2
 #define XML_MINOR_VERSION 2
-#define XML_MICRO_VERSION 1
+#define XML_MICRO_VERSION 3
 
 #ifdef __cplusplus
 }

Modules/expat/loadlibrary.c

+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 2016 - 2017, Steve Holme, <steve_holme@hotmail.com>.
+ *
+ * All rights reserved.
+ *
+ * Permission to  use, copy,  modify, and distribute  this software  for any
+ * purpose with  or without fee is  hereby granted, provided that  the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE  SOFTWARE  IS  PROVIDED  "AS  IS",  WITHOUT  WARRANTY  OF  ANY  KIND,
+ * EXPRESS  OR IMPLIED,  INCLUDING  BUT  NOT LIMITED  TO  THE WARRANTIES  OF
+ * MERCHANTABILITY, FITNESS FOR A  PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+ * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR  OTHERWISE, ARISING FROM, OUT OF OR  IN CONNECTION WITH
+ * THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Except as contained in this notice,  the name of a copyright holder shall
+ * not be used in advertising or otherwise to promote the sale, use or other
+ * dealings  in this  Software without  prior written  authorization of  the
+ * copyright holder.
+ *
+ ***************************************************************************/
+
+#if defined(_WIN32)
+
+#include <windows.h>
+#include <tchar.h>
+
+
+HMODULE _Expat_LoadLibrary(LPCTSTR filename);
+
+
+#if !defined(LOAD_WITH_ALTERED_SEARCH_PATH)
+#define LOAD_WITH_ALTERED_SEARCH_PATH  0x00000008
+#endif
+
+#if !defined(LOAD_LIBRARY_SEARCH_SYSTEM32)
+#define LOAD_LIBRARY_SEARCH_SYSTEM32   0x00000800
+#endif
+
+/* We use our own typedef here since some headers might lack these */
+typedef HMODULE (APIENTRY *LOADLIBRARYEX_FN)(LPCTSTR, HANDLE, DWORD);
+
+/* See function definitions in winbase.h */
+#ifdef UNICODE
+#  ifdef _WIN32_WCE
+#    define LOADLIBARYEX  L"LoadLibraryExW"
+#  else
+#    define LOADLIBARYEX  "LoadLibraryExW"
+#  endif
+#else
+#  define LOADLIBARYEX    "LoadLibraryExA"
+#endif
+
+
+/*
+ * _Expat_LoadLibrary()
+ *
+ * This is used to dynamically load DLLs using the most secure method available
+ * for the version of Windows that we are running on.
+ *
+ * Parameters:
+ *
+ * filename  [in] - The filename or full path of the DLL to load. If only the
+ *                  filename is passed then the DLL will be loaded from the
+ *                  Windows system directory.
+ *
+ * Returns the handle of the module on success; otherwise NULL.
+ */
+HMODULE _Expat_LoadLibrary(LPCTSTR filename)
+{
+  HMODULE hModule = NULL;
+  LOADLIBRARYEX_FN pLoadLibraryEx = NULL;
+
+  /* Get a handle to kernel32 so we can access it's functions at runtime */
+  HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32"));
+  if(!hKernel32)
+    return NULL;
+
+  /* Attempt to find LoadLibraryEx() which is only available on Windows 2000
+     and above */
+  pLoadLibraryEx = (LOADLIBRARYEX_FN) GetProcAddress(hKernel32, LOADLIBARYEX);
+
+  /* Detect if there's already a path in the filename and load the library if
+     there is. Note: Both back slashes and forward slashes have been supported
+     since the earlier days of DOS at an API level although they are not
+     supported by command prompt */
+  if(_tcspbrk(filename, TEXT("\\/"))) {
+    /** !checksrc! disable BANNEDFUNC 1 **/
+    hModule = pLoadLibraryEx ?
+      pLoadLibraryEx(filename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
+      LoadLibrary(filename);
+  }
+  /* Detect if KB2533623 is installed, as LOAD_LIBARY_SEARCH_SYSTEM32 is only
+     supported on Windows Vista, Windows Server 2008, Windows 7 and Windows
+     Server 2008 R2 with this patch or natively on Windows 8 and above */
+  else if(pLoadLibraryEx && GetProcAddress(hKernel32, "AddDllDirectory")) {
+    /* Load the DLL from the Windows system directory */
+    hModule = pLoadLibraryEx(filename, NULL, LOAD_LIBRARY_SEARCH_SYSTEM32);
+  }
+  else {
+    /* Attempt to get the Windows system path */
+    UINT systemdirlen = GetSystemDirectory(NULL, 0);
+    if(systemdirlen) {
+      /* Allocate space for the full DLL path (Room for the null terminator
+         is included in systemdirlen) */
+      size_t filenamelen = _tcslen(filename);
+      TCHAR *path = malloc(sizeof(TCHAR) * (systemdirlen + 1 + filenamelen));
+      if(path && GetSystemDirectory(path, systemdirlen)) {
+        /* Calculate the full DLL path */
+        _tcscpy(path + _tcslen(path), TEXT("\\"));
+        _tcscpy(path + _tcslen(path), filename);
+
+        /* Load the DLL from the Windows system directory */
+        /** !checksrc! disable BANNEDFUNC 1 **/
+        hModule = pLoadLibraryEx ?
+          pLoadLibraryEx(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) :
+          LoadLibrary(path);
+
+      }
+      free(path);
+    }
+  }
+
+  return hModule;
+}
+
+#else /* defined(_WIN32) */
+
+/* ISO C requires a translation unit to contain at least one declaration
+   [-Wempty-translation-unit] */
+typedef int _TRANSLATION_UNIT_LOAD_LIBRARY_C_NOT_EMTPY;
+
+#endif /* defined(_WIN32) */

Modules/expat/siphash.h

@@ -2,9 +2,8 @@
  * siphash.h - SipHash-2-4 in a single header file
  * --------------------------------------------------------------------------
  * Derived by William Ahern from the reference implementation[1] published[2]
- * by Jean-Philippe Aumasson and Daniel J. Berstein. Licensed in kind.
  * by Jean-Philippe Aumasson and Daniel J. Berstein.
- * Minimal changes by Sebastian Pipping on top, details below.
+ * Minimal changes by Sebastian Pipping and Victor Stinner on top, see below.
  * Licensed under the CC0 Public Domain Dedication license.
  *
  * 1. https://www.131002.net/siphash/siphash24.c
@@ -12,14 +11,25 @@
  * --------------------------------------------------------------------------
  * HISTORY:
  *
- * 2017-06-10  (Sebastian Pipping)
+ * 2017-07-25  (Vadim Zeitlin)
+ *   - Fix use of SIPHASH_MAIN macro
+ *
+ * 2017-07-05  (Sebastian Pipping)
+ *   - Use _SIP_ULL macro to not require a C++11 compiler if compiled as C++
+ *   - Add const qualifiers at two places
+ *   - Ensure <=80 characters line length (assuming tab width 4)
+ *
+ * 2017-06-23  (Victor Stinner)
+ *   - Address Win64 compile warnings
+ *
+ * 2017-06-18  (Sebastian Pipping)
  *   - Clarify license note in the header
  *   - Address C89 issues:
  *     - Stop using inline keyword (and let compiler decide)
- *     - Turn integer suffix ULL to UL
  *     - Replace _Bool by int
  *     - Turn macro siphash24 into a function
  *     - Address invalid conversion (void pointer) by explicit cast
+ *   - Address lack of stdint.h for Visual Studio 2003 to 2008
  *   - Always expose sip24_valid (for self-tests)
  *
  * 2012-11-04 - Born.  (William Ahern)
@@ -76,7 +86,23 @@
 #define SIPHASH_H
 
 #include <stddef.h> /* size_t */
-#include <stdint.h> /* uint64_t uint32_t uint8_t */
+
+#if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER < 1600)
+  /* For vs2003/7.1 up to vs2008/9.0; _MSC_VER 1600 is vs2010/10.0 */
+  typedef unsigned __int8   uint8_t;
+  typedef unsigned __int32 uint32_t;
+  typedef unsigned __int64 uint64_t;
+#else
+ #include <stdint.h> /* uint64_t uint32_t uint8_t */
+#endif
+
+
+/*
+ * Workaround to not require a C++11 compiler for using ULL suffix
+ * if this code is included and compiled as C++; related GCC warning is:
+ * warning: use of C++11 long long integer constant [-Wlong-long]
+ */
+#define _SIP_ULL(high, low)  (((uint64_t)high << 32) | low)
 
 
 #define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b))))
@@ -158,11 +184,12 @@ static void sip_round(struct siphash *H, const int rounds) {
 } /* sip_round() */
 
 
-static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) {
-	H->v0 = 0x736f6d6570736575UL ^ key->k[0];
-	H->v1 = 0x646f72616e646f6dUL ^ key->k[1];
-	H->v2 = 0x6c7967656e657261UL ^ key->k[0];
-	H->v3 = 0x7465646279746573UL ^ key->k[1];
+static struct siphash *sip24_init(struct siphash *H,
+		const struct sipkey *key) {
+	H->v0 = _SIP_ULL(0x736f6d65U, 0x70736575U) ^ key->k[0];
+	H->v1 = _SIP_ULL(0x646f7261U, 0x6e646f6dU) ^ key->k[1];
+	H->v2 = _SIP_ULL(0x6c796765U, 0x6e657261U) ^ key->k[0];
+	H->v3 = _SIP_ULL(0x74656462U, 0x79746573U) ^ key->k[1];
 
 	H->p = H->buf;
 	H->c = 0;
@@ -173,7 +200,8 @@ static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) {
 
 #define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)])
 
-static struct siphash *sip24_update(struct siphash *H, const void *src, size_t len) {
+static struct siphash *sip24_update(struct siphash *H, const void *src,
+		size_t len) {
 	const unsigned char *p = (const unsigned char *)src, *pe = p + len;
 	uint64_t m;
 
@@ -198,7 +226,7 @@ static struct siphash *sip24_update(struct siphash *H, const void *src, size_t l
 
 
 static uint64_t sip24_final(struct siphash *H) {
-	char left = (char)(H->p - H->buf);
+	const char left = (char)(H->p - H->buf);
 	uint64_t b = (H->c + left) << 56;
 
 	switch (left) {
@@ -222,7 +250,8 @@ static uint64_t sip24_final(struct siphash *H) {
 } /* sip24_final() */
 
 
-static uint64_t siphash24(const void *src, size_t len, const struct sipkey *key) {
+static uint64_t siphash24(const void *src, size_t len,
+		const struct sipkey *key) {
 	struct siphash state = SIPHASH_INITIALIZER;
 	return sip24_final(sip24_update(sip24_init(&state, key), src, len));
 } /* siphash24() */
@@ -310,7 +339,8 @@ static int sip24_valid(void) {
 	struct sipkey k;
 	size_t i;
 
-	sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017");
+	sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011"
+			"\012\013\014\015\016\017");
 
 	for (i = 0; i < sizeof in; ++i) {
 		in[i] = (unsigned char)i;
@@ -323,12 +353,12 @@ static int sip24_valid(void) {
 } /* sip24_valid() */
 
 
-#if SIPHASH_MAIN
+#ifdef SIPHASH_MAIN
 
 #include <stdio.h>
 
 int main(void) {
-	int ok = sip24_valid();
+	const int ok = sip24_valid();
 
 	if (ok)
 		puts("OK");

Modules/expat/xmlrole.c

@@ -170,7 +170,14 @@ prolog1(PROLOG_STATE *state,
   case XML_TOK_COMMENT:
     return XML_ROLE_COMMENT;
   case XML_TOK_BOM:
-    return XML_ROLE_NONE;
+    /* This case can never arise.  To reach this role function, the
+     * parse must have passed through prolog0 and therefore have had
+     * some form of input, even if only a space.  At that point, a
+     * byte order mark is no longer a valid character (though
+     * technically it should be interpreted as a non-breaking space),
+     * so will be rejected by the tokenizing stages.
+     */
+    return XML_ROLE_NONE; /* LCOV_EXCL_LINE */
   case XML_TOK_DECL_OPEN:
     if (!XmlNameMatchesAscii(enc,
                              ptr + 2 * MIN_BYTES_PER_CHAR(enc),
@@ -1285,6 +1292,26 @@ declClose(PROLOG_STATE *state,
   return common(state, tok);
 }
 
+/* This function will only be invoked if the internal logic of the
+ * parser has broken down.  It is used in two cases:
+ *
+ * 1: When the XML prolog has been finished.  At this point the
+ * processor (the parser level above these role handlers) should
+ * switch from prologProcessor to contentProcessor and reinitialise
+ * the handler function.
+ *
+ * 2: When an error has been detected (via common() below).  At this
+ * point again the processor should be switched to errorProcessor,
+ * which will never call a handler.
+ *
+ * The result of this is that error() can only be called if the
+ * processor switch failed to happen, which is an internal error and
+ * therefore we shouldn't be able to provoke it simply by using the
+ * library.  It is a necessary backstop, however, so we merely exclude
+ * it from the coverage statistics.
+ *
+ * LCOV_EXCL_START
+ */
 static int PTRCALL
 error(PROLOG_STATE *UNUSED_P(state),
       int UNUSED_P(tok),
@@ -1294,6 +1321,7 @@ error(PROLOG_STATE *UNUSED_P(state),
 {
   return XML_ROLE_NONE;
 }
+/* LCOV_EXCL_STOP */
 
 static int FASTCALL
 common(PROLOG_STATE *state, int tok)

Modules/expat/xmltok.c

@@ -1019,7 +1019,11 @@ streqci(const char *s1, const char *s2)
     if (ASCII_a <= c1 && c1 <= ASCII_z)
       c1 += ASCII_A - ASCII_a;
     if (ASCII_a <= c2 && c2 <= ASCII_z)
-      c2 += ASCII_A - ASCII_a;
+      /* The following line will never get executed.  streqci() is
+       * only called from two places, both of which guarantee to put
+       * upper-case strings into s2.
+       */
+      c2 += ASCII_A - ASCII_a; /* LCOV_EXCL_LINE */
     if (c1 != c2)
       return 0;
     if (!c1)
@@ -1291,7 +1295,7 @@ XmlUtf8Encode(int c, char *buf)
   };
 
   if (c < 0)
-    return 0;
+    return 0; /* LCOV_EXCL_LINE: this case is always eliminated beforehand */
   if (c < min2) {
     buf[0] = (char)(c | UTF8_cval1);
     return 1;
@@ -1314,7 +1318,7 @@ XmlUtf8Encode(int c, char *buf)
     buf[3] = (char)((c & 0x3f) | 0x80);
     return 4;
   }
-  return 0;
+  return 0; /* LCOV_EXCL_LINE: this case too is eliminated before calling */
 }
 
 int FASTCALL
@@ -1465,6 +1469,9 @@ XmlInitUnknownEncoding(void *mem,
     else if (c < 0) {
       if (c < -4)
         return 0;
+      /* Multi-byte sequences need a converter function */
+      if (!convert)
+        return 0;
       e->normal.type[i] = (unsigned char)(BT_LEAD2 - (c + 2));
       e->utf8[i][0] = 0;
       e->utf16[i] = 0;

Modules/expat/xmltok_impl.c

@@ -1198,8 +1198,14 @@ PREFIX(attributeValueTok)(const ENCODING *enc, const char *ptr,
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
-  else if (! HAS_CHAR(enc, ptr, end))
-    return XML_TOK_PARTIAL;
+  else if (! HAS_CHAR(enc, ptr, end)) {
+    /* This line cannot be executed.  The incoming data has already
+     * been tokenized once, so incomplete characters like this have
+     * already been eliminated from the input.  Retaining the paranoia
+     * check is still valuable, however.
+     */
+    return XML_TOK_PARTIAL; /* LCOV_EXCL_LINE */
+  }
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
@@ -1258,8 +1264,14 @@ PREFIX(entityValueTok)(const ENCODING *enc, const char *ptr,
   const char *start;
   if (ptr >= end)
     return XML_TOK_NONE;
-  else if (! HAS_CHAR(enc, ptr, end))
-    return XML_TOK_PARTIAL;
+  else if (! HAS_CHAR(enc, ptr, end)) {
+    /* This line cannot be executed.  The incoming data has already
+     * been tokenized once, so incomplete characters like this have
+     * already been eliminated from the input.  Retaining the paranoia
+     * check is still valuable, however.
+     */
+    return XML_TOK_PARTIAL; /* LCOV_EXCL_LINE */
+  }
   start = ptr;
   while (HAS_CHAR(enc, ptr, end)) {
     switch (BYTE_TYPE(enc, ptr)) {
@@ -1614,6 +1626,14 @@ PREFIX(predefinedEntityName)(const ENCODING *UNUSED_P(enc), const char *ptr,
   return 0;
 }
 
+/* This function does not appear to be called from anywhere within the
+ * library code.  It is used via the macro XmlSameName(), which is
+ * defined but never used.  Since it appears in the encoding function
+ * table, removing it is not a thing to be undertaken lightly.  For
+ * the moment, we simply exclude it from coverage tests.
+ *
+ * LCOV_EXCL_START
+ */
 static int PTRCALL
 PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2)
 {
@@ -1677,14 +1697,21 @@ PREFIX(sameName)(const ENCODING *enc, const char *ptr1, const char *ptr2)
   }
   /* not reached */
 }
+/* LCOV_EXCL_STOP */
 
 static int PTRCALL
 PREFIX(nameMatchesAscii)(const ENCODING *UNUSED_P(enc), const char *ptr1,
                          const char *end1, const char *ptr2)
 {
   for (; *ptr2; ptr1 += MINBPC(enc), ptr2++) {
-    if (end1 - ptr1 < MINBPC(enc))
-      return 0;
+    if (end1 - ptr1 < MINBPC(enc)) {
+      /* This line cannot be executed.  THe incoming data has already
+       * been tokenized once, so imcomplete characters like this have
+       * already been eliminated from the input.  Retaining the
+       * paranoia check is still valuable, however.
+       */
+      return 0; /* LCOV_EXCL_LINE */
+    }
     if (!CHAR_MATCHES(enc, ptr1, *ptr2))
       return 0;
   }

PCbuild/_elementtree.vcxproj

@@ -87,6 +87,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_elementtree.c" />
+    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />

PCbuild/_elementtree.vcxproj.filters

@@ -33,6 +33,9 @@
     <ClInclude Include="..\Modules\expat\latin1tab.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="..\Modules\expat\loadlibrary.c">
+      <Filter>Header Files</Filter>
+    </ClInclude>
     <ClInclude Include="..\Modules\expat\macconfig.h">
       <Filter>Header Files</Filter>
     </ClInclude>

PCbuild/pyexpat.vcxproj

@@ -68,6 +68,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\pyexpat.c" />
+    <ClCompile Include="..\Modules\expat\loadlibrary.c" />
     <ClCompile Include="..\Modules\expat\xmlparse.c" />
     <ClCompile Include="..\Modules\expat\xmlrole.c" />
     <ClCompile Include="..\Modules\expat\xmltok.c" />

PCbuild/pyexpat.vcxproj.filters

@@ -20,6 +20,9 @@
     <ClCompile Include="..\Modules\pyexpat.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Modules\expat\loadlibrary.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\Modules\expat\xmlparse.c">
       <Filter>Source Files</Filter>
     </ClCompile>

setup.py

@@ -1525,6 +1525,9 @@ class PyBuildExt(build_ext):
             expat_inc = [os.path.join(os.getcwd(), srcdir, 'Modules', 'expat')]
             define_macros = [
                 ('HAVE_EXPAT_CONFIG_H', '1'),
+                # bpo-30947: Python uses best available entropy sources to
+                # call XML_SetHashSalt(), expat entropy sources are not needed
+                ('XML_POOR_ENTROPY', '1'),
             ]
             expat_lib = []
             expat_sources = ['expat/xmlparse.c',