Issue #16113: integrade SHA-3 (Keccak) patch from http://hg.python.org/sandbox/cheimes

Doc/library/hashlib.rst

@@ -51,9 +51,13 @@ concatenation of the data fed to it so far using the :meth:`digest` or
 .. index:: single: OpenSSL; (use in module hashlib)
 
 Constructors for hash algorithms that are always present in this module are
-:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, and
-:func:`sha512`.  Additional algorithms may also be available depending upon the
-OpenSSL library that Python uses on your platform.
+:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`,
+:func:`sha512`, :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, and
+:func:`sha3_512`. Additional algorithms may also be available depending upon
+the OpenSSL library that Python uses on your platform.
+
+   .. versionchanged:: 3.4
+      Add sha3 family of hash algorithms.
 
 For example, to obtain the digest of the byte string ``b'Nobody inspects the
 spammish repetition'``::

Doc/license.rst

@@ -658,6 +658,25 @@ The :mod:`select` and contains the following notice for the kqueue interface::
   SUCH DAMAGE.
 
 
+SHA-3
+-----
+
+The module :mod:`_sha3` and :mod:`hashlib` are using the reference
+implementation of Keccak. The files at :file:`Modules/_sha3/keccak/` contain
+the following note::
+
+  The Keccak sponge function, designed by Guido Bertoni, Joan Daemen,
+  Michaël Peeters and Gilles Van Assche. For more information, feedback or
+  questions, please refer to our website: http://keccak.noekeon.org/
+
+  Implementation by the designers,
+  hereby denoted as "the implementer".
+
+  To the extent possible under law, the implementer has waived all copyright
+  and related or neighboring rights to the source code in this file.
+  http://creativecommons.org/publicdomain/zero/1.0/
+
+
 strtod and dtoa
 ---------------
 

Doc/whatsnew/3.4.rst

@@ -101,7 +101,7 @@ Implementation improvements:
 
 Significantly Improved Library Modules:
 
-* None yet.
+* SHA-3 (Keccak) support for :mod:`hashlib`.
 
 Security improvements:
 

Lib/hashlib.py

@@ -54,7 +54,8 @@ More condensed:
 
 # This tuple and __get_builtin_constructor() must be modified if a new
 # always available algorithm is added.
-__always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
+__always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512',
+                      'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512')
 
 algorithms_guaranteed = set(__always_supported)
 algorithms_available = set(__always_supported)
@@ -85,6 +86,18 @@ def __get_builtin_constructor(name):
                 return _sha512.sha512
             elif bs == '384':
                 return _sha512.sha384
+        elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
+                      'SHA3_224', 'SHA3_256', 'SHA3_384', 'SHA3_512'}:
+            import _sha3
+            bs = name[5:]
+            if bs == '224':
+                return _sha3.sha3_224
+            elif bs == '256':
+                return _sha3.sha3_256
+            elif bs == '384':
+                return _sha3.sha3_384
+            elif bs == '512':
+                return _sha3.sha3_512
     except ImportError:
         pass  # no extension module, this hash is unsupported.
 

Lib/test/test_hashlib.py

@@ -36,7 +36,10 @@ def hexstr(s):
 class HashLibTestCase(unittest.TestCase):
     supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1',
                              'sha224', 'SHA224', 'sha256', 'SHA256',
-                             'sha384', 'SHA384', 'sha512', 'SHA512' )
+                             'sha384', 'SHA384', 'sha512', 'SHA512',
+                             'sha3_224', 'sha3_256', 'sha3_384',
+                             'sha3_512', 'SHA3_224', 'SHA3_256',
+                             'SHA3_384', 'SHA3_512' )
 
     # Issue #14693: fallback modules are always compiled under POSIX
     _warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG
@@ -93,6 +96,12 @@ class HashLibTestCase(unittest.TestCase):
         if _sha512:
             self.constructors_to_test['sha384'].add(_sha512.sha384)
             self.constructors_to_test['sha512'].add(_sha512.sha512)
+        _sha3 = self._conditional_import_module('_sha3')
+        if _sha3:
+            self.constructors_to_test['sha3_224'].add(_sha3.sha3_224)
+            self.constructors_to_test['sha3_256'].add(_sha3.sha3_256)
+            self.constructors_to_test['sha3_384'].add(_sha3.sha3_384)
+            self.constructors_to_test['sha3_512'].add(_sha3.sha3_512)
 
         super(HashLibTestCase, self).__init__(*args, **kwargs)
 
@@ -158,6 +167,7 @@ class HashLibTestCase(unittest.TestCase):
             self.assertEqual(m1.digest(), m2.digest())
 
     def check(self, name, data, digest):
+        digest = digest.lower()
         constructors = self.constructors_to_test[name]
         # 2 is for hashlib.name(...) and hashlib.new(name, ...)
         self.assertGreaterEqual(len(constructors), 2)
@@ -183,6 +193,10 @@ class HashLibTestCase(unittest.TestCase):
         self.check_no_unicode('sha256')
         self.check_no_unicode('sha384')
         self.check_no_unicode('sha512')
+        self.check_no_unicode('sha3_224')
+        self.check_no_unicode('sha3_256')
+        self.check_no_unicode('sha3_384')
+        self.check_no_unicode('sha3_512')
 
     def test_case_md5_0(self):
         self.check('md5', b'', 'd41d8cd98f00b204e9800998ecf8427e')
@@ -318,11 +332,122 @@ class HashLibTestCase(unittest.TestCase):
           "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973eb"+
           "de0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b")
 
+    # SHA-3 family
+    def test_case_sha3_224_0(self):
+        self.check('sha3_224', b"",
+          "F71837502BA8E10837BDD8D365ADB85591895602FC552B48B7390ABD")
+
+    def test_case_sha3_224_1(self):
+        self.check('sha3_224', bytes.fromhex("CC"),
+          "A9CAB59EB40A10B246290F2D6086E32E3689FAF1D26B470C899F2802")
+
+    def test_case_sha3_224_2(self):
+        self.check('sha3_224', bytes.fromhex("41FB"),
+          "615BA367AFDC35AAC397BC7EB5D58D106A734B24986D5D978FEFD62C")
+
+    def test_case_sha3_224_3(self):
+        self.check('sha3_224', bytes.fromhex(
+            "433C5303131624C0021D868A30825475E8D0BD3052A022180398F4CA4423B9"+
+            "8214B6BEAAC21C8807A2C33F8C93BD42B092CC1B06CEDF3224D5ED1EC29784"+
+            "444F22E08A55AA58542B524B02CD3D5D5F6907AFE71C5D7462224A3F9D9E53"+
+            "E7E0846DCBB4CE"),
+          "62B10F1B6236EBC2DA72957742A8D4E48E213B5F8934604BFD4D2C3A")
+
+    @bigmemtest(size=_4G + 5, memuse=1)
+    def test_case_sha3_224_huge(self, size):
+        if size == _4G + 5:
+            try:
+                self.check('sha3_224', b'A'*size,
+                           '58ef60057c9dddb6a87477e9ace5a26f0d9db01881cf9b10a9f8c224')
+            except OverflowError:
+                pass # 32-bit arch
+
+
+    def test_case_sha3_256_0(self):
+        self.check('sha3_256', b"",
+          "C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470")
+
+    def test_case_sha3_256_1(self):
+        self.check('sha3_256', bytes.fromhex("CC"),
+          "EEAD6DBFC7340A56CAEDC044696A168870549A6A7F6F56961E84A54BD9970B8A")
+
+    def test_case_sha3_256_2(self):
+        self.check('sha3_256', bytes.fromhex("41FB"),
+          "A8EACEDA4D47B3281A795AD9E1EA2122B407BAF9AABCB9E18B5717B7873537D2")
+
+    def test_case_sha3_256_3(self):
+        self.check('sha3_256', bytes.fromhex(
+            "433C5303131624C0021D868A30825475E8D0BD3052A022180398F4CA4423B9"+
+            "8214B6BEAAC21C8807A2C33F8C93BD42B092CC1B06CEDF3224D5ED1EC29784"+
+            "444F22E08A55AA58542B524B02CD3D5D5F6907AFE71C5D7462224A3F9D9E53"+
+            "E7E0846DCBB4CE"),
+          "CE87A5173BFFD92399221658F801D45C294D9006EE9F3F9D419C8D427748DC41")
+
+
+    def test_case_sha3_384_0(self):
+        self.check('sha3_384', b"",
+          "2C23146A63A29ACF99E73B88F8C24EAA7DC60AA771780CCC006AFBFA8FE2479B"+
+          "2DD2B21362337441AC12B515911957FF")
+
+    def test_case_sha3_384_1(self):
+        self.check('sha3_384', bytes.fromhex("CC"),
+          "1B84E62A46E5A201861754AF5DC95C4A1A69CAF4A796AE405680161E29572641"+
+          "F5FA1E8641D7958336EE7B11C58F73E9")
+
+    def test_case_sha3_384_2(self):
+        self.check('sha3_384', bytes.fromhex("41FB"),
+          "495CCE2714CD72C8C53C3363D22C58B55960FE26BE0BF3BBC7A3316DD563AD1D"+
+          "B8410E75EEFEA655E39D4670EC0B1792")
+
+    def test_case_sha3_384_3(self):
+        self.check('sha3_384', bytes.fromhex(
+            "433C5303131624C0021D868A30825475E8D0BD3052A022180398F4CA4423B9"+
+            "8214B6BEAAC21C8807A2C33F8C93BD42B092CC1B06CEDF3224D5ED1EC29784"+
+            "444F22E08A55AA58542B524B02CD3D5D5F6907AFE71C5D7462224A3F9D9E53"+
+            "E7E0846DCBB4CE"),
+          "135114508DD63E279E709C26F7817C0482766CDE49132E3EDF2EEDD8996F4E35"+
+          "96D184100B384868249F1D8B8FDAA2C9")
+
+
+    def test_case_sha3_512_0(self):
+        self.check('sha3_512', b"",
+          "0EAB42DE4C3CEB9235FC91ACFFE746B29C29A8C366B7C60E4E67C466F36A4304"+
+          "C00FA9CAF9D87976BA469BCBE06713B435F091EF2769FB160CDAB33D3670680E")
+
+    def test_case_sha3_512_1(self):
+        self.check('sha3_512', bytes.fromhex("CC"),
+          "8630C13CBD066EA74BBE7FE468FEC1DEE10EDC1254FB4C1B7C5FD69B646E4416"+
+          "0B8CE01D05A0908CA790DFB080F4B513BC3B6225ECE7A810371441A5AC666EB9")
+
+    def test_case_sha3_512_2(self):
+        self.check('sha3_512', bytes.fromhex("41FB"),
+          "551DA6236F8B96FCE9F97F1190E901324F0B45E06DBBB5CDB8355D6ED1DC34B3"+
+          "F0EAE7DCB68622FF232FA3CECE0D4616CDEB3931F93803662A28DF1CD535B731")
+
+    def test_case_sha3_512_3(self):
+        self.check('sha3_512', bytes.fromhex(
+            "433C5303131624C0021D868A30825475E8D0BD3052A022180398F4CA4423B9"+
+            "8214B6BEAAC21C8807A2C33F8C93BD42B092CC1B06CEDF3224D5ED1EC29784"+
+            "444F22E08A55AA58542B524B02CD3D5D5F6907AFE71C5D7462224A3F9D9E53"+
+            "E7E0846DCBB4CE"),
+          "527D28E341E6B14F4684ADB4B824C496C6482E51149565D3D17226828884306B"+
+          "51D6148A72622C2B75F5D3510B799D8BDC03EAEDE453676A6EC8FE03A1AD0EAB")
+
+
     def test_gil(self):
         # Check things work fine with an input larger than the size required
         # for multithreaded operation (which is hardwired to 2048).
         gil_minsize = 2048
 
+        for name in self.supported_hash_names:
+            m = hashlib.new(name)
+            m.update(b'1')
+            m.update(b'#' * gil_minsize)
+            m.update(b'1')
+
+            m = hashlib.new(name, b'x' * gil_minsize)
+            m.update(b'1')
+
         m = hashlib.md5()
         m.update(b'1')
         m.update(b'#' * gil_minsize)

Modules/_hashopenssl.c

@@ -17,24 +17,6 @@
 #include "structmember.h"
 #include "hashlib.h"
 
-#ifdef WITH_THREAD
-#include "pythread.h"
-    #define ENTER_HASHLIB(obj) \
-        if ((obj)->lock) { \
-            if (!PyThread_acquire_lock((obj)->lock, 0)) { \
-                Py_BEGIN_ALLOW_THREADS \
-                PyThread_acquire_lock((obj)->lock, 1); \
-                Py_END_ALLOW_THREADS \
-            } \
-        }
-    #define LEAVE_HASHLIB(obj) \
-        if ((obj)->lock) { \
-            PyThread_release_lock((obj)->lock); \
-        }
-#else
-    #define ENTER_HASHLIB(obj)
-    #define LEAVE_HASHLIB(obj)
-#endif
 
 /* EVP is the preferred interface to hashing in OpenSSL */
 #include <openssl/evp.h>
@@ -43,10 +25,6 @@
 
 #define MUNCH_SIZE INT_MAX
 
-/* TODO(gps): We should probably make this a module or EVPobject attribute
- * to allow the user to optimize based on the platform they're using. */
-#define HASHLIB_GIL_MINSIZE 2048
-
 #ifndef HASH_OBJ_CONSTRUCTOR
 #define HASH_OBJ_CONSTRUCTOR 0
 #endif

Modules/_sha3/cleanup.py

+#!/usr/bin/env python
+# Copyright (C) 2012   Christian Heimes (christian@python.org)
+# Licensed to PSF under a Contributor Agreement.
+#
+# cleanup Keccak sources
+
+import os
+import re
+
+CPP1 = re.compile("^//(.*)")
+CPP2 = re.compile("\ //(.*)")
+
+STATICS = ("void ", "int ", "HashReturn ", "const UINT64 ", "UINT16 ")
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+KECCAK = os.path.join(HERE, "keccak")
+
+def getfiles():
+    for name in os.listdir(KECCAK):
+        name = os.path.join(KECCAK, name)
+        if os.path.isfile(name):
+            yield name
+
+def cleanup(f):
+    buf = []
+    for line in f:
+        # mark all functions and global data as static
+        if line.startswith(STATICS):
+            buf.append("static " + line)
+            continue
+        # remove UINT64 typedef, we have our own
+        if line.startswith("typedef unsigned long long int"):
+            buf.append("/* %s */\n" % line.strip())
+            continue
+        # remove #include "brg_endian.h"
+        if "brg_endian.h" in line:
+            buf.append("/* %s */\n" % line.strip())
+            continue
+        # transform C++ comments into ANSI C comments
+        line = CPP1.sub(r"/* \1 */", line)
+        line = CPP2.sub(r" /* \1 */", line)
+        buf.append(line)
+    return "".join(buf)
+
+for name in getfiles():
+    with open(name) as f:
+        res = cleanup(f)
+    with open(name, "w") as f:
+        f.write(res)

Modules/_sha3/keccak/KeccakF-1600-32.macros

+/*
+The Keccak sponge function, designed by Guido Bertoni, Joan Daemen,
+Michaël Peeters and Gilles Van Assche. For more information, feedback or
+questions, please refer to our website: http://keccak.noekeon.org/
+
+Implementation by the designers,
+hereby denoted as "the implementer".
+
+To the extent possible under law, the implementer has waived all copyright
+and related or neighboring rights to the source code in this file.
+http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+#ifdef UseSchedule
+    #if (UseSchedule == 1)
+        #include "KeccakF-1600-32-s1.macros"
+    #elif (UseSchedule == 2)
+        #include "KeccakF-1600-32-s2.macros"
+    #elif (UseSchedule == 3)
+        #include "KeccakF-1600-32-rvk.macros"
+    #else
+        #error "This schedule is not supported."
+    #endif
+#else
+    #include "KeccakF-1600-32-s1.macros"
+#endif

Modules/_sha3/keccak/KeccakF-1600-int-set.h

+#define ProvideFast576
+#define ProvideFast832
+#define ProvideFast1024
+#define ProvideFast1088
+#define ProvideFast1152
+#define ProvideFast1344

Modules/_sha3/keccak/KeccakF-1600-interface.h

+/*
+The Keccak sponge function, designed by Guido Bertoni, Joan Daemen,
+Michaël Peeters and Gilles Van Assche. For more information, feedback or
+questions, please refer to our website: http://keccak.noekeon.org/
+
+Implementation by the designers,
+hereby denoted as "the implementer".
+
+To the extent possible under law, the implementer has waived all copyright
+and related or neighboring rights to the source code in this file.
+http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+#ifndef _KeccakPermutationInterface_h_
+#define _KeccakPermutationInterface_h_
+
+#include "KeccakF-1600-int-set.h"
+
+static void KeccakInitialize( void );
+static void KeccakInitializeState(unsigned char *state);
+static void KeccakPermutation(unsigned char *state);
+#ifdef ProvideFast576
+static void KeccakAbsorb576bits(unsigned char *state, const unsigned char *data);
+#endif
+#ifdef ProvideFast832
+static void KeccakAbsorb832bits(unsigned char *state, const unsigned char *data);
+#endif
+#ifdef ProvideFast1024
+static void KeccakAbsorb1024bits(unsigned char *state, const unsigned char *data);
+#endif
+#ifdef ProvideFast1088
+static void KeccakAbsorb1088bits(unsigned char *state, const unsigned char *data);
+#endif
+#ifdef ProvideFast1152
+static void KeccakAbsorb1152bits(unsigned char *state, const unsigned char *data);
+#endif
+#ifdef ProvideFast1344
+static void KeccakAbsorb1344bits(unsigned char *state, const unsigned char *data);
+#endif
+static void KeccakAbsorb(unsigned char *state, const unsigned char *data, unsigned int laneCount);
+#ifdef ProvideFast1024
+static void KeccakExtract1024bits(const unsigned char *state, unsigned char *data);
+#endif
+static void KeccakExtract(const unsigned char *state, unsigned char *data, unsigned int laneCount);
+
+#endif

Modules/_sha3/keccak/KeccakF-1600-opt32-settings.h

+/*
+#define Unrolling 2
+#define UseBebigokimisa
+#define UseInterleaveTables
+#define UseSchedule 3
+*/

Modules/_sha3/keccak/KeccakF-1600-opt64-settings.h

+/*
+#define Unrolling 24
+#define UseBebigokimisa
+#define UseSSE
+#define UseOnlySIMD64
+#define UseMMX
+#define UseSHLD
+#define UseXOP
+*/

Modules/_sha3/keccak/KeccakF-1600-unrolling.macros

+/*
+The Keccak sponge function, designed by Guido Bertoni, Joan Daemen,
+Michaël Peeters and Gilles Van Assche. For more information, feedback or
+questions, please refer to our website: http://keccak.noekeon.org/
+
+Implementation by the designers,
+hereby denoted as "the implementer".
+
+To the extent possible under law, the implementer has waived all copyright
+and related or neighboring rights to the source code in this file.
+http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+#if (Unrolling == 24)
+#define rounds \
+    prepareTheta \
+    thetaRhoPiChiIotaPrepareTheta( 0, A, E) \
+    thetaRhoPiChiIotaPrepareTheta( 1, E, A) \
+    thetaRhoPiChiIotaPrepareTheta( 2, A, E) \
+    thetaRhoPiChiIotaPrepareTheta( 3, E, A) \
+    thetaRhoPiChiIotaPrepareTheta( 4, A, E) \
+    thetaRhoPiChiIotaPrepareTheta( 5, E, A) \
+    thetaRhoPiChiIotaPrepareTheta( 6, A, E) \
+    thetaRhoPiChiIotaPrepareTheta( 7, E, A) \
+    thetaRhoPiChiIotaPrepareTheta( 8, A, E) \
+    thetaRhoPiChiIotaPrepareTheta( 9, E, A) \
+    thetaRhoPiChiIotaPrepareTheta(10, A, E) \
+    thetaRhoPiChiIotaPrepareTheta(11, E, A) \
+    thetaRhoPiChiIotaPrepareTheta(12, A, E) \
+    thetaRhoPiChiIotaPrepareTheta(13, E, A) \
+    thetaRhoPiChiIotaPrepareTheta(14, A, E) \
+    thetaRhoPiChiIotaPrepareTheta(15, E, A) \
+    thetaRhoPiChiIotaPrepareTheta(16, A, E) \
+    thetaRhoPiChiIotaPrepareTheta(17, E, A) \
+    thetaRhoPiChiIotaPrepareTheta(18, A, E) \
+    thetaRhoPiChiIotaPrepareTheta(19, E, A) \
+    thetaRhoPiChiIotaPrepareTheta(20, A, E) \
+    thetaRhoPiChiIotaPrepareTheta(21, E, A) \
+    thetaRhoPiChiIotaPrepareTheta(22, A, E) \
+    thetaRhoPiChiIota(23, E, A) \
+    copyToState(state, A)
+#elif (Unrolling == 12)
+#define rounds \
+    prepareTheta \
+    for(i=0; i<24; i+=12) { \
+        thetaRhoPiChiIotaPrepareTheta(i   , A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+ 1, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+ 2, A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+ 3, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+ 4, A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+ 5, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+ 6, A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+ 7, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+ 8, A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+ 9, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+10, A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+11, E, A) \
+    } \
+    copyToState(state, A)
+#elif (Unrolling == 8)
+#define rounds \
+    prepareTheta \
+    for(i=0; i<24; i+=8) { \
+        thetaRhoPiChiIotaPrepareTheta(i  , A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+1, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+2, A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+3, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+4, A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+5, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+6, A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+7, E, A) \
+    } \
+    copyToState(state, A)
+#elif (Unrolling == 6)
+#define rounds \
+    prepareTheta \
+    for(i=0; i<24; i+=6) { \
+        thetaRhoPiChiIotaPrepareTheta(i  , A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+1, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+2, A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+3, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+4, A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+5, E, A) \
+    } \
+    copyToState(state, A)
+#elif (Unrolling == 4)
+#define rounds \
+    prepareTheta \
+    for(i=0; i<24; i+=4) { \
+        thetaRhoPiChiIotaPrepareTheta(i  , A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+1, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+2, A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+3, E, A) \
+    } \
+    copyToState(state, A)
+#elif (Unrolling == 3)
+#define rounds \
+    prepareTheta \
+    for(i=0; i<24; i+=3) { \
+        thetaRhoPiChiIotaPrepareTheta(i  , A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+1, E, A) \
+        thetaRhoPiChiIotaPrepareTheta(i+2, A, E) \
+        copyStateVariables(A, E) \
+    } \
+    copyToState(state, A)
+#elif (Unrolling == 2)
+#define rounds \
+    prepareTheta \
+    for(i=0; i<24; i+=2) { \
+        thetaRhoPiChiIotaPrepareTheta(i  , A, E) \
+        thetaRhoPiChiIotaPrepareTheta(i+1, E, A) \
+    } \
+    copyToState(state, A)
+#elif (Unrolling == 1)
+#define rounds \
+    prepareTheta \
+    for(i=0; i<24; i++) { \
+        thetaRhoPiChiIotaPrepareTheta(i  , A, E) \
+        copyStateVariables(A, E) \
+    } \
+    copyToState(state, A)
+#else
+#error "Unrolling is not correctly specified!"
+#endif

Modules/_sha3/keccak/KeccakNISTInterface.c

+/*
+The Keccak sponge function, designed by Guido Bertoni, Joan Daemen,
+Michaël Peeters and Gilles Van Assche. For more information, feedback or
+questions, please refer to our website: http://keccak.noekeon.org/
+
+Implementation by the designers,
+hereby denoted as "the implementer".
+
+To the extent possible under law, the implementer has waived all copyright
+and related or neighboring rights to the source code in this file.
+http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+#include <string.h>
+#include "KeccakNISTInterface.h"
+#include "KeccakF-1600-interface.h"
+
+static HashReturn Init(hashState *state, int hashbitlen)
+{
+    switch(hashbitlen) {
+        case 0: /*  Default parameters, arbitrary length output */
+            InitSponge((spongeState*)state, 1024, 576);
+            break;
+        case 224:
+            InitSponge((spongeState*)state, 1152, 448);
+            break;
+        case 256:
+            InitSponge((spongeState*)state, 1088, 512);
+            break;
+        case 384:
+            InitSponge((spongeState*)state, 832, 768);
+            break;
+        case 512:
+            InitSponge((spongeState*)state, 576, 1024);
+            break;
+        default:
+            return BAD_HASHLEN;
+    }
+    state->fixedOutputLength = hashbitlen;
+    return SUCCESS;
+}
+
+static HashReturn Update(hashState *state, const BitSequence *data, DataLength databitlen)
+{
+    if ((databitlen % 8) == 0)
+        return Absorb((spongeState*)state, data, databitlen);
+    else {
+        HashReturn ret = Absorb((spongeState*)state, data, databitlen - (databitlen % 8));
+        if (ret == SUCCESS) {
+            unsigned char lastByte; 
+            /*  Align the last partial byte to the least significant bits */
+            lastByte = data[databitlen/8] >> (8 - (databitlen % 8));
+            return Absorb((spongeState*)state, &lastByte, databitlen % 8);
+        }
+        else
+            return ret;
+    }
+}
+
+static HashReturn Final(hashState *state, BitSequence *hashval)
+{
+    return Squeeze(state, hashval, state->fixedOutputLength);
+}
+
+/*
+static HashReturn Hash(int hashbitlen, const BitSequence *data, DataLength databitlen, BitSequence *hashval)
+{
+    hashState state;
+    HashReturn result;
+
+    if ((hashbitlen != 224) && (hashbitlen != 256) && (hashbitlen != 384) && (hashbitlen != 512))
+        return BAD_HASHLEN; *  Only the four fixed output lengths available through this API *
+    result = Init(&state, hashbitlen);
+    if (result != SUCCESS)
+        return result;
+    result = Update(&state, data, databitlen);
+    if (result != SUCCESS)
+        return result;
+    result = Final(&state, hashval);
+    return result;
+}
+*/
+

Modules/_sha3/keccak/KeccakNISTInterface.h

+/*
+The Keccak sponge function, designed by Guido Bertoni, Joan Daemen,
+Michaël Peeters and Gilles Van Assche. For more information, feedback or
+questions, please refer to our website: http://keccak.noekeon.org/
+
+Implementation by the designers,
+hereby denoted as "the implementer".
+
+To the extent possible under law, the implementer has waived all copyright
+and related or neighboring rights to the source code in this file.
+http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+#ifndef _KeccakNISTInterface_h_
+#define _KeccakNISTInterface_h_
+
+#include "KeccakSponge.h"
+
+typedef unsigned char BitSequence;
+typedef unsigned long long DataLength;
+typedef enum { SUCCESS = 0, FAIL = 1, BAD_HASHLEN = 2 } HashReturn;
+
+typedef spongeState hashState;
+
+/**
+  * Function to initialize the state of the Keccak[r, c] sponge function.
+  * The rate r and capacity c values are determined from @a hashbitlen.
+  * @param  state       Pointer to the state of the sponge function to be initialized.
+  * @param  hashbitlen  The desired number of output bits, 
+  *                     or 0 for Keccak[] with default parameters
+  *                     and arbitrarily-long output.
+  * @pre    The value of hashbitlen must be one of 0, 224, 256, 384 and 512.
+  * @return SUCCESS if successful, BAD_HASHLEN if the value of hashbitlen is incorrect.
+  */
+static HashReturn Init(hashState *state, int hashbitlen);
+/**
+  * Function to give input data for the sponge function to absorb.
+  * @param  state       Pointer to the state of the sponge function initialized by Init().
+  * @param  data        Pointer to the input data. 
+  *                     When @a databitLen is not a multiple of 8, the last bits of data must be
+  *                     in the most significant bits of the last byte.
+  * @param  databitLen  The number of input bits provided in the input data.
+  * @pre    In the previous call to Absorb(), databitLen was a multiple of 8.
+  * @return SUCCESS if successful, FAIL otherwise.
+  */
+static HashReturn Update(hashState *state, const BitSequence *data, DataLength databitlen);
+/**
+  * Function to squeeze output data from the sponge function.
+  * If @a hashbitlen was not 0 in the call to Init(), the number of output bits is equal to @a hashbitlen.
+  * If @a hashbitlen was 0 in the call to Init(), the output bits must be extracted using the Squeeze() function.
+  * @param  state       Pointer to the state of the sponge function initialized by Init().
+  * @param  hashval     Pointer to the buffer where to store the output data.
+  * @return SUCCESS if successful, FAIL otherwise.
+  */
+static HashReturn Final(hashState *state, BitSequence *hashval);
+/**
+  * Function to compute a hash using the Keccak[r, c] sponge function.
+  * The rate r and capacity c values are determined from @a hashbitlen.
+  * @param  hashbitlen  The desired number of output bits.
+  * @param  data        Pointer to the input data. 
+  *                     When @a databitLen is not a multiple of 8, the last bits of data must be
+  *                     in the most significant bits of the last byte.
+  * @param  databitLen  The number of input bits provided in the input data.
+  * @param  hashval     Pointer to the buffer where to store the output data.
+  * @pre    The value of hashbitlen must be one of 224, 256, 384 and 512.
+  * @return SUCCESS if successful, BAD_HASHLEN if the value of hashbitlen is incorrect.
+  */
+/*
+static HashReturn Hash(int hashbitlen, const BitSequence *data, DataLength databitlen, BitSequence *hashval);
+*/
+
+#endif

Modules/_sha3/keccak/KeccakSponge.h

+/*
+The Keccak sponge function, designed by Guido Bertoni, Joan Daemen,
+Michaël Peeters and Gilles Van Assche. For more information, feedback or
+questions, please refer to our website: http://keccak.noekeon.org/
+
+Implementation by the designers,
+hereby denoted as "the implementer".
+
+To the extent possible under law, the implementer has waived all copyright
+and related or neighboring rights to the source code in this file.
+http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+#ifndef _KeccakSponge_h_
+#define _KeccakSponge_h_
+
+#define KeccakPermutationSize 1600
+#define KeccakPermutationSizeInBytes (KeccakPermutationSize/8)
+#define KeccakMaximumRate 1536
+#define KeccakMaximumRateInBytes (KeccakMaximumRate/8)
+
+#if defined(__GNUC__)
+#define ALIGN __attribute__ ((aligned(32)))
+#elif defined(_MSC_VER)
+#define ALIGN __declspec(align(32))
+#else
+#define ALIGN
+#endif
+
+ALIGN typedef struct spongeStateStruct {
+    ALIGN unsigned char state[KeccakPermutationSizeInBytes];
+    ALIGN unsigned char dataQueue[KeccakMaximumRateInBytes];
+    unsigned int rate;
+    unsigned int capacity;
+    unsigned int bitsInQueue;
+    unsigned int fixedOutputLength;
+    int squeezing;
+    unsigned int bitsAvailableForSqueezing;
+} spongeState;
+
+/**
+  * Function to initialize the state of the Keccak[r, c] sponge function.
+  * The sponge function is set to the absorbing phase.
+  * @param  state       Pointer to the state of the sponge function to be initialized.
+  * @param  rate        The value of the rate r.
+  * @param  capacity    The value of the capacity c.
+  * @pre    One must have r+c=1600 and the rate a multiple of 64 bits in this implementation.
+  * @return Zero if successful, 1 otherwise.
+  */
+static int InitSponge(spongeState *state, unsigned int rate, unsigned int capacity);
+/**
+  * Function to give input data for the sponge function to absorb.
+  * @param  state       Pointer to the state of the sponge function initialized by InitSponge().
+  * @param  data        Pointer to the input data. 
+  *                     When @a databitLen is not a multiple of 8, the last bits of data must be
+  *                     in the least significant bits of the last byte.
+  * @param  databitLen  The number of input bits provided in the input data.
+  * @pre    In the previous call to Absorb(), databitLen was a multiple of 8.
+  * @pre    The sponge function must be in the absorbing phase,
+  *         i.e., Squeeze() must not have been called before.
+  * @return Zero if successful, 1 otherwise.
+  */
+static int Absorb(spongeState *state, const unsigned char *data, unsigned long long databitlen);
+/**
+  * Function to squeeze output data from the sponge function.
+  * If the sponge function was in the absorbing phase, this function 
+  * switches it to the squeezing phase.
+  * @param  state       Pointer to the state of the sponge function initialized by InitSponge().
+  * @param  output      Pointer to the buffer where to store the output data.
+  * @param  outputLength    The number of output bits desired.
+  *                     It must be a multiple of 8.
+  * @return Zero if successful, 1 otherwise.
+  */
+static int Squeeze(spongeState *state, unsigned char *output, unsigned long long outputLength);
+
+#endif

Modules/hashlib.h

@@ -26,3 +26,36 @@
             return NULL; \
         } \
     } while(0);
+
+/*
+ * Helper code to synchronize access to the hash object when the GIL is
+ * released around a CPU consuming hashlib operation. All code paths that
+ * access a mutable part of obj must be enclosed in a ENTER_HASHLIB /
+ * LEAVE_HASHLIB block or explicitly acquire and release the lock inside
+ * a PY_BEGIN / END_ALLOW_THREADS block if they wish to release the GIL for
+ * an operation.
+ */
+
+#ifdef WITH_THREAD
+#include "pythread.h"
+    #define ENTER_HASHLIB(obj) \
+        if ((obj)->lock) { \
+            if (!PyThread_acquire_lock((obj)->lock, 0)) { \
+                Py_BEGIN_ALLOW_THREADS \
+                PyThread_acquire_lock((obj)->lock, 1); \
+                Py_END_ALLOW_THREADS \
+            } \
+        }
+    #define LEAVE_HASHLIB(obj) \
+        if ((obj)->lock) { \
+            PyThread_release_lock((obj)->lock); \
+        }
+#else
+    #define ENTER_HASHLIB(obj)
+    #define LEAVE_HASHLIB(obj)
+#endif
+
+/* TODO(gps): We should probably make this a module or EVPobject attribute
+ * to allow the user to optimize based on the platform they're using. */
+#define HASHLIB_GIL_MINSIZE 2048
+