Replace MD5 file hashing by SHA-1, both because it's faster (by 25% on 64 bit...

Replace MD5 file hashing by SHA-1, both because it's faster (by 25% on 64 bit Linux) and because MD5 is no longer allowed in US FIPS 140-2 environments.
Closes #2790.

CHANGES.rst

@@ -39,6 +39,10 @@ Bugs fixed
 Other changes
 -------------
 
+* Source file fingerprinting now uses SHA-1 instead of MD5 since the latter
+  tends to be slower and less widely supported these days.
+  (Github issue #2790)
+
 * Support for Python 2.6 was removed.
 
 

Cython/Build/Dependencies.py

@@ -115,7 +115,7 @@ def nonempty(it, error_msg="expected non-empty iterator"):
 def file_hash(filename):
     path = os.path.normpath(filename)
     prefix = ('%d:%s' % (len(path), path)).encode("UTF-8")
-    m = hashlib.md5(prefix)
+    m = hashlib.sha1(prefix)
     with open(path, 'rb') as f:
         data = f.read(65000)
         while data:
@@ -634,7 +634,7 @@ class DependencyTree(object):
         incorporate everything that has an influence on the generated code.
         """
         try:
-            m = hashlib.md5(__version__.encode('UTF-8'))
+            m = hashlib.sha1(__version__.encode('UTF-8'))
             m.update(file_hash(filename).encode('UTF-8'))
             for x in sorted(self.all_dependencies(filename)):
                 if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'):

Cython/Build/Inline.py

@@ -3,11 +3,7 @@ from __future__ import absolute_import
 import sys, os, re, inspect
 import imp
 
-try:
-    import hashlib
-except ImportError:
-    import md5 as hashlib
-
+import hashlib
 from distutils.core import Distribution, Extension
 from distutils.command.build_ext import build_ext
 
@@ -184,7 +180,7 @@ def cython_inline(code, get_type=unsafe_type,
     arg_names = sorted(kwds)
     arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
     key = orig_code, arg_sigs, sys.version_info, sys.executable, language_level, Cython.__version__
-    module_name = "_cython_inline_" + hashlib.md5(_unicode(key).encode('utf-8')).hexdigest()
+    module_name = "_cython_inline_" + hashlib.sha1(_unicode(key).encode('utf-8')).hexdigest()
 
     if module_name in sys.modules:
         module = sys.modules[module_name]

Cython/Build/IpythonMagic.py

@@ -62,11 +62,7 @@ try:
 except NameError:   # Python 3
     from imp import reload
 
-try:
-    import hashlib
-except ImportError:
-    import md5 as hashlib
-
+import hashlib
 from distutils.core import Distribution, Extension
 from distutils.command.build_ext import build_ext
 
@@ -308,7 +304,7 @@ class CythonMagics(Magics):
         if args.name:
             module_name = py3compat.unicode_to_str(args.name)
         else:
-            module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
+            module_name = "_cython_magic_" + hashlib.sha1(str(key).encode('utf-8')).hexdigest()
         html_file = os.path.join(lib_dir, module_name + '.html')
         module_path = os.path.join(lib_dir, module_name + self.so_ext)
 

Cython/Compiler/Code.py

@@ -13,22 +13,17 @@ cython.declare(os=object, re=object, operator=object, textwrap=object,
                DebugFlags=object, basestring=object, defaultdict=object,
                closing=object, partial=object)
 
+import hashlib
+import operator
 import os
 import re
 import shutil
-import sys
-import operator
 import textwrap
 from string import Template
 from functools import partial
 from contextlib import closing
 from collections import defaultdict
 
-try:
-    import hashlib
-except ImportError:
-    import md5 as hashlib
-
 from . import Naming
 from . import Options
 from . import DebugFlags
@@ -1881,7 +1876,7 @@ class CCodeWriter(object):
         include_dir = self.globalstate.common_utility_include_dir
         if include_dir and len(code) > 1024:
             include_file = "%s_%s.h" % (
-                name, hashlib.md5(code.encode('utf8')).hexdigest())
+                name, hashlib.sha1(code.encode('utf8')).hexdigest())
             path = os.path.join(include_dir, include_file)
             if not os.path.exists(path):
                 tmp_path = '%s.tmp%s' % (path, os.getpid())

Cython/Compiler/ParseTreeTransforms.py

@@ -1700,7 +1700,7 @@ if VALUE is not None:
                     e.type.create_to_py_utility_code(env)
                     e.type.create_from_py_utility_code(env)
             all_members_names = sorted([e.name for e in all_members])
-            checksum = '0x%s' % hashlib.md5(' '.join(all_members_names).encode('utf-8')).hexdigest()[:7]
+            checksum = '0x%s' % hashlib.sha1(' '.join(all_members_names).encode('utf-8')).hexdigest()[:7]
             unpickle_func_name = '__pyx_unpickle_%s' % node.class_name
 
             # TODO(robertwb): Move the state into the third argument