Always regenerate .c/cpp output files when changing the Cython version. This...

Always regenerate .c/cpp output files when changing the Cython version. This is probably what users expect since silently keeping outdated files is error prone.

Cython/Build/Dependencies.py

@@ -1063,7 +1063,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
                     with open(c_file+'.dep', 'w') as outfile:
                         outfile.write(depline)
 
-                if os.path.exists(c_file):
+                # Missing files and those generated by other Cython versions should always be recreated.
+                if Utils.file_generated_by_this_cython(c_file):
                     c_timestamp = os.path.getmtime(c_file)
                 else:
                     c_timestamp = -1

Cython/Compiler/Code.py

@@ -2134,7 +2134,7 @@ class CCodeWriter(object):
             self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname))
 
     def put_generated_by(self):
-        self.putln("/* Generated by Cython %s */" % Version.watermark)
+        self.putln(Utils.GENERATED_BY_MARKER)
         self.putln("")
 
     def put_h_guard(self, guard):

Cython/Utils.py

@@ -42,6 +42,9 @@ _CACHE_NAME_PATTERN = re.compile(r"^__(.+)_cache$")
 
 modification_time = os.path.getmtime
 
+GENERATED_BY_MARKER = "/* Generated by Cython %s */" % cython_version
+GENERATED_BY_MARKER_BYTES = GENERATED_BY_MARKER.encode('us-ascii')
+
 
 class _TryFinallyGeneratorContextManager(object):
     """
@@ -193,6 +196,17 @@ def is_cython_generated_file(path, allow_failed=False, if_not_found=True):
     )
 
 
+def file_generated_by_this_cython(path):
+    file_content = b''
+    if os.path.exists(path):
+        try:
+            with open(path, "rb") as f:
+                file_content = f.read(len(GENERATED_BY_MARKER_BYTES))
+        except (OSError, IOError):
+            pass  # Probably just doesn't exist any more
+    return file_content and file_content.startswith(GENERATED_BY_MARKER_BYTES)
+
+
 def file_newer_than(path, time):
     ftime = modification_time(path)
     return ftime > time

tests/build/cythonize_newer_files.srctree

+"""
+PYTHON test.py
+"""
+
+######## a.pyx ########
+
+######## test.py ########
+
+import os.path
+
+from Cython.Utils import GENERATED_BY_MARKER_BYTES, clear_function_caches, clear_method_caches
+from Cython.Build.Dependencies import cythonize, DependencyTree
+import Cython.Build.Dependencies
+
+getmtime = os.path.getmtime
+
+def wait_for_newer_mtime(filename, old_mtime):
+    mtime = old_mtime
+    while mtime <= old_mtime:
+        os.utime(filename, None)
+        mtime = getmtime(filename)
+    return mtime
+
+
+# test the mtime waiting itself
+with open("test_file.txt", 'wb') as f:
+    pass
+orig_mtime = getmtime("test_file.txt")
+wait_for_newer_mtime("test_file.txt", orig_mtime)
+assert orig_mtime < getmtime("test_file.txt")
+
+
+def fresh_cythonize(*args):
+    clear_function_caches()
+    clear_method_caches(DependencyTree.timestamp)
+    Cython.Build.Dependencies._dep_tree = None
+    cythonize(*args)
+
+
+assert not os.path.exists("a.c")
+
+# new
+fresh_cythonize("*.pyx")
+assert os.path.isfile("a.c")
+mtime = getmtime("a.c")
+
+# already exists
+fresh_cythonize("*.pyx")
+assert mtime == getmtime("a.c")
+
+# outdated
+wait_for_newer_mtime("a.pyx", mtime)
+assert mtime < getmtime("a.pyx")
+fresh_cythonize("*.pyx")
+new_mtime = getmtime("a.c")
+assert mtime < new_mtime
+
+# now up to date
+fresh_cythonize("*.pyx")
+assert new_mtime == getmtime("a.c")
+
+# different Cython version (even though newer)
+marker = b"/* Generated by Cython "
+assert GENERATED_BY_MARKER_BYTES.startswith(marker)  # safety belt
+with open("a.c", "rb") as f:
+    content = f.read()
+
+assert content.startswith(GENERATED_BY_MARKER_BYTES)
+content = marker + b"123" + content[len(marker):]
+
+with open("a.c", "wb") as f:
+    f.write(content)
+wait_for_newer_mtime("a.c", new_mtime)
+
+other_cython_mtime = getmtime("a.c")
+assert mtime < new_mtime < other_cython_mtime
+
+fresh_cythonize("*.pyx")
+latest_mtime = getmtime("a.c")
+assert mtime < new_mtime < other_cython_mtime <= latest_mtime
+
+with open("a.c", "rb") as f:
+    assert f.read(len(GENERATED_BY_MARKER_BYTES)) == GENERATED_BY_MARKER_BYTES  # file was rewritten
+
+# now up to date
+fresh_cythonize("*.pyx")
+assert mtime < new_mtime < other_cython_mtime <= latest_mtime == getmtime("a.c")