fix for files with coding cookies and BOMs

1613ed81 · Benjamin Peterson · 689a5580 · 1613ed81
Commit 1613ed81 authored Mar 18, 2010 by Benjamin Peterson
Hide whitespace changes
Inline Side-by-side

Showing with 5 additions and 3 deletions

Lib/tokenize.py Lib/tokenize.py +5 -3

No files found.
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -333,9 +333,11 @@ def detect_encoding(readline):
            # This behaviour mimics the Python interpreter
            raise SyntaxError("unknown encoding: " + encoding)
-        if bom_found and codec.name != 'utf-8':
+        if bom_found:
-            # This behaviour mimics the Python interpreter
+            if codec.name != 'utf-8':
-            raise SyntaxError('encoding problem: utf-8')
+                # This behaviour mimics the Python interpreter
+                raise SyntaxError('encoding problem: utf-8')
+            encoding += '-sig'
        return encoding
    first = read_or_stop()