Commit 81dd8b95 authored by Benjamin Peterson's avatar Benjamin Peterson

use some more itertools magic to make '' be yielded after readline is done

parent 21db77e3
...@@ -379,10 +379,11 @@ def tokenize(readline): ...@@ -379,10 +379,11 @@ def tokenize(readline):
""" """
# This import is here to avoid problems when the itertools module is not # This import is here to avoid problems when the itertools module is not
# built yet and tokenize is imported. # built yet and tokenize is imported.
from itertools import chain from itertools import chain, repeat
encoding, consumed = detect_encoding(readline) encoding, consumed = detect_encoding(readline)
rl_iter = iter(readline, "") rl_gen = iter(readline, b"")
return _tokenize(chain(consumed, rl_iter).__next__, encoding) empty = repeat(b"")
return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
def _tokenize(readline, encoding): def _tokenize(readline, encoding):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment