Commit 2baa1889 authored by Stefan Behnel's avatar Stefan Behnel

minor source modernisations and cleanups in Scanning.py

parent 73aa3651
......@@ -4,6 +4,9 @@ import cython
from ..Plex.Scanners cimport Scanner
cdef get_lexicon()
cdef initial_compile_time_env()
cdef class Method:
cdef object name
cdef object __name__
......
......@@ -5,13 +5,15 @@
from __future__ import absolute_import
import cython
cython.declare(EncodedString=object, make_lexicon=object, lexicon=object,
any_string_prefix=unicode, IDENT=unicode,
print_function=object, error=object, warning=object,
os=object, platform=object)
import os
import platform
import cython
cython.declare(EncodedString=object, any_string_prefix=unicode, IDENT=unicode,
print_function=object, error=object, warning=object)
from .. import Utils
from ..Plex.Scanners import Scanner
from ..Plex.Errors import UnrecognizedInput
......@@ -28,12 +30,14 @@ scanner_dump_file = None
lexicon = None
def get_lexicon():
global lexicon
if not lexicon:
lexicon = make_lexicon()
return lexicon
#------------------------------------------------------------------
py_reserved_words = [
......@@ -49,6 +53,7 @@ pyx_reserved_words = py_reserved_words + [
"cimport", "DEF", "IF", "ELIF", "ELSE"
]
class Method(object):
def __init__(self, name):
......@@ -58,6 +63,7 @@ class Method(object):
def __call__(self, stream, text):
return getattr(stream, self.name)(text)
#------------------------------------------------------------------
class CompileTimeScope(object):
......@@ -88,6 +94,7 @@ class CompileTimeScope(object):
else:
raise
def initial_compile_time_env():
benv = CompileTimeScope()
names = ('UNAME_SYSNAME', 'UNAME_NODENAME', 'UNAME_RELEASE',
......@@ -116,6 +123,7 @@ def initial_compile_time_env():
denv = CompileTimeScope(benv)
return denv
#------------------------------------------------------------------
class SourceDescriptor(object):
......@@ -166,6 +174,7 @@ class SourceDescriptor(object):
except AttributeError:
return False
class FileSourceDescriptor(SourceDescriptor):
"""
Represents a code source. A code source is a more generic abstraction
......@@ -235,6 +244,7 @@ class FileSourceDescriptor(SourceDescriptor):
def __repr__(self):
return "<FileSourceDescriptor:%s>" % self.filename
class StringSourceDescriptor(SourceDescriptor):
"""
Instances of this class can be used instead of a filenames if the
......@@ -275,6 +285,7 @@ class StringSourceDescriptor(SourceDescriptor):
def __repr__(self):
return "<StringSourceDescriptor:%s>" % self.name
#------------------------------------------------------------------
class PyrexScanner(Scanner):
......@@ -284,8 +295,8 @@ class PyrexScanner(Scanner):
# compile_time_eval boolean In a true conditional compilation context
# compile_time_expr boolean In a compile-time expression context
def __init__(self, file, filename, parent_scanner = None,
scope = None, context = None, source_encoding=None, parse_comments=True, initial_pos=None):
def __init__(self, file, filename, parent_scanner=None,
scope=None, context=None, source_encoding=None, parse_comments=True, initial_pos=None):
Scanner.__init__(self, get_lexicon(), file, filename, initial_pos)
if parent_scanner:
self.context = parent_scanner.context
......@@ -299,8 +310,8 @@ class PyrexScanner(Scanner):
self.compile_time_env = initial_compile_time_env()
self.compile_time_eval = 1
self.compile_time_expr = 0
if hasattr(context.options, 'compile_time_env') and \
context.options.compile_time_env is not None:
if (hasattr(context.options, 'compile_time_env') and
context.options.compile_time_env is not None):
self.compile_time_env.update(context.options.compile_time_env)
self.parse_comments = parse_comments
self.source_encoding = source_encoding
......@@ -326,11 +337,11 @@ class PyrexScanner(Scanner):
return self.indentation_stack[-1]
def open_bracket_action(self, text):
self.bracket_nesting_level = self.bracket_nesting_level + 1
self.bracket_nesting_level += 1
return text
def close_bracket_action(self, text):
self.bracket_nesting_level = self.bracket_nesting_level - 1
self.bracket_nesting_level -= 1
return text
def newline_action(self, text):
......@@ -406,6 +417,7 @@ class PyrexScanner(Scanner):
sy, systring = self.read()
except UnrecognizedInput:
self.error("Unrecognized character")
return # just a marker, error() always raises
if sy == IDENT:
if systring in self.keywords:
if systring == u'print' and print_function in self.context.future_directives:
......@@ -445,21 +457,21 @@ class PyrexScanner(Scanner):
# This method should be added to Plex
self.queue.insert(0, (token, value))
def error(self, message, pos = None, fatal = True):
def error(self, message, pos=None, fatal=True):
if pos is None:
pos = self.position()
if self.sy == 'INDENT':
err = error(pos, "Possible inconsistent indentation")
error(pos, "Possible inconsistent indentation")
err = error(pos, message)
if fatal: raise err
def expect(self, what, message = None):
def expect(self, what, message=None):
if self.sy == what:
self.next()
else:
self.expected(what, message)
def expect_keyword(self, what, message = None):
def expect_keyword(self, what, message=None):
if self.sy == IDENT and self.systring == what:
self.next()
else:
......@@ -476,12 +488,10 @@ class PyrexScanner(Scanner):
self.error("Expected '%s', found '%s'" % (what, found))
def expect_indent(self):
self.expect('INDENT',
"Expected an increase in indentation level")
self.expect('INDENT', "Expected an increase in indentation level")
def expect_dedent(self):
self.expect('DEDENT',
"Expected a decrease in indentation level")
self.expect('DEDENT', "Expected a decrease in indentation level")
def expect_newline(self, message="Expected a newline", ignore_semicolon=False):
# Expect either a newline or end of file
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment