Use unicode_literals throughout compiler code.

Makefile

@@ -53,6 +53,8 @@ export GOPATH := $(ROOT_DIR)/build
 export PYTHONPATH := $(ROOT_DIR)/$(PY_DIR)
 export PATH := $(ROOT_DIR)/build/bin:$(PATH)
 
+PYTHONPARSER_SRCS := $(patsubst third_party/%,$(PY_DIR)/%,$(wildcard third_party/pythonparser/*.py))
+
 COMPILER_BIN := build/bin/grumpc
 COMPILER_SRCS := $(addprefix $(PY_DIR)/grumpy/compiler/,$(notdir $(shell find compiler -name '*.py' -not -name '*_test.py'))) $(PY_DIR)/grumpy/__init__.py
 COMPILER_TESTS := $(patsubst %.py,grumpy/%,$(filter-out compiler/expr_visitor_test.py compiler/stmt_test.py,$(wildcard compiler/*_test.py)))
@@ -62,7 +64,7 @@ COMPILER_PASS_FILES := $(patsubst %,$(PY_DIR)/%.pass,$(COMPILER_TESTS))
 COMPILER_EXPR_VISITOR_PASS_FILES := $(patsubst %,$(PY_DIR)/grumpy/compiler/expr_visitor_test.%of32.pass,$(shell seq 32))
 COMPILER_STMT_PASS_FILES := $(patsubst %,$(PY_DIR)/grumpy/compiler/stmt_test.%of16.pass,$(shell seq 16))
 COMPILER_D_FILES := $(patsubst %,$(PY_DIR)/%.d,$(COMPILER_TESTS))
-COMPILER := $(COMPILER_BIN) $(COMPILER_SRCS)
+COMPILER := $(COMPILER_BIN) $(COMPILER_SRCS) $(PYTHONPARSER_SRCS)
 
 RUNNER_BIN := build/bin/grumprun
 RUNTIME_SRCS := $(addprefix build/src/grumpy/,$(notdir $(wildcard runtime/*.go)))
@@ -73,7 +75,7 @@ RUNNER = $(RUNNER_BIN) $(COMPILER) $(RUNTIME) $(STDLIB)
 
 GRUMPY_STDLIB_SRCS := $(shell find lib -name '*.py')
 GRUMPY_STDLIB_PACKAGES := $(foreach x,$(GRUMPY_STDLIB_SRCS),$(patsubst lib/%.py,%,$(patsubst lib/%/__init__.py,%,$(x))))
-THIRD_PARTY_STDLIB_SRCS := $(shell find third_party -name '*.py')
+THIRD_PARTY_STDLIB_SRCS := $(shell find third_party/pypy -name '*.py') $(shell find third_party/stdlib -name '*.py')
 THIRD_PARTY_STDLIB_PACKAGES := $(foreach x,$(THIRD_PARTY_STDLIB_SRCS),$(patsubst third_party/stdlib/%.py,%,$(patsubst third_party/pypy/%.py,%,$(patsubst third_party/pypy/%/__init__.py,%,$(patsubst third_party/stdlib/%/__init__.py,%,$(x))))))
 STDLIB_SRCS := $(GRUMPY_STDLIB_SRCS) $(THIRD_PARTY_STDLIB_SRCS)
 STDLIB_PACKAGES := $(GRUMPY_STDLIB_PACKAGES) $(THIRD_PARTY_STDLIB_PACKAGES)
@@ -148,7 +150,7 @@ $(COMPILER_PASS_FILES): %.pass: %.py $(COMPILER)
 	@touch $@
 	@echo compiler/`basename $*` PASS
 
-$(COMPILER_D_FILES): $(PY_DIR)/%.d: $(PY_DIR)/%.py $(COMPILER_SRCS)
+$(COMPILER_D_FILES): $(PY_DIR)/%.d: $(PY_DIR)/%.py $(COMPILER_SRCS) $(PYTHONPARSER_SRCS)
 	@$(PYTHON) -m modulefinder $< | awk '{if (match($$2, /^grumpy\>/)) { print "$(PY_DIR)/$*.pass: " substr($$3, length("$(ROOT_DIR)/") + 1) }}' > $@
 
 -include $(COMPILER_D_FILES)
@@ -268,6 +270,10 @@ $(eval $(foreach x,$(STDLIB_TESTS),$(call GRUMPY_STDLIB_TEST,$(x))))
 $(PY_DIR)/weetest.py: lib/weetest.py
 	@cp -f $< $@
 
+$(PYTHONPARSER_SRCS): $(PY_DIR)/%: third_party/%
+	@mkdir -p $(@D)
+	@cp -f $< $@
+
 $(patsubst %_test,build/%.go,$(ACCEPT_TESTS)): build/%.go: %_test.py $(COMPILER)
 	@mkdir -p $(@D)
 	@$(COMPILER_BIN) $< > $@

compiler/block.py

@@ -16,11 +16,15 @@
 
 """Classes for analyzing and storing the state of Python code blocks."""
 
+from __future__ import unicode_literals
+
 import abc
 import ast
 import collections
 import re
 
+from pythonparser import source
+
 from grumpy.compiler import expr
 from grumpy.compiler import util
 
@@ -56,7 +60,7 @@ class Block(object):
   _filename = None
   _full_package_name = None
   _libroot = None
-  _lines = None
+  _buffer = None
   _runtime = None
   _strings = None
   imports = None
@@ -96,8 +100,8 @@ class Block(object):
     return self._module_block._filename  # pylint: disable=protected-access
 
   @property
-  def lines(self):
-    return self._module_block._lines  # pylint: disable=protected-access
+  def buffer(self):
+    return self._module_block._buffer  # pylint: disable=protected-access
 
   @property
   def strings(self):
@@ -220,14 +224,14 @@ class ModuleBlock(Block):
     imports: A dict mapping fully qualified Go package names to Package objects.
   """
 
-  def __init__(self, full_package_name, runtime, libroot, filename, lines,
+  def __init__(self, full_package_name, runtime, libroot, filename, src,
                future_features):
     super(ModuleBlock, self).__init__(None, '<module>')
     self._full_package_name = full_package_name
     self._runtime = runtime
     self._libroot = libroot
     self._filename = filename
-    self._lines = lines
+    self._buffer = source.Buffer(src)
     self._strings = set()
     self.imports = {}
     self._future_features = future_features

compiler/block_test.py

@@ -16,6 +16,8 @@
 
 """Tests Package, Block, BlockVisitor and related classes."""
 
+from __future__ import unicode_literals
+
 import ast
 import textwrap
 import unittest
@@ -106,7 +108,7 @@ class BlockTest(unittest.TestCase):
   def _ResolveName(self, b, name):
     writer = util.Writer()
     b.resolve_name(writer, name)
-    return writer.out.getvalue()
+    return writer.getvalue()
 
 
 class BlockVisitorTest(unittest.TestCase):
@@ -243,7 +245,7 @@ class FunctionBlockVisitorTest(unittest.TestCase):
 
 
 def _MakeModuleBlock():
-  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [],
+  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', '',
                            stmt.FutureFeatures())
 
 

compiler/expr.py

@@ -16,6 +16,8 @@
 
 """Classes representing generated expressions."""
 
+from __future__ import unicode_literals
+
 import abc
 
 from grumpy.compiler import util

compiler/expr_visitor.py

@@ -16,6 +16,8 @@
 
 """Visitor class for traversing Python expressions."""
 
+from __future__ import unicode_literals
+
 import ast
 import contextlib
 import textwrap
@@ -462,10 +464,10 @@ class ExprVisitor(ast.NodeVisitor):
           self.writer.write('return πg.NewGenerator(πF, func(πSent *πg.Object) '
                             '(*πg.Object, *πg.BaseException) {')
           with self.writer.indent_block():
-            self.writer.write_block(func_block, visitor.writer.out.getvalue())
+            self.writer.write_block(func_block, visitor.writer.getvalue())
           self.writer.write('}).ToObject(), nil')
         else:
-          self.writer.write_block(func_block, visitor.writer.out.getvalue())
+          self.writer.write_block(func_block, visitor.writer.getvalue())
       self.writer.write('}), πF.Globals()).ToObject()')
     return result
 

compiler/expr_visitor_test.py

@@ -16,6 +16,8 @@
 
 """Tests for ExprVisitor."""
 
+from __future__ import unicode_literals
+
 import ast
 import subprocess
 import textwrap
@@ -35,11 +37,13 @@ def _MakeExprTest(expr):
   return Test
 
 
-def _MakeLiteralTest(lit):
+def _MakeLiteralTest(lit, expected=None):
+  if expected is None:
+    expected = lit
   def Test(self):
     status, output = _GrumpRun('print repr({}),'.format(lit))
     self.assertEqual(0, status, output)
-    self.assertEqual(eval('repr({})'.format(lit)), output.strip())  # pylint: disable=eval-used
+    self.assertEqual(expected, output.strip())  # pylint: disable=eval-used
   return Test
 
 
@@ -130,9 +134,9 @@ class ExprVisitorTest(unittest.TestCase):
   testCompareNotInTuple = _MakeExprTest('10 < 12 not in (1, 2, 3)')
 
   testDictEmpty = _MakeLiteralTest('{}')
-  testDictNonEmpty = _MakeLiteralTest('{"foo": 42, "bar": 43}')
+  testDictNonEmpty = _MakeLiteralTest("{'foo': 42, 'bar': 43}")
 
-  testSetNoneEmpty = _MakeLiteralTest('{"foo", "bar"}')
+  testSetNonEmpty = _MakeLiteralTest("{'foo', 'bar'}", "set(['foo', 'bar'])")
 
   testDictCompFor = _MakeExprTest('{x: str(x) for x in range(3)}')
   testDictCompForIf = _MakeExprTest(
@@ -181,16 +185,15 @@ class ExprVisitorTest(unittest.TestCase):
 
   testNumInt = _MakeLiteralTest('42')
   testNumLong = _MakeLiteralTest('42L')
-  testNumIntLarge = _MakeLiteralTest('12345678901234567890')
+  testNumIntLarge = _MakeLiteralTest('12345678901234567890',
+                                     '12345678901234567890L')
   testNumFloat = _MakeLiteralTest('102.1')
-  testNumFloatOnlyDecimal = _MakeLiteralTest('.5')
-  # TODO: Current Grumpy's repr on float has different behavior than CPython.
-  # so skip these for now.
-  testNumFloatNoDecimal = unittest.expectedFailure(_MakeLiteralTest('5.'))
-  testNumFloatSci = unittest.expectedFailure(_MakeLiteralTest('1e6'))
-  testNumFloatSciCap = unittest.expectedFailure(_MakeLiteralTest('1E6'))
-  testNumFloatSciCapPlus = unittest.expectedFailure(_MakeLiteralTest('1E+6'))
-  testNumFloatSciMinus = _MakeLiteralTest('1e-6')
+  testNumFloatOnlyDecimal = _MakeLiteralTest('.5', '0.5')
+  testNumFloatNoDecimal = _MakeLiteralTest('5.', '5')
+  testNumFloatSci = _MakeLiteralTest('1e6', '1e+06')
+  testNumFloatSciCap = _MakeLiteralTest('1E6', '1e+06')
+  testNumFloatSciCapPlus = _MakeLiteralTest('1E+6', '1e+06')
+  testNumFloatSciMinus = _MakeLiteralTest('1e-06')
   testNumComplex = _MakeLiteralTest('3j')
 
   testSubscriptDictStr = _MakeExprTest('{"foo": 42}["foo"]')
@@ -205,14 +208,14 @@ class ExprVisitorTest(unittest.TestCase):
   testSubscriptMultiDimSlice = _MakeSliceTest(
       "'foo','bar':'baz':'qux'", "('foo', slice('bar', 'baz', 'qux'))")
 
-  testStrEmpty = _MakeLiteralTest('""')
-  testStrAscii = _MakeLiteralTest('"abc"')
-  testStrUtf8 = _MakeLiteralTest(r'"\tfoo\n\xcf\x80"')
-  testStrQuoted = _MakeLiteralTest('\'"foo"\'')
-  testStrUtf16 = _MakeLiteralTest(r'u"\u0432\u043e\u043b\u043d"')
+  testStrEmpty = _MakeLiteralTest("''")
+  testStrAscii = _MakeLiteralTest("'abc'")
+  testStrUtf8 = _MakeLiteralTest(r"'\tfoo\n\xcf\x80'")
+  testStrQuoted = _MakeLiteralTest('\'"foo"\'', '\'"foo"\'')
+  testStrUtf16 = _MakeLiteralTest("u'\\u0432\\u043e\\u043b\\u043d'")
 
-  testTupleEmpty = _MakeLiteralTest(())
-  testTupleNonEmpty = _MakeLiteralTest((1, 2, 3))
+  testTupleEmpty = _MakeLiteralTest('()')
+  testTupleNonEmpty = _MakeLiteralTest('(1, 2, 3)')
 
   testUnaryOpNot = _MakeExprTest('not True')
   testUnaryOpInvert = _MakeExprTest('~4')
@@ -223,7 +226,7 @@ class ExprVisitorTest(unittest.TestCase):
 
 
 def _MakeModuleBlock():
-  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [],
+  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', '',
                            stmt.FutureFeatures())
 
 
@@ -235,7 +238,7 @@ def _ParseAndVisitExpr(expr):
   writer = util.Writer()
   visitor = expr_visitor.ExprVisitor(_MakeModuleBlock(), writer)
   visitor.visit(_ParseExpr(expr))
-  return writer.out.getvalue()
+  return writer.getvalue()
 
 
 def _GrumpRun(cmd):

compiler/shard_test.py

@@ -14,6 +14,8 @@
 
 """Wrapper for unit tests that loads a subset of all test methods."""
 
+from __future__ import unicode_literals
+
 import argparse
 import random
 import re

compiler/stmt.py

@@ -16,6 +16,8 @@
 
 """Visitor class for traversing Python statements."""
 
+from __future__ import unicode_literals
+
 import ast
 import string
 import textwrap
@@ -221,7 +223,7 @@ class StatementVisitor(ast.NodeVisitor):
       with self.writer.indent_block():
         self.writer.write_temp_decls(body_visitor.block)
         self.writer.write_block(body_visitor.block,
-                                body_visitor.writer.out.getvalue())
+                                body_visitor.writer.getvalue())
       tmpl = textwrap.dedent("""\
           }).Eval(πF, πF.Globals(), nil, nil)
           if πE != nil {
@@ -815,6 +817,6 @@ class StatementVisitor(ast.NodeVisitor):
 
   def _write_py_context(self, lineno):
     if lineno:
-      line = self.block.lines[lineno - 1].strip()
+      line = self.block.buffer.source_line(lineno).strip()
       self.writer.write('// line {}: {}'.format(lineno, line))
       self.writer.write('πF.SetLineno({})'.format(lineno))

compiler/stmt_test.py

@@ -16,6 +16,8 @@
 
 """Tests for StatementVisitor."""
 
+from __future__ import unicode_literals
+
 import ast
 import re
 import subprocess
@@ -573,7 +575,7 @@ class StatementVisitorTest(unittest.TestCase):
         'exc', 'tb', handlers), [1, 2])
     expected = re.compile(r'ResolveGlobal\(.*foo.*\bIsInstance\(.*'
                           r'goto Label1.*goto Label2', re.DOTALL)
-    self.assertRegexpMatches(visitor.writer.out.getvalue(), expected)
+    self.assertRegexpMatches(visitor.writer.getvalue(), expected)
 
   def testWriteExceptDispatcherBareExceptionNotLast(self):
     visitor = stmt.StatementVisitor(_MakeModuleBlock())
@@ -593,11 +595,11 @@ class StatementVisitorTest(unittest.TestCase):
         r'ResolveGlobal\(.*foo.*\bif .*\bIsInstance\(.*\{.*goto Label1.*'
         r'ResolveGlobal\(.*bar.*\bif .*\bIsInstance\(.*\{.*goto Label2.*'
         r'\bRaise\(exc\.ToObject\(\), nil, tb\.ToObject\(\)\)', re.DOTALL)
-    self.assertRegexpMatches(visitor.writer.out.getvalue(), expected)
+    self.assertRegexpMatches(visitor.writer.getvalue(), expected)
 
 
 def _MakeModuleBlock():
-  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [],
+  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', '',
                            stmt.FutureFeatures())
 
 
@@ -605,7 +607,7 @@ def _ParseAndVisit(source):
   mod = ast.parse(source)
   future_features = stmt.visit_future(mod)
   b = block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>',
-                        source.split('\n'), future_features)
+                        source, future_features)
   visitor = stmt.StatementVisitor(b)
   visitor.visit(mod)
   return visitor

compiler/util.py

@@ -16,9 +16,13 @@
 
 """Utilities for generating Go code."""
 
+from __future__ import unicode_literals
+
+import codecs
 import contextlib
 import cStringIO
 import string
+import StringIO
 import textwrap
 
 
@@ -44,9 +48,12 @@ class Writer(object):
   """Utility class for writing blocks of Go code to a file-like object."""
 
   def __init__(self, out=None):
-    self.out = out or cStringIO.StringIO()
+    self.out = codecs.getwriter('utf8')(out or cStringIO.StringIO())
     self.indent_level = 0
 
+  def getvalue(self):
+    return self.out.getvalue().decode('utf8')
+
   @contextlib.contextmanager
   def indent_block(self, n=1):
     """A context manager that indents by n on entry and dedents on exit."""
@@ -127,7 +134,7 @@ class Writer(object):
 
 def go_str(value):
   """Returns value as a valid Go string literal."""
-  io = cStringIO.StringIO()
+  io = StringIO.StringIO()
   io.write('"')
   for c in value:
     if c in _ESCAPES:

compiler/util_test.py

@@ -16,6 +16,8 @@
 
 """Tests Writer and other utils."""
 
+from __future__ import unicode_literals
+
 import unittest
 
 from grumpy.compiler import block
@@ -31,14 +33,14 @@ class WriterTest(unittest.TestCase):
     with writer.indent_block(n=2):
       writer.write('bar')
     writer.write('baz')
-    self.assertEqual(writer.out.getvalue(), 'foo\n\t\tbar\nbaz\n')
+    self.assertEqual(writer.getvalue(), 'foo\n\t\tbar\nbaz\n')
 
   def testWriteBlock(self):
     writer = util.Writer()
     mod_block = block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>',
-                                  [], stmt.FutureFeatures())
+                                  '', stmt.FutureFeatures())
     writer.write_block(mod_block, 'BODY')
-    output = writer.out.getvalue()
+    output = writer.getvalue()
     dispatch = 'switch πF.State() {\n\tcase 0:\n\tdefault: panic'
     self.assertIn(dispatch, output)
     self.assertIn('return nil, nil\n}', output)
@@ -46,48 +48,48 @@ class WriterTest(unittest.TestCase):
   def testWriteImportBlockEmptyImports(self):
     writer = util.Writer()
     writer.write_import_block({})
-    self.assertEqual(writer.out.getvalue(), '')
+    self.assertEqual(writer.getvalue(), '')
 
   def testWriteImportBlockImportsSorted(self):
     writer = util.Writer()
     imports = {name: block.Package(name) for name in ('a', 'b', 'c')}
     writer.write_import_block(imports)
-    self.assertEqual(writer.out.getvalue(),
+    self.assertEqual(writer.getvalue(),
                      'import (\n\tπ_a "a"\n\tπ_b "b"\n\tπ_c "c"\n)\n')
 
   def testWriteMultiline(self):
     writer = util.Writer()
     writer.indent(2)
     writer.write('foo\nbar\nbaz\n')
-    self.assertEqual(writer.out.getvalue(), '\t\tfoo\n\t\tbar\n\t\tbaz\n')
+    self.assertEqual(writer.getvalue(), '\t\tfoo\n\t\tbar\n\t\tbaz\n')
 
   def testWritePyContext(self):
     writer = util.Writer()
     writer.write_py_context(12, 'print "foo"')
-    self.assertEqual(writer.out.getvalue(), '// line 12: print "foo"\n')
+    self.assertEqual(writer.getvalue(), '// line 12: print "foo"\n')
 
   def testWriteSkipBlankLine(self):
     writer = util.Writer()
     writer.write('foo\n\nbar')
-    self.assertEqual(writer.out.getvalue(), 'foo\nbar\n')
+    self.assertEqual(writer.getvalue(), 'foo\nbar\n')
 
   def testWriteTmpl(self):
     writer = util.Writer()
     writer.write_tmpl('$foo, $bar\n$baz', foo=1, bar=2, baz=3)
-    self.assertEqual(writer.out.getvalue(), '1, 2\n3\n')
+    self.assertEqual(writer.getvalue(), '1, 2\n3\n')
 
   def testIndent(self):
     writer = util.Writer()
     writer.indent(2)
     writer.write('foo')
-    self.assertEqual(writer.out.getvalue(), '\t\tfoo\n')
+    self.assertEqual(writer.getvalue(), '\t\tfoo\n')
 
   def testDedent(self):
     writer = util.Writer()
     writer.indent(4)
     writer.dedent(3)
     writer.write('foo')
-    self.assertEqual(writer.out.getvalue(), '\tfoo\n')
+    self.assertEqual(writer.getvalue(), '\tfoo\n')
 
 
 if __name__ == '__main__':

third_party/pythonparser/LICENSE.txt

+Copyright (c) 2015 whitequark <whitequark@whitequark.org>
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

third_party/pythonparser/README.md

+The source code in this directory is forked from
+[github.com/m-labs/pythonparser](https://github.com/m-labs/pythonparser).
+There are very light modifications to the source code so that it will work with
+Grumpy.

tools/grumpc

@@ -17,6 +17,8 @@
 
 """A Python -> Go transcompiler."""
 
+from __future__ import unicode_literals
+
 import argparse
 import ast
 import sys
@@ -59,8 +61,7 @@ def main(args):
 
   full_package_name = args.modname.replace('.', '/')
   mod_block = block.ModuleBlock(full_package_name, args.runtime, args.libroot,
-                                args.filename, py_contents.split('\n'),
-                                future_features)
+                                args.filename, py_contents, future_features)
   mod_block.add_native_import('grumpy')
   visitor = stmt.StatementVisitor(mod_block)
   # Indent so that the module body is aligned with the goto labels.
@@ -89,7 +90,7 @@ def main(args):
     for s in sorted(mod_block.strings):
       writer.write('ß{} := πg.InternStr({})'.format(s, util.go_str(s)))
     writer.write_temp_decls(mod_block)
-    writer.write_block(mod_block, visitor.writer.out.getvalue())
+    writer.write_block(mod_block, visitor.writer.getvalue())
   writer.write('}')
   writer.write('var Code *πg.Code')