Simplify dispatch and block body logic. Eliminate Block type. Unify functions...

Simplify dispatch and block body logic. Eliminate Block type. Unify functions with and without a dispatch function. Now only generators need a closure for holding state since the dispatcher is in a loop instead of an inner function.

compiler/expr_visitor.py

@@ -420,28 +420,20 @@ class ExprVisitor(ast.NodeVisitor):
           filename=util.go_str(self.block.filename), args=func_args.expr,
           flags=' | '.join(flags) if flags else 0)
       with self.writer.indent_block():
-        # Declare the local variables used in this function.
         for var in func_block.vars.values():
           if var.type != block.Var.TYPE_GLOBAL:
             fmt = 'var {0} *πg.Object = {1}; _ = {0}'
             self.writer.write(fmt.format(
                 util.adjust_local_name(var.name), var.init_expr))
-        body = visitor.writer.out.getvalue()
-        if func_block.checkpoints:
-          self.writer.write_block(func_block, body)
-          if func_block.is_generator:
-            self.writer.write('return πg.NewGenerator('
-                              'πBlock, πF.Globals()).ToObject(), nil')
-          else:
-            self.writer.write('return πBlock.Exec(πF)')
+        self.writer.write_temp_decls(func_block)
+        if func_block.is_generator:
+          self.writer.write('return πg.NewGenerator(πF, func(πSent *πg.Object) '
+                            '(*πg.Object, *πg.BaseException) {')
+          with self.writer.indent_block():
+            self.writer.write_block(func_block, visitor.writer.out.getvalue())
+          self.writer.write('}).ToObject(), nil')
         else:
-          assert not func_block.is_generator
-          self.writer.write('var πE *πg.BaseException\n_ = πE')
-          self.writer.write_temp_decls(func_block)
-          # There's no goto labels so align with the rest of the function.
-          with self.writer.indent_block(-1):
-            self.writer.write(body)
-          self.writer.write('return πg.None, nil')
+          self.writer.write_block(func_block, visitor.writer.out.getvalue())
       self.writer.write('}), πF.Globals()).ToObject()')
     return result
 

compiler/stmt.py

@@ -119,10 +119,10 @@ class StatementVisitor(ast.NodeVisitor):
                              filename=util.go_str(self.block.filename),
                              cls=cls.expr)
       with self.writer.indent_block():
+        self.writer.write_temp_decls(body_visitor.block)
         self.writer.write_block(body_visitor.block,
                                 body_visitor.writer.out.getvalue())
       tmpl = textwrap.dedent("""\
-          \treturn πBlock.Exec(πF)
           }).Eval(πF, πF.Globals(), nil, nil)
           if πE != nil {
           \treturn nil, πE
@@ -183,11 +183,13 @@ class StatementVisitor(ast.NodeVisitor):
           if $n, πE = πg.Next(πF, $i); πE != nil {
           \tisStop, exc := πg.IsInstance(πF, πE.ToObject(), πg.StopIterationType.ToObject())
           \tif exc != nil {
-          \t\treturn nil, exc
+          \t\tπE = exc
+          \t\tcontinue
           \t}
           \tif !isStop {
-          \t\treturn nil, πE
+          \t\tcontinue
           \t}
+          \tπE = nil
           \tπF.RestoreExc(nil, nil)
           \tgoto Label$orelse
           }""")
@@ -322,23 +324,20 @@ class StatementVisitor(ast.NodeVisitor):
       if node.tback:
         assert node.inst, 'raise had tback but no inst'
       self._write_py_context(node.lineno)
-      self.writer.write('return nil, πF.Raise({}, {}, {})'.format(
+      self.writer.write('πE = πF.Raise({}, {}, {})'.format(
           t.expr, inst.expr, tb.expr))
+      self.writer.write('continue')
 
   def visit_Return(self, node):
     assert isinstance(self.block, block.FunctionBlock)
     self._write_py_context(node.lineno)
-    if self.block.is_generator:
-      if node.value:
-        raise util.ParseError(node, 'returning a value in a generator function')
-      self.writer.write('return nil, πF.Raise('
-                        'πg.StopIterationType.ToObject(), nil, nil)')
-      return
+    if self.block.is_generator and node.value:
+      raise util.ParseError(node, 'returning a value in a generator function')
     if node.value:
       with self.expr_visitor.visit(node.value) as value:
         self.writer.write('return {}, nil'.format(value.expr))
     else:
-      self.writer.write('return πg.None, nil')
+      self.writer.write('return nil, nil')
 
   def visit_TryExcept(self, node):  # pylint: disable=g-doc-args
     # The general structure generated by this method is shown below:
@@ -414,12 +413,14 @@ class StatementVisitor(ast.NodeVisitor):
     with self.block.alloc_temp('*πg.BaseException') as exc,\
         self.block.alloc_temp('*πg.Traceback') as tb:
       self.writer.write_label(finally_label)
+      self.writer.write('πE = nil')
       self.writer.write('{}, {} = πF.RestoreExc(nil, nil)'.format(
           exc.expr, tb.expr))
       self._visit_each(node.finalbody)
       self.writer.write_tmpl(textwrap.dedent("""\
           if $exc != nil {
-          \treturn nil, πF.Raise($exc.ToObject(), nil, $tb.ToObject())
+          \tπE = πF.Raise($exc.ToObject(), nil, $tb.ToObject())
+          \tcontinue
           }"""), exc=exc.expr, tb=tb.expr)
 
   def visit_While(self, node):
@@ -499,11 +500,11 @@ class StatementVisitor(ast.NodeVisitor):
             if $exc != nil {
             \t$t = $exc.Type()
             \tif $swallow_exc, πE = $exit_func.Call(πF, πg.Args{$mgr, $t.ToObject(), $exc.ToObject(), $tb.ToObject()}, nil); πE != nil {
-            \t\treturn nil, πE
+            \t\tcontinue
             \t}
             } else {
             \tif $swallow_exc, πE = $exit_func.Call(πF, πg.Args{$mgr, πg.None, πg.None, πg.None}, nil); πE != nil {
-            \t\treturn nil, πE
+            \t\tcontinue
             \t}
             }
         """
@@ -519,7 +520,8 @@ class StatementVisitor(ast.NodeVisitor):
             swallow_exc_bool, 'πg.IsTrue(πF, {})', swallow_exc.expr)
         self.writer.write_tmpl(textwrap.dedent("""\
             if $exc != nil && $swallow_exc != true {
-            \treturn nil, πF.Raise(nil, nil, nil)
+            \tπE = πF.Raise(nil, nil, nil)
+            \tcontinue
             }"""), exc=exc.expr, swallow_exc=swallow_exc_bool.expr)
 
   def _assign_target(self, target, value):
@@ -636,6 +638,7 @@ class StatementVisitor(ast.NodeVisitor):
       self.block.bind_var(self.writer, except_node.name.id,
                           '{}.ToObject()'.format(exc))
     self._visit_each(except_node.body)
+    self.writer.write('πE = nil')
     self.writer.write('πF.RestoreExc(nil, nil)')
 
   def _write_except_dispatcher(self, exc, tb, handlers):
@@ -672,8 +675,9 @@ class StatementVisitor(ast.NodeVisitor):
         self.writer.write('goto Label{}'.format(handler_labels[-1]))
     if handlers[-1].type:
       # There's no bare except, so the fallback is to re-raise.
-      stmt = 'return nil, πF.Raise({}.ToObject(), nil, {}.ToObject())'
-      self.writer.write(stmt.format(exc, tb))
+      self.writer.write(
+          'πE = πF.Raise({}.ToObject(), nil, {}.ToObject())'.format(exc, tb))
+      self.writer.write('continue')
     return handler_labels
 
   def _write_py_context(self, lineno):

compiler/util.py

@@ -62,11 +62,8 @@ class Writer(object):
       block_: The Block object representing the code block.
       body: String containing Go code making up the body of the code block.
     """
-    self.write('var πE *πg.BaseException\n_ = πE')
-    self.write_temp_decls(block_)
-    # Write the function body.
-    self.write('πBlock := πg.NewBlock(func(πF *πg.Frame, πSent *πg.Object) '
-               '(*πg.Object, *πg.BaseException) {')
+    self.write('var πE *πg.BaseException; _ = πE')
+    self.write('for ; πF.State() >= 0; πF.PopCheckpoint() {')
     with self.indent_block():
       self.write('switch πF.State() {')
       self.write('case 0:')
@@ -74,12 +71,12 @@ class Writer(object):
         self.write_tmpl('case $state: goto Label$state', state=checkpoint)
       self.write('default: panic("unexpected function state")')
       self.write('}')
-    # Assume that body is aligned with goto labels.
-    self.write(body)
-    with self.indent_block():
-      self.write('return {}, nil'.format(
-          'nil' if block_.is_generator else 'πg.None'))
-    self.write('})')
+      # Assume that body is aligned with goto labels.
+      with self.indent_block(-1):
+        self.write(body)
+      self.write('return nil, nil')
+    self.write('}')
+    self.write('return nil, πE')
 
   def write_import_block(self, imports):
     if not imports:
@@ -103,13 +100,13 @@ class Writer(object):
   def write_checked_call2(self, result, call, *args, **kwargs):
     return self.write_tmpl(textwrap.dedent("""\
         if $result, πE = $call; πE != nil {
-        \treturn nil, πE
+        \tcontinue
         }"""), result=result.name, call=call.format(*args, **kwargs))
 
   def write_checked_call1(self, call, *args, **kwargs):
     return self.write_tmpl(textwrap.dedent("""\
         if raised := $call; raised != nil {
-        \treturn nil, raised
+        \tcontinue
         }"""), call=call.format(*args, **kwargs))
 
   def write_temp_decls(self, block_):

compiler/util_test.py

@@ -39,7 +39,7 @@ class WriterTest(unittest.TestCase):
     output = writer.out.getvalue()
     dispatch = 'switch πF.State() {\n\tcase 0:\n\tdefault: panic'
     self.assertIn(dispatch, output)
-    self.assertIn('return πg.None, nil\n}', output)
+    self.assertIn('return nil, nil\n}', output)
 
   def testWriteImportBlockEmptyImports(self):
     writer = util.Writer()

runtime/block.go

-// Copyright 2016 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package grumpy
-
-// Block is a handle to code that runs in a new scope such as a function, class
-// or module.
-type Block struct {
-	// fn is a closure that executes the body of the code block. It may be
-	// re-entered multiple times, e.g. for exception handling.
-	fn func(*Frame, *Object) (*Object, *BaseException)
-}
-
-// NewBlock creates a Block object.
-func NewBlock(fn func(*Frame, *Object) (*Object, *BaseException)) *Block {
-	return &Block{fn}
-}
-
-// Exec runs b in the context of a new child frame of back.
-func (b *Block) Exec(f *Frame) (*Object, *BaseException) {
-	return b.execInternal(f, nil)
-}
-
-func (b *Block) execInternal(f *Frame, sendValue *Object) (*Object, *BaseException) {
-	// Re-enter function body while we have checkpoint handlers left.
-	for {
-		ret, raised := b.fn(f, sendValue)
-		if raised == nil || len(f.checkpoints) == 0 {
-			return ret, raised
-		}
-		f.state = f.PopCheckpoint()
-	}
-}

runtime/block_test.go

-// Copyright 2016 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package grumpy
-
-import (
-	"reflect"
-	"testing"
-)
-
-func TestBlockExecTryExcept(t *testing.T) {
-	type blockArgs struct {
-		f *Frame
-		e *BaseException
-	}
-	args := []blockArgs{}
-	b := NewBlock(func(f *Frame, _ *Object) (*Object, *BaseException) {
-		e, _ := f.ExcInfo()
-		switch f.State() {
-		case 0:
-			goto Start
-		case 1:
-			goto Except
-		default:
-			t.Fatalf("got invalid state %d", f.State())
-		}
-	Start:
-		args = append(args, blockArgs{f, e})
-		f.PushCheckpoint(1)
-		return nil, f.RaiseType(RuntimeErrorType, "foo")
-	Except:
-		f.RestoreExc(nil, nil)
-		args = append(args, blockArgs{f, e})
-		return None, nil
-	})
-	b.Exec(newFrame(nil))
-	wantExc := mustCreateException(RuntimeErrorType, "foo")
-	if len(args) != 2 {
-		t.Errorf("called %d times, want 2", len(args))
-	} else if args[0].f == nil || args[0].f != args[1].f {
-		t.Errorf("got frames %v %v, want non-nil but identical frames", args[0].f, args[1].f)
-	} else if args[0].e != nil {
-		t.Errorf("call 0 raised %v, want nil", args[0].e)
-	} else if !exceptionsAreEquivalent(args[1].e, wantExc) {
-		t.Errorf("call 1 raised %v, want %v", args[1].e, wantExc)
-	}
-}
-
-func TestBlockExecRaises(t *testing.T) {
-	var f1, f2 *Frame
-	b1 := NewBlock(func(f *Frame, _ *Object) (*Object, *BaseException) {
-		f1 = f
-		return nil, f.RaiseType(ValueErrorType, "bar")
-	})
-	b2 := NewBlock(func(f *Frame, _ *Object) (*Object, *BaseException) {
-		f2 = f
-		return b1.Exec(f)
-	})
-	b2.Exec(newFrame(nil))
-	e, tb := f1.ExcInfo()
-	wantExc := mustCreateException(ValueErrorType, "bar")
-	if !exceptionsAreEquivalent(e, wantExc) {
-		t.Errorf("raised %v, want %v", e, wantExc)
-	}
-	wantTraceback := newTraceback(f1, nil)
-	if !reflect.DeepEqual(tb, wantTraceback) {
-		t.Errorf("exception traceback was %+v, want %+v", tb, wantTraceback)
-	}
-}

runtime/code.go

@@ -122,6 +122,9 @@ func (c *Code) Eval(f *Frame, globals *Dict, args Args, kwargs KWArgs) (*Object,
 		// Restore exc_info to what it was when we left the previous
 		// frame.
 		f.RestoreExc(oldExc, oldTraceback)
+		if ret == nil {
+			ret = None
+		}
 	} else {
 		_, tb := f.ExcInfo()
 		tb = newTraceback(f, tb)

runtime/code_test.go

@@ -30,6 +30,9 @@ func TestNewCode(t *testing.T) {
 	fn := func(f *Frame, args []*Object) (*Object, *BaseException) {
 		return NewTuple(Args(args).makeCopy()...).ToObject(), nil
 	}
+	nilFn := func(*Frame, []*Object) (*Object, *BaseException) {
+		return nil, nil
+	}
 	cases := []invokeTestCase{
 		invokeTestCase{args: wrapArgs(NewCode("f1", "foo.py", nil, 0, fn)), want: NewTuple().ToObject()},
 		invokeTestCase{args: wrapArgs(NewCode("f2", "foo.py", []FunctionArg{{"a", nil}}, 0, fn), 123), want: newTestTuple(123).ToObject()},
@@ -49,6 +52,7 @@ func TestNewCode(t *testing.T) {
 		invokeTestCase{args: wrapArgs(NewCode("f6", "foo.py", []FunctionArg{{"a", nil}}, CodeFlagKWArg, fn), "bar"), want: newTestTuple("bar", NewDict()).ToObject()},
 		invokeTestCase{args: wrapArgs(NewCode("f6", "foo.py", []FunctionArg{{"a", nil}}, CodeFlagKWArg, fn)), kwargs: wrapKWArgs("a", "apple", "b", "bear"), want: newTestTuple("apple", newTestDict("b", "bear")).ToObject()},
 		invokeTestCase{args: wrapArgs(NewCode("f6", "foo.py", []FunctionArg{{"a", nil}}, CodeFlagKWArg, fn), "bar"), kwargs: wrapKWArgs("b", "baz", "c", "qux"), want: newTestTuple("bar", newTestDict("b", "baz", "c", "qux")).ToObject()},
+		invokeTestCase{args: wrapArgs(NewCode("f7", "foo.py", nil, 0, nilFn)), want: None},
 	}
 	for _, cas := range cases {
 		if err := runInvokeTestCase(testFunc.ToObject(), &cas); err != "" {

runtime/frame.go

@@ -89,11 +89,14 @@ func (f *Frame) PushCheckpoint(state RunState) {
 
 // PopCheckpoint removes the last element of f's checkpoint stack and returns
 // it.
-func (f *Frame) PopCheckpoint() RunState {
-	last := len(f.checkpoints) - 1
-	ret := f.checkpoints[last]
-	f.checkpoints = f.checkpoints[:last]
-	return ret
+func (f *Frame) PopCheckpoint() {
+	numCheckpoints := len(f.checkpoints)
+	if numCheckpoints == 0 {
+		f.state = -1
+	} else {
+		f.state = f.checkpoints[numCheckpoints-1]
+		f.checkpoints = f.checkpoints[:numCheckpoints-1]
+	}
 }
 
 // Raise creates an exception and sets the exc info indicator in a way that is

runtime/frame_test.go

@@ -68,6 +68,7 @@ func TestFramePopCheckpoint(t *testing.T) {
 		want    RunState
 		wantTop RunState
 	}{
+		{nil, testRunStateInvalid, testRunStateInvalid},
 		{[]RunState{testRunStateDone}, testRunStateDone, testRunStateInvalid},
 		{[]RunState{testRunStateDone, testRunStateStart}, testRunStateStart, testRunStateDone},
 	}
@@ -76,7 +77,8 @@ func TestFramePopCheckpoint(t *testing.T) {
 		for _, state := range cas.states {
 			f.PushCheckpoint(state)
 		}
-		if got := f.PopCheckpoint(); got != cas.want {
+		f.PopCheckpoint()
+		if got := f.State(); got != cas.want {
 			t.Errorf("%#v.Pop() = %v, want %v", f, got, cas.want)
 		} else if numCheckpoints := len(f.checkpoints); numCheckpoints == 0 && cas.wantTop != testRunStateInvalid {
 			t.Errorf("%#v.Pop() left checkpoint stack empty, wanted top to be %v", f, cas.wantTop)

runtime/generator.go

@@ -38,15 +38,13 @@ type Generator struct {
 	Object
 	mutex sync.Mutex
 	state generatorState
-	block *Block
 	frame *Frame
+	fn    func(*Object) (*Object, *BaseException)
 }
 
 // NewGenerator returns a new Generator object that runs the given Block b.
-func NewGenerator(b *Block, globals *Dict) *Generator {
-	f := newFrame(nil)
-	f.globals = globals
-	return &Generator{Object: Object{typ: GeneratorType}, block: b, frame: f}
+func NewGenerator(f *Frame, fn func(*Object) (*Object, *BaseException)) *Generator {
+	return &Generator{Object: Object{typ: GeneratorType}, frame: f, fn: fn}
 }
 
 func toGeneratorUnsafe(o *Object) *Generator {
@@ -79,17 +77,13 @@ func (g *Generator) resume(f *Frame, sendValue *Object) (*Object, *BaseException
 		return nil, raised
 	}
 	g.frame.pushFrame(f)
-	// Yield statements push a checkpoint but upon first entry into the
-	// generator, the stack will be empty so don't pop in that case.
-	if oldState != generatorStateCreated {
-		g.frame.state = g.frame.PopCheckpoint()
-	}
-	result, raised := g.block.execInternal(g.frame, sendValue)
+	result, raised := g.fn(sendValue)
 	g.mutex.Lock()
 	if result == nil && raised == nil {
 		raised = f.Raise(StopIterationType.ToObject(), nil, nil)
 	}
 	if raised == nil {
+		g.frame.PopCheckpoint()
 		g.state = generatorStateReady
 	} else {
 		g.state = generatorStateDone

runtime/generator_test.go

@@ -19,19 +19,20 @@ import (
 )
 
 func TestGeneratorNext(t *testing.T) {
+	f := newFrame(nil)
 	var recursive *Object
-	recursiveBlock := NewBlock(func(f *Frame, _ *Object) (*Object, *BaseException) {
+	recursiveFn := func(*Object) (*Object, *BaseException) {
 		next, raised := GetAttr(f, recursive, NewStr("next"), nil)
 		if raised != nil {
 			return nil, raised
 		}
 		return next.Call(f, nil, nil)
-	})
-	recursive = NewGenerator(recursiveBlock, NewDict()).ToObject()
-	empty := NewBlock(func(f *Frame, _ *Object) (*Object, *BaseException) {
+	}
+	recursive = NewGenerator(f, recursiveFn).ToObject()
+	emptyFn := func(*Object) (*Object, *BaseException) {
 		return nil, nil
-	})
-	exhausted := NewGenerator(empty, NewDict()).ToObject()
+	}
+	exhausted := NewGenerator(newFrame(nil), emptyFn).ToObject()
 	mustNotRaise(ListType.Call(newFrame(nil), Args{exhausted}, nil))
 	cases := []invokeTestCase{
 		invokeTestCase{args: wrapArgs(recursive), wantExc: mustCreateException(ValueErrorType, "generator already executing")},
@@ -45,12 +46,12 @@ func TestGeneratorNext(t *testing.T) {
 }
 
 func TestGeneratorSend(t *testing.T) {
-	empty := NewBlock(func(f *Frame, _ *Object) (*Object, *BaseException) {
+	emptyFn := func(*Object) (*Object, *BaseException) {
 		return nil, nil
-	})
+	}
 	cases := []invokeTestCase{
-		invokeTestCase{args: wrapArgs(NewGenerator(empty, NewDict()), 123), wantExc: mustCreateException(TypeErrorType, "can't send non-None value to a just-started generator")},
-		invokeTestCase{args: wrapArgs(NewGenerator(empty, NewDict()), "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "'send' of 'generator' requires 2 arguments")},
+		invokeTestCase{args: wrapArgs(NewGenerator(newFrame(nil), emptyFn), 123), wantExc: mustCreateException(TypeErrorType, "can't send non-None value to a just-started generator")},
+		invokeTestCase{args: wrapArgs(NewGenerator(newFrame(nil), emptyFn), "foo", "bar"), wantExc: mustCreateException(TypeErrorType, "'send' of 'generator' requires 2 arguments")},
 	}
 	for _, cas := range cases {
 		if err := runInvokeMethodTestCase(GeneratorType, "send", &cas); err != "" {
@@ -60,7 +61,8 @@ func TestGeneratorSend(t *testing.T) {
 }
 
 func TestGeneratorSimple(t *testing.T) {
-	b := NewBlock(func(f *Frame, _ *Object) (*Object, *BaseException) {
+	f := newFrame(nil)
+	fn := func(*Object) (*Object, *BaseException) {
 		switch f.State() {
 		case 0:
 			goto Start
@@ -79,9 +81,9 @@ func TestGeneratorSimple(t *testing.T) {
 		return NewStr("bar").ToObject(), nil
 	Yield2:
 		return nil, nil
-	})
+	}
 	cas := &invokeTestCase{
-		args: wrapArgs(NewGenerator(b, NewDict())),
+		args: wrapArgs(NewGenerator(f, fn)),
 		want: newTestList("foo", "bar").ToObject(),
 	}
 	if err := runInvokeTestCase(ListType.ToObject(), cas); err != "" {

tools/grumpc

@@ -79,11 +79,10 @@ def main(args):
   with writer.indent_block():
     for s in sorted(mod_block.strings):
       writer.write('ß{} := πg.InternStr({})'.format(s, util.go_str(s)))
+    writer.write_temp_decls(mod_block)
     writer.write_block(mod_block, visitor.writer.out.getvalue())
-  writer.write(textwrap.dedent("""\
-      \treturn πBlock.Exec(πF)
-      }
-      var Code *πg.Code"""))
+  writer.write('}')
+  writer.write('var Code *πg.Code')
 
   if has_main:
     writer.write_tmpl(textwrap.dedent("""\