Add pydeps tool to analyze script imports

Makefile

@@ -120,7 +120,7 @@ ACCEPT_PY_PASS_FILES := $(patsubst %,build/%_py.pass,$(filter-out %/native_test,
 BENCHMARKS := $(patsubst %.py,%,$(wildcard benchmarks/*.py))
 BENCHMARK_BINS := $(patsubst %,build/%_benchmark,$(BENCHMARKS))
 
-TOOL_BINS = $(patsubst %,build/bin/%,benchcmp coverparse diffrange)
+TOOL_BINS = $(patsubst %,build/bin/%,benchcmp coverparse diffrange pydeps)
 
 GOLINT_BIN = build/bin/golint
 PYLINT_BIN = build/bin/pylint
@@ -222,7 +222,7 @@ $(PYLINT_BIN):
 	@cd build/third_party/pylint-1.6.4 && $(PYTHON) setup.py install --prefix $(ROOT_DIR)/build
 
 pylint: $(PYLINT_BIN)
-	@$(PYTHON) $(PYLINT_BIN) compiler/*.py $(addprefix tools/,benchcmp coverparse diffrange grumpc grumprun)
+	@$(PYTHON) $(PYLINT_BIN) compiler/*.py $(addprefix tools/,benchcmp coverparse diffrange grumpc grumprun pydeps)
 
 lint: golint pylint
 
@@ -247,9 +247,9 @@ build/src/__python__/$(2)/module.go: $(1) $(COMPILER) | $(filter-out $(STDLIB_SR
 	@mkdir -p build/src/__python__/$(2)
 	@$(COMPILER_BIN) -modname=$(notdir $(2)) $(1) > $$@
 
-build/src/__python__/$(2)/module.d: $(1)
+build/src/__python__/$(2)/module.d: $(1) build/bin/pydeps $(PYTHONPARSER_SRCS) $(COMPILER)
 	@mkdir -p build/src/__python__/$(2)
-	@$(PYTHON) -m modulefinder -p $(ROOT_DIR)/lib:$(ROOT_DIR)/third_party/stdlib:$(ROOT_DIR)/third_party/pypy $$< | awk '{if (($$$$1 == "m" || $$$$1 == "P") && $$$$2 != "__main__" && $$$$2 != "$(2)") {gsub(/\./, "/", $$$$2); print "$(PKG_DIR)/__python__/$(2).a: $(PKG_DIR)/__python__/" $$$$2 ".a"}}' > $$@
+	@build/bin/pydeps $$< | awk '{gsub(/\./, "/", $$$$0); print "$(PKG_DIR)/__python__/$(2).a: $(PKG_DIR)/__python__/" $$$$0 ".a"}' > $$@
 
 $(PKG_DIR)/__python__/$(2).a: build/src/__python__/$(2)/module.go $(RUNTIME)
 	@mkdir -p $(PKG_DIR)/__python__/$(dir $(2))

compiler/stmt.py

@@ -370,12 +370,16 @@ class StatementVisitor(algorithm.Visitor):
 
   def visit_Import(self, node):
     self._write_py_context(node.lineno)
-    for imp in util.ImportVisitor().visit(node):
+    visitor = util.ImportVisitor()
+    visitor.visit(node)
+    for imp in visitor.imports:
       self._import_and_bind(imp)
 
   def visit_ImportFrom(self, node):
     self._write_py_context(node.lineno)
-    for imp in util.ImportVisitor().visit(node):
+    visitor = util.ImportVisitor()
+    visitor.visit(node)
+    for imp in visitor.imports:
       if imp.is_native:
         values = [b.value for b in imp.bindings]
         with self._import_native(imp.name, values) as mod:

compiler/util.py

@@ -86,8 +86,10 @@ class ImportVisitor(algorithm.Visitor):
 
   # pylint: disable=invalid-name,missing-docstring,no-init
 
+  def __init__(self):
+    self.imports = []
+
   def visit_Import(self, node):
-    imports = []
     for alias in node.names:
       if alias.name.startswith(_NATIVE_MODULE_PREFIX):
         raise ImportError(
@@ -97,8 +99,7 @@ class ImportVisitor(algorithm.Visitor):
         imp.add_binding(Import.MODULE, alias.asname, Import.LEAF)
       else:
         imp.add_binding(Import.MODULE, alias.name.split('.')[-1], Import.ROOT)
-      imports.append(imp)
-    return imports
+      self.imports.append(imp)
 
   def visit_ImportFrom(self, node):
     if any(a.name == '*' for a in node.names):
@@ -107,26 +108,25 @@ class ImportVisitor(algorithm.Visitor):
       raise ImportError(node, msg)
 
     if node.module == '__future__':
-      return []
+      return
 
     if node.module.startswith(_NATIVE_MODULE_PREFIX):
       imp = Import(node.module[len(_NATIVE_MODULE_PREFIX):], is_native=True)
       for alias in node.names:
         asname = alias.asname or alias.name
         imp.add_binding(Import.MEMBER, asname, alias.name)
-      return [imp]
+      self.imports.append(imp)
+      return
 
     # NOTE: Assume that the names being imported are all modules within a
     # package. E.g. "from a.b import c" is importing the module c from package
     # a.b, not some member of module b. We cannot distinguish between these
     # two cases at compile time and the Google style guide forbids the latter
     # so we support that use case only.
-    imports = []
     for alias in node.names:
       imp = Import('{}.{}'.format(node.module, alias.name))
       imp.add_binding(Import.MODULE, alias.asname or alias.name, Import.LEAF)
-      imports.append(imp)
-    return imports
+      self.imports.append(imp)
 
 
 class Writer(object):

compiler/util_test.py

@@ -96,7 +96,9 @@ class ImportVisitorTest(unittest.TestCase):
         imp, self._visit_import('from __go__.fmt import Printf as foo'))
 
   def _visit_import(self, source):
-    return util.ImportVisitor().visit(pythonparser.parse(source).body[0])
+    visitor = util.ImportVisitor()
+    visitor.visit(pythonparser.parse(source).body[0])
+    return visitor.imports
 
   def _assert_imports_equal(self, want, got):
     if isinstance(want, util.Import):

third_party/pypy/_sha512.py

@@ -277,17 +277,17 @@ class sha384(sha512):
         return new
 
 def test():
-    import _sha512
+#    import _sha512
 
     a_str = "just a test string"
 
-    assert _sha512.sha512().hexdigest() == sha512().hexdigest()
-    assert _sha512.sha512(a_str).hexdigest() == sha512(a_str).hexdigest()
-    assert _sha512.sha512(a_str*7).hexdigest() == sha512(a_str*7).hexdigest()
+    assert sha512().hexdigest() == sha512().hexdigest()
+    assert sha512(a_str).hexdigest() == sha512(a_str).hexdigest()
+    assert sha512(a_str*7).hexdigest() == sha512(a_str*7).hexdigest()
 
     s = sha512(a_str)
     s.update(a_str)
-    assert _sha512.sha512(a_str+a_str).hexdigest() == s.hexdigest()
+    assert sha512(a_str+a_str).hexdigest() == s.hexdigest()
 
 if __name__ == "__main__":
     test()

tools/pydeps

+#!/usr/bin/env python
+
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Outputs names of modules imported by a script."""
+
+import argparse
+import sys
+
+import pythonparser
+
+from grumpy.compiler import util
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('script')
+
+
+def main(args):
+  with open(args.script) as py_file:
+    py_contents = py_file.read()
+  try:
+    mod = pythonparser.parse(py_contents)
+  except SyntaxError as e:
+    print >> sys.stderr, '{}: line {}: invalid syntax: {}'.format(
+        e.filename, e.lineno, e.text)
+    return 2
+  visitor = util.ImportVisitor()
+  try:
+    visitor.visit(mod)
+  except util.ParseError as e:
+    print >> sys.stderr, str(e)
+    return 2
+  imports = set()
+  for imp in visitor.imports:
+    if not imp.is_native and imp.name not in imports:
+      imports.add(imp.name)
+      print imp.name
+
+
+if __name__ == '__main__':
+  main(parser.parse_args())