Merge pull request #1393 from undingen/jemalloc_prof

jemalloc: build with profiling support

CMakeLists.txt

@@ -149,7 +149,7 @@ ExternalProject_Add(libjemalloc
                     SOURCE_DIR ${CMAKE_SOURCE_DIR}/build_deps/jemalloc
                     DEPENDS gitsubmodules
                     UPDATE_COMMAND autoconf
-                    CONFIGURE_COMMAND ${CMAKE_SOURCE_DIR}/build_deps/jemalloc/configure --prefix=${CMAKE_BINARY_DIR}/jemalloc --enable-autogen --enable-prof-libunwind
+                    CONFIGURE_COMMAND ${CMAKE_SOURCE_DIR}/build_deps/jemalloc/configure --prefix=${CMAKE_BINARY_DIR}/jemalloc --enable-autogen --enable-prof --enable-prof-libunwind
                     BUILD_COMMAND ${MAKE_PRG}
                     INSTALL_COMMAND ${MAKE_PRG} install_bin install_lib
                     LOG_UPDATE ON

docs/MEMORY_PROFILING.md

+There are different ways to do heap profiling but it looks like jemalloc heap profiling works currently best for Pyston. (unfortunately valgrind often crashes currently if the ICs are enabled)
+
+## Prerequisites
+Pyston comes with jemalloc which is already configured with enabled profiling.
+But because we currently don't install the header one needs to run as a workaround `sudo apt-get install libjemalloc-dev` to install the system package and rebuild jemalloc afterwards.
+
+## Usage
+The jemalloc documentation about profiling can be found [here](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling) and the one about leak checking [here](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Leak-Checking).
+
+an example usage is:
+```
+~/pyston$ MALLOC_CONF="prof:true,prof_prefix:jeprof.out" ./pyston_release minibenchmarks/raytrace.py
+~/pyston$ build_deps/jemalloc/bin/pprof pyston_release jeprof.out.*
+(pprof) top3
+Total: 4.5 MB
+     1.0  22.2%  22.2%      1.0  22.2% pyston::BST_FunctionDef::operator new (inline)
+     0.5  11.1%  33.3%      0.5  11.1% pyston::DenseMap::allocateBuckets (inline)
+     0.5  11.1%  44.5%      0.5  11.1% pyston::CFG::addBlock
+```
+

test/extra/avro_test.py

@@ -5,10 +5,10 @@ from test_helper import create_virtenv, run_test
 
 ENV_NAME = "avro_test_env_" + os.path.basename(sys.executable)
 PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
-NOSETESTS_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "nosetests"))
+PYTEST_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "py.test"))
 AVRO_DIR = os.path.abspath(os.path.join(ENV_NAME, "avro-1.7.7"))
 
-packages = ["nose==1.3.7", "avro==1.7.7"] 
+packages = ["pytest==2.8.7", "py==1.4.29", "avro==1.7.7"] 
 create_virtenv(ENV_NAME, packages, force_create = True)
 
 url = "https://pypi.python.org/packages/source/a/avro/avro-1.7.7.tar.gz"
@@ -18,7 +18,6 @@ subprocess.check_call(["tar", "-zxf", "avro-1.7.7.tar.gz"], cwd=ENV_NAME)
 env = os.environ
 env["PYTHONPATH"] = os.path.abspath(os.path.join(ENV_NAME, "lib/python2.7/site-packages"))
 
-# this tests also fail when run in cpython with nose.
-# pytest makes two of this work but we can't currently run pytest...
-expected = [{'ran': 51, 'errors': 3}]
-run_test([NOSETESTS_EXE], env=env, cwd=AVRO_DIR, expected=expected)
+# cpython has the same number of failures
+expected = [{'failed': 2, 'passed': 47}]
+run_test([PYTEST_EXE], env=env, cwd=AVRO_DIR, expected=expected)

test/extra/paste_test.py

@@ -36,7 +36,7 @@ print ">> "
 # - no sys.settrace 
 # - no shiftjis encoding
 # - slightly different error messages
-expected = [{"failed" : 21, "passed" : 113}]
+expected = [{"failed" : 22, "passed" : 112}]
 run_test([PYTEST_EXE], cwd=PASTE_TEST_DIR, expected=expected)