Merge pull request #208 from dagar/master

initial cmake support

.gitignore

@@ -20,6 +20,8 @@ pyston_release
 pyston_grwl
 pyston_grwl_dbg
 pyston_nosync
+/gc
+/analysis
 
 *.cache
 tests/t.py

.gitmodules

+[submodule "libunwind"]
+	path = libunwind
+	url = git://git.sv.gnu.org/libunwind.git
+[submodule "libpypa"]
+	path = libpypa
+	url = https://github.com/vinzenz/libpypa.git

CMakeLists.txt

+cmake_minimum_required(VERSION 2.8)
+project(pyston)
+
+include(ExternalProject)
+
+set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
+set(DEPS_DIR $ENV{HOME}/pyston_deps)
+
+# set build type to release by default
+set(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "" FORCE)
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: Debug Release." FORCE)
+endif()
+if(NOT ${CMAKE_BUILD_TYPE} STREQUAL "Release" AND NOT ${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+  message(FATAL_ERROR "CMAKE_BUILD_TYPE must be set to Release or Debug")
+endif()
+
+option(ENABLE_GIL "threading use GIL" ON)
+option(ENABLE_GRWL "threading use GRWL" OFF)
+option(ENABLE_INTEL_JIT_EVENTS "LLVM support for Intel JIT Events API" OFF)
+option(ENABLE_LLVM_DEBUG "LLVM debug symbols" OFF)
+option(ENABLE_VALGRIND "pyston valgrind support" OFF)
+
+# initial clang flags (set here so they're used when building llvm)
+set(CLANG_FLAGS "-Qunused-arguments -fcolor-diagnostics" CACHE STRING "Clang specific C and CXX flags")
+if("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CLANG_FLAGS}")
+endif()
+if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CLANG_FLAGS}")
+endif()
+
+# llvm disable debug info unless ENABLE_LLVM_DEBUG is ON
+if(${CMAKE_BUILD_TYPE} STREQUAL "Debug" AND NOT ENABLE_LLVM_DEBUG)
+  set(CMAKE_CXX_FLAGS_DEBUG "-g0" CACHE STRING "" FORCE)
+endif()
+
+execute_process(COMMAND cat llvm_revision.txt WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE LLVMREV OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+# llvm and clang patches
+add_custom_target(llvm_gotorev python ${CMAKE_SOURCE_DIR}/tools/git_svn_gotorev.py ${DEPS_DIR}/llvm-trunk ${LLVMREV} llvm_patches WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+add_custom_target(clang_gotorev python ${CMAKE_SOURCE_DIR}/tools/git_svn_gotorev.py ${DEPS_DIR}/llvm-trunk/tools/clang ${LLVMREV} clang_patches WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+add_custom_target(llvm_up DEPENDS llvm_gotorev clang_gotorev)
+
+# llvm
+set(LLVM_TARGETS_TO_BUILD "host" CACHE STRING "LLVM targets")
+#set(LLVM_EXTERNAL_CLANG_SOURCE_DIR "${CMAKE_SOURCE_DIR}/clang" CACHE String "Clang directory")
+#add_subdirectory(llvm EXCLUDE_FROM_ALL)
+if(ENABLE_INTEL_JIT_EVENTS)
+  set(LLVM_USE_INTEL_JITEVENTS "ON" CACHE STRING "Enable building support for the Intel JIT Events API")
+  set(INTEL_JIT_EVENTS_LIB "inteljitevents")
+  add_definitions(-DENABLE_INTEL_JIT_EVENTS=1)
+endif()
+add_subdirectory(${DEPS_DIR}/llvm-trunk ${CMAKE_BINARY_DIR}/llvm)
+set(CMAKE_MODULE_PATH "${CMAKE_BINARY_DIR}/llvm/share/llvm/cmake/")
+include(LLVMConfig)
+llvm_map_components_to_libnames(LLVM_LIBS core mcjit native bitreader ipo irreader debuginfo instrumentation ${INTEL_JIT_EVENTS_LIB})
+
+# libunwind
+if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
+  set(LIBUNWIND_FLAGS "-g -O0")
+  set(LIBUNWIND_DEBUG "--enable-debug")
+  set(LIBUNWIND_DEBUG_FRAME "--enable-debug-frame")
+endif()
+ExternalProject_Add(libunwind
+                    PREFIX libunwind
+                    SOURCE_DIR ${CMAKE_SOURCE_DIR}/libunwind
+                    UPDATE_COMMAND autoreconf -i
+                    CONFIGURE_COMMAND ${CMAKE_SOURCE_DIR}/libunwind/configure CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=${LIBUNWIND_FLAGS} CXXFLAGS=${LIBUNWIND_FLAGS} --prefix=${CMAKE_BINARY_DIR}/libunwind --enable-shared=0 ${LIBUNWIND_DEBUG} ${LIBUNWIND_DEBUG_FRAME}
+                    LOG_UPDATE ON
+                    LOG_CONFIGURE ON
+                    LOG_BUILD ON
+                    LOG_INSTALL ON)
+
+# libpypa
+add_subdirectory(libpypa)
+
+# valgrind
+if(ENABLE_VALGRIND)
+  find_package(Valgrind REQUIRED)
+  include_directories(${VALGRIND_INCLUDE_DIR})
+  message(STATUS "Including valgrind ${VALGRIND_INCLUDE_DIR}")
+else()
+  add_definitions(-DNVALGRIND)
+endif()
+
+if(ENABLE_GRWL)
+  add_definitions(-DTHREADING_USE_GIL=0 -DTHREADING_USE_GRWL=1)
+else()
+  add_definitions(-DTHREADING_USE_GIL=1 -DTHREADING_USE_GRWL=0)
+endif()
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Werror -Wreturn-type -Wno-sign-compare -Wno-unused -Wno-sign-compare -Wno-unused-parameter -fno-omit-frame-pointer")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} -std=c++11 -fno-rtti -fexceptions -fvisibility-inlines-hidden -ffunction-sections -fdata-sections -Woverloaded-virtual -Wno-invalid-offsetof -Wcast-qual -Wno-sign-conversion -Wnon-virtual-dtor -Winit-self -Wmissing-include-dirs -Wstrict-overflow=5 -Wpointer-arith -Wtype-limits -Wwrite-strings -Wempty-body -Waggregate-return -Wmissing-field-initializers -Wredundant-decls -Winline -Wint-to-pointer-cast -Wlong-long -Wvla")
+
+set(CLANG_FLAGS "${CLANG_FLAGS} -Wimplicit-int -Wstrict-prototypes -Wold-style-definition -Wnested-externs -Wpointer-to-int-cast -Wno-mismatched-tags -Wno-extern-c-compat")
+if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CLANG_FLAGS}")
+endif()
+
+if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-long-long -Wno-aggregate-return -Wno-inline -Wno-redundant-decls -Wno-strict-overflow")
+endif()
+
+add_definitions(${LLVM_DEFINITIONS})
+add_definitions(-DDEFAULT_PYTHON_MAJOR_VERSION=2 -DDEFAULT_PYTHON_MINOR_VERSION=7 -DDEFAULT_PYTHON_MICRO_VERSION=6) # Python 2.7.6
+add_definitions(-DLLVMREV=${LLVMREV})
+
+include_directories(${LLVM_INCLUDE_DIRS})
+include_directories(include)
+
+find_package(LibLZMA REQUIRED)
+link_directories(${CMAKE_BINARY_DIR}/libunwind/lib)
+link_directories(${LLVM_LIBRARY_DIRS})
+
+add_subdirectory(lib_python)
+add_subdirectory(src)
+add_subdirectory(test/test_extension)
+add_subdirectory(test/unittests)
+add_subdirectory(tools)
+
+add_executable(pyston $<TARGET_OBJECTS:PYSTON_MAIN_OBJECT> $<TARGET_OBJECTS:PYSTON_OBJECTS> $<TARGET_OBJECTS:FROM_CPYTHON_OBJECTS>)
+target_link_libraries(pyston stdlib pthread m readline gmp unwind pypa double-conversion ${LLVM_LIBS} ${LIBLZMA_LIBRARIES})
+
+# copy the python standard library (lib_python/2.7) and src/codegen/parse_ast.py to the build directory
+add_custom_command(TARGET pyston POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/lib_python/2.7 ${CMAKE_BINARY_DIR}/lib_python/2.7)
+add_custom_command(TARGET pyston POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/src/codegen/parse_ast.py ${CMAKE_BINARY_DIR}/src/codegen/parse_ast.py)
+
+add_custom_target(astcompare COMMAND ${CMAKE_SOURCE_DIR}/tools/astprint_test.sh
+                  DEPENDS astprint
+                  COMMENT "Running libpypa vs CPython AST result comparison test")
+
+# test
+enable_testing()
+add_test(NAME lint COMMAND python ${CMAKE_SOURCE_DIR}/tools/lint.py WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src)
+add_test(NAME check-format COMMAND ${CMAKE_SOURCE_DIR}/tools/check_format.sh ${LLVM_TOOLS_BINARY_DIR}/clang-format WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src)
+add_test(NAME gc_unittest COMMAND gc_unittest)
+add_test(NAME analysis_unittest COMMAND analysis_unittest)
+add_test(NAME pyston_defaults COMMAND python ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j4 -k ${CMAKE_SOURCE_DIR}/test/tests)
+add_test(NAME pyston_disable_interpreter COMMAND python ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j4 -k -a -n -k ${CMAKE_SOURCE_DIR}/test/tests)
+add_test(NAME pyston_max_compilation_tier COMMAND python ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j4 -a -O -k ${CMAKE_SOURCE_DIR}/test/tests)
+
+# format
+file(GLOB_RECURSE FORMAT_FILES ${CMAKE_SOURCE_DIR}/src/*.h ${CMAKE_SOURCE_DIR}/src/*.cpp)
+add_custom_target(format ${LLVM_TOOLS_BINARY_DIR}/clang-format -style=file -i ${FORMAT_FILES} DEPENDS clang-format WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src)
+add_custom_target(check-format ${CMAKE_SOURCE_DIR}/tools/check_format.sh ${LLVM_TOOLS_BINARY_DIR}/clang-format DEPENDS clang-format WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src)
+
+# lint
+add_custom_target(lint python ${CMAKE_SOURCE_DIR}/tools/lint.py WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src)
+
+# check
+add_custom_target(check-pyston COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure DEPENDS pyston clang-format ext_cpython ext_pyston unittests) 
+
+# {run,dbg,perf,memcheck,memleaks,cachegrind}_TESTNAME
+file(GLOB RUNTARGETS ${CMAKE_SOURCE_DIR}/test/tests/*.py ${CMAKE_SOURCE_DIR}/microbenchmarks/*.py ${CMAKE_SOURCE_DIR}/minibenchmarks/*.py)
+foreach(RUNTARGET ${RUNTARGETS})
+  get_filename_component(BASEFILENAME ${RUNTARGET} NAME_WE)
+  add_custom_target(run_${BASEFILENAME} ./pyston -q ${RUNTARGET} DEPENDS pyston)
+  add_custom_target(dbg_${BASEFILENAME} gdb --ex "set confirm off" --ex "handle SIGUSR2 pass nostop noprint" --ex run --ex "bt 20" --args ./pyston -q ${RUNTARGET} DEPENDS pyston)
+  add_custom_target(perf_${BASEFILENAME} perf record -g -- ./pyston -q -p ${RUNTARGET}
+                    COMMAND perf report -v -n -g flat,1000 | bash ${CMAKE_SOURCE_DIR}/tools/cumulate.sh | less -S)
+
+  if(ENABLE_VALGRIND)
+    add_custom_target(memcheck_${BASEFILENAME} valgrind --tool=memcheck --leak-check=no --db-attach=yes ./pyston ${RUNTARGET} DEPENDS pyston)
+    add_custom_target(memleaks_${BASEFILENAME} valgrind --tool=memcheck --leak-check=full --leak-resolution=low --show-reachable=yes ./pyston ${RUNTARGET} DEPENDS pyston)
+    add_custom_target(cachegrind_${BASEFILENAME} valgrind --tool=cachegrind ./pyston ${RUNTARGET} DEPENDS pyston)
+  endif()
+endforeach()

cmake/FindValgrind.cmake

+# find valgrind header and binary
+include(FindPackageHandleStandardArgs)
+
+find_path(VALGRIND_INCLUDE_DIR
+          NAMES valgrind.h
+          PATHS /usr/include /usr/include/valgrind ${VALGRIND_DIR}/include)
+
+find_program(VALGRIND_BIN NAMES valgrind PATH /usr/bin ${VALGRIND_DIR}/bin)
+
+find_package_handle_standard_args(valgrind REQUIRED_VARS VALGRIND_BIN VALGRIND_INCLUDE_DIR)

lib_python/CMakeLists.txt

+# compile specified files in lib_python/2.7_Modules
+file(GLOB_RECURSE FROM_CPYTHON_SRCS errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c)
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers")
+add_library(FROM_CPYTHON_OBJECTS OBJECT ${FROM_CPYTHON_SRCS})

libpypa @ 1245304c

+Subproject commit 1245304cce0e4f58e9ec311add5f70ef54929ea3

libunwind @ c90a2e02

+Subproject commit c90a2e02b3c1b03362a549a05261a4d0513d6026

src/CMakeLists.txt

+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+set(CMAKE_CXX_FLAGS_DEBUG "-g -DBINARY_SUFFIX= -DBINARY_STRIPPED_SUFFIX=_stripped")
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fstrict-aliasing -enable-tbaa -DNVALGRIND -DBINARY_SUFFIX=_release -DBINARY_STRIPPED_SUFFIX=")
+
+execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE GITREV OUTPUT_STRIP_TRAILING_WHITESPACE)
+set_source_files_properties(jit.cpp PROPERTIES COMPILE_DEFINITIONS "GITREV=${GITREV}")
+
+include_directories(${CMAKE_BINARY_DIR})
+include_directories(${CMAKE_BINARY_DIR}/libunwind/include)
+include_directories(${CMAKE_SOURCE_DIR}/libpypa/src)
+
+file(GLOB ANALYSIS_SRCS analysis/*.cpp)
+file(GLOB ASM_WRITING_SRCS asm_writing/*.cpp)
+file(GLOB CAPI_SRCS capi/*.c capi/*.cpp)
+file(GLOB CODEGEN_IRGEN_SRCS codegen/irgen/*.cpp)
+file(GLOB CODEGEN_OPT_SRCS codegen/opt/*.cpp)
+file(GLOB CODEGEN_SRCS codegen/*.cpp)
+file(GLOB CORE_SRCS core/*.cpp)
+file(GLOB GC_SRCS gc/*.cpp)
+file(GLOB RUNTIME_BUILTIN_MODULES_SRCS runtime/builtin_modules/*.cpp)
+file(GLOB RUNTIME_SRCS runtime/*.cpp)
+
+add_library(PYSTON_OBJECTS OBJECT codegen/profiling/profiling.cpp ${ANALYSIS_SRCS} ${ASM_WRITING_SRCS} ${CAPI_SRCS} ${CODEGEN_SRCS} ${CODEGEN_IRGEN_SRCS} ${CODEGEN_OPT_SRCS} ${CORE_SRCS} ${GC_SRCS} ${RUNTIME_BUILTIN_MODULES_SRCS} ${RUNTIME_SRCS})
+add_dependencies(PYSTON_OBJECTS libunwind pypa ${LLVM_LIBS})
+add_library(PYSTON_MAIN_OBJECT OBJECT jit.cpp)
+add_dependencies(PYSTON_MAIN_OBJECT ${LLVM_LIBS})
+
+# build stdlib
+add_subdirectory(runtime/inline)
+
+add_custom_target(cpplint python ${CMAKE_SOURCE_DIR}/tools/cpplint.py --filter=-whitespace,-build/header_guard,-build/include_order,-readability/todo ${ANALYSIS_SRCS} ${ASM_WRITING_SRCS} ${CAPI_SRCS} ${CODEGEN_SRCS} ${CODEGEN_IRGEN_SRCS} ${CODEGEN_OPT_SRCS} ${CORE_SRCS} ${GC_SRCS} ${RUNTIME_SRCS} ${RUNTIME_BUILTIN_MODULES_SRCS} ${FROM_CPYTHON_SRCS} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src)

src/codegen/parser.cpp

@@ -889,9 +889,6 @@ static std::string getParserCommandLine(const char* fn) {
     // Start by removing the binary name, because the "pyston" binary will break the logic below
     llvm::sys::path::remove_filename(parse_ast_fn);
 
-    while (llvm::sys::path::filename(parse_ast_fn) != "pyston") {
-        llvm::sys::path::remove_filename(parse_ast_fn);
-    }
     llvm::sys::path::append(parse_ast_fn, "src/codegen/parse_ast.py");
 
     return std::string("python -S ") + parse_ast_fn.str().str() + " " + fn;

src/jit.cpp

@@ -119,7 +119,6 @@ int main(int argc, char** argv) {
 
     llvm::SmallString<128> stdlib_dir(self_path);
     llvm::sys::path::remove_filename(stdlib_dir); // executable name
-    llvm::sys::path::remove_filename(stdlib_dir); // "src/" dir
     llvm::sys::path::append(stdlib_dir, "lib_python");
     llvm::sys::path::append(stdlib_dir, "2.7");
     appendToSysPath(stdlib_dir.c_str());

src/runtime/capi.cpp

@@ -937,7 +937,7 @@ extern "C" Py_ssize_t PyNumber_AsSsize_t(PyObject* o, PyObject* exc) {
 }
 
 BoxedModule* importTestExtension(const std::string& name) {
-    std::string pathname_name = "../test/test_extension/" + name + ".pyston.so";
+    std::string pathname_name = "test/test_extension/" + name + ".pyston.so";
     const char* pathname = pathname_name.c_str();
     void* handle = dlopen(pathname, RTLD_NOW);
     if (!handle) {

src/runtime/inline/CMakeLists.txt

+# Copyright (c) 2010-2011, Rob Jansen
+# 
+# To the extent that a federal employee is an author of a portion of
+# this software or a derivative work thereof, no copyright is claimed by
+# the United States Government, as represented by the Secretary of the
+# Navy ("GOVERNMENT") under Title 17, U.S. Code. All Other Rights 
+# Reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the names of the copyright owners nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# GOVERNMENT ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" CONDITION
+# AND DISCLAIMS ANY LIABILITY OF ANY KIND FOR ANY DAMAGES WHATSOEVER
+# RESULTING FROM THE USE OF THIS SOFTWARE.
+
+# adapted from https://github.com/shadow/shadow/blob/master/cmake/LLVMTools.cmake
+
+set(BC_INLINE_SRCS boxing.cpp dict.cpp link_forcer.cpp list.cpp tuple.cpp xrange.cpp)
+include_directories(${CMAKE_SOURCE_DIR}/src)
+
+# set CXX flags
+set(BC_CXX_FLAGS "")
+if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+  separate_arguments(BC_CXX_FLAGS UNIX_COMMAND "${CMAKE_CXX_FLAGS} ${CLANG_FLAGS} ${CMAKE_CXX_FLAGS_DEBUG}")
+else()
+  separate_arguments(BC_CXX_FLAGS UNIX_COMMAND "${CMAKE_CXX_FLAGS} ${CLANG_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE}")
+  set(OPT_OPTIONS "-strip-debug")
+  set(STDLIB_SUFFIX ".release")
+endif()
+
+# set defines
+set(BC_DEFINES "")
+get_directory_property(COMPILE_DEFINITIONS COMPILE_DEFINITIONS)
+foreach(DEFINITION ${COMPILE_DEFINITIONS})
+  list(APPEND BC_DEFINES -D${DEFINITION})
+endforeach(DEFINITION)
+
+# set includes
+set(BC_INCLUDES "")
+get_directory_property(INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES)
+foreach(DIRECTORY ${INCLUDE_DIRECTORIES})
+  list(APPEND BC_INCLUDES -I${DIRECTORY})
+endforeach(DIRECTORY)
+
+set(BC_FILES "")
+set(BC_OBJS "")
+foreach(BC_SRC ${BC_INLINE_SRCS})
+  get_filename_component(BC_SRC_BASENAME ${BC_SRC} NAME)
+  get_filename_component(BC_SRC_FULLPATH ${BC_SRC} ABSOLUTE)
+
+  # generate the bitcode for this file
+  add_custom_command(OUTPUT ${BC_SRC_BASENAME}.bc
+                     COMMAND ${LLVM_TOOLS_BINARY_DIR}/clang++ ${BC_DEFINES} ${BC_CXX_FLAGS} ${BC_INCLUDES} -c ${BC_SRC_FULLPATH} -o ${BC_SRC_BASENAME}.bc -emit-llvm
+                     DEPENDS ${BC_SRC_FULLPATH} clang
+                     COMMENT "Building LLVM bitcode ${BC_SRC_BASENAME}.bc"
+                     VERBATIM)
+  set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${BC_SRC_BASENAME}.bc)
+
+  # publicize the bitcode file
+  add_custom_command(OUTPUT ${BC_SRC_BASENAME}.pub.bc
+                     COMMAND ${CMAKE_BINARY_DIR}/tools/publicize ${BC_SRC_BASENAME}.bc -o ${BC_SRC_BASENAME}.pub.bc
+                     DEPENDS ${BC_SRC_BASENAME}.bc publicize
+                     COMMENT "Publicizing LLVM bitcode ${BC_SRC_BASENAME}.pub.bc"
+                     VERBATIM)
+  set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${BC_SRC_BASENAME}.pub.bc)
+  set_source_files_properties(${BC_SRC_BASENAME}.pub.bc PROPERTIES LANGUAGE CXX)
+
+  # compile into a regular object file
+  add_custom_command(OUTPUT ${BC_SRC_BASENAME}.o
+                     COMMAND ${LLVM_TOOLS_BINARY_DIR}/clang++ ${BC_DEFINES} ${BC_CXX_FLAGS} ${BC_INCLUDES} -c ${BC_SRC_BASENAME}.pub.bc -o ${BC_SRC_BASENAME}.o
+                     DEPENDS ${BC_SRC_BASENAME}.pub.bc clang
+                     COMMENT "Compiling ${BC_SRC_BASENAME}.pub.bc -> ${BC_SRC_BASENAME}.o"
+                     VERBATIM)
+  set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${BC_SRC_BASENAME}.o)
+  set_source_files_properties(${BC_SRC_BASENAME}.o PROPERTIES EXTERNAL_OBJECT true GENERATED true)
+
+  # keep track of every bitcode and object file we created
+  list(APPEND BC_FILES ${BC_SRC_BASENAME}.pub.bc)
+  list(APPEND BC_OBJS ${BC_SRC_BASENAME}.o)
+endforeach(BC_SRC)
+
+# link all the bitcode files together
+add_custom_command(OUTPUT stdlib.unopt.bc
+                   COMMAND ${LLVM_TOOLS_BINARY_DIR}/llvm-link -o ${CMAKE_CURRENT_BINARY_DIR}/stdlib.unopt.bc ${BC_FILES}
+                   DEPENDS ${BC_FILES} llvm-link
+                   COMMENT "Linking stdlib.unopt.bc")
+
+# optimize stdlib
+add_custom_command(OUTPUT stdlib${STDLIB_SUFFIX}.bc
+                   COMMAND ${LLVM_TOOLS_BINARY_DIR}/opt -O3 ${OPT_OPTIONS} stdlib.unopt.bc -o stdlib${STDLIB_SUFFIX}.bc
+                   DEPENDS stdlib.unopt.bc opt
+                   COMMENT "Optimizing stdlib.unopt.bc -> stdlib.bc")
+
+add_custom_command(OUTPUT stdlib.bc.o
+                   COMMAND ${CMAKE_LINKER} -r -b binary stdlib${STDLIB_SUFFIX}.bc -o ${CMAKE_CURRENT_BINARY_DIR}/stdlib.bc.o
+                   DEPENDS stdlib${STDLIB_SUFFIX}.bc
+                   COMMENT "Embedding stdlib${STDLIB_SUFFIX}.bc -> stdlib.bc.o")
+
+if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+  add_custom_command(OUTPUT stdlib.stripped.bc
+                     COMMAND ${LLVM_TOOLS_BINARY_DIR}/opt -strip-debug stdlib.bc -o ${CMAKE_CURRENT_BINARY_DIR}/stdlib.stripped.bc
+                     DEPENDS stdlib.bc opt
+                     COMMENT "Stripping stdlib.bc -> stdlib.stripped.bc")
+
+  add_custom_command(OUTPUT stdlib.stripped.bc.o
+                     COMMAND ${CMAKE_LINKER} -r -b binary stdlib.stripped.bc -o ${CMAKE_CURRENT_BINARY_DIR}/stdlib.stripped.bc.o
+                     DEPENDS stdlib.stripped.bc
+                     COMMENT "Embedding stdlib.stripped.bc -> stdlib.stripped.bc.o")
+  add_library(stdlib stdlib.stripped.bc.o stdlib.bc.o ${BC_OBJS})
+else()
+  add_library(stdlib stdlib.bc.o ${BC_OBJS})
+endif()
+set_property(TARGET stdlib PROPERTY LINKER_LANGUAGE C)
+set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "stdlib.unopt.bc;stdlib.bc;stdlib.bc.o;stdlib.stripped.bc;stdlib.stripped.bc.o;stdlib.release.bc")
+

test/test_extension/CMakeLists.txt

+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers")
+set(CMAKE_SHARED_LIBRARY_PREFIX "")
+set(CMAKE_SHARED_LIBRARY_SUFFIX ".pyston${CMAKE_SHARED_LIBRARY_SUFFIX}")
+
+add_custom_command(OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/build/lib.linux-x86_64-2.7/basic_test.so
+                   COMMAND python setup.py build
+                   DEPENDS basic_test.c descr_test.c
+                   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+
+add_library(basic_test SHARED basic_test.c)
+add_library(descr_test SHARED descr_test.c)
+
+add_custom_target(ext_cpython DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/build/lib.linux-x86_64-2.7/basic_test.so)
+add_custom_target(ext_pyston DEPENDS basic_test descr_test)

test/unittests/CMakeLists.txt

+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-undef")
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+
+add_definitions(-DGTEST_HAS_RTTI=0 ${LLVM_DEFINITIONS})
+include_directories(${CMAKE_SOURCE_DIR}/src)
+include_directories(${LLVM_SOURCE_DIR}/utils/unittest/googletest/include)
+
+add_custom_target(unittests)
+
+macro(add_unittest unittest)
+  add_executable(${unittest}_unittest EXCLUDE_FROM_ALL ${unittest}.cpp $<TARGET_OBJECTS:PYSTON_OBJECTS> $<TARGET_OBJECTS:FROM_CPYTHON_OBJECTS>)
+  target_link_libraries(${unittest}_unittest stdlib gmp readline pypa double-conversion unwind gtest gtest_main ${LLVM_LIBS} ${LIBLZMA_LIBRARIES})
+  add_dependencies(${unittest}_unittest gtest gtest_main)
+  add_dependencies(unittests ${unittest}_unittest)
+endmacro()
+
+add_unittest(gc)
+add_unittest(analysis)
+add_custom_command(TARGET analysis_unittest POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/test/unittests/analysis_listcomp.py ${CMAKE_BINARY_DIR}/test/unittests/analysis_listcomp.py)

test/unittests/analysis.cpp

@@ -22,7 +22,7 @@ protected:
 };
 
 TEST_F(AnalysisTest, augassign) {
-    const std::string fn("../test/unittests/analysis_listcomp.py");
+    const std::string fn("test/unittests/analysis_listcomp.py");
     AST_Module* module = caching_parse(fn.c_str());
     assert(module);
 

tools/CMakeLists.txt

+add_executable(demangle EXCLUDE_FROM_ALL demangle.cpp)
+add_executable(mcjitcache EXCLUDE_FROM_ALL mcjitcache.cpp)
+
+llvm_map_components_to_libnames(LLVM_LIBS_EXTRA BitWriter)
+
+add_executable(publicize publicize.cpp)
+target_link_libraries(publicize curses dl pthread ${LLVM_LIBS} ${LLVM_LIBS_EXTRA}) 
+add_dependencies(publicize ${LLVM_LIBS_EXTRA})
+
+include_directories(${CMAKE_SOURCE_DIR}/src)
+add_executable(astprint EXCLUDE_FROM_ALL astprint.cpp $<TARGET_OBJECTS:PYSTON_OBJECTS> $<TARGET_OBJECTS:FROM_CPYTHON_OBJECTS>)
+target_link_libraries(astprint stdlib gmp readline pypa double-conversion unwind ${LLVM_LIBS} ${LIBLZMA_LIBRARIES})
+  

tools/tester.py

@@ -69,7 +69,7 @@ def get_expected_output(fn):
 
     # TODO don't suppress warnings globally:
     env = dict(os.environ)
-    env["PYTHONPATH"] = "../test/test_extension/build/lib.linux-x86_64-2.7/"
+    env["PYTHONPATH"] = os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/build/lib.linux-x86_64-2.7/"
     p = subprocess.Popen(["python", "-Wignore", fn], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open("/dev/null"), preexec_fn=set_ulimits, env=env)
     out, err = p.communicate()
     code = p.wait()