Initial transpiler PoC. Supports most usual Python structures, and a basic...

Initial transpiler PoC. Supports most usual Python structures, and a basic builtins module is provided.

.clang-format

+---
+Language: Cpp
+BasedOnStyle: LLVM
+IndentWidth: 2
+UseTab: Never
+---
\ No newline at end of file

.clang-tidy

+Checks: '-google-explicit-constructor -misc-no-recursion'
\ No newline at end of file

.gitignore

+.idea
\ No newline at end of file

rt/include/python/builtins.hpp

+//
+// Created by Tom on 07/03/2023.
+//
+
+#ifndef TYPON_BUILTINS_HPP
+#define TYPON_BUILTINS_HPP
+
+#include <string>
+#include <vector>
+#include <iostream>
+#include <ostream>
+#include <unordered_set>
+#include <optional>
+#include <sstream>
+#include <unordered_map>
+
+using namespace std::literals;
+
+template<typename T>
+concept PyIterator = requires(T t) {
+    { t.py_next() } -> std::same_as<std::optional<T>>;
+};
+
+template<typename T>
+concept PyIterable = requires(T t) {
+    { t.py_iter() } -> PyIterator;
+};
+
+template<PyIterable T, PyIterator U>
+U iter(const T &t) {
+    return t.py_iter();
+}
+
+template<typename T>
+concept PyLen = requires(const T &t) {
+    { t.py_len() } -> std::same_as<size_t>;
+};
+
+template<PyLen T>
+size_t len(const T &t) {
+    return t.py_len();
+}
+
+template<typename T>
+void print_to(const T &x, std::ostream &s) {
+    s << x;
+}
+
+template<typename T>
+concept Boolean = std::same_as<T, bool>;
+
+template<typename T>
+concept Printable = requires(const T &x, std::ostream &s) {
+    { x.py_print(s) } -> std::same_as<void>;
+};
+
+template<Printable T>
+void print_to(const T &x, std::ostream &s) {
+    x.py_print(s);
+}
+
+template<Boolean T>
+void print_to(const T &x, std::ostream &s) {
+    s << (x ? "True" : "False");
+}
+
+template<typename T>
+std::string str(const T &x) {
+    std::stringstream s;
+    print_to(x, s);
+    return s.str();
+}
+
+template<typename T>
+class PyList : public std::vector<T> {
+public:
+    PyList(std::vector<T> &&v) : std::vector<T>(std::move(v)) {}
+
+    PyList(std::initializer_list<T> &&v) : std::vector<T>(std::move(v)) {}
+
+    operator std::vector<T>() const {
+        return std::vector<T>(this->begin(), this->end());
+    }
+
+    operator std::vector<T> &() {
+        return *reinterpret_cast<std::vector<T> *>(this);
+    }
+
+    size_t py_len() const {
+        return this->size();
+    }
+
+    void py_print(std::ostream &s) const {
+        s << '[';
+        if (this->size() > 0) {
+            print_to(this->operator[](0), s);
+            for (size_t i = 1; i < this->size(); i++) {
+                s << ", ";
+                print_to(this->operator[](i), s);
+            }
+        }
+        s << ']';
+    }
+
+    PyList<T> operator+(const PyList<T> &other) const {
+        std::vector<T> v;
+        v.reserve(this->size() + other.size());
+        v.insert(v.end(), this->begin(), this->end());
+        v.insert(v.end(), other.begin(), other.end());
+        return PyList<T>(std::move(v));
+    }
+
+    PyList<T> operator*(size_t n) const {
+        std::vector<T> v;
+        v.reserve(this->size() * n);
+        for (size_t i = 0; i < n; i++) {
+            v.insert(v.end(), this->begin(), this->end());
+        }
+        return PyList<T>(std::move(v));
+    }
+};
+
+
+template<typename T>
+class PySet : public std::unordered_set<T> {
+public:
+    PySet(std::unordered_set<T> &&s) : std::unordered_set<T>(std::move(s)) {}
+
+    PySet(std::initializer_list<T> &&s) : std::unordered_set<T>(std::move(s)) {}
+
+    operator std::unordered_set<T>() const {
+        return std::unordered_set<T>(this->begin(), this->end());
+    }
+
+    size_t py_len() const {
+        return this->size();
+    }
+
+    bool py_contains(const T &t) const {
+        return this->find(t) != this->end();
+    }
+
+    class iterator {
+    public:
+        using value_type = T;
+        using difference_type = std::ptrdiff_t;
+        using pointer = T *;
+        using reference = T &;
+        using iterator_category = std::forward_iterator_tag;
+
+        iterator(typename std::unordered_set<T>::iterator it) : _it(it) {}
+
+        iterator &operator++() {
+            _it++;
+            return *this;
+        }
+
+        iterator operator++(int) {
+            iterator tmp = *this;
+            _it++;
+            return tmp;
+        }
+
+        bool operator==(const iterator &rhs) const {
+            return _it == rhs._it;
+        }
+
+        bool operator!=(const iterator &rhs) const {
+            return _it != rhs._it;
+        }
+
+        const T &operator*() const {
+            return *_it;
+        }
+
+        const T *operator->() const {
+            return &*_it;
+        }
+
+    private:
+        typename std::unordered_set<T>::iterator _it;
+    };
+
+    iterator py_iter() const {
+        return this->begin();
+    }
+
+    void py_print(std::ostream &s) const {
+        s << '{';
+        if (this->size() > 0) {
+            print_to(*this->begin(), s);
+            for (auto it = ++this->begin(); it != this->end(); it++) {
+                s << ", ";
+                print_to(*it, s);
+            }
+        }
+        s << '}';
+    }
+};
+
+template<typename K, typename V>
+class PyDict : public std::unordered_map<K, V> {
+public:
+    PyDict(std::unordered_map<K, V> &&m) : std::unordered_map<K, V>(std::move(m)) {}
+
+    PyDict(std::initializer_list<std::pair<const K, V>> m) : std::unordered_map<K, V>(m) {}
+
+    operator std::unordered_map<K, V>() const {
+        return std::unordered_map<K, V>(this->begin(), this->end());
+    }
+
+    operator std::unordered_map<K, V> &() {
+        return *reinterpret_cast<std::unordered_map<K, V> *>(this);
+    }
+
+    size_t py_len() const {
+        return this->size();
+    }
+
+    bool py_contains(const K &k) const {
+        return this->find(k) != this->end();
+    }
+
+    class iterator {
+    public:
+        using value_type = std::pair<K, V>;
+        using difference_type = std::ptrdiff_t;
+        using pointer = std::pair<K, V> *;
+        using reference = std::pair<K, V> &;
+        using iterator_category = std::forward_iterator_tag;
+
+        iterator(typename std::unordered_map<K, V>::iterator it) : _it(it) {}
+
+        iterator &operator++() {
+            _it++;
+            return *this;
+        }
+
+        iterator operator++(int) {
+            iterator tmp = *this;
+            _it++;
+            return tmp;
+        }
+
+        bool operator==(const iterator &rhs) const {
+            return _it == rhs._it;
+        }
+
+        bool operator!=(const iterator &rhs) const {
+            return _it != rhs._it;
+        }
+
+        const std::pair<K, V> &operator*() const {
+            return *_it;
+        }
+
+        const std::pair<K, V> *operator->() const {
+            return &*_it;
+        }
+
+    private:
+        typename std::unordered_map<K, V>::iterator _it;
+    };
+
+    iterator py_iter() const {
+        return this->begin();
+    }
+
+    void py_print(std::ostream &s) const {
+        s << '{';
+        if (this->size() > 0) {
+            print_to(this->begin()->first, s);
+            s << ": ";
+            print_to(this->begin()->second, s);
+            for (auto it = ++this->begin(); it != this->end(); it++) {
+                s << ", ";
+                print_to(it->first, s);
+                s << ": ";
+                print_to(it->second, s);
+            }
+        }
+        s << '}';
+    }
+};
+
+template<typename K, typename V>
+PyDict(std::initializer_list<std::pair<K, V>>) -> PyDict<K, V>;
+
+void print() {
+    std::cout << '\n';
+}
+
+template<typename T, typename ... Args>
+void print(T const &head, Args const &... args) {
+    print_to(head, std::cout);
+    (((std::cout << ' '), print_to(args, std::cout)), ...);
+    std::cout << '\n';
+}
+
+bool is_cpp() {
+    return true;
+}
+
+#endif //TYPON_BUILTINS_HPP

trans/.gitignore

+__pycache__
\ No newline at end of file

trans/main.py

+print("Main")
\ No newline at end of file

trans/requirements.txt

 clang-format==15.0.7

trans/test_runner.py

+# coding: utf-8
+from os import system
+from pathlib import Path
+
+from transpiler import transpile
+from transpiler.format import format_code
+
+# print(format_code("int x = 2 + ((3 * 5));;;"))
+
+def run_tests():
+    for path in Path('tests').glob('*.py'):
+        print(path.name)
+        with open(path, "r", encoding="utf-8") as f:
+            res = format_code(transpile(f.read()))
+        name_cpp = path.with_suffix('.cpp')
+        with open(name_cpp, "w", encoding="utf-8") as fcpp:
+            fcpp.write(res)
+        name_bin = path.with_suffix('').as_posix()
+        cmd = f"bash -c 'g++ -I../rt/include -std=c++20 -o {name_bin} {name_cpp.as_posix()}'"
+        commands = [
+            cmd,
+            f"bash -c 'PYTHONPATH=. python3 ./{path.as_posix()}'",
+            f"bash -c './{name_bin}'",
+            f"scp ./{name_bin} tom@192.168.139.128:/tmp/{name_bin}",  # TODO: temporary test suite. Will fix.
+            f"bash -c 'ssh tom@192.168.139.128 \"chmod +x /tmp/{name_bin} && /tmp/{name_bin}\"'"
+        ]
+        for cmd in commands:
+            if system(cmd) != 0:
+                print(f"Error running command: {cmd}")
+                break
+
+
+if __name__ == "__main__":
+    run_tests()

trans/tests/.gitignore

+*
+!/**/
+!*.*
+*.cpp
\ No newline at end of file

trans/tests/builtins_test.py

+# coding: utf-8
+
+from typon import is_cpp
+
+if __name__ == "__main__":
+    # todo: 0x55 & 7 == 5
+    print("C++   " if is_cpp() else "Python",
+          "res=", 5, ".", True, [4, 5, 6], {7, 8, 9}, [1, 2] + [3, 4], [5, 6] * 3, {1: 7, 9: 3}, 0x55 & 7 == 5)
+    print()
\ No newline at end of file

trans/tests/naive_fibonacci_fork.py

+from typon import fork, sync
+
+
+#def fibo(n: int) -> int:
+#    if n < 2:
+#        return n
+#    a = fork(lambda: fibo(n - 1))
+#    b = fork(lambda: fibo(n - 2))
+#    sync()
+#    return a + b
+
+
+if __name__ == "__main__":
+    print("res=", 5, ".")
\ No newline at end of file

trans/transpiler/__init__.py

+# coding: utf-8
+import ast
+from itertools import chain, zip_longest
+from typing import *
+
+
+def compare_ast(node1: Union[ast.expr, list[ast.expr]], node2: Union[ast.expr, list[ast.expr]]) -> bool:
+    if type(node1) is not type(node2):
+        return False
+
+    if isinstance(node1, ast.AST):
+        for k, v in vars(node1).items():
+            if k in {"lineno", "end_lineno", "col_offset", "end_col_offset", "ctx"}:
+                continue
+            if not compare_ast(v, getattr(node2, k)):
+                return False
+        return True
+
+    elif isinstance(node1, list) and isinstance(node2, list):
+        return all(compare_ast(n1, n2) for n1, n2 in zip_longest(node1, node2))
+    else:
+        return node1 == node2
+
+
+def flatmap(f, items):
+    return chain.from_iterable(map(f, items))
+
+
+def join(sep: str, items: Iterable[Iterable[str]]) -> Iterable[str]:
+    items = iter(items)
+    try:
+        yield from next(items)
+        for item in items:
+            yield sep
+            yield from item
+    except StopIteration:
+        return
+
+
+def transpile(source):
+    tree = ast.parse(source)
+    # print(ast.unparse(tree))
+    return "\n".join(filter(None, map(str, TyponVisitor().visit(tree))))
+
+
+SYMBOLS = {
+    ast.Eq: "==",
+    ast.NotEq: '!=',
+    ast.Pass: '/* pass */',
+    ast.Mult: '*',
+    ast.Add: '+',
+    ast.Sub: '-',
+    ast.Div: '/',
+    ast.FloorDiv: '/',
+    ast.Mod: '%',
+    ast.Lt: '<',
+    ast.Gt: '>',
+    ast.GtE: '>=',
+    ast.LtE: '<=',
+    ast.LShift: '<<',
+    ast.RShift: '>>',
+    ast.BitXor: '^',
+    ast.BitOr: '|',
+    ast.BitAnd: '&',
+    ast.Not: '!',
+    ast.IsNot: '!=',
+    ast.USub: '-',
+    ast.And: '&&',
+    ast.Or: '||'
+}
+"""Mapping of Python AST nodes to C++ symbols."""
+
+PRECEDENCE = [
+    ("*", "/", "%",),
+    ("+", "-"),
+    ("<<", ">>"),
+    ("<", "<=", ">", ">="),
+    ("==", "!="),
+    ("&",),
+    ("^",),
+    ("|",),
+    ("&&",),
+    ("||",),
+]
+"""Precedence of C++ operators."""
+
+PRECEDENCE_LEVELS = {op: i for i, ops in enumerate(PRECEDENCE) for op in ops}
+"""Mapping of C++ operators to their precedence level."""
+
+MAPPINGS = {
+    "True": "true",
+    "False": "false",
+    "None": "nullptr"
+}
+"""Mapping of Python builtin constants to C++ equivalents."""
+
+
+class NodeVisitor:
+    def visit(self, node):
+        """Visit a node."""
+        if type(node) in SYMBOLS:
+            yield SYMBOLS[type(node)]
+        else:
+            for parent in node.__class__.__mro__:
+                if visitor := getattr(self, 'visit_' + parent.__name__, None):
+                    yield from visitor(node)
+                    break
+            else:
+                raise NotImplementedError(node.__class__.__mro__, ast.dump(node))
+
+    def process_args(self, node: ast.arguments) -> (str, str):
+        for field in ("posonlyargs", "vararg", "kwonlyargs", "kw_defaults", "kwarg", "defaults"):
+            if getattr(node, field, None):
+                raise NotImplementedError(node, field)
+
+        if not node.args:
+            return "", "()"
+        f_args = [(arg, f"T{i + 1}") for i, arg in enumerate(node.args)]
+        return (
+            "<" + ", ".join(f"typename {t}" for _, t in f_args) + ">",
+            "(" + ", ".join(f"{t} {next(self.visit(n))}" for n, t in f_args) + ")"
+        )
+
+    def visit_arg(self, node: ast.arg) -> Iterable[str]:
+        # TODO: identifiers
+        yield node.arg
+
+
+class ExpressionVisitor(NodeVisitor):
+    def __init__(self, precedence: int = 0):
+        self._precedence = precedence
+
+    def visit_Tuple(self, node: ast.Tuple) -> Iterable[str]:
+        yield f"std::make_tuple({', '.join(flatmap(self.visit, node.elts))})"
+
+    def visit_Constant(self, node: ast.Constant) -> Iterable[str]:
+        if isinstance(node.value, str):
+            # TODO: escape sequences
+            yield f"\"{repr(node.value)[1:-1]}\"s"
+        elif isinstance(node.value, bool):
+            yield str(node.value).lower()
+        elif isinstance(node.value, int):
+            # TODO: bigints
+            yield str(node.value)
+        else:
+            raise NotImplementedError(node, type(node))
+
+    def visit_Name(self, node: ast.Name) -> Iterable[str]:
+        if node.id.startswith("__") and node.id.endswith("__"):
+            return f"py_{node.id[2:-2]}"
+        yield MAPPINGS.get(node.id, node.id)
+
+    def visit_Compare(self, node: ast.Compare) -> Iterable[str]:
+        # TODO: operator precedence
+        operands = [node.left, *node.comparators]
+        yield from self.visit_binary_operation(node.ops[0], operands[0], operands[1])
+        for (left, right), op in zip(zip(operands[1:], operands[2:]), node.ops[1:]):
+            # TODO: cleaner code
+            yield " && "
+            yield from self.visit_binary_operation(op, left, right)
+
+    def visit_Call(self, node: ast.Call) -> Iterable[str]:
+        if getattr(node, "keywords", None):
+            raise NotImplementedError(node, "keywords")
+        if getattr(node, "starargs", None):
+            raise NotImplementedError(node, "varargs")
+        if getattr(node, "kwargs", None):
+            raise NotImplementedError(node, "kwargs")
+        yield from self.visit(node.func)
+        yield "("
+        yield from join(", ", map(self.visit, node.args))
+        yield ")"
+
+    def visit_Lambda(self, node: ast.Lambda) -> Iterable[str]:
+        yield "[]"
+        templ, args = self.process_args(node.args)
+        yield templ
+        yield args
+        yield "{"
+        yield "return"
+        yield from self.visit(node.body)
+        yield ";"
+        yield "}"
+
+    def visit_BinOp(self, node: ast.BinOp) -> Iterable[str]:
+        yield from self.visit_binary_operation(node.op, node.left, node.right)
+
+    def visit_binary_operation(self, op, left: ast.AST, right: ast.AST) -> Iterable[str]:
+        # TODO: precedence
+        op = SYMBOLS[type(op)]
+        inner = ExpressionVisitor(PRECEDENCE_LEVELS[op])
+        prio = inner._precedence > self._precedence != 0
+        if prio:
+            yield "("
+        yield from inner.visit(left)
+        yield op
+        yield from inner.visit(right)
+        if prio:
+            yield ")"
+
+    def visit_Attribute(self, node: ast.Attribute) -> Iterable[str]:
+        yield from self.visit(node.value)
+        yield "."
+        yield node.attr
+
+    def visit_List(self, node: ast.List) -> Iterable[str]:
+        yield "PyList{"
+        yield from join(", ", map(self.visit, node.elts))
+        yield "}"
+
+    def visit_Set(self, node: ast.Set) -> Iterable[str]:
+        yield "PySet{"
+        yield from join(", ", map(self.visit, node.elts))
+        yield "}"
+
+    def visit_Dict(self, node: ast.Dict) -> Iterable[str]:
+        def visit_item(key, value):
+            yield "std::pair {"
+            yield from self.visit(key)
+            yield ", "
+            yield from self.visit(value)
+            yield "}"
+
+        yield "PyDict{"
+        yield from join(", ", map(visit_item, node.keys, node.values))
+        yield "}"
+
+    def visit_Subscript(self, node: ast.Subscript) -> Iterable[str]:
+        yield from self.visit(node.value)
+        yield "["
+        yield from self.visit(node.slice)
+        yield "]"
+
+    def visit_UnaryOp(self, node: ast.UnaryOp) -> Iterable[str]:
+        yield from self.visit(node.op)
+        yield from self.visit(node.operand)
+
+    def visit_IfExp(self, node: ast.IfExp) -> Iterable[str]:
+        yield from self.visit(node.test)
+        yield " ? "
+        yield from self.visit(node.body)
+        yield " : "
+        yield from self.visit(node.orelse)
+
+
+class TyponVisitor(NodeVisitor):
+    def visit_Module(self, node: ast.Module) -> Iterable[str]:
+        stmt: ast.AST
+        yield "#include <python/builtins.hpp>"
+        for stmt in node.body:
+            yield from self.visit(stmt)
+
+    def visit_Expr(self, node: ast.Expr) -> Iterable[str]:
+        yield from ExpressionVisitor().visit(node.value)
+        yield ";"
+
+    def visit_Import(self, node: ast.Import) -> Iterable[str]:
+        for name in node.names:
+            if name == "typon":
+                yield "// typon import"
+            else:
+                raise NotImplementedError(node)
+
+    def visit_ImportFrom(self, node: ast.ImportFrom) -> Iterable[str]:
+        if node.module == "typon":
+            yield "// typon import"
+        else:
+            raise NotImplementedError(node)
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> Iterable[str]:
+        templ, args = self.process_args(node.args)
+        if templ:
+            yield "template"
+            yield templ
+        yield f"auto {node.name}"
+        yield args
+        yield "{"
+        for child in node.body:
+            yield from self.visit(child)
+        yield "}"
+
+    def visit_If(self, node: ast.If) -> Iterable[str]:
+        if not node.orelse and compare_ast(node.test, ast.parse('__name__ == "__main__"', mode="eval").body):
+            yield "int main() {"
+            for child in node.body:
+                yield from self.visit(child)
+            yield "}"
+            return
+
+        yield "if ("
+        yield from ExpressionVisitor().visit(node.test)
+        yield ") {"
+        for child in node.body:
+            yield from self.visit(child)
+        yield "}"
+        if node.orelse:
+            yield "else "
+            if isinstance(node.orelse, ast.If):
+                yield from self.visit(node.orelse)
+            else:
+                yield "{"
+                for child in node.orelse:
+                    yield from self.visit(child)
+                yield "}"
+
+    def visit_Return(self, node: ast.Return) -> Iterable[str]:
+        yield "return "
+        if node.value:
+            yield from ExpressionVisitor().visit(node.value)
+        yield ";"
+
+    def visit_While(self, node: ast.While) -> Iterable[str]:
+        yield "while ("
+        yield from ExpressionVisitor().visit(node.test)
+        yield ") {"
+        for child in node.body:
+            yield from self.visit(child)
+        yield "}"
+        if node.orelse:
+            raise NotImplementedError(node, "orelse")
+
+    def visit_lvalue(self, lvalue: ast.expr) -> Iterable[str]:
+        if isinstance(lvalue, ast.Tuple):
+            yield f"std::tie({', '.join(flatmap(ExpressionVisitor().visit, lvalue.elts))})"
+        elif isinstance(lvalue, (ast.Name, ast.Subscript)):
+            yield from ExpressionVisitor().visit(lvalue)
+        else:
+            raise NotImplementedError(lvalue)
+
+    def visit_Assign(self, node: ast.Assign) -> Iterable[str]:
+        if len(node.targets) != 1:
+            raise NotImplementedError(node)
+        yield from self.visit_lvalue(node.targets[0])
+        yield " = "
+        yield from ExpressionVisitor().visit(node.value)
+        yield ";"
+
+    def visit_For(self, node: ast.For) -> Iterable[str]:
+        if not isinstance(node.target, ast.Name):
+            raise NotImplementedError(node)
+        yield f"for (auto {node.target.id} : "
+        yield from ExpressionVisitor().visit(node.iter)
+        yield ") {"
+        for child in node.body:
+            yield from self.visit(child)
+        yield "}"
+        if node.orelse:
+            raise NotImplementedError(node, "orelse")

trans/transpiler/format.py

+# coding: utf-8
+import subprocess
+
+import clang_format
+
+clang = clang_format._get_executable("clang-format")  # noqa
+
+
+def format_code(code: str) -> str:
+    return subprocess.check_output([clang, "-style=LLVM"], input=code.encode("utf-8")).decode("utf-8")

trans/typon/__init__.py

+from typing import Callable, TypeVar
+
+T = TypeVar("T")
+
+
+def fork(_f: Callable[[], T]) -> T:
+    # stub
+    pass
+
+
+def sync():
+    # stub
+    pass
+
+
+def is_cpp():
+    return False