Add doc

docs/.gitignore

+/build
\ No newline at end of file

docs/Makefile

+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/source/conf.py

+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'Typon'
+copyright = '2023, Nexedi'
+author = 'Nexedi'
+release = '0.1'
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ['myst_parser']
+source_suffix = ['.rst', '.md']
+
+templates_path = ['_templates']
+exclude_patterns = []
+
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = 'furo'
+html_title = 'Typon'
+
+html_static_path = ['_static']
+
+html_theme_options = {
+    "footer_icons": [
+        {
+            "name": "GitLab",
+            "url": "https://lab.nexedi.com/zdimension/typon",
+            "html": """
+                <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 16 16">
+  <g id="LOGO" transform="matrix(0.083313003182, 0, 0, 0.083313003182, 183.289983120911, -181.990001822971)" style="transform-origin: -208.489px 189.99px;">
+    <path d="m 282.83,170.73 -0.27,-0.69 -26.14,-68.22 a 6.81,6.81 0 0 0 -2.69,-3.24 7,7 0 0 0 -8,0.43 7,7 0 0 0 -2.32,3.52 l -17.65,54 h -71.47 l -17.65,-54 a 6.86,6.86 0 0 0 -2.32,-3.53 7,7 0 0 0 -8,-0.43 6.87,6.87 0 0 0 -2.69,3.24 L 97.44,170 l -0.26,0.69 a 48.54,48.54 0 0 0 16.1,56.1 l 0.09,0.07 0.24,0.17 39.82,29.82 19.7,14.91 12,9.06 a 8.07,8.07 0 0 0 9.76,0 l 12,-9.06 19.7,-14.91 40.06,-30 0.1,-0.08 a 48.56,48.56 0 0 0 16.08,-56.04 z" />
+  </g>
+</svg>
+            """,
+            "class": "",
+        },
+    ]
+}
\ No newline at end of file

docs/source/gettingstarted/cli.md

+# Command-line usage
+
+Typon can be used from the command-line to compile a Python file:
+
+```shell
+typon [-o/--output output] [-d/--debug] [-v/--verbose] input
+```
+
+Once generated, the C++ code file can be compiled using your compiler of choice:
+
+```shell
+$(CXX) -O3 $(typon --cpp-flags) input.cpp
+```
+
+The Typon runtime is header-only, so no additional linking is required.
\ No newline at end of file

docs/source/index.md

+---
+hide-toc: true
+---
+
+# The Typon compiler
+
+```{toctree} 
+:hidden:
+:caption: Getting started
+
+gettingstarted/cli
+```
+
+```{toctree} 
+:hidden:
+:caption: Language
+
+language/differences
+language/typesystem
+language/concurrency
+```
+
+```{toctree} 
+:hidden:
+:caption: Interoperability
+
+interoperability/cpython
+```
+
+## What's this?
+
+Typon is a work-in-progress **Python-to-C++ compiler**.
+
+Untyped, idiomatic Python code can be compiled to C++, which can then be compiled to native code using a C++ compiler.
+
+## Supported features
+
+Typon's goal is to support as big a subset of Python as possible while still being able to process *idiomatic* Python
+code. This means:
+
+- **type inference and type checking**: a successful Typon build means no type errors in the C++ code
+- **support for nested functions and closures**
+- **automatic memory management**: objects are garbage-collected using reference counting
+- **interoperability with native Python**: Typon can call Python functions and use Python objects (CPython will then be
+  used, only for that use)
+- **first-class types and functions**: Typon supports passing and storing types and functions as values
+
+## What's new?
+
+You may be thinking "why yet another Python compiler?". Typon was born out of the observation that none of the existing
+Python compilers focused on concurrency and asynchronous programming.
+
+## Comparison
+
+Typon is not the first Python compiler, many other projects have similar goals. Here is a comparison with some of them:
+
+|                               | Cython              | Nuitka                                       | Codon        | PyPy   | Shed Skin    | Typon                           |
+|-------------------------------|---------------------|----------------------------------------------|--------------|--------|--------------|---------------------------------|
+| Status                        | Mature              | Mature                                       | Mature       | Mature | Experimental | Experimental                    |
+| Platform support              | Cross               | Cross                                        | Cross + WASM | Cross  | ?            | Linux only (relies on io-uring) |
+| Native codegen                | AOT                 | n/a (AOT + Python)                           | AOT          | JIT    | AOT          | AOT                             |
+| Required type annotations     | Function signatures | No                                           | ?            | No     | No           | Class fields                    |
+| Type inference                | ? (unclear)         |                                              |              | n/a    | One-way      | Bidirectional (Hindley-Milner)  |
+| Native execution              | ✓                   | Partial -- calls CPython API for many things | ✓            |        | ✓            | ✓                               |
+| First-class types             |                     | ✓                                            |              | ✓      |              | ✓                               |
+| Closures, bound methods       | ✓                   | ✓                                            | ✓            | ✓      |              | ✓                               |
+| Interoperability with CPython | ✓                   | ✓                                            | ✓            | ✓      |              | ✓                               |
+| Async, I/O coroutines         | Not for `cdef`      | ✓                                            |              | ✓      |              | ✓                               |
+
+(non-exhaustive table, incomplete)
\ No newline at end of file

docs/source/interoperability/cpython.md

+# Importing Python modules
+
+When importing a module outside from Typon's scope, if a module matching that name exists in Python's scope, then that
+module and optionally the specified functions are hoisted to the `import` statement's scope (as in Python).
+
+```py
+from numpy import square
+import math
+
+if __name__ == "__main__":
+    x = [1, 2, 3, 4]
+    y: list[int] = square(x)
+    print(x, y)
+    f: int = math.factorial(5)
+    print("5! =", f)
+```
+
+Typon cannot currently determine the type signature of imported functions, so they must be either be annotated, or be
+used in a context where their type can be fully inferred.
+
+Typon values (C++ objects) are converted to their Python equivalents when passed to Python functions, and vice-versa. 
+At the moment, collections are passed by copy; passing by reference is not yet supported.
\ No newline at end of file

docs/source/language/concurrency.md

+# Concurrency features
+
+The C++ runtime used by generated programs relies on a custom-made continuation-stealing concurrency runtime using cutting-edge C++20 coroutines,
+featuring both `fork`/`sync` structured concurrency and `future`-based unbounded
+concurrency.
+
+## Concurrency runtime
+
+```{include} ../../../README.md
+:start-after: "### Status"
+:end-before: "## `typon/compiler`, A Python to C++ compiler"
+```
\ No newline at end of file

docs/source/language/differences.md

+# Differences with pure Python
+
+Although Typon strives to accept as much idiomatic Python as possible, there are some differences and unsupported features:
+
+- heterogeneous lists and tuples are not supported
+```py
+l = ["abc", 123, False]  # unsupported
+d = {5: "abc", True: 123}  # unsupported
+```
+
+- dynamic class manipulation (monkey-patching, `__slots__`, `__dict__`) is not supported
+- generator expressions are not supported yet
+- `int` is fixed-size
+- garbage collection is done using reference counting **→ Boehm will be used at a later point**
+- `try` blocks are not supported yet
\ No newline at end of file

docs/source/language/typesystem.md

+# Type system
+
+Typon reuses the notational conventions from [PEP 484](https://www.python.org/dev/peps/pep-0484/), and the types defined in the `typing` module.
+
+The type system as it is now is probably not sound, but this is something that will evolve in the future.
+
+## Algebraic Data Types
+
+Typon supports ADTs in the form of tuples (product types) and unions (sum types):
+
+```py
+from typing import Optional
+
+a = (1, "a", True)  # tuple[int, str, bool]
+
+def f(x: Optional[int] = None):  # alias for Union[int, NoneType]
+    pass
+    
+def g(y: str | int):  # alias for Union[str, int]
+    pass
+```
+
+## Inference
+
+Currently, a derivative of Hindley-Milner is used to infer types. A unification-based system has some disadvantages though,
+notably non-locality of errors and the difficulty of modeling inheritance, and as such this will probably be changed to
+something based on [bidirectional typing](https://arxiv.org/pdf/1908.05839.pdf).
+
+However, even with the current system, the type inference is quite powerful, and Python programs can be compiled often with no changes at all,
+with an immediate performance bump.
\ No newline at end of file

trans/tests/a_a_a_a_a_a_errtest.py

-import sys
-import math
-from typing import Callable
-
-# def f(x: Callable[[int], int]):
-#     return x(5)
-
-x = 4
-
-def f():
-    if True:
-        next = 5
-    x = 7
-
-    def g():
-        nonlocal x
-        x = 6
-
-    return 123
-
-# def h(x):
-#     y = len(x)
-#     z = x[4]
-
-
-if __name__ == "__main__":
-    #print(f(lambda n: n + 1)) # todo
-    print(f())
-
-
-    for i in range(10):
-        if i > 4:
-            break
-    else:
-        print("else")
\ No newline at end of file

trans/tests/a_a_a_a_calcbasic.py → trans/tests/calcbasic.py

@@ -79,9 +79,9 @@ def tokenize(inp: str):
         tok: Token
 
         if next in ops_syms:
-            tok = Token(TokenType.OPERATION, read(), 0)
+            tok = Token(TokenType.OPERATION, read(), 0.)
         elif next in "()":
-            tok = Token(TokenType.PARENTHESIS, read(), 0)
+            tok = Token(TokenType.PARENTHESIS, read(), 0.)
         elif next in "0123456789.":
             tok = read_number()
         else:
@@ -92,50 +92,51 @@ def tokenize(inp: str):
     return tokens
 
 
-def parse(tokens: list[Token]):
-    index = 0
+class Parser:
+    tokens: list[Token]
+    index: int
 
-    def has():
-        return index < len(tokens)
+    def __init__(self, tokens):
+        self.tokens = tokens
+        self.index = 0
+
+    def has(self):
+        return self.index < len(self.tokens)
 
-    def current():
-        if not has():
+    def current(self):
+        if not self.has():
             raise Exception("expected token, got EOL")
-        return tokens[index]
+        return self.tokens[self.index]
 
-    def match(type: TokenType, val: Optional[str] = None):
-        return has() and tokens[index].type == type and (val is None or tokens[index].val == val)
+    def match(self, type: TokenType, val: Optional[str] = None):
+        return self.has() and self.tokens[self.index].type == type and (val is None or self.tokens[self.index].val == val)
 
-    def accept(type: TokenType, val: Optional[str] = None):
-        nonlocal index
-        if match(type, val):
-            index += 1
+    def accept(self, type: TokenType, val: Optional[str] = None):
+        if self.match(type, val):
+            self.index += 1
             return True
         return False
 
-    def expect(type: TokenType, val: Optional[str] = None):
-        nonlocal index
-        if match(type, val):
-            index += 1
-            return tokens[index - 1]
-        if not has():
+    def expect(self, type: TokenType, val: Optional[str] = None):
+        if self.match(type, val):
+            self.index += 1
+            return self.tokens[self.index - 1]
+        if not self.has():
             raise Exception("expected {}, got EOL".format(type))
         else:
-            raise Exception("expected {}, got {}".format(type, current().type))
+            raise Exception("expected {}, got {}".format(type, self.current().type))
 
-    parse_term: Callable[[], None]
-
-    def parse_bin(priority=0):
+    def parse_bin(self, priority=0):
         if priority >= MAX_PRIORITY:
-            return parse_term()
+            return self.parse_term()
 
-        left = parse_bin(priority + 1)
+        left = self.parse_bin(priority + 1)
         ops = ops_by_priority[priority]
 
-        while has() and current().type == TokenType.OPERATION:
+        while self.has() and self.current().type == TokenType.OPERATION:
             for op in ops:
-                if accept(TokenType.OPERATION, op.symbol):
-                    right = parse_bin(priority + 1)
+                if self.accept(TokenType.OPERATION, op.symbol):
+                    right = self.parse_bin(priority + 1)
                     left = op.perform(left, right)
                     break
             else:
@@ -143,23 +144,23 @@ def parse(tokens: list[Token]):
 
         return left
 
-    def parse_expr():
-        return parse_bin()
+    def parse_expr(self):
+        return self.parse_bin()
 
-    def parse_term():
-        token = current()
+    def parse_term(self):
+        token = self.current()
 
         if token.type == TokenType.NUMBER:
-            return expect(TokenType.NUMBER).num
-        elif accept(TokenType.PARENTHESIS, "("):
-            val = parse_expr()
-            expect(TokenType.PARENTHESIS, ")")
+            return self.expect(TokenType.NUMBER).num
+        elif self.accept(TokenType.PARENTHESIS, "("):
+            val = self.parse_expr()
+            self.expect(TokenType.PARENTHESIS, ")")
             return val
         else:
             raise Exception("expected term, got {}".format(token.type))
 
-    return parse_expr()
-
+def parse(tokens):
+    return Parser(tokens).parse_expr()
 
 if __name__ == "__main__":
     while True:

trans/tests/a_a_check_socket_listening.py → trans/tests/check_socket_listening.py

-# norun
-# nocompile
-
-import sys
-from socket import socket, getaddrinfo, AF_UNIX, SOCK_STREAM
-
-if __name__ == "__main__":
-    s: socket
-    if len(sys.argv) == 3:
-        host = sys.argv[1]
-        port = sys.argv[2]
-        family, _, _, _, addr = getaddrinfo(host, int(port))[0]
-        s = socket(family, SOCK_STREAM)
-        s.connect(addr)
-    elif len(sys.argv) == 2:
-        path = sys.argv[1]
-        addr: str
-        if path.startswith('@'):
-            addr = '\0' + path[1:]
-        else:
-            addr = path
-        s = socket(AF_UNIX, SOCK_STREAM)
-        s.connect(addr)
-    else:
-        print("Usage:", sys.argv[0] , "host port | path | @abstract")
-        sys.exit(1)
-
+# norun
+# nocompile
+
+import sys
+from socket import socket, getaddrinfo, AF_UNIX, SOCK_STREAM
+
+if __name__ == "__main__":
+    s: socket
+    if len(sys.argv) == 3:
+        host = sys.argv[1]
+        port = sys.argv[2]
+        family, _, _, _, addr = getaddrinfo(host, int(port))[0]
+        s = socket(family, SOCK_STREAM)
+        s.connect(addr)
+    elif len(sys.argv) == 2:
+        path = sys.argv[1]
+        addr: str
+        if path.startswith('@'):
+            addr = '\0' + path[1:]
+        else:
+            addr = path
+        s = socket(AF_UNIX, SOCK_STREAM)
+        s.connect(addr)
+    else:
+        print("Usage:", sys.argv[0] , "host port | path | @abstract")
+        sys.exit(1)
+
     s.close()
\ No newline at end of file

trans/tests/gen_test.py

@@ -10,4 +10,4 @@ def fib(upto):
 if __name__ == "__main__":
     f = fib(50)
     for i in range(15):
-        print(next(f, None))
\ No newline at end of file
+        print(next(f))
\ No newline at end of file

trans/tests/a_a_a_a_usertype.py → trans/tests/usertype.py

@@ -14,7 +14,7 @@ class Person:
 
 def creer():
     return Person("jean", 123)
-# todo: https://lab.nexedi.com/xavier_thompson/typon-snippets/blob/master/dot/dot.cpp
+
 if __name__ == "__main__":
     y = Person
     x = creer()