Continue work on making the compiler more ergonomic

trans/__main__.py

+"""
+usage:
+typon --cpp-flags
+or
+typon in.py [-o out.py] [-v]
+"""
+
+import argparse
+import logging
+from pathlib import Path
+import sys
+
+compiler_path = Path(__file__).parent
+runtime_path = compiler_path.parent / "rt"
+
+sys.path.insert(0, str(compiler_path))
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("input", help="input file", nargs="?" if "--cpp-flags" in sys.argv else 1)
+parser.add_argument("-o", "--output", help="output file")
+parser.add_argument("--cpp-flags", help="print cpp flags", action="store_true")
+parser.add_argument(
+    '-d', '--debug',
+    help="Print lots of debugging statements",
+    action="store_const", dest="loglevel", const=logging.DEBUG,
+    default=logging.WARNING,
+)
+parser.add_argument(
+    '-v', '--verbose',
+    help="Be verbose",
+    action="store_const", dest="loglevel", const=logging.INFO,
+)
+
+args = parser.parse_args()
+logging.basicConfig(level=args.loglevel)
+
+if args.cpp_flags:
+    import pybind11.commands
+    import sysconfig
+
+    include_dirs = [
+        str(runtime_path / "include"),
+        sysconfig.get_path("include"),
+        sysconfig.get_path("platinclude"),
+        pybind11.commands.get_include()
+    ]
+    include_dirs = list(dict.fromkeys(include_dirs))
+    cpp_flags = [
+        *["-I" + d for d in include_dirs],
+        "-pthread", "-luring", "-lfmt", "-lssl", "-lcrypto", "-lpython3.10"
+    ]
+    print(" ".join(cpp_flags))
+    exit(0)
+
+path = Path(args.input[0])
+with open(path, "r", encoding="utf-8") as f:
+    code = f.read()
+
+from .transpiler import transpile
+from .transpiler.format import format_code
+
+raw_cpp = transpile(code, path.name, path)
+formatted = format_code(raw_cpp)
+
+output_name = args.output or path.with_suffix('.cpp')
+
+with open(output_name, "w", encoding="utf-8") as f:
+    f.write(formatted)
+
+# TODO
+"""
+webserver => investiguer
+scanfs => fork
+promesse => faire
+self/this => bind/dot
+stocker smart ptr dans closures
+"""

trans/main.py

-print("Main")
-
-# TODO
-"""
-webserver => investiguer
-scanfs => fork
-promesse => faire
-self/this => bind/dot
-stocker smart ptr dans closures
-"""
\ No newline at end of file

trans/test_runner.py

@@ -11,6 +11,10 @@ from dotenv import load_dotenv
 
 load_dotenv()
 
+# todo: promise https://lab.nexedi.com/nexedi/slapos.toolbox/blob/master/slapos/promise/plugin/check_socket_listening.py
+# todo: scan fs https://lab.nexedi.com/xavier_thompson/scan-filesystem/blob/master/rust/src/main.rs
+# todo: refs https://lab.nexedi.com/xavier_thompson/typon-snippets/tree/master/references
+
 parser = argparse.ArgumentParser()
 parser.add_argument("-c", "--compile", action="store_true")
 args = parser.parse_args()

trans/tests/a_a_a_errtest.py

 import sys
 import math
 x = 5
-x: str = "str"
+#x: str = "str"
 
+y = "ab"
 
 if __name__ == "__main__":
-    y = (6).x
\ No newline at end of file
+    pass
\ No newline at end of file

trans/tests/a_scanfs.py

 # coding: utf-8
+# norun
+from __future__ import annotations
 
 import hashlib
 import io
 import json
 import os
 import stat
-from typing import Self
 
 class StatResult:
     st_mode: int

trans/tests/webserver.py

@@ -4,6 +4,7 @@
 import sys
 from socket import socket, SOCK_STREAM, AF_INET6, SOL_SOCKET, SO_REUSEADDR
 from typon import fork
+#fork = lambda x: x()
 
 BACKLOG = 1024
 PORT = 8000

trans/transpiler/__init__.py

@@ -12,20 +12,19 @@ colorama.init()
 from transpiler.consts import MAPPINGS
 from transpiler.exceptions import CompileError
 from transpiler.phases.desugar_with import DesugarWith
-# from transpiler.phases import initial_pytype
 from transpiler.phases.emit_cpp.file import FileVisitor
 from transpiler.phases.if_main import IfMainVisitor
 from transpiler.phases.typing.block import ScoperBlockVisitor
 from transpiler.phases.typing.scope import Scope
+from transpiler.utils import highlight
 
 from itertools import islice
 import sys
 
 import colorful as cf
 
-from transpiler.utils import highlight
-
 
+from logging import debug
 
 
 def exception_hook(exc_type, exc_value, tb):
@@ -59,6 +58,11 @@ def exception_hook(exc_type, exc_value, tb):
 
     if last_node is not None:
         print()
+        if not hasattr(last_node, "lineno"):
+            print(cf.red("Error: "), cf.white("No line number available"))
+            last_node.lineno = 1
+            print(ast.unparse(last_node))
+            return
         print(f"In file \"{cf.white(last_file)}\", line {last_node.lineno}")
         print(f"From {last_node.lineno}:{last_node.col_offset} to {last_node.end_lineno}:{last_node.end_col_offset}")
         with open(last_file, "r", encoding="utf-8") as f:
@@ -145,6 +149,7 @@ def transpile(source, name="<module>", path=None):
     TB = f"transpiling module {cf.white(name)}"
     res = ast.parse(source, type_comments=True)
     # res = initial_pytype.run(source, res)
+
     res = DesugarWith().visit(res)
     IfMainVisitor().visit(res)
     ScoperBlockVisitor().visit(res)
@@ -153,11 +158,11 @@ def transpile(source, name="<module>", path=None):
 
     # display each scope
     def disp_scope(scope, indent=0):
-        print("  " * indent, scope.kind)
+        debug("  " * indent, scope.kind)
         for child in scope.children:
             disp_scope(child, indent + 1)
         for var in scope.vars.items():
-            print("  " * (indent + 1), var)
+            debug("  " * (indent + 1), var)
 
     # disp_scope(res.scope)
 

trans/transpiler/phases/desugar_with/__init__.py

@@ -8,22 +8,27 @@ def process(items: list[ast.withitem], body: list[ast.stmt]) -> PlainBlock:
     first, *rest = items
     val, name = first.context_expr, first.optional_vars
     cm_name = ast.Name(id=f"cm_{hash(first)}")
+    end_node = name or first.context_expr
+    with_lineno = {"lineno": first.context_expr.lineno, "col_offset": first.context_expr.col_offset,
+                   "end_lineno": end_node.end_lineno, "end_col_offset": end_node.end_col_offset}
     res = [
-        ast.Assign(targets=[cm_name], value=val)
+        ast.Assign(targets=[cm_name], value=val, **with_lineno)
     ]
-    enter_call = ast.Call(func=ast.Attribute(value=cm_name, attr="__enter__"), args=[], keywords=[])
+    enter_call = ast.Call(func=ast.Attribute(value=cm_name, attr="__enter__", **with_lineno), args=[], keywords=[],
+                          **with_lineno)
     if name:
-        res.append(ast.Assign(targets=[name], value=enter_call))
+        res.append(ast.Assign(targets=[name], value=enter_call, **with_lineno))
     else:
-        res.append(ast.Expr(value=enter_call))
+        res.append(ast.Expr(value=enter_call, **with_lineno))
     if rest:
         res.append(process(rest, body))
     else:
         res.append(PlainBlock(body))
-    res.append(ast.Expr(value=ast.Call(func=ast.Attribute(value=cm_name, attr="__exit__"), args=[], keywords=[])))
+    res.append(ast.Expr(
+        value=ast.Call(func=ast.Attribute(value=cm_name, attr="__exit__"), args=[], keywords=[], **with_lineno)))
     return PlainBlock(res)
 
 
 class DesugarWith(ast.NodeTransformer):
     def visit_With(self, node: ast.With):
-        return process(node.items, node.body)
\ No newline at end of file
+        return process(node.items, node.body)

trans/transpiler/phases/emit_cpp/module.py

@@ -70,14 +70,14 @@ class ModuleVisitor(BlockVisitor):
         yield f"}} {alias};"
 
     def visit_ImportFrom(self, node: ast.ImportFrom) -> Iterable[str]:
-        if node.module_obj.is_python:
+        if node.module in {"typon", "typing", "__future__"}:
+            yield ""
+        elif node.module_obj.is_python:
             for alias in node.names:
                 fty = alias.item_obj
                 assert isinstance(fty, FunctionType)
 
                 yield from self.emit_python_func(node.module, alias.name, alias.asname or alias.name, fty)
-        elif node.module in {"typon", "typing", "__future__"}:
-            yield ""
         else:
             yield from self.import_module(node.module)
             for alias in node.names:

trans/transpiler/phases/typing/__init__.py

 import ast
 from pathlib import Path
-
+from logging import debug
 from transpiler.phases.typing.scope import VarKind, VarDecl, ScopeKind, Scope
 from transpiler.phases.typing.stdlib import PRELUDE, StdlibVisitor
 from transpiler.phases.typing.types import TY_TYPE, TY_INT, TY_STR, TY_BOOL, TY_COMPLEX, TY_NONE, FunctionType, \
@@ -53,14 +53,14 @@ def discover_module(path: Path, scope):
             scope.vars[child.name] = make_mod_decl(child.name, mod_scope)
         elif child.name == "__init__.py":
             StdlibVisitor(scope).visit(ast.parse(child.read_text()))
-            print(f"Visited {child}")
+            debug(f"Visited {child}")
         elif child.suffix == ".py":
             mod_scope = PRELUDE.child(ScopeKind.GLOBAL)
             StdlibVisitor(mod_scope).visit(ast.parse(child.read_text()))
             if child.stem[-1] == "_":
                 child = child.with_name(child.stem[:-1])
             scope.vars[child.stem] = make_mod_decl(child.name, mod_scope)
-            print(f"Visited {child}")
+            debug(f"Visited {child}")
 
 
 def make_mod_decl(child, mod_scope):
@@ -69,5 +69,5 @@ def make_mod_decl(child, mod_scope):
 
 discover_module(typon_std, PRELUDE)
 
-print("Stdlib visited!")
+debug("Stdlib visited!")
 #exit()
\ No newline at end of file

trans/transpiler/phases/typing/block.py

@@ -3,7 +3,7 @@ import dataclasses
 import importlib
 from dataclasses import dataclass
 
-from transpiler.utils import highlight
+from transpiler.utils import highlight, linenodata
 from transpiler.phases.typing import make_mod_decl
 from transpiler.phases.typing.common import ScoperVisitor
 from transpiler.phases.typing.expr import ScoperExprVisitor
@@ -95,9 +95,10 @@ class ScoperBlockVisitor(ScoperVisitor):
             raise NotImplementedError(node)
         ty = self.visit_annotation(node.annotation)
         node.is_declare = self.visit_assign_target(node.target, ty)
-        ty_val = self.get_type(node.value)
-        TB = f"unifying annotation {highlight(node.annotation)} with value {highlight(node.value)} of type {highlight(ty_val)}"
-        ty.unify(ty_val)
+        if node.value is not None:
+            ty_val = self.get_type(node.value)
+            TB = f"unifying annotation {highlight(node.annotation)} with value {highlight(node.value)} of type {highlight(ty_val)}"
+            ty.unify(ty_val)
 
     def visit_assign_target(self, target, decl_val: BaseType) -> bool:
         TB = f"analyzing assignment target {highlight(target)} with value {highlight(decl_val)}"
@@ -241,7 +242,10 @@ class ScoperBlockVisitor(ScoperVisitor):
                 self.scope.global_scope.vars[name] = VarDecl(VarKind.LOCAL, None)
 
     def visit_AugAssign(self, node: ast.AugAssign):
-        equivalent = ast.Assign(targets=[node.target], value=ast.BinOp(left=node.target, op=node.op, right=node.value))
+        equivalent = ast.Assign(
+            targets=[node.target],
+            value=ast.BinOp(left=node.target, op=node.op, right=node.value, **linenodata(node)),
+            **linenodata(node))
         self.visit(equivalent)
 
     def visit(self, node: ast.AST):

trans/transpiler/phases/typing/exceptions.py

@@ -88,10 +88,6 @@ class ArgumentCountMismatchError(CompileError):
     func: TypeOperator
     arguments: TypeOperator
 
-    def __setattr__(self, key, value):
-        print(key, value)
-        super().__setattr__(key, value)
-
     def __str__(self) -> str:
         fcount = str(len(self.func.args))
         if self.func.variadic:

trans/transpiler/phases/typing/stdlib.py

@@ -3,7 +3,7 @@ import dataclasses
 from abc import ABCMeta
 from dataclasses import dataclass, field
 from typing import Optional, List, Dict
-
+from logging import debug
 from transpiler.phases.typing.annotations import TypeAnnotationVisitor
 from transpiler.phases.typing.common import PRELUDE
 from transpiler.phases.typing.expr import ScoperExprVisitor
@@ -49,10 +49,10 @@ class StdlibVisitor(NodeVisitorSeq):
         if existing := self.scope.get(node.name):
             ty = existing.type
         else:
-            class TheType(TypeOperator):
+            class BuiltinClassType(TypeOperator):
                 def __init__(self, *args):
                     super().__init__(args, node.name)
-            ty = TypeType(TheType)
+            ty = TypeType(BuiltinClassType)
             self.scope.vars[node.name] = VarDecl(VarKind.LOCAL, ty)
         typevars = []
         for b in node.bases:
@@ -118,11 +118,11 @@ class StdlibVisitor(NodeVisitorSeq):
             try:
                 res = self.expr().visit(oper)
             except:
-                print("Type of", ast.unparse(oper), ":=", "INVALID")
+                debug(f"Type of {ast.unparse(oper)} := INVALID")
             else:
                 raise AssertionError(f"Assertion should fail, got {res} for {ast.unparse(oper)}")
         else:
-            print("Type of", ast.unparse(node.test), ":=", self.expr().visit(node.test))
+            debug(f"Type of {ast.unparse(node.test)} := {self.expr().visit(node.test)}")
 
     def visit_Call(self, node: ast.Call) -> BaseType:
         ty_op = self.visit(node.func)

trans/transpiler/phases/typing/types.py

@@ -134,10 +134,10 @@ class TypeOperator(BaseType, ABC):
 
     @staticmethod
     def make_type(name: str):
-        class TheType(TypeOperator):
+        class BuiltinType(TypeOperator):
             def __init__(self):
                 super().__init__([], name)
-        return TheType()
+        return BuiltinType()
 
     def __init_subclass__(cls, **kwargs):
         super().__init_subclass__(**kwargs)

trans/transpiler/utils.py

@@ -27,6 +27,11 @@ def highlight(code, full=False):
     """
     Syntax highlights code as Python using colorama
     """
+    TB = f"syntax highlighting {code}"
+    if code is None:
+        return cf.yellow("<None>")
+    if type(code) == list:
+        return repr([highlight(x) for x in code])
     from transpiler.phases.typing import BaseType
     if isinstance(code, ast.AST):
         return cf.italic_grey60(f"[{type(code).__name__}] ") + highlight(ast.unparse(code))
@@ -71,3 +76,6 @@ class UnsupportedNodeError(Exception):
 
     def __str__(self) -> str:
         return f"Unsupported node: {self.node.__class__.__mro__} {ast.dump(self.node)}"
+
+def linenodata(node):
+    return {k: getattr(node, k) for k in ("lineno", "end_lineno", "col_offset", "end_col_offset")}
\ No newline at end of file