Commit 3121b290 authored by Kirill Smelkov's avatar Kirill Smelkov

golang: Teach pychan to work with channels of C types, not only PyObjects

Introduce notion of data type (dtype similarly to NumPy) into pychan and
teach it to accept for send objects only matching that dtype. Likewise
teach pychan to decode raw bytes received from underlying channel into
Python object correspodningg to pychan dtype. For C dtypes, e.g.
'C.int', 'C.double' etc, contrary to chan of python objects, the
transfer can be done without depending on Python GIL. This way channels
of such C-level dtypes can be used to implement interaction in between
Python and nogil worlds.
parent 2c8063f4
...@@ -99,6 +99,14 @@ channels. For example:: ...@@ -99,6 +99,14 @@ channels. For example::
# default case # default case
... ...
By default `chan` creates new channel that can carry arbitrary Python objects.
However type of channel elements can be specified via `chan(dtype=X)` - for
example `chan(dtype='C.int')` creates new channel whose elements are C
integers. `chan.nil(X)` creates typed nil channel. `Cython/nogil API`_
explains how channels with non-Python dtypes, besides in-Python usage, can be
additionally used for interaction in between Python and nogil worlds.
Methods Methods
------- -------
...@@ -238,6 +246,13 @@ can be used to multiplex on several channels. For example:: ...@@ -238,6 +246,13 @@ can be used to multiplex on several channels. For example::
# default case # default case
... ...
Channels created from Python are represented by `pychan` cdef class. Python
channels that carry non-Python elements (`pychan.dtype != DTYPE_PYOBJECT`) can
be converted to Cython/nogil `chan[T]` via `pychan.chan_*()`. For example
`pychan.chan_int()` converts Python channel created via `pychan(dtype='C.int')`
into `chan[int]`. This provides interaction mechanism in between *nogil* and
Python worlds.
`panic` stops normal execution of current goroutine by throwing a C-level `panic` stops normal execution of current goroutine by throwing a C-level
exception. On Python/C boundaries C-level exceptions have to be converted to exception. On Python/C boundaries C-level exceptions have to be converted to
Python-level exceptions with `topyexc`. For example:: Python-level exceptions with `topyexc`. For example::
......
...@@ -110,6 +110,7 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil: ...@@ -110,6 +110,7 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil:
_chan *ch _chan *ch
_chanop op _chanop op
unsigned flags unsigned flags
unsigned user
void *ptxrx void *ptxrx
uint64_t itxrx uint64_t itxrx
cbool *rxok cbool *rxok
...@@ -127,9 +128,40 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil: ...@@ -127,9 +128,40 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil:
cdef void topyexc() except * cdef void topyexc() except *
cpdef pypanic(arg) cpdef pypanic(arg)
# pychan is chan<object> # pychan is python wrapper over chan<object> or chan<structZ|bool|int|double|...>
from cython cimport final from cython cimport final
# DType describes type of channel elements.
# TODO consider supporting NumPy dtypes too.
cdef enum DType:
DTYPE_PYOBJECT = 0 # chan[object]
DTYPE_STRUCTZ = 1 # chan[structZ]
DTYPE_BOOL = 2 # chan[bool]
DTYPE_INT = 3 # chan[int]
DTYPE_DOUBLE = 4 # chan[double]
DTYPE_NTYPES = 5
# pychan wraps a channel into python object.
#
# Type of channel can be either channel of python objects, or channel of
# C-level objects. If channel elements are C-level objects, the channel - even
# via pychan wrapper - can be used to interact with nogil world.
#
# There can be multiple pychan(s) wrapping a particular raw channel.
@final @final
cdef class pychan: cdef class pychan:
cdef _chan *_ch cdef _chan *_ch
cdef DType dtype # type of channel elements
# pychan.nil(X) creates new nil pychan with element type X.
@staticmethod # XXX needs to be `cpdef nil()` but cython:
cdef pychan _nil(object dtype) # "static cpdef methods not yet supported"
# chan_X returns ._ch wrapped into typesafe pyx/nogil-level chan[X].
# chan_X panics if channel type != X.
# X can be any C-level type, but not PyObject.
cdef nogil:
chan[structZ] chan_structZ (pychan pych)
chan[cbool] chan_bool (pychan pych)
chan[int] chan_int (pychan pych)
chan[double] chan_double (pychan pych)
This diff is collapsed.
...@@ -210,3 +210,72 @@ def test_select_win_while_queue(): ...@@ -210,3 +210,72 @@ def test_select_win_while_queue():
def test_select_inplace(): def test_select_inplace():
with nogil: with nogil:
_test_select_inplace() _test_select_inplace()
# helpers for pychan(dtype=X) py <-> c tests.
def pychan_structZ_recv(pychan pych):
with nogil: _pychan_structZ_recv(pych)
return None
def pychan_structZ_send(pychan pych, obj):
if obj is not None:
raise TypeError("cannot convert %r to structZ" % (obj,))
cdef structZ _
with nogil:
_pychan_structZ_send(pych, _)
def pychan_structZ_close(pychan pych):
with nogil: _pychan_structZ_close(pych)
def pychan_bool_recv(pychan pych):
with nogil: _ = _pychan_bool_recv(pych)
return _
def pychan_bool_send(pychan pych, cbool obj):
with nogil: _pychan_bool_send(pych, obj)
def pychan_bool_close(pychan pych):
with nogil: _pychan_bool_close(pych)
def pychan_int_recv(pychan pych):
with nogil: _ = _pychan_int_recv(pych)
return _
def pychan_int_send(pychan pych, int obj):
with nogil: _pychan_int_send(pych, obj)
def pychan_int_close(pychan pych):
with nogil: _pychan_int_close(pych)
def pychan_double_recv(pychan pych):
with nogil: _ = _pychan_double_recv(pych)
return _
def pychan_double_send(pychan pych, double obj):
with nogil: _pychan_double_send(pych, obj)
def pychan_double_close(pychan pych):
with nogil: _pychan_double_close(pych)
cdef nogil:
structZ _pychan_structZ_recv(pychan pych) except +topyexc:
return pych.chan_structZ().recv()
void _pychan_structZ_send(pychan pych, structZ obj) except +topyexc:
pych.chan_structZ().send(obj)
void _pychan_structZ_close(pychan pych) except +topyexc:
pych.chan_structZ().close()
cbool _pychan_bool_recv(pychan pych) except +topyexc:
return pych.chan_bool().recv()
void _pychan_bool_send(pychan pych, cbool obj) except +topyexc:
pych.chan_bool().send(obj)
void _pychan_bool_close(pychan pych) except +topyexc:
pych.chan_bool().close()
int _pychan_int_recv(pychan pych) except +topyexc:
return pych.chan_int().recv()
void _pychan_int_send(pychan pych, int obj) except +topyexc:
pych.chan_int().send(obj)
void _pychan_int_close(pychan pych) except +topyexc:
pych.chan_int().close()
double _pychan_double_recv(pychan pych) except +topyexc:
return pych.chan_double().recv()
void _pychan_double_send(pychan pych, double obj) except +topyexc:
pych.chan_double().send(obj)
void _pychan_double_close(pychan pych) except +topyexc:
pych.chan_double().close()
...@@ -29,6 +29,7 @@ from subprocess import Popen, PIPE ...@@ -29,6 +29,7 @@ from subprocess import Popen, PIPE
from six.moves import range as xrange from six.moves import range as xrange
import gc, weakref import gc, weakref
from golang import _golang_test
from golang._golang_test import pywaitBlocked as waitBlocked, pylen_recvq as len_recvq, \ from golang._golang_test import pywaitBlocked as waitBlocked, pylen_recvq as len_recvq, \
pylen_sendq as len_sendq, pypanicWhenBlocked as panicWhenBlocked pylen_sendq as len_sendq, pypanicWhenBlocked as panicWhenBlocked
...@@ -738,15 +739,160 @@ def _test_blockforever(): ...@@ -738,15 +739,160 @@ def _test_blockforever():
with panics("t: blocks forever"): select((z.send, 1), z.recv) with panics("t: blocks forever"): select((z.send, 1), z.recv)
def test_chan_misc(): # verify chan(dtype=X) functionality.
nilch = nilchan def test_chan_dtype_invalid():
with raises(TypeError) as exc:
chan(dtype="BadType")
assert exc.value.args == ("pychan: invalid dtype: 'BadType'",)
chantypev = [
# dtype obj zero-obj
('object', 'abc', None),
('C.structZ', None, None),
('C.bool', True, False),
('C.int', 4, 0),
('C.double', 3.14, 0.0),
]
@mark.parametrize('dtype,obj,zobj', chantypev)
def test_chan_dtype(dtype, obj, zobj):
# py -> py (pysend/pyrecv; buffered)
ch = chan(1, dtype=dtype)
ch.send(obj)
obj2, ok = ch.recv_()
assert ok == True
assert type(obj2) is type(obj)
assert obj2 == obj
# send with different type - rejected
for (dtype2, obj2, _) in chantypev:
if dtype2 == dtype or dtype == "object":
continue # X -> X; object accepts *,
if (dtype2, dtype) == ('C.int', 'C.double'): # int -> double ok
continue
with raises(TypeError) as exc:
ch.send(obj2)
# XXX we can implement vvv, but it will potentially hide cause error
# XXX (or use raise from?)
#assert exc.value.args == ("type mismatch: expect %s; got %r" % (dtype, obj2),)
with raises(TypeError) as exc:
select((ch.send, obj2))
# py -> py (pyclose/pyrecv)
ch.close()
obj2, ok = ch.recv_()
assert ok == False
assert type(obj2) is type(zobj)
assert obj2 == zobj
# below tests are for py <-> c interaction
if dtype == "object":
return
ctype = dtype[2:] # C.int -> int
ch = chan(dtype=dtype) # recreate after close; mode=synchronous
# recv/send/close via C
def crecv(ch):
return getattr(_golang_test, "pychan_%s_recv" % ctype)(ch)
def csend(ch, obj):
getattr(_golang_test, "pychan_%s_send" % ctype)(ch, obj)
def cclose(ch):
getattr(_golang_test, "pychan_%s_close" % ctype)(ch)
# py -> c (pysend/crecv)
rx = chan()
def _():
_ = crecv(ch)
rx.send(_)
go(_)
ch.send(obj)
obj2 = rx.recv()
assert type(obj2) is type(obj)
assert obj2 == obj
# py -> c (pyselect/crecv)
rx = chan()
def _():
_ = crecv(ch)
rx.send(_)
go(_)
_, _rx = select(
(ch.send, obj), # 0
)
assert (_, _rx) == (0, None)
obj2 = rx.recv()
assert type(obj2) is type(obj)
assert obj2 == obj
# py -> c (pyclose/crecv)
rx = chan()
def _():
_ = crecv(ch)
rx.send(_)
go(_)
ch.close()
obj2 = rx.recv()
assert type(obj2) is type(zobj)
assert obj2 == zobj
assert nilch == nilch # nil == nil ch = chan(dtype=dtype) # recreate after close
# py <- c (pyrecv/csend)
def _():
csend(ch, obj)
go(_)
obj2 = ch.recv()
assert type(obj2) is type(obj)
assert obj2 == obj
# py <- c (pyselect/csend)
def _():
csend(ch, obj)
go(_)
_, _rx = select(
ch.recv, # 0
)
assert _ == 0
obj2 = _rx
assert type(obj2) is type(obj)
assert obj2 == obj
# py <- c (pyrecv/cclose)
def _():
cclose(ch)
go(_)
obj2 = ch.recv()
assert type(obj2) is type(zobj)
assert obj2 == zobj
@mark.parametrize('dtype', [_[0] for _ in chantypev])
def test_chan_dtype_misc(dtype):
nilch = chan.nil(dtype)
# nil repr
if dtype == "object":
assert repr(nilch) == "nilchan"
else:
assert repr(nilch) == ("chan.nil(%r)" % dtype)
# optimization: nil[X]() -> always same object
nilch_ = chan.nil(dtype)
assert nilch is nilch_
if dtype == "object":
assert nilch is nilchan
assert hash(nilch) == hash(nilchan)
assert nilch == nilch # nil[X] == nil[X]
assert nilch == nilchan # nil[X] == nil[*]
assert nilchan == nilch # nil[*] == nil[X]
# channels can be compared, different channels differ # channels can be compared, different channels differ
assert nilch != None # just in case assert nilch != None # just in case
ch1 = chan() ch1 = chan(dtype=dtype)
ch2 = chan() ch2 = chan(dtype=dtype)
ch3 = chan() ch3 = chan()
assert ch1 != ch2; assert ch1 == ch1 assert ch1 != ch2; assert ch1 == ch1
assert ch1 != ch3; assert ch2 == ch2 assert ch1 != ch3; assert ch2 == ch2
...@@ -758,6 +904,43 @@ def test_chan_misc(): ...@@ -758,6 +904,43 @@ def test_chan_misc():
assert nilch != ch2 assert nilch != ch2
assert nilch != ch3 assert nilch != ch3
# .nil on chan instance XXX doesn't work (yet ?)
"""
ch = chan() # non-nil chan object instance
with raises(AttributeError):
ch.nil
"""
# nil[X] vs nil[Y]
for (dtype2, _, _) in chantypev:
nilch2 = chan.nil(dtype2)
# nil[*] stands for untyped nil - it is equal to nil[X] for ∀ X
if dtype == "object" or dtype2 == "object":
if dtype != dtype2:
assert nilch is not nilch2
assert hash(nilch) == hash(nilch2)
assert (nilch == nilch2) == True
assert (nilch2 == nilch) == True
assert (nilch != nilch2) == False
assert (nilch2 != nilch) == False
continue
# nil[X] == nil[X]
if dtype == dtype2:
assert hash(nilch) == hash(nilch2)
assert (nilch == nilch2) == True
assert (nilch2 == nilch) == True
assert (nilch != nilch2) == False
assert (nilch2 != nilch) == False
continue
# nil[X] != nil[Y]
assert nilch is not nilch2
assert (nilch == nilch2) == False
assert (nilch2 == nilch) == False
assert (nilch != nilch2) == True
assert (nilch2 != nilch) == True
def test_func(): def test_func():
# test how @func(cls) works # test how @func(cls) works
......
...@@ -164,7 +164,9 @@ typedef struct _selcase { ...@@ -164,7 +164,9 @@ typedef struct _selcase {
_chan *ch; // channel _chan *ch; // channel
enum _chanop op : 8; // chansend/chanrecv/default enum _chanop op : 8; // chansend/chanrecv/default
enum _selflags flags : 8; // e.g. _INPLACE_DATA enum _selflags flags : 8; // e.g. _INPLACE_DATA
unsigned :16; unsigned user : 8; // arbitrary value that can be set by user
// (e.g. pyselect stores channel type here)
unsigned : 8;
union { union {
void *ptxrx; // chansend: ptx; chanrecv: prx void *ptxrx; // chansend: ptx; chanrecv: prx
uint64_t itxrx; // used instead of .ptxrx if .flags&_INPLACE_DATA != 0 uint64_t itxrx; // used instead of .ptxrx if .flags&_INPLACE_DATA != 0
...@@ -191,6 +193,7 @@ _selcase _selsend(_chan *ch, const void *ptx) { ...@@ -191,6 +193,7 @@ _selcase _selsend(_chan *ch, const void *ptx) {
.ch = ch, .ch = ch,
.op = _CHANSEND, .op = _CHANSEND,
.flags = (enum _selflags)0, .flags = (enum _selflags)0,
.user = 0xff,
.ptxrx = (void *)ptx, .ptxrx = (void *)ptx,
.rxok = NULL, .rxok = NULL,
}; };
...@@ -204,6 +207,7 @@ _selcase _selrecv(_chan *ch, void *prx) { ...@@ -204,6 +207,7 @@ _selcase _selrecv(_chan *ch, void *prx) {
.ch = ch, .ch = ch,
.op = _CHANRECV, .op = _CHANRECV,
.flags = (enum _selflags)0, .flags = (enum _selflags)0,
.user = 0xff,
.ptxrx = prx, .ptxrx = prx,
.rxok = NULL, .rxok = NULL,
}; };
...@@ -217,6 +221,7 @@ _selcase _selrecv_(_chan *ch, void *prx, bool *pok) { ...@@ -217,6 +221,7 @@ _selcase _selrecv_(_chan *ch, void *prx, bool *pok) {
.ch = ch, .ch = ch,
.op = _CHANRECV, .op = _CHANRECV,
.flags = (enum _selflags)0, .flags = (enum _selflags)0,
.user = 0xff,
.ptxrx = prx, .ptxrx = prx,
.rxok = pok, .rxok = pok,
}; };
......
...@@ -860,6 +860,7 @@ const _selcase _default = { ...@@ -860,6 +860,7 @@ const _selcase _default = {
.ch = NULL, .ch = NULL,
.op = _DEFAULT, .op = _DEFAULT,
.flags = (_selflags)0, .flags = (_selflags)0,
.user = 0xff,
.ptxrx = NULL, .ptxrx = NULL,
.rxok = NULL, .rxok = NULL,
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment