Fix tests in Python 3.

Fixes the tests for Python 3, except for
lib/tests/test_zodb.py::test_zstor_2zurl which require NEO support
for Python 3.

bigarray/tests/test_basic.py

@@ -107,7 +107,7 @@ def test_bigarray_noobject(testbig):
 
     # NOTE str & unicode are fixed-size types - if size is not explicitly given
     # it will become S0 or U0
-    obj_dtypev = [numpy.object, 'O', 'i4, O', [('x', 'i4'), ('y', 'i4, O')]]
+    obj_dtypev = [object, 'O', 'i4, O', [('x', 'i4'), ('y', 'i4, O')]]
     for dtype_ in obj_dtypev:
         with raises(TypeError):
             BigArray((1,), dtype_, Zh)

lib/tests/test_zodb.py

@@ -33,7 +33,6 @@ from golang import defer, func
 from pytest import raises
 import pytest; xfail = pytest.mark.xfail
 from ZEO.ClientStorage import ClientStorage as ZEOStorage
-from neo.client.Storage import Storage as NEOStorage
 import os
 from six.moves.urllib.parse import quote_plus
 
@@ -494,6 +493,7 @@ def test_zstor_2zurl(tmpdir, neo_ssl_dict):
     # neo returns new NEO client for specified cluster name and master address.
     # NOTE, similarly to ZEO, the client is returned without waiting until server nodes are connected.
     def neo(cluster_name, master_addr, ssl=0):
+        from neo.client.Storage import Storage as NEOStorage
         kwargs = dict(master_nodes=master_addr, name=cluster_name)
         if ssl:
             kwargs.update(neo_ssl_dict)

setup.py

@@ -401,7 +401,7 @@ setup(
                      'pytest',
                      'scipy',
                      'neoppod',     # lib/tests/test_zodb.py
-                     'ZEO',         # lib/tests/test_zodb.py
+                     'ZEO[test]',         # lib/tests/test_zodb.py
                     ],
     },
 

wcfs/__init__.py

@@ -67,6 +67,7 @@ The following environment variables can be used to control wcfs.py client:
 from __future__ import print_function, absolute_import
 
 import os, sys, hashlib, subprocess, stat
+import six
 import logging; log = logging.getLogger('wcfs')
 from os.path import dirname
 from stat import S_ISDIR
@@ -504,6 +505,9 @@ def _mntpt_4zurl(zurl):
     # masters, we still have them associated with the same wcfs mountpoint.
     zurl = zurl_normalize_main(zurl)
 
+    if isinstance(zurl, six.text_type):
+      zurl = zurl.encode("utf-8")
+
     m = hashlib.sha1()
     m.update(zurl)
 

wcfs/client/_wcfs.pxd

@@ -19,6 +19,7 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 # cython: language_level=2
+# cython: c_string_encoding=utf8
 # distutils: language=c++
 
 # Package _wcfs provides Python-wrappers for C++ wcfs client package.

wcfs/client/_wcfs.pyx

@@ -19,6 +19,7 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 # cython: language_level=2
+# cython: c_string_type=str, c_string_encoding=utf8
 # cython: auto_pickle=False
 # distutils: language=c++
 

wcfs/internal/pycompat/pycompat.go

@@ -3,7 +3,9 @@
 package pycompat
 
 import (
+	"fmt"
 	"math/big"
+	pickle "github.com/kisielk/og-rek"
 )
 
 // Int64 tries to convert unpickled Python value to int64.
@@ -12,14 +14,31 @@ import (
 //
 // XXX + support for float?
 func Int64(xv interface{}) (v int64, ok bool) {
-        switch v := xv.(type) {
-        case int64:
-                return v, true
-        case *big.Int:
-                if v.IsInt64() {
-                        return v.Int64(), true
-                }
-        }
-
-        return 0, false
+  switch v := xv.(type) {
+  case int64:
+    return v, true
+  case *big.Int:
+    if v.IsInt64() {
+      return v.Int64(), true
+    }
+  }
+
+  return 0, false
 }
+
+// Xstrbytes verifies and extacts str|bytes from unpickled value.
+func Xstrbytes(x interface{}) (string, error) {
+	var s string
+	switch x := x.(type) {
+	default:
+		return "", fmt.Errorf("expect str|bytes; got %T", x)
+
+	case string:
+		s = x
+
+	case pickle.Bytes:
+		s = string(x)
+	}
+
+	return s, nil
+}
\ No newline at end of file

wcfs/internal/wcfs_test.pyx

@@ -18,6 +18,7 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 # cython: language_level=2
+# cython: c_string_type=str, c_string_encoding=utf8
 # distutils: language=c++
 
 """Module wcfs_test.pyx complements wcfs_test.py with things that cannot be

wcfs/internal/xbtree.py

@@ -720,7 +720,7 @@ def __walkBFS(tree): # i[] of [](of _NodeInRange on each level)
             assert isinstance(rn.node, (Tree, Bucket))
             if isinstance(rn.node, Tree):
                 v = (rn.range.klo,) + rn.node.keyv + (rn.range.khi,)
-                rv = zip(v[:-1], v[1:])  # (klo,k1), (k1,k2), ..., (kN,khi)
+                rv = list(zip(v[:-1], v[1:]))  # (klo,k1), (k1,k2), ..., (kN,khi)
                 assert len(rv) == len(rn.node.children)
 
                 for i in range(len(rv)):
@@ -771,7 +771,7 @@ def __zwalkBFS(ztree): # i[] of [](of _NodeInRange on each level)
                     panic("bad tree kind %r" % kind)
 
                 v = [rn.range.klo] + keyv + [rn.range.khi]
-                rv = zip(v[:-1], v[1:])  # (klo,k1), (k1,k2), ..., (kN,khi)
+                rv = list(zip(v[:-1], v[1:]))  # (klo,k1), (k1,k2), ..., (kN,khi)
                 assert len(rv) == len(children)
 
                 for i in range(len(rv)):

wcfs/internal/xbtree/xbtreetest/treeenv.go

@@ -31,6 +31,7 @@ import (
 	"lab.nexedi.com/kirr/neo/go/zodb"
 
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
+	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/pycompat"
 	"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
 )
 
@@ -302,8 +303,8 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
 	defer zblkdir.PDeactivate()
 
 	for xname, xzblk := range zblkdir.Data {
-		name, ok := xname.(string)
-		if !ok {
+		name, err := pycompat.Xstrbytes(xname)
+		if err != nil {
 			exc.Raisef("root['treegen/values']: key [%q]: expected str, got %T", xname, xname)
 		}
 

wcfs/internal/xbtree/xbtreetest/treegen.py

@@ -191,13 +191,14 @@ class ZCtx(object):
         valdict = zctx.root.get('treegen/values', None)
         if valdict is None:
             valdict = zctx.root['treegen/values'] = PersistentMapping()
-        valv = b'abcdefghij'
+        valv = 'abcdefghij'
         for v in valv:
+            v_data = v.encode('ascii')
             zblk = valdict.get(v, None)
-            if zblk is not None and zblk.loadblkdata() == v:
+            if zblk is not None and zblk.loadblkdata() == v_data:
                 continue
             zblk = ZBlk()
-            zblk.setblkdata(v)
+            zblk.setblkdata(v_data)
             valdict[v] = zblk
         zctx.valdict = valdict
         commit('treegen/values: init %r' % valv, skipIfEmpty=True)
@@ -259,7 +260,7 @@ def TreesSrv(zstor, r):
             t, D = treetxt.split()
             assert D.startswith('D')
             kv = kvDecode(t[1:], zctx.vdecode)
-            zv = _kvDecode(D[1:], kdecode=lambda ktxt: ktxt, vdecode=lambda vtxt: vtxt)
+            zv = _kvDecode(D[1:], kdecode=lambda ktxt: ktxt, vdecode=lambda vtxt: vtxt.encode('ascii'))
             patch(ztree,    diff(ztree, kv), kv)
 
             # ~ patch(valdict, diff(valdict,zv))  but sets zblk.value on change

wcfs/internal/zdata/zblk.go

@@ -97,8 +97,8 @@ func (zb *zBlk0State) PyGetState() interface{} {
 
 // PySetState implements zodb.PyStateful.
 func (zb *zBlk0State) PySetState(pystate interface{}) error {
-	blkdata, ok := pystate.(string)
-	if !ok {
+	blkdata, err := pycompat.Xstrbytes(pystate)
+	if err != nil {
 		return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate))
 	}
 
@@ -143,8 +143,8 @@ func (zd *zDataState) PyGetState() interface{} {
 
 // PySetState implements zodb.PyStateful.
 func (zd *zDataState) PySetState(pystate interface{}) error {
-	data, ok := pystate.(string)
-	if !ok {
+	data, err := pycompat.Xstrbytes(pystate)
+	if err != nil {
 		return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate))
 	}
 

wcfs/wcfs_test.py

@@ -41,7 +41,8 @@ from persistent.timestamp import TimeStamp
 from ZODB.utils import z64, u64, p64
 
 import sys, os, os.path, subprocess
-from thread import get_ident as gettid
+import six
+from six.moves._thread import get_ident as gettid
 from time import gmtime
 from errno import EINVAL, ENOTCONN
 from resource import setrlimit, getrlimit, RLIMIT_MEMLOCK
@@ -481,7 +482,7 @@ class tDB(tWCFS):
     def change(t, zf, changeDelta):
         assert isinstance(zf, ZBigFile)
         zfDelta = t._changed.setdefault(zf, {})
-        for blk, data in changeDelta.iteritems():
+        for blk, data in six.iteritems(changeDelta):
             data = b(data)
             assert len(data) <= zf.blksize
             zfDelta[blk] = data
@@ -520,7 +521,7 @@ class tDB(tWCFS):
             dfile = DFile()
             zconns.add(zf._p_jar)
             zfh = zf.fileh_open(_use_wcfs=False)
-            for blk, data in zfDelta.iteritems():
+            for blk, data in six.iteritems(zfDelta):
                 dfile.ddata[blk] = data
                 data += b'\0'*(zf.blksize - len(data))  # trailing \0
                 vma = zfh.mmap(blk, 1)
@@ -793,7 +794,7 @@ class tFile:
                 try:
                     b = read_exfault_nogil(blkview[0:1])
                 except SegmentationFault:
-                    b = 'FAULT'
+                    b = b'FAULT'
                 t._blkaccess(blk)
                 have_read.send(b)
             go(_)
@@ -805,7 +806,7 @@ class tFile:
                 raise ctx.err()
             b = _rx
 
-            ev.append('read ' + b)
+            ev.append(b'read ' + b)
         ev = doCheckingPin(_, pinokByWLink, pinfunc)
 
         # XXX hack - wlinks are notified and emit events simultaneously - we
@@ -813,7 +814,7 @@ class tFile:
         # are inside (i.e. read is stuck until pins are acknowledged).
         # Better do explicit check in tracetest style.
         assert ev[0]  == 'read pre', ev
-        assert ev[-1] == 'read ' + dataok[0], ev
+        assert ev[-1] == b'read ' + dataok[0:1], ev
         ev = ev[1:-1]
         if not shouldPin:
             assert ev == []
@@ -843,7 +844,7 @@ class tFile:
         assert st.st_blksize == t.blksize
         assert st.st_size == len(dataokv)*t.blksize
         if mtime is not None:
-            assert st.st_mtime == tidtime(mtime)
+            assert st.st_mtime == pytest.approx(tidtime(mtime))
 
         cachev = t.cached()
         for blk, dataok in enumerate(dataokv):
@@ -1100,7 +1101,7 @@ def _expectPin(twlink, ctx, zf, expect): # -> []SrvReq
     expected = set()    # of expected pin messages
     for blk, at in expect.items():
         hat = h(at) if at is not None else 'head'
-        msg = b"pin %s #%d @%s" % (h(zf._p_oid), blk, hat)
+        msg = "pin %s #%d @%s" % (h(zf._p_oid), blk, hat)
         assert msg not in expected
         expected.add(msg)
 
@@ -1372,7 +1373,7 @@ def test_wcfs_watch_robust():
     def _(ctx):
         req = wl.recvReq(ctx)
         assert req is not None
-        assert req.msg == b"pin %s #%d @%s" % (h(zf._p_oid), 2, h(at1))
+        assert req.msg == "pin %s #%d @%s" % (h(zf._p_oid), 2, h(at1))
         # don't reply to req - close instead
         wl.closeWrite()
     wg.go(_)
@@ -1385,8 +1386,8 @@ def test_wcfs_watch_robust():
 
     # invalid requests -> wcfs replies error
     wl = t.openwatch()
-    assert wl.sendReq(timeout(), b'bla bla') ==  \
-            b'error bad watch: not a watch request: "bla bla"'
+    assert wl.sendReq(timeout(), 'bla bla') ==  \
+            'error bad watch: not a watch request: "bla bla"'
 
     # invalid request not following frame structure -> fatal + wcfs must close watch link
     assert wl.fatalv == []
@@ -1397,7 +1398,7 @@ def test_wcfs_watch_robust():
     )
     if _ == 0:
         raise RuntimeError("%s: did not rx EOF after bad frame " % wl)
-    assert wl.fatalv == [b'error: invalid frame: "zzz hello\\n" (invalid stream)']
+    assert wl.fatalv == ['error: invalid frame: "zzz hello\\n" (invalid stream)']
     wl.close()
 
     # watch with @at < δtail.tail -> rejected
@@ -1476,7 +1477,7 @@ def test_wcfs_pintimeout_kill():
     def _(ctx):
         req = wl.recvReq(ctx)
         assert req is not None
-        assert req.msg == b"pin %s #%d @%s" % (h(zf._p_oid), 2, h(at1))
+        assert req.msg == "pin %s #%d @%s" % (h(zf._p_oid), 2, h(at1))
 
         # sleep > wcfs pin timeout - wcfs must kill us
         _, _rx = select(
@@ -1512,9 +1513,9 @@ def test_wcfs_watch_setup_ahead():
         defer(wl.close)
 
         wat = tidfromtime(tidtime(at1) + 1*dt)  # > at1, but < at2
-        rx = wl.sendReq(ctx, b"watch %s @%s" % (h(zf._p_oid), h(wat)))
+        rx = wl.sendReq(ctx, "watch %s @%s" % (h(zf._p_oid), h(wat)))
         assert ready(committing)
-        assert rx == b"ok"
+        assert rx == "ok"
     wg.go(_)
 
     # T2: sleep(10·dt); commit
@@ -1738,7 +1739,7 @@ def test_wcfs_remmap_on_pin():
         assert at    == at1
         mm.map_into_ro(f._blk(blk), f1.f.fileno(), blk*f.blksize)
 
-    f._assertBlk(2, 'hello', {wl: {2:at1}}, pinfunc=_)     # NOTE not world
+    f._assertBlk(2, b'hello', {wl: {2:at1}}, pinfunc=_)     # NOTE not world
 
 
 # verify that pin message is not sent for the same blk@at twice.