Merge pull request #390 from iovisor/tables_cleanup

Cleanup tables classes in bcc python module

src/cc/bpf_common.cc

@@ -134,6 +134,18 @@ int bpf_table_type_id(void *program, size_t id) {
   return mod->table_type(id);
 }
 
+size_t bpf_table_max_entries(void *program, const char *table_name) {
+  auto mod = static_cast<ebpf::BPFModule *>(program);
+  if (!mod) return 0;
+  return mod->table_max_entries(table_name);
+}
+
+size_t bpf_table_max_entries_id(void *program, size_t id) {
+  auto mod = static_cast<ebpf::BPFModule *>(program);
+  if (!mod) return 0;
+  return mod->table_max_entries(id);
+}
+
 const char * bpf_table_name(void *program, size_t id) {
   auto mod = static_cast<ebpf::BPFModule *>(program);
   if (!mod) return nullptr;

src/cc/bpf_common.h

@@ -42,6 +42,8 @@ int bpf_table_fd(void *program, const char *table_name);
 int bpf_table_fd_id(void *program, size_t id);
 int bpf_table_type(void *program, const char *table_name);
 int bpf_table_type_id(void *program, size_t id);
+size_t bpf_table_max_entries(void *program, const char *table_name);
+size_t bpf_table_max_entries_id(void *program, size_t id);
 const char * bpf_table_name(void *program, size_t id);
 const char * bpf_table_key_desc(void *program, const char *table_name);
 const char * bpf_table_key_desc_id(void *program, size_t id);

src/cc/bpf_module.cc

@@ -522,6 +522,15 @@ int BPFModule::table_type(size_t id) const {
   return (*tables_)[id].type;
 }
 
+size_t BPFModule::table_max_entries(const string &name) const {
+  return table_max_entries(table_id(name));
+}
+
+size_t BPFModule::table_max_entries(size_t id) const {
+  if (id >= tables_->size()) return 0;
+  return (*tables_)[id].max_entries;
+}
+
 const char * BPFModule::table_name(size_t id) const {
   if (id >= tables_->size()) return nullptr;
   return (*tables_)[id].name.c_str();

src/cc/bpf_module.h

@@ -70,6 +70,8 @@ class BPFModule {
   const char * table_name(size_t id) const;
   int table_type(const std::string &name) const;
   int table_type(size_t id) const;
+  size_t table_max_entries(const std::string &name) const;
+  size_t table_max_entries(size_t id) const;
   const char * table_key_desc(size_t id) const;
   const char * table_key_desc(const std::string &name) const;
   size_t table_key_size(size_t id) const;

src/python/MANIFEST

 # file GENERATED by distutils, do NOT edit
 setup.py
 bcc/__init__.py
+bcc/table.py
+bcc/libbcc.py

src/python/bcc/__init__.py

@@ -14,7 +14,6 @@
 
 from __future__ import print_function
 import atexit
-from collections import MutableMapping
 import ctypes as ct
 import fcntl
 import json
@@ -26,95 +25,8 @@ import struct
 import sys
 basestring = (unicode if sys.version_info[0] < 3 else str)
 
-lib = ct.CDLL("libbcc.so.0")
-
-# keep in sync with bpf_common.h
-lib.bpf_module_create_b.restype = ct.c_void_p
-lib.bpf_module_create_b.argtypes = [ct.c_char_p, ct.c_char_p, ct.c_uint]
-lib.bpf_module_create_c.restype = ct.c_void_p
-lib.bpf_module_create_c.argtypes = [ct.c_char_p, ct.c_uint,
-        ct.POINTER(ct.c_char_p), ct.c_int]
-lib.bpf_module_create_c_from_string.restype = ct.c_void_p
-lib.bpf_module_create_c_from_string.argtypes = [ct.c_char_p, ct.c_uint,
-        ct.POINTER(ct.c_char_p), ct.c_int]
-lib.bpf_module_destroy.restype = None
-lib.bpf_module_destroy.argtypes = [ct.c_void_p]
-lib.bpf_module_license.restype = ct.c_char_p
-lib.bpf_module_license.argtypes = [ct.c_void_p]
-lib.bpf_module_kern_version.restype = ct.c_uint
-lib.bpf_module_kern_version.argtypes = [ct.c_void_p]
-lib.bpf_num_functions.restype = ct.c_ulonglong
-lib.bpf_num_functions.argtypes = [ct.c_void_p]
-lib.bpf_function_name.restype = ct.c_char_p
-lib.bpf_function_name.argtypes = [ct.c_void_p, ct.c_ulonglong]
-lib.bpf_function_start.restype = ct.c_void_p
-lib.bpf_function_start.argtypes = [ct.c_void_p, ct.c_char_p]
-lib.bpf_function_size.restype = ct.c_size_t
-lib.bpf_function_size.argtypes = [ct.c_void_p, ct.c_char_p]
-lib.bpf_table_id.restype = ct.c_ulonglong
-lib.bpf_table_id.argtypes = [ct.c_void_p, ct.c_char_p]
-lib.bpf_table_fd.restype = ct.c_int
-lib.bpf_table_fd.argtypes = [ct.c_void_p, ct.c_char_p]
-lib.bpf_table_type_id.restype = ct.c_int
-lib.bpf_table_type_id.argtypes = [ct.c_void_p, ct.c_ulonglong]
-lib.bpf_table_key_desc.restype = ct.c_char_p
-lib.bpf_table_key_desc.argtypes = [ct.c_void_p, ct.c_char_p]
-lib.bpf_table_leaf_desc.restype = ct.c_char_p
-lib.bpf_table_leaf_desc.argtypes = [ct.c_void_p, ct.c_char_p]
-lib.bpf_table_key_snprintf.restype = ct.c_int
-lib.bpf_table_key_snprintf.argtypes = [ct.c_void_p, ct.c_ulonglong,
-        ct.c_char_p, ct.c_ulonglong, ct.c_void_p]
-lib.bpf_table_leaf_snprintf.restype = ct.c_int
-lib.bpf_table_leaf_snprintf.argtypes = [ct.c_void_p, ct.c_ulonglong,
-        ct.c_char_p, ct.c_ulonglong, ct.c_void_p]
-lib.bpf_table_key_sscanf.restype = ct.c_int
-lib.bpf_table_key_sscanf.argtypes = [ct.c_void_p, ct.c_ulonglong,
-        ct.c_char_p, ct.c_void_p]
-lib.bpf_table_leaf_sscanf.restype = ct.c_int
-lib.bpf_table_leaf_sscanf.argtypes = [ct.c_void_p, ct.c_ulonglong,
-        ct.c_char_p, ct.c_void_p]
-
-# keep in sync with libbpf.h
-lib.bpf_get_next_key.restype = ct.c_int
-lib.bpf_get_next_key.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p]
-lib.bpf_lookup_elem.restype = ct.c_int
-lib.bpf_lookup_elem.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p]
-lib.bpf_update_elem.restype = ct.c_int
-lib.bpf_update_elem.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p,
-        ct.c_ulonglong]
-lib.bpf_delete_elem.restype = ct.c_int
-lib.bpf_delete_elem.argtypes = [ct.c_int, ct.c_void_p]
-lib.bpf_open_raw_sock.restype = ct.c_int
-lib.bpf_open_raw_sock.argtypes = [ct.c_char_p]
-lib.bpf_attach_socket.restype = ct.c_int
-lib.bpf_attach_socket.argtypes = [ct.c_int, ct.c_int]
-lib.bpf_prog_load.restype = ct.c_int
-lib.bpf_prog_load.argtypes = [ct.c_int, ct.c_void_p, ct.c_size_t,
-        ct.c_char_p, ct.c_uint, ct.c_char_p, ct.c_uint]
-lib.bpf_attach_kprobe.restype = ct.c_void_p
-_CB_TYPE = ct.CFUNCTYPE(None, ct.py_object, ct.c_int,
-        ct.c_ulonglong, ct.POINTER(ct.c_ulonglong))
-_RAW_CB_TYPE = ct.CFUNCTYPE(None, ct.py_object, ct.c_void_p, ct.c_int)
-lib.bpf_attach_kprobe.argtypes = [ct.c_int, ct.c_char_p, ct.c_char_p, ct.c_int,
-        ct.c_int, ct.c_int, _CB_TYPE, ct.py_object]
-lib.bpf_detach_kprobe.restype = ct.c_int
-lib.bpf_detach_kprobe.argtypes = [ct.c_char_p]
-lib.bpf_attach_uprobe.restype = ct.c_void_p
-_CB_TYPE = ct.CFUNCTYPE(None, ct.py_object, ct.c_int,
-        ct.c_ulonglong, ct.POINTER(ct.c_ulonglong))
-_RAW_CB_TYPE = ct.CFUNCTYPE(None, ct.py_object, ct.c_void_p, ct.c_int)
-lib.bpf_attach_uprobe.argtypes = [ct.c_int, ct.c_char_p, ct.c_char_p, ct.c_int,
-        ct.c_int, ct.c_int, _CB_TYPE, ct.py_object]
-lib.bpf_detach_uprobe.restype = ct.c_int
-lib.bpf_detach_uprobe.argtypes = [ct.c_char_p]
-lib.bpf_open_perf_buffer.restype = ct.c_void_p
-lib.bpf_open_perf_buffer.argtypes = [_RAW_CB_TYPE, ct.py_object, ct.c_int, ct.c_int]
-lib.perf_reader_poll.restype = ct.c_int
-lib.perf_reader_poll.argtypes = [ct.c_int, ct.POINTER(ct.c_void_p), ct.c_int]
-lib.perf_reader_free.restype = None
-lib.perf_reader_free.argtypes = [ct.c_void_p]
-lib.perf_reader_fd.restype = int
-lib.perf_reader_fd.argtypes = [ct.c_void_p]
+from .libbcc import lib, _CB_TYPE
+from .table import Table
 
 open_kprobes = {}
 open_uprobes = {}
@@ -124,7 +36,6 @@ KALLSYMS = "/proc/kallsyms"
 ksym_addrs = []
 ksym_names = []
 ksym_loaded = 0
-stars_max = 40
 
 @atexit.register
 def cleanup_kprobes():
@@ -149,288 +60,20 @@ class BPF(object):
     SCHED_CLS = 3
     SCHED_ACT = 4
 
-    HASH = 1
-    ARRAY = 2
-    PROG_ARRAY = 3
-    PERF_EVENT_ARRAY = 4
-
     _probe_repl = re.compile("[^a-zA-Z0-9_]")
     _libsearch_cache = {}
     _lib_load_address_cache = {}
     _lib_symbol_cache = {}
 
+    # defined for compatibility reasons, to be removed
+    Table = Table
+
     class Function(object):
         def __init__(self, bpf, name, fd):
             self.bpf = bpf
             self.name = name
             self.fd = fd
 
-    class Table(MutableMapping):
-        def __init__(self, bpf, map_id, map_fd, keytype, leaftype):
-            self.bpf = bpf
-            self.map_id = map_id
-            self.map_fd = map_fd
-            self.Key = keytype
-            self.Leaf = leaftype
-            self.ttype = lib.bpf_table_type_id(self.bpf.module, self.map_id)
-            self._cbs = {}
-
-        def key_sprintf(self, key):
-            key_p = ct.pointer(key)
-            buf = ct.create_string_buffer(ct.sizeof(self.Key) * 8)
-            res = lib.bpf_table_key_snprintf(self.bpf.module, self.map_id,
-                    buf, len(buf), key_p)
-            if res < 0:
-                raise Exception("Could not printf key")
-            return buf.value
-
-        def leaf_sprintf(self, leaf):
-            leaf_p = ct.pointer(leaf)
-            buf = ct.create_string_buffer(ct.sizeof(self.Leaf) * 8)
-            res = lib.bpf_table_leaf_snprintf(self.bpf.module, self.map_id,
-                    buf, len(buf), leaf_p)
-            if res < 0:
-                raise Exception("Could not printf leaf")
-            return buf.value
-
-        def key_scanf(self, key_str):
-            key = self.Key()
-            key_p = ct.pointer(key)
-            res = lib.bpf_table_key_sscanf(self.bpf.module, self.map_id,
-                    key_str, key_p)
-            if res < 0:
-                raise Exception("Could not scanf key")
-            return key
-
-        def leaf_scanf(self, leaf_str):
-            leaf = self.Leaf()
-            leaf_p = ct.pointer(leaf)
-            res = lib.bpf_table_leaf_sscanf(self.bpf.module, self.map_id,
-                    leaf_str, leaf_p)
-            if res < 0:
-                raise Exception("Could not scanf leaf")
-            return leaf
-
-        def open_perf_buffer(self, callback):
-            """open_perf_buffers(callback)
-
-            Opens a set of per-cpu ring buffer to receive custom perf event
-            data from the bpf program. The callback will be invoked for each
-            event submitted from the kernel, up to millions per second.
-            """
-
-            for i in range(0, multiprocessing.cpu_count()):
-                self._open_perf_buffer(i, callback)
-
-        def _open_perf_buffer(self, cpu, callback):
-            fn = _RAW_CB_TYPE(lambda _, data, size: callback(cpu, data, size))
-            reader = lib.bpf_open_perf_buffer(fn, None, -1, cpu)
-            if not reader:
-                raise Exception("Could not open perf buffer")
-            fd = lib.perf_reader_fd(reader)
-            self[self.Key(cpu)] = self.Leaf(fd)
-            open_kprobes[(id(self), cpu)] = reader
-            # keep a refcnt
-            self._cbs[cpu] = fn
-
-        def close_perf_buffer(self, key):
-            reader = open_kprobes.get((id(self), key))
-            if reader:
-                lib.perf_reader_free(reader)
-                del(open_kprobes[(id(self), key)])
-            del self._cbs[key]
-
-        def __getitem__(self, key):
-            key_p = ct.pointer(key)
-            leaf = self.Leaf()
-            leaf_p = ct.pointer(leaf)
-            res = lib.bpf_lookup_elem(self.map_fd,
-                    ct.cast(key_p, ct.c_void_p),
-                    ct.cast(leaf_p, ct.c_void_p))
-            if res < 0:
-                raise KeyError
-            return leaf
-
-        def __setitem__(self, key, leaf):
-            key_p = ct.pointer(key)
-            leaf_p = ct.pointer(leaf)
-            res = lib.bpf_update_elem(self.map_fd,
-                    ct.cast(key_p, ct.c_void_p),
-                    ct.cast(leaf_p, ct.c_void_p), 0)
-            if res < 0:
-                raise Exception("Could not update table")
-
-        def __len__(self):
-            i = 0
-            for k in self: i += 1
-            return i
-
-        def __delitem__(self, key):
-            key_p = ct.pointer(key)
-            ttype = lib.bpf_table_type_id(self.bpf.module, self.map_id)
-            # Deleting from array type maps does not have an effect, so
-            # zero out the entry instead.
-            if ttype in (BPF.ARRAY, BPF.PROG_ARRAY, BPF.PERF_EVENT_ARRAY):
-                leaf = self.Leaf()
-                leaf_p = ct.pointer(leaf)
-                res = lib.bpf_update_elem(self.map_fd,
-                        ct.cast(key_p, ct.c_void_p),
-                        ct.cast(leaf_p, ct.c_void_p), 0)
-                if res < 0:
-                    raise Exception("Could not clear item")
-                if ttype == BPF.PERF_EVENT_ARRAY:
-                    self.close_perf_buffer(key)
-            else:
-                res = lib.bpf_delete_elem(self.map_fd,
-                        ct.cast(key_p, ct.c_void_p))
-                if res < 0:
-                    raise KeyError
-
-        # override the MutableMapping's implementation of these since they
-        # don't handle KeyError nicely
-        def itervalues(self):
-            for key in self:
-                # a map entry may be deleted in between discovering the key and
-                # fetching the value, suppress such errors
-                try:
-                    yield self[key]
-                except KeyError:
-                    pass
-
-        def iteritems(self):
-            for key in self:
-                try:
-                    yield (key, self[key])
-                except KeyError:
-                    pass
-
-        def items(self):
-            return [item for item in self.iteritems()]
-
-        def values(self):
-            return [value for value in self.itervalues()]
-
-        def clear(self):
-            # default clear uses popitem, which can race with the bpf prog
-            for k in self.keys():
-                self.__delitem__(k)
-
-        @staticmethod
-        def _stars(val, val_max, width):
-            i = 0
-            text = ""
-            while (1):
-                if (i > (width * val / val_max) - 1) or (i > width - 1):
-                    break
-                text += "*"
-                i += 1
-            if val > val_max:
-                text = text[:-1] + "+"
-            return text
-
-        def print_log2_hist(self, val_type="value", section_header="Bucket ptr",
-                section_print_fn=None):
-            """print_log2_hist(val_type="value", section_header="Bucket ptr",
-                               section_print_fn=None)
-
-            Prints a table as a log2 histogram. The table must be stored as
-            log2. The val_type argument is optional, and is a column header.
-            If the histogram has a secondary key, multiple tables will print
-            and section_header can be used as a header description for each.
-            If section_print_fn is not None, it will be passed the bucket value
-            to format into a string as it sees fit.
-            """
-            if isinstance(self.Key(), ct.Structure):
-                tmp = {}
-                f1 = self.Key._fields_[0][0]
-                f2 = self.Key._fields_[1][0]
-                for k, v in self.items():
-                    bucket = getattr(k, f1)
-                    vals = tmp[bucket] = tmp.get(bucket, [0] * 65)
-                    slot = getattr(k, f2)
-                    vals[slot] = v.value
-                for bucket, vals in tmp.items():
-                    if section_print_fn:
-                        print("\n%s = %s" % (section_header,
-                            section_print_fn(bucket)))
-                    else:
-                        print("\n%s = %r" % (section_header, bucket))
-                    self._print_log2_hist(vals, val_type, 0)
-            else:
-                vals = [0] * 65
-                for k, v in self.items():
-                    vals[k.value] = v.value
-                self._print_log2_hist(vals, val_type, 0)
-
-        def _print_log2_hist(self, vals, val_type, val_max):
-            global stars_max
-            log2_dist_max = 64
-            idx_max = -1
-
-            for i, v in enumerate(vals):
-                if v > 0: idx_max = i
-                if v > val_max: val_max = v
-
-            if idx_max <= 32:
-                header = "     %-19s : count     distribution"
-                body = "%10d -> %-10d : %-8d |%-*s|"
-                stars = stars_max
-            else:
-                header = "               %-29s : count     distribution"
-                body = "%20d -> %-20d : %-8d |%-*s|"
-                stars = int(stars_max / 2)
-
-            if idx_max > 0:
-                print(header % val_type);
-            for i in range(1, idx_max + 1):
-                low = (1 << i) >> 1
-                high = (1 << i) - 1
-                if (low == high):
-                    low -= 1
-                val = vals[i]
-                print(body % (low, high, val, stars,
-                              self._stars(val, val_max, stars)))
-
-
-        def __iter__(self):
-            return BPF.Table.Iter(self, self.Key)
-
-        def iter(self): return self.__iter__()
-        def keys(self): return self.__iter__()
-
-        class Iter(object):
-            def __init__(self, table, keytype):
-                self.Key = keytype
-                self.table = table
-                k = self.Key()
-                kp = ct.pointer(k)
-                # if 0 is a valid key, try a few alternatives
-                if k in table:
-                    ct.memset(kp, 0xff, ct.sizeof(k))
-                    if k in table:
-                        ct.memset(kp, 0x55, ct.sizeof(k))
-                        if k in table:
-                            raise Exception("Unable to allocate iterator")
-                self.key = k
-            def __iter__(self):
-                return self
-            def __next__(self):
-                return self.next()
-            def next(self):
-                self.key = self.table.next(self.key)
-                return self.key
-
-        def next(self, key):
-            next_key = self.Key()
-            next_key_p = ct.pointer(next_key)
-            key_p = ct.pointer(key)
-            res = lib.bpf_get_next_key(self.map_fd,
-                    ct.cast(key_p, ct.c_void_p),
-                    ct.cast(next_key_p, ct.c_void_p))
-            if res < 0:
-                raise StopIteration()
-            return next_key
-
     @staticmethod
     def _find_file(filename):
         """ If filename is invalid, search in ./ of argv[0] """
@@ -593,7 +236,7 @@ class BPF(object):
             if not leaf_desc:
                 raise Exception("Failed to load BPF Table %s leaf desc" % name)
             leaftype = BPF._decode_table_type(json.loads(leaf_desc.decode()))
-        return BPF.Table(self, map_id, map_fd, keytype, leaftype)
+        return Table(self, map_id, map_fd, keytype, leaftype)
 
     def __getitem__(self, key):
         if key not in self.tables:

src/python/bcc/libbcc.py

+# Copyright 2015 PLUMgrid
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ctypes as ct
+
+lib = ct.CDLL("libbcc.so.0")
+
+# keep in sync with bpf_common.h
+lib.bpf_module_create_b.restype = ct.c_void_p
+lib.bpf_module_create_b.argtypes = [ct.c_char_p, ct.c_char_p, ct.c_uint]
+lib.bpf_module_create_c.restype = ct.c_void_p
+lib.bpf_module_create_c.argtypes = [ct.c_char_p, ct.c_uint,
+        ct.POINTER(ct.c_char_p), ct.c_int]
+lib.bpf_module_create_c_from_string.restype = ct.c_void_p
+lib.bpf_module_create_c_from_string.argtypes = [ct.c_char_p, ct.c_uint,
+        ct.POINTER(ct.c_char_p), ct.c_int]
+lib.bpf_module_destroy.restype = None
+lib.bpf_module_destroy.argtypes = [ct.c_void_p]
+lib.bpf_module_license.restype = ct.c_char_p
+lib.bpf_module_license.argtypes = [ct.c_void_p]
+lib.bpf_module_kern_version.restype = ct.c_uint
+lib.bpf_module_kern_version.argtypes = [ct.c_void_p]
+lib.bpf_num_functions.restype = ct.c_ulonglong
+lib.bpf_num_functions.argtypes = [ct.c_void_p]
+lib.bpf_function_name.restype = ct.c_char_p
+lib.bpf_function_name.argtypes = [ct.c_void_p, ct.c_ulonglong]
+lib.bpf_function_start.restype = ct.c_void_p
+lib.bpf_function_start.argtypes = [ct.c_void_p, ct.c_char_p]
+lib.bpf_function_size.restype = ct.c_size_t
+lib.bpf_function_size.argtypes = [ct.c_void_p, ct.c_char_p]
+lib.bpf_table_id.restype = ct.c_ulonglong
+lib.bpf_table_id.argtypes = [ct.c_void_p, ct.c_char_p]
+lib.bpf_table_fd.restype = ct.c_int
+lib.bpf_table_fd.argtypes = [ct.c_void_p, ct.c_char_p]
+lib.bpf_table_type_id.restype = ct.c_int
+lib.bpf_table_type_id.argtypes = [ct.c_void_p, ct.c_ulonglong]
+lib.bpf_table_max_entries_id.restype = ct.c_ulonglong
+lib.bpf_table_max_entries_id.argtypes = [ct.c_void_p, ct.c_ulonglong]
+lib.bpf_table_key_desc.restype = ct.c_char_p
+lib.bpf_table_key_desc.argtypes = [ct.c_void_p, ct.c_char_p]
+lib.bpf_table_leaf_desc.restype = ct.c_char_p
+lib.bpf_table_leaf_desc.argtypes = [ct.c_void_p, ct.c_char_p]
+lib.bpf_table_key_snprintf.restype = ct.c_int
+lib.bpf_table_key_snprintf.argtypes = [ct.c_void_p, ct.c_ulonglong,
+        ct.c_char_p, ct.c_ulonglong, ct.c_void_p]
+lib.bpf_table_leaf_snprintf.restype = ct.c_int
+lib.bpf_table_leaf_snprintf.argtypes = [ct.c_void_p, ct.c_ulonglong,
+        ct.c_char_p, ct.c_ulonglong, ct.c_void_p]
+lib.bpf_table_key_sscanf.restype = ct.c_int
+lib.bpf_table_key_sscanf.argtypes = [ct.c_void_p, ct.c_ulonglong,
+        ct.c_char_p, ct.c_void_p]
+lib.bpf_table_leaf_sscanf.restype = ct.c_int
+lib.bpf_table_leaf_sscanf.argtypes = [ct.c_void_p, ct.c_ulonglong,
+        ct.c_char_p, ct.c_void_p]
+
+# keep in sync with libbpf.h
+lib.bpf_get_next_key.restype = ct.c_int
+lib.bpf_get_next_key.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p]
+lib.bpf_lookup_elem.restype = ct.c_int
+lib.bpf_lookup_elem.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p]
+lib.bpf_update_elem.restype = ct.c_int
+lib.bpf_update_elem.argtypes = [ct.c_int, ct.c_void_p, ct.c_void_p,
+        ct.c_ulonglong]
+lib.bpf_delete_elem.restype = ct.c_int
+lib.bpf_delete_elem.argtypes = [ct.c_int, ct.c_void_p]
+lib.bpf_open_raw_sock.restype = ct.c_int
+lib.bpf_open_raw_sock.argtypes = [ct.c_char_p]
+lib.bpf_attach_socket.restype = ct.c_int
+lib.bpf_attach_socket.argtypes = [ct.c_int, ct.c_int]
+lib.bpf_prog_load.restype = ct.c_int
+lib.bpf_prog_load.argtypes = [ct.c_int, ct.c_void_p, ct.c_size_t,
+        ct.c_char_p, ct.c_uint, ct.c_char_p, ct.c_uint]
+lib.bpf_attach_kprobe.restype = ct.c_void_p
+_CB_TYPE = ct.CFUNCTYPE(None, ct.py_object, ct.c_int,
+        ct.c_ulonglong, ct.POINTER(ct.c_ulonglong))
+_RAW_CB_TYPE = ct.CFUNCTYPE(None, ct.py_object, ct.c_void_p, ct.c_int)
+lib.bpf_attach_kprobe.argtypes = [ct.c_int, ct.c_char_p, ct.c_char_p, ct.c_int,
+        ct.c_int, ct.c_int, _CB_TYPE, ct.py_object]
+lib.bpf_detach_kprobe.restype = ct.c_int
+lib.bpf_detach_kprobe.argtypes = [ct.c_char_p]
+lib.bpf_attach_uprobe.restype = ct.c_void_p
+lib.bpf_attach_uprobe.argtypes = [ct.c_int, ct.c_char_p, ct.c_char_p, ct.c_int,
+        ct.c_int, ct.c_int, _CB_TYPE, ct.py_object]
+lib.bpf_detach_uprobe.restype = ct.c_int
+lib.bpf_detach_uprobe.argtypes = [ct.c_char_p]
+lib.bpf_open_perf_buffer.restype = ct.c_void_p
+lib.bpf_open_perf_buffer.argtypes = [_RAW_CB_TYPE, ct.py_object, ct.c_int, ct.c_int]
+lib.perf_reader_poll.restype = ct.c_int
+lib.perf_reader_poll.argtypes = [ct.c_int, ct.POINTER(ct.c_void_p), ct.c_int]
+lib.perf_reader_free.restype = None
+lib.perf_reader_free.argtypes = [ct.c_void_p]
+lib.perf_reader_fd.restype = int
+lib.perf_reader_fd.argtypes = [ct.c_void_p]

src/python/bcc/table.py

+# Copyright 2015 PLUMgrid
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import MutableMapping
+import ctypes as ct
+
+from .libbcc import lib, _RAW_CB_TYPE
+
+BPF_MAP_TYPE_HASH = 1
+BPF_MAP_TYPE_ARRAY = 2
+BPF_MAP_TYPE_PROG_ARRAY = 3
+BPF_MAP_TYPE_PERF_EVENT_ARRAY = 4
+
+stars_max = 40
+
+# helper functions, consider moving these to a utils module
+def _stars(val, val_max, width):
+    i = 0
+    text = ""
+    while (1):
+        if (i > (width * val / val_max) - 1) or (i > width - 1):
+            break
+        text += "*"
+        i += 1
+    if val > val_max:
+        text = text[:-1] + "+"
+    return text
+
+
+def _print_log2_hist(vals, val_type):
+    global stars_max
+    log2_dist_max = 64
+    idx_max = -1
+    val_max = 0
+
+    for i, v in enumerate(vals):
+        if v > 0: idx_max = i
+        if v > val_max: val_max = v
+
+    if idx_max <= 32:
+        header = "     %-19s : count     distribution"
+        body = "%10d -> %-10d : %-8d |%-*s|"
+        stars = stars_max
+    else:
+        header = "               %-29s : count     distribution"
+        body = "%20d -> %-20d : %-8d |%-*s|"
+        stars = int(stars_max / 2)
+
+    if idx_max > 0:
+        print(header % val_type);
+    for i in range(1, idx_max + 1):
+        low = (1 << i) >> 1
+        high = (1 << i) - 1
+        if (low == high):
+            low -= 1
+        val = vals[i]
+        print(body % (low, high, val, stars,
+                      _stars(val, val_max, stars)))
+
+
+def Table(bpf, map_id, map_fd, keytype, leaftype):
+    """Table(bpf, map_id, map_fd, keytype, leaftype)
+
+    Create a python object out of a reference to a bpf table handle"""
+
+    ttype = lib.bpf_table_type_id(bpf.module, map_id)
+    t = None
+    if ttype == BPF_MAP_TYPE_HASH:
+        t = HashTable(bpf, map_id, map_fd, keytype, leaftype)
+    elif ttype == BPF_MAP_TYPE_ARRAY:
+        t = Array(bpf, map_id, map_fd, keytype, leaftype)
+    elif ttype == BPF_MAP_TYPE_PROG_ARRAY:
+        t = ProgArray(bpf, map_id, map_fd, keytype, leaftype)
+    elif ttype == BPF_MAP_TYPE_PERF_EVENT_ARRAY:
+        t = PerfEventArray(bpf, map_id, map_fd, keytype, leaftype)
+    if t == None:
+        raise Exception("Unknown table type %d" % ttype)
+    return t
+
+
+class TableBase(MutableMapping):
+
+    def __init__(self, bpf, map_id, map_fd, keytype, leaftype):
+        self.bpf = bpf
+        self.map_id = map_id
+        self.map_fd = map_fd
+        self.Key = keytype
+        self.Leaf = leaftype
+        self.ttype = lib.bpf_table_type_id(self.bpf.module, self.map_id)
+        self._cbs = {}
+
+    def key_sprintf(self, key):
+        key_p = ct.pointer(key)
+        buf = ct.create_string_buffer(ct.sizeof(self.Key) * 8)
+        res = lib.bpf_table_key_snprintf(self.bpf.module, self.map_id,
+                buf, len(buf), key_p)
+        if res < 0:
+            raise Exception("Could not printf key")
+        return buf.value
+
+    def leaf_sprintf(self, leaf):
+        leaf_p = ct.pointer(leaf)
+        buf = ct.create_string_buffer(ct.sizeof(self.Leaf) * 8)
+        res = lib.bpf_table_leaf_snprintf(self.bpf.module, self.map_id,
+                buf, len(buf), leaf_p)
+        if res < 0:
+            raise Exception("Could not printf leaf")
+        return buf.value
+
+    def key_scanf(self, key_str):
+        key = self.Key()
+        key_p = ct.pointer(key)
+        res = lib.bpf_table_key_sscanf(self.bpf.module, self.map_id,
+                key_str, key_p)
+        if res < 0:
+            raise Exception("Could not scanf key")
+        return key
+
+    def leaf_scanf(self, leaf_str):
+        leaf = self.Leaf()
+        leaf_p = ct.pointer(leaf)
+        res = lib.bpf_table_leaf_sscanf(self.bpf.module, self.map_id,
+                leaf_str, leaf_p)
+        if res < 0:
+            raise Exception("Could not scanf leaf")
+        return leaf
+
+    def __getitem__(self, key):
+        key_p = ct.pointer(key)
+        leaf = self.Leaf()
+        leaf_p = ct.pointer(leaf)
+        res = lib.bpf_lookup_elem(self.map_fd,
+                ct.cast(key_p, ct.c_void_p),
+                ct.cast(leaf_p, ct.c_void_p))
+        if res < 0:
+            raise KeyError
+        return leaf
+
+    def __setitem__(self, key, leaf):
+        key_p = ct.pointer(key)
+        leaf_p = ct.pointer(leaf)
+        res = lib.bpf_update_elem(self.map_fd,
+                ct.cast(key_p, ct.c_void_p),
+                ct.cast(leaf_p, ct.c_void_p), 0)
+        if res < 0:
+            raise Exception("Could not update table")
+
+    # override the MutableMapping's implementation of these since they
+    # don't handle KeyError nicely
+    def itervalues(self):
+        for key in self:
+            # a map entry may be deleted in between discovering the key and
+            # fetching the value, suppress such errors
+            try:
+                yield self[key]
+            except KeyError:
+                pass
+
+    def iteritems(self):
+        for key in self:
+            try:
+                yield (key, self[key])
+            except KeyError:
+                pass
+
+    def items(self):
+        return [item for item in self.iteritems()]
+
+    def values(self):
+        return [value for value in self.itervalues()]
+
+    def clear(self):
+        # default clear uses popitem, which can race with the bpf prog
+        for k in self.keys():
+            self.__delitem__(k)
+
+    def __iter__(self):
+        return TableBase.Iter(self, self.Key)
+
+    def iter(self): return self.__iter__()
+    def keys(self): return self.__iter__()
+
+    class Iter(object):
+        def __init__(self, table, keytype):
+            self.Key = keytype
+            self.table = table
+            k = self.Key()
+            kp = ct.pointer(k)
+            # if 0 is a valid key, try a few alternatives
+            if k in table:
+                ct.memset(kp, 0xff, ct.sizeof(k))
+                if k in table:
+                    ct.memset(kp, 0x55, ct.sizeof(k))
+                    if k in table:
+                        raise Exception("Unable to allocate iterator")
+            self.key = k
+        def __iter__(self):
+            return self
+        def __next__(self):
+            return self.next()
+        def next(self):
+            self.key = self.table.next(self.key)
+            return self.key
+
+    def next(self, key):
+        next_key = self.Key()
+        next_key_p = ct.pointer(next_key)
+        key_p = ct.pointer(key)
+        res = lib.bpf_get_next_key(self.map_fd,
+                ct.cast(key_p, ct.c_void_p),
+                ct.cast(next_key_p, ct.c_void_p))
+        if res < 0:
+            raise StopIteration()
+        return next_key
+
+    def print_log2_hist(self, val_type="value", section_header="Bucket ptr",
+            section_print_fn=None):
+        """print_log2_hist(val_type="value", section_header="Bucket ptr",
+                           section_print_fn=None)
+
+        Prints a table as a log2 histogram. The table must be stored as
+        log2. The val_type argument is optional, and is a column header.
+        If the histogram has a secondary key, multiple tables will print
+        and section_header can be used as a header description for each.
+        If section_print_fn is not None, it will be passed the bucket value
+        to format into a string as it sees fit.
+        """
+        if isinstance(self.Key(), ct.Structure):
+            tmp = {}
+            f1 = self.Key._fields_[0][0]
+            f2 = self.Key._fields_[1][0]
+            for k, v in self.items():
+                bucket = getattr(k, f1)
+                vals = tmp[bucket] = tmp.get(bucket, [0] * 65)
+                slot = getattr(k, f2)
+                vals[slot] = v.value
+            for bucket, vals in tmp.items():
+                if section_print_fn:
+                    print("\n%s = %s" % (section_header,
+                        section_print_fn(bucket)))
+                else:
+                    print("\n%s = %r" % (section_header, bucket))
+                _print_log2_hist(vals, val_type)
+        else:
+            vals = [0] * 65
+            for k, v in self.items():
+                vals[k.value] = v.value
+            _print_log2_hist(vals, val_type)
+
+
+class HashTable(TableBase):
+    def __init__(self, *args, **kwargs):
+        super(HashTable, self).__init__(*args, **kwargs)
+
+    def __len__(self):
+        i = 0
+        for k in self: i += 1
+        return i
+
+    def __delitem__(self, key):
+        key_p = ct.pointer(key)
+        res = lib.bpf_delete_elem(self.map_fd, ct.cast(key_p, ct.c_void_p))
+        if res < 0:
+            raise KeyError
+
+
+class ArrayBase(TableBase):
+    def __init__(self, *args, **kwargs):
+        super(ArrayBase, self).__init__(*args, **kwargs)
+        self.max_entries = int(lib.bpf_table_max_entries_id(self.bpf.module,
+                self.map_id))
+
+    def _normalize_key(self, key):
+        if isinstance(key, int):
+            if key < 0:
+                key = len(self) + key
+            key = self.Key(key)
+        if not isinstance(key, ct._SimpleCData):
+            raise IndexError("Array index must be an integer type")
+        if key.value >= len(self):
+            raise IndexError("Array index out of range")
+        return key
+
+    def __len__(self):
+        return self.max_entries
+
+    def __getitem__(self, key):
+        key = self._normalize_key(key)
+        return super(ArrayBase, self).__getitem__(key)
+
+    def __setitem__(self, key, leaf):
+        key = self._normalize_key(key)
+        super(ArrayBase, self).__setitem__(key, leaf)
+
+    def __delitem__(self, key):
+        key = self._normalize_key(key)
+        key_p = ct.pointer(key)
+
+        # Deleting from array type maps does not have an effect, so
+        # zero out the entry instead.
+        leaf = self.Leaf()
+        leaf_p = ct.pointer(leaf)
+        res = lib.bpf_update_elem(self.map_fd, ct.cast(key_p, ct.c_void_p),
+                ct.cast(leaf_p, ct.c_void_p), 0)
+        if res < 0:
+            raise Exception("Could not clear item")
+
+    def __iter__(self):
+        return ArrayBase.Iter(self, self.Key)
+
+    class Iter(object):
+        def __init__(self, table, keytype):
+            self.Key = keytype
+            self.table = table
+            self.i = -1
+
+        def __iter__(self):
+            return self
+        def __next__(self):
+            return self.next()
+        def next(self):
+            self.i += 1
+            if self.i == len(self.table):
+                raise StopIteration()
+            return self.Key(self.i)
+
+class Array(ArrayBase):
+    def __init__(self, *args, **kwargs):
+        super(Array, self).__init__(*args, **kwargs)
+
+
+class ProgArray(ArrayBase):
+    def __init__(self, *args, **kwargs):
+        super(ProgArray, self).__init__(*args, **kwargs)
+
+    def __setitem__(self, key, leaf):
+        if isinstance(leaf, int):
+            leaf = self.Leaf(leaf)
+        if isinstance(leaf, self.bpf.Function):
+            leaf = self.Leaf(leaf.fd)
+        super(ProgArray, self).__setitem__(key, leaf)
+
+
+class PerfEventArray(ArrayBase):
+    def __init__(self, *args, **kwargs):
+        super(PerfEventArray, self).__init__(*args, **kwargs)
+
+    def __delitem__(self, key):
+        super(PerfEventArray, self).__init__(key)
+        self.close_perf_buffer(key)
+
+    def open_perf_buffer(self, callback):
+        """open_perf_buffers(callback)
+
+        Opens a set of per-cpu ring buffer to receive custom perf event
+        data from the bpf program. The callback will be invoked for each
+        event submitted from the kernel, up to millions per second.
+        """
+
+        for i in range(0, multiprocessing.cpu_count()):
+            self._open_perf_buffer(i, callback)
+
+    def _open_perf_buffer(self, cpu, callback):
+        fn = _RAW_CB_TYPE(lambda _, data, size: callback(cpu, data, size))
+        reader = lib.bpf_open_perf_buffer(fn, None, -1, cpu)
+        if not reader:
+            raise Exception("Could not open perf buffer")
+        fd = lib.perf_reader_fd(reader)
+        self[self.Key(cpu)] = self.Leaf(fd)
+        open_kprobes[(id(self), cpu)] = reader
+        # keep a refcnt
+        self._cbs[cpu] = fn
+
+    def close_perf_buffer(self, key):
+        reader = open_kprobes.get((id(self), key))
+        if reader:
+            lib.perf_reader_free(reader)
+            del(open_kprobes[(id(self), key)])
+        del self._cbs[key]

tests/cc/test_array.py

@@ -21,5 +21,20 @@ class TestArray(TestCase):
                 self.assertEqual(v.value, 1000)
         self.assertEqual(len(t1), 128)
 
+    def test_native_type(self):
+        b = BPF(text="""BPF_TABLE("array", int, u64, table1, 128);""")
+        t1 = b["table1"]
+        t1[0] = c_ulonglong(100)
+        t1[-2] = c_ulonglong(37)
+        t1[127] = c_ulonglong(1000)
+        for i, v in t1.items():
+            if i.value == 0:
+                self.assertEqual(v.value, 100)
+            if i.value == 127:
+                self.assertEqual(v.value, 1000)
+        self.assertEqual(len(t1), 128)
+        self.assertEqual(t1[-2].value, 37)
+        self.assertEqual(t1[-1].value, t1[127].value)
+
 if __name__ == "__main__":
     main()

tests/cc/test_trace3.py

@@ -35,6 +35,7 @@ class TestBlkRequest(TestCase):
         for key, leaf in self.latency.items():
             print("latency %u:" % key.value, "count %u" % leaf.value)
         sys.stdout.flush()
+        self.assertEqual(len(list(self.latency.keys())), len(self.latency))
 
 if __name__ == "__main__":
     main()

tests/cc/test_trace4.py

@@ -12,7 +12,7 @@ class TestKprobeRgx(TestCase):
         self.b = BPF(text="""
         typedef struct { int idx; } Key;
         typedef struct { u64 val; } Val;
-        BPF_TABLE("array", Key, Val, stats, 3);
+        BPF_TABLE("hash", Key, Val, stats, 3);
         int hello(void *ctx) {
           stats.lookup_or_init(&(Key){1}, &(Val){0})->val++;
           return 0;