Python 3 compatibility fixes around string handling (#986)

This fixes the bcc module and all the affected tools for issues related to string handling in Python 3. Specifically, when passing Python strings to C libraries they are encoded as ASCII, and when constructing Python strings from C strings, they are decoded first.

src/python/bcc/__init__.py

@@ -759,11 +759,12 @@ class BPF(object):
     def get_user_functions_and_addresses(name, sym_re):
         addresses = []
         def sym_cb(sym_name, addr):
-            if re.match(sym_re, sym_name):
-                addresses.append((sym_name, addr))
+            dname = sym_name.decode()
+            if re.match(sym_re, dname):
+                addresses.append((dname, addr))
             return 0
 
-        res = lib.bcc_foreach_symbol(name, _SYM_CB_TYPE(sym_cb))
+        res = lib.bcc_foreach_symbol(name.encode('ascii'), _SYM_CB_TYPE(sym_cb))
         if res < 0:
             raise Exception("Error %d enumerating symbols in %s" % (res, name))
         return addresses

src/python/bcc/usdt.py

@@ -138,7 +138,8 @@ class USDT(object):
                     "either a pid or a binary path must be specified")
 
     def enable_probe(self, probe, fn_name):
-        if lib.bcc_usdt_enable_probe(self.context, probe, fn_name) != 0:
+        if lib.bcc_usdt_enable_probe(self.context, probe.encode('ascii'),
+                fn_name.encode('ascii')) != 0:
             raise USDTException(
                     ("failed to enable probe '%s'; a possible cause " +
                      "can be that the probe requires a pid to enable") %
@@ -146,7 +147,8 @@ class USDT(object):
                   )
 
     def enable_probe_or_bail(self, probe, fn_name):
-        if lib.bcc_usdt_enable_probe(self.context, probe, fn_name) != 0:
+        if lib.bcc_usdt_enable_probe(self.context, probe.encode('ascii'),
+                fn_name.encode('ascii')) != 0:
             print(
 """Error attaching USDT probes: the specified pid might not contain the
 given language's runtime, or the runtime was not built with the required
@@ -175,7 +177,7 @@ tplist tool.""")
     def attach_uprobes(self, bpf):
         probes = self.enumerate_active_probes()
         for (binpath, fn_name, addr, pid) in probes:
-            bpf.attach_uprobe(name=binpath, fn_name=fn_name,
+            bpf.attach_uprobe(name=binpath.decode(), fn_name=fn_name.decode(),
                               addr=addr, pid=pid)
 
     def enumerate_active_probes(self):

tools/bashreadline.py

@@ -56,7 +56,7 @@ print("%-9s %-6s %s" % ("TIME", "PID", "COMMAND"))
 
 def print_event(cpu, data, size):
     event = ct.cast(data, ct.POINTER(Data)).contents
-    print("%-9s %-6d %s" % (strftime("%H:%M:%S"), event.pid, event.str))
+    print("%-9s %-6d %s" % (strftime("%H:%M:%S"), event.pid, event.str.decode()))
 
 b["events"].open_perf_buffer(print_event)
 while 1:

tools/biosnoop.py

@@ -175,7 +175,8 @@ def print_event(cpu, data, size):
         delta = float(delta) + (event.ts - prev_ts)
 
     print("%-14.9f %-14.14s %-6s %-7s %-2s %-9s %-7s %7.2f" % (
-        delta / 1000000, event.name, event.pid, event.disk_name, rwflg, val,
+        delta / 1000000, event.name.decode(), event.pid,
+        event.disk_name.decode(), rwflg, val,
         event.len, float(event.delta) / 1000000))
 
     prev_ts = event.ts

tools/biotop.py

@@ -212,9 +212,9 @@ while 1:
 
         # print line
         avg_ms = (float(v.us) / 1000) / v.io
-        print("%-6d %-16s %1s %-3d %-3d %-8s %5s %7s %6.2f" % (k.pid, k.name,
-            "W" if k.rwflag else "R", k.major, k.minor, diskname, v.io,
-            v.bytes / 1024, avg_ms))
+        print("%-6d %-16s %1s %-3d %-3d %-8s %5s %7s %6.2f" % (k.pid,
+            k.name.decode(), "W" if k.rwflag else "R", k.major, k.minor,
+            diskname, v.io, v.bytes / 1024, avg_ms))
 
         line += 1
         if line >= maxrows:

tools/bitesize.py

@@ -72,4 +72,5 @@ dist = b.get_table("dist")
 try:
     sleep(99999999)
 except KeyboardInterrupt:
-    dist.print_log2_hist("Kbytes", "Process Name")
+    dist.print_log2_hist("Kbytes", "Process Name",
+            section_print_fn=bytes.decode)

tools/btrfsslower.py

@@ -311,12 +311,12 @@ def print_event(cpu, data, size):
 
     if (csv):
         print("%d,%s,%d,%s,%d,%d,%d,%s" % (
-            event.ts_us, event.task, event.pid, type, event.size,
-            event.offset, event.delta_us, event.file))
+            event.ts_us, event.task.decode(), event.pid, type, event.size,
+            event.offset, event.delta_us, event.file.decode()))
         return
     print("%-8s %-14.14s %-6s %1s %-7s %-8d %7.2f %s" % (strftime("%H:%M:%S"),
-        event.task, event.pid, type, event.size, event.offset / 1024,
-        float(event.delta_us) / 1000, event.file))
+        event.task.decode(), event.pid, type, event.size, event.offset / 1024,
+        float(event.delta_us) / 1000, event.file.decode()))
 
 # initialize BPF
 b = BPF(text=bpf_text)

tools/cachetop.py

@@ -72,7 +72,7 @@ def get_processes_stats(
     counts = bpf.get_table("counts")
     stats = defaultdict(lambda: defaultdict(int))
     for k, v in counts.items():
-        stats["%d-%d-%s" % (k.pid, k.uid, k.comm)][k.ip] = v.value
+        stats["%d-%d-%s" % (k.pid, k.uid, k.comm.decode())][k.ip] = v.value
     stats_list = []
 
     for pid, count in sorted(stats.items(), key=lambda stat: stat[0]):

tools/capable.py

@@ -148,7 +148,8 @@ def print_event(cpu, data, size):
     else:
         name = "?"
     print("%-9s %-6d %-6d %-16s %-4d %-20s %d" % (strftime("%H:%M:%S"),
-        event.uid, event.pid, event.comm, event.cap, name, event.audit))
+        event.uid, event.pid, event.comm.decode(), event.cap, name,
+        event.audit))
 
 # loop with callback to print_event
 b["events"].open_perf_buffer(print_event)

tools/dcsnoop.py

@@ -147,8 +147,8 @@ start_ts = time.time()
 def print_event(cpu, data, size):
     event = ct.cast(data, ct.POINTER(Data)).contents
     print("%-11.6f %-6d %-16s %1s %s" % (
-            time.time() - start_ts, event.pid, event.comm, mode_s[event.type],
-            event.filename))
+            time.time() - start_ts, event.pid, event.comm.decode(),
+            mode_s[event.type], event.filename.decode()))
 
 # header
 print("%-11s %-6s %-16s %1s %s" % ("TIME(s)", "PID", "COMM", "T", "FILE"))

tools/ext4dist.py

@@ -196,7 +196,7 @@ while (1):
     if args.interval and (not args.notimestamp):
         print(strftime("%H:%M:%S:"))
 
-    dist.print_log2_hist(label, "operation")
+    dist.print_log2_hist(label, "operation", section_print_fn=bytes.decode)
     dist.clear()
 
     countdown -= 1

tools/ext4slower.py

@@ -305,12 +305,12 @@ def print_event(cpu, data, size):
 
     if (csv):
         print("%d,%s,%d,%s,%d,%d,%d,%s" % (
-            event.ts_us, event.task, event.pid, type, event.size,
-            event.offset, event.delta_us, event.file))
+            event.ts_us, event.task.decode(), event.pid, type, event.size,
+            event.offset, event.delta_us, event.file.decode()))
         return
     print("%-8s %-14.14s %-6s %1s %-7s %-8d %7.2f %s" % (strftime("%H:%M:%S"),
-        event.task, event.pid, type, event.size, event.offset / 1024,
-        float(event.delta_us) / 1000, event.file))
+        event.task.decode(), event.pid, type, event.size, event.offset / 1024,
+        float(event.delta_us) / 1000, event.file.decode()))
 
 # initialize BPF
 b = BPF(text=bpf_text)

tools/filelife.py

@@ -132,7 +132,7 @@ print("%-8s %-6s %-16s %-7s %s" % ("TIME", "PID", "COMM", "AGE(s)", "FILE"))
 def print_event(cpu, data, size):
     event = ct.cast(data, ct.POINTER(Data)).contents
     print("%-8s %-6d %-16s %-7.2f %s" % (strftime("%H:%M:%S"), event.pid,
-        event.comm, float(event.delta) / 1000, event.fname))
+        event.comm.decode(), float(event.delta) / 1000, event.fname.decode()))
 
 b["events"].open_perf_buffer(print_event)
 while 1:

tools/fileslower.py

@@ -235,13 +235,13 @@ def print_event(cpu, data, size):
     event = ct.cast(data, ct.POINTER(Data)).contents
 
     ms = float(event.delta_us) / 1000
-    name = event.name
+    name = event.name.decode()
     if event.name_len > DNAME_INLINE_LEN:
         name = name[:-3] + "..."
 
     print("%-8.3f %-14.14s %-6s %1s %-7s %7.2f %s" % (
-        time.time() - start_ts, event.comm, event.pid, mode_s[event.mode],
-        event.sz, ms, name))
+        time.time() - start_ts, event.comm.decode(), event.pid,
+        mode_s[event.mode], event.sz, ms, name))
 
 b["events"].open_perf_buffer(print_event, page_cnt=64)
 while 1:

tools/gethostlatency.py

@@ -123,7 +123,7 @@ print("%-9s %-6s %-16s %10s %s" % ("TIME", "PID", "COMM", "LATms", "HOST"))
 def print_event(cpu, data, size):
     event = ct.cast(data, ct.POINTER(Data)).contents
     print("%-9s %-6d %-16s %10.2f %s" % (strftime("%H:%M:%S"), event.pid,
-        event.comm, (event.delta / 1000000), event.host))
+        event.comm.decode(), (event.delta / 1000000), event.host.decode()))
 
 # loop with callback to print_event
 b["events"].open_perf_buffer(print_event)

tools/hardirqs.py

@@ -148,7 +148,7 @@ while (1):
     else:
         print("%-26s %11s" % ("HARDIRQ", "TOTAL_" + label))
         for k, v in sorted(dist.items(), key=lambda dist: dist[1].value):
-            print("%-26s %11d" % (k.name, v.value / factor))
+            print("%-26s %11d" % (k.name.decode(), v.value / factor))
     dist.clear()
 
     countdown -= 1

tools/killsnoop.py

@@ -131,7 +131,7 @@ def print_event(cpu, data, size):
         return
 
     print("%-9s %-6d %-16s %-4d %-6d %d" % (strftime("%H:%M:%S"),
-        event.pid, event.comm, event.sig, event.tpid, event.ret))
+        event.pid, event.comm.decode(), event.sig, event.tpid, event.ret))
 
 # loop with callback to print_event
 b["events"].open_perf_buffer(print_event)

tools/mdflush.py

@@ -60,8 +60,8 @@ print("%-8s %-6s %-16s %s" % ("TIME", "PID", "COMM", "DEVICE"))
 # process event
 def print_event(cpu, data, size):
     event = ct.cast(data, ct.POINTER(Data)).contents
-    print("%-8s %-6d %-16s %s" % (strftime("%H:%M:%S"), event.pid, event.comm,
-        event.disk))
+    print("%-8s %-6d %-16s %s" % (strftime("%H:%M:%S"), event.pid,
+        event.comm.decode(), event.disk.decode()))
 
 # read events
 b["events"].open_perf_buffer(print_event)

tools/offcputime.py

@@ -276,7 +276,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
             print("    --")
         for addr in user_stack:
             print("    %s" % b.sym(addr, k.tgid))
-        print("    %-16s %s (%d)" % ("-", k.name, k.pid))
+        print("    %-16s %s (%d)" % ("-", k.name.decode(), k.pid))
         print("        %d\n" % v.value)
 
 if missing_stacks > 0:

tools/offwaketime.py

@@ -306,7 +306,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
 
     else:
         # print wakeup name then stack in reverse order
-        print("    %-16s %s %s" % ("waker:", k.waker, k.t_pid))
+        print("    %-16s %s %s" % ("waker:", k.waker.decode(), k.t_pid))
         for addr in waker_user_stack:
             print("    %s" % b.sym(addr, k.tgid))
         if args.delimited:
@@ -324,7 +324,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
             print("    -")
         for addr in target_user_stack:
             print("    %s" % b.sym(addr, k.tgid))
-        print("    %-16s %s %s" % ("target:", k.target, k.w_pid))
+        print("    %-16s %s %s" % ("target:", k.target.decode(), k.w_pid))
         print("        %d\n" % v.value)
 
 if missing_stacks > 0:

tools/oomkill.py

@@ -68,7 +68,8 @@ def print_event(cpu, data, size):
         avgline = stats.read().rstrip()
     print(("%s Triggered by PID %d (\"%s\"), OOM kill of PID %d (\"%s\")"
         ", %d pages, loadavg: %s") % (strftime("%H:%M:%S"), event.fpid,
-        event.fcomm, event.tpid, event.tcomm, event.pages, avgline))
+        event.fcomm.decode(), event.tpid, event.tcomm.decode(), event.pages,
+        avgline))
 
 # initialize BPF
 b = BPF(text=bpf_text)

tools/opensnoop.py

@@ -175,7 +175,7 @@ def print_event(cpu, data, size):
 
     print("%-6d %-16s %4d %3d %s" %
           (event.id & 0xffffffff if args.tid else event.id >> 32,
-           event.comm, fd_s, err, event.fname))
+           event.comm.decode(), fd_s, err, event.fname.decode()))
 
 # loop with callback to print_event
 b["events"].open_perf_buffer(print_event, page_cnt=64)

tools/profile.py

@@ -292,7 +292,7 @@ for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
             print("    --")
         for addr in user_stack:
             print("    %s" % b.sym(addr, k.pid))
-        print("    %-16s %s (%d)" % ("-", k.name, k.pid))
+        print("    %-16s %s (%d)" % ("-", k.name.decode(), k.pid))
         print("        %d\n" % v.value)
 
 # check missing

tools/slabratetop.py

@@ -120,7 +120,7 @@ while 1:
     line = 0
     for k, v in reversed(sorted(counts.items(),
                                 key=lambda counts: counts[1].size)):
-        print("%-32s %6d %10d" % (k.name, v.count, v.size))
+        print("%-32s %6d %10d" % (k.name.decode(), v.count, v.size))
 
         line += 1
         if line >= maxrows:

tools/sslsniff.py

@@ -200,10 +200,11 @@ def print_event(cpu, data, size, rw):
                 " bytes lost) " + "-" * 5
 
     print("%-12s %-18.9f %-16s %-6d %-6d\n%s\n%s\n%s\n\n" % (rw, time_s,
-                                                             event.comm,
+                                                             event.comm.decode(),
                                                              event.pid,
                                                              event.len,
-                                                             s_mark, event.v0,
+                                                             s_mark,
+                                                             event.v0.decode(),
                                                              e_mark))
 
 b["perf_SSL_write"].open_perf_buffer(print_event_write)

tools/stacksnoop.py

@@ -115,7 +115,7 @@ def print_event(cpu, data, size):
 
     if verbose:
         print("%-18.9f %-12.12s %-6d %-3d %s" %
-              (ts, event.comm, event.pid, cpu, function))
+              (ts, event.comm.decode(), event.pid, cpu, function))
     else:
         print("%-18.9f %s" % (ts, function))
 

tools/statsnoop.py

@@ -155,8 +155,8 @@ def print_event(cpu, data, size):
     if args.timestamp:
         print("%-14.9f" % (float(event.ts_ns - start_ts) / 1000000000), end="")
 
-    print("%-6d %-16s %4d %3d %s" % (event.pid, event.comm,
-        fd_s, err, event.fname))
+    print("%-6d %-16s %4d %3d %s" % (event.pid, event.comm.decode(),
+        fd_s, err, event.fname.decode()))
 
 # loop with callback to print_event
 b["events"].open_perf_buffer(print_event, page_cnt=64)

tools/tcpaccept.py

@@ -159,8 +159,9 @@ def print_ipv4_event(cpu, data, size):
         if start_ts == 0:
             start_ts = event.ts_us
         print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
-    print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid, event.task,
-        event.ip, inet_ntop(AF_INET, pack("I", event.daddr)),
+    print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
+        event.task.decode(), event.ip,
+        inet_ntop(AF_INET, pack("I", event.daddr)),
         inet_ntop(AF_INET, pack("I", event.saddr)), event.lport))
 
 def print_ipv6_event(cpu, data, size):
@@ -170,8 +171,8 @@ def print_ipv6_event(cpu, data, size):
         if start_ts == 0:
             start_ts = event.ts_us
         print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
-    print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid, event.task,
-        event.ip, inet_ntop(AF_INET6, event.daddr),
+    print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
+        event.task.decode(), event.ip, inet_ntop(AF_INET6, event.daddr),
         inet_ntop(AF_INET6, event.saddr), event.lport))
 
 # initialize BPF

tools/tcpconnect.py

@@ -201,8 +201,9 @@ def print_ipv4_event(cpu, data, size):
         if start_ts == 0:
             start_ts = event.ts_us
         print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
-    print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid, event.task,
-        event.ip, inet_ntop(AF_INET, pack("I", event.saddr)),
+    print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
+        event.task.decode(), event.ip,
+        inet_ntop(AF_INET, pack("I", event.saddr)),
         inet_ntop(AF_INET, pack("I", event.daddr)), event.dport))
 
 def print_ipv6_event(cpu, data, size):
@@ -213,7 +214,7 @@ def print_ipv6_event(cpu, data, size):
             start_ts = event.ts_us
         print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
     print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
-        event.task, event.ip, inet_ntop(AF_INET6, event.saddr),
+        event.task.decode(), event.ip, inet_ntop(AF_INET6, event.saddr),
         inet_ntop(AF_INET6, event.daddr), event.dport))
 
 # initialize BPF

tools/tcpconnlat.py

@@ -199,8 +199,9 @@ def print_ipv4_event(cpu, data, size):
         if start_ts == 0:
             start_ts = event.ts_us
         print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
-    print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid, event.task,
-        event.ip, inet_ntop(AF_INET, pack("I", event.saddr)),
+    print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid,
+        event.task.decode(), event.ip,
+        inet_ntop(AF_INET, pack("I", event.saddr)),
         inet_ntop(AF_INET, pack("I", event.daddr)), event.dport,
         float(event.delta_us) / 1000))
 
@@ -211,8 +212,8 @@ def print_ipv6_event(cpu, data, size):
         if start_ts == 0:
             start_ts = event.ts_us
         print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
-    print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid, event.task,
-        event.ip, inet_ntop(AF_INET6, event.saddr),
+    print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid,
+        event.task.decode(), event.ip, inet_ntop(AF_INET6, event.saddr),
         inet_ntop(AF_INET6, event.daddr), event.dport,
         float(event.delta_us) / 1000))
 

tools/tcplife.py

@@ -305,7 +305,7 @@ def print_ipv4_event(cpu, data, size):
             print("%.6f," % delta_s, end="")
         else:
             print("%-9.6f " % delta_s, end="")
-    print(format_string % (event.pid, event.task,
+    print(format_string % (event.pid, event.task.decode(),
         "4" if args.wide or args.csv else "",
         inet_ntop(AF_INET, pack("I", event.saddr)), event.ports >> 32,
         inet_ntop(AF_INET, pack("I", event.daddr)), event.ports & 0xffffffff,
@@ -327,7 +327,7 @@ def print_ipv6_event(cpu, data, size):
             print("%.6f," % delta_s, end="")
         else:
             print("%-9.6f " % delta_s, end="")
-    print(format_string % (event.pid, event.task,
+    print(format_string % (event.pid, event.task.decode(),
         "6" if args.wide or args.csv else "",
         inet_ntop(AF_INET6, event.saddr), event.ports >> 32,
         inet_ntop(AF_INET6, event.daddr), event.ports & 0xffffffff,

tools/trace.py

@@ -486,13 +486,13 @@ BPF_PERF_OUTPUT(%s);
                 msg = self._format_message(bpf, event.tgid, values)
                 if not Probe.print_time:
                     print("%-6d %-6d %-12s %-16s %s" %
-                          (event.tgid, event.pid, event.comm,
+                          (event.tgid, event.pid, event.comm.decode(),
                            self._display_function(), msg))
                 else:
                     time = strftime("%H:%M:%S") if Probe.use_localtime else \
                            Probe._time_off_str(event.timestamp_ns)
                     print("%-8s %-6d %-6d %-12s %-16s %s" %
-                          (time[:8], event.tgid, event.pid, event.comm,
+                          (time[:8], event.tgid, event.pid, event.comm.decode(),
                            self._display_function(), msg))
 
                 if self.kernel_stack:

tools/ttysnoop.py

@@ -111,7 +111,7 @@ if not args.noclear:
 # process event
 def print_event(cpu, data, size):
     event = ct.cast(data, ct.POINTER(Data)).contents
-    print("%s" % event.buf[0:event.count], end="")
+    print("%s" % event.buf[0:event.count].decode(), end="")
     sys.stdout.flush()
 
 # loop with callback to print_event

tools/wakeuptime.py

@@ -193,23 +193,23 @@ while (1):
     for k, v in sorted(counts.items(), key=lambda counts: counts[1].value):
         if folded:
             # print folded stack output
-            line = k.waker + ";"
+            line = k.waker.decode() + ";"
             for i in reversed(range(0, maxdepth)):
                 if k.ret[i] == 0:
                     continue
                 line = line + b.ksym(k.ret[i])
                 if i != 0:
                     line = line + ";"
-            print("%s;%s %d" % (line, k.target, v.value))
+            print("%s;%s %d" % (line, k.target.decode(), v.value))
         else:
             # print default multi-line stack output
-            print("    %-16s %s" % ("target:", k.target))
+            print("    %-16s %s" % ("target:", k.target.decode()))
             for i in range(0, maxdepth):
                 if k.ret[i] == 0:
                     break
                 print("    %-16x %s" % (k.ret[i],
                     b.ksym(k.ret[i])))
-            print("    %-16s %s" % ("waker:", k.waker))
+            print("    %-16s %s" % ("waker:", k.waker.decode()))
             print("        %d\n" % v.value)
     counts.clear()