Address round 2 of comments in https://github.com/iovisor/bcc/pull/936

- Specify when `--binary` is needed (statically vs dynamically-linked
  binaries).

- Make `-h`, `_examples.txt`, and man page have concrete examples and
  be more user-friendly.

man/man8/deadlock_detector.8

@@ -38,7 +38,11 @@ CONFIG_BPF and bcc
 show this help message and exit
 .TP
 \--binary BINARY
-If set, use this as the path to the binary for the process.
+If set, trace the mutexes from the binary at this path. For
+statically-linked binaries, this argument is not required.
+For dynamically-linked binaries, this argument is required and should be the
+path of the pthread library the binary is using.
+Example: /lib/x86_64-linux-gnu/libpthread.so.0
 .TP
 \--dump-graph DUMP_GRAPH
 If set, this will dump the mutex graph to the specified file.
@@ -47,40 +51,50 @@ If set, this will dump the mutex graph to the specified file.
 Print statistics about the mutex wait graph.
 .TP
 \--lock-symbols LOCK_SYMBOLS
-Comma-separated list of lock symbols to trace. Default is pthread_mutex_lock
+Comma-separated list of lock symbols to trace. Default is pthread_mutex_lock.
+These symbols cannot be inlined in the binary.
 .TP
 \--unlock-symbols UNLOCK_SYMBOLS
-Comma-separated list of unlock symbols to trace. Default is pthread_mutex_unlock
+Comma-separated list of unlock symbols to trace. Default is
+pthread_mutex_unlock. These symbols cannot be inlined in the binary.
 .TP
 pid
 Pid to trace
 .SH EXAMPLES
 .TP
-Find potential deadlocks in a process:
+Find potential deadlocks in PID 181. The --binary argument is not needed for \
+statically-linked binaries.
 #
-.B deadlock_detector $(pidof binary)
+.B deadlock_detector 181
 .TP
-Find potential deadlocks in a process and dump the mutex wait graph to a file:
+Find potential deadlocks in PID 181. If the process was created from a \
+dynamically-linked executable, the --binary argument is required and must be \
+the path of the pthread library:
 #
-.B deadlock_detector $(pidof binary) --dump-graph graph.json
+.B deadlock_detector 181 --binary /lib/x86_64-linux-gnu/libpthread.so.0
 .TP
-Find potential deadlocks in a process and print mutex wait graph statistics:
+Find potential deadlocks in PID 181. If the process was created from a \
+statically-linked executable, optionally pass the location of the binary. \
+On older kernels without https://lkml.org/lkml/2017/1/13/585, binaries that \
+contain `:` in the path cannot be attached with uprobes. As a workaround, we \
+can create a symlink to the binary, and provide the symlink name instead with \
+the `--binary` option:
 #
-.B deadlock_detector $(pidof binary) --verbose
+.B deadlock_detector 181 --binary /usr/local/bin/lockinversion
 .TP
-Find potential deadlocks in a process with custom mutexes:
+Find potential deadlocks in PID 181 and dump the mutex wait graph to a file:
 #
-.B deadlock_detector $(pidof binary)
-.B      --lock-symbols custom_mutex1_lock,custom_mutex2_lock
-.B      --unlock_symbols custom_mutex1_unlock,custom_mutex2_unlock
+.B deadlock_detector 181 --dump-graph graph.json
 .TP
-Find potential deadlocks in a process, and provide the path to the binary. On \
-older kernels without https://lkml.org/lkml/2017/1/13/585, binaries that \
-contain `:` in the path cannot be attached with uprobes. As a workaround, we \
-can create a symlink to the binary, and provide the symlink name instead with \
-the `--binary` option:
+Find potential deadlocks in PID 181 and print mutex wait graph statistics:
+#
+.B deadlock_detector 181 --verbose
+.TP
+Find potential deadlocks in PID 181 with custom mutexes:
 #
-.B deadlock_detector $(pidof binary) --binary /path/to/program
+.B deadlock_detector 181
+.B      --lock-symbols custom_mutex1_lock,custom_mutex2_lock
+.B      --unlock_symbols custom_mutex1_unlock,custom_mutex2_unlock
 .SH OUTPUT
 This program does not output any fields. Rather, it will keep running until
 it finds a potential deadlock, or the user hits Ctrl-C. If the program finds

tools/deadlock_detector.py

@@ -387,11 +387,36 @@ def strlist(s):
 
 
 def main():
+    examples = '''Examples:
+    deadlock_detector 181        # Analyze PID 181
+
+    deadlock_detector 181 --binary /lib/x86_64-linux-gnu/libpthread.so.0
+                                 # Analyze PID 181 and locks from this binary.
+                                 # If tracing a process that is running from
+                                 # a dynamically-linked binary, this argument
+                                 # is required and should be the path to the
+                                 # pthread library.
+
+    deadlock_detector 181 --verbose
+                                 # Analyze PID 181 and print statistics about
+                                 # the mutex wait graph.
+
+    deadlock_detector 181 --lock-symbols my_mutex_lock1,my_mutex_lock2 \\
+        --unlock-symbols my_mutex_unlock1,my_mutex_unlock2
+                                 # Analyze PID 181 and trace custom mutex
+                                 # symbols instead of pthread mutexes.
+
+    deadlock_detector 181 --dump-graph graph.json
+                                 # Analyze PID 181 and dump the mutex wait
+                                 # graph to graph.json.
+    '''
     parser = argparse.ArgumentParser(
         description=(
             'Detect potential deadlocks (lock inversions) in a running binary.'
-            ' Must be run as root.'
-        )
+            '\nMust be run as root.'
+        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=examples,
     )
     parser.add_argument('pid', type=int, help='Pid to trace')
     # Binaries with `:` in the path will fail to attach uprobes on kernels
@@ -401,7 +426,11 @@ def main():
         '--binary',
         type=str,
         default='',
-        help='If set, use this as the path to the binary for the process.',
+        help='If set, trace the mutexes from the binary at this path. '
+        'For statically-linked binaries, this argument is not required. '
+        'For dynamically-linked binaries, this argument is required and '
+        'should be the path of the pthread library the binary is using. '
+        'Example: /lib/x86_64-linux-gnu/libpthread.so.0',
     )
     parser.add_argument(
         '--dump-graph',
@@ -419,14 +448,14 @@ def main():
         type=strlist,
         default=['pthread_mutex_lock'],
         help='Comma-separated list of lock symbols to trace. Default is '
-        'pthread_mutex_lock',
+        'pthread_mutex_lock. These symbols cannot be inlined in the binary.',
     )
     parser.add_argument(
         '--unlock-symbols',
         type=strlist,
         default=['pthread_mutex_unlock'],
         help='Comma-separated list of unlock symbols to trace. Default is '
-        'pthread_mutex_unlock',
+        'pthread_mutex_unlock. These symbols cannot be inlined in the binary.',
     )
     args = parser.parse_args()
     if not args.binary:

tools/deadlock_detector_example.txt

@@ -35,7 +35,7 @@ after the mutex has been created. As a result, this tool will not find
 potential deadlocks that involve only one mutex.
 
 
-# ./deadlock_detector.py $(pidof lockinversion)
+# ./deadlock_detector.py 181
 Tracing... Hit Ctrl-C to end.
 ----------------
 Potential Deadlock Detected!
@@ -239,7 +239,31 @@ uses a similar format as ThreadSanitizer
 (https://github.com/google/sanitizers/wiki/ThreadSanitizerDeadlockDetector).
 
 
-# ./deadlock_detector.py $(pidof program) --dump-graph graph.json --verbose
+# ./deadlock_detector.py 181 --binary /usr/local/bin/lockinversion
+
+Tracing... Hit Ctrl-C to end.
+^C
+
+If the traced process is instantiated from a statically-linked executable,
+this argument is optional, and the program will determine the path of the
+executable from the pid. However, on older kernels without this patch
+("uprobe: Find last occurrence of ':' when parsing uprobe PATH:OFFSET",
+https://lkml.org/lkml/2017/1/13/585), binaries that contain `:` in the path
+cannot be attached with uprobes. As a workaround, we can create a symlink
+to the binary, and provide the symlink name instead to the `--binary` option.
+
+
+# ./deadlock_detector.py 181 --binary /lib/x86_64-linux-gnu/libpthread.so.0
+
+Tracing... Hit Ctrl-C to end.
+^C
+
+If the traced process is instantiated from a dynamically-linked executable,
+this argument is required and needs to be the path to the pthread shared
+library used by the executable.
+
+
+# ./deadlock_detector.py 181 --dump-graph graph.json --verbose
 
 Tracing... Hit Ctrl-C to end.
 Mutexes: 0, Edges: 0
@@ -260,7 +284,7 @@ serialize the graph to analyze it later, you can pass the `--dump-graph FILE`
 flag, and the program will serialize the graph in json.
 
 
-# ./deadlock_detector.py $(pidof program) --lock-symbols custom_mutex1_lock,custom_mutex2_lock --unlock_symbols custom_mutex1_unlock,custom_mutex2_unlock --verbose
+# ./deadlock_detector.py 181 --lock-symbols custom_mutex1_lock,custom_mutex2_lock --unlock_symbols custom_mutex1_unlock,custom_mutex2_unlock --verbose
 
 Tracing... Hit Ctrl-C to end.
 Mutexes: 0, Edges: 0
@@ -281,21 +305,6 @@ Note that if the symbols are inlined in the binary, then this program can result
 in false positives.
 
 
-# ./deadlock_detector.py $(pidof program) --binary /path/to/my/program
-
-Tracing... Hit Ctrl-C to end.
-^C
-
-You can optionally pass the path to the binary for your process.
-
-By default, the path to the binary is not required and all the user needs to
-provide is the pid of the process. However, on older kernels without this patch
-("uprobe: Find last occurrence of ':' when parsing uprobe PATH:OFFSET",
-https://lkml.org/lkml/2017/1/13/585), binaries that contain `:` in the path
-cannot be attached with uprobes. As a workaround, we can create a symlink
-to the binary, and provide the symlink name instead with the `--binary` option.
-
-
 USAGE message:
 
 # ./deadlock_detector.py -h
@@ -305,23 +314,52 @@ usage: deadlock_detector.py [-h] [--binary BINARY] [--dump-graph DUMP_GRAPH]
                             [--unlock-symbols UNLOCK_SYMBOLS]
                             pid
 
-Detect potential deadlocks (lock inversions) in a running binary. Must be run
-as root.
+Detect potential deadlocks (lock inversions) in a running binary.
+Must be run as root.
 
 positional arguments:
   pid                   Pid to trace
 
 optional arguments:
   -h, --help            show this help message and exit
-  --binary BINARY       If set, use this as the path to the binary for the
-                        process.
+  --binary BINARY       If set, trace the mutexes from the binary at this
+                        path. For statically-linked binaries, this argument is
+                        not required. For dynamically-linked binaries, this
+                        argument is required and should be the path of the
+                        pthread library the binary is using. Example:
+                        /lib/x86_64-linux-gnu/libpthread.so.0
   --dump-graph DUMP_GRAPH
                         If set, this will dump the mutex graph to the
                         specified file.
   --verbose             Print statistics about the mutex wait graph.
   --lock-symbols LOCK_SYMBOLS
                         Comma-separated list of lock symbols to trace. Default
-                        is pthread_mutex_lock
+                        is pthread_mutex_lock. These symbols cannot be inlined
+                        in the binary.
   --unlock-symbols UNLOCK_SYMBOLS
                         Comma-separated list of unlock symbols to trace.
-                        Default is pthread_mutex_unlock
+                        Default is pthread_mutex_unlock. These symbols cannot
+                        be inlined in the binary.
+
+Examples:
+    deadlock_detector 181        # Analyze PID 181
+
+    deadlock_detector 181 --binary /lib/x86_64-linux-gnu/libpthread.so.0
+                                 # Analyze PID 181 and locks from this binary.
+                                 # If tracing a process that is running from
+                                 # a dynamically-linked binary, this argument
+                                 # is required and should be the path to the
+                                 # pthread library.
+
+    deadlock_detector 181 --verbose
+                                 # Analyze PID 181 and print statistics about
+                                 # the mutex wait graph.
+
+    deadlock_detector 181 --lock-symbols my_mutex_lock1,my_mutex_lock2 \
+        --unlock-symbols my_mutex_unlock1,my_mutex_unlock2
+                                 # Analyze PID 181 and trace custom mutex
+                                 # symbols instead of pthread mutexes.
+
+    deadlock_detector 181 --dump-graph graph.json
+                                 # Analyze PID 181 and dump the mutex wait
+                                 # graph to graph.json.