Merge tag 'perf-core-for-mingo-4.14-20170816' of...

Merge tag 'perf-core-for-mingo-4.14-20170816' of git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux into perf/core

Pull perf core improvements and fixes:

New features:

- Support exporting Intel PT data to sqlite3 with python perf scripts,
  this is in addition to the postgresql support that was already there (Adrian Hunter)

Infrastructure changes:

- Handle perf tool builds with less features in perf shell tests, such
  as those with NO_LIBDWARF=1 or even without 'perf probe' (Arnaldo Carvalho de Melo)

- Replace '|&' with '2>&1 |' to work with more shells in the just
  introduced perf test shell harness (Kim Phillips)

Architecture related fixes:

- Fix endianness problem when loading parameters in the BPF prologue
  generated by perf, noticed using 'perf test BPF' in s390x systems (Wang Nan, Thomas Richter)
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>

tools/perf/Documentation/intel-pt.txt

@@ -104,9 +104,9 @@ system, asynchronous, interrupt, transaction abort, trace begin, trace end, and
 in transaction, respectively.
 
 While it is possible to create scripts to analyze the data, an alternative
-approach is available to export the data to a postgresql database.  Refer to
-script export-to-postgresql.py for more details, and to script
-call-graph-from-postgresql.py for an example of using the database.
+approach is available to export the data to a sqlite or postgresql database.
+Refer to script export-to-sqlite.py or export-to-postgresql.py for more details,
+and to script call-graph-from-sql.py for an example of using the database.
 
 There is also script intel-pt-events.py which provides an example of how to
 unpack the raw data for power events and PTWRITE.

tools/perf/scripts/python/bin/export-to-sqlite-record

+#!/bin/bash
+
+#
+# export perf data to a sqlite3 database. Can cover
+# perf ip samples (excluding the tracepoints). No special
+# record requirements, just record what you want to export.
+#
+perf record $@

tools/perf/scripts/python/bin/export-to-sqlite-report

+#!/bin/bash
+# description: export perf data to a sqlite3 database
+# args: [database name] [columns] [calls]
+n_args=0
+for i in "$@"
+do
+    if expr match "$i" "-" > /dev/null ; then
+	break
+    fi
+    n_args=$(( $n_args + 1 ))
+done
+if [ "$n_args" -gt 3 ] ; then
+    echo "usage: export-to-sqlite-report [database name] [columns] [calls]"
+    exit
+fi
+if [ "$n_args" -gt 2 ] ; then
+    dbname=$1
+    columns=$2
+    calls=$3
+    shift 3
+elif [ "$n_args" -gt 1 ] ; then
+    dbname=$1
+    columns=$2
+    shift 2
+elif [ "$n_args" -gt 0 ] ; then
+    dbname=$1
+    shift
+fi
+perf script $@ -s "$PERF_EXEC_PATH"/scripts/python/export-to-sqlite.py $dbname $columns $calls

tools/perf/scripts/python/call-graph-from-postgresql.py → tools/perf/scripts/python/call-graph-from-sql.py

 #!/usr/bin/python2
-# call-graph-from-postgresql.py: create call-graph from postgresql database
-# Copyright (c) 2014, Intel Corporation.
+# call-graph-from-sql.py: create call-graph from sql database
+# Copyright (c) 2014-2017, Intel Corporation.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms and conditions of the GNU General Public License,
@@ -11,18 +11,19 @@
 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 # more details.
 
-# To use this script you will need to have exported data using the
-# export-to-postgresql.py script.  Refer to that script for details.
+# To use this script you will need to have exported data using either the
+# export-to-sqlite.py or the export-to-postgresql.py script.  Refer to those
+# scripts for details.
 #
-# Following on from the example in the export-to-postgresql.py script, a
+# Following on from the example in the export scripts, a
 # call-graph can be displayed for the pt_example database like this:
 #
-#	python tools/perf/scripts/python/call-graph-from-postgresql.py pt_example
+#	python tools/perf/scripts/python/call-graph-from-sql.py pt_example
 #
-# Note this script supports connecting to remote databases by setting hostname,
-# port, username, password, and dbname e.g.
+# Note that for PostgreSQL, this script supports connecting to remote databases
+# by setting hostname, port, username, password, and dbname e.g.
 #
-#	python tools/perf/scripts/python/call-graph-from-postgresql.py "hostname=myhost username=myuser password=mypassword dbname=pt_example"
+#	python tools/perf/scripts/python/call-graph-from-sql.py "hostname=myhost username=myuser password=mypassword dbname=pt_example"
 #
 # The result is a GUI window with a tree representing a context-sensitive
 # call-graph.  Expanding a couple of levels of the tree and adjusting column
@@ -160,7 +161,7 @@ class TreeItem():
 				  '( SELECT short_name FROM dsos WHERE id = ( SELECT dso_id FROM symbols WHERE id = ( SELECT symbol_id FROM call_paths WHERE id = call_path_id ) ) ), '
 				  '( SELECT ip FROM call_paths where id = call_path_id ) '
 				  'FROM calls WHERE parent_call_path_id = ' + str(self.call_path_id) + ' AND comm_id = ' + str(self.comm_id) + ' AND thread_id = ' + str(self.thread_id) +
-				  'ORDER BY call_path_id')
+				  ' ORDER BY call_path_id')
 		if not ret:
 			raise Exception("Query failed: " + query.lastError().text())
 		last_call_path_id = 0
@@ -291,29 +292,40 @@ class MainWindow(QMainWindow):
 
 if __name__ == '__main__':
 	if (len(sys.argv) < 2):
-		print >> sys.stderr, "Usage is: call-graph-from-postgresql.py <database name>"
+		print >> sys.stderr, "Usage is: call-graph-from-sql.py <database name>"
 		raise Exception("Too few arguments")
 
 	dbname = sys.argv[1]
 
-	db = QSqlDatabase.addDatabase('QPSQL')
-
-	opts = dbname.split()
-	for opt in opts:
-		if '=' in opt:
-			opt = opt.split('=')
-			if opt[0] == 'hostname':
-				db.setHostName(opt[1])
-			elif opt[0] == 'port':
-				db.setPort(int(opt[1]))
-			elif opt[0] == 'username':
-				db.setUserName(opt[1])
-			elif opt[0] == 'password':
-				db.setPassword(opt[1])
-			elif opt[0] == 'dbname':
-				dbname = opt[1]
-		else:
-			dbname = opt
+	is_sqlite3 = False
+	try:
+		f = open(dbname)
+		if f.read(15) == "SQLite format 3":
+			is_sqlite3 = True
+		f.close()
+	except:
+		pass
+
+	if is_sqlite3:
+		db = QSqlDatabase.addDatabase('QSQLITE')
+	else:
+		db = QSqlDatabase.addDatabase('QPSQL')
+		opts = dbname.split()
+		for opt in opts:
+			if '=' in opt:
+				opt = opt.split('=')
+				if opt[0] == 'hostname':
+					db.setHostName(opt[1])
+				elif opt[0] == 'port':
+					db.setPort(int(opt[1]))
+				elif opt[0] == 'username':
+					db.setUserName(opt[1])
+				elif opt[0] == 'password':
+					db.setPassword(opt[1])
+				elif opt[0] == 'dbname':
+					dbname = opt[1]
+			else:
+				dbname = opt
 
 	db.setDatabaseName(dbname)
 	if not db.open():

tools/perf/scripts/python/export-to-postgresql.py

@@ -59,7 +59,7 @@ import datetime
 #	pt_example=# \q
 #
 # An example of using the database is provided by the script
-# call-graph-from-postgresql.py.  Refer to that script for details.
+# call-graph-from-sql.py.  Refer to that script for details.
 #
 # Tables:
 #
@@ -340,7 +340,8 @@ if branches:
 		'to_sym_offset	bigint,'
 		'to_ip		bigint,'
 		'branch_type	integer,'
-		'in_tx		boolean)')
+		'in_tx		boolean,'
+		'call_path_id	bigint)')
 else:
 	do_query(query, 'CREATE TABLE samples ('
 		'id		bigint		NOT NULL,'

tools/perf/tests/bpf-script-test-prologue.c

@@ -26,9 +26,11 @@ static void (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) =
 	(void *) 6;
 
 SEC("func=null_lseek file->f_mode offset orig")
-int bpf_func__null_lseek(void *ctx, int err, unsigned long f_mode,
+int bpf_func__null_lseek(void *ctx, int err, unsigned long _f_mode,
 			 unsigned long offset, unsigned long orig)
 {
+	fmode_t f_mode = (fmode_t)_f_mode;
+
 	if (err)
 		return 0;
 	if (f_mode & FMODE_WRITE)

tools/perf/tests/shell/lib/probe.sh

+# Arnaldo Carvalho de Melo <acme@kernel.org>, 2017
+
+skip_if_no_perf_probe() {
+	perf probe 2>&1 | grep -q 'is not a perf-command' && return 2
+	return 0
+}

tools/perf/tests/shell/lib/probe_vfs_getname.sh

 # Arnaldo Carvalho de Melo <acme@kernel.org>, 2017
 
-perf probe -l | grep -q probe:vfs_getname
+perf probe -l 2>&1 | grep -q probe:vfs_getname
 had_vfs_getname=$?
 
 cleanup_probe_vfs_getname() {
@@ -12,17 +12,12 @@ cleanup_probe_vfs_getname() {
 add_probe_vfs_getname() {
 	local verbose=$1
 	if [ $had_vfs_getname -eq 1 ] ; then
-               line=$(perf probe -L getname_flags | egrep 'result.*=.*filename;' | sed -r 's/[[:space:]]+([[:digit:]]+)[[:space:]]+result->uptr.*/\1/')
-               perf probe $verbose "vfs_getname=getname_flags:${line} pathname=result->name:string"
+		line=$(perf probe -L getname_flags 2>&1 | egrep 'result.*=.*filename;' | sed -r 's/[[:space:]]+([[:digit:]]+)[[:space:]]+result->uptr.*/\1/')
+		perf probe $verbose "vfs_getname=getname_flags:${line} pathname=result->name:string"
 	fi
 }
 
 skip_if_no_debuginfo() {
-	add_probe_vfs_getname -v 2>&1 | grep -q "^Failed to find the path for kernel" && return 2
-	return 1
-}
-
-skip_if_no_debuginfo() {
-	add_probe_vfs_getname -v 2>&1 | grep -q "^Failed to find the path for kernel" && return 2
+	add_probe_vfs_getname -v 2>&1 | egrep -q "^(Failed to find the path for kernel|Debuginfo-analysis is not supported)" && return 2
 	return 1
 }

tools/perf/tests/shell/probe_vfs_getname.sh

@@ -2,6 +2,10 @@
 #
 # Arnaldo Carvalho de Melo <acme@kernel.org>, 2017
 
+. $(dirname $0)/lib/probe.sh
+
+skip_if_no_perf_probe || exit 2
+
 . $(dirname $0)/lib/probe_vfs_getname.sh
 
 add_probe_vfs_getname || skip_if_no_debuginfo

tools/perf/tests/shell/record+script_probe_vfs_getname.sh

@@ -7,6 +7,10 @@
 
 # Arnaldo Carvalho de Melo <acme@kernel.org>, 2017
 
+. $(dirname $0)/lib/probe.sh
+
+skip_if_no_perf_probe || exit 2
+
 . $(dirname $0)/lib/probe_vfs_getname.sh
 
 perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX)

tools/perf/tests/shell/trace+probe_libc_inet_pton.sh

@@ -8,6 +8,8 @@
 
 # Arnaldo Carvalho de Melo <acme@kernel.org>, 2017
 
+. $(dirname $0)/lib/probe.sh
+
 trace_libc_inet_pton_backtrace() {
 	idx=0
 	expected[0]="PING.*bytes"
@@ -20,7 +22,7 @@ trace_libc_inet_pton_backtrace() {
 	expected[7]="getaddrinfo[[:space:]]\(/usr/lib.*/libc-[0-9]+\.[0-9]+\.so\)$"
 	expected[8]=".*\(.*/bin/ping.*\)$"
 
-	perf trace --no-syscalls -e probe_libc:inet_pton/max-stack=3/ ping -6 -c 1 ::1 |& grep -v ^$ | while read line ; do
+	perf trace --no-syscalls -e probe_libc:inet_pton/max-stack=3/ ping -6 -c 1 ::1 2>&1 | grep -v ^$ | while read line ; do
 		echo $line
 		echo "$line" | egrep -q "${expected[$idx]}"
 		if [ $? -ne 0 ] ; then
@@ -32,6 +34,7 @@ trace_libc_inet_pton_backtrace() {
 	done
 }
 
+skip_if_no_perf_probe && \
 perf probe -q /lib64/libc-*.so inet_pton && \
 trace_libc_inet_pton_backtrace
 err=$?

tools/perf/tests/shell/trace+probe_vfs_getname.sh

@@ -8,6 +8,10 @@
 
 # Arnaldo Carvalho de Melo <acme@kernel.org>, 2017
 
+. $(dirname $0)/lib/probe.sh
+
+skip_if_no_perf_probe || exit 2
+
 . $(dirname $0)/lib/probe_vfs_getname.sh
 
 file=$(mktemp /tmp/temporary_file.XXXXX)

tools/perf/util/bpf-prologue.c

@@ -58,6 +58,46 @@ check_pos(struct bpf_insn_pos *pos)
 	return 0;
 }
 
+/*
+ * Convert type string (u8/u16/u32/u64/s8/s16/s32/s64 ..., see
+ * Documentation/trace/kprobetrace.txt) to size field of BPF_LDX_MEM
+ * instruction (BPF_{B,H,W,DW}).
+ */
+static int
+argtype_to_ldx_size(const char *type)
+{
+	int arg_size = type ? atoi(&type[1]) : 64;
+
+	switch (arg_size) {
+	case 8:
+		return BPF_B;
+	case 16:
+		return BPF_H;
+	case 32:
+		return BPF_W;
+	case 64:
+	default:
+		return BPF_DW;
+	}
+}
+
+static const char *
+insn_sz_to_str(int insn_sz)
+{
+	switch (insn_sz) {
+	case BPF_B:
+		return "BPF_B";
+	case BPF_H:
+		return "BPF_H";
+	case BPF_W:
+		return "BPF_W";
+	case BPF_DW:
+		return "BPF_DW";
+	default:
+		return "UNKNOWN";
+	}
+}
+
 /* Give it a shorter name */
 #define ins(i, p) append_insn((i), (p))
 
@@ -258,9 +298,14 @@ gen_prologue_slowpath(struct bpf_insn_pos *pos,
 	}
 
 	/* Final pass: read to registers */
-	for (i = 0; i < nargs; i++)
-		ins(BPF_LDX_MEM(BPF_DW, BPF_PROLOGUE_START_ARG_REG + i,
+	for (i = 0; i < nargs; i++) {
+		int insn_sz = (args[i].ref) ? argtype_to_ldx_size(args[i].type) : BPF_DW;
+
+		pr_debug("prologue: load arg %d, insn_sz is %s\n",
+			 i, insn_sz_to_str(insn_sz));
+		ins(BPF_LDX_MEM(insn_sz, BPF_PROLOGUE_START_ARG_REG + i,
 				BPF_REG_FP, -BPF_REG_SIZE * (i + 1)), pos);
+	}
 
 	ins(BPF_JMP_IMM(BPF_JA, BPF_REG_0, 0, JMP_TO_SUCCESS_CODE), pos);