Broaden use of doc_extract code, put in ccanlint, and fix ccanlint

compile.

tools/Makefile

-ALL_TOOLS = tools/ccan_depends tools/run_tests tools/doc_extract tools/namespacize
+ALL_TOOLS = tools/ccan_depends tools/run_tests tools/doc_extract tools/namespacize tools/ccanlint/ccanlint
 
 .PHONY: tools
 tools: $(ALL_TOOLS)
@@ -7,7 +7,7 @@ tools/ccan_depends: tools/ccan_depends.o tools/depends.o ccan/str_talloc/str_tal
 
 tools/run_tests: tools/run_tests.o tools/depends.o ccan/str_talloc/str_talloc.o ccan/grab_file/grab_file.o ccan/tap/tap.o ccan/noerr/noerr.o ccan/talloc/talloc.o
 
-tools/doc_extract: tools/doc_extract.o ccan/str_talloc/str_talloc.o ccan/grab_file/grab_file.o ccan/noerr/noerr.o ccan/talloc/talloc.o
+tools/doc_extract: tools/doc_extract.o tools/doc_extract-core.o ccan/str_talloc/str_talloc.o ccan/grab_file/grab_file.o ccan/noerr/noerr.o ccan/talloc/talloc.o
 
 tools/namespacize: tools/namespacize.o tools/depends.o ccan/str_talloc/str_talloc.o ccan/grab_file/grab_file.o ccan/noerr/noerr.o ccan/talloc/talloc.o
 

tools/ccanlint/Makefile

@@ -23,6 +23,7 @@ tools/ccanlint/ccanlint: \
 	$(OBJS)			\
 	tools/ccanlint/ccanlint.o \
 	tools/ccanlint/file_analysis.o \
+	tools/doc_extract-core.o \
 	ccan/str_talloc/str_talloc.o ccan/grab_file/grab_file.o \
 	ccan/talloc/talloc.o ccan/noerr/noerr.o
 

tools/ccanlint/ccanlint.h

 #ifndef CCAN_LINT_H
 #define CCAN_LINT_H
-#include <list/list.h>
+#include <ccan/list/list.h>
 #include <stdbool.h>
+#include "../doc_extract.h"
 
 struct manifest {
 	char *basename;
@@ -54,11 +55,16 @@ struct ccan_file {
 
 	unsigned int num_lines;
 	char **lines;
+
+	struct list_head *doc_sections;
 };
 
 /* Use this rather than accessing f->lines directly: loads on demand. */
 char **get_ccan_file_lines(struct ccan_file *f);
 
+/* Similarly for ->doc_sections */
+struct list_head *get_ccan_file_docs(struct ccan_file *f);
+
 /* Call the reporting on every line in the file.  sofar contains
  * previous results. */
 char *report_on_lines(struct list_head *files,

tools/ccanlint/file_analysis.c

 #include "ccanlint.h"
-#include "get_file_lines.h"
-#include <talloc/talloc.h>
-#include <str/str.h>
-#include <str_talloc/str_talloc.h>
-#include <grab_file/grab_file.h>
-#include <noerr/noerr.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
+#include <ccan/str_talloc/str_talloc.h>
+#include <ccan/grab_file/grab_file.h>
+#include <ccan/noerr/noerr.h>
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -24,6 +23,15 @@ char **get_ccan_file_lines(struct ccan_file *f)
 	return f->lines;
 }
 
+struct list_head *get_ccan_file_docs(struct ccan_file *f)
+{
+	if (!f->doc_sections) {
+		get_ccan_file_lines(f);
+		f->doc_sections = extract_doc_sections(f->lines, f->num_lines);
+	}
+	return f->doc_sections;
+}
+
 static void add_files(struct manifest *m, const char *dir)
 {
 	DIR *d;
@@ -47,6 +55,7 @@ static void add_files(struct manifest *m, const char *dir)
 
 		f = talloc(m, struct ccan_file);
 		f->lines = NULL;
+		f->doc_sections = NULL;
 		f->name = talloc_asprintf(f, "%s%s", dir, ent->d_name);
 		if (lstat(f->name, &st) != 0)
 			err(1, "lstat %s", f->name);

tools/ccanlint/get_file_lines.h

-#ifndef GET_FILE_LINES_H
-#define GET_FILE_LINES_H
-
-char **get_file_lines(void *ctx, const char *name, unsigned int *num_lines);
-
-#endif /* GET_FILE_LINES_H */

tools/ccanlint/has_main_header.c

@@ -8,9 +8,9 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <err.h>
-#include <str/str.h>
-#include <talloc/talloc.h>
-#include <noerr/noerr.h>
+#include <ccan/str/str.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/noerr/noerr.h>
 
 static void *check_has_main_header(struct manifest *m)
 {

tools/ccanlint/has_tests.c

@@ -8,7 +8,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <err.h>
-#include <talloc/talloc.h>
+#include <ccan/talloc/talloc.h>
 
 static char test_is_not_dir[] = "test is not a directory";
 

tools/ccanlint/idempotent.c

 #include "ccanlint.h"
-#include <talloc/talloc.h>
-#include <str/str.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>

tools/ccanlint/no_info.c

@@ -9,7 +9,7 @@
 #include <stdio.h>
 #include <err.h>
 #include <string.h>
-#include <noerr/noerr.h>
+#include <ccan/noerr/noerr.h>
 
 static void *check_no_info(struct manifest *m)
 {

tools/ccanlint/trailing_whitespace.c

 /* Trailing whitespace test.  Almost embarrassing, but trivial. */
 #include "ccanlint.h"
-#include <talloc/talloc.h>
-#include <str/str.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
 
 static char *report_on_trailing_whitespace(const char *line)
 {

tools/doc_extract-core.c

+/* This merely extracts, doesn't do XML or anything. */
+#include <err.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdbool.h>
+#include <ctype.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
+#include "doc_extract.h"
+#include "tools.h"
+
+static char **grab_doc(char **lines, unsigned int num)
+{
+	char **ret;
+	unsigned int i;
+	bool printing = false;
+
+	ret = talloc_array(NULL, char *, num+1);
+
+	num = 0;
+	for (i = 0; lines[i]; i++) {
+		if (streq(lines[i], "/**")) {
+			printing = true;
+			if (num != 0)
+				talloc_append_string(ret[num-1], "\n");
+		} else if (streq(lines[i], " */")) 
+			printing = false;
+		else if (printing) {
+			if (strstarts(lines[i], " * "))
+				ret[num++] = talloc_strdup(ret, lines[i]+3);
+			else if (strstarts(lines[i], " *"))
+				ret[num++] = talloc_strdup(ret, lines[i]+2);
+			else
+				errx(1, "Malformed line %u", i);
+		}
+	}
+	ret[num] = NULL;
+	return ret;
+}
+
+static bool is_blank(const char *line)
+{
+	return line && line[strspn(line, " \t\n")] == '\0';
+}
+
+static bool is_section(const char *line, bool one_liner)
+{
+	unsigned int len;
+
+	if (!isupper(line[0]))
+		return false;
+	len = strspn(line, IDENT_CHARS);
+	if (line[len] != ':')
+		return false;
+
+	/* If it can be a one-liner, a space is sufficient.*/
+	if (one_liner)
+		return (line[len+1] == ' ' || line[len+1] == '\t');
+
+	return line[len] == ':' && is_blank(line+len+1);
+}
+
+/* Summary line is form '<identifier> - ' */
+static bool is_summary_line(const char *line)
+{
+	unsigned int id_len;
+
+	id_len = strspn(line, IDENT_CHARS);
+	if (id_len == 0)
+		return false;
+	if (!strstarts(line + id_len, " - "))
+		return false;
+
+	return true;
+}
+
+static struct doc_section *new_section(struct list_head *list,
+				       const char *function,
+				       const char *type)
+{
+	struct doc_section *d = talloc(list, struct doc_section);
+	d->function = function;
+	d->type = type;
+	d->lines = NULL;
+	d->num_lines = 0;
+	list_add_tail(list, &d->list);
+	return d;
+}
+
+static void add_line(struct doc_section *curr, const char *line)
+{
+	curr->lines = talloc_realloc(curr, curr->lines, char *,
+				     curr->num_lines+1);
+	curr->lines[curr->num_lines++] = talloc_strdup(curr->lines, line);
+}
+
+struct list_head *extract_doc_sections(char **rawlines, unsigned int num)
+{
+	char **lines = grab_doc(rawlines, num);
+	const char *function = NULL;
+	struct doc_section *curr = NULL;
+	unsigned int i;
+	struct list_head *list;
+
+	list = talloc(NULL, struct list_head);
+	list_head_init(list);
+
+	for (i = 0; lines[i]; i++) {
+		if (is_summary_line(lines[i])) {
+			function = talloc_strndup(list, lines[i],
+						  strcspn(lines[i], " "));
+			curr = new_section(list, function, "summary");
+			add_line(curr, strstr(lines[i], " - ") + 3);
+			curr = new_section(list, function, "description");
+		} else if (is_section(lines[i], false)) {
+			char *type = talloc_strndup(curr, lines[i],
+						    strcspn(lines[i], ":"));
+			curr = new_section(list, function, type);
+		} else if (is_section(lines[i], true)) {
+			unsigned int sectlen = strcspn(lines[i], ":");
+			char *type = talloc_strndup(curr, lines[i], sectlen);
+			curr = new_section(list, function, type);
+			add_line(curr, lines[i] + sectlen + 1
+				 + strspn(lines[i] + sectlen + 1, " \t"));
+		} else {
+			if (!curr)
+				continue;
+			add_line(curr, lines[i]);
+		}
+	}
+	talloc_free(lines);
+	return list;
+}

tools/doc_extract.c

 /* This merely extracts, doesn't do XML or anything. */
 #include <err.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <stdbool.h>
-#include <ccan/talloc/talloc.h>
+#include <stdio.h>
 #include <ccan/str/str.h>
 #include <ccan/str_talloc/str_talloc.h>
+#include <ccan/talloc/talloc.h>
 #include <ccan/grab_file/grab_file.h>
-#include "tools.h"
-
-static char **grab_doc(const char *fname)
-{
-	char *file;
-	char **lines, **ret;
-	unsigned int i, num;
-	bool printing = false;
-
-	file = grab_file(NULL, fname, NULL);
-	if (!file)
-		err(1, "Reading file %s", fname);
-	lines = strsplit(file, file, "\n", &num);
-	ret = talloc_array(NULL, char *, num+1);
-
-	num = 0;
-	for (i = 0; lines[i]; i++) {
-		if (streq(lines[i], "/**")) {
-			printing = true;
-			if (num != 0)
-				talloc_append_string(ret[num-1], "\n");
-		} else if (streq(lines[i], " */")) 
-			printing = false;
-		else if (printing) {
-			if (strstarts(lines[i], " * "))
-				ret[num++] = talloc_strdup(ret, lines[i]+3);
-			else if (strstarts(lines[i], " *"))
-				ret[num++] = talloc_strdup(ret, lines[i]+2);
-			else
-				errx(1, "Malformed line %s:%u", fname, i);
-		}
-	}
-	ret[num] = NULL;
-	talloc_free(file);
-	return ret;
-}
-
-static bool is_blank(const char *line)
-{
-	return line && line[strspn(line, " \t\n")] == '\0';
-}
-
-static bool is_section(const char *line, bool maybe_one_liner)
-{
-	unsigned int len;
-
-	len = strcspn(line, " \t\n:");
-	if (len == 0)
-		return false;
-
-	if (line[len] != ':')
-		return false;
-
-	/* If it can be a one-liner, a space is sufficient.*/
-	if (maybe_one_liner && (line[len+1] == ' ' || line[len+1] == '\t'))
-		return true;
-
-	return line[len] == ':' && is_blank(line+len+1);
-}
-
-/* Summary line is form '<identifier> - ' */
-static bool is_summary_line(const char *line)
-{
-	unsigned int id_len;
-
-	id_len = strspn(line, IDENT_CHARS);
-	if (id_len == 0)
-		return false;
-	if (!strstarts(line + id_len, " - "))
-		return false;
-
-	return true;
-}
-
-static bool end_section(const char *line)
-{
-	return !line || is_section(line, true) || is_summary_line(line);
-}
-
-static unsigned int find_section(char **lines, const char *name,
-				 bool maybe_one_liner)
-{
-	unsigned int i;
-
-	for (i = 0; lines[i]; i++) {
-		if (!is_section(lines[i], maybe_one_liner))
-			continue;
-		if (strncasecmp(lines[i], name, strlen(name)) != 0)
-			continue;
-		if (lines[i][strlen(name)] == ':')
-			break;
-	}
-	return i;
-}
-
-/* function is NULL if we don't care. */
-static unsigned int find_summary(char **lines, const char *function)
-{
-	unsigned int i;
-
-	for (i = 0; lines[i]; i++) {
-		if (!is_summary_line(lines[i]))
-			continue;
-		if (function) {
-			if (!strstarts(lines[i], function))
-				continue;
-			if (!strstarts(lines[i] + strlen(function), " - "))
-				continue;
-		}
-		break;
-	}
-	return i;
-}
-
+#include "doc_extract.h"
 
 int main(int argc, char *argv[])
 {
@@ -145,88 +26,49 @@ int main(int argc, char *argv[])
 
 	type = argv[1];
 	for (i = 2; i < argc; i++) {
-		unsigned int line;
-		char **lines = grab_doc(argv[i]);
-
-		if (!lines[0])
+		char *file, **lines;
+		unsigned int num;
+		struct list_head *list;
+		struct doc_section *d;
+
+		file = grab_file(NULL, argv[i], NULL);
+		if (!file)
+			err(1, "Reading file %s", argv[i]);
+		lines = strsplit(file, file, "\n", &num);
+
+		list = extract_doc_sections(lines, num);
+		if (list_empty(list))
 			errx(1, "No documentation in file %s", argv[i]);
-
-		if (function) {
-			/* Allow us to trawl multiple files for a function */
-			line = find_summary(lines, function);
-			if (!lines[line])
-				continue;
-
-			/* Trim to just this function then. */
-			lines += line;
-			lines[find_summary(lines+1, NULL)] = NULL;
-		}
-		/* Simple one-line fields. */
-		if (streq(type, "author")
-		    || streq(type, "maintainer")
-		    || streq(type, "licence")) {
-			line = find_section(lines, type, true);
-			if (lines[line]) {
-				const char *p = strchr(lines[line], ':') + 1;
-				p += strspn(p, " \t\n");
-				if (p[0] == '\0') {
-					/* Must be on next line. */
-					if (end_section(lines[line+1]))
-						errx(1, "Malformed %s", type);
-					puts(lines[line+1]);
-				} else
-					puts(p);
-			}
-		} else if (streq(type, "summary")) {
-			/* Summary comes after - on first line. */
-			char *dash;
-
-			dash = strchr(lines[0], '-');
-			if (!dash)
-				errx(1, "Malformed first line: no -");
-			dash += strspn(dash, "- ");
-			puts(dash);
-		} else if (streq(type, "description")) {
-			line = 1;
-			while (is_blank(lines[line]))
-				line++;
-
-			while (!end_section(lines[line]))
-				puts(lines[line++]);
-		} else if (streq(type, "example")) {
-			line = find_section(lines, type, false);
-			if (lines[line]) {
-				unsigned int strip;
-				line++;
-
-				while (is_blank(lines[line]))
-					line++;
-
-				/* Examples can be indented.  Take cue
-				 * from first non-blank line. */
-				if (lines[line])
-					strip = strspn(lines[line], " \t");
-
-				while (!end_section(lines[line])) {
-					if (strspn(lines[line], " \t") >= strip)
-						puts(lines[line] + strip);
-					else
-						puts(lines[line]);
-					line++;
+		talloc_free(file);
+
+		if (streq(type, "functions")) {
+			const char *last = NULL;
+			list_for_each(list, d, list) {
+				if (d->function) {
+					if (!last || !streq(d->function, last))
+						printf("%s\n", d->function);
+					last = d->function;
 				}
 			}
-		} else if (streq(type, "functions")) {
-			while (lines[line = find_summary(lines, NULL)]) {
-				const char *dash = strstr(lines[line], " - ");
-				printf("%.*s\n",
-				       dash - lines[line], lines[line]);
-				lines += line+1;
+		} else {
+			unsigned int j;
+			list_for_each(list, d, list) {
+				if (function) {
+					if (!d->function)
+						continue;
+					if (!streq(d->function, function))
+						continue;
+				}
+				if (strcasecmp(type, "all") == 0)
+					printf("%s:\n", d->type);
+				else if (strcasecmp(d->type, type) != 0)
+					continue;
+
+				for (j = 0; j < d->num_lines; j++)
+					printf("%s\n", d->lines[j]);
 			}
-		} else if (streq(type, "all")) {
-			for (line = 0; lines[line]; line++)
-				puts(lines[line]);
-		} else
-			errx(1, "Unknown type '%s'", type);
+		}
+		talloc_free(list);
 	}
 	return 0;
 }

tools/doc_extract.h

+#ifndef _DOC_EXTRACT_CORE_H
+#define _DOC_EXTRACT_CORE_H
+#include <stdbool.h>
+#include <ccan/list/list.h>
+
+struct doc_section {
+	struct list_node list;
+	const char *function;
+	const char *type;
+	unsigned int num_lines;
+	char **lines;
+};
+
+struct list_head *extract_doc_sections(char **rawlines, unsigned int num);
+#endif /* _DOC_EXTRACT_CORE_H */