[PATCH] Fix early parallel make failures

From: Sam Ravnborg <sam@ravnborg.org>

Ingo said:

  Starting at around 2.6.4-rc2-mm1, I keep seeing 'scripts/fixdep: Text
  file busy' messages when doing a -j10 bzImage build - which seems to
  suggest that by the time fixdep is used by the build system it's not
  built yet.

Sam said:

I was pretty sure it was something I had caused, so I gave it a spin.  What
actually happened was that we tried to build the target 'silentoldconfig'
in parrallel with 'scripts'.  Since 'silentoldconfig' started a new make
and then the config target needed 'scripts' we saw two parallel runs.

The way I decided to fix it was to split scripts/ in two parts.  The first
part is now the very basic stuff - moved to scripts/basic/.  The second
part is dependent on kernel config etc.  and kept in scripts/

In the 2.7 timeframe i will redo this initial stuff - it's becoming too
messy for anyone to understand today.


Description:

Fix dependencies in early phases of kernel build.  This solves a few
problems nively: modpost is no longer rebuild twicewhen reaching the
'target' state 'make -j10' now works nicely again

The patch is rather large due to the following file moves:
mkdir scripts/basic
mv scripts/fixdep.c        scripts/basic
mv scripts/split-include.c scripts/basic
mv scripts/docproc.c       scripts/basic

Documentation/DocBook/Makefile

@@ -47,7 +47,7 @@ installmandocs: mandocs
 ###
 #External programs used
 KERNELDOC = scripts/kernel-doc
-DOCPROC   = scripts/docproc
+DOCPROC   = scripts/basic/docproc
 SPLITMAN  = $(PERL) $(srctree)/scripts/split-man
 MAKEMAN   = $(PERL) $(srctree)/scripts/makeman
 

Makefile

@@ -304,17 +304,10 @@ RCS_TAR_IGNORE := --exclude SCCS --exclude BitKeeper --exclude .svn --exclude CV
 # ===========================================================================
 # Rules shared between *config targets and build targets
 
-# Helpers built in scripts/
-
-scripts/docproc scripts/split-include : scripts ;
-
-.PHONY: scripts scripts/fixdep
-scripts:
-	$(Q)$(MAKE) $(build)=scripts
-
-scripts/fixdep:
-	$(Q)$(MAKE) $(build)=scripts $@
-
+# Basic helpers built in scripts/
+.PHONY: scripts_basic
+scripts_basic:
+	$(Q)$(MAKE) $(build)=scripts/basic
 
 # To make sure we do not include .config for any of the *config targets
 # catch them early, and hand them over to scripts/kconfig/Makefile
@@ -358,9 +351,9 @@ ifeq ($(config-targets),1)
 # *config targets only - make sure prerequisites are updated, and descend
 # in scripts/kconfig to make the *config target
 
-%config: scripts/fixdep FORCE
+config: scripts_basic FORCE
 	$(Q)$(MAKE) $(build)=scripts/kconfig $@
-config : scripts/fixdep FORCE
+%config: scripts_basic FORCE
 	$(Q)$(MAKE) $(build)=scripts/kconfig $@
 
 else
@@ -368,6 +361,16 @@ else
 # Build targets only - this includes vmlinux, arch specific targets, clean
 # targets and others. In general all targets except *config targets.
 
+# Additional helpers built in scripts/
+# Carefully list dependencies so we do not try to build scripts twice
+# in parrallel
+.PHONY: scripts
+scripts: scripts_basic include/config/MARKER
+	$(Q)$(MAKE) $(build)=$(@)
+
+scripts_basic: include/linux/autoconf.h
+
+
 # That's our default target when none is given on the command line
 # Note that 'modules' will be added as a prerequisite as well, 
 # in the CONFIG_MODULES part below
@@ -400,7 +403,9 @@ include .config
 # with it and forgot to run make oldconfig
 include/linux/autoconf.h: .config
 	$(Q)$(MAKE) -f $(srctree)/Makefile silentoldconfig
-
+else
+# Dummy target needed, because used as prerequisite
+include/linux/autoconf.h: ;
 endif
 
 include $(srctree)/arch/$(ARCH)/Makefile
@@ -633,9 +638,9 @@ include/asm:
 
 # 	Split autoconf.h into include/linux/config/*
 
-include/config/MARKER: scripts/split-include include/linux/autoconf.h
+include/config/MARKER: include/linux/autoconf.h
 	@echo '  SPLIT   include/linux/autoconf.h -> include/config/*'
-	@scripts/split-include include/linux/autoconf.h include/config
+	@scripts/basic/split-include include/linux/autoconf.h include/config
 	@touch $@
 
 # Generate some files
@@ -929,7 +934,7 @@ help:
 
 # Documentation targets
 # ---------------------------------------------------------------------------
-%docs: scripts/docproc FORCE
+%docs: scripts FORCE
 	$(Q)$(MAKE) $(build)=Documentation/DocBook $@
 
 # Scripts to check various things for consistency
@@ -982,7 +987,7 @@ if_changed_dep = $(if $(strip $? $(filter-out FORCE $(wildcard $^),$^)\
 	@set -e; \
 	$(if $($(quiet)cmd_$(1)),echo '  $(subst ','\'',$($(quiet)cmd_$(1)))';) \
 	$(cmd_$(1)); \
-	scripts/fixdep $(depfile) $@ '$(subst $$,$$$$,$(subst ','\'',$(cmd_$(1))))' > $(@D)/.$(@F).tmp; \
+	scripts/basic/fixdep $(depfile) $@ '$(subst $$,$$$$,$(subst ','\'',$(cmd_$(1))))' > $(@D)/.$(@F).tmp; \
 	rm -f $(depfile); \
 	mv -f $(@D)/.$(@F).tmp $(@D)/.$(@F).cmd)
 

scripts/Makefile

@@ -2,14 +2,10 @@
 # scripts contains sources for various helper programs used throughout
 # the kernel for the build process.
 # ---------------------------------------------------------------------------
-# fix-dep: 	 Used to generate dependency information during build process
-# split-include: Divide all config symbols up in a number of files in
-#                include/config/...
 # docproc: 	 Preprocess .tmpl file in order to generate .sgml docs
 # conmakehash:	 Create arrays for initializing the kernel console tables
 
-host-progs	:= fixdep split-include conmakehash docproc kallsyms modpost \
-		   mk_elfconfig pnmtologo bin2c
+host-progs	:= conmakehash kallsyms modpost mk_elfconfig pnmtologo bin2c
 always		:= $(host-progs) empty.o
 
 modpost-objs	:= modpost.o file2alias.o sumversion.o
@@ -17,10 +13,7 @@ modpost-objs	:= modpost.o file2alias.o sumversion.o
 subdir-$(CONFIG_MODVERSIONS)	+= genksyms
 
 # Let clean descend into subdirs
-subdir-	+= lxdialog kconfig
-
-# fixdep is needed to compile other host programs
-$(addprefix $(obj)/,$(filter-out fixdep,$(always)) $(subdir-y)): $(obj)/fixdep
+subdir-	+= basic lxdialog kconfig
 
 # dependencies on generated files need to be listed explicitly
 

scripts/Makefile.build

@@ -162,7 +162,7 @@ define rule_cc_o_c
 	$(if $($(quiet)cmd_cc_o_c),echo '  $($(quiet)cmd_cc_o_c)';)	  \
 	$(cmd_cc_o_c);							  \
 	$(cmd_modversions)						  \
-	scripts/fixdep $(depfile) $@ '$(cmd_cc_o_c)' > $(@D)/.$(@F).tmp;  \
+	scripts/basic/fixdep $(depfile) $@ '$(cmd_cc_o_c)' > $(@D)/.$(@F).tmp;  \
 	rm -f $(depfile);						  \
 	mv -f $(@D)/.$(@F).tmp $(@D)/.$(@F).cmd
 endef

scripts/Makefile.lib

@@ -249,7 +249,7 @@ if_changed_dep = $(if $(strip $? $(filter-out FORCE $(wildcard $^),$^)\
 	@set -e; \
 	$(if $($(quiet)cmd_$(1)),echo '  $(subst ','\'',$($(quiet)cmd_$(1)))';) \
 	$(cmd_$(1)); \
-	scripts/fixdep $(depfile) $@ '$(subst $$,$$$$,$(subst ','\'',$(cmd_$(1))))' > $(@D)/.$(@F).tmp; \
+	scripts/basic/fixdep $(depfile) $@ '$(subst $$,$$$$,$(subst ','\'',$(cmd_$(1))))' > $(@D)/.$(@F).tmp; \
 	rm -f $(depfile); \
 	mv -f $(@D)/.$(@F).tmp $(@D)/.$(@F).cmd)
 

scripts/basic/Makefile

+###
+# Makefile.basic list the most basic programs used during the build process.
+# The programs listed herein is what is needed to do the basic stuff,
+# such as splitting .config and fix dependency file.
+# This initial step is needed to avoid files to be recompiled
+# when kernel configuration changes (which is what happens when
+# .config is included by main Makefile.
+# ---------------------------------------------------------------------------
+# fixdep: 	 Used to generate dependency information during build process
+# split-include: Divide all config symbols up in a number of files in
+#                include/config/...
+# docproc:	 Used in Documentation/docbook
+
+host-progs	:= fixdep split-include docproc
+always		:= $(host-progs)
+
+# fixdep is needed to compile other host programs
+$(addprefix $(obj)/,$(filter-out fixdep,$(always))): $(obj)/fixdep

scripts/docproc.c → scripts/basic/docproc.c

@@ -28,7 +28,7 @@
  *		!Ifilename
  *		!Dfilename
  *		!Ffilename
- *		
+ *
  */
 
 #include <stdio.h>
@@ -125,7 +125,7 @@ void add_new_symbol(struct symfile *sym, char * symname)
 struct symfile * add_new_file(char * filename)
 {
 	symfilelist[symfilecnt++].filename = strdup(filename);
-	return &symfilelist[symfilecnt - 1];					
+	return &symfilelist[symfilecnt - 1];
 }
 /* Check if file already are present in the list */
 struct symfile * filename_exist(char * filename)
@@ -149,8 +149,8 @@ void noaction2(char * file, char * line)   { file = file; line = line; }
 /* Echo the line without further action */
 void printline(char * line)               { printf("%s", line); }
 
-/* 
- * Find all symbols exported with EXPORT_SYMBOL and EXPORT_SYMBOL_GPL 
+/*
+ * Find all symbols exported with EXPORT_SYMBOL and EXPORT_SYMBOL_GPL
  * in filename.
  * All symbols located are stored in symfilelist.
  */
@@ -210,7 +210,7 @@ void docfunctions(char * filename, char * type)
 	int symcnt = 0;
 	int idx = 0;
 	char **vec;
-	
+
 	for (i=0; i <= symfilecnt; i++)
 		symcnt += symfilelist[i].symbolcnt;
 	vec = malloc((2 + 2 * symcnt + 2) * sizeof(char*));
@@ -250,7 +250,7 @@ void singfunc(char * filename, char * line)
 	vec[idx++] = KERNELDOC;
 	vec[idx++] = DOCBOOK;
 
-        /* Split line up in individual parameters preceeded by FUNCTION */        
+        /* Split line up in individual parameters preceeded by FUNCTION */
         for (i=0; line[i]; i++) {
                 if (isspace(line[i])) {
                         line[i] = '\0';
@@ -318,7 +318,7 @@ void parse_file(FILE *infile)
 	}
 	fflush(stdout);
 }
-		
+
 
 int main(int argc, char *argv[])
 {
@@ -359,7 +359,7 @@ int main(int argc, char *argv[])
 		externalfunctions = extfunc;
 		symbolsonly       = printline;
 		singlefunctions   = singfunc;
-		
+
 		parse_file(infile);
 	}
 	else if (strcmp("depend", argv[1]) == 0)

scripts/fixdep.c → scripts/basic/fixdep.c

 /*
- * "Optimize" a list of dependencies as spit out by gcc -MD 
+ * "Optimize" a list of dependencies as spit out by gcc -MD
  * for the kernel build
  * ===========================================================================
  *
@@ -11,7 +11,7 @@
  *
  *
  * Introduction:
- * 
+ *
  * gcc produces a very nice and correct list of dependencies which
  * tells make when to remake a file.
  *
@@ -23,21 +23,21 @@
  * regenerated.  make notices that and will rebuild every file which
  * includes autoconf.h, i.e. basically all files. This is extremely
  * annoying if the user just changed CONFIG_HIS_DRIVER from n to m.
- * 
+ *
  * So we play the same trick that "mkdep" played before. We replace
  * the dependency on linux/autoconf.h by a dependency on every config
  * option which is mentioned in any of the listed prequisites.
- *  
+ *
  * To be exact, split-include populates a tree in include/config/,
  * e.g. include/config/his/driver.h, which contains the #define/#undef
  * for the CONFIG_HIS_DRIVER option.
  *
  * So if the user changes his CONFIG_HIS_DRIVER option, only the objects
  * which depend on "include/linux/config/his/driver.h" will be rebuilt,
- * so most likely only his driver ;-) 
+ * so most likely only his driver ;-)
  *
  * The idea above dates, by the way, back to Michael E Chastain, AFAIK.
- * 
+ *
  * So to get dependencies right, there two issues:
  * o if any of the files the compiler read changed, we need to rebuild
  * o if the command line given to the compile the file changed, we
@@ -89,7 +89,7 @@
  * we cannot miss a rebuild. Since people tend to not mention totally
  * unrelated CONFIG_ options all over the place, it's not an
  * efficiency problem either.
- * 
+ *
  * (Note: it'd be easy to port over the complete mkdep state machine,
  *  but I don't think the added complexity is worth it)
  */
@@ -233,7 +233,7 @@ void parse_config_file(char *map, size_t len)
 		}
 		continue;
 
-	found: 
+	found:
 		use_config(p+7, q-p-7);
 	}
 }
@@ -243,10 +243,10 @@ int strrcmp(char *s, char *sub)
 {
 	int slen = strlen(s);
 	int sublen = strlen(sub);
-  
+
 	if (sublen > slen)
 		return 1;
-	
+
 	return memcmp(s + slen - sublen, sub, sublen);
 }
 
@@ -273,7 +273,7 @@ void do_config_file(char *filename)
 		close(fd);
 		return;
 	}
-	
+
 	parse_config_file(map, st.st_size);
 
 	munmap(map, st.st_size);
@@ -344,7 +344,7 @@ void print_deps(void)
 		close(fd);
 		return;
 	}
-	
+
 	parse_dep_file(map, st.st_size);
 
 	munmap(map, st.st_size);
@@ -369,7 +369,7 @@ int main(int argc, char *argv[])
 
 	if (argc != 4)
 		usage();
-		
+
 	depfile = argv[1];
 	target = argv[2];
 	cmdline = argv[3];

scripts/split-include.c → scripts/basic/split-include.c

@@ -96,7 +96,7 @@ int main(int argc, const char * argv [])
     /* Change to output directory. */
     if (chdir(str_dir_config) != 0)
 	ERROR_EXIT(str_dir_config);
-	
+
     /* Put initial separator into target list. */
     ptarget = list_target;
     *ptarget++ = '\n';