[PATCH] Fix early parallel make failures

From: Sam Ravnborg <sam@ravnborg.org>

Ingo said:

  Starting at around 2.6.4-rc2-mm1, I keep seeing 'scripts/fixdep: Text
  file busy' messages when doing a -j10 bzImage build - which seems to
  suggest that by the time fixdep is used by the build system it's not
  built yet.

Sam said:

I was pretty sure it was something I had caused, so I gave it a spin.  What
actually happened was that we tried to build the target 'silentoldconfig'
in parrallel with 'scripts'.  Since 'silentoldconfig' started a new make
and then the config target needed 'scripts' we saw two parallel runs.

The way I decided to fix it was to split scripts/ in two parts.  The first
part is now the very basic stuff - moved to scripts/basic/.  The second
part is dependent on kernel config etc.  and kept in scripts/

In the 2.7 timeframe i will redo this initial stuff - it's becoming too
messy for anyone to understand today.


Description:

Fix dependencies in early phases of kernel build.  This solves a few
problems nively: modpost is no longer rebuild twicewhen reaching the
'target' state 'make -j10' now works nicely again

The patch is rather large due to the following file moves:
mkdir scripts/basic
mv scripts/fixdep.c        scripts/basic
mv scripts/split-include.c scripts/basic
mv scripts/docproc.c       scripts/basic

Documentation/DocBook/Makefile

@@ -47,7 +47,7 @@ installmandocs: mandocs
 ###
 #External programs used
 KERNELDOC = scripts/kernel-doc
-DOCPROC   = scripts/docproc
+DOCPROC   = scripts/basic/docproc
 SPLITMAN  = $(PERL) $(srctree)/scripts/split-man
 MAKEMAN   = $(PERL) $(srctree)/scripts/makeman
 

Makefile

@@ -304,17 +304,10 @@ RCS_TAR_IGNORE := --exclude SCCS --exclude BitKeeper --exclude .svn --exclude CV
 # ===========================================================================
 # Rules shared between *config targets and build targets
 
-# Helpers built in scripts/
-
-scripts/docproc scripts/split-include : scripts ;
-
-.PHONY: scripts scripts/fixdep
-scripts:
-	$(Q)$(MAKE) $(build)=scripts
-
-scripts/fixdep:
-	$(Q)$(MAKE) $(build)=scripts $@
-
+# Basic helpers built in scripts/
+.PHONY: scripts_basic
+scripts_basic:
+	$(Q)$(MAKE) $(build)=scripts/basic
 
 # To make sure we do not include .config for any of the *config targets
 # catch them early, and hand them over to scripts/kconfig/Makefile
@@ -358,9 +351,9 @@ ifeq ($(config-targets),1)
 # *config targets only - make sure prerequisites are updated, and descend
 # in scripts/kconfig to make the *config target
 
-%config: scripts/fixdep FORCE
+config: scripts_basic FORCE
 	$(Q)$(MAKE) $(build)=scripts/kconfig $@
-config : scripts/fixdep FORCE
+%config: scripts_basic FORCE
 	$(Q)$(MAKE) $(build)=scripts/kconfig $@
 
 else
@@ -368,6 +361,16 @@ else
 # Build targets only - this includes vmlinux, arch specific targets, clean
 # targets and others. In general all targets except *config targets.
 
+# Additional helpers built in scripts/
+# Carefully list dependencies so we do not try to build scripts twice
+# in parrallel
+.PHONY: scripts
+scripts: scripts_basic include/config/MARKER
+	$(Q)$(MAKE) $(build)=$(@)
+
+scripts_basic: include/linux/autoconf.h
+
+
 # That's our default target when none is given on the command line
 # Note that 'modules' will be added as a prerequisite as well, 
 # in the CONFIG_MODULES part below
@@ -400,7 +403,9 @@ include .config
 # with it and forgot to run make oldconfig
 include/linux/autoconf.h: .config
 	$(Q)$(MAKE) -f $(srctree)/Makefile silentoldconfig
-
+else
+# Dummy target needed, because used as prerequisite
+include/linux/autoconf.h: ;
 endif
 
 include $(srctree)/arch/$(ARCH)/Makefile
@@ -633,9 +638,9 @@ include/asm:
 
 # 	Split autoconf.h into include/linux/config/*
 
-include/config/MARKER: scripts/split-include include/linux/autoconf.h
+include/config/MARKER: include/linux/autoconf.h
 	@echo '  SPLIT   include/linux/autoconf.h -> include/config/*'
-	@scripts/split-include include/linux/autoconf.h include/config
+	@scripts/basic/split-include include/linux/autoconf.h include/config
 	@touch $@
 
 # Generate some files
@@ -929,7 +934,7 @@ help:
 
 # Documentation targets
 # ---------------------------------------------------------------------------
-%docs: scripts/docproc FORCE
+%docs: scripts FORCE
 	$(Q)$(MAKE) $(build)=Documentation/DocBook $@
 
 # Scripts to check various things for consistency
@@ -982,7 +987,7 @@ if_changed_dep = $(if $(strip $? $(filter-out FORCE $(wildcard $^),$^)\
 	@set -e; \
 	$(if $($(quiet)cmd_$(1)),echo '  $(subst ','\'',$($(quiet)cmd_$(1)))';) \
 	$(cmd_$(1)); \
-	scripts/fixdep $(depfile) $@ '$(subst $$,$$$$,$(subst ','\'',$(cmd_$(1))))' > $(@D)/.$(@F).tmp; \
+	scripts/basic/fixdep $(depfile) $@ '$(subst $$,$$$$,$(subst ','\'',$(cmd_$(1))))' > $(@D)/.$(@F).tmp; \
 	rm -f $(depfile); \
 	mv -f $(@D)/.$(@F).tmp $(@D)/.$(@F).cmd)
 

scripts/Makefile

@@ -2,14 +2,10 @@
 # scripts contains sources for various helper programs used throughout
 # the kernel for the build process.
 # ---------------------------------------------------------------------------
-# fix-dep: 	 Used to generate dependency information during build process
-# split-include: Divide all config symbols up in a number of files in
-#                include/config/...
 # docproc: 	 Preprocess .tmpl file in order to generate .sgml docs
 # conmakehash:	 Create arrays for initializing the kernel console tables
 
-host-progs	:= fixdep split-include conmakehash docproc kallsyms modpost \
-		   mk_elfconfig pnmtologo bin2c
+host-progs	:= conmakehash kallsyms modpost mk_elfconfig pnmtologo bin2c
 always		:= $(host-progs) empty.o
 
 modpost-objs	:= modpost.o file2alias.o sumversion.o
@@ -17,10 +13,7 @@ modpost-objs	:= modpost.o file2alias.o sumversion.o
 subdir-$(CONFIG_MODVERSIONS)	+= genksyms
 
 # Let clean descend into subdirs
-subdir-	+= lxdialog kconfig
-
-# fixdep is needed to compile other host programs
-$(addprefix $(obj)/,$(filter-out fixdep,$(always)) $(subdir-y)): $(obj)/fixdep
+subdir-	+= basic lxdialog kconfig
 
 # dependencies on generated files need to be listed explicitly
 

scripts/Makefile.build

@@ -162,7 +162,7 @@ define rule_cc_o_c
 	$(if $($(quiet)cmd_cc_o_c),echo '  $($(quiet)cmd_cc_o_c)';)	  \
 	$(cmd_cc_o_c);							  \
 	$(cmd_modversions)						  \
-	scripts/fixdep $(depfile) $@ '$(cmd_cc_o_c)' > $(@D)/.$(@F).tmp;  \
+	scripts/basic/fixdep $(depfile) $@ '$(cmd_cc_o_c)' > $(@D)/.$(@F).tmp;  \
 	rm -f $(depfile);						  \
 	mv -f $(@D)/.$(@F).tmp $(@D)/.$(@F).cmd
 endef

scripts/Makefile.lib

@@ -249,7 +249,7 @@ if_changed_dep = $(if $(strip $? $(filter-out FORCE $(wildcard $^),$^)\
 	@set -e; \
 	$(if $($(quiet)cmd_$(1)),echo '  $(subst ','\'',$($(quiet)cmd_$(1)))';) \
 	$(cmd_$(1)); \
-	scripts/fixdep $(depfile) $@ '$(subst $$,$$$$,$(subst ','\'',$(cmd_$(1))))' > $(@D)/.$(@F).tmp; \
+	scripts/basic/fixdep $(depfile) $@ '$(subst $$,$$$$,$(subst ','\'',$(cmd_$(1))))' > $(@D)/.$(@F).tmp; \
 	rm -f $(depfile); \
 	mv -f $(@D)/.$(@F).tmp $(@D)/.$(@F).cmd)
 

scripts/basic/Makefile

+###
+# Makefile.basic list the most basic programs used during the build process.
+# The programs listed herein is what is needed to do the basic stuff,
+# such as splitting .config and fix dependency file.
+# This initial step is needed to avoid files to be recompiled
+# when kernel configuration changes (which is what happens when
+# .config is included by main Makefile.
+# ---------------------------------------------------------------------------
+# fixdep: 	 Used to generate dependency information during build process
+# split-include: Divide all config symbols up in a number of files in
+#                include/config/...
+# docproc:	 Used in Documentation/docbook
+
+host-progs	:= fixdep split-include docproc
+always		:= $(host-progs)
+
+# fixdep is needed to compile other host programs
+$(addprefix $(obj)/,$(filter-out fixdep,$(always))): $(obj)/fixdep