Commit 51e46c7a authored by Kees Cook's avatar Kees Cook Committed by Jonathan Corbet

docs, parallelism: Rearrange how jobserver reservations are made

Rasmus correctly observed that the existing jobserver reservation only
worked if no other build targets were specified. The correct approach
is to hold the jobserver slots until sphinx has finished. To fix this,
the following changes are made:

- refactor (and rename) scripts/jobserver-exec to set an environment
  variable for the maximally reserved jobserver slots and exec a
  child, to release the slots on exit.

- create Documentation/scripts/parallel-wrapper.sh which examines both
  $PARALLELISM and the detected "-jauto" logic from Documentation/Makefile
  to decide sphinx's final -j argument.

- chain these together in Documentation/Makefile
Suggested-by: default avatarRasmus Villemoes <linux@rasmusvillemoes.dk>
Link: https://lore.kernel.org/lkml/eb25959a-9ec4-3530-2031-d9d716b40b20@rasmusvillemoes.dkSigned-off-by: default avatarKees Cook <keescook@chromium.org>
Link: https://lore.kernel.org/r/20191121205929.40371-4-keescook@chromium.orgSigned-off-by: default avatarJonathan Corbet <corbet@lwn.net>
parent dffd0114
......@@ -33,8 +33,6 @@ ifeq ($(HAVE_SPHINX),0)
else # HAVE_SPHINX
export SPHINX_PARALLEL = $(shell perl -e 'open IN,"sphinx-build --version 2>&1 |"; while (<IN>) { if (m/([\d\.]+)/) { print "auto" if ($$1 >= "1.7") } ;} close IN')
# User-friendly check for pdflatex and latexmk
HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi)
HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi)
......@@ -67,8 +65,9 @@ quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(BUILDDIR)/$3/$4)
cmd_sphinx = $(MAKE) BUILDDIR=$(abspath $(BUILDDIR)) $(build)=Documentation/media $2 && \
PYTHONDONTWRITEBYTECODE=1 \
BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(srctree)/$(src)/$5/$(SPHINX_CONF)) \
$(PYTHON) $(srctree)/scripts/jobserver-exec \
$(SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \
$(SPHINXBUILD) \
-j $(shell python $(srctree)/scripts/jobserver-count $(SPHINX_PARALLEL)) \
-b $2 \
-c $(abspath $(srctree)/$(src)) \
-d $(abspath $(BUILDDIR)/.doctrees/$3) \
......
#!/bin/sh
# SPDX-License-Identifier: GPL-2.0+
#
# Figure out if we should follow a specific parallelism from the make
# environment (as exported by scripts/jobserver-exec), or fall back to
# the "auto" parallelism when "-jN" is not specified at the top-level
# "make" invocation.
sphinx="$1"
shift || true
parallel="$PARALLELISM"
if [ -z "$parallel" ] ; then
# If no parallelism is specified at the top-level make, then
# fall back to the expected "-jauto" mode that the "htmldocs"
# target has had.
auto=$(perl -e 'open IN,"'"$sphinx"' --version 2>&1 |";
while (<IN>) {
if (m/([\d\.]+)/) {
print "auto" if ($1 >= "1.7")
}
}
close IN')
if [ -n "$auto" ] ; then
parallel="$auto"
fi
fi
# Only if some parallelism has been determined do we add the -jN option.
if [ -n "$parallel" ] ; then
parallel="-j$parallel"
fi
exec "$sphinx" "$parallel" "$@"
......@@ -2,17 +2,17 @@
# SPDX-License-Identifier: GPL-2.0+
#
# This determines how many parallel tasks "make" is expecting, as it is
# not exposed via an special variables.
# not exposed via an special variables, reserves them all, runs a subprocess
# with PARALLELISM environment variable set, and releases the jobs back again.
#
# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
from __future__ import print_function
import os, sys, fcntl, errno
# Default parallelism is "1" unless overridden on the command-line.
default="1"
if len(sys.argv) > 1:
default=sys.argv[1]
import os, sys, errno
import subprocess
# Extract and prepare jobserver file descriptors from envirnoment.
claim = 0
jobs = b""
try:
# Fetch the make environment options.
flags = os.environ['MAKEFLAGS']
......@@ -29,31 +29,38 @@ try:
# on an unexpecting process with the same reader fd.
reader = os.open("/proc/self/fd/%d" % (reader),
os.O_RDONLY | os.O_NONBLOCK)
except (KeyError, IndexError, ValueError, IOError, OSError) as e:
print(e, file=sys.stderr)
# Read out as many jobserver slots as possible.
while True:
try:
slot = os.read(reader, 8)
jobs += slot
except (OSError, IOError) as e:
if e.errno == errno.EWOULDBLOCK:
# Stop at the end of the jobserver queue.
break
# If something went wrong, give back the jobs.
if len(jobs):
os.write(writer, jobs)
raise e
# Add a bump for our caller's reserveration, since we're just going
# to sit here blocked on our child.
claim = len(jobs) + 1
except (KeyError, IndexError, ValueError, OSError, IOError) as e:
# Any missing environment strings or bad fds should result in just
# using the default specified parallelism.
print(default)
sys.exit(0)
# not being parallel.
pass
# Read out as many jobserver slots as possible.
jobs = b""
while True:
try:
slot = os.read(reader, 1)
jobs += slot
except (OSError, IOError) as e:
if e.errno == errno.EWOULDBLOCK:
# Stop when reach the end of the jobserver queue.
break
raise e
# Return all the reserved slots.
os.write(writer, jobs)
# We can only claim parallelism if there was a jobserver (i.e. a top-level
# "-jN" argument) and there were no other failures. Otherwise leave out the
# environment variable and let the child figure out what is best.
if claim > 0:
os.environ['PARALLELISM'] = '%d' % (claim)
# If the jobserver was (impossibly) full or communication failed, use default.
if len(jobs) < 1:
print(default)
sys.exit(0)
rc = subprocess.call(sys.argv[1:])
# Return all the reserved slots.
if len(jobs):
os.write(writer, jobs)
# Report available slots (with a bump for our caller's reserveration).
print(len(jobs) + 1)
sys.exit(rc)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment