Commit 5cb43f32 authored by Kirill Smelkov's avatar Kirill Smelkov

openssl: Allow it to be build with make -jN N > 1

There is a story for OpenSSL build to be broken for `make -j`. Gentoo-
and various other people proposed fixes, at least from 2008, but for
some reasons (patch does not work with BSD make, upstream forgot to
apply, etc) they are not applied to main codebase.

What we can do is to "cherry-pick" the patch from Gentoo to speed-up
OpenSSL build. On a typical 8-cores i7 node, with the following test
buildout:

    [buildout]
    extends =
      slapos/stack/slapos.cfg
      slapos/component/openssl/buildout.cfg

after full build

                    time (rm -rf parts/openssl && ./bin/buildout install openssl)

    MAKEFLAGS=-j1   1m48s
    MAKEFLAGS=-j8   0m53s

and the timing does not subtract time to e.g. download the package from
network, so the building speedup itself is even more.

/cc @kazuhiko
parent e8f90a8d
......@@ -22,6 +22,7 @@ patch-binary = ${patch:location}/bin/patch
patches =
${:_profile_base_location_}/openssl-nodoc.patch#dd1345ae7ed18ccf176bc6b77c516f98
${:_profile_base_location_}/openssl-exlibs.patch#fba5c873cf974ba80a973be41da3c738
${:_profile_base_location_}/openssl-1.0.1m-parallel-build.patch#a1ddd93b5b296473c7446131deb31f93
patch-options = -p1
configure-command = ./config
configure-options =
......@@ -33,10 +34,6 @@ configure-options =
shared no-idea no-mdc2 no-rc5 zlib
-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${buildout:parts-directory}/${:_buildout_section_name_}/lib
&& make depend
# it seems that parallel build sometimes fails for openssl.
make-options =
-j1
make-targets =
all install_sw && rm -f ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/* && for i in ${ca-certificates:location}/certs/*/*.crt; do ln -sv $i ${buildout:parts-directory}/${:_buildout_section_name_}/etc/ssl/certs/`${buildout:parts-directory}/${:_buildout_section_name_}/bin/openssl x509 -hash -noout -in $i`.0; done; true
environment =
......
https://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/dev-libs/openssl/files/
http://rt.openssl.org/Ticket/Display.html?id=2084
--- openssl-1.0.1m/crypto/Makefile
+++ openssl-1.0.1m/crypto/Makefile
@@ -85,11 +85,11 @@
@if [ -z "$(THIS)" ]; then $(MAKE) -f $(TOP)/Makefile reflect THIS=$@; fi
subdirs:
- @target=all; $(RECURSIVE_MAKE)
+ +@target=all; $(RECURSIVE_MAKE)
files:
$(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO
- @target=files; $(RECURSIVE_MAKE)
+ +@target=files; $(RECURSIVE_MAKE)
links:
@$(PERL) $(TOP)/util/mklink.pl ../include/openssl $(EXHEADER)
@@ -100,7 +100,7 @@
# lib: $(LIB): are splitted to avoid end-less loop
lib: $(LIB)
@touch lib
-$(LIB): $(LIBOBJ)
+$(LIB): $(LIBOBJ) | subdirs
$(AR) $(LIB) $(LIBOBJ)
[ -z "$(FIPSLIBDIR)" ] || $(AR) $(LIB) $(FIPSLIBDIR)fipscanister.o
$(RANLIB) $(LIB) || echo Never mind.
@@ -111,7 +111,7 @@
fi
libs:
- @target=lib; $(RECURSIVE_MAKE)
+ +@target=lib; $(RECURSIVE_MAKE)
install:
@[ -n "$(INSTALLTOP)" ] # should be set by top Makefile...
@@ -120,7 +120,7 @@
(cp $$i $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i; \
chmod 644 $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i ); \
done;
- @target=install; $(RECURSIVE_MAKE)
+ +@target=install; $(RECURSIVE_MAKE)
lint:
@target=lint; $(RECURSIVE_MAKE)
--- openssl-1.0.1m/crypto/objects/Makefile
+++ openssl-1.0.1m/crypto/objects/Makefile
@@ -44,11 +44,11 @@
# objects.pl both reads and writes obj_mac.num
obj_mac.h: objects.pl objects.txt obj_mac.num
$(PERL) objects.pl objects.txt obj_mac.num obj_mac.h
- @sleep 1; touch obj_mac.h; sleep 1
-obj_xref.h: objxref.pl obj_xref.txt obj_mac.num
+# This doesn't really need obj_mac.h, but since that rule reads & writes
+# obj_mac.num, we can't run in parallel with it.
+obj_xref.h: objxref.pl obj_xref.txt obj_mac.num obj_mac.h
$(PERL) objxref.pl obj_mac.num obj_xref.txt > obj_xref.h
- @sleep 1; touch obj_xref.h; sleep 1
files:
$(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO
--- openssl-1.0.1m/engines/Makefile
+++ openssl-1.0.1m/engines/Makefile
@@ -72,7 +72,7 @@
all: lib subdirs
-lib: $(LIBOBJ)
+lib: $(LIBOBJ) | subdirs
@if [ -n "$(SHARED_LIBS)" ]; then \
set -e; \
for l in $(LIBNAMES); do \
@@ -89,7 +89,7 @@
subdirs:
echo $(EDIRS)
- @target=all; $(RECURSIVE_MAKE)
+ +@target=all; $(RECURSIVE_MAKE)
files:
$(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO
@@ -128,7 +128,7 @@
mv -f $(INSTALL_PREFIX)$(INSTALLTOP)/$(LIBDIR)/engines/$$pfx$$l$$sfx.new $(INSTALL_PREFIX)$(INSTALLTOP)/$(LIBDIR)/engines/$$pfx$$l$$sfx ); \
done; \
fi
- @target=install; $(RECURSIVE_MAKE)
+ +@target=install; $(RECURSIVE_MAKE)
tags:
ctags $(SRC)
--- openssl-1.0.1m/Makefile.org
+++ openssl-1.0.1m/Makefile.org
@@ -273,17 +273,17 @@
build_libs: build_crypto build_ssl build_engines
build_crypto:
- @dir=crypto; target=all; $(BUILD_ONE_CMD)
-build_ssl:
- @dir=ssl; target=all; $(BUILD_ONE_CMD)
-build_engines:
- @dir=engines; target=all; $(BUILD_ONE_CMD)
-build_apps:
- @dir=apps; target=all; $(BUILD_ONE_CMD)
-build_tests:
- @dir=test; target=all; $(BUILD_ONE_CMD)
-build_tools:
- @dir=tools; target=all; $(BUILD_ONE_CMD)
+ +@dir=crypto; target=all; $(BUILD_ONE_CMD)
+build_ssl: build_crypto
+ +@dir=ssl; target=all; $(BUILD_ONE_CMD)
+build_engines: build_crypto
+ +@dir=engines; target=all; $(BUILD_ONE_CMD)
+build_apps: build_libs
+ +@dir=apps; target=all; $(BUILD_ONE_CMD)
+build_tests: build_libs
+ +@dir=test; target=all; $(BUILD_ONE_CMD)
+build_tools: build_libs
+ +@dir=tools; target=all; $(BUILD_ONE_CMD)
all_testapps: build_libs build_testapps
build_testapps:
@@ -538,9 +538,9 @@
dist_pem_h:
(cd crypto/pem; $(MAKE) -e $(BUILDENV) pem.h; $(MAKE) clean)
-install: all install_docs install_sw
+install: install_docs install_sw
-install_sw:
+install_dirs:
@$(PERL) $(TOP)/util/mkdir-p.pl $(INSTALL_PREFIX)$(INSTALLTOP)/bin \
$(INSTALL_PREFIX)$(INSTALLTOP)/$(LIBDIR) \
$(INSTALL_PREFIX)$(INSTALLTOP)/$(LIBDIR)/engines \
@@ -549,12 +549,19 @@
$(INSTALL_PREFIX)$(OPENSSLDIR)/misc \
$(INSTALL_PREFIX)$(OPENSSLDIR)/certs \
$(INSTALL_PREFIX)$(OPENSSLDIR)/private
+ @$(PERL) $(TOP)/util/mkdir-p.pl \
+ $(INSTALL_PREFIX)$(MANDIR)/man1 \
+ $(INSTALL_PREFIX)$(MANDIR)/man3 \
+ $(INSTALL_PREFIX)$(MANDIR)/man5 \
+ $(INSTALL_PREFIX)$(MANDIR)/man7
+
+install_sw: install_dirs
@set -e; headerlist="$(EXHEADER)"; for i in $$headerlist;\
do \
(cp $$i $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i; \
chmod 644 $(INSTALL_PREFIX)$(INSTALLTOP)/include/openssl/$$i ); \
done;
- @set -e; target=install; $(RECURSIVE_BUILD_CMD)
+ +@set -e; target=install; $(RECURSIVE_BUILD_CMD)
@set -e; liblist="$(LIBS)"; for i in $$liblist ;\
do \
if [ -f "$$i" ]; then \
@@ -634,12 +641,7 @@
done; \
done
-install_docs:
- @$(PERL) $(TOP)/util/mkdir-p.pl \
- $(INSTALL_PREFIX)$(MANDIR)/man1 \
- $(INSTALL_PREFIX)$(MANDIR)/man3 \
- $(INSTALL_PREFIX)$(MANDIR)/man5 \
- $(INSTALL_PREFIX)$(MANDIR)/man7
+install_docs: install_dirs
@pod2man="`cd ./util; ./pod2mantest $(PERL)`"; \
here="`pwd`"; \
filecase=; \
--- openssl-1.0.1m/Makefile.shared
+++ openssl-1.0.1m/Makefile.shared
@@ -105,6 +105,7 @@
SHAREDFLAGS="$${SHAREDFLAGS:-$(CFLAGS) $(SHARED_LDFLAGS)}"; \
LIBPATH=`for x in $$LIBDEPS; do echo $$x; done | sed -e 's/^ *-L//;t' -e d | uniq`; \
LIBPATH=`echo $$LIBPATH | sed -e 's/ /:/g'`; \
+ [ -e $$SHLIB$$SHLIB_SOVER$$SHLIB_SUFFIX ] && exit 0; \
LD_LIBRARY_PATH=$$LIBPATH:$$LD_LIBRARY_PATH \
$${SHAREDCMD} $${SHAREDFLAGS} \
-o $$SHLIB$$SHLIB_SOVER$$SHLIB_SUFFIX \
@@ -122,6 +123,7 @@
done; \
fi; \
if [ -n "$$SHLIB_SOVER" ]; then \
+ [ -e "$$SHLIB$$SHLIB_SUFFIX" ] || \
( $(SET_X); rm -f $$SHLIB$$SHLIB_SUFFIX; \
ln -s $$prev $$SHLIB$$SHLIB_SUFFIX ); \
fi; \
--- openssl-1.0.1m/test/Makefile
+++ openssl-1.0.1m/test/Makefile
@@ -130,7 +130,7 @@
tags:
ctags $(SRC)
-tests: exe apps $(TESTS)
+tests: exe $(TESTS)
apps:
@(cd ..; $(MAKE) DIRS=apps all)
@@ -388,118 +388,118 @@
link_app.$${shlib_target}
$(RSATEST)$(EXE_EXT): $(RSATEST).o $(DLIBCRYPTO)
- @target=$(RSATEST); $(BUILD_CMD)
+ +@target=$(RSATEST); $(BUILD_CMD)
$(BNTEST)$(EXE_EXT): $(BNTEST).o $(DLIBCRYPTO)
- @target=$(BNTEST); $(BUILD_CMD)
+ +@target=$(BNTEST); $(BUILD_CMD)
$(ECTEST)$(EXE_EXT): $(ECTEST).o $(DLIBCRYPTO)
- @target=$(ECTEST); $(BUILD_CMD)
+ +@target=$(ECTEST); $(BUILD_CMD)
$(EXPTEST)$(EXE_EXT): $(EXPTEST).o $(DLIBCRYPTO)
- @target=$(EXPTEST); $(BUILD_CMD)
+ +@target=$(EXPTEST); $(BUILD_CMD)
$(IDEATEST)$(EXE_EXT): $(IDEATEST).o $(DLIBCRYPTO)
- @target=$(IDEATEST); $(BUILD_CMD)
+ +@target=$(IDEATEST); $(BUILD_CMD)
$(MD2TEST)$(EXE_EXT): $(MD2TEST).o $(DLIBCRYPTO)
- @target=$(MD2TEST); $(BUILD_CMD)
+ +@target=$(MD2TEST); $(BUILD_CMD)
$(SHATEST)$(EXE_EXT): $(SHATEST).o $(DLIBCRYPTO)
- @target=$(SHATEST); $(BUILD_CMD)
+ +@target=$(SHATEST); $(BUILD_CMD)
$(SHA1TEST)$(EXE_EXT): $(SHA1TEST).o $(DLIBCRYPTO)
- @target=$(SHA1TEST); $(BUILD_CMD)
+ +@target=$(SHA1TEST); $(BUILD_CMD)
$(SHA256TEST)$(EXE_EXT): $(SHA256TEST).o $(DLIBCRYPTO)
- @target=$(SHA256TEST); $(BUILD_CMD)
+ +@target=$(SHA256TEST); $(BUILD_CMD)
$(SHA512TEST)$(EXE_EXT): $(SHA512TEST).o $(DLIBCRYPTO)
- @target=$(SHA512TEST); $(BUILD_CMD)
+ +@target=$(SHA512TEST); $(BUILD_CMD)
$(RMDTEST)$(EXE_EXT): $(RMDTEST).o $(DLIBCRYPTO)
- @target=$(RMDTEST); $(BUILD_CMD)
+ +@target=$(RMDTEST); $(BUILD_CMD)
$(MDC2TEST)$(EXE_EXT): $(MDC2TEST).o $(DLIBCRYPTO)
- @target=$(MDC2TEST); $(BUILD_CMD)
+ +@target=$(MDC2TEST); $(BUILD_CMD)
$(MD4TEST)$(EXE_EXT): $(MD4TEST).o $(DLIBCRYPTO)
- @target=$(MD4TEST); $(BUILD_CMD)
+ +@target=$(MD4TEST); $(BUILD_CMD)
$(MD5TEST)$(EXE_EXT): $(MD5TEST).o $(DLIBCRYPTO)
- @target=$(MD5TEST); $(BUILD_CMD)
+ +@target=$(MD5TEST); $(BUILD_CMD)
$(HMACTEST)$(EXE_EXT): $(HMACTEST).o $(DLIBCRYPTO)
- @target=$(HMACTEST); $(BUILD_CMD)
+ +@target=$(HMACTEST); $(BUILD_CMD)
$(WPTEST)$(EXE_EXT): $(WPTEST).o $(DLIBCRYPTO)
- @target=$(WPTEST); $(BUILD_CMD)
+ +@target=$(WPTEST); $(BUILD_CMD)
$(RC2TEST)$(EXE_EXT): $(RC2TEST).o $(DLIBCRYPTO)
- @target=$(RC2TEST); $(BUILD_CMD)
+ +@target=$(RC2TEST); $(BUILD_CMD)
$(BFTEST)$(EXE_EXT): $(BFTEST).o $(DLIBCRYPTO)
- @target=$(BFTEST); $(BUILD_CMD)
+ +@target=$(BFTEST); $(BUILD_CMD)
$(CASTTEST)$(EXE_EXT): $(CASTTEST).o $(DLIBCRYPTO)
- @target=$(CASTTEST); $(BUILD_CMD)
+ +@target=$(CASTTEST); $(BUILD_CMD)
$(RC4TEST)$(EXE_EXT): $(RC4TEST).o $(DLIBCRYPTO)
- @target=$(RC4TEST); $(BUILD_CMD)
+ +@target=$(RC4TEST); $(BUILD_CMD)
$(RC5TEST)$(EXE_EXT): $(RC5TEST).o $(DLIBCRYPTO)
- @target=$(RC5TEST); $(BUILD_CMD)
+ +@target=$(RC5TEST); $(BUILD_CMD)
$(DESTEST)$(EXE_EXT): $(DESTEST).o $(DLIBCRYPTO)
- @target=$(DESTEST); $(BUILD_CMD)
+ +@target=$(DESTEST); $(BUILD_CMD)
$(RANDTEST)$(EXE_EXT): $(RANDTEST).o $(DLIBCRYPTO)
- @target=$(RANDTEST); $(BUILD_CMD)
+ +@target=$(RANDTEST); $(BUILD_CMD)
$(DHTEST)$(EXE_EXT): $(DHTEST).o $(DLIBCRYPTO)
- @target=$(DHTEST); $(BUILD_CMD)
+ +@target=$(DHTEST); $(BUILD_CMD)
$(DSATEST)$(EXE_EXT): $(DSATEST).o $(DLIBCRYPTO)
- @target=$(DSATEST); $(BUILD_CMD)
+ +@target=$(DSATEST); $(BUILD_CMD)
$(METHTEST)$(EXE_EXT): $(METHTEST).o $(DLIBCRYPTO)
- @target=$(METHTEST); $(BUILD_CMD)
+ +@target=$(METHTEST); $(BUILD_CMD)
$(SSLTEST)$(EXE_EXT): $(SSLTEST).o $(DLIBSSL) $(DLIBCRYPTO)
- @target=$(SSLTEST); $(FIPS_BUILD_CMD)
+ +@target=$(SSLTEST); $(FIPS_BUILD_CMD)
$(ENGINETEST)$(EXE_EXT): $(ENGINETEST).o $(DLIBCRYPTO)
- @target=$(ENGINETEST); $(BUILD_CMD)
+ +@target=$(ENGINETEST); $(BUILD_CMD)
$(EVPTEST)$(EXE_EXT): $(EVPTEST).o $(DLIBCRYPTO)
- @target=$(EVPTEST); $(BUILD_CMD)
+ +@target=$(EVPTEST); $(BUILD_CMD)
$(EVPEXTRATEST)$(EXE_EXT): $(EVPEXTRATEST).o $(DLIBCRYPTO)
- @target=$(EVPEXTRATEST); $(BUILD_CMD)
+ +@target=$(EVPEXTRATEST); $(BUILD_CMD)
$(ECDSATEST)$(EXE_EXT): $(ECDSATEST).o $(DLIBCRYPTO)
- @target=$(ECDSATEST); $(BUILD_CMD)
+ +@target=$(ECDSATEST); $(BUILD_CMD)
$(ECDHTEST)$(EXE_EXT): $(ECDHTEST).o $(DLIBCRYPTO)
- @target=$(ECDHTEST); $(BUILD_CMD)
+ +@target=$(ECDHTEST); $(BUILD_CMD)
$(IGETEST)$(EXE_EXT): $(IGETEST).o $(DLIBCRYPTO)
- @target=$(IGETEST); $(BUILD_CMD)
+ +@target=$(IGETEST); $(BUILD_CMD)
$(JPAKETEST)$(EXE_EXT): $(JPAKETEST).o $(DLIBCRYPTO)
- @target=$(JPAKETEST); $(BUILD_CMD)
+ +@target=$(JPAKETEST); $(BUILD_CMD)
$(ASN1TEST)$(EXE_EXT): $(ASN1TEST).o $(DLIBCRYPTO)
- @target=$(ASN1TEST); $(BUILD_CMD)
+ +@target=$(ASN1TEST); $(BUILD_CMD)
$(SRPTEST)$(EXE_EXT): $(SRPTEST).o $(DLIBCRYPTO)
- @target=$(SRPTEST); $(BUILD_CMD)
+ +@target=$(SRPTEST); $(BUILD_CMD)
$(HEARTBEATTEST)$(EXE_EXT): $(HEARTBEATTEST).o $(DLIBCRYPTO)
- @target=$(HEARTBEATTEST); $(BUILD_CMD_STATIC)
+ +@target=$(HEARTBEATTEST); $(BUILD_CMD_STATIC)
$(CONSTTIMETEST)$(EXE_EXT): $(CONSTTIMETEST).o
- @target=$(CONSTTIMETEST) $(BUILD_CMD)
+ +@target=$(CONSTTIMETEST) $(BUILD_CMD)
#$(AESTEST).o: $(AESTEST).c
# $(CC) -c $(CFLAGS) -DINTERMEDIATE_VALUE_KAT -DTRACE_KAT_MCT $(AESTEST).c
@@ -512,7 +512,7 @@
# fi
dummytest$(EXE_EXT): dummytest.o $(DLIBCRYPTO)
- @target=dummytest; $(BUILD_CMD)
+ +@target=dummytest; $(BUILD_CMD)
# DO NOT DELETE THIS LINE -- make depend depends on it.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment