Update Release Candidate

component/cmake/buildout.cfg

 [buildout]
 extends =
   ../ncurses/buildout.cfg
+  ../openssl/buildout.cfg
 
 parts =
   cmake
@@ -11,5 +12,5 @@ shared = true
 url = https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz
 md5sum = 79bd7e65cd81ea3aa2619484ad6ff25a
 environment =
-  CMAKE_INCLUDE_PATH=${ncurses:location}/include
-  CMAKE_LIBRARY_PATH=${ncurses:location}/lib
+  CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
+  CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib

component/golang/buildout.cfg

@@ -68,6 +68,14 @@ md5sum = 6132109d4050da349eadc9f7b0304ef4
 environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
 
+[golang1.13]
+<= golang-common
+url = https://dl.google.com/go/go1.13.9.src.tar.gz
+md5sum = 4ad8b04f962be93a32f3021e6f35b3b9
+
+# go1.13 needs go1.4 to bootstrap
+environment-extra =
+  GOROOT_BOOTSTRAP=${golang14:location}
 
 # ---- infrastructure to build Go workspaces / projects ----
 

component/haproxy/buildout.cfg

@@ -13,8 +13,8 @@ parts = haproxy
 [haproxy]
 recipe = slapos.recipe.cmmi
 shared = true
-url = http://www.haproxy.org/download/1.8/src/haproxy-1.8.23.tar.gz
-md5sum = 6c35b83a9969449c4b79783a2119551e
+url = http://www.haproxy.org/download/1.8/src/haproxy-1.8.25.tar.gz
+md5sum = ef2164ca3b1ea9011aa271a8cbe030a4
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/kumo/buildout.cfg

 [buildout]
 extends =
+  ../defaults.cfg
   ../tokyocabinet/buildout.cfg
   ../messagepack/buildout.cfg
   ../openssl/buildout.cfg

component/libmicrohttpd/buildout.cfg

+[buildout]
+extends =
+  ../libiconv/buildout.cfg
+  ../gnutls/buildout.cfg
+  ../curl/buildout.cfg
+
+parts =
+  libmicrohttpd
+
+
+[libmicrohttpd]
+recipe = slapos.recipe.cmmi
+url = https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.70.tar.gz
+md5sum = dcd6045ecb4ea18c120afedccbd1da74
+configure-options =
+  --with-libcurl=${curl:location}
+  --with-gnutls=${gnutls:location}
+  --with-libiconv-prefix=${libiconv:location}
+  --without-libintl-prefix

component/libtiff/buildout.cfg

@@ -19,7 +19,7 @@ configure-options =
   --disable-lzma
 patch-options = -p1
 patches =
-  ${:_profile_base_location_}/tiff_4.0.8-3.debian.patch#9f52aed13d6ae864b3fb46372fe97780
+  ${:_profile_base_location_}/debian_4.0.8-2+deb9u5.patch#dc9c9cb7f4f9a00100f908e640895185
 environment =
   CPPFLAGS=-I${libjpeg:location}/include -I${jbigkit:location}/include -I${zlib:location}/include
   LDFLAGS=-L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib -L${jbigkit:location}/lib -Wl,-rpath=${jbigkit:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib

component/libtiff/debian_4.0.8-2+deb9u5.patch

+diff -pur tiff-4.0.4/tools/tiffsplit.c tiff-4.0.4_patch/tools/tiffsplit.c
+--- tiff-4.0.4/tools/tiffsplit.c	2015-05-28 15:10:26.000000000 +0200
++++ tiff-4.0.4_patch/tools/tiffsplit.c	2016-02-12 19:15:30.532005041 +0100
+@@ -179,8 +179,9 @@ tiffcp(TIFF* in, TIFF* out)
+ 		    TIFFSetField(out, TIFFTAG_JPEGTABLES, count, table);
+ 		}
+ 	}
++	uint32 count = 0;
+         CopyField(TIFFTAG_PHOTOMETRIC, shortv);
+-	CopyField(TIFFTAG_PREDICTOR, shortv);
++	CopyField2(TIFFTAG_PREDICTOR, count, shortv);
+ 	CopyField(TIFFTAG_THRESHHOLDING, shortv);
+ 	CopyField(TIFFTAG_FILLORDER, shortv);
+ 	CopyField(TIFFTAG_ORIENTATION, shortv);
+@@ -188,7 +189,7 @@ tiffcp(TIFF* in, TIFF* out)
+ 	CopyField(TIFFTAG_MAXSAMPLEVALUE, shortv);
+ 	CopyField(TIFFTAG_XRESOLUTION, floatv);
+ 	CopyField(TIFFTAG_YRESOLUTION, floatv);
+-	CopyField(TIFFTAG_GROUP3OPTIONS, longv);
++	CopyField2(TIFFTAG_GROUP3OPTIONS, count, longv);
+ 	CopyField(TIFFTAG_GROUP4OPTIONS, longv);
+ 	CopyField(TIFFTAG_RESOLUTIONUNIT, shortv);
+ 	CopyField(TIFFTAG_PLANARCONFIG, shortv);
+From 02669064e927074819ce1ed39aba0fccaa167717 Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Mon, 29 May 2017 10:12:54 +0000
+Subject: [PATCH] * libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter
+ clamping to avoid int32 overflow in TIFFYCbCrtoRGB(). Fixes
+ https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844 Credit to OSS Fuzz
+
+---
+ ChangeLog           | 7 +++++++
+ libtiff/tif_color.c | 6 +++---
+ 2 files changed, 10 insertions(+), 3 deletions(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index ee8d9d08..61116596 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,3 +1,10 @@
++2017-05-29  Even Rouault <even.rouault at spatialys.com>
++
++	* libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter clamping to avoid
++	int32 overflow in TIFFYCbCrtoRGB().
++	Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844
++	Credit to OSS Fuzz
++
+ 2017-05-21  Bob Friesenhahn  <bfriesen@simple.dallas.tx.us>
+ 
+ 	* configure.ac: libtiff 4.0.8 released.
+diff --git a/libtiff/tif_color.c b/libtiff/tif_color.c
+index 055ed3b2..10a5e66e 100644
+--- a/libtiff/tif_color.c
++++ b/libtiff/tif_color.c
+@@ -275,10 +275,10 @@ TIFFYCbCrToRGBInit(TIFFYCbCrToRGB* ycbcr, float *luma, float *refBlackWhite)
+       for (i = 0, x = -128; i < 256; i++, x++) {
+ 	    int32 Cr = (int32)CLAMPw(Code2V(x, refBlackWhite[4] - 128.0F,
+ 			    refBlackWhite[5] - 128.0F, 127),
+-                            -128.0F * 64, 128.0F * 64);
++                            -128.0F * 32, 128.0F * 32);
+ 	    int32 Cb = (int32)CLAMPw(Code2V(x, refBlackWhite[2] - 128.0F,
+ 			    refBlackWhite[3] - 128.0F, 127),
+-                            -128.0F * 64, 128.0F * 64);
++                            -128.0F * 32, 128.0F * 32);
+ 
+ 	    ycbcr->Cr_r_tab[i] = (int32)((D1*Cr + ONE_HALF)>>SHIFT);
+ 	    ycbcr->Cb_b_tab[i] = (int32)((D3*Cb + ONE_HALF)>>SHIFT);
+@@ -286,7 +286,7 @@ TIFFYCbCrToRGBInit(TIFFYCbCrToRGB* ycbcr, float *luma, float *refBlackWhite)
+ 	    ycbcr->Cb_g_tab[i] = D4*Cb + ONE_HALF;
+ 	    ycbcr->Y_tab[i] =
+ 		    (int32)CLAMPw(Code2V(x + 128, refBlackWhite[0], refBlackWhite[1], 255),
+-                                  -128.0F * 64, 128.0F * 64);
++                                  -128.0F * 32, 128.0F * 32);
+       }
+     }
+ 
+From 468988860e0dae62ebbf991627c74bcbb4bd256f Mon Sep 17 00:00:00 2001
+From: erouault <erouault>
+Date: Mon, 29 May 2017 11:29:06 +0000
+Subject: [PATCH] * libtiff/tif_getimage.c: initYCbCrConversion(): stricter
+ validation for refBlackWhite coefficients values. To avoid invalid
+ float->int32 conversion (when refBlackWhite[0] == 2147483648.f) Fixes
+ https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1907 Credit to OSS Fuzz
+
+---
+ ChangeLog              | 8 ++++++++
+ libtiff/tif_getimage.c | 2 +-
+ 2 files changed, 9 insertions(+), 1 deletion(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index a2ddaac2..04881ba7 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,5 +1,13 @@
+ 2017-05-29  Even Rouault <even.rouault at spatialys.com>
+ 
++	* libtiff/tif_getimage.c: initYCbCrConversion(): stricter validation for
++	refBlackWhite coefficients values. To avoid invalid float->int32 conversion
++	(when refBlackWhite[0] == 2147483648.f)
++	Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1907
++	Credit to OSS Fuzz
++
++2017-05-29  Even Rouault <even.rouault at spatialys.com>
++
+ 	* libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter clamping to avoid
+ 	int32 overflow in TIFFYCbCrtoRGB().
+ 	Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844
+diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
+index dc373abc..a209a7a7 100644
+--- a/libtiff/tif_getimage.c
++++ b/libtiff/tif_getimage.c
+@@ -2241,7 +2241,7 @@ DECLARESepPutFunc(putseparate8bitYCbCr11tile)
+ 
+ static int isInRefBlackWhiteRange(float f)
+ {
+-    return f >= (float)(-0x7FFFFFFF + 128) && f <= (float)0x7FFFFFFF;
++    return f > (float)(-0x7FFFFFFF + 128) && f < (float)0x7FFFFFFF;
+ }
+ 
+ static int
+commit 40448d58fbfad52d2dde5bd18daa30b17fe35fcd
+Author: erouault <erouault>
+Date:   Thu Jun 1 12:44:04 2017 +0000
+
+    * libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
+    and use it in TIFFReadDirectory() so as to ignore fields whose tag is a
+    codec-specified tag but this codec is not enabled. This avoids TIFFGetField()
+    to behave differently depending on whether the codec is enabled or not, and
+    thus can avoid stack based buffer overflows in a number of TIFF utilities
+    such as tiffsplit, tiffcmp, thumbnail, etc.
+    Patch derived from 0063-Handle-properly-CODEC-specific-tags.patch
+    (http://bugzilla.maptools.org/show_bug.cgi?id=2580) by Raphaël Hertzog.
+    Fixes:
+    http://bugzilla.maptools.org/show_bug.cgi?id=2580
+    http://bugzilla.maptools.org/show_bug.cgi?id=2693
+    http://bugzilla.maptools.org/show_bug.cgi?id=2625 (CVE-2016-10095)
+    http://bugzilla.maptools.org/show_bug.cgi?id=2564 (CVE-2015-7554)
+    http://bugzilla.maptools.org/show_bug.cgi?id=2561 (CVE-2016-5318)
+    http://bugzilla.maptools.org/show_bug.cgi?id=2499 (CVE-2014-8128)
+    http://bugzilla.maptools.org/show_bug.cgi?id=2441
+    http://bugzilla.maptools.org/show_bug.cgi?id=2433
+
+diff --git a/ChangeLog b/ChangeLog
+index 04881ba7..ebd1a3c0 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,3 +1,23 @@
++2017-06-01  Even Rouault <even.rouault at spatialys.com>
++
++	* libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
++	and use it in TIFFReadDirectory() so as to ignore fields whose tag is a
++	codec-specified tag but this codec is not enabled. This avoids TIFFGetField()
++	to behave differently depending on whether the codec is enabled or not, and
++	thus can avoid stack based buffer overflows in a number of TIFF utilities
++	such as tiffsplit, tiffcmp, thumbnail, etc.
++	Patch derived from 0063-Handle-properly-CODEC-specific-tags.patch
++	(http://bugzilla.maptools.org/show_bug.cgi?id=2580) by Raphaël Hertzog.
++	Fixes:
++	http://bugzilla.maptools.org/show_bug.cgi?id=2580
++	http://bugzilla.maptools.org/show_bug.cgi?id=2693
++	http://bugzilla.maptools.org/show_bug.cgi?id=2625 (CVE-2016-10095)
++	http://bugzilla.maptools.org/show_bug.cgi?id=2564 (CVE-2015-7554)
++	http://bugzilla.maptools.org/show_bug.cgi?id=2561 (CVE-2016-5318)
++	http://bugzilla.maptools.org/show_bug.cgi?id=2499 (CVE-2014-8128)
++	http://bugzilla.maptools.org/show_bug.cgi?id=2441
++	http://bugzilla.maptools.org/show_bug.cgi?id=2433
++
+ 2017-05-29  Even Rouault <even.rouault at spatialys.com>
+ 
+ 	* libtiff/tif_getimage.c: initYCbCrConversion(): stricter validation for
+diff --git a/libtiff/tif_dir.h b/libtiff/tif_dir.h
+index 6af5f3dc..5a380767 100644
+--- a/libtiff/tif_dir.h
++++ b/libtiff/tif_dir.h
+@@ -1,4 +1,4 @@
+-/* $Id: tif_dir.h,v 1.54 2011-02-18 20:53:05 fwarmerdam Exp $ */
++/* $Id: tif_dir.h,v 1.55 2017-06-01 12:44:04 erouault Exp $ */
+ 
+ /*
+  * Copyright (c) 1988-1997 Sam Leffler
+@@ -291,6 +291,7 @@ struct _TIFFField {
+ extern int _TIFFMergeFields(TIFF*, const TIFFField[], uint32);
+ extern const TIFFField* _TIFFFindOrRegisterField(TIFF *, uint32, TIFFDataType);
+ extern  TIFFField* _TIFFCreateAnonField(TIFF *, uint32, TIFFDataType);
++extern int _TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag);
+ 
+ #if defined(__cplusplus)
+ }
+diff --git a/libtiff/tif_dirinfo.c b/libtiff/tif_dirinfo.c
+index 23ad0020..4904f540 100644
+--- a/libtiff/tif_dirinfo.c
++++ b/libtiff/tif_dirinfo.c
+@@ -1,4 +1,4 @@
+-/* $Id: tif_dirinfo.c,v 1.126 2016-11-18 02:52:13 bfriesen Exp $ */
++/* $Id: tif_dirinfo.c,v 1.127 2017-06-01 12:44:04 erouault Exp $ */
+ 
+ /*
+  * Copyright (c) 1988-1997 Sam Leffler
+@@ -956,6 +956,109 @@ TIFFMergeFieldInfo(TIFF* tif, const TIFFFieldInfo info[], uint32 n)
+ 	return 0;
+ }
+ 
++int
++_TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag)
++{
++	/* Filter out non-codec specific tags */
++	switch (tag) {
++	    /* Shared tags */
++	    case TIFFTAG_PREDICTOR:
++	    /* JPEG tags */
++	    case TIFFTAG_JPEGTABLES:
++	    /* OJPEG tags */
++	    case TIFFTAG_JPEGIFOFFSET:
++	    case TIFFTAG_JPEGIFBYTECOUNT:
++	    case TIFFTAG_JPEGQTABLES:
++	    case TIFFTAG_JPEGDCTABLES:
++	    case TIFFTAG_JPEGACTABLES:
++	    case TIFFTAG_JPEGPROC:
++	    case TIFFTAG_JPEGRESTARTINTERVAL:
++	    /* CCITT* */
++	    case TIFFTAG_BADFAXLINES:
++	    case TIFFTAG_CLEANFAXDATA:
++	    case TIFFTAG_CONSECUTIVEBADFAXLINES:
++	    case TIFFTAG_GROUP3OPTIONS:
++	    case TIFFTAG_GROUP4OPTIONS:
++		break;
++	    default:
++		return 1;
++	}
++	/* Check if codec specific tags are allowed for the current
++	 * compression scheme (codec) */
++	switch (tif->tif_dir.td_compression) {
++	    case COMPRESSION_LZW:
++		if (tag == TIFFTAG_PREDICTOR)
++		    return 1;
++		break;
++	    case COMPRESSION_PACKBITS:
++		/* No codec-specific tags */
++		break;
++	    case COMPRESSION_THUNDERSCAN:
++		/* No codec-specific tags */
++		break;
++	    case COMPRESSION_NEXT:
++		/* No codec-specific tags */
++		break;
++	    case COMPRESSION_JPEG:
++		if (tag == TIFFTAG_JPEGTABLES)
++		    return 1;
++		break;
++	    case COMPRESSION_OJPEG:
++		switch (tag) {
++		    case TIFFTAG_JPEGIFOFFSET:
++		    case TIFFTAG_JPEGIFBYTECOUNT:
++		    case TIFFTAG_JPEGQTABLES:
++		    case TIFFTAG_JPEGDCTABLES:
++		    case TIFFTAG_JPEGACTABLES:
++		    case TIFFTAG_JPEGPROC:
++		    case TIFFTAG_JPEGRESTARTINTERVAL:
++			return 1;
++		}
++		break;
++	    case COMPRESSION_CCITTRLE:
++	    case COMPRESSION_CCITTRLEW:
++	    case COMPRESSION_CCITTFAX3:
++	    case COMPRESSION_CCITTFAX4:
++		switch (tag) {
++		    case TIFFTAG_BADFAXLINES:
++		    case TIFFTAG_CLEANFAXDATA:
++		    case TIFFTAG_CONSECUTIVEBADFAXLINES:
++			return 1;
++		    case TIFFTAG_GROUP3OPTIONS:
++			if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX3)
++			    return 1;
++			break;
++		    case TIFFTAG_GROUP4OPTIONS:
++			if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX4)
++			    return 1;
++			break;
++		}
++		break;
++	    case COMPRESSION_JBIG:
++		/* No codec-specific tags */
++		break;
++	    case COMPRESSION_DEFLATE:
++	    case COMPRESSION_ADOBE_DEFLATE:
++		if (tag == TIFFTAG_PREDICTOR)
++		    return 1;
++		break;
++	   case COMPRESSION_PIXARLOG:
++		if (tag == TIFFTAG_PREDICTOR)
++		    return 1;
++		break;
++	    case COMPRESSION_SGILOG:
++	    case COMPRESSION_SGILOG24:
++		/* No codec-specific tags */
++		break;
++	    case COMPRESSION_LZMA:
++		if (tag == TIFFTAG_PREDICTOR)
++		    return 1;
++		break;
++
++	}
++	return 0;
++}
++
+ /* vim: set ts=8 sts=8 sw=8 noet: */
+ 
+ /*
+diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
+index 772ebaf7..acde78b5 100644
+--- a/libtiff/tif_dirread.c
++++ b/libtiff/tif_dirread.c
+@@ -1,4 +1,4 @@
+-/* $Id: tif_dirread.c,v 1.208 2017-04-27 15:46:22 erouault Exp $ */
++/* $Id: tif_dirread.c,v 1.209 2017-06-01 12:44:04 erouault Exp $ */
+ 
+ /*
+  * Copyright (c) 1988-1997 Sam Leffler
+@@ -3580,6 +3580,10 @@ TIFFReadDirectory(TIFF* tif)
+ 							goto bad;
+ 						dp->tdir_tag=IGNORE;
+ 						break;
++                                        default:
++                                            if( !_TIFFCheckFieldIsValidForCodec(tif, dp->tdir_tag) )
++                                                dp->tdir_tag=IGNORE;
++                                            break;
+ 				}
+ 			}
+ 		}
+From fe8d7165956b88df4837034a9161dc5fd20cf67a Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Mon, 26 Jun 2017 15:19:59 +0000
+Subject: [PATCH] * libtiff/tif_jbig.c: fix memory leak in error code path of
+ JBIGDecode() Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2706 Reported
+ by team OWL337
+
+* libtiff/tif_jpeg.c: error out at decoding time if anticipated libjpeg
+---
+ ChangeLog          | 8 +++++++-
+ libtiff/tif_jbig.c | 1 +
+ 2 files changed, 8 insertions(+), 1 deletion(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index bc5096e7..ecd70534 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,3 +1,9 @@
++2017-06-26  Even Rouault <even.rouault at spatialys.com>
++
++	* libtiff/tif_jbig.c: fix memory leak in error code path of JBIGDecode()
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2706
++	Reported by team OWL337
++
+ 2017-06-01  Even Rouault <even.rouault at spatialys.com>
+ 
+ 	* libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
+diff --git a/libtiff/tif_jbig.c b/libtiff/tif_jbig.c
+index 5f5f75e2..c75f31d9 100644
+--- a/libtiff/tif_jbig.c
++++ b/libtiff/tif_jbig.c
+@@ -94,6 +94,7 @@ static int JBIGDecode(TIFF* tif, uint8* buffer, tmsize_t size, uint16 s)
+ 			     jbg_strerror(decodeStatus)
+ #endif
+ 			     );
++		jbg_dec_free(&decoder);
+ 		return 0;
+ 	}
+ 
+From 1077fad562e03d1cad591dd10163dd80ad63ab0e Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Fri, 30 Jun 2017 13:11:18 +0000
+Subject: [PATCH] * libtiff/tif_read.c, tiffiop.h: add a
+ _TIFFReadEncodedStripAndAllocBuffer() function, variant of
+ TIFFReadEncodedStrip() that allocates the decoded buffer only after a first
+ successful TIFFFillStrip(). This avoids excessive memory allocation on
+ corrupted files. * libtiff/tif_getimage.c: use
+ _TIFFReadEncodedStripAndAllocBuffer(). Fixes
+ http://bugzilla.maptools.org/show_bug.cgi?id=2708 and
+ https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2433 . Credit to OSS
+ Fuzz
+
+---
+ ChangeLog              | 11 +++++++
+ libtiff/tif_getimage.c | 59 ++++++++++++++++++++++----------------
+ libtiff/tif_read.c     | 78 +++++++++++++++++++++++++++++++++++++++++++-------
+ libtiff/tiffiop.h      |  5 ++++
+ 4 files changed, 118 insertions(+), 35 deletions(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index c969f9e2..6f085e09 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,3 +1,14 @@
++2017-06-30  Even Rouault <even.rouault at spatialys.com>
++
++	* libtiff/tif_read.c, tiffiop.h: add a _TIFFReadEncodedStripAndAllocBuffer()
++	function, variant of TIFFReadEncodedStrip() that allocates the
++	decoded buffer only after a first successful TIFFFillStrip(). This avoids
++	excessive memory allocation on corrupted files.
++	* libtiff/tif_getimage.c: use _TIFFReadEncodedStripAndAllocBuffer(). 
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2708 and
++	https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2433 .
++	Credit to OSS Fuzz
++
+ 2017-06-26  Even Rouault <even.rouault at spatialys.com>
+ 
+ 	* libtiff/tif_jbig.c: fix memory leak in error code path of JBIGDecode()
+diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
+index cee8e930..cc6e8f30 100644
+--- a/libtiff/tif_getimage.c
++++ b/libtiff/tif_getimage.c
+@@ -905,26 +905,22 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ 	tileContigRoutine put = img->put.contig;
+ 	uint32 row, y, nrow, nrowsub, rowstoread;
+ 	tmsize_t pos;
+-	unsigned char* buf;
++	unsigned char* buf = NULL;
+ 	uint32 rowsperstrip;
+ 	uint16 subsamplinghor,subsamplingver;
+ 	uint32 imagewidth = img->width;
+ 	tmsize_t scanline;
+ 	int32 fromskew, toskew;
+ 	int ret = 1, flip;
++        tmsize_t maxstripsize;
+ 
+ 	TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING, &subsamplinghor, &subsamplingver);
+ 	if( subsamplingver == 0 ) {
+ 		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Invalid vertical YCbCr subsampling");
+ 		return (0);
+ 	}
+-
+-	buf = (unsigned char*) _TIFFmalloc(TIFFStripSize(tif));
+-	if (buf == 0) {
+-		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
+-		return (0);
+-	}
+-	_TIFFmemset(buf, 0, TIFFStripSize(tif));
++	
++	maxstripsize = TIFFStripSize(tif);
+ 
+ 	flip = setorientation(img);
+ 	if (flip & FLIP_VERTICALLY) {
+@@ -946,11 +942,12 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ 		nrowsub = nrow;
+ 		if ((nrowsub%subsamplingver)!=0)
+ 			nrowsub+=subsamplingver-nrowsub%subsamplingver;
+-		if (TIFFReadEncodedStrip(tif,
++		if (_TIFFReadEncodedStripAndAllocBuffer(tif,
+ 		    TIFFComputeStrip(tif,row+img->row_offset, 0),
+-		    buf,
++		    (void**)(&buf),
++                    maxstripsize,
+ 		    ((row + img->row_offset)%rowsperstrip + nrowsub) * scanline)==(tmsize_t)(-1)
+-		    && img->stoponerr)
++		    && (buf == NULL || img->stoponerr))
+ 		{
+ 			ret = 0;
+ 			break;
+@@ -994,8 +991,8 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ {
+ 	TIFF* tif = img->tif;
+ 	tileSeparateRoutine put = img->put.separate;
+-	unsigned char *buf;
+-	unsigned char *p0, *p1, *p2, *pa;
++	unsigned char *buf = NULL;
++	unsigned char *p0 = NULL, *p1 = NULL, *p2 = NULL, *pa = NULL;
+ 	uint32 row, y, nrow, rowstoread;
+ 	tmsize_t pos;
+ 	tmsize_t scanline;
+@@ -1014,15 +1011,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ 		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
+ 		return (0);
+ 	}
+-	p0 = buf = (unsigned char *)_TIFFmalloc(bufsize);
+-	if (buf == 0) {
+-		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for tile buffer");
+-		return (0);
+-	}
+-	_TIFFmemset(buf, 0, bufsize);
+-	p1 = p0 + stripsize;
+-	p2 = p1 + stripsize;
+-	pa = (alpha?(p2+stripsize):NULL);
+ 
+ 	flip = setorientation(img);
+ 	if (flip & FLIP_VERTICALLY) {
+@@ -1040,7 +1028,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+           case PHOTOMETRIC_MINISBLACK:
+           case PHOTOMETRIC_PALETTE:
+             colorchannels = 1;
+-            p2 = p1 = p0;
+             break;
+ 
+           default:
+@@ -1056,7 +1043,31 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ 		rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
+ 		nrow = (row + rowstoread > h ? h - row : rowstoread);
+ 		offset_row = row + img->row_offset;
+-		if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
++                if( buf == NULL )
++                {
++                    if (_TIFFReadEncodedStripAndAllocBuffer(
++                            tif, TIFFComputeStrip(tif, offset_row, 0),
++                            (void**) &buf, bufsize,
++                            ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
++                        && (buf == NULL || img->stoponerr))
++                    {
++                            ret = 0;
++                            break;
++                    }
++                    p0 = buf;
++                    if( colorchannels == 1 )
++                    {
++                        p2 = p1 = p0;
++                        pa = (alpha?(p0+3*stripsize):NULL);
++                    }
++                    else
++                    {
++                        p1 = p0 + stripsize;
++                        p2 = p1 + stripsize;
++                        pa = (alpha?(p2+stripsize):NULL);
++                    }
++                }
++		else if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
+ 		    p0, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+ 		    && img->stoponerr)
+ 		{
+diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
+index fc0072e7..047305ab 100644
+--- a/libtiff/tif_read.c
++++ b/libtiff/tif_read.c
+@@ -442,18 +442,17 @@ TIFFReadScanline(TIFF* tif, void* buf, uint32 row, uint16 sample)
+ }
+ 
+ /*
+- * Read a strip of data and decompress the specified
+- * amount into the user-supplied buffer.
++ * Calculate the strip size according to the number of
++ * rows in the strip (check for truncated last strip on any
++ * of the separations).
+  */
+-tmsize_t
+-TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
++static tmsize_t TIFFReadEncodedStripGetStripSize(TIFF* tif, uint32 strip, uint16* pplane)
+ {
+ 	static const char module[] = "TIFFReadEncodedStrip";
+ 	TIFFDirectory *td = &tif->tif_dir;
+ 	uint32 rowsperstrip;
+ 	uint32 stripsperplane;
+ 	uint32 stripinplane;
+-	uint16 plane;
+ 	uint32 rows;
+ 	tmsize_t stripsize;
+ 	if (!TIFFCheckRead(tif,0))
+@@ -465,23 +464,37 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+ 		    (unsigned long)td->td_nstrips);
+ 		return((tmsize_t)(-1));
+ 	}
+-	/*
+-	 * Calculate the strip size according to the number of
+-	 * rows in the strip (check for truncated last strip on any
+-	 * of the separations).
+-	 */
++
+ 	rowsperstrip=td->td_rowsperstrip;
+ 	if (rowsperstrip>td->td_imagelength)
+ 		rowsperstrip=td->td_imagelength;
+ 	stripsperplane= TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip);
+ 	stripinplane=(strip%stripsperplane);
+-	plane=(uint16)(strip/stripsperplane);
++	if( pplane ) *pplane=(uint16)(strip/stripsperplane);
+ 	rows=td->td_imagelength-stripinplane*rowsperstrip;
+ 	if (rows>rowsperstrip)
+ 		rows=rowsperstrip;
+ 	stripsize=TIFFVStripSize(tif,rows);
+ 	if (stripsize==0)
+ 		return((tmsize_t)(-1));
++	return stripsize;
++}
++
++/*
++ * Read a strip of data and decompress the specified
++ * amount into the user-supplied buffer.
++ */
++tmsize_t
++TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
++{
++	static const char module[] = "TIFFReadEncodedStrip";
++	TIFFDirectory *td = &tif->tif_dir;
++	tmsize_t stripsize;
++	uint16 plane;
++
++	stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
++	if (stripsize==((tmsize_t)(-1)))
++		return((tmsize_t)(-1));
+ 
+     /* shortcut to avoid an extra memcpy() */
+     if( td->td_compression == COMPRESSION_NONE &&
+@@ -510,6 +523,49 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+ 	return(stripsize);
+ }
+ 
++/* Variant of TIFFReadEncodedStrip() that does 
++ * * if *buf == NULL, *buf = _TIFFmalloc(bufsizetoalloc) only after TIFFFillStrip() has
++ *   suceeded. This avoid excessive memory allocation in case of truncated
++ *   file.
++ * * calls regular TIFFReadEncodedStrip() if *buf != NULL
++ */
++tmsize_t
++_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
++                                    void **buf, tmsize_t bufsizetoalloc,
++                                    tmsize_t size_to_read)
++{
++    tmsize_t this_stripsize;
++    uint16 plane;
++
++    if( *buf != NULL )
++    {
++        return TIFFReadEncodedStrip(tif, strip, *buf, size_to_read);
++    }
++
++    this_stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
++    if (this_stripsize==((tmsize_t)(-1)))
++            return((tmsize_t)(-1));
++
++    if ((size_to_read!=(tmsize_t)(-1))&&(size_to_read<this_stripsize))
++            this_stripsize=size_to_read;
++    if (!TIFFFillStrip(tif,strip))
++            return((tmsize_t)(-1));
++
++    *buf = _TIFFmalloc(bufsizetoalloc);
++    if (*buf == NULL) {
++            TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
++            return((tmsize_t)(-1));
++    }
++    _TIFFmemset(*buf, 0, bufsizetoalloc);
++
++    if ((*tif->tif_decodestrip)(tif,*buf,this_stripsize,plane)<=0)
++            return((tmsize_t)(-1));
++    (*tif->tif_postdecode)(tif,*buf,this_stripsize);
++    return(this_stripsize);
++
++
++}
++
+ static tmsize_t
+ TIFFReadRawStrip1(TIFF* tif, uint32 strip, void* buf, tmsize_t size,
+     const char* module)
+diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
+index 846ade03..7f0b90f7 100644
+--- a/libtiff/tiffiop.h
++++ b/libtiff/tiffiop.h
+@@ -365,6 +365,11 @@ extern void* _TIFFCheckRealloc(TIFF*, void*, tmsize_t, tmsize_t, const char*);
+ extern double _TIFFUInt64ToDouble(uint64);
+ extern float _TIFFUInt64ToFloat(uint64);
+ 
++extern tmsize_t
++_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
++                                    void **buf, tmsize_t bufsizetoalloc,
++                                    tmsize_t size_to_read);
++
+ extern int TIFFInitDumpMode(TIFF*, int);
+ #ifdef PACKBITS_SUPPORT
+ extern int TIFFInitPackBits(TIFF*, int);
+From 6173a57d39e04d68b139f8c1aa499a24dbe74ba1 Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Fri, 30 Jun 2017 17:29:44 +0000
+Subject: [PATCH] * libtiff/tif_dirwrite.c: in
+ TIFFWriteDirectoryTagCheckedXXXX() functions associated with LONG8/SLONG8
+ data type, replace assertion that the file is BigTIFF, by a non-fatal error.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2712 Reported by team
+ OWL337
+
+---
+ ChangeLog              |  8 ++++++++
+ libtiff/tif_dirwrite.c | 20 ++++++++++++++++----
+ 2 files changed, 24 insertions(+), 4 deletions(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index 6f085e09..77a64385 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,5 +1,13 @@
+ 2017-06-30  Even Rouault <even.rouault at spatialys.com>
+ 
++	* libtiff/tif_dirwrite.c: in TIFFWriteDirectoryTagCheckedXXXX()
++	functions associated with LONG8/SLONG8 data type, replace assertion that
++	the file is BigTIFF, by a non-fatal error.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2712
++	Reported by team OWL337
++
++2017-06-30  Even Rouault <even.rouault at spatialys.com>
++
+ 	* libtiff/tif_read.c, tiffiop.h: add a _TIFFReadEncodedStripAndAllocBuffer()
+ 	function, variant of TIFFReadEncodedStrip() that allocates the
+ 	decoded buffer only after a first successful TIFFFillStrip(). This avoids
+diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
+index 2967da58..8d6686ba 100644
+--- a/libtiff/tif_dirwrite.c
++++ b/libtiff/tif_dirwrite.c
+@@ -2111,7 +2111,10 @@ TIFFWriteDirectoryTagCheckedLong8(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, ui
+ {
+ 	uint64 m;
+ 	assert(sizeof(uint64)==8);
+-	assert(tif->tif_flags&TIFF_BIGTIFF);
++	if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
++		TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","LONG8 not allowed for ClassicTIFF");
++		return(0);
++	}
+ 	m=value;
+ 	if (tif->tif_flags&TIFF_SWAB)
+ 		TIFFSwabLong8(&m);
+@@ -2124,7 +2127,10 @@ TIFFWriteDirectoryTagCheckedLong8Array(TIFF* tif, uint32* ndir, TIFFDirEntry* di
+ {
+ 	assert(count<0x20000000);
+ 	assert(sizeof(uint64)==8);
+-	assert(tif->tif_flags&TIFF_BIGTIFF);
++	if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
++		TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","LONG8 not allowed for ClassicTIFF");
++		return(0);
++	}
+ 	if (tif->tif_flags&TIFF_SWAB)
+ 		TIFFSwabArrayOfLong8(value,count);
+ 	return(TIFFWriteDirectoryTagData(tif,ndir,dir,tag,TIFF_LONG8,count,count*8,value));
+@@ -2136,7 +2142,10 @@ TIFFWriteDirectoryTagCheckedSlong8(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, u
+ {
+ 	int64 m;
+ 	assert(sizeof(int64)==8);
+-	assert(tif->tif_flags&TIFF_BIGTIFF);
++	if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
++		TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","SLONG8 not allowed for ClassicTIFF");
++		return(0);
++	}
+ 	m=value;
+ 	if (tif->tif_flags&TIFF_SWAB)
+ 		TIFFSwabLong8((uint64*)(&m));
+@@ -2149,7 +2158,10 @@ TIFFWriteDirectoryTagCheckedSlong8Array(TIFF* tif, uint32* ndir, TIFFDirEntry* d
+ {
+ 	assert(count<0x20000000);
+ 	assert(sizeof(int64)==8);
+-	assert(tif->tif_flags&TIFF_BIGTIFF);
++	if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
++		TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","SLONG8 not allowed for ClassicTIFF");
++		return(0);
++	}
+ 	if (tif->tif_flags&TIFF_SWAB)
+ 		TIFFSwabArrayOfLong8((uint64*)value,count);
+ 	return(TIFFWriteDirectoryTagData(tif,ndir,dir,tag,TIFF_SLONG8,count,count*8,value));
+From 69bfeec247899776b1b396651adb47436e5f1556 Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Sat, 15 Jul 2017 11:13:46 +0000
+Subject: [PATCH] * tools/tiff2pdf.c: prevent heap buffer overflow write in
+ "Raw" mode on PlanarConfig=Contig input images. Fixes
+ http://bugzilla.maptools.org/show_bug.cgi?id=2715 Reported by team OWL337
+
+---
+ ChangeLog        | 7 +++++++
+ tools/tiff2pdf.c | 7 ++++++-
+ 2 files changed, 13 insertions(+), 1 deletion(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index b4771234..1b5490f3 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,3 +1,10 @@
++2017-07-15  Even Rouault <even.rouault at spatialys.com>
++
++	* tools/tiff2pdf.c: prevent heap buffer overflow write in "Raw"
++	mode on PlanarConfig=Contig input images.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2715
++	Reported by team OWL337
++
+ 2017-06-30  Even Rouault <even.rouault at spatialys.com>
+ 
+ 	* libtiff/tif_dirwrite.c: in TIFFWriteDirectoryTagCheckedXXXX()
+diff --git a/tools/tiff2pdf.c b/tools/tiff2pdf.c
+index db196e04..cd1e2358 100644
+--- a/tools/tiff2pdf.c
++++ b/tools/tiff2pdf.c
+@@ -1737,7 +1737,12 @@ void t2p_read_tiff_data(T2P* t2p, TIFF* input){
+ 	    return;
+ 
+ 	t2p->pdf_transcode = T2P_TRANSCODE_ENCODE;
+-	if(t2p->pdf_nopassthrough==0){
++        /* It seems that T2P_TRANSCODE_RAW mode doesn't support separate->contig */
++        /* conversion. At least t2p_read_tiff_size and t2p_read_tiff_size_tile */
++        /* do not take into account the number of samples, and thus */
++        /* that can cause heap buffer overflows such as in */
++        /* http://bugzilla.maptools.org/show_bug.cgi?id=2715 */
++	if(t2p->pdf_nopassthrough==0 && t2p->tiff_planar!=PLANARCONFIG_SEPARATE){
+ #ifdef CCITT_SUPPORT
+ 		if(t2p->tiff_compression==COMPRESSION_CCITTFAX4  
+ 			){
+From dc02f9050311a90b3c0655147cee09bfa7081cfc Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Sat, 15 Jul 2017 13:23:09 +0000
+Subject: [PATCH] * libtiff/tif_read.c: add protection against excessive memory
+ allocation attempts in TIFFReadDirEntryArray() on short files. Effective for
+ mmap'ed case. And non-mmap'ed case, but restricted to 64bit builds. Fixes
+ http://bugzilla.maptools.org/show_bug.cgi?id=2675
+
+---
+ ChangeLog             |  8 +++++
+ libtiff/tif_dirread.c | 89 ++++++++++++++++++++++++++++++++++++++++++++++++---
+ 2 files changed, 92 insertions(+), 5 deletions(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index 932ddee5..e8a2be5b 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,5 +1,13 @@
+ 2017-07-15  Even Rouault <even.rouault at spatialys.com>
+ 
++	* libtiff/tif_read.c: add protection against excessive memory
++	allocation attempts in TIFFReadDirEntryArray() on short files.
++	Effective for mmap'ed case. And non-mmap'ed case, but restricted
++	to 64bit builds.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2675
++
++2017-07-15  Even Rouault <even.rouault at spatialys.com>
++
+ 	* tools/tiff2pdf.c: prevent heap buffer overflow write in "Raw"
+ 	mode on PlanarConfig=Contig input images.
+ 	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2715
+diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
+index 99ca5200..411d1a45 100644
+--- a/libtiff/tif_dirread.c
++++ b/libtiff/tif_dirread.c
+@@ -765,6 +765,65 @@ static enum TIFFReadDirEntryErr TIFFRead
+ 	}
+ }
+ 
++#define INITIAL_THRESHOLD (1024 * 1024)
++#define THRESHOLD_MULTIPLIER 10
++#define MAX_THRESHOLD (THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * INITIAL_THRESHOLD)
++
++static enum TIFFReadDirEntryErr TIFFReadDirEntryDataAndRealloc(
++                    TIFF* tif, uint64 offset, tmsize_t size, void** pdest)
++{
++#if SIZEOF_VOIDP == 8 || SIZEOF_SIZE_T == 8
++        tmsize_t threshold = INITIAL_THRESHOLD;
++#endif
++        tmsize_t already_read = 0;
++
++        assert( !isMapped(tif) );
++
++        if (!SeekOK(tif,offset))
++                return(TIFFReadDirEntryErrIo);
++
++        /* On 64 bit processes, read first a maximum of 1 MB, then 10 MB, etc */
++        /* so as to avoid allocating too much memory in case the file is too */
++        /* short. We could ask for the file size, but this might be */
++        /* expensive with some I/O layers (think of reading a gzipped file) */
++        /* Restrict to 64 bit processes, so as to avoid reallocs() */
++        /* on 32 bit processes where virtual memory is scarce.  */
++        while( already_read < size )
++        {
++            void* new_dest;
++            tmsize_t bytes_read;
++            tmsize_t to_read = size - already_read;
++#if SIZEOF_VOIDP == 8 || SIZEOF_SIZE_T == 8
++            if( to_read >= threshold && threshold < MAX_THRESHOLD )
++            {
++                to_read = threshold;
++                threshold *= THRESHOLD_MULTIPLIER;
++            }
++#endif
++
++            new_dest = (uint8*) _TIFFrealloc(
++                            *pdest, already_read + to_read);
++            if( new_dest == NULL )
++            {
++                TIFFErrorExt(tif->tif_clientdata, tif->tif_name,
++                            "Failed to allocate memory for %s "
++                            "(%ld elements of %ld bytes each)",
++                            "TIFFReadDirEntryArray",
++                             (long) 1, (long) already_read + to_read);
++                return TIFFReadDirEntryErrAlloc;
++            }
++            *pdest = new_dest;
++
++            bytes_read = TIFFReadFile(tif,
++                (char*)*pdest + already_read, to_read);
++            already_read += bytes_read;
++            if (bytes_read != to_read) {
++                return TIFFReadDirEntryErrIo;
++            }
++        }
++        return TIFFReadDirEntryErrOk;
++}
++
+ static enum TIFFReadDirEntryErr TIFFReadDirEntryArray(TIFF* tif, TIFFDirEntry* direntry, uint32* count, uint32 desttypesize, void** value)
+ {
+ 	int typesize;
+@@ -791,9 +851,22 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryArrayWithLimit(
+ 	*count=(uint32)direntry->tdir_count;
+ 	datasize=(*count)*typesize;
+ 	assert((tmsize_t)datasize>0);
+-	data=_TIFFCheckMalloc(tif, *count, typesize, "ReadDirEntryArray");
+-	if (data==0)
+-		return(TIFFReadDirEntryErrAlloc);
++
++	if( isMapped(tif) && datasize > tif->tif_size )
++		return TIFFReadDirEntryErrIo;
++
++	if( !isMapped(tif) &&
++		(((tif->tif_flags&TIFF_BIGTIFF) && datasize > 8) ||
++		(!(tif->tif_flags&TIFF_BIGTIFF) && datasize > 4)) )
++	{
++		data = NULL;
++	}
++	else
++	{
++		data=_TIFFCheckMalloc(tif, *count, typesize, "ReadDirEntryArray");
++		if (data==0)
++			return(TIFFReadDirEntryErrAlloc);
++	}
+ 	if (!(tif->tif_flags&TIFF_BIGTIFF))
+ 	{
+ 		if (datasize<=4)
+@@ -804,7 +877,10 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryArrayWithLimit(
+ 			uint32 offset = direntry->tdir_offset.toff_long;
+ 			if (tif->tif_flags&TIFF_SWAB)
+ 				TIFFSwabLong(&offset);
+-			err=TIFFReadDirEntryData(tif,(uint64)offset,(tmsize_t)datasize,data);
++			if( isMapped(tif) )
++				err=TIFFReadDirEntryData(tif,(uint64)offset,(tmsize_t)datasize,data);
++			else
++				err=TIFFReadDirEntryDataAndRealloc(tif,(uint64)offset,(tmsize_t)datasize,&data);
+ 			if (err!=TIFFReadDirEntryErrOk)
+ 			{
+ 				_TIFFfree(data);
+@@ -822,7 +898,10 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryArrayWithLimit(
+ 			uint64 offset = direntry->tdir_offset.toff_long8;
+ 			if (tif->tif_flags&TIFF_SWAB)
+ 				TIFFSwabLong8(&offset);
+-			err=TIFFReadDirEntryData(tif,offset,(tmsize_t)datasize,data);
++			if( isMapped(tif) )
++				err=TIFFReadDirEntryData(tif,(uint64)offset,(tmsize_t)datasize,data);
++			else
++				err=TIFFReadDirEntryDataAndRealloc(tif,(uint64)offset,(tmsize_t)datasize,&data);
+ 			if (err!=TIFFReadDirEntryErrOk)
+ 			{
+ 				_TIFFfree(data);
+From f91ca83a21a6a583050e5a5755ce1441b2bf1d7e Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Wed, 23 Aug 2017 13:21:41 +0000
+Subject: [PATCH] * libtiff/tif_dirwrite.c: replace assertion related to not
+ finding the SubIFD tag by runtime check. Fixes
+ http://bugzilla.maptools.org/show_bug.cgi?id=2727 Reported by team OWL337
+
+---
+ ChangeLog              | 7 +++++++
+ libtiff/tif_dirwrite.c | 7 ++++++-
+ 2 files changed, 13 insertions(+), 1 deletion(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index 3da2b704..87554768 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,3 +1,10 @@
++2017-08-23  Even Rouault <even.rouault at spatialys.com>
++
++	* libtiff/tif_dirwrite.c: replace assertion related to not finding the
++	SubIFD tag by runtime check.
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2727
++	Reported by team OWL337
++
+ 2017-07-15  Even Rouault <even.rouault at spatialys.com>
+ 
+ 	* libtiff/tif_read.c: add protection against excessive memory
+diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
+index 38edb3fb..a85f0627 100644
+--- a/libtiff/tif_dirwrite.c
++++ b/libtiff/tif_dirwrite.c
+@@ -821,7 +821,12 @@ TIFFWriteDirectorySec(TIFF* tif, int isimage, int imagedone, uint64* pdiroff)
+ 			TIFFDirEntry* nb;
+ 			for (na=0, nb=dir; ; na++, nb++)
+ 			{
+-				assert(na<ndir);
++				if( na == ndir )
++                                {
++                                    TIFFErrorExt(tif->tif_clientdata,module,
++                                                 "Cannot find SubIFD tag");
++                                    goto bad;
++                                }
+ 				if (nb->tdir_tag==TIFFTAG_SUBIFD)
+ 					break;
+ 			}
+From b6af137bf9ef852f1a48a50a5afb88f9e9da01cc Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Wed, 23 Aug 2017 13:33:42 +0000
+Subject: [PATCH] * libtiff/tif_dirwrite.c: replace assertion to tag value not
+ fitting on uint32 when selecting the value of SubIFD tag by runtime check (in
+ TIFFWriteDirectoryTagSubifd()). Fixes
+ http://bugzilla.maptools.org/show_bug.cgi?id=2728 Reported by team OWL337
+
+SubIFD tag by runtime check (in TIFFWriteDirectorySec())
+---
+ ChangeLog              | 10 +++++++++-
+ libtiff/tif_dirwrite.c |  9 ++++++++-
+ 2 files changed, 17 insertions(+), 2 deletions(-)
+
+diff --git a/ChangeLog b/ChangeLog
+index 87554768..58d5e0cc 100644
+--- a/ChangeLog
++++ b/ChangeLog
+@@ -1,7 +1,15 @@
++2017-08-23  Even Rouault <even.rouault at spatialys.com>
++
++	* libtiff/tif_dirwrite.c: replace assertion to tag value not fitting
++	on uint32 when selecting the value of SubIFD tag by runtime check
++	(in TIFFWriteDirectoryTagSubifd()).
++	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2728
++	Reported by team OWL337
++
+ 2017-08-23  Even Rouault <even.rouault at spatialys.com>
+ 
+ 	* libtiff/tif_dirwrite.c: replace assertion related to not finding the
+-	SubIFD tag by runtime check.
++	SubIFD tag by runtime check (in TIFFWriteDirectorySec())
+ 	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2727
+ 	Reported by team OWL337
+ 
+diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
+index a85f0627..cad0a498 100644
+--- a/libtiff/tif_dirwrite.c
++++ b/libtiff/tif_dirwrite.c
+@@ -1949,7 +1949,14 @@ TIFFWriteDirectoryTagSubifd(TIFF* tif, uint32* ndir, TIFFDirEntry* dir)
+ 		for (p=0; p < tif->tif_dir.td_nsubifd; p++)
+ 		{
+                         assert(pa != 0);
+-			assert(*pa <= 0xFFFFFFFFUL);
++
++                        /* Could happen if an classicTIFF has a SubIFD of type LONG8 (which is illegal) */
++                        if( *pa > 0xFFFFFFFFUL)
++                        {
++                            TIFFErrorExt(tif->tif_clientdata,module,"Illegal value for SubIFD tag");
++                            _TIFFfree(o);
++                            return(0);
++                        }
+ 			*pb++=(uint32)(*pa++);
+ 		}
+ 		n=TIFFWriteDirectoryTagCheckedIfdArray(tif,ndir,dir,TIFFTAG_SUBIFD,tif->tif_dir.td_nsubifd,o);
+From c6f41df7b581402dfba3c19a1e3df4454c551a01 Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Sun, 31 Dec 2017 15:09:41 +0100
+Subject: [PATCH] libtiff/tif_print.c: TIFFPrintDirectory(): fix null pointer dereference on corrupted file. Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2770
+
+---
+ libtiff/tif_print.c | 8 ++++----
+ 1 file changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/libtiff/tif_print.c b/libtiff/tif_print.c
+index 9959d35..8deceb2 100644
+--- a/libtiff/tif_print.c
++++ b/libtiff/tif_print.c
+@@ -667,13 +667,13 @@ TIFFPrintDirectory(TIFF* tif, FILE* fd, long flags)
+ #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+ 			fprintf(fd, "    %3lu: [%8I64u, %8I64u]\n",
+ 			    (unsigned long) s,
+-			    (unsigned __int64) td->td_stripoffset[s],
+-			    (unsigned __int64) td->td_stripbytecount[s]);
++			    td->td_stripoffset ? (unsigned __int64) td->td_stripoffset[s] : 0,
++			    td->td_stripbytecount ? (unsigned __int64) td->td_stripbytecount[s] : 0);
+ #else
+ 			fprintf(fd, "    %3lu: [%8llu, %8llu]\n",
+ 			    (unsigned long) s,
+-			    (unsigned long long) td->td_stripoffset[s],
+-			    (unsigned long long) td->td_stripbytecount[s]);
++			    td->td_stripoffset ? (unsigned long long) td->td_stripoffset[s] : 0,
++			    td->td_stripbytecount ? (unsigned long long) td->td_stripbytecount[s] : 0);
+ #endif
+ 	}
+ }
+--
+libgit2 0.26.0
+
+diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c
+index 2ccaf44..cbf2b69 100644
+--- a/libtiff/tif_dir.c
++++ b/libtiff/tif_dir.c
+@@ -1065,6 +1065,9 @@ _TIFFVGetField(TIFF* tif, uint32 tag, va_list ap)
+ 			if (td->td_samplesperpixel - td->td_extrasamples > 1) {
+ 				*va_arg(ap, uint16**) = td->td_transferfunction[1];
+ 				*va_arg(ap, uint16**) = td->td_transferfunction[2];
++			} else {
++				*va_arg(ap, uint16**) = NULL;
++				*va_arg(ap, uint16**) = NULL;
+ 			}
+ 			break;
+ 		case TIFFTAG_REFERENCEBLACKWHITE:
+diff --git a/tools/tiff2pdf.c b/tools/tiff2pdf.c
+index d1a9b09..484776c 100644
+--- a/tools/tiff2pdf.c
++++ b/tools/tiff2pdf.c
+@@ -237,7 +237,7 @@ typedef struct {
+ 	float tiff_whitechromaticities[2];
+ 	float tiff_primarychromaticities[6];
+ 	float tiff_referenceblackwhite[2];
+-	float* tiff_transferfunction[3];
++	uint16* tiff_transferfunction[3];
+ 	int pdf_image_interpolate;	/* 0 (default) : do not interpolate,
+ 					   1 : interpolate */
+ 	uint16 tiff_transferfunctioncount;
+@@ -1047,6 +1047,8 @@ void t2p_read_tiff_init(T2P* t2p, TIFF* input){
+ 	uint16 pagen=0;
+ 	uint16 paged=0;
+ 	uint16 xuint16=0;
++	uint16 tiff_transferfunctioncount=0;
++	uint16* tiff_transferfunction[3];
+ 
+ 	directorycount=TIFFNumberOfDirectories(input);
+ 	t2p->tiff_pages = (T2P_PAGE*) _TIFFmalloc(TIFFSafeMultiply(tmsize_t,directorycount,sizeof(T2P_PAGE)));
+@@ -1147,26 +1149,48 @@ void t2p_read_tiff_init(T2P* t2p, TIFF* input){
+                 }
+ #endif
+ 		if (TIFFGetField(input, TIFFTAG_TRANSFERFUNCTION,
+-                                 &(t2p->tiff_transferfunction[0]),
+-                                 &(t2p->tiff_transferfunction[1]),
+-                                 &(t2p->tiff_transferfunction[2]))) {
+-			if((t2p->tiff_transferfunction[1] != (float*) NULL) &&
+-                           (t2p->tiff_transferfunction[2] != (float*) NULL) &&
+-                           (t2p->tiff_transferfunction[1] !=
+-                            t2p->tiff_transferfunction[0])) {
+-				t2p->tiff_transferfunctioncount = 3;
+-				t2p->tiff_pages[i].page_extra += 4;
+-				t2p->pdf_xrefcount += 4;
+-			} else {
+-				t2p->tiff_transferfunctioncount = 1;
+-				t2p->tiff_pages[i].page_extra += 2;
+-				t2p->pdf_xrefcount += 2;
+-			}
+-			if(t2p->pdf_minorversion < 2)
+-				t2p->pdf_minorversion = 2;
++                                 &(tiff_transferfunction[0]),
++                                 &(tiff_transferfunction[1]),
++                                 &(tiff_transferfunction[2]))) {
++
++                        if((tiff_transferfunction[1] != (uint16*) NULL) &&
++                           (tiff_transferfunction[2] != (uint16*) NULL)
++                          ) {
++                            tiff_transferfunctioncount=3;
++                        } else {
++                            tiff_transferfunctioncount=1;
++                        }
+                 } else {
+-			t2p->tiff_transferfunctioncount=0;
++			tiff_transferfunctioncount=0;
+ 		}
++
++                if (i > 0){
++                    if (tiff_transferfunctioncount != t2p->tiff_transferfunctioncount){
++                        TIFFError(
++                            TIFF2PDF_MODULE,
++                            "Different transfer function on page %d",
++                            i);
++                        t2p->t2p_error = T2P_ERR_ERROR;
++                        return;
++                    }
++                }
++
++                t2p->tiff_transferfunctioncount = tiff_transferfunctioncount;
++                t2p->tiff_transferfunction[0] = tiff_transferfunction[0];
++                t2p->tiff_transferfunction[1] = tiff_transferfunction[1];
++                t2p->tiff_transferfunction[2] = tiff_transferfunction[2];
++                if(tiff_transferfunctioncount == 3){
++                        t2p->tiff_pages[i].page_extra += 4;
++                        t2p->pdf_xrefcount += 4;
++                        if(t2p->pdf_minorversion < 2)
++                                t2p->pdf_minorversion = 2;
++                } else if (tiff_transferfunctioncount == 1){
++                        t2p->tiff_pages[i].page_extra += 2;
++                        t2p->pdf_xrefcount += 2;
++                        if(t2p->pdf_minorversion < 2)
++                                t2p->pdf_minorversion = 2;
++                }
++
+ 		if( TIFFGetField(
+ 			input, 
+ 			TIFFTAG_ICCPROFILE, 
+@@ -1827,10 +1851,9 @@ void t2p_read_tiff_data(T2P* t2p, TIFF* input){
+ 			 &(t2p->tiff_transferfunction[0]),
+ 			 &(t2p->tiff_transferfunction[1]),
+ 			 &(t2p->tiff_transferfunction[2]))) {
+-		if((t2p->tiff_transferfunction[1] != (float*) NULL) &&
+-                   (t2p->tiff_transferfunction[2] != (float*) NULL) &&
+-                   (t2p->tiff_transferfunction[1] !=
+-                    t2p->tiff_transferfunction[0])) {
++		if((t2p->tiff_transferfunction[1] != (uint16*) NULL) &&
++                   (t2p->tiff_transferfunction[2] != (uint16*) NULL)
++                  ) {
+ 			t2p->tiff_transferfunctioncount=3;
+ 		} else {
+ 			t2p->tiff_transferfunctioncount=1;
+From 3719385a3fac5cfb20b487619a5f08abbf967cf8 Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Sun, 11 Mar 2018 11:14:01 +0100
+Subject: [PATCH] ChopUpSingleUncompressedStrip: avoid memory exhaustion (CVE-2017-11613)
+
+In ChopUpSingleUncompressedStrip(), if the computed number of strips is big
+enough and we are in read only mode, validate that the file size is consistent
+with that number of strips to avoid useless attempts at allocating a lot of
+memory for the td_stripbytecount and td_stripoffset arrays.
+
+Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2724
+---
+ libtiff/tif_dirread.c | 11 +++++++++++
+ 1 file changed, 11 insertions(+)
+
+diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
+index 3fc0c8e..1a3259c 100644
+--- a/libtiff/tif_dirread.c
++++ b/libtiff/tif_dirread.c
+@@ -5653,6 +5653,17 @@ ChopUpSingleUncompressedStrip(TIFF* tif)
+         if( nstrips == 0 )
+             return;
+ 
++        /* If we are going to allocate a lot of memory, make sure that the */
++        /* file is as big as needed */
++        if( tif->tif_mode == O_RDONLY &&
++            nstrips > 1000000 &&
++            (tif->tif_dir.td_stripoffset[0] >= TIFFGetFileSize(tif) ||
++             tif->tif_dir.td_stripbytecount[0] >
++                    TIFFGetFileSize(tif) - tif->tif_dir.td_stripoffset[0]) )
++        {
++            return;
++        }
++
+ 	newcounts = (uint64*) _TIFFCheckMalloc(tif, nstrips, sizeof (uint64),
+ 				"for chopped \"StripByteCounts\" array");
+ 	newoffsets = (uint64*) _TIFFCheckMalloc(tif, nstrips, sizeof (uint64),
+--
+libgit2 0.27.0
+
+From 7a092f8af2568d61993a8cc2e7a35a998d7d37be Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Sat, 17 Mar 2018 09:36:29 +0100
+Subject: [PATCH] ChopUpSingleUncompressedStrip: avoid memory exhaustion (CVE-2017-11613)
+
+Rework fix done in 3719385a3fac5cfb20b487619a5f08abbf967cf8 to work in more
+cases like https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=6979.
+Credit to OSS Fuzz
+
+Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2724
+---
+ libtiff/tif_dirread.c | 5 ++---
+ 1 file changed, 2 insertions(+), 3 deletions(-)
+
+diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
+index 1a3259c..6baa7b3 100644
+--- a/libtiff/tif_dirread.c
++++ b/libtiff/tif_dirread.c
+@@ -5657,9 +5657,8 @@ ChopUpSingleUncompressedStrip(TIFF* tif)
+         /* file is as big as needed */
+         if( tif->tif_mode == O_RDONLY &&
+             nstrips > 1000000 &&
+-            (tif->tif_dir.td_stripoffset[0] >= TIFFGetFileSize(tif) ||
+-             tif->tif_dir.td_stripbytecount[0] >
+-                    TIFFGetFileSize(tif) - tif->tif_dir.td_stripoffset[0]) )
++            (offset >= TIFFGetFileSize(tif) ||
++             stripbytes > (TIFFGetFileSize(tif) - offset) / (nstrips - 1)) )
+         {
+             return;
+         }
+--
+libgit2 0.27.0
+
+From be4c85b16e8801a16eec25e80eb9f3dd6a96731b Mon Sep 17 00:00:00 2001
+From: Hugo Lefeuvre <hle@debian.org>
+Date: Sun, 8 Apr 2018 14:07:08 -0400
+Subject: [PATCH] Fix NULL pointer dereference in TIFFPrintDirectory
+
+The TIFFPrintDirectory function relies on the following assumptions,
+supposed to be guaranteed by the specification:
+
+(a) A Transfer Function field is only present if the TIFF file has
+    photometric type < 3.
+
+(b) If SamplesPerPixel > Color Channels, then the ExtraSamples field
+    has count SamplesPerPixel - (Color Channels) and contains
+    information about supplementary channels.
+
+While respect of (a) and (b) are essential for the well functioning of
+TIFFPrintDirectory, no checks are realized neither by the callee nor
+by TIFFPrintDirectory itself. Hence, following scenarios might happen
+and trigger the NULL pointer dereference:
+
+(1) TIFF File of photometric type 4 or more has illegal Transfer
+    Function field.
+
+(2) TIFF File has photometric type 3 or less and defines a
+    SamplesPerPixel field such that SamplesPerPixel > Color Channels
+    without defining all extra samples in the ExtraSamples fields.
+
+In this patch, we address both issues with respect of the following
+principles:
+
+(A) In the case of (1), the defined transfer table should be printed
+    safely even if it isn't 'legal'. This allows us to avoid expensive
+    checks in TIFFPrintDirectory. Also, it is quite possible that
+    an alternative photometric type would be developed (not part of the
+    standard) and would allow definition of Transfer Table. We want
+    libtiff to be able to handle this scenario out of the box.
+
+(B) In the case of (2), the transfer table should be printed at its
+    right size, that is if TIFF file has photometric type Palette
+    then the transfer table should have one row and not three, even
+    if two extra samples are declared.
+
+In order to fulfill (A) we simply add a new 'i < 3' end condition to
+the broken TIFFPrintDirectory loop. This makes sure that in any case
+where (b) would be respected but not (a), everything stays fine.
+
+(B) is fulfilled by the loop condition
+'i < td->td_samplesperpixel - td->td_extrasamples'. This is enough as
+long as (b) is respected.
+
+Naturally, we also make sure (b) is respected. This is done in the
+TIFFReadDirectory function by making sure any non-color channel is
+counted in ExtraSamples.
+
+This commit addresses CVE-2018-7456.
+---
+ libtiff/tif_dirread.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ libtiff/tif_print.c   |  2 +-
+ 2 files changed, 63 insertions(+), 1 deletion(-)
+
+diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
+index 6baa7b3..af5b84a 100644
+--- a/libtiff/tif_dirread.c
++++ b/libtiff/tif_dirread.c
+@@ -166,6 +166,7 @@ static int TIFFFetchStripThing(TIFF* tif, TIFFDirEntry* dir, uint32 nstrips, uin
+ static int TIFFFetchSubjectDistance(TIFF*, TIFFDirEntry*);
+ static void ChopUpSingleUncompressedStrip(TIFF*);
+ static uint64 TIFFReadUInt64(const uint8 *value);
++static int _TIFFGetMaxColorChannels(uint16 photometric);
+ 
+ static int _TIFFFillStrilesInternal( TIFF *tif, int loadStripByteCount );
+ 
+@@ -3484,6 +3485,35 @@ static void TIFFReadDirEntryOutputErr(TIFF* tif, enum TIFFReadDirEntryErr err, c
+ }
+ 
+ /*
++ * Return the maximum number of color channels specified for a given photometric
++ * type. 0 is returned if photometric type isn't supported or no default value
++ * is defined by the specification.
++ */
++static int _TIFFGetMaxColorChannels( uint16 photometric )
++{
++    switch (photometric) {
++	case PHOTOMETRIC_PALETTE:
++	case PHOTOMETRIC_MINISWHITE:
++	case PHOTOMETRIC_MINISBLACK:
++            return 1;
++	case PHOTOMETRIC_YCBCR:
++	case PHOTOMETRIC_RGB:
++	case PHOTOMETRIC_CIELAB:
++            return 3;
++	case PHOTOMETRIC_SEPARATED:
++	case PHOTOMETRIC_MASK:
++            return 4;
++	case PHOTOMETRIC_LOGL:
++	case PHOTOMETRIC_LOGLUV:
++	case PHOTOMETRIC_CFA:
++	case PHOTOMETRIC_ITULAB:
++	case PHOTOMETRIC_ICCLAB:
++	default:
++            return 0;
++    }
++}
++
++/*
+  * Read the next TIFF directory from a file and convert it to the internal
+  * format. We read directories sequentially.
+  */
+@@ -3499,6 +3529,7 @@ TIFFReadDirectory(TIFF* tif)
+ 	uint32 fii=FAILED_FII;
+         toff_t nextdiroff;
+     int bitspersample_read = FALSE;
++        int color_channels;
+ 
+ 	tif->tif_diroff=tif->tif_nextdiroff;
+ 	if (!TIFFCheckDirOffset(tif,tif->tif_nextdiroff))
+@@ -4003,6 +4034,37 @@ TIFFReadDirectory(TIFF* tif)
+ 			}
+ 		}
+ 	}
++
++	/*
++	 * Make sure all non-color channels are extrasamples.
++	 * If it's not the case, define them as such.
++	 */
++        color_channels = _TIFFGetMaxColorChannels(tif->tif_dir.td_photometric);
++        if (color_channels && tif->tif_dir.td_samplesperpixel - tif->tif_dir.td_extrasamples > color_channels) {
++                uint16 old_extrasamples;
++                uint16 *new_sampleinfo;
++
++                TIFFWarningExt(tif->tif_clientdata,module, "Sum of Photometric type-related "
++                    "color channels and ExtraSamples doesn't match SamplesPerPixel. "
++                    "Defining non-color channels as ExtraSamples.");
++
++                old_extrasamples = tif->tif_dir.td_extrasamples;
++                tif->tif_dir.td_extrasamples = (tif->tif_dir.td_samplesperpixel - color_channels);
++
++                // sampleinfo should contain information relative to these new extra samples
++                new_sampleinfo = (uint16*) _TIFFcalloc(tif->tif_dir.td_extrasamples, sizeof(uint16));
++                if (!new_sampleinfo) {
++                    TIFFErrorExt(tif->tif_clientdata, module, "Failed to allocate memory for "
++                                "temporary new sampleinfo array (%d 16 bit elements)",
++                                tif->tif_dir.td_extrasamples);
++                    goto bad;
++                }
++
++                memcpy(new_sampleinfo, tif->tif_dir.td_sampleinfo, old_extrasamples * sizeof(uint16));
++                _TIFFsetShortArray(&tif->tif_dir.td_sampleinfo, new_sampleinfo, tif->tif_dir.td_extrasamples);
++                _TIFFfree(new_sampleinfo);
++        }
++
+ 	/*
+ 	 * Verify Palette image has a Colormap.
+ 	 */
+diff --git a/libtiff/tif_print.c b/libtiff/tif_print.c
+index 8deceb2..1d86adb 100644
+--- a/libtiff/tif_print.c
++++ b/libtiff/tif_print.c
+@@ -546,7 +546,7 @@ TIFFPrintDirectory(TIFF* tif, FILE* fd, long flags)
+ 				uint16 i;
+ 				fprintf(fd, "    %2ld: %5u",
+ 				    l, td->td_transferfunction[0][l]);
+-				for (i = 1; i < td->td_samplesperpixel; i++)
++				for (i = 1; i < td->td_samplesperpixel - td->td_extrasamples && i < 3; i++)
+ 					fprintf(fd, " %5u",
+ 					    td->td_transferfunction[i][l]);
+ 				fputc('\n', fd);
+--
+libgit2 0.27.0
+
+From 9171da596c88e6a2dadcab4a3a89dddd6e1b4655 Mon Sep 17 00:00:00 2001
+From: Nathan Baker <elitebadger@gmail.com>
+Date: Thu, 25 Jan 2018 21:28:15 +0000
+Subject: [PATCH] Add workaround to pal2rgb buffer overflow.
+
+---
+ tools/pal2rgb.c | 17 +++++++++++++++--
+ 1 file changed, 15 insertions(+), 2 deletions(-)
+
+diff --git a/tools/pal2rgb.c b/tools/pal2rgb.c
+index 0423598..01fcf94 100644
+--- a/tools/pal2rgb.c
++++ b/tools/pal2rgb.c
+@@ -184,8 +184,21 @@ main(int argc, char* argv[])
+ 	{ unsigned char *ibuf, *obuf;
+ 	  register unsigned char* pp;
+ 	  register uint32 x;
+-	  ibuf = (unsigned char*)_TIFFmalloc(TIFFScanlineSize(in));
+-	  obuf = (unsigned char*)_TIFFmalloc(TIFFScanlineSize(out));
++	  tmsize_t tss_in = TIFFScanlineSize(in);
++	  tmsize_t tss_out = TIFFScanlineSize(out);
++	  if (tss_out / tss_in < 3) {
++		/*
++		 * BUG 2750: The following code does not know about chroma
++		 * subsampling of JPEG data. It assumes that the output buffer is 3x
++		 * the length of the input buffer due to exploding the palette into
++		 * RGB tuples. If this assumption is incorrect, it could lead to a
++		 * buffer overflow. Go ahead and fail now to prevent that.
++		 */
++		fprintf(stderr, "Could not determine correct image size for output. Exiting.\n");
++		return -1;
++      }
++	  ibuf = (unsigned char*)_TIFFmalloc(tss_in);
++	  obuf = (unsigned char*)_TIFFmalloc(tss_out);
+ 	  switch (config) {
+ 	  case PLANARCONFIG_CONTIG:
+ 		for (row = 0; row < imagelength; row++) {
+--
+libgit2 0.27.0
+
+From 681748ec2f5ce88da5f9fa6831e1653e46af8a66 Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Sun, 14 Oct 2018 16:38:29 +0200
+Subject: [PATCH] JBIG: fix potential out-of-bounds write in JBIGDecode()
+
+JBIGDecode doesn't check if the user provided buffer is large enough
+to store the JBIG decoded image, which can potentially cause out-of-bounds
+write in the buffer.
+This issue was reported and analyzed by Thomas Dullien.
+
+Also fixes a (harmless) potential use of uninitialized memory when
+tif->tif_rawsize > tif->tif_rawcc
+
+And in case libtiff is compiled with CHUNKY_STRIP_READ_SUPPORT, make sure
+that whole strip data is provided to JBIGDecode()
+
+Index: tiff-4.0.8/libtiff/tif_jbig.c
+===================================================================
+--- tiff-4.0.8.orig/libtiff/tif_jbig.c	2018-10-25 15:35:44.812533616 +0200
++++ tiff-4.0.8/libtiff/tif_jbig.c	2018-10-25 15:35:44.804533616 +0200
+@@ -53,17 +53,18 @@
+ 	struct jbg_dec_state decoder;
+ 	int decodeStatus = 0;
+ 	unsigned char* pImage = NULL;
+-	(void) size, (void) s;
++	unsigned long decodedSize;
++	(void) s;
+ 
+ 	if (isFillOrder(tif, tif->tif_dir.td_fillorder))
+ 	{
+-		TIFFReverseBits(tif->tif_rawdata, tif->tif_rawdatasize);
++		TIFFReverseBits(tif->tif_rawcp, tif->tif_rawcc);
+ 	}
+ 
+ 	jbg_dec_init(&decoder);
+ 
+ #if defined(HAVE_JBG_NEWLEN)
+-	jbg_newlen(tif->tif_rawdata, (size_t)tif->tif_rawdatasize);
++	jbg_newlen(tif->tif_rawcp, (size_t)tif->tif_rawcc);
+ 	/*
+ 	 * I do not check the return status of jbg_newlen because even if this
+ 	 * function fails it does not necessarily mean that decoding the image
+@@ -76,8 +77,8 @@
+ 	 */
+ #endif /* HAVE_JBG_NEWLEN */
+ 
+-	decodeStatus = jbg_dec_in(&decoder, (unsigned char*)tif->tif_rawdata,
+-				  (size_t)tif->tif_rawdatasize, NULL);
++	decodeStatus = jbg_dec_in(&decoder, (unsigned char*)tif->tif_rawcp,
++				  (size_t)tif->tif_rawcc, NULL);
+ 	if (JBG_EOK != decodeStatus)
+ 	{
+ 		/*
+@@ -98,9 +99,28 @@
+ 		return 0;
+ 	}
+ 
++	decodedSize = jbg_dec_getsize(&decoder);
++	if( (tmsize_t)decodedSize < size )
++	{
++	    TIFFWarningExt(tif->tif_clientdata, "JBIG",
++	                   "Only decoded %lu bytes, whereas %lu requested",
++	                   decodedSize, (unsigned long)size);
++	}
++	else if( (tmsize_t)decodedSize > size )
++	{
++	    TIFFErrorExt(tif->tif_clientdata, "JBIG",
++	                 "Decoded %lu bytes, whereas %lu were requested",
++	                 decodedSize, (unsigned long)size);
++	    jbg_dec_free(&decoder);
++	    return 0;
++	}
+ 	pImage = jbg_dec_getimage(&decoder, 0);
+-	_TIFFmemcpy(buffer, pImage, jbg_dec_getsize(&decoder));
++	_TIFFmemcpy(buffer, pImage, decodedSize);
+ 	jbg_dec_free(&decoder);
++
++        tif->tif_rawcp += tif->tif_rawcc;
++        tif->tif_rawcc = 0;
++
+ 	return 1;
+ }
+ 
+Index: tiff-4.0.8/libtiff/tif_read.c
+===================================================================
+--- tiff-4.0.8.orig/libtiff/tif_read.c	2018-10-25 15:30:38.184542808 +0200
++++ tiff-4.0.8/libtiff/tif_read.c	2018-10-25 15:36:32.076532199 +0200
+@@ -329,6 +329,12 @@
+             return 0;
+         whole_strip = tif->tif_dir.td_stripbytecount[strip] < 10
+                 || isMapped(tif);
++        if( td->td_compression == COMPRESSION_JBIG )
++        {
++            /* Ideally plugins should have a way to declare they don't support
++             * chunk strip */
++            whole_strip = 1;
++        }
+ #else
+         whole_strip = 1;
+ #endif
+From 473851d211cf8805a161820337ca74cc9615d6ef Mon Sep 17 00:00:00 2001
+From: Nathan Baker <nathanb@lenovo-chrome.com>
+Date: Tue, 6 Feb 2018 10:13:57 -0500
+Subject: [PATCH] Fix for bug 2772
+
+It is possible to craft a TIFF document where the IFD list is circular,
+leading to an infinite loop while traversing the chain. The libtiff
+directory reader has a failsafe that will break out of this loop after
+reading 65535 directory entries, but it will continue processing,
+consuming time and resources to process what is essentially a bogus TIFF
+document.
+
+This change fixes the above behavior by breaking out of processing when
+a TIFF document has >= 65535 directories and terminating with an error.
+
+---
+ contrib/addtiffo/tif_overview.c | 14 +++++++++++++-
+ tools/tiff2pdf.c                | 10 ++++++++++
+ tools/tiffcrop.c                | 13 +++++++++++--
+ 3 files changed, 34 insertions(+), 3 deletions(-)
+
+Index: tiff-4.0.8/contrib/addtiffo/tif_overview.c
+===================================================================
+--- tiff-4.0.8.orig/contrib/addtiffo/tif_overview.c	2018-10-28 12:17:34.733012009 +0100
++++ tiff-4.0.8/contrib/addtiffo/tif_overview.c	2018-10-28 12:17:34.721012009 +0100
+@@ -65,6 +65,8 @@
+ #  define MAX(a,b)      ((a>b) ? a : b)
+ #endif
+ 
++#define TIFF_DIR_MAX  65534
++
+ void TIFFBuildOverviews( TIFF *, int, int *, int, const char *,
+                          int (*)(double,void*), void * );
+ 
+@@ -91,6 +93,7 @@
+ {
+     toff_t	nBaseDirOffset;
+     toff_t	nOffset;
++    tdir_t	iNumDir;
+ 
+     (void) bUseSubIFDs;
+ 
+@@ -147,7 +150,16 @@
+         return 0;
+ 
+     TIFFWriteDirectory( hTIFF );
+-    TIFFSetDirectory( hTIFF, (tdir_t) (TIFFNumberOfDirectories(hTIFF)-1) );
++    iNumDir = TIFFNumberOfDirectories(hTIFF);
++    if( iNumDir > TIFF_DIR_MAX )
++    {
++        TIFFErrorExt( TIFFClientdata(hTIFF),
++                      "TIFF_WriteOverview",
++                      "File `%s' has too many directories.\n",
++                      TIFFFileName(hTIFF) );
++        exit(-1);
++    }
++    TIFFSetDirectory( hTIFF, (tdir_t) (iNumDir - 1) );
+ 
+     nOffset = TIFFCurrentDirOffset( hTIFF );
+ 
+Index: tiff-4.0.8/tools/tiff2pdf.c
+===================================================================
+--- tiff-4.0.8.orig/tools/tiff2pdf.c	2018-10-28 12:17:34.733012009 +0100
++++ tiff-4.0.8/tools/tiff2pdf.c	2018-10-28 12:19:07.665009223 +0100
+@@ -68,6 +68,8 @@
+ 
+ #define PS_UNIT_SIZE	72.0F
+ 
++#define TIFF_DIR_MAX    65534
++
+ /* This type is of PDF color spaces. */
+ typedef enum {
+ 	T2P_CS_BILEVEL = 0x01,	/* Bilevel, black and white */
+@@ -1051,6 +1053,14 @@
+ 	uint16* tiff_transferfunction[3];
+ 
+ 	directorycount=TIFFNumberOfDirectories(input);
++        if(directorycount > TIFF_DIR_MAX) {
++                TIFFError(
++                        TIFF2PDF_MODULE,
++                        "TIFF contains too many directories, %s",
++                        TIFFFileName(input));
++                t2p->t2p_error = T2P_ERR_ERROR;
++                return;
++        }
+ 	t2p->tiff_pages = (T2P_PAGE*) _TIFFmalloc(TIFFSafeMultiply(tmsize_t,directorycount,sizeof(T2P_PAGE)));
+ 	if(t2p->tiff_pages==NULL){
+ 		TIFFError(
+Index: tiff-4.0.8/tools/tiffcrop.c
+===================================================================
+--- tiff-4.0.8.orig/tools/tiffcrop.c	2018-10-28 12:17:34.733012009 +0100
++++ tiff-4.0.8/tools/tiffcrop.c	2018-10-28 12:17:34.725012009 +0100
+@@ -217,6 +217,8 @@
+ #define DUMP_TEXT   1
+ #define DUMP_RAW    2
+ 
++#define TIFF_DIR_MAX  65534
++
+ /* Offsets into buffer for margins and fixed width and length segments */
+ struct offset {
+   uint32  tmargin;
+@@ -2233,7 +2235,7 @@
+     pageNum = -1;
+   else
+     total_images = 0;
+-  /* read multiple input files and write to output file(s) */
++  /* Read multiple input files and write to output file(s) */
+   while (optind < argc - 1)
+     {
+     in = TIFFOpen (argv[optind], "r");
+@@ -2241,7 +2243,14 @@
+       return (-3);
+ 
+     /* If only one input file is specified, we can use directory count */
+-    total_images = TIFFNumberOfDirectories(in); 
++    total_images = TIFFNumberOfDirectories(in);
++    if (total_images > TIFF_DIR_MAX)
++      {
++      TIFFError (TIFFFileName(in), "File contains too many directories");
++      if (out != NULL)
++        (void) TIFFClose(out);
++      return (1);
++      }
+     if (image_count == 0)
+       {
+       dirnum = 0;
+From 58a898cb4459055bb488ca815c23b880c242a27d Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Sat, 12 May 2018 15:32:31 +0200
+Subject: [PATCH] LZWDecodeCompat(): fix potential index-out-of-bounds write.  Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2780 / CVE-2018-8905
+
+The fix consists in using the similar code LZWDecode() to validate we
+don't write outside of the output buffer.
+
+ libtiff/tif_lzw.c | 17 ++++++++++++-----
+ 1 file changed, 12 insertions(+), 5 deletions(-)
+
+Index: tiff-4.0.8/libtiff/tif_lzw.c
+===================================================================
+--- tiff-4.0.8.orig/libtiff/tif_lzw.c	2018-10-28 12:59:52.864935922 +0100
++++ tiff-4.0.8/libtiff/tif_lzw.c	2018-10-28 13:01:50.136932407 +0100
+@@ -603,6 +603,7 @@
+ 	char *tp;
+ 	unsigned char *bp;
+ 	int code, nbits;
++	int len;
+ 	long nextbits, nextdata, nbitsmask;
+ 	code_t *codep, *free_entp, *maxcodep, *oldcodep;
+ 
+@@ -751,13 +752,18 @@
+ 				}  while (--occ);
+ 				break;
+ 			}
+-			assert(occ >= codep->length);
+-			op += codep->length;
+-			occ -= codep->length;
+-			tp = op;
++                        len = codep->length;
++                        tp = op + len;
+ 			do {
+-				*--tp = codep->value;
+-			} while( (codep = codep->next) != NULL );
++                                int t;
++                                --tp;
++                                t = codep->value;
++                                codep = codep->next;
++                                *tp = (char)t;
++                        } while (codep && tp > op);
++                        assert(occ >= len);
++                        op += len;
++                        occ -= len;
+ 		} else {
+ 			*op++ = (char)code;
+ 			occ--;
+From f1b94e8a3ba49febdd3361c0214a1d1149251577 Mon Sep 17 00:00:00 2001
+From: Young_X <YangX92@hotmail.com>
+Date: Sat, 8 Sep 2018 14:36:12 +0800
+Subject: [PATCH] only read/write TIFFTAG_GROUP3OPTIONS or
+ TIFFTAG_GROUP4OPTIONS if compression is COMPRESSION_CCITTFAX3 or
+ COMPRESSION_CCITTFAX4
+
+---
+ tools/pal2rgb.c | 18 +++++++++++++++++-
+ tools/tiff2bw.c | 18 +++++++++++++++++-
+ 2 files changed, 34 insertions(+), 2 deletions(-)
+
+diff --git a/tools/pal2rgb.c b/tools/pal2rgb.c
+index 01fcf941..01d8502e 100644
+--- a/tools/pal2rgb.c
++++ b/tools/pal2rgb.c
+@@ -402,7 +402,23 @@ cpTags(TIFF* in, TIFF* out)
+ {
+     struct cpTag *p;
+     for (p = tags; p < &tags[NTAGS]; p++)
+-	cpTag(in, out, p->tag, p->count, p->type);
++    {
++        if( p->tag == TIFFTAG_GROUP3OPTIONS )
++        {
++            uint16 compression;
++            if( !TIFFGetField(in, TIFFTAG_COMPRESSION, &compression) ||
++                    compression != COMPRESSION_CCITTFAX3 )
++                continue;
++        }
++        if( p->tag == TIFFTAG_GROUP4OPTIONS )
++        {
++            uint16 compression;
++            if( !TIFFGetField(in, TIFFTAG_COMPRESSION, &compression) ||
++                    compression != COMPRESSION_CCITTFAX4 )
++                continue;
++        }
++        cpTag(in, out, p->tag, p->count, p->type);
++    }
+ }
+ #undef NTAGS
+ 
+diff --git a/tools/tiff2bw.c b/tools/tiff2bw.c
+index 05faba87..5bef3142 100644
+--- a/tools/tiff2bw.c
++++ b/tools/tiff2bw.c
+@@ -450,7 +450,23 @@ cpTags(TIFF* in, TIFF* out)
+ {
+     struct cpTag *p;
+     for (p = tags; p < &tags[NTAGS]; p++)
+-	cpTag(in, out, p->tag, p->count, p->type);
++    {
++        if( p->tag == TIFFTAG_GROUP3OPTIONS )
++        {
++            uint16 compression;
++            if( !TIFFGetField(in, TIFFTAG_COMPRESSION, &compression) ||
++                    compression != COMPRESSION_CCITTFAX3 )
++                continue;
++        }
++        if( p->tag == TIFFTAG_GROUP4OPTIONS )
++        {
++            uint16 compression;
++            if( !TIFFGetField(in, TIFFTAG_COMPRESSION, &compression) ||
++                    compression != COMPRESSION_CCITTFAX4 )
++                continue;
++        }
++        cpTag(in, out, p->tag, p->count, p->type);
++    }
+ }
+ #undef NTAGS
+ 
+-- 
+2.18.1
+
+From de144fd228e4be8aa484c3caf3d814b6fa88c6d9 Mon Sep 17 00:00:00 2001
+From: Even Rouault <even.rouault@spatialys.com>
+Date: Sat, 12 May 2018 14:24:15 +0200
+Subject: [PATCH] TIFFWriteDirectorySec: avoid assertion. Fixes
+ http://bugzilla.maptools.org/show_bug.cgi?id=2795. CVE-2018-10963
+
+---
+ libtiff/tif_dirwrite.c | 7 +++++--
+ 1 file changed, 5 insertions(+), 2 deletions(-)
+
+diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
+index 2430de6d..c15a28db 100644
+--- a/libtiff/tif_dirwrite.c
++++ b/libtiff/tif_dirwrite.c
+@@ -695,8 +695,11 @@ TIFFWriteDirectorySec(TIFF* tif, int isimage, int imagedone, uint64* pdiroff)
+ 								}
+ 								break;
+ 							default:
+-								assert(0);   /* we should never get here */
+-								break;
++								TIFFErrorExt(tif->tif_clientdata,module,
++								            "Cannot write tag %d (%s)",
++								            TIFFFieldTag(o),
++                                                                            o->field_name ? o->field_name : "unknown");
++								goto bad;
+ 						}
+ 					}
+ 				}
+-- 
+2.18.1
+
+diff --git a/tools/tiffcp.c b/tools/tiffcp.c
+index 2f406e2d7065d31e2fccee0c08871cdab1d07fb7..8c81aa4f2d3adb332d00b317c887842f9c9f2b52 100644
+--- a/tools/tiffcp.c
++++ b/tools/tiffcp.c
+@@ -43,6 +43,7 @@
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
++#include <limits.h>
+ 
+ #include <ctype.h>
+ 
+@@ -1391,7 +1392,7 @@ DECLAREreadFunc(readSeparateTilesIntoBuffer)
+ 	int status = 1;
+ 	uint32 imagew = TIFFRasterScanlineSize(in);
+ 	uint32 tilew = TIFFTileRowSize(in);
+-	int iskew  = imagew - tilew*spp;
++	int iskew;
+ 	tsize_t tilesize = TIFFTileSize(in);
+ 	tdata_t tilebuf;
+ 	uint8* bufp = (uint8*) buf;
+@@ -1399,6 +1400,12 @@ DECLAREreadFunc(readSeparateTilesIntoBuffer)
+ 	uint32 row;
+ 	uint16 bps = 0, bytes_per_sample;
+ 
++	if (spp > (INT_MAX / tilew))
++	{
++		TIFFError(TIFFFileName(in), "Error, cannot handle that much samples per tile row (Tile Width * Samples/Pixel)");
++		return 0;
++	}
++	iskew = imagew - tilew*spp;
+ 	tilebuf = _TIFFmalloc(tilesize);
+ 	if (tilebuf == 0)
+ 		return 0;
+diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
+index c15a28dbd8fcb99b81fa5a1d44fcbcda881f42a7..ef30c869d30e210d90be16ce91f44087925fbad3 100644
+--- a/libtiff/tif_dirwrite.c
++++ b/libtiff/tif_dirwrite.c
+@@ -1895,12 +1895,14 @@ TIFFWriteDirectoryTagTransferfunction(TIFF* tif, uint32* ndir, TIFFDirEntry* dir
+ 		n=3;
+ 	if (n==3)
+ 	{
+-		if (!_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[2],m*sizeof(uint16)))
++		if (tif->tif_dir.td_transferfunction[2] == NULL ||
++		    !_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[2],m*sizeof(uint16)))
+ 			n=2;
+ 	}
+ 	if (n==2)
+ 	{
+-		if (!_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[1],m*sizeof(uint16)))
++		if (tif->tif_dir.td_transferfunction[1] == NULL ||
++		    !_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[1],m*sizeof(uint16)))
+ 			n=1;
+ 	}
+ 	if (n==0)
+diff --git a/tools/ppm2tiff.c b/tools/ppm2tiff.c
+index af6e41243bb4edbc229af473222964b2dac2e75c..c2d59257b9e7847719281e3ec8559a0359767b12 100644
+--- a/tools/ppm2tiff.c
++++ b/tools/ppm2tiff.c
+@@ -72,15 +72,16 @@ BadPPM(char* file)
+ 	exit(-2);
+ }
+ 
++
++#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
++#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
++
+ static tmsize_t
+ multiply_ms(tmsize_t m1, tmsize_t m2)
+ {
+-	tmsize_t bytes = m1 * m2;
+-
+-	if (m1 && bytes / m1 != m2)
+-		bytes = 0;
+-
+-	return bytes;
++        if( m1 == 0 || m2 > TIFF_TMSIZE_T_MAX / m1 )
++            return 0;
++        return m1 * m2;
+ }
+ 
+ int
+diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c
+index 6f0b48798bdeee91729c10e1fbcf9786234be5f3..078fbcec20677f19f7f967a4834011fe60df1df3 100644
+--- a/libtiff/tif_dir.c
++++ b/libtiff/tif_dir.c
+@@ -287,6 +287,18 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
+                 _TIFFfree(td->td_smaxsamplevalue);
+                 td->td_smaxsamplevalue = NULL;
+             }
++            /* Test if 3 transfer functions instead of just one are now needed
++               See http://bugzilla.maptools.org/show_bug.cgi?id=2820 */
++            if( td->td_transferfunction[0] != NULL && (v - td->td_extrasamples > 1) &&
++                !(td->td_samplesperpixel - td->td_extrasamples > 1))
++            {
++                    TIFFWarningExt(tif->tif_clientdata,module,
++                        "SamplesPerPixel tag value is changing, "
++                        "but TransferFunction was read with a different value. Cancelling it");
++                    TIFFClrFieldBit(tif,FIELD_TRANSFERFUNCTION);
++                    _TIFFfree(td->td_transferfunction[0]);
++                    td->td_transferfunction[0] = NULL;
++            }
+         }
+ 		td->td_samplesperpixel = (uint16) v;
+ 		break;
+@@ -363,6 +375,16 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
+ 		_TIFFsetShortArray(&td->td_colormap[2], va_arg(ap, uint16*), v32);
+ 		break;
+ 	case TIFFTAG_EXTRASAMPLES:
++            if ( td->td_transferfunction[0] != NULL && (td->td_samplesperpixel - v > 1) &&
++                 !(td->td_samplesperpixel - td->td_extrasamples > 1))
++            {
++                    TIFFWarningExt(tif->tif_clientdata,module,
++                        "ExtraSamples tag value is changing, "
++                        "but TransferFunction was read with a different value. Cancelling it");
++                    TIFFClrFieldBit(tif,FIELD_TRANSFERFUNCTION);
++                    _TIFFfree(td->td_transferfunction[0]);
++                    td->td_transferfunction[0] = NULL;
++            }
+ 		if (!setExtraSamples(td, ap, &v))
+ 			goto badvalue;
+ 		break;
+diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c
+index 078fbcec20677f19f7f967a4834011fe60df1df3..028ea54a256b4123ac320138aaedd1b356c2132f 100644
+--- a/libtiff/tif_dir.c
++++ b/libtiff/tif_dir.c
+@@ -90,13 +90,15 @@ setDoubleArrayOneValue(double** vpp, double value, size_t nmemb)
+  * Install extra samples information.
+  */
+ static int
+-setExtraSamples(TIFFDirectory* td, va_list ap, uint32* v)
++setExtraSamples(TIFF* tif, va_list ap, uint32* v)
+ {
+ /* XXX: Unassociated alpha data == 999 is a known Corel Draw bug, see below */
+ #define EXTRASAMPLE_COREL_UNASSALPHA 999 
+ 
+ 	uint16* va;
+ 	uint32 i;
++        TIFFDirectory* td = &tif->tif_dir;
++        static const char module[] = "setExtraSamples";
+ 
+ 	*v = (uint16) va_arg(ap, uint16_vap);
+ 	if ((uint16) *v > td->td_samplesperpixel)
+@@ -118,6 +120,18 @@ setExtraSamples(TIFFDirectory* td, va_list ap, uint32* v)
+ 				return 0;
+ 		}
+ 	}
++
++        if ( td->td_transferfunction[0] != NULL && (td->td_samplesperpixel - *v > 1) &&
++                !(td->td_samplesperpixel - td->td_extrasamples > 1))
++        {
++                TIFFWarningExt(tif->tif_clientdata,module,
++                    "ExtraSamples tag value is changing, "
++                    "but TransferFunction was read with a different value. Cancelling it");
++                TIFFClrFieldBit(tif,FIELD_TRANSFERFUNCTION);
++                _TIFFfree(td->td_transferfunction[0]);
++                td->td_transferfunction[0] = NULL;
++        }
++
+ 	td->td_extrasamples = (uint16) *v;
+ 	_TIFFsetShortArray(&td->td_sampleinfo, va, td->td_extrasamples);
+ 	return 1;
+@@ -375,17 +389,7 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
+ 		_TIFFsetShortArray(&td->td_colormap[2], va_arg(ap, uint16*), v32);
+ 		break;
+ 	case TIFFTAG_EXTRASAMPLES:
+-            if ( td->td_transferfunction[0] != NULL && (td->td_samplesperpixel - v > 1) &&
+-                 !(td->td_samplesperpixel - td->td_extrasamples > 1))
+-            {
+-                    TIFFWarningExt(tif->tif_clientdata,module,
+-                        "ExtraSamples tag value is changing, "
+-                        "but TransferFunction was read with a different value. Cancelling it");
+-                    TIFFClrFieldBit(tif,FIELD_TRANSFERFUNCTION);
+-                    _TIFFfree(td->td_transferfunction[0]);
+-                    td->td_transferfunction[0] = NULL;
+-            }
+-		if (!setExtraSamples(td, ap, &v))
++		if (!setExtraSamples(tif, ap, &v))
+ 			goto badvalue;
+ 		break;
+ 	case TIFFTAG_MATTEING:
+diff --git a/libtiff/tif_aux.c b/libtiff/tif_aux.c
+index 90d30214c6f50d60b5a896d644942cb04ff6e8ea..3e9bda43d5f8b20274ad306499bb85c5a06c73e9 100644
+--- a/libtiff/tif_aux.c
++++ b/libtiff/tif_aux.c
+@@ -59,18 +59,57 @@ _TIFFMultiply64(TIFF* tif, uint64 first, uint64 second, const char* where)
+ 	return bytes;
+ }
+ 
++tmsize_t
++_TIFFMultiplySSize(TIFF* tif, tmsize_t first, tmsize_t second, const char* where)
++{
++    if( first <= 0 || second <= 0 )
++    {
++        if( tif != NULL && where != NULL )
++        {
++            TIFFErrorExt(tif->tif_clientdata, where,
++                        "Invalid argument to _TIFFMultiplySSize() in %s", where);
++        }
++        return 0;
++    }
++
++    if( first > TIFF_TMSIZE_T_MAX / second )
++    {
++        if( tif != NULL && where != NULL )
++        {
++            TIFFErrorExt(tif->tif_clientdata, where,
++                        "Integer overflow in %s", where);
++        }
++        return 0;
++    }
++    return first * second;
++}
++
++tmsize_t _TIFFCastUInt64ToSSize(TIFF* tif, uint64 val, const char* module)
++{
++    if( val > (uint64)TIFF_TMSIZE_T_MAX )
++    {
++        if( tif != NULL && module != NULL )
++        {
++            TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
++        }
++        return 0;
++    }
++    return (tmsize_t)val;
++}
++
+ void*
+ _TIFFCheckRealloc(TIFF* tif, void* buffer,
+ 		  tmsize_t nmemb, tmsize_t elem_size, const char* what)
+ {
+ 	void* cp = NULL;
+-	tmsize_t bytes = nmemb * elem_size;
+-
++        tmsize_t count = _TIFFMultiplySSize(tif, nmemb, elem_size, NULL);
+ 	/*
+-	 * XXX: Check for integer overflow.
++	 * Check for integer overflow.
+ 	 */
+-	if (nmemb && elem_size && bytes / elem_size == nmemb)
+-		cp = _TIFFrealloc(buffer, bytes);
++	if (count != 0)
++	{
++		cp = _TIFFrealloc(buffer, count);
++	}
+ 
+ 	if (cp == NULL) {
+ 		TIFFErrorExt(tif->tif_clientdata, tif->tif_name,
+diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
+index 599bf12798044f0ff49405de30300fe68f1d17e3..c88b5fa67dc640e1849f9b33bfbc887d0a66d15d 100644
+--- a/libtiff/tif_getimage.c
++++ b/libtiff/tif_getimage.c
+@@ -755,9 +755,8 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ 	uint32 leftmost_tw;
+ 
+ 	tilesize = TIFFTileSize(tif);  
+-	bufsize = TIFFSafeMultiply(tmsize_t,alpha?4:3,tilesize);
++	bufsize = _TIFFMultiplySSize(tif, alpha?4:3,tilesize, "gtTileSeparate");
+ 	if (bufsize == 0) {
+-		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtTileSeparate");
+ 		return (0);
+ 	}
+ 	buf = (unsigned char*) _TIFFmalloc(bufsize);
+@@ -1006,9 +1005,8 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+         uint16 colorchannels;
+ 
+ 	stripsize = TIFFStripSize(tif);  
+-	bufsize = TIFFSafeMultiply(tmsize_t,alpha?4:3,stripsize);
++	bufsize = _TIFFMultiplySSize(tif,alpha?4:3,stripsize, "gtStripSeparate");
+ 	if (bufsize == 0) {
+-		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
+ 		return (0);
+ 	}
+ 
+diff --git a/libtiff/tif_luv.c b/libtiff/tif_luv.c
+index 6a63eadcab341f1c70cd1d34c27713da8ca0dd76..6fe485884ceb73e48985bc02d620ae7ac3a7eed0 100644
+--- a/libtiff/tif_luv.c
++++ b/libtiff/tif_luv.c
+@@ -1264,16 +1264,10 @@ LogL16GuessDataFmt(TIFFDirectory *td)
+ 	return (SGILOGDATAFMT_UNKNOWN);
+ }
+ 
+-
+-#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
+-#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
+-
+ static tmsize_t
+ multiply_ms(tmsize_t m1, tmsize_t m2)
+ {
+-        if( m1 == 0 || m2 > TIFF_TMSIZE_T_MAX / m1 )
+-            return 0;
+-        return m1 * m2;
++        return _TIFFMultiplySSize(NULL, m1, m2, NULL);
+ }
+ 
+ static int
+diff --git a/libtiff/tif_pixarlog.c b/libtiff/tif_pixarlog.c
+index b1e48d99c9246191253f86083b7046a316ec5171..a2149c852bc425c08677081d20d6eff81755cb95 100644
+--- a/libtiff/tif_pixarlog.c
++++ b/libtiff/tif_pixarlog.c
+@@ -636,15 +636,10 @@ PixarLogGuessDataFmt(TIFFDirectory *td)
+ 	return guess;
+ }
+ 
+-#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
+-#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
+-
+ static tmsize_t
+ multiply_ms(tmsize_t m1, tmsize_t m2)
+ {
+-        if( m1 == 0 || m2 > TIFF_TMSIZE_T_MAX / m1 )
+-            return 0;
+-        return m1 * m2;
++        return _TIFFMultiplySSize(NULL, m1, m2, NULL);
+ }
+ 
+ static tmsize_t
+diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
+index 1f5362a7a4269821ada45eb31ec597d65508c592..c436b0d5ecc2f580d542d5d0bd58cf513b7896c7 100644
+--- a/libtiff/tif_read.c
++++ b/libtiff/tif_read.c
+@@ -31,9 +31,6 @@
+ #include "tiffiop.h"
+ #include <stdio.h>
+ 
+-#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
+-#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
+-
+ int TIFFFillStrip(TIFF* tif, uint32 strip);
+ int TIFFFillTile(TIFF* tif, uint32 tile);
+ static int TIFFStartStrip(TIFF* tif, uint32 strip);
+@@ -51,6 +48,8 @@ TIFFReadRawTile1(TIFF* tif, uint32 tile, void* buf, tmsize_t size, const char* m
+ #define THRESHOLD_MULTIPLIER 10
+ #define MAX_THRESHOLD (THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * INITIAL_THRESHOLD)
+ 
++#define TIFF_INT64_MAX ((((int64)0x7FFFFFFF) << 32) | 0xFFFFFFFF)
++
+ /* Read 'size' bytes in tif_rawdata buffer starting at offset 'rawdata_offset'
+  * Returns 1 in case of success, 0 otherwise. */
+ static int TIFFReadAndRealloc( TIFF* tif, tmsize_t size,
+@@ -716,23 +716,8 @@ TIFFReadRawStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+ 		return ((tmsize_t)(-1));
+ 	}
+ 	bytecount = td->td_stripbytecount[strip];
+-	if ((int64)bytecount <= 0) {
+-#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+-		TIFFErrorExt(tif->tif_clientdata, module,
+-			     "%I64u: Invalid strip byte count, strip %lu",
+-			     (unsigned __int64) bytecount,
+-			     (unsigned long) strip);
+-#else
+-		TIFFErrorExt(tif->tif_clientdata, module,
+-			     "%llu: Invalid strip byte count, strip %lu",
+-			     (unsigned long long) bytecount,
+-			     (unsigned long) strip);
+-#endif
+-		return ((tmsize_t)(-1));
+-	}
+-	bytecountm = (tmsize_t)bytecount;
+-	if ((uint64)bytecountm!=bytecount) {
+-		TIFFErrorExt(tif->tif_clientdata, module, "Integer overflow");
++        bytecountm = _TIFFCastUInt64ToSSize(tif, bytecount, module);
++	if (bytecountm == 0) {
+ 		return ((tmsize_t)(-1));
+ 	}
+ 	if (size != (tmsize_t)(-1) && size < bytecountm)
+@@ -756,7 +756,7 @@ TIFFFillStrip(TIFF* tif, uint32 strip)
+ 	if ((tif->tif_flags&TIFF_NOREADRAW)==0)
+ 	{
+ 		uint64 bytecount = td->td_stripbytecount[strip];
+-		if ((int64)bytecount <= 0) {
++		if( bytecount == 0 || bytecount > (uint64)TIFF_INT64_MAX ) {
+ #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+ 			TIFFErrorExt(tif->tif_clientdata, module,
+ 				"Invalid strip byte count %I64u, strip %lu",
+@@ -783,7 +783,7 @@ TIFFFillStrip(TIFF* tif, uint32 strip)
+ 			    (bytecount - 4096) / 10 > (uint64)stripsize  )
+ 			{
+ 				uint64 newbytecount = (uint64)stripsize * 10 + 4096;
+-				if( (int64)newbytecount >= 0 )
++				if( newbytecount == 0 || newbytecount > (uint64)TIFF_INT64_MAX )
+ 				{
+ #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+ 					TIFFWarningExt(tif->tif_clientdata, module,
+@@ -1104,10 +1104,8 @@ TIFFReadRawTile(TIFF* tif, uint32 tile,
+ 	bytecount64 = td->td_stripbytecount[tile];
+ 	if (size != (tmsize_t)(-1) && (uint64)size < bytecount64)
+ 		bytecount64 = (uint64)size;
+-	bytecountm = (tmsize_t)bytecount64;
+-	if ((uint64)bytecountm!=bytecount64)
+-	{
+-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
++	bytecountm = _TIFFCastUInt64ToSSize(tif, bytecount64, module);
++        if( bytecountm == 0 ) {
+ 		return ((tmsize_t)(-1));
+ 	}
+ 	return (TIFFReadRawTile1(tif, tile, buf, bytecountm, module));
+@@ -1129,7 +1129,7 @@ TIFFFillTile(TIFF* tif, uint32 tile)
+ 	if ((tif->tif_flags&TIFF_NOREADRAW)==0)
+ 	{
+ 		uint64 bytecount = td->td_stripbytecount[tile];
+-		if ((int64)bytecount <= 0) {
++		if( bytecount == 0 || bytecount > (uint64)TIFF_INT64_MAX ) {
+ #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+ 			TIFFErrorExt(tif->tif_clientdata, module,
+ 				"%I64u: Invalid tile byte count, tile %lu",
+@@ -1143,6 +1143,39 @@ TIFFFillTile(TIFF* tif, uint32 tile)
+ #endif
+ 			return (0);
+ 		}
++
++		/* To avoid excessive memory allocations: */
++		/* Byte count should normally not be larger than a number of */
++		/* times the uncompressed size plus some margin */
++                if( bytecount > 1024 * 1024 )
++                {
++			/* 10 and 4096 are just values that could be adjusted. */
++			/* Hopefully they are safe enough for all codecs */
++			tmsize_t stripsize = TIFFTileSize(tif);
++			if( stripsize != 0 &&
++			    (bytecount - 4096) / 10 > (uint64)stripsize  )
++			{
++				uint64 newbytecount = (uint64)stripsize * 10 + 4096;
++				if( newbytecount == 0 || newbytecount > (uint64)TIFF_INT64_MAX )
++				{
++#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
++					TIFFWarningExt(tif->tif_clientdata, module,
++					  "Too large tile byte count %I64u, tile %lu. Limiting to %I64u",
++					     (unsigned __int64) bytecount,
++					     (unsigned long) tile,
++					     (unsigned __int64) newbytecount);
++#else
++					TIFFErrorExt(tif->tif_clientdata, module,
++					  "Too large tile byte count %llu, tile %lu. Limiting to %llu",
++					     (unsigned long long) bytecount,
++					     (unsigned long) tile,
++					     (unsigned long long) newbytecount);
++#endif
++					bytecount = newbytecount;
++				}
++			}
++		}
++
+ 		if (isMapped(tif) &&
+ 		    (isFillOrder(tif, td->td_fillorder)
+ 		     || (tif->tif_flags & TIFF_NOBITREV))) {
+diff --git a/libtiff/tif_strip.c b/libtiff/tif_strip.c
+index 2f3cd883b4106f7b43ee0dcec6c47a560cf3d135..c08c60a7928d1b27a4d66b385acd46faea992603 100644
+--- a/libtiff/tif_strip.c
++++ b/libtiff/tif_strip.c
+@@ -131,15 +131,8 @@ TIFFVStripSize(TIFF* tif, uint32 nrows)
+ {
+ 	static const char module[] = "TIFFVStripSize";
+ 	uint64 m;
+-	tmsize_t n;
+ 	m=TIFFVStripSize64(tif,nrows);
+-	n=(tmsize_t)m;
+-	if ((uint64)n!=m)
+-	{
+-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
+-		n=0;
+-	}
+-	return(n);
++        return _TIFFCastUInt64ToSSize(tif, m, module);
+ }
+ 
+ /*
+@@ -213,15 +206,8 @@ TIFFStripSize(TIFF* tif)
+ {
+ 	static const char module[] = "TIFFStripSize";
+ 	uint64 m;
+-	tmsize_t n;
+ 	m=TIFFStripSize64(tif);
+-	n=(tmsize_t)m;
+-	if ((uint64)n!=m)
+-	{
+-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
+-		n=0;
+-	}
+-	return(n);
++	return _TIFFCastUInt64ToSSize(tif, m, module);
+ }
+ 
+ /*
+@@ -332,14 +318,8 @@ TIFFScanlineSize(TIFF* tif)
+ {
+ 	static const char module[] = "TIFFScanlineSize";
+ 	uint64 m;
+-	tmsize_t n;
+ 	m=TIFFScanlineSize64(tif);
+-	n=(tmsize_t)m;
+-	if ((uint64)n!=m) {
+-		TIFFErrorExt(tif->tif_clientdata,module,"Integer arithmetic overflow");
+-		n=0;
+-	}
+-	return(n);
++	return _TIFFCastUInt64ToSSize(tif, m, module);
+ }
+ 
+ /*
+@@ -368,15 +348,8 @@ TIFFRasterScanlineSize(TIFF* tif)
+ {
+ 	static const char module[] = "TIFFRasterScanlineSize";
+ 	uint64 m;
+-	tmsize_t n;
+ 	m=TIFFRasterScanlineSize64(tif);
+-	n=(tmsize_t)m;
+-	if ((uint64)n!=m)
+-	{
+-		TIFFErrorExt(tif->tif_clientdata,module,"Integer arithmetic overflow");
+-		n=0;
+-	}
+-	return(n);
++	return _TIFFCastUInt64ToSSize(tif, m, module);
+ }
+ 
+ /* vim: set ts=8 sts=8 sw=8 noet: */
+diff --git a/libtiff/tif_tile.c b/libtiff/tif_tile.c
+index 58fe9354a387c7fa9b9d0333b54e4d4d163d4777..661cc771548bc15d5de572b4f04bb9d91c20bc40 100644
+--- a/libtiff/tif_tile.c
++++ b/libtiff/tif_tile.c
+@@ -183,15 +183,8 @@ TIFFTileRowSize(TIFF* tif)
+ {
+ 	static const char module[] = "TIFFTileRowSize";
+ 	uint64 m;
+-	tmsize_t n;
+ 	m=TIFFTileRowSize64(tif);
+-	n=(tmsize_t)m;
+-	if ((uint64)n!=m)
+-	{
+-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
+-		n=0;
+-	}
+-	return(n);
++	return _TIFFCastUInt64ToSSize(tif, m, module);
+ }
+ 
+ /*
+@@ -250,15 +243,8 @@ TIFFVTileSize(TIFF* tif, uint32 nrows)
+ {
+ 	static const char module[] = "TIFFVTileSize";
+ 	uint64 m;
+-	tmsize_t n;
+ 	m=TIFFVTileSize64(tif,nrows);
+-	n=(tmsize_t)m;
+-	if ((uint64)n!=m)
+-	{
+-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
+-		n=0;
+-	}
+-	return(n);
++	return _TIFFCastUInt64ToSSize(tif, m, module);
+ }
+ 
+ /*
+@@ -274,15 +260,8 @@ TIFFTileSize(TIFF* tif)
+ {
+ 	static const char module[] = "TIFFTileSize";
+ 	uint64 m;
+-	tmsize_t n;
+ 	m=TIFFTileSize64(tif);
+-	n=(tmsize_t)m;
+-	if ((uint64)n!=m)
+-	{
+-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
+-		n=0;
+-	}
+-	return(n);
++	return _TIFFCastUInt64ToSSize(tif, m, module);
+ }
+ 
+ /*
+diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
+index dd6fb095aff046bd7b9a68532dcf3e00ad5ac02d..60ccc87b319050b08dd94a74f9a1aaaef66cdb54 100644
+--- a/libtiff/tiffiop.h
++++ b/libtiff/tiffiop.h
+@@ -78,6 +78,9 @@ extern int snprintf(char* str, size_t size, const char* format, ...);
+ #define	FALSE	0
+ #endif
+ 
++#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
++#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
++
+ typedef struct client_info {
+     struct client_info *next;
+     void *data;
+@@ -260,7 +263,7 @@ struct tiff {
+ #define TIFFhowmany8_64(x) (((x)&0x07)?((uint64)(x)>>3)+1:(uint64)(x)>>3)
+ #define TIFFroundup_64(x, y) (TIFFhowmany_64(x,y)*(y))
+ 
+-/* Safe multiply which returns zero if there is an integer overflow */
++/* Safe multiply which returns zero if there is an *unsigned* integer overflow. This macro is not safe for *signed* integer types */
+ #define TIFFSafeMultiply(t,v,m) ((((t)(m) != (t)0) && (((t)(((v)*(m))/(m))) == (t)(v))) ? (t)((v)*(m)) : (t)0)
+ 
+ #define TIFFmax(A,B) ((A)>(B)?(A):(B))
+@@ -359,6 +362,8 @@ extern TIFFErrorHandlerExt _TIFFerrorHandlerExt;
+ 
+ extern uint32 _TIFFMultiply32(TIFF*, uint32, uint32, const char*);
+ extern uint64 _TIFFMultiply64(TIFF*, uint64, uint64, const char*);
++extern tmsize_t _TIFFMultiplySSize(TIFF*, tmsize_t, tmsize_t, const char*);
++extern tmsize_t _TIFFCastUInt64ToSSize(TIFF*, uint64, const char*);
+ extern void* _TIFFCheckMalloc(TIFF*, tmsize_t, tmsize_t, const char*);
+ extern void* _TIFFCheckRealloc(TIFF*, void*, tmsize_t, tmsize_t, const char*);
+ 
+diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
+index c88b5fa67dc640e1849f9b33bfbc887d0a66d15d..4da785d3a1fc237cc95ab93a2eb0969556457b13 100644
+--- a/libtiff/tif_getimage.c
++++ b/libtiff/tif_getimage.c
+@@ -936,16 +936,23 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ 	fromskew = (w < imagewidth ? imagewidth - w : 0);
+ 	for (row = 0; row < h; row += nrow)
+ 	{
++		uint32 temp;
+ 		rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
+ 		nrow = (row + rowstoread > h ? h - row : rowstoread);
+ 		nrowsub = nrow;
+ 		if ((nrowsub%subsamplingver)!=0)
+ 			nrowsub+=subsamplingver-nrowsub%subsamplingver;
++		temp = (row + img->row_offset)%rowsperstrip + nrowsub;
++		if( scanline > 0 && temp > (size_t)(TIFF_TMSIZE_T_MAX / scanline) )
++		{
++			TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in gtStripContig");
++			return 0;
++		}
+ 		if (_TIFFReadEncodedStripAndAllocBuffer(tif,
+ 		    TIFFComputeStrip(tif,row+img->row_offset, 0),
+ 		    (void**)(&buf),
+                     maxstripsize,
+-		    ((row + img->row_offset)%rowsperstrip + nrowsub) * scanline)==(tmsize_t)(-1)
++		    temp * scanline)==(tmsize_t)(-1)
+ 		    && (buf == NULL || img->stoponerr))
+ 		{
+ 			ret = 0;
+@@ -1038,15 +1045,22 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ 	fromskew = (w < imagewidth ? imagewidth - w : 0);
+ 	for (row = 0; row < h; row += nrow)
+ 	{
++                uint32 temp;
+ 		rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
+ 		nrow = (row + rowstoread > h ? h - row : rowstoread);
+ 		offset_row = row + img->row_offset;
++                temp = (row + img->row_offset)%rowsperstrip + nrow;
++                if( scanline > 0 && temp > (size_t)(TIFF_TMSIZE_T_MAX / scanline) )
++                {
++                        TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in gtStripSeparate");
++                        return 0;
++                }
+                 if( buf == NULL )
+                 {
+                     if (_TIFFReadEncodedStripAndAllocBuffer(
+                             tif, TIFFComputeStrip(tif, offset_row, 0),
+                             (void**) &buf, bufsize,
+-                            ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
++                            temp * scanline)==(tmsize_t)(-1)
+                         && (buf == NULL || img->stoponerr))
+                     {
+                             ret = 0;
+@@ -1066,7 +1080,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+                     }
+                 }
+ 		else if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
+-		    p0, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
++		    p0, temp * scanline)==(tmsize_t)(-1)
+ 		    && img->stoponerr)
+ 		{
+ 			ret = 0;
+@@ -1074,7 +1088,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ 		}
+ 		if (colorchannels > 1 
+                     && TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 1),
+-                                            p1, ((row + img->row_offset)%rowsperstrip + nrow) * scanline) == (tmsize_t)(-1)
++                                            p1, temp * scanline) == (tmsize_t)(-1)
+ 		    && img->stoponerr)
+ 		{
+ 			ret = 0;
+@@ -1082,7 +1096,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ 		}
+ 		if (colorchannels > 1 
+                     && TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 2),
+-                                            p2, ((row + img->row_offset)%rowsperstrip + nrow) * scanline) == (tmsize_t)(-1)
++                                            p2, temp * scanline) == (tmsize_t)(-1)
+ 		    && img->stoponerr)
+ 		{
+ 			ret = 0;
+@@ -1091,7 +1105,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
+ 		if (alpha)
+ 		{
+ 			if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, colorchannels),
+-			    pa, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
++			    pa, temp * scanline)==(tmsize_t)(-1)
+ 			    && img->stoponerr)
+ 			{
+ 				ret = 0;

component/libtiff/tiff_4.0.8-3.debian.patch

-diff -pur tiff-4.0.4/tools/tiffsplit.c tiff-4.0.4_patch/tools/tiffsplit.c
---- tiff-4.0.4/tools/tiffsplit.c	2015-05-28 15:10:26.000000000 +0200
-+++ tiff-4.0.4_patch/tools/tiffsplit.c	2016-02-12 19:15:30.532005041 +0100
-@@ -179,8 +179,9 @@ tiffcp(TIFF* in, TIFF* out)
- 		    TIFFSetField(out, TIFFTAG_JPEGTABLES, count, table);
- 		}
- 	}
-+	uint32 count = 0;
-         CopyField(TIFFTAG_PHOTOMETRIC, shortv);
--	CopyField(TIFFTAG_PREDICTOR, shortv);
-+	CopyField2(TIFFTAG_PREDICTOR, count, shortv);
- 	CopyField(TIFFTAG_THRESHHOLDING, shortv);
- 	CopyField(TIFFTAG_FILLORDER, shortv);
- 	CopyField(TIFFTAG_ORIENTATION, shortv);
-@@ -188,7 +189,7 @@ tiffcp(TIFF* in, TIFF* out)
- 	CopyField(TIFFTAG_MAXSAMPLEVALUE, shortv);
- 	CopyField(TIFFTAG_XRESOLUTION, floatv);
- 	CopyField(TIFFTAG_YRESOLUTION, floatv);
--	CopyField(TIFFTAG_GROUP3OPTIONS, longv);
-+	CopyField2(TIFFTAG_GROUP3OPTIONS, count, longv);
- 	CopyField(TIFFTAG_GROUP4OPTIONS, longv);
- 	CopyField(TIFFTAG_RESOLUTIONUNIT, shortv);
- 	CopyField(TIFFTAG_PLANARCONFIG, shortv);
-From 02669064e927074819ce1ed39aba0fccaa167717 Mon Sep 17 00:00:00 2001
-From: erouault <erouault>
-Date: Mon, 29 May 2017 10:12:54 +0000
-Subject: [PATCH] * libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter
- clamping to avoid int32 overflow in TIFFYCbCrtoRGB(). Fixes
- https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844 Credit to OSS Fuzz
-
----
- ChangeLog           | 7 +++++++
- libtiff/tif_color.c | 6 +++---
- 2 files changed, 10 insertions(+), 3 deletions(-)
-
-diff --git a/ChangeLog b/ChangeLog
-index ee8d9d08..61116596 100644
---- a/ChangeLog
-+++ b/ChangeLog
-@@ -1,3 +1,10 @@
-+2017-05-29  Even Rouault <even.rouault at spatialys.com>
-+
-+	* libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter clamping to avoid
-+	int32 overflow in TIFFYCbCrtoRGB().
-+	Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844
-+	Credit to OSS Fuzz
-+
- 2017-05-21  Bob Friesenhahn  <bfriesen@simple.dallas.tx.us>
- 
- 	* configure.ac: libtiff 4.0.8 released.
-diff --git a/libtiff/tif_color.c b/libtiff/tif_color.c
-index 055ed3b2..10a5e66e 100644
---- a/libtiff/tif_color.c
-+++ b/libtiff/tif_color.c
-@@ -275,10 +275,10 @@ TIFFYCbCrToRGBInit(TIFFYCbCrToRGB* ycbcr, float *luma, float *refBlackWhite)
-       for (i = 0, x = -128; i < 256; i++, x++) {
- 	    int32 Cr = (int32)CLAMPw(Code2V(x, refBlackWhite[4] - 128.0F,
- 			    refBlackWhite[5] - 128.0F, 127),
--                            -128.0F * 64, 128.0F * 64);
-+                            -128.0F * 32, 128.0F * 32);
- 	    int32 Cb = (int32)CLAMPw(Code2V(x, refBlackWhite[2] - 128.0F,
- 			    refBlackWhite[3] - 128.0F, 127),
--                            -128.0F * 64, 128.0F * 64);
-+                            -128.0F * 32, 128.0F * 32);
- 
- 	    ycbcr->Cr_r_tab[i] = (int32)((D1*Cr + ONE_HALF)>>SHIFT);
- 	    ycbcr->Cb_b_tab[i] = (int32)((D3*Cb + ONE_HALF)>>SHIFT);
-@@ -286,7 +286,7 @@ TIFFYCbCrToRGBInit(TIFFYCbCrToRGB* ycbcr, float *luma, float *refBlackWhite)
- 	    ycbcr->Cb_g_tab[i] = D4*Cb + ONE_HALF;
- 	    ycbcr->Y_tab[i] =
- 		    (int32)CLAMPw(Code2V(x + 128, refBlackWhite[0], refBlackWhite[1], 255),
--                                  -128.0F * 64, 128.0F * 64);
-+                                  -128.0F * 32, 128.0F * 32);
-       }
-     }
- 
-From 468988860e0dae62ebbf991627c74bcbb4bd256f Mon Sep 17 00:00:00 2001
-From: erouault <erouault>
-Date: Mon, 29 May 2017 11:29:06 +0000
-Subject: [PATCH] * libtiff/tif_getimage.c: initYCbCrConversion(): stricter
- validation for refBlackWhite coefficients values. To avoid invalid
- float->int32 conversion (when refBlackWhite[0] == 2147483648.f) Fixes
- https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1907 Credit to OSS Fuzz
-
----
- ChangeLog              | 8 ++++++++
- libtiff/tif_getimage.c | 2 +-
- 2 files changed, 9 insertions(+), 1 deletion(-)
-
-diff --git a/ChangeLog b/ChangeLog
-index a2ddaac2..04881ba7 100644
---- a/ChangeLog
-+++ b/ChangeLog
-@@ -1,5 +1,13 @@
- 2017-05-29  Even Rouault <even.rouault at spatialys.com>
- 
-+	* libtiff/tif_getimage.c: initYCbCrConversion(): stricter validation for
-+	refBlackWhite coefficients values. To avoid invalid float->int32 conversion
-+	(when refBlackWhite[0] == 2147483648.f)
-+	Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1907
-+	Credit to OSS Fuzz
-+
-+2017-05-29  Even Rouault <even.rouault at spatialys.com>
-+
- 	* libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter clamping to avoid
- 	int32 overflow in TIFFYCbCrtoRGB().
- 	Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844
-diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
-index dc373abc..a209a7a7 100644
---- a/libtiff/tif_getimage.c
-+++ b/libtiff/tif_getimage.c
-@@ -2241,7 +2241,7 @@ DECLARESepPutFunc(putseparate8bitYCbCr11tile)
- 
- static int isInRefBlackWhiteRange(float f)
- {
--    return f >= (float)(-0x7FFFFFFF + 128) && f <= (float)0x7FFFFFFF;
-+    return f > (float)(-0x7FFFFFFF + 128) && f < (float)0x7FFFFFFF;
- }
- 
- static int
-commit 40448d58fbfad52d2dde5bd18daa30b17fe35fcd
-Author: erouault <erouault>
-Date:   Thu Jun 1 12:44:04 2017 +0000
-
-    * libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
-    and use it in TIFFReadDirectory() so as to ignore fields whose tag is a
-    codec-specified tag but this codec is not enabled. This avoids TIFFGetField()
-    to behave differently depending on whether the codec is enabled or not, and
-    thus can avoid stack based buffer overflows in a number of TIFF utilities
-    such as tiffsplit, tiffcmp, thumbnail, etc.
-    Patch derived from 0063-Handle-properly-CODEC-specific-tags.patch
-    (http://bugzilla.maptools.org/show_bug.cgi?id=2580) by Raphaël Hertzog.
-    Fixes:
-    http://bugzilla.maptools.org/show_bug.cgi?id=2580
-    http://bugzilla.maptools.org/show_bug.cgi?id=2693
-    http://bugzilla.maptools.org/show_bug.cgi?id=2625 (CVE-2016-10095)
-    http://bugzilla.maptools.org/show_bug.cgi?id=2564 (CVE-2015-7554)
-    http://bugzilla.maptools.org/show_bug.cgi?id=2561 (CVE-2016-5318)
-    http://bugzilla.maptools.org/show_bug.cgi?id=2499 (CVE-2014-8128)
-    http://bugzilla.maptools.org/show_bug.cgi?id=2441
-    http://bugzilla.maptools.org/show_bug.cgi?id=2433
-
-diff --git a/ChangeLog b/ChangeLog
-index 04881ba7..ebd1a3c0 100644
---- a/ChangeLog
-+++ b/ChangeLog
-@@ -1,3 +1,23 @@
-+2017-06-01  Even Rouault <even.rouault at spatialys.com>
-+
-+	* libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
-+	and use it in TIFFReadDirectory() so as to ignore fields whose tag is a
-+	codec-specified tag but this codec is not enabled. This avoids TIFFGetField()
-+	to behave differently depending on whether the codec is enabled or not, and
-+	thus can avoid stack based buffer overflows in a number of TIFF utilities
-+	such as tiffsplit, tiffcmp, thumbnail, etc.
-+	Patch derived from 0063-Handle-properly-CODEC-specific-tags.patch
-+	(http://bugzilla.maptools.org/show_bug.cgi?id=2580) by Raphaël Hertzog.
-+	Fixes:
-+	http://bugzilla.maptools.org/show_bug.cgi?id=2580
-+	http://bugzilla.maptools.org/show_bug.cgi?id=2693
-+	http://bugzilla.maptools.org/show_bug.cgi?id=2625 (CVE-2016-10095)
-+	http://bugzilla.maptools.org/show_bug.cgi?id=2564 (CVE-2015-7554)
-+	http://bugzilla.maptools.org/show_bug.cgi?id=2561 (CVE-2016-5318)
-+	http://bugzilla.maptools.org/show_bug.cgi?id=2499 (CVE-2014-8128)
-+	http://bugzilla.maptools.org/show_bug.cgi?id=2441
-+	http://bugzilla.maptools.org/show_bug.cgi?id=2433
-+
- 2017-05-29  Even Rouault <even.rouault at spatialys.com>
- 
- 	* libtiff/tif_getimage.c: initYCbCrConversion(): stricter validation for
-diff --git a/libtiff/tif_dir.h b/libtiff/tif_dir.h
-index 6af5f3dc..5a380767 100644
---- a/libtiff/tif_dir.h
-+++ b/libtiff/tif_dir.h
-@@ -1,4 +1,4 @@
--/* $Id: tif_dir.h,v 1.54 2011-02-18 20:53:05 fwarmerdam Exp $ */
-+/* $Id: tif_dir.h,v 1.55 2017-06-01 12:44:04 erouault Exp $ */
- 
- /*
-  * Copyright (c) 1988-1997 Sam Leffler
-@@ -291,6 +291,7 @@ struct _TIFFField {
- extern int _TIFFMergeFields(TIFF*, const TIFFField[], uint32);
- extern const TIFFField* _TIFFFindOrRegisterField(TIFF *, uint32, TIFFDataType);
- extern  TIFFField* _TIFFCreateAnonField(TIFF *, uint32, TIFFDataType);
-+extern int _TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag);
- 
- #if defined(__cplusplus)
- }
-diff --git a/libtiff/tif_dirinfo.c b/libtiff/tif_dirinfo.c
-index 23ad0020..4904f540 100644
---- a/libtiff/tif_dirinfo.c
-+++ b/libtiff/tif_dirinfo.c
-@@ -1,4 +1,4 @@
--/* $Id: tif_dirinfo.c,v 1.126 2016-11-18 02:52:13 bfriesen Exp $ */
-+/* $Id: tif_dirinfo.c,v 1.127 2017-06-01 12:44:04 erouault Exp $ */
- 
- /*
-  * Copyright (c) 1988-1997 Sam Leffler
-@@ -956,6 +956,109 @@ TIFFMergeFieldInfo(TIFF* tif, const TIFFFieldInfo info[], uint32 n)
- 	return 0;
- }
- 
-+int
-+_TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag)
-+{
-+	/* Filter out non-codec specific tags */
-+	switch (tag) {
-+	    /* Shared tags */
-+	    case TIFFTAG_PREDICTOR:
-+	    /* JPEG tags */
-+	    case TIFFTAG_JPEGTABLES:
-+	    /* OJPEG tags */
-+	    case TIFFTAG_JPEGIFOFFSET:
-+	    case TIFFTAG_JPEGIFBYTECOUNT:
-+	    case TIFFTAG_JPEGQTABLES:
-+	    case TIFFTAG_JPEGDCTABLES:
-+	    case TIFFTAG_JPEGACTABLES:
-+	    case TIFFTAG_JPEGPROC:
-+	    case TIFFTAG_JPEGRESTARTINTERVAL:
-+	    /* CCITT* */
-+	    case TIFFTAG_BADFAXLINES:
-+	    case TIFFTAG_CLEANFAXDATA:
-+	    case TIFFTAG_CONSECUTIVEBADFAXLINES:
-+	    case TIFFTAG_GROUP3OPTIONS:
-+	    case TIFFTAG_GROUP4OPTIONS:
-+		break;
-+	    default:
-+		return 1;
-+	}
-+	/* Check if codec specific tags are allowed for the current
-+	 * compression scheme (codec) */
-+	switch (tif->tif_dir.td_compression) {
-+	    case COMPRESSION_LZW:
-+		if (tag == TIFFTAG_PREDICTOR)
-+		    return 1;
-+		break;
-+	    case COMPRESSION_PACKBITS:
-+		/* No codec-specific tags */
-+		break;
-+	    case COMPRESSION_THUNDERSCAN:
-+		/* No codec-specific tags */
-+		break;
-+	    case COMPRESSION_NEXT:
-+		/* No codec-specific tags */
-+		break;
-+	    case COMPRESSION_JPEG:
-+		if (tag == TIFFTAG_JPEGTABLES)
-+		    return 1;
-+		break;
-+	    case COMPRESSION_OJPEG:
-+		switch (tag) {
-+		    case TIFFTAG_JPEGIFOFFSET:
-+		    case TIFFTAG_JPEGIFBYTECOUNT:
-+		    case TIFFTAG_JPEGQTABLES:
-+		    case TIFFTAG_JPEGDCTABLES:
-+		    case TIFFTAG_JPEGACTABLES:
-+		    case TIFFTAG_JPEGPROC:
-+		    case TIFFTAG_JPEGRESTARTINTERVAL:
-+			return 1;
-+		}
-+		break;
-+	    case COMPRESSION_CCITTRLE:
-+	    case COMPRESSION_CCITTRLEW:
-+	    case COMPRESSION_CCITTFAX3:
-+	    case COMPRESSION_CCITTFAX4:
-+		switch (tag) {
-+		    case TIFFTAG_BADFAXLINES:
-+		    case TIFFTAG_CLEANFAXDATA:
-+		    case TIFFTAG_CONSECUTIVEBADFAXLINES:
-+			return 1;
-+		    case TIFFTAG_GROUP3OPTIONS:
-+			if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX3)
-+			    return 1;
-+			break;
-+		    case TIFFTAG_GROUP4OPTIONS:
-+			if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX4)
-+			    return 1;
-+			break;
-+		}
-+		break;
-+	    case COMPRESSION_JBIG:
-+		/* No codec-specific tags */
-+		break;
-+	    case COMPRESSION_DEFLATE:
-+	    case COMPRESSION_ADOBE_DEFLATE:
-+		if (tag == TIFFTAG_PREDICTOR)
-+		    return 1;
-+		break;
-+	   case COMPRESSION_PIXARLOG:
-+		if (tag == TIFFTAG_PREDICTOR)
-+		    return 1;
-+		break;
-+	    case COMPRESSION_SGILOG:
-+	    case COMPRESSION_SGILOG24:
-+		/* No codec-specific tags */
-+		break;
-+	    case COMPRESSION_LZMA:
-+		if (tag == TIFFTAG_PREDICTOR)
-+		    return 1;
-+		break;
-+
-+	}
-+	return 0;
-+}
-+
- /* vim: set ts=8 sts=8 sw=8 noet: */
- 
- /*
-diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
-index 772ebaf7..acde78b5 100644
---- a/libtiff/tif_dirread.c
-+++ b/libtiff/tif_dirread.c
-@@ -1,4 +1,4 @@
--/* $Id: tif_dirread.c,v 1.208 2017-04-27 15:46:22 erouault Exp $ */
-+/* $Id: tif_dirread.c,v 1.209 2017-06-01 12:44:04 erouault Exp $ */
- 
- /*
-  * Copyright (c) 1988-1997 Sam Leffler
-@@ -3580,6 +3580,10 @@ TIFFReadDirectory(TIFF* tif)
- 							goto bad;
- 						dp->tdir_tag=IGNORE;
- 						break;
-+                                        default:
-+                                            if( !_TIFFCheckFieldIsValidForCodec(tif, dp->tdir_tag) )
-+                                                dp->tdir_tag=IGNORE;
-+                                            break;
- 				}
- 			}
- 		}
-From fe8d7165956b88df4837034a9161dc5fd20cf67a Mon Sep 17 00:00:00 2001
-From: Even Rouault <even.rouault@spatialys.com>
-Date: Mon, 26 Jun 2017 15:19:59 +0000
-Subject: [PATCH] * libtiff/tif_jbig.c: fix memory leak in error code path of
- JBIGDecode() Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2706 Reported
- by team OWL337
-
-* libtiff/tif_jpeg.c: error out at decoding time if anticipated libjpeg
----
- ChangeLog          | 8 +++++++-
- libtiff/tif_jbig.c | 1 +
- 2 files changed, 8 insertions(+), 1 deletion(-)
-
-diff --git a/ChangeLog b/ChangeLog
-index bc5096e7..ecd70534 100644
---- a/ChangeLog
-+++ b/ChangeLog
-@@ -1,3 +1,9 @@
-+2017-06-26  Even Rouault <even.rouault at spatialys.com>
-+
-+	* libtiff/tif_jbig.c: fix memory leak in error code path of JBIGDecode()
-+	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2706
-+	Reported by team OWL337
-+
- 2017-06-01  Even Rouault <even.rouault at spatialys.com>
- 
- 	* libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
-diff --git a/libtiff/tif_jbig.c b/libtiff/tif_jbig.c
-index 5f5f75e2..c75f31d9 100644
---- a/libtiff/tif_jbig.c
-+++ b/libtiff/tif_jbig.c
-@@ -94,6 +94,7 @@ static int JBIGDecode(TIFF* tif, uint8* buffer, tmsize_t size, uint16 s)
- 			     jbg_strerror(decodeStatus)
- #endif
- 			     );
-+		jbg_dec_free(&decoder);
- 		return 0;
- 	}
- 
-From 1077fad562e03d1cad591dd10163dd80ad63ab0e Mon Sep 17 00:00:00 2001
-From: Even Rouault <even.rouault@spatialys.com>
-Date: Fri, 30 Jun 2017 13:11:18 +0000
-Subject: [PATCH] * libtiff/tif_read.c, tiffiop.h: add a
- _TIFFReadEncodedStripAndAllocBuffer() function, variant of
- TIFFReadEncodedStrip() that allocates the decoded buffer only after a first
- successful TIFFFillStrip(). This avoids excessive memory allocation on
- corrupted files. * libtiff/tif_getimage.c: use
- _TIFFReadEncodedStripAndAllocBuffer(). Fixes
- http://bugzilla.maptools.org/show_bug.cgi?id=2708 and
- https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2433 . Credit to OSS
- Fuzz
-
----
- ChangeLog              | 11 +++++++
- libtiff/tif_getimage.c | 59 ++++++++++++++++++++++----------------
- libtiff/tif_read.c     | 78 +++++++++++++++++++++++++++++++++++++++++++-------
- libtiff/tiffiop.h      |  5 ++++
- 4 files changed, 118 insertions(+), 35 deletions(-)
-
-diff --git a/ChangeLog b/ChangeLog
-index c969f9e2..6f085e09 100644
---- a/ChangeLog
-+++ b/ChangeLog
-@@ -1,3 +1,14 @@
-+2017-06-30  Even Rouault <even.rouault at spatialys.com>
-+
-+	* libtiff/tif_read.c, tiffiop.h: add a _TIFFReadEncodedStripAndAllocBuffer()
-+	function, variant of TIFFReadEncodedStrip() that allocates the
-+	decoded buffer only after a first successful TIFFFillStrip(). This avoids
-+	excessive memory allocation on corrupted files.
-+	* libtiff/tif_getimage.c: use _TIFFReadEncodedStripAndAllocBuffer(). 
-+	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2708 and
-+	https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2433 .
-+	Credit to OSS Fuzz
-+
- 2017-06-26  Even Rouault <even.rouault at spatialys.com>
- 
- 	* libtiff/tif_jbig.c: fix memory leak in error code path of JBIGDecode()
-diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
-index cee8e930..cc6e8f30 100644
---- a/libtiff/tif_getimage.c
-+++ b/libtiff/tif_getimage.c
-@@ -905,26 +905,22 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
- 	tileContigRoutine put = img->put.contig;
- 	uint32 row, y, nrow, nrowsub, rowstoread;
- 	tmsize_t pos;
--	unsigned char* buf;
-+	unsigned char* buf = NULL;
- 	uint32 rowsperstrip;
- 	uint16 subsamplinghor,subsamplingver;
- 	uint32 imagewidth = img->width;
- 	tmsize_t scanline;
- 	int32 fromskew, toskew;
- 	int ret = 1, flip;
-+        tmsize_t maxstripsize;
- 
- 	TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING, &subsamplinghor, &subsamplingver);
- 	if( subsamplingver == 0 ) {
- 		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Invalid vertical YCbCr subsampling");
- 		return (0);
- 	}
--
--	buf = (unsigned char*) _TIFFmalloc(TIFFStripSize(tif));
--	if (buf == 0) {
--		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
--		return (0);
--	}
--	_TIFFmemset(buf, 0, TIFFStripSize(tif));
-+	
-+	maxstripsize = TIFFStripSize(tif);
- 
- 	flip = setorientation(img);
- 	if (flip & FLIP_VERTICALLY) {
-@@ -946,11 +942,12 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
- 		nrowsub = nrow;
- 		if ((nrowsub%subsamplingver)!=0)
- 			nrowsub+=subsamplingver-nrowsub%subsamplingver;
--		if (TIFFReadEncodedStrip(tif,
-+		if (_TIFFReadEncodedStripAndAllocBuffer(tif,
- 		    TIFFComputeStrip(tif,row+img->row_offset, 0),
--		    buf,
-+		    (void**)(&buf),
-+                    maxstripsize,
- 		    ((row + img->row_offset)%rowsperstrip + nrowsub) * scanline)==(tmsize_t)(-1)
--		    && img->stoponerr)
-+		    && (buf == NULL || img->stoponerr))
- 		{
- 			ret = 0;
- 			break;
-@@ -994,8 +991,8 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
- {
- 	TIFF* tif = img->tif;
- 	tileSeparateRoutine put = img->put.separate;
--	unsigned char *buf;
--	unsigned char *p0, *p1, *p2, *pa;
-+	unsigned char *buf = NULL;
-+	unsigned char *p0 = NULL, *p1 = NULL, *p2 = NULL, *pa = NULL;
- 	uint32 row, y, nrow, rowstoread;
- 	tmsize_t pos;
- 	tmsize_t scanline;
-@@ -1014,15 +1011,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
- 		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
- 		return (0);
- 	}
--	p0 = buf = (unsigned char *)_TIFFmalloc(bufsize);
--	if (buf == 0) {
--		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for tile buffer");
--		return (0);
--	}
--	_TIFFmemset(buf, 0, bufsize);
--	p1 = p0 + stripsize;
--	p2 = p1 + stripsize;
--	pa = (alpha?(p2+stripsize):NULL);
- 
- 	flip = setorientation(img);
- 	if (flip & FLIP_VERTICALLY) {
-@@ -1040,7 +1028,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
-           case PHOTOMETRIC_MINISBLACK:
-           case PHOTOMETRIC_PALETTE:
-             colorchannels = 1;
--            p2 = p1 = p0;
-             break;
- 
-           default:
-@@ -1056,7 +1043,31 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
- 		rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
- 		nrow = (row + rowstoread > h ? h - row : rowstoread);
- 		offset_row = row + img->row_offset;
--		if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
-+                if( buf == NULL )
-+                {
-+                    if (_TIFFReadEncodedStripAndAllocBuffer(
-+                            tif, TIFFComputeStrip(tif, offset_row, 0),
-+                            (void**) &buf, bufsize,
-+                            ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
-+                        && (buf == NULL || img->stoponerr))
-+                    {
-+                            ret = 0;
-+                            break;
-+                    }
-+                    p0 = buf;
-+                    if( colorchannels == 1 )
-+                    {
-+                        p2 = p1 = p0;
-+                        pa = (alpha?(p0+3*stripsize):NULL);
-+                    }
-+                    else
-+                    {
-+                        p1 = p0 + stripsize;
-+                        p2 = p1 + stripsize;
-+                        pa = (alpha?(p2+stripsize):NULL);
-+                    }
-+                }
-+		else if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
- 		    p0, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
- 		    && img->stoponerr)
- 		{
-diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
-index fc0072e7..047305ab 100644
---- a/libtiff/tif_read.c
-+++ b/libtiff/tif_read.c
-@@ -442,18 +442,17 @@ TIFFReadScanline(TIFF* tif, void* buf, uint32 row, uint16 sample)
- }
- 
- /*
-- * Read a strip of data and decompress the specified
-- * amount into the user-supplied buffer.
-+ * Calculate the strip size according to the number of
-+ * rows in the strip (check for truncated last strip on any
-+ * of the separations).
-  */
--tmsize_t
--TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
-+static tmsize_t TIFFReadEncodedStripGetStripSize(TIFF* tif, uint32 strip, uint16* pplane)
- {
- 	static const char module[] = "TIFFReadEncodedStrip";
- 	TIFFDirectory *td = &tif->tif_dir;
- 	uint32 rowsperstrip;
- 	uint32 stripsperplane;
- 	uint32 stripinplane;
--	uint16 plane;
- 	uint32 rows;
- 	tmsize_t stripsize;
- 	if (!TIFFCheckRead(tif,0))
-@@ -465,23 +464,37 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
- 		    (unsigned long)td->td_nstrips);
- 		return((tmsize_t)(-1));
- 	}
--	/*
--	 * Calculate the strip size according to the number of
--	 * rows in the strip (check for truncated last strip on any
--	 * of the separations).
--	 */
-+
- 	rowsperstrip=td->td_rowsperstrip;
- 	if (rowsperstrip>td->td_imagelength)
- 		rowsperstrip=td->td_imagelength;
- 	stripsperplane= TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip);
- 	stripinplane=(strip%stripsperplane);
--	plane=(uint16)(strip/stripsperplane);
-+	if( pplane ) *pplane=(uint16)(strip/stripsperplane);
- 	rows=td->td_imagelength-stripinplane*rowsperstrip;
- 	if (rows>rowsperstrip)
- 		rows=rowsperstrip;
- 	stripsize=TIFFVStripSize(tif,rows);
- 	if (stripsize==0)
- 		return((tmsize_t)(-1));
-+	return stripsize;
-+}
-+
-+/*
-+ * Read a strip of data and decompress the specified
-+ * amount into the user-supplied buffer.
-+ */
-+tmsize_t
-+TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
-+{
-+	static const char module[] = "TIFFReadEncodedStrip";
-+	TIFFDirectory *td = &tif->tif_dir;
-+	tmsize_t stripsize;
-+	uint16 plane;
-+
-+	stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
-+	if (stripsize==((tmsize_t)(-1)))
-+		return((tmsize_t)(-1));
- 
-     /* shortcut to avoid an extra memcpy() */
-     if( td->td_compression == COMPRESSION_NONE &&
-@@ -510,6 +523,49 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
- 	return(stripsize);
- }
- 
-+/* Variant of TIFFReadEncodedStrip() that does 
-+ * * if *buf == NULL, *buf = _TIFFmalloc(bufsizetoalloc) only after TIFFFillStrip() has
-+ *   suceeded. This avoid excessive memory allocation in case of truncated
-+ *   file.
-+ * * calls regular TIFFReadEncodedStrip() if *buf != NULL
-+ */
-+tmsize_t
-+_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
-+                                    void **buf, tmsize_t bufsizetoalloc,
-+                                    tmsize_t size_to_read)
-+{
-+    tmsize_t this_stripsize;
-+    uint16 plane;
-+
-+    if( *buf != NULL )
-+    {
-+        return TIFFReadEncodedStrip(tif, strip, *buf, size_to_read);
-+    }
-+
-+    this_stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
-+    if (this_stripsize==((tmsize_t)(-1)))
-+            return((tmsize_t)(-1));
-+
-+    if ((size_to_read!=(tmsize_t)(-1))&&(size_to_read<this_stripsize))
-+            this_stripsize=size_to_read;
-+    if (!TIFFFillStrip(tif,strip))
-+            return((tmsize_t)(-1));
-+
-+    *buf = _TIFFmalloc(bufsizetoalloc);
-+    if (*buf == NULL) {
-+            TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
-+            return((tmsize_t)(-1));
-+    }
-+    _TIFFmemset(*buf, 0, bufsizetoalloc);
-+
-+    if ((*tif->tif_decodestrip)(tif,*buf,this_stripsize,plane)<=0)
-+            return((tmsize_t)(-1));
-+    (*tif->tif_postdecode)(tif,*buf,this_stripsize);
-+    return(this_stripsize);
-+
-+
-+}
-+
- static tmsize_t
- TIFFReadRawStrip1(TIFF* tif, uint32 strip, void* buf, tmsize_t size,
-     const char* module)
-diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
-index 846ade03..7f0b90f7 100644
---- a/libtiff/tiffiop.h
-+++ b/libtiff/tiffiop.h
-@@ -365,6 +365,11 @@ extern void* _TIFFCheckRealloc(TIFF*, void*, tmsize_t, tmsize_t, const char*);
- extern double _TIFFUInt64ToDouble(uint64);
- extern float _TIFFUInt64ToFloat(uint64);
- 
-+extern tmsize_t
-+_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
-+                                    void **buf, tmsize_t bufsizetoalloc,
-+                                    tmsize_t size_to_read);
-+
- extern int TIFFInitDumpMode(TIFF*, int);
- #ifdef PACKBITS_SUPPORT
- extern int TIFFInitPackBits(TIFF*, int);
-From 6173a57d39e04d68b139f8c1aa499a24dbe74ba1 Mon Sep 17 00:00:00 2001
-From: Even Rouault <even.rouault@spatialys.com>
-Date: Fri, 30 Jun 2017 17:29:44 +0000
-Subject: [PATCH] * libtiff/tif_dirwrite.c: in
- TIFFWriteDirectoryTagCheckedXXXX() functions associated with LONG8/SLONG8
- data type, replace assertion that the file is BigTIFF, by a non-fatal error.
- Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2712 Reported by team
- OWL337
-
----
- ChangeLog              |  8 ++++++++
- libtiff/tif_dirwrite.c | 20 ++++++++++++++++----
- 2 files changed, 24 insertions(+), 4 deletions(-)
-
-diff --git a/ChangeLog b/ChangeLog
-index 6f085e09..77a64385 100644
---- a/ChangeLog
-+++ b/ChangeLog
-@@ -1,5 +1,13 @@
- 2017-06-30  Even Rouault <even.rouault at spatialys.com>
- 
-+	* libtiff/tif_dirwrite.c: in TIFFWriteDirectoryTagCheckedXXXX()
-+	functions associated with LONG8/SLONG8 data type, replace assertion that
-+	the file is BigTIFF, by a non-fatal error.
-+	Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2712
-+	Reported by team OWL337
-+
-+2017-06-30  Even Rouault <even.rouault at spatialys.com>
-+
- 	* libtiff/tif_read.c, tiffiop.h: add a _TIFFReadEncodedStripAndAllocBuffer()
- 	function, variant of TIFFReadEncodedStrip() that allocates the
- 	decoded buffer only after a first successful TIFFFillStrip(). This avoids
-diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
-index 2967da58..8d6686ba 100644
---- a/libtiff/tif_dirwrite.c
-+++ b/libtiff/tif_dirwrite.c
-@@ -2111,7 +2111,10 @@ TIFFWriteDirectoryTagCheckedLong8(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, ui
- {
- 	uint64 m;
- 	assert(sizeof(uint64)==8);
--	assert(tif->tif_flags&TIFF_BIGTIFF);
-+	if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
-+		TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","LONG8 not allowed for ClassicTIFF");
-+		return(0);
-+	}
- 	m=value;
- 	if (tif->tif_flags&TIFF_SWAB)
- 		TIFFSwabLong8(&m);
-@@ -2124,7 +2127,10 @@ TIFFWriteDirectoryTagCheckedLong8Array(TIFF* tif, uint32* ndir, TIFFDirEntry* di
- {
- 	assert(count<0x20000000);
- 	assert(sizeof(uint64)==8);
--	assert(tif->tif_flags&TIFF_BIGTIFF);
-+	if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
-+		TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","LONG8 not allowed for ClassicTIFF");
-+		return(0);
-+	}
- 	if (tif->tif_flags&TIFF_SWAB)
- 		TIFFSwabArrayOfLong8(value,count);
- 	return(TIFFWriteDirectoryTagData(tif,ndir,dir,tag,TIFF_LONG8,count,count*8,value));
-@@ -2136,7 +2142,10 @@ TIFFWriteDirectoryTagCheckedSlong8(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, u
- {
- 	int64 m;
- 	assert(sizeof(int64)==8);
--	assert(tif->tif_flags&TIFF_BIGTIFF);
-+	if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
-+		TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","SLONG8 not allowed for ClassicTIFF");
-+		return(0);
-+	}
- 	m=value;
- 	if (tif->tif_flags&TIFF_SWAB)
- 		TIFFSwabLong8((uint64*)(&m));
-@@ -2149,7 +2158,10 @@ TIFFWriteDirectoryTagCheckedSlong8Array(TIFF* tif, uint32* ndir, TIFFDirEntry* d
- {
- 	assert(count<0x20000000);
- 	assert(sizeof(int64)==8);
--	assert(tif->tif_flags&TIFF_BIGTIFF);
-+	if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
-+		TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","SLONG8 not allowed for ClassicTIFF");
-+		return(0);
-+	}
- 	if (tif->tif_flags&TIFF_SWAB)
- 		TIFFSwabArrayOfLong8((uint64*)value,count);
- 	return(TIFFWriteDirectoryTagData(tif,ndir,dir,tag,TIFF_SLONG8,count,count*8,value));

component/proxysql/buildout.cfg

+[buildout]
+extends =
+  ../autoconf/buildout.cfg
+  ../automake/buildout.cfg
+  ../libtool/buildout.cfg
+  ../cmake/buildout.cfg
+  ../openssl/buildout.cfg
+  ../patch/buildout.cfg
+  ../git/buildout.cfg
+  ../openssl/buildout.cfg
+  ../bzip2/buildout.cfg
+  ../perl/buildout.cfg
+  ../gnutls/buildout.cfg
+  ../curl/buildout.cfg
+  ../gnutls/buildout.cfg
+  ../libzip/buildout.cfg
+  ../m4/buildout.cfg
+  ../pcre/buildout.cfg
+  ../jemalloc/buildout.cfg
+  ../libmicrohttpd/buildout.cfg
+
+parts =
+  proxysql
+
+[proxysql]
+recipe = slapos.recipe.cmmi
+version = v2.0.12
+url = https://github.com/sysown/proxysql/archive/${:version}.tar.gz
+md5sum = 70ec17fe73703a25730fdd44b6bc3ef5
+prefix = @@LOCATION@@
+# Patch installation path for SlapOS
+pre-configure =
+  mkdir -p ${:prefix}/bin ${:prefix}/etc/init.d ${:prefix}/lib/systemd/system
+  sed -ri "
+    s#(\s)/usr/bin#\1${:prefix}/bin#g
+    s#(\s)/etc#\1${:prefix}/etc#g
+    s#(\s)/usr/lib#\1${:prefix}/lib#
+    s#(\s)/var/lib#\1${:prefix}/lib#g
+    s#(\s)useradd#\1echo useradd#g
+    s#(\s)systemctl#\1echo systemctl#g
+    s#(\s)chkconfig#\1echo chkconfig#g
+    s#(\s)update-rc.d#\1echo update-rc.d#g" Makefile
+
+configure-command = true
+environment =
+  GIT_VERSION=${:version}
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${libgcrypt:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
+  PATH=${m4:location}/bin:${libtool:location}/bin:${libgcrypt:location}/bin:${curl:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${bzip2:location}/bin:${autoconf:location}/bin:${git:location}/bin:${automake:location}/bin:${patch:location}/bin:${cmake:location}/bin:%(PATH)s
+  CXXFLAGS=-I${openssl:location}/include -I${gnutls:location}/include -I${zlib:location}/include
+  CFLAGS=-I${gnutls:location}/include
+  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath=${curl:location}/lib -L${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${curl:location}/lib -L${pcre:location}/lib -L${jemalloc:location}/lib -L${libmicrohttpd:location}/lib
+  CMAKE_INCLUDE_PATH=${openssl:location}/include:${gnutls:location}/include:${curl:location}/include:${pcre:location}/include:${jemalloc:location}/include:${libmicrohttpd:location}/include
+  CMAKE_LIBRARY_PATH=${openssl:location}/lib:${gnutls:location}/lib:${curl:location}/lib:${pcre:location}/lib:${jemalloc:location}/lib:${libmicrohttpd:location}/lib
+  LIBTOOL=libtool
+  ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${libtool:location}/share/aclocal

component/python-manuel/buildout.cfg

+[buildout]
+extends = ../patch/buildout.cfg
+parts = manuel
+
+[manuel]
+recipe = zc.recipe.egg
+egg = manuel
+find-links = https://github.com/benji-york/manuel/tarball/${:revision}/manuel-${versions:manuel}.tar.gz
+revision = aeea5ce7e853b8dabb815efb83141c6ddeb62904
+
+[versions]
+manuel = 1.10.2.dev0

component/replication-manager/buildout.cfg

+[buildout]
+extends =
+  ../../component/golang/buildout.cfg
+
+parts =
+  gowork
+
+[gowork]
+golang  = ${golang1.13:location}
+install =
+buildflags =  -v --tags server --ldflags "-extldflags 'static' -w -s -X main.GoOS=linux -X main.GoArch=amd64 -X main.Version=2.1 -X main.FullVersion=$FULLVERSION -X main.Build=$(date +%FT%T%z) -X main.WithProvisioning=ON -X main.WithOpenSVC=OFF -X main.WithHaproxy=ON -X main.WithMaxscale=ON  -X main.WithMariadbshardproxy=ON -X  main.WithProxysql=ON -X  main.WithSphinx=ON -X main.WithArbitration=OFF -X main.WithArbitrationClient=ON -X main.WithMonitoring=ON -X main.WithHttp=ON -X main.WithBackup=ON -X main.WithMail=ON -X main.WithEnforce=ON -X main.WithDeprecate=ON"
+
+[gowork.goinstall]
+depends_gitfetch  =
+    ${git.signal18.io_signal18_repman:recipe}
+
+command = set -e
+  . ${gowork:env.sh}
+  cd ${git.signal18.io_signal18_repman:location}
+  export GO111MODULE=on
+  export FULLVERSION=$(git describe --tags)
+  go build ${gowork:buildflags} -o ${gowork:bin}/replication-manager
+  chmod -R u+w .
+# Remove binary files provided with replication manager else testnode will complain with 'libXXX => not found'.
+  rm -f ${git.signal18.io_signal18_repman:location}/share/amd64/darwin/*
+  rm -f ${git.signal18.io_signal18_repman:location}/share/amd64/linux/*
+
+[git.signal18.io_signal18_repman]
+<= go-git-package
+go.importpath = github.com/signal18/replication-manager
+repository =  https://github.com/signal18/replication-manager
+branch = 2.1
+revision = 9167a82c81af8f7be41cf51bc9be8a37dc3d8c03
\ No newline at end of file

component/restic/buildout.cfg

+[buildout]
+extends =
+  ../../component/golang/buildout.cfg
+
+parts =
+  restic
+
+[gowork]
+golang  = ${golang1.13:location}
+
+
+[restic]
+recipe  = plone.recipe.command
+update-command = ${:command}
+stop-on-error = True
+# GO111MODULE=on enables go modules support
+# the chmod is needed as modules are fetched with u-w
+command =
+  . ${gowork:env.sh} &&
+  cd ${git.github.com_restic_restic:location} &&
+  export GO111MODULE=on &&
+  go run build.go -o ${:output} &&
+  chmod -R u+w .
+output = ${gowork:bin}/restic
+location = ${:output}
+
+
+[git.github.com_restic_restic]
+<= go-git-package
+go.importpath = github.com/restic/restic
+repository =  https://github.com/restic/restic
+revision = v0.9.6
\ No newline at end of file

component/rina-tools/buildout.cfg

 [buildout]
 extends =
+  ../defaults.cfg
   ../automake/buildout.cfg
   ../git/buildout.cfg
   ../libtool/buildout.cfg
@@ -7,6 +8,9 @@ extends =
 
 parts = rina-tools
 
+[gcc]
+min_version = 0
+
 [irati-stack]
 recipe = slapos.recipe.build:gitclone
 repository = https://github.com/jmuchemb/irati-stack.git

component/socat/buildout.cfg

 [buildout]
+extends =
+  ../openssl/buildout.cfg
+
 parts =
   socat
 
@@ -7,3 +10,5 @@ recipe = slapos.recipe.cmmi
 url = http://www.dest-unreach.org/socat/download/socat-${:version}.tar.gz
 version = 1.7.3.2
 md5sum = aec3154f7854580cfab0c2d81e910519
+environment =
+  LDFLAGS=-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
\ No newline at end of file

component/sysbench/buildout.cfg

+[buildout]
+extends =
+  ../autoconf/buildout.cfg
+  ../automake/buildout.cfg
+  ../libtool/buildout.cfg
+  ../gettext/buildout.cfg
+  ../m4/buildout.cfg
+  ../mariadb/buildout.cfg
+
+parts =
+  sysbench
+
+[sysbench]
+recipe = slapos.recipe.cmmi
+shared = true
+url = https://github.com/akopytov/sysbench/archive/1.0.19.tar.gz
+md5sum = 2912bfe7238cac7351459019a84e2557
+pre-configure =
+  aclocal -I${pkgconfig:location}/share/aclocal -I${libtool:location}/share/aclocal -I${gettext:location}/share/aclocal
+  ./autogen.sh
+configure-options =
+  --disable-static
+  --with-mysql-includes=${mariadb:location}/include/mysql
+  --with-mysql-libs=${mariadb:location}/lib
+environment =
+  PATH=${m4:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${pkgconfig:location}/bin:${libtool:location}/bin:%(PATH)s
+  CPPFLAGS=-I${gettext:location}/include -I${mariadb:location}/include
+  LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${mariadb:location}/lib -Wl,-rpath=${mariadb:location}/lib
+  ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${gettext:location}/share/aclocal:${libtool:location}/share/aclocal
\ No newline at end of file

component/zstd/buildout.cfg

@@ -10,8 +10,8 @@ parts =
 recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/facebook/zstd/releases/download/v${:version}/zstd-${:version}.tar.gz
-version = 1.4.4
-md5sum = 487f7ee1562dee7c1c8adf85e2a63df9
+version = 1.4.5
+md5sum = dd0b53631303b8f972dafa6fd34beb0c
 location = @@LOCATION@@
 configure-command = :
 environment =

software/build-rina/software.cfg

@@ -8,9 +8,6 @@ parts =
   template
 download-cache = ${:directory}/download-cache
 
-[gcc]
-min_version = 0
-
 [template]
 recipe = slapos.recipe.template:jinja2
 # XXX: "template.cfg" is hardcoded in instanciation recipe

software/buildout-testing/runTestSuite.in

@@ -4,6 +4,7 @@ import argparse, os, re, subprocess, sys
 from time import gmtime, strftime, time
 from erp5.util import taskdistribution
 from erp5.util.testsuite import SubprocessError, TestSuite
+from pkg_resources import get_distribution
 from zc.buildout.buildout import Buildout
 
 if str is bytes:
@@ -40,17 +41,17 @@ class DummyTestResult:
 class BuildoutTestSuite(TestSuite):
 
   RUN_RE = re.compile(
-    r'Ran (?P<all_tests>\d+) tests with'
-    ' (?P<failures>\d+) failures,'
-    ' (?P<errors>\d+) errors and'
-    ' (?P<skips>\d+) skipped in')
+    br'Ran (?P<all_tests>\d+) tests with'
+    br' (?P<failures>\d+) failures,'
+    br' (?P<errors>\d+) errors and'
+    br' (?P<skips>\d+) skipped in')
 
   def run(self, test):
     start = time()
     try:
       status_dict = self.spawn(os.path.join('bin', 'zope-testrunner'),
         '--test-path', os.path.join(test_dict[test], 'src'))
-    except SubprocessError, e:
+    except SubprocessError as e:
       status_dict = e.status_dict
     end = time()
     status_dict.update(
@@ -93,35 +94,38 @@ def main():
   else:
     test_result = DummyTestResult(list(test_dict))
 
-  fd = os.open('buildout.cfg', os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0666)
+  fd = os.open('buildout.cfg', os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
   try:
     os.write(fd, str2bytes("""\
 [buildout]
-extends = %s
 develop =%s
-parts = test
+parts = testrunner
 newest = false
+versions = versions
 
 [versions]
 %s
+zope.exceptions = 4.3
+zope.interface = 4.7.2
+zope.testing = 4.7
 
-[bootstrap]
+[testrunner]
 recipe = zc.recipe.egg
-eggs = zc.buildout
-
-[test]
-recipe = zc.recipe.egg
-eggs +=
+eggs =
+  ${:recipe}
+  zope.testing
   zope.testrunner
 scripts =
   zope-testrunner
-""" % (os.path.join(slapos_buildout, 'buildout.cfg'),
-       ''.join('\n  ' + x for x in test_dict.itervalues()),
-       '\n'.join(x + ' =' for x in test_dict))))
+extra-paths =
+  %s
+""" % (''.join('\n  ' + x for x in test_dict.values()),
+       '\n'.join(x + ' =' for x in test_dict),
+       get_distribution('manuel').location,
+       )))
   finally:
     os.close(fd)
-  Buildout('buildout.cfg', {}).install(['bootstrap'])
-  subprocess.check_call((os.path.join('bin', 'buildout'),))
+  Buildout('buildout.cfg', {}).install(None)
 
   test_suite = BuildoutTestSuite(1)
   while 1:

software/buildout-testing/software.cfg

 [buildout]
 extends =
+  ../../component/python-manuel/buildout.cfg
   ../../stack/slapos.cfg
 parts =
   slapos-cookbook
@@ -14,6 +15,7 @@ git-executable = ${git:location}/bin/git
 recipe = zc.recipe.egg
 eggs = erp5.util
   zc.buildout
+  ${manuel:egg}
 scripts = ${:interpreter}
 interpreter = ${:_buildout_section_name_}
 

software/erp5/software-shared.cfg

+[buildout]
+extends = software.cfg
+shared-parts = /opt/slapgrid/shared-parts
+eggs-directory = /opt/slapgrid/shared-eggs
+abi-tag-eggs = true

software/neoppod/software-zodb5.cfg

@@ -6,7 +6,7 @@ eggs += mock
 ZEO-patches =
 
 [versions]
-ZODB = 5.5.1
+ZODB = 5.6.0
 ZEO = 5.2.0
 transaction = 2.4.0
 

software/repman/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[instance.cfg]
+filename = instance.cfg.in
+md5sum = b41f521b5f7980c64260ed0e5c494450
+
+[instance-repman.cfg]
+_update_hash_filename_ = instance-repman.cfg.jinja2.in
+md5sum = 7dbaace0d7db0e26d582ad17f36ac9cd
+
+[config-toml.in]
+_update_hash_filename_ = templates/config.toml.in
+md5sum = 5cfa75ca5a0048a050c0041dfe541f3d
+
+[config-cluster-toml.in]
+_update_hash_filename_ = templates/cluster-config.toml.in
+md5sum = d2e79a9435082d9420281b4f59a5d464
+
+[nginx.conf.in]
+_update_hash_filename_ = templates/nginx.conf.in
+md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
+
+[template-mariadb.cfg]
+_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
+md5sum = 189ccee60d0fb53e29431a45e0816bc1
+
+[template-my-cnf]
+_update_hash_filename_ = templates/my.cnf.in
+md5sum = f3661b788099bb31d71ba6e7d36836d9
+
+[template-mariadb-initial-setup]
+_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
+md5sum = 9be53e2e92333b93e92556b8a01d9c42
+
+[mariadb-init-root-sql]
+_update_hash_filename_ = templates/mariadb_init_root.sql.in
+md5sum = d927b5d36410bb02717d5ca125525785
+
+[init-root-wrapper-in]
+_update_hash_filename_ = templates/init_root_wrapper.in
+md5sum = 83ef59b5afaf4454d368823c33aef9cb
+
+[repman-manager-sh.in]
+_update_hash_filename_ = templates/repman-manager.sh.in
+md5sum = 50503bec392e31126328f51eadc11634
+
+[dbjobs-in]
+_update_hash_filename_ = templates/dbjobs.in
+md5sum = d2ebd2ec55bf8489789a52c808729925
+
+[mysqld-need-start.sh.in]
+_update_hash_filename_ = templates/mysqld-need-start.sh.in
+md5sum = e9bcee5dc1318fe3acda2663472214f5
+
+[proxy-need-start-stop.sh.in]
+_update_hash_filename_ = templates/proxy-need-start-stop.sh.in
+md5sum = 455aaf369bf5141758dc57f2c0e67b08

software/repman/instance-mariadb.cfg.jinja2.in

+{% set part_list = [] -%}
+{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
+{% set use_ipv6 = slapparameter_dict.get('use-ipv6', True) -%}
+{% set port = slapparameter_dict['tcp-port'] %}
+{% set host = (ipv4_set | list)[0] -%}
+{% if use_ipv6 -%}
+{%   set ip = (ipv6_set | list)[0] -%}
+{%   set host = '[' ~ ip ~ ']' -%}
+{% else -%}
+{%   set ip = (ipv4_set | list)[0] -%}
+{% endif -%}
+{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
+{% set database_list = slapparameter_dict['database-list'] -%}
+
+# XXX- TODO: add cron with check db need restart then restart
+# API check restard needed: http://repman/api/clusters/{clusterName}/servers/{serverName}/{serverPort}/need-restart
+
+[{{ section('publish') }}]
+recipe = slapos.cookbook:publish.serialised
+-extends = publish-early
+database-host = {{ host }}:{{ port }}
+monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
+partition-path = ${buildout:directory}
+receiver-port = ${dbjob-parameter:socat-port}
+
+[publish-early]
+recipe = slapos.cookbook:publish-early
+-init =
+
+database-list = {{ dumps(database_list) }}
+
+[jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+mode = 644
+
+[jinja2-template-executable]
+< = jinja2-template-base
+mode = 755
+
+[simplefile]
+< = jinja2-template-base
+template = inline:{{ '{{ content }}' }}
+
+{% macro simplefile(section_name, file_path, content, mode='') -%}
+{%   set content_section_name = section_name ~ '-content' -%}
+[{{  content_section_name }}]
+content = {{ dumps(content) }}
+
+[{{  section(section_name) }}]
+< = simplefile
+rendered = {{ file_path }}
+context = key content {{content_section_name}}:content
+mode = {{ mode }}
+{%- endmacro %}
+
+[my-cnf-parameters]
+socket = ${directory:run}/mariadb.sock
+ip = {{ ip }}
+data-directory = ${directory:srv}/mariadb
+pid-file = ${directory:run}/mariadb.pid
+plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
+groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
+innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
+innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
+innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}
+innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
+innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
+relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
+ssl-crt = ${directory:mariadb-ssl}/crt.pem
+ssl-key = ${directory:mariadb-ssl}/key.pem
+ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
+
+[my-cnf]
+< = jinja2-template-base
+rendered = ${directory:etc}/mariadb.cnf
+template = {{ parameter_dict['template-my-cnf'] }}
+context = section parameter_dict my-cnf-parameters
+
+[init-root-sql]
+< = jinja2-template-base
+rendered = ${directory:etc}/.init-root.sql
+template = {{ parameter_dict['template-mariadb-init-root'] }}
+context = section parameter_dict init-script-parameters
+mode = 600
+
+[init-script-parameters]
+password = {{ slapparameter_dict['root-password'] }}
+database-list = {{ dumps(database_list) }}
+mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
+root-user = repman
+heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
+
+[init-script]
+< = jinja2-template-executable
+# XXX: is there a better location ?
+rendered = ${directory:etc}/mariadb_initial_setup.sql
+template = {{ parameter_dict['template-mariadb-initial-setup'] }}
+context = section parameter_dict init-script-parameters
+
+[update-mysql]
+recipe = slapos.cookbook:generic.mysql.wrap_update_mysql
+output = ${directory:bin}/mariadb_update
+binary = ${binary-wrap-mysql_upgrade:wrapper-path}
+mysql = ${binary-wrap-mysql:wrapper-path}
+init-script = ${init-script:rendered}
+mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
+
+[{{ section('update-mysql-script') }}]
+< = jinja2-template-executable
+rendered = ${directory:scripts}/mariadb_update
+init-password = ${directory:etc}/.init-passwd.done
+upgrade-done = ${directory:lib}/mariadb-update-done
+context =
+  key init_password_done :init-password
+  key upgrade_done       :upgrade-done
+  key init_root_sql      init-root-sql:rendered
+  key mysql_update       update-mysql:output
+  raw mysql_conf         ${directory:etc}/mysql/my.cnf
+  raw dash_bin           {{ dash }}
+  raw mysql_bin          {{ parameter_dict['mariadb-location'] }}/bin/mysql
+template = {{ parameter_dict['template-init-root-wrapper'] }}
+
+[mysqld]
+< = jinja2-template-executable
+rendered = ${directory:bin}/mysqld
+template = {{ parameter_dict['template-mysqld-wrapper'] }}
+context =
+  key defaults_file install-mysql-config:config
+  key datadir my-cnf-parameters:data-directory
+  key environ :environ
+environ =
+  GRN_PLUGINS_PATH='${my-cnf-parameters:groonga-plugins-path}'
+  ODBCSYSINI='${directory:etc}'
+  LD_LIBRARY_PATH=$${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}'{{ parameter_dict['unixodbc-location'] }}/lib'
+  {%- for variable in slapparameter_dict.get('environment-variables', ()) %}
+  {{ variable }}
+  {%- endfor %}
+
+[ca-mysqld]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+key-file = ${my-cnf-parameters:ssl-key}
+cert-file = ${my-cnf-parameters:ssl-crt}
+executable = ${mysqld:rendered}
+wrapper = ${directory:controller}/mariadb
+
+{% import "supervisord_lib" as supervisord_lib with context %}
+{{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
+{% do part_list.append("supervisord-mariadb-ctl") -%}
+{% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}",
+  "stopwaitsecs": 86400, "environment": [],
+  "stdout_logfile": "${directory:log}/mariadb_stdout.log",
+  "stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
+
+{{ supervisord_lib.supervisord_program("mariadb", maradb_program_dict) }}
+{% do part_list.append("supervisord-mariadb") %}
+
+[odbc-ini-text]
+text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }}
+
+[{{ section('odbc-ini') }}]
+< = jinja2-template-base
+rendered = ${directory:etc}/odbc.ini
+template = inline:{% raw -%}
+  {{ parameter_dict['text'].decode('base64') }}
+  {%- endraw %}
+context = section parameter_dict odbc-ini-text
+
+[{{ section('logrotate-entry-mariadb') }}]
+< = logrotate-entry-base
+name = mariadb
+log = ${dbjob-parameter:log-dir}/errors.log ${dbjob-parameter:log-dir}/sql-errors
+post = "${binary-wrap-mysql:wrapper-path}" -B -e "FLUSH LOGS"
+
+[{{ section('binary-link') }}]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:bin}
+link-binary = {{ dumps(parameter_dict['link-binary']) }}
+
+[binary-wrap-base]
+recipe = slapos.cookbook:wrapper
+# Note: --defaults-file must be the first argument, otherwise wrapped binary
+# will reject it.
+command-line = 
+  "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
+  --defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args}
+wrapper-path = ${directory:bin}/${:command}
+extra-args =
+
+[binary-wrap-mysql]
+<= binary-wrap-base
+command = mysql
+
+[binary-wrap-mysqldump]
+<= binary-wrap-base
+command = mysqldump
+
+[binary-wrap-mysql_upgrade]
+<= binary-wrap-base
+command = mysql_upgrade
+extra-args = --skip-write-binlog
+
+[binary-wrap-mysqladmin]
+<= binary-wrap-base
+command = mysqladmin
+
+[binary-wrap-mysql_tzinfo_to_sql]
+<= binary-wrap-base
+command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
+command = mysql_tzinfo_to_sql
+
+[binary-wrap-pt-digest]
+<= binary-wrap-base
+command-line = "{{ parameter_dict['percona-tools-location'] }}/bin/${:command}"
+command = pt-query-digest
+
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = ${buildout:directory}/bin
+etc = ${buildout:directory}/etc
+scripts = ${:etc}/run
+services = ${:etc}/service
+controller = ${:etc}/controller
+plugin = ${:etc}/plugin
+srv = ${buildout:directory}/srv
+tmp = ${buildout:directory}/tmp
+backup = ${:srv}/backup
+mariadb-backup-full = ${:backup}/mariadb-full
+mariadb-backup-incremental = ${:backup}/mariadb-incremental
+mariadb-ssl = ${:etc}/mariadb-ssl
+var = ${buildout:directory}/var
+lib = ${:var}/lib
+mysql = ${:lib}/mysql
+log = ${:var}/log
+run = ${:var}/run
+config-tmp = ${:tmp}/config
+custom = ${directory:etc}/mysql/custom
+
+[dbjob-parameter]
+bash-bin = {{ bash_bin }}
+db-user = ${init-script-parameters:root-user}
+db-password = ${init-script-parameters:password}
+mysql-dir = ${directory:mysql}
+dbjob-cnf = ${directory:etc}/mysql/my.cnf
+log-dir = ${directory:mysql}/.system/logs
+tmp-dir = ${directory:tmp}
+mysqld-socket  = ${my-cnf-parameters:socket}
+socat-port = {{ int(port) + 9 }}
+restart-script = ${mysqld-restart-script:rendered}
+socat-location = {{ parameter_dict['socat-location'] }}
+mysql-location = {{ parameter_dict['mariadb-location'] }}
+gzip-location = {{ parameter_dict['gzip-location'] }}
+ip = {{ ip }}
+host = {{ host }}
+port = {{ port }}
+use-ipv6 = {{ dumps(use_ipv6) }}
+
+[dbjobs-executable]
+< = jinja2-template-executable
+rendered = ${directory:bin}/dbjobs
+context =
+  section parameter_dict dbjob-parameter
+template = {{ parameter_dict['dbjobs-template'] }}
+
+[{{ section('dbjobs-cron-entry') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = dbjobs
+frequency = * * * * *
+command = ${dbjobs-executable:rendered}
+
+[mysqld-restart-script]
+< = jinja2-template-executable
+rendered = ${directory:bin}/mysqld_restart
+template = inline:#!/bin/sh
+  # This script is disabled on SlapOS
+  echo "RESTART FROM dbjbos WAS DISABLED"
+
+[{{ section('mariadb-need-start') }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = mariadb-need-start
+frequency = * * * * *
+command = ${template-mysqld-need-start:rendered}
+
+[template-mysqld-need-start]
+< = jinja2-template-executable
+rendered = ${directory:bin}/mysqld_need_start
+template = {{ parameter_dict['template-mysqld-need-start'] }}
+context =
+  key mariadb_controller mariadb-ctl-bin:wrapper-path
+  key update_config      mysql-get-config:rendered
+  raw username           {{ slapparameter_dict['repman-user'] }}
+  raw repman_url         {{ slapparameter_dict['repman-url'] }}
+  raw jq_bin             {{ jq_bin }}
+  raw cluster            {{ slapparameter_dict['cluster'] }}
+  raw db_host            {{ host }}
+  raw db_port            {{ port }}
+  raw bash_bin           {{ bash_bin }}
+  raw curl_bin           {{ curl_bin }}
+
+# Donwnload mariadb configuration from repman
+[mysql-get-config]
+< = jinja2-template-executable
+rendered = ${directory:bin}/mysqld-update-config
+cluster = {{ slapparameter_dict['cluster'] }}
+template = inline:#!{{ bash_bin }}
+  cd ${directory:config-tmp} &&
+  {{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
+  tar -xzf config.tar.gz
+  cp -r data/.system ${directory:mysql}
+  rm -rf ${directory:etc}/mysql
+  cp -r etc/mysql ${directory:etc}
+  ln -sf ${directory:mysql}/.system ${directory:var}/system
+  ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf
+
+[{{ section('install-mysql-config') }}]
+recipe = plone.recipe.command
+stop-on-error = true
+config = ${directory:etc}/mysql/my.cnf
+command = ${mysql-get-config:rendered}
+update-command = ${:command}
+
+[dash]
+dash = {{ dumps(dash) }}
+
+[{{ section('promise-check-computer-memory') }}]
+<= monitor-promise-base
+module = check_command_execute
+name = check-computer-memory.py
+config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
+
+[{{ section('promise') }}]
+<= monitor-promise-base
+module = check_command_execute
+name = mariadb.py
+config-command = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "{{ ip }}" "{{ port }}"
+
+[monitor-instance-parameter]
+monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
+monitor-httpd-port = {{ port + 1 }}
+monitor-title = {{ slapparameter_dict['name'] }}
+password = {{ slapparameter_dict['monitor-passwd'] }}
+
+[buildout]
+extends =
+  {{ template_monitor }}
+parts +=
+  {{ part_list | join('\n  ') }}

software/repman/instance-repman-input-schema.json

+{
+	"type": "object",
+	"$schema": "http://json-schema.org/draft-04/schema",
+	"title": "Input Parameters",
+	"properties": {
+		"slave-frontend": {
+			"title": "Web frontend",
+			"description": "Front end used to provide web access for internal services at the kvm.",
+			"properties": {
+				"slave-domain": {
+					"title": "Slave frontend domain",
+					"description": "Unique domain name for this slave frontend.",
+					"type": "string",
+					"default": ""
+				},
+				"instance-guid": {
+					"title": "Main Frontend Instance ID",
+					"description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
+					"type": "string",
+					"default": ""
+				},
+				"frontend-software-type": {
+					"title": "Frontend Software Type",
+					"description": "Type of the frontend instance, like \"frontend\".",
+					"type": "string"
+				},
+				"frontend-software-url": {
+					"title": "Frontend Software URL",
+					"description": "Software Release URL of the frontend instance, like \"http://example.com/path/to/software.cfg\".",
+					"type": "string",
+					"format": "uri",
+					"default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
+				}
+			},
+			"type": "object"
+		},
+		"computer-memory-percent-threshold": {
+			"title": "Computer memory percent threshold.",
+			"description": "Computer memory percent threshold.",
+			"type": "int",
+			"default": 80
+		},
+		"monitor-interface-url": {
+			"title": "Monitor Web Interface URL",
+			"description": "Give Url of HTML web interface that will be used to render this monitor instance.",
+			"type": "string",
+			"format": "uri",
+			"default": "https://monitor.app.officejs.com"
+		},
+		"monitor-cors-domains": {
+			"title": "Monitor CORS domains",
+			"description": "List of cors domains separated with space. Needed for ajax query on this monitor instance from a different domain.",
+			"type": "string",
+			"default": "monitor.app.officejs.com"
+		},
+		"mail-from": {
+			"title": "Mail From",
+			"description": "Mail From address",
+			"type": "string",
+			"default": ""
+		},
+		"mail-smtp-addr": {
+			"title": "Mail SMTP address",
+			"description": "Mail SMTP address. Default: localhost:25",
+			"type": "string",
+			"default": "localhost:25"
+		},
+		"mail-smtp-password": {
+			"title": "Mail SMTP password",
+			"description": "Mail SMTP password",
+			"type": "string",
+			"default": ""
+		},
+		"mail-smtp-user": {
+			"title": "Mail SMTP User",
+			"description": "Mail SMTP User",
+			"type": "string",
+			"default": ""
+		},
+		"mail-to": {
+			"title": "Mail To",
+			"description": "",
+			"type": "string",
+			"default": ""
+		},
+		"tags": {
+			"title": "Provisioning db tags",
+			"description": "Provisioning db tags",
+			"type": "string",
+			"default": "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake"
+		},
+		"http-session-lifetime": {
+			"title": "Web Session life time in s",
+			"description": "Web interface Session life time in seconds. Default 86400",
+			"type": "integer",
+			"default": 86400
+		},
+		"http-refresh-interval": {
+			"title": "Web refresh interval in s",
+			"description": "Web interface refresh interval in s. Default 4s.",
+			"type": "integer",
+			"default": 4
+		},
+		"autorejoin": {
+			"title": "Automatic rejoin a failed master",
+			"description": "Automatic rejoin a failed master (default true)",
+			"type": "boolean",
+			"default": true
+		},
+		"autoseed": {
+			"title": "Automatic join a standalone node",
+			"description": "Automatic join a standalone node",
+			"type": "boolean",
+			"default": true
+		},
+		"repman-cluster-dict": {
+			"title": "Replication Manager clusters definition",
+			"description": "Replication Manager clusters definition",
+			"patternProperties": {
+				".*": {
+					"properties": {
+						"name": {
+							"title": "Name of the cluster",
+							"description": "Name of the cluster: Should not contains spaces or any special characters.",
+							"type": "string",
+							"default": ""
+						},
+						"database-amount": {
+							"title": "Amount of databases for cluster",
+							"description": "Database amount to deploy with this cluster. Minimal amount is 2 required to enable replication.",
+							"type": "integer",
+							"default": 2,
+							"minimum": 2
+						},
+						"-sla-0-computer_guid": {
+							"title": "Prefered Master Computer Guid",
+							"description": "Computer Guid for prefered Master database.",
+							"type": "string",
+							"default": ""
+						},
+						"-sla-1-computer_guid": {
+							"title": "Slave 1 Computer Guid",
+							"description": "Computer Guid for the first slave database.",
+							"type": "string",
+							"default": ""
+						},
+						"-sla-2-computer_guid": {
+							"title": "Slave 2 Computer Guid",
+							"description": "Computer Guid for the second slave database.",
+							"type": "string",
+							"default": ""
+						},
+						"-sla-3-computer_guid": {
+							"title": "Slave 3 Computer Guid",
+							"description": "Computer Guid for the third slave database.",
+							"type": "string",
+							"default": ""
+						},
+						"-sla-4-computer_guid": {
+							"title": "Slave 4 Computer Guid",
+							"description": "Computer Guid for the fourth slave database.",
+							"type": "string",
+							"default": ""
+						},
+						"proxysql-user": {
+							"title": "Proxysql username",
+							"description": "Proxysql external user, default is 'external'.",
+							"type": "string",
+							"default": "external"
+						},
+						"proxy-tags": {
+							"title": "Proxy tags",
+							"description": "playbook configuration tags. Default: pkg,masterslave,linux,noreadwritesplit",
+							"type": "string",
+							"default": "pkg,masterslave,linux,noreadwritesplit"
+						},
+						"logical-backup-cron": {
+							"title": "Mysqldump backup cron definition.",
+							"description": "Logical backup cron expression represents a set of times, using cron format.",
+							"type": "string",
+							"default": "0 21 * * *"
+						},
+						"physical-backup-cron": {
+							"title": "Mariabackup cron definition",
+							"description": "Physical backup cron expression represents a set of times, using cron format.",
+							"type": "string",
+							"default": "0 1 * * *"
+						},
+						"proxy-cpu-cores": {
+							"title": "Proxy Cpu cores",
+							"description": "Proxy Cpu cores. Default: 2",
+							"type": "integer",
+							"default": 2,
+							"minimum": 1
+						},
+						"proxy-memory": {
+							"title": "Proxy Memory usage in giga bytes",
+							"description": "Proxy Memory usage in giga bytes. Default: 1G",
+							"type": "integer",
+							"default": 1,
+							"minimum": 1
+						},
+						"db-cpu-cores": {
+							"title": "Database Cpu cores",
+							"description": "Database Cpu cores. Default: 2",
+							"type": "integer",
+							"default": 2,
+							"minimum": 1
+						},
+						"db-disk-iops": {
+							"title": "Database Rnd IO/s",
+							"description": "Rnd IO/s in seconds for micro service VM (default 300).",
+							"type": "integer",
+							"default": 300
+						},
+						"db-memory": {
+							"title": "Database memory in M",
+							"description": "Memory in M for micro service VM (default 256)",
+							"type": "integer",
+							"default": 256,
+							"minimum": 256,
+							"multipleOf": 256
+						},
+						"db-memory-shared-pct": {
+							"title": "Percent memory shared per buffer",
+							"description": "Percent memory shared per buffer (default \"threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0\")",
+							"type": "string",
+							"default": "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0"
+						},
+						"db-memory-threaded-pct": {
+							"title": "Percent memory allocted per threads",
+							"description": "Percent memory allocted per threads. (default \"tmp:70,join:20,sort:10\")",
+							"type": "string",
+							"default": "tmp:70,join:20,sort:10"
+						},
+						"innodb-file-per-table": {
+							"title": "enable Innodb file per table",
+							"description": "enable Innodb file per table. Possible value: 0=disabled, 1=enabled",
+							"type": "integer",
+							"default": 1,
+							"minimum": 0,
+							"maximum": 1
+						},
+						"use-ipv6": {
+							"title": "Mariadb server listen on IPv6",
+							"description": "Listen on IPv6 instead of IPv4.",
+							"type": "boolean",
+							"default": true
+						},
+						"failover-mode": {
+							"title": "Failover mode",
+							"description": "Failover is manual or automatic (default \"manual\").",
+							"type": "string",
+							"default": "manual",
+              "enum": [
+                "manual",
+                "automatic"
+              ]
+
+						},
+						"failover-limit": {
+							"title": "Failover amount limit",
+							"description": "Failover is canceld if already failover this number of time (0: unlimited) (default 5).",
+							"type": "integer",
+							"default": 5
+						},
+						"failover-falsepositive-heartbeat": {
+							"title": "Failover check slaves do not receive heartbeat",
+							"description": "Failover checks that slaves do not receive heartbeat (default true).",
+							"type": "boolean",
+							"default": true
+						},
+						"failover-falsepositive-heartbeat-timeout": {
+							"title": "Failover check slaves do not receive heartbeat timeout",
+							"description": "Failover checks that slaves do not receive heartbeat detection timeout  (default 3).",
+							"type": "integer",
+							"default": 3
+						},
+						"failover-falsepositive-ping-counter": {
+							"title": "Failover amount of ping failures",
+							"description": "Failover after this number of ping failures (interval 1s) (default 5).",
+							"type": "integer",
+							"default": 5
+						},
+						"failover-max-slave-delay": {
+							"title": "Failover election ignore slave with replication delay",
+							"description": "Election ignore slave with replication delay over this time in sec (default 30).",
+							"type": "integer",
+							"default": 30
+						},
+						"failover-readonly-state": {
+							"title": "Failover Switchover set slaves as read-only",
+							"description": "Failover Switchover set slaves as read-only (default true).",
+							"type": "boolean",
+							"default": true
+						},
+						"failover-restart-unsafe": {
+							"title": "Failover when cluster down if a slave is start first",
+							"description": "Failover when cluster down if a slave is start first.",
+							"type": "boolean",
+							"default": false
+						},
+						"failover-time-limit": {
+							"title": "Cancel failover if time in sec not passed after previous failover",
+							"description": "Failover is canceled if timer in sec is not passed with previous failover (0: do not wait).",
+							"type": "integer",
+							"default": 0
+						},
+						"switchover-at-equal-gtid": {
+							"title": "Switchover only when slaves are fully in sync",
+							"description": "Switchover only when slaves are fully in sync.",
+							"type": "boolean",
+							"default": false
+						},
+						"switchover-slave-wait-catch": {
+							"title": "Switchover wait for slave to catch with replication",
+							"description": "Switchover wait for slave to catch with replication, not needed in GTID mode but enable to detect possible issues like witing on old master (default true).",
+							"type": "boolean",
+							"default": true
+						},
+						"switchover-wait-kill": {
+							"title": "Switchover wait ms before killing threads on demoted master",
+							"description": "Switchover wait this many milliseconds before killing threads on demoted master (default 5000).",
+							"type": "integer",
+							"default": 5000
+						},
+						"switchover-wait-trx": {
+							"title": "Cancel switchover after timeout seconds if can't aquire FTWRL",
+							"description": "Switchover is cancel after this timeout in second if can't aquire FTWRL (default 10).",
+							"type": "integer",
+							"default": 10
+						},
+						"switchover-wait-write-query": {
+							"title": "Cancel switchover after timeout if a write query is running",
+							"description": "Switchover is canceled if a write query is running for this time (default 10).",
+							"type": "integer",
+							"default": 10
+						}
+					},
+					"type": "object"
+				}
+			},
+			"type": "object"
+		}
+	}
+}
\ No newline at end of file

software/repman/instance-repman-output-schema.json

+{
+  "name": "Output Parameters",
+  "properties": {
+    "backend-url": {
+      "title": "Backend URL",
+      "description": "URL used to connect directly to backend without frontend. Requires IPv6.",
+      "type": "string",
+      "format": "uri"
+    },
+    "url": {
+      "title": "URL",
+      "description": "URL used to connect to the service.",
+      "type": "string",
+      "format": "uri"
+    },
+    "repman-password": {
+      "title": "Repman password",
+      "description": "Password for Replication Manager service.",
+      "type": "string"
+    }
+  }
+}

software/repman/instance-repman.cfg.jinja2.in

+{% set publish_dict = {} -%}
+{% set part_list = [] -%}
+{% set monitor_base_url_dict = {} -%}
+{% set mariadb_dict = {} -%}
+{% set mariadb_server_list = [] -%}
+{% set receiver_port_list = [] -%}
+{% set mariadb_path_list = [] -%}
+{% set ip = (ipv6_set | list)[0] -%}
+{% set ipv4 = (ipv4_set | list)[0] -%}
+{% set cluster_list = [] -%}
+{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake" -%}
+{% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+run = ${:var}/run
+scripts = ${:etc}/run
+service = ${:etc}/service
+controller = ${:etc}/controller
+promise = ${:etc}/promise
+log = ${:var}/log
+data = ${:var}/lib
+nginx-prefix = ${:var}/nginx
+tmp = ${:home}/tmp
+
+{% import "supervisord_lib" as supervisord_lib with context %}
+{% set proxysql_controller = "proxysql-ctl" -%}
+{{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
+{% do part_list.append("supervisord-proxysql-ctl") -%}
+
+[request-common]
+recipe = slapos.cookbook:request.serialised
+software-url = ${slap-connection:software-release-url}
+server-url = ${slap-connection:server-url}
+key-file = ${slap-connection:key-file}
+cert-file = ${slap-connection:cert-file}
+computer-id = ${slap-connection:computer-id}
+partition-id = ${slap-connection:partition-id}
+
+[download-proxy-config]
+recipe = slapos.recipe.template:jinja2
+template = inline:#!{{ bash_bin }}
+  NAME=$1
+  HOST=$2
+  PORT=$3
+  CONFIG=$4
+  if [ -z "$CONFIG" ]; then
+    CONFIG="${repman:proxies}/proxysql-$NAME.cnf"
+  fi
+  mkdir -p ${repman:config-tmp}/proxies
+  cd ${repman:config-tmp}/proxies
+  {{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
+  tar -xzf proxies-$NAME.tar.gz
+  cp conf/proxysql.cnf $CONFIG
+rendered = ${directory:bin}/update-proxysql-config
+mode = 755
+
+{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
+{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
+  "database-amount": 2, "proxysql-user": "external", "proxy-tags": "pkg,masterslave,linux,noreadwritesplit",
+  "logical-backup-cron": "0 21 * * *", "physical-backup-cron": "0 1 * * *"}} -%}
+{% for name, parameter_dict in slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).items() -%}
+
+{% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
+{% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
+
+{% set database_list = parameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
+# Request mariadb instances
+{% set db_amount = parameter_dict.get('database-amount', 2) -%}
+{% if db_amount < 2 -%}
+{% set db_amount = 2 -%}
+{% endif -%}
+
+{% for i in range(0, db_amount) -%}
+{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
+{% set section = 'request-mariadb-' ~ i -%}
+{% set dbname = 'Mariadb-' ~ i -%}
+
+[{{ section }}]
+<= request-common
+software-type = mariadb
+name = {{ dbname }}
+sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
+{% for key, value in mariadb_dict.items() -%}
+config-{{ key }} = {{ dumps(value) }}
+{% endfor -%}
+config-monitor-passwd = ${publish-early:monitor-password}
+config-root-password = ${publish-early:db-root-password}
+config-repman-user = ${repman-parameter:username}
+config-heartbeat-user = ${repman-parameter:heartbeat-user}
+#config-repman-passwd =  ${repman-parameter:password}
+config-repman-url = ${nginx-parameter:backend-url}
+config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
+config-cluster = {{ name }}
+config-name = {{ dbname }}
+config-database-list = {{ dumps(database_list) }}
+return = 
+  database-host
+  receiver-port
+  monitor-base-url
+  partition-path
+
+{% do part_list.append(section) -%}
+{% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
+{% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
+{% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
+{% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
+
+{% endfor -%}
+
+# Manage Replication Manager clusters
+[{{name}}-admin-port]
+recipe = slapos.cookbook:free_port
+ip = {{ ipv4 }}
+minimum = 6032
+maximum = 6132
+
+[{{name}}-port]
+recipe = slapos.cookbook:free_port
+ip = {{ ipv4 }}
+minimum = 7032
+maximum = 7132
+
+{% set prefered_master = parameter_dict.get("db-prefered-master") -%}
+[{{ name ~ '-cluster-parameter' }}]
+{% for key, value in parameter_dict.items() -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
+proxy-port = {{ '${' ~ name ~ '-port:port}' }}
+proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
+db-user = repman
+db-password = ${publish-early:db-root-password}
+db-list = {{ mariadb_server_list | join(',') }}
+
+heartbeat-user = ${repman-parameter:heartbeat-user}
+heartbeat-password = ${publish-early:db-root-password}
+partition-list = {{ mariadb_path_list | join(',') }}
+{% if prefered_master -%}
+db-prefered-master = {{ prefered_master }}
+{% else -%}
+# First database is the prefered master
+db-prefered-master = {{ mariadb_server_list[0] }}
+{% endif -%}
+proxysql-servers = {{ ipv4 }}
+proxysql-servers-ipv6 = [{{ ip }}]
+password = ${repman-parameter:password}
+proxysql-partition = ${buildout:directory}
+receiver-port-list = {{ receiver_port_list | join(',') }}
+proxy-tags = {{ parameter_dict.get("proxy-tags", "pkg,masterslave,linux,noreadwritesplit") }}
+logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
+physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
+proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }}
+proxy-memory = {{ parameter_dict.get("proxy-memory", 1) }}
+db-cpu-cores = {{ parameter_dict.get("db-cpu-cores", 2) }}
+db-disk-iops = {{ parameter_dict.get("db-disk-iops", 300) }}
+db-memory = {{ parameter_dict.get("db-memory", 256) }}
+db-memory-shared-pct = {{ parameter_dict.get("db-memory-shared-pct", "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0") }}
+db-memory-threaded-pct = {{ parameter_dict.get("db-memory-threaded-pct", "tmp:70,join:20,sort:10") }}
+# failover
+failover-mode = {{ parameter_dict.get('failover-mode', 'manual') }}
+failover-limit = {{ parameter_dict.get('failover-limit', 5) }}
+failover-falsepositive-heartbeat = {{ parameter_dict.get('failover-falsepositive-heartbeat', True) }}
+failover-falsepositive-heartbeat-timeout = {{ parameter_dict.get('failover-falsepositive-heartbeat-timeout', 3) }}
+failover-falsepositive-ping-counter = {{ parameter_dict.get('failover-falsepositive-ping-counter', 5) }}
+failover-max-slave-delay = {{ parameter_dict.get('failover-max-slave-delay', 30) }}
+failover-readonly-state = {{ parameter_dict.get('failover-readonly-state', True) }}
+failover-restart-unsafe = {{ parameter_dict.get('failover-restart-unsafe', False) }}
+failover-time-limit = {{ parameter_dict.get('failover-time-limit', 0) }}
+#switchover
+switchover-at-equal-gtid = {{ parameter_dict.get('switchover-at-equal-gtid', False) }}
+switchover-slave-wait-catch = {{ parameter_dict.get('switchover-slave-wait-catch', True) }}
+switchover-wait-kill = {{ parameter_dict.get('switchover-wait-kill', 5000) }}
+switchover-wait-trx = {{ parameter_dict.get('switchover-wait-trx', 10) }}
+switchover-wait-write-query = {{ parameter_dict.get('switchover-wait-write-query', 10) }}
+
+[{{ 'config-' ~ name }}]
+recipe = slapos.recipe.template:jinja2
+template = {{ config_cluster_toml_in }}
+rendered = ${repman:clusters}/config-{{ name }}.toml
+extra-context =
+context =
+  section parameter_dict {{ name ~ '-cluster-parameter' }}
+
+# Donwnload mariadb configuration from repman
+[config-proxysql-{{ name }}]
+recipe = plone.recipe.command
+# if Repman is not started, cannot download config from server
+stop-on-error = false
+config = ${repman:proxies}/proxysql-{{ name }}.cnf
+data = ${repman:proxy-data}/{{ name }}
+command =
+  mkdir -p ${:data} &&
+  ${download-proxy-config:rendered} {{ name }} {{ ipv4 }} {{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }} ${:config}
+update-command = ${:command}
+
+[proxysql-{{ name }}-wrapper]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ proxysql_location }}/bin/proxysql -f
+  -c ${config-proxysql-{{ name }}:config}
+  -D ${config-proxysql-{{ name }}:data}
+  --reload
+#  -S /tmp/proxysql_admin.sock
+wrapper-path = ${directory:controller}/proxysql-{{ name }}
+wait-for-files =
+  ${repman:bootstrap}/{{ name }}_bootstrapped
+depends =
+  {{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }}
+  {{ '${proxysql-' ~ name ~ '-promise:recipe}' }}
+  {{ '${proxysql-' ~ name ~ '-ipv6-promise:recipe}' }}
+
+[proxysql-{{ name }}-admin-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = proxysql-{{ name }}-admin-port-listening.py
+config-hostname=  {{ ipv4 }}
+config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }}
+
+[proxysql-{{ name }}-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = proxysql-{{ name }}-port-listening.py
+config-hostname=  {{ ipv4 }}
+config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
+
+[proxysql-{{ name }}-ipv6-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = proxysql-{{ name }}-ipv6-port-listening.py
+config-hostname=  {{ ip }}
+config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
+
+{% set service_name = "proxysql-" ~ name -%}
+{% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
+  "stopwaitsecs": 60, "environment": [],
+  "stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
+  "stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
+
+{{ supervisord_lib.supervisord_program(service_name, proxysql_dict) }}
+{% do part_list.append("supervisord-" ~ service_name) %}
+
+{% do part_list.append('config-' ~ name) -%}
+{% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
+
+{% set publish_database_list = [] -%}
+{% set publish_database_v6_list = [] -%}
+{% for database in database_list -%}
+{%   if database.get('user') -%}
+{%     do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
+{%     do publish_database_v6_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
+{%   else -%}
+{%     do publish_database_list.append("mysql://" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
+{%     do publish_database_v6_list.append("mysql://[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
+{%   endif -%}
+{% endfor -%}
+
+{% do publish_dict.__setitem__(name ~ '-database-list', "!py!['" ~ publish_database_list | join("', '") ~ "']") -%}
+{% do publish_dict.__setitem__(name ~ '-database-list-v6', "!py!['" ~ publish_database_v6_list | join("', '") ~ "']") -%}
+{% endfor -%}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[instance-parameter]
+<= slap-configuration
+# repman monitor seems to use a fixed port
+repman-secure-port = 10005
+repman-port = 10001
+nginx-ssl-port  = 10006
+nginx-port  = 10007
+
+
+[repman]
+recipe   = slapos.cookbook:mkdirectory
+etc      = ${directory:etc}/repman
+data-dir = ${directory:var}/lib
+root-dir = ${directory:srv}/repman
+clusters = ${:etc}/cluster.d
+proxies  = ${:etc}/proxy
+proxy-data = ${:data-dir}/proxy
+config-tmp = ${directory:tmp}/config
+bootstrap = ${:etc}/bootstrap
+proxies-log = ${directory:log}/proxy
+
+[nginx-parameter]
+ipv6 = ${instance-parameter:ipv6-random}
+port = ${instance-parameter:nginx-port}
+ssl-port = ${instance-parameter:nginx-ssl-port}
+ssl-certificate = ${ca-nginx:cert-file}
+ssl-key = ${ca-nginx:key-file}
+pid-file = ${directory:run}/nginx.pid
+access-log = ${directory:log}/nginx_access.log
+error-log = ${directory:log}/nginx_error.log
+repman-secure-url = https://${repman-parameter:ipv4}:${repman-parameter:secure-port}
+repman-url = http://${repman-parameter:ipv4}:${repman-parameter:port}
+config-file = ${directory:etc}/nginx.conf
+backend-ssl-url = https://[${:ipv6}]:${:ssl-port}
+backend-url = http://[${:ipv6}]:${:port}
+
+[repman-password]
+recipe = slapos.cookbook:generate.password
+bytes = 12
+
+[gen-root-password]
+recipe = slapos.cookbook:generate.password
+bytes = 12
+
+[repman-parameter]
+log = ${directory:log}/repman.log
+http-root = ${repman:root-dir}/dashboard
+share-dir = ${repman:root-dir}/share
+secure-port = ${instance-parameter:repman-secure-port}
+port = ${instance-parameter:repman-port}
+ipv4 = ${instance-parameter:ipv4-random}
+username = admin
+heartbeat-user = heartbeat
+password = ${publish-early:repman-password}
+cluster-d = ${repman:clusters}
+autorejoin = {{ slapparameter_dict.get("autorejoin", True) }}
+autoseed = {{ slapparameter_dict.get("autoseed", True) }}
+mysql-bin-dir = {{ mariadb_location }}/bin
+mysqlbinlog-path = {{ mariadb_location }}/bin/mysqlbinlog
+mysqlclient-path = {{ mariadb_location }}/bin/mysql
+mysqldump-path = {{ mariadb_location }}/bin/mysqldump
+haproxy-bin   = {{ haproxy_location }}/sbin/haproxy
+sysbench-bin  = {{ sysbench_location }}/bin/sysbench
+restic-bin    = {{ restic_bin_location }}
+
+mail-from = {{ slapparameter_dict.get("mail-from", "mrm@localhost") }}
+mail-smtp-addr = {{ slapparameter_dict.get("mail-smtp-addr", "localhost:25") }}
+mail-smtp-password = {{ slapparameter_dict.get("mail-smtp-password", "") }}
+mail-smtp-user = {{ slapparameter_dict.get("mail-smtp-user", "") }}
+mail-to = {{ slapparameter_dict.get("mail-to", "") }}
+enabled-tags = {{ slapparameter_dict.get("tags", tags) }}
+
+http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }}
+http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }}
+
+[repman-config-folder]
+recipe = plone.recipe.command
+repman-location = {{ repman_src_location }}
+command =
+  cd ${:repman-location}
+  {{ rsync_location }}/bin/rsync -av share ${repman:root-dir}/
+  {{ rsync_location }}/bin/rsync -av  dashboard ${repman:root-dir}/
+
+update-command = ${:command}
+  
+[replication-manager-reload]
+recipe = slapos.recipe.template:jinja2
+template = {{ template_repman_manager_sh }}
+cluster-list = {{ dumps(slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).keys() ) }}
+context =
+  section parameter_dict repman
+  key username repman-parameter:username
+  key password repman-parameter:password
+  key secure_url nginx-parameter:backend-ssl-url
+  key cluster_name_list :cluster-list
+  raw jq_bin {{ jq_bin }}
+  raw curl_bin {{ curl_bin }}
+  raw bash_bin {{ bash_bin }}
+rendered = ${directory:scripts}/repman-reload
+mode = 755
+
+[replication-manager]
+recipe = slapos.cookbook:wrapper
+command-line =
+   {{ gowork_bin }}/replication-manager
+   --monitoring-basedir=${repman:root-dir}
+   --monitoring-sharedir=${repman-parameter:share-dir}
+   --http-root=${repman-parameter:http-root}
+   --monitoring-datadir=${repman:data-dir}
+   --config=${repman-config.toml:rendered}
+   --log-file=${repman-parameter:log}
+   --memprofile=${directory:tmp}/repmgr.mprof
+   monitor
+
+wrapper-path = ${directory:service}/replication-manager
+# setup repman instance folder
+depends =
+  ${repman-config-folder:recipe}
+  ${replication-manager-reload:recipe}
+  ${repman-listen-promise:recipe}
+  ${repman-listen-ssl-promise:recipe}
+
+[repman-config.toml]
+recipe = slapos.recipe.template:jinja2
+template = {{ config_toml_in }}
+rendered = ${repman:etc}/config.toml
+extra-context =
+context =
+  section parameter_dict repman-parameter
+
+[repman-listen-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = repman_service_listen.py
+config-hostname = ${repman-parameter:ipv4}
+config-port = ${repman-parameter:port}
+
+[repman-listen-ssl-promise]
+<= monitor-promise-base
+module = check_port_listening
+name = repman_service_ssl_listen.py
+config-hostname = ${repman-parameter:ipv4}
+config-port = ${repman-parameter:secure-port}
+
+[nginx-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ nginx_conf_in }}
+rendered = ${nginx-parameter:config-file}
+context =
+    section parameter_dict nginx-parameter
+
+[nginx-launcher]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ nginx_bin }}
+  -p ${directory:nginx-prefix}
+  -c ${nginx-conf:rendered}
+wrapper-path = ${directory:bin}/nginx-start
+wait-for-files =
+  ${ca-directory:certs}/nginx.key
+  ${ca-directory:certs}/nginx.crt
+  ${nginx-graceful-wrapper:rendered}
+
+[nginx-graceful-wrapper]
+recipe = slapos.recipe.template:jinja2
+template = inline:#!{{ bash_bin }}
+  kill -USR1 "$(cat ${nginx-parameter:pid-file})"
+rendered = ${directory:scripts}/nginx-graceful
+context =
+mode = 755
+
+[ca-nginx]
+<= certificate-authority
+recipe = slapos.cookbook:certificate_authority.request
+cert-file = ${ca-directory:certs}/nginx.crt
+key-file = ${ca-directory:certs}/nginx.key
+executable = ${nginx-launcher:wrapper-path}
+wrapper = ${directory:bin}/ca-nginx
+
+[ca-nginx-service]
+recipe = slapos.cookbook:wrapper
+command-line = ${ca-nginx:wrapper}
+wrapper-path = ${directory:services}/nginx
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+[logrotate-entry-nginx]
+<= logrotate-entry-base
+name = nginx
+log = ${nginx-parameter:access-log} ${nginx-parameter:error-log}
+post = kill -USR1 $(cat ${nginx-parameter:pid-file})
+
+[publish-early]
+recipe = slapos.cookbook:publish-early
+-init =
+  monitor-password monitor-htpasswd:passwd
+  db-root-password gen-root-password:passwd
+  repman-password repman-password:passwd
+
+[publish-connection-parameter]
+<= monitor-publish
+-extends = publish-early
+recipe = slapos.cookbook:publish
+backend-url = ${nginx-parameter:backend-ssl-url}
+url = ${repman-frontend:connection-secure_access}
+username = ${repman-parameter:username}
+{% for name, value in publish_dict.items() -%}
+{{   name }} = {{ value }}
+{% endfor %}
+
+
+[monitor-instance-parameter]
+monitor-httpd-port = 8060
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }} 
+username = admin
+password = ${publish-early:monitor-password}
+
+[monitor-base-url-dict]
+{% for key, value in monitor_base_url_dict.items() -%}
+{{ key }} = {{ value }}
+{% endfor %}
+
+[repman-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Replication Manager Frontend
+# XXX We have hardcoded SR URL here.
+software-url = {{ frontend_parameter_dict.get('frontend-software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg') }}
+{% if frontend_parameter_dict.get('frontend-software-type', '') -%}
+software-type ={{ frontend_parameter_dict['frontend-software-type'] }}
+{% endif -%}
+slave = true
+config-url = ${nginx-parameter:backend-ssl-url}
+config-domain = {{ frontend_parameter_dict.get('slave-domain', '') }}
+return = domain secure_access
+
+[repman-frontend-promise]
+<= monitor-promise-base
+module = check_url_available
+name = check_repman_frontend.py
+config-url = https://${repman-frontend:connection-domain}
+config-check-secure = 1
+
+[repman-backend-promise]
+<= monitor-promise-base
+module = check_url_available
+name = check_repman_backend.py
+config-url = ${nginx-parameter:backend-ssl-url}
+config-check-secure = 1
+
+[template-proxysql-need-stop-start]
+recipe = slapos.recipe.template:jinja2
+rendered = ${directory:bin}/proxysql_check_stop_start
+template = {{ template_proxy_need_stopstart }}
+mode = 755
+cluster-list = !py![{{ cluster_list | join(', ') }}]
+context =
+  key proxysql_controller {{proxysql_controller}}-bin:wrapper-path
+  key repman_url          nginx-parameter:backend-url
+  key get_proxy_config    download-proxy-config:rendered
+  key cluster_list        :cluster-list
+  raw jq_bin              {{ jq_bin }}
+  raw bash_bin           {{ bash_bin }}
+  raw curl_bin            {{ curl_bin }}
+
+[proxy-need-stop-start]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = proxysql-need-stop-start
+frequency = * * * * *
+command = ${template-proxysql-need-stop-start:rendered}
+
+#############################
+#
+# Deploy replication-manager instance
+#
+#############################
+[buildout]
+extends =
+  {{ template_monitor }}
+parts =
+  replication-manager
+  monitor-base
+  logrotate-entry-nginx
+  ca-nginx-service
+  publish-connection-parameter
+  repman-frontend-promise
+  repman-backend-promise
+  proxy-need-stop-start
+
+# Complete parts with sections
+  {{ part_list | join('\n  ') }}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true

software/repman/instance.cfg.in

+[buildout]
+parts = switch-softwaretype
+
+eggs-directory = {{ buildout_egg_directory }}
+develop-eggs-directory = {{ buildout_develop_directory }}
+offline = true
+
+
+[switch-softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+default = template-instance-repman.cfg:rendered
+RootSoftwareInstance = ${:default}
+mariadb = template-instance-mariadb.cfg:rendered
+
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+
+[jinja2-template-base]
+recipe  = slapos.recipe.template:jinja2
+mode    = 0644
+extensions = jinja2.ext.do
+rendered= ${buildout:directory}/${:_buildout_section_name_}
+supervisord-lib = {{ supervisord_lib }}
+import-list =
+  file supervisord_lib :supervisord-lib
+context =
+    key slapparameter_dict      slap-configuration:configuration
+    key computer_id             slap-configuration:computer
+    key ipv4_set                slap-configuration:ipv4
+    key ipv6_set                slap-configuration:ipv6
+    raw buildout_directory      {{ buildout_directory }}
+    raw buildout_bin_directory  {{ buildout_bin_directory }}
+    raw eggs_directory          {{ buildout_egg_directory }}
+    raw develop_eggs_directory  {{ buildout_develop_directory }}
+    raw mariadb_location        {{ mariadb_location }}
+    raw supervisord_lib         {{ supervisord_lib }}
+    raw supervisord_conf        {{ supervisord_conf }}
+    raw template_monitor        {{ template_monitor_cfg }}
+
+# program binaries
+    raw bash_bin                {{ bash_location }}/bin/bash
+    raw jq_bin                  {{ jq_location }}/jq
+    raw curl_bin                {{ curl_location }}/bin/curl
+
+    ${:extra-context}
+extra-context =
+
+[template-instance-repman.cfg]
+<= jinja2-template-base
+template= {{ template_repman_cfg }}
+extra-context =
+    raw gowork_bin                  {{ gowork_bin }}
+    raw haproxy_location            {{ haproxy_location }}
+    raw nginx_bin                   {{ nginx_location }}/sbin/nginx
+    raw repman_src_location         {{ repman_src_location }}
+
+# config files
+    raw config_toml_in             {{ config_toml_in }}
+    raw config_cluster_toml_in     {{ config_cluster_toml_in }}
+    raw nginx_conf_in              {{ nginx_conf_in }}
+    raw rsync_location             {{ rsync_location }}
+    raw restic_bin_location        {{ restic_bin_location }}
+    raw sysbench_location          {{ sysbench_location }}
+    raw proxysql_location          {{ proxysql_location }}
+    raw template_repman_manager_sh {{ template_repman_manager_sh }}
+    raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
+
+[template-mariadb-parameters]
+bash = {{ bash_location }}
+dash-location = {{ dash_location }}
+gzip-location = {{ gzip_location }}
+mariadb-location = {{ mariadb_location }}
+template-my-cnf = {{ template_my_cnf }}
+template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
+template-mariadb-init-root = {{ template_init_root_sql }}
+template-init-root-wrapper = {{ template_init_root_wrapper }}
+template-mysqld-wrapper = {{ template_mysqld_wrapper }}
+template-mysqld-need-start = {{ mysqld_start_template }}
+link-binary = {{ dumps(mariadb_link_binary) }}
+check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
+bin-directory = {{ bin_directory }}
+percona-tools-location = {{ percona_toolkit_location }}
+unixodbc-location = {{ unixodbc_location }}
+curl-location = {{ curl_location }}
+dbjobs-template = {{ dbjobs_in }}
+socat-location = {{ socat_location }}
+mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
+mroonga-mariadb-plugin-dir = {{ mroonga_mariadb_plugin_dir }}
+groonga-plugins-path = {{ groonga_plugin_dir }}:{{ groonga_mysql_normalizer_plugin_dir }}
+
+[template-instance-mariadb.cfg]
+<= jinja2-template-base
+template = {{ template_mariadb }}
+filename = instance-mariadb.cfg
+extra-context =
+    section parameter_dict template-mariadb-parameters
+
+

software/repman/software.cfg

+[buildout]
+
+extends =
+  buildout.hash.cfg
+  ../../component/restic/buildout.cfg
+  ../../component/replication-manager/buildout.cfg
+  ../../component/mariadb/buildout.cfg
+  ../../component/nginx/buildout.cfg
+  ../../component/haproxy/buildout.cfg
+  ../../component/logrotate/buildout.cfg
+  ../../component/percona-toolkit/buildout.cfg
+  ../../component/gzip/buildout.cfg
+  ../../component/sed/buildout.cfg
+  ../../component/coreutils/buildout.cfg
+  ../../component/grep/buildout.cfg
+  ../../component/sysbench/buildout.cfg
+  ../../component/proxysql/buildout.cfg
+  ../../component/socat/buildout.cfg
+  ../../component/rsync/buildout.cfg
+  ../../stack/supervisord/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+  ../neoppod/software-common.cfg
+
+parts =
+  slapos-cookbook
+  mroonga-mariadb
+  instance.cfg
+  template-mariadb.cfg
+  template-mysqld-wrapper
+  gowork
+
+[instance.cfg]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:directory}/instance.cfg
+template = ${:_profile_base_location_}/${:filename}
+mode = 0644
+context =
+    key bash_location bash:location
+    key bin_directory buildout:bin-directory
+    key config_toml_in config-toml.in:target
+    key config_cluster_toml_in config-cluster-toml.in:target
+    key coreutils_location coreutils:location
+    key curl_location curl:location
+    key buildout_egg_directory buildout:eggs-directory
+    key buildout_develop_directory buildout:develop-eggs-directory
+    key buildout_directory buildout:directory
+    key buildout_bin_directory buildout:bin-directory
+    key dbjobs_in dbjobs-in:target
+    key dash_location dash:location
+    key jq_location jq-binary:location
+    key logrotate_cfg template-logrotate-base:rendered
+    key gowork_bin gowork:bin
+    key gzip_location gzip:location
+    key haproxy_location haproxy:location
+    key template_monitor monitor2-template:rendered
+    key mariadb_link_binary template-mariadb.cfg:link-binary
+    key mariadb_location mariadb:location
+    key mysqld_start_template mysqld-need-start.sh.in:target
+    key mroonga_mariadb_install_sql mroonga-mariadb:install-sql
+    key mroonga_mariadb_plugin_dir mroonga-mariadb:plugin-dir
+    key groonga_plugin_dir groonga:groonga-plugin-dir
+    key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
+    key nginx_conf_in nginx.conf.in:target
+    key nginx_location nginx:location
+    key percona_toolkit_location percona-toolkit:location
+    key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
+    key repman_src_location git.signal18.io_signal18_repman:location
+    key rsync_location rsync:location
+    key restic_bin_location restic:location
+    key socat_location socat:location
+    key supervisord_lib supervisord-library:target
+    key supervisord_conf supervisord-conf:target
+    key template_repman_manager_sh repman-manager-sh.in:target
+    key template_mariadb template-mariadb.cfg:target
+    key template_mariadb_initial_setup template-mariadb-initial-setup:target
+    key template_monitor_cfg monitor2-template:rendered
+    key template_my_cnf template-my-cnf:target
+    key template_mysqld_wrapper template-mysqld-wrapper:rendered
+    key template_init_root_sql mariadb-init-root-sql:target
+    key template_init_root_wrapper init-root-wrapper-in:target
+    key template_repman_cfg instance-repman.cfg:target
+    key unixodbc_location unixodbc:location
+    key sysbench_location sysbench:location
+    key proxysql_location proxysql:location
+
+
+[jq-binary]
+recipe = hexagonit.recipe.download
+url = https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+md5sum = 1fffde9f3c7944f063265e9a5e67ae4f
+filename = jq
+mode = 0755
+download-only = true
+
+[download-file]
+recipe  = slapos.recipe.build:download
+url     = ${:_profile_base_location_}/${:_update_hash_filename_}
+destination = ${buildout:directory}/${:_buildout_section_name_}
+
+[config-toml.in]
+<= download-file
+
+[config-cluster-toml.in]
+<= download-file
+
+[instance-repman.cfg]
+<= download-file
+
+[repman-manager-sh.in]
+<= download-file
+
+[template-mariadb.cfg]
+<= download-file
+link-binary =
+  ${coreutils:location}/bin/basename
+  ${coreutils:location}/bin/cat
+  ${coreutils:location}/bin/cp
+  ${coreutils:location}/bin/ls
+  ${coreutils:location}/bin/tr
+  ${coreutils:location}/bin/uname
+  ${gettext:location}/lib/gettext/hostname
+  ${grep:location}/bin/grep
+  ${sed:location}/bin/sed
+  ${mariadb:location}/bin/mysqlbinlog
+
+[template-mariadb-initial-setup]
+<= download-file
+
+[template-my-cnf]
+<= download-file
+
+[mariadb-init-root-sql]
+<= download-file
+
+[init-root-wrapper-in]
+<= download-file
+
+[nginx.conf.in]
+<= download-file
+
+[dbjobs-in]
+<= download-file
+
+[mysqld-need-start.sh.in]
+<= download-file
+
+[proxy-need-start-stop.sh.in]
+<= download-file
+
+# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
+[versions]
+slapos.recipe.template = 4.4
+rubygemsrecipe  = 0.2.2+slapos001

software/repman/software.cfg.json

+{
+  "name": "Replication Manager",
+  "description": "Replication Manager",
+  "serialisation": "xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "serialisation": "json-in-xml",
+      "description": "Replication Manager",
+      "request": "instance-repman-input-schema.json",
+      "response": "instance-repman-output-schema.json",
+      "index": 0
+    }
+  }
+}

software/repman/templates/cluster-config.toml.in

+{% macro setbool(value) -%}
+{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
+{% endmacro -%}
+[{{ parameter_dict['name'] }}]
+title = "{{ parameter_dict['name'] }}"
+monitoring-save-config = true
+db-servers-hosts = "{{ parameter_dict['db-list'] }}"
+db-servers-prefered-master = "{{ parameter_dict['db-prefered-master'] }}"
+db-servers-credential = "{{ parameter_dict['db-user'] }}:{{ parameter_dict['db-password'] }}"
+replication-credential = "{{ parameter_dict['db-user'] }}:{{ parameter_dict['db-password'] }}"
+monitoring-write-heartbeat-credential="{{ parameter_dict['heartbeat-user'] }}:{{ parameter_dict['heartbeat-password'] }}"
+db-servers-connect-timeout = 1
+slapos-db-partitions = "{{ parameter_dict['partition-list'] }}"
+slapos-proxysql-partitions = "{{ parameter_dict['proxysql-partition'] }}"
+
+proxysql = true
+proxysql-port = {{ parameter_dict['proxy-port'] }}
+proxysql-servers = "{{ parameter_dict['proxysql-servers'] }}"
+proxysql-servers-ipv6 = "{{ parameter_dict['proxysql-servers-ipv6'] }}"
+proxysql-user = "{{ parameter_dict['proxysql-user'] }}"
+proxysql-bootstrap = true
+proxysql-admin-port = {{ parameter_dict['proxy-admin-port'] }}
+proxysql-password = "{{ parameter_dict['password'] }}"
+
+prov-proxy-tags = "{{ parameter_dict['proxy-tags'] }}"
+
+monitoring-scheduler = true
+scheduler-db-servers-logical-backup  = true
+scheduler-db-servers-logical-backup-cron = "0 {{ parameter_dict['logical-backup-cron'] }}"
+scheduler-db-servers-logs   =  true
+scheduler-db-servers-logs-cron = "0 0 23 * * *"
+scheduler-db-servers-logs-table-keep = 4
+scheduler-db-servers-logs-table-rotate  = true
+scheduler-db-servers-logs-table-rotate-cron = "0 0 23 * * *"
+scheduler-db-servers-optimize  = true
+scheduler-db-servers-optimize-cron = "0 0 3 1 * 5"
+scheduler-db-servers-physical-backup = true
+scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backup-cron'] }}"
+backup-physical-type = "mariabackup"
+backup-logical-type = "mysqldump"
+scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
+
+prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
+prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
+prov-db-cpu-cores = {{ parameter_dict['db-cpu-cores'] }}
+prov-db-disk-iops = {{ parameter_dict['db-disk-iops'] }}
+prov-db-memory = {{ parameter_dict['db-memory'] }}
+prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
+prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
+
+test-inject-traffic = true
+# failover
+failover-mode = "{{ parameter_dict['failover-mode'] }}"
+failover-limit = {{ parameter_dict['failover-limit'] }}
+failover-falsepositive-heartbeat = {{ setbool(parameter_dict['failover-falsepositive-heartbeat']) }}
+failover-falsepositive-heartbeat-timeout = {{ parameter_dict['failover-falsepositive-heartbeat-timeout'] }}
+failover-falsepositive-ping-counter = {{ parameter_dict['failover-falsepositive-ping-counter'] }}
+failover-max-slave-delay = {{ parameter_dict['failover-max-slave-delay'] }}
+failover-readonly-state = {{ setbool(parameter_dict['failover-readonly-state']) }}
+failover-restart-unsafe = {{ setbool(parameter_dict['failover-restart-unsafe']) }}
+failover-time-limit = {{ parameter_dict['failover-time-limit'] }}
+#switchover
+switchover-at-equal-gtid = {{ setbool(parameter_dict['switchover-at-equal-gtid']) }}
+switchover-slave-wait-catch = {{ setbool(parameter_dict['switchover-slave-wait-catch']) }}
+switchover-wait-kill = {{ parameter_dict['switchover-wait-kill'] }}
+switchover-wait-trx = {{ parameter_dict['switchover-wait-trx'] }}
+switchover-wait-write-query = {{ parameter_dict['switchover-wait-write-query'] }}
+
+

software/repman/templates/config.toml.in

+{% macro setbool(value) -%}
+{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
+{% endmacro -%}
+[Default]
+api-bind = "{{ parameter_dict['ipv4'] }}"
+http-bind-address = "{{ parameter_dict['ipv4'] }}"
+http-server = true
+http-session-lifetime = {{ parameter_dict['http-session-lifetime'] }}
+http-refresh-interval = {{ int(parameter_dict['http-refresh-interval'])*1000 }}
+monitoring-save-config = false
+api-https-bind = true
+api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
+include = "{{ parameter_dict['cluster-d'] }}"
+autorejoin = {{ setbool(parameter_dict['autorejoin']) }}
+autoseed = {{ setbool(parameter_dict['autoseed']) }}
+{% if parameter_dict['autoseed'] in ['true', 'True'] -%}
+autorejoin-logical-backup = true
+{% endif -%}
+db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
+# Database list of hosts to ignore in election
+#db-servers-ignored-hosts =
+# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
+monitoring-address = "{{ parameter_dict['ipv4'] }}"
+monitoring-wait-retry = 40
+
+#haproxy = true
+#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
+# HaProxy input bind address for read (default "0.0.0.0")
+#haproxy-ip-read-bind =
+# HaProxy input bind address for write (default "0.0.0.0")
+#haproxy-ip-write-bind =
+# HaProxy load balance read port to all nodes (default 3307)
+#haproxy-read-port =
+# HaProxy hosts (default "127.0.0.1")
+#haproxy-servers =
+# HaProxy statistics port (default 1988)
+#haproxy-stat-port =
+#HaProxy read-write port to leader (default 3306)
+#haproxy-write-port =
+
+# Use restic to archive and restore backups
+backup = true
+backup-restic = true
+backup-restic-binary-path = "{{ parameter_dict['restic-bin'] }}"
+backup-restic-aws = false
+backup-restic-password = "{{ parameter_dict['password'] }}"
+
+backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
+backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
+backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
+
+# Mail configuration
+# Alert email sender (default "mrm@localhost")
+mail-from = "{{ parameter_dict['mail-from'] }}"
+# Alert email SMTP server address, in host:[port] format (default "localhost:25")
+mail-smtp-addr = "{{ parameter_dict['mail-smtp-addr'] }}"
+mail-smtp-password = "{{ parameter_dict['mail-smtp-password'] }}"
+mail-smtp-user = "{{ parameter_dict['mail-smtp-user'] }}"
+# Alert email recipients, separated by commas
+mail-to = "{{ parameter_dict['mail-to'] }}"
+
+prov-orchestrator = "slapos"
+prov-db-tags = "{{ parameter_dict['enabled-tags'] }}"
+
+sysbench-binary-path = "{{ parameter_dict['sysbench-bin'] }}"
+# Number of threads to run benchmark (default 4)
+sysbench-threads = 4
+# Time to run benchmark (default 100)
+sysbench-time = 100
+sysbench-v1 = true
+

software/repman/templates/dbjobs.in

+#!/bin/bash
+
+USER={{ parameter_dict['db-user'] }}
+PASSWORD={{ parameter_dict['db-password'] }}
+ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
+SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
+BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
+DATADIR={{ parameter_dict['mysql-dir'] }}/
+{% if parameter_dict['use-ipv6'] == True -%}
+{% set listen = "TCP6-LISTEN" -%}
+{% else -%}
+{% set listen = "TCP-LISTEN" -%}
+{% endif -%}
+export PATH={{ parameter_dict['socat-location'] }}/bin:{{ parameter_dict['mysql-location'] }}/bin:{{ parameter_dict['gzip-location'] }}/bin:$PATH
+JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "reseedxtrabackup" "reseedmariabackup" "reseedmysqldump" "flashbackxtrabackup" "flashbackmariadbackup" "flashbackmysqldump" "stop" "start")
+
+doneJob()
+{
+  mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
+}
+
+pauseJob()
+{
+  mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
+}
+
+partialRestore()
+{
+ mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
+ for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
+ mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
+
+
+  for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
+    cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
+    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo  engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
+    mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
+    mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
+    mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
+    mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
+    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
+  done
+  for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
+    mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
+    mysql --defaults-file=/etc/mysql/dbjob.cnf  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
+  done
+  for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
+    mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
+    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
+  done
+ done
+ for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
+    mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
+    mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
+ done
+ cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk  -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
+ mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}   -e"flush privileges;start slave;"
+}
+
+kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
+
+for job in "${JOBS[@]}"
+do
+
+ TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
+
+ ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
+ ID=($(echo $TASK | awk -F@ '{ print $1 }'))
+ #purge de past
+ mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
+
+  if [ "$ADDRESS" == "" ]; then
+    echo "No $job needed"
+  else
+    echo "Processing $job"
+    case "$job" in
+      reseedmysqldump)
+       echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
+       pauseJob
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
+        mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
+      ;;
+      flashbackmysqldump)
+       echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
+       pauseJob
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
+       mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }}  -e 'start slave;'
+      ;;
+      reseedmariabackup)
+       rm -rf $BACKUPDIR
+       mkdir $BACKUPDIR
+       echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
+       pauseJob
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | mbstream -x -C $BACKUPDIR
+       # mbstream -p, --parallel
+       mariabackup --prepare --export --target-dir=$BACKUPDIR
+       partialRestore
+      ;;
+      flashbackmariadbackup)
+       rm -rf $BACKUPDIR
+       mkdir $BACKUPDIR
+       echo "Waiting backup." >  {{ parameter_dict['log-dir'] }}/dbjob.out
+       pauseJob
+       socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | xbstream -x -C $BACKUPDIR
+       mariabackup --prepare --export --target-dir=$BACKUPDIR
+       partialRestore
+      ;;
+      mariabackup)
+       cd {{ parameter_dict['tmp-dir'] }}
+       mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}'  --no-version-check  --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
+      ;;
+      error)
+       cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
+      ;;
+      slowquery)
+       cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
+      ;;
+      optimize)
+       mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
+      ;;
+      restart)
+        {{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out
+      ;;
+  esac
+  doneJob
+  fi
+
+done

software/repman/templates/init_root_wrapper.in

+#!/bin/bash
+
+run_mysql () {
+  {{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
+  --protocol=socket -uroot -hlocalhost $@
+}
+
+if [ ! -f "{{ init_password_done }}" ]; then
+  for i in {30..0}; do
+    if echo 'SELECT 1' | run_mysql  &> /dev/null; then
+      break
+    fi
+    echo 'MySQL init process in progress...'
+    sleep 1
+  done
+  if [ "$i" = 0 ]; then
+    echo >&2 'MySQL init process failed.'
+    exit 1
+  fi
+  echo "Setting mariabdb root password...";
+  run_mysql  < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
+  echo "done"
+fi
+
+# Run mariadb_upgrade when replication is bootstrapped will break replication topology.
+# skip when already upgraded until we have a good solution.
+if [ ! -f "{{ upgrade_done }}" ]; then
+  {{ mysql_update }}
+  if [ $? -eq 0 ]; then
+    touch {{ upgrade_done }};
+  fi
+fi
\ No newline at end of file

software/repman/templates/mariadb_init_root.sql.in

+--  What's done in this file shouldn't be replicated
+--  or products like mysql-fabric won't work
+SET @@SESSION.SQL_LOG_BIN=0;
+
+CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
+CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
+CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
+
+CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
+CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
+GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
+
+DROP DATABASE IF EXISTS test ;
+FLUSH PRIVILEGES ;
\ No newline at end of file

software/repman/templates/mariadb_initial_setup.sql.in

+SET @@SESSION.SQL_LOG_BIN=0;
+
+USE mysql;
+{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
+{% if mroonga %}
+SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
+{% endif %}
+DROP FUNCTION IF EXISTS sphinx_snippets;
+#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
+
+{% macro database(name, user, password) -%}
+CREATE DATABASE IF NOT EXISTS `{{ name }}`;
+{% if user -%}
+GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
+GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
+GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
+{%- endif %}
+{% endmacro -%}
+
+{% for entry in parameter_dict['database-list'] -%}
+{{   database(entry['name'], entry.get('user'), entry.get('password')) }}
+{% endfor -%}
\ No newline at end of file

software/repman/templates/my.cnf.in

+{% set socket = parameter_dict['socket'] -%}
+# ERP5 buildout my.cnf template based on my-huge.cnf shipped with mysql
+# The MySQL server
+[mysqld]
+# ERP5 by default requires InnoDB storage. MySQL by default fallbacks to using
+# different engine, like MyISAM. Such behaviour generates problems only, when
+# tables requested as InnoDB are silently created with MyISAM engine.
+#
+# Loud fail is really required in such case.
+# Already present in REPMAN
+#sql_mode="NO_ENGINE_SUBSTITUTION"
+
+socket = {{ socket }}
+datadir = {{ parameter_dict['data-directory'] }}
+pid_file = {{ parameter_dict['pid-file'] }}
+
+{% set innodb_buffer_pool_size = parameter_dict['innodb-buffer-pool-size'] -%}
+{% if innodb_buffer_pool_size %}innodb_buffer_pool_size = {{ innodb_buffer_pool_size }}{% endif %}
+{% set innodb_buffer_pool_instances = parameter_dict['innodb-buffer-pool-instances'] -%}
+{% if innodb_buffer_pool_instances %}innodb_buffer_pool_instances = {{ innodb_buffer_pool_instances }}{% endif %}
+{% set innodb_log_file_size = parameter_dict['innodb-log-file-size'] -%}
+{% if innodb_log_file_size %} innodb_log_file_size = {{ innodb_log_file_size }}{% endif %}
+{% set innodb_log_buffer_size = parameter_dict['innodb-log-buffer-size'] -%}
+{% if innodb_log_buffer_size %} innodb_log_buffer_size = {{ innodb_log_buffer_size }}{% endif %}
+
+# very important to allow parallel indexing
+# Note: this is compatible with binlog-based incremental backups, because ERP5
+# doesn't use "insert ... select" (in any number of queries) pattern.
+# innodb_locks_unsafe_for_binlog = 1
+
+#plugin_load = ha_mroonga
+plugin-dir = {{ parameter_dict['plugin-directory'] }}
+
+{% if 'ssl-key' in parameter_dict -%}
+ssl
+ssl-cert = {{ parameter_dict['ssl-crt'] }}
+ssl-key = {{ parameter_dict['ssl-key'] }}
+{%   if 'ssl-ca-crt' in parameter_dict -%}
+ssl-ca = {{ parameter_dict['ssl-ca-crt'] }}
+{%-  endif %}
+{%   if 'ssl-crl' in parameter_dict -%}
+ssl-crl = {{ parameter_dict['ssl-crl'] }}
+{%-  endif %}
+{%   if 'ssl-cipher' in parameter_dict -%}
+ssl-cipher = {{ parameter_dict['ssl-cipher'] }}
+{%-  endif %}
+{%- endif %}
+
+
+# Some dangerous settings you may want to uncomment temporarily
+# if you only want performance or less disk access.
+{% set x = '' if parameter_dict['relaxed-writes'] else '#' -%}
+{{x}}innodb_flush_log_at_trx_commit = 0
+{{x}}innodb_flush_method = nosync
+{{x}}innodb_doublewrite = 0
+{{x}}sync_frm = 0
+
+# skip_character_set_client_handshake
+
+
+[client]
+socket = {{ socket }}
+user = root
+
+[mysql]
+no_auto_rehash
+
+[mysqlhotcopy]
+interactive_timeout
+
+[mysqldump]
+max_allowed_packet = 128M

software/repman/templates/mysqld-need-start.sh.in

+#!{{ bash_bin }}
+
+curl () {
+  {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
+}
+
+# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
+
+# Checking if mariadb start is needed
+#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
+CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
+
+if [ $CODE -eq 200 ]; then
+  echo "$CODE: Updating mysql configuration..."
+  # update mysql configuration
+  {{ update_config }}
+  echo "$CODE: Starting mariadb service..."
+  # print current status, can be useful for debug...
+  {{ mariadb_controller }} status mariadb
+  {{ mariadb_controller }} start mariadb
+  sleep 5
+  # check again if the service is still up...
+  {{ mariadb_controller }} status mariadb
+fi

software/repman/templates/nginx.conf.in

+pid {{ parameter_dict['pid-file'] }};
+error_log {{ parameter_dict['error-log'] }};
+
+daemon off;
+
+events {
+  worker_connections 1024;
+  accept_mutex off;
+}
+
+http {
+     default_type application/octet-stream;
+     access_log {{ parameter_dict['access-log'] }} combined;
+     client_max_body_size 10M;
+     map $http_upgrade $connection_upgrade {
+        default upgrade;
+        ''      close;
+     }
+
+     server {
+        listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['ssl-port'] }} ssl;
+        server_name _;
+        ssl_certificate     {{ parameter_dict['ssl-certificate'] }};
+        ssl_certificate_key {{ parameter_dict['ssl-key'] }};
+        ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
+        ssl_ciphers         ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5;
+        ssl_prefer_server_ciphers on;
+        keepalive_timeout 90s;
+
+        location / {
+            proxy_redirect off;
+            proxy_set_header   X-Forwarded-Proto $scheme;
+            proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+            proxy_set_header   X-Forwarded-Host  $http_host;
+            proxy_connect_timeout 200;
+
+            proxy_pass {{ parameter_dict['repman-secure-url'] }};
+        }
+    }
+
+    server {
+        listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['port'] }};
+        server_name _;
+
+        location / {
+            proxy_redirect off;
+            proxy_set_header   X-Forwarded-Proto $scheme;
+            proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+            proxy_set_header   X-Forwarded-Host  $http_host;
+            proxy_connect_timeout 200;
+
+            proxy_pass {{ parameter_dict['repman-url'] }};
+        }
+    }
+}

software/repman/templates/proxy-need-start-stop.sh.in

+#!{{ bash_bin }}
+
+curl () {
+  {{ curl_bin }} -k --silent "$@"
+}
+
+check_start_cluster () {
+  NAME=$1
+  HOST=$2
+  PORT=$3
+
+  CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-start)
+
+  if [ $CODE -eq 200 ]; then
+    echo "$CODE: Starting proxysql $HOST:$PORT..."
+    {{ proxysql_controller }} start proxysql-$NAME
+    sleep 1
+    # check again if the service is still up...
+    {{ proxysql_controller }} status proxysql-$NAME
+  fi
+}
+
+check_stop_cluster () {
+  NAME=$1
+  HOST=$2
+  PORT=$3
+
+  CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-stop)
+
+  if [ $CODE -eq 200 ]; then
+    echo "$CODE: updating proxysql config $HOST:$PORT..."
+    {{ get_proxy_config }} $NAME $HOST $PORT
+    echo "$CODE: Stoping proxysql $HOST:$PORT..."
+    {{ proxysql_controller }} stop proxysql-$NAME
+    sleep 1
+    # check again if the service is stopped...
+    {{ proxysql_controller }} status proxysql-$NAME
+  fi
+}
+
+{% for cluster_dict in cluster_list -%}
+check_start_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
+check_stop_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
+{% endfor -%}

software/repman/templates/repman-manager.sh.in

+#!{{ bash_bin }}
+
+#set -e
+
+curl () {
+  {{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
+}
+
+get_token () {
+  curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
+}
+
+wait_database () {
+  NAME=$1
+  for retry in {1..50}; do
+    echo ">> Wait until $NAME databases are ready...";
+    CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases);
+    if [ $CODE -eq 504 ]; then
+      # We have a timeout try again
+      echo ">> [$retry] Timeout on {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases, trying again...";
+    else
+      if [ $CODE -eq 200 ]; then
+        break;
+      else
+        if [ $CODE -eq 401 ]; then
+          # try again with new token
+          TOKEN=$(get_token | {{ jq_bin }} -r '.token')
+        fi
+      fi
+      echo ">> [$retry] waitdatabases returned code $CODE...";
+    fi
+    sleep 30
+  done
+}
+
+activate_proxy () {
+  NAME=$1
+  URL="{{ secure_url }}/api/clusters/$NAME/settings/actions/switch/database-hearbeat"
+  echo ">> Calling $URL...";
+  CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" $URL)
+  if [ $CODE -eq 200 ]; then
+    return 0;
+  else
+    echo ">> ERROR: failed to activate proxy: $URL returned code $CODE"
+    return 1;
+  fi
+}
+
+TOKEN=$(get_token | {{ jq_bin }} -r '.token')
+
+# Always reload cluster configuration to apply recent changes
+{% for name in cluster_name_list -%}
+# reload {{ name }} settings
+echo "Reloading settings for {{ name }}..."
+curl -H "Authorization: Bearer ${TOKEN}" \
+  {{ secure_url }}/api/clusters/{{ name }}/settings/actions/reload
+
+# Start Replication on {{ name }}
+if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
+
+  wait_database {{ name }}
+
+  echo "Bootstrap replication on {{ name }}..."
+  TOKEN=$(get_token | {{ jq_bin }} -r '.token')
+
+  curl -H "Authorization: Bearer ${TOKEN}" \
+    {{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
+  CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
+  SUCCESS=0
+  if [ $CODE -eq 200 ]; then
+    activate_proxy {{ name }}
+    if [ $? -eq 0 ]; then
+      # Mark boostrap done!
+      echo "Cluster {{ name }} replication bootstrapped"
+      echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
+    fi
+  else
+    echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
+  fi
+fi
+
+{% endfor %}
\ No newline at end of file

software/repman/test/README.md

+Tests for Replication Manager software release

software/repman/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.repman'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' Replication Manager",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'supervisor',
+        'pexpect',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/repman/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from __future__ import unicode_literals
+
+import os
+import textwrap
+import logging
+import tempfile
+import time
+from six.moves.urllib.parse import urlparse, urljoin
+
+import pexpect
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestRepman(SlapOSInstanceTestCase):
+  __partition_reference__ = 'R'  # solve path too long for postgresql and unicorn
+
+  @classmethod
+  def getInstanceSoftwareType(cls):
+    return 'default'
+
+  def setUp(self):
+    self.backend_url = self.computer_partition.getConnectionParameterDict(
+    )['backend-url']
+
+  def test_http_get(self):
+    resp = requests.get(self.backend_url, verify=False)
+    self.assertTrue(
+      resp.status_code in [requests.codes.ok, requests.codes.found])

software/slapos-sr-testing/software.cfg

@@ -148,6 +148,11 @@ setup = ${slapos-repository:location}/software/cloudooo/test/
 egg = slapos.test.dream
 setup = ${slapos-repository:location}/software/dream/test/
 
+[slapos.test.repman-setup]
+<= setup-develop-egg
+egg = slapos.test.repman
+setup = ${slapos-repository:location}/software/repman/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
@@ -193,6 +198,7 @@ eggs =
   ${slapos.test.cloudooo-setup:egg}
   ${slapos.test.dream-setup:egg}
   ${slapos.test.metabase-setup:egg}
+  ${slapos.test.repman-setup:egg}
   ${backports.lzma:egg}
 entry-points =
   runTestSuite=erp5.util.testsuite:runTestSuite
@@ -259,6 +265,7 @@ extra =
   ${slapos.test.gitlab-setup:setup}
   ${slapos.test.cloudooo-setup:setup}
   ${slapos.test.dream-setup:setup}
+  ${slapos.test.repman-setup:setup}
 
 [versions]
 # slapos.core is used from the clone always

stack/caucase/buildout.cfg

@@ -35,6 +35,6 @@ mode = 0644
 depends = ${caucase-jinja2-library-eggs:eggs}
 
 [versions]
-caucase = 0.9.5
+caucase = 0.9.7
 pem = 17.1.0
 PyJWT = 1.7.1

stack/erp5/buildout.cfg

@@ -610,7 +610,7 @@ zope.app.testing = 3.8.1
 
 # Pinned versions
 APacheDEX = 1.6.2
-Pillow = 5.2.0
+Pillow = 6.2.2
 Products.CMFActionIcons = 2.1.3
 Products.DCWorkflowGraph = 0.4.1
 # Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one.
@@ -706,7 +706,7 @@ fpconst = 0.7.2
 graphviz = 0.5.2
 
 # Required by:
-# Pillow==4.0.0
+# Pillow==6.2.2
 olefile = 0.44
 
 # Required by:

stack/slapos.cfg

@@ -95,6 +95,7 @@ eggs =
   ${python-cachecontrol:egg}
   ${python-cliff:egg}
   ${python-cryptography:egg}
+  ${jsonschema:egg}
   pyOpenSSL
   slapos.cookbook
 
@@ -107,6 +108,11 @@ eggs =
   ${python-cryptography:egg}
   slapos.toolbox
 
+[jsonschema]
+recipe = zc.recipe.egg:custom
+egg = ${:_buildout_section_name_}
+setup-eggs = setuptools_scm
+
 # Install a slapos command with networkcache enabled in ${buildout:bin-directory}
 [slapos-command]
 recipe = zc.recipe.egg
@@ -230,6 +236,10 @@ attrs = 18.2.0
 # jsonschema==3.0.0a3
 pyrsistent = 0.14.5
 
+# Required by:
+# jsonschema==3.0.2
+setuptools-scm = 3.5.0
+
 # Required by:
 # cryptography==1.8.1
 ipaddress = 1.0.18

stack/supervisord/README.rst

+Supervisord process manager
+
+How to use
+==========
+
+Supervisord stack provides a library which can be called in your instance slapos. This stack can be used to run sub services in a partition.
+
+To use:
+
+ * extend ``stack/supervisord/buildout.cfg`` in your software.cfg file.
+ * provide ``supervisord-library:target`` and ``supervisord-conf:target`` to your instance template which require to use supervisord controller.
+ * add ``{% import "supervisord" as supervisord with context %}`` to instance template which call supervisord library. See example below:
+
+**software.cfg**
+::
+
+  [template-instance]
+  recipe = slapos.recipe.template:jinja2
+  context =
+      key buildout_bin_directory buildout:bin-directory
+      key supervisord        supervisord-library:target
+      key supervisord_conf   supervisord-conf:target
+
+**instance.cfg.in**
+::
+
+    [template-custom-instance.cfg]
+    recipe  = slapos.recipe.template:jinja2
+    supervisord-lib = {{ supervisord }}
+    import-list =
+      file supervisord :supervisord-lib
+    context =
+      raw buildout_bin_directory  {{ buildout_bin_directory }}
+      raw supervisord_conf        {{ supervisord_conf }}
+
+
+**custom-instance.cfg**
+::
+
+  {% import "supervisord" as supervisord with context %}
+  {{ supervisord.supervisord("custom-controller", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
+
+  # add program to service controller
+  {% set program_dict = {"name": "mariadb", "command": "${mariadb-service:wrapper}",
+    "stopwaitsecs": 300, "environment": []} %}
+
+  {{ supervisord.supervisord_program("mariadb", program_dict) }}
+
+  ...
+  
+  [buildout]
+  
+  parts = 
+    ...
+    supervisord-custom-controller
+    supervisord-mariadb
+
+
+Supervisord inside partition
+============================
+
+Check supervisord controlled services status:
+::
+
+  $ instance/slappartXX/bin/custom-controller status
+  mariadb                          RUNNING   pid 5511, uptime 6:04:54
+
+
+`supervisord_program` parameters and defaults:
+
+.. code-block:: python
+
+  program_dict = {
+    "name": "NAME",
+    "command": "WRAPPER_PATH",
+    "stopwaitsecs": 60,
+    "environment": ['PATH="/usr/bin/:/partition/bin/:$PATH"', 'MAKEFLAGS="-j2"'],
+    "autostart": True,
+    "autorestart": False,
+    "startsecs": 0,
+    "startretries": 0,
+    "stopsignal": "TERM",
+    "stdout_logfile": "NONE",
+    "stderr_logfile": "NONE"
+  }
+

stack/supervisord/buildout.cfg

+[buildout]
+
+extends =
+  ../slapos.cfg
+  buildout.hash.cfg
+
+parts =
+  supervisord-conf
+  supervisord-library
+
+[supervisord-download-base]
+recipe = slapos.recipe.build:download
+mode = 0644
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+
+
+[supervisord-eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${slapos-cookbook:eggs}
+  supervisor
+scripts =
+  supervisord
+  supervisorctl
+
+[supervisord-library]
+<= supervisord-download-base
+filename = supervisord.jinja2.in
+depends = ${supervisord-eggs:recipe}
+
+[supervisord-conf]
+<= supervisord-download-base
+filename = supervisord.conf.in

stack/supervisord/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[supervisord-library]
+_update_hash_filename_ = supervisord.jinja2.in
+md5sum = 163c9f60e4ad3842162cbb11d771b7b8
+
+[supervisord-conf]
+_update_hash_filename_ = supervisord.conf.in
+md5sum = d624f65151233493c6dbdafa83ae8cbd
+

stack/supervisord/supervisord.conf.in

+[unix_http_server]
+file = {{ parameter_dict['socket-path'] }}
+chmod=0700
+
+[include]
+files = {{ parameter_dict['include-dir'] }}/*.conf
+
+[supervisorctl]
+serverurl = unix://{{ parameter_dict['socket-path'] }}
+
+[supervisord]
+loglevel = {{ parameter_dict['log-level'] }}
+logfile_maxbytes = 2MB
+nodaemon = false
+logfile-backups = 3
+logfile = {{ parameter_dict['log-file'] }}
+pidfile = {{ parameter_dict['pid-file'] }}
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

stack/supervisord/supervisord.jinja2.in

+{% macro supervisord(
+  name,
+  buildout_bin_directory,
+  supervisord_conf,
+  use_service_hash=False
+) -%}
+
+[controller-directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+var = ${buildout:directory}/var
+log = ${:var}/log
+run = ${:var}/run
+supervisord = ${:etc}/supervisord-{{ name }}.conf.d
+
+[controller-parameters]
+socket-path = ${controller-directory:run}/{{ name }}.socket
+include-dir = ${controller-directory:supervisord}
+log-file    = ${controller-directory:log}/supervisord-{{ name }}.log
+log-level   = info
+pid-file    = ${controller-directory:run}/supervisord-{{ name }}.pid
+
+[supervisord-controller-conf]
+recipe = slapos.recipe.template:jinja2
+template = {{ supervisord_conf }}
+context =
+  section parameter_dict controller-parameters
+rendered = ${controller-directory:etc}/supervisord-{{ name }}.conf
+
+[supervisord-{{ name }}]
+recipe = slapos.cookbook:wrapper
+command-line = {{ buildout_bin_directory }}/supervisord -c ${supervisord-controller-conf:rendered} --nodaemon
+wrapper-path = ${directory:services}/supervisord-{{ name }}
+{% if use_service_hash -%}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+{% endif -%}
+depends = 
+  {{ '${' ~ name ~ '-bin:recipe}' }}
+
+[{{ name }}-bin]
+recipe = slapos.cookbook:wrapper
+command-line = {{ buildout_bin_directory }}/supervisorctl -c ${supervisord-controller-conf:rendered}
+wrapper-path = ${directory:bin}/{{ name }}
+
+{%- endmacro %}
+
+
+{% macro supervisord_program(
+  name,
+  parameter_dict
+) -%}
+
+
+[supervisord-{{ name }}]
+recipe = slapos.recipe.template:jinja2
+template = inline:[program:{{ parameter_dict['name'] }}]
+  directory = ${buildout:directory}
+  command = {{ parameter_dict['command'] }}
+  process_name = {{ parameter_dict['name'] }}
+  autostart = {{ parameter_dict.get('autostart', True) }}
+  autorestart = {{ parameter_dict.get('autorestart', False) }}
+  startsecs = {{ parameter_dict.get('startsecs', 0) }}
+  startretries = {{ parameter_dict.get('startretries', 0) }}
+  exitcodes = {{ parameter_dict.get('exitcodes', 0) }}
+  stopsignal = {{ parameter_dict.get('stopsignal', 'TERM') }}
+  stopwaitsecs = {{ parameter_dict.get('stopwaitsecs', 60) }}
+  serverurl=AUTO
+  redirect_stderr=true
+  stdout_logfile = {{ parameter_dict.get('stdout_logfile', 'NONE') }}
+  stdout_logfile_maxbytes = 1000KB
+  stdout_logfile_backups = 1
+  stderr_logfile = {{ parameter_dict.get('stderr_logfile', 'NONE') }}
+  stderr_logfile_maxbytes = 1000KB
+  stderr_logfile_backups = 1
+  environment = {{ parameter_dict['environment'] | join(',') }}
+
+rendered = ${controller-directory:supervisord}/{{ name }}.conf
+
+{%- endmacro %}

[Jérome Perrin]

In this release KVM and gitlab are broken ( for what seems to be a buildout bug, because for gitlab a workaround like jerome/slapos@5806ed5c seems to fix ).

@rafael @tomo How should we do when we need to release but master is broken ?

PS: is anybody working on repairing the state on master ? += and extends have always been buggy in buildout, maybe we should apply this gitlab patch and do same in kvm ?

[Rafael Monnerat]

How should we do when we need to release but master is broken ?

It is a bit complex but couldn't it get fixed by who broken it or make a workarround? KVM needs to be stable for a project so, in any case, it should be fixed somehow.

[Thomas Gambier]

How should we do when we need to release but master is broken ?

It's OK to merge on 1.0 branch even if all software are not working. The purpose of merging in 1.0 is to be able to tag and generate a SR in slapos master. We won't generate a SR in slapos master for broken SR for sure.

And of course, master should be fixed in the meantime.

PS: is anybody working on repairing the state on master ? += and extends have always been buggy in buildout, maybe we should apply this gitlab patch and do same in kvm ?

You can have a look at https://www.erp5.com/group_section/forum/buildout--extends-and---cheat-sheet-O0wP6uGjM1