Skip to content

slapos
slapos
Commit f581d489 authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

Update Release Candidate 

Conflicts:
	component/groonga/buildout.cfg
	component/mariadb/buildout.cfg
	software/caucase/software.cfg
	software/jupyter/instance.cfg.in
	software/jupyter/software.cfg
	stack/caucase/buildout.hash.cfg
parents 53366193 b8a90cb0
Hide whitespace changes
Inline Side-by-side
Showing with 1905 additions and 2367 deletions
component/apache/buildout.cfg
View file @ f581d489
......@@ -34,9 +34,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
[apache]
# inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
recipe = slapos.recipe.cmmi
version = 2.4.26
version = 2.4.27
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = d4d47749a44461cb2e6c9d78a22b522b
md5sum = 97b6bbfa83c866dbe20ef317e3afd108
pre-configure =
cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ &&
cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util
......@@ -118,9 +118,9 @@ environment =
[apache-2.2]
# inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
recipe = slapos.recipe.cmmi
version = 2.2.32
version = 2.2.34
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = d590ca1a8ce577b6bdfbd738d2de4741
md5sum = c0bf1bdb779a3b5e6e294caf9ca20441
patch-options = -p1
configure-options = --disable-static
--enable-authn-alias
......
component/file/buildout.cfg
View file @ f581d489
......@@ -8,8 +8,8 @@ extends =
[file]
recipe = slapos.recipe.cmmi
url = ftp://ftp.astron.com/pub/file/file-5.23.tar.gz
md5sum = 61db35209ce71a6d576392ce6e1d2f80
url = http://ftp.icm.edu.pl/packages/file/file-5.32.tar.gz
md5sum = 4f2503752ff041895090ed6435610435
configure-options =
--disable-static
environment =
......
component/gnutls/buildout.cfg
View file @ f581d489
......@@ -14,6 +14,7 @@ url = http://www.gnupg.org/ftp/gcrypt/${:_buildout_section_name_}/${:_buildout_s
configure-options = ${:configure-options-extra}
--enable-option-checking=fatal
--disable-dependency-tracking
--disable-doc
environment = ${:environment-extra}
PATH=${gettext:location}/bin:%(PATH)s
environment-extra =
......@@ -23,25 +24,22 @@ environment-extra =
version = 1.27
md5sum = 5217ef3e76a7275a2a3b569a12ddc989
configure-options-extra =
--disable-doc
--disable-tests
[libgcrypt]
<= gpg-common
version = 1.7.7
md5sum = d1769481b1b506a632fd66c5e5f62e41
version = 1.8.1
md5sum = b21817f9d850064d2177285f1073ec55
configure-options-extra =
--with-gpg-error-prefix=${libgpg-error:location}
--disable-doc
environment-extra =
LDFLAGS=-Wl,-rpath=${libgpg-error:location}/lib
[gnutls]
<= gpg-common
url = http://www.gnupg.org/ftp/gcrypt/gnutls/v3.5/gnutls-3.5.13.tar.xz
md5sum = 4fd41ad86572933c2379b4cc321a0959
url = http://www.gnupg.org/ftp/gcrypt/gnutls/v3.5/gnutls-3.5.15.tar.xz
md5sum = bcdcbc65c50a7499617ad9f4d0058de9
configure-options-extra =
--disable-doc
--disable-static
--disable-libdane
--disable-guile
......
component/groonga/buildout.cfg
View file @ f581d489
......@@ -14,8 +14,8 @@ extends =
[groonga]
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/groonga/groonga-7.0.4.tar.gz
md5sum = 90c8ddf58fecdf7dba1a053adc9b9974
url = http://packages.groonga.org/source/groonga/groonga-7.0.5.tar.gz
md5sum = d42dd98c5272ed2b9ab50ded54eb818b
# temporary patch to respect more tokens in natural language mode.
patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
......
component/jupyter/buildout.cfg 0 → 100644
View file @ f581d489
[buildout]
extends =
../numpy/openblas.cfg
../matplotlib/buildout.cfg
../ipython/buildout.cfg
../python-pyzmq/buildout.cfg
../python-2.7/buildout.cfg
../scipy/buildout.cfg
../scikit-learn/buildout.cfg
../pandas/buildout.cfg
../seaborn/buildout.cfg
parts =
jupyter
jupyter-notebook-scripts
[jupyter-env]
<= numpy-env
[jupyter]
recipe = zc.recipe.egg:scripts
eggs =
# jupter base egg
jupyter
# explicit eggs to generate scripts from them
jupyter_client
jupyter_console
jupyter_core
nbconvert
nbformat
notebook
# additional eggs
requests
astor
${matplotlib:egg}
${scipy:egg}
${pandas:egg}
${scikit-learn:egg}
${seaborn:egg}
${statsmodels:egg}
patsy
# additional eggs for jupyterlab
jupyterlab
jupyterlab-launcher
interpreter = pythonjupyter
environment = jupyter-env
setup-eggs =
${numpy:egg}
${python-pyzmq:egg}
${ipython:egg}
scripts =
jupyter-kernelspec
pythonjupyter
jupyter
jupyter-trust
jupyter-nbconvert
jupyter-console
jupyter-migrate
jupyter-troubleshoot
jupyter-run
[jupyter-notebook-initialized-scripts]
recipe = zc.recipe.egg:scripts
eggs = ${jupyter:eggs}
interpreter = pythonjupyter
environment = jupyter-env
scripts =
jupyter-nbextension
jupyter-notebook
jupyter-serverextension
jupyter-lab
jupyter-labextension
jupyter-labhub
initialization =
# https://github.com/ipython/ipython/issues/5420
import os
os.environ['PYTHONPATH'] = ':'.join(sys.path)
component/libtiff/buildout.cfg
View file @ f581d489
......@@ -10,15 +10,15 @@ parts =
[libtiff]
recipe = slapos.recipe.cmmi
url = http://download.osgeo.org/libtiff/tiff-4.0.7.tar.gz
md5sum = 77ae928d2c6b7fb46a21c3a29325157b
url = http://download.osgeo.org/libtiff/tiff-4.0.8.tar.gz
md5sum = 2a7d1c1318416ddf36d5f6fa4600069b
configure-options =
--disable-static
--without-x
--disable-lzma
patch-options = -p1
patches =
${:_profile_base_location_}/tiff_4.0.7-6.debian.patch#b9249bb352e693b7975f9d343018994d
${:_profile_base_location_}/tiff_4.0.8-3.debian.patch#9f52aed13d6ae864b3fb46372fe97780
environment =
CPPFLAGS=-I${libjpeg:location}/include -I${jbigkit:location}/include -I${zlib:location}/include
LDFLAGS=-L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib -L${jbigkit:location}/lib -Wl,-rpath=${jbigkit:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
......
component/libtiff/tiff_4.0.7-6.debian.patch deleted 100644 → 0
View file @ 53366193
commit 8c101323f5789ef6a0db952d53794e9c4ba48207
Author: erouault <erouault>
Date: Fri Dec 2 21:56:56 2016 +0000
* libtiff/tif_read.c, libtiff/tiffiop.h: fix uint32 overflow in
TIFFReadEncodedStrip() that caused an integer division by zero.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2596
diff --git a/ChangeLog b/ChangeLog
index 46a5d7c..668b66a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2016-12-02 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_read.c, libtiff/tiffiop.h: fix uint32 overflow in
+ TIFFReadEncodedStrip() that caused an integer division by zero.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2596
+
2016-11-19 Bob Friesenhahn <bfriesen@simple.dallas.tx.us>
* libtiff 4.0.7 released.
diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
index 8003592..29a311d 100644
--- a/libtiff/tif_read.c
+++ b/libtiff/tif_read.c
@@ -1,4 +1,4 @@
-/* $Id: tif_read.c,v 1.49 2016-07-10 18:00:21 erouault Exp $ */
+/* $Id: tif_read.c,v 1.50 2016-12-02 21:56:56 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -346,7 +346,7 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
rowsperstrip=td->td_rowsperstrip;
if (rowsperstrip>td->td_imagelength)
rowsperstrip=td->td_imagelength;
- stripsperplane=((td->td_imagelength+rowsperstrip-1)/rowsperstrip);
+ stripsperplane= TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip);
stripinplane=(strip%stripsperplane);
plane=(uint16)(strip/stripsperplane);
rows=td->td_imagelength-stripinplane*rowsperstrip;
diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
index 8bcd0c1..5294ee7 100644
--- a/libtiff/tiffiop.h
+++ b/libtiff/tiffiop.h
@@ -1,4 +1,4 @@
-/* $Id: tiffiop.h,v 1.89 2016-01-23 21:20:34 erouault Exp $ */
+/* $Id: tiffiop.h,v 1.90 2016-12-02 21:56:56 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -250,6 +250,10 @@ struct tiff {
#define TIFFhowmany_32(x, y) (((uint32)x < (0xffffffff - (uint32)(y-1))) ? \
((((uint32)(x))+(((uint32)(y))-1))/((uint32)(y))) : \
0U)
+/* Variant of TIFFhowmany_32() that doesn't return 0 if x close to MAXUINT. */
+/* Caution: TIFFhowmany_32_maxuint_compat(x,y)*y might overflow */
+#define TIFFhowmany_32_maxuint_compat(x, y) \
+ (((uint32)(x) / (uint32)(y)) + ((((uint32)(x) % (uint32)(y)) != 0) ? 1 : 0))
#define TIFFhowmany8_32(x) (((x)&0x07)?((uint32)(x)>>3)+1:(uint32)(x)>>3)
#define TIFFroundup_32(x, y) (TIFFhowmany_32(x,y)*(y))
#define TIFFhowmany_64(x, y) ((((uint64)(x))+(((uint64)(y))-1))/((uint64)(y)))
commit b412777317cabbf8ed89ca38fb180991cca89b8c
Author: erouault <erouault>
Date: Fri Dec 2 22:13:32 2016 +0000
* tools/tiffcp.c: avoid uint32 underflow in cpDecodedStrips that
can cause various issues, such as buffer overflows in the library.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2598
diff --git a/ChangeLog b/ChangeLog
index 668b66a..0f154d6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
2016-12-02 Even Rouault <even.rouault at spatialys.com>
+ * tools/tiffcp.c: avoid uint32 underflow in cpDecodedStrips that
+ can cause various issues, such as buffer overflows in the library.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2598
+
+2016-12-02 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_read.c, libtiff/tiffiop.h: fix uint32 overflow in
TIFFReadEncodedStrip() that caused an integer division by zero.
Reported by Agostino Sarubbo.
diff --git a/tools/tiffcp.c b/tools/tiffcp.c
index 338a3d1..6dfb9a9 100644
--- a/tools/tiffcp.c
+++ b/tools/tiffcp.c
@@ -1,4 +1,4 @@
-/* $Id: tiffcp.c,v 1.55 2016-10-08 15:54:57 erouault Exp $ */
+/* $Id: tiffcp.c,v 1.56 2016-12-02 22:13:32 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -985,7 +985,7 @@ DECLAREcpFunc(cpDecodedStrips)
tstrip_t s, ns = TIFFNumberOfStrips(in);
uint32 row = 0;
_TIFFmemset(buf, 0, stripsize);
- for (s = 0; s < ns; s++) {
+ for (s = 0; s < ns && row < imagelength; s++) {
tsize_t cc = (row + rowsperstrip > imagelength) ?
TIFFVStripSize(in, imagelength - row) : stripsize;
if (TIFFReadEncodedStrip(in, s, buf, cc) < 0
commit da99990ba6e1203798a59eb836fc6433ed6e3d66
Author: erouault <erouault>
Date: Fri Dec 2 23:05:51 2016 +0000
* libtiff/tif_pixarlog.c, libtiff/tif_luv.c: fix heap-based buffer
overflow on generation of PixarLog / LUV compressed files, with
ColorMap, TransferFunction attached and nasty plays with bitspersample.
The fix for LUV has not been tested, but suffers from the same kind
of issue of PixarLog.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2604
diff --git a/ChangeLog b/ChangeLog
index 0f154d6..93c01f8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_pixarlog.c, libtiff/tif_luv.c: fix heap-based buffer
+ overflow on generation of PixarLog / LUV compressed files, with
+ ColorMap, TransferFunction attached and nasty plays with bitspersample.
+ The fix for LUV has not been tested, but suffers from the same kind
+ of issue of PixarLog.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2604
+
2016-12-02 Even Rouault <even.rouault at spatialys.com>
* tools/tiffcp.c: avoid uint32 underflow in cpDecodedStrips that
diff --git a/libtiff/tif_luv.c b/libtiff/tif_luv.c
index ca08f30..f42ac01 100644
--- a/libtiff/tif_luv.c
+++ b/libtiff/tif_luv.c
@@ -1,4 +1,4 @@
-/* $Id: tif_luv.c,v 1.43 2016-09-04 21:32:56 erouault Exp $ */
+/* $Id: tif_luv.c,v 1.44 2016-12-02 23:05:51 erouault Exp $ */
/*
* Copyright (c) 1997 Greg Ward Larson
@@ -158,6 +158,7 @@
typedef struct logLuvState LogLuvState;
struct logLuvState {
+ int encoder_state; /* 1 if encoder correctly initialized */
int user_datafmt; /* user data format */
int encode_meth; /* encoding method */
int pixel_size; /* bytes per pixel */
@@ -1552,6 +1553,7 @@ LogLuvSetupEncode(TIFF* tif)
td->td_photometric, "must be either LogLUV or LogL");
break;
}
+ sp->encoder_state = 1;
return (1);
notsupported:
TIFFErrorExt(tif->tif_clientdata, module,
@@ -1563,19 +1565,27 @@ notsupported:
static void
LogLuvClose(TIFF* tif)
{
+ LogLuvState* sp = (LogLuvState*) tif->tif_data;
TIFFDirectory *td = &tif->tif_dir;
+ assert(sp != 0);
/*
* For consistency, we always want to write out the same
* bitspersample and sampleformat for our TIFF file,
* regardless of the data format being used by the application.
* Since this routine is called after tags have been set but
* before they have been recorded in the file, we reset them here.
+ * Note: this is really a nasty approach. See PixarLogClose
*/
- td->td_samplesperpixel =
- (td->td_photometric == PHOTOMETRIC_LOGL) ? 1 : 3;
- td->td_bitspersample = 16;
- td->td_sampleformat = SAMPLEFORMAT_INT;
+ if( sp->encoder_state )
+ {
+ /* See PixarLogClose. Might avoid issues with tags whose size depends
+ * on those below, but not completely sure this is enough. */
+ td->td_samplesperpixel =
+ (td->td_photometric == PHOTOMETRIC_LOGL) ? 1 : 3;
+ td->td_bitspersample = 16;
+ td->td_sampleformat = SAMPLEFORMAT_INT;
+ }
}
static void
diff --git a/libtiff/tif_pixarlog.c b/libtiff/tif_pixarlog.c
index f4af2ba..9836dce 100644
--- a/libtiff/tif_pixarlog.c
+++ b/libtiff/tif_pixarlog.c
@@ -1,4 +1,4 @@
-/* $Id: tif_pixarlog.c,v 1.48 2016-09-23 22:12:18 erouault Exp $ */
+/* $Id: tif_pixarlog.c,v 1.49 2016-12-02 23:05:51 erouault Exp $ */
/*
* Copyright (c) 1996-1997 Sam Leffler
@@ -1233,8 +1233,10 @@ PixarLogPostEncode(TIFF* tif)
static void
PixarLogClose(TIFF* tif)
{
+ PixarLogState* sp = (PixarLogState*) tif->tif_data;
TIFFDirectory *td = &tif->tif_dir;
+ assert(sp != 0);
/* In a really sneaky (and really incorrect, and untruthful, and
* troublesome, and error-prone) maneuver that completely goes against
* the spirit of TIFF, and breaks TIFF, on close, we covertly
@@ -1243,8 +1245,19 @@ PixarLogClose(TIFF* tif)
* readers that don't know about PixarLog, or how to set
* the PIXARLOGDATFMT pseudo-tag.
*/
- td->td_bitspersample = 8;
- td->td_sampleformat = SAMPLEFORMAT_UINT;
+
+ if (sp->state&PLSTATE_INIT) {
+ /* We test the state to avoid an issue such as in
+ * http://bugzilla.maptools.org/show_bug.cgi?id=2604
+ * What appends in that case is that the bitspersample is 1 and
+ * a TransferFunction is set. The size of the TransferFunction
+ * depends on 1<<bitspersample. So if we increase it, an access
+ * out of the buffer will happen at directory flushing.
+ * Another option would be to clear those targs.
+ */
+ td->td_bitspersample = 8;
+ td->td_sampleformat = SAMPLEFORMAT_UINT;
+ }
}
static void
commit c533d200ecc45e00892a94f9bb2e762a5aa0b2ce
Author: erouault <erouault>
Date: Sat Dec 3 11:02:15 2016 +0000
* libtiff/tif_dirread.c: modify ChopUpSingleUncompressedStrip() to
instanciate compute ntrips as TIFFhowmany_32(td->td_imagelength, rowsperstrip),
instead of a logic based on the total size of data. Which is faulty is
the total size of data is not sufficient to fill the whole image, and thus
results in reading outside of the StripByCounts/StripOffsets arrays when
using TIFFReadScanline().
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2608.
* libtiff/tif_strip.c: revert the change in TIFFNumberOfStrips() done
for http://bugzilla.maptools.org/show_bug.cgi?id=2587 / CVE-2016-9273 since
the above change is a better fix that makes it unnecessary.
diff --git a/ChangeLog b/ChangeLog
index 93c01f8..9dbc7a0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,20 @@
2016-12-03 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_dirread.c: modify ChopUpSingleUncompressedStrip() to
+ instanciate compute ntrips as TIFFhowmany_32(td->td_imagelength, rowsperstrip),
+ instead of a logic based on the total size of data. Which is faulty is
+ the total size of data is not sufficient to fill the whole image, and thus
+ results in reading outside of the StripByCounts/StripOffsets arrays when
+ using TIFFReadScanline().
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2608.
+
+ * libtiff/tif_strip.c: revert the change in TIFFNumberOfStrips() done
+ for http://bugzilla.maptools.org/show_bug.cgi?id=2587 / CVE-2016-9273 since
+ the above change is a better fix that makes it unnecessary.
+
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_pixarlog.c, libtiff/tif_luv.c: fix heap-based buffer
overflow on generation of PixarLog / LUV compressed files, with
ColorMap, TransferFunction attached and nasty plays with bitspersample.
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 01070f2..f290528 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -1,4 +1,4 @@
-/* $Id: tif_dirread.c,v 1.204 2016-11-16 15:14:15 erouault Exp $ */
+/* $Id: tif_dirread.c,v 1.205 2016-12-03 11:02:15 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -5502,8 +5502,7 @@ ChopUpSingleUncompressedStrip(TIFF* tif)
uint64 rowblockbytes;
uint64 stripbytes;
uint32 strip;
- uint64 nstrips64;
- uint32 nstrips32;
+ uint32 nstrips;
uint32 rowsperstrip;
uint64* newcounts;
uint64* newoffsets;
@@ -5534,18 +5533,17 @@ ChopUpSingleUncompressedStrip(TIFF* tif)
return;
/*
- * never increase the number of strips in an image
+ * never increase the number of rows per strip
*/
if (rowsperstrip >= td->td_rowsperstrip)
return;
- nstrips64 = TIFFhowmany_64(bytecount, stripbytes);
- if ((nstrips64==0)||(nstrips64>0xFFFFFFFF)) /* something is wonky, do nothing. */
- return;
- nstrips32 = (uint32)nstrips64;
+ nstrips = TIFFhowmany_32(td->td_imagelength, rowsperstrip);
+ if( nstrips == 0 )
+ return;
- newcounts = (uint64*) _TIFFCheckMalloc(tif, nstrips32, sizeof (uint64),
+ newcounts = (uint64*) _TIFFCheckMalloc(tif, nstrips, sizeof (uint64),
"for chopped \"StripByteCounts\" array");
- newoffsets = (uint64*) _TIFFCheckMalloc(tif, nstrips32, sizeof (uint64),
+ newoffsets = (uint64*) _TIFFCheckMalloc(tif, nstrips, sizeof (uint64),
"for chopped \"StripOffsets\" array");
if (newcounts == NULL || newoffsets == NULL) {
/*
@@ -5562,18 +5560,18 @@ ChopUpSingleUncompressedStrip(TIFF* tif)
* Fill the strip information arrays with new bytecounts and offsets
* that reflect the broken-up format.
*/
- for (strip = 0; strip < nstrips32; strip++) {
+ for (strip = 0; strip < nstrips; strip++) {
if (stripbytes > bytecount)
stripbytes = bytecount;
newcounts[strip] = stripbytes;
- newoffsets[strip] = offset;
+ newoffsets[strip] = stripbytes ? offset : 0;
offset += stripbytes;
bytecount -= stripbytes;
}
/*
* Replace old single strip info with multi-strip info.
*/
- td->td_stripsperimage = td->td_nstrips = nstrips32;
+ td->td_stripsperimage = td->td_nstrips = nstrips;
TIFFSetField(tif, TIFFTAG_ROWSPERSTRIP, rowsperstrip);
_TIFFfree(td->td_stripbytecount);
diff --git a/libtiff/tif_strip.c b/libtiff/tif_strip.c
index b6098dd..6e9f2ef 100644
--- a/libtiff/tif_strip.c
+++ b/libtiff/tif_strip.c
@@ -1,4 +1,4 @@
-/* $Id: tif_strip.c,v 1.37 2016-11-09 23:00:49 erouault Exp $ */
+/* $Id: tif_strip.c,v 1.38 2016-12-03 11:02:15 erouault Exp $ */
/*
* Copyright (c) 1991-1997 Sam Leffler
@@ -63,15 +63,6 @@ TIFFNumberOfStrips(TIFF* tif)
TIFFDirectory *td = &tif->tif_dir;
uint32 nstrips;
- /* If the value was already computed and store in td_nstrips, then return it,
- since ChopUpSingleUncompressedStrip might have altered and resized the
- since the td_stripbytecount and td_stripoffset arrays to the new value
- after the initial affectation of td_nstrips = TIFFNumberOfStrips() in
- tif_dirread.c ~line 3612.
- See http://bugzilla.maptools.org/show_bug.cgi?id=2587 */
- if( td->td_nstrips )
- return td->td_nstrips;
-
nstrips = (td->td_rowsperstrip == (uint32) -1 ? 1 :
TIFFhowmany_32(td->td_imagelength, td->td_rowsperstrip));
if (td->td_planarconfig == PLANARCONFIG_SEPARATE)
commit 43576568ed4af4bd43409b7ff36939340141dfd6
Author: erouault <erouault>
Date: Sat Dec 3 11:15:18 2016 +0000
* libtiff/tif_ojpeg.c: make OJPEGDecode() early exit in case of failure in
OJPEGPreDecode(). This will avoid a divide by zero, and potential other issues.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2611
diff --git a/ChangeLog b/ChangeLog
index 9dbc7a0..5b23665 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
2016-12-03 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_ojpeg.c: make OJPEGDecode() early exit in case of failure in
+ OJPEGPreDecode(). This will avoid a divide by zero, and potential other issues.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2611
+
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_dirread.c: modify ChopUpSingleUncompressedStrip() to
instanciate compute ntrips as TIFFhowmany_32(td->td_imagelength, rowsperstrip),
instead of a logic based on the total size of data. Which is faulty is
diff --git a/libtiff/tif_ojpeg.c b/libtiff/tif_ojpeg.c
index 30a1812..93839d8 100644
--- a/libtiff/tif_ojpeg.c
+++ b/libtiff/tif_ojpeg.c
@@ -1,4 +1,4 @@
-/* $Id: tif_ojpeg.c,v 1.65 2016-09-04 21:32:56 erouault Exp $ */
+/* $Id: tif_ojpeg.c,v 1.66 2016-12-03 11:15:18 erouault Exp $ */
/* WARNING: The type of JPEG encapsulation defined by the TIFF Version 6.0
specification is now totally obsolete and deprecated for new applications and
@@ -244,6 +244,7 @@ typedef enum {
typedef struct {
TIFF* tif;
+ int decoder_ok;
#ifndef LIBJPEG_ENCAP_EXTERNAL
JMP_BUF exit_jmpbuf;
#endif
@@ -722,6 +723,7 @@ OJPEGPreDecode(TIFF* tif, uint16 s)
}
sp->write_curstrile++;
}
+ sp->decoder_ok = 1;
return(1);
}
@@ -784,8 +786,14 @@ OJPEGPreDecodeSkipScanlines(TIFF* tif)
static int
OJPEGDecode(TIFF* tif, uint8* buf, tmsize_t cc, uint16 s)
{
+ static const char module[]="OJPEGDecode";
OJPEGState* sp=(OJPEGState*)tif->tif_data;
(void)s;
+ if( !sp->decoder_ok )
+ {
+ TIFFErrorExt(tif->tif_clientdata,module,"Cannot decode: decoder not correctly initialized");
+ return 0;
+ }
if (sp->libjpeg_jpeg_query_style==0)
{
if (OJPEGDecodeRaw(tif,buf,cc)==0)
commit 4307dad7fc153baefeb6bb677060e9c5157e5d42
Author: erouault <erouault>
Date: Sat Dec 3 11:35:56 2016 +0000
* tools/tiffcrop.c: fix readContigStripsIntoBuffer() in -i (ignore) mode so
that the output buffer is correctly incremented to avoid write outside bounds.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2620
diff --git a/ChangeLog b/ChangeLog
index 5b23665..d6a416b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
2016-12-03 Even Rouault <even.rouault at spatialys.com>
+ * tools/tiffcrop.c: fix readContigStripsIntoBuffer() in -i (ignore) mode so
+ that the output buffer is correctly incremented to avoid write outside bounds.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2620
+
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_ojpeg.c: make OJPEGDecode() early exit in case of failure in
OJPEGPreDecode(). This will avoid a divide by zero, and potential other issues.
Reported by Agostino Sarubbo.
diff --git a/tools/tiffcrop.c b/tools/tiffcrop.c
index 722b132..bdcbd63 100644
--- a/tools/tiffcrop.c
+++ b/tools/tiffcrop.c
@@ -1,4 +1,4 @@
-/* $Id: tiffcrop.c,v 1.46 2016-11-18 14:58:46 erouault Exp $ */
+/* $Id: tiffcrop.c,v 1.47 2016-12-03 11:35:56 erouault Exp $ */
/* tiffcrop.c -- a port of tiffcp.c extended to include manipulations of
* the image data through additional options listed below
@@ -3698,7 +3698,7 @@ static int readContigStripsIntoBuffer (TIFF* in, uint8* buf)
(unsigned long) strip, (unsigned long)rows);
return 0;
}
- bufp += bytes_read;
+ bufp += stripsize;
}
return 1;
commit 17d56c24c10ed300233164cc51380979124d6dd8
Author: erouault <erouault>
Date: Sat Dec 3 12:19:32 2016 +0000
* tools/tiffcrop.c: add 3 extra bytes at end of strip buffer in
readSeparateStripsIntoBuffer() to avoid read outside of heap allocated buffer.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2621
diff --git a/ChangeLog b/ChangeLog
index d6a416b..50db803 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
2016-12-03 Even Rouault <even.rouault at spatialys.com>
+ * tools/tiffcrop.c: add 3 extra bytes at end of strip buffer in
+ readSeparateStripsIntoBuffer() to avoid read outside of heap allocated buffer.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2621
+
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
* tools/tiffcrop.c: fix readContigStripsIntoBuffer() in -i (ignore) mode so
that the output buffer is correctly incremented to avoid write outside bounds.
Reported by Agostino Sarubbo.
diff --git a/tools/tiffcrop.c b/tools/tiffcrop.c
index bdcbd63..9122aab 100644
--- a/tools/tiffcrop.c
+++ b/tools/tiffcrop.c
@@ -1,4 +1,4 @@
-/* $Id: tiffcrop.c,v 1.47 2016-12-03 11:35:56 erouault Exp $ */
+/* $Id: tiffcrop.c,v 1.48 2016-12-03 12:19:32 erouault Exp $ */
/* tiffcrop.c -- a port of tiffcp.c extended to include manipulations of
* the image data through additional options listed below
@@ -4815,10 +4815,17 @@ static int readSeparateStripsIntoBuffer (TIFF *in, uint8 *obuf, uint32 length,
nstrips = TIFFNumberOfStrips(in);
strips_per_sample = nstrips /spp;
+ /* Add 3 padding bytes for combineSeparateSamples32bits */
+ if( (size_t) stripsize > 0xFFFFFFFFU - 3U )
+ {
+ TIFFError("readSeparateStripsIntoBuffer", "Integer overflow when calculating buffer size.");
+ exit(-1);
+ }
+
for (s = 0; (s < spp) && (s < MAX_SAMPLES); s++)
{
srcbuffs[s] = NULL;
- buff = _TIFFmalloc(stripsize);
+ buff = _TIFFmalloc(stripsize + 3);
if (!buff)
{
TIFFError ("readSeparateStripsIntoBuffer",
@@ -4827,6 +4834,9 @@ static int readSeparateStripsIntoBuffer (TIFF *in, uint8 *obuf, uint32 length,
_TIFFfree (srcbuffs[i]);
return 0;
}
+ buff[stripsize] = 0;
+ buff[stripsize+1] = 0;
+ buff[stripsize+2] = 0;
srcbuffs[s] = buff;
}
commit fc9eedf265394eb8a5633160a8fcdb7ece072701
Author: erouault <erouault>
Date: Sat Dec 3 13:00:03 2016 +0000
* tools/tiffcrop.c: fix integer division by zero when BitsPerSample is missing.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2619
diff --git a/ChangeLog b/ChangeLog
index 50db803..2940828 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
2016-12-03 Even Rouault <even.rouault at spatialys.com>
+ * tools/tiffcrop.c: fix integer division by zero when BitsPerSample is missing.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2619
+
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
* tools/tiffcrop.c: add 3 extra bytes at end of strip buffer in
readSeparateStripsIntoBuffer() to avoid read outside of heap allocated buffer.
Reported by Agostino Sarubbo.
diff --git a/tools/tiffcrop.c b/tools/tiffcrop.c
index 9122aab..21dd087 100644
--- a/tools/tiffcrop.c
+++ b/tools/tiffcrop.c
@@ -1,4 +1,4 @@
-/* $Id: tiffcrop.c,v 1.48 2016-12-03 12:19:32 erouault Exp $ */
+/* $Id: tiffcrop.c,v 1.49 2016-12-03 13:00:04 erouault Exp $ */
/* tiffcrop.c -- a port of tiffcp.c extended to include manipulations of
* the image data through additional options listed below
@@ -1164,7 +1164,7 @@ writeBufferToSeparateStrips (TIFF* out, uint8* buf,
tdata_t obuf;
(void) TIFFGetFieldDefaulted(out, TIFFTAG_ROWSPERSTRIP, &rowsperstrip);
- (void) TIFFGetField(out, TIFFTAG_BITSPERSAMPLE, &bps);
+ (void) TIFFGetFieldDefaulted(out, TIFFTAG_BITSPERSAMPLE, &bps);
bytes_per_sample = (bps + 7) / 8;
if( width == 0 ||
(uint32)bps * (uint32)spp > TIFF_UINT32_MAX / width ||
@@ -4760,7 +4760,7 @@ static int readSeparateStripsIntoBuffer (TIFF *in, uint8 *obuf, uint32 length,
int i, bytes_per_sample, bytes_per_pixel, shift_width, result = 1;
uint32 j;
int32 bytes_read = 0;
- uint16 bps, planar;
+ uint16 bps = 0, planar;
uint32 nstrips;
uint32 strips_per_sample;
uint32 src_rowsize, dst_rowsize, rows_processed, rps;
@@ -4780,7 +4780,7 @@ static int readSeparateStripsIntoBuffer (TIFF *in, uint8 *obuf, uint32 length,
}
memset (srcbuffs, '\0', sizeof(srcbuffs));
- TIFFGetField(in, TIFFTAG_BITSPERSAMPLE, &bps);
+ TIFFGetFieldDefaulted(in, TIFFTAG_BITSPERSAMPLE, &bps);
TIFFGetFieldDefaulted(in, TIFFTAG_PLANARCONFIG, &planar);
TIFFGetFieldDefaulted(in, TIFFTAG_ROWSPERSTRIP, &rps);
if (rps > length)
commit a1d523c27dafafadf589c77e834c52661f1c78fc
Author: erouault <erouault>
Date: Sat Dec 3 14:18:48 2016 +0000
* tools/tiffinfo.c: fix null pointer dereference in -r mode when the image has
no StripByteCount tag.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2594
diff --git a/ChangeLog b/ChangeLog
index dd27c7f..e41d00c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
2016-12-03 Even Rouault <even.rouault at spatialys.com>
+ * tools/tiffinfo.c: fix null pointer dereference in -r mode when the image has
+ no StripByteCount tag.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2594
+
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
* tools/tiffcrop.c: fix integer division by zero when BitsPerSample is missing.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2619
diff --git a/tools/tiffinfo.c b/tools/tiffinfo.c
index b02c7d4..4d58055 100644
--- a/tools/tiffinfo.c
+++ b/tools/tiffinfo.c
@@ -1,4 +1,4 @@
-/* $Id: tiffinfo.c,v 1.25 2016-11-12 20:06:05 bfriesen Exp $ */
+/* $Id: tiffinfo.c,v 1.26 2016-12-03 14:18:49 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -417,7 +417,7 @@ TIFFReadRawData(TIFF* tif, int bitrev)
uint64* stripbc=NULL;
TIFFGetField(tif, TIFFTAG_STRIPBYTECOUNTS, &stripbc);
- if (nstrips > 0) {
+ if (stripbc != NULL && nstrips > 0) {
uint32 bufsize = (uint32) stripbc[0];
tdata_t buf = _TIFFmalloc(bufsize);
tstrip_t s;
commit 6d3ef98b2415b2edfa36a5ba600d5a824c094309
Author: erouault <erouault>
Date: Sat Dec 3 14:42:40 2016 +0000
* tools/tiffcp.c: avoid potential division by zero if BitsPerSamples tag is
missing.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2597
diff --git a/ChangeLog b/ChangeLog
index e41d00c..0d7b12d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
2016-12-03 Even Rouault <even.rouault at spatialys.com>
+ * tools/tiffcp.c: avoid potential division by zero if BitsPerSamples tag is
+ missing.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2597
+
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
* tools/tiffinfo.c: fix null pointer dereference in -r mode when the image has
no StripByteCount tag.
Reported by Agostino Sarubbo.
diff --git a/tools/tiffcp.c b/tools/tiffcp.c
index 6dfb9a9..c8e48c3 100644
--- a/tools/tiffcp.c
+++ b/tools/tiffcp.c
@@ -1,4 +1,4 @@
-/* $Id: tiffcp.c,v 1.56 2016-12-02 22:13:32 erouault Exp $ */
+/* $Id: tiffcp.c,v 1.57 2016-12-03 14:42:40 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -1378,7 +1378,7 @@ DECLAREreadFunc(readSeparateTilesIntoBuffer)
uint8* bufp = (uint8*) buf;
uint32 tw, tl;
uint32 row;
- uint16 bps, bytes_per_sample;
+ uint16 bps = 0, bytes_per_sample;
tilebuf = _TIFFmalloc(tilesize);
if (tilebuf == 0)
@@ -1387,6 +1387,12 @@ DECLAREreadFunc(readSeparateTilesIntoBuffer)
(void) TIFFGetField(in, TIFFTAG_TILEWIDTH, &tw);
(void) TIFFGetField(in, TIFFTAG_TILELENGTH, &tl);
(void) TIFFGetField(in, TIFFTAG_BITSPERSAMPLE, &bps);
+ if( bps == 0 )
+ {
+ TIFFError(TIFFFileName(in), "Error, cannot read BitsPerSample");
+ status = 0;
+ goto done;
+ }
assert( bps % 8 == 0 );
bytes_per_sample = bps/8;
commit 9991b31a7c651e7b87a3ccd73b3dc5c67dcfdd60
Author: erouault <erouault>
Date: Sat Dec 3 15:30:31 2016 +0000
* tools/tif_dir.c: when TIFFGetField(, TIFFTAG_NUMBEROFINKS, ) is called,
limit the return number of inks to SamplesPerPixel, so that code that parses
ink names doesn't go past the end of the buffer.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2599
Reported by Agostino Sarubbo.
diff --git a/ChangeLog b/ChangeLog
index 0d7b12d..fb9fc0e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2016-12-03 Even Rouault <even.rouault at spatialys.com>
+ * tools/tif_dir.c: when TIFFGetField(, TIFFTAG_NUMBEROFINKS, ) is called,
+ limit the return number of inks to SamplesPerPixel, so that code that parses
+ ink names doesn't go past the end of the buffer.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2599
+
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
* tools/tiffcp.c: avoid potential division by zero if BitsPerSamples tag is
missing.
Reported by Agostino Sarubbo.
diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c
index ad21655..2574e74 100644
--- a/libtiff/tif_dir.c
+++ b/libtiff/tif_dir.c
@@ -1,4 +1,4 @@
-/* $Id: tif_dir.c,v 1.127 2016-10-25 21:35:15 erouault Exp $ */
+/* $Id: tif_dir.c,v 1.128 2016-12-03 15:30:31 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -854,6 +854,32 @@ _TIFFVGetField(TIFF* tif, uint32 tag, va_list ap)
if( fip == NULL ) /* cannot happen since TIFFGetField() already checks it */
return 0;
+ if( tag == TIFFTAG_NUMBEROFINKS )
+ {
+ int i;
+ for (i = 0; i < td->td_customValueCount; i++) {
+ uint16 val;
+ TIFFTagValue *tv = td->td_customValues + i;
+ if (tv->info->field_tag != tag)
+ continue;
+ val = *(uint16 *)tv->value;
+ /* Truncate to SamplesPerPixel, since the */
+ /* setting code for INKNAMES assume that there are SamplesPerPixel */
+ /* inknames. */
+ /* Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2599 */
+ if( val > td->td_samplesperpixel )
+ {
+ TIFFWarningExt(tif->tif_clientdata,"_TIFFVGetField",
+ "Truncating NumberOfInks from %u to %u",
+ val, td->td_samplesperpixel);
+ val = td->td_samplesperpixel;
+ }
+ *va_arg(ap, uint16*) = val;
+ return 1;
+ }
+ return 0;
+ }
+
/*
* We want to force the custom code to be used for custom
* fields even if the tag happens to match a well known
commit 307a31765cb01245e3655ce168385dd7d51e14bd
Author: erouault <erouault>
Date: Sat Dec 3 15:44:15 2016 +0000
* tools/tiffcp.c: avoid potential division by zero if BitsPerSamples tag is
missing.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2607
diff --git a/ChangeLog b/ChangeLog
index ac2d922..94be038 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
2016-12-03 Even Rouault <even.rouault at spatialys.com>
+ * tools/tiffcp.c: avoid potential division by zero if BitsPerSamples tag is
+ missing.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2607
+
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
* tools/tif_dir.c: when TIFFGetField(, TIFFTAG_NUMBEROFINKS, ) is called,
limit the return number of inks to SamplesPerPixel, so that code that parses
ink names doesn't go past the end of the buffer.
diff --git a/tools/tiffcp.c b/tools/tiffcp.c
index c8e48c3..142cbb0 100644
--- a/tools/tiffcp.c
+++ b/tools/tiffcp.c
@@ -1,4 +1,4 @@
-/* $Id: tiffcp.c,v 1.57 2016-12-03 14:42:40 erouault Exp $ */
+/* $Id: tiffcp.c,v 1.58 2016-12-03 15:44:15 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -1569,7 +1569,7 @@ DECLAREwriteFunc(writeBufferToSeparateTiles)
uint8* bufp = (uint8*) buf;
uint32 tl, tw;
uint32 row;
- uint16 bps, bytes_per_sample;
+ uint16 bps = 0, bytes_per_sample;
obuf = _TIFFmalloc(TIFFTileSize(out));
if (obuf == NULL)
@@ -1578,6 +1578,12 @@ DECLAREwriteFunc(writeBufferToSeparateTiles)
(void) TIFFGetField(out, TIFFTAG_TILELENGTH, &tl);
(void) TIFFGetField(out, TIFFTAG_TILEWIDTH, &tw);
(void) TIFFGetField(out, TIFFTAG_BITSPERSAMPLE, &bps);
+ if( bps == 0 )
+ {
+ TIFFError(TIFFFileName(out), "Error, cannot read BitsPerSample");
+ _TIFFfree(obuf);
+ return 0;
+ }
assert( bps % 8 == 0 );
bytes_per_sample = bps/8;
commit b34209fede77aa203cd5adcd9638ccc70731a50f
Author: erouault <erouault>
Date: Sat Dec 3 16:40:01 2016 +0000
* tools/tiffcp.c: fix uint32 underflow/overflow that can cause heap-based
buffer overflow.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2610
diff --git a/ChangeLog b/ChangeLog
index 94be038..8ee76c0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
2016-12-03 Even Rouault <even.rouault at spatialys.com>
+ * tools/tiffcp.c: fix uint32 underflow/overflow that can cause heap-based
+ buffer overflow.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2610
+
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
* tools/tiffcp.c: avoid potential division by zero if BitsPerSamples tag is
missing.
Reported by Agostino Sarubbo.
diff --git a/tools/tiffcp.c b/tools/tiffcp.c
index 142cbb0..6d96bb8 100644
--- a/tools/tiffcp.c
+++ b/tools/tiffcp.c
@@ -1,4 +1,4 @@
-/* $Id: tiffcp.c,v 1.58 2016-12-03 15:44:15 erouault Exp $ */
+/* $Id: tiffcp.c,v 1.59 2016-12-03 16:40:01 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -1163,7 +1163,7 @@ bad:
static void
cpStripToTile(uint8* out, uint8* in,
- uint32 rows, uint32 cols, int outskew, int inskew)
+ uint32 rows, uint32 cols, int outskew, int64 inskew)
{
while (rows-- > 0) {
uint32 j = cols;
@@ -1320,7 +1320,7 @@ DECLAREreadFunc(readContigTilesIntoBuffer)
tdata_t tilebuf;
uint32 imagew = TIFFScanlineSize(in);
uint32 tilew = TIFFTileRowSize(in);
- int iskew = imagew - tilew;
+ int64 iskew = (int64)imagew - (int64)tilew;
uint8* bufp = (uint8*) buf;
uint32 tw, tl;
uint32 row;
@@ -1348,7 +1348,7 @@ DECLAREreadFunc(readContigTilesIntoBuffer)
status = 0;
goto done;
}
- if (colb + tilew > imagew) {
+ if (colb > iskew) {
uint32 width = imagew - colb;
uint32 oskew = tilew - width;
cpStripToTile(bufp + colb,
commit 18bca4cf3057681689efb502175cbe5f01cb68c3
Author: erouault <erouault>
Date: Sat Dec 3 16:50:02 2016 +0000
* tools/tiffcp.c: replace assert( (bps % 8) == 0 ) by a non assert check.
Reported by Agostino Sarubbo.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2605
diff --git a/ChangeLog b/ChangeLog
index 8ee76c0..025eb72 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
2016-12-03 Even Rouault <even.rouault at spatialys.com>
+ * tools/tiffcp.c: replace assert( (bps % 8) == 0 ) by a non assert check.
+ Reported by Agostino Sarubbo.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2605
+
+2016-12-03 Even Rouault <even.rouault at spatialys.com>
+
* tools/tiffcp.c: fix uint32 underflow/overflow that can cause heap-based
buffer overflow.
Reported by Agostino Sarubbo.
diff --git a/tools/tiffcp.c b/tools/tiffcp.c
index 6d96bb8..49c9d37 100644
--- a/tools/tiffcp.c
+++ b/tools/tiffcp.c
@@ -1,4 +1,4 @@
-/* $Id: tiffcp.c,v 1.59 2016-12-03 16:40:01 erouault Exp $ */
+/* $Id: tiffcp.c,v 1.60 2016-12-03 16:50:02 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -45,7 +45,6 @@
#include <string.h>
#include <ctype.h>
-#include <assert.h>
#ifdef HAVE_UNISTD_H
# include <unistd.h>
@@ -1393,7 +1392,12 @@ DECLAREreadFunc(readSeparateTilesIntoBuffer)
status = 0;
goto done;
}
- assert( bps % 8 == 0 );
+ if( (bps % 8) != 0 )
+ {
+ TIFFError(TIFFFileName(in), "Error, cannot handle BitsPerSample that is not a multiple of 8");
+ status = 0;
+ goto done;
+ }
bytes_per_sample = bps/8;
for (row = 0; row < imagelength; row += tl) {
@@ -1584,7 +1588,12 @@ DECLAREwriteFunc(writeBufferToSeparateTiles)
_TIFFfree(obuf);
return 0;
}
- assert( bps % 8 == 0 );
+ if( (bps % 8) != 0 )
+ {
+ TIFFError(TIFFFileName(out), "Error, cannot handle BitsPerSample that is not a multiple of 8");
+ _TIFFfree(obuf);
+ return 0;
+ }
bytes_per_sample = bps/8;
for (row = 0; row < imagelength; row += tl) {
commit 8c9dbee088d8b43cdae47b9c5f711058bd1f17f1
Author: erouault <erouault>
Date: Tue Dec 13 18:15:48 2016 +0000
* libtiff/tif_fax3.h: revert change done on 2016-01-09 that made
Param member of TIFFFaxTabEnt structure a uint16 to reduce size of
the binary. It happens that the Hylafax software uses the tables that
follow this typedef (TIFFFaxMainTable, TIFFFaxWhiteTable,
TIFFFaxBlackTable), also they are not in a public libtiff header.
Raised by Lee Howard.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2636
diff --git a/ChangeLog b/ChangeLog
index ee6fd802..558db20e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2016-12-13 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_fax3.h: revert change done on 2016-01-09 that made
+ Param member of TIFFFaxTabEnt structure a uint16 to reduce size of
+ the binary. It happens that the Hylafax software uses the tables that
+ follow this typedef (TIFFFaxMainTable, TIFFFaxWhiteTable,
+ TIFFFaxBlackTable), also they are not in a public libtiff header.
+ Raised by Lee Howard.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2636
+
2016-12-03 Even Rouault <even.rouault at spatialys.com>
* tools/tiffcp.c: replace assert( (bps % 8) == 0 ) by a non assert check.
diff --git a/libtiff/tif_fax3.h b/libtiff/tif_fax3.h
index e0b2ca6b..45ce43f1 100644
--- a/libtiff/tif_fax3.h
+++ b/libtiff/tif_fax3.h
@@ -1,4 +1,4 @@
-/* $Id: tif_fax3.h,v 1.11 2016-01-23 21:20:34 erouault Exp $ */
+/* $Id: tif_fax3.h,v 1.12 2016-12-13 18:15:48 erouault Exp $ */
/*
* Copyright (c) 1990-1997 Sam Leffler
@@ -81,10 +81,12 @@ extern void _TIFFFax3fillruns(unsigned char*, uint32*, uint32*, uint32);
#define S_MakeUp 11
#define S_EOL 12
+/* WARNING: do not change the layout of this structure as the Halyfax software */
+/* really depends on it. See http://bugzilla.maptools.org/show_bug.cgi?id=2636 */
typedef struct { /* state table entry */
unsigned char State; /* see above */
unsigned char Width; /* width of code in bits */
- uint16 Param; /* unsigned 16-bit run length in bits */
+ uint32 Param; /* unsigned 32-bit run length in bits (holds on 16 bit actually, but cannot be changed. See above warning) */
} TIFFFaxTabEnt;
extern const TIFFFaxTabEnt TIFFFaxMainTable[];
From c7153361a4041260719b340f73f2f76b0969235c Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Tue, 20 Dec 2016 17:28:17 +0000
Subject: [PATCH] * tools/tiff2pdf.c: avoid potential heap-based overflow in
t2p_readwrite_pdf_image_tile(). Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2640
---
ChangeLog | 6 ++++++
tools/tiff2pdf.c | 2 +-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index 6be3602..91ba4e6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2016-12-20 Even Rouault <even.rouault at spatialys.com>
+
+ * tools/tiff2pdf.c: avoid potential heap-based overflow in
+ t2p_readwrite_pdf_image_tile().
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2640
+
2016-12-13 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_fax3.h: revert change done on 2016-01-09 that made
diff --git a/tools/tiff2pdf.c b/tools/tiff2pdf.c
index 47d7629..db196e0 100644
--- a/tools/tiff2pdf.c
+++ b/tools/tiff2pdf.c
@@ -2895,7 +2895,7 @@ tsize_t t2p_readwrite_pdf_image_tile(T2P
return(0);
}
if(TIFFGetField(input, TIFFTAG_JPEGTABLES, &count, &jpt) != 0) {
- if (count >= 4) {
+ if (count > 4) {
/* Ignore EOI marker of JpegTables */
_TIFFmemcpy(buffer, jpt, count - 2);
bufferoffset += count - 2;
From 5c080298d59efa53264d7248bbe3a04660db6ef7 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 19:25:44 +0000
Subject: [PATCH] * tools/tiffcp.c: error out cleanly in cpContig2SeparateByRow
and cpSeparate2ContigByRow if BitsPerSample != 8 to avoid heap based
overflow. Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2656 and
http://bugzilla.maptools.org/show_bug.cgi?id=2657
---
ChangeLog | 7 +++++++
tools/tiffcp.c | 24 ++++++++++++++++++++++--
2 files changed, 29 insertions(+), 2 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index f78cad0..064f25b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
+ * tools/tiffcp.c: error out cleanly in cpContig2SeparateByRow and
+ cpSeparate2ContigByRow if BitsPerSample != 8 to avoid heap based overflow.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2656 and
+ http://bugzilla.maptools.org/show_bug.cgi?id=2657
+
2016-12-20 Even Rouault <even.rouault at spatialys.com>
* tools/tiff2pdf.c: avoid potential heap-based overflow in
diff --git a/tools/tiffcp.c b/tools/tiffcp.c
index bdf754c..8bbcd52 100644
--- a/tools/tiffcp.c
+++ b/tools/tiffcp.c
@@ -591,7 +591,7 @@ static copyFunc pickCopyFunc(TIFF*, TIFF*, uint16, uint16);
static int
tiffcp(TIFF* in, TIFF* out)
{
- uint16 bitspersample, samplesperpixel = 1;
+ uint16 bitspersample = 1, samplesperpixel = 1;
uint16 input_compression, input_photometric = PHOTOMETRIC_MINISBLACK;
copyFunc cf;
uint32 width, length;
@@ -1067,6 +1067,16 @@ DECLAREcpFunc(cpContig2SeparateByRow)
register uint32 n;
uint32 row;
tsample_t s;
+ uint16 bps = 0;
+
+ (void) TIFFGetField(in, TIFFTAG_BITSPERSAMPLE, &bps);
+ if( bps != 8 )
+ {
+ TIFFError(TIFFFileName(in),
+ "Error, can only handle BitsPerSample=8 in %s",
+ "cpContig2SeparateByRow");
+ return 0;
+ }
inbuf = _TIFFmalloc(scanlinesizein);
outbuf = _TIFFmalloc(scanlinesizeout);
@@ -1120,6 +1130,16 @@ DECLAREcpFunc(cpSeparate2ContigByRow)
register uint32 n;
uint32 row;
tsample_t s;
+ uint16 bps = 0;
+
+ (void) TIFFGetField(in, TIFFTAG_BITSPERSAMPLE, &bps);
+ if( bps != 8 )
+ {
+ TIFFError(TIFFFileName(in),
+ "Error, can only handle BitsPerSample=8 in %s",
+ "cpSeparate2ContigByRow");
+ return 0;
+ }
inbuf = _TIFFmalloc(scanlinesizein);
outbuf = _TIFFmalloc(scanlinesizeout);
@@ -1784,7 +1804,7 @@ pickCopyFunc(TIFF* in, TIFF* out, uint16 bitspersample, uint16 samplesperpixel)
uint32 w, l, tw, tl;
int bychunk;
- (void) TIFFGetField(in, TIFFTAG_PLANARCONFIG, &shortv);
+ (void) TIFFGetFieldDefaulted(in, TIFFTAG_PLANARCONFIG, &shortv);
if (shortv != config && bitspersample != 8 && samplesperpixel > 1) {
fprintf(stderr,
"%s: Cannot handle different planar configuration w/ bits/sample != 8\n",
From 47f2fb61a3a64667bce1a8398a8fcb1b348ff122 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 12:15:01 +0000
Subject: [PATCH] * libtiff/tif_jpeg.c: avoid integer division by zero in
JPEGSetupEncode() when horizontal or vertical sampling is set to 0. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2653
---
ChangeLog | 6 ++++++
libtiff/tif_jpeg.c | 7 +++++++
2 files changed, 13 insertions(+)
diff --git a/ChangeLog b/ChangeLog
index c82bc76..a7208f5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_jpeg.c: avoid integer division by zero in
+ JPEGSetupEncode() when horizontal or vertical sampling is set to 0.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2653
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* tools/tiffcp.c: error out cleanly in cpContig2SeparateByRow and
cpSeparate2ContigByRow if BitsPerSample != 8 to avoid heap based overflow.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2656 and
diff --git a/libtiff/tif_jpeg.c b/libtiff/tif_jpeg.c
index 38595f9..6c17c38 100644
--- a/libtiff/tif_jpeg.c
+++ b/libtiff/tif_jpeg.c
@@ -1626,6 +1626,13 @@ JPEGSetupEncode(TIFF* tif)
case PHOTOMETRIC_YCBCR:
sp->h_sampling = td->td_ycbcrsubsampling[0];
sp->v_sampling = td->td_ycbcrsubsampling[1];
+ if( sp->h_sampling == 0 || sp->v_sampling == 0 )
+ {
+ TIFFErrorExt(tif->tif_clientdata, module,
+ "Invalig horizontal/vertical sampling value");
+ return (0);
+ }
+
/*
* A ReferenceBlackWhite field *must* be present since the
* default value is inappropriate for YCbCr. Fill in the
From 3cfd62d77c2a7e147a05bd678524c345fa9c2bb8 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 13:28:01 +0000
Subject: [PATCH] * libtiff/tif_dirread.c: avoid division by floating point 0
in TIFFReadDirEntryCheckedRational() and TIFFReadDirEntryCheckedSrational(),
and return 0 in that case (instead of infinity as before presumably)
Apparently some sanitizers do not like those divisions by zero. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2644
---
ChangeLog | 8 ++++++++
libtiff/tif_dirread.c | 10 ++++++++--
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 6a752cd..722a405 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_dirread.c: avoid division by floating point 0 in
+ TIFFReadDirEntryCheckedRational() and TIFFReadDirEntryCheckedSrational(),
+ and return 0 in that case (instead of infinity as before presumably)
+ Apparently some sanitizers do not like those divisions by zero.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2644
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_jpeg.c: avoid integer division by zero in
JPEGSetupEncode() when horizontal or vertical sampling is set to 0.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2653
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 570d0c3..8a1e42a 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -2872,7 +2872,10 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryCheckedRational(TIFF* tif, TIFFD
m.l = direntry->tdir_offset.toff_long8;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabArrayOfLong(m.i,2);
- if (m.i[0]==0)
+ /* Not completely sure what we should do when m.i[1]==0, but some */
+ /* sanitizers do not like division by 0.0: */
+ /* http://bugzilla.maptools.org/show_bug.cgi?id=2644 */
+ if (m.i[0]==0 || m.i[1]==0)
*value=0.0;
else
*value=(double)m.i[0]/(double)m.i[1];
@@ -2900,7 +2903,10 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryCheckedSrational(TIFF* tif, TIFF
m.l=direntry->tdir_offset.toff_long8;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabArrayOfLong(m.i,2);
- if ((int32)m.i[0]==0)
+ /* Not completely sure what we should do when m.i[1]==0, but some */
+ /* sanitizers do not like division by 0.0: */
+ /* http://bugzilla.maptools.org/show_bug.cgi?id=2644 */
+ if ((int32)m.i[0]==0 || m.i[1]==0)
*value=0.0;
else
*value=(double)((int32)m.i[0])/(double)m.i[1];
From 3144e57770c1e4d26520d8abee750f8ac8b75490 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 16:09:02 +0000
Subject: [PATCH] * libtiff/tif_dir.c, tif_dirread.c, tif_dirwrite.c: implement
various clampings of double to other data types to avoid undefined behaviour
if the output range isn't big enough to hold the input value. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2643
http://bugzilla.maptools.org/show_bug.cgi?id=2642
http://bugzilla.maptools.org/show_bug.cgi?id=2646
http://bugzilla.maptools.org/show_bug.cgi?id=2647
---
ChangeLog | 10 ++++++
libtiff/tif_dir.c | 18 +++++++---
libtiff/tif_dirread.c | 10 +++++-
libtiff/tif_dirwrite.c | 90 ++++++++++++++++++++++++++++++++++++++++++++------
4 files changed, 113 insertions(+), 15 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 722a405..6517640 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,15 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_dir.c, tif_dirread.c, tif_dirwrite.c: implement various clampings
+ of double to other data types to avoid undefined behaviour if the output range
+ isn't big enough to hold the input value.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2643
+ http://bugzilla.maptools.org/show_bug.cgi?id=2642
+ http://bugzilla.maptools.org/show_bug.cgi?id=2646
+ http://bugzilla.maptools.org/show_bug.cgi?id=2647
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_dirread.c: avoid division by floating point 0 in
TIFFReadDirEntryCheckedRational() and TIFFReadDirEntryCheckedSrational(),
and return 0 in that case (instead of infinity as before presumably)
diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c
index 68a55af..a04d28f 100644
--- a/libtiff/tif_dir.c
+++ b/libtiff/tif_dir.c
@@ -31,6 +31,7 @@
* (and also some miscellaneous stuff)
*/
#include "tiffiop.h"
+#include <float.h>
/*
* These are used in the backwards compatibility code...
@@ -154,6 +155,15 @@ checkInkNamesString(TIFF* tif, uint32 slen, const char* s)
return (0);
}
+static float TIFFClampDoubleToFloat( double val )
+{
+ if( val > FLT_MAX )
+ return FLT_MAX;
+ if( val < -FLT_MAX )
+ return -FLT_MAX;
+ return (float)val;
+}
+
static int
_TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
{
@@ -312,13 +322,13 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
dblval = va_arg(ap, double);
if( dblval < 0 )
goto badvaluedouble;
- td->td_xresolution = (float) dblval;
+ td->td_xresolution = TIFFClampDoubleToFloat( dblval );
break;
case TIFFTAG_YRESOLUTION:
dblval = va_arg(ap, double);
if( dblval < 0 )
goto badvaluedouble;
- td->td_yresolution = (float) dblval;
+ td->td_yresolution = TIFFClampDoubleToFloat( dblval );
break;
case TIFFTAG_PLANARCONFIG:
v = (uint16) va_arg(ap, uint16_vap);
@@ -327,10 +337,10 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
td->td_planarconfig = (uint16) v;
break;
case TIFFTAG_XPOSITION:
- td->td_xposition = (float) va_arg(ap, double);
+ td->td_xposition = TIFFClampDoubleToFloat( va_arg(ap, double) );
break;
case TIFFTAG_YPOSITION:
- td->td_yposition = (float) va_arg(ap, double);
+ td->td_yposition = TIFFClampDoubleToFloat( va_arg(ap, double) );
break;
case TIFFTAG_RESOLUTIONUNIT:
v = (uint16) va_arg(ap, uint16_vap);
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 8a1e42a..77b0f37 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -40,6 +40,7 @@
*/
#include "tiffiop.h"
+#include <float.h>
#define IGNORE 0 /* tag placeholder used below */
#define FAILED_FII ((uint32) -1)
@@ -2406,7 +2407,14 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryFloatArray(TIFF* tif, TIFFDirEnt
ma=(double*)origdata;
mb=data;
for (n=0; n<count; n++)
- *mb++=(float)(*ma++);
+ {
+ double val = *ma++;
+ if( val > FLT_MAX )
+ val = FLT_MAX;
+ else if( val < -FLT_MAX )
+ val = -FLT_MAX;
+ *mb++=(float)val;
+ }
}
break;
}
diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
index c9e871b..2967da5 100644
--- a/libtiff/tif_dirwrite.c
+++ b/libtiff/tif_dirwrite.c
@@ -30,6 +30,7 @@
* Directory Write Support Routines.
*/
#include "tiffiop.h"
+#include <float.h>
#ifdef HAVE_IEEEFP
#define TIFFCvtNativeToIEEEFloat(tif, n, fp)
@@ -939,6 +940,69 @@ TIFFWriteDirectorySec(TIFF* tif, int isimage, int imagedone, uint64* pdiroff)
return(0);
}
+static float TIFFClampDoubleToFloat( double val )
+{
+ if( val > FLT_MAX )
+ return FLT_MAX;
+ if( val < -FLT_MAX )
+ return -FLT_MAX;
+ return (float)val;
+}
+
+static int8 TIFFClampDoubleToInt8( double val )
+{
+ if( val > 127 )
+ return 127;
+ if( val < -128 || val != val )
+ return -128;
+ return (int8)val;
+}
+
+static int16 TIFFClampDoubleToInt16( double val )
+{
+ if( val > 32767 )
+ return 32767;
+ if( val < -32768 || val != val )
+ return -32768;
+ return (int16)val;
+}
+
+static int32 TIFFClampDoubleToInt32( double val )
+{
+ if( val > 0x7FFFFFFF )
+ return 0x7FFFFFFF;
+ if( val < -0x7FFFFFFF-1 || val != val )
+ return -0x7FFFFFFF-1;
+ return (int32)val;
+}
+
+static uint8 TIFFClampDoubleToUInt8( double val )
+{
+ if( val < 0 )
+ return 0;
+ if( val > 255 || val != val )
+ return 255;
+ return (uint8)val;
+}
+
+static uint16 TIFFClampDoubleToUInt16( double val )
+{
+ if( val < 0 )
+ return 0;
+ if( val > 65535 || val != val )
+ return 65535;
+ return (uint16)val;
+}
+
+static uint32 TIFFClampDoubleToUInt32( double val )
+{
+ if( val < 0 )
+ return 0;
+ if( val > 0xFFFFFFFFU || val != val )
+ return 0xFFFFFFFFU;
+ return (uint32)val;
+}
+
static int
TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, uint16 tag, uint32 count, double* value)
{
@@ -959,7 +1023,7 @@ TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* di
if (tif->tif_dir.td_bitspersample<=32)
{
for (i = 0; i < count; ++i)
- ((float*)conv)[i] = (float)value[i];
+ ((float*)conv)[i] = TIFFClampDoubleToFloat(value[i]);
ok = TIFFWriteDirectoryTagFloatArray(tif,ndir,dir,tag,count,(float*)conv);
}
else
@@ -971,19 +1035,19 @@ TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* di
if (tif->tif_dir.td_bitspersample<=8)
{
for (i = 0; i < count; ++i)
- ((int8*)conv)[i] = (int8)value[i];
+ ((int8*)conv)[i] = TIFFClampDoubleToInt8(value[i]);
ok = TIFFWriteDirectoryTagSbyteArray(tif,ndir,dir,tag,count,(int8*)conv);
}
else if (tif->tif_dir.td_bitspersample<=16)
{
for (i = 0; i < count; ++i)
- ((int16*)conv)[i] = (int16)value[i];
+ ((int16*)conv)[i] = TIFFClampDoubleToInt16(value[i]);
ok = TIFFWriteDirectoryTagSshortArray(tif,ndir,dir,tag,count,(int16*)conv);
}
else
{
for (i = 0; i < count; ++i)
- ((int32*)conv)[i] = (int32)value[i];
+ ((int32*)conv)[i] = TIFFClampDoubleToInt32(value[i]);
ok = TIFFWriteDirectoryTagSlongArray(tif,ndir,dir,tag,count,(int32*)conv);
}
break;
@@ -991,19 +1055,19 @@ TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* di
if (tif->tif_dir.td_bitspersample<=8)
{
for (i = 0; i < count; ++i)
- ((uint8*)conv)[i] = (uint8)value[i];
+ ((uint8*)conv)[i] = TIFFClampDoubleToUInt8(value[i]);
ok = TIFFWriteDirectoryTagByteArray(tif,ndir,dir,tag,count,(uint8*)conv);
}
else if (tif->tif_dir.td_bitspersample<=16)
{
for (i = 0; i < count; ++i)
- ((uint16*)conv)[i] = (uint16)value[i];
+ ((uint16*)conv)[i] = TIFFClampDoubleToUInt16(value[i]);
ok = TIFFWriteDirectoryTagShortArray(tif,ndir,dir,tag,count,(uint16*)conv);
}
else
{
for (i = 0; i < count; ++i)
- ((uint32*)conv)[i] = (uint32)value[i];
+ ((uint32*)conv)[i] = TIFFClampDoubleToUInt32(value[i]);
ok = TIFFWriteDirectoryTagLongArray(tif,ndir,dir,tag,count,(uint32*)conv);
}
break;
@@ -2094,6 +2094,7 @@ TIFFWriteDirectoryTagCheckedSlong8Array(
static int
TIFFWriteDirectoryTagCheckedRational(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, uint16 tag, double value)
{
+ static const char module[] = "TIFFWriteDirectoryTagCheckedRational";
uint32 m[2];
assert(value>=0.0);
assert(sizeof(uint32)==4);
@@ -2102,7 +2102,12 @@ TIFFWriteDirectoryTagCheckedRational(TIF
m[0]=0;
m[1]=1;
}
- else if (value==(double)(uint32)value)
+ else if( value != value )
+ {
+ TIFFErrorExt(tif->tif_clientdata,module,"Not-a-number value is illegal");
+ return 0;
+ }
+ else if (value <= 0xFFFFFFFFU && value==(double)(uint32)value)
{
m[0]=(uint32)value;
m[1]=1;
@@ -2143,12 +2212,13 @@ TIFFWriteDirectoryTagCheckedRationalArray(TIFF* tif, uint32* ndir, TIFFDirEntry*
}
for (na=value, nb=m, nc=0; nc<count; na++, nb+=2, nc++)
{
- if (*na<=0.0)
+ if (*na<=0.0 || *na != *na)
{
nb[0]=0;
nb[1]=1;
}
- else if (*na==(float)(uint32)(*na))
+ else if (*na >= 0 && *na <= (float)0xFFFFFFFFU &&
+ *na==(float)(uint32)(*na))
{
nb[0]=(uint32)(*na);
nb[1]=1;
From 0a76a8c765c7b8327c59646284fa78c3c27e5490 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 16:13:50 +0000
Subject: [PATCH] * libtiff/tif_jpeg.c: validate BitsPerSample in
JPEGSetupEncode() to avoid undefined behaviour caused by invalid shift
exponent. Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2648
---
ChangeLog | 6 ++++++
libtiff/tif_jpeg.c | 7 +++++++
2 files changed, 13 insertions(+)
diff --git a/ChangeLog b/ChangeLog
index 6517640..8e202a2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_jpeg.c: validate BitsPerSample in JPEGSetupEncode() to avoid
+ undefined behaviour caused by invalid shift exponent.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2648
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_dir.c, tif_dirread.c, tif_dirwrite.c: implement various clampings
of double to other data types to avoid undefined behaviour if the output range
isn't big enough to hold the input value.
diff --git a/libtiff/tif_jpeg.c b/libtiff/tif_jpeg.c
index 6c17c38..192989a 100644
--- a/libtiff/tif_jpeg.c
+++ b/libtiff/tif_jpeg.c
@@ -1632,6 +1632,13 @@ JPEGSetupEncode(TIFF* tif)
"Invalig horizontal/vertical sampling value");
return (0);
}
+ if( td->td_bitspersample > 16 )
+ {
+ TIFFErrorExt(tif->tif_clientdata, module,
+ "BitsPerSample %d not allowed for JPEG",
+ td->td_bitspersample);
+ return (0);
+ }
/*
* A ReferenceBlackWhite field *must* be present since the
From 66e7bd59520996740e4df5495a830b42fae48bc4 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 16:33:34 +0000
Subject: [PATCH] * libtiff/tif_read.c: avoid potential undefined behaviour on
signed integer addition in TIFFReadRawStrip1() in isMapped() case. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2650
---
ChangeLog | 6 ++++++
libtiff/tif_read.c | 27 ++++++++++++++++++---------
2 files changed, 24 insertions(+), 9 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 8e202a2..3e31464 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_read.c: avoid potential undefined behaviour on signed integer
+ addition in TIFFReadRawStrip1() in isMapped() case.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2650
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_jpeg.c: validate BitsPerSample in JPEGSetupEncode() to avoid
undefined behaviour caused by invalid shift exponent.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2648
diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
index 52bbf50..b7aacbd 100644
--- a/libtiff/tif_read.c
+++ b/libtiff/tif_read.c
@@ -420,16 +420,25 @@ TIFFReadRawStrip1(TIFF* tif, uint32 strip, void* buf, tmsize_t size,
return ((tmsize_t)(-1));
}
} else {
- tmsize_t ma,mb;
+ tmsize_t ma;
tmsize_t n;
- ma=(tmsize_t)td->td_stripoffset[strip];
- mb=ma+size;
- if ((td->td_stripoffset[strip] > (uint64)TIFF_TMSIZE_T_MAX)||(ma>tif->tif_size))
- n=0;
- else if ((mb<ma)||(mb<size)||(mb>tif->tif_size))
- n=tif->tif_size-ma;
- else
- n=size;
+ if ((td->td_stripoffset[strip] > (uint64)TIFF_TMSIZE_T_MAX)||
+ ((ma=(tmsize_t)td->td_stripoffset[strip])>tif->tif_size))
+ {
+ n=0;
+ }
+ else if( ma > TIFF_TMSIZE_T_MAX - size )
+ {
+ n=0;
+ }
+ else
+ {
+ tmsize_t mb=ma+size;
+ if (mb>tif->tif_size)
+ n=tif->tif_size-ma;
+ else
+ n=size;
+ }
if (n!=size) {
#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
TIFFErrorExt(tif->tif_clientdata, module,
From 48780b4fcc425cddc4ef8ffdf536f96a0d1b313b Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 16:38:26 +0000
Subject: [PATCH] =?UTF-8?q?*=20libtiff/tif=5Fgetimage.c:=20add=20explicit?=
=?UTF-8?q?=20uint32=20cast=20in=20putagreytile=20to=20avoid=20UndefinedBe?=
=?UTF-8?q?haviorSanitizer=20warning.=20Patch=20by=20Nicol=C3=A1s=20Pe?=
=?UTF-8?q?=C3=B1a.=20Fixes=20http://bugzilla.maptools.org/show=5Fbug.cgi?=
=?UTF-8?q?=3Fid=3D2658?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
ChangeLog | 7 +++++++
libtiff/tif_getimage.c | 2 +-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index 3e31464..6a342e5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_getimage.c: add explicit uint32 cast in putagreytile to
+ avoid UndefinedBehaviorSanitizer warning.
+ Patch by Nicolás Peña.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2658
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_read.c: avoid potential undefined behaviour on signed integer
addition in TIFFReadRawStrip1() in isMapped() case.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2650
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index fed31f1..2fa1775 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -1305,7 +1305,7 @@ DECLAREContigPutFunc(putagreytile)
while (h-- > 0) {
for (x = w; x-- > 0;)
{
- *cp++ = BWmap[*pp][0] & (*(pp+1) << 24 | ~A1);
+ *cp++ = BWmap[*pp][0] & ((uint32)*(pp+1) << 24 | ~A1);
pp += samplesperpixel;
}
cp += toskew;
From d60332057b9575ada4f264489582b13e30137be1 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 19:02:49 +0000
Subject: [PATCH] * libtiff/tiffiop.h, tif_unix.c, tif_win32.c, tif_vms.c: add
_TIFFcalloc()
* libtiff/tif_read.c: TIFFReadBufferSetup(): use _TIFFcalloc() to zero
initialize tif_rawdata.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2651
---
ChangeLog | 8 ++++++++
libtiff/tif_read.c | 4 +++-
libtiff/tif_unix.c | 8 ++++++++
libtiff/tif_vms.c | 8 ++++++++
libtiff/tif_win32.c | 8 ++++++++
libtiff/tiffio.h | 1 +
6 files changed, 36 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index 6a342e5..abd75d7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tiffiop.h, tif_unix.c, tif_win32.c, tif_vms.c: add _TIFFcalloc()
+
+ * libtiff/tif_read.c: TIFFReadBufferSetup(): use _TIFFcalloc() to zero
+ initialize tif_rawdata.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2651
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_getimage.c: add explicit uint32 cast in putagreytile to
avoid UndefinedBehaviorSanitizer warning.
Patch by Nicolás Peña.
diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
index 277fdd6..4535ccb 100644
--- a/libtiff/tif_read.c
+++ b/libtiff/tif_read.c
@@ -985,7 +985,9 @@ TIFFReadBufferSetup(TIFF* tif, void* bp, tmsize_t size)
"Invalid buffer size");
return (0);
}
- tif->tif_rawdata = (uint8*) _TIFFmalloc(tif->tif_rawdatasize);
+ /* Initialize to zero to avoid uninitialized buffers in case of */
+ /* short reads (http://bugzilla.maptools.org/show_bug.cgi?id=2651) */
+ tif->tif_rawdata = (uint8*) _TIFFcalloc(1, tif->tif_rawdatasize);
tif->tif_flags |= TIFF_MYBUFFER;
}
if (tif->tif_rawdata == NULL) {
diff --git a/libtiff/tif_unix.c b/libtiff/tif_unix.c
index 7c7bc96..89dd32e 100644
--- a/libtiff/tif_unix.c
+++ b/libtiff/tif_unix.c
@@ -316,6 +316,14 @@ _TIFFmalloc(tmsize_t s)
return (malloc((size_t) s));
}
+void* _TIFFcalloc(tmsize_t nmemb, tmsize_t siz)
+{
+ if( nmemb == 0 || siz == 0 )
+ return ((void *) NULL);
+
+ return calloc((size_t) nmemb, (size_t)siz);
+}
+
void
_TIFFfree(void* p)
{
diff --git a/libtiff/tif_win32.c b/libtiff/tif_win32.c
index d730b3a..3e9001b 100644
--- a/libtiff/tif_win32.c
+++ b/libtiff/tif_win32.c
@@ -360,6 +360,14 @@ _TIFFmalloc(tmsize_t s)
return (malloc((size_t) s));
}
+void* _TIFFcalloc(tmsize_t nmemb, tmsize_t siz)
+{
+ if( nmemb == 0 || siz == 0 )
+ return ((void *) NULL);
+
+ return calloc((size_t) nmemb, (size_t)siz);
+}
+
void
_TIFFfree(void* p)
{
diff --git a/libtiff/tiffio.h b/libtiff/tiffio.h
index 732da17..fbd9171 100644
--- a/libtiff/tiffio.h
+++ b/libtiff/tiffio.h
@@ -293,6 +293,7 @@ extern TIFFCodec* TIFFGetConfiguredCODECs(void);
*/
extern void* _TIFFmalloc(tmsize_t s);
+extern void* _TIFFcalloc(tmsize_t nmemb, tmsize_t siz);
extern void* _TIFFrealloc(void* p, tmsize_t s);
extern void _TIFFmemset(void* p, int v, tmsize_t c);
extern void _TIFFmemcpy(void* d, const void* s, tmsize_t c);
From 8283e4d1b7e53340684d12932880cbcbaf23a8c1 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Thu, 12 Jan 2017 17:43:25 +0000
Subject: [PATCH] =?UTF-8?q?*=20libtiff/tif=5Fojpeg.c:=20fix=20leak=20in=20?=
=?UTF-8?q?OJPEGReadHeaderInfoSecTablesAcTable=20when=20read=20fails.=20Pa?=
=?UTF-8?q?tch=20by=20Nicol=C3=A1s=20Pe=C3=B1a.=20Fixes=20http://bugzilla.?=
=?UTF-8?q?maptools.org/show=5Fbug.cgi=3Fid=3D2659?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
ChangeLog | 7 +++++++
libtiff/tif_ojpeg.c | 3 +++
2 files changed, 10 insertions(+)
diff --git a/ChangeLog b/ChangeLog
index 6e6f3b0..12e0370 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2017-01-12 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_ojpeg.c: fix leak in OJPEGReadHeaderInfoSecTablesAcTable
+ when read fails.
+ Patch by Nicolás Peña.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2659
+
2017-01-11 Even Rouault <even.rouault at spatialys.com>
* libtiff/tiffiop.h, tif_unix.c, tif_win32.c, tif_vms.c: add _TIFFcalloc()
diff --git a/libtiff/tif_ojpeg.c b/libtiff/tif_ojpeg.c
index f19e8fd..b92f0eb 100644
--- a/libtiff/tif_ojpeg.c
+++ b/libtiff/tif_ojpeg.c
@@ -1918,7 +1918,10 @@ OJPEGReadHeaderInfoSecTablesAcTable(TIFF* tif)
rb[sizeof(uint32)+5+n]=o[n];
p=(uint32)TIFFReadFile(tif,&(rb[sizeof(uint32)+21]),q);
if (p!=q)
+ {
+ _TIFFfree(rb);
return(0);
+ }
sp->actable[m]=rb;
sp->sos_tda[m]=(sp->sos_tda[m]|m);
}
From 2ea32f7372b65c24b2816f11c04bf59b5090d05b Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Thu, 12 Jan 2017 19:23:20 +0000
Subject: [PATCH] * libtiff/tif_ojpeg.c: fix leak in
OJPEGReadHeaderInfoSecTablesQTable, OJPEGReadHeaderInfoSecTablesDcTable and
OJPEGReadHeaderInfoSecTablesAcTable
---
ChangeLog | 3 ++-
libtiff/tif_ojpeg.c | 6 ++++++
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index 12e0370..cd2fa17 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
2017-01-12 Even Rouault <even.rouault at spatialys.com>
- * libtiff/tif_ojpeg.c: fix leak in OJPEGReadHeaderInfoSecTablesAcTable
+ * libtiff/tif_ojpeg.c: fix leak in OJPEGReadHeaderInfoSecTablesQTable,
+ OJPEGReadHeaderInfoSecTablesDcTable and OJPEGReadHeaderInfoSecTablesAcTable
when read fails.
Patch by Nicolás Peña.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2659
diff --git a/libtiff/tif_ojpeg.c b/libtiff/tif_ojpeg.c
index b92f0eb..5f6c684 100644
--- a/libtiff/tif_ojpeg.c
+++ b/libtiff/tif_ojpeg.c
@@ -1790,7 +1790,10 @@ OJPEGReadHeaderInfoSecTablesQTable(TIFF* tif)
TIFFSeekFile(tif,sp->qtable_offset[m],SEEK_SET);
p=(uint32)TIFFReadFile(tif,&ob[sizeof(uint32)+5],64);
if (p!=64)
+ {
+ _TIFFfree(ob);
return(0);
+ }
sp->qtable[m]=ob;
sp->sof_tq[m]=m;
}
@@ -1854,7 +1857,10 @@ OJPEGReadHeaderInfoSecTablesDcTable(TIFF* tif)
rb[sizeof(uint32)+5+n]=o[n];
p=(uint32)TIFFReadFile(tif,&(rb[sizeof(uint32)+21]),q);
if (p!=q)
+ {
+ _TIFFfree(rb);
return(0);
+ }
sp->dctable[m]=rb;
sp->sos_tda[m]=(m<<4);
}
component/libtiff/tiff_4.0.8-3.debian.patch 0 → 100644
View file @ f581d489
diff -pur tiff-4.0.4/tools/tiffsplit.c tiff-4.0.4_patch/tools/tiffsplit.c
--- tiff-4.0.4/tools/tiffsplit.c 2015-05-28 15:10:26.000000000 +0200
+++ tiff-4.0.4_patch/tools/tiffsplit.c 2016-02-12 19:15:30.532005041 +0100
@@ -179,8 +179,9 @@ tiffcp(TIFF* in, TIFF* out)
TIFFSetField(out, TIFFTAG_JPEGTABLES, count, table);
}
}
+ uint32 count = 0;
CopyField(TIFFTAG_PHOTOMETRIC, shortv);
- CopyField(TIFFTAG_PREDICTOR, shortv);
+ CopyField2(TIFFTAG_PREDICTOR, count, shortv);
CopyField(TIFFTAG_THRESHHOLDING, shortv);
CopyField(TIFFTAG_FILLORDER, shortv);
CopyField(TIFFTAG_ORIENTATION, shortv);
@@ -188,7 +189,7 @@ tiffcp(TIFF* in, TIFF* out)
CopyField(TIFFTAG_MAXSAMPLEVALUE, shortv);
CopyField(TIFFTAG_XRESOLUTION, floatv);
CopyField(TIFFTAG_YRESOLUTION, floatv);
- CopyField(TIFFTAG_GROUP3OPTIONS, longv);
+ CopyField2(TIFFTAG_GROUP3OPTIONS, count, longv);
CopyField(TIFFTAG_GROUP4OPTIONS, longv);
CopyField(TIFFTAG_RESOLUTIONUNIT, shortv);
CopyField(TIFFTAG_PLANARCONFIG, shortv);
From 02669064e927074819ce1ed39aba0fccaa167717 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Mon, 29 May 2017 10:12:54 +0000
Subject: [PATCH] * libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter
clamping to avoid int32 overflow in TIFFYCbCrtoRGB(). Fixes
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844 Credit to OSS Fuzz
---
ChangeLog | 7 +++++++
libtiff/tif_color.c | 6 +++---
2 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index ee8d9d08..61116596 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2017-05-29 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter clamping to avoid
+ int32 overflow in TIFFYCbCrtoRGB().
+ Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844
+ Credit to OSS Fuzz
+
2017-05-21 Bob Friesenhahn <bfriesen@simple.dallas.tx.us>
* configure.ac: libtiff 4.0.8 released.
diff --git a/libtiff/tif_color.c b/libtiff/tif_color.c
index 055ed3b2..10a5e66e 100644
--- a/libtiff/tif_color.c
+++ b/libtiff/tif_color.c
@@ -275,10 +275,10 @@ TIFFYCbCrToRGBInit(TIFFYCbCrToRGB* ycbcr, float *luma, float *refBlackWhite)
for (i = 0, x = -128; i < 256; i++, x++) {
int32 Cr = (int32)CLAMPw(Code2V(x, refBlackWhite[4] - 128.0F,
refBlackWhite[5] - 128.0F, 127),
- -128.0F * 64, 128.0F * 64);
+ -128.0F * 32, 128.0F * 32);
int32 Cb = (int32)CLAMPw(Code2V(x, refBlackWhite[2] - 128.0F,
refBlackWhite[3] - 128.0F, 127),
- -128.0F * 64, 128.0F * 64);
+ -128.0F * 32, 128.0F * 32);
ycbcr->Cr_r_tab[i] = (int32)((D1*Cr + ONE_HALF)>>SHIFT);
ycbcr->Cb_b_tab[i] = (int32)((D3*Cb + ONE_HALF)>>SHIFT);
@@ -286,7 +286,7 @@ TIFFYCbCrToRGBInit(TIFFYCbCrToRGB* ycbcr, float *luma, float *refBlackWhite)
ycbcr->Cb_g_tab[i] = D4*Cb + ONE_HALF;
ycbcr->Y_tab[i] =
(int32)CLAMPw(Code2V(x + 128, refBlackWhite[0], refBlackWhite[1], 255),
- -128.0F * 64, 128.0F * 64);
+ -128.0F * 32, 128.0F * 32);
}
}
From 468988860e0dae62ebbf991627c74bcbb4bd256f Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Mon, 29 May 2017 11:29:06 +0000
Subject: [PATCH] * libtiff/tif_getimage.c: initYCbCrConversion(): stricter
validation for refBlackWhite coefficients values. To avoid invalid
float->int32 conversion (when refBlackWhite[0] == 2147483648.f) Fixes
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1907 Credit to OSS Fuzz
---
ChangeLog | 8 ++++++++
libtiff/tif_getimage.c | 2 +-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index a2ddaac2..04881ba7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2017-05-29 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_getimage.c: initYCbCrConversion(): stricter validation for
+ refBlackWhite coefficients values. To avoid invalid float->int32 conversion
+ (when refBlackWhite[0] == 2147483648.f)
+ Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1907
+ Credit to OSS Fuzz
+
+2017-05-29 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter clamping to avoid
int32 overflow in TIFFYCbCrtoRGB().
Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index dc373abc..a209a7a7 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -2241,7 +2241,7 @@ DECLARESepPutFunc(putseparate8bitYCbCr11tile)
static int isInRefBlackWhiteRange(float f)
{
- return f >= (float)(-0x7FFFFFFF + 128) && f <= (float)0x7FFFFFFF;
+ return f > (float)(-0x7FFFFFFF + 128) && f < (float)0x7FFFFFFF;
}
static int
commit 40448d58fbfad52d2dde5bd18daa30b17fe35fcd
Author: erouault <erouault>
Date: Thu Jun 1 12:44:04 2017 +0000
* libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
and use it in TIFFReadDirectory() so as to ignore fields whose tag is a
codec-specified tag but this codec is not enabled. This avoids TIFFGetField()
to behave differently depending on whether the codec is enabled or not, and
thus can avoid stack based buffer overflows in a number of TIFF utilities
such as tiffsplit, tiffcmp, thumbnail, etc.
Patch derived from 0063-Handle-properly-CODEC-specific-tags.patch
(http://bugzilla.maptools.org/show_bug.cgi?id=2580) by Raphaël Hertzog.
Fixes:
http://bugzilla.maptools.org/show_bug.cgi?id=2580
http://bugzilla.maptools.org/show_bug.cgi?id=2693
http://bugzilla.maptools.org/show_bug.cgi?id=2625 (CVE-2016-10095)
http://bugzilla.maptools.org/show_bug.cgi?id=2564 (CVE-2015-7554)
http://bugzilla.maptools.org/show_bug.cgi?id=2561 (CVE-2016-5318)
http://bugzilla.maptools.org/show_bug.cgi?id=2499 (CVE-2014-8128)
http://bugzilla.maptools.org/show_bug.cgi?id=2441
http://bugzilla.maptools.org/show_bug.cgi?id=2433
diff --git a/ChangeLog b/ChangeLog
index 04881ba7..ebd1a3c0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+2017-06-01 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
+ and use it in TIFFReadDirectory() so as to ignore fields whose tag is a
+ codec-specified tag but this codec is not enabled. This avoids TIFFGetField()
+ to behave differently depending on whether the codec is enabled or not, and
+ thus can avoid stack based buffer overflows in a number of TIFF utilities
+ such as tiffsplit, tiffcmp, thumbnail, etc.
+ Patch derived from 0063-Handle-properly-CODEC-specific-tags.patch
+ (http://bugzilla.maptools.org/show_bug.cgi?id=2580) by Raphaël Hertzog.
+ Fixes:
+ http://bugzilla.maptools.org/show_bug.cgi?id=2580
+ http://bugzilla.maptools.org/show_bug.cgi?id=2693
+ http://bugzilla.maptools.org/show_bug.cgi?id=2625 (CVE-2016-10095)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2564 (CVE-2015-7554)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2561 (CVE-2016-5318)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2499 (CVE-2014-8128)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2441
+ http://bugzilla.maptools.org/show_bug.cgi?id=2433
+
2017-05-29 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_getimage.c: initYCbCrConversion(): stricter validation for
diff --git a/libtiff/tif_dir.h b/libtiff/tif_dir.h
index 6af5f3dc..5a380767 100644
--- a/libtiff/tif_dir.h
+++ b/libtiff/tif_dir.h
@@ -1,4 +1,4 @@
-/* $Id: tif_dir.h,v 1.54 2011-02-18 20:53:05 fwarmerdam Exp $ */
+/* $Id: tif_dir.h,v 1.55 2017-06-01 12:44:04 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -291,6 +291,7 @@ struct _TIFFField {
extern int _TIFFMergeFields(TIFF*, const TIFFField[], uint32);
extern const TIFFField* _TIFFFindOrRegisterField(TIFF *, uint32, TIFFDataType);
extern TIFFField* _TIFFCreateAnonField(TIFF *, uint32, TIFFDataType);
+extern int _TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag);
#if defined(__cplusplus)
}
diff --git a/libtiff/tif_dirinfo.c b/libtiff/tif_dirinfo.c
index 23ad0020..4904f540 100644
--- a/libtiff/tif_dirinfo.c
+++ b/libtiff/tif_dirinfo.c
@@ -1,4 +1,4 @@
-/* $Id: tif_dirinfo.c,v 1.126 2016-11-18 02:52:13 bfriesen Exp $ */
+/* $Id: tif_dirinfo.c,v 1.127 2017-06-01 12:44:04 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -956,6 +956,109 @@ TIFFMergeFieldInfo(TIFF* tif, const TIFFFieldInfo info[], uint32 n)
return 0;
}
+int
+_TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag)
+{
+ /* Filter out non-codec specific tags */
+ switch (tag) {
+ /* Shared tags */
+ case TIFFTAG_PREDICTOR:
+ /* JPEG tags */
+ case TIFFTAG_JPEGTABLES:
+ /* OJPEG tags */
+ case TIFFTAG_JPEGIFOFFSET:
+ case TIFFTAG_JPEGIFBYTECOUNT:
+ case TIFFTAG_JPEGQTABLES:
+ case TIFFTAG_JPEGDCTABLES:
+ case TIFFTAG_JPEGACTABLES:
+ case TIFFTAG_JPEGPROC:
+ case TIFFTAG_JPEGRESTARTINTERVAL:
+ /* CCITT* */
+ case TIFFTAG_BADFAXLINES:
+ case TIFFTAG_CLEANFAXDATA:
+ case TIFFTAG_CONSECUTIVEBADFAXLINES:
+ case TIFFTAG_GROUP3OPTIONS:
+ case TIFFTAG_GROUP4OPTIONS:
+ break;
+ default:
+ return 1;
+ }
+ /* Check if codec specific tags are allowed for the current
+ * compression scheme (codec) */
+ switch (tif->tif_dir.td_compression) {
+ case COMPRESSION_LZW:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+ case COMPRESSION_PACKBITS:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_THUNDERSCAN:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_NEXT:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_JPEG:
+ if (tag == TIFFTAG_JPEGTABLES)
+ return 1;
+ break;
+ case COMPRESSION_OJPEG:
+ switch (tag) {
+ case TIFFTAG_JPEGIFOFFSET:
+ case TIFFTAG_JPEGIFBYTECOUNT:
+ case TIFFTAG_JPEGQTABLES:
+ case TIFFTAG_JPEGDCTABLES:
+ case TIFFTAG_JPEGACTABLES:
+ case TIFFTAG_JPEGPROC:
+ case TIFFTAG_JPEGRESTARTINTERVAL:
+ return 1;
+ }
+ break;
+ case COMPRESSION_CCITTRLE:
+ case COMPRESSION_CCITTRLEW:
+ case COMPRESSION_CCITTFAX3:
+ case COMPRESSION_CCITTFAX4:
+ switch (tag) {
+ case TIFFTAG_BADFAXLINES:
+ case TIFFTAG_CLEANFAXDATA:
+ case TIFFTAG_CONSECUTIVEBADFAXLINES:
+ return 1;
+ case TIFFTAG_GROUP3OPTIONS:
+ if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX3)
+ return 1;
+ break;
+ case TIFFTAG_GROUP4OPTIONS:
+ if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX4)
+ return 1;
+ break;
+ }
+ break;
+ case COMPRESSION_JBIG:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_DEFLATE:
+ case COMPRESSION_ADOBE_DEFLATE:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+ case COMPRESSION_PIXARLOG:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+ case COMPRESSION_SGILOG:
+ case COMPRESSION_SGILOG24:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_LZMA:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+
+ }
+ return 0;
+}
+
/* vim: set ts=8 sts=8 sw=8 noet: */
/*
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 772ebaf7..acde78b5 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -1,4 +1,4 @@
-/* $Id: tif_dirread.c,v 1.208 2017-04-27 15:46:22 erouault Exp $ */
+/* $Id: tif_dirread.c,v 1.209 2017-06-01 12:44:04 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -3580,6 +3580,10 @@ TIFFReadDirectory(TIFF* tif)
goto bad;
dp->tdir_tag=IGNORE;
break;
+ default:
+ if( !_TIFFCheckFieldIsValidForCodec(tif, dp->tdir_tag) )
+ dp->tdir_tag=IGNORE;
+ break;
}
}
}
From fe8d7165956b88df4837034a9161dc5fd20cf67a Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Mon, 26 Jun 2017 15:19:59 +0000
Subject: [PATCH] * libtiff/tif_jbig.c: fix memory leak in error code path of
JBIGDecode() Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2706 Reported
by team OWL337
* libtiff/tif_jpeg.c: error out at decoding time if anticipated libjpeg
---
ChangeLog | 8 +++++++-
libtiff/tif_jbig.c | 1 +
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index bc5096e7..ecd70534 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2017-06-26 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_jbig.c: fix memory leak in error code path of JBIGDecode()
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2706
+ Reported by team OWL337
+
2017-06-01 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
diff --git a/libtiff/tif_jbig.c b/libtiff/tif_jbig.c
index 5f5f75e2..c75f31d9 100644
--- a/libtiff/tif_jbig.c
+++ b/libtiff/tif_jbig.c
@@ -94,6 +94,7 @@ static int JBIGDecode(TIFF* tif, uint8* buffer, tmsize_t size, uint16 s)
jbg_strerror(decodeStatus)
#endif
);
+ jbg_dec_free(&decoder);
return 0;
}
From 1077fad562e03d1cad591dd10163dd80ad63ab0e Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Fri, 30 Jun 2017 13:11:18 +0000
Subject: [PATCH] * libtiff/tif_read.c, tiffiop.h: add a
_TIFFReadEncodedStripAndAllocBuffer() function, variant of
TIFFReadEncodedStrip() that allocates the decoded buffer only after a first
successful TIFFFillStrip(). This avoids excessive memory allocation on
corrupted files. * libtiff/tif_getimage.c: use
_TIFFReadEncodedStripAndAllocBuffer(). Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2708 and
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2433 . Credit to OSS
Fuzz
---
ChangeLog | 11 +++++++
libtiff/tif_getimage.c | 59 ++++++++++++++++++++++----------------
libtiff/tif_read.c | 78 +++++++++++++++++++++++++++++++++++++++++++-------
libtiff/tiffiop.h | 5 ++++
4 files changed, 118 insertions(+), 35 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index c969f9e2..6f085e09 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2017-06-30 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_read.c, tiffiop.h: add a _TIFFReadEncodedStripAndAllocBuffer()
+ function, variant of TIFFReadEncodedStrip() that allocates the
+ decoded buffer only after a first successful TIFFFillStrip(). This avoids
+ excessive memory allocation on corrupted files.
+ * libtiff/tif_getimage.c: use _TIFFReadEncodedStripAndAllocBuffer().
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2708 and
+ https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2433 .
+ Credit to OSS Fuzz
+
2017-06-26 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_jbig.c: fix memory leak in error code path of JBIGDecode()
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index cee8e930..cc6e8f30 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -905,26 +905,22 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
tileContigRoutine put = img->put.contig;
uint32 row, y, nrow, nrowsub, rowstoread;
tmsize_t pos;
- unsigned char* buf;
+ unsigned char* buf = NULL;
uint32 rowsperstrip;
uint16 subsamplinghor,subsamplingver;
uint32 imagewidth = img->width;
tmsize_t scanline;
int32 fromskew, toskew;
int ret = 1, flip;
+ tmsize_t maxstripsize;
TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING, &subsamplinghor, &subsamplingver);
if( subsamplingver == 0 ) {
TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Invalid vertical YCbCr subsampling");
return (0);
}
-
- buf = (unsigned char*) _TIFFmalloc(TIFFStripSize(tif));
- if (buf == 0) {
- TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
- return (0);
- }
- _TIFFmemset(buf, 0, TIFFStripSize(tif));
+
+ maxstripsize = TIFFStripSize(tif);
flip = setorientation(img);
if (flip & FLIP_VERTICALLY) {
@@ -946,11 +942,12 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
nrowsub = nrow;
if ((nrowsub%subsamplingver)!=0)
nrowsub+=subsamplingver-nrowsub%subsamplingver;
- if (TIFFReadEncodedStrip(tif,
+ if (_TIFFReadEncodedStripAndAllocBuffer(tif,
TIFFComputeStrip(tif,row+img->row_offset, 0),
- buf,
+ (void**)(&buf),
+ maxstripsize,
((row + img->row_offset)%rowsperstrip + nrowsub) * scanline)==(tmsize_t)(-1)
- && img->stoponerr)
+ && (buf == NULL || img->stoponerr))
{
ret = 0;
break;
@@ -994,8 +991,8 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
{
TIFF* tif = img->tif;
tileSeparateRoutine put = img->put.separate;
- unsigned char *buf;
- unsigned char *p0, *p1, *p2, *pa;
+ unsigned char *buf = NULL;
+ unsigned char *p0 = NULL, *p1 = NULL, *p2 = NULL, *pa = NULL;
uint32 row, y, nrow, rowstoread;
tmsize_t pos;
tmsize_t scanline;
@@ -1014,15 +1011,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
return (0);
}
- p0 = buf = (unsigned char *)_TIFFmalloc(bufsize);
- if (buf == 0) {
- TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for tile buffer");
- return (0);
- }
- _TIFFmemset(buf, 0, bufsize);
- p1 = p0 + stripsize;
- p2 = p1 + stripsize;
- pa = (alpha?(p2+stripsize):NULL);
flip = setorientation(img);
if (flip & FLIP_VERTICALLY) {
@@ -1040,7 +1028,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
case PHOTOMETRIC_MINISBLACK:
case PHOTOMETRIC_PALETTE:
colorchannels = 1;
- p2 = p1 = p0;
break;
default:
@@ -1056,7 +1043,31 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
nrow = (row + rowstoread > h ? h - row : rowstoread);
offset_row = row + img->row_offset;
- if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
+ if( buf == NULL )
+ {
+ if (_TIFFReadEncodedStripAndAllocBuffer(
+ tif, TIFFComputeStrip(tif, offset_row, 0),
+ (void**) &buf, bufsize,
+ ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+ && (buf == NULL || img->stoponerr))
+ {
+ ret = 0;
+ break;
+ }
+ p0 = buf;
+ if( colorchannels == 1 )
+ {
+ p2 = p1 = p0;
+ pa = (alpha?(p0+3*stripsize):NULL);
+ }
+ else
+ {
+ p1 = p0 + stripsize;
+ p2 = p1 + stripsize;
+ pa = (alpha?(p2+stripsize):NULL);
+ }
+ }
+ else if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
p0, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
&& img->stoponerr)
{
diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
index fc0072e7..047305ab 100644
--- a/libtiff/tif_read.c
+++ b/libtiff/tif_read.c
@@ -442,18 +442,17 @@ TIFFReadScanline(TIFF* tif, void* buf, uint32 row, uint16 sample)
}
/*
- * Read a strip of data and decompress the specified
- * amount into the user-supplied buffer.
+ * Calculate the strip size according to the number of
+ * rows in the strip (check for truncated last strip on any
+ * of the separations).
*/
-tmsize_t
-TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+static tmsize_t TIFFReadEncodedStripGetStripSize(TIFF* tif, uint32 strip, uint16* pplane)
{
static const char module[] = "TIFFReadEncodedStrip";
TIFFDirectory *td = &tif->tif_dir;
uint32 rowsperstrip;
uint32 stripsperplane;
uint32 stripinplane;
- uint16 plane;
uint32 rows;
tmsize_t stripsize;
if (!TIFFCheckRead(tif,0))
@@ -465,23 +464,37 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
(unsigned long)td->td_nstrips);
return((tmsize_t)(-1));
}
- /*
- * Calculate the strip size according to the number of
- * rows in the strip (check for truncated last strip on any
- * of the separations).
- */
+
rowsperstrip=td->td_rowsperstrip;
if (rowsperstrip>td->td_imagelength)
rowsperstrip=td->td_imagelength;
stripsperplane= TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip);
stripinplane=(strip%stripsperplane);
- plane=(uint16)(strip/stripsperplane);
+ if( pplane ) *pplane=(uint16)(strip/stripsperplane);
rows=td->td_imagelength-stripinplane*rowsperstrip;
if (rows>rowsperstrip)
rows=rowsperstrip;
stripsize=TIFFVStripSize(tif,rows);
if (stripsize==0)
return((tmsize_t)(-1));
+ return stripsize;
+}
+
+/*
+ * Read a strip of data and decompress the specified
+ * amount into the user-supplied buffer.
+ */
+tmsize_t
+TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+{
+ static const char module[] = "TIFFReadEncodedStrip";
+ TIFFDirectory *td = &tif->tif_dir;
+ tmsize_t stripsize;
+ uint16 plane;
+
+ stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
+ if (stripsize==((tmsize_t)(-1)))
+ return((tmsize_t)(-1));
/* shortcut to avoid an extra memcpy() */
if( td->td_compression == COMPRESSION_NONE &&
@@ -510,6 +523,49 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
return(stripsize);
}
+/* Variant of TIFFReadEncodedStrip() that does
+ * * if *buf == NULL, *buf = _TIFFmalloc(bufsizetoalloc) only after TIFFFillStrip() has
+ * suceeded. This avoid excessive memory allocation in case of truncated
+ * file.
+ * * calls regular TIFFReadEncodedStrip() if *buf != NULL
+ */
+tmsize_t
+_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
+ void **buf, tmsize_t bufsizetoalloc,
+ tmsize_t size_to_read)
+{
+ tmsize_t this_stripsize;
+ uint16 plane;
+
+ if( *buf != NULL )
+ {
+ return TIFFReadEncodedStrip(tif, strip, *buf, size_to_read);
+ }
+
+ this_stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
+ if (this_stripsize==((tmsize_t)(-1)))
+ return((tmsize_t)(-1));
+
+ if ((size_to_read!=(tmsize_t)(-1))&&(size_to_read<this_stripsize))
+ this_stripsize=size_to_read;
+ if (!TIFFFillStrip(tif,strip))
+ return((tmsize_t)(-1));
+
+ *buf = _TIFFmalloc(bufsizetoalloc);
+ if (*buf == NULL) {
+ TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
+ return((tmsize_t)(-1));
+ }
+ _TIFFmemset(*buf, 0, bufsizetoalloc);
+
+ if ((*tif->tif_decodestrip)(tif,*buf,this_stripsize,plane)<=0)
+ return((tmsize_t)(-1));
+ (*tif->tif_postdecode)(tif,*buf,this_stripsize);
+ return(this_stripsize);
+
+
+}
+
static tmsize_t
TIFFReadRawStrip1(TIFF* tif, uint32 strip, void* buf, tmsize_t size,
const char* module)
diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
index 846ade03..7f0b90f7 100644
--- a/libtiff/tiffiop.h
+++ b/libtiff/tiffiop.h
@@ -365,6 +365,11 @@ extern void* _TIFFCheckRealloc(TIFF*, void*, tmsize_t, tmsize_t, const char*);
extern double _TIFFUInt64ToDouble(uint64);
extern float _TIFFUInt64ToFloat(uint64);
+extern tmsize_t
+_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
+ void **buf, tmsize_t bufsizetoalloc,
+ tmsize_t size_to_read);
+
extern int TIFFInitDumpMode(TIFF*, int);
#ifdef PACKBITS_SUPPORT
extern int TIFFInitPackBits(TIFF*, int);
From 6173a57d39e04d68b139f8c1aa499a24dbe74ba1 Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Fri, 30 Jun 2017 17:29:44 +0000
Subject: [PATCH] * libtiff/tif_dirwrite.c: in
TIFFWriteDirectoryTagCheckedXXXX() functions associated with LONG8/SLONG8
data type, replace assertion that the file is BigTIFF, by a non-fatal error.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2712 Reported by team
OWL337
---
ChangeLog | 8 ++++++++
libtiff/tif_dirwrite.c | 20 ++++++++++++++++----
2 files changed, 24 insertions(+), 4 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 6f085e09..77a64385 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2017-06-30 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_dirwrite.c: in TIFFWriteDirectoryTagCheckedXXXX()
+ functions associated with LONG8/SLONG8 data type, replace assertion that
+ the file is BigTIFF, by a non-fatal error.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2712
+ Reported by team OWL337
+
+2017-06-30 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_read.c, tiffiop.h: add a _TIFFReadEncodedStripAndAllocBuffer()
function, variant of TIFFReadEncodedStrip() that allocates the
decoded buffer only after a first successful TIFFFillStrip(). This avoids
diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
index 2967da58..8d6686ba 100644
--- a/libtiff/tif_dirwrite.c
+++ b/libtiff/tif_dirwrite.c
@@ -2111,7 +2111,10 @@ TIFFWriteDirectoryTagCheckedLong8(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, ui
{
uint64 m;
assert(sizeof(uint64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","LONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
m=value;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabLong8(&m);
@@ -2124,7 +2127,10 @@ TIFFWriteDirectoryTagCheckedLong8Array(TIFF* tif, uint32* ndir, TIFFDirEntry* di
{
assert(count<0x20000000);
assert(sizeof(uint64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","LONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabArrayOfLong8(value,count);
return(TIFFWriteDirectoryTagData(tif,ndir,dir,tag,TIFF_LONG8,count,count*8,value));
@@ -2136,7 +2142,10 @@ TIFFWriteDirectoryTagCheckedSlong8(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, u
{
int64 m;
assert(sizeof(int64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","SLONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
m=value;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabLong8((uint64*)(&m));
@@ -2149,7 +2158,10 @@ TIFFWriteDirectoryTagCheckedSlong8Array(TIFF* tif, uint32* ndir, TIFFDirEntry* d
{
assert(count<0x20000000);
assert(sizeof(int64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","SLONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabArrayOfLong8((uint64*)value,count);
return(TIFFWriteDirectoryTagData(tif,ndir,dir,tag,TIFF_SLONG8,count,count*8,value));
component/mariadb/1a910088ec55c61434029d8e1cc3c6192ac508d7.diff 0 → 100644
View file @ f581d489
diff --git a/mrn_mysql_compat.h b/mrn_mysql_compat.h
index d2b227ff..337de968 100644
--- a/mrn_mysql_compat.h
+++ b/mrn_mysql_compat.h
@@ -362,6 +362,7 @@
#if defined(MRN_MARIADB_P) && \
((MYSQL_VERSION_ID >= 100207) || \
+ ((MYSQL_VERSION_ID >= 100126) && (MYSQL_VERSION_ID < 100200)) || \
((MYSQL_VERSION_ID >= 50557) && (MYSQL_VERSION_ID < 100000)))
# define mrn_create_partition_name(out, \
out_length, \
component/mariadb/buildout.cfg
View file @ f581d489
......@@ -17,14 +17,17 @@ extends =
../readline/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
# The following lines are only for mariarocks.cfg
# to be extended last without touching 'parts'.
../gcc/buildout.cfg
parts =
mariadb
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-10.1.25/source/mariadb-10.1.25.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
md5sum = 6349e524bc449767c4650196d4141a02
url = https://downloads.mariadb.org/f/mariadb-10.1.26/source/mariadb-10.1.26.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
md5sum = bb88afb72434c6d567c742896dd50d41
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
......@@ -55,15 +58,17 @@ configure-options =
-DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
-DWITHOUT_MROONGA_STORAGE_ENGINE=1
-DWITHOUT_DAEMON_EXAMPLE=1
-DCMAKE_C_FLAGS="-I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
-DCMAKE_CXX_FLAGS="-I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
-DCMAKE_INSTALL_RPATH=${:CMAKE_INSTALL_RPATH}
CMAKE_INSTALL_RPATH=${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
-DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
-DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
-DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
CMAKE_CFLAGS = -I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include
CMAKE_LIBRARY_PATH = ${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib${:extra_library_path}
extra_library_path =
environment =
CMAKE_PROGRAM_PATH=${cmake:location}/bin
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include
CMAKE_LIBRARY_PATH=${bzip2:location}/lib:${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -L${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
PATH=${patch:location}/bin:%(PATH)s
post-install =
mkdir -p ${:location}/include/wsrep &&
......@@ -73,12 +78,17 @@ post-install =
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/mroonga/mroonga-7.04.tar.gz
md5sum = 1d889cdd0e4f3ac0a84d18d16ab920d6
url = http://packages.groonga.org/source/mroonga/mroonga-7.05.tar.gz
md5sum = d289667a60a6aa78d8bc276ab61b2996
pre-configure = set -e
rm -rf fake_mariadb_source
mkdir -p fake_mariadb_source
ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql
test -d ${mariadb:location}/include/mysql/server/private && (
mkdir -p fake_mariadb_source/sql
cp -a ${mariadb:location}/include/mysql/server/private/* fake_mariadb_source/sql
cp -a ${mariadb:location}/include/mysql/server/*.h fake_mariadb_source/sql
cp -a ${mariadb:location}/include/mysql/server/mysql fake_mariadb_source/sql
) || ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql
cp -ar ${mariadb:location}/include fake_mariadb_source/include
! test -f fake_mariadb_source/include/mysql/mariadb_stmt.h ||
sed -i -E -e '/^enum enum_(cursor|indicator)_type$/,/};/d' fake_mariadb_source/include/mysql/mariadb_stmt.h
......@@ -89,7 +99,8 @@ configure-options =
--disable-document
patch-options = -p1
patches =
${:_profile_base_location_}/mroonga_boolean.patch#29f85fc98e85d39b31e53b1e1683d288
${:_profile_base_location_}/mroonga_boolean.patch#c818568fe35ca6a4298f18e575d962a0
${:_profile_base_location_}/1a910088ec55c61434029d8e1cc3c6192ac508d7.diff#a8b8df4dd870a1a269dfd9dc6941467b
pre-build =
sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
environment =
......
component/mariadb/mariarocks.cfg
View file @ f581d489
# Do not extend any file that touch buildout:parts.
[mariadb]
url = https://downloads.mariadb.com/MariaDB/mariadb-10.2.6/source/mariadb-10.2.6.tar.gz
md5sum = 3d454cdadbd3208e1c9c23e5338a62b0
url = https://downloads.mariadb.org/f/mariadb-10.2.8/source/mariadb-10.2.8.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
md5sum = f93cbd5bfde3c0d082994764ff7db580
stable-patches =
configure-options +=
-DPLUGIN_DAEMON_EXAMPLE=NO
......@@ -8,4 +10,4 @@ configure-options +=
-DPLUGIN_MROONGA=NO
-DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
-DCMAKE_INSTALL_RPATH=${gcc:location}/lib:${gcc:location}/lib64:${:CMAKE_INSTALL_RPATH}
extra_library_path = :${gcc:location}/lib:${gcc:location}/lib64
component/mariadb/mroonga_boolean.patch
View file @ f581d489
diff --git a/ha_mroonga.cpp b/ha_mroonga.cpp
index 06691fb..b59e685 100644
--- a/ha_mroonga.cpp
+++ b/ha_mroonga.cpp
@@ -8283,7 +8283,7 @@ grn_rc ha_mroonga::generic_ft_init_ext_prepare_expression_in_boolean_mode(
--- mroonga-7.05/lib/mrn_query_parser.cpp~ 2017-07-27 07:47:36.000000000 +0200
+++ mroonga-7.05/lib/mrn_query_parser.cpp 2017-08-14 11:19:25.339015061 +0200
@@ -52,7 +52,7 @@
const char *keyword = NULL;
uint keyword_length = 0;
- grn_operator default_operator = GRN_OP_OR;
+ grn_operator default_operator = GRN_OP_ADJUST;
grn_expr_flags expression_flags = 0;
generic_ft_init_ext_parse_pragma(info,
key,
const char *raw_query = NULL;
size_t raw_query_length = 0;
- grn_operator default_operator = GRN_OP_OR;
+ grn_operator default_operator = GRN_OP_ADJUST;
grn_expr_flags expression_flags = 0;
parse_pragma(query,
query_length,
component/nginx/buildout.cfg
View file @ f581d489
......@@ -4,6 +4,7 @@ extends =
../git/buildout.cfg
../libexpat/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../pcre/buildout.cfg
../zlib/buildout.cfg
......@@ -13,6 +14,10 @@ parts = nginx-output
recipe = slapos.recipe.cmmi
url = http://nginx.org/download/nginx-1.10.1.tar.gz
md5sum = 088292d9caf6059ef328aa7dda332e44
patch-options = -p0
patch-binary = ${patch:location}/bin/patch
patches =
http://nginx.org/download/patch.2017.ranges.txt#40bf9f37c881cb3b10cfefd84ca92f6a
[nginx]
<= nginx-common
......
component/pcre/buildout.cfg
View file @ f581d489
......@@ -4,8 +4,8 @@ parts =
[pcre]
recipe = slapos.recipe.cmmi
url = http://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.40.tar.bz2
md5sum = 41a842bf7dcecd6634219336e2167d1d
url = https://ftp.pcre.org/pub/pcre/pcre-8.41.tar.bz2
md5sum = c160d22723b1670447341b08c58981c1
configure-options =
--disable-static
--enable-unicode-properties
component/percona-toolkit/buildout.cfg
View file @ f581d489
......@@ -8,9 +8,9 @@ parts =
[percona-toolkit]
recipe = slapos.recipe.cmmi
version = 2.2.15
url = http://www.percona.com/redir/downloads/percona-toolkit/${:version}/tarball/percona-toolkit-${:version}.tar.gz
md5sum = 022f40dadaea9025820530ea1f986192
version = 3.0.3
url = https://www.percona.com/downloads/percona-toolkit/${:version}/source/tarball/percona-toolkit-${:version}.tar.gz
md5sum = 8af181994fdf9aa984475637861098e9
depends =
${perl-DBI:location}
${perl-DBD-mariadb:location}
......
component/postgresql/buildout.cfg
View file @ f581d489
......@@ -35,5 +35,5 @@ environment =
[postgresql92]
<= postgresql-common
url = http://ftp.postgresql.org/pub/source/v9.2.18/postgresql-9.2.18.tar.bz2
md5sum = fd175eb5f29557c6ef2eeaf340330f9a
url = http://ftp.postgresql.org/pub/source/v9.2.18/postgresql-9.2.22.tar.bz2
md5sum = c5d3fb5229baf9e94ee2287980c55321
component/python-2.7/buildout.cfg
View file @ f581d489
......@@ -13,7 +13,6 @@ extends =
../zlib/buildout.cfg
../file/buildout.cfg
../xz-utils/buildout.cfg
../gcc/buildout.cfg
parts =
python2.7
......
component/python-mysqlclient/buildout.cfg
View file @ f581d489
......@@ -16,6 +16,7 @@ recipe = zc.recipe.egg:custom
egg = mysqlclient
patches =
https://github.com/PyMySQL/mysqlclient-python/commit/1693848c9f6ca863868d94d63499830f7f4f3a1f.diff#a493a91f9263243eb331fcab9901b8b0
${:_profile_base_location_}/mysqlclient-1.3.10-mariadb-10.2.8.patch#807b694fcd7ade4da4bcca321b1a29d2
patch-options = -p1
patch-binary = ${patch:location}/bin/patch
environment = python-mysqlclient-env
......
component/python-mysqlclient/mysqlclient-1.3.10-mariadb-10.2.8.patch 0 → 100644
View file @ f581d489
diff -ur mysqlclient-1.3.10.orig/_mysql.c mysqlclient-1.3.10/_mysql.c
--- mysqlclient-1.3.10.orig/_mysql.c 2017-01-04 13:47:08.000000000 +0100
+++ mysqlclient-1.3.10/_mysql.c 2017-08-21 10:53:21.014929937 +0200
@@ -1060,7 +1060,7 @@
if (self && PyModule_Check((PyObject*)self))
self = NULL;
if (self && self->open) {
-#if MYSQL_VERSION_ID >= 50707 && !defined(MARIADB_BASE_VERSION)
+#if MYSQL_VERSION_ID >= 50707 && (!defined(MARIADB_BASE_VERSION) && !defined(MARIADB_VERSION_ID))
len = mysql_real_escape_string_quote(&(self->connection), out, in, size, '\'');
#else
len = mysql_real_escape_string(&(self->connection), out, in, size);
@@ -1118,7 +1118,7 @@
out = PyBytes_AS_STRING(str);
check_server_init(NULL);
if (self && self->open) {
-#if MYSQL_VERSION_ID >= 50707 && !defined(MARIADB_BASE_VERSION)
+#if MYSQL_VERSION_ID >= 50707 && (!defined(MARIADB_BASE_VERSION) && !defined(MARIADB_VERSION_ID))
len = mysql_real_escape_string_quote(&(self->connection), out+1, in, size, '\'');
#else
len = mysql_real_escape_string(&(self->connection), out+1, in, size);
diff -ur mysqlclient-1.3.10.orig/setup_posix.py mysqlclient-1.3.10/setup_posix.py
--- mysqlclient-1.3.10.orig/setup_posix.py 2016-07-26 10:12:24.000000000 +0200
+++ mysqlclient-1.3.10/setup_posix.py 2017-08-21 10:54:59.643071374 +0200
@@ -63,7 +63,7 @@
if extra_compile_args[i] == '-arch':
extra_link_args += ['-arch', extra_compile_args[i + 1]]
- include_dirs = [dequote(i[2:])
+ include_dirs = [dequote(os.path.isdir(i[2:]+'/server') and i[2:]+'/server' or i[2:])
for i in mysql_config('include') if i.startswith('-I')]
if static:
slapos/recipe/check_port_listening/__init__.py → slapos/recipe/check_port_listening.py
View file @ f581d489
......@@ -26,24 +26,35 @@
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
from zc.buildout.easy_install import _safe_arg, script_header
import sys
template = script_header + r"""
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
import socket
import sys
addr = "%(hostname)s", %(port)s
try:
socket.create_connection(addr).close()
except (socket.error, socket.timeout):
sys.stderr.write("%%s on %%s isn't listening\n" %% addr)
sys.exit(127)
"""
class Recipe(GenericBaseRecipe):
"""
Check listening port promise
"""
def install(self):
config = dict(
hostname=self.options['hostname'],
port=self.options['port'],
python_path=sys.executable,
)
vnc_promise = self.createExecutable(
self.options['path'],
self.substituteTemplate(
self.getTemplateFilename('socket_connection_attempt.py.in'),
config))
promise = self.createExecutable(self.options['path'], template % {
'python': _safe_arg(sys.executable),
'dash_S': '', # BBB buildout 1.x
'hostname': self.options['hostname'],
'port': self.options['port'],
})
return [vnc_promise]
return [promise]
slapos/recipe/check_port_listening/template/socket_connection_attempt.py.in deleted 100644 → 0
View file @ 53366193
#!%(python_path)s
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
import socket
import sys
hostname = "%(hostname)s"
port = %(port)s
try:
s = socket.create_connection((hostname, port))
s.close()
except (socket.error, socket.timeout):
sys.stderr.write("%(port)s on %(hostname)s isn't listening\n")
sys.exit(127)
software/agent/software.cfg
View file @ f581d489
......@@ -51,29 +51,29 @@ pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.download = 1.0
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.71
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
paramiko = 2.0.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
rpdb = 0.1.5
software/apache-frontend/software.cfg
View file @ f581d489
......@@ -11,7 +11,7 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.71
smmap = 0.9.0
numpy = 1.11.2
pyasn1 = 0.2.3
......@@ -19,41 +19,41 @@ pyasn1 = 0.2.3
websockify = 0.8.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
dnspython = 1.15.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
erp5.util = 0.4.49
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
passlib = 1.6.5
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
paramiko = 2.0.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
pycurl = 7.43.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
rpdb = 0.1.5
software/caucase/instance-caucase-input-schema.json
View file @ f581d489
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"extends": "./schema-definitions.json#",
"title": "Input Parameters",
"properties": {
"server-port": {
"title": "http port to use",
"description": "Caucase http port to use.",
"type": "integer",
"default": 8009,
"minimum": 1,
"maximum": 65535
"allOf": [{
"$ref": "#/definitions/tcpv4port"
}, {
"title": "http port to use",
"description": "Caucase http port to use.",
"default": 8009
}]
},
"server-https-port": {
"title": "https port to use",
"description": "Caucase port to use for https connexion.",
"type": "integer",
"default": 8010,
"minimum": 1,
"maximum": 65535
"allOf": [{
"$ref": "#/definitions/tcpv4port"
}, {
"title": "https port to use",
"description": "Caucase port to use for https connexion.",
"default": 8010
}]
},
"external-url": {
"title": "External http url",
......@@ -61,6 +64,13 @@
"description": "The time in seconds before a generated certificate will be deleted on CA server. Set 0 to never delete. Default: 30*24*60*60 seconds (30 days)",
"default": 5184000,
"type": "integer"
},
"auto-sign-csr-amount": {
"title": "Number of CSR to sign automatically",
"description": "The number of CSR to sign automatically at startup. Has no effect if there is more than the specified value of csr submitted to caucase. This value should be as lowest as possible",
"default": 1,
"minimum": 1,
"type": "integer"
}
}
}
\ No newline at end of file
software/caucase/software.cfg
View file @ f581d489
......@@ -5,6 +5,6 @@ extends =
parts =
slapos-cookbook
cacause-extra-eggs
caucase-extra-eggs
instance-caucase
software/dream/software.cfg
View file @ f581d489
......@@ -48,7 +48,7 @@ scipy = 0.13.3
simpy = 3.0.5
zope.dottedname = 4.1.0
tablib = 0.10.0
mysqlclient = 1.3.10+SlapOSPatched001
mysqlclient = 1.3.10+SlapOSPatched002
# indirect dependancies
cp.recipe.cmd = 0.5
......
software/erp5/software-erp5-component.cfg
View file @ f581d489
......@@ -4,6 +4,7 @@
[buildout]
extends =
software.cfg
../../component/mariadb/mariarocks.cfg
[slapos.cookbook-repository]
branch = erp5-component
......
software/erp5testnode/buildout.hash.cfg 0 → 100644
View file @ f581d489
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
filename = instance.cfg
md5sum = 307663d73ef3ef94b02567ecd322252e
[template-default]
filename = instance-default.cfg
md5sum = 76f63d443c5fdcea6fac68791c6bb65b
software/erp5testnode/instance-default.cfg
View file @ f581d489
......@@ -17,6 +17,7 @@ parts =
monitor-base
monitor-publish
testnode-frontend
resiliency-exclude-file
[monitor-publish]
recipe = slapos.cookbook:publish
......@@ -162,6 +163,12 @@ software = $${rootdirectory:srv}/software
shellinabox = $${rootdirectory:srv}/shellinabox
ca-dir = $${rootdirectory:srv}/ca
[resiliency-exclude-file]
# Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template
input = inline: **
output = $${directory:srv}/exporter.exclude
[testnode-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
......
software/erp5testnode/software.cfg
View file @ f581d489
[buildout]
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../component/git/buildout.cfg
../../component/lxml-python/buildout.cfg
......@@ -49,17 +50,43 @@ recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
mode = 0644
md5sum = 307663d73ef3ef94b02567ecd322252e
[template-default]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-default.cfg
output = ${buildout:directory}/template-default.cfg
mode = 0644
md5sum = 05519f3887a309d3ec069e0aa9f52ebc
[versions]
PyXML = 0.8.5
erp5.util = 0.4.49
slapos.recipe.template = 3.0
ipython = 5.3.0
apache-libcloud = 2.1.0
gitdb2 = 2.0.2
pyasn1 = 0.3.2
smmap2 = 2.0.3
# Required by:
# slapos.toolbox==0.70
GitPython = 2.1.5
# Required by:
# slapos.toolbox==0.70
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.70
dnspython = 1.15.0
# Required by:
# slapos.toolbox==0.70
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.70
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.70
passlib = 1.7.1
software/ipython_notebook/template/ERP5kernel.py.jinja → software/jupyter/ERP5kernel.py
View file @ f581d489
#!{{ python_executable }}
from ipykernel.kernelbase import Kernel
from ipykernel.kernelapp import IPKernelApp
from IPython.core.display import HTML
import requests
import json
import sys
# erp5_url from buildout
erp5_url = "{{ erp5_url }}"
if not erp5_url:
erp5_url = None
else:
erp5_url = "%s/erp5/Base_executeJupyter" % erp5_url
erp5_url = None
if len(sys.argv) > 1:
erp5_url = "%s/erp5/Base_executeJupyter" % (sys.argv[1],)
class MagicInfo:
"""
......@@ -61,7 +57,7 @@ class ERP5Kernel(Kernel):
language = 'ERP5'
language_version = '0.1'
language_info = {'mimetype': 'text/plain', 'name':'python'}
banner = "ERP5 integration with ipython notebook"
banner = "ERP5 integration with jupyter notebook"
def __init__(self, user=None, password=None, url=None, status_code=None,
*args, **kwargs):
......
software/jupyter/buildout.hash.cfg 0 → 100644
View file @ f581d489
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance-jupyter-notebook]
filename = instance.cfg.in
md5sum = 89ef078efd20c5e03d1cf7228568b865
[jupyter-notebook-config]
filename = jupyter_notebook_config.py.jinja
md5sum = 720e90a829c63371696bc3009917a743
[jupyter-set-password]
filename = jupyter_set_password.cgi.jinja
md5sum = b8d31441780b524a7e52d1710dd78385
[erp5-kernel]
filename = ERP5kernel.py
md5sum = 7d5309fe79afbcb455c0d8181b42e56c
[kernel-json]
filename = kernel.json.jinja
md5sum = 33547be93a67530165e079dc3ecfdac3
[custom-js]
filename = custom.js
md5sum = 0bf9e2eb1718b14307265fe05a167018
software/ipython_notebook/template/custom.js.jinja → software/jupyter/custom.js
View file @ f581d489
......@@ -81,8 +81,8 @@
* @static
*/
$([IPython.events]).on('notebook_loaded.Notebook', function(){
var kernelname = IPython.notebook.kernel_selector.current_selection;
$([jupyter.events]).on('notebook_loaded.Notebook', function(){
var kernelname = jupyter.notebook.kernel_selector.current_selection;
var display_text="<div class='output_subarea output_text output_result'>\
<pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
......
software/ipython_notebook/instance.cfg.in → software/jupyter/instance.cfg.in
View file @ f581d489
[buildout]
parts =
instance
ipython_notebook
jupyter_notebook
read-knowledge0
publish-connection-parameter
jupyter-notebook-config
erp5-kernel
kernel-json
custom-js
......@@ -33,7 +34,7 @@ port = 8888
host = ${slapconfiguration:ipv6-random}
cert_file = ${generate-certificate:cert_file}
key_file = ${generate-certificate:key_file}
logfile = ${directory:log}/ipython_notebook.log
logfile = ${directory:log}/jupyter_notebook.log
notebook_dir = ${directory:notebook_dir}
[dynamic-jinja2-template-base]
......@@ -52,13 +53,13 @@ command =
-out ${instance-parameter:cert_file}
fi
update-command = ${:command}
cert_file = ${directory:etc}/ipython_notebook_cert.crt
key_file = ${directory:etc}/ipython_notebook_cert.key
cert_file = ${directory:etc}/jupyter_cert.crt
key_file = ${directory:etc}/jupyter_cert.key
[instance]
recipe = slapos.cookbook:wrapper
command-line =
{{ bin_directory }}/ipython notebook
{{ bin_directory }}/jupyter-lab
--no-browser
--ip=${instance-parameter:host}
--port=${instance-parameter:port}
......@@ -66,24 +67,19 @@ command-line =
--certfile=${instance-parameter:cert_file}
--keyfile=${instance-parameter:key_file}
--notebook-dir=${instance-parameter:notebook_dir}
--logfile=${instance-parameter:logfile}
--config=${ipython-notebook-config:rendered}
# Add 'log-level' alias to 'DEBUG' so as to log requests being done while running
# of service ipython_notebook.
--log-level="DEBUG"
wrapper-path = ${directory:service}/ipython_notebook
wrapper-path = ${directory:service}/jupyter-lab
parameters-extra = true
# Explicitly define IPython Directory to be used while starting ipython notebook
# Also explicitly force IPython to use en_US.utf8 encoding
environment =
IPYTHONDIR=${directory:ipython_dir}
JUPYTER_CONFIG_DIR=${directory:ipython_dir}
JUPYTER_PATH=${directory:jupyter_dir}
JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
LANG=C.UTF-8
[ipython-notebook-config]
[jupyter-notebook-config]
<= dynamic-jinja2-template-base
template = {{ ipython_notebook_config_location }}/{{ ipython_notebook_config_filename }}
rendered = ${directory:etc}/ipython_notebook_config.py
template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
mode = 0744
context =
raw config_cfg ${buildout:directory}/knowledge0.cfg
......@@ -98,12 +94,16 @@ service = ${:etc}/service
promise = ${:etc}/promise/
log = ${:var}/log
notebook_dir = ${:var}/notebooks
# Add folders to explicitly define ipython directory
ipython_dir = ${:home}/ipython
ipython_kernel_dir = ${:ipython_dir}/kernels
erp5_kernel_dir = ${:ipython_kernel_dir}/ERP5
[ipython_notebook]
# Add folders to explicitly define jupyter directory
jupyter_dir = ${:home}/jupyter
jupyter_config_dir = ${:jupyter_dir}/etc
jupyter_kernel_dir = ${:jupyter_dir}/kernels
jupyter_runtime_dir = ${:jupyter_dir}/runtime
jupyter_custom_dir = ${:jupyter_config_dir}/custom
jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
[jupyter_notebook]
# This part is called like this because knowledge0.write uses the part name for
# the section name in the config file.
recipe = slapos.cookbook:zero-knowledge.write
......@@ -126,16 +126,13 @@ instance-configuration =
[publish-connection-parameter]
recipe = slapos.cookbook:publish.serialised
url = https://[${instance-parameter:host}]:${instance-parameter:port}
jupyter-classic-url = https://[${instance-parameter:host}]:${instance-parameter:port}/tree
jupyterlab-url = https://[${instance-parameter:host}]:${instance-parameter:port}/lab
[erp5-kernel]
<= dynamic-jinja2-template-base
template = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
rendered = ${directory:erp5_kernel_dir}/ERP5kernel.py
# Use ipython as executable python as we'll be needing requests library in kernel
context =
raw python_executable {{ bin_directory }}/ipython
key erp5_url slapconfiguration:configuration.erp5-url
recipe = slapos.cookbook:symbolic.link
link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
target-directory = ${directory:erp5_kernel_dir}
[kernel-json]
<= dynamic-jinja2-template-base
......@@ -144,13 +141,13 @@ rendered = ${directory:erp5_kernel_dir}/kernel.json
# Use python2.7 executable bin file for kernel config
context =
raw python_executable {{ python_executable }}
key kernel_dir erp5-kernel:rendered
raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
key erp5_url slapconfiguration:configuration.erp5-url
raw display_name ERP5
raw language_name python
[custom-js]
<= dynamic-jinja2-template-base
template = {{ custom_js_location }}/{{ custom_js_filename }}
rendered = ${directory:ipython_dir}/profile_default/static/custom/custom.js
mode = 0744
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:jupyter_custom_dir}
link-binary = {{ custom_js_location }}/custom.js
software/ipython_notebook/template/ipython_notebook_config.py.jinja → software/jupyter/jupyter_notebook_config.py.jinja
View file @ f581d489
......@@ -25,14 +25,14 @@ c = get_config()
parser = ConfigParser.ConfigParser()
parser.read(knowledge_0)
if not parser.has_section("ipython_notebook"):
parser.add_section("ipython_notebook")
if not parser.has_section("jupyter_notebook"):
parser.add_section("jupyter_notebook")
if not parser.has_option("ipython_notebook", "password") or \
parser.get("ipython_notebook", "password") == "":
parser.set("ipython_notebook", "password", random_password())
if not parser.has_option("jupyter_notebook", "password") or \
parser.get("jupyter_notebook", "password") == "":
parser.set("jupyter_notebook", "password", random_password())
c.NotebookApp.password = passwd(parser.get("ipython_notebook", "password"))
c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
with open(knowledge_0, 'w') as file:
parser.write(file)
\ No newline at end of file
parser.write(file)
software/ipython_notebook/template/ipython_set_password.cgi.jinja → software/jupyter/jupyter_set_password.cgi.jinja
View file @ f581d489
......@@ -24,16 +24,16 @@ if not os.path.exists(config_file):
parser = ConfigParser.ConfigParser()
parser.read(config_file)
if not parser.has_section("ipython_notebook"):
parser.add_section("ipython_notebook")
if not parser.has_section("jupyter_notebook"):
parser.add_section("jupyter_notebook")
if not parser.has_option("ipython_notebook", "password"):
parser.set("ipython_notebook", "password", "")
if not parser.has_option("jupyter_notebook", "password"):
parser.set("jupyter_notebook", "password", "")
if "password" in form:
parser.set("ipython_notebook", "password", passwd(form["password"].value))
parser.set("jupyter_notebook", "password", passwd(form["password"].value))
# subprocess.call('{{ httpd_graceful }}')
# TODO: we should restart ipython
# TODO: we should restart jupyter
with open(config_file, 'w') as file:
parser.write(file)
......@@ -43,7 +43,7 @@ print "<html><head>"
print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
print "<link rel=\"stylesheet\" href=\"static/style.css\">"
print "</head><body>"
print "<h1>IPython Notebook Password :</h1>"
print "<h1>Jupyter Notebook Password :</h1>"
print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
......
software/ipython_notebook/template/kernel.json.jinja → software/jupyter/kernel.json.jinja
View file @ f581d489
......@@ -2,6 +2,7 @@
"argv": [
"{{python_executable}}",
"{{kernel_dir}}",
"{{erp5_url}}",
"-f",
"{connection_file}"
],
......
software/ipython_notebook/software.cfg → software/jupyter/software.cfg
View file @ f581d489
......@@ -3,59 +3,40 @@ versions = versions
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../component/ipython/buildout.cfg
../../component/scipy/buildout.cfg
../../component/scikit-learn/buildout.cfg
../../component/pandas/buildout.cfg
../../component/openssl/buildout.cfg
../../component/seaborn/buildout.cfg
../../component/jupyter/buildout.cfg
../../stack/monitor/buildout.cfg
parts =
slapos-cookbook
ipython-notebook
instance-jupyter
[ipython-notebook]
; In the ipython notebook software, we use more eggs than in the minimal
; ipython notebook component
eggs +=
requests
astor
${scipy:egg}
${pandas:egg}
${scikit-learn:egg}
${seaborn:egg}
${statsmodels:egg}
patsy
jupyter
jupyter-notebook-initialized-scripts
instance-jupyter-notebook
[jupyter]
python_executable = ${buildout:bin-directory}/${:interpreter}
[download-file-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/template/${:filename}
url = ${:_profile_base_location_}/${:filename}
download-only = true
mode = 0644
[ipython-notebook-config]
[jupyter-notebook-config]
<= download-file-base
filename = ipython_notebook_config.py.jinja
[ipython-notebook-set-password]
[jupyter-set-password]
<= download-file-base
filename = ipython_set_password.cgi.jinja
[erp5-kernel]
<= download-file-base
filename = ERP5kernel.py.jinja
[kernel-json]
<= download-file-base
filename = kernel.json.jinja
[custom-js]
<= download-file-base
filename = custom.js.jinja
[instance-jupyter]
[instance-jupyter-notebook]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/template.cfg
......@@ -65,11 +46,11 @@ context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key openssl_output openssl-output:openssl
key python_executable python2.7:executable
key ipython_notebook_config_location ipython-notebook-config:location
key ipython_notebook_config_filename ipython-notebook-config:filename
key ipython_notebook_set_password_location ipython-notebook-set-password:location
key ipython_notebook_set_password_filename ipython-notebook-set-password:filename
key python_executable jupyter:python_executable
key jupyter_config_location jupyter-notebook-config:location
key jupyter_config_filename jupyter-notebook-config:filename
key jupyter_set_password_location jupyter-set-password:location
key jupyter_set_password_filename jupyter-set-password:filename
key erp5_kernel_location erp5-kernel:location
key erp5_kernel_filename erp5-kernel:filename
key kernel_json_location kernel-json:location
......@@ -90,6 +71,8 @@ ipython-genutils = 0.1.0
ipywidgets = 6.0.0
jupyter-client = 5.0.0
jupyter-core = 4.3.0
jupyterlab = 0.26.3
jupyterlab-launcher = 0.3.1
matplotlib = 2.0.0
mistune = 0.7.3
nbformat = 4.3.0
......@@ -162,3 +145,10 @@ subprocess32 = 3.2.7
# Required by:
# prompt-toolkit==1.0.13
wcwidth = 0.1.7
jupyter = 1.0.0
jupyter-console = 5.1.0
# Required by:
# jupyter==1.0.0
qtconsole = 4.3.0
software/kvm/software.cfg
View file @ f581d489
......@@ -5,7 +5,7 @@ extends = common.cfg
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1
slapos.toolbox = 0.69
slapos.toolbox = 0.71
erp5.util = 0.4.49
apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0
......@@ -16,23 +16,23 @@ smmap = 0.9.0
# websockify = 0.8.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
dnspython = 1.14.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
lockfile = 0.12.2
# Required by:
......@@ -40,9 +40,9 @@ lockfile = 0.12.2
numpy = 1.11.2rc1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
paramiko = 2.0.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
passlib = 1.6.5
\ No newline at end of file
software/nayuos/README.md
View file @ f581d489
# NayuOS
This is a SlapOS recipe to build NayuOS. It needs to be put in the <code>/srv/slapgrid/\<part\>/srv/runner/project/slapos/software/</code> directory. The created directory is called <code>\<nayuos_build_dirname\></code> in this documentation.
This is a SlapOS recipe to build NayuOS.
## License
......@@ -8,42 +8,63 @@ GPL v2 or later
## Requirements
* sudo on the host (for now)
* some environment variables need to be authorized to be propagated when cros_sdk calls sudo:
* sudo on the host
* environment variables need to be authorized to be propagated when cros_sdk calls sudo: `: Defaults env_keep += "CROS_CACHEDIR DEPOT_TOOLS"`
* slapuser with sudo rights to execute the cros_sdk scripts (needed to access the chroot environment provided by Chromium OS)
in /etc/sudoers (replace slapuser9 by your user, and release-R48-7647.B by the release you have chosen): `: slapuser9 ALL=NOPASSWD: /srv/slapgrid/slappart9/srv/runner/instance/slappart0/parts/chromiumos/release-R48-7647.B/chromite/bin/cros_sdk, /srv/slapgrid/slappart9/srv/runner/instance/slappart0/wrapper_bin/wrapper_cros_sdk, /bin/kill`
: Defaults env_keep += "CROS_CACHEDIR DEPOT_TOOLS"
It's useful to have the right to kill cros_sdk processes, when needed. ;)
* slapuser with sudo rights to execute the cros_sdk scripts (needed to access the chroot environment provided by Chromium OS)
in /etc/sudoers (replace slapuser9 by your user, and release-R48-7647.B by the release you have chosen):
## Technical notes
: slapuser9 ALL=NOPASSWD: /srv/slapgrid/slappart9/srv/runner/instance/slappart0/parts/chromiumos/release-R48-7647.B/chromite/bin/cros_sdk, /srv/slapgrid/slappart9/srv/runner/instance/slappart0/wrapper_bin/wrapper_cros_sdk, /bin/kill
After any change to the build process it is necessary to delete (using sudo)
the building environment at `~/srv/runner/instance/slappart0/parts/chromiumos/<TAG>`.
BEWARE that the web runner is serving images for the [official website](https://nayuos.nexedi.com).
NayuOS and ChromiumOS is "just" a version of Gentoo. Thus it uses `ebuild` packages
and anything installable in Gentoo can be installed to NayuOS too. Of course only
during OS build phase and one has to count with limited space.
### Upgrading (building new image)
Please read **Requirements** section carefully. After selecting your desired
`release` from the list <https://chromium.googlesource.com/chromiumos/manifest/+refs>
it is **necessary** to add `sudo` rules for that release as shown there.
It's useful to have the right to kill cros_sdk processes, when needed. ;)
## Input
In the vifib parameters (softinst\<nb\>.host.vifib.net \> Services \> Parameters):
* board / ex: peppy, swanky, ... (choosing daisy will accept all licenses for the daisy board build only, in order to use Mali drivers, see [chromium mailing list](https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/Pf9ZG2itxWM))
* branch / ex: release-R46-7390.B (you can find the release in the [Chromium OS source tree](https://chromium.googlesource.com/chromiumos/manifest/+refs))
* keep_cache / yes|no (choosing "no" saves about 15Go of disk space per board, choosing "yes" will makes next build faster and less expensive in term of needed ressources because of not rebuilding everything)
* **board** / ex: peppy, swanky, ... Complete list of devices and board names on [chromiumOS developer guide](http://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices)
* **branch** / ex: release-R46-7390.B (you can find the release in the [Chromium OS source tree](https://chromium.googlesource.com/chromiumos/manifest/+refs))
* **keep_cache** / yes|no (choosing "no" saves about 15Go of disk space per board, choosing "yes" will makes next build faster and less expensive in term of needed ressources because of not rebuilding everything)
//Choosing board daisy will accept all licenses for the daisy board build only, in order to use Mali drivers, see [chromium mailing list](https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/Pf9ZG2itxWM)
## Output
The image will be produced in:
<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/images/</code>
and the logs are in:
<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/var/log/cros_sources_dl.log</code> and <code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/var/log/cros_build.log</code>
The script that download the sources and build is located in
<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/etc/run</code>
Software release produces a build script `<instance_partition>/etc/run/cros_full_build`.
Build produces
* Image: `<instance_partition>/parts/chromiumos/images/`
* Compilation logs: `<instance_partition>/var/log/cros_sources_dl.log`
* Build logs: `<instance_partition>/var/log/cros_build.log`
## External documents
* [ <code>repo</code> command reference ](https://source.android.com/source/using-repo.html)
* [ NayuOS official website ](https://www.nayuos.com)
* [ `repo` command reference ](https://source.android.com/source/using-repo.html)
* [ NayuOS official website ](https://nayuos.nexedi.com)
* [ Board names list ](http://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices)
* [ crouton for chroot ](https://github.com/dnschneid/crouton) ([warning about verified boot](https://github.com/dnschneid/crouton/blob/2a1fc9da380650f47e2bcf37d00962bfb68c4830/installer/main.sh#L517-L536))
## Notes for possible improvements
* [ Running virtual machines on your chromebook ](https://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices/running-virtual-machines-on-your-chromebook)
* to have a more common User Agent (the one of ChromiumOS/NayuOS is quite rare and identifies the user, see [studies of the EFF](https://panopticlick.eff.org/static/browser-uniqueness.pdf)), it seems possible to change the User-Agent flag for guest mode in the getOffTheRecord function, and adding a line (key "kUserAgent" , value "some common user agent" string). Then rebuild Chromium and [add it to NayuOS](https://www.chromium.org/chromium-os/developer-guide#TOC-Making-changes-to-the-Chromium-web-).
* remove need of root priviledge for entering the chroot, maybe by using fakeroot in 'scripts/wrapper_sudo.in'?
* change more options on Chromium OS "Privacy" part by default: there are [a few options](https://support.google.com/chromebook/answer/114836) which still use Google services
* provide ChromiumOS package manager [ chromebrew ](https://skycocker.github.io/chromebrew/) by default
software/nayuos/instance.cfg
View file @ f581d489
# Jinja2 template of a buildout file to create runnable scripts
#
# Received variables
# - software_dir: location where this repository was cloned to
# - instance_dir: location where this 'instance.cfg' is gettin rendered (same as ${buildout:directory})
[buildout]
parts =
parameters
promise-sudo-on-host
template-full-build-script
template-sudo-wrapper
template-cros-sdk-wrapper
# eggs given by software.cfg
# standard declaration of eggs directories
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
......@@ -16,15 +23,18 @@ develop-eggs-directory = {{ develop_eggs_directory }}
recipe = slapos.recipe.build:gitclone
repository = https://chromium.googlesource.com/chromium/tools/depot_tools.git
branch = master
git-executable = {{ git_path }}/bin/git
[customize-path]
# add depot tools directory (for cros_sdk binary among others) and git directory to the path
command =
export PATH="${directory:wrapper_dir}":"{{ git_path }}":"{{ curl_path }}":"${depot-tools:location}":"$PATH";
export PATH="${directory:wrapper_dir}":"{{ git_path }}/bin":"{{ curl_path }}/bin":"${depot-tools:location}":"{{thin_provisioning_tools}}/sbin":"$PATH";
[nayuos-ebuilds]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nayuos-ebuilds.git
branch = master
git-executable = {{ git_path }}/bin/git
############################################################################################
......@@ -44,20 +54,25 @@ cert = ${slap_connection:cert_file}
recipe = slapos.cookbook:mkdirectory
log = ${buildout:directory}/var/log
run = ${buildout:directory}/etc/run
promise = ${buildout:directory}/etc/promise
wrapper_dir = ${buildout:directory}/wrapper_bin
cros_location = ${buildout:directory}/parts/chromiumos
ebuilds_dir = ${nayuos-ebuilds:location}
scripts_dir = {{ scripts_dir }}
logo_dir = {{ logo_dir }}
[bin]
# dummy section to hold references to frequently used binaries
wrapper_cros_sdk=${directory:wrapper_dir}/wrapper_cros_sdk
bash=/bin/bash
sudo=/usr/bin/sudo
[promise-sudo-on-host]
# assert sudo is installed, as it is required to enter the chroot 'cros_sdk'
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:promise}/${:_buildout_section_name_}
command-line = sudo -V
[template-sudo-wrapper]
recipe = slapos.recipe.template:jinja2
template = {{ scripts_dir }}/wrapper_sudo.in
template = {{ software_dir }}/scripts/wrapper_sudo.in
rendered = ${directory:wrapper_dir}/sudo
md5sum = ded5a92be4e37ec32eb9d3087d3e19bd
mode = 0700
......@@ -68,33 +83,37 @@ context =
[template-cros-sdk-wrapper]
recipe = slapos.recipe.template:jinja2
template = {{ scripts_dir }}/wrapper_cros_sdk.in
template = {{ software_dir }}/scripts/wrapper_cros_sdk.in
rendered = ${bin:wrapper_cros_sdk}
md5sum = 7159fe3d5b85a283733cf686c4ee0a74
mode = 0700
context =
# XXX bash path is the one from the host
key bash_path bin:bash
raw git_path {{ git_path }}
raw curl_path {{ curl_path }}
raw git_path {{ git_path }}/bin
raw curl_path {{ curl_path }}/bin
[template-full-build-script]
# create the rendered script in the buildout-directory/etc/run
# (use jinja for templating)
recipe = slapos.recipe.template:jinja2
template = {{ scripts_dir }}/cros_full_build.in
template = {{ software_dir }}/scripts/cros_full_build.in
rendered = ${directory:run}/cros_full_build
md5sum = 75599e6b8418a5f3756c7c7b26600399
md5sum = b2ef515a871b18216baf187485be74f8
mode = 0700
context =
key bash_path bin:bash
key instance_log_dir directory:log
key cros_location directory:cros_location
key export_path_cmd customize-path:command
key branch parameters:configuration.branch
key boards_list parameters:configuration.boards
key keep_cache parameters:configuration.keep_cache
key ebuilds_dir directory:ebuilds_dir
key scripts_dir directory:scripts_dir
key logo_dir directory:logo_dir
raw nayu_dev_packages net-libs/nodejs net-misc/re6stnet dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup
key bash_path bin:bash
key instance_log_dir directory:log
key cros_location directory:cros_location
key export_path_cmd customize-path:command
key branch parameters:configuration.branch
key boards_list parameters:configuration.boards
key keep_cache parameters:configuration.keep_cache
key ebuilds_dir nayuos-ebuilds:location
raw scripts_dir {{ software_dir }}/scripts
raw logo_dir {{ software_dir }}/logo
# packages to be copied from our overlay into ChromiuOS' overlay
# those packages will be marked as dependency of root filesystem
raw nayu_dev_rootfs_packages app-misc/nayuos-chromium-policy
# those packages will be marked as dependency of the chromium build itself
raw nayu_dev_packages net-misc/re6stnet dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup
software/nayuos/scripts/cros_full_build.in
View file @ f581d489
#! {{ bash_path }}
function increase_revision_number() {
# this increase the revision number an ebuild
# (except for revision number 9999 which is a value for "live ebuild", for dev revisions)
# see https://devmanual.gentoo.org/general-concepts/ebuild-revisions/
if [ -z "$1" ] ; then
echo "ERROR: No parameter given to function: increase_revision_number."
echo "Usage: increase_revision_number /path/to/dir/containing/ebuild"
exit 1
elif [ -d "$1" ]; then
DIR="$1"
CURRENT_FILE=""
CURRENT_REVISION=0
# get the biggest revision number
# XXX: may break if there are many version of an ebuild
# (eg: bash-4.3_p42-r4.ebuild and bash-3.7-r8.ebuild)
for FILENAME in $(find "${DIR}" -maxdepth 1 -regextype sed -regex .*-r[0-9]*\.ebuild); do
echo $FILENAME
REVISION="$(echo ${FILENAME} | rev | cut -d- -f1 | rev | tr -d [:alpha:] | tr -d [:punct:])"
echo "${REVISION}"
if [ "$CURRENT_REVISION" -lt "$REVISION" ] ; then
CURRENT_REVISION=${REVISION}
CURRENT_FILE=${FILENAME}
fi
done
if [ "$CURRENT_REVISION" -ne "9999" ]; then
# increase REVISION
NEW_REVISION=$((CURRENT_REVISION+1))
NEW_FILE="$(echo ${CURRENT_FILE} | sed -r 's/-r'${CURRENT_REVISION}'/-r'${NEW_REVISION}'/')"
echo "changing revision number: ${CURRENT_FILE} -> ${NEW_FILE}"
mv "${CURRENT_FILE}" "${NEW_FILE}"
fi
fi
}
function insert_packages_into_ebuild() {
# Insert packages (in form of "category/package") into RDEPEND part of ebuild.
# Optionally increasing version of given ebuild.
EBUILD=$1
changed=0
shift
while [ ! -z "$1" ]
do
package=$1
if [[ $( grep "${package}" "${EBUILD}" ) ]] ; then
echo "no need to change ${EBUILD} file to add ${package}..." >> "${BUILD_LOG}"
else
printf "\n\nRDEPEND=\"\${RDEPEND}\n ${package}\"\n">> ${EBUILD}
changed=1
fi
shift
done;
if [ $changed -eq 1 ]
then
# if we inserted at least one package then increase the version to be sure
# it gets selected in case of more overlays with the same package
increase_revision_number $(dirname $EBUILD)
fi
}
function latest_ebuild() {
# find the latest ebuild file (previously we hardcoded the versions)
# @argument ebuild directory to find the file in
find $1 -type f -name '*ebuild' | sort -r | head -1
}
######################## Download of sources using the "repo" command ########################
DL_LOG="{{ instance_log_dir }}/cros_sources_dl.log"
BRANCH="{{ branch }}"
CHROMIUM_OVERLAY="{{ cros_location }}/{{ branch }}/src/third_party/chromiumos-overlay"
PORTAGE_STABLE="{{ cros_location }}/{{ branch }}/src/third_party/portage-stable"
{{ export_path_cmd }}
echo "getting Chromium OS sources..." >> $DL_LOG
......@@ -22,6 +94,8 @@ repo init -u https://chromium.googlesource.com/chromiumos/manifest.git -b {{ bra
# in case changes were already made, stash changes to be able to pull
cd ${CHROMIUM_OVERLAY}
git stash
cd ${PORTAGE_STABLE}
git stash
repo sync >> $DL_LOG || { echo "Problem while downloading Chromium OS sources (repo sync). Exiting." ; exit 1 ; }
......@@ -38,10 +112,10 @@ cros_sdk --download || { echo "Problem while entering chroot or downloading chro
# it is needed by re6st
# just need to change -sqlite by sqlite on the right line
# same for IPv6 support
PACKAGE_USE_FILE=${CHROMIUM_OVERLAY}/profiles/targets/chromeos/package.use
PACKAGE_USE_FILE="${CHROMIUM_OVERLAY}/profiles/targets/chromeos/package.use"
line_number=$(sed -n '/dev-lang\/python/=' ${PACKAGE_USE_FILE})
if [ $line_number ] ; then
echo $line_number
echo "$line_number"
sed -i "${line_number}s/-sqlite/sqlite/" ${PACKAGE_USE_FILE}
if [[ ! $(sed -n ${line_number}p ${PACKAGE_USE_FILE} | grep ipv6) ]]; then
sed -i "${line_number}s/$/ ipv6/" ${PACKAGE_USE_FILE}
......@@ -51,57 +125,169 @@ fi
# add some ebuilds, specific to NayuOS
for category in $( ls {{ ebuilds_dir }} ); do
echo ${category}
cp -R {{ ebuilds_dir }}/${category}/* ${CHROMIUM_OVERLAY}/${category}/
cp -R "{{ ebuilds_dir }}/${category}/"* "${CHROMIUM_OVERLAY}/${category}/"
done
install -m 770 "{{ scripts_dir }}/test_nayuos_image" "{{ cros_location }}/{{ branch }}/src/scripts/test_nayuos_image"
### packages management ###
BASE_CHROMEOS_DEV_ROOT_EBUILD=${CHROMIUM_OVERLAY}/chromeos-base/chromeos-dev-root/chromeos-dev-root-0.0.1.ebuild
VIRTUAL_CHROMEOS_OS_DEV_EBUILD=${CHROMIUM_OVERLAY}/virtual/target-chromium-os-dev/target-chromium-os-dev-1.ebuild
# change the virtual ebuild responsible for installing all packages to add the ones
# needed for NayuOS
for package in {{ nayu_dev_packages }} ; do
echo $package
if [[ $( grep "${package}" "${VIRTUAL_CHROMEOS_OS_DEV_EBUILD}" ) ]] ; then
echo "no need to change ${VIRTUAL_CHROMEOS_OS_DEV_EBUILD} file to add ${package}..." >> "${BUILD_LOG}"
else
printf "\n\nRDEPEND=\"\${RDEPEND}\n ${package}\"\n">> ${VIRTUAL_CHROMEOS_OS_DEV_EBUILD}
fi
done
# make virtual ebuild responsible for installing all packages dependent on NayuOS own packages
insert_packages_into_ebuild \
$(latest_ebuild ${CHROMIUM_OVERLAY}/virtual/target-chromium-os-dev) \
net-misc/re6stnet dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup
# insert policies into /etc/chromium/policies/recommended with NayuOS customizations
CHROMEOS_DEV_ROOT_EBIULD=
insert_packages_into_ebuild \
$(latest_ebuild ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-dev-root) \
app-misc/nayuos-chromium-policy
# do not install the Upstart init script that starts ssh daemon at boot time
rm ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/files/openssh-server.conf
sed -i -n '/src_install/q;p' ${CHROMIUM_OVERLAY}/chromeos-base/openssh-server-init/openssh-server-init-0.0.1.ebuild
sed -i -n '/src_install/q;p' ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/chromeos-sshd-init-0.0.1.ebuild
find ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init* -name openssh-server.conf -delete
# strip away the src_install part because it only sets up booting scripts
# and increase version in case of other overlays with the same packages
sed -i -n '/src_install/q;p' $(latest_ebuild ${CHROMIUM_OVERLAY}/chromeos-base/openssh-server-init)
increase_revision_number ${CHROMIUM_OVERLAY}/chromeos-base/openssh-server-init/
sed -i -n '/src_install/q;p' $(latest_ebuild ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init)
increase_revision_number ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/
# bashrc modifications
BASH_EBUILD_DIR={{ cros_location }}/{{ branch }}/src/third_party/portage-stable/app-shells/bash
if [[ $(grep "git --exec-path" "${BASH_EBUILD_DIR}/files/dot-bashrc") ]] ; then
echo "git alias already set"
else
echo "# git quickfix for finding right git executables
if [ -d /usr/local/libexec/git-core/ ] ; then
alias git='git --exec-path=/usr/local/libexec/git-core/'
BASH_EBUILD_DIR="${PORTAGE_STABLE}/app-shells/bash"
if ! grep "BEGIN NayuOS configuration" "${BASH_EBUILD_DIR}/files/dot-bashrc" > /dev/null ; then
cat >> "${BASH_EBUILD_DIR}/files/dot-bashrc" <<EOF
# ----- BEGIN NayuOS configuration -----
# use vim as default editor if nano does not exist
which nano &> /dev/null || export EDITOR=vim
# git quickfix for finding right git executables
export GIT_EXEC_PATH=/usr/local/libexec/git-core
# configure .gitconfig once
if [ ! -e ~/.gitconfig ] ; then
which less &> /dev/null && git config --global core.pager less
fi
# git quickfix for finding 'less' and using it as pager
less_path=\$(which less 2>/dev/null)
if [ "\${less_path}" ] ; then
git config --global core.pager \${less_path}
fi" >> ${BASH_EBUILD_DIR}/files/dot-bashrc
# go to ~/Download
[ "\$(pwd)" = / ] && cd ~/Downloads
alias ls='ls -CF --color=auto'
alias l='ls -lF'
alias lh='ls -hAl'
alias la='ls -la'
alias ll='ls -l'
tree() {
python -Sc 'import sys, os
aa = sys.argv[1:] or ["."]
for a in aa:
o = len(a.split(os.sep))
for r, dd, ff in os.walk(a):
dp = len(r.split(os.sep)) - o + 1
print "%s- %s" % ((dp - 1) * " ", os.path.basename(r))
for f in ff:
print "%s- %s" % (dp * " ", f)' "\$@"
}
showzip() {
[ \$# = 1 ] || { echo "Usage: showzip ZIPFILE" >&2 ; return 1 ; }
python -Sc 'import sys, zipfile
zfile = zipfile.ZipFile(sys.argv[1])
zfile.printdir()
zfile.close()' "\$@"
}
zip() {
[ \$# -gt 1 ] || { echo "Usage: zip ZIPFILE FILE..." >&2 ; return 1 ; }
python -Sc 'import sys, zipfile, os
if os.path.lexists(sys.argv[1]):
sys.stderr.write("zip: file %r already exists\n" % sys.argv[1])
sys.exit(1)
zfile = zipfile.ZipFile(sys.argv[1], "w", zipfile.ZIP_DEFLATED)
def arc(arg):
if os.path.islink(arg):
sys.stderr.write("zip: symlink %r ignored\n" % arg)
elif os.path.isdir(arg):
for r, _, ff in os.walk(arg):
zfile.write(r)
for f in ff: arc(os.path.join(r, f))
else:
zfile.write(arg)
for arg in sys.argv[2:]: arc(arg)
zfile.close()' "\$@"
}
unzip() {
[ \$# != 0 ] || { echo "Usage: unzip ZIPFILE [TARGET]" >&2 ; return 1 ; }
python -Sc 'import sys, zipfile
zfile = zipfile.ZipFile(sys.argv[1])
zfile.extractall(sys.argv[2] if len(sys.argv) > 2 else ".")
zfile.close()' "\$@"
}
encmount() {
if [ -z \$1 ] ; then
echo "Please, give a device as parameter, ex: /dev/sda"
return 1
fi
if [ ! -d /media/removable/key ] ; then
echo "New dir at /media/removable/key"
sudo mkdir /media/removable/key
fi
echo "Mounting \$1 on /media/removable/key"
sudo cryptsetup luksOpen \$1 key
sudo mount -o noexec /dev/mapper/key /media/removable/key
}
__safewrite() {
local cmd=safewrite
local usage="Usage: \$cmd DST < stdin
Where DST is the destination file
Copy stdin to DST. Data is flushed regularly to avoid some eventual crash during cache sync.
Example:
gunzip myimage.img.gz -c | sudosafewrite /dev/sda"
local KiB=1024
local MiB=\$((1024 * \$KiB))
local bs=4096
local count=\$((50 * \$MiB / \$bs))
local infostep=50
local infounit=MiB
local safewrite_seek=0
local safewrite_out=
local safewrite_length=\$infostep
[ \$# = 2 ] || { echo "\$usage" >&2 ; return 2 ; }
[ \$1 != - ] || { echo "DST should not be '-'" >&2 ; return 2 ; }
while [ \$? = 0 ] ; do
echo "+ copying until \$safewrite_length \$infounit"
safewrite_out=\$(\$1 bs=\$bs count=\$count seek=\$safewrite_seek of="\$2" 2>&1) || { echo "\$safewrite_out" ; return 1 ; }
safewrite_length=\$((\$safewrite_length + \$infostep))
safewrite_seek=\$((\$safewrite_seek + \$count))
echo "\$safewrite_out"
! echo "\$safewrite_out" | grep '(0 B) copied' > /dev/null
done
return 0
}
safewrite() { __safewrite dd "\$@" ; }
sudodd() { sudo dd "\$@" ; }
sudosafewrite() { __safewrite sudodd "\$@" ; }
# ----- END NayuOS configuration -----
EOF
fi
# increase the revision number
# XXX: remove hardcoded revision numbers
cd ${CHROMIUM_OVERLAY}/chromeos-base/openssh-server-init/
ln -s openssh-server-init-0.0.1-r24.ebuild openssh-server-init-0.0.1-r25.ebuild
cd ${CHROMIUM_OVERLAY}/chromeos-base/chromeos-sshd-init/
ln -s chromeos-sshd-init-0.0.1-r5.ebuild chromeos-sshd-init-0.0.1-r6.ebuild
cd ${BASH_EBUILD_DIR}
ln -s bash-4.2_p48-r1.ebuild bash-4.2_p48-r2.ebuild
increase_revision_number "${BASH_EBUILD_DIR}"
######################################## Build ##############################################
BOARDS="{{ boards_list }}"
......@@ -118,7 +304,7 @@ for board in ${BOARDS} ; do
# preparing packages (for chroot and image)
date >> "${BUILD_LOG}"
echo "building packages for a ${board}-flavoured Chromium OS..." >> "${BUILD_LOG}"
cros_sdk -- ./build_packages --board=${board} >> "${BUILD_LOG}"
cros_sdk --nouse-image -- ./build_packages --board=${board} >> "${BUILD_LOG}"
# change boot pictures
cros_sdk -- cros_workon --board=${board} start chromiumos-assets
......@@ -131,20 +317,22 @@ for board in ${BOARDS} ; do
NAYU_IMAGE_LOCATION=/tmp/${board}.nayuos.img
# rebuild packages with boot pictures
cros_sdk -- ./build_packages --board=${board} >> "${BUILD_LOG}"
cros_sdk --nouse-image-- ./build_packages --board=${board} >> "${BUILD_LOG}"
# NayuOS images
date >> "${BUILD_LOG}"
echo "building image" >> "${BUILD_LOG}"
cros_sdk -- ./build_image --board=${board} dev >> "${BUILD_LOG}" \
cros_sdk --nouse-image -- ./build_image --board=${board} dev >> "${BUILD_LOG}" \
&& cros_sdk -- rm -f $NAYU_IMAGE_LOCATION && cros_sdk -- touch $NAYU_IMAGE_LOCATION \
&& cros_sdk -- cros flash --board=${board} file://$NAYU_IMAGE_LOCATION >> "${BUILD_LOG}" \
&& cros_sdk -- ./test_nayuos_image ${board} > "${TEST_LOG}" \
|| { echo "An error occured while building ${board} NayuOS image. Exiting." ; exit 1 ;}
# && cros_sdk -- ./test_nayuos_image ${board} > "${TEST_LOG}" \
# save ~15Go/device but delete cache (next build will be as long)
# save a lot of space on the server but delete cache and build files
# (it means that the next build will be as long and use as much resources as this one)
if [ ${KEEP_CACHE,,} == "no" ] ; then
cros_sdk -- sudo rm -R /var/cache/chromeos-chrome/chrome-src/src/out_${board}
cros_sdk -- sudo rm -R "/var/cache/chromeos-chrome/chrome-src/src/out_${board}"
cros_sdk -- sudo rm -R "/build/${board}"
fi
if [ ${board} == daisy ]; then
......
software/nayuos/scripts/test_nayuos_image
View file @ f581d489
......@@ -14,8 +14,10 @@ ORIGINAL_GRANDENET_SCRIPT=~/trunk/src/third_party/chromiumos-overlay/net-misc/re
GRANDENET_SCRIPT="usr/local/bin/grandenet"
INIT_SSH_SERVER="etc/init/openssh-server.conf"
BASHRC="etc/skel/.bashrc"
EXPECTED_ALIAS="alias git='git --exec-path=/usr/local/libexec/git-core/'"
GIT_CORE="usr/local/libexec/git-core/"
GIT_EXPECTED_EXPORT="export GIT_EXEC_PATH=/usr/local/libexec/git-core"
VIRTUALENV_BIN="usr/local/bin/virtualenv"
CHROMIUM_POLICY="etc/chromium/policies/recommended/nayuos_policy.json"
HAS_FAILED=0
......@@ -32,14 +34,17 @@ function print_result() {
# MOUNT IMAGE AND GET INFO
install -d ${MOUNTPOINT}
install -d "${MOUNTPOINT}"
./mount_gpt_image.sh --safe -f $( ./get_latest_image.sh --board=${BOARD} ) -r ${MOUNTPOINT}
echo $(ls "${MOUNTPOINT}/usr/local")
if [[ $(ls "${MOUNTPOINT}/usr/local") ]] ; then
my_diff=$(diff ${ORIGINAL_GRANDENET_SCRIPT} "${MOUNTPOINT}/${GRANDENET_SCRIPT}")
my_diff=$(diff "${ORIGINAL_GRANDENET_SCRIPT}" "${MOUNTPOINT}/${GRANDENET_SCRIPT}")
opensshd_config=$(ls "${MOUNTPOINT}/${INIT_SSH_SERVER}")
gitalias=$(grep "${EXPECTED_ALIAS}" "${MOUNTPOINT}/${BASHRC}")
gitcore=$(ls "${MOUNTPOINT}/${GIT_CORE}")
gitexport=$(grep "${GIT_EXPECTED_EXPORT}" "${MOUNTPOINT}/${BASHRC}")
virtualenvbin=$(ls "${MOUNTPOINT}/${VIRTUALENV_BIN}")
chromium_policy=$(ls "${MOUNTPOINT}/${CHROMIUM_POLICY}")
else
is_empty=1
fi
......@@ -50,7 +55,7 @@ rmdir ${MOUNTPOINT}
# PRINT RESULTS
echo "* test if /usr/local exists"
if [[ ${is_empty} == 1 ]] ; then
if [[ "${is_empty}" == "1" ]] ; then
print_result ${FAILURE} "/usr/local is empty."
else
print_result ${SUCCESS} "/usr/local is not empty."
......@@ -71,10 +76,18 @@ else
print_result ${SUCCESS} "opensshd config removed."
fi
echo "* test git quick fix for option --exec-path"
echo "* test if git core directory is not empty"
if [[ "${gitcore}" == "" ]] ; then
print_result ${FAILURE} "No git file in ${MOUNTPOINT}/${GIT_CORE}."
else
print_result ${SUCCESS} "git core directory exists and is not empty."
fi
echo "* test bashrc changes for git paths quick fix"
if [[ ${gitalias} == "" ]] ; then
print_result ${FAILURE} "Expected alias for git command not in ${MOUNTPOINT}/${BASHRC}. Should be: ${EXPECTED_ALIAS}"
if [[ "${gitexport}" == "" ]] ; then
print_result ${FAILURE} "Expected alias for git command not in ${MOUNTPOINT}/${BASHRC}. There should be: ${GIT_EXPECTED_EXPORT}"
else
print_result ${SUCCESS} "git alias is correct."
fi
......@@ -86,6 +99,14 @@ else
else
print_result ${FAILURE} "Expected virtualenv binary not in ${MOUNTPOINT}/${VIRTUALENV_BIN}."
fi
echo "* test if Chromium policies are installed"
if [[ ${chromium_policy} ]] ; then
print_result ${SUCCESS} "Chromium policies are installed."
else
print_result ${FAILURE} "Expected Chromium policies file not in ${MOUNTPOINT}/${CHROMIUM_POLICY}."
fi
fi
exit ${HAS_FAILED}
software/nayuos/software.cfg
View file @ f581d489
......@@ -2,48 +2,55 @@
extends =
# basic needs
../../stack/slapos.cfg
../../component/automake/buildout.cfg
../../component/libexpat/buildout.cfg
../../component/libaio/buildout.cfg
../../component/boost-lib/buildout.cfg
parts +=
# use stack/slapos.cfg
slapos-cookbook
verify-packages-exist-on-host
template-instance
install-eggs-for-the-instance
git
curl
thin-provisioning-tools
versions = versions
[versions]
slapos.recipe.template = 3.0
[verify-packages-exist-on-host]
recipe = plone.recipe.command
stop-on-error = true
# run the same command when installing and on updates
update-command = ${:command}
command = sudo -V
[template-instance]
# create the instance.cfg file in the buildout directory
# (use jinja for templating)
# jinja2 render instance.cfg file in the buildout directory
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg
rendered = ${buildout:directory}/instance.cfg
md5sum = be6bd665c0b356a84090d200ea14b33c
mode = 0644
scripts_dir = ${:_profile_base_location_}/scripts
logo_dir = ${:_profile_base_location_}/logo
curl_path = ${curl:location}/bin
git_path = ${git:location}/bin
md5sum = 22be269701616bb6f5453eb769e0cdaa
context =
# for access to the eggs from the instance
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key scripts_dir :scripts_dir
key logo_dir :logo_dir
key curl_path :curl_path
key git_path :git_path
key software_dir :_profile_base_location_
key instance_dir buildout:directory
key curl_path curl:location
key git_path git:location
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key thin_provisioning_tools thin-provisioning-tools:location
[install-eggs-for-the-instance]
# after installation of the eggs, the recipes
# will be available (added to sys.path)
recipe = zc.recipe.egg
eggs = slapos.recipe.build
plone.recipe.command
[thin-provisioning-tools]
recipe = slapos.recipe.cmmi
location = ${buildout:parts-directory}/${:_buildout_section_name_}
url = https://github.com/jthornber/thin-provisioning-tools/archive/v0.7.1.tar.gz
md5sum = 44cc03dd39a55b2f0364b330850d63fd
pre-configure = ${autoconf:location}/bin/autoconf
make-options = PREFIX=${:location}
# environment has to define CXXFLAGS and not CPPFLAGS because of the ./configure
# which has kind-of special behaviour
environment =
PATH=${m4:location}/bin:${automake:location}/bin:${autoconf:location}/bin:%(PATH)s
CXXFLAGS=-I${libexpat:location}/include -I${libaio:location}/include -I${boost-lib:location}/include
LDFLAGS=-L${libexpat:location}/lib -L${libaio:location}/lib -L${boost-lib:location}/lib
dependencies =
automake
libexpat
libaio
boost-lib
software/neoppod/software-common.cfg
View file @ f581d489
......@@ -113,44 +113,44 @@ apache-libcloud = 1.5.0
ecdsa = 0.13
gitdb2 = 2.0.0
msgpack-python = 0.4.8
mysqlclient = 1.3.10+SlapOSPatched001
mysqlclient = 1.3.10+SlapOSPatched002
persistent = 4.2.3
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.71
smmap2 = 2.0.1
transaction = 1.7.0
zodbpickle = 0.6.0
zodbtools = 0.0.0.dev3
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
GitPython = 2.1.3
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
PyRSS2Gen = 1.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
dnspython = 1.14.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
feedparser = 5.2.1
# slapos.toolbox==0.69
# slapos.toolbox==0.71
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
paramiko = 2.1.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
passlib = 1.7.1
software/nginx-push-stream/software.cfg
View file @ f581d489
......@@ -46,4 +46,4 @@ mode = 0644
[versions]
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.71
software/re6stnet/software.cfg
View file @ f581d489
......@@ -112,15 +112,15 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.71
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
atomize = 0.2.0
# Required by:
......@@ -128,11 +128,11 @@ atomize = 0.2.0
backports.ssl-match-hostname = 3.4.0.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
feedparser = 5.1.3
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
lockfile = 0.12.2
# Required by:
......@@ -140,10 +140,10 @@ lockfile = 0.12.2
miniupnpc = 1.9
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
paramiko = 2.0.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
rpdb = 0.1.5
software/slapos-master/buildout.hash.cfg
View file @ f581d489
......@@ -15,11 +15,11 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 50834c59ded302034cafcce7315a55c3
md5sum = aa10af616493b97d2b630c46d869a9d7
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = 5562e6738470b0fc96211f46bf8d41bc
md5sum = c5ce18fa4d4be9b9a2d789f3bbd37840
[template-apache-backend-conf]
filename = apache-backend.conf.in
......
software/slapos-master/instance-balancer.cfg.in
View file @ f581d489
......@@ -204,7 +204,7 @@ crl =
{% do part_list.append('logrotate-' ~ base_name) -%}
{% do apache_service_log_list.__setitem__(family_name, base_name) -%}
[logrotate-{{ base_name }}]
< = logrotate-entry-base
< = logrotate-entry-base
name = {{ base_name }}
log = ${apache-conf-parameter-dict:log-dir}/{{ base_name }}-error.log ${apache-conf-parameter-dict:log-dir}/{{ base_name }}-access.log
post = test ! -s ${apache-conf-parameter-dict:pid-file} || {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf-parameter-dict:pid-file} -s USR1
......
software/slapos-master/instance-erp5.cfg.in
View file @ f581d489
......@@ -9,6 +9,7 @@
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set monitor_base_url_dict = {} -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
[request-common]
<= request-common-base
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
......@@ -36,12 +37,13 @@ config-{{ k }} = {{ dumps(v) }}
{% for k, v in key_config.iteritems() -%}
config-{{ k }} = {{ '${' ~ v ~ '}' }}
{% endfor -%}
config-name = {{ name }}
{% endmacro -%}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{% if has_posftix -%}
{{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
{%- else %}
......@@ -199,6 +201,7 @@ config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longreque
config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
config-name = {{ partition_name }}
{% endfor -%}
{# if not explicitly configured, connect jupyter to first zope family, which -#}
......@@ -310,7 +313,10 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-shared-certificate-authority-path = ${directory:ca-dir}
config-monitor-passwd = ${monitor-htpasswd:passwd}
config-name = ${:name}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
[request-frontend-base]
{% if has_frontend -%}
......@@ -366,7 +372,7 @@ parts +=
monitor-base
[monitor-conf-parameters]
monitor-title = ERP5 monitor
monitor-title = ERP5
password = ${monitor-htpasswd:passwd}
[monitor-base-url-dict]
......
software/slaprunner/buildout.hash.cfg
View file @ f581d489
......@@ -31,7 +31,7 @@ md5sum = 5cfa49bcf20612844e1c50a85740d0b3
[template-runner-export-script]
filename = template/runner-export.sh.jinja2
md5sum = ef87691a9675beb646ae1c99ef69e551
md5sum = 94c0eddb2af3290942f64e04f95a707c
[instance-runner-export]
filename = instance-runner-export.cfg.in
......
software/slaprunner/software.cfg
View file @ f581d489
......@@ -17,41 +17,41 @@ gunicorn = 19.4.5
prettytable = 0.7.2
pycurl = 7.43.0
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.71
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
PyRSS2Gen = 1.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
dnspython = 1.14.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
erp5.util = 0.4.49
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
paramiko = 2.0.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.71
passlib = 1.6.5
software/slaprunner/template/runner-export.sh.jinja2
View file @ f581d489
......@@ -20,7 +20,7 @@ tmp_directory='{{ directory["tmp"] }}'
rsync () {
set -x
'{{ rsync_binary }}' -rlptgov --stats --safe-links --delete --delete-excluded "$@"
'{{ rsync_binary }}' -rlptgov --stats --safe-links --ignore-missing-args --delete --delete-excluded "$@"
set +x
}
......
software/wendelin/software.cfg
View file @ f581d489
......@@ -11,7 +11,8 @@ parts +=
msgpack-python
ipython
wendelin.core
ipython-notebook
jupyter
jupyter-notebook-initialized-scripts
wendelin_test_suite_runner
wendelin_testrunner
......@@ -59,15 +60,15 @@ initialization =
repository_id_list += wendelin
[local-bt5-repository]
# we need to override it
list = ${erp5:location}/bt5 ${erp5:location}/product/ERP5/bootstrap ${wendelin:location}/bt5/
list += ${wendelin:location}/bt5
# Jupyter is by default enabled in Wendelin
[erp5-defaults]
jupyter-enable-default = true
[wendelin]
<= erp5
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
repository = https://lab.nexedi.com/nexedi/wendelin.git
branch = master
......
stack/boinc/buildout.cfg
View file @ f581d489
......@@ -69,7 +69,7 @@ pycrypto = 2.6
apache-libcloud = 0.12.4
async = 0.6.1
gitdb = 0.5.4
mysqlclient = 1.3.10+SlapOSPatched001
mysqlclient = 1.3.10+SlapOSPatched002
plone.recipe.command = 1.1
slapos.recipe.template = 2.4.2
slapos.toolbox = 0.40.4
......
stack/caucase/buildout.cfg
View file @ f581d489
......@@ -88,17 +88,18 @@ context =
[versions]
Flask-User = 0.6.11
SQLAlchemy = 1.1.9
caucase = 0.1.3
apache-libcloud = 2.1.0
bcrypt = 3.1.3
caucase = 0.1.4
futures = 3.1.1
gitdb2 = 2.0.2
gunicorn = 19.7.1
slapos.recipe.template = 3.0
APacheDEX = 1.6.2
bcrypt = 3.1.3
slapos.toolbox = 0.71
smmap2 = 2.0.3
# Required by:
# caucase==0.1.3
# caucase==0.1.4
Flask-AlchemyDumps = 0.0.10
# Required by:
......@@ -122,6 +123,18 @@ Flask-Script = 2.0.5
# Flask-User==0.6.11
Flask-WTF = 0.14.2
# Required by:
# slapos.toolbox==0.71
GitPython = 2.1.5
# Required by:
# slapos.toolbox==0.71
PyRSS2Gen = 1.1
# Required by:
# Flask-AlchemyDumps==0.0.10
SQLAlchemy = 1.1.11
# Required by:
# Flask-AlchemyDumps==0.0.10
Unipath = 1.1
......@@ -130,14 +143,45 @@ Unipath = 1.1
# Flask-WTF==0.14.2
WTForms = 2.1
# Required by:
# slapos.toolbox==0.71
atomize = 0.2.0
# Required by:
# Flask-Mail==0.9.1
blinker = 1.4
# Required by:
# caucase==0.1.3
# slapos.toolbox==0.71
dnspython = 1.15.0
# Required by:
# slapos.toolbox==0.71
erp5.util = 0.4.49
# Required by:
# slapos.toolbox==0.71
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.71
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.71
passlib = 1.7.1
# Required by:
# caucase==0.1.4
pem = 16.1.0
# caucase==0.1.4
pyasn1 = 0.2.3
# Required by:
# caucase==0.1.4
pyasn1-modules = 0.0.9
# Required by:
# Flask-User==0.6.11
pycryptodome = 3.4.6
stack/caucase/buildout.hash.cfg
View file @ f581d489
......@@ -28,7 +28,7 @@ md5sum = a317d2f948cd3d16c860d05cc07ecf42
[template-caucase]
filename = instance-caucase.cfg.jinja2.in
md5sum = 1988d9c98c2331f49335a1b9a4c26026
md5sum = 7db9e8bf23cf4689e7986c381b94d2cb
[instance-caucase]
filename = instance.cfg.in
......
stack/caucase/instance-caucase.cfg.jinja2.in
View file @ f581d489
......@@ -24,6 +24,12 @@ crl-life-period = {{ slapparameter_dict.get('crl-life-period', 0.02) }}
ca-life-period = {{ slapparameter_dict.get('ca-life-period', 10) }}
# time before clean certificate on CA: 60*24*60*60
crt-keep-time = {{ slapparameter_dict.get('crt-keep-time', 5184000) }}
# number of csr to sign automaticaly, minimum value is 1
{% if int(slapparameter_dict.get('auto-sign-csr-amount', 1)) < 1 -%}
auto-sign-csr-amount = 1
{% else -%}
auto-sign-csr-amount = {{ slapparameter_dict.get('auto-sign-csr-amount', 1) }}
{% endif -%}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -135,6 +141,8 @@ input = inline:
ca-life-period ${ca-parameters:ca-life-period}
# time before clean certificate on CA: 60*24*60*60
crt-keep-time ${ca-parameters:crt-keep-time}
# number of csr to sign automaticaly
auto-sign-csr-amount ${ca-parameters:auto-sign-csr-amount}
output = ${directory:etc}/ca.conf
mode = 700
......@@ -211,6 +219,7 @@ curl_path = {{ curl_executable_location }}
recipe = slapos.cookbook:publish.serialised
http-url = ${caucase-server:http-url}
https-url = ${caucase-server:url}
init-user = admin
{% do part_list.append('publish-connection-parameter') -%}
{% endif -%}
......
stack/erp5/buildout.cfg
View file @ f581d489
......@@ -52,6 +52,7 @@ extends =
../../component/coreutils/buildout.cfg
../../component/grep/buildout.cfg
../../component/dash/buildout.cfg
../../component/bash/buildout.cfg
../../component/wget/buildout.cfg
../../component/aspell/buildout.cfg
../../component/cloudooo/buildout.cfg
......@@ -61,8 +62,8 @@ extends =
../../component/userhosts/buildout.cfg
../../component/postfix/buildout.cfg
../monitor/buildout.cfg
../../software/ipython_notebook/software.cfg
../../software/caucase/software.cfg
../../software/jupyter/software.cfg
../../software/neoppod/software-common.cfg
# keep neoppod extends last
......@@ -100,6 +101,7 @@ parts +=
percona-toolkit
zabbix-agent
dash
bash
wget
userhosts
postfix
......@@ -139,15 +141,16 @@ parts +=
template
# jupyter
ipython-notebook
instance-jupyter
jupyter
jupyter-notebook-initialized-scripts
instance-jupyter-notebook
# override python2.7 to add SlapOS libstdc++ in RPATH.
[python2.7]
extra-ldflags = -Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64
# override instance-jupyter not to render into default template.cfg
[instance-jupyter]
# override instance-jupyter-notebook not to render into default template.cfg
[instance-jupyter-notebook]
rendered = ${buildout:directory}/template-jupyter.cfg
[instance-caucase]
......@@ -167,6 +170,10 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
<= download-base
mode = 755
[mariadb-slowquery-check-script]
<= download-base
mode = 755
[mariadb-slow-query-report-script]
<= download-base
mode = 755
......@@ -231,6 +238,8 @@ context =
key mariadb_link_binary template-mariadb:link-binary
key zope_link_binary template-zope:link-binary
key apache_location apache:location
key apdex_result_check_script apdex-result-check-script:target
key mariadb_slowquery_check_script mariadb-slowquery-check-script:target
key aspell_location aspell:location
key bin_directory buildout:bin-directory
key buildout_bin_directory buildout:bin-directory
......@@ -241,6 +250,7 @@ context =
key curl_location curl:location
key cyrus_sasl_location cyrus-sasl:location
key dash_location dash:location
key bash_location bash:location
key dbus_glib_location dbus-glib:location
key dbus_location dbus:location
key dcron_location dcron:location
......@@ -297,7 +307,7 @@ context =
key template_create_erp5_site_real template-create-erp5-site-real:target
key template_erp5 template-erp5:target
key template_haproxy_cfg template-haproxy-cfg:target
key template_jupyter_cfg instance-jupyter:rendered
key template_jupyter_cfg instance-jupyter-notebook:rendered
key template_kumofs template-kumofs:target
key template_mariadb template-mariadb:target
key template_mariadb_initial_setup template-mariadb-initial-setup:target
......@@ -350,6 +360,9 @@ link-binary =
[template-balancer]
<= download-base
[apdex-result-check-script]
<= download-base
[template-haproxy-cfg]
<= download-base
......@@ -371,7 +384,7 @@ recipe = plone.recipe.command
stop-on-error = true
genbt5list = ${erp5:location}/product/ERP5/bin/genbt5list
command =
${buildout:executable} ${:genbt5list} ${local-bt5-repository:list}
echo '${local-bt5-repository:list}' |xargs ${buildout:executable} ${:genbt5list}
update-command = ${:command}
[erp5_repository_list]
......@@ -668,7 +681,7 @@ Products.DCWorkflowGraph = 0.4.1
# Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one.
Products.ExternalEditor = 1.1.1
Products.GenericSetup = 1.8.6
Products.LongRequestLogger = 2.0.0
Products.LongRequestLogger = 2.1.0
# Products.MimetypesRegistry 2.1 requires AccessControl>=3.0.0Acquisition.
Products.MimetypesRegistry = 2.0.10
Products.PluginRegistry = 1.4
......
stack/erp5/buildout.hash.cfg
View file @ f581d489
......@@ -19,19 +19,23 @@ md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
[mariadb-slow-query-report-script]
filename = mysql-querydigest.sh.in
md5sum = dc974bd74cf967ae6250d81322629c44
md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb
[mariadb-slowquery-check-script]
filename = instance-mariadb-check-slowquery-result.sh.in
md5sum = 356e0e2db1da0e8b479908fb739e5cc0
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = 3b6c19686ac52c2d452597dbacc88888
md5sum = 7ee2e801dda1181d1b42281e6466fc4d
[template-kumofs]
filename = instance-kumofs.cfg.in
md5sum = 253027f37c4e5bd908e6af551848f03a
md5sum = 091d3c3feb2d543d176b0fadb11c07dc
[template-cloudooo]
filename = instance-cloudoo.cfg.in
md5sum = 43253cbfd5d361047ba6a99b116b2370
md5sum = 76f9e8c8cdc352081e34539d8fc17026
[template-zope-conf]
filename = zope.conf.in
......@@ -75,7 +79,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
[template]
filename = instance.cfg.in
md5sum = e364ea67bfe786b6b6ebd6c4f0cd628a
md5sum = 7b4ebfd2ef31ed57a7129020c8ccde67
[monitor-template-dummy]
filename = dummy.cfg
......@@ -83,19 +87,23 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 13638031b6b6c9ad9c0a9c4e6d9a202a
md5sum = 909c8eb4f1f2f2d58ad982cec67228bc
[template-zeo]
filename = instance-zeo.cfg.in
md5sum = 2248d249cfc53c73e7c65665d4d53be0
md5sum = d1f33d406d528ae27d973e2dd0efb1ba
[template-zope]
filename = instance-zope.cfg.in
md5sum = 6a64d1615c3ef9f6311c863d5aa0c58f
md5sum = a2377d5c53fd2a441ea713b428e4844b
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = f2fb0c537c124622fe8e89afe0188519
md5sum = d14ee7f13e2bd815cc96e28101e59670
[apdex-result-check-script]
filename = instance-balancer-check-apachedex-result.sh.in
md5sum = 421c68c97cadc49911382cd3185288a1
[template-haproxy-cfg]
filename = haproxy.cfg.in
......
stack/erp5/instance-balancer-check-apachedex-result.sh.in 0 → 100644
View file @ f581d489
#!{{ bash }}
set -e
APACHEDEX_FILE='{{ apdex_file }}/ApacheDex-'$(date +%Y-%m-%d)'.html'
APACHEDEX_REPORT_JSON_FILE={{ apdex_status_file }}
DESIRED_THRESHOLD={{ user_threshold }}
# Check if the file is there
if [ ! -s "$APACHEDEX_FILE" ]; then
# If file doesn't exists create one
# If it is empty check for modification time
if [ ! -f "$APACHEDEX_FILE" ]; then
touch $APACHEDEX_FILE
else
MODIFIED_DATE=`stat -c '%Z' $APACHEDEX_FILE`
CURRENT_DATE=`date +%s`
if [[ `echo "$CURRENT_DATE - $MODIFIED_DATE" | bc` -gt 108000 ]]
then
echo "File modification date is greater than 30 hours"
JSON_CONTENT=`cat $APACHEDEX_REPORT_STATUS_FILE`
MESSAGE=`echo $JSON_CONTENT | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
echo $MESSAGE
exit 2
else
echo "File is empty for now"
fi
fi
else
# Check if the result exists
{
REGEX="Overall<\/h2><table .*><tr>[[:space:]]<th>apdex<\/th><th>.*?<tr>[[:space:]]<td [^<]*>(.*?)%<\/td>"
FILE_CONTENT=`cat $APACHEDEX_FILE`
if [[ $FILE_CONTENT =~ $REGEX ]]
then
RESULT=${BASH_REMATCH[1]}
RESULT=${RESULT:-0}
if [[ `echo "$RESULT > $DESIRED_THRESHOLD" | bc` -eq 1 ]]
then
echo "Your score is $RESULT %, Thanks for keeping it all clean"
else
echo "Threshold is lower than exptected: Expected was $DESIRED_THRESHOLD % and we current is $RESULT %"
exit 2
fi
else
echo "No threshold found in the result"
fi
} || {
echo "Cannot parse the apdex result"
}
fi
stack/erp5/instance-balancer.cfg.in
View file @ f581d489
......@@ -219,7 +219,7 @@ path = ${directory:promise}/apache
hostname = {{ ipv4 }}
port = {{ apache_dict.values()[0][0] }}
[publish]
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
{% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%}
{{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
......@@ -260,7 +260,7 @@ cert =
crl =
{%- endif %}
[logrotate-apache]
[{{ section('logrotate-apache') }}]
< = logrotate-entry-base
name = apache
log = ${apache-conf-parameter-dict:error-log} ${apache-conf-parameter-dict:access-log}
......@@ -271,7 +271,7 @@ recipe = slapos.cookbook:mkdirectory
apache-conf = ${:etc}/apache
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
promise = ${directory:etc}/promise
promise = ${:etc}/promise
services = ${:etc}/run
var = ${buildout:directory}/var
run = ${:var}/run
......@@ -284,39 +284,39 @@ newcerts = ${:ca-dir}/newcerts
crl = ${:ca-dir}/crl
apachedex = ${monitor-directory:private}/apachedex
[monitor-generate-apachedex-report]
[{{ section('monitor-generate-apachedex-report') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${monitor-directory:reports}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}"
command = apachedex_every_3_hour
[apachedex-parameters]
default_parameter =
# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
apache-log-list = ${apache-conf-parameter-dict:access-log}
default = ${monitor-directory:etc}/apdex_default
base-list = ${monitor-directory:etc}/apdex_base_list
skip-base-list = ${monitor-directory:etc}/apdex_skip_base_list
erp5-base-list = ${monitor-directory:etc}/apdex_erp5_base_list
configuration = {{ slapparameter_dict['apachedex-configuration'] }}
promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
[{{ section('monitor-promise-apachedex-result') }}]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['apdex-result-check-script'] }}
rendered = ${monitor-directory:promises}/check-apachedex-result
status-file = ${monitor-directory:private}/apachedex.report.json
context =
raw bash {{ parameter_dict['bash'] }}/bin/bash
raw user_threshold ${apachedex-parameters:promise-threshold}
key apdex_file directory:apachedex
key apdex_status_file :status-file
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }}
monitor-title = Balancer monitor
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
instance-configuration =
file apachedex-default ${apachedex-parameters:default}
file apachedex-base-list ${apachedex-parameters:base-list}
file apachedex-skip-base-list ${apachedex-parameters:skip-base-list}
file apachedex-erp5-base-list ${apachedex-parameters:erp5-base-list}
[buildout]
extends =
{{ logrotate_cfg }}
{{ parameter_dict['template-monitor'] }}
parts +=
publish
logrotate-apache
monitor-generate-apachedex-report
{{ part_list | join('\n ') }}
stack/erp5/instance-cloudoo.cfg.in
View file @ f581d489
......@@ -111,7 +111,7 @@ framebuffer = ${:srv}/framebuffer
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ tcpv4_port + 2 }}
monitor-title = Cloudooo monitor
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
[binary-link]
......
stack/erp5/instance-erp5.cfg.in
View file @ f581d489
{% import "root_common" as root_common with context %}
{% import "root_common" as root_common with context -%}
{% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
{% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
{% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
......@@ -10,6 +10,7 @@
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set monitor_base_url_dict = {} -%}
{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
{% set crl_update_period = slapparameter_dict.get('caucase', {}).pop('crl-update-periodicity', 'daily') -%}
[request-common]
<= request-common-base
......@@ -38,12 +39,13 @@ config-{{ k }} = {{ dumps(v) }}
{% for k, v in key_config.iteritems() -%}
config-{{ k }} = {{ '${' ~ v ~ '}' }}
{% endfor -%}
config-name = {{ name }}
{% endmacro -%}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{% if has_posftix -%}
{{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
{%- else %}
......@@ -57,7 +59,7 @@ connection-url = smtp://127.0.0.2:0/
[request-caucase]
connection-http-url = {{ caucase_url }}
{%- else %}
{{ request('caucase', 'caucase', 'caucase', {'server-port': 8890, 'server-https-port': 8891}, {'http-url': True, 'https-url': False}) }}
{{ request('caucase', 'caucase', 'caucase', {'server-port': 8890, 'server-https-port': 8891, 'auto-sign-csr-amount': 2}, {'http-url': True, 'https-url': False}) }}
{% endif -%}
{# ZODB -#}
......@@ -274,10 +276,14 @@ config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}
config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }}
config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
config-monitor-passwd = ${monitor-htpasswd:passwd}
config-name = ${:name}
config-caucase-url = ${request-caucase:connection-http-url}
config-crl-update-periodicity = {{ crl_update_period }}
config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
[request-frontend-base]
{% if has_frontend -%}
......@@ -331,7 +337,7 @@ parts +=
monitor-base
[monitor-conf-parameters]
monitor-title = ERP5 monitor
monitor-title = ERP5
password = ${monitor-htpasswd:passwd}
[monitor-base-url-dict]
......
stack/erp5/instance-kumofs.cfg.in
View file @ f581d489
......@@ -112,5 +112,5 @@ port = ${kumofs-instance:manager-port}
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ tcpv4_port + 4 }}
monitor-title = Kumofs monitor
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
stack/erp5/instance-mariadb-check-slowquery-result.sh.in 0 → 100644
View file @ f581d489
#!{{ bash }}
set -e
DIGEST_FILE='{{ slow_query_digest }}/slowquery_digest.txt'
SLOW_QUERY_STATUS_FILE='{{ slow_query_status }}'
DESIRED_MAX_QUERY_THRESHOLD={{ max_queries_threshold }}
DESIRED_SLOW_QUERY_THRESHOLD={{ slowest_queries_threshold }}
# Check if the file is there
if [ ! -s "$DIGEST_FILE" ]; then
# If file doesn't exists create one
# If it is empty check for modification time
if [ ! -f "$DIGEST_FILE" ]; then
touch $DIGEST_FILE
else
MODIFIED_DATE=`stat -c '%Z' $DIGEST_FILE`
CURRENT_DATE=`date +%s`
if [[ `echo "$CURRENT_DATE - $MODIFIED_DATE" | bc` -gt 108000 ]]
then
echo "File modification date is greater than 30 hours"
JSON_CONTENT=`cat $SLOW_QUERY_STATUS_FILE`
MESSAGE=`echo $JSON_CONTENT | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
echo $MESSAGE
exit 2
else
echo "File is empty for now"
fi
fi
else
# Check if the result exists
{
# get the total number of queries ran and the max time
# TODO: improve regex
# TODO: improve the parameters (currently we are using threshold on queries and max execute time)
# # Overall: (.*) total,(?:.*\n){4}# Exec time(?: *\d*m?s){2} *(.*?)m?s
REGEX="# Overall: (.*) total,.*# Exec time *[[:digit:]]*m?s *[[:digit:]]*m?s *([[:digit:]]*)m?s"
FILE_CONTENT=`cat $DIGEST_FILE`
if [[ $FILE_CONTENT =~ $REGEX ]]
then
TOTAL_QUERIES_EXEC=${BASH_REMATCH[1]}
SLOWEST_QUERY_TIME=${BASH_REMATCH[2]}
HAS_K="${TOTAL_QUERIES_EXEC: -1}"
if [[ "$HAS_K" == "k" ]]
then
PRE="${TOTAL_QUERIES_EXEC::-1}"
TOTAL_QUERIES_EXEC=$(echo "scale=4; ${PRE:-0}*1000" | bc)
else
TOTAL_QUERIES_EXEC=${TOTAL_QUERIES_EXEC:-0}
fi
# TODO: support ms
SLOWEST_QUERY_TIME="${SLOWEST_QUERY_TIME:-0}"
if [[ `echo "$TOTAL_QUERIES_EXEC < $DESIRED_MAX_QUERY_THRESHOLD" | bc` -eq 1 && `echo "$SLOWEST_QUERY_TIME < $DESIRED_SLOW_QUERY_THRESHOLD" | bc` -eq 1 ]]
then
echo "Total number of slow queries are: $TOTAL_QUERIES_EXEC"
echo "Time taken by slowest query is: $SLOWEST_QUERY_TIME"
echo "Thanks for keeping it all clean"
else
echo "Ops! One of the two expected parameters did not meet"
echo "Time taken by slowest query is: $SLOWEST_QUERY_TIME s and required maximum is $DESIRED_SLOW_QUERY_THRESHOLD s"
echo "Total slow queries are $TOTAL_QUERIES_EXEC and expected maximum value is $DESIRED_MAX_QUERY_THRESHOLD"
exit 2
fi
else
echo "No threshold found in the result"
fi
} || {
echo "Cannot parse the result"
}
fi
stack/erp5/instance-mariadb.cfg.in
View file @ f581d489
......@@ -282,14 +282,30 @@ context =
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
rendered = ${monitor-directory:reports}/${:filename}
output-folder = ${directory:srv}/monitor/
filename = mariadb_slow_query_every_23_hour
mode = 755
context =
raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
raw dash {{ parameter_dict['dash-location'] }}/bin/dash
key output_folder :output-folder
key output_folder monitor-directory:private
[slow-query-digest-parameters]
max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
[{{ section('monitor-promise-slowquery-result') }}]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slowquery-check-script'] }}
rendered = ${monitor-directory:promises}/mariadb-slow-queries-result
status-file = ${monitor-directory:private}/mariadb_slow_query.report.json
context =
raw default_threshold 4000
raw bash {{ parameter_dict['bash'] }}/bin/bash
key slow_query_digest monitor-directory:private
key slow_query_status :status-file
key max_queries_threshold slow-query-digest-parameters:max_queries_threshold
key slowest_queries_threshold slow-query-digest-parameters:slowest_queries_threshold
[{{ section('promise') }}]
recipe = slapos.cookbook:wrapper
......@@ -297,11 +313,10 @@ command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened"
wrapper-path = ${directory:promise}/mariadb
parameters-extra = true
{% set monitor_passwd = slapparameter_dict['monitor-passwd'] -%}
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ port + 1 }}
monitor-title = Mariadb monitor
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
[buildout]
......
stack/erp5/instance-zeo.cfg.in
View file @ f581d489
......@@ -185,7 +185,7 @@ tidstorage = {{ tidstorage_repozo_path }}
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }}
monitor-title = ZODB monitor
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
[buildout]
......
stack/erp5/instance-zope.cfg.in
View file @ f581d489
......@@ -394,7 +394,7 @@ bt5-repository-url = {{ dumps(slapparameter_dict['bt5-repository-url']) }}
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }}
monitor-title = Zope monitor
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
[buildout]
......
stack/erp5/instance.cfg.in
View file @ f581d489
......@@ -73,7 +73,7 @@ extra-context =
[dynamic-template-erp5-parameters]
jupyter-enable-default = {{ jupyter_enable_default }}
local-bt5-repository = {{ local_bt5_repository }}
local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
template-monitor = {{ dumps(template_monitor) }}
[dynamic-template-erp5]
......@@ -93,6 +93,7 @@ openssl-location = {{ openssl_location }}
[dynamic-template-balancer-parameters]
apache = {{ apache_location }}
apdex-result-check-script = {{ apdex_result_check_script }}
openssl = {{ openssl_location }}
haproxy = {{ haproxy_location }}
bin-directory = {{ bin_directory }}
......@@ -101,6 +102,7 @@ run-apachedex-location = {{ bin_directory }}/runApacheDex
6tunnel = {{ sixtunnel_location }}
curl-location = {{ curl_location }}
dash = {{ dash_location }}
bash = {{ bash_location }}
template-haproxy-cfg = {{ template_haproxy_cfg }}
template-apache-conf = {{ template_apache_conf }}
template-monitor = {{ dumps(template_monitor) }}
......@@ -169,6 +171,7 @@ extra-context =
section parameter_dict dynamic-template-kumofs-parameters
[dynamic-template-mariadb-parameters]
bash = {{ bash_location }}
coreutils-location = {{ coreutils_location }}
dash-location = {{ dash_location }}
findutils-location = {{ findutils_location }}
......@@ -180,6 +183,7 @@ link-binary = {{ dumps(mariadb_link_binary) }}
bin-directory = {{ bin_directory }}
mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }}
mariadb-slowquery-check-script = {{ mariadb_slowquery_check_script}}
percona-tools-location = {{ percona_toolkit_location }}
template-monitor = {{ template_monitor }}
......
stack/erp5/mysql-querydigest.sh.in
View file @ f581d489
......@@ -10,6 +10,8 @@ if [ ! -d "$OUTPUT_FOLDER" ]; then
exit 0
fi
OUTPUT_FILE=${OUTPUT_FOLDER}/slowquery_digest.txt
TODAY=`date +%Y%m%d`
SLOW_LOG=$SLOW_QUERY_PATH-$TODAY
......@@ -19,4 +21,5 @@ if [ ! -f "$SLOW_LOG" ]; then
exit 1
fi
eval $PT_QUERY_EXEC $SLOW_LOG
$PT_QUERY_EXEC $SLOW_LOG > $OUTPUT_FILE
echo "ok"
stack/lamp.cfg
View file @ f581d489
......@@ -42,7 +42,7 @@ keep-compile-dir = false
apache-libcloud = 0.9.1
async = 0.6.1
gitdb = 0.5.4
mysqlclient = 1.3.10+SlapOSPatched001
mysqlclient = 1.3.10+SlapOSPatched002
plone.recipe.command = 1.1
slapos.recipe.template = 2.3
slapos.toolbox = 0.40.4
......
stack/lamp/buildout.cfg
View file @ f581d489
......@@ -185,7 +185,7 @@ eggs =
apache-libcloud = 0.12.3
async = 0.6.1
gitdb = 0.5.4
mysqlclient = 1.3.10+SlapOSPatched001
mysqlclient = 1.3.10+SlapOSPatched002
pycrypto = 2.6
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 2.4.2
......
stack/monitor/buildout.cfg
View file @ f581d489
......@@ -131,5 +131,5 @@ depends =
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
pycurl = 7.43.0
slapos.toolbox = 0.69
slapos.toolbox = 0.71
stack/slapos.cfg
View file @ f581d489
......@@ -116,7 +116,7 @@ click = 6.7
cliff = 2.4.0
cmd2 = 0.7.0
collective.recipe.template = 2.0
cryptography = 1.8.1
cryptography = 2.0.3
decorator = 4.0.11
idna = 2.2
inotifyx = 0.2.2
......@@ -129,7 +129,7 @@ pbr = 2.0.0
plone.recipe.command = 1.1
prettytable = 0.7.2
psutil = 5.2.0
pyOpenSSL = 16.2.0
pyOpenSSL = 17.2.0
pyparsing = 2.2.0
pytz = 2016.10
requests = 2.13.0
......@@ -182,7 +182,7 @@ pycparser = 2.17
# Required by:
# slapos.core==1.3.18
supervisor = 3.3.1
supervisor = 3.3.3
# Required by:
# slapos.core==1.3.18
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7