Skip to content

M
mariadb
Commit 14cf7336 authored by stewart@willster.(none)'s avatar stewart@willster.(none)
Browse files
Options

Merge ssmith@bk-internal.mysql.com:/home/bk/mysql-5.1-new-ndb 

into  willster.(none):/home/stewart/Documents/MySQL/5.1/ndb
parents c6377e66 14ff8874
Hide whitespace changes
Inline Side-by-side
Showing with 1559 additions and 483 deletions
storage/ndb/tools/Makefile.am
View file @ 14cf7336
......@@ -14,7 +14,7 @@
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
dist_bin_SCRIPTS = ndb_size.pl ndb_error_reporter
dist_pkgdata_DATA = ndb_size.tmpl
dist_pkgdata_DATA =
ndbtools_PROGRAMS = \
ndb_test_platform \
......
storage/ndb/tools/ndb_size.pl
View file @ 14cf7336
#!/usr/bin/perl -w
# Copyright (C) 2005-2007 MySQL AB
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
use strict;
use DBI;
use POSIX;
use HTML::Template;
use Getopt::Long;
# MySQL Cluster size estimator
# ----------------------------
#
# (C)2005 MySQL AB
#
#
# The purpose of this tool is to work out storage requirements
# from an existing MySQL database.
#
......@@ -25,157 +37,578 @@ use HTML::Template;
#
# BUGS
# ----
# - enum/set is 0 byte storage! Woah - efficient!
# - DECIMAL is 0 byte storage. A bit too efficient.
# - some float stores come out weird (when there's a comma e.g. 'float(4,1)')
# - no disk data values
# - computes the storage requirements of views (and probably MERGE)
# - ignores character sets.
# - ignores character sets?
package MySQL::NDB::Size::Parameter;
use Class::MethodMaker [
scalar => 'name',
scalar => 'default',
scalar => 'mem_per_item',
scalar => [{ -default => '' },'unit'],
hash => [ qw ( value
ver_mem_per_item
mem_total ) ],
new => [ -hash => 'new' ],
];
1;
package MySQL::NDB::Size::Report;
use Class::MethodMaker [
scalar => [ qw( database
dsn ) ],
array => 'versions',
hash => [qw( tables
parameters
supporting_tables) ],
new => [ -hash => 'new' ],
];
1;
package MySQL::NDB::Size::Column;
my $template = HTML::Template->new(filename => 'ndb_size.tmpl',
die_on_bad_params => 0)
or die "Could not open ndb_size.tmpl.";
use Class::MethodMaker [
new => [ -hash => 'new' ],
scalar => [ qw( name
type
is_varsize
size
Key) ],
hash => 'dm_overhead',
scalar => [{ -default => 4 },'align'],
scalar => [{ -default => 0 },'null_bits'],
];
my $dbh;
# null_bits:
# 0 if not nullable, 1 if nullable
# + additional if bitfield as these are stored in the null bits
# if is_varsize, null_bits are in varsize part.
if(@ARGV < 3 || $ARGV[0] eq '--usage' || $ARGV[0] eq '--help')
# dm is default DataMemory value. Automatically 4byte aligned
# ver_dm is DataMemory value for specific versions.
# an entry in ver_dm OVERRIDES the dm value.
# e.g. if way column stored changed in new version.
#
# if is_varsize, dm/ver_dm is in varsized part.
sub ver_dm_exists
{
print STDERR "Usage:\n";
print STDERR "\tndb_size.pl database hostname user password\n\n";
print STDERR "If you need to specify a port number, use host:port\n\n";
exit(1);
my ($self,$ver)= @_;
return exists($self->{ver_dm}{$ver});
}
use Data::Dumper;
sub ver_dm
{
my $database= $ARGV[0];
my $hostname= $ARGV[1];
my $user= $ARGV[2];
my $password= $ARGV[3];
my $dsn = "DBI:mysql:database=$database;host=$hostname";
$dbh= DBI->connect($dsn, $user, $password) or exit(1);
$template->param(db => $database);
$template->param(dsn => $dsn);
my ($self,$ver,$val)= @_;
if(@_ > 2)
{
$self->{ver_dm}{$ver}=
$self->align * POSIX::floor(($val+$self->align-1)/$self->align);
}
return $self->{ver_dm}{$ver};
}
my @releases = ({rel=>'4.1'},{rel=>'5.0'},{rel=>'5.1'}); #,{rel=>'5.1-dd'});
$template->param(releases => \@releases);
sub dm
{
my ($self,$val)= @_;
if(@_ > 1)
{
$self->{dm}=
$self->align * POSIX::floor(($val+$self->align-1)/$self->align)
}
return $self->{dm};
}
my $tables = $dbh->selectall_arrayref("show tables");
package MySQL::NDB::Size::Index;
my @table_size;
use Class::MethodMaker [
new => [ -hash => 'new' ],
hash => [ qw( ver_dm
ver_im ) ],
scalar => [ qw( name
type
comment
columns
unique
dm
im) ],
scalar => [ { -default=> 4 },'align'],
scalar => [ { -default=> 0 },'is_supporting_table' ],
];
my @dbDataMemory;
my @dbIndexMemory;
my @NoOfAttributes;
my @NoOfIndexes;
my @NoOfTables;
$NoOfTables[$_]{val} = @{$tables} foreach 0..$#releases;
package MySQL::NDB::Size::Table;
# The following are computed by compute_row_size:
# row_dm_size DataMemory Size per row
# row_vdm_size Varsized DataMemory Size per row
# row_ddm_size Disk Data size per row (on disk size)
#
# These are hashes of versions. If an entry in (dm|vdm|ddm)_versions exists,
# then this parameter is calculated.
#
# Specific per-row overhead is in row_(dm|vdm|ddm)_overhead.
# e.g. if there is a varsized column, we have a vdm overhead for the
# varsized part of the row, otherwise vdm_size==0
# Any supporting tables - e.g. BLOBs have their name in supporting_tables
# These tables should then be looked up in the main report object.
# The main report object also has a supporting_tables hash used for
# excluding these from the main list of tables.
use POSIX;
use Class::MethodMaker [
new => [ -hash => 'new' ],
array => [ qw( supporting_tables
dm_versions
vdm_versions
ddm_versions ) ],
scalar => [ qw( name
rows ) ],
hash => [ qw( columns
indexes
indexed_columns
row_im_size
row_dm_size
row_vdm_size
row_dm_overhead
row_vdm_overhead
row_ddm_overhead) ],
scalar => [ { -default=> 8192 },'im_pagesize'],
scalar => [ { -default=> 0 },'im_page_overhead'],
scalar => [ { -default=> 32768 },'dm_pagesize' ],
scalar => [ { -default=> 128 },'dm_page_overhead' ],
scalar => [ { -default=> 32768 },'vdm_pagesize' ],
scalar => [ { -default=> 128 },'vdm_page_overhead' ],
hash => [ # these are computed
qw(
dm_null_bytes
vdm_null_bytes
dm_needed
vdm_needed
im_needed
im_rows_per_page
dm_rows_per_page
vdm_rows_per_page) ],
scalar => [ { -default=> 4 },'align'],
];
sub compute_row_size
{
my ($self, $releases) = @_;
sub align {
my($to,@unaligned) = @_;
my @aligned;
foreach my $x (@unaligned) {
push @aligned, $to * POSIX::floor(($x+$to-1)/$to);
my %row_dm_size;
my %row_vdm_size;
my %row_im_size;
my %dm_null_bits;
my %vdm_null_bits;
my $no_varsize= 0;
foreach my $c (keys %{$self->columns})
{
if($self->columns->{$c}->is_varsize)
{
$no_varsize++;
foreach my $ver ($self->vdm_versions)
{
if($self->columns->{$c}->ver_dm_exists($ver))
{
$row_vdm_size{$ver}+= $self->columns->{$c}->ver_dm($ver);
$vdm_null_bits{$ver}+= $self->columns->{$c}->null_bits();
}
else
{
$row_vdm_size{$ver}+= $self->columns->{$c}->dm;
$vdm_null_bits{$ver}+= $self->columns->{$c}->null_bits();
}
}
}
foreach my $ver ($self->dm_versions)
{
if($self->columns->{$c}->ver_dm_exists($ver))
{
next if $self->columns->{$c}->is_varsize;
$row_dm_size{$ver}+= $self->columns->{$c}->ver_dm($ver);
$dm_null_bits{$ver}+= $self->columns->{$c}->null_bits();
}
else
{
$row_dm_size{$ver}+= $self->columns->{$c}->dm||0;
$dm_null_bits{$ver}+= $self->columns->{$c}->null_bits()||0;
}
}
}
foreach ($self->row_dm_overhead_keys())
{
$row_dm_size{$_}+= $self->row_dm_overhead->{$_}
if exists($row_dm_size{$_});
}
return @aligned;
foreach ($self->row_vdm_overhead_keys())
{
$row_vdm_size{$_}+= $self->row_vdm_overhead->{$_}
if exists($row_vdm_size{$_});
}
# now we compute size of indexes for dm
foreach my $i (keys %{$self->indexes})
{
foreach my $ver ($self->dm_versions)
{
$row_dm_size{$ver}+= $self->indexes->{$i}->dm() || 0;
}
}
# now we compute size of indexes for im
while(my ($iname, $i) = $self->indexes_each())
{
foreach my $ver ($self->dm_versions)
{
if($i->ver_im_exists($ver))
{
$row_im_size{$ver}+= $i->ver_im->{$ver};
}
else
{
$row_im_size{$ver}+= $i->im() || 0;
}
}
}
# 32-bit align the null part
foreach my $k (keys %dm_null_bits)
{
$dm_null_bits{$k}=
$self->align * POSIX::floor((ceil($dm_null_bits{$k}/8)+$self->align-1)
/$self->align);
}
foreach my $k (keys %vdm_null_bits)
{
$vdm_null_bits{$k}=
$self->align * POSIX::floor((ceil($vdm_null_bits{$k}/8)+$self->align-1)
/$self->align);
}
# Finally set things
$self->dm_null_bytes(%dm_null_bits);
$self->vdm_null_bytes(%vdm_null_bits);
# add null bytes to dm/vdm size
foreach my $k (keys %row_dm_size)
{
$row_dm_size{$k}+=$dm_null_bits{$k}||0;
}
foreach my $k (keys %row_vdm_size)
{
$row_vdm_size{$k}+=$vdm_null_bits{$k}||0;
}
$self->row_dm_size(%row_dm_size);
$self->row_vdm_size(%row_vdm_size);
$self->row_im_size(%row_im_size);
}
sub compute_estimate
{
my ($self) = @_;
foreach my $ver (@{$self->dm_versions})
{
$self->dm_rows_per_page_set($ver =>
floor(
($self->dm_pagesize - $self->dm_page_overhead)
/
$self->row_dm_size->{$ver}
)
);
}
foreach my $ver (@{$self->vdm_versions})
{
next if ! $self->row_vdm_size_exists($ver);
$self->vdm_rows_per_page_set($ver =>
floor(
($self->vdm_pagesize - $self->vdm_page_overhead)
/
$self->row_vdm_size->{$ver}
)
);
}
$self->im_page_overhead(0) if !$self->im_page_overhead();
foreach my $ver (@{$self->dm_versions})
{
$self->im_rows_per_page_set($ver =>
floor(
($self->im_pagesize - $self->im_page_overhead)
/
$self->row_im_size->{$ver}
)
);
}
$self->dm_needed_set($_ => $self->dm_pagesize()
*
POSIX::ceil(
$self->rows
/
($self->dm_rows_per_page->{$_})
)
)
foreach $self->dm_versions;
$self->vdm_needed_set($_ => (!$self->vdm_rows_per_page->{$_})? 0 :
$self->vdm_pagesize()
*
POSIX::ceil(
$self->rows
/
($self->vdm_rows_per_page->{$_})
)
)
foreach $self->vdm_versions;
$self->im_needed_set($_ => $self->im_pagesize()
*
POSIX::ceil(
$self->rows
/
($self->im_rows_per_page->{$_})
)
)
foreach $self->dm_versions;
}
package main;
my ($dbh,$database,$hostname,$user,$password,$help,$savequeries,$loadqueries,$debug,$format);
GetOptions('database|d=s'=>\$database,
'hostname=s'=>\$hostname,
'user|u=s'=>\$user,
'password|p=s'=>\$password,
'savequeries|s=s'=>\$savequeries,
'loadqueries|l=s'=>\$loadqueries,
'help|usage|h!'=>\$help,
'debug'=>\$debug,
'format|f=s'=>\$format,
);
my $report= new MySQL::NDB::Size::Report;
if($help || !$database)
{
print STDERR "Usage:\n";
print STDERR "\tndb_size.pl --database=<db name> [--hostname=<host>]"
."[--user=<user>] [--password=<password>] [--help|-h] [--format=(html|text)] [--loadqueries=<file>] [--savequeries=<file>]\n\n";
print STDERR "\t--hostname=<host>:<port> can be used to designate a "
."specific port\n";
print STDERR "\t--hostname defaults to localhost\n";
print STDERR "\t--user and --password default to empty string\n";
print STDERR "\t--format=(html|text) Output format\n";
print STDERR "\t--savequeries=<file> saves all queries to the DB into <file>\n";
print STDERR "\t--loadqueries=<file> loads query results from <file>. Doesn't connect to DB.\n";
exit(1);
}
$hostname= 'localhost' unless $hostname;
my %queries; # used for loadqueries/savequeries
if(!$loadqueries)
{
my $dsn = "DBI:mysql:database=$database;host=$hostname";
$dbh= DBI->connect($dsn, $user, $password) or exit(1);
$report->database($database);
$report->dsn($dsn);
}
else
{
open Q,"< $loadqueries";
my @q= <Q>;
my $VAR1;
my $e= eval join("",@q) or die $@;
%queries= %$e;
close Q;
$report->database("file:$loadqueries");
$report->dsn("file:$loadqueries");
}
$report->versions('4.1','5.0','5.1');
my $tables;
if($loadqueries)
{
$tables= $queries{"show tables"};
}
else
{
$tables= $dbh->selectall_arrayref("show tables");
$queries{"show tables"}= $tables;
}
sub do_table {
my $table= shift;
my $t= shift;
my $info= shift;
my %indexes= %{$_[0]};
my @count= @{$_[1]};
my @columns;
my %columnsize; # used for index calculations
# We now work out the DataMemory usage
# sizes for 4.1, 5.0, 5.1 and 5.1-dd
my @totalsize= (0,0,0,0);
@totalsize= @totalsize[0..$#releases]; # limit to releases we're outputting
my $nrvarsize= 0;
$t->dm_versions($report->versions);
$t->vdm_versions('5.1');
$t->ddm_versions('5.1');
foreach(keys %$info)
foreach my $colname (keys %$info)
{
my @realsize = (0,0,0,0);
my @varsize = (0,0,0,0);
my $type;
my $size;
my $name= $_;
my $is_varsize= 0;
my $col= new MySQL::NDB::Size::Column(name => $colname);
my ($type, $size);
$col->Key($$info{$colname}{Key})
if(defined($$info{$colname}{Key}) &&$$info{$colname}{Key} ne '');
if($$info{$_}{Type} =~ /^(.*?)\((\d+)\)/)
$col->null_bits(defined($$info{$colname}{Null})
&& $$info{$colname}{Null} eq 'YES');
if(defined($$info{$colname}{Type})
&& $$info{$colname}{Type} =~ /^(.*?)\((.+)\)/)
{
$type= $1;
$size= $2;
}
elsif(exists($$info{$colname}{type}))
{
# we have a Column object..
$type= $$info{$colname}{type};
$size= $$info{$colname}{size};
}
else
{
$type= $$info{$_}{Type};
$type= $$info{$colname}{Type};
}
$col->type($type);
$col->size($size);
if($type =~ /tinyint/)
{@realsize=(1,1,1,1)}
{$col->dm(1)}
elsif($type =~ /smallint/)
{@realsize=(2,2,2,2)}
{$col->dm(2)}
elsif($type =~ /mediumint/)
{@realsize=(3,3,3,3)}
{$col->dm(3)}
elsif($type =~ /bigint/)
{@realsize=(8,8,8,8)}
{$col->dm(8)}
elsif($type =~ /int/)
{@realsize=(4,4,4,4)}
{$col->dm(4)}
elsif($type =~ /float/)
{
if($size<=24)
{@realsize=(4,4,4,4)}
if(!defined($size) || $size<=24)
{$col->dm(4)}
else
{@realsize=(8,8,8,8)}
{$col->dm(8)}
}
elsif($type =~ /double/ || $type =~ /real/)
{@realsize=(8,8,8,8)}
{$col->dm(8)}
elsif($type =~ /bit/)
{
my $a=($size+7)/8;
@realsize = ($a,$a,$a,$a);
# bitfields stored in null bits
$col->null_bits($size+($col->null_bits()||0));
}
elsif($type =~ /datetime/)
{@realsize=(8,8,8,8)}
{$col->dm(8)}
elsif($type =~ /timestamp/)
{@realsize=(4,4,4,4)}
{$col->dm(4)}
elsif($type =~ /date/ || $type =~ /time/)
{@realsize=(3,3,3,3)}
{$col->dm(3)}
elsif($type =~ /year/)
{@realsize=(1,1,1,1)}
{$col->dm(1)}
elsif($type =~ /enum/ || $type =~ /set/)
{
# I *think* this is correct..
my @items= split ',',$size;
$col->dm(ceil((scalar @items)/256));
}
elsif($type =~ /varchar/ || $type =~ /varbinary/)
{
my $fixed=$size+ceil($size/256);
my @dynamic=$dbh->selectrow_array("select avg(length(`"
.$name
."`)) from `".$table.'`');
$dynamic[0]=0 if !$dynamic[0];
$dynamic[0]+=ceil($dynamic[0]/256); # size bit
$nrvarsize++;
$is_varsize= 1;
$varsize[3]= ceil($dynamic[0]);
@realsize= ($fixed,$fixed,ceil($dynamic[0]),$fixed);
$col->dm_overhead_set('length' => ceil($size/256));
$col->dm($fixed);
if(!$col->Key()) # currently keys must be non varsized
{
my $sql= "select avg(length(`"
.$colname
."`)) from `".$t->name().'`';
my @dynamic;
if($loadqueries)
{
@dynamic= @{$queries{$sql}};
}
else
{
@dynamic= $dbh->selectrow_array($sql);
$queries{$sql}= \@dynamic;
}
$dynamic[0]=0 if ! defined($dynamic[0]) || !@dynamic;
$dynamic[0]+=ceil($size/256); # size bit
$col->is_varsize(1);
$col->ver_dm('5.1',ceil($dynamic[0]));
}
}
elsif($type =~ /binary/ || $type =~ /char/)
{@realsize=($size,$size,$size,$size)}
{$col->dm($size)}
elsif($type =~ /text/ || $type =~ /blob/)
{
@realsize=(8+256,8+256,8+256,8+256);
$col->dm_overhead_set('length' => 8);
$col->dm(8+256);
my $blobhunk= 2000;
$blobhunk= 8000 if $type=~ /longblob/;
$blobhunk= 4000 if $type=~ /mediumblob/;
my @blobsize=$dbh->selectrow_array("select SUM(CEILING(".
"length(`$name`)/$blobhunk))".
"from `".$table."`");
my $sql= "select SUM(CEILING(".
"length(`$colname`)/$blobhunk))"
."from `".$t->name."`";
my @blobsize;
if($loadqueries)
{
@blobsize= @{$queries{$sql}};
}
else
{
@blobsize= $dbh->selectrow_array($sql);
$queries{$sql}= \@blobsize;
}
$blobsize[0]=0 if !defined($blobsize[0]);
#$NoOfTables[$_]{val} += 1 foreach 0..$#releases; # blob uses table
do_table($table."\$BLOB_$name",
# Is a supporting table, add it to the lists:
$report->supporting_tables_set($t->name()."\$BLOB_$colname" => 1);
$t->supporting_tables_push($t->name()."\$BLOB_$colname");
my $st= new MySQL::NDB::Size::Table(name =>
$t->name()."\$BLOB_$colname",
rows => $blobsize[0],
row_dm_overhead =>
{ '4.1' => 12,
'5.0' => 12,
'5.1' => 16,
},
row_vdm_overhead =>
{ '5.1' => 8 },
row_ddm_overhead =>
{ '5.1' => 8 },
);
do_table($st,
{'PK'=>{Type=>'int'},
'DIST'=>{Type=>'int'},
'PART'=>{Type=>'int'},
......@@ -195,26 +628,12 @@ sub do_table {
\@blobsize);
}
@realsize= @realsize[0..$#releases];
@realsize= align(4,@realsize);
$totalsize[$_]+=$realsize[$_] foreach 0..$#totalsize;
my @realout;
push @realout,{val=>$_} foreach @realsize;
push @columns, {
name=>$name,
type=>$type,
is_varsize=>$is_varsize,
size=>$size,
key=>$$info{$_}{Key},
datamemory=>\@realout,
};
$columnsize{$name}= \@realsize; # used for index calculations
$col->type($type);
$col->size($size);
$t->columns_set( $colname => $col );
}
$report->tables_set( $t->name => $t );
# And now... the IndexMemory usage.
#
# Firstly, we assemble some information about the indexes.
......@@ -222,170 +641,1058 @@ sub do_table {
# we can still connect to pre-5.0 mysqlds.
if(!defined($indexes{PRIMARY})) {
my @usage= ({val=>8},{val=>8},{val=>8},{val=>8});
@usage= @usage[0..$#releases];
$indexes{PRIMARY}= {
type=>'BTREE',
unique=>1,
comment=>'Hidden pkey created by NDB',
columns=>['HIDDEN_NDB_PKEY'],
};
push @columns, {
name=>'HIDDEN_NDB_PKEY',
type=>'bigint',
size=>8,
key=>'PRI',
datamemory=>\@usage,
};
$columnsize{'HIDDEN_NDB_PKEY'}= [8,8,8];
}
my $i= new MySQL::NDB::Size::Index(
name => 'PRIMARY',
unique => 1,
comment =>'Hidden pkey created by NDB',
type =>'BTREE',
columns => ['HIDDEN_NDB_PKEY'],
);
my @IndexDataMemory= ({val=>0},{val=>0},{val=>0},{val=>0});
my @RowIndexMemory= ({val=>0},{val=>0},{val=>0},{val=>0});
@IndexDataMemory= @IndexDataMemory[0..$#releases];
@RowIndexMemory= @RowIndexMemory[0..$#releases];
$i->im(16);
$i->dm(16);
$i->ver_im('4.1',25+8);
my @indexes;
foreach my $index (keys %indexes) {
my $im41= 25;
$im41+=$columnsize{$_}[0] foreach @{$indexes{$index}{columns}};
my @im = ({val=>$im41},{val=>25},{val=>25}); #,{val=>25});
my @dm = ({val=>10},{val=>10},{val=>10}); #,{val=>10});
push @indexes, {
name=>$index,
type=>$indexes{$index}{type},
columns=>join(',',@{$indexes{$index}{columns}}),
indexmemory=>\@im,
datamemory=>\@dm,
};
$IndexDataMemory[$_]{val}+=$dm[$_]{val} foreach 0..$#releases;
$RowIndexMemory[$_]{val}+=$im[$_]{val} foreach 0..$#releases;
$t->indexes_set('PRIMARY' => $i);
$t->indexed_columns_set('HIDDEN_NDB_PKEY' => 1);
$t->columns_set('HIDDEN_NDB_PKEY' =>
new MySQL::NDB::Size::Column(
name => 'HIDDEN_NDB_PKEY',
type => 'bigint',
dm => 8,
Key => 'PRI'));
}
# total size + 16 bytes overhead
my @TotalDataMemory;
my @RowOverhead = ({val=>16},{val=>16},{val=>16}); #,{val=>24});
# 5.1 has ptr to varsize page, and per-varsize overhead
my @nrvarsize_mem= ({val=>0},{val=>0},
{val=>8}); #,{val=>0});
my @indexes;
# We model the PRIMARY first as needed for secondary uniq indexes
if(defined($indexes{'PRIMARY'}))
{
my @a= align(4,$nrvarsize*2);
$nrvarsize_mem[2]{val}+=$a[0]+$nrvarsize*4;
my $index= 'PRIMARY';
my $i= new MySQL::NDB::Size::Index(
name => $index,
unique => $indexes{$index}{unique},
comment => $indexes{$index}{comment},
type => $indexes{$index}{type},
columns => [@{$indexes{$index}{columns}}],
);
my $im41= 25; # keep old estimate for 4.1
$im41+= $t->columns->{$_}->dm foreach @{$indexes{$index}{columns}};
$i->im(16); # estimate from Johan
$i->dm(16) if $indexes{$index}{unique}; # estimate from Johan
$i->ver_im('4.1',$im41);
$t->indexes_set($index => $i);
$t->indexed_columns_set($_ => 1)
foreach @{$indexes{$index}{columns}};
}
$TotalDataMemory[$_]{val}=$IndexDataMemory[$_]{val}+$totalsize[$_]+$RowOverhead[$_]{val}+$nrvarsize_mem[$_]{val} foreach 0..$#releases;
my @RowDataMemory;
push @RowDataMemory,{val=>$_} foreach @totalsize;
my @RowPerPage;
push @RowPerPage,{val=>(floor((32768-128)/$TotalDataMemory[$_]{val}))} foreach 0..$#TotalDataMemory;
my @RowPerIndexPage;
push @RowPerIndexPage,{val=>(floor(8192/$RowIndexMemory[$_]{val}))} foreach 0..$#TotalDataMemory;
my @DataMemory;
push @DataMemory,{val=>ceil(($count[0]/($RowPerPage[$_]{val})))*32} foreach 0..$#RowPerPage;
my @IndexMemory;
push @IndexMemory,{val=>ceil(($count[0]/($RowPerIndexPage[$_]{val})))*8} foreach 0..$#RowPerPage;
my $count= $count[0];
my @counts;
$counts[$_]{val}= $count foreach 0..$#releases;
my @nrvarsize_rel= ({val=>0},{val=>0},
{val=>$nrvarsize}); #,{val=>0});
push @table_size, {
table=>$table,
indexes=>\@indexes,
columns=>\@columns,
count=>\@counts,
RowOverhead=>\@RowOverhead,
RowDataMemory=>\@RowDataMemory,
nrvarsize=>\@nrvarsize_rel,
nrvarsize_mem=>\@nrvarsize_mem,
releases=>\@releases,
IndexDataMemory=>\@IndexDataMemory,
TotalDataMemory=>\@TotalDataMemory,
RowPerPage=>\@RowPerPage,
DataMemory=>\@DataMemory,
RowIndexMemory=>\@RowIndexMemory,
RowPerIndexPage=>\@RowPerIndexPage,
IndexMemory=>\@IndexMemory,
};
$dbDataMemory[$_]{val} += $DataMemory[$_]{val} foreach 0..$#releases;
$dbIndexMemory[$_]{val} += $IndexMemory[$_]{val} foreach 0..$#releases;
$NoOfAttributes[$_]{val} += @columns foreach 0..$#releases;
$NoOfIndexes[$_]{val} += @indexes foreach 0..$#releases;
}
foreach my $index (keys %indexes) {
next if $index eq 'PRIMARY';
if(!$indexes{$index}{unique})
{
my $i= new MySQL::NDB::Size::Index(
name => $index,
unique => $indexes{$index}{unique},
comment => $indexes{$index}{comment},
type => $indexes{$index}{type},
columns => [@{$indexes{$index}{columns}}],
);
$i->dm(16);
$t->indexes_set($index => $i);
$t->indexed_columns_set($_ => 1)
foreach @{$indexes{$index}{columns}};
}
else
{
my $i= new MySQL::NDB::Size::Index(
name => $index,
unique => $indexes{$index}{unique},
comment => $indexes{$index}{comment},
type => $indexes{$index}{type},
columns => [@{$indexes{$index}{columns}},
@{$t->indexes->{'PRIMARY'}->columns()}],
);
$i->is_supporting_table(1);
$t->indexes_set($index => $i);
my %idxcols;
foreach(@{$i->columns()})
{
$idxcols{$_} = $t->columns->{$_}
}
# Is a supporting table, add it to the lists:
my $idxname= $t->name().'_'.join('_',@{$indexes{$index}{columns}}).
"\$unique";
$report->supporting_tables_set($idxname => 1);
$t->supporting_tables_push($idxname);
$t->indexed_columns_set($_ => 1)
foreach @{$indexes{$index}{columns}};
my $st= new MySQL::NDB::Size::Table(name => $idxname,
rows => $count[0],
row_dm_overhead =>
{ '4.1' => 12,
'5.0' => 12,
'5.1' => 16+4,
},
row_vdm_overhead =>
{ '5.1' => 8 },
row_ddm_overhead =>
{ '5.1' => 8 },
);
do_table($st,
\%idxcols,
{
'PRIMARY' => {
'unique' => 0,#$indexes{$index}{unique},
'columns' => [@{$indexes{$index}{columns}}],
'type' => 'BTREE',
}
},
\@count);
}
}
$t->compute_row_size($report->versions);
} # do_table
foreach(@{$tables})
{
my $table= @{$_}[0];
my $info= $dbh->selectall_hashref('describe `'.$table.'`',"Field");
my @count = $dbh->selectrow_array('select count(*) from `'.$table.'`');
my $info;
{
my $sql= 'describe `'.$table.'`';
if($loadqueries)
{
$info= $queries{$sql};
}
else
{
$info= $dbh->selectall_hashref($sql,"Field");
$queries{$sql}= $info;
}
}
my @count;
{
my $sql= 'select count(*) from `'.$table.'`';
if($loadqueries)
{
@count= @{$queries{$sql}};
}
else
{
@count= $dbh->selectrow_array($sql);
$queries{$sql}= \@count;
}
}
my %indexes;
{
my $sth= $dbh->prepare("show index from `".$table.'`');
$sth->execute;
while(my $i = $sth->fetchrow_hashref)
{
my @show_indexes;
{
my $sql= "show index from `".$table.'`';
if($loadqueries)
{
@show_indexes= @{$queries{$sql}};
}
else
{
my $sth= $dbh->prepare($sql);
$sth->execute;
while(my $i = $sth->fetchrow_hashref)
{
push @show_indexes, $i;
}
$queries{$sql}= \@show_indexes;
}
}
foreach my $i(@show_indexes)
{
$indexes{${%$i}{Key_name}}= {
type=>${%$i}{Index_type},
unique=>!${%$i}{Non_unique},
comment=>${%$i}{Comment},
} if !defined($indexes{${%$i}{Key_name}});
$indexes{${%$i}{Key_name}}{columns}[${%$i}{Seq_in_index}-1]=
$indexes{${%$i}{Key_name}}{columns}[${%$i}{Seq_in_index}-1]=
${%$i}{Column_name};
}
}
do_table($table, $info, \%indexes, \@count);
}
my @NoOfTriggers;
# for unique hash indexes
$NoOfTriggers[$_]{val} += $NoOfIndexes[$_]{val}*3 foreach 0..$#releases;
# for ordered index
$NoOfTriggers[$_]{val} += $NoOfIndexes[$_]{val} foreach 0..$#releases;
my @ParamMemory;
foreach (0..$#releases) {
$ParamMemory[0]{releases}[$_]{val}= POSIX::ceil(200*$NoOfAttributes[$_]{val}/1024);
$ParamMemory[0]{name}= 'Attributes';
$ParamMemory[1]{releases}[$_]{val}= 20*$NoOfTables[$_]{val};
$ParamMemory[1]{name}= 'Tables';
$ParamMemory[2]{releases}[$_]{val}= 10*$NoOfIndexes[$_]{val};
$ParamMemory[2]{name}= 'OrderedIndexes';
$ParamMemory[3]{releases}[$_]{val}= 15*$NoOfIndexes[$_]{val};
$ParamMemory[3]{name}= 'UniqueHashIndexes';
}
$template->param(tables => \@table_size);
$template->param(Parameters => [{name=>'DataMemory (kb)',
releases=>\@dbDataMemory},
{name=>'IndexMemory (kb)',
releases=>\@dbIndexMemory},
{name=>'MaxNoOfTables',
releases=>\@NoOfTables},
{name=>'MaxNoOfAttributes',
releases=>\@NoOfAttributes},
{name=>'MaxNoOfOrderedIndexes',
releases=>\@NoOfIndexes},
{name=>'MaxNoOfUniqueHashIndexes',
releases=>\@NoOfIndexes},
{name=>'MaxNoOfTriggers',
releases=>\@NoOfTriggers}
]
);
$template->param(ParamMemory => \@ParamMemory);
print $template->output;
my $t= new MySQL::NDB::Size::Table(name => $table,
rows => $count[0],
row_dm_overhead =>
{ '4.1' => 12,
'5.0' => 12,
'5.1' => 16,
},
row_vdm_overhead => { '5.1' => 8 },
row_ddm_overhead => { '5.1' => 8 },
);
do_table($t, $info, \%indexes, \@count);
}
# compute table estimates
while(my ($tname,$t)= $report->tables_each())
{
$t->compute_estimate();
}
# Now parameters....
$report->parameters_set('NoOfTables' =>
new MySQL::NDB::Size::Parameter(name=>'NoOfTables',
mem_per_item=>20,
default=>128)
);
$report->parameters->{'NoOfTables'}->value_set($_ => scalar @{$report->tables_keys()})
foreach $report->versions;
$report->parameters_set('NoOfAttributes' =>
new MySQL::NDB::Size::Parameter(name=>'NoOfAttributes',
mem_per_item=>0.2,
default=>1000)
);
{
my $attr= 0;
while(my ($tname,$t)= $report->tables_each())
{
$attr+= scalar @{$t->columns_keys()};
}
$report->parameters->{'NoOfAttributes'}->value_set($_ => $attr)
foreach $report->versions;
}
$report->parameters_set('NoOfOrderedIndexes' =>
new MySQL::NDB::Size::Parameter(name=>'NoOfOrderedIndexes',
mem_per_item=>10,
default=>128)
);
{
my $attr= 0;
while(my ($tname,$t)= $report->tables_each())
{
next if $report->supporting_tables_exists($tname);
$attr+= scalar @{$t->indexes_keys()};
}
$report->parameters->{'NoOfOrderedIndexes'}->value_set($_ => $attr)
foreach $report->versions;
}
$report->parameters_set('NoOfUniqueHashIndexes' =>
new MySQL::NDB::Size::Parameter(name=>'NoOfUniqueHashIndexes',
mem_per_item=>15,
default=>64)
);
{
my $attr= 0;
while(my ($tname,$t)= $report->tables_each())
{
next if not $tname =~ /\$unique$/;
$attr++;
}
$report->parameters->{'NoOfUniqueHashIndexes'}->value_set($_ => $attr)
foreach $report->versions;
}
# Size of trigger is not documented
$report->parameters_set('NoOfTriggers' =>
new MySQL::NDB::Size::Parameter(name=>'NoOfTriggers',
mem_per_item=>0,
default=>768)
);
{
$report->parameters->{'NoOfTriggers'}->value_set(
$_ => (
(3*
$report->parameters->{'NoOfUniqueHashIndexes'}->value->{$_})
+
$report->parameters->{'NoOfOrderedIndexes'}->value->{$_}
+
(4* # for backups (3) and replication (1??)
$report->parameters->{'NoOfTables'}->value->{$_})
)
)
foreach $report->versions;
}
# DataMemory is in bytes...
$report->parameters_set('DataMemory' =>
new MySQL::NDB::Size::Parameter(name=>'DataMemory',
mem_per_item=>1024,
unit=>'KB',
default=>80*1024)
);
$report->parameters_set('IndexMemory' =>
new MySQL::NDB::Size::Parameter(name=>'IndexMemory',
mem_per_item=>1024,
unit=>'KB',
default=>18*1024)
);
{
foreach my $ver ($report->versions)
{
my $dm=0;
my $im=0;
while(my ($tname,$t)= $report->tables_each())
{
$dm+=$t->dm_needed->{$ver};
$dm+=$t->vdm_needed->{$ver} || 0;
$im+=$t->im_needed->{$ver};
}
$report->parameters->{'DataMemory'}->value_set($ver => $dm/1024);
$report->parameters->{'IndexMemory'}->value_set($ver => $im/1024);
}
}
if($savequeries)
{
open Q, "> $savequeries";
print Q Dumper(\%queries);
close Q;
}
use Data::Dumper;
if($debug)
{
eval 'print STDERR Dumper($report)';
}
if($format eq 'text')
{
my $text_out= new MySQL::NDB::Size::Output::Text($report);
$text_out->output();
}
elsif($format eq 'html')
{
my $html_out= new MySQL::NDB::Size::Output::HTML($report);
$html_out->output();
}
else
{
# default to text output
my $text_out= new MySQL::NDB::Size::Output::Text($report);
$text_out->output();
}
package MySQL::NDB::Size::Output::Text;
use Data::Dumper;
sub new { bless { report=> $_[1] }, $_[0]}
sub ul
{
my $s= $_[1]."\n";
$s.='-' foreach (1..length($_[1]));
return $s.="\n";
}
sub output
{
my $self= shift;
my $r= $self->{report};
print $self->ul("ndb_size.pl report for database ". $r->database().
" (".(($r->tables_count()||0)-($r->supporting_tables_count()||0)).
" tables)");
print "Connected to: ".$r->dsn()."\n\n";
print "Including information for versions: ".
join(', ',@{$r->versions})."\n\n";
foreach my $tname (@{$r->tables_keys()})
{
my $t= $r->tables->{$tname};
# next if $r->supporting_tables_exists($tname);
print $self->ul($tname)."\n";
# format strings
my $f= "%25s ";
my $v= "%10s ";
# Columns
print "DataMemory for Columns (* means varsized DataMemory):\n";
printf $f.'%20s %9s %5s','Column Name','Type','Varsized', 'Key';
printf $v, $_ foreach @{$r->versions};
print "\n";
my %dm_totals;
my %vdm_totals;
while(my ($cname, $c)= $t->columns_each())
{
$c->type =~ /^([^\(]*)/g;
printf $f.'%20s %9s %5s',
$cname,
$1.(
( $c->size and not $c->type() =~ /(enum|set)/)
? '('.$c->size.')'
:'' ),
($c->is_varsize)? 'Y':' ',
(defined($c->Key))?$c->Key:' ';
foreach(@{$r->versions})
{
if($c->ver_dm_exists($_))
{
printf $v, $c->ver_dm($_).(($c->is_varsize)?'*':'');
if($c->is_varsize())
{
$vdm_totals{$_}+= $c->ver_dm($_);
}
else
{
$dm_totals{$_}+= $c->ver_dm($_);
}
}
else
{
printf $v, $c->dm||'N/A';
$dm_totals{$_}+=$c->dm||0;
}
}
print "\n";
}
printf $f.'%20s %9s %5s','','','', '';
printf $v, '--' foreach @{$t->dm_versions};
print "\n";
printf $f.'%20s %9s %5s','Fixed Size Columns DM/Row','','','';
printf $v, $dm_totals{$_} foreach @{$r->versions};
print "\n";
printf $f.'%20s %9s %5s','Varsize Columns DM/Row','','','';
printf $v, $vdm_totals{$_} || 0 foreach @{$r->versions};
print "\n";
# DM for Indexes
print "\n\nDataMemory for Indexes:\n";
printf $f.'%20s ','Index Name','Type';
printf $v, $_ foreach @{$r->versions};
print "\n";
my %idx_dm_totals;
while(my ($iname, $i)= $t->indexes_each())
{
printf $f.'%20s ',$iname,$i->type();
foreach(@{$r->versions})
{
if($i->ver_dm_exists($_))
{
printf $v, $i->ver_dm($_).(($i->is_varsize)?'*':'');
$idx_dm_totals{$_}+= $i->ver_dm($_);
}
else
{
printf $v, ((defined($i->dm))?$i->dm:'N/A');
$idx_dm_totals{$_}+= $i->dm if defined($i->dm);
}
}
print "\n";
}
printf $f.'%20s ','','';
printf $v, '--' foreach @{$r->versions};
print "\n";
printf $f.'%20s ','Total Index DM/Row','';
printf $v, (defined($idx_dm_totals{$_}))?$idx_dm_totals{$_}:0
foreach @{$r->versions};
print "\n\n";
if(@{$t->supporting_tables()})
{
print "\n\nSupporting Tables DataMemory/Row";
my %supp_total;
foreach(@{$t->supporting_tables()})
{
print "\n";
printf $f, $_;
my $st= $r->tables->{$_};
printf $v, $st->row_dm_size->{$_} foreach @{$st->dm_versions};
$supp_total{$_}+=$st->row_dm_size->{$_}
foreach @{$st->dm_versions};
}
print "\n";
printf $f, '';
printf $v, '--' foreach @{$t->dm_versions};
print "\n";
printf $f, 'This DataMemory/Row';
printf $v, $t->row_dm_size->{$_} foreach @{$t->dm_versions};
$supp_total{$_}+=$t->row_dm_size->{$_}
foreach @{$t->dm_versions};
print "\n";
printf $f, 'Total DM/Row';
printf $v, $supp_total{$_} foreach @{$t->dm_versions};
print " Includes DM in other tables\n";
}
# IM for Columns
print "\n\nIndexMemory for Indexes:\n";
printf $f,'Index Name';
printf $v, $_ foreach @{$r->versions};
print "\n";
my %im_totals;
foreach my $iname (@{$t->indexes_keys()})
{
my $i= $t->indexes->{$iname};
next if $i->is_supporting_table();
printf $f, $iname;
foreach(@{$r->versions})
{
if(!defined($i->im))
{
printf $v,'N/A';
next;
}
if($i->ver_im_exists($_))
{
printf $v, $i->ver_im->{$_};
$im_totals{$_}+= $i->ver_im->{$_};
}
else
{
printf $v, $i->im;
$im_totals{$_}+=$i->im;
}
}
print "\n";
}
printf $f,'';
printf $v, '--' foreach @{$t->dm_versions};
print "\n";
printf $f,'Indexes IM/Row';
printf $v, $im_totals{$_} foreach @{$r->versions};
print "\n";
if(@{$t->supporting_tables()})
{
print "\n\nSupporting Tables IndexMemory/Row";
my %supp_total;
foreach(@{$t->supporting_tables()})
{
print "\n";
my $st= $r->tables->{$_};
foreach(@{$st->indexes_keys()})
{
printf $f, $st->name() if $_ eq 'PRIMARY';
printf $f, $st->name().$_ if $_ ne 'PRIMARY';
my $sti= $st->indexes->{$_};
printf $v, ($sti->ver_im_exists($_))
?$sti->ver_im->{$_}
:$sti->im() foreach @{$st->dm_versions};
$supp_total{$_}+= ($sti->ver_im_exists($_))
?$sti->ver_im->{$_}
:$sti->im() foreach @{$st->dm_versions};
}
}
print "\n";
printf $f, '';
printf $v, '--' foreach @{$t->dm_versions};
print "\n";
print "\n";
printf $f, 'Total Suppt IM/Row';
printf $v, $supp_total{$_} foreach @{$t->dm_versions};
print "\n";
}
print "\n\n\nSummary (for THIS table):\n";
printf $f, '';
printf $v, $_ foreach @{$r->versions};
print "\n";
printf $f, 'Fixed Overhead DM/Row';
printf $v, $t->row_dm_overhead->{$_} foreach @{$t->dm_versions};
print "\n";
printf $f, 'NULL Bytes/Row';
printf $v, $t->dm_null_bytes->{$_}||0 foreach @{$t->dm_versions};
print "\n";
printf $f, 'DataMemory/Row';
printf $v, $t->row_dm_size->{$_} foreach @{$t->dm_versions};
print " (Includes overhead, bitmap and indexes)\n";
print "\n";
printf $f, 'Varsize Overhead DM/Row';
printf $v, $t->row_vdm_overhead->{$_}||0 foreach @{$t->dm_versions};
print "\n";
printf $f, 'Varsize NULL Bytes/Row';
printf $v, $t->vdm_null_bytes->{$_}||0 foreach @{$t->dm_versions};
print "\n";
printf $f, 'Avg Varside DM/Row';
printf $v, (exists($t->row_vdm_size->{$_})?
$t->row_vdm_size->{$_}: 0)
foreach @{$r->versions};
print "\n\n";
printf $f, 'No. Rows';
printf $v, $t->rows foreach @{$r->versions};
print "\n\n";
printf $f, 'Rows/'.($t->dm_pagesize()/1024).'kb DM Page';
printf $v, $t->dm_rows_per_page->{$_} foreach @{$r->versions};
print "\n";
printf $f, 'Fixedsize DataMemory (KB)';
printf $v, $t->dm_needed->{$_}/1024 foreach @{$r->versions};
print "\n\n";
printf $f, 'Rows/'.($t->vdm_pagesize()/1024).'kb Varsize DM Page';
printf $v, $t->vdm_rows_per_page->{$_}||0 foreach @{$r->versions};
print "\n";
printf $f, 'Varsize DataMemory (KB)';
printf $v, ($t->vdm_needed->{$_}||0)/1024 foreach @{$r->versions};
print "\n\n";
printf $f, 'Rows/'.($t->im_pagesize()/1024).'kb IM Page';
printf $v, $t->im_rows_per_page->{$_} foreach @{$r->versions};
print "\n";
printf $f, 'IndexMemory (KB)';
printf $v, $t->im_needed->{$_}/1024 foreach @{$r->versions};
print "\n\n\n";
}
print "\n\n\n";
print $self->ul("Parameter Minimum Requirements");
print "* indicates greater than default\n\n";
printf "%25s ","Parameter";
printf "%15s ",'Default' ;
printf "%15s%1s ",$_,'' foreach @{$r->versions};
print "\n";
while( my ($pname, $p)= $r->parameters_each())
{
printf "%25s ",$pname.(($p->unit)?' ('.$p->unit.')':'');
printf "%15u ", $p->default;
printf "%15u%1s ", $p->value->{$_},
($p->value->{$_} > $p->default)?'*':''
foreach @{$r->versions};
print "\n";
}
print "\n\n\n";
}
sub table
{
my $self= shift;
my $t= shift;
}
package MySQL::NDB::Size::Output::HTML;
sub new { bless { report=> $_[1] }, $_[0]}
sub tag
{
my ($self,$tag,$content)= @_;
return "<$tag>$content</$tag>\n";
}
sub h1 { my ($self,$t)= @_; return $self->tag('h1',$t); }
sub h2 { my ($self,$t)= @_; return $self->tag('h2',$t); }
sub h3 { my ($self,$t)= @_; return $self->tag('h3',$t); }
sub h4 { my ($self,$t)= @_; return $self->tag('h4',$t); }
sub p { my ($self,$t)= @_; return $self->tag('p',$t); }
sub b { my ($self,$t)= @_; return $self->tag('b',$t); }
sub th
{
my ($self)= shift;
my $c;
$c.=$self->tag('th',$_) foreach @_;
return $self->tag('tr',$c);
}
sub tr
{
my ($self)= shift;
my $c;
$c.=$self->tag('td',$_) foreach @_;
return $self->tag('tr',$c);
}
sub td { my ($self,$t)= @_; return $self->tag('td',$t); }
sub ul
{
my ($self)= shift;
my $c;
$c.= " ".$self->li($_) foreach @_;
return $self->tag('ul',$c);
}
sub li { my ($self,$t)= @_; return $self->tag('li',$t); }
sub href
{
my ($self,$href,$t)= @_;
$href =~ s/\$/__/g;
return "<a href=\"$href\">$t</a>";
}
sub aname
{
my ($self,$href,$t)= @_;
$href =~ s/\$/__/g;
return "<a id=\"$href\">$t</a>";
}
sub output
{
my $self= shift;
my $r= $self->{report};
print <<ENDHTML;
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8"/>
<meta name="keywords" content="MySQL Cluster" />
ENDHTML
print "<title>MySQL Cluster size estimate for ".$r->database()."</title>";
print <<ENDHTML;
<style type="text/css">
table { border-collapse: collapse }
td,th { border: 1px solid black }
</style>
</head>
<body>
ENDHTML
print $self->h1("ndb_size.pl report for database ". $r->database().
" (".(($r->tables_count()||0)-($r->supporting_tables_count()||0)).
" tables)");
print $self->p("Connected to: ".$r->dsn());
print $self->p("Including information for versions: ".
join(', ',@{$r->versions}));
if(@{$r->tables_keys()})
{
print $self->h2("Table List");
my @tlist;
foreach(sort @{$r->tables_keys()})
{
push @tlist, $self->href("#$_",$_);
}
print $self->ul(@tlist);
}
foreach my $tname (sort @{$r->tables_keys()})
{
my $t= $r->tables->{$tname};
print $self->h2($self->aname($tname,$tname));
# format strings
my $f= "%25s ";
my $v= "%10s ";
# Columns
print $self->h3("DataMemory for Columns");
print $self->p("* means varsized DataMemory");
print "<table>\n";
print $self->th('Column Name','Type','Varsized', 'Key',
@{$r->versions});
my %dm_totals;
my %vdm_totals;
while(my ($cname, $c)= $t->columns_each())
{
$c->type =~ /^([^\(]*)/g;
my @verinfo;
foreach(@{$r->versions})
{
if($c->ver_dm_exists($_))
{
push @verinfo, $c->ver_dm($_).(($c->is_varsize)?'*':'');
if($c->is_varsize())
{
$vdm_totals{$_}+= $c->ver_dm($_);
}
else
{
$dm_totals{$_}+= $c->ver_dm($_);
}
}
else
{
push @verinfo, $c->dm||'N/A';
$dm_totals{$_}+=$c->dm||0;
}
}
print $self->tr(
$cname,
$1.(
( $c->size and not $c->type() =~ /(enum|set)/)
? '('.$c->size.')'
:'' ),
($c->is_varsize)? 'Y':' ',
(defined($c->Key))?$c->Key:' ',@verinfo);
}
{
my @dmtot;
push @dmtot, $self->b($dm_totals{$_}) foreach @{$r->versions};
print $self->tr($self->b('Fixed Size Columns DM/Row'),'','','',
@dmtot);
}
{
my @vdmtot;
push @vdmtot, $self->b($vdm_totals{$_} || 0)
foreach @{$r->versions};
print $self->tr($self->b('Varsize Columns DM/Row'),'','','',
@vdmtot);
}
print "</table>\n";
# DM for Indexes
print $self->h3('DataMemory for Indexes');
print "<table>\n";
print $self->th('Index Name','Type',@{$r->versions});
my %idx_dm_totals;
while(my ($iname, $i)= $t->indexes_each())
{
my @verinfo;
foreach(@{$r->versions})
{
if($i->ver_dm_exists($_))
{
push @verinfo, $i->ver_dm($_).(($i->is_varsize)?'*':'');
$idx_dm_totals{$_}+= $i->ver_dm($_);
}
else
{
push @verinfo, ((defined($i->dm))?$i->dm:'N/A');
$idx_dm_totals{$_}+= $i->dm if defined($i->dm);
}
}
printf $self->tr($iname,$i->type(),@verinfo);
}
{
my @idxtot;
push @idxtot, $self->b((defined($idx_dm_totals{$_}))
? $idx_dm_totals{$_}:0)
foreach @{$r->versions};
print $self->tr($self->b('Total Index DM/Row'),'',
@idxtot);
}
print "</table>";
if(@{$t->supporting_tables()})
{
print $self->h3("Supporting Tables DataMemory/Row");
my %supp_total;
print "<table>";
print $self->th('Table',@{$r->versions});
foreach(@{$t->supporting_tables()})
{
my $st= $r->tables->{$_};
my @stdm;
push @stdm, $st->row_dm_size->{$_} foreach @{$st->dm_versions};
print $self->tr($_,@stdm);
$supp_total{$_}+=$st->row_dm_size->{$_}
foreach @{$st->dm_versions};
}
{
my @rdmtot;
push @rdmtot, $self->b($t->row_dm_size->{$_})
foreach @{$t->dm_versions};
print $self->tr($self->b('This DataMemory/Row'),@rdmtot);
}
$supp_total{$_}+=$t->row_dm_size->{$_}
foreach @{$t->dm_versions};
{
my @tdmr;
push @tdmr, $self->b($supp_total{$_})
foreach @{$t->dm_versions};
print $self->tr($self->b('Total DM/Row (inludes DM in other tables)'),@tdmr);
}
print "</table>";
}
# IM for Columns
print $self->h3("IndexMemory for Indexes");
print "<table>\n";
print $self->th('Index Name', @{$r->versions});
my %im_totals;
foreach my $iname (@{$t->indexes_keys()})
{
my $i= $t->indexes->{$iname};
next if $i->is_supporting_table();
my @verinfo;
foreach(@{$r->versions})
{
if(!defined($i->im))
{
push @verinfo,'N/A';
next;
}
if($i->ver_im_exists($_))
{
push @verinfo, $i->ver_im->{$_};
$im_totals{$_}+= $i->ver_im->{$_};
}
else
{
push @verinfo, $i->im;
$im_totals{$_}+=$i->im;
}
}
print $self->tr($iname, @verinfo);
}
{
my @v;
push @v, $self->b($im_totals{$_}) foreach @{$r->versions};
printf $self->tr('Indexes IM/Row',@v);
}
print "</table>\n";
if(@{$t->supporting_tables()})
{
print $self->h3("Supporting Tables IndexMemory/Row");
print "<table>\n";
my %supp_total;
foreach(@{$t->supporting_tables()})
{
my $st= $r->tables->{$_};
foreach(@{$st->indexes_keys()})
{
my @r;
push @r, $st->name() if $_ eq 'PRIMARY';
push @r, $st->name().$_ if $_ ne 'PRIMARY';
my $sti= $st->indexes->{$_};
push @r, ($sti->ver_im_exists($_))
?$sti->ver_im->{$_}
:$sti->im() foreach @{$st->dm_versions};
$supp_total{$_}+= ($sti->ver_im_exists($_))
?$sti->ver_im->{$_}
:$sti->im() foreach @{$st->dm_versions};
print $self->tr(@r);
}
}
{
my @r;
push @r, $self->b($supp_total{$_}) foreach @{$t->dm_versions};
print $self->tr($self->b('Total Suppt IM/Row'),@r);
}
print "</table>\n";
}
print $self->h3("Summary (for THIS table)");
print $self->h4("Fixed Sized Part");
print "<table>\n";
print $self->tr('',@{$r->versions});
{ my @r;
push @r, $t->row_dm_overhead->{$_} foreach @{$t->dm_versions};
print $self->tr('Fixed Overhead DM/Row',@r);
}
{ my @r;
push @r, $t->dm_null_bytes->{$_}||0 foreach @{$t->dm_versions};
print $self->tr('NULL Bytes/Row',@r);
}
{ my @r;
push @r, $t->row_dm_size->{$_} foreach @{$t->dm_versions};
print $self->tr('DataMemory/Row (incl overhead, bitmap, indexes)',
@r);
}
print "</table>\n";
print $self->h4("Variable Sized Part");
print "<table>\n";
{ my @r;
push @r, $t->row_vdm_overhead->{$_}||0 foreach @{$t->dm_versions};
print $self->tr('Varsize Overhead DM/Row',@r);
}
{ my @r;
push @r, $t->vdm_null_bytes->{$_}||0 foreach @{$t->dm_versions};
print $self->tr('Varsize NULL Bytes/Row',@r);
}
{ my @r;
push @r, (exists($t->row_vdm_size->{$_})?
$t->row_vdm_size->{$_}: 0)
foreach @{$r->versions};
print $self->tr('Avg Varside DM/Row',@r);
}
print "</table>\n";
print $self->h4("Memory Calculations");
print "<table>\n";
{ my @r;
push @r, $t->rows foreach @{$r->versions};
print $self->tr('No. Rows',@r);
}
{ my @r;
push @r, $t->dm_rows_per_page->{$_} foreach @{$r->versions};
print $self->tr('Rows/'.($t->dm_pagesize()/1024).'kb DM Page',@r);
}
{ my @r;
push @r, $t->dm_needed->{$_}/1024 foreach @{$r->versions};
print $self->tr('Fixedsize DataMemory (KB)',@r);
}
{ my @r;
push @r, $t->vdm_rows_per_page->{$_}||0 foreach @{$r->versions};
print $self->tr('Rows/'.($t->vdm_pagesize()/1024).
'kb Varsize DM Page', @r);
}
{ my @r;
push @r, ($t->vdm_needed->{$_}||0)/1024 foreach @{$r->versions};
print $self->tr('Varsize DataMemory (KB)', @r);
}
{ my @r;
push @r, $t->im_rows_per_page->{$_} foreach @{$r->versions};
print $self->tr('Rows/'.($t->im_pagesize()/1024).'kb IM Page', @r);
}
{ my @r;
push @r, $t->im_needed->{$_}/1024 foreach @{$r->versions};
print $self->tr('IndexMemory (KB)', @r);
}
print "</table><hr/>\n\n";
}
print $self->h1("Parameter Minimum Requirements");
print $self->p("* indicates greater than default");
print "<table>\n";
print $self->th("Parameter",'Default',@{$r->versions});
while( my ($pname, $p)= $r->parameters_each())
{
my @r;
push @r, $p->value->{$_}.
(($p->value->{$_} > $p->default)?'*':'')
foreach @{$r->versions};
print $self->tr($pname.(($p->unit)?' ('.$p->unit.')':''),
$p->default,
@r);
}
print "</table></body></html>";
}
sub table
{
my $self= shift;
my $t= shift;
}
storage/ndb/tools/ndb_size.tmpl deleted 100644 → 0
View file @ c6377e66
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8"/>
<meta name="keywords" content="MySQL Cluster" />
<title>MySQL Cluster size estimate for <TMPL_VAR NAME="db" ESCAPE="HTML"></title>
<style type="text/css">
table { border-collapse: collapse }
td,th { border: 1px solid black }
</style>
</head>
<body>
<h1>MySQL Cluster analysis for <TMPL_VAR NAME="db" escape="html"></h1>
<p>This is an automated analysis of the <TMPL_VAR NAME="DSN" escape="html"> database for migration into <a href="http://www.mysql.com/">MySQL</a> Cluster. No warranty is made to the accuracy of the information.</p>
<p>This information should be valid for MySQL 4.1 and 5.0. Since 5.1 is not a final release yet, the numbers should be used as a guide only.</p>
<p>5.1-dd is for tables stored on disk. The ndb_size.pl estimates are <b>experimental</b> and should not be trusted. Notably we don't take into account indexed columns being in DataMemory versus non-indexed on disk.</p>
<h2>Parameter Settings</h2>
<p><b>NOTE</b> the configuration parameters below do not take into account system tables and other requirements.</p>
<table>
<tr>
<th>Parameter</th>
<TMPL_LOOP NAME=releases>
<th><TMPL_VAR NAME=rel></th>
</TMPL_LOOP>
</tr>
<TMPL_LOOP NAME=Parameters>
<tr>
<td><TMPL_VAR NAME=name></td>
<TMPL_LOOP NAME=releases>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
</TMPL_LOOP>
</table>
<h2>Memory usage because of parameters</h2>
<p>Usage is in kilobytes. Actual usage will vary as you should set the parameters larger than those listed in the table above.</p>
<table>
<tr>
<th>Parameter</th>
<TMPL_LOOP NAME=releases>
<th><TMPL_VAR NAME=rel></th>
</TMPL_LOOP>
</tr>
<TMPL_LOOP NAME=ParamMemory>
<tr>
<td><TMPL_VAR NAME=name></td>
<TMPL_LOOP NAME=releases>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
</TMPL_LOOP>
</table>
<h2>Table List</h2>
<ul>
<TMPL_LOOP NAME="tables">
<li><a href="#<TMPL_VAR NAME="table">"><TMPL_VAR NAME="table"></a></li>
</TMPL_LOOP>
</ul>
<hr/>
<TMPL_LOOP NAME="tables">
<h2><a name="<TMPL_VAR NAME="table">"><TMPL_VAR NAME="table"></a></h2>
<table>
<tr>
<th>Column</th>
<th>Type</th>
<th>VARSIZE</th>
<th>Size</th>
<th>Key</th>
<TMPL_LOOP NAME=releases>
<th><TMPL_VAR NAME=rel> NDB Size</th>
</TMPL_LOOP>
</tr>
<TMPL_LOOP NAME="columns">
<tr>
<td><TMPL_VAR NAME=name></td>
<td><TMPL_VAR NAME=type></td>
<td><TMPL_IF NAME=is_varsize>YES<TMPL_ELSE>&nbsp;</TMPL_IF></td>
<td><TMPL_VAR NAME=size></td>
<td><TMPL_VAR NAME=key></td>
<TMPL_LOOP NAME=datamemory>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
</TMPL_LOOP>
</table>
<p>&nbsp;</p>
<h3>Indexes</h3>
<p>We assume that indexes are ORDERED (not created USING HASH). If order is not required, 10 bytes of data memory can be saved per row if the index is created USING HASH</p>
<table>
<tr>
<th>Index</th>
<th>Type</th>
<th>Columns</th>
<TMPL_LOOP NAME=releases>
<th><TMPL_VAR NAME=rel> IdxMem</th>
</TMPL_LOOP>
<TMPL_LOOP NAME=releases>
<th><TMPL_VAR NAME=rel> DatMem</th>
</TMPL_LOOP>
</tr>
<TMPL_LOOP NAME="indexes">
<tr>
<td><TMPL_VAR NAME=name></td>
<td><TMPL_VAR NAME=type></td>
<td><TMPL_VAR NAME=columns></td>
<TMPL_LOOP NAME=indexmemory>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
<TMPL_LOOP NAME=datamemory>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
</TMPL_LOOP>
</table>
<h3>DataMemory Usage</h3>
<table>
<tr>
<th>&nbsp;</th>
<TMPL_LOOP NAME=releases>
<th><TMPL_VAR NAME=rel></th>
</TMPL_LOOP>
</tr>
<tr>
<th>Nr Varsized Attributes</th>
<TMPL_LOOP NAME=nrvarsize>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
<tr>
<th>Row Overhead</th>
<TMPL_LOOP NAME=RowOverhead>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
<tr>
<th>Varsized Overhead</th>
<TMPL_LOOP NAME=nrvarsize_mem>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
<tr>
<th>Column DataMemory/Row</th>
<TMPL_LOOP NAME=RowDataMemory>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
<tr>
<th>Index DataMemory/Row</th>
<TMPL_LOOP NAME=IndexDataMemory>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
<tr>
<th>Total DataMemory/Row</th>
<TMPL_LOOP NAME=TotalDataMemory>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
<tr>
<th>Rows per 32kb page</th>
<TMPL_LOOP NAME=RowPerPage>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
<tr>
<th>Current number of rows</th>
<TMPL_LOOP NAME=count>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
<tr>
<th>Total DataMemory (kb)</th>
<TMPL_LOOP NAME=DataMemory>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
</table>
<h3>IndexMemory Usage</h3>
<table>
<tr>
<th>&nbsp;</th>
<TMPL_LOOP NAME=releases>
<th><TMPL_VAR NAME=rel></th>
</TMPL_LOOP>
</tr>
<tr>
<th>IndexMemory/Row</th>
<TMPL_LOOP NAME=RowIndexMemory>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
<tr>
<th>Rows per 8kb page</th>
<TMPL_LOOP NAME=RowPerIndexPage>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
<tr>
<th>Current number of rows</th>
<TMPL_LOOP NAME=count>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
<tr>
<th>Total IndexMemory (kb)</th>
<TMPL_LOOP NAME=IndexMemory>
<td><TMPL_VAR NAME=val></td>
</TMPL_LOOP>
</tr>
</table>
<hr/>
</TMPL_LOOP>
<p>This is the output of ndb_size.pl.</p>
</body>
</html>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7