Skip to content

O
olapy
Commit 217c6233 authored by mouadh's avatar mouadh
Browse files
Options

remove memory bench

parent 8cd36207
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 808 deletions
olapy/core/mdx/executor/execute_config_file.py
View file @ 217c6233
from __future__ import absolute_import, division, print_function from __future__ import absolute_import, division, print_function
from ..tools.mem_bench import memory_usage
from ..tools.connection import MyDB from ..tools.connection import MyDB
import pandas.io.sql as psql import pandas.io.sql as psql
import os import os
...@@ -21,7 +20,6 @@ def _load_table_config_file(executer_instance, cube_obj): ...@@ -21,7 +20,6 @@ def _load_table_config_file(executer_instance, cube_obj):
db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube) db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
memory_usage("1 - before executing query //// _load_table_config_file")
for dimension in cube_obj.dimensions: for dimension in cube_obj.dimensions:
df = psql.read_sql_query("SELECT * FROM {0}".format(dimension.name), df = psql.read_sql_query("SELECT * FROM {0}".format(dimension.name),
...@@ -43,8 +41,6 @@ def _load_table_config_file(executer_instance, cube_obj): ...@@ -43,8 +41,6 @@ def _load_table_config_file(executer_instance, cube_obj):
col for col in df.columns if col.lower()[-2:] != 'id' col for col in df.columns if col.lower()[-2:] != 'id'
]] ]]
memory_usage("2 - after query, before fetchall /////// _load_table_config_file")
return tables return tables
# excel client # excel client
...@@ -60,7 +56,6 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj): ...@@ -60,7 +56,6 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube) db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
# load facts table # load facts table
memory_usage("1 - before executing query //// _construct_star_schema_config_file")
fusion = psql.read_sql_query( fusion = psql.read_sql_query(
"SELECT * FROM {0}".format(executer_instance.facts), db.engine) "SELECT * FROM {0}".format(executer_instance.facts), db.engine)
...@@ -83,8 +78,6 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj): ...@@ -83,8 +78,6 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
# remove suffixe from dimension and keep the same column name for facts # remove suffixe from dimension and keep the same column name for facts
suffixes=('', '_y')) suffixes=('', '_y'))
memory_usage("2 - after query, before fetchall /////// _construct_star_schema_config_file")
# measures in config-file only # measures in config-file only
if cubes_obj.facts[0].measures: if cubes_obj.facts[0].measures:
executer_instance.measures = cubes_obj.facts[0].measures executer_instance.measures = cubes_obj.facts[0].measures
...@@ -109,14 +102,10 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj): ...@@ -109,14 +102,10 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
if cubes_obj.facts[0].columns: if cubes_obj.facts[0].columns:
all_columns += cubes_obj.facts[0].columns all_columns += cubes_obj.facts[0].columns
memory_usage("1 - before executing query //// 1111 _construct_web_star_schema_config_file ")
fusion = psql.read_sql_query( fusion = psql.read_sql_query(
"SELECT * FROM {0}".format(executer_instance.facts), db.engine) "SELECT * FROM {0}".format(executer_instance.facts), db.engine)
memory_usage("2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file")
tables = {} tables = {}
memory_usage("1 - before executing query //// 3333333333 _construct_web_star_schema_config_file ")
for table in cubes_obj.tables: for table in cubes_obj.tables:
tab = psql.read_sql_query("SELECT * FROM {0}".format(table.name), tab = psql.read_sql_query("SELECT * FROM {0}".format(table.name),
...@@ -142,14 +131,11 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj): ...@@ -142,14 +131,11 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
all_columns += list(tab.columns) all_columns += list(tab.columns)
tables.update({table.name: tab}) tables.update({table.name: tab})
memory_usage("2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file")
# measures in config-file only # measures in config-file only
if cubes_obj.facts[0].measures: if cubes_obj.facts[0].measures:
executer_instance.measures = cubes_obj.facts[0].measures executer_instance.measures = cubes_obj.facts[0].measures
all_columns += cubes_obj.facts[0].measures all_columns += cubes_obj.facts[0].measures
memory_usage("1 - before executing query //// 55555555 _construct_web_star_schema_config_file ")
for fact_key, dimension_and_key in cubes_obj.facts[0].keys.items(): for fact_key, dimension_and_key in cubes_obj.facts[0].keys.items():
dimension_name = dimension_and_key.split('.')[0] dimension_name = dimension_and_key.split('.')[0]
if dimension_name in tables.keys(): if dimension_name in tables.keys():
...@@ -165,7 +151,5 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj): ...@@ -165,7 +151,5 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
# remove suffixe from dimension and keep the same column name for facts # remove suffixe from dimension and keep the same column name for facts
suffixes=('', '_y')) suffixes=('', '_y'))
memory_usage("2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file")
return fusion[[column for column in all_columns if 'id' != column[-2:]]] return fusion[[column for column in all_columns if 'id' != column[-2:]]]
olapy/core/mdx/executor/execute_db.py
View file @ 217c6233
...@@ -2,7 +2,6 @@ from __future__ import absolute_import, division, print_function ...@@ -2,7 +2,6 @@ from __future__ import absolute_import, division, print_function
from sqlalchemy import inspect from sqlalchemy import inspect
import pandas as pd import pandas as pd
from ..tools.mem_bench import memory_usage
from ..tools.connection import MyDB from ..tools.connection import MyDB
import pandas.io.sql as psql import pandas.io.sql as psql
...@@ -77,7 +76,6 @@ def _load_tables_db(executer_instance): ...@@ -77,7 +76,6 @@ def _load_tables_db(executer_instance):
db = MyDB(db_config_file_path=executer_instance.DATA_FOLDER,db=executer_instance.cube) db = MyDB(db_config_file_path=executer_instance.DATA_FOLDER,db=executer_instance.cube)
inspector = inspect(db.engine) inspector = inspect(db.engine)
memory_usage("1 - before executing query //// _load_tables_db")
for table_name in inspector.get_table_names(): for table_name in inspector.get_table_names():
# value = psql.read_sql_query( # value = psql.read_sql_query(
...@@ -93,21 +91,6 @@ def _load_tables_db(executer_instance): ...@@ -93,21 +91,6 @@ def _load_tables_db(executer_instance):
col for col in value.columns if col.lower()[-3:] != '_id' col for col in value.columns if col.lower()[-3:] != '_id'
]] ]]
# tables = {}
# db = MyDB(db_config_file_path=executer_instance.DATA_FOLDER, db=executer_instance.cube)
# # inspector = inspect(db.engine)
# cursor = db.engine.cursor()
# cursor.execute("""SELECT table_name FROM information_schema.tables
# WHERE table_schema = 'public'""")
#
# for table_name in cursor.fetchall():
# value = psql.read_sql_query(
# 'SELECT * FROM "{0}" '.format(table_name[0]), db.engine)
#
# tables[table_name[0]] = value[[
# col for col in value.columns if col.lower()[-3:] != '_id'
# ]]
memory_usage("2 - after query, before fetchall /////// _load_tables_db")
return tables return tables
...@@ -119,7 +102,6 @@ def _construct_star_schema_db(executer_instance): ...@@ -119,7 +102,6 @@ def _construct_star_schema_db(executer_instance):
:return: star schema DataFrame :return: star schema DataFrame
""" """
db = MyDB(db=executer_instance.cube) db = MyDB(db=executer_instance.cube)
memory_usage("1 - before executing query //// _construct_star_schema_db")
# load facts table # load facts table
with db.engine as connection: with db.engine as connection:
fusion = psql.read_sql_query( fusion = psql.read_sql_query(
...@@ -136,5 +118,4 @@ def _construct_star_schema_db(executer_instance): ...@@ -136,5 +118,4 @@ def _construct_star_schema_db(executer_instance):
print('No common column') print('No common column')
pass pass
memory_usage("2 - after query, before fetchall /////// _construct_star_schema_db")
return fusion return fusion
olapy/core/mdx/tools/bech_mem_opt2.txt deleted 100644 → 0
View file @ 8cd36207
Memory summary:1 - before executing query //// _load_tables_db
VM: 700.28Mb
Memory summary:2 - after query, before fetchall /////// _load_tables_db
VM: 2781.79Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 2782.04Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 2782.29Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 2782.29Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 2784.29Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 2784.29Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 2784.29Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 2784.04Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 2783.79Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 2783.79Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 2784.29Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 2784.29Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 2784.29Mb
Memory summary:1 - before executing query //// _load_tables_db
types | # objects | total size
======= | =========== | ============
dict | 7429 | 8.72 MB
str | 58990 | 8.30 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// _load_tables_db
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
dict | 7802 | 8.88 MB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 592 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 179 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 2.01 GB
<class 'pandas.core.series.Series | 618 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 630 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 185 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 2.01 GB
<class 'pandas.core.series.Series | 604 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 2.01 GB
<class 'pandas.core.series.Series | 592 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 181 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 181 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
<class 'pandas.core.series.Series | 629 | 2.01 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 2.01 GB
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 2.01 GB
<class 'pandas.core.series.Series | 615 | 2.01 GB
olapy/core/mdx/tools/bech_mem_opt3.txt deleted 100644 → 0
View file @ 8cd36207
Memory summary:1 - before executing query //// _load_tables_db
VM: 700.27Mb
Memory summary:2 - after query, before fetchall /////// _load_tables_db
VM: 2719.36Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 2719.36Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 2719.86Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 2719.86Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 2721.61Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 2721.61Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 2721.61Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 2721.61Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 2721.36Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 2721.36Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 2721.86Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 2721.86Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 2721.86Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 2721.86Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 2721.36Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 2721.36Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 2721.86Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 2721.86Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 2721.86Mb
****************************************
types | # objects | total size
======= | =========== | ============
dict | 7430 | 8.72 MB
str | 58996 | 8.30 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// _load_tables_db
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 1.93 GB
dict | 7803 | 8.86 MB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 592 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 179 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 180 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 180 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 1.93 GB
<class 'pandas.core.series.Series | 618 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 630 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 185 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 1.93 GB
<class 'pandas.core.series.Series | 604 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 1.93 GB
<class 'pandas.core.series.Series | 592 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 181 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 181 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
<class 'pandas.core.series.Series | 629 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
<class 'pandas.core.series.Series | 615 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 603 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 180 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 181 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 181 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
<class 'pandas.core.series.Series | 629 | 1.93 GB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 1.93 GB
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 1.93 GB
<class 'pandas.core.series.Series | 615 | 1.93 GB
----------------------------
olapy/core/mdx/tools/bech_mem_with_psycopg.txt deleted 100644 → 0
View file @ 8cd36207
Memory summary:1 - before executing query //// _load_tables_db
VM: 699.77Mb
Memory summary:2 - after query, before fetchall /////// _load_tables_db
VM: 1763.52Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 1763.52Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 1763.77Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 1763.77Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
VM: 1764.27Mb
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
VM: 1764.27Mb
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
VM: 1764.52Mb
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
VM: 1764.52Mb
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
VM: 1764.52Mb
**********************************
types | # objects | total size
======= | =========== | ============
dict | 7235 | 8.59 MB
str | 58455 | 8.18 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// _load_tables_db
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 539.66 MB
dict | 7649 | 8.78 MB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 592 | 538.26 MB
<class 'pandas.core.frame.DataFrame | 179 | 538.22 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 539.70 MB
<class 'pandas.core.frame.DataFrame | 180 | 539.66 MB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 618 | 539.70 MB
<class 'pandas.core.frame.DataFrame | 180 | 539.66 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 540.42 MB
<class 'pandas.core.series.Series | 618 | 539.70 MB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 630 | 540.47 MB
<class 'pandas.core.frame.DataFrame | 185 | 540.42 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 185 | 540.83 MB
<class 'pandas.core.series.Series | 604 | 539.02 MB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 180 | 538.50 MB
<class 'pandas.core.series.Series | 592 | 538.26 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 540.06 MB
<class 'pandas.core.frame.DataFrame | 181 | 539.94 MB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 540.06 MB
<class 'pandas.core.frame.DataFrame | 181 | 539.94 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 540.71 MB
<class 'pandas.core.series.Series | 629 | 540.06 MB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 540.83 MB
<class 'pandas.core.frame.DataFrame | 186 | 540.71 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 541.11 MB
<class 'pandas.core.series.Series | 615 | 539.39 MB
----------------------------
Memory summary:1 - before executing query //// 1111 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 603 | 538.62 MB
<class 'pandas.core.frame.DataFrame | 180 | 538.50 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 222222222222 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 540.06 MB
<class 'pandas.core.frame.DataFrame | 181 | 539.94 MB
----------------------------
Memory summary:1 - before executing query //// 3333333333 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 629 | 540.06 MB
<class 'pandas.core.frame.DataFrame | 181 | 539.94 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 44444444 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 540.71 MB
<class 'pandas.core.series.Series | 629 | 540.06 MB
----------------------------
Memory summary:1 - before executing query //// 55555555 _construct_web_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.series.Series | 641 | 540.83 MB
<class 'pandas.core.frame.DataFrame | 186 | 540.71 MB
----------------------------
Memory summary:2 - after query, before fetchall /////// 6666666666 _construct_star_schema_config_file
types | # objects | total size
===================================== | =========== | ============
<class 'pandas.core.frame.DataFrame | 186 | 541.11 MB
<class 'pandas.core.series.Series | 615 | 539.39 MB
olapy/core/mdx/tools/mem_bench.py deleted 100644 → 0
View file @ 8cd36207
import os
from os.path import expanduser
from pympler import summary, muppy
import psutil
def get_virtual_memory_usage_kb():
"""
The process's current virtual memory size in Kb, as a float.
"""
return float(psutil.Process().memory_info_ex().vms) / 1024.0
def memory_usage(where):
"""
Print out a basic summary of memory usage.
"""
with open(os.path.join(expanduser('~'), 'bech_mem.txt'), mode='a+') as file:
mem_summary = summary.summarize(muppy.get_objects())
file.write("Memory summary:" + where + '\n\n')
print("Memory summary:" + where )
summary.print_(mem_summary, limit=2)
print('----------------------------')
file.write("VM: %.2fMb" % (get_virtual_memory_usage_kb() / 1024.0) + '\n\n')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7