Skip to content

O
olapy
Commit cd49e537 authored by mouadh's avatar mouadh
Browse files
Options

format

parent 09c6d728
Hide whitespace changes
Inline Side-by-side
Showing with 265 additions and 172 deletions
micro_bench/__init__.py
View file @ cd49e537
......@@ -23,7 +23,8 @@ PROFILING_LINES = 15
def main():
file = open('bench_result' + str(datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")), 'w')
file = open('bench_result' +
str(datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")), 'w')
gen = CubeGen(number_dimensions=3, rows_length=1000, columns_length=5)
gen.generate_csv(gen.generate_cube(3, 1000))
XmlaProviderService.discover_tools.change_catalogue(CUBE_NAME)
......@@ -52,14 +53,18 @@ def main():
WHERE ([Measures].[Amount])
CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
file.write("Query 1 :\n" + cmd + "\n----------------------------------------------------------\n\n")
file.write(
"Query 1 :\n" + cmd +
"\n----------------------------------------------------------\n\n")
t.add_row(['Query 1', mbench.bench(conn, cmd, CUBE_NAME)])
cmd = """SELECT
NON EMPTY Hierarchize(AddCalculatedMembers(DrilldownMember({{{
[table0].[table0].[All table0A].Members}}}, {
[table0].[table0].[table0A].[""" + str(XmlaProviderService.discover_tools.star_schema_dataframe.table0A[1]) + """]})))
[table0].[table0].[table0A].[""" + str(
XmlaProviderService.discover_tools.star_schema_dataframe.table0A[
1]) + """]})))
DIMENSION PROPERTIES PARENT_UNIQUE_NAME,HIERARCHY_UNIQUE_NAME
ON COLUMNS
FROM [""" + CUBE_NAME + """]
......@@ -67,12 +72,18 @@ def main():
CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS
"""
file.write("Query 2 :\n" + cmd + "\n----------------------------------------------------------\n\n")
file.write(
"Query 2 :\n" + cmd +
"\n----------------------------------------------------------\n\n")
t.add_row(['Query 2', mbench.bench(conn, cmd, CUBE_NAME)])
tup = "[table0].[table0].[table0A].[" + str(XmlaProviderService.discover_tools.star_schema_dataframe.table0A[0]) + "]"
tup = "[table0].[table0].[table0A].[" + str(
XmlaProviderService.discover_tools.star_schema_dataframe.table0A[
0]) + "]"
for d in range(REFINEMENT_LVL):
tup += ",\n[table0].[table0].[table0A].[" + str(XmlaProviderService.discover_tools.star_schema_dataframe.table0A[d + 1]) + "]"
tup += ",\n[table0].[table0].[table0A].[" + str(
XmlaProviderService.discover_tools.star_schema_dataframe.table0A[
d + 1]) + "]"
cmd = """
SELECT NON EMPTY Hierarchize(AddCalculatedMembers(DrilldownMember({{{
......@@ -86,15 +97,25 @@ def main():
CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS
"""
file.write("Query 3 :\n" + cmd + "\n----------------------------------------------------------\n\n")
file.write(
"Query 3 :\n" + cmd +
"\n----------------------------------------------------------\n\n")
t.add_row(['Query 3', mbench.bench(conn, cmd, CUBE_NAME)])
file.write(str(t) + "\n\n")
try:
file.write('******************************************************************************\n')
file.write('* mondrian with "warehouse" Cube (note the same as olapy but resemble to it) *\n')
file.write('* (olapy warehouse"s cube has more rows) *\n')
file.write('******************************************************************************\n\n')
file.write(
'******************************************************************************\n'
)
file.write(
'* mondrian with "warehouse" Cube (note the same as olapy but resemble to it) *\n'
)
file.write(
'* (olapy warehouse"s cube has more rows) *\n'
)
file.write(
'******************************************************************************\n\n'
)
t = PrettyTable(['Query', 'mondrian', 'olapy'])
p2 = xmla.XMLAProvider()
......@@ -111,9 +132,13 @@ def main():
FROM
[Warehouse]"""
file.write("Query 1 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
t.add_row(['Query 1', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(conn, cmd, 'foodmart')])
file.write(
"Query 1 :\n" + cmd2 +
"\n----------------------------------------------------------\n\n")
t.add_row([
'Query 1', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(
conn, cmd, 'foodmart')
])
cmd = """SELECT NON EMPTY Hierarchize(AddCalculatedMembers(DrilldownMember({{{
[Product].[Product].[All brand_name].Members}}}, {
......@@ -131,8 +156,13 @@ def main():
DIMENSION PROPERTIES PARENT_UNIQUE_NAME ON 0
FROM [Warehouse]"""
file.write("Query 2 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
t.add_row(['Query 2', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(conn, cmd, 'foodmart')])
file.write(
"Query 2 :\n" + cmd2 +
"\n----------------------------------------------------------\n\n")
t.add_row([
'Query 2', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(
conn, cmd, 'foodmart')
])
cmd = """SELECT NON EMPTY CrossJoin(Hierarchize(AddCalculatedMembers({
[Product].[Product].[All brand_name].Members})), Hierarchize(AddCalculatedMembers({
......@@ -151,8 +181,13 @@ def main():
DIMENSION PROPERTIES PARENT_UNIQUE_NAME ON 0
FROM [Warehouse]"""
file.write("Query 3 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
t.add_row(['Query 3', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(conn, cmd, 'foodmart')])
file.write(
"Query 3 :\n" + cmd2 +
"\n----------------------------------------------------------\n\n")
t.add_row([
'Query 3', mbench.bench(c2, cmd2, 'FoodMart'), mbench.bench(
conn, cmd, 'foodmart')
])
file.write(str(t) + "\n\n")
......@@ -167,16 +202,23 @@ def main():
t = PrettyTable(['Query', 'olapy', 'icCube'])
p2 = xmla.XMLAProvider()
c2 = p2.connect(location="http://localhost:8282/icCube/xmla", username="demo",
password="demo")
c2 = p2.connect(
location="http://localhost:8282/icCube/xmla",
username="demo",
password="demo")
cmd = """SELECT
FROM [Sales]
WHERE ([Measures].[Amount])
CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
file.write("Query 1 :\n" + cmd + "\n----------------------------------------------------------\n\n")
t.add_row(['Query 1', mbench.bench(conn, cmd, 'sales'), mbench.bench(c2, cmd, 'Sales (Excel)')])
file.write(
"Query 1 :\n" + cmd +
"\n----------------------------------------------------------\n\n")
t.add_row([
'Query 1', mbench.bench(conn, cmd, 'sales'), mbench.bench(
c2, cmd, 'Sales (Excel)')
])
cmd = """SELECT
NON EMPTY Hierarchize(AddCalculatedMembers({DrilldownLevel({
......@@ -194,8 +236,13 @@ def main():
FROM [Sales]
WHERE ([Measures].[Amount])
CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
file.write("Query 2 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
t.add_row(['Query 2', mbench.bench(conn, cmd2, 'sales'), mbench.bench(c2, cmd, 'Sales (Excel)')])
file.write(
"Query 2 :\n" + cmd2 +
"\n----------------------------------------------------------\n\n")
t.add_row([
'Query 2', mbench.bench(conn, cmd2, 'sales'), mbench.bench(
c2, cmd, 'Sales (Excel)')
])
cmd = """SELECT
NON EMPTY Hierarchize(AddCalculatedMembers(DrilldownMember({{DrilldownMember({{DrilldownLevel({
......@@ -224,8 +271,13 @@ def main():
FROM [sales]
WHERE ([Measures].[Amount])
CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
file.write("Query 3 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
t.add_row(['Query 3', mbench.bench(conn, cmd2, 'sales'), mbench.bench(c2, cmd, 'Sales (Excel)')])
file.write(
"Query 3 :\n" + cmd2 +
"\n----------------------------------------------------------\n\n")
t.add_row([
'Query 3', mbench.bench(conn, cmd2, 'sales'), mbench.bench(
c2, cmd, 'Sales (Excel)')
])
cmd = """SELECT
NON EMPTY CrossJoin(Hierarchize(AddCalculatedMembers(DrilldownMember
......@@ -257,8 +309,13 @@ def main():
WHERE ([Measures].[Amount])
CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
file.write("Query 4 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
t.add_row(['Query 4', mbench.bench(conn, cmd2, 'sales'), mbench.bench(c2, cmd, 'Sales (Excel)')])
file.write(
"Query 4 :\n" + cmd2 +
"\n----------------------------------------------------------\n\n")
t.add_row([
'Query 4', mbench.bench(conn, cmd2, 'sales'), mbench.bench(
c2, cmd, 'Sales (Excel)')
])
cmd = """SELECT
NON EMPTY CrossJoin(CrossJoin(Hierarchize(AddCalculatedMembers
......@@ -307,8 +364,13 @@ def main():
WHERE ([Measures].[Amount])
CELL PROPERTIES VALUE, FORMAT_STRING, LANGUAGE, BACK_COLOR, FORE_COLOR, FONT_FLAGS"""
file.write("Query 5 :\n" + cmd2 + "\n----------------------------------------------------------\n\n")
t.add_row(['Query 5', mbench.bench(conn, cmd2, 'sales'), mbench.bench(c2, cmd, 'Sales (Excel)')])
file.write(
"Query 5 :\n" + cmd2 +
"\n----------------------------------------------------------\n\n")
t.add_row([
'Query 5', mbench.bench(conn, cmd2, 'sales'), mbench.bench(
c2, cmd, 'Sales (Excel)')
])
file.write(str(t) + "\n\n")
......@@ -316,8 +378,8 @@ def main():
print('Make sure icCube is running and containing sales Excel cube')
pass
file.write('---------------- Profiling olapy Query 5 ------------------ \n\n')
file.write(
'---------------- Profiling olapy Query 5 ------------------ \n\n')
cProfile.run("""cmd = '''
SELECT
NON EMPTY CrossJoin(CrossJoin(Hierarchize(AddCalculatedMembers(DrilldownMember({{DrilldownMember({{{
......@@ -347,19 +409,19 @@ request.Command = Command(Statement = cmd)
request.Properties = Propertielist(PropertyList = Property(Catalog='sales'))
XmlaProviderService().Execute(XmlaProviderService(),request)""",
"{}.profile".format(__file__)
)
"{}.profile".format(__file__))
s = pstats.Stats("{}.profile".format(__file__), stream=file)
s.strip_dirs()
s.sort_stats("time").print_stats(PROFILING_LINES)
try:
os.system('gprof2dot -f pstats __init__.py.profile | dot -Tpng -o profile.png')
os.system(
'gprof2dot -f pstats main.py.profile | dot -Tpng -o profile.png')
except:
print('make sure gprof2dot and graphviz are installed')
os.remove('__init__.py.profile ')
os.remove('main.py.profile ')
gen.remove_temp_cube()
file.close()
server.stop()
......
micro_bench/cube_generator.py
View file @ cd49e537
......@@ -7,7 +7,8 @@ import shutil
from olapy.core.mdx.executor.execute import MdxEngine
CUBE_NAME ="temp_cube"
CUBE_NAME = "temp_cube"
class CubeGen:
"""
......@@ -21,7 +22,8 @@ class CubeGen:
# We have to generate DataFrames and save them to csv format because XmlaProviderService and
# MdxEngine classes use those files
def __init__(self, number_dimensions=1, rows_length=1000, columns_length=2):
def __init__(self, number_dimensions=1, rows_length=1000,
columns_length=2):
self.number_dimensions = number_dimensions
self.rows_length = rows_length
self.columns_length = columns_length
......@@ -38,14 +40,20 @@ class CubeGen:
facts = pd.DataFrame()
for idx, dim in enumerate(range(self.number_dimensions)):
table_name = 'table' + str(idx)
table_values = pd.DataFrame(np.random.randint(min_val, max_val, size=(self.rows_length, self.columns_length)),
columns=list(
table_name + col for col in string.ascii_uppercase[:self.columns_length]))
table_values = pd.DataFrame(
np.random.randint(
min_val,
max_val,
size=(self.rows_length, self.columns_length)),
columns=list(
table_name + col
for col in string.ascii_uppercase[:self.columns_length]))
table_values.index.name = table_name + "_id"
tables[table_name] = table_values.reset_index()
facts[table_name + "_id"] = tables[table_name][table_name + "_id"]
facts['Amount'] = np.random.randint(300, 1000, size=(self.rows_length, 1))
facts['Amount'] = np.random.randint(
300, 1000, size=(self.rows_length, 1))
tables['Facts'] = facts
return tables
......@@ -57,19 +65,22 @@ class CubeGen:
:param tables: dict of DataFrames
"""
cube_path = os.path.join(
os.path.abspath(
os.path.join(os.path.dirname(__file__), "..")), MdxEngine.CUBE_FOLDER)
os.path.abspath(os.path.join(os.path.dirname(__file__), "..")),
MdxEngine.CUBE_FOLDER)
if not os.path.isdir(os.path.join(cube_path, CUBE_NAME)):
os.makedirs(os.path.join(cube_path, CUBE_NAME))
cube_path = os.path.join(cube_path, CUBE_NAME)
for (table_name, table_value) in tables.items():
table_value.to_csv(os.path.join(os.path.join(cube_path, table_name + '.csv')), sep=";", index=False)
table_value.to_csv(
os.path.join(os.path.join(cube_path, table_name + '.csv')),
sep=";",
index=False)
@staticmethod
def remove_temp_cube():
"""Remove the temporary cube."""
cube_path = os.path.join(
os.path.abspath(
os.path.join(os.path.dirname(__file__), "..")), MdxEngine.CUBE_FOLDER)
os.path.abspath(os.path.join(os.path.dirname(__file__), "..")),
MdxEngine.CUBE_FOLDER)
if os.path.isdir(os.path.join(cube_path, CUBE_NAME)):
shutil.rmtree(os.path.join(cube_path, CUBE_NAME))
micro_bench/micro_bench.py
View file @ cd49e537
......@@ -21,4 +21,5 @@ class MicBench:
to one million
:return: float execution time in seconds
"""
return Timer(lambda: connection.Execute(query, Catalog=cube)).timeit(number=number)
return Timer(lambda: connection.Execute(query, Catalog=cube)).timeit(
number=number)
olapy/core/mdx/executor/execute.py
View file @ cd49e537
......@@ -41,6 +41,7 @@ class MdxEngine:
# (before instantiate MdxEngine I need to access cubes information)
csv_files_cubes = []
postgres_db_cubes = []
# to show just config file's dimensions
def __init__(self,
......@@ -113,7 +114,8 @@ class MdxEngine:
try:
db = MyDB(db_config_file_path=olapy_data_location)
# TODO this work only with postgres
result = db.engine.execute('SELECT datname FROM pg_database WHERE datistemplate = false;')
result = db.engine.execute(
'SELECT datname FROM pg_database WHERE datistemplate = false;')
available_tables = result.fetchall()
# cursor.execute("""SELECT datname FROM pg_database
# WHERE datistemplate = false;""")
......@@ -164,9 +166,10 @@ class MdxEngine:
config_file_parser = ConfigParser(self.cube_path)
tables = {}
if self.client == 'excel' and config_file_parser.config_file_exist(client_type=self.client
) and self.cube in config_file_parser.get_cubes_names(client_type=self.client
):
if self.client == 'excel' and config_file_parser.config_file_exist(
client_type=self.
client) and self.cube in config_file_parser.get_cubes_names(
client_type=self.client):
# for web (config file) we need only star_schema_dataframes, not all tables
for cubes in config_file_parser.construct_cubes():
......@@ -188,7 +191,8 @@ class MdxEngine:
# if web, get measures from config file
config_file_parser = ConfigParser(self.cube_path)
if self.client == 'web' and config_file_parser.config_file_exist('web'):
if self.client == 'web' and config_file_parser.config_file_exist(
'web'):
for cubes in config_file_parser.construct_cubes(self.client):
# update facts name
......@@ -214,19 +218,19 @@ class MdxEngine:
fusion = None
config_file_parser = ConfigParser(self.cube_path)
if config_file_parser.config_file_exist(
self.client
) and self.cube in config_file_parser.get_cubes_names(
client_type=self.client):
self.
client) and self.cube in config_file_parser.get_cubes_names(
client_type=self.client):
for cubes in config_file_parser.construct_cubes(self.client):
# TODO cubes.source == 'csv'
if cubes.source == 'postgres':
# TODO one config file (I will try to merge dimensions between them in web part)
if self.client == 'web':
fusion = _construct_web_star_schema_config_file(
self, cubes)
fusion = _construct_web_star_schema_config_file(self,
cubes)
else:
fusion = _construct_star_schema_config_file(
self, cubes)
fusion = _construct_star_schema_config_file(self,
cubes)
elif self.cube in self.csv_files_cubes:
fusion = _construct_star_schema_csv_files(self)
......@@ -256,7 +260,8 @@ class MdxEngine:
:return: path to the cube
"""
if MdxEngine.DATA_FOLDER is not None:
return os.path.join(MdxEngine.DATA_FOLDER, MdxEngine.CUBE_FOLDER, self.cube)
return os.path.join(MdxEngine.DATA_FOLDER, MdxEngine.CUBE_FOLDER,
self.cube)
return os.path.join(self.cube_path, self.cube)
# TODO temporary function
......@@ -309,7 +314,7 @@ class MdxEngine:
for tup_att in tup[0].replace('.Members', '').split('.') if tup_att
]
for tup in re.compile(regex).findall(
query.encode("utf-8",'replace')[start:stop])
query.encode("utf-8", 'replace')[start:stop])
if len(tup[0].split('.')) > 1]
# TODO temporary function
......@@ -603,9 +608,8 @@ class MdxEngine:
:return: updated columns_to_keep
"""
if len(
tuple_as_list
) == 3 and tuple_as_list[-1] in self.tables_loaded[tuple_as_list[0]].columns:
if len(tuple_as_list) == 3 and tuple_as_list[-1] in self.tables_loaded[
tuple_as_list[0]].columns:
# in case of [Geography].[Geography].[Country]
cols = [tuple_as_list[-1]]
else:
......
olapy/core/mdx/executor/execute_config_file.py
View file @ cd49e537
......@@ -4,6 +4,7 @@ from ..tools.connection import MyDB
import pandas.io.sql as psql
import os
# split execution into three part (execute from config files,
# execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model)
......@@ -18,7 +19,9 @@ def _load_table_config_file(executer_instance, cube_obj):
# just one facts table right now
executer_instance.facts = cube_obj.facts[0].table_name
db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
db = MyDB(
db_config_file_path=os.path.dirname(executer_instance.cube_path),
db=executer_instance.cube)
for dimension in cube_obj.dimensions:
......@@ -35,7 +38,9 @@ def _load_table_config_file(executer_instance, cube_obj):
table_name = dimension.name
# rename columns if value not None
df.rename(columns=(dict((k, v) for k, v in dimension.columns.items() if v)), inplace=True)
df.rename(
columns=(dict((k, v) for k, v in dimension.columns.items() if v)),
inplace=True)
tables[table_name] = df[[
col for col in df.columns if col.lower()[-2:] != 'id'
......@@ -43,6 +48,7 @@ def _load_table_config_file(executer_instance, cube_obj):
return tables
# excel client
def _construct_star_schema_config_file(executer_instance, cubes_obj):
"""
......@@ -53,7 +59,9 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
:return: star schema DataFrame
"""
executer_instance.facts = cubes_obj.facts[0].table_name
db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
db = MyDB(
db_config_file_path=os.path.dirname(executer_instance.cube_path),
db=executer_instance.cube)
# load facts table
fusion = psql.read_sql_query(
......@@ -74,8 +82,11 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
# TODO CHOSE BETWEEN THOSES DF
fusion = fusion.merge(
df, left_on=fact_key, right_on=dimension_and_key.split('.')[1], how='left',
# remove suffixe from dimension and keep the same column name for facts
df,
left_on=fact_key,
right_on=dimension_and_key.split('.')[1],
how='left',
# remove suffixe from dimension and keep the same column name for facts
suffixes=('', '_y'))
# measures in config-file only
......@@ -84,6 +95,7 @@ def _construct_star_schema_config_file(executer_instance, cubes_obj):
return fusion
# web client
def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
"""
......@@ -96,7 +108,9 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
all_columns = []
executer_instance.facts = cubes_obj.facts[0].table_name
db = MyDB(db_config_file_path=os.path.dirname(executer_instance.cube_path), db=executer_instance.cube)
db = MyDB(
db_config_file_path=os.path.dirname(executer_instance.cube_path),
db=executer_instance.cube)
# load facts table
if cubes_obj.facts[0].columns:
......@@ -147,9 +161,11 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
# TODO check merge (how)
fusion = fusion.merge(
df, left_on=fact_key, right_on=dimension_and_key.split('.')[1], how='left',
df,
left_on=fact_key,
right_on=dimension_and_key.split('.')[1],
how='left',
# remove suffixe from dimension and keep the same column name for facts
suffixes=('', '_y'))
return fusion[[column for column in all_columns if 'id' != column[-2:]]]
olapy/core/mdx/executor/execute_csv_files.py
View file @ cd49e537
......@@ -8,6 +8,7 @@ import pandas as pd
# execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model)
def _load_tables_csv_files(executer_instance):
"""
Load tables from csv files.
......
olapy/core/mdx/executor/execute_db.py
View file @ cd49e537
......@@ -9,8 +9,6 @@ import pandas.io.sql as psql
# execute csv files if they respect olapy's start schema model,
# and execute data base tables if they respect olapy's start schema model)
# class StringFolder(object):
# """
# Class that will fold strings. See 'fold_string'.
......@@ -68,6 +66,7 @@ import pandas.io.sql as psql
# TODO try pandas.read_sql_table and pandas.read_sql
def _load_tables_db(executer_instance):
"""
Load tables from database.
......@@ -75,7 +74,9 @@ def _load_tables_db(executer_instance):
:return: tables dict with table name as key and dataframe as value
"""
tables = {}
db = MyDB(db_config_file_path=executer_instance.DATA_FOLDER,db=executer_instance.cube)
db = MyDB(
db_config_file_path=executer_instance.DATA_FOLDER,
db=executer_instance.cube)
inspector = inspect(db.engine)
for table_name in inspector.get_table_names():
......@@ -84,9 +85,12 @@ def _load_tables_db(executer_instance):
# 'SELECT * FROM "{0}"'.format(table_name), db.engine)
# results = db.engine.execute('SELECT * FROM "{0}"'.format(table_name))
results = db.engine.execution_options(stream_results=True).execute('SELECT * FROM "{0}"'.format(table_name))
results = db.engine.execution_options(stream_results=True).execute(
'SELECT * FROM "{0}"'.format(table_name))
# Fetch all the results of the query
value = pd.DataFrame(iter(results),columns=results.keys()) # Pass results as an iterator
value = pd.DataFrame(
iter(results),
columns=results.keys()) # Pass results as an iterator
# with string_folding_wrapper we loose response time
# value = pd.DataFrame(string_folding_wrapper(results),columns=results.keys())
tables[table_name] = value[[
......
olapy/core/mdx/tools/config_file_parser.py
View file @ cd49e537
......@@ -246,7 +246,7 @@ class ConfigParser:
"""
def __init__(self,
cube_path = None,
cube_path=None,
file_name='cubes-config.xml',
web_config_file_name='web_cube_config.xml'):
"""
......@@ -262,7 +262,8 @@ class ConfigParser:
home_directory = expanduser("~")
if cube_path is None:
self.cube_path = os.path.join(home_directory, 'olapy-data', 'cubes')
self.cube_path = os.path.join(home_directory, 'olapy-data',
'cubes')
else:
self.cube_path = cube_path
......@@ -303,7 +304,7 @@ class ConfigParser:
else:
return False
def get_cubes_names(self,client_type):
def get_cubes_names(self, client_type):
"""
Get all cubes names in the config file.
......@@ -359,10 +360,10 @@ class ConfigParser:
# column_new_name = [key.attrib['column_new_name'] for key in xml_dimension.findall('name')],
displayName=xml_dimension.find('displayName').text,
columns=OrderedDict(
(column_name.text , None if not column_name.attrib else column_name.attrib['column_new_name'])
(column_name.text, None if not column_name.attrib
else column_name.attrib['column_new_name'])
for column_name in xml_dimension.findall(
'columns/name')
))
'columns/name')))
for xml_dimension in tree.xpath(
'/cubes/cube/dimensions/dimension')
]
......@@ -449,8 +450,7 @@ class ConfigParser:
global_table={
'columns':
dashboard.find('Global_table/columns').text.split(','),
'rows':
dashboard.find('Global_table/rows').text.split(',')
'rows': dashboard.find('Global_table/rows').text.split(',')
},
pie_charts=dashboard.find('PieCharts').text.split(','),
bar_charts=dashboard.find('BarCharts').text.split(','),
......
olapy/core/mdx/tools/connection.py
View file @ cd49e537
from __future__ import absolute_import, division, print_function
# import psycopg2 as pg
from sqlalchemy import create_engine
# postgres connection
from olapy_config_file_parser import DbConfigParser
from .olapy_config_file_parser import DbConfigParser
class MyDB(object):
"""Connect to sql database (postgres only right now)."""
def __init__(self,db_config_file_path=None,db=None):
def __init__(self, db_config_file_path=None, db=None):
# TODO temporary
db_config = DbConfigParser(config_path=db_config_file_path)
......@@ -21,18 +23,19 @@ class MyDB(object):
# first i want to show all databases to user (in excel)
# self.engine = pg.connect("user={0} password={1} host='{2}'".
# format(username, password, host))
self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
username, password, 'postgres', host, port))
self.engine = create_engine(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
username, password, 'postgres', host, port))
else:
# and then we connect to the user db
self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
username, password, db, host, port))
self.engine = create_engine(
'postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
username, password, db, host, port))
# self.connection = pg.connect(
# "user={0} password={1} dbname='{2}' host='{3}'".format(
# username, password, db, host))
def __del__(self):
if hasattr(self, 'connection'):
self.engine.dispose()
olapy/core/mdx/tools/olapy_config_file_parser.py
View file @ cd49e537
......@@ -4,12 +4,11 @@ import os
from lxml import etree
class DbConfigParser:
# TODO one config file (I will try to merge dimensions between them in web part)
def __init__(self,
config_path = None,
file_name='olapy-config.xml'):
def __init__(self, config_path=None, file_name='olapy-config.xml'):
"""
:param cube_path: path to cube (csv folders)
......@@ -46,10 +45,9 @@ class DbConfigParser:
return [
{
# 'sgbd': db.find('sgbd').text,
'user_name': db.find('user_name').text,
'password': db.find('password').text,
'host': db.find('host').text,
'port': db.find('port').text,
}
for db in tree.xpath('/olapy/database')
'user_name': db.find('user_name').text,
'password': db.find('password').text,
'host': db.find('host').text,
'port': db.find('port').text,
} for db in tree.xpath('/olapy/database')
]
olapy/core/services/models.py
View file @ cd49e537
......@@ -2,12 +2,11 @@ from __future__ import absolute_import, division, print_function
from spyne import ComplexModel, Integer, Unicode, XmlAttribute
# NOTE : I didn't respect python naming convention here
# because we need to create the xmla response (generated by spyne) with the same variable names,
# thus xmla requests from excel can be reached
class Tuple(object):
"""Tuple description (used by spyne)."""
......@@ -110,7 +109,7 @@ class Propertielist(ComplexModel):
class Command(ComplexModel):
"""Command description (used by spyne)."""
_type_info = {'Statement': Unicode,}
_type_info = {'Statement': Unicode, }
class ExecuteRequest(ComplexModel):
......
olapy/core/services/xmla.py
View file @ cd49e537
......@@ -54,12 +54,11 @@ class XmlaProviderService(ServiceBase):
discover_tools = XmlaDiscoverTools()
sessio_id = discover_tools.session_id
@rpc(
DiscoverRequest,
_returns=AnyXml,
_body_style="bare",
_out_header=Session,
_throws=InvalidCredentialsError)
@rpc(DiscoverRequest,
_returns=AnyXml,
_body_style="bare",
_out_header=Session,
_throws=InvalidCredentialsError)
def Discover(ctx, request):
"""
The first principle function of xmla protocol.
......@@ -76,12 +75,11 @@ class XmlaProviderService(ServiceBase):
ctx.out_header = Session(SessionId=str(XmlaProviderService.sessio_id))
config_parser = ConfigParser(discover_tools.executer.cube_path)
if config_parser.xmla_authentication(
) and ctx.transport.req_env['QUERY_STRING'] != 'admin':
if config_parser.xmla_authentication() and ctx.transport.req_env[
'QUERY_STRING'] != 'admin':
raise InvalidCredentialsError(
fault_string=
'You do not have permission to access this resource',
fault_string='You do not have permission to access this resource',
fault_object=None)
# TODO call (labster) login function or create login with token (according to labster db)
......@@ -143,11 +141,10 @@ class XmlaProviderService(ServiceBase):
# Execute function must take 2 argument ( JUST 2 ! ) Command and Properties
# we encapsulate them in ExecuteRequest object
@rpc(
ExecuteRequest,
_returns=AnyXml,
_body_style="bare",
_out_header=Session)
@rpc(ExecuteRequest,
_returns=AnyXml,
_body_style="bare",
_out_header=Session)
def Execute(ctx, request):
"""
The second principle function of xmla protocol.
......@@ -235,7 +232,7 @@ application = Application(
wsgi_application = WsgiApplication(application)
def start_server(host='0.0.0.0',port=8000,write_on_file=False):
def start_server(host='0.0.0.0', port=8000, write_on_file=False):
"""
Start the xmla server.
......
olapy/core/services/xmla_discover_tools.py
View file @ cd49e537
......@@ -85,7 +85,7 @@ class XmlaDiscoverTools():
<Value>{5}</Value>
</row>
""".format(PropertyName, PropertyDescription, PropertyType,
PropertyAccessType, IsRequired, Value)
PropertyAccessType, IsRequired, Value)
else:
rows = """
<row>
......@@ -139,7 +139,7 @@ class XmlaDiscoverTools():
{1}
</root>
</return>
""".format(xsd,rows))
""".format(xsd, rows))
if request.Restrictions.RestrictionList.PropertyName == 'Catalog':
if request.Properties.PropertyList.Catalog is not None:
......@@ -195,7 +195,7 @@ class XmlaDiscoverTools():
'MdpropMdxNamedSets', 'int', 'Read', 'false',
'15')
return get_props(discover_preperties_xsd, '','', '', '', '','')
return get_props(discover_preperties_xsd, '', '', '', '', '', '')
def discover_schema_rowsets_response(self, request):
if request.Restrictions.RestrictionList.SchemaName == "MDSCHEMA_HIERARCHIES" and \
......@@ -1929,11 +1929,11 @@ class XmlaDiscoverTools():
# french caracteres
# TODO encode dataframe
if type(df.iloc[0][0]) == unicode:
column_attribut = df.iloc[0][0].encode('utf-8','replace')
column_attribut = df.iloc[0][0].encode('utf-8',
'replace')
else:
column_attribut = df.iloc[0][0]
rows += """
<row>
<CATALOG_NAME>{0}</CATALOG_NAME>
......@@ -1956,11 +1956,8 @@ class XmlaDiscoverTools():
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION>
</row>
""".format(self.selected_catalogue,
table_name,
df.columns[0],
column_attribut)
""".format(self.selected_catalogue, table_name,
df.columns[0], column_attribut)
rows += """
<row>
......@@ -2006,7 +2003,8 @@ class XmlaDiscoverTools():
# french caracteres
# TODO encode dataframe
if type(df.iloc[0][0]) == unicode:
column_attribut = df.iloc[0][0].encode('utf-8','replace')
column_attribut = df.iloc[0][0].encode('utf-8',
'replace')
else:
column_attribut = df.iloc[0][0]
......@@ -2032,10 +2030,8 @@ class XmlaDiscoverTools():
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION>
</row>
""".format(self.selected_catalogue,
table_name,
df.columns[0],
column_attribut)
""".format(self.selected_catalogue, table_name,
df.columns[0], column_attribut)
rows += """
<row>
......
olapy/core/services/xmla_execute_tools.py
View file @ cd49e537
......@@ -104,11 +104,11 @@ class XmlaExecuteTools():
first_att = 3
# query with on columns and on rows (without measure)
elif mdx_execution_result['columns_desc']['columns'] and mdx_execution_result['columns_desc']['rows']:
elif mdx_execution_result['columns_desc'][
'columns'] and mdx_execution_result['columns_desc']['rows']:
# ['Geography','America']
tuples = [
zip(
*[[[key] + list(row)
zip(*[[[key] + list(row)
for row in splited_df[key].itertuples(index=False)]
for key in splited_df.keys()
if key is not self.executer.facts])
......@@ -120,8 +120,7 @@ class XmlaExecuteTools():
else:
# ['Geography','Amount','America']
tuples = [
zip(
*[[[key] + [mes] + list(row)
zip(*[[[key] + [mes] + list(row)
for row in splited_df[key].itertuples(index=False)]
for key in splited_df.keys()
if key is not self.executer.facts])
......@@ -155,7 +154,10 @@ class XmlaExecuteTools():
# french caracteres
# TODO encode dataframe
if type(tuple_without_minus_1[-1]) == unicode:
tuple_without_minus_1 = [x.encode('utf-8','replace') for x in tuple_without_minus_1]
tuple_without_minus_1 = [
x.encode('utf-8', 'replace')
for x in tuple_without_minus_1
]
axis0 += """
<Member Hierarchy="[{0}].[{0}]">
......@@ -254,7 +256,8 @@ class XmlaExecuteTools():
# TODO must be OPTIMIZED every time!!!!!
dfs = self.split_dataframe(mdx_execution_result)
if mdx_execution_result['columns_desc']['rows'] and mdx_execution_result['columns_desc']['columns']:
if mdx_execution_result['columns_desc'][
'rows'] and mdx_execution_result['columns_desc']['columns']:
return """
{0}
......@@ -306,12 +309,11 @@ class XmlaExecuteTools():
"""
columns_loop = []
if (
(len(mdx_execution_result['columns_desc']['columns'].keys()) == 0)
if ((len(mdx_execution_result['columns_desc']['columns'].keys()) == 0)
^
(len(mdx_execution_result['columns_desc']['rows'].keys()) == 0)
) and self.executer.facts in mdx_execution_result['columns_desc']['all'].keys(
):
(len(mdx_execution_result['columns_desc']['rows'].keys()) == 0
)) and self.executer.facts in mdx_execution_result[
'columns_desc']['all'].keys():
# iterate DataFrame horizontally
columns_loop = itertools.chain(*[
......@@ -332,7 +334,7 @@ class XmlaExecuteTools():
cell_data = ""
index = 0
for value in columns_loop:
if np.isnan(value) :
if np.isnan(value):
value = ''
cell_data += """
<Cell CellOrdinal="{0}">
......@@ -392,9 +394,9 @@ class XmlaExecuteTools():
to_write = "[{0}].[{0}]".format(dim_diff)
if dim_diff == 'Measures':
# if measures > 1 we don't have to write measure
if self.executer.facts in mdx_execution_result['columns_desc']['all'] and len(
mdx_execution_result['columns_desc']['all']
[self.executer.facts]) > 1:
if self.executer.facts in mdx_execution_result['columns_desc'][
'all'] and len(mdx_execution_result['columns_desc'][
'all'][self.executer.facts]) > 1:
continue
else:
to_write = "[Measures]"
......@@ -452,9 +454,9 @@ class XmlaExecuteTools():
"""
hierarchy_info = ""
# measure must be written at the top
if self.executer.facts in mdx_execution_result['columns_desc'][mdx_query_axis].keys(
) and len(mdx_execution_result['columns_desc'][mdx_query_axis]
[self.executer.facts]) > 1:
if self.executer.facts in mdx_execution_result['columns_desc'][
mdx_query_axis].keys() and len(mdx_execution_result[
'columns_desc'][mdx_query_axis][self.executer.facts]) > 1:
hierarchy_info += """
<HierarchyInfo name="{0}">
<UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/>
......@@ -556,15 +558,18 @@ class XmlaExecuteTools():
for dim_diff in list(
set(self.executer.get_all_tables_names(ignore_fact=True)) -
set(table_name
for table_name in mdx_execution_result['columns_desc']
['all'])):
for table_name in mdx_execution_result['columns_desc'][
'all'])):
# TODO encode dataframe
# french caracteres
if type(self.executer.tables_loaded[dim_diff].iloc[0][0]) == unicode:
column_attribut = self.executer.tables_loaded[dim_diff].iloc[0][0].encode('utf-8','replace')
if type(self.executer.tables_loaded[dim_diff].iloc[0][
0]) == unicode:
column_attribut = self.executer.tables_loaded[dim_diff].iloc[
0][0].encode('utf-8', 'replace')
else:
column_attribut = self.executer.tables_loaded[dim_diff].iloc[0][0]
column_attribut = self.executer.tables_loaded[dim_diff].iloc[
0][0]
tuple += """
<Member Hierarchy="[{0}].[{0}]">
......
tests/test_execute_mdx.py
View file @ cd49e537
from __future__ import absolute_import, division, print_function
import pandas as pd
from olapy.core.mdx.executor.execute import MdxEngine
......
tests/test_xmla_notox.py
View file @ cd49e537
......@@ -198,8 +198,7 @@ def test_query2(conn):
mems.append(
Member(
_Hierarchy='[Geography].[Geography]',
UName=
'[Geography].[Geography].[Country].[America].[United States]',
UName='[Geography].[Geography].[Country].[America].[United States]',
Caption='United States',
LName='[Geography].[Geography].[Country]',
LNum='1',
......@@ -209,14 +208,12 @@ def test_query2(conn):
mems.append(
Member(
_Hierarchy='[Geography].[Geography]',
UName=
'[Geography].[Geography].[City].[America].[United States].[New York]',
UName='[Geography].[Geography].[City].[America].[United States].[New York]',
Caption='New York',
LName='[Geography].[Geography].[City]',
LNum='2',
DisplayInfo='131076',
PARENT_UNIQUE_NAME=
'[Geography].[Geography].[Continent].[America].[United States]',
PARENT_UNIQUE_NAME='[Geography].[Geography].[Continent].[America].[United States]',
HIERARCHY_UNIQUE_NAME='[Geography].[Geography]'))
mems.append(
Member(
......@@ -250,14 +247,12 @@ def test_query2(conn):
mems.append(
Member(
_Hierarchy='[Geography].[Geography]',
UName=
'[Geography].[Geography].[City].[Europe].[Spain].[Barcelona]',
UName='[Geography].[Geography].[City].[Europe].[Spain].[Barcelona]',
Caption='Barcelona',
LName='[Geography].[Geography].[City]',
LNum='2',
DisplayInfo='131076',
PARENT_UNIQUE_NAME=
'[Geography].[Geography].[Continent].[Europe].[Spain]',
PARENT_UNIQUE_NAME='[Geography].[Geography].[Continent].[Europe].[Spain]',
HIERARCHY_UNIQUE_NAME='[Geography].[Geography]'))
mems.append(
Member(
......@@ -267,8 +262,7 @@ def test_query2(conn):
LName='[Geography].[Geography].[City]',
LNum='2',
DisplayInfo='131076',
PARENT_UNIQUE_NAME=
'[Geography].[Geography].[Continent].[Europe].[Spain]',
PARENT_UNIQUE_NAME='[Geography].[Geography].[Continent].[Europe].[Spain]',
HIERARCHY_UNIQUE_NAME='[Geography].[Geography]'))
mems.append(
Member(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7