Skip to content

O
olapy
Commit 64486df0 authored by Stefane Fermigier's avatar Stefane Fermigier
Browse files
Options

"On rows".

parent 26bd503c
Hide whitespace changes
Inline Side-by-side
Showing with 439 additions and 255 deletions
olapy/core/mdx/executor/execute.py
View file @ 64486df0
......@@ -127,6 +127,23 @@ class MdxEngine:
"""
return os.path.join(self.cube_path, self.cube)
# TODO temporary function
def get_tuples(self, query, start=None, stop=None):
# TODO use grako instead and remove regex
regex = "(\[[\w\d ]+\](\.\[[\w\d\.\- ]+\])*\.?((Members)|(\[Q\d\]))?)"
if start is not None:
start = query.index(start)
if stop is not None:
stop = query.index(stop)
# clean the query (from All, Members...)
return [[
tup_att.replace('All ', '').replace('[', "").replace("]", "")
for tup_att in tup[0].replace('.Members', '').split('.')
] for tup in re.compile(regex).findall(query[start:stop])
if len(tup[0].split('.')) > 1]
# TODO temporary function
def decorticate_query(self, query):
"""
......@@ -135,16 +152,38 @@ class MdxEngine:
:return: all tuples in the query
"""
# TODO use grako instead and remove regex
regex = "(\[[\w\d ]+\](\.\[[\w\d\.\- ]+\])*\.?((Members)|(\[Q\d\]))?)"
# clean the query
tuples_on_mdx_query = [[
tup_att.replace('All ', '').replace('[', "").replace("]", "")
for tup_att in tup[0].replace('.Members', '').split('.')
] for tup in re.compile(regex).findall(query)
if len(tup[0].split('.')) > 1]
tuples_on_mdx_query = self.get_tuples(query)
on_rows = []
on_columns = []
on_where = []
return tuples_on_mdx_query
# ON ROWS
if 'ON ROWS' in query:
stop = 'ON ROWS'
if 'ON COLUMNS' in query:
start = 'ON COLUMNS'
else:
start = 'SELECT'
on_rows = self.get_tuples(query, start, stop)
# ON COLUMNS
if 'ON COLUMNS' in query:
start = 'SELECT'
stop = 'ON COLUMNS'
on_columns = self.get_tuples(query, start, stop)
# WHERE
if 'WHERE' in query:
start = 'FROM'
on_where = self.get_tuples(query, start)
return {
'all': tuples_on_mdx_query,
'columns': on_columns,
'rows': on_rows,
'where': on_where
}
def change_measures(self, tuples_on_mdx):
"""
......@@ -161,6 +200,7 @@ class MdxEngine:
]
def get_tables_and_columns(self, tuple_as_list):
# TODO update docstring
"""
get used dimensions and columns in the MDX Query (useful for DataFrame -> xmla response transformation)
......@@ -178,23 +218,31 @@ class MdxEngine:
Facts : ['Amount','Count']
}
"""
tables_columns = OrderedDict()
axes = {}
# TODO optimize
measures = []
for tupl in tuple_as_list:
for axis, tuples in tuple_as_list.items():
measures = []
tables_columns = OrderedDict()
# if we have measures in columns or rows axes like :
# SELECT {[Measures].[Amount],[Measures].[Count]} ON COLUMNS
# we have to add measures directly to tables_columns
if tupl[0].upper() == 'MEASURES':
measures.append(tupl[-1])
tables_columns.update({self.facts: measures})
else:
tables_columns.update({
tupl[0]:
self.tables_loaded[tupl[0]].columns[:len(tupl[2:])]
})
return tables_columns
for tupl in tuples:
if tupl[0].upper() == 'MEASURES':
if tupl[-1] not in measures:
measures.append(tupl[-1])
tables_columns.update({self.facts: measures})
else:
continue
else:
tables_columns.update({
tupl[0]:
self.tables_loaded[tupl[0]].columns[:len(tupl[2:])]
})
axes.update({axis: tables_columns})
return axes
def execute_one_tuple(self, tuple_as_list, Dataframe_in, columns_to_keep):
"""
......@@ -397,24 +445,21 @@ class MdxEngine:
"""
# use measures that exists on where or insides axes
all_tuples = self.decorticate_query(self.mdx_query)
query_axes = self.decorticate_query(self.mdx_query)
if self.change_measures(all_tuples):
self.measures = self.change_measures(all_tuples)
if self.change_measures(query_axes['all']):
self.measures = self.change_measures(query_axes['all'])
# get only used columns and dimensions for all query
start_df = self.load_star_schema_dataframe
tables_n_columns = self.get_tables_and_columns(all_tuples)
tables_n_columns = self.get_tables_and_columns(query_axes)
columns_to_keep = {
table: columns
for table, columns in tables_n_columns.items()
if table != self.facts
}
columns_to_keep = OrderedDict(
(table, columns) for table, columns in tables_n_columns['all'].items() if table != self.facts)
# if we have measures on axes we have to ignore them
tuples_on_mdx_query = [
tup for tup in all_tuples if tup[0].upper() != 'MEASURES'
tup for tup in query_axes['all'] if tup[0].upper() != 'MEASURES'
]
# if we have tuples in axes
# to avoid prob with query like this: SELECT FROM [Sales] WHERE ([Measures].[Amount])
......@@ -469,6 +514,7 @@ class MdxEngine:
df = pd.concat(self.add_missed_column(df, next_df))
# TODO groupby in web demo (remove it for more performance)
# TODO margins=True for columns total !!!!!
return {
'result':
df.drop_duplicates().replace(np.nan, -1).groupby(cols).sum(),
......
olapy/core/services/xmla.py
View file @ 64486df0
......@@ -81,22 +81,27 @@ class XmlaProviderService(ServiceBase):
return discover_tools.discover_mdschema_measures__response(request)
elif request.RequestType == "MDSCHEMA_DIMENSIONS":
return discover_tools.discover_mdschema_dimensions_response(request)
return discover_tools.discover_mdschema_dimensions_response(
request)
elif request.RequestType == "MDSCHEMA_HIERARCHIES":
return discover_tools.discover_mdschema_hierarchies_response(request)
return discover_tools.discover_mdschema_hierarchies_response(
request)
elif request.RequestType == "MDSCHEMA_LEVELS":
return discover_tools.discover_mdschema_levels__response(request)
elif request.RequestType == "MDSCHEMA_MEASUREGROUPS":
return discover_tools.discover_mdschema_measuresgroups_response(request)
return discover_tools.discover_mdschema_measuresgroups_response(
request)
elif request.RequestType == "MDSCHEMA_MEASUREGROUP_DIMENSIONS":
return discover_tools.discover_mdschema_measuresgroups_dimensions_response(request)
return discover_tools.discover_mdschema_measuresgroups_dimensions_response(
request)
elif request.RequestType == "MDSCHEMA_PROPERTIES":
return discover_tools.discover_mdschema_properties_response(request)
return discover_tools.discover_mdschema_properties_response(
request)
elif request.RequestType == "MDSCHEMA_MEMBERS":
return discover_tools.discover_mdschema_members_response(request)
......@@ -127,7 +132,8 @@ class XmlaProviderService(ServiceBase):
</return>
""")
else:
XmlaProviderService.discover_tools.change_catalogue(request.Properties.PropertyList.Catalog)
XmlaProviderService.discover_tools.change_catalogue(
request.Properties.PropertyList.Catalog)
executer = XmlaProviderService.discover_tools.executer
executer.mdx_query = request.Command.Statement
df = executer.execute_mdx()
......@@ -143,33 +149,34 @@ class XmlaProviderService(ServiceBase):
<Cube>
<CubeName>Sales</CubeName>
<LastDataUpdate
xmlns="http://schemas.microsoft.com/analysisservices/2003/engine">{6}</LastDataUpdate>
xmlns="http://schemas.microsoft.com/analysisservices/2003/engine">{7}</LastDataUpdate>
<LastSchemaUpdate
xmlns="http://schemas.microsoft.com/analysisservices/2003/engine">{6}</LastSchemaUpdate>
xmlns="http://schemas.microsoft.com/analysisservices/2003/engine">{7}</LastSchemaUpdate>
</Cube>
</CubeInfo>
<AxesInfo>
{1}
</AxesInfo>
{2}
</AxesInfo>
{3}
</OlapInfo>
<Axes>
{3}
{4}
{5}
</Axes>
<CellData>
{5}
{6}
</CellData>
</root>
</return>
""".format(execute_xsd,
xmla_tools.generate_axes_info(df),
xmla_tools.generate_axes_info_slicer(df),
xmla_tools.generate_cell_info(),
xmla_tools.generate_xs0(df),
xmla_tools.generate_slicer_axis(df),
xmla_tools.generate_cell_data(df),
datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
))
datetime.now().strftime('%Y-%m-%dT%H:%M:%S')))
application = Application(
......
olapy/core/services/xmla_discover_tools.py
View file @ 64486df0
......@@ -16,13 +16,16 @@ class XmlaDiscoverTools():
"""
XmlaDiscoverTools for generating xmla discover responses
"""
def __init__(self):
# right now the catalogue_name and cube name are the same
self.catalogues = MdxEngine.get_cubes_names()
self.selected_catalogue = self.catalogues[0]
self.executer = MdxEngine(self.selected_catalogue)
self.star_schema_dataframe = self.executer.load_star_schema_dataframe[
[col for col in self.executer.load_star_schema_dataframe.columns if col[-3:] != "_id"]]
self.star_schema_dataframe = self.executer.load_star_schema_dataframe[[
col for col in self.executer.load_star_schema_dataframe.columns
if col[-3:] != "_id"
]]
self.SessionId = uuid.uuid1()
def change_catalogue(self, new_catalogue):
......@@ -37,7 +40,11 @@ class XmlaDiscoverTools():
self.selected_catalogue = new_catalogue
self.executer = MdxEngine(new_catalogue)
self.star_schema_dataframe = self.executer.load_star_schema_dataframe[
[col for col in self.executer.load_star_schema_dataframe.columns if col[-3:] != "_id"]]
[
col
for col in self.executer.load_star_schema_dataframe.columns
if col[-3:] != "_id"
]]
def discover_datasources_response(self):
return etree.fromstring("""
......@@ -60,7 +67,8 @@ class XmlaDiscoverTools():
</return>""")
def discover_properties_response(self, request):
def get_props(xsd, PropertyName, PropertyDescription, PropertyType, PropertyAccessType, IsRequired, Value):
def get_props(xsd, PropertyName, PropertyDescription, PropertyType,
PropertyAccessType, IsRequired, Value):
return etree.fromstring("""
<return>
<root xmlns="urn:schemas-microsoft-com:xml-analysis:rowset"
......@@ -77,7 +85,8 @@ class XmlaDiscoverTools():
</row>
</root>
</return>
""".format(PropertyName, PropertyDescription, PropertyType, PropertyAccessType, IsRequired, Value))
""".format(PropertyName, PropertyDescription, PropertyType,
PropertyAccessType, IsRequired, Value))
if request.Restrictions.RestrictionList.PropertyName == 'Catalog':
if request.Properties.PropertyList.Catalog is not None:
......@@ -85,48 +94,53 @@ class XmlaDiscoverTools():
value = self.selected_catalogue
else:
value = "olapy Unspecified Catalog"
return get_props(discover_preperties_xsd, 'Catalog', 'Catalog', 'string', 'ReadWrite', 'false', value)
return get_props(discover_preperties_xsd, 'Catalog', 'Catalog',
'string', 'ReadWrite', 'false', value)
elif request.Restrictions.RestrictionList.PropertyName == 'ServerName':
return get_props(discover_preperties_xsd, 'ServerName', 'ServerName', 'string', 'Read', 'false', 'Mouadh')
return get_props(discover_preperties_xsd, 'ServerName',
'ServerName', 'string', 'Read', 'false', 'Mouadh')
elif request.Restrictions.RestrictionList.PropertyName == 'ProviderVersion':
return get_props(discover_preperties_xsd, 'ProviderVersion', 'ProviderVersion', 'string', 'Read', 'false',
return get_props(discover_preperties_xsd, 'ProviderVersion',
'ProviderVersion', 'string', 'Read', 'false',
'0.02 08-Mar-2016 08:41:28 GMT')
elif request.Restrictions.RestrictionList.PropertyName == 'MdpropMdxSubqueries':
if 'Unspecified' in request.Properties.PropertyList.Catalog:
return get_props(discover_preperties_xsd, 'MdpropMdxSubqueries', 'MdpropMdxSubqueries', 'int', 'Read',
'false', '15')
return get_props(discover_preperties_xsd,
'MdpropMdxSubqueries', 'MdpropMdxSubqueries',
'int', 'Read', 'false', '15')
if request.Properties.PropertyList.Catalog is not None:
self.change_catalogue(request.Properties.PropertyList.Catalog)
return get_props(discover_preperties_xsd, 'MdpropMdxSubqueries', 'MdpropMdxSubqueries', 'int', 'Read',
'false', '15')
return get_props(discover_preperties_xsd,
'MdpropMdxSubqueries', 'MdpropMdxSubqueries',
'int', 'Read', 'false', '15')
elif request.Restrictions.RestrictionList.PropertyName == 'MdpropMdxDrillFunctions':
if 'Unspecified' in request.Properties.PropertyList.Catalog:
return get_props(discover_preperties_xsd, 'MdpropMdxDrillFunctions', 'MdpropMdxDrillFunctions', 'int',
'Read',
'false', '3')
return get_props(
discover_preperties_xsd, 'MdpropMdxDrillFunctions',
'MdpropMdxDrillFunctions', 'int', 'Read', 'false', '3')
if request.Properties.PropertyList.Catalog is not None:
self.change_catalogue(request.Properties.PropertyList.Catalog)
return get_props(discover_preperties_xsd, 'MdpropMdxDrillFunctions', 'MdpropMdxDrillFunctions', 'int',
'Read',
'false', '3')
return get_props(
discover_preperties_xsd, 'MdpropMdxDrillFunctions',
'MdpropMdxDrillFunctions', 'int', 'Read', 'false', '3')
elif request.Restrictions.RestrictionList.PropertyName == 'MdpropMdxNamedSets':
if 'Unspecified' in request.Properties.PropertyList.Catalog:
return get_props(discover_preperties_xsd, 'MdpropMdxNamedSets', 'MdpropMdxNamedSets', 'int',
'Read',
'false', '15')
return get_props(discover_preperties_xsd, 'MdpropMdxNamedSets',
'MdpropMdxNamedSets', 'int', 'Read', 'false',
'15')
if request.Properties.PropertyList.Catalog is not None:
self.change_catalogue(request.Properties.PropertyList.Catalog)
return get_props(discover_preperties_xsd, 'MdpropMdxNamedSets', 'MdpropMdxNamedSets', 'int',
'Read',
'false', '15')
return get_props(discover_preperties_xsd, 'MdpropMdxNamedSets',
'MdpropMdxNamedSets', 'int', 'Read', 'false',
'15')
return etree.fromstring("""
<return>
......@@ -1693,7 +1707,8 @@ class XmlaDiscoverTools():
</row>
</root>
</return>
""".format(self.selected_catalogue, self.selected_catalogue))
""".format(self.selected_catalogue,
self.selected_catalogue))
return etree.fromstring("""
<return>
<root xmlns="urn:schemas-microsoft-com:xml-analysis:rowset"
......@@ -1761,7 +1776,8 @@ class XmlaDiscoverTools():
if request.Restrictions.RestrictionList.CUBE_NAME == self.selected_catalogue:
if request.Restrictions.RestrictionList.MEASURE_VISIBILITY == 3:
if request.Properties.PropertyList.Catalog is not None:
self.change_catalogue(request.Properties.PropertyList.Catalog)
self.change_catalogue(
request.Properties.PropertyList.Catalog)
for mes in self.executer.measures:
measures += """
<row>
......@@ -1796,10 +1812,12 @@ class XmlaDiscoverTools():
if request.Restrictions.RestrictionList.CUBE_NAME == self.selected_catalogue:
if request.Restrictions.RestrictionList.CATALOG_NAME == self.selected_catalogue:
if request.Properties.PropertyList.Catalog is not None:
self.change_catalogue(request.Properties.PropertyList.Catalog)
self.change_catalogue(
request.Properties.PropertyList.Catalog)
rows = ""
ord = 1
for tables in self.executer.get_all_tables_names(ignore_fact=True):
for tables in self.executer.get_all_tables_names(
ignore_fact=True):
rows += """
<row>
<CATALOG_NAME>{0}</CATALOG_NAME>
......@@ -1815,9 +1833,7 @@ class XmlaDiscoverTools():
<IS_READWRITE>false</IS_READWRITE>
<DIMENSION_UNIQUE_SETTINGS>1</DIMENSION_UNIQUE_SETTINGS>
<DIMENSION_IS_VISIBLE>true</DIMENSION_IS_VISIBLE>
</row>""".format(self.selected_catalogue,
tables,
ord)
</row>""".format(self.selected_catalogue, tables, ord)
ord += 1
rows += """
......@@ -1880,9 +1896,8 @@ class XmlaDiscoverTools():
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION>
</row>
""".format(
self.selected_catalogue, table_name, df.columns[0],
df.iloc[0][0])
""".format(self.selected_catalogue, table_name,
df.columns[0], df.iloc[0][0])
# self.executer.get_attribute_column_rm_id(tables, column, 0))
rows += """
......@@ -1907,7 +1922,8 @@ class XmlaDiscoverTools():
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION>
</row>
""".format(self.selected_catalogue, self.executer.measures[0])
""".format(self.selected_catalogue,
self.executer.measures[0])
return etree.fromstring("""
<return>
......@@ -1946,10 +1962,8 @@ class XmlaDiscoverTools():
<HIERARCHY_ORIGIN>1</HIERARCHY_ORIGIN>
<INSTANCE_SELECTION>0</INSTANCE_SELECTION>
</row>
""".format(self.selected_catalogue,
table_name,
df.columns[0],
df.iloc[0][0])
""".format(self.selected_catalogue, table_name,
df.columns[0], df.iloc[0][0])
rows += """
<row>
......@@ -1989,9 +2003,11 @@ class XmlaDiscoverTools():
if request.Restrictions.RestrictionList.CUBE_NAME == self.selected_catalogue:
if request.Restrictions.RestrictionList.CATALOG_NAME == self.selected_catalogue:
if request.Properties.PropertyList.Catalog is not None:
self.change_catalogue(request.Properties.PropertyList.Catalog)
self.change_catalogue(
request.Properties.PropertyList.Catalog)
rows = ""
for tables in self.executer.get_all_tables_names(ignore_fact=True):
for tables in self.executer.get_all_tables_names(
ignore_fact=True):
l_nb = 0
for col in self.executer.tables_loaded[tables].columns:
rows += """
......@@ -2013,7 +2029,8 @@ class XmlaDiscoverTools():
<LEVEL_KEY_CARDINALITY>1</LEVEL_KEY_CARDINALITY>
<LEVEL_ORIGIN>2</LEVEL_ORIGIN>
</row>
""".format(self.selected_catalogue, tables, col, l_nb)
""".format(self.selected_catalogue, tables, col,
l_nb)
l_nb += 1
rows += """
......@@ -2074,7 +2091,8 @@ class XmlaDiscoverTools():
if request.Properties.PropertyList.Catalog is not None:
self.change_catalogue(request.Properties.PropertyList.Catalog)
rows = ""
for tables in self.executer.get_all_tables_names(ignore_fact=True):
for tables in self.executer.get_all_tables_names(
ignore_fact=True):
rows += """
<row>
<CATALOG_NAME>{0}</CATALOG_NAME>
......@@ -2210,7 +2228,8 @@ class XmlaDiscoverTools():
</row>
</root>
</return>
""".format(self.selected_catalogue, mdschema_properties_PROPERTIES_xsd))
""".format(self.selected_catalogue,
mdschema_properties_PROPERTIES_xsd))
elif request.Restrictions.RestrictionList.PROPERTY_TYPE == 1:
return etree.fromstring("""
<return>
......@@ -2228,7 +2247,8 @@ class XmlaDiscoverTools():
if request.Properties.PropertyList.Catalog is not None:
self.change_catalogue(request.Properties.PropertyList.Catalog)
if request.Restrictions.RestrictionList.TREE_OP == 8:
separed_tuple = request.Restrictions.RestrictionList.MEMBER_UNIQUE_NAME.split(".")
separed_tuple = request.Restrictions.RestrictionList.MEMBER_UNIQUE_NAME.split(
".")
joined = ".".join(separed_tuple[:-1])
# exple
# separed_tuple -> [Product].[Product].[Company].[Crazy Development]
......@@ -2262,5 +2282,7 @@ class XmlaDiscoverTools():
</root>
</return>
""".format(self.selected_catalogue, separed_tuple[
0], joined, request.Restrictions.RestrictionList.MEMBER_UNIQUE_NAME, ''.join(
c for c in separed_tuple[-1] if c not in '[]')))
0], joined, request.Restrictions.RestrictionList.
MEMBER_UNIQUE_NAME, ''.join(
c for c in separed_tuple[-1]
if c not in '[]')))
olapy/core/services/xmla_execute_tools.py
View file @ 64486df0
......@@ -2,6 +2,8 @@ from __future__ import absolute_import, division, print_function
from collections import OrderedDict
import itertools
class XmlaExecuteTools():
"""
......@@ -51,14 +53,14 @@ class XmlaExecuteTools():
"""
# TODO new version with facts as splited df maybe
DataFrames = OrderedDict()
[DataFrames.update({key: mdx_execution_result['result'].reset_index()[list(value)]}) for key, value in
mdx_execution_result['columns_desc'].items() if key != self.executer.facts]
return DataFrames
return OrderedDict(
(key, mdx_execution_result['result'].reset_index()[list(value)])
for key, value in mdx_execution_result['columns_desc']['all']
.items())
def get_tuple_without_nan(self, tuple):
"""
remove nan from tuple
remove nan from tuple.
example
in :
......@@ -73,53 +75,127 @@ class XmlaExecuteTools():
:return: tuple as list without -1
"""
for index, att in enumerate(tuple[::-1]):
if att != -1:
return tuple[:len(tuple) - index]
return tuple
def check_measures_only_selected(self, mdx_execution_result):
"""
check if mdx query contains only measures
:param mdx_execution_result: mdx_execute() result
:return: True | False
"""
return len(mdx_execution_result['columns_desc'].keys()) == 1 and mdx_execution_result['columns_desc'].keys()[
0] == self.executer.facts
return tuple
def generate_xs0_measures_only(self, mdx_execution_result):
def generate_xs0_one_axis(self,
mdx_execution_result,
splited_df,
mdx_query_axis='all',
axis="Axis0"):
"""
generate xs0 if only measures exists in the mdx query
:param mdx_execution_result: mdx_execute() result
:return: xs0 xml as string
:param mdx_execution_result:
:param splited_df:
:return:
"""
axis0 = ""
if len(mdx_execution_result['columns_desc'][self.executer.facts]) > 1:
for column in mdx_execution_result['result'].columns:
# only measure selected
if mdx_execution_result['columns_desc'][
mdx_query_axis].keys() == [self.executer.facts]:
if len(mdx_execution_result['columns_desc'][mdx_query_axis][
self.executer.facts]) == 1:
# to ignore for tupls in itertools.chain(*tuples)
tuples = []
else:
# ['Facts', 'Amount', 'Amount']
tuples = [[[[self.executer.facts] + [mes] + [mes]]]
for mes in self.executer.measures]
first_att = 3
# query with on columns and on rows (without measure)
elif mdx_execution_result['columns_desc'][
'columns'] and mdx_execution_result['columns_desc']['rows']:
# ['Geography','America']
tuples = [
zip(* [[[key] + list(row)
for row in splited_df[key].itertuples(index=False)]
for key in splited_df.keys()
if key is not self.executer.facts])
]
first_att = 2
# query with on columns and on rows (many measures selected)
else:
# ['Geography','Amount','America']
tuples = [
zip(* [[[key] + [mes] + list(row)
for row in splited_df[key].itertuples(index=False)]
for key in splited_df.keys()
if key is not self.executer.facts])
for mes in self.executer.measures
]
first_att = 3
for tupls in itertools.chain(*tuples):
axis0 += "<Tuple>\n"
# [u'Geography', u'Amount', 'America']
# tupls[0][1] --> Measure
if tupls[0][1] in self.executer.measures and len(
self.executer.measures) > 1:
axis0 += """
<Member Hierarchy="[Measures]">
<UName>[Measures].[{0}]</UName>
<Caption>{0}</Caption>
<LName>[Measures]</LName>
<LNum>0</LNum>
<DisplayInfo>0</DisplayInfo>
<HIERARCHY_UNIQUE_NAME>[Measures]</HIERARCHY_UNIQUE_NAME>
</Member>
""".format(tupls[0][1])
if len(tupls) == 1:
axis0 += "</Tuple>\n"
continue
for tupl in tupls:
tuple_without_minus_1 = self.get_tuple_without_nan(tupl)
axis0 += """
<Member Hierarchy="[{0}].[{0}]">
<UName>[{0}].[{0}].[{1}].{2}</UName>
<Caption>{3}</Caption>
<LName>[{0}].[{0}].[{1}]</LName>
<LNum>{4}</LNum>
<DisplayInfo>131076</DisplayInfo>""".format(
tuple_without_minus_1[0], splited_df[tuple_without_minus_1[
0]].columns[len(tuple_without_minus_1) - first_att],
'.'.join([
'[' + str(i) + ']'
for i in tuple_without_minus_1[first_att - 1:]
]), tuple_without_minus_1[-1],
len(tuple_without_minus_1) - first_att)
# PARENT_UNIQUE_NAME must be before HIERARCHY_UNIQUE_NAME
if len(tuple_without_minus_1[first_att - 1:]) > 1:
axis0 += """
<PARENT_UNIQUE_NAME>[{0}].[{0}].[{1}].{2}</PARENT_UNIQUE_NAME>""".format(
tuple_without_minus_1[0],
splited_df[tuple_without_minus_1[0]].columns[0],
'.'.join([
'[' + str(i) + ']'
for i in tuple_without_minus_1[first_att - 1:-1]
]))
axis0 += """
<Tuple>
<Member Hierarchy="[Measures]">
<UName>[Measures].[{0}]</UName>
<Caption>{0}</Caption>
<LName>[Measures]</LName>
<LNum>0</LNum>
<DisplayInfo>0</DisplayInfo>
<HIERARCHY_UNIQUE_NAME>[Measures]</HIERARCHY_UNIQUE_NAME>
</Member>
</Tuple>
""".format(column)
<HIERARCHY_UNIQUE_NAME>[{0}].[{0}]</HIERARCHY_UNIQUE_NAME>
</Member>
""".format(tuple_without_minus_1[0])
axis0 += "</Tuple>\n"
if axis0:
axis0 = """
<Axis name="Axis0">
<Axis name="{0}">
<Tuples>
{0}
{1}
</Tuples>
</Axis>
""".format(axis0)
""".format(axis, axis0)
return axis0
......@@ -175,79 +251,49 @@ class XmlaExecuteTools():
:param mdx_execution_result: mdx_execute() result
:return: xs0 xml as string
"""
# TODO must be OPTIMIZED every time !!!!!
# only measures selected
if self.check_measures_only_selected(mdx_execution_result):
return self.generate_xs0_measures_only(mdx_execution_result)
# TODO must be OPTIMIZED every time!!!!!
dfs = self.split_DataFrame(mdx_execution_result)
axis0 = ""
keys_without_fact = [key for key in mdx_execution_result['columns_desc'].keys() if
key != self.executer.facts]
# every selected measure
for mes in self.executer.measures:
for row in zip(*([list(row) for row in dfs[key].itertuples(index=False)] for key in dfs.keys())):
axis0 += "<Tuple>\n"
if len(self.executer.measures) > 1:
axis0 += """
<Member Hierarchy="[Measures]">
<UName>[Measures].[{0}]</UName>
<Caption>{0}</Caption>
<LName>[Measures]</LName>
<LNum>0</LNum>
<DisplayInfo>0</DisplayInfo>
<HIERARCHY_UNIQUE_NAME>[Measures]</HIERARCHY_UNIQUE_NAME>
</Member>
""".format(mes)
for index, tupl in enumerate(row):
tuple_without_minus_1 = self.get_tuple_without_nan(tupl)
# tuple without parent
axis0 += """
<Member Hierarchy="[{0}].[{0}]">
<UName>[{0}].[{0}].[{1}].{2}</UName>
<Caption>{3}</Caption>
<LName>[{0}].[{0}].[{1}]</LName>
<LNum>{4}</LNum>
<DisplayInfo>131076</DisplayInfo>""".format(
keys_without_fact[index],
dfs[keys_without_fact[index]].columns[len(tuple_without_minus_1) - 1],
'.'.join(['[' + str(i) + ']' for i in tuple_without_minus_1]),
tuple_without_minus_1[-1],
len(tuple_without_minus_1) - 1
)
# PARENT_UNIQUE_NAME must be before HIERARCHY_UNIQUE_NAME
if len(tuple_without_minus_1) > 1:
axis0 += """
<PARENT_UNIQUE_NAME>[{0}].[{0}].[{1}].{2}</PARENT_UNIQUE_NAME>""".format(
keys_without_fact[index],
dfs[keys_without_fact[index]].columns[0],
'.'.join(['[' + str(i) + ']' for i in tuple_without_minus_1[:-1]])
)
axis0 += """
<HIERARCHY_UNIQUE_NAME>[{0}].[{0}]</HIERARCHY_UNIQUE_NAME>
</Member>
""".format(keys_without_fact[index])
axis0 += "</Tuple>\n"
if axis0:
axis0 = """
<Axis name="Axis0">
<Tuples>
{0}
</Tuples>
</Axis>
""".format(axis0)
return axis0
if mdx_execution_result['columns_desc'][
'rows'] and mdx_execution_result['columns_desc']['columns']:
return """
{0}
{1}
""".format(
self.generate_xs0_one_axis(
mdx_execution_result,
dfs,
mdx_query_axis='columns',
axis="Axis0"),
self.generate_xs0_one_axis(
mdx_execution_result,
dfs,
mdx_query_axis='rows',
axis="Axis1"))
# only one measure selected
elif not mdx_execution_result['columns_desc'][
'rows'] and not mdx_execution_result['columns_desc']['columns'] and\
mdx_execution_result['columns_desc']['where']:
return """
{0}
""".format(
self.generate_xs0_one_axis(
mdx_execution_result,
dfs,
mdx_query_axis='where',
axis="Axis0"))
# one axis
return self.generate_xs0_one_axis(
mdx_execution_result, dfs, mdx_query_axis='columns', axis="Axis0")
# TODO maybe fusion with generate xs0 for less iteration
def generate_cell_data(self, mdx_execution_result):
"""
examle of CellData::
<Cell CellOrdinal="0">
<Value xsi:type="xsi:long">768</Value>
</Cell>
......@@ -271,7 +317,81 @@ class XmlaExecuteTools():
index += 1
return cell_data
def generate_axes_info(self, mdx_execution_result):
def generate_axes_info_slicer(self, mdx_execution_result):
"""
Not used dimensions
example AxisInfo::
<AxesInfo>
<AxisInfo name="SlicerAxis">
<HierarchyInfo name="[Time].[Time]">
<UName name="[Time].[Time].[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<Caption name="[Time].[Time].[MEMBER_CAPTION]" type="xs:string"/>
<LName name="[Time].[Time].[LEVEL_UNIQUE_NAME]" type="xs:string"/>
<LNum name="[Time].[Time].[LEVEL_NUMBER]" type="xs:int"/>
<DisplayInfo name="[Time].[Time].[DISPLAY_INFO]" type="xs:unsignedInt"/>
</HierarchyInfo>
<HierarchyInfo name="[Measures]">
<UName name="[Measures].[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<Caption name="[Measures].[MEMBER_CAPTION]" type="xs:string"/>
<LName name="[Measures].[LEVEL_UNIQUE_NAME]" type="xs:string"/>
<LNum name="[Measures].[LEVEL_NUMBER]" type="xs:int"/>
<DisplayInfo name="[Measures].[DISPLAY_INFO]" type="xs:unsignedInt"/>
</HierarchyInfo>
</AxisInfo>
</AxesInfo>
:param mdx_execution_result: mdx_execute() result
:return: AxisInfo as string
"""
all_dimensions_names = self.executer.get_all_tables_names(
ignore_fact=True)
all_dimensions_names.append('Measures')
hierarchy_info_slicer = ""
slicer_list = list(
set(all_dimensions_names) - set([
table_name
for table_name in mdx_execution_result['columns_desc']['all']
]))
# we have to write measures after dimensions !
if 'Measures' in slicer_list:
slicer_list.insert(
len(slicer_list),
slicer_list.pop(slicer_list.index('Measures')))
for dim_diff in slicer_list:
to_write = "[{0}].[{0}]".format(dim_diff)
if dim_diff == 'Measures':
# if measures > 1 we don't have to write measure
if len(self.executer.measures) > 1:
continue
else:
to_write = "[Measures]"
hierarchy_info_slicer += """
<HierarchyInfo name="{0}">
<UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<Caption name="{0}.[MEMBER_CAPTION]" type="xs:string"/>
<LName name="{0}.[LEVEL_UNIQUE_NAME]" type="xs:string"/>
<LNum name="{0}.[LEVEL_NUMBER]" type="xs:int"/>
<DisplayInfo name="{0}.[DISPLAY_INFO]" type="xs:unsignedInt"/>
</HierarchyInfo>
""".format(to_write)
if hierarchy_info_slicer:
hierarchy_info_slicer = "<AxisInfo name='SlicerAxis'>\n" + hierarchy_info_slicer + "\n</AxisInfo>\n"
return hierarchy_info_slicer
def generate_one_axis_info(self,
mdx_execution_result,
mdx_query_axis='columns',
Axis='Axis0'):
"""
example AxisInfo::
......@@ -298,81 +418,67 @@ class XmlaExecuteTools():
<HIERARCHY_UNIQUE_NAME name="[Product].[Product].[HIERARCHY_UNIQUE_NAME]" type="xs:string"/>
</HierarchyInfo>
</AxisInfo>
<AxisInfo name="SlicerAxis">
<HierarchyInfo name="[Time].[Time]">
<UName name="[Time].[Time].[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<Caption name="[Time].[Time].[MEMBER_CAPTION]" type="xs:string"/>
<LName name="[Time].[Time].[LEVEL_UNIQUE_NAME]" type="xs:string"/>
<LNum name="[Time].[Time].[LEVEL_NUMBER]" type="xs:int"/>
<DisplayInfo name="[Time].[Time].[DISPLAY_INFO]" type="xs:unsignedInt"/>
</HierarchyInfo>
<HierarchyInfo name="[Measures]">
<UName name="[Measures].[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<Caption name="[Measures].[MEMBER_CAPTION]" type="xs:string"/>
<LName name="[Measures].[LEVEL_UNIQUE_NAME]" type="xs:string"/>
<LNum name="[Measures].[LEVEL_NUMBER]" type="xs:int"/>
<DisplayInfo name="[Measures].[DISPLAY_INFO]" type="xs:unsignedInt"/>
</HierarchyInfo>
</AxisInfo>
</AxesInfo>
:param mdx_execution_result: mdx_execute() result
:return: AxisInfo as string
:param mdx_execution_result:
:param mdx_query_axis: columns or rows (columns by default)
:param Axis: Axis0 or Axis1 (Axis0 by default)
:return:
"""
# TODO reduce complexity
all_dimensions_names = self.executer.get_all_tables_names(ignore_fact=True)
all_dimensions_names = self.executer.get_all_tables_names(
ignore_fact=True)
hierarchy_info = ""
all_dimensions_names.append('Measures')
for table_name in mdx_execution_result['columns_desc']:
for table_name in mdx_execution_result['columns_desc'][mdx_query_axis]:
to_write = "[{0}].[{0}]".format(table_name)
# measures must be added to axis0 if measures selected > 1
if table_name == self.executer.facts and len(mdx_execution_result['columns_desc'][table_name]) > 1:
if table_name == self.executer.facts and len(mdx_execution_result[
'columns_desc'][mdx_query_axis][table_name]) > 1:
to_write = "[Measures]"
all_dimensions_names.remove('Measures')
elif table_name == self.executer.facts:
continue
hierarchy_info += """
<HierarchyInfo name="{0}">
<UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<Caption name="{0}.[MEMBER_CAPTION]" type="xs:string"/>
<LName name="{0}.[LEVEL_UNIQUE_NAME]" type="xs:string"/>
<LNum name="{0}.[LEVEL_NUMBER]" type="xs:int"/>
<DisplayInfo name="{0}.[DISPLAY_INFO]" type="xs:unsignedInt"/>
<PARENT_UNIQUE_NAME name="{0}.[PARENT_UNIQUE_NAME]" type="xs:string"/>
<HIERARCHY_UNIQUE_NAME name="{0}.[HIERARCHY_UNIQUE_NAME]" type="xs:string"/>
</HierarchyInfo>
<HierarchyInfo name="{0}">
<UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<Caption name="{0}.[MEMBER_CAPTION]" type="xs:string"/>
<LName name="{0}.[LEVEL_UNIQUE_NAME]" type="xs:string"/>
<LNum name="{0}.[LEVEL_NUMBER]" type="xs:int"/>
<DisplayInfo name="{0}.[DISPLAY_INFO]" type="xs:unsignedInt"/>
<PARENT_UNIQUE_NAME name="{0}.[PARENT_UNIQUE_NAME]" type="xs:string"/>
<HIERARCHY_UNIQUE_NAME name="{0}.[HIERARCHY_UNIQUE_NAME]" type="xs:string"/>
</HierarchyInfo>
""".format(to_write)
if hierarchy_info:
hierarchy_info = "<AxisInfo name='Axis0'>\n" + hierarchy_info + "\n</AxisInfo>\n"
hierarchy_info = """
<AxisInfo name='{0}'>
{1}
</AxisInfo>
""".format(Axis, hierarchy_info)
hierarchy_info_slicer = ""
slicer_list = list(set(all_dimensions_names) - set(
[table_name for table_name in mdx_execution_result['columns_desc']]))
return hierarchy_info
# we have to write measures after dimensions !
if 'Measures' in slicer_list:
slicer_list.insert(len(slicer_list), slicer_list.pop(slicer_list.index('Measures')))
for dim_diff in slicer_list:
to_write = "[{0}].[{0}]".format(dim_diff)
if dim_diff == 'Measures':
to_write = "[Measures]"
hierarchy_info_slicer += """
<HierarchyInfo name="{0}">
<UName name="{0}.[MEMBER_UNIQUE_NAME]" type="xs:string"/>
<Caption name="{0}.[MEMBER_CAPTION]" type="xs:string"/>
<LName name="{0}.[LEVEL_UNIQUE_NAME]" type="xs:string"/>
<LNum name="{0}.[LEVEL_NUMBER]" type="xs:int"/>
<DisplayInfo name="{0}.[DISPLAY_INFO]" type="xs:unsignedInt"/>
</HierarchyInfo>
""".format(to_write)
def generate_axes_info(self, mdx_execution_result):
"""
:param mdx_execution_result: mdx_execute() result
:return: AxisInfo as string
"""
if hierarchy_info_slicer:
hierarchy_info_slicer = "<AxisInfo name='SlicerAxis'>\n" + hierarchy_info_slicer + "\n</AxisInfo>\n"
if mdx_execution_result['columns_desc']['rows']:
return """
{0}
{1}
""".format(
self.generate_one_axis_info(
mdx_execution_result,
mdx_query_axis='columns',
Axis='Axis0'),
self.generate_one_axis_info(
mdx_execution_result, mdx_query_axis='rows', Axis='Axis1'))
return hierarchy_info + '\n' + hierarchy_info_slicer
return self.generate_one_axis_info(mdx_execution_result)
def generate_cell_info(self):
return """
......@@ -416,8 +522,13 @@ class XmlaExecuteTools():
"""
tuple = ""
# not used dimensions
for dim_diff in list(set(self.executer.get_all_tables_names(ignore_fact=True)) - set(
[table_name for table_name in mdx_execution_result['columns_desc']])):
for dim_diff in list(
set(self.executer.get_all_tables_names(ignore_fact=True)) -
set([
table_name
for table_name in mdx_execution_result['columns_desc'][
'all']
])):
tuple += """
<Member Hierarchy="[{0}].[{0}]">
<UName>[{0}].[{0}].[{1}].[{2}]</UName>
......
tests/test_xmla_notox.py
View file @ 64486df0
......@@ -266,8 +266,6 @@ def test_query2(conn):
DisplayInfo='131076',
PARENT_UNIQUE_NAME='[Geography].[Geography].[Continent].[Europe]',
HIERARCHY_UNIQUE_NAME='[Geography].[Geography]'))
print([Member(**dict(co)) for co in columns])
print(mems)
assert [Member(**dict(co)) for co in columns] == mems
assert values == [768, 768, 768, 255, 4, 3, 2, 1, 248]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7