Commit 71fe3e64 authored by mouadh's avatar mouadh

change to sqlalchemy

parent 99d84a3b
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!-- this config file will be deleted ASAP -->
<olapy> <olapy>
<database> <database>
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
<user_name>postgres</user_name> <user_name>postgres</user_name>
<password>root</password> <password>root</password>
<host>localhost</host> <host>localhost</host>
<port>5432</port>
</database> </database>
......
...@@ -95,12 +95,15 @@ class MdxEngine: ...@@ -95,12 +95,15 @@ class MdxEngine:
# get postgres databases # get postgres databases
try: try:
db = MyDB() db = MyDB()
cursor = db.connection.cursor() connection = db.engine
cursor.execute("""SELECT datname FROM pg_database # TODO this work only with postgres
WHERE datistemplate = false;""") result = connection.execute('SELECT datname FROM pg_database WHERE datistemplate = false;')
available_tables = result.fetchall()
# cursor.execute("""SELECT datname FROM pg_database
# WHERE datistemplate = false;""")
MdxEngine.postgres_db_cubes = [ MdxEngine.postgres_db_cubes = [
database[0] for database in cursor.fetchall() database[0] for database in available_tables
] ]
except Exception: except Exception:
......
...@@ -90,13 +90,13 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj): ...@@ -90,13 +90,13 @@ def _construct_web_star_schema_config_file(executer_instance, cubes_obj):
all_columns += cubes_obj.facts[0].columns all_columns += cubes_obj.facts[0].columns
fusion = psql.read_sql_query( fusion = psql.read_sql_query(
"SELECT * FROM {0}".format(executer_instance.facts), db.connection) "SELECT * FROM {0}".format(executer_instance.facts), db.engine)
tables = {} tables = {}
for table in cubes_obj.tables: for table in cubes_obj.tables:
tab = psql.read_sql_query("SELECT * FROM {0}".format(table.name), tab = psql.read_sql_query("SELECT * FROM {0}".format(table.name),
db.connection) db.engine)
try: try:
if table.columns: if table.columns:
......
from __future__ import absolute_import, division, print_function from __future__ import absolute_import, division, print_function
from sqlalchemy import inspect
from ..tools.connection import MyDB from ..tools.connection import MyDB
import pandas.io.sql as psql import pandas.io.sql as psql
...@@ -12,15 +14,13 @@ def _load_tables_db(executer_instance): ...@@ -12,15 +14,13 @@ def _load_tables_db(executer_instance):
""" """
tables = {} tables = {}
db = MyDB(db=executer_instance.cube) db = MyDB(db=executer_instance.cube)
cursor = db.connection.cursor() inspector = inspect(db.engine)
cursor.execute("""SELECT table_name FROM information_schema.tables
WHERE table_schema = 'public'""")
for table_name in cursor.fetchall(): for table_name in inspector.get_table_names():
value = psql.read_sql_query( value = psql.read_sql_query(
'SELECT * FROM "{0}" '.format(table_name[0]), db.connection) 'SELECT * FROM "{0}"'.format(table_name), db.engine)
tables[table_name[0]] = value[[ tables[table_name] = value[[
col for col in value.columns if col.lower()[-3:] != '_id' col for col in value.columns if col.lower()[-3:] != '_id'
]] ]]
return tables return tables
...@@ -37,16 +37,15 @@ def _construct_star_schema_db(executer_instance): ...@@ -37,16 +37,15 @@ def _construct_star_schema_db(executer_instance):
# load facts table # load facts table
fusion = psql.read_sql_query( fusion = psql.read_sql_query(
'SELECT * FROM "{0}" '.format(executer_instance.facts), db.connection) 'SELECT * FROM "{0}" '.format(executer_instance.facts), db.engine)
inspector = inspect(db.engine)
cursor = db.connection.cursor() for db_table_name in inspector.get_table_names():
cursor.execute("""SELECT table_name FROM information_schema.tables
WHERE table_schema = 'public'""")
for db_table_name in cursor.fetchall():
try: try:
fusion = fusion.merge( fusion = fusion.merge(
psql.read_sql_query("SELECT * FROM {0}".format( psql.read_sql_query("SELECT * FROM {0}".format(
db_table_name[0]), db.connection)) db_table_name[0]), db.engine))
except: except:
print('No common column') print('No common column')
pass pass
......
import psycopg2 as pg import psycopg2 as pg
from sqlalchemy import create_engine
# postgres connection # postgres connection
from olapy_config_file_parser import DbConfigParser from olapy_config_file_parser import DbConfigParser
...@@ -24,20 +25,24 @@ class MyDB(object): ...@@ -24,20 +25,24 @@ class MyDB(object):
username = db_credentials['user_name'] username = db_credentials['user_name']
password = db_credentials['password'] password = db_credentials['password']
host = db_credentials['host'] host = db_credentials['host']
port = db_credentials['port']
if db is None: if db is None:
# first i want to show all databases to user (in excel) # first i want to show all databases to user (in excel)
self.connection = pg.connect("user={0} password={1} host='{2}'". self.engine = pg.connect("user={0} password={1} host='{2}'".
format(username, password, host)) format(username, password, host))
self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
username, password, 'postgres', host, port))
else: else:
# and then we connect to the user db # and then we connect to the user db
try: self.engine = create_engine('postgresql+psycopg2://{0}:{1}@{3}:{4}/{2}'.format(
self.connection = pg.connect( username, password, db, host, port))
"user={0} password={1} dbname='{2}' host='{3}'".format( # self.connection = pg.connect(
username, password, db, host)) # "user={0} password={1} dbname='{2}' host='{3}'".format(
except: # username, password, db, host))
print("can't connect")
def __del__(self): def __del__(self):
if hasattr(self, 'connection'): if hasattr(self, 'connection'):
self.connection.close() self.engine.dispose()
...@@ -43,15 +43,13 @@ class DbConfigParser: ...@@ -43,15 +43,13 @@ class DbConfigParser:
parser = etree.XMLParser() parser = etree.XMLParser()
tree = etree.parse(config_file, parser) tree = etree.parse(config_file, parser)
try: return [
return [ {
{ # 'sgbd': db.find('sgbd').text,
# 'sgbd': db.find('sgbd').text, 'user_name': db.find('user_name').text,
'user_name': db.find('user_name').text, 'password': db.find('password').text,
'password': db.find('password').text, 'host': db.find('host').text,
'host': db.find('host').text, 'port': db.find('port').text,
} }
for db in tree.xpath('/olapy/database') for db in tree.xpath('/olapy/database')
] ]
except:
raise ('missed name or source tags')
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment