From eabb6b07f484b1c54ca8def2b28248b21f501f04 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 17 Mar 2021 09:13:36 -0400 Subject: [PATCH 01/37] update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b6e4761..e4a1406 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] From d1fffc287c50c3056e40748f233d8412e19030a8 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 17 Mar 2021 09:50:18 -0400 Subject: [PATCH 02/37] draft --- pydb/__init__.py | 0 pydb/pydb.py | 150 +++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 28 +++++++++ 3 files changed, 178 insertions(+) create mode 100644 pydb/__init__.py create mode 100644 pydb/pydb.py create mode 100644 setup.py diff --git a/pydb/__init__.py b/pydb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pydb/pydb.py b/pydb/pydb.py new file mode 100644 index 0000000..d63af5a --- /dev/null +++ b/pydb/pydb.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +""" +Module for interacting with the database +""" +# -*- encoding: utf-8 -*- + +#============================================================================================ +# Imports +#============================================================================================ +# Standard imports +import logging + +# Third-party imports +import cx_Oracle + +# Local imports + + +class DBUtil: + """ + Class for interacting with an oracle database + """ + + def __init__(self, oracle_config: dict, logging_formatter: logging.Formatter = None): + self.oracle_config = oracle_config + + self.logger = logging.getLogger(__name__) + stream_handler = logging.StreamHandler() + stream_handler.setLevel(logging.INFO) + + if logging_formatter: + stream_handler.setFormatter(logging_formatter) + else: + stream_format = logging.Formatter( + '{"log_level": "%(levelname)s", ' + '"app_file_line": "%(name)s:%(lineno)d", ' + '"message": %(message)s}' + ) + stream_handler.setFormatter(stream_format) + self.logger.addHandler(stream_handler) + self.logger.setLevel(logging.INFO) + + def get_session_pool(self): + """ + Function for creating a session pool with the database + """ + if self.oracle_config.get('pool') is None: + host = self.oracle_config.get('host') + port = self.oracle_config.get('port') + instance = self.oracle_config.get('instance') + + try: + dsn_str = cx_Oracle.makedsn(host, port, service_name=instance) + pool = cx_Oracle.SessionPool( + user=self.oracle_config.get('user'), + password=self.oracle_config.get('pwd'), + dsn=dsn_str, + min=2, + max=5, + increment=1, + threaded=True, + encoding="UTF-8" + ) + self.oracle_config['pool'] = pool + return pool + + except cx_Oracle.DatabaseError as err: + obj, = err.args + self.logger.error("Error creating pool") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) + raise Exception(f"Error creating pool: {obj.message}") + else: + return self.oracle_config['pool'] + + def create_connection(self, pool): + """ + Function for creating a connection with the database from a session pool + """ + try: + connection = pool.acquire() + return connection + + except cx_Oracle.DatabaseError as err: + obj, = err.args + self.logger.error("Error acquiring database connection from the session pool") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) + raise Exception("Error acquiring database connection from the session pool") + + @staticmethod + def make_dict(cursor): + """ + Function for converting a query result row into a dictionary + """ + column_names = [d[0] for d in cursor.description] + + def create_row(*args): + return dict(zip(column_names, args)) + return create_row + + def execute_query(self, pool, query_string: str, args=None) -> dict: + """ + Function for executing a query against the database via the session pool + """ + try: + connection = self.create_connection(pool) + cursor = connection.cursor() + if args is not None: + cursor.execute(query_string, args) + else: + cursor.execute(query_string) + cursor.rowfactory = self.make_dict(cursor) + query_result = cursor.fetchall() + return query_result + + except cx_Oracle.DatabaseError as err: + obj, = err.args + self.logger.error("Error in execute_query") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) + raise Exception(f"Error executing query: {query_string}") + + finally: + cursor.close() + pool.release(connection) + + def execute_update(self, pool, query_string, args=None): + """ + Function for executing an insert/update query against the database via the session pool + """ + try: + connection = self.create_connection(pool) + cursor = connection.cursor() + if args is not None: + cursor.execute(query_string, args) + else: + cursor.execute(query_string) + connection.commit() + + except cx_Oracle.DatabaseError as err: + obj, = err.args + self.logger.error("Error in execute_update") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) + raise Exception(f"Error executing update: {query_string}") + + finally: + cursor.close() + pool.release(connection) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..cd00dcd --- /dev/null +++ b/setup.py @@ -0,0 +1,28 @@ +import setuptools + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +setuptools.setup( + name="pydb", + version="0.0.1", + author="Michael Kerry", + author_email="michael_kerry@harvard.edu", + description="A package to facilitate connecting to an oracle DB from a python application", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/huit/pydb", + project_urls={ + "Bug Tracker": "https://github.com/huit/pydb/issues", + }, + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + install_requires=[ + "cx-Oracle==8.1.0", + ], + packages=setuptools.find_packages(), + python_requires=">=3.7", +) From 2638a4b5c15971f8d538f75384ea9f10bc36cde7 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 17 Mar 2021 10:05:40 -0400 Subject: [PATCH 03/37] update docs; refactor --- pydb/pydb.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pydb/pydb.py b/pydb/pydb.py index d63af5a..65c8420 100644 --- a/pydb/pydb.py +++ b/pydb/pydb.py @@ -22,7 +22,19 @@ class DBUtil: """ def __init__(self, oracle_config: dict, logging_formatter: logging.Formatter = None): + """ + Setup for oracle db connections. oracle_config must be a python dictionary with the following fields: + host + port + instance (service name) + user + pwd + + :param oracle_config: + :param logging_formatter: + """ self.oracle_config = oracle_config + self.pool = None self.logger = logging.getLogger(__name__) stream_handler = logging.StreamHandler() @@ -44,7 +56,9 @@ def get_session_pool(self): """ Function for creating a session pool with the database """ - if self.oracle_config.get('pool') is None: + if self.pool: + return self.pool + else: host = self.oracle_config.get('host') port = self.oracle_config.get('port') instance = self.oracle_config.get('instance') @@ -61,7 +75,7 @@ def get_session_pool(self): threaded=True, encoding="UTF-8" ) - self.oracle_config['pool'] = pool + self.pool = pool return pool except cx_Oracle.DatabaseError as err: @@ -70,8 +84,6 @@ def get_session_pool(self): self.logger.error("Context: %s", obj.context) self.logger.error("Message: %s", obj.message) raise Exception(f"Error creating pool: {obj.message}") - else: - return self.oracle_config['pool'] def create_connection(self, pool): """ From b01f92331f3f92c982977cc8c03d0368851854e6 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 17 Mar 2021 10:11:26 -0400 Subject: [PATCH 04/37] rename for clarity --- pydb/pydb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydb/pydb.py b/pydb/pydb.py index 65c8420..7008ef3 100644 --- a/pydb/pydb.py +++ b/pydb/pydb.py @@ -16,7 +16,7 @@ # Local imports -class DBUtil: +class PyDB: """ Class for interacting with an oracle database """ From a8023acc3b7e870aca5ff202bad600998e7aa926 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 17 Mar 2021 10:33:23 -0400 Subject: [PATCH 05/37] cleanup --- pydb/pydb.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pydb/pydb.py b/pydb/pydb.py index 7008ef3..742b1c6 100644 --- a/pydb/pydb.py +++ b/pydb/pydb.py @@ -34,7 +34,7 @@ def __init__(self, oracle_config: dict, logging_formatter: logging.Formatter = N :param logging_formatter: """ self.oracle_config = oracle_config - self.pool = None + self.pool = self.get_session_pool() self.logger = logging.getLogger(__name__) stream_handler = logging.StreamHandler() @@ -85,12 +85,12 @@ def get_session_pool(self): self.logger.error("Message: %s", obj.message) raise Exception(f"Error creating pool: {obj.message}") - def create_connection(self, pool): + def create_connection(self): """ Function for creating a connection with the database from a session pool """ try: - connection = pool.acquire() + connection = self.get_session_pool().acquire() return connection except cx_Oracle.DatabaseError as err: @@ -111,12 +111,12 @@ def create_row(*args): return dict(zip(column_names, args)) return create_row - def execute_query(self, pool, query_string: str, args=None) -> dict: + def execute_query(self, query_string: str, args=None) -> dict: """ Function for executing a query against the database via the session pool """ try: - connection = self.create_connection(pool) + connection = self.create_connection() cursor = connection.cursor() if args is not None: cursor.execute(query_string, args) @@ -135,14 +135,14 @@ def execute_query(self, pool, query_string: str, args=None) -> dict: finally: cursor.close() - pool.release(connection) + self.get_session_pool().release(connection) - def execute_update(self, pool, query_string, args=None): + def execute_update(self, query_string, args=None): """ Function for executing an insert/update query against the database via the session pool """ try: - connection = self.create_connection(pool) + connection = self.create_connection() cursor = connection.cursor() if args is not None: cursor.execute(query_string, args) @@ -159,4 +159,4 @@ def execute_update(self, pool, query_string, args=None): finally: cursor.close() - pool.release(connection) + self.get_session_pool().release(connection) From 75264285f68e4c66b98b7d72fa5d7071b37f8186 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 17 Mar 2021 10:42:24 -0400 Subject: [PATCH 06/37] rename for clarity --- pydb/{pydb.py => oracle_db.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename pydb/{pydb.py => oracle_db.py} (98%) diff --git a/pydb/pydb.py b/pydb/oracle_db.py similarity index 98% rename from pydb/pydb.py rename to pydb/oracle_db.py index 742b1c6..8ab2d82 100644 --- a/pydb/pydb.py +++ b/pydb/oracle_db.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Module for interacting with the database +Module for interacting with an oracle database """ # -*- encoding: utf-8 -*- @@ -16,7 +16,7 @@ # Local imports -class PyDB: +class OracleDB: """ Class for interacting with an oracle database """ From df48b87bf28ff888b4b0938c3c47f5489a941319 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 17 Mar 2021 11:21:02 -0400 Subject: [PATCH 07/37] revise logger setup --- pydb/oracle_db.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py index 8ab2d82..f7b9e58 100644 --- a/pydb/oracle_db.py +++ b/pydb/oracle_db.py @@ -21,7 +21,7 @@ class OracleDB: Class for interacting with an oracle database """ - def __init__(self, oracle_config: dict, logging_formatter: logging.Formatter = None): + def __init__(self, oracle_config: dict, logging_format: logging.Formatter = None): """ Setup for oracle db connections. oracle_config must be a python dictionary with the following fields: host @@ -31,26 +31,29 @@ def __init__(self, oracle_config: dict, logging_formatter: logging.Formatter = N pwd :param oracle_config: - :param logging_formatter: + :param logging_format: """ self.oracle_config = oracle_config self.pool = self.get_session_pool() - self.logger = logging.getLogger(__name__) + self.logger = self.setup_logger(logging_format) + + @staticmethod + def setup_logger(logging_format: logging.Formatter = None) -> logging.Logger: + logger = logging.getLogger(__name__) stream_handler = logging.StreamHandler() stream_handler.setLevel(logging.INFO) - if logging_formatter: - stream_handler.setFormatter(logging_formatter) - else: - stream_format = logging.Formatter( + if logging_format is None: + logging_format = logging.Formatter( '{"log_level": "%(levelname)s", ' '"app_file_line": "%(name)s:%(lineno)d", ' '"message": %(message)s}' ) - stream_handler.setFormatter(stream_format) - self.logger.addHandler(stream_handler) - self.logger.setLevel(logging.INFO) + stream_handler.setFormatter(logging_format) + logger.addHandler(stream_handler) + logger.setLevel(logging.INFO) + return logger def get_session_pool(self): """ From 8b7da3259e734f36ee2f6673f93759d24d30b43e Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 17 Mar 2021 14:06:49 -0400 Subject: [PATCH 08/37] revise logger setup --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index cd00dcd..2a11aa6 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ ], install_requires=[ "cx-Oracle==8.1.0", + 'pylog @ https://github.com/huit/pylog/archive/v0.0.1.tar.gz#egg=pylog', ], packages=setuptools.find_packages(), python_requires=">=3.7", From db2132317609b8cc10890d6e5393a98647dcb9e3 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 17 Mar 2021 14:08:21 -0400 Subject: [PATCH 09/37] revise logger setup --- pydb/oracle_db.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py index f7b9e58..1a51de8 100644 --- a/pydb/oracle_db.py +++ b/pydb/oracle_db.py @@ -13,6 +13,8 @@ # Third-party imports import cx_Oracle +from pylog.pylog import get_common_logger_for_module + # Local imports @@ -21,7 +23,7 @@ class OracleDB: Class for interacting with an oracle database """ - def __init__(self, oracle_config: dict, logging_format: logging.Formatter = None): + def __init__(self, oracle_config: dict, logging_level: int = 50, logging_format: logging.Formatter = None): """ Setup for oracle db connections. oracle_config must be a python dictionary with the following fields: host @@ -31,29 +33,13 @@ def __init__(self, oracle_config: dict, logging_format: logging.Formatter = None pwd :param oracle_config: - :param logging_format: + :param logging_level: defaults to logging.CRITICAL + :param logging_format: defaults to None here, which translates to the pylog.get_commong_logging_format """ self.oracle_config = oracle_config self.pool = self.get_session_pool() - self.logger = self.setup_logger(logging_format) - - @staticmethod - def setup_logger(logging_format: logging.Formatter = None) -> logging.Logger: - logger = logging.getLogger(__name__) - stream_handler = logging.StreamHandler() - stream_handler.setLevel(logging.INFO) - - if logging_format is None: - logging_format = logging.Formatter( - '{"log_level": "%(levelname)s", ' - '"app_file_line": "%(name)s:%(lineno)d", ' - '"message": %(message)s}' - ) - stream_handler.setFormatter(logging_format) - logger.addHandler(stream_handler) - logger.setLevel(logging.INFO) - return logger + self.logger = get_common_logger_for_module(module_name=__name__, level=logging_level, log_format=logging_format) def get_session_pool(self): """ From 5c9a7c44beabd39edc45aff252684efb0c0f2066 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 17 Mar 2021 14:34:59 -0400 Subject: [PATCH 10/37] fix bug --- pydb/oracle_db.py | 55 +++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py index 1a51de8..cd250cf 100644 --- a/pydb/oracle_db.py +++ b/pydb/oracle_db.py @@ -37,10 +37,37 @@ def __init__(self, oracle_config: dict, logging_level: int = 50, logging_format: :param logging_format: defaults to None here, which translates to the pylog.get_commong_logging_format """ self.oracle_config = oracle_config - self.pool = self.get_session_pool() + self.pool = self.set_up_session_pool() self.logger = get_common_logger_for_module(module_name=__name__, level=logging_level, log_format=logging_format) + def set_up_session_pool(self): + host = self.oracle_config.get('host') + port = self.oracle_config.get('port') + instance = self.oracle_config.get('instance') + + try: + dsn_str = cx_Oracle.makedsn(host, port, service_name=instance) + pool = cx_Oracle.SessionPool( + user=self.oracle_config.get('user'), + password=self.oracle_config.get('pwd'), + dsn=dsn_str, + min=2, + max=5, + increment=1, + threaded=True, + encoding="UTF-8" + ) + self.pool = pool + return pool + + except cx_Oracle.DatabaseError as err: + obj, = err.args + self.logger.error("Error creating pool") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) + raise Exception(f"Error creating pool: {obj.message}") + def get_session_pool(self): """ Function for creating a session pool with the database @@ -48,31 +75,7 @@ def get_session_pool(self): if self.pool: return self.pool else: - host = self.oracle_config.get('host') - port = self.oracle_config.get('port') - instance = self.oracle_config.get('instance') - - try: - dsn_str = cx_Oracle.makedsn(host, port, service_name=instance) - pool = cx_Oracle.SessionPool( - user=self.oracle_config.get('user'), - password=self.oracle_config.get('pwd'), - dsn=dsn_str, - min=2, - max=5, - increment=1, - threaded=True, - encoding="UTF-8" - ) - self.pool = pool - return pool - - except cx_Oracle.DatabaseError as err: - obj, = err.args - self.logger.error("Error creating pool") - self.logger.error("Context: %s", obj.context) - self.logger.error("Message: %s", obj.message) - raise Exception(f"Error creating pool: {obj.message}") + self.set_up_session_pool() def create_connection(self): """ From fc74b9c696db43f9a6ca9c7d93ead12bfa6c52f0 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 17 Mar 2021 17:17:23 -0400 Subject: [PATCH 11/37] add interface --- pydb/database.py | 33 +++++++++++++++++++++++++++++++++ pydb/oracle_db.py | 11 ++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 pydb/database.py diff --git a/pydb/database.py b/pydb/database.py new file mode 100644 index 0000000..adc6eef --- /dev/null +++ b/pydb/database.py @@ -0,0 +1,33 @@ +import abc + +from enum import Enum + + +class Database(Enum): + """ + not implemented: other databases + """ + ORACLE = "oracle" + + +class DBInterface(metaclass=abc.ABCMeta): + @classmethod + def __subclasshook__(cls, subclass): + return (hasattr(subclass, 'execute_query') and + callable(subclass.execute_query) and + hasattr(subclass, 'execute_update') and + callable(subclass.execute_update) and + hasattr(subclass, 'health_check') and + callable(subclass.health_check)) + + @abc.abstractmethod + def execute_query(self, query_string: str, args=None) -> dict: + raise NotImplementedError + + @abc.abstractmethod + def execute_update(self, query_string, args=None): + raise NotImplementedError + + @abc.abstractmethod + def health_check(): + raise NotImplementedError diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py index cd250cf..62f74d5 100644 --- a/pydb/oracle_db.py +++ b/pydb/oracle_db.py @@ -15,10 +15,12 @@ from pylog.pylog import get_common_logger_for_module +from .database import DBInterface + # Local imports -class OracleDB: +class OracleDB(DBInterface): """ Class for interacting with an oracle database """ @@ -152,3 +154,10 @@ def execute_update(self, query_string, args=None): finally: cursor.close() self.get_session_pool().release(connection) + + def health_check(self): + """ + provides a means to verify DB connectivity with a simple query + :return: + """ + return self.execute_query("SELECT 1 FROM DUAL") From d6b2c86b7cf2c2a0c98fa99301ca30e7b96f0342 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 18 Mar 2021 08:56:42 -0400 Subject: [PATCH 12/37] refactor init params and handling --- pydb/oracle_db.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py index 62f74d5..3294f0b 100644 --- a/pydb/oracle_db.py +++ b/pydb/oracle_db.py @@ -25,34 +25,36 @@ class OracleDB(DBInterface): Class for interacting with an oracle database """ - def __init__(self, oracle_config: dict, logging_level: int = 50, logging_format: logging.Formatter = None): + def __init__(self, host: str, port: int, service: str, user: str, pwd: str, logging_level: int = 50, logging_format: logging.Formatter = None): """ Setup for oracle db connections. oracle_config must be a python dictionary with the following fields: - host - port - instance (service name) - user - pwd - :param oracle_config: + :param host: + :param port: + :param service: + :param user: + :param pwd: + :param logging_level: + :param logging_format: :param logging_level: defaults to logging.CRITICAL :param logging_format: defaults to None here, which translates to the pylog.get_commong_logging_format """ - self.oracle_config = oracle_config + + self.host = host + self.port = port + self.service = service + self.user = user + self.pwd = pwd self.pool = self.set_up_session_pool() self.logger = get_common_logger_for_module(module_name=__name__, level=logging_level, log_format=logging_format) def set_up_session_pool(self): - host = self.oracle_config.get('host') - port = self.oracle_config.get('port') - instance = self.oracle_config.get('instance') - try: - dsn_str = cx_Oracle.makedsn(host, port, service_name=instance) + dsn_str = cx_Oracle.makedsn(self.host, self.port, service_name=self.instance) pool = cx_Oracle.SessionPool( - user=self.oracle_config.get('user'), - password=self.oracle_config.get('pwd'), + user=self.user, + password=self.pwd, dsn=dsn_str, min=2, max=5, From f51d83ce9036a13f42a126e6dfe6592894db93dd Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 18 Mar 2021 08:57:40 -0400 Subject: [PATCH 13/37] cleanup --- pydb/oracle_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py index 3294f0b..6947363 100644 --- a/pydb/oracle_db.py +++ b/pydb/oracle_db.py @@ -51,7 +51,7 @@ def __init__(self, host: str, port: int, service: str, user: str, pwd: str, logg def set_up_session_pool(self): try: - dsn_str = cx_Oracle.makedsn(self.host, self.port, service_name=self.instance) + dsn_str = cx_Oracle.makedsn(self.host, self.port, service_name=self.service) pool = cx_Oracle.SessionPool( user=self.user, password=self.pwd, From a7e27dc60e4f59e64bdefc89e6e2bd2a383a1ea8 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 18 Mar 2021 15:08:01 -0400 Subject: [PATCH 14/37] add cleanup method --- pydb/database.py | 8 +++++++- pydb/oracle_db.py | 13 +++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/pydb/database.py b/pydb/database.py index adc6eef..a9d63e3 100644 --- a/pydb/database.py +++ b/pydb/database.py @@ -18,7 +18,9 @@ def __subclasshook__(cls, subclass): hasattr(subclass, 'execute_update') and callable(subclass.execute_update) and hasattr(subclass, 'health_check') and - callable(subclass.health_check)) + callable(subclass.health_check) and + hasattr(subclass, 'cleanup') and + callable(subclass.cleanup)) @abc.abstractmethod def execute_query(self, query_string: str, args=None) -> dict: @@ -31,3 +33,7 @@ def execute_update(self, query_string, args=None): @abc.abstractmethod def health_check(): raise NotImplementedError + + @abc.abstractmethod + def cleanup(): + raise NotImplementedError diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py index 6947363..50808d3 100644 --- a/pydb/oracle_db.py +++ b/pydb/oracle_db.py @@ -163,3 +163,16 @@ def health_check(self): :return: """ return self.execute_query("SELECT 1 FROM DUAL") + + def cleanup(self): + if self.pool is not None: + self.logger.info("Active session pool found. Attempting to close session pool.") + try: + self.pool.close(force=True) + self.logger.info("Session pool successfully closed.") + + except cx_Oracle.Error as err: + self.logger.error("Unable to close the active session.", exc_info=True) + obj, = err.args + self.logger.error("Context:", obj.context) + self.logger.error("Message:", obj.message) From 701d5b289959068e402ec2d168ac8f2cd0eaf3d6 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Fri, 19 Mar 2021 16:24:06 -0400 Subject: [PATCH 15/37] rename Database enum to DatabaseType; add sql alchemy --- pydb/database.py | 18 ++++++- pydb/oracle_db.py | 6 ++- pydb/sql_alchemy_oracle_db.py | 88 +++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 pydb/sql_alchemy_oracle_db.py diff --git a/pydb/database.py b/pydb/database.py index a9d63e3..7a0dc37 100644 --- a/pydb/database.py +++ b/pydb/database.py @@ -3,11 +3,12 @@ from enum import Enum -class Database(Enum): +class DatabaseType(Enum): """ not implemented: other databases """ ORACLE = "oracle" + SQL_ALCHEMY_ORACLE = "sql_alchemy_oracle" class DBInterface(metaclass=abc.ABCMeta): @@ -20,7 +21,12 @@ def __subclasshook__(cls, subclass): hasattr(subclass, 'health_check') and callable(subclass.health_check) and hasattr(subclass, 'cleanup') and - callable(subclass.cleanup)) + callable(subclass.cleanup) and + hasattr(subclass, 'create_connection') and + callable(subclass.create_connection) and + hasattr(subclass, 'get_session') and + callable(subclass.get_session) + or NotImplemented) @abc.abstractmethod def execute_query(self, query_string: str, args=None) -> dict: @@ -37,3 +43,11 @@ def health_check(): @abc.abstractmethod def cleanup(): raise NotImplementedError + + @abc.abstractmethod + def create_connection(): + raise NotImplementedError + + @abc.abstractmethod + def get_session(): + raise NotImplementedError diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py index 50808d3..3ed9a95 100644 --- a/pydb/oracle_db.py +++ b/pydb/oracle_db.py @@ -25,7 +25,8 @@ class OracleDB(DBInterface): Class for interacting with an oracle database """ - def __init__(self, host: str, port: int, service: str, user: str, pwd: str, logging_level: int = 50, logging_format: logging.Formatter = None): + def __init__(self, host: str, port: int, service: str, user: str, pwd: str, + logging_level: int = 50, logging_format: logging.Formatter = None): """ Setup for oracle db connections. oracle_config must be a python dictionary with the following fields: @@ -176,3 +177,6 @@ def cleanup(self): obj, = err.args self.logger.error("Context:", obj.context) self.logger.error("Message:", obj.message) + + def get_session(self): + return None diff --git a/pydb/sql_alchemy_oracle_db.py b/pydb/sql_alchemy_oracle_db.py new file mode 100644 index 0000000..3c95eb1 --- /dev/null +++ b/pydb/sql_alchemy_oracle_db.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +import cx_Oracle +import logging + +from sqlalchemy import create_engine, text +from sqlalchemy.pool import NullPool +from sqlalchemy.orm import sessionmaker + +from pylog.pylog import get_common_logger_for_module + + +class SqlAlchemyOracleUtil: + def __init__(self, host: str, port: int, service: str, user: str, pwd: str, + logging_level: int = 50, logging_format: logging.Formatter = None): + self.logger = get_common_logger_for_module(module_name=__name__, level=logging_level, log_format=logging_format) + self.host = host + self.port = port + self.service = service + self.user = user + self.pwd = pwd + self.engine = self.setup_engine() + + def setup_engine(self): + try: + dsn_str = cx_Oracle.makedsn(self.host, self.port, service_name=self.service) + pool = cx_Oracle.SessionPool( + user=self.user, password=self.pwd, dsn=dsn_str, + min=2, max=5, increment=1, threaded=True + ) + engine = create_engine("oracle://", + creator=pool.acquire, + poolclass=NullPool, + echo=True, + max_identifier_length=128) + + self.logger.info("Setup sqlalchemy engine successfully") + return engine + except Exception as err: + obj, = err.args + self.logger.critical("Cannot create sql alchemy engine") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) + raise Exception(f"Cannot create sql alchemy engine: {obj.message}") + + def get_engine(self): + if self.engine is None: + self.engine = self.setup_engine() + return self.engine + + def get_session(self): + Session = sessionmaker(bind=self.get_engine()) + return Session() + + def create_connection(self): + return self.engine.connect() + + def health_check(self): + with self.create_connection() as conn: + return conn.scalar("select 1 from dual") + + def execute_query(self, query_string: str, args: dict): + with self.create_connection() as conn: + statement = text(query_string) + if args is not None and len(args.keys()) > 0: + return conn.execute(statement, args) + else: + return conn.execute(statement) + + def execute_update(self, query_string: str, args: dict): + with self.create_connection() as conn: + statement = text(query_string) + trans = conn.begin() + if args is not None and len(args.keys()) > 0: + conn.execute(statement, args) + else: + conn.execute(statement) + trans.commit() + + def cleanup(self): + if self.engine is not None: + self.logger.info("sql alchemy engine found") + try: + self.engine.dispose() + except Exception as err: + obj, = err.args + self.logger.critical("Cannot dispose sql alchemy engine") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) From c1d062874135a7ded0c704504a2e11919d11a9e9 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Fri, 19 Mar 2021 16:24:58 -0400 Subject: [PATCH 16/37] update setup --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 2a11aa6..d953059 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ ], install_requires=[ "cx-Oracle==8.1.0", + 'sqlalchemy==1.4.1', 'pylog @ https://github.com/huit/pylog/archive/v0.0.1.tar.gz#egg=pylog', ], packages=setuptools.find_packages(), From deb22af8a781b2d780de9f89c58bf09cf81ad9f1 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Fri, 19 Mar 2021 16:27:00 -0400 Subject: [PATCH 17/37] update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d953059..3b94e09 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="pydb", - version="0.0.1", + version="0.0.2", author="Michael Kerry", author_email="michael_kerry@harvard.edu", description="A package to facilitate connecting to an oracle DB from a python application", From 29c3b16bcb3a7460a63b144e51b72e7588cf8fc7 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Fri, 19 Mar 2021 16:29:04 -0400 Subject: [PATCH 18/37] rename --- pydb/sql_alchemy_oracle_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydb/sql_alchemy_oracle_db.py b/pydb/sql_alchemy_oracle_db.py index 3c95eb1..400ae1c 100644 --- a/pydb/sql_alchemy_oracle_db.py +++ b/pydb/sql_alchemy_oracle_db.py @@ -9,7 +9,7 @@ from pylog.pylog import get_common_logger_for_module -class SqlAlchemyOracleUtil: +class SqlAlchemyOracleDB: def __init__(self, host: str, port: int, service: str, user: str, pwd: str, logging_level: int = 50, logging_format: logging.Formatter = None): self.logger = get_common_logger_for_module(module_name=__name__, level=logging_level, log_format=logging_format) From 549ca1adb0cfa9a2640d2153bd8319ecc78ee7a5 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 24 Mar 2021 13:16:21 -0400 Subject: [PATCH 19/37] update handling of execute_query --- pydb/sql_alchemy_oracle_db.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pydb/sql_alchemy_oracle_db.py b/pydb/sql_alchemy_oracle_db.py index 400ae1c..32f1a1d 100644 --- a/pydb/sql_alchemy_oracle_db.py +++ b/pydb/sql_alchemy_oracle_db.py @@ -58,13 +58,16 @@ def health_check(self): with self.create_connection() as conn: return conn.scalar("select 1 from dual") - def execute_query(self, query_string: str, args: dict): + def execute_query(self, query_string: str, args: dict = None) -> list: with self.create_connection() as conn: statement = text(query_string) + if args is not None and len(args.keys()) > 0: - return conn.execute(statement, args) + query_result = conn.execute(statement, args).fetchall() else: - return conn.execute(statement) + query_result = conn.execute(statement).fetchall() + + return [dict(row) for row in query_result] def execute_update(self, query_string: str, args: dict): with self.create_connection() as conn: From eaa53f6f960dac2ee59fa5ed20e3275f2ac25418 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 24 Mar 2021 13:45:29 -0400 Subject: [PATCH 20/37] update documentation --- README.md | 68 ++++++++++++++++++++++++++++++++++- pydb/database.py | 32 ++++++++++++++++- pydb/sql_alchemy_oracle_db.py | 48 ++++++++++++++++++++++++- 3 files changed, 145 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 33519d2..6b6257a 100644 --- a/README.md +++ b/README.md @@ -1 +1,67 @@ -# pydb \ No newline at end of file +# pydb + +a tool for facilitating connection to databases with python and performing basic operations + +## Requirements + + python >= 3.7 + (see setup.py for additional packages) + +## Installation and setup + +In a suitable python3 (>=3.7) virtual env, using pip: + + pip install https://github.com/huit/pydb/archive/v0.0.2.tar.gz#egg=pydb + # import the module for the specific type of db you'd like to use + from pydb.oracle_db import OracleDB + +* creating an OracleDB instance requires host, port, service, user, pwd. + * other db types may have other requirements - see specific module for details +* logging_level is optional, and will default to `logging.CRITICAL` +* logging_format is optional, and will default to pylog default formatting +* see https://github.com/huit/pylog for details + + + # where 8003 is a valid port + db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="username", pwd="pwd") + +## Basic operations + + create_connection() + provides a connection to the db host for more 'direct' access + + get_session() + Specific to SQL Alchemy; allows interaction with SQL Alchemy entities + see https://docs.sqlalchemy.org/en/14/orm/session.html?highlight=session#module-sqlalchemy.orm.session + + execute_query(self, query_string: str, args=None) -> list + executes a sql query, return a list of dictionaries representing rows + + execute_update(self, query_string, args=None) + used to execute an insert, update, or delete sql statement + + health_check() + Performs a basic query against the db to ensure connectivity + + cleanup() + Attempts to release any 'live' objects/connections to the host + +## Example + +Given a valid connection, and a table called `EMP`... + +To query the `EMP` table for all records: + + result = db.execute_query("select * from emp") + +To query the `EMP` table for a specific record: + + result = db.execute_query("select * from emp where ename= :ename", {'ename':'JOHNSON'}) + +Results for the individual rows would be in the following form: + + {'EMPNO': 7935, 'ENAME': 'JOHNSON', 'JOB': 'CLERK', 'MGR': 7839, 'HIREDATE': datetime.datetime(1981, 5, 1, 0, 0), 'SAL': 2850.0, 'COMM': None, 'DEPTNO': 30} + +Row results may vary somewhat depending on the exact module... e.g., for SqlAlchemyOracleDB you would receive the following: + + {'empno': 7935, 'ename': 'JOHNSON', 'job': 'CLERK', 'mgr': 7839, 'hiredate': datetime.datetime(1981, 5, 1, 0, 0), 'sal': Decimal('2850'), 'comm': None, 'deptno': 30} \ No newline at end of file diff --git a/pydb/database.py b/pydb/database.py index 7a0dc37..c40b38b 100644 --- a/pydb/database.py +++ b/pydb/database.py @@ -29,25 +29,55 @@ def __subclasshook__(cls, subclass): or NotImplemented) @abc.abstractmethod - def execute_query(self, query_string: str, args=None) -> dict: + def execute_query(self, query_string: str, args=None) -> list: + """ + executes a sql query, return a list of dictionaries representing rows + :param self: + :param query_string: + :param args: + :return: + """ raise NotImplementedError @abc.abstractmethod def execute_update(self, query_string, args=None): + """ + used to execute an insert, update, or delete sql statement + :param self: + :param query_string: + :param args: + :return: + """ raise NotImplementedError @abc.abstractmethod def health_check(): + """ + Performs a basic query against the db to ensure connectivity + :return: + """ raise NotImplementedError @abc.abstractmethod def cleanup(): + """ + Attempts to release any 'live' objects/connections to the host + :return: + """ raise NotImplementedError @abc.abstractmethod def create_connection(): + """ + provides a connection to the db host for more 'direct' access + :return: + """ raise NotImplementedError @abc.abstractmethod def get_session(): + """ + Specific to SQL Alchemy; allows interaction with SQL Alchemy entities + :return: + """ raise NotImplementedError diff --git a/pydb/sql_alchemy_oracle_db.py b/pydb/sql_alchemy_oracle_db.py index 32f1a1d..25f4ebe 100644 --- a/pydb/sql_alchemy_oracle_db.py +++ b/pydb/sql_alchemy_oracle_db.py @@ -10,8 +10,21 @@ class SqlAlchemyOracleDB: + """ + Module for interacting with an Oracle DB via SQL Alchemy + """ def __init__(self, host: str, port: int, service: str, user: str, pwd: str, logging_level: int = 50, logging_format: logging.Formatter = None): + """ + initialize object with values required to create a connection to an Oracle DB + :param host: + :param port: + :param service: + :param user: + :param pwd: + :param logging_level: + :param logging_format: + """ self.logger = get_common_logger_for_module(module_name=__name__, level=logging_level, log_format=logging_format) self.host = host self.port = port @@ -21,6 +34,10 @@ def __init__(self, host: str, port: int, service: str, user: str, pwd: str, self.engine = self.setup_engine() def setup_engine(self): + """ + Create a SQL Alchemy engine to connect to Oracle DB with a connection pool + :return: + """ try: dsn_str = cx_Oracle.makedsn(self.host, self.port, service_name=self.service) pool = cx_Oracle.SessionPool( @@ -48,17 +65,35 @@ def get_engine(self): return self.engine def get_session(self): + """ + Specific to SQL Alchemy; allows interaction with SQL Alchemy entities + :return: + """ Session = sessionmaker(bind=self.get_engine()) return Session() def create_connection(self): - return self.engine.connect() + """ + provides a connection to the db host for more 'direct' access + :return: + """ + return self.get_engine().connect() def health_check(self): + """ + Performs a basic query against the db to ensure connectivity + """ with self.create_connection() as conn: return conn.scalar("select 1 from dual") def execute_query(self, query_string: str, args: dict = None) -> list: + """ + executes a sql query, return a list of dictionaries representing rows + + :param query_string: str + :param args: dict + :return: list of dict + """ with self.create_connection() as conn: statement = text(query_string) @@ -70,6 +105,13 @@ def execute_query(self, query_string: str, args: dict = None) -> list: return [dict(row) for row in query_result] def execute_update(self, query_string: str, args: dict): + """ + used to execute an insert, update, or delete sql statement + + :param query_string: str + :param args: dict + :return: + """ with self.create_connection() as conn: statement = text(query_string) trans = conn.begin() @@ -80,6 +122,10 @@ def execute_update(self, query_string: str, args: dict): trans.commit() def cleanup(self): + """ + release any existing connections; to be called prior to exiting program + :return: + """ if self.engine is not None: self.logger.info("sql alchemy engine found") try: From 62641f91d4acbdf34cfb9834be80b0e5e6196ab7 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 24 Mar 2021 13:46:37 -0400 Subject: [PATCH 21/37] update documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b6257a..37703ac 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,6 @@ Results for the individual rows would be in the following form: {'EMPNO': 7935, 'ENAME': 'JOHNSON', 'JOB': 'CLERK', 'MGR': 7839, 'HIREDATE': datetime.datetime(1981, 5, 1, 0, 0), 'SAL': 2850.0, 'COMM': None, 'DEPTNO': 30} -Row results may vary somewhat depending on the exact module... e.g., for SqlAlchemyOracleDB you would receive the following: +Row results may vary somewhat depending on the exact module... e.g., for SqlAlchemyOracleDB the following would be received: {'empno': 7935, 'ename': 'JOHNSON', 'job': 'CLERK', 'mgr': 7839, 'hiredate': datetime.datetime(1981, 5, 1, 0, 0), 'sal': Decimal('2850'), 'comm': None, 'deptno': 30} \ No newline at end of file From f92e138084b3b36b5e9248773762fc52e6b6dfe1 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 24 Mar 2021 14:39:10 -0400 Subject: [PATCH 22/37] update documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 37703ac..6060091 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ In a suitable python3 (>=3.7) virtual env, using pip: * see https://github.com/huit/pylog for details - # where 8003 is a valid port db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="username", pwd="pwd") + # where 8003 is a valid port ## Basic operations From a261ec417788305955e56eac0d68e564d7b9e6a2 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 24 Mar 2021 14:50:41 -0400 Subject: [PATCH 23/37] update documentation formatting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6060091..011d965 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ In a suitable python3 (>=3.7) virtual env, using pip: db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="username", pwd="pwd") - # where 8003 is a valid port + (where 8003 is a valid port) ## Basic operations From 197e80319c7f9b1cd88f6532d3fca1365e9543bb Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 24 Mar 2021 14:51:37 -0400 Subject: [PATCH 24/37] update documentation formatting --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 011d965..bd67b26 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,10 @@ In a suitable python3 (>=3.7) virtual env, using pip: * logging_format is optional, and will default to pylog default formatting * see https://github.com/huit/pylog for details - +``` db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="username", pwd="pwd") - (where 8003 is a valid port) - + # where 8003 is a valid port) +``` ## Basic operations create_connection() From c0d30ccca15250e16f073d04e01be003d200eee8 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 24 Mar 2021 14:52:05 -0400 Subject: [PATCH 25/37] update documentation formatting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bd67b26..4695d7e 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ In a suitable python3 (>=3.7) virtual env, using pip: ``` db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="username", pwd="pwd") - # where 8003 is a valid port) + # where 8003 is a valid port ``` ## Basic operations From 52a8db367ad959c3234f8dcb7db2dfda9dc5f5d2 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 24 Mar 2021 14:52:28 -0400 Subject: [PATCH 26/37] update documentation formatting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4695d7e..34872dd 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ In a suitable python3 (>=3.7) virtual env, using pip: * see https://github.com/huit/pylog for details ``` - db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="username", pwd="pwd") - # where 8003 is a valid port +db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="username", pwd="pwd") +# where 8003 is a valid port ``` ## Basic operations From dec71249db5deb234d788654d05fee040d99300f Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 24 Mar 2021 14:55:08 -0400 Subject: [PATCH 27/37] update documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 34872dd..4bfdc24 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="userna Performs a basic query against the db to ensure connectivity cleanup() - Attempts to release any 'live' objects/connections to the host + Attempts to release any 'live' objects/connections to the host - to be run before exiting program ## Example From 38009eab0b97cb4b6c41620dc732a62c77c4049a Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Wed, 24 Mar 2021 14:55:34 -0400 Subject: [PATCH 28/37] update documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4bfdc24..66d48fb 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="userna cleanup() Attempts to release any 'live' objects/connections to the host - to be run before exiting program -## Example +## Examples Given a valid connection, and a table called `EMP`... From e8d1027a6dabf0ae7e57a304926ce336dcc7a2bf Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 25 Mar 2021 09:38:33 -0400 Subject: [PATCH 29/37] update pylog version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3b94e09..e4470ac 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ install_requires=[ "cx-Oracle==8.1.0", 'sqlalchemy==1.4.1', - 'pylog @ https://github.com/huit/pylog/archive/v0.0.1.tar.gz#egg=pylog', + 'pylog @ https://github.com/huit/pylog/archive/v0.0.2.tar.gz#egg=pylog', ], packages=setuptools.find_packages(), python_requires=">=3.7", From c5b9294f8eb47324c698d8d187f5072659d87070 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 25 Mar 2021 12:18:20 -0400 Subject: [PATCH 30/37] update pylog version url --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e4470ac..d6e0bab 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ install_requires=[ "cx-Oracle==8.1.0", 'sqlalchemy==1.4.1', - 'pylog @ https://github.com/huit/pylog/archive/v0.0.2.tar.gz#egg=pylog', + 'pylog @ https://github.com/huit/pylog/archive/refs/tags/v0.0.2.tar.gz#egg=pylog', ], packages=setuptools.find_packages(), python_requires=">=3.7", From 71eeed00c72c44183df6c67671f8ad96d3351d1a Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 25 Mar 2021 14:01:23 -0400 Subject: [PATCH 31/37] move creation of logger --- pydb/oracle_db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py index 3ed9a95..e034c44 100644 --- a/pydb/oracle_db.py +++ b/pydb/oracle_db.py @@ -46,9 +46,10 @@ def __init__(self, host: str, port: int, service: str, user: str, pwd: str, self.service = service self.user = user self.pwd = pwd + self.logger = get_common_logger_for_module(module_name=__name__, level=logging_level, log_format=logging_format) + self.pool = self.set_up_session_pool() - self.logger = get_common_logger_for_module(module_name=__name__, level=logging_level, log_format=logging_format) def set_up_session_pool(self): try: From 32eadebbf95956ed423f66f498191c3c0840c07b Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 25 Mar 2021 14:01:51 -0400 Subject: [PATCH 32/37] fix line spacing --- README.md | 37 ++++++++++++++++++++++++++++--------- pydb/oracle_db.py | 1 - 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 66d48fb..aea1d7f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ a tool for facilitating connection to databases with python and performing basic operations +## Purpose and intended audience + + + ## Requirements python >= 3.7 @@ -10,11 +14,11 @@ a tool for facilitating connection to databases with python and performing basic ## Installation and setup In a suitable python3 (>=3.7) virtual env, using pip: - +``` pip install https://github.com/huit/pydb/archive/v0.0.2.tar.gz#egg=pydb # import the module for the specific type of db you'd like to use from pydb.oracle_db import OracleDB - +``` * creating an OracleDB instance requires host, port, service, user, pwd. * other db types may have other requirements - see specific module for details * logging_level is optional, and will default to `logging.CRITICAL` @@ -51,17 +55,32 @@ db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="userna Given a valid connection, and a table called `EMP`... To query the `EMP` table for all records: - +``` result = db.execute_query("select * from emp") - +``` To query the `EMP` table for a specific record: - +``` result = db.execute_query("select * from emp where ename= :ename", {'ename':'JOHNSON'}) - +``` Results for the individual rows would be in the following form: - +``` {'EMPNO': 7935, 'ENAME': 'JOHNSON', 'JOB': 'CLERK', 'MGR': 7839, 'HIREDATE': datetime.datetime(1981, 5, 1, 0, 0), 'SAL': 2850.0, 'COMM': None, 'DEPTNO': 30} - +``` Row results may vary somewhat depending on the exact module... e.g., for SqlAlchemyOracleDB the following would be received: +``` + {'empno': 7935, 'ename': 'JOHNSON', 'job': 'CLERK', 'mgr': 7839, 'hiredate': datetime.datetime(1981, 5, 1, 0, 0), 'sal': Decimal('2850'), 'comm': None, 'deptno': 30} +``` + +### integrated example +``` +pip install https://github.com/huit/pydb/archive/v0.0.2.tar.gz#egg=pydb + +from pydb.oracle_db import OracleDB +db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="username", pwd="pwd") + +result = db.execute_query("select * from emp where ename= :ename", {'ename':'JOHNSON'}) +print(result) + +{'EMPNO': 7935, 'ENAME': 'JOHNSON', 'JOB': 'CLERK', 'MGR': 7839, 'HIREDATE': datetime.datetime(1981, 5, 1, 0, 0), 'SAL': 2850.0, 'COMM': None, 'DEPTNO': 30} - {'empno': 7935, 'ename': 'JOHNSON', 'job': 'CLERK', 'mgr': 7839, 'hiredate': datetime.datetime(1981, 5, 1, 0, 0), 'sal': Decimal('2850'), 'comm': None, 'deptno': 30} \ No newline at end of file +``` \ No newline at end of file diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py index e034c44..61ff52f 100644 --- a/pydb/oracle_db.py +++ b/pydb/oracle_db.py @@ -50,7 +50,6 @@ def __init__(self, host: str, port: int, service: str, user: str, pwd: str, self.pool = self.set_up_session_pool() - def set_up_session_pool(self): try: dsn_str = cx_Oracle.makedsn(self.host, self.port, service_name=self.service) From e3bc4524373cd84422903dc557ce3bd212640cf8 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 25 Mar 2021 15:21:37 -0400 Subject: [PATCH 33/37] update docs --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index aea1d7f..1215938 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ a tool for facilitating connection to databases with python and performing basic In a suitable python3 (>=3.7) virtual env, using pip: ``` - pip install https://github.com/huit/pydb/archive/v0.0.2.tar.gz#egg=pydb + pip install https://github.com/huit/pydb/archive/refs/tags/v0.0.2.tar.gz#egg=pydb # import the module for the specific type of db you'd like to use from pydb.oracle_db import OracleDB ``` @@ -56,31 +56,32 @@ Given a valid connection, and a table called `EMP`... To query the `EMP` table for all records: ``` - result = db.execute_query("select * from emp") +result = db.execute_query("select * from emp") ``` To query the `EMP` table for a specific record: ``` - result = db.execute_query("select * from emp where ename= :ename", {'ename':'JOHNSON'}) +result = db.execute_query("select * from emp where ename= :ename", {'ename':'JOHNSON'}) ``` Results for the individual rows would be in the following form: ``` - {'EMPNO': 7935, 'ENAME': 'JOHNSON', 'JOB': 'CLERK', 'MGR': 7839, 'HIREDATE': datetime.datetime(1981, 5, 1, 0, 0), 'SAL': 2850.0, 'COMM': None, 'DEPTNO': 30} +{'EMPNO': 7935, 'ENAME': 'JOHNSON', 'JOB': 'CLERK', 'MGR': 7839, 'HIREDATE': datetime.datetime(1981, 5, 1, 0, 0), 'SAL': 2850.0, 'COMM': None, 'DEPTNO': 30} ``` Row results may vary somewhat depending on the exact module... e.g., for SqlAlchemyOracleDB the following would be received: ``` - {'empno': 7935, 'ename': 'JOHNSON', 'job': 'CLERK', 'mgr': 7839, 'hiredate': datetime.datetime(1981, 5, 1, 0, 0), 'sal': Decimal('2850'), 'comm': None, 'deptno': 30} +{'empno': 7935, 'ename': 'JOHNSON', 'job': 'CLERK', 'mgr': 7839, 'hiredate': datetime.datetime(1981, 5, 1, 0, 0), 'sal': Decimal('2850'), 'comm': None, 'deptno': 30} ``` ### integrated example ``` -pip install https://github.com/huit/pydb/archive/v0.0.2.tar.gz#egg=pydb +pip install https://github.com/huit/pydb/archive/refs/tags/v0.0.2.tar.gz#egg=pydb from pydb.oracle_db import OracleDB db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="username", pwd="pwd") result = db.execute_query("select * from emp where ename= :ename", {'ename':'JOHNSON'}) print(result) - +``` +produces +``` {'EMPNO': 7935, 'ENAME': 'JOHNSON', 'JOB': 'CLERK', 'MGR': 7839, 'HIREDATE': datetime.datetime(1981, 5, 1, 0, 0), 'SAL': 2850.0, 'COMM': None, 'DEPTNO': 30} - ``` \ No newline at end of file From ce67314cfe1d9a976101d33963becc304baebbc9 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 25 Mar 2021 15:21:52 -0400 Subject: [PATCH 34/37] cleanup per PR feedback --- pydb/database.py | 15 --------------- pydb/oracle_db.py | 23 ++++++----------------- pydb/sql_alchemy_oracle_db.py | 2 +- 3 files changed, 7 insertions(+), 33 deletions(-) diff --git a/pydb/database.py b/pydb/database.py index c40b38b..1cb84f8 100644 --- a/pydb/database.py +++ b/pydb/database.py @@ -12,21 +12,6 @@ class DatabaseType(Enum): class DBInterface(metaclass=abc.ABCMeta): - @classmethod - def __subclasshook__(cls, subclass): - return (hasattr(subclass, 'execute_query') and - callable(subclass.execute_query) and - hasattr(subclass, 'execute_update') and - callable(subclass.execute_update) and - hasattr(subclass, 'health_check') and - callable(subclass.health_check) and - hasattr(subclass, 'cleanup') and - callable(subclass.cleanup) and - hasattr(subclass, 'create_connection') and - callable(subclass.create_connection) and - hasattr(subclass, 'get_session') and - callable(subclass.get_session) - or NotImplemented) @abc.abstractmethod def execute_query(self, query_string: str, args=None) -> list: diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py index 61ff52f..c282ab0 100644 --- a/pydb/oracle_db.py +++ b/pydb/oracle_db.py @@ -48,7 +48,7 @@ def __init__(self, host: str, port: int, service: str, user: str, pwd: str, self.pwd = pwd self.logger = get_common_logger_for_module(module_name=__name__, level=logging_level, log_format=logging_format) - self.pool = self.set_up_session_pool() + self._pool = self.set_up_session_pool() def set_up_session_pool(self): try: @@ -63,7 +63,6 @@ def set_up_session_pool(self): threaded=True, encoding="UTF-8" ) - self.pool = pool return pool except cx_Oracle.DatabaseError as err: @@ -73,22 +72,12 @@ def set_up_session_pool(self): self.logger.error("Message: %s", obj.message) raise Exception(f"Error creating pool: {obj.message}") - def get_session_pool(self): - """ - Function for creating a session pool with the database - """ - if self.pool: - return self.pool - else: - self.set_up_session_pool() - def create_connection(self): """ Function for creating a connection with the database from a session pool """ try: - connection = self.get_session_pool().acquire() - return connection + return self._pool.acquire() except cx_Oracle.DatabaseError as err: obj, = err.args @@ -132,7 +121,7 @@ def execute_query(self, query_string: str, args=None) -> dict: finally: cursor.close() - self.get_session_pool().release(connection) + self._pool.release(connection) def execute_update(self, query_string, args=None): """ @@ -156,7 +145,7 @@ def execute_update(self, query_string, args=None): finally: cursor.close() - self.get_session_pool().release(connection) + self._pool.release(connection) def health_check(self): """ @@ -166,10 +155,10 @@ def health_check(self): return self.execute_query("SELECT 1 FROM DUAL") def cleanup(self): - if self.pool is not None: + if self._pool is not None: self.logger.info("Active session pool found. Attempting to close session pool.") try: - self.pool.close(force=True) + self._pool.close(force=True) self.logger.info("Session pool successfully closed.") except cx_Oracle.Error as err: diff --git a/pydb/sql_alchemy_oracle_db.py b/pydb/sql_alchemy_oracle_db.py index 25f4ebe..ab87054 100644 --- a/pydb/sql_alchemy_oracle_db.py +++ b/pydb/sql_alchemy_oracle_db.py @@ -9,7 +9,7 @@ from pylog.pylog import get_common_logger_for_module -class SqlAlchemyOracleDB: +class SqlAlchemyOracleDB(DBInterface): """ Module for interacting with an Oracle DB via SQL Alchemy """ From aa683cee9f525e95d5e23f902def775ed979c7f8 Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 25 Mar 2021 16:16:38 -0400 Subject: [PATCH 35/37] add import --- pydb/sql_alchemy_oracle_db.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pydb/sql_alchemy_oracle_db.py b/pydb/sql_alchemy_oracle_db.py index ab87054..4d2e113 100644 --- a/pydb/sql_alchemy_oracle_db.py +++ b/pydb/sql_alchemy_oracle_db.py @@ -8,6 +8,8 @@ from pylog.pylog import get_common_logger_for_module +from .database import DBInterface + class SqlAlchemyOracleDB(DBInterface): """ From 1a9ea4144aa328fcb8a7c823d139ce2455b4fa0b Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 25 Mar 2021 18:06:13 -0400 Subject: [PATCH 36/37] update readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1215938..309fd17 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,9 @@ a tool for facilitating connection to databases with python and performing basic ## Purpose and intended audience - +For many APIs the responses are based upon data residing in a relational database; this module provides a standardized +way for python programmers to access a database, via straight SQL calls to `execute_query()` and `execute_update()` and +also permits users to access SQLAlchemy for ORM-based interactions with database objects. ## Requirements From 1318bb9d3f0f913f4a0c59e4de05ec39c6e649bb Mon Sep 17 00:00:00 2001 From: Michael Kerry Date: Thu, 25 Mar 2021 18:22:38 -0400 Subject: [PATCH 37/37] Update README.md --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 309fd17..c9293c3 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,13 @@ db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="userna Specific to SQL Alchemy; allows interaction with SQL Alchemy entities see https://docs.sqlalchemy.org/en/14/orm/session.html?highlight=session#module-sqlalchemy.orm.session - execute_query(self, query_string: str, args=None) -> list + execute_query(self, query_string: str, args: dict = None) -> list executes a sql query, return a list of dictionaries representing rows + 'args' represents a dictionary of parameterized values for the query; see examples below - execute_update(self, query_string, args=None) + execute_update(self, query_string, args: dict = None) used to execute an insert, update, or delete sql statement + 'args' represents a dictionary of parameterized values for the query; see examples below health_check() Performs a basic query against the db to ensure connectivity @@ -86,4 +88,4 @@ print(result) produces ``` {'EMPNO': 7935, 'ENAME': 'JOHNSON', 'JOB': 'CLERK', 'MGR': 7839, 'HIREDATE': datetime.datetime(1981, 5, 1, 0, 0), 'SAL': 2850.0, 'COMM': None, 'DEPTNO': 30} -``` \ No newline at end of file +```