From e41508844fff9282521554c68fa09381fd9abbda Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Sun, 27 Nov 2022 08:30:44 -0700 Subject: [PATCH 01/15] Working to catch more API exceptions renamed: johnnyfive/old_email.py -> ToyModels/old_email.py modified: johnnyfive/gmail.py modified: johnnyfive/utils.py --- {johnnyfive => ToyModels}/old_email.py | 0 johnnyfive/gmail.py | 43 ++++++++++++++++---------- johnnyfive/utils.py | 31 ++++++++++++++----- 3 files changed, 49 insertions(+), 25 deletions(-) rename {johnnyfive => ToyModels}/old_email.py (100%) diff --git a/johnnyfive/old_email.py b/ToyModels/old_email.py similarity index 100% rename from johnnyfive/old_email.py rename to ToyModels/old_email.py diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index 89dbb2a..b680d50 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -25,12 +25,12 @@ # 3rd Party Libraries from bs4 import BeautifulSoup -from googleapiclient.discovery import build -from googleapiclient.errors import UnknownApiNameOrVersion -from googleapiclient.errors import HttpError -from google_auth_oauthlib.flow import InstalledAppFlow -from google.auth.transport.requests import Request -from google.oauth2.credentials import Credentials +import googleapiclient.discovery +import googleapiclient.errors +import google_auth_oauthlib.flow +import google.auth.exceptions +import google.auth.transport.requests +import google.oauth2.credentials # Internal Imports from johnnyfive import utils @@ -160,7 +160,7 @@ def send(self): .send(userId="me", body=sendable_message) .execute ) - except (HttpError, ConnectionError) as error: + except (googleapiclient.errors.HttpError, ConnectionError) as error: warnings.warn(f"An error occurred within GmailMessage.send():\n{error}") return None @@ -207,7 +207,7 @@ def __init__(self, label=None, after=None, before=None, interactive=False): .execute ) self.message_list = results.get("messages", []) - except (HttpError, ConnectionError) as error: + except (googleapiclient.errors.HttpError, ConnectionError) as error: warnings.warn( f"An error occurred within GetMessages.__init__():\n{error}" ) @@ -238,7 +238,7 @@ def render_message(self, message_id): headers = payload["headers"] # If exception, print message and return empty values - except (HttpError, ConnectionError) as error: + except (googleapiclient.errors.HttpError, ConnectionError) as error: warnings.warn( f"An error occurred within GetMessages.render_message():\n{error}" ) @@ -318,7 +318,7 @@ def update_msg_labels(self, message_id, add_labels=None, remove_labels=None): .execute ) # If exception, print message - except (HttpError, ConnectionError) as error: + except (googleapiclient.errors.HttpError, ConnectionError) as error: warnings.warn( f"An error occurred within GetMessages.update_msg_labels():\n{error}" ) @@ -351,7 +351,7 @@ def _lableId_from_labelName(self, name): self.service.users().labels().list(userId="me").execute ) self.label_list = results.get("labels", []) - except (HttpError, ConnectionError) as error: + except (googleapiclient.errors.HttpError, ConnectionError) as error: warnings.warn( f"An error occurred within GetMessages._labelId_from_labelName():\n{error}" ) @@ -394,7 +394,9 @@ def setup_gmail(interactive=False): # Read in the credential token creds = None if os.path.exists(token_fn := utils.Paths.gmail_token): - creds = Credentials.from_authorized_user_file(token_fn, SCOPES) + creds = google.oauth2.credentials.Credentials.from_authorized_user_file( + token_fn, SCOPES + ) # If there are no (valid) credentials available... if not creds or not creds.valid: @@ -402,13 +404,17 @@ def setup_gmail(interactive=False): # If just expired, refresh and move on if creds and creds.expired and creds.refresh_token: try: - utils.safe_service_connect(creds.refresh, Request()) - except (HttpError, ConnectionError) as error: + utils.safe_service_connect( + creds.refresh, google.auth.transport.requests.Request() + ) + except (googleapiclient.errors.HttpError, ConnectionError) as error: warnings.warn(f"An error occurred within setup_gmail():\n{error}") + except google.auth.exceptions.RefreshError as error: + warnings.warn(f"Some sort of refresh error occurred:\n{error}") # If running in `interactive`, lauch browser to log in elif interactive: - flow = InstalledAppFlow.from_client_secrets_file( + flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file( utils.Paths.gmail_creds, SCOPES ) creds = flow.run_local_server(port=0) @@ -429,8 +435,11 @@ def setup_gmail(interactive=False): # Try building the GMail API service. If error, print error & return None try: # Call the Gmail API - return build("gmail", "v1", credentials=creds) - except (HttpError, UnknownApiNameOrVersion) as error: + return googleapiclient.discovery.build("gmail", "v1", credentials=creds) + except ( + googleapiclient.errors.HttpError, + googleapiclient.errors.UnknownApiNameOrVersion, + ) as error: # TODO(developer) - Handle errors from gmail API. warnings.warn(f"An error occurred within setup_gmail():\n{error}") return None diff --git a/johnnyfive/utils.py b/johnnyfive/utils.py index d75ed39..9cb07d1 100644 --- a/johnnyfive/utils.py +++ b/johnnyfive/utils.py @@ -24,12 +24,13 @@ import warnings # 3rd Party Libraries -from googleapiclient.errors import HttpError -from google.auth.exceptions import TransportError +import atlassian.errors +import googleapiclient.errors +import google.auth.exceptions import httplib2 from pkg_resources import resource_filename import requests -from slack_sdk.errors import SlackApiError +import slack_sdk.errors # Lowell Libraries import ligmos @@ -220,7 +221,7 @@ def safe_service_connect(func, *args, pause=5, nretries=5, **kwargs): # This is a network error... retry except ( ConnectionError, - TransportError, + google.auth.exceptions.TransportError, httplib2.error.ServerNotFoundError, ) as exception: print( @@ -245,16 +246,30 @@ def safe_service_connect(func, *args, pause=5, nretries=5, **kwargs): break # Gmail service error, no retry and pass the exception upward - except HttpError as exception: + except googleapiclient.errors.HttpError as exception: warnings.warn( - f"Caught Gmail error... passing up. {type(exception).__name__}" + f"Caught Gmail HTTP error... passing up. {type(exception).__name__}" ) raise exception # Slack service error, no retry and pass the exception upward - except SlackApiError as exception: + except slack_sdk.errors.SlackApiError as exception: warnings.warn( - f"Caught Slack error... passing up. {type(exception).__name__}" + f"Caught Slack API error... passing up. {type(exception).__name__}" + ) + raise exception + + # =========================================== + # Specific service errors + except atlassian.errors.ApiError as exception: + warnings.warn( + f"Caught Atlassian API Error... passing up. {type(exception).__name__}" + ) + raise exception + + except google.auth.exceptions.RefreshError as exception: + warnings.warn( + f"Caught Google Refresh Error... passing up. {type(exception).__name__}" ) raise exception From f32b5c13a8a0a89cc2bf03403296a32f0ec69955 Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Mon, 28 Nov 2022 15:24:00 -0700 Subject: [PATCH 02/15] Working modified: johnnyfive/confluence.py modified: johnnyfive/gmail.py modified: johnnyfive/slack.py modified: johnnyfive/utils.py modified: setup.py --- johnnyfive/confluence.py | 176 ++++++++++++++--------------- johnnyfive/gmail.py | 235 ++++++++++++++++++++++++--------------- johnnyfive/slack.py | 12 +- johnnyfive/utils.py | 90 +++++++++------ setup.py | 2 +- 5 files changed, 302 insertions(+), 213 deletions(-) diff --git a/johnnyfive/confluence.py b/johnnyfive/confluence.py index 89f89c2..421c9e1 100644 --- a/johnnyfive/confluence.py +++ b/johnnyfive/confluence.py @@ -18,11 +18,11 @@ import warnings # 3rd Party Libraries -from atlassian import Confluence +import atlassian import requests # Internal Imports -from johnnyfive import utils +import johnnyfive.utils # Set API Components @@ -36,31 +36,31 @@ class ConfluencePage: Parameters ---------- - space : `str` + space : str The name of the Confluence space for this page - page_title : `str` + page_title : str The page title - instance : ``, optional - An existing Confluence influence, to be used in the case of many - instances of this class used in short order [Default: None] + instance : :class:`atlassian.Confluence`, optional + An existing Confluence object instance to be used instead of + reinstantiating a new Confluence object for communication and + authentication. [Default: None] """ def __init__(self, space, page_title, instance=None, use_oauth=False): self.space = space self.title = page_title - self.instance = ( + self.confluence = ( setup_confluence(use_oauth=use_oauth) - if not isinstance(instance, Confluence) + if not isinstance(instance, atlassian.Confluence) else instance ) - self.uname = self.instance.username self.space_perms = self._set_permdict() # Set the class metadata based on this page self._set_metadata() def add_comment(self, comment): - """add_comment Add a comment to the Confluence page + """Add a comment to the Confluence page Sometimes it's helpful to include a comment at the bottom of the Confluence page. These will be signed by Nanni. This method adds @@ -68,16 +68,18 @@ def add_comment(self, comment): Parameters ---------- - comment : `str` + comment : str The comment to be left on the page. """ if not self._check_perm("COMMENT", "add a comment"): return - utils.safe_service_connect(self.instance.add_comment, self.page_id, comment) + johnnyfive.utils.safe_service_connect( + self.confluence.add_comment, self.page_id, comment + ) def add_label(self, label): - """add_label Add a label to the Confluence page + """Add a label to the Confluence page Sometimes it's helpful to have a label on a Confluence page for searching or sorting. This method adds such to a page. @@ -90,30 +92,32 @@ def add_label(self, label): if not self._check_perm("EDITSPACE", "add a label"): return - utils.safe_service_connect(self.instance.set_page_label, self.page_id, label) + johnnyfive.utils.safe_service_connect( + self.confluence.set_page_label, self.page_id, label + ) def attach_file(self, filename, name=None, content_type=None, comment=None): - """attach_file Attach a file to this page + """Attach a file to this page Wrapper for the Confluence method attach_file() that includes the page ID of this object and is wrapped in utils.safe_service_connect(). Parameters ---------- - filename : `str` + filename : str Filename of the attachment - name : `str`, optional + name : str, optional Display name for this attachment [Default: None] - content_type : `str`, optional + content_type : str, optional MIME content type [Default: None] - comment : `str`, optional + comment : str, optional Additional comment or description to be included [Default: None] """ if not self._check_perm("CREATEATTACHMENT", "create an attachment"): return - utils.safe_service_connect( - self.instance.attach_file, + johnnyfive.utils.safe_service_connect( + self.confluence.attach_file, filename, name=name, content_type=content_type, @@ -122,17 +126,17 @@ def attach_file(self, filename, name=None, content_type=None, comment=None): ) def create(self, page_body, parent_id=None): - """create Create a brand new Confluence page + """Create a brand new Confluence page Summon from the depths of computing a new page. Parameters ---------- - page_body : `str` + page_body : str The body of the new Confluence page. - parent_id : `str`, optional + parent_id : str, optional The parent page to place this under. If none given, the new page - will be created at the root of `self.space`. [Default: None] + will be created at the root of ``self.space``. [Default: None] """ if not self._check_perm("EDITSPACE", "create a page"): return @@ -142,8 +146,8 @@ def create(self, page_body, parent_id=None): print("Can't create a page that already exists!") return - utils.safe_service_connect( - self.instance.create_page, + johnnyfive.utils.safe_service_connect( + self.confluence.create_page, self.space, self.title, page_body, @@ -155,58 +159,58 @@ def create(self, page_body, parent_id=None): self._set_metadata() def delete_attachment(self, filename): - """delete_attachment Delete an attachment from this page + """Delete an attachment from this page Wrapper for the Confluence method delete_attachment() that includes the page ID of this object and is wrapped in utils.safe_service_connect(). An attahment may be deleted using either the filename, or by passing - `None` or "" to the filename and specifying the attachment_id. + ``None`` or "" to the filename and specifying the attachment_id. Parameters ---------- - filename : `str` + filename : str Filename of the attachment to delete """ if not self._check_perm("REMOVEATTACHMENT", "remove an attachment"): return - utils.safe_service_connect( - self.instance.delete_attachment, self.page_id, filename + johnnyfive.utils.safe_service_connect( + self.confluence.delete_attachment, self.page_id, filename ) def get_page_attachments(self, limit=200): - """get_page_attachments _summary_ + """Retrieve the page attachments - Return a list of page attachment IDs, up to `limit` in length. + Return a list of page attachment IDs, up to ``limit`` in length. Parameters ---------- - limit : `int`, optional + limit : int, optional The number of attachments to return [Default: 200] Returns ------- - `list` + list List of Confluence attachment IDs """ - return utils.safe_service_connect( - self.instance.get_attachments_from_content, self.page_id, limit=limit + return johnnyfive.utils.safe_service_connect( + self.confluence.get_attachments_from_content, self.page_id, limit=limit ) def get_page_contents(self): - """get_page_contents Retrieve the page contents in HTML-ish format + """Retrieve the page contents in HTML-ish format Either for curiosity or for modification, get the page contents, which live in the `body.storage` portion of the `get_page_by_id` response. Returns ------- - `str` + str The HTML-ish body of the confluence page. """ - contents = utils.safe_service_connect( - self.instance.get_page_by_id, self.page_id, expand="body.storage" + contents = johnnyfive.utils.safe_service_connect( + self.confluence.get_page_by_id, self.page_id, expand="body.storage" ) # Extract the contents from the return object return contents["body"]["storage"]["value"] @@ -220,37 +224,32 @@ def smite(self): if not self._check_perm("REMOVEPAGE", "remove a page"): return - utils.safe_service_connect(self.instance.remove_page, self.page_id) + johnnyfive.utils.safe_service_connect(self.confluence.remove_page, self.page_id) self._set_metadata() def update_contents(self, body): - """update_contents Update the contents of the Confluence page + """Update the contents of the Confluence page Update the page by replacing the existing content with new. The idea - for this method is to be used in concert with `get_page_contents` to - obtain a page, modify it, then replace it. + for this method is to be used in concert with :func:``get_page_contents`` + to obtain a page, modify it, then replace it. Parameters ---------- - body : `str` + body : str The new page contents to upload to Confluence. - - Returns - ------- - _type_ - _description_ """ if not self._check_perm("EDITSPACE", "update a page"): return - utils.safe_service_connect( - self.instance.update_page, self.page_id, self.title, body + johnnyfive.utils.safe_service_connect( + self.confluence.update_page, self.page_id, self.title, body ) def _check_perm(self, perm_key, perm_action): - """_check_perm Check the perm_dict for a particular action + """Check the premissions dictionary for a particular action - Check the `perm_ley` in the permissions dictionary to see whether the + Check the ``perm_key`` in the permissions dictionary to see whether the requested action is permitted. The wrinkle is that if the user does not have permission to see @@ -261,14 +260,14 @@ def _check_perm(self, perm_key, perm_action): Parameters ---------- - perm_key : `str` + perm_key : str The key in perm_dict to look for - perm_action : `str` + perm_action : str The action that is requested by the calling function. Returns ------- - `bool` + bool True for perform action, False for not """ perm_val = self.space_perms.get(perm_key, None) @@ -276,9 +275,9 @@ def _check_perm(self, perm_key, perm_action): # If the value is explicitely False, warn as such if perm_val is False: warnings.warn( - f"User {self.uname} does not have permission " + f"User {self.confluence.username} does not have permission " f"to {perm_action} in space {self.space}.", - utils.PermissionWarning, + johnnyfive.utils.PermissionWarning, ) return False @@ -286,58 +285,58 @@ def _check_perm(self, perm_key, perm_action): if perm_val is None: warnings.warn( "Permissions check is disabled... hoping for the best.", - utils.PermissionWarning, + johnnyfive.utils.PermissionWarning, ) return True def _set_metadata(self): - """_set_metadata Set the various instance metadata + """Set the various instance metadata Especially after a page is created or deleted, this method updates the various instance attributes to keep current. """ - self.exists = utils.safe_service_connect( - self.instance.page_exists, self.space, self.title + self.exists = johnnyfive.utils.safe_service_connect( + self.confluence.page_exists, self.space, self.title ) # Page-Specific Information self.page_id = ( None if not self.exists - else utils.safe_service_connect( - self.instance.get_page_id, self.space, self.title + else johnnyfive.utils.safe_service_connect( + self.confluence.get_page_id, self.space, self.title ) ) self.attachment_url = ( None if not self.exists - else f"{self.instance.url}download/attachments/{self.page_id}/" + else f"{self.confluence.url}download/attachments/{self.page_id}/" ) def _set_permdict(self): - """_set_permdict Create a dictionary of permissions + """Create a dictionary of permissions This method creates a dictionary of permissions for this user in this space. Each item in the dictionary is boolean based on the results of - the method confluence.get_space_permissions(). + the method :func:`confluence.get_space_permissions`. Returns ------- - `dict` + dict The dictionary of permissions (boolean) """ - perms = utils.safe_service_connect( - self.instance.get_space_permissions, self.space + perms = johnnyfive.utils.safe_service_connect( + self.confluence.get_space_permissions, self.space ) # Check to see if the authenticated user can view permissions if not perms: warnings.warn( - f"User {self.uname} needs permission to view " + f"User {self.confluence.username} needs permission to view " f"permissions in space {self.space}. Contact " "your Confluence administrator.", - utils.PermissionWarning, + johnnyfive.utils.PermissionWarning, ) perm_dict = {} @@ -345,7 +344,7 @@ def _set_permdict(self): # Set this permission as false... will update to True if needed perm_dict[perm["type"]] = False for space_perm in perm["spacePermissions"]: - if space_perm["userName"] == self.uname: + if space_perm["userName"] == self.confluence.username: perm_dict[perm["type"]] = True return perm_dict @@ -353,32 +352,35 @@ def _set_permdict(self): # Internal Functions =========================================================# def setup_confluence(use_oauth=False): - """setup_confluence Set up the Confluence class instance + """Set up the Confluence class instance Reads in the confluence.conf configuration file, which contains the URL, username, and password (and/or OAUTH info). - NOTE: For Confluence install version >= 7.9, can use OAUTH for - authentication instead of username/password. + .. note:: + For Confluence install version >= 7.9, can use OAUTH for + authentication instead of username/password. Parameters ---------- - use_oauth : `bool`, optional + use_oauth : bool, optional Use the OAUTH authentication scheme? [Default: False] Returns ------- - confluence : `atlassian.Confluence` + confluence : :class:`atlassian.Confluence` Confluence class, initialized with credentials """ # Read the setup - setup = utils.read_ligmos_conffiles("confluenceSetup") + setup = johnnyfive.utils.read_ligmos_conffiles("confluenceSetup") # If we are using OAUTH, instantiate a Confluence object with it if use_oauth: - s = requests.Session() - s.headers["Authorization"] = f"Bearer {setup.access_token}" - return Confluence(url=setup.host, session=s) + session = requests.Session() + session.headers["Authorization"] = f"Bearer {setup.access_token}" + return atlassian.Confluence(url=setup.host, session=session) # Else, return a Confluence object instantiated with username/password - return Confluence(url=setup.host, username=setup.user, password=setup.password) + return atlassian.Confluence( + url=setup.host, username=setup.user, password=setup.password + ) diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index b680d50..9993394 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -18,7 +18,7 @@ # Built-In Libraries import base64 -from email import mime +import email.mime import mimetypes import os import warnings @@ -33,7 +33,7 @@ import google.oauth2.credentials # Internal Imports -from johnnyfive import utils +import johnnyfive.utils # This scope is for sending email using the OAuth2 library @@ -45,7 +45,7 @@ class GmailMessage: - """GmailMessage Class for a single Gmail Message + """Class for a single Gmail Message _extended_summary_ @@ -62,6 +62,7 @@ class GmailMessage: Display Name of the sender (i.e. which bot) [Default: None] fromaddr : `str`, optional Sender email address [Default: Value from [gmailSetup]] + logger : """ def __init__( @@ -72,31 +73,35 @@ def __init__( fromname=None, fromaddr=None, interactive=False, + logger=None, ): + # Set the logger, if passed + self.logger = logger + # Load default `fromaddr`` if None passed in if not fromaddr: - fromaddr = utils.read_ligmos_conffiles("gmailSetup").user + fromaddr = johnnyfive.utils.read_ligmos_conffiles("gmailSetup").user # Initialize the Gmail connection - self.service = setup_gmail(interactive=interactive) + self.service = setup_gmail(interactive=interactive, logger=self.logger) # Build the container for a multipart MIME message - self.message = mime.multipart.MIMEMultipart() + self.message = email.mime.multipart.MIMEMultipart() self.message["to"] = toaddr if isinstance(toaddr, str) else ",".join(toaddr) self.message["from"] = f"{fromname} <{fromaddr}>" if fromname else fromaddr self.message["subject"] = subject # Place the text into the message - self.message.attach(mime.text.MIMEText(message_text)) + self.message.attach(email.mime.text.MIMEText(message_text)) def add_attachment(self, file): - """add_attachment _summary_ + """Add an attachment to the GMAIL message _extended_summary_ Parameters ---------- - file : `str` + file : str Filename of the attachment """ # For the attachment, guess the MIME type for reading it in @@ -110,16 +115,16 @@ def add_attachment(self, file): main_type, sub_type = content_type.split("/", 1) if main_type == "text": with open(file, "rb") as fp: - attachment = mime.text.MIMEText(fp.read(), _subtype=sub_type) + attachment = email.mime.text.MIMEText(fp.read(), _subtype=sub_type) elif main_type == "image": with open(file, "rb") as fp: - attachment = mime.image.MIMEImage(fp.read(), _subtype=sub_type) + attachment = email.mime.image.MIMEImage(fp.read(), _subtype=sub_type) elif main_type == "audio": with open(file, "rb") as fp: - attachment = mime.audio.MIMEAudio(fp.read(), _subtype=sub_type) + attachment = email.mime.audio.MIMEAudio(fp.read(), _subtype=sub_type) else: with open(file, "rb") as fp: - attachment = mime.base.MIMEBase(main_type, sub_type) + attachment = email.mime.base.MIMEBase(main_type, sub_type) attachment.set_payload(fp.read()) # Add the attachment to the email message @@ -129,7 +134,7 @@ def add_attachment(self, file): self.message.attach(attachment) def send(self): - """send Send the GmailMessage + """Send the GmailMessage _extended_summary_ @@ -138,7 +143,7 @@ def send(self): n_tries : `int`, optional The number of retry attemps at sending this message [Default: 5] - Returns + Returns ------- `dict` The sent message object @@ -150,52 +155,61 @@ def send(self): # If Gmail `Resource` was not returned earlier, try again if not self.service: - self.service = setup_gmail() + self.service = setup_gmail(logger=self.logger) # Try to send the message (API: users.messages.send) try: - return utils.safe_service_connect( + return johnnyfive.utils.safe_service_connect( self.service.users() .messages() .send(userId="me", body=sendable_message) - .execute + .execute, + logger=self.logger, ) except (googleapiclient.errors.HttpError, ConnectionError) as error: - warnings.warn(f"An error occurred within GmailMessage.send():\n{error}") + johnnyfive.utils.proper_print( + f"An error occurred within GmailMessage.send(): {error}", + "except", + self.logger, + ) return None class GetMessages: - """GetMessages Get Gmail messages corresponding to given criteria + """Get Gmail messages corresponding to given criteria _extended_summary_ Parameters ---------- - label : `str`, optional + label : str, optional The Gmail label of messages to find [Default: None] - after : `str`, optional + after : str, optional Date after which to search for messages. Must be in YYYY/MM/DD format. [Default: None] - before : `str`, optional + before : str, optional Date before which to search for messages. Must be in YYYY/MM/DD format. [Default: None] + logger : """ - def __init__(self, label=None, after=None, before=None, interactive=False): + def __init__( + self, label=None, after=None, before=None, interactive=False, logger=None + ): # Initialize basic stuff self.label_list = None self.message_list = [] + self.logger = logger # Initialize the Gmail connection - self.service = setup_gmail(interactive=interactive) + self.service = setup_gmail(interactive=interactive, logger=self.logger) self.label_id = self._lableId_from_labelName(label) - self.query = build_query(after_date=after, before_date=before) + self.query = self.build_query(after_date=after, before_date=before) # Get the list of matching messages (API: users.messages.list) if self.label_id: try: - results = utils.safe_service_connect( + results = johnnyfive.utils.safe_service_connect( self.service.users() .messages() .list( @@ -204,43 +218,49 @@ def __init__(self, label=None, after=None, before=None, interactive=False): q=self.query, maxResults=500, ) - .execute + .execute, + logger=self.logger, ) self.message_list = results.get("messages", []) except (googleapiclient.errors.HttpError, ConnectionError) as error: - warnings.warn( - f"An error occurred within GetMessages.__init__():\n{error}" + johnnyfive.utils.proper_print( + f"An error occurred within GetMessages.__init__(): {error}", + "except", + self.logger, ) def render_message(self, message_id): - """render_message Retrieve and render a message by ID# + """Retrieve and render a message by ID# Gmail mnessages are stored in a JSON-like structure that must be parsed out to get the tasty nougat center. Parameters ---------- - message_id : `str` - The ['id'] field of an entry in self.message_list + message_id : str + The ``['id']`` field of an entry in self.message_list Returns ------- - `dict` + dict Dictionary containing the subject, sender, date, and body of the message. """ try: # Get the message, then start parsing (API: users.messages.get) - results = utils.safe_service_connect( - self.service.users().messages().get(userId="me", id=message_id).execute + results = johnnyfive.utils.safe_service_connect( + self.service.users().messages().get(userId="me", id=message_id).execute, + logger=self.logger, ) payload = results["payload"] headers = payload["headers"] # If exception, print message and return empty values except (googleapiclient.errors.HttpError, ConnectionError) as error: - warnings.warn( - f"An error occurred within GetMessages.render_message():\n{error}" + johnnyfive.utils.proper_print( + f"An error occurred within GetMessages.render_message(): {error}", + "except", + self.logger, ) payload = None @@ -271,26 +291,26 @@ def render_message(self, message_id): return dict(subject=subject, sender=sender, date=date, body=body) def update_msg_labels(self, message_id, add_labels=None, remove_labels=None): - """update_msg_labels Update the labels for a message by ID# + """Update the labels for a message by ID# _extended_summary_ Parameters ---------- - message_id : `str` - The ['id'] field of an entry in self.message_list - add_labels : `list`, optional + message_id : str + The ``['id']`` field of an entry in self.message_list + add_labels : list, optional The list of label IDs to add to this message [Default: None] - remove_labels : `list`, optional + remove_labels : list, optional The list of label IDs to remove from this message [Default: None] Returns ------- - `Any` + Any Uh, the Message object from Gmail... probably just return nothing? """ if not add_labels and not remove_labels: - print("No labels to change.") + johnnyfive.utils.proper_print("No labels to change.", "info", self.logger) return None # Convert Label Names to Label IDs @@ -311,33 +331,36 @@ def update_msg_labels(self, message_id, add_labels=None, remove_labels=None): try: # Modify message lables (API: users.messages.modify) - return utils.safe_service_connect( + return johnnyfive.utils.safe_service_connect( self.service.users() .messages() .modify(userId="me", id=message_id, body=body) - .execute + .execute, + logger=self.logger, ) # If exception, print message except (googleapiclient.errors.HttpError, ConnectionError) as error: - warnings.warn( - f"An error occurred within GetMessages.update_msg_labels():\n{error}" + johnnyfive.utils.proper_print( + f"An error occurred within GetMessages.update_msg_labels(): {error}", + "except", + self.logger, ) # If unsuccessful in connecting, return None return None def _lableId_from_labelName(self, name): - """_lableId_from_labelName Get the Label ID from the Label Name + """Get the Label ID from the Label Name _extended_summary_ Parameters ---------- - name : `str` + name : str Label name Returns ------- - `str` + str Label ID """ if not self.service: @@ -347,19 +370,24 @@ def _lableId_from_labelName(self, name): if not self.label_list: # Get the list of labels for the "me" account (API: users.labels.list) try: - results = utils.safe_service_connect( - self.service.users().labels().list(userId="me").execute + results = johnnyfive.utils.safe_service_connect( + self.service.users().labels().list(userId="me").execute, + logger=self.logger, ) self.label_list = results.get("labels", []) except (googleapiclient.errors.HttpError, ConnectionError) as error: - warnings.warn( - f"An error occurred within GetMessages._labelId_from_labelName():\n{error}" + johnnyfive.utils.proper_print( + f"An error occurred within GetMessages._labelId_from_labelName(): {error}", + "except", + self.logger, ) self.label_list = [] # If there are no labels, return None if not self.label_list: - print("Whoops, no labels found.") + johnnyfive.utils.proper_print( + "Whoops, no labels found.", "warn", self.logger + ) return None label_id = None @@ -370,10 +398,35 @@ def _lableId_from_labelName(self, name): return label_id + @staticmethod + def build_query(after_date=None, before_date=None): + """build_query Build the query string for users.messages.list + + _extended_summary_ + + Parameters + ---------- + after_date : `str` + Date after which to search for messages. + before_date : `str` + Date before which to search for messages. + + Returns + ------- + `str` + The appropriate query string + """ + q = "" + if after_date: + q = q + f" after:{after_date}" + if before_date: + q = q + f" before:{before_date}" + return q + # Newer OAUTH Routines =======================================================# -def setup_gmail(interactive=False): - """setup_gmail Initialize the GMail API (via OAuth) +def setup_gmail(interactive=False, logger=None): + """Initialize the GMail API (via OAuth) [extended_summary] @@ -382,18 +435,19 @@ def setup_gmail(interactive=False): Parameters ---------- - interactive : `bool`, optional + interactive : bool, optional Is this session interactive? Relates to how to deal with toke refresh. [Default: False] + logger : Returns ------- - `googleapiclient.discovery.Resource` + :obj:`googleapiclient.discovery.Resource` The GMail API service object for consumption by other routines """ # Read in the credential token creds = None - if os.path.exists(token_fn := utils.Paths.gmail_token): + if os.path.exists(token_fn := johnnyfive.utils.Paths.gmail_token): creds = google.oauth2.credentials.Credentials.from_authorized_user_file( token_fn, SCOPES ) @@ -404,18 +458,25 @@ def setup_gmail(interactive=False): # If just expired, refresh and move on if creds and creds.expired and creds.refresh_token: try: - utils.safe_service_connect( - creds.refresh, google.auth.transport.requests.Request() + johnnyfive.utils.safe_service_connect( + creds.refresh, + google.auth.transport.requests.Request(), + logger=logger, ) except (googleapiclient.errors.HttpError, ConnectionError) as error: - warnings.warn(f"An error occurred within setup_gmail():\n{error}") + johnnyfive.utils.proper_print( + f"An error occurred within setup_gmail(): {error}", "warn", logger + ) except google.auth.exceptions.RefreshError as error: - warnings.warn(f"Some sort of refresh error occurred:\n{error}") + johnnyfive.utils.proper_print( + f"Some sort of refresh error occurred:\n{error}", "warn", logger + ) # If running in `interactive`, lauch browser to log in elif interactive: + johnnyfive.utils.proper_print("If interactive...", "info", logger) flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file( - utils.Paths.gmail_creds, SCOPES + johnnyfive.utils.Paths.gmail_creds, SCOPES ) creds = flow.run_local_server(port=0) @@ -435,37 +496,35 @@ def setup_gmail(interactive=False): # Try building the GMail API service. If error, print error & return None try: # Call the Gmail API + johnnyfive.utils.proper_print("Calling the GMAIL API...", "info", logger) return googleapiclient.discovery.build("gmail", "v1", credentials=creds) except ( googleapiclient.errors.HttpError, googleapiclient.errors.UnknownApiNameOrVersion, ) as error: # TODO(developer) - Handle errors from gmail API. - warnings.warn(f"An error occurred within setup_gmail():\n{error}") + johnnyfive.utils.proper_print( + f"An error occurred within setup_gmail():\n{error}", "except", logger + ) return None -# Utility Functions ==========================================================# -def build_query(after_date=None, before_date=None): - """build_query Build the query string for users.messages.list +def authenticate_gmail(logger=None): + """Console Script for authenticating Gmail - _extended_summary_ + This is the command-line script for doing the interactive authentication + for Gmail needed to keep the tokens, etc. up to date. When/If there is + a ``RefreshError`` kicked by one of the classes in this module, this script + needs to be run on the command line `interactively` to remove the existing + token and re-authenticate the user via a web browser. - Parameters - ---------- - after_date : `str` - Date after which to search for messages. - before_date : `str` - Date before which to search for messages. + Console script:: + + j5_authenticate_gmail - Returns - ------- - `str` - The appropriate query string """ - q = "" - if after_date: - q = q + f" after:{after_date}" - if before_date: - q = q + f" before:{before_date}" - return q + johnnyfive.utils.proper_print("Authenticate GMail...", "info", logger) + # Remove the existing GMAIL TOKEN file, if extant... + johnnyfive.utils.Paths.gmail_token.unlink(missing_ok=True) + # Run setup + setup_gmail(interactive=True) diff --git a/johnnyfive/slack.py b/johnnyfive/slack.py index c23d4a0..96a8e36 100644 --- a/johnnyfive/slack.py +++ b/johnnyfive/slack.py @@ -24,7 +24,7 @@ import slack_sdk # Internal Imports -from johnnyfive import utils +import johnnyfive.utils # Set API Components @@ -66,7 +66,7 @@ def send_message(self, message): response = None try: # Call the conversations.list method using the WebClient - response = utils.safe_service_connect( + response = johnnyfive.utils.safe_service_connect( self.client.chat_postMessage, channel=self.channel_id, text=message @@ -99,7 +99,7 @@ def upload_file(self, file, title=None): """ response = None try: - response = utils.safe_service_connect( + response = johnnyfive.utils.safe_service_connect( self.client.files_upload, channels=self.channel_id, file=file, @@ -128,7 +128,9 @@ def _read_channels(self, name): try: # Call the conversations.list() method using the WebClient - result = utils.safe_service_connect(self.client.conversations_list) + result = johnnyfive.utils.safe_service_connect( + self.client.conversations_list + ) for _ in result: if conversation_id is not None: break @@ -160,7 +162,7 @@ def setup_slack(): The logging thingie """ # Read the setup - setup = utils.read_ligmos_conffiles("slackSetup") + setup = johnnyfive.utils.read_ligmos_conffiles("slackSetup") # SlackWebClient instantiates a client that can call API methods # When using Bolt, you can use either `app.client` or the `client` passed to listeners. diff --git a/johnnyfive/utils.py b/johnnyfive/utils.py index 9cb07d1..8080673 100644 --- a/johnnyfive/utils.py +++ b/johnnyfive/utils.py @@ -19,6 +19,7 @@ # Built-In Libraries import argparse import os +import pathlib import shutil import time import warnings @@ -39,7 +40,7 @@ # Set API Components -__all__ = ["PermissionWarning", "print_dict", "safe_service_connect"] +__all__ = ["PermissionWarning", "print_dict", "safe_service_connect", "proper_print"] class PermissionWarning(UserWarning): @@ -56,10 +57,10 @@ class Paths: """ # Main data & config directories - config = resource_filename("johnnyfive", "config") - images = resource_filename("johnnyfive", "images") - gmail_token = os.path.join(config, "gmail_token.json") - gmail_creds = os.path.join(config, "gmail_credentials.json") + config = pathlib.Path(resource_filename("johnnyfive", "config")) + images = pathlib.Path(resource_filename("johnnyfive", "images")) + gmail_token = config.joinpath("gmail_token.json") + gmail_creds = config.joinpath("gmail_credentials.json") class authTarget(ligmos.utils.classes.baseTarget): @@ -78,17 +79,6 @@ def __init__(self): self.tokenSecret = None -def authenticate_gmail(): - """authenticate_gmail Console Script for authenticating Gmail - - This will be a command-line script for doing the interactive authentication - for Gmail needed to keep the tokens, etc. up to date. - - TODO: Actually implement this function! - """ - print("Whee! We're going to authenticate gamil!") - - def install_conffiles(args=None): """install_conffiles Console Script for installing configuration files @@ -187,7 +177,7 @@ def print_dict(dd, indent=0, di=4): print(f"{' '*indent}{key:12s}: {value}") -def safe_service_connect(func, *args, pause=5, nretries=5, **kwargs): +def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs): """safe_service_connect Safely connect to Service (error-catching) Wrapper for Service-connection functions to catch errors that might be @@ -206,12 +196,15 @@ def safe_service_connect(func, *args, pause=5, nretries=5, **kwargs): nretries : `int`, optional The total number of times to retry connecting before returning None [Default: 10] + logger : Returns ------- `Any` The return value of `func` -- or None if unable to run `func` """ + + # Now, for the actual function... for i in range(1, nretries + 1): # Nominal function return @@ -224,12 +217,17 @@ def safe_service_connect(func, *args, pause=5, nretries=5, **kwargs): google.auth.exceptions.TransportError, httplib2.error.ServerNotFoundError, ) as exception: - print( - f"\nWarning: Execution of `{func.__name__}` failed because of:\n{exception}" + proper_print( + f"\nWarning: Execution of `{func.__name__}` failed because of:\n{exception}", + "warn", + logger, ) + if (i := i + 1) <= nretries: - print( - f"Waiting {pause} seconds before starting attempt #{i}/{nretries}" + proper_print( + f"Waiting {pause} seconds before starting attempt #{i}/{nretries}", + "info", + logger, ) time.sleep(pause) else: @@ -239,39 +237,67 @@ def safe_service_connect(func, *args, pause=5, nretries=5, **kwargs): # This is for a Service error (premissions, etc.), no retry except requests.exceptions.HTTPError as exception: - print( - f"\nWarning: Execution of `{func.__name__}` failed because of:\n{exception}" - "\nAborting..." + proper_print( + f"\nWarning: Execution of `{func.__name__}` failed because of:\n{exception}", + "except", + logger, ) + proper_print("Aborting...", "except", logger) break # Gmail service error, no retry and pass the exception upward except googleapiclient.errors.HttpError as exception: - warnings.warn( - f"Caught Gmail HTTP error... passing up. {type(exception).__name__}" + proper_print( + f"Caught Gmail HTTP error... passing up. {type(exception).__name__}", + "except", + logger, ) raise exception # Slack service error, no retry and pass the exception upward except slack_sdk.errors.SlackApiError as exception: - warnings.warn( - f"Caught Slack API error... passing up. {type(exception).__name__}" + proper_print( + f"Caught Slack API error... passing up. {type(exception).__name__}", + "except", + logger, ) raise exception # =========================================== # Specific service errors except atlassian.errors.ApiError as exception: - warnings.warn( - f"Caught Atlassian API Error... passing up. {type(exception).__name__}" + proper_print( + f"Caught Atlassian API Error... passing up. {type(exception).__name__}", + "except", + logger, ) raise exception except google.auth.exceptions.RefreshError as exception: - warnings.warn( - f"Caught Google Refresh Error... passing up. {type(exception).__name__}" + proper_print( + f"Caught Google Refresh Error... passing up. {type(exception).__name__}", + "except", + logger, ) raise exception # If not successful, return None return None + + +def proper_print(msg, level, logger=None): + if level == "info": + if logger is None: + print(msg) + else: + logger.info(msg) + elif level == "warn": + if logger is None: + warnings.warn(msg) + else: + logger.warning(msg) + elif level == "except": + if logger is None: + warnings.warn(f"EXCEPTION: {msg}") + else: + logger.exception(msg) diff --git a/setup.py b/setup.py index 9284d4d..0b01335 100644 --- a/setup.py +++ b/setup.py @@ -179,7 +179,7 @@ entry_points={ # Optional 'console_scripts': [ 'j5_install_conf=johnnyfive.utils:install_conffiles', - 'j5_authenticate_gmail=johnnyfive.utils:authenticate_gmail', + 'j5_authenticate_gmail=johnnyfive.gmail:authenticate_gmail', ] }, From a07419c343feed6cde7c80a6267d03252087fc08 Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Tue, 29 Nov 2022 11:55:06 -0700 Subject: [PATCH 03/15] Removed all ``print`` statements modified: johnnyfive/confluence.py modified: johnnyfive/gmail.py modified: johnnyfive/utils.py --- johnnyfive/confluence.py | 74 ++++++++++++++++++++-------- johnnyfive/gmail.py | 64 +++++++++++++----------- johnnyfive/utils.py | 104 +++++++++++++++++++++++---------------- 3 files changed, 151 insertions(+), 91 deletions(-) diff --git a/johnnyfive/confluence.py b/johnnyfive/confluence.py index 421c9e1..edff212 100644 --- a/johnnyfive/confluence.py +++ b/johnnyfive/confluence.py @@ -15,7 +15,6 @@ """ # Built-In Libraries -import warnings # 3rd Party Libraries import atlassian @@ -44,13 +43,21 @@ class ConfluencePage: An existing Confluence object instance to be used instead of reinstantiating a new Confluence object for communication and authentication. [Default: None] + use_oauth : bool, optional + Use OAUTH authentication instead of username/password? [Default: False] + logger : """ - def __init__(self, space, page_title, instance=None, use_oauth=False): + def __init__(self, space, page_title, instance=None, use_oauth=False, logger=None): + + # Initialize instance variables self.space = space self.title = page_title + self.logger = logger + + # Set up the Confluence object instance self.confluence = ( - setup_confluence(use_oauth=use_oauth) + setup_confluence(use_oauth=use_oauth, logger=self.logger) if not isinstance(instance, atlassian.Confluence) else instance ) @@ -75,7 +82,7 @@ def add_comment(self, comment): return johnnyfive.utils.safe_service_connect( - self.confluence.add_comment, self.page_id, comment + self.confluence.add_comment, self.page_id, comment, logger=self.logger ) def add_label(self, label): @@ -93,7 +100,7 @@ def add_label(self, label): return johnnyfive.utils.safe_service_connect( - self.confluence.set_page_label, self.page_id, label + self.confluence.set_page_label, self.page_id, label, logger=self.logger ) def attach_file(self, filename, name=None, content_type=None, comment=None): @@ -123,6 +130,7 @@ def attach_file(self, filename, name=None, content_type=None, comment=None): content_type=content_type, page_id=self.page_id, comment=comment, + logger=self.logger, ) def create(self, page_body, parent_id=None): @@ -143,9 +151,13 @@ def create(self, page_body, parent_id=None): # Check if it exists before we try anything if self.exists: - print("Can't create a page that already exists!") + johnnyfive.utils.proper_print( + "Can't create a page that already exists!", "info", self.logger + ) return + print(f" ***** logger: {type(self.logger)}") + johnnyfive.utils.safe_service_connect( self.confluence.create_page, self.space, @@ -154,6 +166,7 @@ def create(self, page_body, parent_id=None): parent_id=parent_id, representation="wiki", editor="v1", + logger=self.logger, ) # Set the instance metadata (exists, page_id, etc.) self._set_metadata() @@ -176,7 +189,10 @@ def delete_attachment(self, filename): return johnnyfive.utils.safe_service_connect( - self.confluence.delete_attachment, self.page_id, filename + self.confluence.delete_attachment, + self.page_id, + filename, + logger=self.logger, ) def get_page_attachments(self, limit=200): @@ -195,7 +211,10 @@ def get_page_attachments(self, limit=200): List of Confluence attachment IDs """ return johnnyfive.utils.safe_service_connect( - self.confluence.get_attachments_from_content, self.page_id, limit=limit + self.confluence.get_attachments_from_content, + self.page_id, + limit=limit, + logger=self.logger, ) def get_page_contents(self): @@ -210,7 +229,10 @@ def get_page_contents(self): The HTML-ish body of the confluence page. """ contents = johnnyfive.utils.safe_service_connect( - self.confluence.get_page_by_id, self.page_id, expand="body.storage" + self.confluence.get_page_by_id, + self.page_id, + expand="body.storage", + logger=self.logger, ) # Extract the contents from the return object return contents["body"]["storage"]["value"] @@ -224,7 +246,9 @@ def smite(self): if not self._check_perm("REMOVEPAGE", "remove a page"): return - johnnyfive.utils.safe_service_connect(self.confluence.remove_page, self.page_id) + johnnyfive.utils.safe_service_connect( + self.confluence.remove_page, self.page_id, logger=self.logger + ) self._set_metadata() def update_contents(self, body): @@ -243,7 +267,11 @@ def update_contents(self, body): return johnnyfive.utils.safe_service_connect( - self.confluence.update_page, self.page_id, self.title, body + self.confluence.update_page, + self.page_id, + self.title, + body, + logger=self.logger, ) def _check_perm(self, perm_key, perm_action): @@ -274,18 +302,20 @@ def _check_perm(self, perm_key, perm_action): # If the value is explicitely False, warn as such if perm_val is False: - warnings.warn( + johnnyfive.utils.proper_print( f"User {self.confluence.username} does not have permission " f"to {perm_action} in space {self.space}.", - johnnyfive.utils.PermissionWarning, + "warn", + self.logger, ) return False # If value is None, no permission check was performed, proceed if perm_val is None: - warnings.warn( + johnnyfive.utils.proper_print( "Permissions check is disabled... hoping for the best.", - johnnyfive.utils.PermissionWarning, + "warn", + self.logger, ) return True @@ -297,7 +327,7 @@ def _set_metadata(self): various instance attributes to keep current. """ self.exists = johnnyfive.utils.safe_service_connect( - self.confluence.page_exists, self.space, self.title + self.confluence.page_exists, self.space, self.title, logger=self.logger ) # Page-Specific Information @@ -305,7 +335,7 @@ def _set_metadata(self): None if not self.exists else johnnyfive.utils.safe_service_connect( - self.confluence.get_page_id, self.space, self.title + self.confluence.get_page_id, self.space, self.title, logger=self.logger ) ) self.attachment_url = ( @@ -327,16 +357,17 @@ def _set_permdict(self): The dictionary of permissions (boolean) """ perms = johnnyfive.utils.safe_service_connect( - self.confluence.get_space_permissions, self.space + self.confluence.get_space_permissions, self.space, logger=self.logger ) # Check to see if the authenticated user can view permissions if not perms: - warnings.warn( + johnnyfive.utils.proper_print( f"User {self.confluence.username} needs permission to view " f"permissions in space {self.space}. Contact " "your Confluence administrator.", - johnnyfive.utils.PermissionWarning, + "warn", + self.logger, ) perm_dict = {} @@ -351,7 +382,7 @@ def _set_permdict(self): # Internal Functions =========================================================# -def setup_confluence(use_oauth=False): +def setup_confluence(use_oauth=False, logger=None): """Set up the Confluence class instance Reads in the confluence.conf configuration file, which contains the URL, @@ -365,6 +396,7 @@ def setup_confluence(use_oauth=False): ---------- use_oauth : bool, optional Use the OAUTH authentication scheme? [Default: False] + logger : Returns ------- diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index 9993394..7853147 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -21,7 +21,6 @@ import email.mime import mimetypes import os -import warnings # 3rd Party Libraries from bs4 import BeautifulSoup @@ -114,18 +113,18 @@ def add_attachment(self, file): # Case out the content type main_type, sub_type = content_type.split("/", 1) if main_type == "text": - with open(file, "rb") as fp: - attachment = email.mime.text.MIMEText(fp.read(), _subtype=sub_type) + with open(file, "rb") as f_obj: + attachment = email.mime.text.MIMEText(f_obj.read(), _subtype=sub_type) elif main_type == "image": - with open(file, "rb") as fp: - attachment = email.mime.image.MIMEImage(fp.read(), _subtype=sub_type) + with open(file, "rb") as f_obj: + attachment = email.mime.image.MIMEImage(f_obj.read(), _subtype=sub_type) elif main_type == "audio": - with open(file, "rb") as fp: - attachment = email.mime.audio.MIMEAudio(fp.read(), _subtype=sub_type) + with open(file, "rb") as f_obj: + attachment = email.mime.audio.MIMEAudio(f_obj.read(), _subtype=sub_type) else: - with open(file, "rb") as fp: + with open(file, "rb") as f_obj: attachment = email.mime.base.MIMEBase(main_type, sub_type) - attachment.set_payload(fp.read()) + attachment.set_payload(f_obj.read()) # Add the attachment to the email message attachment.add_header( @@ -203,6 +202,14 @@ def __init__( # Initialize the Gmail connection self.service = setup_gmail(interactive=interactive, logger=self.logger) + + # If we cannot connect to GMail, return here with an empty message_list + if self.service is None: + johnnyfive.utils.proper_print( + "Cannot connect to GMail!", "error", self.logger + ) + return + self.label_id = self._lableId_from_labelName(label) self.query = self.build_query(after_date=after, before_date=before) @@ -269,13 +276,13 @@ def render_message(self, message_id): return dict(subject="", sender="", date="", body="") # Look for Subject and Sender Email in the headers - for d in headers: - if d["name"] == "Subject": - subject = d["value"] - if d["name"] == "From": - sender = d["value"] - if d["name"] == "Date": - date = d["value"] + for head_dict in headers: + if head_dict["name"] == "Subject": + subject = head_dict["value"] + if head_dict["name"] == "From": + sender = head_dict["value"] + if head_dict["name"] == "Date": + date = head_dict["value"] # The Body of the message is in Encrypted format -- decode it. # Get the data and decode it with base 64 decoder. @@ -416,12 +423,12 @@ def build_query(after_date=None, before_date=None): `str` The appropriate query string """ - q = "" + query = "" if after_date: - q = q + f" after:{after_date}" + query = query + f" after:{after_date}" if before_date: - q = q + f" before:{before_date}" - return q + query = query + f" before:{before_date}" + return query # Newer OAUTH Routines =======================================================# @@ -468,9 +475,7 @@ def setup_gmail(interactive=False, logger=None): f"An error occurred within setup_gmail(): {error}", "warn", logger ) except google.auth.exceptions.RefreshError as error: - johnnyfive.utils.proper_print( - f"Some sort of refresh error occurred:\n{error}", "warn", logger - ) + return None # If running in `interactive`, lauch browser to log in elif interactive: @@ -482,12 +487,15 @@ def setup_gmail(interactive=False, logger=None): # Otherwise, raise an exception and specify to run interactively else: - raise ValueError( - "\nNo Gmail credentials found. You may need to run:\n" - "j5_install_conf\n" - "or to authenticate user, run (NOT in a container):\n" - "j5_authenticate_gmail" + johnnyfive.utils.proper_print( + "No Gmail credentials found. You may need to run:\n" + "\t`j5_install_conf`\n" + "\tor to authenticate user, run (NOT in a container):\n" + "\t`j5_authenticate_gmail`", + "error", + logger, ) + return None # Save the credentials for the next run with open(token_fn, "w", encoding="utf-8") as token: diff --git a/johnnyfive/utils.py b/johnnyfive/utils.py index 8080673..d028375 100644 --- a/johnnyfive/utils.py +++ b/johnnyfive/utils.py @@ -40,13 +40,7 @@ # Set API Components -__all__ = ["PermissionWarning", "print_dict", "safe_service_connect", "proper_print"] - - -class PermissionWarning(UserWarning): - """PermissionWarning - Subclass of UserWarning that is more specific to the case of permissions - """ +__all__ = ["safe_service_connect", "print_dict", "proper_print"] # Classes to hold useful information @@ -64,7 +58,7 @@ class Paths: class authTarget(ligmos.utils.classes.baseTarget): - """authTarget Extension of LIGMOS baseTarget + """Extension of LIGMOS baseTarget class Adds specified attributes used in JohnnyFive to silence LIGMOS's "Setting orphan object key" messages @@ -80,15 +74,15 @@ def __init__(self): def install_conffiles(args=None): - """install_conffiles Console Script for installing configuration files + """Console Script for installing configuration files This function is designed to install the (secret) configuration files - (e.g., gmail_credentials.json or johnnyfive.conf) into the proper - config/ directory buried wherever on the filesystem. + (`e.g.`, ``gmail_credentials.json`` or ``johnnyfive.conf``) into the proper + ``config/`` directory buried wherever on the filesystem. Parameters ---------- - args : `Any`, optional + args : Any, optional The arguments passed from the command line [Default: None] """ # Use argparse for the Command-Line Script @@ -119,7 +113,7 @@ def install_conffiles(args=None): def read_ligmos_conffiles(confname, conffile="johnnyfive.conf"): - """read_ligmos_conffiles Read a configuration file using LIGMOS + """Read a configuration file using LIGMOS Having this as a separate function may be a bit of an overkill, but it makes it easier to keep the ligmos imports only in one place, and @@ -127,14 +121,14 @@ def read_ligmos_conffiles(confname, conffile="johnnyfive.conf"): Parameters ---------- - confname : `str` + confname : str Name of the table within the configuration file to parse - conffile : `str` + conffile : str Name of the configuration file to parse Returns ------- - `ligmos.utils.classes.baseTarget` + :class:`ligmos.utils.classes.baseTarget` An object with arrtibutes matching the keys in the associated configuration file. """ @@ -146,7 +140,7 @@ def read_ligmos_conffiles(confname, conffile="johnnyfive.conf"): def print_dict(dd, indent=0, di=4): - """print_dict Print a dictionary in tree format + """Print a dictionary in tree format You know how sometimes you get these nested dictionaries, and they're a pain to visually parse? This routine prints out the contents of a @@ -157,11 +151,11 @@ def print_dict(dd, indent=0, di=4): Parameters ---------- - dd : `dict` + dd : dict The dictionary to print - indent : `int`, optional + indent : int, optional The initial indentation for the tree [Default: 0] - di: `int`, optional + di: int, optional The incremental indentation for each layer of the tree [Default: 4] """ if not isinstance(dd, dict): @@ -178,30 +172,30 @@ def print_dict(dd, indent=0, di=4): def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs): - """safe_service_connect Safely connect to Service (error-catching) + """Safely connect to Service (includes error-catching) Wrapper for Service-connection functions to catch errors that might be - kicked (ConnectionTimeout, for instance). + kicked (``ConnectionTimeout``, for instance). - This function performs a semi-infinite loop, pausing for `pause` seconds - after each failed function call, up to a maximum of `nretries` retries. + This function performs a semi-infinite loop, pausing for ``pause`` seconds + after each failed function call, up to a maximum of ``nretries`` retries. Parameters ---------- - func : `method` + func : :obj:`method` The Service connection method to be wrapped - pause : `int` or `float`, optional + pause : :obj:`int` or :obj:`float`, optional The number of seconds to wait in between retries to connect. [Default: 5] - nretries : `int`, optional + nretries : int, optional The total number of times to retry connecting before returning None [Default: 10] logger : Returns ------- - `Any` - The return value of `func` -- or None if unable to run `func` + Any + The return value of ``func`` -- or None if unable to run ``func`` """ # Now, for the actual function... @@ -218,31 +212,36 @@ def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs httplib2.error.ServerNotFoundError, ) as exception: proper_print( - f"\nWarning: Execution of `{func.__name__}` failed because of:\n{exception}", - "warn", + f"Execution of `{func.__name__}` failed because of network error." + f"\n{exception}", + "error", logger, ) - if (i := i + 1) <= nretries: + if i < nretries: proper_print( - f"Waiting {pause} seconds before starting attempt #{i}/{nretries}", + f"Waiting {pause} seconds before starting attempt #{i+1}/{nretries}", "info", logger, ) time.sleep(pause) else: - raise ConnectionError( - f"Could not connect to service after {nretries} attempts." - ) from exception + proper_print( + f"Could not connect to service after {nretries} attempts.", + "error", + logger, + ) + break # This is for a Service error (premissions, etc.), no retry except requests.exceptions.HTTPError as exception: proper_print( - f"\nWarning: Execution of `{func.__name__}` failed because of:\n{exception}", + f"Execution of `{func.__name__}` failed because of HTTP error." + f"\n{exception}", "except", logger, ) - proper_print("Aborting...", "except", logger) + proper_print("Aborting...", "error", logger) break # Gmail service error, no retry and pass the exception upward @@ -263,8 +262,7 @@ def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs ) raise exception - # =========================================== - # Specific service errors + # Confluence service error, no retry and pass the excepetion upward except atlassian.errors.ApiError as exception: proper_print( f"Caught Atlassian API Error... passing up. {type(exception).__name__}", @@ -273,10 +271,14 @@ def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs ) raise exception + # Google RefreshError occurs when the gmail_token.json to too old except google.auth.exceptions.RefreshError as exception: proper_print( - f"Caught Google Refresh Error... passing up. {type(exception).__name__}", - "except", + "Google Token Refresh Error.\n" + f"\tDescription: {exception.args[0]}\n" + "\tIf the reason is 'Token has been expired or revoked', then run\n" + "\t`j5_authenticate_gmail` to refresh the token.", + "error", logger, ) raise exception @@ -286,6 +288,19 @@ def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs def proper_print(msg, level, logger=None): + """Log if logger, else print to stdout + + _extended_summary_ + + Parameters + ---------- + msg : str + The message to convey + level : str + The logging level. One of [``info``,``warn``,``except``] + logger : _type_, optional + The logger to use, if any [Default: None] + """ if level == "info": if logger is None: print(msg) @@ -296,6 +311,11 @@ def proper_print(msg, level, logger=None): warnings.warn(msg) else: logger.warning(msg) + elif level == "error": + if logger is None: + warnings.warn(f"EXCEPTION: {msg}") + else: + logger.error(msg) elif level == "except": if logger is None: warnings.warn(f"EXCEPTION: {msg}") From 0e08c4370916a17fbf6aa9b12bdc681ce020e371 Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Tue, 29 Nov 2022 16:38:41 -0700 Subject: [PATCH 04/15] More error catching modified: johnnyfive/confluence.py modified: johnnyfive/gmail.py modified: johnnyfive/slack.py modified: johnnyfive/utils.py --- johnnyfive/confluence.py | 15 +++++++++------ johnnyfive/gmail.py | 9 ++++++--- johnnyfive/slack.py | 6 +++--- johnnyfive/utils.py | 30 ++++++++++++++++-------------- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/johnnyfive/confluence.py b/johnnyfive/confluence.py index edff212..da21bdd 100644 --- a/johnnyfive/confluence.py +++ b/johnnyfive/confluence.py @@ -45,7 +45,8 @@ class ConfluencePage: authentication. [Default: None] use_oauth : bool, optional Use OAUTH authentication instead of username/password? [Default: False] - logger : + logger : :obj:`logging.Logger`, optional + The logger object for logging [Default: None] """ def __init__(self, space, page_title, instance=None, use_oauth=False, logger=None): @@ -133,7 +134,7 @@ def attach_file(self, filename, name=None, content_type=None, comment=None): logger=self.logger, ) - def create(self, page_body, parent_id=None): + def create(self, page_body, parent_id=None, representation="wiki"): """Create a brand new Confluence page Summon from the depths of computing a new page. @@ -145,6 +146,9 @@ def create(self, page_body, parent_id=None): parent_id : str, optional The parent page to place this under. If none given, the new page will be created at the root of ``self.space``. [Default: None] + representation : str, optional + The Confluence strorage representation to use. [Default: "wiki"] + Use "storage" for XML-based documents """ if not self._check_perm("EDITSPACE", "create a page"): return @@ -156,15 +160,13 @@ def create(self, page_body, parent_id=None): ) return - print(f" ***** logger: {type(self.logger)}") - johnnyfive.utils.safe_service_connect( self.confluence.create_page, self.space, self.title, page_body, parent_id=parent_id, - representation="wiki", + representation=representation, editor="v1", logger=self.logger, ) @@ -396,7 +398,8 @@ def setup_confluence(use_oauth=False, logger=None): ---------- use_oauth : bool, optional Use the OAUTH authentication scheme? [Default: False] - logger : + logger : :obj:`logging.Logger`, optional + The logger object for logging [Default: None] Returns ------- diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index 7853147..2a3c16c 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -61,7 +61,8 @@ class GmailMessage: Display Name of the sender (i.e. which bot) [Default: None] fromaddr : `str`, optional Sender email address [Default: Value from [gmailSetup]] - logger : + logger : :obj:`logging.Logger`, optional + The logger object for logging [Default: None] """ def __init__( @@ -189,7 +190,8 @@ class GetMessages: before : str, optional Date before which to search for messages. Must be in YYYY/MM/DD format. [Default: None] - logger : + logger : :obj:`logging.Logger`, optional + The logger object for logging [Default: None] """ def __init__( @@ -445,7 +447,8 @@ def setup_gmail(interactive=False, logger=None): interactive : bool, optional Is this session interactive? Relates to how to deal with toke refresh. [Default: False] - logger : + logger : :obj:`logging.Logger`, optional + The logger object for logging [Default: None] Returns ------- diff --git a/johnnyfive/slack.py b/johnnyfive/slack.py index 96a8e36..887aed1 100644 --- a/johnnyfive/slack.py +++ b/johnnyfive/slack.py @@ -149,7 +149,7 @@ def _read_channels(self, name): # Internal Functions =========================================================# -def setup_slack(): +def setup_slack(logger=None): """setup_slack Setup the Slack WebClient for posting _extended_summary_ @@ -158,8 +158,8 @@ def setup_slack(): ------- client : `slack_sdk.web.client.WebClient` The WebClient object needed for reading and writing - logger : `logging.Logger` - The logging thingie + logger : :obj:`logging.Logger`, optional + The logger object for logging [Default: None] """ # Read the setup setup = johnnyfive.utils.read_ligmos_conffiles("slackSetup") diff --git a/johnnyfive/utils.py b/johnnyfive/utils.py index d028375..e20183b 100644 --- a/johnnyfive/utils.py +++ b/johnnyfive/utils.py @@ -190,7 +190,8 @@ def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs nretries : int, optional The total number of times to retry connecting before returning None [Default: 10] - logger : + logger : :obj:`logging.Logger`, optional + The logger object for logging [Default: None] Returns ------- @@ -237,21 +238,22 @@ def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs except requests.exceptions.HTTPError as exception: proper_print( f"Execution of `{func.__name__}` failed because of HTTP error." - f"\n{exception}", - "except", + f"\n{type(exception).__name__} {exception.args}", + "error", logger, ) - proper_print("Aborting...", "error", logger) + proper_print("Aborting...", "except", logger) + raise exception break - # Gmail service error, no retry and pass the exception upward - except googleapiclient.errors.HttpError as exception: - proper_print( - f"Caught Gmail HTTP error... passing up. {type(exception).__name__}", - "except", - logger, - ) - raise exception + # # Gmail service error, no retry and pass the exception upward + # except googleapiclient.errors.HttpError as exception: + # proper_print( + # f"Caught Gmail HTTP error... passing up. {type(exception).__name__}", + # "except", + # logger, + # ) + # raise exception # Slack service error, no retry and pass the exception upward except slack_sdk.errors.SlackApiError as exception: @@ -298,8 +300,8 @@ def proper_print(msg, level, logger=None): The message to convey level : str The logging level. One of [``info``,``warn``,``except``] - logger : _type_, optional - The logger to use, if any [Default: None] + logger : :obj:`logging.Logger`, optional + The logger object for logging [Default: None] """ if level == "info": if logger is None: From b19a158f1ebc179b016e8244a31188660c69153e Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Mon, 19 Jun 2023 17:52:32 -0700 Subject: [PATCH 05/15] Fixing imports modified: README.md modified: johnnyfive/gmail.py --- README.md | 2 +- johnnyfive/gmail.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e38d49b..2d9c701 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ import johnnyfive page = johnnyfive.ConfluencePage(space, page_title, instance=None, use_oauth=False) # Gmail API -message = johnnyfive.GmailMessage(toaddr, subject, message_text, fromname=None fromaddr=None, interactive=False) +message = johnnyfive.GmailMessage(toaddr, subject, message_text, fromname=None, fromaddr=None, interactive=False) message_list = johnnyfive.GetMessages(label=None, after=None, before=None, interactive=False) # Slack API diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index 2a3c16c..9736986 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -18,7 +18,11 @@ # Built-In Libraries import base64 -import email.mime +import email.mime.audio +import email.mime.base +import email.mime.image +import email.mime.multipart +import email.mime.text import mimetypes import os From c4b8e56fdf4ff0b0860eb86e3bd1d4005e7b72fe Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Thu, 28 Sep 2023 14:24:12 -0700 Subject: [PATCH 06/15] Remove use of pkg_resources The `pkg_resources` package is being deprecated in favor of `importlib` modules `resources` and `metadata`. This commit updates this package to be in line with this change. modified: johnnyfive/confluence.py modified: johnnyfive/gmail.py modified: johnnyfive/utils.py --- johnnyfive/confluence.py | 1 - johnnyfive/gmail.py | 1 - johnnyfive/utils.py | 12 +++++------- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/johnnyfive/confluence.py b/johnnyfive/confluence.py index da21bdd..c5e3f0e 100644 --- a/johnnyfive/confluence.py +++ b/johnnyfive/confluence.py @@ -50,7 +50,6 @@ class ConfluencePage: """ def __init__(self, space, page_title, instance=None, use_oauth=False, logger=None): - # Initialize instance variables self.space = space self.title = page_title diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index 9736986..738a2d0 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -468,7 +468,6 @@ def setup_gmail(interactive=False, logger=None): # If there are no (valid) credentials available... if not creds or not creds.valid: - # If just expired, refresh and move on if creds and creds.expired and creds.refresh_token: try: diff --git a/johnnyfive/utils.py b/johnnyfive/utils.py index e20183b..0ee813a 100644 --- a/johnnyfive/utils.py +++ b/johnnyfive/utils.py @@ -18,8 +18,8 @@ # Built-In Libraries import argparse +from importlib import resources import os -import pathlib import shutil import time import warnings @@ -29,7 +29,6 @@ import googleapiclient.errors import google.auth.exceptions import httplib2 -from pkg_resources import resource_filename import requests import slack_sdk.errors @@ -51,10 +50,10 @@ class Paths: """ # Main data & config directories - config = pathlib.Path(resource_filename("johnnyfive", "config")) - images = pathlib.Path(resource_filename("johnnyfive", "images")) - gmail_token = config.joinpath("gmail_token.json") - gmail_creds = config.joinpath("gmail_credentials.json") + config = resources.files("johnnyfive") / "config" + images = resources.files("johnnyfive") / "images" + gmail_token = config / "gmail_token.json" + gmail_creds = config / "gmail_credentials.json" class authTarget(ligmos.utils.classes.baseTarget): @@ -201,7 +200,6 @@ def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs # Now, for the actual function... for i in range(1, nretries + 1): - # Nominal function return try: return func(*args, **kwargs) From 986a30bb02d8b75fe63ec30142e0c713138bcada Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Mon, 4 Dec 2023 09:25:05 -0700 Subject: [PATCH 07/15] Fix bug in `gmail.GetMessages.render_message()` In the case where a GMAIL message has multiple "parts" (i.e., HTML, images, etc.), the payload['body'] dictionary element is empty, and what would be there is as a list in the 'parts' element. In the simple case where the GMAIL message is text only, there is only one part (as is the case with relay DOL messages -- the test case when developing this class). This commit adds a check for the 'parts' dictionary element, and selects the first PART, if such a list exists. Otherwise, the usual behavior of extracting the payload['body'] dictionary element holds. This works with the hand-delivered DOL messages of the past few days (the mail relay is down due to IT issues), but no guarantee is made or implied that this fix will be appropriate or work in all use cases of this j5 class. modified: johnnyfive/gmail.py --- johnnyfive/gmail.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index 738a2d0..f5f761b 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -292,7 +292,13 @@ def render_message(self, message_id): # The Body of the message is in Encrypted format -- decode it. # Get the data and decode it with base 64 decoder. - data = payload["body"]["data"] + + # If more than one part (i.e., HTML or images, etc.), get the first + data = ( + payload["body"]["data"] + if "parts" not in payload + else payload["parts"][0]["body"]["data"] + ) data = data.replace("-", "+").replace("_", "/") decoded_data = base64.b64decode(data) From 166948e0a29b8c6f9d13a15f1e5e2e64e766af43 Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Tue, 18 Feb 2025 14:31:33 -0700 Subject: [PATCH 08/15] Raise errors rather than return `None` modified: johnnyfive/gmail.py modified: johnnyfive/slack.py modified: johnnyfive/utils.py new file: pyproject.toml --- johnnyfive/gmail.py | 14 +++++++------- johnnyfive/slack.py | 2 +- johnnyfive/utils.py | 12 ++++++++++-- pyproject.toml | 0 4 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 pyproject.toml diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index f5f761b..a63a84d 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -176,7 +176,7 @@ def send(self): "except", self.logger, ) - return None + raise johnnyfive.utils.J5Error from error class GetMessages: @@ -364,8 +364,8 @@ def update_msg_labels(self, message_id, add_labels=None, remove_labels=None): "except", self.logger, ) - # If unsuccessful in connecting, return None - return None + # If unsuccessful in connecting, raise + raise johnnyfive.utils.J5Error("Unsuccessful connection") def _lableId_from_labelName(self, name): """Get the Label ID from the Label Name @@ -487,7 +487,7 @@ def setup_gmail(interactive=False, logger=None): f"An error occurred within setup_gmail(): {error}", "warn", logger ) except google.auth.exceptions.RefreshError as error: - return None + raise johnnyfive.utils.J5Error from error # If running in `interactive`, lauch browser to log in elif interactive: @@ -507,13 +507,13 @@ def setup_gmail(interactive=False, logger=None): "error", logger, ) - return None + raise johnnyfive.utils.J5Error # Save the credentials for the next run with open(token_fn, "w", encoding="utf-8") as token: token.write(creds.to_json()) - # Try building the GMail API service. If error, print error & return None + # Try building the GMail API service. If error, print error & raise try: # Call the Gmail API johnnyfive.utils.proper_print("Calling the GMAIL API...", "info", logger) @@ -526,7 +526,7 @@ def setup_gmail(interactive=False, logger=None): johnnyfive.utils.proper_print( f"An error occurred within setup_gmail():\n{error}", "except", logger ) - return None + raise johnnyfive.utils.J5Error from error def authenticate_gmail(logger=None): diff --git a/johnnyfive/slack.py b/johnnyfive/slack.py index 887aed1..b0a0d10 100644 --- a/johnnyfive/slack.py +++ b/johnnyfive/slack.py @@ -69,7 +69,7 @@ def send_message(self, message): response = johnnyfive.utils.safe_service_connect( self.client.chat_postMessage, channel=self.channel_id, - text=message + text=message, # You could also use a blocks[] array to send richer content ) # Print result, which includes information about the message (like TS) diff --git a/johnnyfive/utils.py b/johnnyfive/utils.py index 0ee813a..50f51ed 100644 --- a/johnnyfive/utils.py +++ b/johnnyfive/utils.py @@ -42,6 +42,14 @@ __all__ = ["safe_service_connect", "print_dict", "proper_print"] +# Define error classes +class J5Error(Exception): + """J5Error Class + + Base JohnnyFive error class + """ + + # Classes to hold useful information class Paths: """Paths @@ -283,8 +291,8 @@ def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs ) raise exception - # If not successful, return None - return None + # If not successful, raise error + raise J5Error("Unspecified error") def proper_print(msg, level, logger=None): diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e69de29 From cbd7ffa30c100cf792c7f43575c34fce6df5f43b Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Tue, 18 Feb 2025 15:09:51 -0700 Subject: [PATCH 09/15] Persnickity modified: johnnyfive/__init__.py modified: johnnyfive/classes.py modified: johnnyfive/confluence.py modified: johnnyfive/gmail.py modified: johnnyfive/slack.py modified: johnnyfive/utils.py --- johnnyfive/__init__.py | 3 +- johnnyfive/classes.py | 2 + johnnyfive/confluence.py | 96 ++++++++++++++++++------------- johnnyfive/gmail.py | 121 +++++++++++++++++++++------------------ johnnyfive/slack.py | 42 +++++++------- johnnyfive/utils.py | 51 ++++++++++------- 6 files changed, 177 insertions(+), 138 deletions(-) diff --git a/johnnyfive/__init__.py b/johnnyfive/__init__.py index 2124670..d131c4f 100644 --- a/johnnyfive/__init__.py +++ b/johnnyfive/__init__.py @@ -8,8 +8,7 @@ # # @author: tbowers -"""Init File -""" +"""Init File""" # Imports for signal and log handling diff --git a/johnnyfive/classes.py b/johnnyfive/classes.py index 11f1e94..7818217 100644 --- a/johnnyfive/classes.py +++ b/johnnyfive/classes.py @@ -17,6 +17,8 @@ class emailSNMP(object): + """emailSNMP _summary_""" + def __init__(self): self.host = None self.port = 465 diff --git a/johnnyfive/confluence.py b/johnnyfive/confluence.py index c5e3f0e..43bcfd0 100644 --- a/johnnyfive/confluence.py +++ b/johnnyfive/confluence.py @@ -15,6 +15,7 @@ """ # Built-In Libraries +import logging # 3rd Party Libraries import atlassian @@ -35,21 +36,28 @@ class ConfluencePage: Parameters ---------- - space : str + space : :obj:`str` The name of the Confluence space for this page - page_title : str + page_title : :obj:`str` The page title - instance : :class:`atlassian.Confluence`, optional + instance : :class:`~atlassian.Confluence`, optional An existing Confluence object instance to be used instead of reinstantiating a new Confluence object for communication and authentication. [Default: None] - use_oauth : bool, optional + use_oauth : :obj:`bool`, optional Use OAUTH authentication instead of username/password? [Default: False] - logger : :obj:`logging.Logger`, optional + logger : :obj:`~logging.Logger`, optional The logger object for logging [Default: None] """ - def __init__(self, space, page_title, instance=None, use_oauth=False, logger=None): + def __init__( + self, + space: str, + page_title: str, + instance: atlassian.Confluence = None, + use_oauth: bool = False, + logger: logging.Logger = None, + ): # Initialize instance variables self.space = space self.title = page_title @@ -66,7 +74,7 @@ def __init__(self, space, page_title, instance=None, use_oauth=False, logger=Non # Set the class metadata based on this page self._set_metadata() - def add_comment(self, comment): + def add_comment(self, comment: str): """Add a comment to the Confluence page Sometimes it's helpful to include a comment at the bottom of the @@ -75,7 +83,7 @@ def add_comment(self, comment): Parameters ---------- - comment : str + comment : :obj:`str` The comment to be left on the page. """ if not self._check_perm("COMMENT", "add a comment"): @@ -85,7 +93,7 @@ def add_comment(self, comment): self.confluence.add_comment, self.page_id, comment, logger=self.logger ) - def add_label(self, label): + def add_label(self, label: str): """Add a label to the Confluence page Sometimes it's helpful to have a label on a Confluence page for @@ -93,7 +101,7 @@ def add_label(self, label): Parameters ---------- - label : `str` + label : :obj:`str` The label to be added to the page """ if not self._check_perm("EDITSPACE", "add a label"): @@ -103,7 +111,13 @@ def add_label(self, label): self.confluence.set_page_label, self.page_id, label, logger=self.logger ) - def attach_file(self, filename, name=None, content_type=None, comment=None): + def attach_file( + self, + filename: str, + name: str = None, + content_type: str = None, + comment: str = None, + ): """Attach a file to this page Wrapper for the Confluence method attach_file() that includes the @@ -111,13 +125,13 @@ def attach_file(self, filename, name=None, content_type=None, comment=None): Parameters ---------- - filename : str + filename : :obj:`str` Filename of the attachment - name : str, optional + name : :obj:`str`, optional Display name for this attachment [Default: None] - content_type : str, optional + content_type : :obj:`str`, optional MIME content type [Default: None] - comment : str, optional + comment : :obj:`str`, optional Additional comment or description to be included [Default: None] """ if not self._check_perm("CREATEATTACHMENT", "create an attachment"): @@ -133,19 +147,21 @@ def attach_file(self, filename, name=None, content_type=None, comment=None): logger=self.logger, ) - def create(self, page_body, parent_id=None, representation="wiki"): + def create( + self, page_body: str, parent_id: str = None, representation: str = "wiki" + ): """Create a brand new Confluence page Summon from the depths of computing a new page. Parameters ---------- - page_body : str + page_body : :obj:`str` The body of the new Confluence page. - parent_id : str, optional + parent_id : :obj:`str`, optional The parent page to place this under. If none given, the new page will be created at the root of ``self.space``. [Default: None] - representation : str, optional + representation : :obj:`str`, optional The Confluence strorage representation to use. [Default: "wiki"] Use "storage" for XML-based documents """ @@ -172,7 +188,7 @@ def create(self, page_body, parent_id=None, representation="wiki"): # Set the instance metadata (exists, page_id, etc.) self._set_metadata() - def delete_attachment(self, filename): + def delete_attachment(self, filename: str): """Delete an attachment from this page Wrapper for the Confluence method delete_attachment() that includes the @@ -183,7 +199,7 @@ def delete_attachment(self, filename): Parameters ---------- - filename : str + filename : :obj:`str` Filename of the attachment to delete """ if not self._check_perm("REMOVEATTACHMENT", "remove an attachment"): @@ -196,19 +212,19 @@ def delete_attachment(self, filename): logger=self.logger, ) - def get_page_attachments(self, limit=200): + def get_page_attachments(self, limit: int = 200) -> list[object]: """Retrieve the page attachments Return a list of page attachment IDs, up to ``limit`` in length. Parameters ---------- - limit : int, optional - The number of attachments to return [Default: 200] + limit : :obj:`int`, optional + The number of attachments to return (Default: 200) Returns ------- - list + :obj:`list` List of Confluence attachment IDs """ return johnnyfive.utils.safe_service_connect( @@ -218,7 +234,7 @@ def get_page_attachments(self, limit=200): logger=self.logger, ) - def get_page_contents(self): + def get_page_contents(self) -> str: """Retrieve the page contents in HTML-ish format Either for curiosity or for modification, get the page contents, which @@ -226,7 +242,7 @@ def get_page_contents(self): Returns ------- - str + :obj:`str` The HTML-ish body of the confluence page. """ contents = johnnyfive.utils.safe_service_connect( @@ -252,7 +268,7 @@ def smite(self): ) self._set_metadata() - def update_contents(self, body): + def update_contents(self, body: str): """Update the contents of the Confluence page Update the page by replacing the existing content with new. The idea @@ -261,7 +277,7 @@ def update_contents(self, body): Parameters ---------- - body : str + body : :obj:`str` The new page contents to upload to Confluence. """ if not self._check_perm("EDITSPACE", "update a page"): @@ -275,7 +291,7 @@ def update_contents(self, body): logger=self.logger, ) - def _check_perm(self, perm_key, perm_action): + def _check_perm(self, perm_key: str, perm_action: str) -> bool: """Check the premissions dictionary for a particular action Check the ``perm_key`` in the permissions dictionary to see whether the @@ -289,14 +305,14 @@ def _check_perm(self, perm_key, perm_action): Parameters ---------- - perm_key : str + perm_key : :obj:`str` The key in perm_dict to look for - perm_action : str + perm_action : :obj:`str` The action that is requested by the calling function. Returns ------- - bool + :obj:`bool` True for perform action, False for not """ perm_val = self.space_perms.get(perm_key, None) @@ -345,7 +361,7 @@ def _set_metadata(self): else f"{self.confluence.url}download/attachments/{self.page_id}/" ) - def _set_permdict(self): + def _set_permdict(self) -> dict: """Create a dictionary of permissions This method creates a dictionary of permissions for this user in this @@ -354,7 +370,7 @@ def _set_permdict(self): Returns ------- - dict + :obj:`dict` The dictionary of permissions (boolean) """ perms = johnnyfive.utils.safe_service_connect( @@ -383,7 +399,9 @@ def _set_permdict(self): # Internal Functions =========================================================# -def setup_confluence(use_oauth=False, logger=None): +def setup_confluence( + use_oauth: bool = False, logger: logging.Logger = None +) -> atlassian.Confluence: """Set up the Confluence class instance Reads in the confluence.conf configuration file, which contains the URL, @@ -395,14 +413,14 @@ def setup_confluence(use_oauth=False, logger=None): Parameters ---------- - use_oauth : bool, optional + use_oauth : :obj:`bool`, optional Use the OAUTH authentication scheme? [Default: False] - logger : :obj:`logging.Logger`, optional + logger : :obj:`~logging.Logger`, optional The logger object for logging [Default: None] Returns ------- - confluence : :class:`atlassian.Confluence` + confluence : :class:`~atlassian.Confluence` Confluence class, initialized with credentials """ # Read the setup diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index a63a84d..0c39f10 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -23,6 +23,7 @@ import email.mime.image import email.mime.multipart import email.mime.text +import logging import mimetypes import os @@ -54,30 +55,32 @@ class GmailMessage: Parameters ---------- - toaddr : `str` or `list` + toaddr : :obj:`str` or :obj:`list` The intended recipient(s) of the email message - subject : `str` + subject : :obj:`str` The subject of the email message - message_text : `str` + message_text : :obj:`str` The body text of the email message (as a single string with optional newlines.) - fromname : `str`, optional + fromname : :obj:`str`, optional Display Name of the sender (i.e. which bot) [Default: None] - fromaddr : `str`, optional + fromaddr : :obj:`str`, optional Sender email address [Default: Value from [gmailSetup]] - logger : :obj:`logging.Logger`, optional + interactive : :obj:`bool`, optional + Whether to run this in interactive mode (Default: False) + logger : :obj:`~logging.Logger`, optional The logger object for logging [Default: None] """ def __init__( self, - toaddr, - subject, - message_text, - fromname=None, - fromaddr=None, - interactive=False, - logger=None, + toaddr: str | list, + subject: str, + message_text: str, + fromname: str = None, + fromaddr: str = None, + interactive: bool = False, + logger: logging.Logger = None, ): # Set the logger, if passed self.logger = logger @@ -98,14 +101,14 @@ def __init__( # Place the text into the message self.message.attach(email.mime.text.MIMEText(message_text)) - def add_attachment(self, file): + def add_attachment(self, file: str): """Add an attachment to the GMAIL message _extended_summary_ Parameters ---------- - file : str + file : :obj:`str` Filename of the attachment """ # For the attachment, guess the MIME type for reading it in @@ -137,19 +140,14 @@ def add_attachment(self, file): ) self.message.attach(attachment) - def send(self): + def send(self) -> dict: """Send the GmailMessage _extended_summary_ - Parameters - ---------- - n_tries : `int`, optional - The number of retry attemps at sending this message [Default: 5] - Returns ------- - `dict` + :obj:`dict` The sent message object """ # Take the message object, and 64-bit encode it properly for sending @@ -186,20 +184,27 @@ class GetMessages: Parameters ---------- - label : str, optional + label : :obj:`str`, optional The Gmail label of messages to find [Default: None] - after : str, optional + after : :obj:`str`, optional Date after which to search for messages. Must be in YYYY/MM/DD format. - [Default: None] - before : str, optional + (Default: None) + before : :obj:`str`, optional Date before which to search for messages. Must be in YYYY/MM/DD format. - [Default: None] + (Default: None) + interactive : :obj:`bool`, optional + Whether to run this in interactive mode (Default: False) logger : :obj:`logging.Logger`, optional The logger object for logging [Default: None] """ def __init__( - self, label=None, after=None, before=None, interactive=False, logger=None + self, + label: str = None, + after: str = None, + before: str = None, + interactive: bool = False, + logger: logging.Logger = None, ): # Initialize basic stuff self.label_list = None @@ -242,7 +247,7 @@ def __init__( self.logger, ) - def render_message(self, message_id): + def render_message(self, message_id: str) -> dict: """Retrieve and render a message by ID# Gmail mnessages are stored in a JSON-like structure that must be @@ -250,12 +255,12 @@ def render_message(self, message_id): Parameters ---------- - message_id : str + message_id : :obj:`str` The ``['id']`` field of an entry in self.message_list Returns ------- - dict + :obj:`dict` Dictionary containing the subject, sender, date, and body of the message. """ @@ -279,7 +284,7 @@ def render_message(self, message_id): # Return empty dictionary if unsuccessful in connecting if not payload: - return dict(subject="", sender="", date="", body="") + return {"subject": "", "sender": "", "date": "", "body": ""} # Look for Subject and Sender Email in the headers for head_dict in headers: @@ -307,30 +312,30 @@ def render_message(self, message_id): body = body[0].text # Return a dictionary with the plain-text components of this message - return dict(subject=subject, sender=sender, date=date, body=body) + return {"subject": subject, "sender": sender, "date": date, "body": body} - def update_msg_labels(self, message_id, add_labels=None, remove_labels=None): + def update_msg_labels( + self, + message_id: str, + add_labels: list[str] = None, + remove_labels: list[str] = None, + ): """Update the labels for a message by ID# _extended_summary_ Parameters ---------- - message_id : str + message_id : :obj:`str` The ``['id']`` field of an entry in self.message_list - add_labels : list, optional + add_labels : :obj:`list`, optional The list of label IDs to add to this message [Default: None] - remove_labels : list, optional + remove_labels : :obj:`list`, optional The list of label IDs to remove from this message [Default: None] - Returns - ------- - Any - Uh, the Message object from Gmail... probably just return nothing? """ if not add_labels and not remove_labels: johnnyfive.utils.proper_print("No labels to change.", "info", self.logger) - return None # Convert Label Names to Label IDs add_label_ids, remove_label_ids = [], [] @@ -367,19 +372,19 @@ def update_msg_labels(self, message_id, add_labels=None, remove_labels=None): # If unsuccessful in connecting, raise raise johnnyfive.utils.J5Error("Unsuccessful connection") - def _lableId_from_labelName(self, name): + def _lableId_from_labelName(self, name: str) -> str: """Get the Label ID from the Label Name _extended_summary_ Parameters ---------- - name : str + name : :obj:`str` Label name Returns ------- - str + :obj:`str` Label ID """ if not self.service: @@ -418,21 +423,21 @@ def _lableId_from_labelName(self, name): return label_id @staticmethod - def build_query(after_date=None, before_date=None): + def build_query(after_date: str = None, before_date: str = None) -> str: """build_query Build the query string for users.messages.list _extended_summary_ Parameters ---------- - after_date : `str` + after_date : :obj:`str` Date after which to search for messages. - before_date : `str` + before_date : :obj:`str` Date before which to search for messages. Returns ------- - `str` + :obj:`str` The appropriate query string """ query = "" @@ -444,7 +449,9 @@ def build_query(after_date=None, before_date=None): # Newer OAUTH Routines =======================================================# -def setup_gmail(interactive=False, logger=None): +def setup_gmail( + interactive: bool = False, logger: logging.Logger = None +) -> googleapiclient.discovery.Resource: """Initialize the GMail API (via OAuth) [extended_summary] @@ -454,15 +461,15 @@ def setup_gmail(interactive=False, logger=None): Parameters ---------- - interactive : bool, optional + interactive : :obj:`bool`, optional Is this session interactive? Relates to how to deal with toke refresh. [Default: False] - logger : :obj:`logging.Logger`, optional - The logger object for logging [Default: None] + logger : :obj:`~logging.Logger`, optional + The logger object for logging (Default: None) Returns ------- - :obj:`googleapiclient.discovery.Resource` + :obj:`~googleapiclient.discovery.Resource` The GMail API service object for consumption by other routines """ # Read in the credential token @@ -529,7 +536,7 @@ def setup_gmail(interactive=False, logger=None): raise johnnyfive.utils.J5Error from error -def authenticate_gmail(logger=None): +def authenticate_gmail(logger: logging.Logger = None): """Console Script for authenticating Gmail This is the command-line script for doing the interactive authentication @@ -542,6 +549,10 @@ def authenticate_gmail(logger=None): j5_authenticate_gmail + Parameters + ---------- + logger : :obj:`~logging.Logger`, optional + The logger object for logging (Default: None) """ johnnyfive.utils.proper_print("Authenticate GMail...", "info", logger) # Remove the existing GMAIL TOKEN file, if extant... diff --git a/johnnyfive/slack.py b/johnnyfive/slack.py index b0a0d10..b228f2c 100644 --- a/johnnyfive/slack.py +++ b/johnnyfive/slack.py @@ -18,10 +18,12 @@ """ # Built-In Libraries +import pathlib import warnings # 3rd Party Libraries import slack_sdk +import slack_sdk.errors # Internal Imports import johnnyfive.utils @@ -38,29 +40,29 @@ class SlackChannel: Parameters ---------- - channel_name : `str` + channel_name : :obj:`str` Slack Channel into which to post """ - def __init__(self, channel_name): + def __init__(self, channel_name: str): self.client = setup_slack() # Get the channel ID self.channel_id = self._read_channels(channel_name) - def send_message(self, message): - """send_message Send a (text only) message to the channel + def send_message(self, message: str) -> object: + """Send a (text only) message to the channel _extended_summary_ Parameters ---------- - message : `str` or `blocks[]` array + message : :obj:`str` or `blocks[]` array The message to send to the Slack channel Returns ------- - `Any` + :obj:`~typing.Any` The response from Slack """ response = None @@ -80,21 +82,21 @@ def send_message(self, message): ) return response - def upload_file(self, file, title=None): - """upload_file Upload a file to the channel + def upload_file(self, file: str | pathlib.Path, title: str = None): + """Upload a file to the channel _extended_summary_ Parameters ---------- - file : `str` or `os.PathLike` + file : :obj:`str` or :obj:`~pathlib.Path` The (path and) filename of the file to be uploaded. - title : `str`, optional - The title for the file posted [Default: None] + title : :obj:`str`, optional + The title for the file posted (Default: None) Returns ------- - `Any` + :obj:`~typing.Any` The response from Slack """ response = None @@ -111,17 +113,17 @@ def upload_file(self, file, title=None): ) return response - def _read_channels(self, name): - """_read_channels Return the Channel ID for the names channel + def _read_channels(self, name: str) -> str: + """Return the Channel ID for the names channel Parameters ---------- - name : `str` + name : :obj:`str` The name of the channel Returns ------- - `str` + :obj:`str` The desired Channel ID """ conversation_id = None @@ -149,17 +151,15 @@ def _read_channels(self, name): # Internal Functions =========================================================# -def setup_slack(logger=None): - """setup_slack Setup the Slack WebClient for posting +def setup_slack() -> slack_sdk.web.client.WebClient: + """Setup the Slack WebClient for posting _extended_summary_ Returns ------- - client : `slack_sdk.web.client.WebClient` + client : :obj:`~slack_sdk.web.client.WebClient` The WebClient object needed for reading and writing - logger : :obj:`logging.Logger`, optional - The logger object for logging [Default: None] """ # Read the setup setup = johnnyfive.utils.read_ligmos_conffiles("slackSetup") diff --git a/johnnyfive/utils.py b/johnnyfive/utils.py index 50f51ed..915d775 100644 --- a/johnnyfive/utils.py +++ b/johnnyfive/utils.py @@ -19,14 +19,15 @@ # Built-In Libraries import argparse from importlib import resources +import logging import os import shutil import time +import typing import warnings # 3rd Party Libraries import atlassian.errors -import googleapiclient.errors import google.auth.exceptions import httplib2 import requests @@ -80,7 +81,7 @@ def __init__(self): self.tokenSecret = None -def install_conffiles(args=None): +def install_conffiles(args: object = None): """Console Script for installing configuration files This function is designed to install the (secret) configuration files @@ -89,7 +90,7 @@ def install_conffiles(args=None): Parameters ---------- - args : Any, optional + args : :obj:~`typing.Any`, optional The arguments passed from the command line [Default: None] """ # Use argparse for the Command-Line Script @@ -119,7 +120,9 @@ def install_conffiles(args=None): shutil.copy2(file, Paths.config) -def read_ligmos_conffiles(confname, conffile="johnnyfive.conf"): +def read_ligmos_conffiles( + confname: str, conffile: str = "johnnyfive.conf" +) -> ligmos.utils.classes.baseTarget: """Read a configuration file using LIGMOS Having this as a separate function may be a bit of an overkill, but it @@ -128,14 +131,14 @@ def read_ligmos_conffiles(confname, conffile="johnnyfive.conf"): Parameters ---------- - confname : str + confname : :obj:`str` Name of the table within the configuration file to parse - conffile : str + conffile : :obj:`str` Name of the configuration file to parse Returns ------- - :class:`ligmos.utils.classes.baseTarget` + :class:`~ligmos.utils.classes.baseTarget` An object with arrtibutes matching the keys in the associated configuration file. """ @@ -146,7 +149,7 @@ def read_ligmos_conffiles(confname, conffile="johnnyfive.conf"): return ligconf -def print_dict(dd, indent=0, di=4): +def print_dict(dd: dict, indent: int = 0, di: int = 4): """Print a dictionary in tree format You know how sometimes you get these nested dictionaries, and they're a @@ -158,11 +161,11 @@ def print_dict(dd, indent=0, di=4): Parameters ---------- - dd : dict + dd : :obj:`dict` The dictionary to print - indent : int, optional + indent : :obj:`int`, optional The initial indentation for the tree [Default: 0] - di: int, optional + di: :obj:`int`, optional The incremental indentation for each layer of the tree [Default: 4] """ if not isinstance(dd, dict): @@ -178,7 +181,14 @@ def print_dict(dd, indent=0, di=4): print(f"{' '*indent}{key:12s}: {value}") -def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs): +def safe_service_connect( + func: typing.Callable, + *args, + pause: int | float = 5, + nretries: int = 5, + logger: logging.Logger = None, + **kwargs, +) -> object: """Safely connect to Service (includes error-catching) Wrapper for Service-connection functions to catch errors that might be @@ -189,20 +199,20 @@ def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs Parameters ---------- - func : :obj:`method` + func : :obj:`~typing.Callable` The Service connection method to be wrapped pause : :obj:`int` or :obj:`float`, optional The number of seconds to wait in between retries to connect. [Default: 5] - nretries : int, optional + nretries : :obj:`int`, optional The total number of times to retry connecting before returning None [Default: 10] - logger : :obj:`logging.Logger`, optional + logger : :obj:`~logging.Logger`, optional The logger object for logging [Default: None] Returns ------- - Any + :obj:`~typing.Any` The return value of ``func`` -- or None if unable to run ``func`` """ @@ -250,7 +260,6 @@ def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs ) proper_print("Aborting...", "except", logger) raise exception - break # # Gmail service error, no retry and pass the exception upward # except googleapiclient.errors.HttpError as exception: @@ -295,18 +304,18 @@ def safe_service_connect(func, *args, pause=5, nretries=5, logger=None, **kwargs raise J5Error("Unspecified error") -def proper_print(msg, level, logger=None): +def proper_print(msg: str, level: str, logger: logging.Logger = None): """Log if logger, else print to stdout _extended_summary_ Parameters ---------- - msg : str + msg : :obj:`str` The message to convey - level : str + level : ;obj:`str` The logging level. One of [``info``,``warn``,``except``] - logger : :obj:`logging.Logger`, optional + logger : :obj:`~logging.Logger`, optional The logger object for logging [Default: None] """ if level == "info": From 74448f9ac4ce0a09f6c823010ecfceae20bff21d Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Tue, 18 Feb 2025 15:17:54 -0700 Subject: [PATCH 10/15] Catch JSON error when reading Gmail token modified: johnnyfive/confluence.py modified: johnnyfive/gmail.py --- johnnyfive/confluence.py | 8 ++------ johnnyfive/gmail.py | 34 ++++++++++++++++++++-------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/johnnyfive/confluence.py b/johnnyfive/confluence.py index 43bcfd0..58edd44 100644 --- a/johnnyfive/confluence.py +++ b/johnnyfive/confluence.py @@ -65,7 +65,7 @@ def __init__( # Set up the Confluence object instance self.confluence = ( - setup_confluence(use_oauth=use_oauth, logger=self.logger) + setup_confluence(use_oauth=use_oauth) if not isinstance(instance, atlassian.Confluence) else instance ) @@ -399,9 +399,7 @@ def _set_permdict(self) -> dict: # Internal Functions =========================================================# -def setup_confluence( - use_oauth: bool = False, logger: logging.Logger = None -) -> atlassian.Confluence: +def setup_confluence(use_oauth: bool = False) -> atlassian.Confluence: """Set up the Confluence class instance Reads in the confluence.conf configuration file, which contains the URL, @@ -415,8 +413,6 @@ def setup_confluence( ---------- use_oauth : :obj:`bool`, optional Use the OAUTH authentication scheme? [Default: False] - logger : :obj:`~logging.Logger`, optional - The logger object for logging [Default: None] Returns ------- diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index 0c39f10..b2989ed 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -23,6 +23,7 @@ import email.mime.image import email.mime.multipart import email.mime.text +import json import logging import mimetypes import os @@ -221,7 +222,7 @@ def __init__( ) return - self.label_id = self._lableId_from_labelName(label) + self.label_id = self._label_id_from_name(label) self.query = self.build_query(after_date=after, before_date=before) # Get the list of matching messages (API: users.messages.list) @@ -341,10 +342,10 @@ def update_msg_labels( add_label_ids, remove_label_ids = [], [] if add_labels: for label in add_labels: - add_label_ids.append(self._lableId_from_labelName(label)) + add_label_ids.append(self._label_id_from_name(label)) if remove_labels: for label in remove_labels: - remove_label_ids.append(self._lableId_from_labelName(label)) + remove_label_ids.append(self._label_id_from_name(label)) # Build the label dictionary to send to Gmail body = {} @@ -372,7 +373,7 @@ def update_msg_labels( # If unsuccessful in connecting, raise raise johnnyfive.utils.J5Error("Unsuccessful connection") - def _lableId_from_labelName(self, name: str) -> str: + def _label_id_from_name(self, name: str) -> str: """Get the Label ID from the Label Name _extended_summary_ @@ -475,9 +476,14 @@ def setup_gmail( # Read in the credential token creds = None if os.path.exists(token_fn := johnnyfive.utils.Paths.gmail_token): - creds = google.oauth2.credentials.Credentials.from_authorized_user_file( - token_fn, SCOPES - ) + try: + creds = google.oauth2.credentials.Credentials.from_authorized_user_file( + token_fn, SCOPES + ) + except json.decoder.JSONDecodeError as err: + raise johnnyfive.utils.J5Error( + f"Cannot parse Gmail token in {token_fn}" + ) from err # If there are no (valid) credentials available... if not creds or not creds.valid: @@ -489,12 +495,12 @@ def setup_gmail( google.auth.transport.requests.Request(), logger=logger, ) - except (googleapiclient.errors.HttpError, ConnectionError) as error: + except (googleapiclient.errors.HttpError, ConnectionError) as err: johnnyfive.utils.proper_print( - f"An error occurred within setup_gmail(): {error}", "warn", logger + f"An error occurred within setup_gmail(): {err}", "warn", logger ) - except google.auth.exceptions.RefreshError as error: - raise johnnyfive.utils.J5Error from error + except google.auth.exceptions.RefreshError as err: + raise johnnyfive.utils.J5Error from err # If running in `interactive`, lauch browser to log in elif interactive: @@ -528,12 +534,12 @@ def setup_gmail( except ( googleapiclient.errors.HttpError, googleapiclient.errors.UnknownApiNameOrVersion, - ) as error: + ) as err: # TODO(developer) - Handle errors from gmail API. johnnyfive.utils.proper_print( - f"An error occurred within setup_gmail():\n{error}", "except", logger + f"An error occurred within setup_gmail():\n{err}", "except", logger ) - raise johnnyfive.utils.J5Error from error + raise johnnyfive.utils.J5Error from err def authenticate_gmail(logger: logging.Logger = None): From bc7e5a8288e4e411845e58ae5478da1be68c9c23 Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Tue, 18 Feb 2025 15:30:34 -0700 Subject: [PATCH 11/15] Add more useful error message modified: johnnyfive/gmail.py --- johnnyfive/gmail.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index b2989ed..a35ce94 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -500,7 +500,11 @@ def setup_gmail( f"An error occurred within setup_gmail(): {err}", "warn", logger ) except google.auth.exceptions.RefreshError as err: - raise johnnyfive.utils.J5Error from err + raise johnnyfive.utils.J5Error( + f"{type(err).__name__} {err}\n" + "https://stackoverflow.com/questions/10576386/invalid-grant-trying-to-get-oauth-token-from-google\n" + "Try running j5_authenticate_gmail" + ) from err # If running in `interactive`, lauch browser to log in elif interactive: @@ -512,15 +516,18 @@ def setup_gmail( # Otherwise, raise an exception and specify to run interactively else: - johnnyfive.utils.proper_print( + errmsg = ( "No Gmail credentials found. You may need to run:\n" "\t`j5_install_conf`\n" "\tor to authenticate user, run (NOT in a container):\n" - "\t`j5_authenticate_gmail`", + "\t`j5_authenticate_gmail`" + ) + johnnyfive.utils.proper_print( + errmsg, "error", logger, ) - raise johnnyfive.utils.J5Error + raise johnnyfive.utils.J5Error(errmsg) # Save the credentials for the next run with open(token_fn, "w", encoding="utf-8") as token: From 8820e0b2218abbaa659b500fe3dbf4e768edb8b0 Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Tue, 25 Feb 2025 08:45:52 -0700 Subject: [PATCH 12/15] Additional error catching in utils modified: johnnyfive/utils.py --- johnnyfive/utils.py | 53 ++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/johnnyfive/utils.py b/johnnyfive/utils.py index 915d775..5514d57 100644 --- a/johnnyfive/utils.py +++ b/johnnyfive/utils.py @@ -40,7 +40,7 @@ # Set API Components -__all__ = ["safe_service_connect", "print_dict", "proper_print"] +__all__ = ["safe_service_connect", "print_dict", "proper_print", "J5Error"] # Define error classes @@ -142,11 +142,24 @@ def read_ligmos_conffiles( An object with arrtibutes matching the keys in the associated configuration file. """ - ligconf = ligmos.utils.confparsers.rawParser(os.path.join(Paths.config, conffile)) - ligconf = ligmos.workers.confUtils.assignConf( - ligconf[confname], authTarget, backfill=True - ) - return ligconf + try: + ligconf = ligmos.utils.confparsers.rawParser( + os.path.join(Paths.config, conffile) + ) + ligconf = ligmos.workers.confUtils.assignConf( + ligconf[confname], authTarget, backfill=True + ) + return ligconf + except KeyError as err: + raise J5Error( + f"Configuration key {confname} not present.\n" + "Try installing configuration files via j5 utilities." + ) from err + except Exception as err: + raise J5Error( + "Unexpected error occurred while reading in configuration file.\n" + f"\n{type(err).__name__} {err.args}" + ) def print_dict(dd: dict, indent: int = 0, di: int = 4): @@ -227,10 +240,10 @@ def safe_service_connect( ConnectionError, google.auth.exceptions.TransportError, httplib2.error.ServerNotFoundError, - ) as exception: + ) as err: proper_print( f"Execution of `{func.__name__}` failed because of network error." - f"\n{exception}", + f"\n{err}", "error", logger, ) @@ -251,15 +264,15 @@ def safe_service_connect( break # This is for a Service error (premissions, etc.), no retry - except requests.exceptions.HTTPError as exception: + except requests.exceptions.HTTPError as err: proper_print( f"Execution of `{func.__name__}` failed because of HTTP error." - f"\n{type(exception).__name__} {exception.args}", + f"\n{type(err).__name__} {err.args}", "error", logger, ) proper_print("Aborting...", "except", logger) - raise exception + raise err # # Gmail service error, no retry and pass the exception upward # except googleapiclient.errors.HttpError as exception: @@ -271,34 +284,34 @@ def safe_service_connect( # raise exception # Slack service error, no retry and pass the exception upward - except slack_sdk.errors.SlackApiError as exception: + except slack_sdk.errors.SlackApiError as err: proper_print( - f"Caught Slack API error... passing up. {type(exception).__name__}", + f"Caught Slack API error... passing up. {type(err).__name__}", "except", logger, ) - raise exception + raise err # Confluence service error, no retry and pass the excepetion upward - except atlassian.errors.ApiError as exception: + except atlassian.errors.ApiError as err: proper_print( - f"Caught Atlassian API Error... passing up. {type(exception).__name__}", + f"Caught Atlassian API Error... passing up. {type(err).__name__}", "except", logger, ) - raise exception + raise err # Google RefreshError occurs when the gmail_token.json to too old - except google.auth.exceptions.RefreshError as exception: + except google.auth.exceptions.RefreshError as err: proper_print( "Google Token Refresh Error.\n" - f"\tDescription: {exception.args[0]}\n" + f"\tDescription: {err.args[0]}\n" "\tIf the reason is 'Token has been expired or revoked', then run\n" "\t`j5_authenticate_gmail` to refresh the token.", "error", logger, ) - raise exception + raise err # If not successful, raise error raise J5Error("Unspecified error") From 1e21119b327cf2f154d17bcde7bf9385f62a37a7 Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Mon, 5 May 2025 13:17:16 -0700 Subject: [PATCH 13/15] Add additional exceptions for `safe_service_connect()` Add more exceptions to safely catch when connecting to a service. modified: johnnyfive/utils.py --- johnnyfive/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/johnnyfive/utils.py b/johnnyfive/utils.py index 5514d57..ce2ca96 100644 --- a/johnnyfive/utils.py +++ b/johnnyfive/utils.py @@ -18,6 +18,7 @@ # Built-In Libraries import argparse +import dataclasses from importlib import resources import logging import os @@ -52,6 +53,7 @@ class J5Error(Exception): # Classes to hold useful information +@dataclasses.dataclass class Paths: """Paths @@ -65,6 +67,7 @@ class Paths: gmail_creds = config / "gmail_credentials.json" +@dataclasses.dataclass class authTarget(ligmos.utils.classes.baseTarget): """Extension of LIGMOS baseTarget class @@ -159,7 +162,7 @@ def read_ligmos_conffiles( raise J5Error( "Unexpected error occurred while reading in configuration file.\n" f"\n{type(err).__name__} {err.args}" - ) + ) from err def print_dict(dd: dict, indent: int = 0, di: int = 4): @@ -238,8 +241,10 @@ def safe_service_connect( # This is a network error... retry except ( ConnectionError, + TimeoutError, google.auth.exceptions.TransportError, httplib2.error.ServerNotFoundError, + requests.exceptions.ReadTimeout, ) as err: proper_print( f"Execution of `{func.__name__}` failed because of network error." From 422f3ca440e973d42b148fc1d1887fd1282163b8 Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Thu, 6 Nov 2025 12:14:28 -0700 Subject: [PATCH 14/15] Remove dependence on ligmos Was only using the configuration file bits, so just copied those over. Don't need the large list of dependencies that ligmos has when using just this library. modified: johnnyfive/utils.py modified: setup.py --- johnnyfive/utils.py | 215 ++++++++++++++++++++++++++++++++++---------- setup.py | 8 +- 2 files changed, 172 insertions(+), 51 deletions(-) diff --git a/johnnyfive/utils.py b/johnnyfive/utils.py index ce2ca96..3521898 100644 --- a/johnnyfive/utils.py +++ b/johnnyfive/utils.py @@ -18,10 +18,11 @@ # Built-In Libraries import argparse +import configparser import dataclasses from importlib import resources import logging -import os +import pathlib import shutil import time import typing @@ -34,9 +35,6 @@ import requests import slack_sdk.errors -# Lowell Libraries -import ligmos - # Internal Imports @@ -68,7 +66,25 @@ class Paths: @dataclasses.dataclass -class authTarget(ligmos.utils.classes.baseTarget): +class baseTarget: + """ + Empty class that gets inherited by basically everything since it contains + most/all the usual stuff you'd need to connect to a ... thing. + """ + + def __init__(self): + self.name = None + self.host = None + self.port = 22 + self.type = None + self.user = None + self.protocol = None + self.password = None + self.enabled = False + + +@dataclasses.dataclass +class authTarget(baseTarget): """Extension of LIGMOS baseTarget class Adds specified attributes used in JohnnyFive to silence LIGMOS's @@ -84,6 +100,63 @@ def __init__(self): self.tokenSecret = None +def assignConf(conf, obj, backfill=False, debug=False): + """ + Given an arbitrary class reference and a parsed configuration file (conf), + assign keys from the latter into parameters in the former. + + Assumes that ALL keys in the class are present in the configuration; if + they aren't, then they're set to ```None``` and caught/announced in the + ```KeyError``` exception below. + + If 'backfill' is False, parameters that are in the *configuration file* + but not in the given class are *ignored* completely. If True, + they're added to the given class with a warning. + """ + # Make an instance of our given object/class + classy = obj() + + # Get the list of parameters in the instance (classy) given class (obj) + oparams = list(classy.__dict__.keys()) + + # Now do the same for the configuration object (conf) + cparams = list(conf.keys()) + + # Check to see if there are any that are in the class but not in the conf + # If there are, keydiffs will != [] and they'll be shoved into the class + # with a warning if backfill is True, otherwise they're ignored entirely + keydiffs = list(set(cparams) - set(oparams)) + + for key in classy.__dict__: + try: + # Remember: key is from the input class here + kval = conf[key] + + # Check to see if it's a comma-separated-list, and other parsing + # stuff happens to check for none/true/false + nkval = valChecks(kval) + + # Actually set the parameter (key) in the class (classy) + # to the value that we found/cleaned up (nkval) + setattr(classy, key, nkval) + except KeyError: + # This means that + if debug is True: + print("Missing expected configuration key %s" % (key)) + # Just set it to None and move on with our lives + setattr(classy, key, None) + + if backfill is True: + # If there are any, that is + if keydiffs != []: + for orphan in keydiffs: + orphVal = valChecks(conf[orphan]) + print("Setting orphan object key %s to %s" % (orphan, orphVal)) + setattr(classy, orphan, orphVal) + + return classy + + def install_conffiles(args: object = None): """Console Script for installing configuration files @@ -113,7 +186,7 @@ def install_conffiles(args: object = None): # Now, loop through the files privided for file in res.files: # Skip things that aren't files - if not isinstance(file, str) or not os.path.isfile(file): + if not isinstance(file, str) or not pathlib.Path(file).is_file(): print(f"Argument {file} is not a file... skipping.") continue @@ -125,7 +198,7 @@ def install_conffiles(args: object = None): def read_ligmos_conffiles( confname: str, conffile: str = "johnnyfive.conf" -) -> ligmos.utils.classes.baseTarget: +) -> baseTarget: """Read a configuration file using LIGMOS Having this as a separate function may be a bit of an overkill, but it @@ -141,17 +214,13 @@ def read_ligmos_conffiles( Returns ------- - :class:`~ligmos.utils.classes.baseTarget` + :class:`baseTarget` An object with arrtibutes matching the keys in the associated configuration file. """ try: - ligconf = ligmos.utils.confparsers.rawParser( - os.path.join(Paths.config, conffile) - ) - ligconf = ligmos.workers.confUtils.assignConf( - ligconf[confname], authTarget, backfill=True - ) + ligconf = rawParser(Paths.config / conffile) + ligconf = assignConf(ligconf[confname], authTarget, backfill=True) return ligconf except KeyError as err: raise J5Error( @@ -197,6 +266,58 @@ def print_dict(dd: dict, indent: int = 0, di: int = 4): print(f"{' '*indent}{key:12s}: {value}") +def proper_print(msg: str, level: str, logger: logging.Logger = None): + """Log if logger, else print to stdout + + _extended_summary_ + + Parameters + ---------- + msg : :obj:`str` + The message to convey + level : ;obj:`str` + The logging level. One of [``info``,``warn``,``except``] + logger : :obj:`~logging.Logger`, optional + The logger object for logging [Default: None] + """ + if level == "info": + if logger is None: + print(msg) + else: + logger.info(msg) + elif level == "warn": + if logger is None: + warnings.warn(msg) + else: + logger.warning(msg) + elif level == "error": + if logger is None: + warnings.warn(f"EXCEPTION: {msg}") + else: + logger.error(msg) + elif level == "except": + if logger is None: + warnings.warn(f"EXCEPTION: {msg}") + else: + logger.exception(msg) + + +def rawParser(confname): + """ + A simple minded parsing of the given confname file. + Returns a configparser object. + """ + config = None + try: + config = configparser.ConfigParser() + config.read_file(open(confname, "r")) + except IOError as err: + print("ERROR: Configuration file %s not found!" % (confname)) + print(str(err)) + + return config + + def safe_service_connect( func: typing.Callable, *args, @@ -322,37 +443,37 @@ def safe_service_connect( raise J5Error("Unspecified error") -def proper_print(msg: str, level: str, logger: logging.Logger = None): - """Log if logger, else print to stdout - - _extended_summary_ - - Parameters - ---------- - msg : :obj:`str` - The message to convey - level : ;obj:`str` - The logging level. One of [``info``,``warn``,``except``] - logger : :obj:`~logging.Logger`, optional - The logger object for logging [Default: None] - """ - if level == "info": - if logger is None: - print(msg) - else: - logger.info(msg) - elif level == "warn": - if logger is None: - warnings.warn(msg) +def valChecks(kval): + """ """ + # It'll always be a string by this point, so it should always + # have a .split() method. If not, someone else has mucked about + # with the configuration object before it got here. + kval = kval.strip().split(",") + + # Trim off leading/trailing whitespace for each. Also make sure + # that it's a list, no matter what, so we can itterate over it. + kval = [kv.strip() for kv in kval] + + # kval is now definitely a list + allval = [] + for val in kval: + # Some icky type checks + if val.lower() == "none": + nkval = None + elif val.lower() == "false": + nkval = False + elif val.lower() == "true": + nkval = True else: - logger.warning(msg) - elif level == "error": - if logger is None: - warnings.warn(f"EXCEPTION: {msg}") - else: - logger.error(msg) - elif level == "except": - if logger is None: - warnings.warn(f"EXCEPTION: {msg}") - else: - logger.exception(msg) + nkval = val + # Put it into a list in case there's more than one + allval.append(nkval) + + # If there's just one thing that we found, return it alone. Otherwise + # return the full list of stuff + if len(allval) == 1: + nkval = allval[0] + else: + nkval = allval + + return nkval diff --git a/setup.py b/setup.py index 0b01335..bb23073 100644 --- a/setup.py +++ b/setup.py @@ -102,8 +102,9 @@ # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', ], # This field adds keywords for your project which will appear on the @@ -138,8 +139,7 @@ 'pyjwt', 'python-twitter', 'requests', - 'slack_sdk', - 'ligmos @ git+https://github.com/LowellObservatory/ligmos'], + 'slack_sdk'], # List additional groups of dependencies here (e.g. development # dependencies). Users will be able to install these using the "extras" From c11ecb3a4e352a2b4771efc9dc5f5efbbda95249 Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Tue, 8 Sep 2026 23:11:27 -0700 Subject: [PATCH 15/15] Modernize JohnnyFive service integrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Modernize JohnnyFive packaging for Python 3.14 with a standards-based pyproject.toml, compatible runtime constraints, SPDX MPL-2.0 license identifiers, complete type annotations, and NumPy-style API documentation. Replace the project’s Ligmos dependency surface with the small, local compatibility functionality J5 actually uses, keeping installation and deployment lighter while retaining the established public behavior. Add unit and integration-style tests for the core utilities and service integrations, providing regression coverage for configuration, Gmail message rendering, and Confluence page operations. Make Gmail message rendering resilient to multipart and nested MIME structures. Messages without a directly rendered body no longer cause an IndexError during automated report processing. Move Confluence connectivity to atlassian-python-api v5 and the explicit ConfluenceServer REST client for both password and bearer-token authentication. Remove the deprecated JSON-RPC permission preflight and avoid administrator-only REST permission enumeration. J5 now lets each required REST operation enforce the automation account’s actual privileges. Verify the complete suite in the j5 conda environment, including client construction and REST operation compatibility checks, so the updated integrations remain suitable for current service APIs. This consolidated change captures the Python 3.14 modernization, reduced dependency footprint, service API migrations, documentation, typing, and test coverage in one coherent release update. Co-authored-by: Codex --- MANIFEST.in | 9 +- README.md | 13 +- ToyModels/old_email.py | 66 +++++-- ToyModels/tweetTester.py | 31 +++- examples/gmail_example.py | 14 +- examples/slack_example.py | 8 +- johnnyfive/__init__.py | 38 +++- johnnyfive/classes.py | 19 +- johnnyfive/confluence.py | 122 ++++++------- johnnyfive/gmail.py | 208 +++++++++++++++------- johnnyfive/slack.py | 32 ++-- johnnyfive/utils.py | 127 ++++++++++---- pyproject.toml | 56 ++++++ requirements.txt | 3 + setup.cfg | 12 -- setup.py | 199 --------------------- tests/test_service_integrations.py | 272 +++++++++++++++++++++++++++++ tests/test_utils.py | 83 +++++++++ 18 files changed, 874 insertions(+), 438 deletions(-) create mode 100644 requirements.txt delete mode 100644 setup.cfg delete mode 100644 setup.py create mode 100644 tests/test_service_integrations.py create mode 100644 tests/test_utils.py diff --git a/MANIFEST.in b/MANIFEST.in index 8785f35..00c21ec 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,13 @@ include README.md include LICENSE -include setup.cfg +include pyproject.toml -recursive-include johnnyfive/config * +include johnnyfive/config/README.md +include johnnyfive/config/email.conf-TEMPLATE +include johnnyfive/config/emailFooter.txt +include johnnyfive/config/gmail_credentials.json-TEMPLATE +include johnnyfive/config/gmail_token.json-TEMPLATE +include johnnyfive/config/johnnyfive.conf-TEMPLATE recursive-include johnnyfive/images * global-exclude *.pyc *.o *.so *.DS_Store diff --git a/README.md b/README.md index 2d9c701..a117472 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,13 @@ slack_channel = johnnyfive.SlackChannel(channel_name) - google-api-python-client - google-auth-httplib2 - google-auth-oauthlib +- httplib2 - lxml -- pyjwt -- python-twitter - requests - slack_sdk -- ligmos @ https://github.com/LowellObservatory/ligmos + +JohnnyFive parses its own INI-style configuration file and does not require +`ligmos`. ## Installation @@ -52,6 +53,8 @@ Installable either as a standalone library: Or as a dependancy for other software: -- In your package's `setup.cfg` file, add: +- In your package's `pyproject.toml` file, add: - ```install_requires = JohnnyFive @ git+https://github.com/LowellObservatory/JohnnyFive``` + ```toml + dependencies = ["JohnnyFive @ git+https://github.com/LowellObservatory/JohnnyFive"] + ``` diff --git a/ToyModels/old_email.py b/ToyModels/old_email.py index 023857f..e7831da 100644 --- a/ToyModels/old_email.py +++ b/ToyModels/old_email.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 # # Created on 25 Feb 2020 # @@ -19,11 +17,35 @@ import socket import smtplib from email.message import EmailMessage - - -def sendMail(message, smtploc='localhost', port=25, user=None, passw=None): - """ - This assumes that 'message' is an instance of EmailMessage +from typing import Any + + +def sendMail( + message: EmailMessage, + smtploc: str = "localhost", + port: int | str = 25, + user: str | None = None, + passw: str | None = None, +) -> bool: + """Send an email message via an unencrypted or SSL SMTP connection. + + Parameters + ---------- + message : EmailMessage + Message to send. + smtploc : str, optional + SMTP host name. + port : int | str, optional + SMTP port number. + user : str | None, optional + User name for SSL authentication. + passw : str | None, optional + Password for SSL authentication. + + Returns + ------- + bool + Whether the message was sent successfully. """ # Ultimate return value to know whether we need to try again later success = False @@ -77,8 +99,32 @@ def sendMail(message, smtploc='localhost', port=25, user=None, passw=None): return success -def constructMail(subject, body, fromaddr, toaddr, fromname=None): - """ +def constructMail( + subject: str, + body: str, + fromaddr: str, + toaddr: str, + fromname: str | None = None, +) -> EmailMessage: + """Construct a plain-text email message. + + Parameters + ---------- + subject : str + Email subject. + body : str + Plain-text message body. + fromaddr : str + Sender address. + toaddr : str + Recipient address. + fromname : str | None, optional + Sender display name. + + Returns + ------- + EmailMessage + Configured email message. """ msg = EmailMessage() if fromname is None: diff --git a/ToyModels/tweetTester.py b/ToyModels/tweetTester.py index f3e681c..39f099e 100644 --- a/ToyModels/tweetTester.py +++ b/ToyModels/tweetTester.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 # # Created on 7 Feb 2020 # @@ -15,13 +13,27 @@ from __future__ import division, print_function, absolute_import +import configparser +from pathlib import Path +from typing import Mapping + import twitter -import ligmos +def sendMessage(twit: Mapping[str, str], message: str) -> None: + """Post a message to Twitter using configured API credentials. -def sendMessage(twit, message): - """ + Parameters + ---------- + twit : Mapping[str, str] + Twitter API credential mapping. + message : str + Message to post. + + Returns + ------- + None + The API response is printed for this prototype script. """ api = twitter.Api(consumer_key=twit['apiKey'], consumer_secret=twit['apiSecret'], @@ -41,10 +53,11 @@ def sendMessage(twit, message): if __name__ == "__main__": - confFile = '../config/johnnyfive.conf' - conf = ligmos.utils.confparsers.rawParser(confFile) + conf_file = Path(__file__).parents[1] / "johnnyfive" / "config" / "johnnyfive.conf" + conf = configparser.ConfigParser() + conf.read(conf_file) # Quick and dirty for prototyping, will set up classes later - twit = conf['twitterSetup'] + twit = conf["twitterSetup"] sendMessage(twit) diff --git a/examples/gmail_example.py b/examples/gmail_example.py index 34ddef7..c82ed84 100644 --- a/examples/gmail_example.py +++ b/examples/gmail_example.py @@ -1,13 +1,17 @@ -""" Example for using the Gmail module - -_extended_summary_ -""" +"""Demonstrate the Gmail message API.""" from johnnyfive import gmail as j5g from johnnyfive import utils -def main(interactive=False): +def main(interactive: bool = False) -> None: + """Demonstrate sending a Gmail message with an attachment. + + Parameters + ---------- + interactive : bool, optional + Whether OAuth authentication may open a browser. + """ """main Main Testing Driver """ diff --git a/examples/slack_example.py b/examples/slack_example.py index 9c81725..2905d9f 100644 --- a/examples/slack_example.py +++ b/examples/slack_example.py @@ -1,7 +1,4 @@ -""" Example for using the Slack module - -_extended_summary_ -""" +"""Demonstrate the Slack channel API.""" import os @@ -9,7 +6,8 @@ from johnnyfive import utils # Main Testing Driver ========================================================# -def main(): +def main() -> None: + """Demonstrate sending a Slack message and uploading a file.""" """main Main Testing Driver """ slack_object = j5s.SlackChannel('bot_test') diff --git a/johnnyfive/__init__.py b/johnnyfive/__init__.py index d131c4f..08a3415 100644 --- a/johnnyfive/__init__.py +++ b/johnnyfive/__init__.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 # # Created on 07-Mar-2022 # @@ -13,6 +11,8 @@ # Imports for signal and log handling import os +from types import TracebackType +from typing import IO, Type import warnings __all__ = ["ConfluencePage", "GmailMessage", "GetMessages", "SlackChannel"] @@ -24,9 +24,35 @@ from .utils import * # noqa -def short_warning(message, category, filename, lineno, file=None, line=None): - """ - Return the format for a short warning message. +def short_warning( + message: Warning | str, + category: Type[Warning], + filename: str, + lineno: int, + file: IO[str] | None = None, + line: str | None = None, +) -> str: + """Format a warning as a concise single line. + + Parameters + ---------- + message : Warning | str + Warning text or warning instance. + category : type[Warning] + Warning category. + filename : str + Source filename. + lineno : int + Source line number. + file : IO[str] | None, optional + Unused output stream accepted for the warnings hook protocol. + line : str | None, optional + Unused source line accepted for the warnings hook protocol. + + Returns + ------- + str + Formatted warning line. """ return f" {category.__name__}: {message} ({os.path.split(filename)[1]}:{lineno})\n" diff --git a/johnnyfive/classes.py b/johnnyfive/classes.py index 7818217..092341f 100644 --- a/johnnyfive/classes.py +++ b/johnnyfive/classes.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 # # Created on 25 Feb 2020 # @@ -16,10 +14,19 @@ from __future__ import division, print_function, absolute_import -class emailSNMP(object): - """emailSNMP _summary_""" +class emailSNMP: + """Store SMTP connection and message configuration. - def __init__(self): + Attributes + ---------- + host : str | None + SMTP server host name. + port : int + SMTP server port. + """ + + def __init__(self) -> None: + """Initialize an SMTP configuration with safe defaults.""" self.host = None self.port = 465 self.user = None diff --git a/johnnyfive/confluence.py b/johnnyfive/confluence.py index 58edd44..3893922 100644 --- a/johnnyfive/confluence.py +++ b/johnnyfive/confluence.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 # # Created on 23-Sep-2021 # @@ -16,9 +14,10 @@ # Built-In Libraries import logging +from typing import Any # 3rd Party Libraries -import atlassian +from atlassian.confluence import ConfluenceServer import requests # Internal Imports @@ -32,7 +31,7 @@ class ConfluencePage: """ConfluencePage Class for a single Confluence Page - _extended_summary_ + Provides permission-aware operations for one Confluence page. Parameters ---------- @@ -40,7 +39,7 @@ class ConfluencePage: The name of the Confluence space for this page page_title : :obj:`str` The page title - instance : :class:`~atlassian.Confluence`, optional + instance : :class:`~atlassian.confluence.ConfluenceServer`, optional An existing Confluence object instance to be used instead of reinstantiating a new Confluence object for communication and authentication. [Default: None] @@ -54,10 +53,25 @@ def __init__( self, space: str, page_title: str, - instance: atlassian.Confluence = None, + instance: ConfluenceServer | None = None, use_oauth: bool = False, - logger: logging.Logger = None, - ): + logger: logging.Logger | None = None, + ) -> None: + """Initialize a page wrapper and fetch its metadata. + + Parameters + ---------- + space : str + Confluence space key. + page_title : str + Page title within ``space``. + instance : ConfluenceServer | None, optional + Existing authenticated client. + use_oauth : bool, optional + Whether to create a bearer-token client. + logger : logging.Logger | None, optional + Logger used for service errors. + """ # Initialize instance variables self.space = space self.title = page_title @@ -65,16 +79,14 @@ def __init__( # Set up the Confluence object instance self.confluence = ( - setup_confluence(use_oauth=use_oauth) - if not isinstance(instance, atlassian.Confluence) - else instance + setup_confluence(use_oauth=use_oauth) if instance is None else instance ) self.space_perms = self._set_permdict() # Set the class metadata based on this page self._set_metadata() - def add_comment(self, comment: str): + def add_comment(self, comment: str) -> None: """Add a comment to the Confluence page Sometimes it's helpful to include a comment at the bottom of the @@ -93,7 +105,7 @@ def add_comment(self, comment: str): self.confluence.add_comment, self.page_id, comment, logger=self.logger ) - def add_label(self, label: str): + def add_label(self, label: str) -> None: """Add a label to the Confluence page Sometimes it's helpful to have a label on a Confluence page for @@ -114,10 +126,10 @@ def add_label(self, label: str): def attach_file( self, filename: str, - name: str = None, - content_type: str = None, - comment: str = None, - ): + name: str | None = None, + content_type: str | None = None, + comment: str | None = None, + ) -> None: """Attach a file to this page Wrapper for the Confluence method attach_file() that includes the @@ -148,8 +160,8 @@ def attach_file( ) def create( - self, page_body: str, parent_id: str = None, representation: str = "wiki" - ): + self, page_body: str, parent_id: str | None = None, representation: str = "wiki" + ) -> None: """Create a brand new Confluence page Summon from the depths of computing a new page. @@ -188,7 +200,7 @@ def create( # Set the instance metadata (exists, page_id, etc.) self._set_metadata() - def delete_attachment(self, filename: str): + def delete_attachment(self, filename: str) -> None: """Delete an attachment from this page Wrapper for the Confluence method delete_attachment() that includes the @@ -254,7 +266,7 @@ def get_page_contents(self) -> str: # Extract the contents from the return object return contents["body"]["storage"]["value"] - def smite(self): + def smite(self) -> None: """smite Kill with extreme prejudice Remove the Confluence page and update the instance metadata to reflect @@ -268,7 +280,7 @@ def smite(self): ) self._set_metadata() - def update_contents(self, body: str): + def update_contents(self, body: str) -> None: """Update the contents of the Confluence page Update the page by replacing the existing content with new. The idea @@ -327,17 +339,14 @@ def _check_perm(self, perm_key: str, perm_action: str) -> bool: ) return False - # If value is None, no permission check was performed, proceed + # If value is None, permission preflight is disabled; let the REST + # operation itself enforce the authenticated user's permissions. if perm_val is None: - johnnyfive.utils.proper_print( - "Permissions check is disabled... hoping for the best.", - "warn", - self.logger, - ) + return True return True - def _set_metadata(self): + def _set_metadata(self) -> None: """Set the various instance metadata Especially after a page is created or deleted, this method updates the @@ -361,45 +370,24 @@ def _set_metadata(self): else f"{self.confluence.url}download/attachments/{self.page_id}/" ) - def _set_permdict(self) -> dict: - """Create a dictionary of permissions + def _set_permdict(self) -> dict[str, bool]: + """Disable permission enumeration for REST-based automation clients. - This method creates a dictionary of permissions for this user in this - space. Each item in the dictionary is boolean based on the results of - the method :func:`confluence.get_space_permissions`. + Current Confluence REST deployments can require space-administrator + privileges to enumerate all permissions. J5 only needs the narrower + privileges for each page operation, so it lets those REST operations + perform the authoritative authorization check instead. Returns ------- - :obj:`dict` - The dictionary of permissions (boolean) + dict[str, bool] + Empty map indicating that permission preflight is disabled. """ - perms = johnnyfive.utils.safe_service_connect( - self.confluence.get_space_permissions, self.space, logger=self.logger - ) - - # Check to see if the authenticated user can view permissions - if not perms: - johnnyfive.utils.proper_print( - f"User {self.confluence.username} needs permission to view " - f"permissions in space {self.space}. Contact " - "your Confluence administrator.", - "warn", - self.logger, - ) - - perm_dict = {} - for perm in perms: - # Set this permission as false... will update to True if needed - perm_dict[perm["type"]] = False - for space_perm in perm["spacePermissions"]: - if space_perm["userName"] == self.confluence.username: - perm_dict[perm["type"]] = True - - return perm_dict + return {} # Internal Functions =========================================================# -def setup_confluence(use_oauth: bool = False) -> atlassian.Confluence: +def setup_confluence(use_oauth: bool = False) -> ConfluenceServer: """Set up the Confluence class instance Reads in the confluence.conf configuration file, which contains the URL, @@ -416,19 +404,19 @@ def setup_confluence(use_oauth: bool = False) -> atlassian.Confluence: Returns ------- - confluence : :class:`~atlassian.Confluence` + confluence : :class:`~atlassian.confluence.ConfluenceServer` Confluence class, initialized with credentials """ # Read the setup - setup = johnnyfive.utils.read_ligmos_conffiles("confluenceSetup") + setup = johnnyfive.utils.read_config_section("confluenceSetup") - # If we are using OAUTH, instantiate a Confluence object with it + # If we are using OAuth, instantiate a Server client with its bearer token. if use_oauth: session = requests.Session() session.headers["Authorization"] = f"Bearer {setup.access_token}" - return atlassian.Confluence(url=setup.host, session=session) + return ConfluenceServer(url=setup.host, session=session) - # Else, return a Confluence object instantiated with username/password - return atlassian.Confluence( + # Otherwise, return a Server client instantiated with username/password. + return ConfluenceServer( url=setup.host, username=setup.user, password=setup.password ) diff --git a/johnnyfive/gmail.py b/johnnyfive/gmail.py index a35ce94..38a1f17 100644 --- a/johnnyfive/gmail.py +++ b/johnnyfive/gmail.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 # # Created on 15 Feb 2022 # @@ -27,6 +25,9 @@ import logging import mimetypes import os +from collections.abc import Iterator, Mapping +from pathlib import Path +from typing import Any # 3rd Party Libraries from bs4 import BeautifulSoup @@ -52,7 +53,7 @@ class GmailMessage: """Class for a single Gmail Message - _extended_summary_ + Builds MIME messages and sends them through an authenticated Gmail service. Parameters ---------- @@ -75,20 +76,39 @@ class GmailMessage: def __init__( self, - toaddr: str | list, + toaddr: str | list[str], subject: str, message_text: str, - fromname: str = None, - fromaddr: str = None, + fromname: str | None = None, + fromaddr: str | None = None, interactive: bool = False, - logger: logging.Logger = None, - ): + logger: logging.Logger | None = None, + ) -> None: + """Build a Gmail MIME message and initialize its API service. + + Parameters + ---------- + toaddr : str | list[str] + Recipient address or addresses. + subject : str + Email subject. + message_text : str + Plain-text body. + fromname : str | None, optional + Sender display name. + fromaddr : str | None, optional + Sender address; defaults to the J5 configuration value. + interactive : bool, optional + Whether OAuth authorization may open a browser. + logger : logging.Logger | None, optional + Logger used for service errors. + """ # Set the logger, if passed self.logger = logger # Load default `fromaddr`` if None passed in if not fromaddr: - fromaddr = johnnyfive.utils.read_ligmos_conffiles("gmailSetup").user + fromaddr = johnnyfive.utils.read_config_section("gmailSetup").user # Initialize the Gmail connection self.service = setup_gmail(interactive=interactive, logger=self.logger) @@ -102,10 +122,10 @@ def __init__( # Place the text into the message self.message.attach(email.mime.text.MIMEText(message_text)) - def add_attachment(self, file: str): + def add_attachment(self, file: str | Path) -> None: """Add an attachment to the GMAIL message - _extended_summary_ + The attachment MIME type is inferred from its filename. Parameters ---------- @@ -122,7 +142,7 @@ def add_attachment(self, file: str): # Case out the content type main_type, sub_type = content_type.split("/", 1) if main_type == "text": - with open(file, "rb") as f_obj: + with open(file, encoding="utf-8") as f_obj: attachment = email.mime.text.MIMEText(f_obj.read(), _subtype=sub_type) elif main_type == "image": with open(file, "rb") as f_obj: @@ -141,10 +161,10 @@ def add_attachment(self, file: str): ) self.message.attach(attachment) - def send(self) -> dict: + def send(self) -> dict[str, Any]: """Send the GmailMessage - _extended_summary_ + Encodes the MIME message and sends it through Gmail's API. Returns ------- @@ -181,7 +201,7 @@ def send(self) -> dict: class GetMessages: """Get Gmail messages corresponding to given criteria - _extended_summary_ + Queries Gmail messages and exposes helpers for rendering and relabeling them. Parameters ---------- @@ -201,12 +221,27 @@ class GetMessages: def __init__( self, - label: str = None, - after: str = None, - before: str = None, + label: str | None = None, + after: str | None = None, + before: str | None = None, interactive: bool = False, - logger: logging.Logger = None, - ): + logger: logging.Logger | None = None, + ) -> None: + """Connect to Gmail and collect messages matching search criteria. + + Parameters + ---------- + label : str | None, optional + Gmail label name used to filter messages. + after : str | None, optional + Inclusive lower date bound in ``YYYY/MM/DD`` form. + before : str | None, optional + Exclusive upper date bound in ``YYYY/MM/DD`` form. + interactive : bool, optional + Whether OAuth authorization may open a browser. + logger : logging.Logger | None, optional + Logger used for service errors. + """ # Initialize basic stuff self.label_list = None self.message_list = [] @@ -248,7 +283,7 @@ def __init__( self.logger, ) - def render_message(self, message_id: str) -> dict: + def render_message(self, message_id: str) -> dict[str, str]: """Retrieve and render a message by ID# Gmail mnessages are stored in a JSON-like structure that must be @@ -265,14 +300,17 @@ def render_message(self, message_id: str) -> dict: Dictionary containing the subject, sender, date, and body of the message. """ + payload: Mapping[str, Any] | None = None try: # Get the message, then start parsing (API: users.messages.get) results = johnnyfive.utils.safe_service_connect( - self.service.users().messages().get(userId="me", id=message_id).execute, + self.service.users() + .messages() + .get(userId="me", id=message_id, format="full") + .execute, logger=self.logger, ) - payload = results["payload"] - headers = payload["headers"] + payload = results.get("payload", {}) # If exception, print message and return empty values except (googleapiclient.errors.HttpError, ConnectionError) as error: @@ -281,49 +319,91 @@ def render_message(self, message_id: str) -> dict: "except", self.logger, ) - payload = None - # Return empty dictionary if unsuccessful in connecting if not payload: return {"subject": "", "sender": "", "date": "", "body": ""} - # Look for Subject and Sender Email in the headers - for head_dict in headers: - if head_dict["name"] == "Subject": - subject = head_dict["value"] - if head_dict["name"] == "From": - sender = head_dict["value"] - if head_dict["name"] == "Date": - date = head_dict["value"] - - # The Body of the message is in Encrypted format -- decode it. - # Get the data and decode it with base 64 decoder. - - # If more than one part (i.e., HTML or images, etc.), get the first - data = ( - payload["body"]["data"] - if "parts" not in payload - else payload["parts"][0]["body"]["data"] - ) - data = data.replace("-", "+").replace("_", "/") - decoded_data = base64.b64decode(data) - - # `decoded_data` is in lxml format; parse with BeautifulSoup - body = BeautifulSoup(decoded_data, "lxml").body() - body = body[0].text + headers = { + str(header.get("name", "")).lower(): str(header.get("value", "")) + for header in payload.get("headers", []) + if isinstance(header, Mapping) + } # Return a dictionary with the plain-text components of this message - return {"subject": subject, "sender": sender, "date": date, "body": body} + return { + "subject": headers.get("subject", ""), + "sender": headers.get("from", ""), + "date": headers.get("date", ""), + "body": self._extract_message_body(payload), + } + + @staticmethod + def _iter_message_parts(part: Mapping[str, Any]) -> Iterator[Mapping[str, Any]]: + """Yield a MIME part and all of its nested child parts. + + Parameters + ---------- + part : Mapping[str, Any] + Gmail ``MessagePart`` object to traverse. + + Yields + ------ + Mapping[str, Any] + Each MIME part in depth-first order. + """ + yield part + for child in part.get("parts", []): + if isinstance(child, Mapping): + yield from GetMessages._iter_message_parts(child) + + @classmethod + def _extract_message_body(cls, payload: Mapping[str, Any]) -> str: + """Extract readable inline text from a Gmail MIME payload. + + Plain text is preferred when both ``text/plain`` and ``text/html`` + alternatives are present. Container and attachment parts without + inline ``body.data`` are ignored. + + Parameters + ---------- + payload : Mapping[str, Any] + Top-level Gmail ``MessagePart`` payload. + + Returns + ------- + str + Decoded message text, or an empty string when no readable inline + text part exists. + """ + parts = list(cls._iter_message_parts(payload)) + for mime_type in ("text/plain", "text/html"): + for part in parts: + if part.get("mimeType", "").lower() != mime_type: + continue + body = part.get("body", {}) + data = body.get("data") if isinstance(body, Mapping) else None + if not isinstance(data, str) or not data: + continue + + padded_data = data + "=" * (-len(data) % 4) + decoded = base64.urlsafe_b64decode(padded_data).decode( + "utf-8", errors="replace" + ) + if mime_type == "text/plain": + return decoded + return BeautifulSoup(decoded, "lxml").get_text(separator="\n", strip=True) + + return "" def update_msg_labels( self, message_id: str, - add_labels: list[str] = None, - remove_labels: list[str] = None, - ): + add_labels: list[str] | None = None, + remove_labels: list[str] | None = None, + ) -> dict[str, Any]: """Update the labels for a message by ID# - _extended_summary_ + Label names are resolved to Gmail label IDs before the update. Parameters ---------- @@ -373,10 +453,10 @@ def update_msg_labels( # If unsuccessful in connecting, raise raise johnnyfive.utils.J5Error("Unsuccessful connection") - def _label_id_from_name(self, name: str) -> str: + def _label_id_from_name(self, name: str | None) -> str | None: """Get the Label ID from the Label Name - _extended_summary_ + The label list is retrieved once and cached for the instance. Parameters ---------- @@ -424,10 +504,12 @@ def _label_id_from_name(self, name: str) -> str: return label_id @staticmethod - def build_query(after_date: str = None, before_date: str = None) -> str: + def build_query( + after_date: str | None = None, before_date: str | None = None + ) -> str: """build_query Build the query string for users.messages.list - _extended_summary_ + Date filters are formatted for Gmail's message-list query syntax. Parameters ---------- @@ -451,11 +533,11 @@ def build_query(after_date: str = None, before_date: str = None) -> str: # Newer OAUTH Routines =======================================================# def setup_gmail( - interactive: bool = False, logger: logging.Logger = None + interactive: bool = False, logger: logging.Logger | None = None ) -> googleapiclient.discovery.Resource: """Initialize the GMail API (via OAuth) - [extended_summary] + Creates or refreshes the OAuth credentials used by the Gmail API. NOTE: The first time this is run on a machine, it will open a webpage for authorizing the API. All subsequent runs will be silent. @@ -549,7 +631,7 @@ def setup_gmail( raise johnnyfive.utils.J5Error from err -def authenticate_gmail(logger: logging.Logger = None): +def authenticate_gmail(logger: logging.Logger | None = None) -> None: """Console Script for authenticating Gmail This is the command-line script for doing the interactive authentication diff --git a/johnnyfive/slack.py b/johnnyfive/slack.py index b228f2c..b90b71c 100644 --- a/johnnyfive/slack.py +++ b/johnnyfive/slack.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 # # Created on 14-Feb-2022 # @@ -19,6 +17,7 @@ # Built-In Libraries import pathlib +from typing import Any import warnings # 3rd Party Libraries @@ -36,7 +35,7 @@ class SlackChannel: """SlackChannel Class for communicating with a Slack Channel - _extended_summary_ + Resolves a channel name and provides message and file operations. Parameters ---------- @@ -44,16 +43,23 @@ class SlackChannel: Slack Channel into which to post """ - def __init__(self, channel_name: str): + def __init__(self, channel_name: str) -> None: + """Initialize a channel client. + + Parameters + ---------- + channel_name : str + Human-readable Slack channel name. + """ self.client = setup_slack() # Get the channel ID self.channel_id = self._read_channels(channel_name) - def send_message(self, message: str) -> object: + def send_message(self, message: str) -> Any: """Send a (text only) message to the channel - _extended_summary_ + The Slack API response is returned unchanged. Parameters ---------- @@ -82,10 +88,10 @@ def send_message(self, message: str) -> object: ) return response - def upload_file(self, file: str | pathlib.Path, title: str = None): + def upload_file(self, file: str | pathlib.Path, title: str | None = None) -> Any: """Upload a file to the channel - _extended_summary_ + The Slack API response is returned unchanged. Parameters ---------- @@ -113,7 +119,7 @@ def upload_file(self, file: str | pathlib.Path, title: str = None): ) return response - def _read_channels(self, name: str) -> str: + def _read_channels(self, name: str) -> str | None: """Return the Channel ID for the names channel Parameters @@ -151,10 +157,10 @@ def _read_channels(self, name: str) -> str: # Internal Functions =========================================================# -def setup_slack() -> slack_sdk.web.client.WebClient: +def setup_slack() -> slack_sdk.web.client.WebClient | None: """Setup the Slack WebClient for posting - _extended_summary_ + Reads the configured token and creates a client for Slack API calls. Returns ------- @@ -162,7 +168,7 @@ def setup_slack() -> slack_sdk.web.client.WebClient: The WebClient object needed for reading and writing """ # Read the setup - setup = johnnyfive.utils.read_ligmos_conffiles("slackSetup") + setup = johnnyfive.utils.read_config_section("slackSetup") # SlackWebClient instantiates a client that can call API methods # When using Bolt, you can use either `app.client` or the `client` passed to listeners. diff --git a/johnnyfive/utils.py b/johnnyfive/utils.py index 3521898..9931efb 100644 --- a/johnnyfive/utils.py +++ b/johnnyfive/utils.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- # -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 # # Created on 14-Feb-2022 # @@ -39,7 +37,13 @@ # Set API Components -__all__ = ["safe_service_connect", "print_dict", "proper_print", "J5Error"] +__all__ = [ + "J5Error", + "print_dict", + "proper_print", + "read_config_section", + "safe_service_connect", +] # Define error classes @@ -55,7 +59,7 @@ class J5Error(Exception): class Paths: """Paths - [extended_summary] + Centralizes paths to packaged configuration and image resources. """ # Main data & config directories @@ -72,7 +76,8 @@ class baseTarget: most/all the usual stuff you'd need to connect to a ... thing. """ - def __init__(self): + def __init__(self) -> None: + """Initialize a configuration target with common connection fields.""" self.name = None self.host = None self.port = 22 @@ -85,13 +90,13 @@ def __init__(self): @dataclasses.dataclass class authTarget(baseTarget): - """Extension of LIGMOS baseTarget class + """Configuration target with the credentials used by JohnnyFive. - Adds specified attributes used in JohnnyFive to silence LIGMOS's - "Setting orphan object key" messages + Additional values in the configuration section are retained as attributes. """ - def __init__(self): + def __init__(self) -> None: + """Initialize a configuration target with credential fields.""" super().__init__() self.access_token = None self.apiKey = None @@ -100,8 +105,14 @@ def __init__(self): self.tokenSecret = None -def assignConf(conf, obj, backfill=False, debug=False): - """ +def assignConf( + conf: configparser.SectionProxy, + obj: type[baseTarget], + backfill: bool = False, + debug: bool = False, +) -> baseTarget: + """Copy parsed configuration values to a target instance. + Given an arbitrary class reference and a parsed configuration file (conf), assign keys from the latter into parameters in the former. @@ -112,6 +123,22 @@ def assignConf(conf, obj, backfill=False, debug=False): If 'backfill' is False, parameters that are in the *configuration file* but not in the given class are *ignored* completely. If True, they're added to the given class with a warning. + + Parameters + ---------- + conf : configparser.SectionProxy + Configuration section to convert. + obj : type[baseTarget] + Target class to instantiate. + backfill : bool, optional + Whether to retain keys not predefined by ``obj``. + debug : bool, optional + Whether to print missing predefined keys. + + Returns + ------- + baseTarget + Populated configuration target. """ # Make an instance of our given object/class classy = obj() @@ -157,7 +184,7 @@ def assignConf(conf, obj, backfill=False, debug=False): return classy -def install_conffiles(args: object = None): +def install_conffiles(args: typing.Sequence[str] | None = None) -> None: """Console Script for installing configuration files This function is designed to install the (secret) configuration files @@ -196,14 +223,10 @@ def install_conffiles(args: object = None): shutil.copy2(file, Paths.config) -def read_ligmos_conffiles( +def read_config_section( confname: str, conffile: str = "johnnyfive.conf" ) -> baseTarget: - """Read a configuration file using LIGMOS - - Having this as a separate function may be a bit of an overkill, but it - makes it easier to keep the ligmos imports only in one place, and - simplifies the code elsewhere. + """Read a JohnnyFive configuration section into an attribute object. Parameters ---------- @@ -219,9 +242,8 @@ def read_ligmos_conffiles( configuration file. """ try: - ligconf = rawParser(Paths.config / conffile) - ligconf = assignConf(ligconf[confname], authTarget, backfill=True) - return ligconf + config = rawParser(Paths.config / conffile) + return assignConf(config[confname], authTarget, backfill=True) except KeyError as err: raise J5Error( f"Configuration key {confname} not present.\n" @@ -234,7 +256,18 @@ def read_ligmos_conffiles( ) from err -def print_dict(dd: dict, indent: int = 0, di: int = 4): +def read_ligmos_conffiles( + confname: str, conffile: str = "johnnyfive.conf" +) -> baseTarget: + """Backward-compatible alias for :func:`read_config_section`. + + JohnnyFive no longer depends on ligmos; new code should use + :func:`read_config_section`. + """ + return read_config_section(confname, conffile) + + +def print_dict(dd: dict[str, typing.Any], indent: int = 0, di: int = 4) -> None: """Print a dictionary in tree format You know how sometimes you get these nested dictionaries, and they're a @@ -266,10 +299,12 @@ def print_dict(dd: dict, indent: int = 0, di: int = 4): print(f"{' '*indent}{key:12s}: {value}") -def proper_print(msg: str, level: str, logger: logging.Logger = None): +def proper_print( + msg: str, level: str, logger: logging.Logger | None = None +) -> None: """Log if logger, else print to stdout - _extended_summary_ + Selects a logger method or standard warning/output based on ``level``. Parameters ---------- @@ -302,10 +337,18 @@ def proper_print(msg: str, level: str, logger: logging.Logger = None): logger.exception(msg) -def rawParser(confname): - """ - A simple minded parsing of the given confname file. - Returns a configparser object. +def rawParser(confname: str | pathlib.Path) -> configparser.ConfigParser: + """Parse an INI-style configuration file. + + Parameters + ---------- + confname : str | pathlib.Path + Path to the configuration file. + + Returns + ------- + configparser.ConfigParser + Parsed configuration, which is empty if the file cannot be opened. """ config = None try: @@ -319,13 +362,13 @@ def rawParser(confname): def safe_service_connect( - func: typing.Callable, - *args, + func: typing.Callable[..., typing.Any], + *args: typing.Any, pause: int | float = 5, nretries: int = 5, - logger: logging.Logger = None, - **kwargs, -) -> object: + logger: logging.Logger | None = None, + **kwargs: typing.Any, +) -> typing.Any: """Safely connect to Service (includes error-catching) Wrapper for Service-connection functions to catch errors that might be @@ -443,8 +486,20 @@ def safe_service_connect( raise J5Error("Unspecified error") -def valChecks(kval): - """ """ +def valChecks(kval: str) -> str | bool | None | list[str | bool | None]: + """Convert comma-separated configuration values to Python values. + + Parameters + ---------- + kval : str + Raw configuration value. + + Returns + ------- + str | bool | None | list[str | bool | None] + A scalar for one value or a list for multiple values, with literal + ``true``, ``false``, and ``none`` converted to their Python values. + """ # It'll always be a string by this point, so it should always # have a .split() method. If not, someone else has mucked about # with the configuration object before it got here. diff --git a/pyproject.toml b/pyproject.toml index e69de29..dab3c14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -0,0 +1,56 @@ +[build-system] +requires = ["setuptools>=80"] +build-backend = "setuptools.build_meta" + +[project] +name = "johnnyfive" +version = "0.3" +description = "Helper utilities for Lowell Observatory code notifications" +readme = "README.md" +requires-python = ">=3.14" +license = "MPL-2.0" +license-files = ["LICENSE"] +authors = [ + { name = "Lowell Observatory", email = "rhamilton@lowell.edu" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Build Tools", +] +dependencies = [ + "atlassian-python-api>=5.0.4,<6", + "beautifulsoup4>=4.13.0", + "google-api-python-client>=2.200.0", + "google-auth-httplib2>=0.2.0", + "google-auth-oauthlib>=1.2.0", + "httplib2>=0.22.0", + "lxml>=5.3.0", + "requests>=2.32.0", + "slack-sdk>=3.44.1", +] +keywords = ["confluence", "gmail", "notifications", "slack"] + +[project.urls] +"Bug Reports" = "https://github.com/LowellObservatory/JohnnyFive/issues" +Source = "https://github.com/LowellObservatory/JohnnyFive/" + +[project.scripts] +j5_install_conf = "johnnyfive.utils:install_conffiles" +j5_authenticate_gmail = "johnnyfive.gmail:authenticate_gmail" + +[project.optional-dependencies] +dev = ["check-manifest>=0.50"] +test = ["coverage>=7.0", "pytest>=8.0"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["johnnyfive*"] +namespaces = false diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7be44f7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# Runtime dependencies are declared in pyproject.toml. This convenience file +# installs the project and its development/test tools from a source checkout. +.[dev,test] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 3f625fa..0000000 --- a/setup.cfg +++ /dev/null @@ -1,12 +0,0 @@ -[metadata] -# This includes the license file in the wheel. -license_file = LICENSE - -[bdist_wheel] -# This flag says to generate wheels that support both Python 2 and Python -# 3. If your code will not run unchanged on both Python 2 and 3, you will -# need to generate separate wheels for each Python version that you -# support. Removing this line (or setting universal to 0) will prevent -# bdist_wheel from trying to make a universal wheel. For more see: -# https://packaging.python.org/tutorials/distributing-packages/#wheels -universal=1 diff --git a/setup.py b/setup.py deleted file mode 100644 index bb23073..0000000 --- a/setup.py +++ /dev/null @@ -1,199 +0,0 @@ -"""A setuptools based setup module. - -See: -https://packaging.python.org/en/latest/distributing.html -https://github.com/pypa/sampleproject -""" - -# Always prefer setuptools over distutils -from setuptools import setup, find_packages -# To use a consistent encoding -from codecs import open -from os import path - -here = path.abspath(path.dirname(__file__)) - -# Get the long description from the README file -with open(path.join(here, 'README.md'), encoding='utf-8') as f: - long_description = f.read() - -# Arguments marked as "Required" below must be included for upload to PyPI. -# Fields marked as "Optional" may be commented out. - -setup( - # This is the name of your project. The first time you publish this - # package, this name will be registered for you. It will determine how - # users can install this project, e.g.: - # - # $ pip install sampleproject - # - # And where it will live on PyPI: https://pypi.org/project/sampleproject/ - # - # There are some restrictions on what makes a valid project name - # specification here: - # https://packaging.python.org/specifications/core-metadata/#name - name='johnnyfive', # Required - - # Versions should comply with PEP 440: - # https://www.python.org/dev/peps/pep-0440/ - # - # For a discussion on single-sourcing the version across setup.py and the - # project code, see - # https://packaging.python.org/en/latest/single_source_version.html - version='0.3', # Required - - # This is a one-line description or tagline of what your project does. This - # corresponds to the "Summary" metadata field: - # https://packaging.python.org/specifications/core-metadata/#summary - description='Helper utilities for Lowell Observatory code notifications', - - # This is an optional longer description of your project that represents - # the body of text which users will see when they visit PyPI. - # - # Often, this is the same as your README, so you can just read it in from - # that file directly (as we have already done above) - # - # This field corresponds to the "Description" metadata field: - # https://packaging.python.org/specifications/core-metadata/#description-optional - long_description=long_description, # Optional - - # Denotes that our long_description is in Markdown; valid values are - # text/plain, text/x-rst, and text/markdown - # - # Optional if long_description is written in reStructuredText (rst) but - # required for plain-text or Markdown; if unspecified, "applications should - # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and - # fall back to text/plain if it is not valid rst" (see link below) - # - # This field corresponds to the "Description-Content-Type" metadata field: - # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional - long_description_content_type='text/markdown', # Optional (see note above) - - # This should be a valid link to your project's main homepage. - # - # This field corresponds to the "Home-Page" metadata field: - # https://packaging.python.org/specifications/core-metadata/#home-page-optional - url='https://github.com/LowellObservatory/JohnnyFive', # Optional - - # This should be your name or the name of the organization which owns the - # project. - author='Lowell Observatory', # Optional - - # This should be a valid email address corresponding to the author listed - # above. - author_email='rhamilton@lowell.edu', # Optional - - # Classifiers help users find your project by categorizing it. - # - # For a list of valid classifiers, see https://pypi.org/classifiers/ - classifiers=[ # Optional - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - 'Development Status :: 4 - Beta', - - # Indicate who your project is intended for - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Build Tools', - - # Pick your license as you wish - 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', - - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - ], - - # This field adds keywords for your project which will appear on the - # project page. What does your project relate to? - # - # Note that this is a string of words separated by whitespace, not a list. - keywords='', # Optional - - # You can just specify package directories manually here if your project is - # simple. Or you can use find_packages(). - # - # Alternatively, if you just want to distribute a single Python file, use - # the `py_modules` argument instead as follows, which will expect a file - # called `my_module.py` to exist: - # - # py_modules=["my_module"], - # - packages=find_packages(exclude=['contrib', 'docs', 'tests']), # Required - - # This field lists other packages that your project depends on to run. - # Any package you put here will be installed by pip when your project is - # installed, so they must be valid existing projects. - # - # For an analysis of "install_requires" vs pip's requirements files see: - # https://packaging.python.org/en/latest/requirements.html - install_requires=['atlassian-python-api', - 'beautifulsoup4', - 'google-api-python-client', - 'google-auth-httplib2', - 'google-auth-oauthlib', - 'lxml', - 'pyjwt', - 'python-twitter', - 'requests', - 'slack_sdk'], - - # List additional groups of dependencies here (e.g. development - # dependencies). Users will be able to install these using the "extras" - # syntax, for example: - # - # $ pip install sampleproject[dev] - # - # Similar to `install_requires` above, these must be valid existing - # projects. - extras_require={ # Optional - 'dev': ['check-manifest'], - 'test': ['coverage'], - }, - - # If there are data files included in your packages that need to be - # installed, specify them here. - # - # If using Python 2.6 or earlier, then these have to be included in - # MANIFEST.in as well. - package_data={}, - include_package_data=True, - - # Although 'package_data' is the preferred approach, in some case you may - # need to place data files outside of your packages. See: - # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files - # - # In this case, 'data_file' will be installed into '/my_data' - data_files=[], # Optional - - # To provide executable scripts, use entry points in preference to the - # "scripts" keyword. Entry points provide cross-platform support and allow - # `pip` to create the appropriate form of executable for the target - # platform. - # - # For example, the following would provide a command called `sample` which - # executes the function `main` from this package when invoked: - entry_points={ # Optional - 'console_scripts': [ - 'j5_install_conf=johnnyfive.utils:install_conffiles', - 'j5_authenticate_gmail=johnnyfive.gmail:authenticate_gmail', - ] - }, - - # List additional URLs that are relevant to your project as a dict. - # - # This field corresponds to the "Project-URL" metadata fields: - # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use - # - # Examples listed include a pattern for specifying where the package tracks - # issues, where the source is hosted, where to say thanks to the package - # maintainers, and where to support the project financially. The key is - # what's used to render the link text on PyPI. - project_urls={ # Optional - 'Bug Reports': 'https://github.com/LowellObservatory/JohnnyFive/issues', - 'Source': 'https://github.com/LowellObservatory/JohnnyFive/', - }, -) diff --git a/tests/test_service_integrations.py b/tests/test_service_integrations.py new file mode 100644 index 0000000..068b63c --- /dev/null +++ b/tests/test_service_integrations.py @@ -0,0 +1,272 @@ +"""Hermetic integration tests for J5's external-service wrappers.""" + +from __future__ import annotations + +import base64 +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from johnnyfive import confluence, gmail, slack + + +class FakeSlackClient: + """Minimal Slack client that records wrapper calls.""" + + def __init__(self) -> None: + """Initialize call recording.""" + self.calls: list[tuple[str, dict[str, Any]]] = [] + + def conversations_list(self) -> dict[str, list[dict[str, str]]]: + """Return one available channel.""" + return {"channels": [{"name": "alerts", "id": "C123"}]} + + def chat_postMessage(self, **kwargs: Any) -> dict[str, Any]: + """Record and return a message response.""" + self.calls.append(("message", kwargs)) + return {"ok": True, **kwargs} + + def files_upload(self, **kwargs: Any) -> dict[str, Any]: + """Record and return an upload response.""" + self.calls.append(("upload", kwargs)) + return {"ok": True, **kwargs} + + +def test_slack_channel_routes_messages_and_files(monkeypatch: pytest.MonkeyPatch) -> None: + """Resolve a channel and route wrapper operations to its client.""" + client = FakeSlackClient() + monkeypatch.setattr(slack, "setup_slack", lambda: client) + + channel = slack.SlackChannel("alerts") + + assert channel.send_message("hello") == {"ok": True, "channel": "C123", "text": "hello"} + assert channel.upload_file("report.txt", title="Report")["title"] == "Report" + assert [name for name, _ in client.calls] == ["message", "upload"] + + +class FakeConfluenceClient: + """Minimal Confluence client that records page operations.""" + + username = "bot" + url = "https://confluence.example/" + + def __init__(self) -> None: + """Initialize call recording.""" + self.calls: list[tuple[str, tuple[Any, ...], dict[str, Any]]] = [] + self.permission_queries = 0 + + def get_space_permissions(self, space: str) -> list[dict[str, Any]]: + """Grant the permissions used by this test.""" + self.permission_queries += 1 + return [ + { + "type": permission, + "spacePermissions": [{"userName": self.username}], + } + for permission in ("COMMENT", "EDITSPACE", "CREATEATTACHMENT", "REMOVEATTACHMENT", "REMOVEPAGE") + ] + + def page_exists(self, space: str, title: str) -> bool: + """Report that the page exists.""" + return True + + def get_page_id(self, space: str, title: str) -> str: + """Return a deterministic page ID.""" + return "42" + + def add_comment(self, *args: Any, **kwargs: Any) -> None: + """Record a comment call.""" + self.calls.append(("comment", args, kwargs)) + + def set_page_label(self, *args: Any, **kwargs: Any) -> None: + """Record a label call.""" + self.calls.append(("label", args, kwargs)) + + def get_attachments_from_content(self, *args: Any, **kwargs: Any) -> list[str]: + """Return a deterministic attachment list.""" + return ["attachment-1"] + + def get_page_by_id(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + """Return deterministic page content.""" + return {"body": {"storage": {"value": "

content

"}}} + + +def test_confluence_page_integrates_metadata_and_operations( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Exercise metadata, permissions, and API wrappers through one client.""" + client = FakeConfluenceClient() + monkeypatch.setattr(confluence, "setup_confluence", lambda use_oauth=False: client) + + page = confluence.ConfluencePage("OPS", "Status") + page.add_comment("All clear") + page.add_label("nightly") + + assert page.page_id == "42" + assert page.get_page_attachments() == ["attachment-1"] + assert page.get_page_contents() == "

content

" + assert [name for name, _, _ in client.calls] == ["comment", "label"] + assert page.space_perms == {} + assert client.permission_queries == 0 + + +class FakeConfluenceServer: + """Capture constructor arguments for the explicit v5 Server client.""" + + def __init__(self, **kwargs: Any) -> None: + """Store connection parameters without making a network request.""" + self.kwargs = kwargs + + +def test_setup_confluence_uses_explicit_server_client( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Create v5 Server clients for basic and bearer-token authentication.""" + config = SimpleNamespace( + host="https://confluence.example", + user="bot", + password="password", + access_token="token", + ) + monkeypatch.setattr(confluence, "ConfluenceServer", FakeConfluenceServer) + monkeypatch.setattr( + confluence.johnnyfive.utils, + "read_config_section", + lambda _: config, + ) + + basic_client = confluence.setup_confluence() + oauth_client = confluence.setup_confluence(use_oauth=True) + + assert basic_client.kwargs == { + "url": "https://confluence.example", + "username": "bot", + "password": "password", + } + assert oauth_client.kwargs["url"] == "https://confluence.example" + assert oauth_client.kwargs["session"].headers["Authorization"] == "Bearer token" + + +class FakeGmailRequest: + """Fake request object with Gmail's execute protocol.""" + + def __init__(self, response: dict[str, Any]) -> None: + """Store the response returned by ``execute``.""" + self.response = response + + def execute(self) -> dict[str, Any]: + """Return the configured API response.""" + return self.response + + +class FakeGmailService: + """Minimal Gmail service used to exercise message construction and sending.""" + + def __init__(self) -> None: + """Initialize a sent-message record.""" + self.sent_body: dict[str, str] | None = None + + def users(self) -> FakeGmailService: + """Return the users API facade.""" + return self + + def messages(self) -> FakeGmailService: + """Return the messages API facade.""" + return self + + def send(self, *, userId: str, body: dict[str, str]) -> FakeGmailRequest: + """Record a send request and return its response.""" + assert userId == "me" + self.sent_body = body + return FakeGmailRequest({"id": "message-1"}) + + +class FakeGmailReadService: + """Minimal Gmail service that returns a predefined full message payload.""" + + def __init__(self, response: dict[str, Any]) -> None: + """Store the API response and requested message format.""" + self.response = response + self.request_format: str | None = None + + def users(self) -> FakeGmailReadService: + """Return the users API facade.""" + return self + + def messages(self) -> FakeGmailReadService: + """Return the messages API facade.""" + return self + + def get(self, *, userId: str, id: str, format: str) -> FakeGmailRequest: + """Record the requested format and return the configured message.""" + assert userId == "me" + assert id == "message-2" + self.request_format = format + return FakeGmailRequest(self.response) + + +def test_gmail_message_builds_attachment_and_sends( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Construct a MIME message, attach a file, and send through the fake API.""" + service = FakeGmailService() + attachment = tmp_path / "report.txt" + attachment.write_text("report contents", encoding="utf-8") + monkeypatch.setattr(gmail, "setup_gmail", lambda interactive=False, logger=None: service) + + message = gmail.GmailMessage("bot@example.test", "Report", "Body") + message.add_attachment(attachment) + + assert message.send() == {"id": "message-1"} + assert service.sent_body is not None + assert "raw" in service.sent_body + + +def test_render_message_traverses_nested_parts_and_prefers_plain_text() -> None: + """Render nested multipart content without assuming the first part is text.""" + def encode(value: str) -> str: + """Return unpadded base64url content as Gmail supplies it.""" + return base64.urlsafe_b64encode(value.encode("utf-8")).decode("ascii").rstrip("=") + + service = FakeGmailReadService( + { + "payload": { + "headers": [ + {"name": "Subject", "value": "Nightly report"}, + {"name": "From", "value": "bot@example.test"}, + {"name": "Date", "value": "Mon, 8 Sep 2026 19:31:45 -0700"}, + ], + "mimeType": "multipart/mixed", + "body": {}, + "parts": [ + { + "mimeType": "multipart/alternative", + "body": {}, + "parts": [ + {"mimeType": "text/html", "body": {"data": encode("HTML report")}}, + {"mimeType": "text/plain", "body": {"data": encode("Plain report")}}, + ], + } + ], + } + } + ) + messages = object.__new__(gmail.GetMessages) + messages.service = service + messages.logger = None + + assert messages.render_message("message-2") == { + "subject": "Nightly report", + "sender": "bot@example.test", + "date": "Mon, 8 Sep 2026 19:31:45 -0700", + "body": "Plain report", + } + assert service.request_format == "full" + assert gmail.GetMessages._extract_message_body( + {"mimeType": "text/html", "body": {"data": encode("HTML only")}} + ) == "HTML only" + assert gmail.GetMessages._extract_message_body( + {"mimeType": "multipart/mixed", "body": {}, "parts": []} + ) == "" diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..d922942 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,83 @@ +"""Unit tests for configuration and service utility helpers.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from johnnyfive import utils + + +def test_val_checks_converts_scalars_and_lists() -> None: + """Convert configuration literals and comma-separated values.""" + assert utils.valChecks(" true ") is True + assert utils.valChecks("None") is None + assert utils.valChecks("first, false, second") == ["first", False, "second"] + + +def test_read_config_section_and_legacy_alias( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Parse a section into J5's target object through both public names.""" + (tmp_path / "johnnyfive.conf").write_text( + "[service]\nhost = example.test\nenabled = true\ncustom = one, two\n", + encoding="utf-8", + ) + monkeypatch.setattr(utils.Paths, "config", tmp_path) + + target = utils.read_config_section("service") + legacy_target = utils.read_ligmos_conffiles("service") + + assert target.host == "example.test" + assert target.enabled is True + assert target.custom == ["one", "two"] + assert legacy_target.host == target.host + + +def test_read_config_section_reports_missing_section( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Raise J5Error instead of leaking a missing-section KeyError.""" + (tmp_path / "johnnyfive.conf").write_text("[other]\nvalue = 1\n", encoding="utf-8") + monkeypatch.setattr(utils.Paths, "config", tmp_path) + + with pytest.raises(utils.J5Error, match="Configuration key missing"): + utils.read_config_section("missing") + + +def test_install_conffiles_copies_requested_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Copy configuration files to the configured package directory.""" + source = tmp_path / "source.conf" + destination = tmp_path / "destination" + destination.mkdir() + source.write_text("[service]\nkey = value\n", encoding="utf-8") + monkeypatch.setattr(utils.Paths, "config", destination) + + utils.install_conffiles([str(source)]) + + assert (destination / source.name).read_text(encoding="utf-8") == source.read_text( + encoding="utf-8" + ) + + +def test_safe_service_connect_retries_network_failure( + monkeypatch: pytest.MonkeyPatch +) -> None: + """Retry transient connection failures and return the eventual result.""" + attempts = 0 + + def flaky_service() -> str: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise ConnectionError("temporary outage") + return "connected" + + monkeypatch.setattr(utils.time, "sleep", lambda _: None) + + with pytest.warns(UserWarning, match="network error"): + assert utils.safe_service_connect(flaky_service, pause=0, nretries=2) == "connected" + assert attempts == 2