diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 87487d1..b141973 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: if: github.ref == 'refs/heads/main' runs-on: ubuntu-22.04 - container: python:3.7-buster + container: python:3.12-bookworm steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 31fcab7..8a1ad61 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ on: jobs: Pipeline: runs-on: ubuntu-22.04 - container: python:3.7-buster + container: python:3.12-bookworm steps: - uses: actions/checkout@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 947fa8f..660c328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. Preferably use **Added**, **Changed**, **Removed** and **Fixed** topics in each release or unreleased log for a better organization. +## [2.0.1](https://github.com/quintoandar/validations-engine/releases/tag/2.0.1) +### Changed +Updated project requirements to be compatible with Airflow 2.11.0 constraints. + ## [2.0.0](https://github.com/quintoandar/validations-engine/releases/tag/2.0.0) ### Changed GchatHelper and message classes were added as the main messenger service. So in this release, the errors can't be sent to Slack. In the next minor version (2.1.0), we'll add the option to choose between Slack and GChat services. diff --git a/Makefile b/Makefile index a992235..a83c041 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,7 @@ # Environment ================================================================= +CONSTRAINTS_FILE ?= +PIP_CONSTRAINT_OPT := $(if $(strip $(CONSTRAINTS_FILE)),-c $(CONSTRAINTS_FILE),) + .PHONY: environment ## create virtual environment for validations-engine environment: @@ -10,22 +13,22 @@ environment: .PHONY: requirements-test ## install test requirements requirements-test: - @PYTHONPATH=. python -m pip install -r requirements.test.txt + @PYTHONPATH=. python -m pip install $(PIP_CONSTRAINT_OPT) -r requirements.test.txt .PHONY: requirements-lint ## install lint requirements requirements-lint: - @PYTHONPATH=. python -m pip install -r requirements.lint.txt + @PYTHONPATH=. python -m pip install $(PIP_CONSTRAINT_OPT) -r requirements.lint.txt .PHONY: requirements-dev ## install development requirements requirements-dev: - @PYTHONPATH=. python -m pip install -U -r requirements.dev.txt + @PYTHONPATH=. python -m pip install -U $(PIP_CONSTRAINT_OPT) -r requirements.dev.txt .PHONY: requirements-minimum ## install prod requirements requirements-minimum: - @PYTHONPATH=. python -m pip install -U -r requirements.txt + @PYTHONPATH=. python -m pip install -U $(PIP_CONSTRAINT_OPT) -r requirements.txt .PHONY: requirements-all ## install all requirements @@ -34,7 +37,7 @@ requirements-all: requirements-test requirements-lint requirements-dev requireme .PHONY: requirements-docs ## install docs requirements requirements-docs: - @PYTHONPATH=. python -m pip install -r docs/requirements.docs.txt + @PYTHONPATH=. python -m pip install $(PIP_CONSTRAINT_OPT) -r docs/requirements.docs.txt # Test ======================================================================== @@ -81,7 +84,7 @@ type-check: @echo "Mypy" @echo "====" @echo "" - @python -m mypy validations_engine + @python -m mypy --config-file setup.cfg --package validations_engine .PHONY: checks ## run all code checks @@ -90,7 +93,7 @@ checks: type-check style-check .PHONY: apply-style ## run black to fix code style apply-style: - @python -m black -t py36 --exclude="build/|buck-out/|dist/|_build/|pip/|env/|\.pip/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/" . + @python -m black --exclude="build/|buck-out/|dist/|_build/|pip/|env/|\.pip/|\.git/|\.hg/|\.mypy_cache/|\.tox/|\.venv/" . # Packaging =================================================================== diff --git a/docs/source/conf.py b/docs/source/conf.py index 6c0cd8e..37b9606 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,4 +1,5 @@ """Sphinx Configuration.""" + # -*- coding: utf-8 -*- # # Configuration file for the Sphinx documentation builder. @@ -26,9 +27,9 @@ author = "Data Engineering Team" # The short X.Y version -version = "1.0" +version = "2.0" # The full version, including alpha/beta/rc tags -release = "1.0.0" +release = "2.0.1" # -- General configuration --------------------------------------------------- diff --git a/requirements.dev.txt b/requirements.dev.txt index b61d6b7..5488277 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1 +1 @@ -twine==4.0.1 \ No newline at end of file +twine==6.1.0 diff --git a/requirements.lint.txt b/requirements.lint.txt index f9710f2..34bed48 100644 --- a/requirements.lint.txt +++ b/requirements.lint.txt @@ -1,4 +1,5 @@ -black==22.3.0 -flake8==3.7.9 -flake8-docstrings==1.5.0 -mypy==0.782 \ No newline at end of file +black==25.1.0 +flake8==7.3.0 +flake8-docstrings==1.7.0 +mypy==1.9.0 +types-requests==2.32.0.20250515 diff --git a/requirements.test.txt b/requirements.test.txt index b0c4032..9069721 100644 --- a/requirements.test.txt +++ b/requirements.test.txt @@ -1,5 +1,5 @@ -pytest==5.3.2 -pytest-cov==2.8.1 -pytest-xdist==1.31.0 -pytest-mock==2.0.0 +pytest==8.3.5 +pytest-cov==6.1.1 +pytest-xdist==3.6.1 +pytest-mock==3.14.0 pytest-spark==0.5.2 diff --git a/requirements.txt b/requirements.txt index 5e77405..727711c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -requests==2.28.1 \ No newline at end of file +requests==2.32.5 diff --git a/setup.cfg b/setup.cfg index b59fd13..d9c9a75 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.0.0 +current_version = 2.0.1 commit = True tag = True diff --git a/setup.py b/setup.py index 889732b..269b6ee 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages __package_name__ = "validations_engine" -__version__ = "2.0.0" +__version__ = "2.0.1" __repository_url__ = "https://github.com/quintoandar/validations-engine" diff --git a/validations_engine/GchatHelper.py b/validations_engine/GchatHelper.py index c21adb1..6fcfc24 100644 --- a/validations_engine/GchatHelper.py +++ b/validations_engine/GchatHelper.py @@ -1,4 +1,5 @@ """Gchat communications module.""" + from typing import List import requests import logging diff --git a/validations_engine/SlackHelper.py b/validations_engine/SlackHelper.py index ed55c20..8fdb751 100644 --- a/validations_engine/SlackHelper.py +++ b/validations_engine/SlackHelper.py @@ -1,4 +1,5 @@ """Slack communications module.""" + import copy import logging from typing import Dict, Any, Tuple, List @@ -34,7 +35,7 @@ def send_slack_errors(error_messages: List[Tuple[str, str]]) -> bool: @staticmethod def build_slack_payload( - error_messages: List[Tuple[str, str]] + error_messages: List[Tuple[str, str]], ) -> Dict[str, Dict[str, Any]]: """Builds the message payload from the error messages.""" error_dict = {} # type: ignore diff --git a/validations_engine/__init__.py b/validations_engine/__init__.py index e9c1089..a3d150a 100644 --- a/validations_engine/__init__.py +++ b/validations_engine/__init__.py @@ -1 +1,3 @@ """Top-level package for validations_engine.""" + +__version__ = "2.0.1" diff --git a/validations_engine/base_validation_suites_executor.py b/validations_engine/base_validation_suites_executor.py index 431d3db..98fef01 100644 --- a/validations_engine/base_validation_suites_executor.py +++ b/validations_engine/base_validation_suites_executor.py @@ -1,8 +1,9 @@ """Suites base executor.""" + import inspect import logging from logging import INFO -from typing import Dict, Any, List, Tuple +from typing import Dict, Any, List, Optional, Tuple from validations_engine.message import Message @@ -14,7 +15,7 @@ class BaseValidationSuitesExecutor: GCHAT_MSG_HEADER = "" - def __init__(self, auth: Dict[str, Any] = None) -> None: + def __init__(self, auth: Optional[Dict[str, Any]] = None) -> None: self.auth = auth self._suite_validation_has_failures = False self.errors = [] # type: ignore diff --git a/validations_engine/message.py b/validations_engine/message.py index 82b1be2..8fbad3c 100644 --- a/validations_engine/message.py +++ b/validations_engine/message.py @@ -1,4 +1,5 @@ """Message class.""" + from dataclasses import dataclass from typing import Union diff --git a/validations_engine/validations_engine.py b/validations_engine/validations_engine.py index bacb0e0..6b14450 100644 --- a/validations_engine/validations_engine.py +++ b/validations_engine/validations_engine.py @@ -1,4 +1,5 @@ """Validations Engine main class.""" + import importlib import pathlib import sys @@ -6,7 +7,7 @@ from inspect import isclass from os.path import dirname from types import ModuleType -from typing import List, Any, Dict +from typing import List, Any, Dict, Optional from validations_engine.GchatHelper import GchatHelper from validations_engine.message import Message @@ -100,7 +101,7 @@ def load_validation_suites( return all_validation_suites_classes def set_suites_have_failures( - self, param: bool, messages: List[Message] = None + self, param: bool, messages: Optional[List[Message]] = None ) -> None: """Merges previous state with new one.""" self._suites_have_failures |= param