diff --git a/.circleci/config.yml b/.circleci/config.yml index f59cdb0..1ca4a50 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,11 +34,16 @@ common: &common requires: - run-tests + - python/pip-docs: + name: docs + requires: + - coverage + - python/release: name: release config: .carthorse.yml requires: - - coverage + - docs filters: branches: only: master diff --git a/.gitignore b/.gitignore index 6cb726f..3557e5d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ pip-selfcheck.json .coverage .coverage.* build/ +_build/ dist/ pyvenv.cfg bin/ diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..e6cdbac --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,10 @@ +version: 2 +python: + version: "3.8" + install: + - method: pip + path: . + extra_requirements: + - docs +sphinx: + fail_on_warning: true diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..c146823 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,7 @@ +Changes +======= + +0.12.1 (11 Mar 2019) +-------------------- + +- For this and earlier releases, please see git commit history. diff --git a/README.rst b/README.rst index ea81f43..75aa135 100644 --- a/README.rst +++ b/README.rst @@ -1,35 +1,35 @@ -|CircleCI|_ +mortar_import +============= + +|CircleCI|_ |Docs|_ .. |CircleCI| image:: https://circleci.com/gh/Mortar/mortar_import/tree/master.svg?style=shield .. _CircleCI: https://circleci.com/gh/Mortar/mortar_import/tree/master -mortar_import -============= +.. |Docs| image:: https://readthedocs.org/projects/mortar-import/badge/?version=latest +.. _Docs: http://mortar-import.readthedocs.org/en/latest/ + +Tools for finding differences between existing and imported sets of data and then +applying those differences. + +Quickstart +---------- -Tools for importing data, particularly when using `mortar_mixins`__. +.. hidden-code-block -__ https://github.com/Mortar/mortar_mixins + Some Python -Install from PyPI with pip. +.. code-block:: python -Development ------------ + from mortar_import import Diff -Get a clone of the git repo and then do the following:: + class MyDiff(Diff): + pass - virtualenv . - bin/pip install -e .[build,test] - - sudo -u postgres psql -d postgres -c "create user testuser with password 'testpassword';" - sudo -u postgres createdb -O testuser testdb - sudo -u postgres psql -d testdb -c "CREATE EXTENSION btree_gist;" +Full documentation can be found `here`__. - export DB_URL=postgres://testuser:testpassword@localhost:5432/testdb - bin/pytest --cov +__ https://mortar-import.readthedocs.io/en/latest/ -Releasing ---------- +Releases can be found `here`__. -To make a release, just update the version in ``setup.py`` -and push to https://github.com/Mortar/mortar_import -and Carthorse should take care of the rest. +__ https://pypi.org/project/mortar-import/#history diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..1de0715 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,78 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD ?= sphinx-build +PAPER = + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d _build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf _build/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) _build/html + @echo + @echo "Build finished. The HTML pages are in _build/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) _build/dirhtml + @echo + @echo "Build finished. The HTML pages are in _build/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) _build/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) _build/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) _build/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in _build/htmlhelp." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) _build/latex + @echo + @echo "Build finished; the LaTeX files are in _build/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) _build/changes + @echo + @echo "The overview file is in _build/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) _build/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in _build/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) _build/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in _build/doctest/output.txt." diff --git a/docs/abstract.rst b/docs/abstract.rst new file mode 100644 index 0000000..c6f0c22 --- /dev/null +++ b/docs/abstract.rst @@ -0,0 +1,2 @@ +Abstract +======== diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..60059c2 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,32 @@ +API Reference +============= + +.. currentmodule:: mortar_import + + +Diffs +----- + +.. autoclass:: Diff + :members: + +.. autoclass:: SQLAlchemyDiff + :members: + +.. autoclass:: TemporalDiff + :members: + +Extractors +---------- + +.. autoclass:: DictExtractor + :members: + +.. autoclass:: NamedTupleExtractor + :members: + +Typing +------ + +.. automodule:: mortar_import.typing + :members: diff --git a/docs/changes.rst b/docs/changes.rst new file mode 100644 index 0000000..952c7fe --- /dev/null +++ b/docs/changes.rst @@ -0,0 +1,4 @@ + +.. currentmodule:: mortar_import + +.. include:: ../CHANGELOG.rst diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..221a7dc --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +import datetime +import os +import time + +import pkg_resources + +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +build_date = datetime.datetime.utcfromtimestamp(int(os.environ.get('SOURCE_DATE_EPOCH', time.time()))) + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx' + ] + +intersphinx_mapping = { + 'https://docs.python.org/3/': None, +} + +# General +source_suffix = '.rst' +master_doc = 'index' +project = 'mortar_import' +copyright = '2015-%s Chris Withers' % build_date.year +version = release = pkg_resources.get_distribution(project).version +exclude_trees = ['_build'] +pygments_style = 'sphinx' + +# Options for HTML output +html_theme = 'furo' +htmlhelp_basename = project+'doc' + +# Options for LaTeX output +exclude_patterns = ['**/furo.js.LICENSE.txt'] + +autodoc_member_order = 'bysource' diff --git a/docs/development.rst b/docs/development.rst new file mode 100644 index 0000000..76db26f --- /dev/null +++ b/docs/development.rst @@ -0,0 +1,62 @@ +Development +=========== + +.. highlight:: bash + +If you wish to contribute to this project, then you should fork the +repository found here: + +https://github.com/mortar/mortar_import/ + +Once that has been done and you have a checkout, you can follow these +instructions to perform various development tasks: + +Setting your environment +------------------------- + +The recommended way to set up a development environment is to create +a virtualenv and then install the package in editable form as follows:: + + $ python3 -m venv ~/virtualenvs/mortar_import + $ source ~/virtualenvs/mortar_import/bin/activate + $ pip install -U pip setuptools + $ pip install -U -e .[test,build] + +You'll also need a Postgres database in order to run the tests:: + + sudo -u postgres psql -d postgres -c "create user testuser with password 'testpassword';" + sudo -u postgres createdb -O testuser testdb + sudo -u postgres psql -d testdb -c "CREATE EXTENSION btree_gist;" + + export DB_URL=postgres://testuser:testpassword@localhost:5432/testdb + +Running the tests +----------------- + +Once you've set up a virtualenv, the tests can be run in the activated +virtualenv and from the root of a source checkout as follows:: + + $ pytest + +Building the documentation +-------------------------- + +The Sphinx documentation is built by doing the following from the +directory containing ``setup.py``:: + + $ cd docs + $ make html + +To check that the description that will be used on PyPI renders properly, +do the following:: + + $ python setup.py --long-description | rst2html.py > desc.html + +The resulting ``desc.html`` should be checked by opening in a browser. + +Making a release +---------------- + +To make a release, just update the version in ``setup.py``, update the change log +and push to https://github.com/mortar/mortar_import. +Carthorse should take care of the rest. diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..ffa14e5 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,46 @@ +.. currentmodule:: mortar_import + +.. include:: ../README.rst + :end-before: Full documentation + +.. toctree:: + :hidden: + + use.rst + +The following types of :class:`Diff` are included: + + +.. toctree:: + :maxdepth: 1 + + abstract.rst + sqlalchemy.rst + temporal.rst + + +There is also an API reference: + +.. toctree:: + :maxdepth: 1 + + api.rst + +For details of how to install the package or get involved in its +development, please see the sections below: + +.. toctree:: + :maxdepth: 1 + + installation.rst + development.rst + changes.rst + license.rst + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..0fe9670 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,17 @@ +Installation Instructions +========================= + +If you want to experiment with mortar_import, the easiest way to +install it is to do the following in a virtualenv: + +.. code-block:: bash + + pip install mortar_import + + +If you are using conda, mortar_import can be installed as follows: + + +.. code-block:: bash + + conda install -c conda-forge mortar_import diff --git a/docs/license.rst b/docs/license.rst new file mode 100644 index 0000000..b383c6e --- /dev/null +++ b/docs/license.rst @@ -0,0 +1,5 @@ +======= +License +======= + +.. literalinclude:: ../LICENSE.txt diff --git a/docs/sqlalchemy.rst b/docs/sqlalchemy.rst new file mode 100644 index 0000000..8d477bd --- /dev/null +++ b/docs/sqlalchemy.rst @@ -0,0 +1,2 @@ +SQLAlchemy +========== diff --git a/docs/temporal.rst b/docs/temporal.rst new file mode 100644 index 0000000..69f2354 --- /dev/null +++ b/docs/temporal.rst @@ -0,0 +1,2 @@ +Temporal +======== diff --git a/docs/use.rst b/docs/use.rst new file mode 100644 index 0000000..d4b9a15 --- /dev/null +++ b/docs/use.rst @@ -0,0 +1,2 @@ +Usage +===== diff --git a/mortar_import/__init__.py b/mortar_import/__init__.py index 8b13789..6b0129f 100644 --- a/mortar_import/__init__.py +++ b/mortar_import/__init__.py @@ -1 +1,4 @@ - +from .extractors import DictExtractor, NamedTupleExtractor +from .diff import Diff +from .sqlalchemy import SQLAlchemyDiff +from .temporal import TemporalDiff diff --git a/mortar_import/diff.py b/mortar_import/diff.py index c957ab1..f77b5e4 100644 --- a/mortar_import/diff.py +++ b/mortar_import/diff.py @@ -178,6 +178,11 @@ def compute(self) -> None: self.to_delete.append(Deletion(key, *self.existing_mapping[key])) def apply(self) -> None: + """ + :meth:`compute` this :class:`Diff` if it has not already been computed + and then apply it by + :return: + """ if self.to_add is None: self.compute() for op in 'delete', 'update', 'add': diff --git a/mortar_import/typing.py b/mortar_import/typing.py index 2f90b33..04a1b7d 100644 --- a/mortar_import/typing.py +++ b/mortar_import/typing.py @@ -1,9 +1,15 @@ from typing import TypeVar, Hashable +#: The hashable key extracted from existing and imported objects +#: to decide if they represent the same object. Key = TypeVar('Key', bound=Hashable) +#: An existing object. Existing = TypeVar('Existing') +#: An imported object. Imported = TypeVar('Imported') +#: An extracted existing object. ExtractedExisting = TypeVar('ExtractedExisting') +#: An extracted imported object. ExtractedImported = TypeVar('ExtractedImported') diff --git a/setup.py b/setup.py index 382fa52..8620369 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,9 @@ 'testfixtures', 'mortar_mixins>=3', 'mock', + 'sybil', ], build=['setuptools-git', 'wheel', 'twine'], + docs=['sphinx', 'furo', 'mortar_mixins>=3'], ), )