diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..391fa6a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,34 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true + +# Matches multiple files with brace expansion notation +# Set default charset +[*.{js,py}] +charset = utf-8 + +# 4 space indentation +[*.py] +indent_style = space +indent_size = 4 +max_line_length = 100 + +# Tab indentation (no size specified) +[Makefile] +indent_style = tab + +# Indentation override for all JS under lib directory +[lib/**.js] +indent_style = space +indent_size = 2 + +# Matches the exact files either package.json or .travis.yml +[{package.json,.travis.yml}] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..f841446 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,34 @@ +name: CI + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff pytest pytest-cov + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with ruff + run: | + # stop the build if there are Python syntax errors or undefined names + ruff --format=github --select=E9,F63,F7,F82 --target-version=py37 . + # default set of ruff rules with GitHub Annotations + ruff --format=github --target-version=py37 . + continue-on-error: true + - name: Test with pytest + run: | + pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=coppermind --cov-report=xml --cov-report=html diff --git a/.gitignore b/.gitignore index bfb8e80..09b5a76 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,13 @@ coppermind.conf *.idea .idea +.coverage +coverage.xml +test-results.xml +junit +htmlcov +*_cache +__pycache__ +.tox +*.egg-info +dist diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 0000000..703b4f4 --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,14 @@ +--- +# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart +tasks: + - init: | + pip install -r requirements.txt + pip install -r requirements-dev.txt + pre-commit install + +vscode: + extensions: + - Vue.volar + - EditorConfig.EditorConfig + - davraamides.todotxt-mode + - ms-python.python diff --git a/setup.py b/.pre-commit-config.yaml similarity index 100% rename from setup.py rename to .pre-commit-config.yaml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5f04a4d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -language: python -python: - - "3.4" - - "3.5" - - "3.5" - - "3.6" - - "3.7" - - "3.8" - -# command to install dependencies -services: - - "mongodb" - -install: "pip install -r requirements.txt" -# command to run tests -script: nosetests tests diff --git a/coppermind/__init__.py b/coppermind/__init__.py index 30c6445..fffa064 100644 --- a/coppermind/__init__.py +++ b/coppermind/__init__.py @@ -1 +1 @@ -from .webservice import app as CoppermindWS +from .daemon import Coppermind diff --git a/coppermind/common/db/__init__.py b/coppermind/common/db/__init__.py index 8b13789..e69de29 100644 --- a/coppermind/common/db/__init__.py +++ b/coppermind/common/db/__init__.py @@ -1 +0,0 @@ - diff --git a/coppermind/common/db/base.py b/coppermind/common/db/base.py index 4561d99..eeccc24 100644 --- a/coppermind/common/db/base.py +++ b/coppermind/common/db/base.py @@ -2,12 +2,14 @@ import logging from abc import ABCMeta, abstractmethod +from ..tools import SVCObj + class EbookNotFound(Exception): pass -class BaseDB(metaclass=ABCMeta): +class BaseDB(SVCObj, metaclass=ABCMeta): """ Base for all DB classes. Defines a standard interface allowing multiple DB implementations without requiring @@ -16,9 +18,15 @@ class BaseDB(metaclass=ABCMeta): Unittests for any DB implementation should simply be running the ebook unittest against that implementation """ + @property + def config(self): + """ + Return database configuration + """ + return self.svc.config['db'] @abstractmethod - def get_ebook_file(self, uuid): + def get_ebook_file(self, ebook_id): """ Return actual ebook file by UUID """ @@ -31,9 +39,7 @@ def store_ebook_file(self, ebook_file): def save_ebook(self, ebook, **kwargs): data_file = self.store_ebook_file(fmt=ebook.format, **kwargs) - metadata = ebook.serialize() - metadata['storage'] = {'mongo': data_file} - ebook_uuid = self.save_ebook_metadata(metadata) + ebook_uuid = self.save_ebook_metadata(ebook=ebook) return ebook_uuid @abstractmethod diff --git a/coppermind/common/db/filesystem.py b/coppermind/common/db/filesystem.py new file mode 100644 index 0000000..ea21c54 --- /dev/null +++ b/coppermind/common/db/filesystem.py @@ -0,0 +1,68 @@ +import os +import uuid +import sqlite3 +import yact +from shutil import copy2 +from ..models import Ebook +from ..tools.parser import file_hash +from datetime import datetime +from .base import BaseDB, EbookNotFound + + +class FileSystem(BaseDB): + + def __init__(self): + self._storage_directory = os.path.join(os.path.pardir, 'coppermind-storage') + if not os.path.exists(self._storage_directory): + os.makedirs(self._storage_directory) + self._connection = sqlite3.connect(os.path.join(self._storage_directory, self.config.get('filename', 'coppermind.db')), check_same_thread=False) + with self._connection: + self._connection.execute(""" CREATE TABLE IF NOT EXISTS books ( + id text PRIMARY KEY, + author text NOT NULL, + title text NOT NULL, + format text NOT NULL + ); """) + self._connection.execute(""" CREATE TABLE IF NOT EXISTS storage ( + id text PRIMARY KEY, + sha256sum text NOT NULL, + uri text NOT NULL + ); """) + + def get_ebook_file(self, book_id): + with self._connection: + path = self._connection.execute("SELECT uri FROM storage where id = ?", (book_id,)) + with open(path) as book: + return book.read() + + def store_ebook_file(self, **kwargs): + if 'file' in kwargs: # Assume file-like object + raise NotImplementedError('Epub only for now') + elif 'path' in kwargs: # Assume path to ebook on disk + if os.path.exists(kwargs['path']): + sha256 = kwargs.get('sha256') or file_hash(kwargs['path']) + dest_path = os.path.join(self._storage_directory, sha256) + copy2(kwargs['path'], dest_path) + with self._connection: + self._connection.execute("INSERT INTO storage VALUES(?, ?, ?);", (kwargs['uuid'], sha256, f"file://{dest_path}")) + return f"file://{dest_path}" + + def save_ebook_metadata(self, ebook): + try: + book_id = ebook.uuid + except KeyError: + book_id = str(uuid.uuid4()) + ebook._metadata['uuid'] = book_id + with self._connection: + self._connection.execute("INSERT INTO books VALUES(?, ?, ?, ?) on conflict(id) do nothing;", (book_id, ebook.author, ebook.title, ebook.format)) + return book_id + + def get_ebook(self, identifier): + cursor = self._connection.execute("SELECT books.id, author, title, uri, sha256sum FROM storage JOIN books on books.id = storage.id WHERE sha256sum = ?;", (identifier,)) + results = cursor.fetchone() + if results: + return Ebook.from_dict(dict(zip(('uuid', 'author', 'title', 'path'), results))) + raise EbookNotFound('Unable to locate an ebook for identifier {}'.format(identifier)) + + def search_ebooks(self, **query): + raise NotImplementedError diff --git a/coppermind/common/db/mongo.py b/coppermind/common/db/mongo.py index f32f96d..c1a210c 100644 --- a/coppermind/common/db/mongo.py +++ b/coppermind/common/db/mongo.py @@ -47,4 +47,3 @@ def get_ebook(self, identifier): def search_ebooks(self, **query): raise NotImplementedError - diff --git a/coppermind/common/models/ebook.py b/coppermind/common/models/ebook.py index a4cf244..296c8a5 100644 --- a/coppermind/common/models/ebook.py +++ b/coppermind/common/models/ebook.py @@ -1,5 +1,8 @@ +import uuid import logging from ..tools import ebook_parser +from ..tools import SVCObj +from ..db.base import EbookNotFound class Ebook: @@ -14,6 +17,14 @@ class Ebook: def __init__(self, **ebook_data): self._metadata = ebook_data + def save(self, db): + if not self._metadata.get('uuid'): + try: + book_id = db.get_ebook(self._metadata['sha256sum']) + except EbookNotFound: + self._metadata['uuid'] = str(uuid.uuid4()) + db.save_ebook(self, **self.serialize()) + @classmethod def from_dict(self, ebook_data): """ @@ -37,6 +48,9 @@ def serialize(self): """ return self._metadata + def __repr__(self): + return f"<{self.__class__.__name__}({self._metadata['format']}::{self._metadata['sha256sum']})>" + def __getattr__(self, attr): try: return self._metadata[attr] diff --git a/coppermind/common/tools/parser.py b/coppermind/common/tools/parser.py index 4246442..349c9e0 100644 --- a/coppermind/common/tools/parser.py +++ b/coppermind/common/tools/parser.py @@ -6,7 +6,7 @@ from zipfile import ZipFile -__supported_formats__ = ['EPUB'] # TODO: Should pick from installed parsers +__supported_formats__ = ['EPUB', 'MOBI'] # TODO: Should pick from installed parsers class MissingEbookFile(Exception): @@ -24,17 +24,20 @@ def file_hash(path): return sha256.hexdigest() -def ebook_parser(ebook_file, fmt='EPUB'): +def ebook_parser(ebook_file, fmt=None): """ Given an ebook file, parse metadata and return as dict """ - if os.path.exists(ebook_file): - if fmt.upper() not in __supported_formats__: - raise NotImplementedError('{} not yet implemented'.format(fmt.upper())) - fmt_parser = getattr(sys.modules[__name__], '_{}_parser'.format(fmt.lower())) - return fmt_parser(ebook_file) - else: + if not os.path.exists(ebook_file): raise MissingEbookFile("{} was not found, current path: {}".format(ebook_file, os.curdir)) + if fmt is None: + fmt = ebook_file.split('.')[-1].lower() + if fmt.upper() not in __supported_formats__: + raise NotImplementedError('{} not yet implemented'.format(fmt.upper())) + fmt_parser = getattr(sys.modules[__name__], '_{}_parser'.format(fmt.lower())) + parsed = fmt_parser(ebook_file) + parsed.update({'format': fmt.lower(), 'sha256sum': file_hash(ebook_file), 'path': ebook_file}) + return parsed raise Exception("Why did I get here?") @@ -49,9 +52,10 @@ def _epub_parser(epub): sha256 = file_hash(epub) zf = ZipFile(epub) xml = xmltodict.parse(zf.read('META-INF/container.xml')) - metadata_path = xml['container']['rootfiles']['rootfile']['@full-path'] # TODO: validate this is true for all EPUBs + # TODO: validate this is true for all EPUBs + metadata_path = xml['container']['rootfiles']['rootfile']['@full-path'] raw_metadata = xmltodict.parse(zf.read(metadata_path)) - metadata = {'format': 'epub'} + metadata = {} for k, v in raw_metadata['package']['metadata'].items(): if 'dc:' in k: if 'creator' in k: # Required element, needs additional parsing @@ -63,7 +67,8 @@ def _epub_parser(epub): v = [v] # Just in case we get a single element identifiers = [] for i in v: - identifiers.append({'identifier': i['@opf:scheme'], 'value': i['#text']}) # Support multiple identifiers + # Support multiple identifiers + identifiers.append({'identifier': i['@opf:scheme'], 'value': i['#text']}) v = identifiers metadata[k.split('dc:')[-1]] = v metadata['identifiers'].append({'identifier': 'sha256', 'value': sha256}) diff --git a/coppermind/daemon/__init__.py b/coppermind/daemon/__init__.py index 9ca4913..fffa064 100644 --- a/coppermind/daemon/__init__.py +++ b/coppermind/daemon/__init__.py @@ -1 +1 @@ -from .main import Coppermind +from .daemon import Coppermind diff --git a/coppermind/daemon/daemon.py b/coppermind/daemon/daemon.py index 689392f..c6ed3fc 100644 --- a/coppermind/daemon/daemon.py +++ b/coppermind/daemon/daemon.py @@ -3,7 +3,9 @@ import yact import logging from time import sleep -from coppermind.tools import SVC, SVCObj +from ..common.tools import SVC, SVCObj +from ..common.db.filesystem import FileSystem +from .threads.watch_directory import WatchDirectory class Coppermind(): @@ -21,22 +23,37 @@ def __init__(self): def run(self): self.setup_config() self.setup_logging() + self.setup_db() + self._dir_watcher = WatchDirectory() + self._dir_watcher.start() while not self.svc.shutdown: sleep(1) + self._dir_watcher.join(1) + def setup_db(self): + self.svc.db = FileSystem() def setup_config(self): filename = "coppermind.conf" config = yact.from_file(filename) + self.svc.config = config def setup_logging(self): log = logging.getLogger(__name__) - log.setLevel(getattr(logging, self.svc.config.logging.level.upper())) + log.setLevel(getattr(logging, self.svc.config.get('logging.level').upper())) + logformat = logging.Formatter(fmt='%(asctime)s [%(levelname)s] (%(threadName)-10s) %(message)s', + datefmt='%Y-%m-%d %H:%M:%S') - file_handler = logging.FileHandler(self.svc.config.logging.filename) stream_handler = logging.StreamHandler() - - log.addHandler(file_handler) + stream_handler.setFormatter(logformat) log.addHandler(stream_handler) + try: + file_handler = logging.FileHandler(self.svc.config['logging.filename']) + file_handler.setFormatter(logformat) + log.addHandler(file_handler) + except KeyError: + log.info("No logfile configured, skipping persistent log output") + logging.root.handlers.clear() logging.root = log - logging.debug("Logging Setup Complete") + log.addHandler(stream_handler) + log.debug("Logging Setup Complete") diff --git a/coppermind/daemon/threads/watch_directory.py b/coppermind/daemon/threads/watch_directory.py new file mode 100644 index 0000000..2e3049e --- /dev/null +++ b/coppermind/daemon/threads/watch_directory.py @@ -0,0 +1,45 @@ +import os +import glob +import logging +import threading +from time import sleep +from ...common.tools import SVCObj +from ...common.models import Ebook +from ...common.db.base import EbookNotFound + +logger = logging.getLogger() + +class WatchDirectory(threading.Thread, SVCObj): + """ + Watch a directory for ebooks + """ + def __init__(self): + super().__init__() + self.name = f"{self.__class__.__name__}" + self.daemon = True + + def run(self): + logging.debug(f"Starting up") + folder = self.svc.config.get('watch_directory', '.') + logging.debug(f"Watching directory {folder}") + while not self.svc.shutdown: + try: + sleep(1) + for discovered_file in self._scan(folder): + ebook = Ebook.from_file(discovered_file) + try: + self.svc.db.get_ebook(ebook.serialize()['sha256sum']) + except EbookNotFound: + logging.debug(f"Saving {ebook}") + ebook.save(self.svc.db) + except Exception as e: + logging.warning(f"Unknown failure scanning {folder}: {e}", exc_info=1) + sleep(1) + + def _scan(self, watch_directory, suffixes=None): + _suffixes = suffixes or ['.epub'] + for s in _suffixes: + logging.debug(f"Scanning for files ending in {s} in {watch_directory}") + for f in glob.glob(f"*{s}", root_dir=watch_directory, recursive=True): + logging.debug(f"Found file {f} in {watch_directory}") + yield os.path.join(watch_directory, f) diff --git a/coppermind/daemon/threads/webservice.py b/coppermind/daemon/threads/webservice.py index bfe2604..98d3586 100644 --- a/coppermind/daemon/threads/webservice.py +++ b/coppermind/daemon/threads/webservice.py @@ -1 +1,28 @@ -from flask import request, app +from flask import Flask, jsonify + +app = Flask(__name__.split('.')[0]) +PORT = 9090 +BIND = '0.0.0.0' +DEBUG = True + +app.config.from_object(__name__) + + +@app.route('/') +def index(): + return 'Coppermind' + + +@app.route('/ebooks') +def ebooks(): + return jsonify([]) + + +@app.route('/ebooks/') +def ebook(book_id): + return jsonify({}) + + +@app.route('/upload') +def upload(): + pass diff --git a/coppermind/db/filesystem.py b/coppermind/db/filesystem.py deleted file mode 100644 index d88e8c4..0000000 --- a/coppermind/db/filesystem.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import uuid -import yact -from shutil import copy2 -from ..models import Ebook -from ..tools.parser import file_hash -from datetime import datetime -from .base import BaseDB, EbookNotFound - - -class Filesystem(BaseDB): - def __init__(self): - self.filepath = os.path.join(os.path.pardir, 'coppermind-storage') - if not os.path.exists(self.filepath): - os.makedirs(self.filepath) - self.mapping = yact.from_file('map.yaml', self.filepath) - - def get_ebook_file(self, book_id): - path = self.mapping.get(book_id)['filepath'] - with open(os.path.join(self.filepath, path)) as book: - return book.read() - - def store_ebook_file(self, **kwargs): - if 'file' in kwargs: # Assume file-like object - raise NotImplementedError('Epub only for now') - elif 'path' in kwargs: # Assume path to ebook on disk - if os.path.exists(kwargs['path']): - sha256 = kwargs.get('sha256') or file_hash(kwargs['path']) - copy2(kwargs['path'], os.path.join(self.filepath, sha256, os.path.sep)) - return sha256 - - def save_ebook_metadata(self, ebook): - # if not ebook.get('uuid'): - # mongo_uuid = str(uuid.uuid4()) - # ebook['identifiers'].append({'identifier': 'coppermind_id', 'value': mongo_uuid}) - # ebook['uuid'] = mongo_uuid - # self._connection.metadata.update_one({'uuid': mongo_uuid}, {'$set': ebook}, upsert=True) - # return mongo_uuid - pass - - def get_ebook(self, identifier): - - data = self._connection.metadata.find_one({'identifiers.value': identifier}, {'_id': 0}) - if data: - return Ebook.from_dict(data) - raise EbookNotFound('Unable to locate an ebook for identifier {}'.format(identifier)) - - def search_ebooks(self, **query): - raise NotImplementedError diff --git a/coppermind/webservice/__init__.py b/coppermind/webservice/__init__.py deleted file mode 100644 index c07c459..0000000 --- a/coppermind/webservice/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .app import app diff --git a/coppermind/webservice/app.py b/coppermind/webservice/app.py deleted file mode 100644 index 5b60f6c..0000000 --- a/coppermind/webservice/app.py +++ /dev/null @@ -1,29 +0,0 @@ -from flask import Flask, jsonify -from coppermind.common import - -app = Flask(__name__.split('.')[0]) -PORT = 9090 -BIND = '0.0.0.0' -DEBUG = True - -app.config.from_object(__name__) - - -@app.route('/') -def index(): - return 'Coppermind' - - -@app.route('/ebooks') -def ebooks(): - return jsonify([]) - - -@app.route('/ebooks/') -def ebook(book_id): - return jsonify({}) - - -@app.route('/upload') -def upload(): - pass diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..eb8f208 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "coppermind" +version = "0.0.1" +dependencies = [] + +[tool.setuptools.packages.find] +where = ["coppermind"] +exclude = ["tests*"] + + +[tool.ruff] +line-length = 100 +ignore = [ + "F401" +] diff --git a/readme.rst b/readme.rst index 1da9cfe..0860206 100644 --- a/readme.rst +++ b/readme.rst @@ -17,5 +17,3 @@ Once I have all the basics complete, I may also add in the concept of profiles t __ http://brandonsanderson.com/ .. _Mistborn: http://brandonsanderson.com/books/mistborn/ - - diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..3a2fa3a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +pytest +pytest-cov +ruff +pre-commit diff --git a/requirements.txt b/requirements.txt index 9d875b7..2f3e3bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -pymongo xmltodict yact flask diff --git a/scripts/coppermind b/scripts/coppermind old mode 100644 new mode 100755 diff --git a/scripts/coppermind-ws b/scripts/coppermind-ws deleted file mode 100644 index 0f502ad..0000000 --- a/scripts/coppermind-ws +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python -from coppermind import CoppermindWS - -if __name__ == "__main__": - config = CoppermindWS.config - CoppermindWS.run(config['BIND'], config['PORT'], config['DEBUG']) diff --git a/tests/test_db_mongo.py b/tests/test_db_mongo.py deleted file mode 100644 index 717cf7d..0000000 --- a/tests/test_db_mongo.py +++ /dev/null @@ -1,24 +0,0 @@ -import unittest -import hashlib -from . import test_models_ebook -from coppermind.db.mongo import Mongo - - -class test_DB_Mongo(test_models_ebook.testEbook): - - db = Mongo() - - def setUp(self): - self.db._connection.metadata.create_index('uuid', unique=True) - self.db._connection.data_files.create_index('sha256', unique=True) - - def tearDown(self): - """ - Cleanup db after unittests - """ - self.db._connection.metadata.drop() - self.db._connection.data_files.drop() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_models_ebook.py b/tests/test_models_ebook.py index c6cf146..13ac017 100644 --- a/tests/test_models_ebook.py +++ b/tests/test_models_ebook.py @@ -1,8 +1,9 @@ import os import logging import unittest -from coppermind.models import Ebook -from coppermind.tools.parser import file_hash, _mobi_parser, InvalidEbookFile +from coppermind.common.db.filesystem import Filesystem +from coppermind.common.models import Ebook +from coppermind.common.tools.parser import file_hash, _mobi_parser, InvalidEbookFile fake_ebook = {"title": 'FooBar', @@ -56,6 +57,7 @@ def test_get_ebook(self): ebook = Ebook.from_file(self.sample_ebook) ebook_id = self.db.save_ebook(ebook, path=self.sample_ebook) self.assertIsNotNone(self.db.get_ebook_file(sha256)) + self.assertIsNotNone(self.db.get_ebook_file(ebook_id)) def test_duplicate(self): if not self.db: diff --git a/todo.txt b/todo.txt new file mode 100644 index 0000000..be913e5 --- /dev/null +++ b/todo.txt @@ -0,0 +1,5 @@ +(A) Setup GitPod configuration +Bootstrap test ebooks (gutenberg?) +Ebook storage +Email ebook to address (eg, load to Kindle) +(B) Setup repository automation