Jr/dusting off cobwebs

.editorconfig

@@ -0,0 +1,34 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+# Matches multiple files with brace expansion notation
+# Set default charset
+[*.{js,py}]
+charset = utf-8
+
+# 4 space indentation
+[*.py]
+indent_style = space
+indent_size = 4
+max_line_length = 100
+
+# Tab indentation (no size specified)
+[Makefile]
+indent_style = tab
+
+# Indentation override for all JS under lib directory
+[lib/**.js]
+indent_style = space
+indent_size = 2
+
+# Matches the exact files either package.json or .travis.yml
+[{package.json,.travis.yml}]
+indent_style = space
+indent_size = 2

.github/workflows/ci.yaml

@@ -0,0 +1,34 @@
+name: CI
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ruff pytest pytest-cov
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Lint with ruff
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          ruff --format=github --select=E9,F63,F7,F82 --target-version=py37 .
+          # default set of ruff rules with GitHub Annotations
+          ruff --format=github --target-version=py37 .
+        continue-on-error: true
+      - name: Test with pytest
+        run: |
+          pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=coppermind --cov-report=xml --cov-report=html

.gitignore

@@ -4,3 +4,13 @@
 coppermind.conf
 *.idea
 .idea
+.coverage
+coverage.xml
+test-results.xml
+junit
+htmlcov
+*_cache
+__pycache__
+.tox
+*.egg-info
+dist

.gitpod.yml

@@ -0,0 +1,14 @@
+---
+# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart
+tasks:
+  - init: |
+      pip install -r requirements.txt
+      pip install -r requirements-dev.txt
+      pre-commit install
+
+vscode:
+  extensions:
+    - Vue.volar
+    - EditorConfig.EditorConfig
+    - davraamides.todotxt-mode
+    - ms-python.python

coppermind/__init__.py

@@ -1 +1 @@
-from .webservice import app as CoppermindWS
+from .daemon import Coppermind

coppermind/common/db/__init__.py

@@ -1 +0,0 @@
-

coppermind/common/db/base.py

@@ -2,12 +2,14 @@
 import logging
 from abc import ABCMeta, abstractmethod
 
+from ..tools import SVCObj
+
 
 class EbookNotFound(Exception):
     pass
 
 
-class BaseDB(metaclass=ABCMeta):
+class BaseDB(SVCObj, metaclass=ABCMeta):
     """
     Base for all DB classes. Defines a standard interface
     allowing multiple DB implementations without requiring
@@ -16,9 +18,15 @@ class BaseDB(metaclass=ABCMeta):
     Unittests for any DB implementation should simply be
     running the ebook unittest against that implementation
     """
+    @property
+    def config(self):
+        """
+        Return database configuration
+        """
+        return self.svc.config['db']
 
     @abstractmethod
-    def get_ebook_file(self, uuid):
+    def get_ebook_file(self, ebook_id):
         """
         Return actual ebook file by UUID
         """
@@ -31,9 +39,7 @@ def store_ebook_file(self, ebook_file):
 
     def save_ebook(self, ebook, **kwargs):
         data_file = self.store_ebook_file(fmt=ebook.format, **kwargs)
-        metadata = ebook.serialize()
-        metadata['storage'] = {'mongo': data_file}
-        ebook_uuid = self.save_ebook_metadata(metadata)
+        ebook_uuid = self.save_ebook_metadata(ebook=ebook)
         return ebook_uuid
 
     @abstractmethod

coppermind/common/db/filesystem.py

@@ -0,0 +1,68 @@
+import os
+import uuid
+import sqlite3
+import yact
+from shutil import copy2
+from ..models import Ebook
+from ..tools.parser import file_hash
+from datetime import datetime
+from .base import BaseDB, EbookNotFound
+
+
+class FileSystem(BaseDB):
+
+    def __init__(self):
+        self._storage_directory = os.path.join(os.path.pardir, 'coppermind-storage')
+        if not os.path.exists(self._storage_directory):
+            os.makedirs(self._storage_directory)
+        self._connection = sqlite3.connect(os.path.join(self._storage_directory, self.config.get('filename', 'coppermind.db')), check_same_thread=False)
+        with self._connection:
+            self._connection.execute(""" CREATE TABLE IF NOT EXISTS books (
+                                        id text PRIMARY KEY,
+                                        author text NOT NULL,
+                                        title text NOT NULL,
+                                        format text NOT NULL
+                                         ); """)
+            self._connection.execute(""" CREATE TABLE IF NOT EXISTS storage (
+                                        id text PRIMARY KEY,
+                                        sha256sum text NOT NULL,
+                                        uri text NOT NULL
+                                         ); """)
+
+    def get_ebook_file(self, book_id):
+        with self._connection:
+            path = self._connection.execute("SELECT uri FROM storage where id = ?", (book_id,))
+        with open(path) as book:
+            return book.read()
+
+    def store_ebook_file(self, **kwargs):
+        if 'file' in kwargs:  # Assume file-like object
+            raise NotImplementedError('Epub only for now')
+        elif 'path' in kwargs:  # Assume path to ebook on disk
+            if os.path.exists(kwargs['path']):
+                sha256 = kwargs.get('sha256') or file_hash(kwargs['path'])
+                dest_path = os.path.join(self._storage_directory, sha256)
+                copy2(kwargs['path'], dest_path)
+        with self._connection:
+            self._connection.execute("INSERT INTO storage VALUES(?, ?, ?);", (kwargs['uuid'], sha256, f"file://{dest_path}"))
+        return f"file://{dest_path}"
+
+    def save_ebook_metadata(self, ebook):
+        try:
+            book_id = ebook.uuid
+        except KeyError:
+            book_id = str(uuid.uuid4())
+            ebook._metadata['uuid'] = book_id
+        with self._connection:
+            self._connection.execute("INSERT INTO books VALUES(?, ?, ?, ?) on conflict(id) do nothing;", (book_id, ebook.author, ebook.title, ebook.format))
+        return book_id
+
+    def get_ebook(self, identifier):
+        cursor = self._connection.execute("SELECT books.id, author, title, uri, sha256sum FROM storage JOIN books on books.id = storage.id WHERE sha256sum = ?;", (identifier,))
+        results = cursor.fetchone()
+        if results:
+            return Ebook.from_dict(dict(zip(('uuid', 'author', 'title', 'path'), results)))
+        raise EbookNotFound('Unable to locate an ebook for identifier {}'.format(identifier))
+
+    def search_ebooks(self, **query):
+        raise NotImplementedError

coppermind/common/db/mongo.py

@@ -47,4 +47,3 @@ def get_ebook(self, identifier):
 
     def search_ebooks(self, **query):
         raise NotImplementedError
-

coppermind/common/models/ebook.py

@@ -1,5 +1,8 @@
+import uuid
 import logging
 from ..tools import ebook_parser
+from ..tools import SVCObj
+from ..db.base import EbookNotFound
 
 
 class Ebook:
@@ -14,6 +17,14 @@ class Ebook:
     def __init__(self, **ebook_data):
         self._metadata = ebook_data
 
+    def save(self, db):
+        if not self._metadata.get('uuid'):
+            try:
+                book_id = db.get_ebook(self._metadata['sha256sum'])
+            except EbookNotFound:
+                self._metadata['uuid'] = str(uuid.uuid4())
+        db.save_ebook(self, **self.serialize())
+
     @classmethod
     def from_dict(self, ebook_data):
         """
@@ -37,6 +48,9 @@ def serialize(self):
         """
         return self._metadata
 
+    def __repr__(self):
+        return f"<{self.__class__.__name__}({self._metadata['format']}::{self._metadata['sha256sum']})>"
+
     def __getattr__(self, attr):
         try:
             return self._metadata[attr]

coppermind/common/tools/parser.py

@@ -6,7 +6,7 @@
 from zipfile import ZipFile
 
 
-__supported_formats__ = ['EPUB']  # TODO: Should pick from installed parsers
+__supported_formats__ = ['EPUB', 'MOBI']  # TODO: Should pick from installed parsers
 
 
 class MissingEbookFile(Exception):
@@ -24,17 +24,20 @@ def file_hash(path):
     return sha256.hexdigest()
 
 
-def ebook_parser(ebook_file, fmt='EPUB'):
+def ebook_parser(ebook_file, fmt=None):
     """
     Given an ebook file, parse metadata and return as dict
     """
-    if os.path.exists(ebook_file):
-        if fmt.upper() not in __supported_formats__:
-            raise NotImplementedError('{} not yet implemented'.format(fmt.upper()))
-        fmt_parser = getattr(sys.modules[__name__], '_{}_parser'.format(fmt.lower()))
-        return fmt_parser(ebook_file)
-    else:
+    if not os.path.exists(ebook_file):
         raise MissingEbookFile("{} was not found, current path: {}".format(ebook_file, os.curdir))
+    if fmt is None:
+        fmt = ebook_file.split('.')[-1].lower()
+    if fmt.upper() not in __supported_formats__:
+        raise NotImplementedError('{} not yet implemented'.format(fmt.upper()))
+    fmt_parser = getattr(sys.modules[__name__], '_{}_parser'.format(fmt.lower()))
+    parsed = fmt_parser(ebook_file)
+    parsed.update({'format': fmt.lower(), 'sha256sum': file_hash(ebook_file), 'path': ebook_file})
+    return parsed
     raise Exception("Why did I get here?")
 
 
@@ -49,9 +52,10 @@ def _epub_parser(epub):
     sha256 = file_hash(epub)
     zf = ZipFile(epub)
     xml = xmltodict.parse(zf.read('META-INF/container.xml'))
-    metadata_path = xml['container']['rootfiles']['rootfile']['@full-path']  # TODO: validate this is true for all EPUBs
+    # TODO: validate this is true for all EPUBs
+    metadata_path = xml['container']['rootfiles']['rootfile']['@full-path']
     raw_metadata = xmltodict.parse(zf.read(metadata_path))
-    metadata = {'format': 'epub'}
+    metadata = {}
     for k, v in raw_metadata['package']['metadata'].items():
         if 'dc:' in k:
             if 'creator' in k:  # Required element, needs additional parsing
@@ -63,7 +67,8 @@ def _epub_parser(epub):
                     v = [v]  # Just in case we get a single element
                 identifiers = []
                 for i in v:
-                    identifiers.append({'identifier': i['@opf:scheme'], 'value': i['#text']})  # Support multiple identifiers
+                    # Support multiple identifiers
+                    identifiers.append({'identifier': i['@opf:scheme'], 'value': i['#text']})
                 v = identifiers
             metadata[k.split('dc:')[-1]] = v
     metadata['identifiers'].append({'identifier': 'sha256', 'value': sha256})

coppermind/daemon/__init__.py

@@ -1 +1 @@
-from .main import Coppermind
+from .daemon import Coppermind

coppermind/daemon/daemon.py

@@ -3,7 +3,9 @@
 import yact
 import logging
 from time import sleep
-from coppermind.tools import SVC, SVCObj
+from ..common.tools import SVC, SVCObj
+from ..common.db.filesystem import FileSystem
+from .threads.watch_directory import WatchDirectory
 
 
 class Coppermind():
@@ -21,22 +23,37 @@ def __init__(self):
     def run(self):
         self.setup_config()
         self.setup_logging()
+        self.setup_db()
+        self._dir_watcher = WatchDirectory()
+        self._dir_watcher.start()
         while not self.svc.shutdown:
             sleep(1)
+        self._dir_watcher.join(1)
 
+    def setup_db(self):
+        self.svc.db = FileSystem()
 
     def setup_config(self):
         filename = "coppermind.conf"
         config = yact.from_file(filename)
+        self.svc.config = config
 
     def setup_logging(self):
         log = logging.getLogger(__name__)
-        log.setLevel(getattr(logging, self.svc.config.logging.level.upper()))
+        log.setLevel(getattr(logging, self.svc.config.get('logging.level').upper()))
+        logformat = logging.Formatter(fmt='%(asctime)s [%(levelname)s] (%(threadName)-10s) %(message)s',
+                                      datefmt='%Y-%m-%d %H:%M:%S')
 
-        file_handler = logging.FileHandler(self.svc.config.logging.filename)
         stream_handler = logging.StreamHandler()
-
-        log.addHandler(file_handler)
+        stream_handler.setFormatter(logformat)
         log.addHandler(stream_handler)
+        try:
+            file_handler = logging.FileHandler(self.svc.config['logging.filename'])
+            file_handler.setFormatter(logformat)
+            log.addHandler(file_handler)
+        except KeyError:
+            log.info("No logfile configured, skipping persistent log output")
+        logging.root.handlers.clear()
         logging.root = log
-        logging.debug("Logging Setup Complete")
+        log.addHandler(stream_handler)
+        log.debug("Logging Setup Complete")