Optimize initial import and update to MusicBrainz schema 30

.github/workflows/build-mbslave-container-image.yml

@@ -4,15 +4,17 @@ on:
   push:
     branches:
       - main
+      - dev
     tags:
       - v*
   pull_request:
     branches:
       - main
+      - dev
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: acoustid/mbslave
+  IMAGE_NAME: ${{ github.repository_owner }}/mbslave
 
 jobs:
 
@@ -23,27 +25,38 @@ jobs:
       packages: write
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=sha
 
       - name: Log in to the Container registry
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           file: Dockerfile.mbslave
           context: .
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/test.yml

@@ -7,29 +7,54 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+    container: python:${{ matrix.python_version }}
     strategy:
       matrix:
         python_version: [ '3.8', '3.9', '3.10' ]
+    services:
+      postgresql:
+        image: postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: notapassword
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
-    - uses: actions/checkout@v2
-    - name: Install Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python_version }}
+    - uses: actions/checkout@v4
+    - name: Install dependencies
+      run: |
+        apt-get update
+        apt-get install -y postgresql-client libpq-dev sudo
     - name: Install Poetry
       uses: snok/install-poetry@v1
+      with:
+        version: 1.8.5
+        virtualenvs-create: true
+        virtualenvs-in-project: true
     - name: Install application dependencies
       run: poetry install
     - name: Run tests
-      run: ./check.sh
+      run: |
+        poetry run mbslave init --create-user --create-database --empty
+        ./check.sh
+      env:
+        MBSLAVE_DB_HOST: postgresql
+        MBSLAVE_DB_PORT: 5432
+        MBSLAVE_DB_DB: musicbrainz
+        MBSLAVE_DB_USER: musicbrainz
+        MBSLAVE_DB_PASSWORD: reallynotapassword
+        MBSLAVE_DB_ADMIN_USER: postgres
+        MBSLAVE_DB_ADMIN_PASSWORD: notapassword
 
   mbslave_initdb_test:
     runs-on: ubuntu-latest
+    container: python:3.10
     services:
       postgresql:
         image: postgres
-        ports:
-          - 5432/tcp
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: notapassword
@@ -40,21 +65,25 @@ jobs:
           --health-retries 5
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
-      - name: Install Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.10'
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install -y postgresql-client libpq-dev sudo
       - name: Install Poetry
         uses: snok/install-poetry@v1
+        with:
+          version: 1.8.5
+          virtualenvs-create: true
+          virtualenvs-in-project: true
       - name: Install application dependencies
         run: poetry install
       - name: Run `mbslave init`
         run: poetry run mbslave init --create-user --create-database --empty
         env:
-          MBSLAVE_DB_HOST: 127.0.0.1
-          MBSLAVE_DB_PORT: ${{ job.services.postgresql.ports['5432'] }}
-          MBSLAVE_DB_NAME: musicbrainz
+          MBSLAVE_DB_HOST: postgresql
+          MBSLAVE_DB_PORT: 5432
+          MBSLAVE_DB_DB: musicbrainz
           MBSLAVE_DB_USER: musicbrainz
           MBSLAVE_DB_PASSWORD: reallynotapassword
           MBSLAVE_DB_ADMIN_USER: postgres

.gitignore

@@ -8,3 +8,5 @@
 mbdata.egg-info/
 venv/
 result
+/.idea/
+/.bin/

CHANGELOG.rst

@@ -1,3 +1,12 @@
+Version 30.0.0
+==============
+
+- Massive optimization of initial import by temporarily disabling triggers and constraints (MBS-13492).
+- Updated to MusicBrainz schema 30.
+- Added progress bars (`tqdm`) during data import.
+- Improved Docker environment with `docker-compose.dev.yml` and a more robust entrypoint script.
+- Updated dependencies.
+
 Version 29.1.0
 ==============
 

DEVELOPMENT.rst

@@ -5,15 +5,50 @@ Development Guide
 Development Setup
 =================
 
-Clone the repository and setup virtualenv::
+Clone the repository and install the project using `poetry <https://python-poetry.org/>`__.
+If you don't have poetry installed, you can install it using ``pipx install poetry`` or ``pip install poetry``.
+
+::
 
     git clone https://github.com/acoustid/mbslave.git
     cd mbslave/
-    virtualenv -p python3 venv
-    source venv/bin/activate
-    pip install poetry
     poetry install
 
+Poetry will automatically create a virtual environment for you. You can then run commands
+within this environment using ``poetry run mbslave <command>`` or by activating the shell with ``poetry shell``.
+For example, to see available commands:
+
+::
+
+    poetry run mbslave --help
+
+Development Database (PostgreSQL)
+=================================
+
+For development and testing, you can use the provided ``Dockerfile.postgres`` to start a pre-initialized PostgreSQL database with the MusicBrainz schema.
+
+Using Docker Compose:
+
+::
+
+    docker compose -f docker-compose.dev.yml up -d
+
+This will start a PostgreSQL container on port 5432 with:
+* Database: ``musicbrainz``
+* User: ``musicbrainz``
+* Password: ``musicbrainz``
+
+The database will be automatically initialized with the schema when the container starts.
+
+Running Tests
+=============
+
+You can run the tests using ``pytest``. Some tests require a running development database (see above).
+
+::
+
+    poetry run pytest
+
 Updating SQL files and models
 =============================
 
@@ -37,3 +72,5 @@ Release a new version
     rm -rf dist/
     python setup.py sdist
     twine upload dist/mbslave-*.tar.gz
+
+.. 

Dockerfile.mbslave

@@ -1,7 +1,7 @@
 FROM python:3.10 AS build
 
 RUN python3 -m venv /opt/poetry && \
-    /opt/poetry/bin/pip install poetry && \
+    /opt/poetry/bin/pip install poetry==1.8.5 && \
     ln -s /opt/poetry/bin/poetry /usr/local/bin/ && \
     poetry --version
 
@@ -18,5 +18,7 @@ RUN apt-get update && \
 COPY --from=build /tmp/mbdata/dist/ /tmp/dist/
 RUN python -m pip install "$(ls -1 /tmp/dist/*.whl)[replication,models]" && rm -rf /tmp/dist/
 
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-CMD ["mbslave", "sync", "-r"]
+COPY docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "docker-entrypoint.sh"]

Dockerfile.postgres

@@ -0,0 +1,49 @@
+FROM postgres:16
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python3 \
+    python3-pip \
+    python3-venv \
+    postgresql-server-dev-16 \
+    gcc \
+    python3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m venv /opt/mbslave-venv
+ENV PATH="/opt/mbslave-venv/bin:$PATH"
+
+WORKDIR /usr/src/app
+COPY . .
+
+# Install mbslave and its dependencies
+RUN pip install .
+
+ENV POSTGRES_DB=musicbrainz
+ENV POSTGRES_USER=musicbrainz
+ENV POSTGRES_PASSWORD=musicbrainz
+
+# Create a default config file for mbslave
+RUN cat <<EOF > /etc/mbslave.conf
+[database]
+host=/var/run/postgresql
+port=5432
+name=musicbrainz
+user=musicbrainz
+password=musicbrainz
+admin_user=musicbrainz
+EOF
+
+# Add initialization script
+COPY <<-'INIT_DB' /docker-entrypoint-initdb.d/01-init-mbslave.sh
+#!/bin/bash
+set -e
+
+# mbslave init expects to be able to connect to the database
+# When scripts in /docker-entrypoint-initdb.d/ are run, the server is running
+# and accepting connections.
+# We use PGUSER=postgres to connect as the superuser that exists during init.
+
+PGUSER=postgres MBSLAVE_CONFIG=/etc/mbslave.conf mbslave init --empty
+INIT_DB
+
+RUN chmod +x /docker-entrypoint-initdb.d/01-init-mbslave.sh

README.rst

@@ -20,7 +20,6 @@ Installation
 You need to have `Python 3.x <https://python.org/>`__ installed on your system.
 You can use `pipx <https://pypa.github.io/pipx/>`__ to install this package::
 
-       sudo apt install python3 pipx
        pipx install 'mbslave'
 
 There are two ways to configure the application.
@@ -174,3 +173,5 @@ data import. You can temporarily map all schemas to e.g. "musicbrainz_NEW", impo
 database there and then rename it::
 
     echo 'BEGIN; ALTER SCHEMA musicbrainz RENAME TO musicbrainz_OLD; ALTER SCHEMA musicbrainz_NEW RENAME TO musicbrainz; COMMIT;' | mbslave psql -S
+
+.. 

docker-compose.dev.yml

@@ -0,0 +1,33 @@
+services:
+    db:
+        build:
+            context: .
+            dockerfile: Dockerfile.postgres
+        environment:
+            - POSTGRES_PASSWORD=musicbrainz
+        ports:
+            - "5432:5432"
+        volumes:
+            - db_data:/var/lib/postgresql/data
+        shm_size: 8gb
+        healthcheck:
+            test: [ "CMD-SHELL", "pg_isready -U musicbrainz -d musicbrainz" ]
+            interval: 5s
+            timeout: 5s
+            retries: 5
+
+    mbslave:
+        build:
+            context: .
+            dockerfile: Dockerfile.mbslave
+        environment:
+            - KEEP_ALIVE=true
+        volumes:
+            - mbslave_config:/etc/mbslave
+        depends_on:
+            db:
+                condition: service_healthy
+
+volumes:
+    db_data:
+    mbslave_config:

docker-entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+if [ "$KEEP_ALIVE" = "1" ] || [ "$KEEP_ALIVE" = "true" ] || [ $# -eq 0 ]; then
+    echo "KEEP_ALIVE is set or no command provided, staying up..."
+    # Keep the container running
+    tail -f /dev/null
+else
+    exec "$@"
+fi

mbslave/__init__.py

@@ -1,4 +1,4 @@
 # Copyright (C) 2013 Lukas Lalinsky
 # Distributed under the MIT license, see the LICENSE file for details.
 
-__version__ = "29.1.0"
+__version__ = "30.0.0"