diff --git a/annotationengine/views.py b/annotationengine/views.py index 8e6e608..ab035d0 100644 --- a/annotationengine/views.py +++ b/annotationengine/views.py @@ -22,7 +22,7 @@ def wkb_to_numpy(wkb, convert_to_nm=None): """Fixes single geometry column""" shp = to_shape(wkb) - xyz_voxel = np.array([shp.xy[0][0], shp.xy[1][0], shp.z], dtype=np.int) + xyz_voxel = np.array([shp.xy[0][0], shp.xy[1][0], shp.z], dtype=int) if convert_to_nm is not None: return xyz_voxel * convert_to_nm else: @@ -68,7 +68,13 @@ def aligned_volume_view(aligned_volume_name): .filter(Metadata.deleted == None) .filter(Metadata.valid == True) ) - df = pd.read_sql(query.statement, db.database.engine) + statement = str( + query.statement.compile( + db.database.engine, compile_kwargs={"literal_binds": True} + ) + ) + with db.database.engine.connect() as conn: + df = pd.read_sql(statement, conn.connection) base_user_url = "https://{auth_uri}/api/v1/user/{user_id}" auth_uri = os.environ["AUTH_URI"] base_schema_url = ( @@ -120,7 +126,13 @@ def table_view(aligned_volume_name, table_name): Model = db.database._get_model_from_table_name(table_name) table_size = db.database.get_annotation_table_size(table_name) query = db.database.cached_session.query(Model).limit(15) - top15_df = pd.read_sql(query.statement, db.database.engine) + statement = str( + query.statement.compile( + db.database.engine, compile_kwargs={"literal_binds": True} + ) + ) + with db.database.engine.connect() as conn: + top15_df = pd.read_sql(statement, conn.connection) top15_df = fix_wkb_columns(top15_df) return render_template( "table.html", diff --git a/dev.Dockerfile b/dev.Dockerfile index 313ebb3..ea1b22b 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -1,11 +1,31 @@ FROM tiangolo/uwsgi-nginx-flask:python3.9 -RUN git config --global http.sslVerify false && \ - mkdir -p /home/nginx/.cloudvolume/secrets && chown -R nginx /home/nginx && usermod -d /home/nginx -s /bin/bash nginx +RUN apt-get update && apt-get install -y gcc curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install uv + +# Enable bytecode compilation and avoid hardlinks in mounted cache volumes. +ENV UV_COMPILE_BYTECODE=1 +ENV UV_LINK_MODE=copy +ENV UV_PYTHON_DOWNLOADS=0 + +RUN git config --global http.sslVerify false && \ + mkdir -p /home/nginx/.cloudvolume/secrets && \ + chown -R nginx /home/nginx && \ + usermod -d /home/nginx -s /bin/bash nginx + +WORKDIR /app + +# Install runtime + dev/test dependencies from the lockfile before copying source. +COPY uv.lock pyproject.toml ./ +ENV UV_PROJECT_ENVIRONMENT="/usr/local/" +RUN --mount=type=cache,target=/root/.cache/uv \ + UV_VENV_ARGS="--system-site-packages" uv sync --frozen --no-install-project --group dev + +ENV UWSGI_INI /app/uwsgi.ini +ENV PATH /app/.venv/bin:/home/nginx/google-cloud-sdk/bin:/root/google-cloud-sdk/bin:$PATH +ENV PYTHONNOUSERSITE=1 -RUN python -m pip install --upgrade pip -COPY dev_requirements.txt /app/. -RUN pip install -r dev_requirements.txt COPY timeout.conf /etc/nginx/conf.d/ COPY . /app - \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 86b96a1..fe3a312 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,24 @@ services: depends_on: - db + test: + image: "app:tag" + build: + context: . + dockerfile: dev.Dockerfile + env_file: + - ./.env-docker.dev + environment: + FLASK_CONFIGURATION: docker_testing + AUTH_DISABLED: "true" + volumes: + - .:/app + depends_on: + db: + condition: service_healthy + entrypoint: [] + command: ["python", "-m", "pytest"] + volumes: postgres_data: driver: local diff --git a/scripts/run_tests_with_compose.sh b/scripts/run_tests_with_compose.sh new file mode 100755 index 0000000..0d9d83d --- /dev/null +++ b/scripts/run_tests_with_compose.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Run the test suite inside a Docker container using the docker-compose stack. +# +# Usage: +# ./scripts/run_tests_with_compose.sh [--down] [pytest args...] +# +# Examples: +# ./scripts/run_tests_with_compose.sh # run tests, leave db up +# ./scripts/run_tests_with_compose.sh --down # run tests, then tear down +# ./scripts/run_tests_with_compose.sh -v test/test_api.py # run specific tests with verbose + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$PROJECT_ROOT" + +TEAR_DOWN=false +EXTRA_ARGS=() +for arg in "$@"; do + case "$arg" in + --down) + TEAR_DOWN=true + ;; + *) + EXTRA_ARGS+=("$arg") + ;; + esac +done + +# Prefer docker compose (v2) over docker-compose (v1) +COMPOSE_CMD="docker compose" +if ! docker compose version &>/dev/null; then + COMPOSE_CMD="docker-compose" +fi + +echo "Starting Postgres and running tests in Docker..." +# Pass extra args to pytest; -- separates compose args from the command +$COMPOSE_CMD run --build --rm test python -m pytest "${EXTRA_ARGS[@]}" +RESULT=$? + +if [[ "$TEAR_DOWN" == "true" ]]; then + echo "Tearing down docker-compose services..." + $COMPOSE_CMD down +fi + +exit $RESULT diff --git a/test/conftest.py b/test/conftest.py index 5a9edb5..84a7f9c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -59,7 +59,8 @@ def test_aligned_volume(): @pytest.fixture(scope="module") def client(): - flask_app = create_app(config_name="testing") + config_name = os.environ.get("FLASK_CONFIGURATION", "testing") + flask_app = create_app(config_name=config_name) test_logger.info("Starting test flask app...") # Create a test client using the Flask application configured for testing diff --git a/test/test_views_pandas_compat.py b/test/test_views_pandas_compat.py new file mode 100644 index 0000000..c75cfa7 --- /dev/null +++ b/test/test_views_pandas_compat.py @@ -0,0 +1,57 @@ +""" +Regression test for pd.read_sql compatibility with SQLAlchemy 1.4+ and pandas 2.x. + +pandas 2.3+ requires SQLAlchemy >= 2.0.0. When SQLAlchemy 1.4 is installed, +pandas' import_optional_dependency("sqlalchemy") returns None, so pd.read_sql +falls through to the SQLiteDatabase DBAPI2 path and raises: + TypeError: Query must be a string unless using sqlalchemy. + +The fix is to execute via SQLAlchemy directly and build the DataFrame from the +result, bypassing pd.read_sql entirely: + with engine.connect() as conn: + result = conn.execute(query.statement) + df = pd.DataFrame(result.fetchall(), columns=result.keys()) +""" + +import json +import os +from unittest import mock + +import pytest + +aligned_volume_name = "test_aligned_volume" +table_name = "test_view_table" + + +@pytest.fixture(scope="module", autouse=True) +def create_test_table(client): + data = { + "table_name": table_name, + "schema_type": "synapse", + "metadata": { + "user_id": "1", + "description": "Test table for views", + "flat_segmentation_source": "precomputed://gs://test/image", + "voxel_resolution_x": 4, + "voxel_resolution_y": 4, + "voxel_resolution_z": 40, + }, + } + url = f"/annotation/api/v2/aligned_volume/{aligned_volume_name}/table" + with mock.patch("annotationengine.api.check_aligned_volume") as mock_cv: + mock_cv.return_value = aligned_volume_name + client.post(url, json=data, content_type="application/json", follow_redirects=True) + + +class TestViewPages: + def test_aligned_volume_view_returns_200(self, client): + url = f"/annotation/views/aligned_volume/{aligned_volume_name}" + with mock.patch.dict(os.environ, {"AUTH_URI": "auth.test.example.com"}): + response = client.get(url, follow_redirects=False) + assert response.status_code == 200 + + def test_table_view_returns_200(self, client): + url = f"/annotation/views/aligned_volume/{aligned_volume_name}/table/{table_name}" + with mock.patch.dict(os.environ, {"AUTH_URI": "auth.test.example.com"}): + response = client.get(url, follow_redirects=False) + assert response.status_code == 200