test: migrate tests to mender-server compose files

.env

@@ -1,8 +1,14 @@
-MENDER_SERVER_REGISTRY=docker.io
-MENDER_SERVER_ENTERPRISE_REGISTRY=registry.mender.io
-MENDER_SERVER_REPOSITORY=mendersoftware
-MENDER_SERVER_ENTERPRISE_REPOSITORY=mender-server-enterprise
-MENDER_SERVER_TAG=main
+# Tag for all mender-server images (OS and Enterprise).
+#
+# Deliberately do NOT set MENDER_IMAGE_REGISTRY or MENDER_IMAGE_REPOSITORY here:
+# mender-server's base and enterprise compose files reuse those same two
+# variables with different defaults (docker.io/mendersoftware vs
+# registry.mender.io/mender-server-enterprise), so setting either one globally
+# breaks the other flavour.
+MENDER_IMAGE_TAG=main
+
+# mongod 8.0 requires AVX; pin lower for runners that lack it.
+MONGO_VERSION=7.0
 
 MENDER_CLIENT_REGISTRY=docker.io
 MENDER_CLIENT_ENTERPRISE_REGISTRY=registry.mender.io
@@ -37,11 +43,3 @@ MENDER_CONTAINER_MODULES_VERSION=main
 
 # Version of the Mender Client "bundle"
 MENDER_CLIENT_VERSION=main
-
-# If MENDER_LEGACY_IMAGE is defined, the image reference uses
-# the legacy format:
-# * For enterprise, some images have their -enterprise suffix appended
-# * Workflows worker uses the legacy workflows-worker image
-# This will only work if MENDER_TAG < mender-3.7 and
-# MENDER_ENTERPRISE_REPOSITORY=mendersoftware
-# MENDER_LEGACY_IMAGE="true"

.gitignore

@@ -7,7 +7,12 @@ tags
 /release-state.yml
 /mender-demo-artifact.mender
 
-/docker-compose.testing.yml
+# Lock file created by the container manager when compose is driven from the
+# repo root rather than tests/
+/docker_lock
+
+# Output of the old 'keygen' script, removed along with the demo composition
+/keys-generated/
 
 __pycache__
 .cache

.gitlab-ci-default-pipeline.yml

@@ -7,7 +7,7 @@ include:
     file: '.gitlab-ci-check-python3-format.yml'
 
 variables:
-  LICENSE_HEADERS_IGNORE_FILES_REGEXP: '\./extra/gitdm.*'
+  LICENSE_HEADERS_IGNORE_FILES_REGEXP: '\./(extra/gitdm|tests/mender_server).*'
 
 test:extra-tools:
   image: "python:3"

.gitlab-ci-full-integration-template.yml

@@ -15,6 +15,12 @@ test:integration:$CI_NODE_INDEX:
   # jobs (e.g. a test that blocks forever) at 2h instead of burning the full
   # default budget.
   timeout: 2h
+  variables:
+    # The backend composition comes from the mender-server submodule at
+    # tests/mender_server; without this the compose files are simply absent and
+    # every environment fails to start.
+    GIT_SUBMODULE_STRATEGY: recursive
+    GIT_SUBMODULE_DEPTH: 1
   before_script:
     # These two variables would be set by GitLab CI "parallel" feature, and are used by our Pytest
     # plugin to split the tests among the CI jobs. They get substituted by the generator.
@@ -23,12 +29,20 @@ test:integration:$CI_NODE_INDEX:
     - echo "$REGISTRY_MENDER_IO_PASSWORD" | docker login registry.mender.io --username "$REGISTRY_MENDER_IO_USERNAME" --password-stdin
     - echo "$DOCKER_HUB_PASSWORD" | docker login docker.io --username "$DOCKER_HUB_USERNAME" --password-stdin
     - echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY" --username "$CI_REGISTRY_USER" --password-stdin
+    # apk-requirements.txt carries skopeo, e2fsprogs-extra (debugfs) and iptables,
+    # all of which conftest now checks for before collecting.
     - apk add $(cat ./tests/requirements-system/apk-requirements.txt)
     - apk add py3-virtualenv screen
     - python -m virtualenv /.venv
     - source /.venv/bin/activate
+    # Integration keeps its own testutils, so its own requirements file is the
+    # right one -- nothing here imports from the mender_server submodule, which
+    # is used for its compose files only.
     - pip3 install -r ./tests/requirements-python/python-requirements.txt
     - pip3 install pyyaml
+    # Fail early and loudly if the submodule did not come through.
+    - test -f ./tests/mender_server/docker-compose.yml
+      || { echo "mender_server submodule missing"; exit 1; }
     # Gitlab CI tends to set these DOCKER_ variables internally.
     # dind also creates the unix socket at /var/run/docker.sock
     - unset DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH

.gitlab-ci.yml

@@ -52,7 +52,14 @@ variables:
     description: |
       Number of parallel xdist jobs per CI job for integration tests.
       Tests in parallel = CI_JOBS_IN_PARALLEL_INTEGRATION x XDIST_JOBS_IN_PARALLEL_INTEGRATION
-    value: 4
+
+      Lowered from 4 to 3 for the mender-server composition: an environment is
+      now 18 containers (23 for Enterprise) rather than ~14, mostly because
+      SeaweedFS replaces MinIO with four. 3 workers was measured to run cleanly on
+      8 cores / 11 GB available; 4 was not tested and would put ~92 containers on
+      one runner for the Enterprise setups. Worth re-tuning upwards once there is
+      evidence from the actual runner.
+    value: 3
 
   DOCKER_VERSION:
     description: "Docker version to use in jobs"

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "extra/gitdm/gitdm"]
 	path = extra/gitdm/gitdm
 	url = git://git.lwn.net/gitdm.git
+[submodule "tests/mender_server"]
+	path = tests/mender_server
+	url = https://github.com/mendersoftware/mender-server.git