Integration tests improvements

Makefile

@@ -172,7 +172,17 @@ integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc
 	TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
 	PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \
 	OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \
-	go test -v ./test/integration/installuninstall ./test/integration/managementplane ./test/integration/auxiliarycommandserver ./test/integration/nginxless
+	go test -v -parallel 2 ./test/integration/installuninstall ./test/integration/nginxless
+
+	TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
+	PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \
+	OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \
+	go test -v ./test/integration/managementplane
+
+	TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
+	PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \
+	OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \
+	go test -v ./test/integration/auxiliarycommandserver
 
 upgrade-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc
 	TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-repo" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \

test/docker/nginx-oss/apk/Dockerfile

@@ -6,17 +6,18 @@ ARG ENTRY_POINT
 
 WORKDIR /agent
 COPY ./build /agent/build
-COPY $ENTRY_POINT /agent/entrypoint.sh
 
 RUN set -x \
     && addgroup -g 101 -S nginx \
     && adduser -S -D -H -u 101 -h /var/cache/nginx -s /sbin/nologin -G nginx -g nginx nginx \
-    && apk add ca-certificates \
-               curl \
-               openssl \
-               bash \
-               nginx
+    && apk add --no-cache ca-certificates \
+                          curl \
+                          openssl \
+                          bash \
+                          procps \
+                          nginx
 
+COPY $ENTRY_POINT /agent/entrypoint.sh
 RUN chmod +x /agent/entrypoint.sh
 STOPSIGNAL SIGTERM
 

test/docker/nginx-oss/deb/Dockerfile

@@ -8,28 +8,27 @@ ARG PACKAGE_NAME
 ARG PACKAGES_REPO
 
 WORKDIR /agent
-COPY ./build /agent/build
-COPY $ENTRY_POINT /agent/entrypoint.sh
 
 RUN set -x \
-    && ls /usr/sbin/ \
     && groupadd --system --gid 101 nginx \
     && useradd --system --gid nginx --no-create-home --home /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \
-    && apt-get update \
+    && (apt-get update || (sleep 30 && apt-get update) || (sleep 60 && apt-get update)) \
     && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates \
                                                                         gnupg2 \
-                                                                        git \
-                                                                        make \
                                                                         curl \
                                                                         lsb-release \
                                                                         procps \
-                                                                        nginx
+                                                                        nginx \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # Setup nginx agent repository
 RUN curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \
     && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://${PACKAGES_REPO}/nginx-agent/ubuntu/ `lsb_release -cs` agent\n" > /etc/apt/sources.list.d/nginx-agent.list
 
+COPY $ENTRY_POINT /agent/entrypoint.sh
 RUN chmod +x /agent/entrypoint.sh
+COPY ./build /agent/build
 STOPSIGNAL SIGTERM
 
 EXPOSE 80 443
@@ -44,4 +43,7 @@ RUN apt install -y /agent/build/$PACKAGE_NAME.deb
 
 FROM install-nginx as install-agent-repo
 
-RUN apt-get update && apt-get install -y nginx-agent
+RUN (apt-get update || (sleep 30 && apt-get update) || (sleep 60 && apt-get update)) \
+    && apt-get install -y nginx-agent \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*

test/docker/nginx-oss/rpm/Dockerfile

@@ -8,25 +8,13 @@ ARG OS_VERSION
 ARG OS_RELEASE
 
 WORKDIR /agent
-COPY ./ /agent
-COPY $ENTRY_POINT /agent/entrypoint.sh
-
-RUN if [ "$OS_VERSION" = "7" ] && [ "$OS_RELEASE" = "oraclelinux" ]; \
-        then yum install -y oracle-epel-release-el7; \
-    fi
+COPY ./build /agent/build
 
-RUN if [ "$OS_VERSION" = "2" ] && [ "$OS_RELEASE" = "amazonlinux" ]; \
-        then amazon-linux-extras enable epel && yum clean metadata \
-        && yum install -y epel-release; \
-    fi
-
-RUN if [ "$OS_RELEASE" = "amazonlinux" ]; \
-        then yum install -y shadow-utils; \
-    fi
-
-RUN if [ "$OS_RELEASE" = "centos" ] && [ "$OS_VERSION" = "7" ]; \
-        then yum install -y epel-release; \
-    fi
+RUN set -x \
+    && if [ "$OS_VERSION" = "7" ] && [ "$OS_RELEASE" = "oraclelinux" ]; then yum install -y oracle-epel-release-el7; fi \
+    && if [ "$OS_VERSION" = "2" ] && [ "$OS_RELEASE" = "amazonlinux" ]; then amazon-linux-extras enable epel && yum clean metadata && yum install -y epel-release; fi \
+    && if [ "$OS_RELEASE" = "amazonlinux" ]; then yum install -y shadow-utils; fi \
+    && if [ "$OS_RELEASE" = "centos" ] && [ "$OS_VERSION" = "7" ]; then yum install -y epel-release; fi
 
 RUN if [ "$OS_RELEASE" = "redhatenterprise" ] && [ "$OS_VERSION" != "9" ]; \
         then printf "[nginx] \n\
@@ -42,11 +30,10 @@ RUN set -x \
     && adduser -g nginx --system --no-create-home --home /nonexistent --shell /bin/false --uid 101 nginx 2>/dev/null || true \
     && usermod -s /sbin/nologin nginx 2>/dev/null || true \
     && usermod -L nginx 2>/dev/null || true \
-    && yum install -y git \
-                      wget \
-                      procps \
-                      make \
-                      nginx
+    && yum install -y procps-ng \
+                      nginx \
+    && yum clean all \
+    && rm -rf /var/cache/yum
 
 # Setup nginx agent repository
 RUN if [ "$OS_VERSION" = "2023" ] && [ "$OS_RELEASE" = "amazonlinux" ]; \
@@ -74,6 +61,7 @@ gpgkey=https://nginx.org/keys/nginx_signing.key \n\
 module_hotfixes=true" > /etc/yum.repos.d/nginx-agent.repo; \
     fi
 
+COPY $ENTRY_POINT /agent/entrypoint.sh
 RUN chmod +x /agent/entrypoint.sh
 
 STOPSIGNAL SIGTERM
@@ -87,6 +75,7 @@ FROM install-nginx as install-agent-local
 
 ARG PACKAGE_NAME
 
+COPY ./build /agent/build
 RUN yum localinstall -y /agent/build/${PACKAGE_NAME}.rpm
 
 

test/docker/nginx-plus/deb/Dockerfile

@@ -8,8 +8,6 @@ ARG PACKAGE_NAME
 ARG PACKAGES_REPO
 
 WORKDIR /agent
-COPY ./build /agent/build
-COPY $ENTRY_POINT /agent/entrypoint.sh
 
 ENV PLUS_VERSION=R32
 
@@ -57,11 +55,18 @@ EXPOSE 80
 
 STOPSIGNAL SIGQUIT
 
+COPY $ENTRY_POINT /agent/entrypoint.sh
 RUN chmod +x /agent/entrypoint.sh
-RUN apt install -y /agent/build/${PACKAGE_NAME}.deb
 
 STOPSIGNAL SIGTERM
 
 EXPOSE 80 443
 
 ENTRYPOINT ["/agent/entrypoint.sh"]
+
+FROM install-nginx as install-agent-local
+
+ARG PACKAGE_NAME
+
+COPY ./build /agent/build
+RUN apt install -y /agent/build/$PACKAGE_NAME.deb

test/helpers/test_containers_utils.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/moby/moby/client"
 	"github.com/stretchr/testify/assert"
@@ -34,6 +35,11 @@ const (
 	tagKey                    = "TAG"
 )
 
+const (
+	extractFileMaxAttempts = 10
+	extractFileRetryDelay  = 200 * time.Millisecond
+)
+
 type Parameters struct {
 	NginxConfigPath          string
 	NginxAgentConfigPath     string
@@ -509,8 +515,24 @@ func ExtractFileFromContainer(
 	containerPath string,
 ) string {
 	tb.Helper()
-	fileContent, err := testContainer.CopyFileFromContainer(ctx, containerPath)
-	require.NoError(tb, err)
+
+	var fileContent io.ReadCloser
+	totalTimeout := time.Duration(extractFileMaxAttempts) * extractFileRetryDelay
+
+	assert.Eventually(tb, func() bool {
+		var err error
+		fileContent, err = testContainer.CopyFileFromContainer(ctx, containerPath)
+
+		return err == nil
+	}, totalTimeout, extractFileRetryDelay, "Failed to extract file %s", containerPath)
+
+	if fileContent == nil {
+		tb.Fatalf("Unable to extract file %s", containerPath)
+	}
+
+	defer func() {
+		require.NoError(tb, fileContent.Close())
+	}()
 
 	content, err := io.ReadAll(fileContent)
 	require.NoError(tb, err)

test/integration/auxiliarycommandserver/connection_test.go

@@ -24,6 +24,11 @@ import (
 	"github.com/stretchr/testify/suite"
 )
 
+const (
+	eventuallyTimeout  = 1 * time.Second
+	eventuallyInterval = 100 * time.Millisecond
+)
+
 type AuxiliaryTestSuite struct {
 	suite.Suite
 	teardownTest func(tb testing.TB)
@@ -125,8 +130,10 @@ func (s *AuxiliaryTestSuite) TestAuxiliary_Test3_DataplaneHealthRequest() {
 	s.False(s.T().Failed())
 
 	// Check auxiliary server still only has 1 ManagementPlaneResponses as it didn't send the request
-	utils.ManagementPlaneResponses(s.T(), 0, utils.AuxiliaryMockManagementPlaneAPIAddress)
-	s.False(s.T().Failed())
+	s.Eventually(func() bool {
+		responses := utils.ManagementPlaneResponses(s.T(), 0, utils.AuxiliaryMockManagementPlaneAPIAddress)
+		return len(responses) == 0
+	}, eventuallyTimeout, eventuallyInterval, "Expected no responses from auxiliary server, got some")
 	slog.Info("finished auxiliary command server data plane health request test")
 }
 

test/integration/installuninstall/install_uninstall_test.go

@@ -76,6 +76,7 @@ func installUninstallSetup(tb testing.TB, expectNoErrorsInLogs bool) (testcontai
 }
 
 func TestInstallUninstall(t *testing.T) {
+	t.Parallel()
 	testContainer, teardownTest := installUninstallSetup(t, true)
 	defer teardownTest(t)
 	ctx := context.Background()

test/integration/nginxless/nginx_less_mpi_connection_test.go

@@ -16,6 +16,7 @@ import (
 
 // Verify that the agent sends a connection request to Management Plane even when Nginx is not present
 func TestNginxLessGrpc_Connection(t *testing.T) {
+	t.Parallel()
 	slog.Info("starting nginxless connection test")
 	teardownTest := utils.SetupConnectionTest(t, false, true, false,
 		"../../config/agent/nginx-config-with-grpc-client.conf")

test/integration/upgrade/configs/expected-otel-config.yaml

@@ -44,8 +44,8 @@ service:
   pipelines:
     metrics/default:
       receivers:
-        - container_metrics
         - host_metrics
+        - container_metrics
       processors:
         - batch/default_metrics
       exporters:

test/integration/upgrade/configs/otel/otel-config.yaml

@@ -44,8 +44,8 @@ service:
   pipelines:
     metrics/default:
       receivers:
-        - container_metrics
         - host_metrics
+        - container_metrics
       processors:
         - batch/default_metrics
       exporters:

test/integration/upgrade/upgrade_test.go

@@ -99,6 +99,72 @@ func Test_UpgradeFromV3(t *testing.T) {
 	slog.Info("finished agent v3 upgrade tests")
 }
 
+func Test_UpgradeWithCustomOTELConfig(t *testing.T) {
+	ctx := context.Background()
+
+	containerNetwork := utils.CreateContainerNetwork(ctx, t)
+	utils.SetupMockManagementPlaneGrpc(ctx, t, containerNetwork)
+	defer func(ctx context.Context) {
+		err := utils.MockManagementPlaneGrpcContainer.Terminate(ctx)
+		require.NoError(t, err)
+	}(ctx)
+
+	testContainer, teardownTest := upgradeSetup(t, true, "custom_otel", containerNetwork)
+	defer teardownTest(t)
+
+	slog.Info("starting agent v3 upgrade tests with custom OTEL config")
+
+	// get currently installed agent version
+	oldVersion := agentVersion(ctx, t, testContainer)
+
+	// verify agent upgrade
+	verifyAgentUpgrade(ctx, t, testContainer)
+
+	// verify version of agent
+	verifyAgentVersion(ctx, t, testContainer, oldVersion)
+
+	// Expected files to validate after upgrade
+	files := []helpers.ConfigFileDescriptor{
+		{
+			ContainerPath: agentConfigDir + "/nginx-agent.conf",
+			ExpectedPath:  "./configs/otel/nginx-agent.conf",
+			LogLabel:      "agent config",
+		},
+		{
+			ContainerPath: agentConfigDir + "/my_config.yaml",
+			ExpectedPath:  "./configs/otel/my_config.yaml",
+			LogLabel:      "otel custom config",
+		},
+		{
+			ContainerPath: agentConfigDir + "/opentelemetry-collector-agent.yaml",
+			ExpectedPath:  "./configs/otel/otel-config.yaml",
+			LogLabel:      "otel config",
+		},
+	}
+	// verify agent v3 configs has not changed
+	helpers.ValidateContainerFiles(ctx, t, testContainer, files)
+
+	// Validate agent.log contains OTEL startup log
+	helpers.AssertStringInContainerFile(
+		ctx, t, testContainer, agentLogDir+"/agent.log", "Starting OTel collector",
+	)
+	helpers.AssertStringInContainerFile(
+		ctx,
+		t,
+		testContainer,
+		agentLogDir+"/agent.log",
+		"Merging additional OTel config files",
+	)
+
+	// Validate agent otel log contains specific logs
+	helpers.AssertStringInContainerFile(
+		ctx, t, testContainer, agentLogDir+"/opentelemetry-collector-agent.log",
+		"Everything is ready. Begin running and processing data.",
+	)
+
+	slog.Info("finished agent v3 upgrade tests with custom OTEL config")
+}
+
 func upgradeSetup(tb testing.TB, expectNoErrorsInLogs bool, setupType string,
 	containerNetwork *testcontainers.DockerNetwork,
 ) (testcontainers.Container, func(tb testing.TB)) {
@@ -196,8 +262,14 @@ func upgradeAgent(ctx context.Context, tb testing.TB, testContainer testcontaine
 func verifyAgentVersion(ctx context.Context, tb testing.TB, testContainer testcontainers.Container, oldVersion string) {
 	tb.Helper()
 
-	newVersion := agentVersion(ctx, tb, testContainer)
-	assert.NotEqual(tb, oldVersion, newVersion)
+	var newVersion string
+
+	assert.Eventually(tb, func() bool {
+		newVersion = agentVersion(ctx, tb, testContainer)
+
+		return newVersion != oldVersion
+	}, maxUpgradeTime, 100*time.Millisecond, "agent version not upgraded, still %s after upgrade", oldVersion)
+
 	tb.Logf("agent upgraded to version %s successfully", newVersion)
 }
 

test/integration/utils/grpc_management_plane_utils.go

@@ -126,7 +126,9 @@ func CreateContainerNetwork(ctx context.Context, tb testing.TB) *testcontainers.
 	require.NoError(tb, err)
 	tb.Cleanup(func() {
 		networkErr := containerNetwork.Remove(ctx)
-		tb.Logf("Error removing container network: %v", networkErr)
+		if networkErr != nil {
+			tb.Logf("Error removing container network: %v", networkErr)
+		}
 	})
 
 	return containerNetwork