diff --git a/Makefile b/Makefile index 61cd83b3e..5cd4b9775 100644 --- a/Makefile +++ b/Makefile @@ -172,7 +172,17 @@ integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \ PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \ OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \ - go test -v ./test/integration/installuninstall ./test/integration/managementplane ./test/integration/auxiliarycommandserver ./test/integration/nginxless + go test -v -parallel 2 ./test/integration/installuninstall ./test/integration/nginxless + + TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \ + PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \ + OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \ + go test -v ./test/integration/managementplane + + TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \ + PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \ + OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \ + go test -v ./test/integration/auxiliarycommandserver upgrade-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-repo" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \ diff --git a/test/docker/nginx-oss/apk/Dockerfile b/test/docker/nginx-oss/apk/Dockerfile index a8f6ede59..feb371b75 100644 --- a/test/docker/nginx-oss/apk/Dockerfile +++ b/test/docker/nginx-oss/apk/Dockerfile @@ -6,17 +6,18 @@ ARG ENTRY_POINT WORKDIR /agent COPY ./build /agent/build -COPY $ENTRY_POINT /agent/entrypoint.sh RUN set -x \ && addgroup -g 101 -S nginx \ && adduser -S -D -H -u 101 -h /var/cache/nginx -s /sbin/nologin -G nginx -g nginx nginx \ - && apk add ca-certificates \ - curl \ - openssl \ - bash \ - nginx + && apk add --no-cache ca-certificates \ + curl \ + openssl \ + bash \ + procps \ + nginx +COPY $ENTRY_POINT /agent/entrypoint.sh RUN chmod +x /agent/entrypoint.sh STOPSIGNAL SIGTERM diff --git a/test/docker/nginx-oss/deb/Dockerfile b/test/docker/nginx-oss/deb/Dockerfile index 5cea596c1..cfe1c0d85 100644 --- a/test/docker/nginx-oss/deb/Dockerfile +++ b/test/docker/nginx-oss/deb/Dockerfile @@ -8,28 +8,27 @@ ARG PACKAGE_NAME ARG PACKAGES_REPO WORKDIR /agent -COPY ./build /agent/build -COPY $ENTRY_POINT /agent/entrypoint.sh RUN set -x \ - && ls /usr/sbin/ \ && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ - && apt-get update \ + && (apt-get update || (sleep 30 && apt-get update) || (sleep 60 && apt-get update)) \ && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates \ gnupg2 \ - git \ - make \ curl \ lsb-release \ procps \ - nginx + nginx \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* # Setup nginx agent repository RUN curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \ && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://${PACKAGES_REPO}/nginx-agent/ubuntu/ `lsb_release -cs` agent\n" > /etc/apt/sources.list.d/nginx-agent.list +COPY $ENTRY_POINT /agent/entrypoint.sh RUN chmod +x /agent/entrypoint.sh +COPY ./build /agent/build STOPSIGNAL SIGTERM EXPOSE 80 443 @@ -44,4 +43,7 @@ RUN apt install -y /agent/build/$PACKAGE_NAME.deb FROM install-nginx as install-agent-repo -RUN apt-get update && apt-get install -y nginx-agent +RUN (apt-get update || (sleep 30 && apt-get update) || (sleep 60 && apt-get update)) \ + && apt-get install -y nginx-agent \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* diff --git a/test/docker/nginx-oss/rpm/Dockerfile b/test/docker/nginx-oss/rpm/Dockerfile index 8db9618c0..9f029e424 100644 --- a/test/docker/nginx-oss/rpm/Dockerfile +++ b/test/docker/nginx-oss/rpm/Dockerfile @@ -8,25 +8,13 @@ ARG OS_VERSION ARG OS_RELEASE WORKDIR /agent -COPY ./ /agent -COPY $ENTRY_POINT /agent/entrypoint.sh - -RUN if [ "$OS_VERSION" = "7" ] && [ "$OS_RELEASE" = "oraclelinux" ]; \ - then yum install -y oracle-epel-release-el7; \ - fi +COPY ./build /agent/build -RUN if [ "$OS_VERSION" = "2" ] && [ "$OS_RELEASE" = "amazonlinux" ]; \ - then amazon-linux-extras enable epel && yum clean metadata \ - && yum install -y epel-release; \ - fi - -RUN if [ "$OS_RELEASE" = "amazonlinux" ]; \ - then yum install -y shadow-utils; \ - fi - -RUN if [ "$OS_RELEASE" = "centos" ] && [ "$OS_VERSION" = "7" ]; \ - then yum install -y epel-release; \ - fi +RUN set -x \ + && if [ "$OS_VERSION" = "7" ] && [ "$OS_RELEASE" = "oraclelinux" ]; then yum install -y oracle-epel-release-el7; fi \ + && if [ "$OS_VERSION" = "2" ] && [ "$OS_RELEASE" = "amazonlinux" ]; then amazon-linux-extras enable epel && yum clean metadata && yum install -y epel-release; fi \ + && if [ "$OS_RELEASE" = "amazonlinux" ]; then yum install -y shadow-utils; fi \ + && if [ "$OS_RELEASE" = "centos" ] && [ "$OS_VERSION" = "7" ]; then yum install -y epel-release; fi RUN if [ "$OS_RELEASE" = "redhatenterprise" ] && [ "$OS_VERSION" != "9" ]; \ then printf "[nginx] \n\ @@ -42,11 +30,10 @@ RUN set -x \ && adduser -g nginx --system --no-create-home --home /nonexistent --shell /bin/false --uid 101 nginx 2>/dev/null || true \ && usermod -s /sbin/nologin nginx 2>/dev/null || true \ && usermod -L nginx 2>/dev/null || true \ - && yum install -y git \ - wget \ - procps \ - make \ - nginx + && yum install -y procps-ng \ + nginx \ + && yum clean all \ + && rm -rf /var/cache/yum # Setup nginx agent repository RUN if [ "$OS_VERSION" = "2023" ] && [ "$OS_RELEASE" = "amazonlinux" ]; \ @@ -74,6 +61,7 @@ gpgkey=https://nginx.org/keys/nginx_signing.key \n\ module_hotfixes=true" > /etc/yum.repos.d/nginx-agent.repo; \ fi +COPY $ENTRY_POINT /agent/entrypoint.sh RUN chmod +x /agent/entrypoint.sh STOPSIGNAL SIGTERM @@ -87,6 +75,7 @@ FROM install-nginx as install-agent-local ARG PACKAGE_NAME +COPY ./build /agent/build RUN yum localinstall -y /agent/build/${PACKAGE_NAME}.rpm diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 55c5c773f..d1ae875cb 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -8,8 +8,6 @@ ARG PACKAGE_NAME ARG PACKAGES_REPO WORKDIR /agent -COPY ./build /agent/build -COPY $ENTRY_POINT /agent/entrypoint.sh ENV PLUS_VERSION=R32 @@ -57,11 +55,18 @@ EXPOSE 80 STOPSIGNAL SIGQUIT +COPY $ENTRY_POINT /agent/entrypoint.sh RUN chmod +x /agent/entrypoint.sh -RUN apt install -y /agent/build/${PACKAGE_NAME}.deb STOPSIGNAL SIGTERM EXPOSE 80 443 ENTRYPOINT ["/agent/entrypoint.sh"] + +FROM install-nginx as install-agent-local + +ARG PACKAGE_NAME + +COPY ./build /agent/build +RUN apt install -y /agent/build/$PACKAGE_NAME.deb diff --git a/test/helpers/test_containers_utils.go b/test/helpers/test_containers_utils.go index fbb6ee1a2..f55e7ca3f 100644 --- a/test/helpers/test_containers_utils.go +++ b/test/helpers/test_containers_utils.go @@ -11,6 +11,7 @@ import ( "io" "os" "testing" + "time" "github.com/moby/moby/client" "github.com/stretchr/testify/assert" @@ -34,6 +35,11 @@ const ( tagKey = "TAG" ) +const ( + extractFileMaxAttempts = 10 + extractFileRetryDelay = 200 * time.Millisecond +) + type Parameters struct { NginxConfigPath string NginxAgentConfigPath string @@ -509,8 +515,24 @@ func ExtractFileFromContainer( containerPath string, ) string { tb.Helper() - fileContent, err := testContainer.CopyFileFromContainer(ctx, containerPath) - require.NoError(tb, err) + + var fileContent io.ReadCloser + totalTimeout := time.Duration(extractFileMaxAttempts) * extractFileRetryDelay + + assert.Eventually(tb, func() bool { + var err error + fileContent, err = testContainer.CopyFileFromContainer(ctx, containerPath) + + return err == nil + }, totalTimeout, extractFileRetryDelay, "Failed to extract file %s", containerPath) + + if fileContent == nil { + tb.Fatalf("Unable to extract file %s", containerPath) + } + + defer func() { + require.NoError(tb, fileContent.Close()) + }() content, err := io.ReadAll(fileContent) require.NoError(tb, err) diff --git a/test/integration/auxiliarycommandserver/connection_test.go b/test/integration/auxiliarycommandserver/connection_test.go index 09b103270..5c6b20e4c 100644 --- a/test/integration/auxiliarycommandserver/connection_test.go +++ b/test/integration/auxiliarycommandserver/connection_test.go @@ -24,6 +24,11 @@ import ( "github.com/stretchr/testify/suite" ) +const ( + eventuallyTimeout = 1 * time.Second + eventuallyInterval = 100 * time.Millisecond +) + type AuxiliaryTestSuite struct { suite.Suite teardownTest func(tb testing.TB) @@ -125,8 +130,10 @@ func (s *AuxiliaryTestSuite) TestAuxiliary_Test3_DataplaneHealthRequest() { s.False(s.T().Failed()) // Check auxiliary server still only has 1 ManagementPlaneResponses as it didn't send the request - utils.ManagementPlaneResponses(s.T(), 0, utils.AuxiliaryMockManagementPlaneAPIAddress) - s.False(s.T().Failed()) + s.Eventually(func() bool { + responses := utils.ManagementPlaneResponses(s.T(), 0, utils.AuxiliaryMockManagementPlaneAPIAddress) + return len(responses) == 0 + }, eventuallyTimeout, eventuallyInterval, "Expected no responses from auxiliary server, got some") slog.Info("finished auxiliary command server data plane health request test") } diff --git a/test/integration/installuninstall/install_uninstall_test.go b/test/integration/installuninstall/install_uninstall_test.go index 95c1ab841..e42cc5bed 100644 --- a/test/integration/installuninstall/install_uninstall_test.go +++ b/test/integration/installuninstall/install_uninstall_test.go @@ -76,6 +76,7 @@ func installUninstallSetup(tb testing.TB, expectNoErrorsInLogs bool) (testcontai } func TestInstallUninstall(t *testing.T) { + t.Parallel() testContainer, teardownTest := installUninstallSetup(t, true) defer teardownTest(t) ctx := context.Background() diff --git a/test/integration/nginxless/nginx_less_mpi_connection_test.go b/test/integration/nginxless/nginx_less_mpi_connection_test.go index cf3246f7e..2376c070a 100644 --- a/test/integration/nginxless/nginx_less_mpi_connection_test.go +++ b/test/integration/nginxless/nginx_less_mpi_connection_test.go @@ -16,6 +16,7 @@ import ( // Verify that the agent sends a connection request to Management Plane even when Nginx is not present func TestNginxLessGrpc_Connection(t *testing.T) { + t.Parallel() slog.Info("starting nginxless connection test") teardownTest := utils.SetupConnectionTest(t, false, true, false, "../../config/agent/nginx-config-with-grpc-client.conf") diff --git a/test/integration/upgrade/configs/expected-otel-config.yaml b/test/integration/upgrade/configs/expected-otel-config.yaml index 204f5f4c7..f477b6ed5 100644 --- a/test/integration/upgrade/configs/expected-otel-config.yaml +++ b/test/integration/upgrade/configs/expected-otel-config.yaml @@ -44,8 +44,8 @@ service: pipelines: metrics/default: receivers: - - container_metrics - host_metrics + - container_metrics processors: - batch/default_metrics exporters: diff --git a/test/integration/upgrade/configs/otel/otel-config.yaml b/test/integration/upgrade/configs/otel/otel-config.yaml index 204f5f4c7..f477b6ed5 100644 --- a/test/integration/upgrade/configs/otel/otel-config.yaml +++ b/test/integration/upgrade/configs/otel/otel-config.yaml @@ -44,8 +44,8 @@ service: pipelines: metrics/default: receivers: - - container_metrics - host_metrics + - container_metrics processors: - batch/default_metrics exporters: diff --git a/test/integration/upgrade/upgrade_test.go b/test/integration/upgrade/upgrade_test.go index a0d7b21fb..92ae39f54 100644 --- a/test/integration/upgrade/upgrade_test.go +++ b/test/integration/upgrade/upgrade_test.go @@ -99,6 +99,72 @@ func Test_UpgradeFromV3(t *testing.T) { slog.Info("finished agent v3 upgrade tests") } +func Test_UpgradeWithCustomOTELConfig(t *testing.T) { + ctx := context.Background() + + containerNetwork := utils.CreateContainerNetwork(ctx, t) + utils.SetupMockManagementPlaneGrpc(ctx, t, containerNetwork) + defer func(ctx context.Context) { + err := utils.MockManagementPlaneGrpcContainer.Terminate(ctx) + require.NoError(t, err) + }(ctx) + + testContainer, teardownTest := upgradeSetup(t, true, "custom_otel", containerNetwork) + defer teardownTest(t) + + slog.Info("starting agent v3 upgrade tests with custom OTEL config") + + // get currently installed agent version + oldVersion := agentVersion(ctx, t, testContainer) + + // verify agent upgrade + verifyAgentUpgrade(ctx, t, testContainer) + + // verify version of agent + verifyAgentVersion(ctx, t, testContainer, oldVersion) + + // Expected files to validate after upgrade + files := []helpers.ConfigFileDescriptor{ + { + ContainerPath: agentConfigDir + "/nginx-agent.conf", + ExpectedPath: "./configs/otel/nginx-agent.conf", + LogLabel: "agent config", + }, + { + ContainerPath: agentConfigDir + "/my_config.yaml", + ExpectedPath: "./configs/otel/my_config.yaml", + LogLabel: "otel custom config", + }, + { + ContainerPath: agentConfigDir + "/opentelemetry-collector-agent.yaml", + ExpectedPath: "./configs/otel/otel-config.yaml", + LogLabel: "otel config", + }, + } + // verify agent v3 configs has not changed + helpers.ValidateContainerFiles(ctx, t, testContainer, files) + + // Validate agent.log contains OTEL startup log + helpers.AssertStringInContainerFile( + ctx, t, testContainer, agentLogDir+"/agent.log", "Starting OTel collector", + ) + helpers.AssertStringInContainerFile( + ctx, + t, + testContainer, + agentLogDir+"/agent.log", + "Merging additional OTel config files", + ) + + // Validate agent otel log contains specific logs + helpers.AssertStringInContainerFile( + ctx, t, testContainer, agentLogDir+"/opentelemetry-collector-agent.log", + "Everything is ready. Begin running and processing data.", + ) + + slog.Info("finished agent v3 upgrade tests with custom OTEL config") +} + func upgradeSetup(tb testing.TB, expectNoErrorsInLogs bool, setupType string, containerNetwork *testcontainers.DockerNetwork, ) (testcontainers.Container, func(tb testing.TB)) { @@ -196,8 +262,14 @@ func upgradeAgent(ctx context.Context, tb testing.TB, testContainer testcontaine func verifyAgentVersion(ctx context.Context, tb testing.TB, testContainer testcontainers.Container, oldVersion string) { tb.Helper() - newVersion := agentVersion(ctx, tb, testContainer) - assert.NotEqual(tb, oldVersion, newVersion) + var newVersion string + + assert.Eventually(tb, func() bool { + newVersion = agentVersion(ctx, tb, testContainer) + + return newVersion != oldVersion + }, maxUpgradeTime, 100*time.Millisecond, "agent version not upgraded, still %s after upgrade", oldVersion) + tb.Logf("agent upgraded to version %s successfully", newVersion) } diff --git a/test/integration/utils/grpc_management_plane_utils.go b/test/integration/utils/grpc_management_plane_utils.go index 4ad20a698..af75bf905 100644 --- a/test/integration/utils/grpc_management_plane_utils.go +++ b/test/integration/utils/grpc_management_plane_utils.go @@ -126,7 +126,9 @@ func CreateContainerNetwork(ctx context.Context, tb testing.TB) *testcontainers. require.NoError(tb, err) tb.Cleanup(func() { networkErr := containerNetwork.Remove(ctx) - tb.Logf("Error removing container network: %v", networkErr) + if networkErr != nil { + tb.Logf("Error removing container network: %v", networkErr) + } }) return containerNetwork diff --git a/test/mock/grpc/mock_management_command_service.go b/test/mock/grpc/mock_management_command_service.go index 6a6cea46c..2ceeca4b4 100644 --- a/test/mock/grpc/mock_management_command_service.go +++ b/test/mock/grpc/mock_management_command_service.go @@ -43,6 +43,7 @@ type CommandService struct { externalFileServer string configDirectory string dataPlaneResponses []*mpi.DataPlaneResponse + instanceFilesMutex sync.RWMutex dataPlaneResponsesMutex sync.Mutex updateDataPlaneStatusMutex sync.Mutex connectionMutex sync.Mutex @@ -216,8 +217,11 @@ func (cs *CommandService) handleConfigUploadRequest( instanceID := upload.ConfigUploadRequest.GetOverview().GetConfigVersion().GetInstanceId() overviewFiles := upload.ConfigUploadRequest.GetOverview().GetFiles() + cs.instanceFilesMutex.Lock() + defer cs.instanceFilesMutex.Unlock() + if cs.instanceFiles[instanceID] != nil { - filesToDelete := cs.checkForDeletedFiles(instanceID, overviewFiles) + filesToDelete := cs.checkForDeletedFilesLocked(instanceID, overviewFiles) for _, fileToDelete := range filesToDelete { err := os.Remove(fileToDelete) if err != nil { @@ -228,7 +232,8 @@ func (cs *CommandService) handleConfigUploadRequest( cs.instanceFiles[instanceID] = overviewFiles } -func (cs *CommandService) checkForDeletedFiles(instanceID string, overviewFiles []*mpi.File) []string { +// checkForDeletedFilesLocked must be called while holding instanceFilesMutex +func (cs *CommandService) checkForDeletedFilesLocked(instanceID string, overviewFiles []*mpi.File) []string { filesToDelete := []string{} for _, diskfile := range cs.instanceFiles[instanceID] { @@ -402,6 +407,7 @@ func (cs *CommandService) addConfigApplyEndpoint() { return } + cs.instanceFilesMutex.Lock() if filesUpdated { cs.instanceFiles[instanceID] = updatedConfigFiles } else { @@ -426,6 +432,7 @@ func (cs *CommandService) addConfigApplyEndpoint() { }, }, } + cs.instanceFilesMutex.Unlock() cs.requestChan <- &request @@ -438,6 +445,7 @@ func (cs *CommandService) addConfigEndpoint() { instanceID := c.Param("instanceID") var data map[string]interface{} + cs.instanceFilesMutex.RLock() response := &mpi.GetOverviewResponse{ Overview: &mpi.FileOverview{ ConfigVersion: &mpi.ConfigVersion{ @@ -447,6 +455,7 @@ func (cs *CommandService) addConfigEndpoint() { Files: cs.instanceFiles[instanceID], }, } + cs.instanceFilesMutex.RUnlock() if err := json.Unmarshal([]byte(protojson.Format(response)), &data); err != nil { slog.Error("Failed to return connection", "error", err) diff --git a/test/mock/grpc/mock_management_server.go b/test/mock/grpc/mock_management_server.go index 890e731aa..45dd79dfd 100644 --- a/test/mock/grpc/mock_management_server.go +++ b/test/mock/grpc/mock_management_server.go @@ -18,7 +18,7 @@ import ( "syscall" "time" - "github.com/nginx/agent/v3/api/grpc/mpi/v1" + mpi "github.com/nginx/agent/v3/api/grpc/mpi/v1" "github.com/nginx/agent/v3/internal/config" "buf.build/go/protovalidate" @@ -39,6 +39,7 @@ const ( keepAliveTimeout = 10 * time.Second testTimeout = 100 * time.Millisecond connectionType = "tcp" + requestChanSize = 100 ) var ( @@ -71,7 +72,7 @@ func NewMockManagementServer( externalFileServer *string, ) (*MockManagementServer, error) { var err error - requestChan := make(chan *v1.ManagementPlaneRequest) + requestChan := make(chan *mpi.ManagementPlaneRequest, requestChanSize) commandService := serveCommandService(ctx, apiAddress, agentConfig, requestChan, *configDirectory, *externalFileServer) @@ -98,8 +99,8 @@ func NewMockManagementServer( healthcheck := health.NewServer() healthgrpc.RegisterHealthServer(grpcServer, healthcheck) - v1.RegisterCommandServiceServer(grpcServer, commandService) - v1.RegisterFileServiceServer(grpcServer, fileServer) + mpi.RegisterCommandServiceServer(grpcServer, commandService) + mpi.RegisterFileServiceServer(grpcServer, fileServer) go reportHealth(healthcheck, agentConfig) go func() { @@ -187,7 +188,7 @@ func serveCommandService( ctx context.Context, apiAddress string, agentConfig *config.Config, - requestChan chan *v1.ManagementPlaneRequest, + requestChan chan *mpi.ManagementPlaneRequest, configDirectory string, externalFileServer string, ) *CommandService {