Skip to content

Integration tests improvements #1712

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
craigell wants to merge 19 commits into
base: main
Choose a base branch
from
Draft

Integration tests improvements #1712

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
3a61b40
Parallel tests running
craigell Jun 3, 2026
a760cb2
Changes after linting
craigell Jun 3, 2026
4cce5dc
Adding retry logic to upgrade tests
craigell Jun 4, 2026
2a27744
Merge branch 'main' into integration-timings
craigell Jun 4, 2026
3a0cc0f
Re-enable upgrade test
craigell Jun 4, 2026
877124c
Fix unit test
craigell Jun 5, 2026
ce255f6
PR feedback
craigell Jun 5, 2026
41be4e2
Add reliability to tests
craigell Jun 5, 2026
bd0551f
Restoring test
craigell Jun 5, 2026
2057d59
Merge branch 'main' into integration-timings
craigell Jun 9, 2026
494c4bc
Add assert eventually to test
craigell Jun 9, 2026
5b0d37f
Clean up errors
craigell Jun 9, 2026
1496045
Increase timeout
craigell Jun 9, 2026
74bdbc5
Update expected yaml
craigell Jun 9, 2026
02490be
Update dockerfiles to imprve build times
craigell Jun 11, 2026
2a62c9f
Merge branch 'main' into integration-timings
craigell Jun 11, 2026
a0e5d7d
Update dockerfiles
craigell Jun 12, 2026
d078f89
Fix dockerfiles
craigell Jun 12, 2026
7f9bd6b
Fix upgrade tests
craigell Jun 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,17 @@ integration-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc
TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \
OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \
go test -v ./test/integration/installuninstall ./test/integration/managementplane ./test/integration/auxiliarycommandserver ./test/integration/nginxless
go test -v -parallel 2 ./test/integration/installuninstall ./test/integration/nginxless

TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \
OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \
go test -v ./test/integration/managementplane

TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-local" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
PACKAGES_REPO=$(OSS_PACKAGES_REPO) PACKAGE_NAME=$(PACKAGE_NAME) BASE_IMAGE=$(BASE_IMAGE) DOCKERFILE_PATH=$(DOCKERFILE_PATH) IMAGE_PATH=$(IMAGE_PATH) TAG=${IMAGE_TAG} \
OS_VERSION=$(OS_VERSION) OS_RELEASE=$(OS_RELEASE) \
go test -v ./test/integration/auxiliarycommandserver

upgrade-test: $(SELECTED_PACKAGE) build-mock-management-plane-grpc
TEST_ENV="Container" CONTAINER_OS_TYPE=$(CONTAINER_OS_TYPE) BUILD_TARGET="install-agent-repo" CONTAINER_NGINX_IMAGE_REGISTRY=${CONTAINER_NGINX_IMAGE_REGISTRY} \
Expand Down
33 changes: 32 additions & 1 deletion test/helpers/test_containers_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"io"
"os"
"testing"
"time"

"github.com/moby/moby/client"
"github.com/stretchr/testify/assert"
Expand All @@ -21,6 +22,11 @@ import (

const configFilePermissions = 0o600

const (
extractFileMaxAttempts = 10
extractFileRetryDelay = 100 * time.Millisecond
)

type Parameters struct {
NginxConfigPath string
NginxAgentConfigPath string
Expand Down Expand Up @@ -496,8 +502,33 @@ func ExtractFileFromContainer(
containerPath string,
) string {
tb.Helper()
fileContent, err := testContainer.CopyFileFromContainer(ctx, containerPath)

Comment thread
craigell marked this conversation as resolved.
var (
fileContent io.ReadCloser
err error
)

for attempt := 1; attempt <= extractFileMaxAttempts; attempt++ {
fileContent, err = testContainer.CopyFileFromContainer(ctx, containerPath)
if err == nil {
break
}

if attempt == extractFileMaxAttempts {
break
}

select {
case <-ctx.Done():
require.NoError(tb, ctx.Err())
case <-time.After(extractFileRetryDelay):
}
}

require.NoError(tb, err)
defer func() {
require.NoError(tb, fileContent.Close())
}()

content, err := io.ReadAll(fileContent)
require.NoError(tb, err)
Expand Down
13 changes: 11 additions & 2 deletions test/mock/grpc/mock_management_command_service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type CommandService struct {
externalFileServer string
configDirectory string
dataPlaneResponses []*mpi.DataPlaneResponse
instanceFilesMutex sync.RWMutex
dataPlaneResponsesMutex sync.Mutex
updateDataPlaneStatusMutex sync.Mutex
connectionMutex sync.Mutex
Expand Down Expand Up @@ -216,8 +217,11 @@ func (cs *CommandService) handleConfigUploadRequest(
instanceID := upload.ConfigUploadRequest.GetOverview().GetConfigVersion().GetInstanceId()
overviewFiles := upload.ConfigUploadRequest.GetOverview().GetFiles()

cs.instanceFilesMutex.Lock()
defer cs.instanceFilesMutex.Unlock()

if cs.instanceFiles[instanceID] != nil {
filesToDelete := cs.checkForDeletedFiles(instanceID, overviewFiles)
filesToDelete := cs.checkForDeletedFilesLocked(instanceID, overviewFiles)
for _, fileToDelete := range filesToDelete {
err := os.Remove(fileToDelete)
if err != nil {
Expand All @@ -228,7 +232,8 @@ func (cs *CommandService) handleConfigUploadRequest(
cs.instanceFiles[instanceID] = overviewFiles
}

func (cs *CommandService) checkForDeletedFiles(instanceID string, overviewFiles []*mpi.File) []string {
// checkForDeletedFilesLocked must be called while holding instanceFilesMutex
func (cs *CommandService) checkForDeletedFilesLocked(instanceID string, overviewFiles []*mpi.File) []string {
filesToDelete := []string{}

for _, diskfile := range cs.instanceFiles[instanceID] {
Expand Down Expand Up @@ -401,6 +406,7 @@ func (cs *CommandService) addConfigApplyEndpoint() {
return
}

cs.instanceFilesMutex.Lock()
if filesUpdated {
cs.instanceFiles[instanceID] = updatedConfigFiles
} else {
Expand All @@ -425,6 +431,7 @@ func (cs *CommandService) addConfigApplyEndpoint() {
},
},
}
cs.instanceFilesMutex.Unlock()

cs.requestChan <- &request

Expand All @@ -437,6 +444,7 @@ func (cs *CommandService) addConfigEndpoint() {
instanceID := c.Param("instanceID")
var data map[string]interface{}

cs.instanceFilesMutex.RLock()
response := &mpi.GetOverviewResponse{
Overview: &mpi.FileOverview{
ConfigVersion: &mpi.ConfigVersion{
Expand All @@ -446,6 +454,7 @@ func (cs *CommandService) addConfigEndpoint() {
Files: cs.instanceFiles[instanceID],
},
}
cs.instanceFilesMutex.RUnlock()

if err := json.Unmarshal([]byte(protojson.Format(response)), &data); err != nil {
slog.Error("Failed to return connection", "error", err)
Expand Down
11 changes: 6 additions & 5 deletions test/mock/grpc/mock_management_server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
"syscall"
"time"

"github.com/nginx/agent/v3/api/grpc/mpi/v1"
mpi "github.com/nginx/agent/v3/api/grpc/mpi/v1"
"github.com/nginx/agent/v3/internal/config"

"buf.build/go/protovalidate"
Expand All @@ -39,6 +39,7 @@ const (
keepAliveTimeout = 10 * time.Second
testTimeout = 100 * time.Millisecond
connectionType = "tcp"
requestChanSize = 100
)

var (
Expand Down Expand Up @@ -71,7 +72,7 @@ func NewMockManagementServer(
externalFileServer *string,
) (*MockManagementServer, error) {
var err error
requestChan := make(chan *v1.ManagementPlaneRequest)
requestChan := make(chan *mpi.ManagementPlaneRequest, requestChanSize)

commandService := serveCommandService(ctx, apiAddress, agentConfig, requestChan, *configDirectory,
*externalFileServer)
Expand All @@ -98,8 +99,8 @@ func NewMockManagementServer(
healthcheck := health.NewServer()
healthgrpc.RegisterHealthServer(grpcServer, healthcheck)

v1.RegisterCommandServiceServer(grpcServer, commandService)
v1.RegisterFileServiceServer(grpcServer, fileServer)
mpi.RegisterCommandServiceServer(grpcServer, commandService)
mpi.RegisterFileServiceServer(grpcServer, fileServer)
go reportHealth(healthcheck, agentConfig)

go func() {
Expand Down Expand Up @@ -187,7 +188,7 @@ func serveCommandService(
ctx context.Context,
apiAddress string,
agentConfig *config.Config,
requestChan chan *v1.ManagementPlaneRequest,
requestChan chan *mpi.ManagementPlaneRequest,
configDirectory string,
externalFileServer string,
) *CommandService {
Expand Down
Loading