RC dry-run test fixes: QAN search race + RC CLI version resolution

cli/tests/generic.spec.ts

@@ -16,13 +16,11 @@ test.describe('PMM Client "Generic" CLI tests', { tag: '@generic' }, () => {
   });
 
   let PMM_VERSION = `${process.env.CLIENT_VERSION}`;
-  if (/^https?:/.test(PMM_VERSION)) {
-    // An explicit build URL (feature build) carries the version of the branch it was built
-    // from, which may predate the latest bump on v3. The server under test comes from that
-    // same build, so it is the only valid reference for the client's version.
+  if (/^https?:/.test(PMM_VERSION) || /pmm3-rc/.test(PMM_VERSION)) {
+    // Feature-build / RC clients trail v3 VERSION once an RC branches; take the version from the server.
     PMM_VERSION = JSON.parse(cli.execute('sudo pmm-admin status --json').stdout).pmm_agent_status?.server_version;
     if (!PMM_VERSION) throw new Error('Could not read server version from "pmm-admin status --json"');
-  } else if (/latest-tarball|3-dev-latest|pmm3-rc/.test(PMM_VERSION)) {
+  } else if (/latest-tarball|3-dev-latest/.test(PMM_VERSION)) {
     // TODO: refactor to use docker hub API to remove file-update dependency
     // See: https://github.com/Percona-QA/package-testing/blob/master/playbooks/pmm2-client_integration_upgrade_custom_path.yml#L41
     PMM_VERSION = cli.execute('curl -s https://raw.githubusercontent.com/Percona-Lab/pmm-submodules/v3/VERSION')
@@ -601,8 +599,19 @@ test.describe('PMM Client "Generic" CLI tests', { tag: '@generic' }, () => {
     const newAdminStatus = await cli.exec(`docker exec ${containerName} pmm-admin status`);
     const newVersion = await cli.exec(`docker exec ${containerName} pmm-admin version | grep "Version:"`);
 
+    const versionLookup = process.env.PMM_CLIENT_VERSION?.includes('http')
+      ? undefined
+      : cli.execute('curl --fail --silent --show-error https://raw.githubusercontent.com/Percona-Lab/pmm-submodules/v3/VERSION');
+    if (versionLookup && (versionLookup.code !== 0 || !versionLookup.stdout.trim())) {
+      throw new Error('Could not read the expected upgrade version from v3 VERSION');
+    }
+    const upgradedVersion = versionLookup?.stdout.trim() ?? '';
+
     await newPid.outNotContains(oldPid.stdout);
     await newAdminStatus.outContains('Connected');
-    await newVersion.outContains(PMM_VERSION);
+    expect(newVersion.stdout.trim()).not.toEqual(oldVersion.stdout.trim());
+    if (upgradedVersion) {
+      await newVersion.outContains(upgradedVersion);
+    }
   });
 });

e2e_tests/api/server.api.ts

@@ -1,10 +1,40 @@
-import { APIRequestContext } from '@playwright/test';
+import { APIRequestContext, expect } from '@playwright/test';
+import GrafanaHelper from '@helpers/grafana.helper';
 import { Timeouts } from '@helpers/timeouts';
 import apiEndpoints from '@helpers/apiEndpoints';
 
+interface PmmVersion {
+  major: number;
+  minor: number;
+  patch: number;
+  version: string;
+}
+
+interface VersionResponse {
+  version: string;
+}
+
 export default class ServerApi {
   constructor(private request: APIRequestContext) {}
 
+  getPmmVersion = async (): Promise<PmmVersion> => {
+    const response = await this.request.get(apiEndpoints.server.version, {
+      headers: GrafanaHelper.getAuthHeader(),
+    });
+
+    expect(response.status()).toEqual(200);
+
+    const data = (await response.json()) as VersionResponse;
+    const [versionMajor, versionMinor, versionPatch] = data.version.split('.');
+
+    return {
+      major: parseInt(versionMajor),
+      minor: parseInt(versionMinor),
+      patch: parseInt(versionPatch),
+      version: data.version,
+    };
+  };
+
   waitForReady = async (overallTimeoutMs: Timeouts = Timeouts.ONE_MINUTE): Promise<void> => {
     const pollIntervalMs = Timeouts.FIVE_SECONDS;
     const deadline = Date.now() + overallTimeoutMs;

e2e_tests/fixtures/pmmTest.ts

@@ -22,6 +22,8 @@ import apiEndpoints from '@helpers/apiEndpoints';
 import SettingsPage from '@pages/ha/settings.page';
 import UpdatesPage from '@pages/updates.page';
 import DownloadsPage from '@pages/downloads.page';
+import { serverVersionBelow } from '@helpers/version.helper';
+import { minPmmVersion } from '@helpers/versionGates';
 
 const pmmTest = base.extend<{
   settingsPage: SettingsPage;
@@ -155,4 +157,12 @@ const pmmTest = base.extend<{
   vacuumDashboardPage: async ({ page }, use) => await use(new VacuumDashboard(page)),
 });
 
+pmmTest.beforeEach(async ({ api }, testInfo) => {
+  const testId = testInfo.title.match(/PMM-T\d+/)?.[0];
+  const minVersion = testId ? minPmmVersion[testId] : undefined;
+  if (!minVersion) return;
+
+  pmmTest.skip(serverVersionBelow(await api.serverApi.getPmmVersion(), minVersion), `Requires PMM Server ${minVersion}+`);
+});
+
 export default pmmTest;

e2e_tests/helpers/apiEndpoints.ts

@@ -35,6 +35,7 @@ const apiEndpoints = {
     readyz: '/v1/server/readyz',
     settings: '/v1/server/settings',
     updates: '**/v1/server/updates?force=**',
+    version: '/v1/version',
   },
   users: {
     me: '**/v1/users/me',

e2e_tests/helpers/version.helper.ts

@@ -0,0 +1,10 @@
+export const serverVersionBelow = (
+  version: { major: number; minor: number; patch: number },
+  minVersion: string,
+): boolean => {
+  const [major, minor, patch] = minVersion.split('.').map(Number);
+  if (version.major !== major) return version.major < major;
+  if (version.minor !== minor) return version.minor < minor;
+
+  return version.patch < patch;
+};

e2e_tests/helpers/versionGates.ts

@@ -0,0 +1,10 @@
+// Specs keyed by their PMM-T id skip when the running server is below this version.
+export const minPmmVersion: Record<string, string> = {
+  'PMM-T2202': '3.10.0',
+  'PMM-T2262': '3.10.0',
+  'PMM-T2263': '3.10.0',
+  'PMM-T2265': '3.10.0',
+  'PMM-T2266': '3.10.0',
+  'PMM-T2267': '3.10.0',
+  'PMM-T2268': '3.10.0',
+};