Add --defer-validate-fks to create all FKeys as unvalidated

.github/workflows/run-tests-tiered.yml

@@ -156,6 +156,7 @@ jobs:
           - blob-snapshot-release
           - follow-defer-indexes
           - fk-not-valid
+          - defer-validate-fks
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6

docs/include/clone.rst

@@ -47,5 +47,6 @@
      --endpos                      Stop replaying changes when reaching this LSN
      --defer-indexes               Defer index building until after all table data is copied
      --defer-analyze               Defer ANALYZE until after post-data restore
+     --defer-validate-fks          Create FK constraints as NOT VALID, skipping validation scan
      --use-copy-binary             Use the COPY BINARY format for COPY operations
 

src/bin/pgcopydb/cli_clone_follow.c

@@ -70,6 +70,7 @@
 	"  --endpos                      Stop replaying changes when reaching this LSN\n" \
 	"  --defer-indexes               Defer index building until after all table data is copied\n" \
 	"  --defer-analyze               Defer ANALYZE until after post-data restore\n" \
+	"  --defer-validate-fks          Create FK constraints as NOT VALID, skipping validation scan\n" \
 	"  --use-copy-binary             Use the COPY BINARY format for COPY operations\n" \
 
 CommandLine clone_command =

src/bin/pgcopydb/cli_common.c

@@ -260,7 +260,9 @@ cli_copydb_getenv(CopyDBOptions *options)
 		{ PGCOPYDB_DEFER_INDEXES, ENV_TYPE_BOOL,
 		  &(options->deferIndexes) },
 		{ PGCOPYDB_DEFER_ANALYZE, ENV_TYPE_BOOL,
-		  &(options->deferAnalyze) }
+		  &(options->deferAnalyze) },
+		{ PGCOPYDB_DEFER_VALIDATE_FKS, ENV_TYPE_BOOL,
+		  &(options->deferValidateFKs) }
 	};
 
 	int parserCount = sizeof(parsers) / sizeof(parsers[0]);
@@ -654,6 +656,7 @@ cli_copy_db_getopts(int argc, char **argv)
 		{ "restore-tolerance", required_argument, NULL, 256 },
 		{ "defer-indexes", no_argument, NULL, 257 },
 		{ "defer-analyze", no_argument, NULL, 258 },
+		{ "defer-validate-fks", no_argument, NULL, 259 },
 		{ "help", no_argument, NULL, 'h' },
 		{ NULL, 0, NULL, 0 }
 	};
@@ -1149,6 +1152,13 @@ cli_copy_db_getopts(int argc, char **argv)
 				break;
 			}
 
+			case 259:
+			{
+				options.deferValidateFKs = true;
+				log_trace("--defer-validate-fks");
+				break;
+			}
+
 			case '?':
 			default:
 			{

src/bin/pgcopydb/cli_common.h

@@ -73,6 +73,7 @@ typedef struct CopyDBOptions
 	bool skipXidCheck;
 	bool deferIndexes;
 	bool deferAnalyze;
+	bool deferValidateFKs;
 	bool noRolesPasswords;
 	bool failFast;
 	bool useCopyBinary;

src/bin/pgcopydb/copydb.c

@@ -519,6 +519,7 @@ copydb_init_specs(CopyDataSpec *specs,
 		.skipXidCheck = options->skipXidCheck,
 		.deferIndexes = options->deferIndexes,
 		.deferAnalyze = options->deferAnalyze,
+		.deferValidateFKs = options->deferValidateFKs,
 		.noRolesPasswords = options->noRolesPasswords,
 		.failFast = options->failFast,
 		.useCopyBinary = options->useCopyBinary,

src/bin/pgcopydb/copydb.h

@@ -235,6 +235,7 @@ typedef struct CopyDataSpec
 	bool skipXidCheck;
 	bool deferIndexes;
 	bool deferAnalyze;
+	bool deferValidateFKs;
 	bool noRolesPasswords;
 	bool useCopyBinary;
 

src/bin/pgcopydb/defaults.h

@@ -74,6 +74,7 @@
 #define PGCOPYDB_RESTORE_TOLERANCE "PGCOPYDB_RESTORE_TOLERANCE"
 #define PGCOPYDB_DEFER_INDEXES "PGCOPYDB_DEFER_INDEXES"
 #define PGCOPYDB_DEFER_ANALYZE "PGCOPYDB_DEFER_ANALYZE"
+#define PGCOPYDB_DEFER_VALIDATE_FKS "PGCOPYDB_DEFER_VALIDATE_FKS"
 
 /* default values for the command line options */
 #define DEFAULT_TABLE_JOBS 4

src/bin/pgcopydb/indexes.c

@@ -1483,6 +1483,7 @@ copydb_create_fk_constraints(CopyDataSpec *specs)
 	bool success = true;
 	int notValidCount = 0;
 	int sourceNotValidCount = 0;
+	int deferredNotValidCount = 0;
 
 	for (int i = 0; i < fkArray.count; i++)
 	{
@@ -1515,14 +1516,26 @@ copydb_create_fk_constraints(CopyDataSpec *specs)
 		 * for such constraints, so the constraintDef handles this, but we
 		 * log it explicitly for clarity.
 		 */
+		bool notValid = false;
 		if (!fk->convalidated)
 		{
 			log_notice("FK constraint \"%s\" on %s is NOT VALID on source, "
 					   "creating as NOT VALID on target",
 					   fk->conname, fk->tableQname);
+			notValid = true;
 			sourceNotValidCount++;
 		}
 
+		if (specs->deferValidateFKs)
+		{
+			log_notice("Defer validate is set for FKs, "
+					   "creating FK constraint \"%s\" on %s "
+					   "as NOT VALID on target",
+					   fk->conname, fk->tableQname);
+			notValid = true;
+			deferredNotValidCount++;
+		}
+
 		/*
 		 * Build the ALTER TABLE ADD CONSTRAINT command.
 		 */
@@ -1544,6 +1557,16 @@ copydb_create_fk_constraints(CopyDataSpec *specs)
 			}
 		}
 
+		/*
+		 * Source-NOT-VALID constraints already carry NOT VALID inside
+		 * constraintDef (from pg_get_constraintdef), so we only append it
+		 * here when the source was validated and we're deferring.
+		 */
+		if (specs->deferValidateFKs && fk->convalidated)
+		{
+			appendPQExpBufferStr(cmd, " NOT VALID");
+		}
+
 		if (PQExpBufferBroken(cmd))
 		{
 			log_error("Failed to create query for FK constraint \"%s\": "
@@ -1567,16 +1590,15 @@ copydb_create_fk_constraints(CopyDataSpec *specs)
 
 		log_notice("Creating FK constraint: %s", cmd->data);
 
-		bool notValid = false;
-
 		if (!pgsql_execute(&dst, cmd->data))
 		{
 			/*
 			 * Check if the failure is due to a foreign key violation
-			 * (SQLSTATE 23503). If so, retry with NOT VALID.
+			 * (SQLSTATE 23503). If so, retry with NOT VALID only if
+			 * the original attempt didn't already include NOT VALID.
 			 */
 			if (strcmp(dst.sqlstate,
-					   STR_ERRCODE_FOREIGN_KEY_VIOLATION) == 0)
+					   STR_ERRCODE_FOREIGN_KEY_VIOLATION) == 0 && !notValid)
 			{
 				log_warn("FK constraint \"%s\" on %s has pre-existing data "
 						 "violations, retrying with NOT VALID",
@@ -1703,6 +1725,13 @@ copydb_create_fk_constraints(CopyDataSpec *specs)
 				 notValidCount);
 	}
 
+	if (deferredNotValidCount > 0)
+	{
+		log_warn("%d FK constraint(s) created as NOT VALID due to "
+				 "defer-validate flag being used",
+				 deferredNotValidCount);
+	}
+
 	/* cleanup */
 	for (int i = 0; i < fkArray.count; i++)
 	{
@@ -1716,4 +1745,4 @@ copydb_create_fk_constraints(CopyDataSpec *specs)
 	(void) pgsql_finish(&dst);
 
 	return success;
-}
+}

tests/Makefile

@@ -14,7 +14,7 @@ all: pagila pagila-multi-steps blobs unit filtering filtering-standby extensions
 	 cdc-wal2json cdc-test-decoding cdc-endpos-between-transaction cdc-low-level \
 	 follow-wal2json follow-standby follow-9.6 follow-data-only \
 	 endpos-in-multi-wal-txn exclude-extension \
-	 blob-snapshot-release follow-defer-indexes fk-not-valid;
+	 blob-snapshot-release follow-defer-indexes fk-not-valid defer-validate-fks;
 
 pagila: build
 	$(MAKE) -C $@
@@ -79,6 +79,9 @@ follow-defer-indexes: build
 fk-not-valid: build
 	$(MAKE) -C $@
 
+defer-validate-fks: build
+	$(MAKE) -C $@
+
 timescaledb: build
 	$(MAKE) -C $@
 
@@ -91,4 +94,4 @@ build:
 .PHONY: cdc-wal2json cdc-test-decoding cdc-low-level
 .PHONY: follow-wal2json follow-standby follow-9.6
 .PHONY: endpos-in-multi-wal-txn exclude-extension
-.PHONY: blob-snapshot-release follow-defer-indexes fk-not-valid
+.PHONY: blob-snapshot-release follow-defer-indexes fk-not-valid defer-validate-fks

tests/defer-validate-fks/Dockerfile

@@ -0,0 +1,9 @@
+FROM pagila
+
+COPY --from=pgcopydb /usr/local/bin/pgcopydb /usr/local/bin
+
+WORKDIR /usr/src/pgcopydb
+COPY copydb.sh copydb.sh
+
+USER docker
+CMD ["/usr/src/pgcopydb/copydb.sh"]

tests/defer-validate-fks/Makefile

@@ -0,0 +1,15 @@
+# Copyright (c) 2021 The PostgreSQL Global Development Group.
+# Licensed under the PostgreSQL License.
+
+test: down run down ;
+
+run: build
+	$(DOCKER) compose run test
+
+down:
+	$(DOCKER) compose down
+
+build:
+	$(DOCKER) compose build
+
+.PHONY: down build test

tests/defer-validate-fks/compose.yaml

@@ -0,0 +1,42 @@
+services:
+  source:
+    image: postgres:${PGVERSION:-16}
+    expose:
+      - 5432
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: h4ckm3
+      POSTGRES_HOST_AUTH_METHOD: trust
+    command: >
+      -c ssl=on
+      -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
+      -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
+  target:
+    image: postgres:${PGVERSION:-16}
+    expose:
+      - 5432
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: h4ckm3
+      POSTGRES_HOST_AUTH_METHOD: trust
+    command: >
+      -c ssl=on
+      -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
+      -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
+  test:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    cap_add:
+      - SYS_ADMIN
+      - SYS_PTRACE
+    environment:
+      PGSSLMODE: "require"
+      PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres
+      PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres
+      PGCOPYDB_TABLE_JOBS: 4
+      PGCOPYDB_INDEX_JOBS: 4
+      PGCOPYDB_FAIL_FAST: "true"
+    depends_on:
+      - source
+      - target