Introduce new option --with-copy-binary. (#857)

Using COPY with format BINARY can improve performance characteristics, but there is no way to automatically check for two Postgres instances binary compatibility. As a result, introduce a new option that allow users to benefit from the optimization after having manually verified that the source and target Postgres instances are binary compatible.
dimitri · Aug 6, 2024 · d0a15d8 · d0a15d8
1 parent 941817e
commit d0a15d8
Show file tree

Hide file tree

Showing 17 changed files with 69 additions and 6 deletions.
diff --git a/docs/include/clone.rst b/docs/include/clone.rst
@@ -42,4 +42,5 @@
      --create-slot                 Create the replication slot
      --origin                      Use this Postgres replication origin node name
      --endpos                      Stop replaying changes when reaching this LSN
+     --use-copy-binary             Use the COPY BINARY format for COPY operations
    
diff --git a/docs/include/copy-db.rst b/docs/include/copy-db.rst
@@ -22,4 +22,5 @@
      --resume              Allow resuming operations after a failure
      --not-consistent      Allow taking a new snapshot on the source database
      --snapshot            Use snapshot obtained with pg_export_snapshot
+     --use-copy-binary     Use the COPY BINARY format for COPY operations
    
diff --git a/docs/ref/pgcopydb_clone.rst b/docs/ref/pgcopydb_clone.rst
@@ -671,6 +671,14 @@ The following options are available to ``pgcopydb clone``:
 
   __ https://www.postgresql.org/docs/current/app-pgrecvlogical.html
 
+--use-copy-binary
+
+  Use the COPY WITH (FORMAT BINARY) instead of the COPY command. 
+
+  See also documentation for `COPY`__.
+
+  __ https://www.postgresql.org/docs/current/sql-copy.html
+
 --origin
 
   Logical replication target system needs to track the transactions that
@@ -831,6 +839,12 @@ PGCOPYDB_SKIP_CTID_SPLIT
   then pgcopydb skips the CTID split operation during the clone process,
   same as when using the ``--skip-split-by-ctid`` option.
 
+PGCOPYDB_USE_COPY_BINARY
+
+  When true (or *yes*, or *on*, or 1, same input as a Postgres boolean)
+  then pgcopydb uses the COPY WITH (FORMAT BINARY) instead of the COPY
+  command, same as when using the ``--use-copy-binary`` option.
+
 PGCOPYDB_SNAPSHOT
 
   Postgres snapshot identifier to re-use, see also ``--snapshot``.

diff --git a/docs/ref/pgcopydb_copy.rst b/docs/ref/pgcopydb_copy.rst
@@ -356,6 +356,14 @@ The following options are available to ``pgcopydb copy`` sub-commands:
   ``pg_export_snapshot()`` it is possible for pgcopydb to re-use an already
   exported snapshot.
 
+--use-copy-binary
+
+  Use the COPY WITH (FORMAT BINARY) instead of the COPY command. 
+
+  See also documentation for `COPY`__.
+
+  __ https://www.postgresql.org/docs/current/sql-copy.html
+
 --verbose
 
   Increase current verbosity. The default level of verbosity is INFO. In
@@ -447,6 +455,12 @@ PGCOPYDB_SNAPSHOT
 
   Postgres snapshot identifier to re-use, see also ``--snapshot``.
 
+PGCOPYDB_USE_COPY_BINARY
+
+  When true (or *yes*, or *on*, or 1, same input as a Postgres boolean)
+  then pgcopydb uses the COPY WITH (FORMAT BINARY) instead of the COPY
+  command, same as when using the ``--use-copy-binary`` option.
+
 TMPDIR
 
   The pgcopydb command creates all its work files and directories in

diff --git a/src/bin/pgcopydb/cli_clone_follow.c b/src/bin/pgcopydb/cli_clone_follow.c
@@ -65,6 +65,7 @@
 	"  --create-slot                 Create the replication slot\n" \
 	"  --origin                      Use this Postgres replication origin node name\n" \
 	"  --endpos                      Stop replaying changes when reaching this LSN\n" \
+	"  --use-copy-binary             Use the COPY BINARY format for COPY operations\n" \
 
 CommandLine clone_command =
 	make_command(

diff --git a/src/bin/pgcopydb/cli_common.c b/src/bin/pgcopydb/cli_common.c
@@ -249,7 +249,9 @@ cli_copydb_getenv(CopyDBOptions *options)
 		{ PGCOPYDB_SKIP_CTID_SPLIT, ENV_TYPE_BOOL,
 		  &(options->skipCtidSplit) },
 		{ PGCOPYDB_SKIP_TABLESPACES, ENV_TYPE_BOOL,
-		  &(options->restoreOptions.noTableSpaces) }
+		  &(options->restoreOptions.noTableSpaces) },
+		{ PGCOPYDB_USE_COPY_BINARY, ENV_TYPE_BOOL,
+		  &(options->useCopyBinary) }
 	};
 
 	int parserCount = sizeof(parsers) / sizeof(parsers[0]);
@@ -616,6 +618,7 @@ cli_copy_db_getopts(int argc, char **argv)
 		{ "skip-db-properties", no_argument, NULL, 'g' },
 		{ "skip-split-by-ctid", no_argument, NULL, 'k' },
 		{ "no-tablespaces", no_argument, NULL, 'y' },
+		{ "use-copy-binary", no_argument, NULL, 'n' },
 		{ "filter", required_argument, NULL, 'F' },
 		{ "filters", required_argument, NULL, 'F' },
 		{ "requirements", required_argument, NULL, 'Q' },
@@ -651,7 +654,7 @@ cli_copy_db_getopts(int argc, char **argv)
 	}
 
 	const char *optstring =
-		"S:T:D:J:I:b:L:u:mcAPOXj:xBeMlUagkyF:F:Q:irRCN:fp:ws:o:tE:Vvdzqh";
+		"S:T:D:J:I:b:L:u:mcAPOXj:xBeMlUagkynF:F:Q:irRCN:fp:ws:o:tE:Vvdzqh";
 
 	while ((c = getopt_long(argc, argv,
 							optstring, long_options, &option_index)) != -1)
@@ -1084,6 +1087,13 @@ cli_copy_db_getopts(int argc, char **argv)
 				break;
 			}
 
+			case 'n':
+			{
+				options.useCopyBinary = true;
+				log_trace("--use-copy-binary");
+				break;
+			}
+
 			case '?':
 			default:
 			{

diff --git a/src/bin/pgcopydb/cli_common.h b/src/bin/pgcopydb/cli_common.h
@@ -71,6 +71,7 @@ typedef struct CopyDBOptions
 	bool skipCtidSplit;
 	bool noRolesPasswords;
 	bool failFast;
+	bool useCopyBinary;
 
 	bool restart;
 	bool resume;

diff --git a/src/bin/pgcopydb/cli_copy.c b/src/bin/pgcopydb/cli_copy.c
@@ -52,7 +52,8 @@ static CommandLine copy_db_command =
 		"  --restart             Allow restarting when temp files exist already\n"
 		"  --resume              Allow resuming operations after a failure\n"
 		"  --not-consistent      Allow taking a new snapshot on the source database\n"
-		"  --snapshot            Use snapshot obtained with pg_export_snapshot\n",
+		"  --snapshot            Use snapshot obtained with pg_export_snapshot\n"
+		"  --use-copy-binary     Use the COPY BINARY format for COPY operations\n",
 		cli_copy_db_getopts,
 		cli_clone);
 

diff --git a/src/bin/pgcopydb/copydb.c b/src/bin/pgcopydb/copydb.c
@@ -512,6 +512,7 @@ copydb_init_specs(CopyDataSpec *specs,
 		.skipCtidSplit = options->skipCtidSplit,
 		.noRolesPasswords = options->noRolesPasswords,
 		.failFast = options->failFast,
+		.useCopyBinary = options->useCopyBinary,
 
 		.restart = options->restart,
 		.resume = options->resume,

diff --git a/src/bin/pgcopydb/copydb.h b/src/bin/pgcopydb/copydb.h
@@ -231,6 +231,7 @@ typedef struct CopyDataSpec
 	bool skipDBproperties;
 	bool skipCtidSplit;
 	bool noRolesPasswords;
+	bool useCopyBinary;
 
 	bool restart;
 	bool resume;

diff --git a/src/bin/pgcopydb/defaults.h b/src/bin/pgcopydb/defaults.h
@@ -69,6 +69,7 @@
 #define PGCOPYDB_SKIP_TABLESPACES "PGCOPYDB_SKIP_TABLESPACES"
 #define PGCOPYDB_SKIP_DB_PROPERTIES "PGCOPYDB_SKIP_DB_PROPERTIES"
 #define PGCOPYDB_SKIP_CTID_SPLIT "PGCOPYDB_SKIP_CTID_SPLIT"
+#define PGCOPYDB_USE_COPY_BINARY "PGCOPYDB_USE_COPY_BINARY"
 
 /* default values for the command line options */
 #define DEFAULT_TABLE_JOBS 4

diff --git a/src/bin/pgcopydb/pgsql.c b/src/bin/pgcopydb/pgsql.c
@@ -3095,7 +3095,13 @@ pg_copy_send_query(PGSQL *pgsql, CopyArgs *args, ExecStatusType status)
 							  args->srcAttrList,
 							  args->srcQname);
 		}
+
 		appendPQExpBuffer(sql, "to stdout");
+
+		if (args->useCopyBinary)
+		{
+			appendPQExpBuffer(sql, " with (format binary)");
+		}
 	}
 	else if (status == PGRES_COPY_IN)
 	{
@@ -3110,10 +3116,18 @@ pg_copy_send_query(PGSQL *pgsql, CopyArgs *args, ExecStatusType status)
 			appendPQExpBuffer(sql, "copy %s from stdin", args->dstQname);
 		}
 
-		if (args->freeze)
+		if (args->freeze && args->useCopyBinary)
+		{
+			appendPQExpBuffer(sql, " with (freeze, format binary)");
+		}
+		else if (args->freeze)
 		{
 			appendPQExpBuffer(sql, " with (freeze)");
 		}
+		else if (args->useCopyBinary)
+		{
+			appendPQExpBuffer(sql, " with (format binary)");
+		}
 	}
 	else
 	{

diff --git a/src/bin/pgcopydb/pgsql.h b/src/bin/pgcopydb/pgsql.h
@@ -336,6 +336,7 @@ typedef struct CopyArgs
 	char *logCommand;
 	bool truncate;
 	bool freeze;
+	bool useCopyBinary;
 } CopyArgs;
 
 

diff --git a/src/bin/pgcopydb/table-data.c b/src/bin/pgcopydb/table-data.c
@@ -1129,6 +1129,7 @@ copydb_table_create_lockfile(CopyDataSpec *specs,
 	args->dstAttrList = tableSpecs->sourceTable->attrList;
 	args->truncate = false;     /* default value, see below */
 	args->freeze = tableSpecs->sourceTable->partition.partCount <= 1;
+	args->useCopyBinary = specs->useCopyBinary;
 
 	/*
 	 * Check to see if we want to TRUNCATE the table and benefit from the COPY

diff --git a/tests/endpos-in-multi-wal-txn/copydb.sh b/tests/endpos-in-multi-wal-txn/copydb.sh
@@ -34,7 +34,7 @@ sleep 1
 pgcopydb stream setup
 
 # pgcopydb clone uses the environment variables
-pgcopydb clone
+pgcopydb clone --use-copy-binary
 
 # now that the copying is done, inject some SQL DML changes to the source
 psql -d ${PGCOPYDB_SOURCE_PGURI} -f /usr/src/pgcopydb/multi-wal-txn.sql

diff --git a/tests/follow-9.6/copydb.sh b/tests/follow-9.6/copydb.sh
@@ -53,7 +53,7 @@ psql -d ${PGCOPYDB_SOURCE_PGURI} -f /usr/src/pgcopydb/ddl.sql
 find ${TMPDIR}
 
 # pgcopydb copy db uses the environment variables
-pgcopydb clone --follow --notice
+pgcopydb clone --follow --notice --use-copy-binary
 
 # cleanup
 pgcopydb stream sentinel get

diff --git a/tests/pagila/copydb.sh b/tests/pagila/copydb.sh
@@ -61,6 +61,7 @@ pgcopydb clone --notice \
          --skip-ext-comments \
          --skip-db-properties \
          --estimate-table-sizes \
+         --use-copy-binary \
          --source ${PAGILA_SOURCE_PGURI} \
          --target ${PAGILA_TARGET_PGURI}