diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 9e0bb93f08..bf10d012e4 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -661,9 +661,9 @@ PostgreSQL documentation ...). This will make restoration very slow; it is mainly useful for making dumps that can be loaded into non-PostgreSQL databases. - However, since this option generates a separate command for each row, - an error in reloading a row causes only that row to be lost rather - than the entire table contents. + However, since, by default this option generates a separate command + for each row, an error in reloading a row causes only that row to be + lost rather than the entire table contents. @@ -764,11 +764,10 @@ PostgreSQL documentation than COPY). This will make restoration very slow; it is mainly useful for making dumps that can be loaded into non-PostgreSQL databases. - However, since this option generates a separate command for each row, - an error in reloading a row causes only that row to be lost rather - than the entire table contents. - Note that - the restore might fail altogether if you have rearranged column order. + However, since this option, by default, generates a separate command + for each row, an error in reloading a row causes only that row to be + lost rather than the entire table contents. Note that the restore + might fail altogether if you have rearranged column order. The option is safe against column order changes, though even slower. @@ -914,8 +913,9 @@ PostgreSQL documentation Add ON CONFLICT DO NOTHING to INSERT commands. - This option is not valid unless or - is also specified. + This option is not valid unless , + or + is also specified. @@ -938,6 +938,18 @@ PostgreSQL documentation + + + + + Dump data as INSERT commands (rather than + COPY). Controls the maximum number of rows per + INSERT statement. The value specified must be a + number greater than zero. + + + + diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 4a2e122e2d..646b8ae3ba 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,7 @@ typedef struct _restoreOptions int dropSchema; int disable_dollar_quoting; int dump_inserts; + int rows_per_insert; int column_inserts; int if_exists; int no_comments; /* Skip comments */ @@ -140,6 +141,7 @@ typedef struct _dumpOptions int dumpSections; /* bitmask of chosen sections */ bool aclsSkip; const char *lockWaitTimeout; + int rows_per_insert; /* flags for various command-line long options */ int disable_dollar_quoting; diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 2c2f6fb4a9..decb2846d2 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -169,6 +169,7 @@ InitDumpOptions(DumpOptions *opts) /* set any fields that shouldn't default to zeroes */ opts->include_everything = true; opts->dumpSections = DUMP_UNSECTIONED; + opts->rows_per_insert = 1; } /* diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 2b1a94733b..900fb9e0c4 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -313,6 +313,7 @@ main(int argc, char **argv) int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; + bool got_rows_per_insert = false; static DumpOptions dopt; @@ -359,6 +360,7 @@ main(int argc, char **argv) {"exclude-table-data", required_argument, NULL, 4}, {"if-exists", no_argument, &dopt.if_exists, 1}, {"inserts", no_argument, &dopt.dump_inserts, 1}, + {"rows-per-insert", required_argument, NULL, 8}, {"lock-wait-timeout", required_argument, NULL, 2}, {"no-tablespaces", no_argument, &dopt.outputNoTablespaces, 1}, {"quote-all-identifiers", no_argument, "e_all_identifiers, 1}, @@ -557,6 +559,16 @@ main(int argc, char **argv) dosync = false; break; + case 8: /* rows per insert */ + dopt.rows_per_insert = atoi(optarg); + if (dopt.rows_per_insert <= 0) + { + write_msg(NULL, "rows-per-insert must be a positive number\n"); + exit_nicely(1); + } + got_rows_per_insert = true; + break; + default: fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit_nicely(1); @@ -584,6 +596,10 @@ main(int argc, char **argv) if (dopt.column_inserts) dopt.dump_inserts = 1; + /* --rows-per-insert implies --inserts */ + if (got_rows_per_insert) + dopt.dump_inserts = 1; + /* * Binary upgrade mode implies dumping sequence data even in schema-only * mode. This is not exposed as a separate option, but kept separate @@ -607,8 +623,12 @@ main(int argc, char **argv) if (dopt.if_exists && !dopt.outputClean) exit_horribly(NULL, "option --if-exists requires option -c/--clean\n"); - if (dopt.do_nothing && !(dopt.dump_inserts || dopt.column_inserts)) - exit_horribly(NULL, "option --on-conflict-do-nothing requires option --inserts or --column-inserts\n"); + /* + * --inserts are already implied above if --column-inserts or + * --rows-per-insert were specified. + */ + if (dopt.do_nothing && !dopt.dump_inserts) + exit_horribly(NULL, "option --on-conflict-do-nothing requires option --inserts , --rows-per-insert or --column-inserts\n"); /* Identify archive format to emit */ archiveFormat = parseArchiveFormat(format, &archiveMode); @@ -877,6 +897,7 @@ main(int argc, char **argv) ropt->use_setsessauth = dopt.use_setsessauth; ropt->disable_dollar_quoting = dopt.disable_dollar_quoting; ropt->dump_inserts = dopt.dump_inserts; + ropt->rows_per_insert = dopt.rows_per_insert; ropt->no_comments = dopt.no_comments; ropt->no_publications = dopt.no_publications; ropt->no_security_labels = dopt.no_security_labels; @@ -977,6 +998,7 @@ help(const char *progname) printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); printf(_(" --on-conflict-do-nothing add ON CONFLICT DO NOTHING to INSERT commands\n")); printf(_(" --quote-all-identifiers quote all identifiers, even if not key words\n")); + printf(_(" --rows-per-insert number of row per INSERT command\n")); printf(_(" --section=SECTION dump named section (pre-data, data, or post-data)\n")); printf(_(" --serializable-deferrable wait until the dump can run without anomalies\n")); printf(_(" --snapshot=SNAPSHOT use given snapshot for the dump\n")); @@ -1886,6 +1908,8 @@ dumpTableData_insert(Archive *fout, void *dcontext) int tuple; int nfields; int field; + int rows_per_statement = dopt->rows_per_insert; + int rows_this_statement = 0; appendPQExpBuffer(q, "DECLARE _pg_dump_cursor CURSOR FOR " "SELECT * FROM ONLY %s", @@ -1900,68 +1924,86 @@ dumpTableData_insert(Archive *fout, void *dcontext) res = ExecuteSqlQuery(fout, "FETCH 100 FROM _pg_dump_cursor", PGRES_TUPLES_OK); nfields = PQnfields(res); - for (tuple = 0; tuple < PQntuples(res); tuple++) + + /* + * First time through, we build as much of the INSERT statement as + * possible in "insertStmt", which we can then just print for each + * line. If the table happens to have zero columns then this will + * be a complete statement, otherwise it will end in "VALUES " and + * be ready to have the row's column values printed. + */ + if (insertStmt == NULL) { - /* - * First time through, we build as much of the INSERT statement as - * possible in "insertStmt", which we can then just print for each - * line. If the table happens to have zero columns then this will - * be a complete statement, otherwise it will end in "VALUES(" and - * be ready to have the row's column values appended. - */ - if (insertStmt == NULL) - { - TableInfo *targettab; + TableInfo *targettab; - insertStmt = createPQExpBuffer(); + insertStmt = createPQExpBuffer(); - /* - * When load-via-partition-root is set, get the root table - * name for the partition table, so that we can reload data - * through the root table. - */ - if (dopt->load_via_partition_root && tbinfo->ispartition) - targettab = getRootTableInfo(tbinfo); - else - targettab = tbinfo; + /* + * When load-via-partition-root is set, get the root table + * name for the partition table, so that we can reload data + * through the root table. + */ + if (dopt->load_via_partition_root && tbinfo->ispartition) + targettab = getRootTableInfo(tbinfo); + else + targettab = tbinfo; - appendPQExpBuffer(insertStmt, "INSERT INTO %s ", - fmtQualifiedDumpable(targettab)); + appendPQExpBuffer(insertStmt, "INSERT INTO %s ", + fmtQualifiedDumpable(targettab)); - /* corner case for zero-column table */ - if (nfields == 0) - { - appendPQExpBufferStr(insertStmt, "DEFAULT VALUES;\n"); - } - else + /* corner case for zero-column table */ + if (nfields == 0) + { + appendPQExpBufferStr(insertStmt, "DEFAULT VALUES;\n"); + } + else + { + /* append the list of column names if required */ + if (dopt->column_inserts) { - /* append the list of column names if required */ - if (dopt->column_inserts) + appendPQExpBufferChar(insertStmt, '('); + for (field = 0; field < nfields; field++) { - appendPQExpBufferChar(insertStmt, '('); - for (field = 0; field < nfields; field++) - { - if (field > 0) - appendPQExpBufferStr(insertStmt, ", "); - appendPQExpBufferStr(insertStmt, - fmtId(PQfname(res, field))); - } - appendPQExpBufferStr(insertStmt, ") "); + if (field > 0) + appendPQExpBufferStr(insertStmt, ", "); + appendPQExpBufferStr(insertStmt, + fmtId(PQfname(res, field))); } + appendPQExpBufferStr(insertStmt, ") "); + } - if (tbinfo->needs_override) - appendPQExpBufferStr(insertStmt, "OVERRIDING SYSTEM VALUE "); + if (tbinfo->needs_override) + appendPQExpBufferStr(insertStmt, "OVERRIDING SYSTEM VALUE "); - appendPQExpBufferStr(insertStmt, "VALUES ("); - } + appendPQExpBufferStr(insertStmt, "VALUES "); } + } - archputs(insertStmt->data, fout); + for (tuple = 0; tuple < PQntuples(res); tuple++) + { + /* Write the INSERT if not in the middle of a multi-row INSERT. */ + if (rows_this_statement == 0) + archputs(insertStmt->data, fout); - /* if it is zero-column table then we're done */ + + /* + * If it is zero-column table then we've aleady written the + * complete statement, which will mean we've disobeyed + * --rows-per-insert when it's set greater than 1. We do support + * a way to make this multi-row with: + * SELECT UNION ALL SELECT UNION ALL ... but that's non-standard + * so likely we should avoid it given that using INSERTs is + * mostly only ever needed for cross-database exports. + */ if (nfields == 0) continue; + if (rows_this_statement > 0) + archputs(", (", fout); + else + archputs("(", fout); + + for (field = 0; field < nfields; field++) { if (field > 0) @@ -2027,10 +2069,27 @@ dumpTableData_insert(Archive *fout, void *dcontext) } } - if (!dopt->do_nothing) - archputs(");\n", fout); + rows_this_statement++; + + /* + * If we've put the target number of rows onto this statement then + * we can terminate it now. + */ + if (rows_this_statement == rows_per_statement) + { + /* Reset the row counter */ + rows_this_statement = 0; + if (!dopt->do_nothing) + archputs(");\n", fout); + else + archputs(") ON CONFLICT DO NOTHING;\n", fout); + } else - archputs(") ON CONFLICT DO NOTHING;\n", fout); + { + /* Otherwise, get ready for the next row. */ + archputs(")", fout); + } + } if (PQntuples(res) <= 0) @@ -2041,6 +2100,15 @@ dumpTableData_insert(Archive *fout, void *dcontext) PQclear(res); } + /* Terminate any statements that didn't make the row count.*/ + if (rows_this_statement > 0) + { + if (!dopt->do_nothing) + archputs(";\n", fout); + else + archputs(" ON CONFLICT DO NOTHING;\n", fout); + } + archputs("\n\n", fout); ExecuteSqlStatement(fout, "CLOSE _pg_dump_cursor"); diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl index a875d540b8..ebd83922dd 100644 --- a/src/bin/pg_dump/t/001_basic.pl +++ b/src/bin/pg_dump/t/001_basic.pl @@ -118,8 +118,8 @@ command_fails_like( command_fails_like( [ 'pg_dump', '--on-conflict-do-nothing' ], - qr/\Qpg_dump: option --on-conflict-do-nothing requires option --inserts or --column-inserts\E/, - 'pg_dump: option --on-conflict-do-nothing requires option --inserts or --column-inserts'); + qr/\Qpg_dump: option --on-conflict-do-nothing requires option --inserts , --rows-per-insert or --column-inserts\E/, + 'pg_dump: option --on-conflict-do-nothing requires option --inserts , --rows-per-insert or --column-inserts'); # pg_dumpall command-line argument checks command_fails_like(