diff --git a/src/bin/pg_upgrade/Makefile b/src/bin/pg_upgrade/Makefile index d9c8145..153622d 100644 --- a/src/bin/pg_upgrade/Makefile +++ b/src/bin/pg_upgrade/Makefile @@ -11,8 +11,11 @@ OBJS = check.o controldata.o dump.o exec.o file.o function.o info.o \ option.o page.o parallel.o pg_upgrade.o relfilenode.o server.o \ tablespace.o util.o version.o $(WIN32RES) +SUBDIRS = plugins + override CPPFLAGS := -DDLSUFFIX=\"$(DLSUFFIX)\" -I$(srcdir) -I$(libpq_srcdir) $(CPPFLAGS) +$(recurse) all: pg_upgrade diff --git a/src/bin/pg_upgrade/file.c b/src/bin/pg_upgrade/file.c index 9357ad8..648013e 100644 --- a/src/bin/pg_upgrade/file.c +++ b/src/bin/pg_upgrade/file.c @@ -14,75 +14,63 @@ #include - #ifndef WIN32 static int copy_file(const char *fromfile, const char *tofile, bool force); #else static int win32_pghardlink(const char *src, const char *dst); #endif - /* * copyAndUpdateFile() * - * Copies a relation file from src to dst. If pageConverter is non-NULL, this function - * uses that pageConverter to do a page-by-page conversion. + * Copies a relation file from src to dst. This function uses that pageConverter + * to do a page-by-page or file-by-file conversion. */ const char * copyAndUpdateFile(pageCnvCtx *pageConverter, - const char *src, const char *dst, bool force) + const char *src, const char *dst, const char *type_suffix) { - if (pageConverter == NULL) + int i; + convertPlugin *plugin; + + /* Find apporopiate plugin function from ConverterTable */ + for (i = 0; i < LOAD_PLUGIN_NUM; i++) { -#ifndef WIN32 - if (copy_file(src, dst, force) == -1) -#else - if (CopyFile(src, dst, !force) == 0) -#endif - return getErrorText(); - else - return NULL; + plugin = &(pageConverter->converterTable[i]); + + if (strcmp(plugin->src_type_suffix, type_suffix) == 0) + break; } + + /* If this plugin has convertFile function, invoke it */ + if (plugin->convertFile) + return plugin->convertFile(plugin->pluginData, dst, src); else { - /* - * We have a pageConverter object - that implies that the - * PageLayoutVersion differs between the two clusters so we have to - * perform a page-by-page conversion. - * - * If the pageConverter can convert the entire file at once, invoke - * that plugin function, otherwise, read each page in the relation - * file and call the convertPage plugin function. - */ - -#ifdef PAGE_CONVERSION - if (pageConverter->convertFile) - return pageConverter->convertFile(pageConverter->pluginData, - dst, src); - else -#endif + /* We perform a page-by-page conversion */ + + int src_fd; + int dstfd; + char buf[BLCKSZ]; + ssize_t bytesRead; + const char *msg = NULL; + + if ((src_fd = open(src, O_RDONLY, 0)) < 0) + return "could not open source file"; + + if ((dstfd = open(dst, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR)) < 0) { - int src_fd; - int dstfd; - char buf[BLCKSZ]; - ssize_t bytesRead; - const char *msg = NULL; - - if ((src_fd = open(src, O_RDONLY, 0)) < 0) - return "could not open source file"; - - if ((dstfd = open(dst, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR)) < 0) - { - close(src_fd); - return "could not create destination file"; - } + close(src_fd); + return "could not create destination file"; + } - while ((bytesRead = read(src_fd, buf, BLCKSZ)) == BLCKSZ) - { -#ifdef PAGE_CONVERSION - if ((msg = pageConverter->convertPage(pageConverter->pluginData, buf, buf)) != NULL) - break; -#endif + while ((bytesRead = read(src_fd, buf, BLCKSZ)) == BLCKSZ) + { + if (plugin->convertPage) + if ((msg = plugin->convertPage(plugin->pluginData, + buf, + buf)) != NULL) + break; if (write(dstfd, buf, BLCKSZ) != BLCKSZ) { msg = "could not write new page to destination"; @@ -100,10 +88,8 @@ copyAndUpdateFile(pageCnvCtx *pageConverter, else return NULL; } - } } - /* * linkAndUpdateFile() * @@ -115,10 +101,26 @@ copyAndUpdateFile(pageCnvCtx *pageConverter, */ const char * linkAndUpdateFile(pageCnvCtx *pageConverter, - const char *src, const char *dst) + const char *src, const char *dst, const char *type_suffix) { - if (pageConverter != NULL) - return "Cannot in-place update this cluster, page-by-page conversion is required"; + /* + * Even if link mode, we do conversion actually for additional file such as + * visibility map, free space map. + */ + if (strcmp(type_suffix, "_vm") == 0 || + strcmp(type_suffix, "_fsm") == 0) + { + int i; + convertPlugin *convert_table = pageConverter->converterTable; + + for (i = 0; i < LOAD_PLUGIN_NUM; i++) + { + if (strcmp((convert_table[i]).src_type_suffix, type_suffix) == 0) + return (convert_table[i]).convertFile((convert_table[i]).pluginData, + dst, src); + } + return NULL; + } if (pg_link_file(src, dst) == -1) return getErrorText(); @@ -204,7 +206,6 @@ copy_file(const char *srcfile, const char *dstfile, bool force) } #endif - void check_hard_link(void) { @@ -224,6 +225,23 @@ check_hard_link(void) unlink(new_link_file); } +/* + * This function just copies file. Giving pluginData has information + * about force which will be passed to copy function. + */ +const char * +pg_copy_file(void *pluginData, const char *dstName, const char *srcName) +{ +#ifndef WIN32 + if (copy_file(srcName, dstName, true) == -1) +#else + if (CopyFile(srcName, dstName, false) == 0) +#endif + return getErrorText(); + else + return NULL; +} + #ifdef WIN32 static int win32_pghardlink(const char *src, const char *dst) diff --git a/src/bin/pg_upgrade/page.c b/src/bin/pg_upgrade/page.c index e5686e5..8c53b43 100644 --- a/src/bin/pg_upgrade/page.c +++ b/src/bin/pg_upgrade/page.c @@ -13,36 +13,82 @@ #include "storage/bufpage.h" - -#ifdef PAGE_CONVERSION +#include static void getPageVersion( uint16 *version, const char *pathName); -static pageCnvCtx *loadConverterPlugin( - uint16 newPageVersion, uint16 oldPageVersion); +static bool loadConverterPlugin(pageCnvCtx *converter, int pos, + const char *pluginName); +static void setCopyPluginToPageConverter(convertPlugin *plugin); +static void setupMainPageConverter(pageCnvCtx *converter, int pos); +static void setupVMPageConverter(pageCnvCtx *converter, int pos); +static void setupFSMPageConverter(pageCnvCtx *converter, int pos); +/* + * Set convert plugins that just copy file, doesn't convert actually. + * It's for the case where page conversion is not necessary. + */ +static void +setCopyPluginToPageConverter(convertPlugin *plugin) +{ + plugin->startup = NULL; + plugin->convertFile = (pluginConvertFile) pg_copy_file; + plugin->convertPage = NULL; + plugin->shutdown = NULL; + plugin->pluginData = NULL; +} /* * setupPageConverter() * - * This function determines the PageLayoutVersion of the old cluster and - * the PageLayoutVersion of the new cluster. If the versions differ, this - * function loads a converter plugin and returns a pointer to a pageCnvCtx - * object (in *result) that knows how to convert pages from the old format - * to the new format. If the versions are identical, this function just - * returns a NULL pageCnvCtx pointer to indicate that page-by-page conversion - * is not required. + * This function set up all page converters and return pointer + * to pageCnvCtx. After loading all plugin funcions, invoke start up + * function if that plugin has. */ -pageCnvCtx * +const pageCnvCtx * setupPageConverter(void) { + pageCnvCtx *converter = (pageCnvCtx *) pg_malloc(sizeof(pageCnvCtx)); + int i; + + /* Load convert plugin for main relation file */ + setupMainPageConverter(converter, 0); + + /* Other additional convert plugins are load here */ + setupVMPageConverter(converter, 1); + setupFSMPageConverter(converter, 2); + + /* Invoke all startup plugin functions if plugin has */ + for (i = 0; i < LOAD_PLUGIN_NUM; i++) + { + convertPlugin *plugin = &(converter->converterTable[i]); + if (plugin->startup) + plugin->startup(MIGRATOR_API_VERSION, &plugin->pluginVersion, + converter->newPageVersion, converter->oldPageVersion, + &plugin->pluginData); + } + + return converter; +} + +/* + * setupMainPageConverter() + * + * This function determines the PageLayoutVersion of the old cluster and + * the PageLayoutVersion of the new cluster. If the versions differ, this + * function loads a converter plugin that knows how to convert pages from + * the old format to the new format. If the versions are identical, this + * function just set plugin function that just copies file. + */ +static void +setupMainPageConverter(pageCnvCtx *converter, int pos) +{ uint16 oldPageVersion; uint16 newPageVersion; - pageCnvCtx *converter; - const char *msg; char dstName[MAXPGPATH]; char srcName[MAXPGPATH]; + convertPlugin *plugin = &(converter->converterTable[pos]); snprintf(dstName, sizeof(dstName), "%s/global/%u", new_cluster.pgdata, new_cluster.pg_database_oid); @@ -52,27 +98,114 @@ setupPageConverter(void) getPageVersion(&oldPageVersion, srcName); getPageVersion(&newPageVersion, dstName); - /* - * If the old cluster and new cluster use the same page layouts, then we - * don't need a page converter. - */ + converter->oldPageVersion = oldPageVersion; + converter->newPageVersion = newPageVersion; + plugin->src_type_suffix = ""; + plugin->dst_type_suffix = ""; + if (newPageVersion != oldPageVersion) { + char pluginName[MAXPGPATH]; + + /* + * Try to find a plugin that can convert pages of oldPageVersion into + * pages of newPageVersion. For example, if we oldPageVersion = 3 and + * newPageVersion is 4, we search for a plugin named: + * plugins/convertLayout_3_to_4.dll + */ + + /* + * FIXME: we are searching for plugins relative to the current directory, + * we should really search relative to our own executable instead. + */ + snprintf(pluginName, sizeof(pluginName), "./plugins/convertLayout_%d_to_%d%s", + oldPageVersion, newPageVersion, DLSUFFIX); + /* * The clusters use differing page layouts, see if we can find a * plugin that knows how to convert from the old page layout to the * new page layout. */ - - if ((converter = loadConverterPlugin(newPageVersion, oldPageVersion)) == NULL) + if (!loadConverterPlugin(converter, pos, pluginName)) pg_fatal("could not find plugin to convert from old page layout to new page layout\n"); + } + else + { + /* + * If we don't need to do any conversion then we set plugin function + * which just copies file. + */ + setCopyPluginToPageConverter(plugin); + } +} + +/* + * setupVMPageConverter() + * + * Set up convert plugin function for file having "_vm" type suffix. + */ +static void +setupVMPageConverter(pageCnvCtx *converter, int pos) +{ + convertPlugin *plugin = &(converter->converterTable[pos]); + + /* Set type suffix for visibility map */ + plugin->src_type_suffix = "_vm"; + plugin->dst_type_suffix = "_vm"; + + /* + * Do we need to add frozen bit into visibility map? + */ + if (old_cluster.controldata.cat_ver < VISIBILITY_MAP_FROZEN_BIT_CAT_VER && + new_cluster.controldata.cat_ver >= VISIBILITY_MAP_FROZEN_BIT_CAT_VER) + { + char libpath[MAXPGPATH]; + char pluginName[MAXPGPATH]; + bool *checksum_enabled = pg_malloc(sizeof(bool)); + + get_lib_path(mypath, libpath); + snprintf(pluginName, sizeof(pluginName), "%s/plugins/convertLayoutVM_add_frozenbit%s", + libpath, DLSUFFIX); - return converter; + if (!(loadConverterPlugin(converter, pos, pluginName))) + pg_fatal("could not find additional plugin to convert from old page layout to new page layout\n"); + + *checksum_enabled = false; + + /* Check whether checksum is enabled on both cluster */ + if (old_cluster.controldata.data_checksum_version != 0 && + new_cluster.controldata.data_checksum_version != 0) + *checksum_enabled = true; + + plugin->pluginData = (void *) checksum_enabled; } else - return NULL; + /* + * If we don't need to do any convertion for visibility map then + * we set plugin function which just copies file. + */ + setCopyPluginToPageConverter(plugin); } +/* + * setupFSMPageConverter() + * + * Set up page converter plugin function for the file having "_fsm" type suffix. + */ +static void +setupFSMPageConverter(pageCnvCtx *converter, int pos) +{ + convertPlugin *plugin = &(converter->converterTable[pos]); + + /* Set type suffix for free space map */ + plugin->src_type_suffix = "_fsm"; + plugin->dst_type_suffix = "_fsm"; + + /* + * We doesn't need do any convertion for free space map for now. + */ + setCopyPluginToPageConverter(plugin); +} /* * getPageVersion() @@ -103,62 +236,31 @@ getPageVersion(uint16 *version, const char *pathName) return; } - /* * loadConverterPlugin() * * This function loads a page-converter plugin library and grabs a * pointer to each of the (interesting) functions provided by that - * plugin. The name of the plugin library is derived from the given - * newPageVersion and oldPageVersion. If a plugin is found, this - * function returns a pointer to a pageCnvCtx object (which will contain - * a collection of plugin function pointers). If the required plugin - * is not found, this function returns NULL. + * plugin. The name of the plugin library is given. If a plugin is + * loaded successfully, this function returns true. */ -static pageCnvCtx * -loadConverterPlugin(uint16 newPageVersion, uint16 oldPageVersion) +static bool +loadConverterPlugin(pageCnvCtx *converter, int pos, const char *pluginName) { - char pluginName[MAXPGPATH]; void *plugin; - /* - * Try to find a plugin that can convert pages of oldPageVersion into - * pages of newPageVersion. For example, if we oldPageVersion = 3 and - * newPageVersion is 4, we search for a plugin named: - * plugins/convertLayout_3_to_4.dll - */ - - /* - * FIXME: we are searching for plugins relative to the current directory, - * we should really search relative to our own executable instead. - */ - snprintf(pluginName, sizeof(pluginName), "./plugins/convertLayout_%d_to_%d%s", - oldPageVersion, newPageVersion, DLSUFFIX); - - if ((plugin = pg_dlopen(pluginName)) == NULL) - return NULL; + if ((plugin = dlopen(pluginName, RTLD_NOW | RTLD_GLOBAL)) == NULL) + return false; else { - pageCnvCtx *result = (pageCnvCtx *) pg_malloc(sizeof(*result)); - - result->old.PageVersion = oldPageVersion; - result->new.PageVersion = newPageVersion; + convertPlugin *convert_plugin = &(converter->converterTable[pos]); - result->startup = (pluginStartup) pg_dlsym(plugin, "init"); - result->convertFile = (pluginConvertFile) pg_dlsym(plugin, "convertFile"); - result->convertPage = (pluginConvertPage) pg_dlsym(plugin, "convertPage"); - result->shutdown = (pluginShutdown) pg_dlsym(plugin, "fini"); - result->pluginData = NULL; - - /* - * If the plugin has exported an initializer, go ahead and invoke it. - */ - if (result->startup) - result->startup(MIGRATOR_API_VERSION, &result->pluginVersion, - newPageVersion, oldPageVersion, &result->pluginData); - - return result; + convert_plugin->startup = (pluginStartup) dlsym(plugin, "init"); + convert_plugin->convertFile = (pluginConvertFile) dlsym(plugin, "convertFile"); + convert_plugin->convertPage = (pluginConvertPage) dlsym(plugin, "convertPage"); + convert_plugin->shutdown = (pluginShutdown) dlsym(plugin, "fini"); + convert_plugin->pluginData = NULL; } -} -#endif + return true; +} diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index 984c395..71c69db 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -54,6 +54,7 @@ static void cleanup(void); ClusterInfo old_cluster, new_cluster; OSInfo os_info; +char mypath[MAXPGPATH]; char *output_files[] = { SERVER_LOG_FILE, @@ -76,6 +77,9 @@ main(int argc, char **argv) parseCommandLine(argc, argv); + if (find_my_exec(argv[0], mypath) != 0) + pg_fatal("could not find own program executable\n"); + get_restricted_token(os_info.progname); adjust_data_dir(&old_cluster); diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index bc733c4..5f30623 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -110,6 +110,10 @@ extern char *output_files[]; #define VISIBILITY_MAP_CRASHSAFE_CAT_VER 201107031 /* + * The format of visibility map is changed with this 9.6 commit, + */ +#define VISIBILITY_MAP_FROZEN_BIT_CAT_VER 201602131 +/* * pg_multixact format changed in 9.3 commit 0ac5ad5134f2769ccbaefec73844f85, * ("Improve concurrency of foreign key locking") which also updated catalog * version to this value. pg_upgrade behavior depends on whether old and new @@ -322,6 +326,7 @@ extern UserOpts user_opts; extern ClusterInfo old_cluster, new_cluster; extern OSInfo os_info; +extern char mypath[MAXPGPATH]; /* check.c */ @@ -360,11 +365,15 @@ bool exec_prog(const char *log_file, const char *opt_log_file, bool throw_error, const char *fmt,...) pg_attribute_printf(4, 5); void verify_directories(void); bool pid_lock_file_exists(const char *datadir); - +const char *pg_copy_file(void *pluginData, const char *src, const char *dst); /* file.c */ -#ifdef PAGE_CONVERSION +/* + * We have three kinds of file suffix; "", "vm", "_fsm". + */ +#define LOAD_PLUGIN_NUM 3 /* page converter for VM and others */ + typedef const char *(*pluginStartup) (uint16 migratorVersion, uint16 *pluginVersion, uint16 newPageVersion, uint16 oldPageVersion, void **pluginData); @@ -376,28 +385,32 @@ typedef const char *(*pluginShutdown) (void *pluginData); typedef struct { + uint16 pluginVersion; /* API version of converter plugin */ + char *src_type_suffix; + char *dst_type_suffix; + pluginStartup startup; /* Pointer to plugin's startup function */ + pluginConvertFile convertFile; /* Pointer to plugin's file converter + * function */ + pluginConvertPage convertPage; /* Pointer to plugin's page converter + * function */ + pluginShutdown shutdown; /* Pointer to plugin's shutdown function */ + void *pluginData; /* Plugin data (set by plugin) */ +} convertPlugin; + +typedef struct +{ uint16 oldPageVersion; /* Page layout version of the old cluster */ uint16 newPageVersion; /* Page layout version of the new cluster */ uint16 pluginVersion; /* API version of converter plugin */ - void *pluginData; /* Plugin data (set by plugin) */ - pluginStartup startup; /* Pointer to plugin's startup function */ - pluginConvertFile convertFile; /* Pointer to plugin's file converter - * function */ - pluginConvertPage convertPage; /* Pointer to plugin's page converter - * function */ - pluginShutdown shutdown; /* Pointer to plugin's shutdown function */ + convertPlugin converterTable[LOAD_PLUGIN_NUM]; } pageCnvCtx; const pageCnvCtx *setupPageConverter(void); -#else -/* dummy */ -typedef void *pageCnvCtx; -#endif const char *copyAndUpdateFile(pageCnvCtx *pageConverter, const char *src, - const char *dst, bool force); + const char *dst, const char *type_suffix); const char *linkAndUpdateFile(pageCnvCtx *pageConverter, const char *src, - const char *dst); + const char *dst, const char *type_suffix); void check_hard_link(void); FILE *fopen_priv(const char *path, const char *mode); diff --git a/src/bin/pg_upgrade/plugins/Makefile b/src/bin/pg_upgrade/plugins/Makefile new file mode 100644 index 0000000..fb3f941 --- /dev/null +++ b/src/bin/pg_upgrade/plugins/Makefile @@ -0,0 +1,32 @@ +# src/bin/pg_upgrade/plugins/Makefile + +PGFILEDESC = "page conversion plugins for pg_upgrade" + +subdir = src/bin/pg_upgrade/plugins +top_builddir = ../../../../ +include $(top_builddir)/src/Makefile.global + +#PG_CPPFLAGS=-I$(top_builddir)/src/bin/pg_upgrade +override CPPFLAGS := -DDLSUFFIX=\"$(DLSUFFIX)\" -I$(srcdir) -I../ -I$(libpq_srcdir) $(CPPFLAGS) + +NAME = convertLayoutVM_add_frozenbit +OBJS = convertLayoutVM_add_frozenbit.o +plugindir = $(DESTDIR)$(libdir)/plugins + +all: all-shared-lib + +include $(top_srcdir)/src/Makefile.shlib + +install: all installdirs install-plugins + +installdirs: + $(MKDIR_P) '$(plugindir)' + +install-plugins: + $(INSTALL_SHLIB) $(NAME).so '$(plugindir)' + +uninstall: + rm -f '$(plugindir)/$(NAME).so' + +clean: + rm -f $(OBJS) $(NAME).so \ No newline at end of file diff --git a/src/bin/pg_upgrade/plugins/convertLayoutVM_add_frozenbit.c b/src/bin/pg_upgrade/plugins/convertLayoutVM_add_frozenbit.c new file mode 100644 index 0000000..f0ab08f --- /dev/null +++ b/src/bin/pg_upgrade/plugins/convertLayoutVM_add_frozenbit.c @@ -0,0 +1,159 @@ +/* + * convertLayoutVM_add_frozenbit.c + * + * Page converter plugin for Visibility Map + * + * Copyright (c) 2016, PostgreSQL Global Development Group + * src/bin/pg_upgrade/plugins.c + */ + +#include "postgres_fe.h" + +#include "access/visibilitymap.h" +#include "pg_upgrade.h" +#include "storage/bufpage.h" +#include "storage/checksum.h" +#include "storage/checksum_impl.h" +#include "port.h" + +#include + +/* plugin function */ +const char* convertFile(void *pluginData, + const char *dstName, const char *srcName); + +static const int rewriteVisibilitymap(const char *fromfile, const char *tofile, + bool checksum_enabled); + +/* table for fast rewriting vm file in order to add all-frozen information */ +static const uint16 rewrite_vm_table[256] = { + 0, 1, 4, 5, 16, 17, 20, 21, 64, 65, 68, 69, 80, 81, 84, 85, + 256, 257, 260, 261, 272, 273, 276, 277, 320, 321, 324, 325, 336, 337, 340, 341, + 1024, 1025, 1028, 1029, 1040, 1041, 1044, 1045, 1088, 1089, 1092, 1093, 1104, 1105, 1108, 1109, + 1280, 1281, 1284, 1285, 1296, 1297, 1300, 1301, 1344, 1345, 1348, 1349, 1360, 1361, 1364, 1365, + 4096, 4097, 4100, 4101, 4112, 4113, 4116, 4117, 4160, 4161, 4164, 4165, 4176, 4177, 4180, 4181, + 4352, 4353, 4356, 4357, 4368, 4369, 4372, 4373, 4416, 4417, 4420, 4421, 4432, 4433, 4436, 4437, + 5120, 5121, 5124, 5125, 5136, 5137, 5140, 5141, 5184, 5185, 5188, 5189, 5200, 5201, 5204, 5205, + 5376, 5377, 5380, 5381, 5392, 5393, 5396, 5397, 5440, 5441, 5444, 5445, 5456, 5457, 5460, 5461, + 16384, 16385, 16388, 16389, 16400, 16401, 16404, 16405, 16448, 16449, 16452, 16453, 16464, 16465, 16468, 16469, + 16640, 16641, 16644, 16645, 16656, 16657, 16660, 16661, 16704, 16705, 16708, 16709, 16720, 16721, 16724, 16725, + 17408, 17409, 17412, 17413, 17424, 17425, 17428, 17429, 17472, 17473, 17476, 17477, 17488, 17489, 17492, 17493, + 17664, 17665, 17668, 17669, 17680, 17681, 17684, 17685, 17728, 17729, 17732, 17733, 17744, 17745, 17748, 17749, + 20480, 20481, 20484, 20485, 20496, 20497, 20500, 20501, 20544, 20545, 20548, 20549, 20560, 20561, 20564, 20565, + 20736, 20737, 20740, 20741, 20752, 20753, 20756, 20757, 20800, 20801, 20804, 20805, 20816, 20817, 20820, 20821, + 21504, 21505, 21508, 21509, 21520, 21521, 21524, 21525, 21568, 21569, 21572, 21573, 21584, 21585, 21588, 21589, + 21760, 21761, 21764, 21765, 21776, 21777, 21780, 21781, 21824, 21825, 21828, 21829, 21840, 21841, 21844, 21845 +}; + +/* + * convertVMFile() + * + * This plugin function is loaded by main procedure if required. + * pluginData has the information about that checksum are enabled on both + * cluster or not. If rewriting function failed then return error messages. + */ +const char * +convertFile(void *pluginData, const char *dstName, const char *srcName) +{ + bool checksum_enabled; + + checksum_enabled = *(bool *)pluginData; + + if (rewriteVisibilitymap(srcName, dstName, checksum_enabled) == -1) + { +#ifdef WIN32 + _dosmaperr(GetLastError()); +#endif + return strdup(strerror(errno)); + } + + return NULL; +} + +/* + * rewriteVisibilitymap() + * + * Copies a visibility map file while adding all-frozen bit(0) into each bit. + */ +static const int +rewriteVisibilitymap(const char *fromfile, const char *tofile, bool checksum_enabled) +{ + int src_fd = 0; + int dst_fd = 0; + char buffer[BLCKSZ]; + ssize_t bytesRead; + int rewriteVmBytesPerPage = (BLCKSZ - SizeOfPageHeaderData) / 2; + BlockNumber blkno = 0; + + /* Reset errno */ + errno = 0; + + if ((fromfile == NULL) || (tofile == NULL)) + goto err; + + if ((src_fd = open(fromfile, O_RDONLY, 0)) < 0) + goto err; + + if ((dst_fd = open(tofile, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR)) < 0) + goto err; + + /* Perform data rewriting per page */ + while ((bytesRead = read(src_fd, buffer, BLCKSZ)) == BLCKSZ) + { + char *cur, *end, *blkend; + PageHeaderData pageheader; + uint16 vm_bits; + + /* Save the page header data */ + memcpy(&pageheader, buffer, SizeOfPageHeaderData); + + cur = buffer + SizeOfPageHeaderData; + end = buffer + SizeOfPageHeaderData + rewriteVmBytesPerPage; + blkend = buffer + bytesRead; + + while (blkend >= end) + { + char vmbuf[BLCKSZ]; + char *vmtmp = vmbuf; + + /* Copy page header in advance */ + memcpy(vmbuf, &pageheader, SizeOfPageHeaderData); + + vmtmp += SizeOfPageHeaderData; + + /* Rewrite visibility map bit one by one */ + while (end > cur) + { + /* Write rewritten bit from table and its string representation */ + vm_bits = rewrite_vm_table[(uint8) *cur]; + memcpy(vmtmp, &vm_bits, BITS_PER_HEAPBLOCK); + + cur++; + vmtmp += BITS_PER_HEAPBLOCK; + } + + /* Set new checksum for a visibility map page, If enabled */ + if (checksum_enabled) + ((PageHeader) vmbuf)->pd_checksum = pg_checksum_page(vmbuf, blkno); + + if (write(dst_fd, vmbuf, BLCKSZ) != BLCKSZ) + { + if (errno == 0) + errno = ENOSPC; + goto err; + } + + end += rewriteVmBytesPerPage; + blkno++; + } + } + +err: + if (src_fd != 0) + close(src_fd); + + if (dst_fd != 0) + close(dst_fd); + + return (errno == 0) ? 0 : -1; +} diff --git a/src/bin/pg_upgrade/relfilenode.c b/src/bin/pg_upgrade/relfilenode.c index c059c5b..571393b 100644 --- a/src/bin/pg_upgrade/relfilenode.c +++ b/src/bin/pg_upgrade/relfilenode.c @@ -18,7 +18,7 @@ static void transfer_single_new_db(pageCnvCtx *pageConverter, FileNameMap *maps, int size, char *old_tablespace); static void transfer_relfile(pageCnvCtx *pageConverter, FileNameMap *map, - const char *suffix); + const char *suffix); /* @@ -82,6 +82,10 @@ transfer_all_new_dbs(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr, { int old_dbnum, new_dbnum; + pageCnvCtx *pageConverter = NULL; + + /* Set up page-converter and load necessary plugin */ + pageConverter = (pageCnvCtx *) setupPageConverter(); /* Scan the old cluster databases and transfer their files */ for (old_dbnum = new_dbnum = 0; @@ -92,7 +96,6 @@ transfer_all_new_dbs(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr, *new_db = NULL; FileNameMap *mappings; int n_maps; - pageCnvCtx *pageConverter = NULL; /* * Advance past any databases that exist in the new cluster but not in @@ -115,10 +118,6 @@ transfer_all_new_dbs(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr, if (n_maps) { print_maps(mappings, n_maps, new_db->db_name); - -#ifdef PAGE_CONVERSION - pageConverter = setupPageConverter(); -#endif transfer_single_new_db(pageConverter, mappings, n_maps, old_tablespace); } @@ -144,15 +143,9 @@ get_pg_database_relfilenode(ClusterInfo *cluster) int i_relfile; res = executeQueryOrDie(conn, - "SELECT c.relname, c.relfilenode " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n " - "WHERE c.relnamespace = n.oid AND " - " n.nspname = 'pg_catalog' AND " - " c.relname = 'pg_database' " - "ORDER BY c.relname"); - - i_relfile = PQfnumber(res, "relfilenode"); + "SELECT pg_relation_filenode('pg_database') AS filenode"); + + i_relfile = PQfnumber(res, "filenode"); cluster->pg_database_oid = atooid(PQgetvalue(res, 0, i_relfile)); PQclear(res); @@ -268,15 +261,11 @@ transfer_relfile(pageCnvCtx *pageConverter, FileNameMap *map, /* Copying files might take some time, so give feedback. */ pg_log(PG_STATUS, "%s", old_file); - if ((user_opts.transfer_mode == TRANSFER_MODE_LINK) && (pageConverter != NULL)) - pg_fatal("This upgrade requires page-by-page conversion, " - "you must use copy mode instead of link mode.\n"); - if (user_opts.transfer_mode == TRANSFER_MODE_COPY) { pg_log(PG_VERBOSE, "copying \"%s\" to \"%s\"\n", old_file, new_file); - if ((msg = copyAndUpdateFile(pageConverter, old_file, new_file, true)) != NULL) + if ((msg = copyAndUpdateFile(pageConverter, old_file, new_file, type_suffix)) != NULL) pg_fatal("error while copying relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n", map->nspname, map->relname, old_file, new_file, msg); } @@ -284,7 +273,7 @@ transfer_relfile(pageCnvCtx *pageConverter, FileNameMap *map, { pg_log(PG_VERBOSE, "linking \"%s\" to \"%s\"\n", old_file, new_file); - if ((msg = linkAndUpdateFile(pageConverter, old_file, new_file)) != NULL) + if ((msg = linkAndUpdateFile(pageConverter, old_file, new_file, type_suffix)) != NULL) pg_fatal("error while creating link for relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n", map->nspname, map->relname, old_file, new_file, msg); } diff --git a/src/bin/pg_upgrade/test.sh b/src/bin/pg_upgrade/test.sh index ba79fb3..cd9b17e 100644 --- a/src/bin/pg_upgrade/test.sh +++ b/src/bin/pg_upgrade/test.sh @@ -174,6 +174,11 @@ if "$MAKE" -C "$oldsrc" installcheck; then mv "$temp_root"/dump1.sql "$temp_root"/dump1.sql.orig sed "s;$oldsrc;$newsrc;g" "$temp_root"/dump1.sql.orig >"$temp_root"/dump1.sql fi + + vm_sql="SELECT c.relname, c.relallvisible FROM pg_class as c, pg_namespace as n WHERE c.relnamespace = n.oid AND n.nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') ORDER BY c.relname;" + # Test for rewriting visibility map + vacuumdb -d regression || visibilitymap_vacuum1_status=$? + psql -d regression -c "$vm_sql" > "$temp_root"/vm_test1.txt || visibilitymap_test1_status=$? else make_installcheck_status=$? fi @@ -188,6 +193,14 @@ if [ -n "$pg_dumpall1_status" ]; then echo "pg_dumpall of pre-upgrade database cluster failed" exit 1 fi +if [ -n "$visibilitymap_vacuum1_status" ];then + echo "VACUUM of pre-upgrade database cluster for visibility map test failed" + exit 1 +fi +if [ -n "$visibilitymap_test1_status" ];then + echo "SELECT of pre-upgrade database cluster for visibility map test failed" + exit 1 +fi PGDATA=$BASE_PGDATA @@ -203,6 +216,8 @@ case $testhost in esac pg_dumpall -f "$temp_root"/dump2.sql || pg_dumpall2_status=$? +vacuumdb -d regression || visibilitymap_vacuum2_status=$? +psql -d regression -c "$vm_sql" > "$temp_root"/vm_test2.txt || visibilitymap_test2_status=$? pg_ctl -m fast stop # no need to echo commands anymore @@ -214,11 +229,26 @@ if [ -n "$pg_dumpall2_status" ]; then exit 1 fi +if [ -n "$visibilitymap_vacuum2_status" ];then + echo "VACUUM of post-upgrade database cluster for visibility map test failed" + exit 1 +fi + +if [ -n "$visibilitymap_test2_status" ];then + echo "SELECT of post-upgrade database cluster for visibility map test failed" + exit 1 +fi + case $testhost in MINGW*) cmd /c delete_old_cluster.bat ;; *) sh ./delete_old_cluster.sh ;; esac +if ! diff "$temp_root"/vm_test1.txt "$temp_root"/vm_test2.txt >/dev/null; then + echo "Visibility map rewriting test failed" + exit 1 +fi + if diff "$temp_root"/dump1.sql "$temp_root"/dump2.sql >/dev/null; then echo PASSED exit 0