From 0812565a8c0a4c5e6680a59bdf0da0ba3a53e748 Mon Sep 17 00:00:00 2001 From: John Morris Date: Wed, 28 Jun 2023 17:56:14 -0700 Subject: [PATCH 1/2] Added new APIs to Virtual File Descriptors. --- src/backend/storage/file/fd.c | 388 +++++++++++++++++++++++++++++++++- src/include/storage/fd.h | 46 +++- 2 files changed, 421 insertions(+), 13 deletions(-) diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index 3c2a2fbef7..ff37ca41fd 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -206,6 +206,11 @@ typedef struct vfd /* NB: fileName is malloc'd, and must be free'd when closing the VFD */ int fileFlags; /* open(2) flags for (re)opening the file */ mode_t fileMode; /* mode to pass to open(2) */ + + int errorCode; /* Code of the most recent error */ + char errorMsg[121]; /* The most recent error message */ + bool eof; /* Result of last read */ + off_t offset; /* Current position in file */ } Vfd; /* @@ -1538,8 +1543,8 @@ PathNameOpenFile(const char *fileName, int fileFlags) * it will be interpreted relative to the process' working directory * (which should always be $PGDATA when this code is running). */ -File -PathNameOpenFilePerm(const char *fileName, int fileFlags, mode_t fileMode) +static File +PathNameOpenFilePerm_Internal(const char *fileName, int fileFlags, mode_t fileMode) { char *fnamecopy; File file; @@ -1928,10 +1933,11 @@ PathNameDeleteTemporaryFile(const char *path, bool error_on_failure) /* * close a file when done with it */ -void -FileClose(File file) +static int +FileClose_Internal(File file) { Vfd *vfdP; + int save_errno = 0; Assert(FileIsValid(file)); @@ -1945,6 +1951,7 @@ FileClose(File file) /* close the file */ if (close(vfdP->fd) != 0) { + save_errno = errno; /* * We may need to panic on failure to close non-temporary files; * see LruDelete. @@ -1993,16 +2000,19 @@ FileClose(File file) /* in any case do the unlink */ if (unlink(vfdP->fileName)) + { + save_errno = errno; ereport(LOG, (errcode_for_file_access(), - errmsg("could not delete file \"%s\": %m", vfdP->fileName))); + errmsg("could not delete file \"%s\": %m", vfdP->fileName))); + } /* and last report the stat results */ if (stat_errno == 0) ReportTemporaryFileUsage(vfdP->fileName, filestats.st_size); else { - errno = stat_errno; + save_errno = errno = stat_errno; ereport(LOG, (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", vfdP->fileName))); @@ -2017,6 +2027,14 @@ FileClose(File file) * Return the Vfd slot to the free list */ FreeVfd(file); + + if (save_errno != 0) + { + errno = save_errno; + return -1; + } + + return 0; } /* @@ -2086,8 +2104,8 @@ FileWriteback(File file, off_t offset, off_t nbytes, uint32 wait_event_info) pgstat_report_wait_end(); } -int -FileRead(File file, void *buffer, size_t amount, off_t offset, +static ssize_t +FileRead_Internal(File file, void *buffer, size_t amount, off_t offset, uint32 wait_event_info) { int returnCode; @@ -2142,8 +2160,8 @@ retry: return returnCode; } -int -FileWrite(File file, const void *buffer, size_t amount, off_t offset, +static ssize_t +FileWrite_Internal(File file, const void *buffer, size_t amount, off_t offset, uint32 wait_event_info) { int returnCode; @@ -3974,3 +3992,353 @@ assign_io_direct(const char *newval, void *extra) io_direct_flags = *flags; } + + +/******************************************************************************* +* The following functions add sequential and error handling to the VFD routines. +* They are mostly wrappers around the original VFD routines, +* which have been renamed by appending "_internal". +*/ + +/* Point to the Vfd struct for the given file descriptor */ +static inline Vfd* getVfd(File file) +{ + Assert(file >= 0 && file < MAXIMUM_VFD); + return &VfdCache[file]; +} + +/* Point to the Vfd struct, or a dummy Vfd if the file descriptor is -1 */ +static inline Vfd* getVfdErr(File file) +{ + /* Allocate a static Vfd to handle the file = -1 case */ + static Vfd dummyVfd[1] = {{.fileName = "dummy(-1)"}}; + + if (file == -1) + return dummyVfd; + else + return getVfd(file); +} + +static const char *getName(File file) +{ + return getVfdErr(file)->fileName; +} + +/* + * Open a file + * If an error occurs, returns -1 and set up error information + * so FileError(-1) will return true. Note errno is set for compatibility. + * + * We must be sure to release *all* resources if we fail to open the file. + * It should be the same as though never opened. + */ +File PathNameOpenFilePerm(const char *fileName, int fileFlags, mode_t fileMode) +{ + File file = -1; + bool append; + off_t position = 0; + + debug("FileOpenPerm: fileName=%s fileFlags=0x%x fileMode=0x%x\n", fileName, fileFlags, fileMode); + + /* VFDs don't implement O_APPEND. We will position to FileSize instead. */ + append = (fileFlags & O_APPEND) != 0; + fileFlags &= ~O_APPEND; + + /* Clear any previous error information */ + FileClearError(-1); + + /* Open the VFD */ + file = PathNameOpenFilePerm_Internal(fileName, fileFlags, fileMode); + if (file == -1) + return setFileError(-1, errno, "Unable to open file: %s", fileName); + + /* Position at end of file if appending. This only impacts WriteSeq and ReadSeq. */ + if (append) { + + /* Get the size of the file */ + position = FileSize(file); + if (position == -1) { + FileClose(file); + return setFileError(-1, errno, "Unable to O_APPEND to file: %s", fileName); + } + } + + /* Success!. Save the desired position */ + getVfd(file)->offset = position; + FileClearError(file); + + return file; +} + +/* + * Close a file. Like FileOpen(), the error information is saved in the dummy "-1" file, + * but it can also be accessed using the closed virtual file descriptor. + * + * Close has special error handling. If the vfd already has an error, we don't + * overwrite it. This is because the error may have been set by a previous + * operation on the file, and we don't want to lose that information. + * However, the return value will always be 0 if we closed the file successfully. + */ +int FileClose(File file) +{ + debug("FileClose: name=%s, file=%d\n", getName(file), file); + + /* Save any existing error information */ + copyFileError(-1, file); + + /* Close the file */ + if (FileClose_Internal(file) == -1) + return updateFileError(-1, errno, "Unable to close file: %s", getName(file)); + + debug("FileClose(done): file=%d\n", file); + + return 0; +} + + +ssize_t FileRead(File file, void *buffer, size_t amount, off_t offset, uint32 wait_event_info) +{ + ssize_t actual; + + debug("FileRead: name=%s file=%d amount=%zd offset=%lld\n", getName(file), file, amount, offset); + Assert(offset >= 0); + Assert((ssize_t)amount > 0); + + /* Read the data as requested */ + actual = FileRead_Internal(file, buffer, amount, offset, wait_event_info); + getVfd(file)->eof = (actual == 0); + + /* If successful, update the file offset */ + if (actual >= 0) + getVfd(file)->offset = offset + actual; + + debug("FileRead(done): file=%d name=%s actual=%zd\n", file, getName(file), actual); + return actual; +} + + +ssize_t FileWrite(File file, const void *buffer, size_t amount, off_t offset, uint32 wait_event_info) +{ + ssize_t actual; + + debug("FileWrite: name=%s file=%d amount=%zd offset=%lld\n", getName(file), file, amount, offset); + Assert(offset >= 0 && (ssize_t)amount > 0); + + /* Write the data as requested */ + actual = FileWrite_Internal(file, buffer, amount, offset, wait_event_info); + + /* If successful, update the file offset */ + if (actual >= 0) + getVfd(file)->offset = offset + actual; + + debug("FileWrite(done): file=%d name=%s actual=%zd\n", file, getName(file), actual); + + return actual; +} + + + + +/*======================================================================================== + * Routines to emuulate C library FILE routines (fgetc, fprintf, ...) + */ + +/* + * Similar to fgetc. Probably best if used with buffered files. + */ +ssize_t +FileGetc(File file) +{ + char c; + int ret = (int)FileReadSeq(file, &c, 1, 0); + if (ret <= 0) + ret = EOF; + return ret; +} + +/* Similar to fputc */ +ssize_t FilePutc(int c, File file) +{ + char cbuf; + cbuf = c; + if (FileWriteSeq(file, &cbuf, 1, 0) <= 0) + c = EOF; + return c; +} + +/* + * A temporary equivalent of fprintf. + * This version limits text to what fits in a local buffer. + * Ultimately, we need to update the internal snprintf.c (dopr) to spill + * to temporary files. + */ +ssize_t FilePrintf(File file, const char *format, ...) +{ + va_list args; + char buffer[4*1024]; /* arbitrary size, big enough? */ + int size; + + va_start(args, format); + size = vsnprintf(buffer, sizeof(buffer), format, args); + va_end(args); + + if (size < 0 || size >= sizeof(buffer)) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_RESOURCES), + errmsg("FilePrintf buffer overflow - %d characters exceeded %lu buffer", size, sizeof(buffer))); + + return FileWriteSeq(file, buffer, Min(size, sizeof(buffer)-1), 0); +} + +ssize_t FileScanf(File file, const char *format, ...) +{ + Assert(false); /* Not implemented */ + return -1; +} + +ssize_t FilePuts(File file, const char *string) +{ + return FileWriteSeq(file, string, strlen(string), 0); +} + +/* + * Read sequentially from the file. + */ +ssize_t +FileReadSeq(File file, void *buffer, size_t amount, uint32 wait_event_info) +{ + return FileRead(file, buffer, amount, getVfd(file)->offset, wait_event_info); +} + +/* + * Write sequentially to the file. + */ +ssize_t +FileWriteSeq(File file, const void *buffer, size_t amount, uint32 wait_event_info) +{ + return FileWrite(file, buffer, amount, getVfd(file)->offset, wait_event_info); +} + +/* + * Seek to an absolute position within the file. + * Relative positions can be calculated using FileTell or FileSize. + */ +off_t +FileSeek(File file, off_t offset) +{ + getVfd(file)->offset = offset; + return offset; +} + +/* + * Tell us the current file position + */ +off_t +FileTell(File file) +{ + return getVfd(file)->offset; +} + +/* =================================================================== + * Error handling code. + * These functions are similar to ferror(), but can be accessed even when the file is closed or -1. + * This added feature allows error info to be fetched after a failed "open" or "close" call. + */ + +/* True if an error occurred on the file. (EOF is not an error) */ +bool FileError(File file) +{ + return FileErrorCode(file) != 0; +} + +/* True if the last read generated an EOF */ +int FileEof(File file) +{ + return getVfd(file)->eof; +} + +/* Clears an error, and is true if an error had been encountered */ +bool FileClearError(File file) +{ + Vfd *vfd = getVfdErr(file); + bool hasError = vfd->errorCode != 0; + if (hasError) + { + vfd->errorCode = 0; + vfd->errorMsg[0] = '\0'; + } + return hasError; +} + +/* Get a pointer to the error message */ +const char *FileErrorMsg(File file) +{ + return getVfdErr(file)->errorMsg; +} + +/* + * Get the errno associated the file. + * As a side effect, restores errno to the value it had when the error occurred. + */ +int FileErrorCode(File file) +{ + errno = getVfdErr(file)->errorCode; + return errno; +} + +int setFileError(File file, int err, const char *fmt, ...) +{ + va_list args; + Vfd *vfd = getVfdErr(file); + + /* Save the errno */ + vfd->errorCode = errno; + + /* Format the error message */ + va_start(args, fmt); + vsnprintf(vfd->errorMsg, sizeof(vfd->errorMsg), fmt, args); + va_end(args); + + /* Restore the error code for compatibility */ + errno = vfd->errorCode; + + /* Return -1 to indicate an error */ + return -1; +} + +int updateFileError(File file, int err, const char *fmt, ...) +{ + va_list args; + Vfd *vfd = getVfdErr(file); + + /* if we already have an error, don't overwrite it */ + if (vfd->errorCode != 0) + return -1; + + /* Save the errno */ + vfd->errorCode = errno; + + /* Format the error message */ + va_start(args, fmt); + vsnprintf(vfd->errorMsg, sizeof(vfd->errorMsg), fmt, args); + va_end(args); + + /* Restore the error code for compatibility */ + errno = vfd->errorCode; + + /* Return -1 to indicate an error */ + return -1; +} + +int copyFileError(File dst, File src) +{ + Vfd *vfdDst = getVfdErr(dst); + Vfd *vfdSrc = getVfdErr(src); + + /* Copy the error code and message */ + vfdDst->errorCode = vfdSrc->errorCode; + strncpy(vfdDst->errorMsg, vfdSrc->errorMsg, sizeof(vfdDst->errorMsg)); + + /* Return -1 to indicate an error */ + return -1; +} diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..4b26b501d2 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -109,10 +109,10 @@ extern PGDLLIMPORT int max_safe_fds; extern File PathNameOpenFile(const char *fileName, int fileFlags); extern File PathNameOpenFilePerm(const char *fileName, int fileFlags, mode_t fileMode); extern File OpenTemporaryFile(bool interXact); -extern void FileClose(File file); +extern int FileClose(File file); extern int FilePrefetch(File file, off_t offset, off_t amount, uint32 wait_event_info); -extern int FileRead(File file, void *buffer, size_t amount, off_t offset, uint32 wait_event_info); -extern int FileWrite(File file, const void *buffer, size_t amount, off_t offset, uint32 wait_event_info); +extern ssize_t FileRead(File file, void *buffer, size_t amount, off_t offset, uint32 wait_event_info); +extern ssize_t FileWrite(File file, const void *buffer, size_t amount, off_t offset, uint32 wait_event_info); extern int FileSync(File file, uint32 wait_event_info); extern int FileZero(File file, off_t offset, off_t amount, uint32 wait_event_info); extern int FileFallocate(File file, off_t offset, off_t amount, uint32 wait_event_info); @@ -199,4 +199,44 @@ extern int data_sync_elevel(int elevel); #define PG_TEMP_FILES_DIR "pgsql_tmp" #define PG_TEMP_FILE_PREFIX "pgsql_tmp" +/* Operations on virtual files -- Sequential I/O */ +extern ssize_t FileWriteSeq(File file, const void *buffer, size_t amount, uint32 wait_event_info); +extern ssize_t FileReadSeq(File file, void *buffer, size_t amount, uint32 wait_event_info); +extern off_t FileTell(File file); +extern off_t FileSeek(File file, off_t offset); + +/* Operations on virtual files --- similar to fread/fwrite */ +extern ssize_t FilePrintf(File file, const char *format, ...) pg_attribute_printf(2,3); +extern ssize_t FileScanf(File file, const char *format, ...) pg_attribute_printf(2,3); +extern ssize_t FilePuts(File, const char *string); +extern ssize_t FileGetc(File file); +extern ssize_t FilePutc(int c, File file); + +/* Error handling on virtual files -- similar to feof/ferror */ +extern int FileEof(File file); /* Reset on every read */ +extern bool FileError(File file); /* Persists until cleared */ +extern bool FileClearError(File file); /* Clears both Eof and error */ + +extern const char *FileErrorMsg(File file); +extern int FileErrorCode(File file); + +/* Internal helpers for error handling */ +extern int setFileError(File file, int err, const char *format, ...); +extern int updateFileError(File file, int err, const char *format, ...); +extern int copyFileError(File dst, File src); + +/* Declare a "debug" macro */ +#ifdef DEBUG +#define debug(...) \ + do { \ + int save_errno = errno; \ + /* fprintf(stderr, __VA_ARGS__); */ \ + elog(DEBUG2, args); \ + errno = save_errno; \ + while (0) + +#else +#define debug(...) ((void)0) +#endif + #endif /* FD_H */ -- 2.33.0 From 9387e8fe8ed9064bcfdc6da7d51df432d00c2586 Mon Sep 17 00:00:00 2001 From: John Morris Date: Wed, 28 Jun 2023 19:59:47 -0700 Subject: [PATCH 2/2] Added sequential unit test and fixed error reporting bugs --- src/backend/storage/file/fd.c | 25 +- src/include/storage/fd.h | 13 +- src/test/Makefile | 2 +- src/test/meson.build | 1 + src/test/storage/framework/fileFramework.c | 473 ++++++++++++++++++ src/test/storage/framework/fileFramework.h | 25 + src/test/storage/framework/unitTest.h | 20 + src/test/storage/framework/unitTestInternal.h | 84 ++++ src/test/storage/meson.build | 22 + src/test/storage/vfdTest.c | 23 + 10 files changed, 672 insertions(+), 16 deletions(-) create mode 100644 src/test/storage/framework/fileFramework.c create mode 100644 src/test/storage/framework/fileFramework.h create mode 100644 src/test/storage/framework/unitTest.h create mode 100644 src/test/storage/framework/unitTestInternal.h create mode 100644 src/test/storage/meson.build create mode 100644 src/test/storage/vfdTest.c diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index ff37ca41fd..cd7441eb37 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -4083,6 +4083,10 @@ int FileClose(File file) { debug("FileClose: name=%s, file=%d\n", getName(file), file); + /* If invalid vfd or if already closed, then EBADF */ + if (file < 0 || file >= SizeVfdCache || getVfd(file)->fd == -1) + return setFileError(-1, EBADF, "FileClose: invalid file descriptor %d", file); + /* Save any existing error information */ copyFileError(-1, file); @@ -4262,11 +4266,9 @@ bool FileClearError(File file) { Vfd *vfd = getVfdErr(file); bool hasError = vfd->errorCode != 0; - if (hasError) - { - vfd->errorCode = 0; - vfd->errorMsg[0] = '\0'; - } + vfd->errorCode = 0; + vfd->errorMsg[0] = '\0'; + vfd->eof = false; return hasError; } @@ -4282,17 +4284,18 @@ const char *FileErrorMsg(File file) */ int FileErrorCode(File file) { - errno = getVfdErr(file)->errorCode; - return errno; + int errorCode = getVfdErr(file)->errorCode; + errno = errorCode; + return errorCode; } -int setFileError(File file, int err, const char *fmt, ...) +int setFileError(File file, int errorCode, const char *fmt, ...) { va_list args; Vfd *vfd = getVfdErr(file); /* Save the errno */ - vfd->errorCode = errno; + vfd->errorCode = errorCode; /* Format the error message */ va_start(args, fmt); @@ -4306,7 +4309,7 @@ int setFileError(File file, int err, const char *fmt, ...) return -1; } -int updateFileError(File file, int err, const char *fmt, ...) +int updateFileError(File file, int errorCode, const char *fmt, ...) { va_list args; Vfd *vfd = getVfdErr(file); @@ -4316,7 +4319,7 @@ int updateFileError(File file, int err, const char *fmt, ...) return -1; /* Save the errno */ - vfd->errorCode = errno; + vfd->errorCode = errorCode; /* Format the error message */ va_start(args, fmt); diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 4b26b501d2..69b6ce7111 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -12,6 +12,8 @@ *------------------------------------------------------------------------- */ +//#define DEBUG + /* * calls: * @@ -225,15 +227,18 @@ extern int setFileError(File file, int err, const char *format, ...); extern int updateFileError(File file, int err, const char *format, ...); extern int copyFileError(File dst, File src); +/* Some nicer names */ +static inline File FileOpen(const char *name, int fileFlags) {return PathNameOpenFile(name, fileFlags);} + /* Declare a "debug" macro */ #ifdef DEBUG #define debug(...) \ do { \ - int save_errno = errno; \ - /* fprintf(stderr, __VA_ARGS__); */ \ - elog(DEBUG2, args); \ + int save_errno = errno; \ + fprintf(stderr, __VA_ARGS__); \ + /* elog(DEBUG2, __VA_ARGS__); */ \ errno = save_errno; \ - while (0) + } while (0) #else #define debug(...) ((void)0) diff --git a/src/test/Makefile b/src/test/Makefile index dbd3192874..ed56a2de86 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -12,7 +12,7 @@ subdir = src/test top_builddir = ../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = perl regress isolation modules authentication recovery subscription +SUBDIRS = perl regress isolation modules authentication recovery subscription storage ifeq ($(with_icu),yes) SUBDIRS += icu diff --git a/src/test/meson.build b/src/test/meson.build index 5f3c9c2ba2..f582e6c497 100644 --- a/src/test/meson.build +++ b/src/test/meson.build @@ -5,6 +5,7 @@ subdir('isolation') subdir('authentication') subdir('recovery') +subdir('storage') subdir('subscription') subdir('modules') diff --git a/src/test/storage/framework/fileFramework.c b/src/test/storage/framework/fileFramework.c new file mode 100644 index 0000000000..405859b27b --- /dev/null +++ b/src/test/storage/framework/fileFramework.c @@ -0,0 +1,473 @@ +/**/ + +#include +#include +#include +#include +#include +#include + +#include "c.h" +//#include "storage/iostack_internal.h" +#include "storage/fd.h" +//#include "storage/vfd.h" +#include "utils/wait_event.h" + +#include "fileFramework.h" +#include "unitTestInternal.h" + +typedef uint8_t Byte; + +typedef void IoStack; +typedef IoStack *(*CreateTestStackFn)(size_t blockSize); +void setTestStack(CreateTestStackFn fn, size_t blockSize); +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define ROUNDUP(a,b) (((a) + (b) - 1) / (b) * (b)) +#define ROUNDDOWN(a,b) ((a) / (b) * (b)) +#define PATH_MAX 1024 + +#define countof(array) (sizeof(array)/sizeof(array[0])) + +/* Matrix of file and block sizes for testing. */ +off_t fileSize[] = {0, 1024, 1, 64, 1027, 7*1024, 32*1024 + 127, 6*1024*1024+153}; +size_t blockSize[] = {1024, 4 * 1024, 3 * 1024 + 357, 1024 - 237, 64, 1}; + + +/* Given the position in the seek, generate one byte of data for that position. */ +static inline Byte generateByte(size_t position) +{ + static char data[] = "The cat in the hat jumped over the quick brown fox while the dog ran away with the spoon.\n"; + size_t idx = position % (sizeof(data)-1); // Skip the nil character. + return data[idx]; +} + +/* Fill a buffer with data appropriate to that position in the seek */ +static void generateBuffer(size_t position, Byte *buf, size_t size) +{ + for (size_t i = 0; i < size; i++) + buf[i] = generateByte(position+i); +} + +/* Verify a buffer has appropriate data for that position in the test file. */ +static bool verifyBuffer(size_t position, Byte *buf, size_t size) +{ + for (size_t i = 0; i < size; i++) + { + Byte expected = generateByte(position + i); + if (expected != buf[i]) + debug("verifyBuffer: i=%zu position=%zu buf[i]=%c expected=%c\n", i, position, buf[i], expected); + PG_ASSERT_EQ(expected, buf[i]); + } + return true; +} + +/* + * Create a file and fill it with known data. + * The file contains the same line of text repeated over and over, which + * - makes it easy to verify with a text editor, + * - doesn't align with typical block sizes, and + * - is compressible. + */ +static void generateFile(char *path, off_t size, size_t bufferSize) +{ + Byte *buf; + off_t position; + File file; + + debug("generateFile: path=%s\n", path); + file = FileOpen( path, O_WRONLY|O_CREAT|O_TRUNC); + PG_ASSERT(file != -1); + buf = malloc(bufferSize); /* TODO: make buf be at end of struct */ + + for (position = 0; position < size; position += bufferSize) + { + ssize_t actual; + off_t expected = MIN(bufferSize, size-position); + generateBuffer(position, buf, expected); + actual = FileWriteSeq(file, buf, expected, 0); + PG_ASSERT_EQ(expected, actual); + } + + free(buf); + PG_ASSERT(FileClose(file) == 0); +} + +/* Verify a iostack has the correct data */ +static void verifyFile(char *path, off_t fileSize, ssize_t bufferSize) +{ + File file; + Byte *buf; + + debug("verifyFile: path=%s\n", path); + file = FileOpen(path, O_RDONLY|PG_TESTSTACK); + PG_ASSERT(file >= 0); + PG_ASSERT(!FileEof(file)); + PG_ASSERT(!FileError(file)); + buf = malloc(bufferSize); + + for (off_t actual, position = 0; position < fileSize; position += actual) + { + size_t expected = MIN(bufferSize, fileSize - position); + actual = FileReadSeq(file, buf, bufferSize, 0); + PG_ASSERT_EQ(expected, actual); + PG_ASSERT(verifyBuffer(position, buf, actual)); + PG_ASSERT(!FileEof(file)); + PG_ASSERT(!FileError(file)); + } + + // Read a final EOF. + PG_ASSERT(!FileEof(file)); + FileReadSeq(file, buf, 1, 0); + PG_ASSERT(FileEof(file)); + + PG_ASSERT(FileClose(file) == 0); +} + +/* + * Create a file and fill it with known data using random seeks. + * The file contains the same line of text repeated over and over, which + * - makes it easy to verify output with a text editor, + * - doesn't align with typical block sizes, and + * - is compressible. + */ +static void allocateFile(char *path, off_t size, ssize_t bufferSize) +{ + File file; + Byte *buf; + off_t position; + + debug("allocateFile: path=%s\n", path); + /* Start out by allocating space and filling the file with "X"s. */ + file = FileOpen(path, O_WRONLY|O_CREAT|O_TRUNC|PG_TESTSTACK); + buf = malloc(bufferSize); + memset(buf, 'X', bufferSize); + + for (position = 0; position < size; position += bufferSize) + { + size_t expected = (size_t)MIN(bufferSize, size-position); + size_t actual = FileWrite(file, buf, expected, position, 0); + PG_ASSERT_EQ(actual, expected); + } + + PG_ASSERT(FileClose(file) == 0); + free(buf); +} + +static const int prime = 3197; + +static void generateRandomFile(char *path, off_t size, size_t blockSize) +{ + size_t nrBlocks; + File file; + Byte *buf; + + debug("generateRandomFile: path=%s\n", path); + /* The nr of blocks must be relatively prime to "prime", otherwise we won't visit all the blocks. */ + nrBlocks = (size + blockSize - 1) / blockSize; + PG_ASSERT( nrBlocks == 0 || (nrBlocks % prime) != 0); + + file = FileOpen(path, O_RDWR|PG_TESTSTACK); + PG_ASSERT(file >= 0); + buf = malloc(blockSize); + + + for (off_t idx = 0; idx < nrBlocks; idx++) + { + ssize_t actual, expected; + /* Pick a pseudo-random block and seek to it */ + off_t position = ((idx * prime) % nrBlocks) * blockSize; + //printf("fileSeek - idx = %u blockNr=%u nrBlocks=%u\n", idx, (idx*prime)%nrBlocks, nrBlocks); + + /* Generate data appropriate for that block. */ + expected = (size_t)MIN(blockSize, size - position); + generateBuffer(position, buf, expected); + + /* Write the block */ + actual = FileWrite(file, buf, expected, position, 0); + PG_ASSERT_EQ(expected,actual); + } + + PG_ASSERT(FileClose(file) == 0); +} + +static void appendFile(char *path, off_t fileSize, size_t bufferSize) +{ + File file; + Byte *buf; + ssize_t blockSize; + ssize_t lastSize; + off_t lastBlock; + ssize_t remaining, actual; + + debug("appendFile: path=%s\n", path); + file = FileOpen(path, O_RDWR|O_APPEND|PG_TESTSTACK); + PG_ASSERT(file >= 0); + buf = malloc(bufferSize); + + /* Since we are appending, we are at the end of file - should match file size */ + PG_ASSERT_EQ(fileSize, FileTell(file)); + + /* The requested buffer size should be a multiple of the underlying block size. (Property of unit test) */ + blockSize = 1; //FileBlockSize(file); + PG_ASSERT_EQ(0, bufferSize % blockSize); + + /* If the last block is a partial block, ... */ + lastBlock = ROUNDDOWN(fileSize, blockSize); + lastSize = fileSize - lastBlock; + if (lastSize > 0) + { + /* Rewrite the last block, which is now full */ + generateBuffer(lastBlock, buf, blockSize); + PG_ASSERT_EQ(blockSize, FileWrite(file, buf, blockSize, lastBlock, 0)); + + /* Adjust the write parameters to write whatever is left. It is now block aligned. */ + lastBlock += blockSize; + } + + /* Write whatever remains to the end of file */ + remaining = (fileSize + bufferSize) - lastBlock; + generateBuffer(lastBlock, buf, remaining); + actual = FileWriteSeq(file, buf, remaining, 0); + PG_ASSERT_EQ(remaining, actual); + + /* Close the file and verify it is correct. */ + PG_ASSERT_EQ(0, FileClose(file)); + verifyFile(path, fileSize+bufferSize, bufferSize); +} + +/* + * Verify an ioStack has the correct data through random seeks. + * This should do a complete verification - examining every byte of the file. + */ +static void verifyRandomFile(char *path, off_t size, size_t blockSize) +{ + File file; + Byte *buf; + size_t nrBlocks; + + debug("verifyRandomFile: path=%s\n", path); + file = FileOpen(path, O_RDONLY|PG_TESTSTACK); + PG_ASSERT(file >= 0); + buf = malloc(blockSize); + + nrBlocks = (size + blockSize -1) / blockSize; + PG_ASSERT(nrBlocks == 0 || (nrBlocks % prime) != 0); + for (size_t idx = 0; idx < nrBlocks; idx++) + { + ssize_t actual, expected; + + /* Pick a pseudo-random block and read it */ + off_t position = ((idx * prime) % nrBlocks) * blockSize; + + actual = FileRead(file, buf, blockSize, position, 0); + + /* Verify we read the correct data */ + expected = MIN(blockSize, size-position); + PG_ASSERT_EQ(expected, actual); + PG_ASSERT(verifyBuffer(position, buf, actual)); + } + + PG_ASSERT_EQ(0, FileClose(file)); +} + + +static void deleteFile(char *name) +{ + unlink(name); +} + + +static void regression(char *name, size_t blockSize) +{ + File file; + Byte buf[128]; + Byte *block; + + deleteFile(name); + + /* Shouldn't open a non-existent file - various modes) */ + file = FileOpen(name, O_RDWR|PG_TESTSTACK); + PG_ASSERT_EQ(-1, file); + PG_ASSERT_EQ(ENOENT, errno); + + file = FileOpen(name, O_RDONLY|PG_TESTSTACK); + PG_ASSERT(file == -1); + PG_ASSERT_EQ(errno, ENOENT); + + /* OK to create a file and reopen readonly */ + file = FileOpen(name, O_CREAT | O_WRONLY | O_TRUNC|PG_TESTSTACK); + PG_ASSERT(file >= 0); + PG_ASSERT_EQ(FileClose(file), 0); + + file = FileOpen(name, O_CREAT | O_WRONLY | O_TRUNC | PG_TESTSTACK); + PG_ASSERT(file >= 0); + PG_ASSERT_EQ(FileClose(file), 0); + + /* EBADF if closing an already closed file */ + PG_ASSERT(FileClose(file) != 0 && errno == EBADF); + + /* Should read EOF on empty file */ + file = FileOpen(name, O_RDONLY|PG_TESTSTACK); + PG_ASSERT(0 == FileRead(file, buf, sizeof(buf), 0, 0)); + PG_ASSERT(FileEof(file)); + PG_ASSERT(!FileError(file)); + PG_ASSERT(FileClose(file) == 0); + + /* Should write a block and then read EOF */ + block = calloc(blockSize, 1); + file = FileOpen(name, O_RDWR|PG_TESTSTACK); + PG_ASSERT_EQ(blockSize, FileWriteSeq(file, block, blockSize, 0)); + PG_ASSERT_EQ(0, FileReadSeq(file, block, blockSize, 0)); + PG_ASSERT(FileEof(file)); + PG_ASSERT(!FileError(file)); + PG_ASSERT(FileClose(file) == 0); + free(block); + + deleteFile(name); +} + + +/* + * Run a test on a single configuration determined by file size and buffer size + */ +void singleSeekTest(CreateTestStackFn testStack, char *nameFmt, off_t size, size_t bufferSize) +{ + char fileName[PATH_MAX]; + snprintf(fileName, sizeof(fileName), nameFmt, size, bufferSize); + beginTest(fileName); + + /* Inject the procedure to create an I/O Stack */ + setTestStack(testStack, bufferSize); + + /* create and read back as a stream */ + generateFile(fileName, size, bufferSize); + verifyFile(fileName, size, bufferSize); + + /* Fill in the file with garbage, then write it out as random writes */ + allocateFile(fileName, size, bufferSize); + generateRandomFile(fileName, size, bufferSize); + verifyFile(fileName, size, bufferSize); + + /* append to the file */ + appendFile(fileName, size, bufferSize); + verifyFile(fileName, size+bufferSize, ROUNDDOWN(16*1024, bufferSize)); /* larger buffer */ + + /* Read back as random reads */ + verifyRandomFile(fileName, size+bufferSize, bufferSize); + + regression(fileName, bufferSize); + + /* Clean things up */ + deleteFile(fileName); +} + +/* run a matrix of tests for various file sizes and I/O sizes. All will use a 1K block size. */ +void seekTest(CreateTestStackFn testStack, char *nameFmt) +{ + for (int fileIdx = 0; fileIdx +#include +extern void MemoryContextInit(void); + +#define TEST_DIR "/tmp/pgtest/" + +#define BEGIN do { +#define END } while (0) + +// static const char *expectFmt = "Expected '%s' but got '%s'"; +#define expectFmt "Expected '%s' but got '%s'" + +/* Is an integer expression signed? Note _Generic is a C11 feature. Define as "false" if c99. */ +#define isSigned(x) _Generic(x, \ + char: true, \ + int: true, \ + long: true, \ + long long: true, \ + float: (void)0, \ + double: (void)0, \ + default: false) + + +/* Verify two scaler values are equal */ +#define PG_ASSERT_EQ(a,b) \ + BEGIN \ + uint64_t _a=a; uint64_t _b = b; \ + char _bufa[16], _bufb[16]; \ + if ( _a != _b || (isSigned(a) && (int64_t)_a < 0) != (isSigned(b) && (int64_t)_b < 0)) \ + PG_ASSERT_FMT(expectFmt, PG_INT_TO_STR(a, _a, _bufa), PG_INT_TO_STR(b, _b, _bufb)); \ + END + + +/* Verify two strings are equal */ +#define PG_ASSERT_EQ_STR(stra, strb) \ + BEGIN \ + if (strcmp(stra, strb) != 0) \ + PG_ASSERT_FMT(expectFmt, stra, strb); \ + END + + +/* + * Format a signed/unsigned integer as a string. + * Since we only want to evalate an expression once, accepts both expression (for type) and value. + */ +#define PG_INT_TO_STR(expr, val, buf) \ + isSigned(expr)? (snprintf(buf, sizeof(buf), "%lld", (long long)(val)), buf) \ + : (snprintf(buf, sizeof(buf), "%llu", (unsigned long long)(val)), buf) + + +#define PG_ASSERT_ERRNO(_expectedErrno) \ + BEGIN \ + PG_ASSERT_EQ(_expectedErrno, errno) ; \ + END + +/* Display a formatted message and exit */ +#define PG_ASSERT_FMT(fmt, ...) \ + BEGIN \ + char _buf[256]; \ + snprintf(_buf, sizeof(_buf), fmt, __VA_ARGS__); \ + PG_ASSERT_MSG(_buf); \ + END + +/* Display an unformatted message and exit */ +#define PG_ASSERT_MSG(msg) \ + (fprintf(stderr, "FAILED: %s (%s:%d) %s\n", __func__, __FILE__, __LINE__, msg ), abort()) + +/* Verify the expression is true */ +#define PG_ASSERT(expr) \ + BEGIN \ + if (!(expr)) \ + PG_ASSERT_MSG("'" #expr "' is false"); \ + END + +static inline void beginTestGroup(char *name) {fprintf(stderr, "Begin Testgroup %s\n", name);} +static inline void beginTest(char *name) {fprintf(stderr, " Test %s\n", name);} + +#endif //FILTER_UNITTESTINTERNAL_H diff --git a/src/test/storage/meson.build b/src/test/storage/meson.build new file mode 100644 index 0000000000..732363dd4d --- /dev/null +++ b/src/test/storage/meson.build @@ -0,0 +1,22 @@ +#################################################################### +# Create an archive library containing all the backend object files. +#################################################################### +test_lib = static_library('test_lib', + objects: backend_objs, + ) + + +################################################### +# Individual unit tests in the 'iostack' test suite. +# TODO: refactor so each is a single line? +################################################### + +# Test the basic File layer +storage_vfdtest = executable('storage_vfdtest', + files('vfdTest.c', 'framework/fileFramework.c', ), + include_directories: [include_directories('.'), postgres_inc], + link_args: backend_link_args, + link_with: [test_lib, backend_link_with], + link_depends: backend_link_depends, +) +test('storage/vfd', storage_vfdtest, suite: 'iostack') diff --git a/src/test/storage/vfdTest.c b/src/test/storage/vfdTest.c new file mode 100644 index 0000000000..b01ae19ad1 --- /dev/null +++ b/src/test/storage/vfdTest.c @@ -0,0 +1,23 @@ +/* + * + */ +#include +#include +#include +#include "./framework/fileFramework.h" +#include "./framework/unitTest.h" + + +#define createStack NULL + +void testMain() +{ + system("rm -rf " TEST_DIR "vfd; mkdir -p " TEST_DIR "vfd"); + + + + beginTest("Storage"); + beginTestGroup("Vfd Stack"); + singleSeekTest(createStack, TEST_DIR "vfd/testfile_%u_%u.dat", 1024, 4096); + seekTest(createStack, TEST_DIR "vfd/testfile_%u_%u.dat"); +} -- 2.33.0