diff --git a/Makefile b/Makefile index 2b663f59b..5c50c6d5e 100644 --- a/Makefile +++ b/Makefile @@ -36,17 +36,16 @@ OBJS := src/utils/configuration.o src/utils/json.o src/utils/logger.o \ OBJS += src/archive.o src/backup.o src/catalog.o src/checkdb.o src/configure.o src/data.o \ src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \ src/parsexlog.o src/ptrack.o src/pg_probackup.o src/restore.o src/show.o src/stream.o \ - src/util.o src/validate.o src/datapagemap.o src/catchup.o + src/util.o src/validate.o src/datapagemap.o src/catchup.o \ + src/compatibility/pg-11.o # sources borrowed from postgresql (paths are relative to pg top dir) BORROWED_H_SRC := \ - src/include/portability/instr_time.h \ src/bin/pg_basebackup/receivelog.h \ src/bin/pg_basebackup/streamutil.h \ src/bin/pg_basebackup/walmethods.h BORROWED_C_SRC := \ src/backend/access/transam/xlogreader.c \ - src/backend/utils/hash/pg_crc.c \ src/bin/pg_basebackup/receivelog.c \ src/bin/pg_basebackup/streamutil.c \ src/bin/pg_basebackup/walmethods.c @@ -87,7 +86,6 @@ override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS) PG_LIBS_INTERNAL = $(libpq_pgport) ${PTHREAD_CFLAGS} # additional dependencies on borrowed files -src/archive.o: $(BORROW_DIR)/instr_time.h src/backup.o src/catchup.o src/pg_probackup.o: $(BORROW_DIR)/streamutil.h src/stream.o $(BORROW_DIR)/receivelog.o $(BORROW_DIR)/streamutil.o $(BORROW_DIR)/walmethods.o: $(BORROW_DIR)/receivelog.h $(BORROW_DIR)/receivelog.h: $(BORROW_DIR)/walmethods.h diff --git a/src/archive.c b/src/archive.c index 0ebe5e504..b552689cd 100644 --- a/src/archive.c +++ b/src/archive.c @@ -11,7 +11,7 @@ #include #include "pg_probackup.h" #include "utils/thread.h" -#include "instr_time.h" +#include "portability/instr_time.h" static int push_file_internal(const char *wal_file_name, const char *pg_xlog_dir, diff --git a/src/backup.c b/src/backup.c index 15f1a4d1c..b98935b8e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -1725,7 +1725,7 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c if (S_ISREG(file->mode)) { - file->crc = pgFileGetCRC(full_filename, true, false); + file->crc = pgFileGetCRC32C(full_filename, false); file->write_size = file->size; file->uncompressed_size = file->size; diff --git a/src/catalog.c b/src/catalog.c index b4be159d1..fdfe4340a 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -1067,7 +1067,7 @@ get_backup_filelist(pgBackup *backup, bool strict) files = parray_new(); - INIT_FILE_CRC32(true, content_crc); + INIT_CRC32C(content_crc); while (fgets(buf, lengthof(buf), fp)) { @@ -1089,7 +1089,7 @@ get_backup_filelist(pgBackup *backup, bool strict) hdr_size; pgFile *file; - COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); + COMP_CRC32C(content_crc, buf, strlen(buf)); get_control_value_str(buf, "path", path, sizeof(path),true); get_control_value_int64(buf, "size", &write_size, true); @@ -1141,7 +1141,7 @@ get_backup_filelist(pgBackup *backup, bool strict) parray_append(files, file); } - FIN_FILE_CRC32(true, content_crc); + FIN_CRC32C(content_crc); if (ferror(fp)) elog(ERROR, "Failed to read from file: \"%s\"", backup_filelist_path); @@ -2538,7 +2538,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, setvbuf(out, buf, _IOFBF, BUFFERSZ); if (sync) - INIT_FILE_CRC32(true, backup->content_crc); + INIT_CRC32C(backup->content_crc); /* print each file in the list */ for (i = 0; i < parray_num(files); i++) @@ -2606,13 +2606,13 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root, sprintf(line+len, "}\n"); if (sync) - COMP_FILE_CRC32(true, backup->content_crc, line, strlen(line)); + COMP_CRC32C(backup->content_crc, line, strlen(line)); fprintf(out, "%s", line); } if (sync) - FIN_FILE_CRC32(true, backup->content_crc); + FIN_CRC32C(backup->content_crc); if (fflush(out) != 0) elog(ERROR, "Cannot flush file list \"%s\": %s", diff --git a/src/compatibility/pg-11.c b/src/compatibility/pg-11.c new file mode 100644 index 000000000..52f4b551c --- /dev/null +++ b/src/compatibility/pg-11.c @@ -0,0 +1,98 @@ +/*------------------------------------------------------------------------- + * + * pg-11.c + * PostgreSQL <= 11 compatibility + * + * Portions Copyright (c) 2022, Postgres Professional + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + *------------------------------------------------------------------------- + */ + +#include + +#if PG_VERSION_NUM < 120000 + +#include "c.h" +#include "utils/pg_crc.h" + +/* From postgresql src/backend/utils/hash/pg_crc.c */ + +/* + * Lookup table for calculating CRC-32 using Sarwate's algorithm. + * + * This table is based on the polynomial + * x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1. + * (This is the same polynomial used in Ethernet checksums, for instance.) + * Using Williams' terms, this is the "normal", not "reflected" version. + */ + +const uint32 pg_crc32_table[256] = { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, + 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, + 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, + 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, + 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, + 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, + 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, + 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, + 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, + 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, + 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, + 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, + 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, + 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, + 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, + 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, + 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, + 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, + 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, + 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, + 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, + 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, + 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, + 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, + 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, + 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, + 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, + 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D +}; + +#endif diff --git a/src/compatibility/pg-11.h b/src/compatibility/pg-11.h new file mode 100644 index 000000000..63a83070a --- /dev/null +++ b/src/compatibility/pg-11.h @@ -0,0 +1,67 @@ +/*------------------------------------------------------------------------- + * + * pg-11.h + * PostgreSQL <= 11 compatibility + * + * Copyright (c) 2022, Postgres Professional + * + * When PG-11 reaches the end of support, we will need to remove + * *_CRC32_COMPAT macros and use *_CRC32C instead. + * And this file will be removed. + *------------------------------------------------------------------------- + */ + +#ifndef PG11_COMPAT_H +#define PG11_COMPAT_H + +#include "utils/pgut.h" + +#if PG_VERSION_NUM >= 120000 + +#define INIT_CRC32_COMPAT(backup_version, crc) \ +do { \ + Assert(backup_version >= 20025); \ + INIT_CRC32C(crc); \ +} while (0) + +#define COMP_CRC32_COMPAT(backup_version, crc, data, len) \ +do { \ + Assert(backup_version >= 20025); \ + COMP_CRC32C((crc), (data), (len)); \ +} while (0) + +#define FIN_CRC32_COMPAT(backup_version, crc) \ +do { \ + Assert(backup_version >= 20025); \ + FIN_CRC32C(crc); \ +} while (0) + +#else /* PG_VERSION_NUM < 120000 */ + +#define INIT_CRC32_COMPAT(backup_version, crc) \ +do { \ + if (backup_version <= 20021 || backup_version >= 20025) \ + INIT_CRC32C(crc); \ + else \ + INIT_TRADITIONAL_CRC32(crc); \ +} while (0) + +#define COMP_CRC32_COMPAT(backup_version, crc, data, len) \ +do { \ + if (backup_version <= 20021 || backup_version >= 20025) \ + COMP_CRC32C((crc), (data), (len)); \ + else \ + COMP_TRADITIONAL_CRC32(crc, data, len); \ +} while (0) + +#define FIN_CRC32_COMPAT(backup_version, crc) \ +do { \ + if (backup_version <= 20021 || backup_version >= 20025) \ + FIN_CRC32C(crc); \ + else \ + FIN_TRADITIONAL_CRC32(crc); \ +} while (0) + +#endif /* PG_VERSION_NUM < 120000 */ + +#endif /* PG11_COMPAT_H */ diff --git a/src/data.c b/src/data.c index 17ae4b91a..f50749497 100644 --- a/src/data.c +++ b/src/data.c @@ -24,6 +24,9 @@ #include "utils/thread.h" +/* for crc32_compat macros */ +#include "compatibility/pg-11.h" + /* Union to ease operations on relation pages */ typedef struct DataPage { @@ -32,7 +35,7 @@ typedef struct DataPage } DataPage; static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader *bph, - pg_crc32 *crc, bool use_crc32c); + pg_crc32 *crc, uint32 backup_version); #ifdef HAVE_LIBZ /* Implementation of zlib compression method */ @@ -448,7 +451,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum, write_buffer_size = compressed_size + sizeof(BackupPageHeader); /* Update CRC */ - COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size); + COMP_CRC32C(*crc, write_buffer, write_buffer_size); /* write data page */ if (fio_fwrite(out, write_buffer, write_buffer_size) != write_buffer_size) @@ -529,7 +532,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat file->read_size = 0; file->write_size = 0; file->uncompressed_size = 0; - INIT_FILE_CRC32(true, file->crc); + INIT_CRC32C(file->crc); /* * Read each page, verify checksum and write it to backup. @@ -628,7 +631,7 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat cleanup: /* finish CRC calculation */ - FIN_FILE_CRC32(true, file->crc); + FIN_CRC32C(file->crc); /* dump page headers */ write_page_headers(headers, file, hdr_map, is_merge); @@ -805,7 +808,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, file->crc = fio_get_crc32(FIO_DB_HOST, from_fullpath, false); /* ...and checksum is the same... */ - if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) + if (EQ_CRC32C(file->crc, prev_file->crc)) { file->write_size = BYTES_INVALID; return; /* ...skip copying file. */ @@ -1018,7 +1021,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers * or when merging something. Align read_len only when restoring * or merging old backups. */ - if (get_page_header(in, from_fullpath, &(page).bph, NULL, false)) + if (get_page_header(in, from_fullpath, &(page).bph, NULL, backup_version)) { cur_pos_in += sizeof(BackupPageHeader); @@ -1389,7 +1392,7 @@ backup_non_data_file_internal(const char *from_fullpath, ssize_t read_len = 0; char *buf = NULL; - INIT_FILE_CRC32(true, file->crc); + INIT_CRC32C(file->crc); /* reset size summary */ file->read_size = 0; @@ -1485,7 +1488,7 @@ backup_non_data_file_internal(const char *from_fullpath, strerror(errno)); /* update CRC */ - COMP_FILE_CRC32(true, file->crc, buf, read_len); + COMP_CRC32C(file->crc, buf, read_len); file->read_size += read_len; } @@ -1501,7 +1504,7 @@ backup_non_data_file_internal(const char *from_fullpath, cleanup: /* finish CRC calculation and store into pgFile */ - FIN_FILE_CRC32(true, file->crc); + FIN_CRC32C(file->crc); if (in && fclose(in)) elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno)); @@ -1678,7 +1681,6 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, bool is_valid = true; FILE *in; pg_crc32 crc; - bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; BackupPageHeader2 *headers = NULL; int n_hdr = -1; off_t cur_pos_in = 0; @@ -1702,7 +1704,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, } /* calc CRC of backup file */ - INIT_FILE_CRC32(use_crc32c, crc); + INIT_CRC32_COMPAT(backup_version, crc); /* read and validate pages one by one */ while (true) @@ -1718,7 +1720,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, if (interrupted || thread_interrupted) elog(ERROR, "Interrupted during data file validation"); - /* newer backups have page headers in separate storage */ + /* newer backups (post 2.4.0) have page headers in separate storage */ if (headers) { n_hdr++; @@ -1747,10 +1749,10 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, cur_pos_in = headers[n_hdr].pos; } } - /* old backups rely on header located directly in data file */ + /* old backups (pre 2.4.0) rely on header located directly in data file */ else { - if (get_page_header(in, fullpath, &(compressed_page).bph, &crc, use_crc32c)) + if (get_page_header(in, fullpath, &(compressed_page).bph, &crc, backup_version)) { /* Backward compatibility kludge, TODO: remove in 3.0 * for some reason we padded compressed pages in old versions @@ -1790,9 +1792,9 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, cur_pos_in += read_len; if (headers) - COMP_FILE_CRC32(use_crc32c, crc, &compressed_page, read_len); + COMP_CRC32_COMPAT(backup_version, crc, &compressed_page, read_len); else - COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); + COMP_CRC32_COMPAT(backup_version, crc, compressed_page.data, read_len); if (compressed_size != BLCKSZ || page_may_be_compressed(compressed_page.data, file->compress_alg, @@ -1861,7 +1863,7 @@ validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, } } - FIN_FILE_CRC32(use_crc32c, crc); + FIN_CRC32_COMPAT(backup_version, crc); fclose(in); if (crc != file->crc) @@ -2017,7 +2019,7 @@ get_lsn_map(const char *fullpath, uint32 checksum_version, /* Every page in data file contains BackupPageHeader, extract it */ bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, - pg_crc32 *crc, bool use_crc32c) + pg_crc32 *crc, uint32 backup_version) { /* read BackupPageHeader */ size_t read_len = fread(bph, 1, sizeof(BackupPageHeader), in); @@ -2044,7 +2046,7 @@ get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, * the problem of backward compatibility for backups of old versions */ if (crc) - COMP_FILE_CRC32(use_crc32c, *crc, bph, read_len); + COMP_CRC32_COMPAT(backup_version, *crc, bph, read_len); if (bph->block == 0 && bph->compressed_size == 0) elog(ERROR, "Empty block in file \"%s\"", fullpath); @@ -2363,6 +2365,8 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, * array of headers. * TODO: some access optimizations would be great here: * less fseeks, buffering, descriptor sharing, etc. + * + * Used for post 2.4.0 backups */ BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict) @@ -2437,9 +2441,9 @@ get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, b } /* validate checksum */ - INIT_FILE_CRC32(true, hdr_crc); - COMP_FILE_CRC32(true, hdr_crc, headers, read_len); - FIN_FILE_CRC32(true, hdr_crc); + INIT_CRC32C(hdr_crc); + COMP_CRC32C(hdr_crc, headers, read_len); + FIN_CRC32C(hdr_crc); if (hdr_crc != file->hdr_crc) { @@ -2486,9 +2490,9 @@ write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, read_len = (file->n_headers + 1) * sizeof(BackupPageHeader2); /* calculate checksums */ - INIT_FILE_CRC32(true, file->hdr_crc); - COMP_FILE_CRC32(true, file->hdr_crc, headers, read_len); - FIN_FILE_CRC32(true, file->hdr_crc); + INIT_CRC32C(file->hdr_crc); + COMP_CRC32C(file->hdr_crc, headers, read_len); + FIN_CRC32C(file->hdr_crc); zheaders = pgut_malloc(read_len * 2); memset(zheaders, 0, read_len * 2); diff --git a/src/dir.c b/src/dir.c index 0bcd60169..53f92ef74 100644 --- a/src/dir.c +++ b/src/dir.c @@ -203,26 +203,23 @@ pgFileInit(const char *rel_path) * obvious about it. */ pg_crc32 -pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) +pgFileGetCRC32C(const char *file_path, bool missing_ok) { FILE *fp; pg_crc32 crc = 0; char *buf; size_t len = 0; - INIT_FILE_CRC32(use_crc32c, crc); + INIT_CRC32C(crc); /* open file in binary read mode */ fp = fopen(file_path, PG_BINARY_R); if (fp == NULL) { - if (errno == ENOENT) + if (missing_ok && errno == ENOENT) { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } + FIN_CRC32C(crc); + return crc; } elog(ERROR, "Cannot open file \"%s\": %s", @@ -234,7 +231,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) buf = pgut_malloc(STDIO_BUFSIZE); /* calc CRC of file */ - for (;;) + do { if (interrupted) elog(ERROR, "interrupted during CRC calculation"); @@ -244,19 +241,75 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) if (ferror(fp)) elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); - /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); + COMP_CRC32C(crc, buf, len); + } + while (!feof(fp)); + + FIN_CRC32C(crc); + fclose(fp); + pg_free(buf); + + return crc; +} + +#if PG_VERSION_NUM < 120000 +/* + * Read the local file to compute its CRC using traditional algorithm. + * (*_TRADITIONAL_CRC32 macros) + * This was used only in version 2.0.22--2.0.24 + * And never used for PG >= 12 + * To be removed with end of PG-11 support + */ +pg_crc32 +pgFileGetCRC32(const char *file_path, bool missing_ok) +{ + FILE *fp; + pg_crc32 crc = 0; + char *buf; + size_t len = 0; + + INIT_TRADITIONAL_CRC32(crc); + + /* open file in binary read mode */ + fp = fopen(file_path, PG_BINARY_R); + if (fp == NULL) + { + if (missing_ok && errno == ENOENT) + { + FIN_TRADITIONAL_CRC32(crc); + return crc; + } - if (feof(fp)) - break; + elog(ERROR, "Cannot open file \"%s\": %s", + file_path, strerror(errno)); } - FIN_FILE_CRC32(use_crc32c, crc); + /* disable stdio buffering */ + setvbuf(fp, NULL, _IONBF, BUFSIZ); + buf = pgut_malloc(STDIO_BUFSIZE); + + /* calc CRC of file */ + do + { + if (interrupted) + elog(ERROR, "interrupted during CRC calculation"); + + len = fread(buf, 1, STDIO_BUFSIZE, fp); + + if (ferror(fp)) + elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); + + COMP_TRADITIONAL_CRC32(crc, buf, len); + } + while (!feof(fp)); + + FIN_TRADITIONAL_CRC32(crc); fclose(fp); pg_free(buf); return crc; } +#endif /* PG_VERSION_NUM < 120000 */ /* * Read the local file to compute its CRC. @@ -265,7 +318,7 @@ pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) * obvious about it. */ pg_crc32 -pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) +pgFileGetCRC32Cgz(const char *file_path, bool missing_ok) { gzFile fp; pg_crc32 crc = 0; @@ -273,19 +326,16 @@ pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) int err; char *buf; - INIT_FILE_CRC32(use_crc32c, crc); + INIT_CRC32C(crc); /* open file in binary read mode */ fp = gzopen(file_path, PG_BINARY_R); if (fp == NULL) { - if (errno == ENOENT) + if (missing_ok && errno == ENOENT) { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } + FIN_CRC32C(crc); + return crc; } elog(ERROR, "Cannot open file \"%s\": %s", @@ -311,16 +361,16 @@ pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) { const char *err_str = NULL; - err_str = gzerror(fp, &err); - elog(ERROR, "Cannot read from compressed file %s", err_str); + err_str = gzerror(fp, &err); + elog(ERROR, "Cannot read from compressed file %s", err_str); } } /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); + COMP_CRC32C(crc, buf, len); } - FIN_FILE_CRC32(use_crc32c, crc); + FIN_CRC32C(crc); gzclose(fp); pg_free(buf); @@ -1758,7 +1808,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ /* Add metadata to backup_content.control */ file = pgFileNew(database_map_path, DATABASE_MAP, true, 0, FIO_BACKUP_HOST); - file->crc = pgFileGetCRC(database_map_path, true, false); + file->crc = pgFileGetCRC32C(database_map_path, false); file->write_size = file->size; file->uncompressed_size = file->read_size; diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 7ce455459..55194a247 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -207,28 +207,6 @@ typedef enum ForkName ptrack } ForkName; -#define INIT_FILE_CRC32(use_crc32c, crc) \ -do { \ - if (use_crc32c) \ - INIT_CRC32C(crc); \ - else \ - INIT_TRADITIONAL_CRC32(crc); \ -} while (0) -#define COMP_FILE_CRC32(use_crc32c, crc, data, len) \ -do { \ - if (use_crc32c) \ - COMP_CRC32C((crc), (data), (len)); \ - else \ - COMP_TRADITIONAL_CRC32(crc, data, len); \ -} while (0) -#define FIN_FILE_CRC32(use_crc32c, crc) \ -do { \ - if (use_crc32c) \ - FIN_CRC32C(crc); \ - else \ - FIN_TRADITIONAL_CRC32(crc); \ -} while (0) - #define pg_off_t unsigned long long @@ -1045,8 +1023,11 @@ extern pgFile *pgFileNew(const char *path, const char *rel_path, extern pgFile *pgFileInit(const char *rel_path); extern void pgFileFree(void *file); -extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok); -extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok); +extern pg_crc32 pgFileGetCRC32C(const char *file_path, bool missing_ok); +#if PG_VERSION_NUM < 120000 +extern pg_crc32 pgFileGetCRC32(const char *file_path, bool missing_ok); +#endif +extern pg_crc32 pgFileGetCRC32Cgz(const char *file_path, bool missing_ok); extern int pgFileMapComparePath(const void *f1, const void *f2); extern int pgFileCompareName(const void *f1, const void *f2); diff --git a/src/restore.c b/src/restore.c index 28a79f1ed..e4479a242 100644 --- a/src/restore.c +++ b/src/restore.c @@ -2010,7 +2010,6 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, { int i; int j; -// pg_crc32 crc; parray *database_map = NULL; parray *dbOid_exclude_list = NULL; pgFile *database_map_file = NULL; @@ -2040,13 +2039,6 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); - /* check database_map CRC */ -// crc = pgFileGetCRC(database_map_path, true, true, NULL, FIO_LOCAL_HOST); -// -// if (crc != database_map_file->crc) -// elog(ERROR, "Invalid CRC of backup file \"%s\" : %X. Expected %X", -// database_map_file->path, crc, database_map_file->crc); - /* get database_map from file */ database_map = read_database_map(backup); diff --git a/src/stream.c b/src/stream.c index b10eb7308..e2e016f4d 100644 --- a/src/stream.c +++ b/src/stream.c @@ -2,7 +2,7 @@ * * stream.c: pg_probackup specific code for WAL streaming * - * Portions Copyright (c) 2015-2022, Postgres Professional + * Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -689,7 +689,7 @@ add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos if (existing_file) { if (do_crc) - (*existing_file)->crc = pgFileGetCRC(wal_segment_fullpath, true, false); + (*existing_file)->crc = pgFileGetCRC32C(wal_segment_fullpath, false); (*existing_file)->write_size = xlog_seg_size; (*existing_file)->uncompressed_size = xlog_seg_size; @@ -697,7 +697,7 @@ add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos } if (do_crc) - file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); + file->crc = pgFileGetCRC32C(wal_segment_fullpath, false); /* Should we recheck it using stat? */ file->write_size = xlog_seg_size; @@ -728,7 +728,7 @@ add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir) /* calculate crc */ if (do_crc) - file->crc = pgFileGetCRC(fullpath, true, false); + file->crc = pgFileGetCRC32C(fullpath, false); file->write_size = file->size; file->uncompressed_size = file->size; diff --git a/src/util.c b/src/util.c index b58d88f96..e16241a70 100644 --- a/src/util.c +++ b/src/util.c @@ -304,7 +304,7 @@ get_pgcontrol_checksum(const char *pgdata_path) /* First fetch file... */ buffer = slurpFile(FIO_BACKUP_HOST, pgdata_path, XLOG_CONTROL_FILE, &size, false); - + elog(WARNING, "checking %s", pgdata_path); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); diff --git a/src/utils/file.c b/src/utils/file.c index 86977a19a..27b5edf86 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -1428,9 +1428,9 @@ fio_get_crc32(fio_location location, const char *file_path, bool decompress) else { if (decompress) - return pgFileGetCRCgz(file_path, true, true); + return pgFileGetCRC32Cgz(file_path, true); else - return pgFileGetCRC(file_path, true, true); + return pgFileGetCRC32C(file_path, true); } } @@ -2082,7 +2082,7 @@ fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, Assert(hdr.size <= sizeof(buf)); IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + COMP_CRC32C(file->crc, buf, hdr.size); /* lazily open backup file */ if (!out) @@ -2270,8 +2270,6 @@ fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, Assert(hdr.size <= sizeof(buf)); IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); - if (fio_fseek(out, blknum * BLCKSZ) < 0) { elog(ERROR, "Cannot seek block %u of \"%s\": %s", @@ -2635,7 +2633,7 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, if (file) { file->read_size += hdr.size; - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); + COMP_CRC32C(file->crc, buf, hdr.size); } } else @@ -3366,9 +3364,9 @@ fio_communicate(int in, int out) case FIO_GET_CRC32: /* calculate crc32 for a file */ if (hdr.arg == 1) - crc = pgFileGetCRCgz(buf, true, true); + crc = pgFileGetCRC32Cgz(buf, true); else - crc = pgFileGetCRC(buf, true, true); + crc = pgFileGetCRC32C(buf, true); IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); break; case FIO_GET_CHECKSUM_MAP: @@ -3606,9 +3604,9 @@ pioLocalDrive_pioGetCRC32(VSelf, path_t path, bool compressed, err_i *err) elog(VERBOSE, "Local Drive calculate crc32 for '%s', compressed=%d", path, compressed); if (compressed) - return pgFileGetCRCgz(path, true, true); + return pgFileGetCRC32Cgz(path, true); else - return pgFileGetCRC(path, true, true); + return pgFileGetCRC32C(path, true); } static bool diff --git a/src/utils/pgut.h b/src/utils/pgut.h index 638259a3c..72ac20379 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -3,7 +3,7 @@ * pgut.h * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2021, Postgres Professional + * Portions Copyright (c) 2017-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -11,6 +11,7 @@ #ifndef PGUT_H #define PGUT_H +#include #include "postgres_fe.h" #include "libpq-fe.h" diff --git a/src/validate.c b/src/validate.c index 79a450ac8..e0aea8702 100644 --- a/src/validate.c +++ b/src/validate.c @@ -3,7 +3,7 @@ * validate.c: validate backup files. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -341,14 +341,22 @@ pgBackupValidateFiles(void *arg) * Starting from 2.0.25 we calculate crc of pg_control differently. */ if (arguments->backup_version >= 20025 && - strcmp(file->name, "pg_control") == 0 && - !file->external_dir_num) + strcmp(file->rel_path, XLOG_CONTROL_FILE) == 0 && + file->external_dir_num == 0) crc = get_pgcontrol_checksum(arguments->base_path); else - crc = pgFileGetCRC(file_fullpath, - arguments->backup_version <= 20021 || - arguments->backup_version >= 20025, - false); +#if PG_VERSION_NUM >= 120000 + { + Assert(arguments->backup_version >= 20025); + crc = pgFileGetCRC32C(file_fullpath, false); + } +#else /* PG_VERSION_NUM < 120000 */ + if (arguments->backup_version <= 20021 || arguments->backup_version >= 20025) + crc = pgFileGetCRC32C(file_fullpath, false); + else + crc = pgFileGetCRC32(file_fullpath, false); +#endif /* PG_VERSION_NUM < 120000 */ + if (crc != file->crc) { elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", @@ -720,8 +728,6 @@ validate_tablespace_map(pgBackup *backup, bool no_validate) pgFile **tablespace_map = NULL; pg_crc32 crc; parray *files = get_backup_filelist(backup, true); - bool use_crc32c = parse_program_version(backup->program_version) <= 20021 || - parse_program_version(backup->program_version) >= 20025; parray_qsort(files, pgFileCompareRelPathWithExternal); join_path_components(map_path, backup->database_dir, PG_TABLESPACE_MAP_FILE); @@ -746,7 +752,16 @@ validate_tablespace_map(pgBackup *backup, bool no_validate) /* check tablespace map checksumms */ if (!no_validate) { - crc = pgFileGetCRC(map_path, use_crc32c, false); +#if PG_VERSION_NUM >= 120000 + Assert(parse_program_version(backup->program_version) >= 20025); + crc = pgFileGetCRC32C(map_path, false); +#else /* PG_VERSION_NUM < 120000 */ + if (parse_program_version(backup->program_version) <= 20021 + || parse_program_version(backup->program_version) >= 20025) + crc = pgFileGetCRC32C(map_path, false); + else + crc = pgFileGetCRC32(map_path, false); +#endif /* PG_VERSION_NUM < 120000 */ if ((*tablespace_map)->crc != crc) elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, "