diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 8ee22ca65..9ccecd620 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -1141,11 +1141,10 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, T /* in util.c */ extern TimeLineID get_current_timeline(PGconn *conn); -extern TimeLineID get_current_timeline_from_control(fio_location location, const char *pgdata_path, bool safe); +extern TimeLineID get_current_timeline_from_control(fio_location location, const char *pgdata_path); extern XLogRecPtr get_checkpoint_location(PGconn *conn); extern uint64 get_system_identifier(fio_location location, const char *pgdata_path, bool safe); extern uint64 get_remote_system_identifier(PGconn *conn); -extern uint32 get_data_checksum_version(bool safe); extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); extern uint32 get_xlog_seg_size(const char *pgdata_path); extern void get_redo(fio_location location, const char *pgdata_path, RedoParams *redo); diff --git a/src/util.c b/src/util.c index a54e34e08..782c859aa 100644 --- a/src/util.c +++ b/src/util.c @@ -96,24 +96,6 @@ checkControlFile(ControlFileData *ControlFile) "the PostgreSQL installation would be incompatible with this data directory."); } -/* - * Verify control file contents in the buffer src, and copy it to *ControlFile. - */ -static void -digestControlFile(ControlFileData *ControlFile, char *src, size_t size) -{ - int ControlFileSize = PG_CONTROL_FILE_SIZE; - - if (size != ControlFileSize) - elog(ERROR, "unexpected control file size %d, expected %d", - (int) size, ControlFileSize); - - memcpy(ControlFile, src, sizeof(ControlFileData)); - - /* Additional checks on control file */ - checkControlFile(ControlFile); -} - /* * Write ControlFile to pg_control */ @@ -164,7 +146,7 @@ get_current_timeline(PGconn *conn) if (PQresultStatus(res) == PGRES_TUPLES_OK) val = PQgetvalue(res, 0, 0); else - return get_current_timeline_from_control(FIO_DB_HOST, instance_config.pgdata, false); + return get_current_timeline_from_control(FIO_DB_HOST, instance_config.pgdata); if (!parse_uint32(val, &tli, 0)) { @@ -172,28 +154,62 @@ get_current_timeline(PGconn *conn) elog(WARNING, "Invalid value of timeline_id %s", val); /* TODO 3.0 remove it and just error out */ - return get_current_timeline_from_control(FIO_DB_HOST, instance_config.pgdata, false); + return get_current_timeline_from_control(FIO_DB_HOST, instance_config.pgdata); } return tli; } +static err_i +get_control_file(fio_location location, path_t pgdata_path, path_t file, + ControlFileData *control, bool safe) +{ + pioDrive_i drive; + char fullpath[MAXPGPATH]; + ft_bytes_t bytes; + err_i err; + + fobj_reset_err(&err); + + join_path_components(fullpath, pgdata_path, file); + + drive = pioDriveForLocation(location); + bytes = $i(pioReadFile, drive, .path = fullpath, .err = &err); + if ($haserr(err) && safe) + { + ft_logerr(FT_WARNING, $errmsg(err), "Could not get control file"); + memset(control, 0, sizeof(ControlFileData)); + return $noerr(); + } + if ($haserr(err)) + return $err(RT, "Could not get control file: {cause}", + cause(err.self)); + + if (bytes.len != PG_CONTROL_FILE_SIZE) + return $err(RT, "unexpected control file size: {size}, expected {wantedSz}", + size(bytes.len), wantedSz(PG_CONTROL_FILE_SIZE)); + + memcpy(control, bytes.ptr, sizeof(ControlFileData)); + ft_bytes_free(&bytes); + + /* Additional checks on control file */ + checkControlFile(control); + + return $noerr(); +} + /* Get timeline from pg_control file */ TimeLineID -get_current_timeline_from_control(fio_location location, const char *pgdata_path, bool safe) +get_current_timeline_from_control(fio_location location, const char *pgdata_path) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; + err_i err; - /* First fetch file... */ - buffer = slurpFile(location, pgdata_path, XLOG_CONTROL_FILE, - &size, safe); - if (safe && buffer == NULL) - return 0; - - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); + err = get_control_file(location, pgdata_path, XLOG_CONTROL_FILE, + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Getting current timeline"); return ControlFile.checkPointCopy.ThisTimeLineID; } @@ -223,16 +239,14 @@ get_checkpoint_location(PGconn *conn) uint64 get_system_identifier(fio_location location, const char *pgdata_path, bool safe) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; + err_i err; - /* First fetch file... */ - buffer = slurpFile(location, pgdata_path, XLOG_CONTROL_FILE, &size, safe); - if (safe && buffer == NULL) - return 0; - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); + err = get_control_file(location, pgdata_path, XLOG_CONTROL_FILE, + &ControlFile, safe); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Getting system identifier"); return ControlFile.system_identifier; } @@ -262,14 +276,14 @@ uint32 get_xlog_seg_size(const char *pgdata_path) { #if PG_VERSION_NUM >= 110000 + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; + err_i err; - /* First fetch file... */ - buffer = slurpFile(FIO_DB_HOST, pgdata_path, XLOG_CONTROL_FILE, &size, false); - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); + err = get_control_file(FIO_DB_HOST, pgdata_path, XLOG_CONTROL_FILE, + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Trying to fetch segment size"); return ControlFile.xlog_seg_size; #else @@ -277,36 +291,17 @@ get_xlog_seg_size(const char *pgdata_path) #endif } -uint32 -get_data_checksum_version(bool safe) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(FIO_DB_HOST, instance_config.pgdata, XLOG_CONTROL_FILE, - &size, safe); - if (buffer == NULL) - return 0; - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.data_checksum_version; -} - pg_crc32c get_pgcontrol_checksum(const char *pgdata_path) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; + err_i err; - /* First fetch file... */ - buffer = slurpFile(FIO_BACKUP_HOST, pgdata_path, XLOG_CONTROL_FILE, &size, false); - elog(WARNING, "checking %s", pgdata_path); - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); + err = get_control_file(FIO_BACKUP_HOST, pgdata_path, XLOG_CONTROL_FILE, + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Getting pgcontrol checksum"); return ControlFile.crc; } @@ -314,15 +309,14 @@ get_pgcontrol_checksum(const char *pgdata_path) void get_redo(fio_location location, const char *pgdata_path, RedoParams *redo) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(location, pgdata_path, XLOG_CONTROL_FILE, &size, false); + err_i err; - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); + err = get_control_file(location, pgdata_path, XLOG_CONTROL_FILE, + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Fetching redo lsn"); redo->lsn = ControlFile.checkPointCopy.redo; redo->tli = ControlFile.checkPointCopy.ThisTimeLineID; @@ -352,14 +346,15 @@ void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; - char fullpath[MAXPGPATH]; + char fullpath[MAXPGPATH]; + err_i err; - /* First fetch file content */ - buffer = slurpFile(FIO_DB_HOST, instance_config.pgdata, XLOG_CONTROL_FILE, &size, false); - digestControlFile(&ControlFile, buffer, size); + err = get_control_file(FIO_DB_HOST, instance_config.pgdata, XLOG_CONTROL_FILE, + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Set min recovery point"); elog(LOG, "Current minRecPoint %X/%X", (uint32) (ControlFile.minRecoveryPoint >> 32), @@ -383,8 +378,6 @@ set_min_recovery_point(pgFile *file, const char *backup_path, /* Update pg_control checksum in backup_list */ file->crc = ControlFile.crc; - - pg_free(buffer); } /* @@ -394,22 +387,21 @@ void copy_pgcontrol_file(fio_location from_location, const char *from_fullpath, fio_location to_location, const char *to_fullpath, pgFile *file) { + FOBJ_FUNC_ARP(); ControlFileData ControlFile; - char *buffer; - size_t size; + err_i err; - buffer = slurpFile(from_location, from_fullpath, "", &size, false); - - digestControlFile(&ControlFile, buffer, size); + err = get_control_file(from_location, from_fullpath, "", + &ControlFile, false); + if ($haserr(err)) + ft_logerr(FT_FATAL, $errmsg(err), "Fetching control file"); file->crc = ControlFile.crc; - file->read_size = (int64_t)size; - file->write_size = (int64_t)size; - file->uncompressed_size = (int64_t)size; + file->read_size = PG_CONTROL_FILE_SIZE; + file->write_size = PG_CONTROL_FILE_SIZE; + file->uncompressed_size = PG_CONTROL_FILE_SIZE; writeControlFile(to_location, to_fullpath, &ControlFile); - - pg_free(buffer); } /* diff --git a/src/utils/file.c b/src/utils/file.c index f51773e70..ef07ffb4c 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -3605,6 +3605,7 @@ fio_communicate(int in, int out) fio_header hdr; pioDrive_i drive; pio_stat_t st; + ft_bytes_t bytes; int rc; int tmp_fd; pg_crc32 crc; @@ -3725,6 +3726,27 @@ fio_communicate(int in, int out) hdr.size = 0; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; + case FIO_READ_FILE_AT_ONCE: + bytes = $i(pioReadFile, drive, .path = buf, + .binary = hdr.arg != 0, .err = &err); + if ($haserr(err)) + { + const char *msg = $errmsg(err); + hdr.arg = getErrno(err); + hdr.size = strlen(msg) + 1; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(out, msg, hdr.size), hdr.size); + } + else + { + hdr.arg = 0; + hdr.size = bytes.len; + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + if (bytes.len > 0) + IO_CHECK(fio_write_all(out, bytes.ptr, bytes.len), bytes.len); + } + ft_bytes_free(&bytes); + break; case FIO_ACCESS: /* Check presence of file with specified name */ hdr.size = 0; hdr.arg = access(buf, hdr.arg) < 0 ? errno : 0; @@ -4163,6 +4185,89 @@ pioLocalDrive_pioRemoveDir(VSelf, const char *root, bool root_as_well) { parray_free(files); } +static ft_bytes_t +pioLocalDrive_pioReadFile(VSelf, path_t path, bool binary, err_i* err) +{ + FOBJ_FUNC_ARP(); + Self(pioLocalDrive); + pioFile_i fl; + pio_stat_t st; + ft_bytes_t res = ft_bytes(NULL, 0); + size_t amount; + + fobj_reset_err(err); + + st = $(pioStat, self, .path = path, .follow_symlink = true, .err = err); + if ($haserr(*err)) + { + $iresult(*err); + return res; + } + if (st.pst_kind != PIO_KIND_REGULAR) + { + *err = $err(RT, "File {path:q} is not regular: {kind}", path(path), + kind(pio_file_kind2str(st.pst_kind, path))); + $iresult(*err); + return res; + } + + /* forbid too large file because of remote protocol */ + if (st.pst_size >= INT32_MAX) + { + *err = $err(RT, "File {path:q} is too large: {size}", path(path), + size(st.pst_size), errNo(ENOMEM)); + $iresult(*err); + return res; + } + if (binary) + res = ft_bytes_alloc(st.pst_size); + else + { + res = ft_bytes_alloc(st.pst_size + 1); + res.len -= 1; + } + + /* + * rely on "local file is read whole at once always". + * Is it true? + */ + fl = $(pioOpen, self, .path = path, .flags = O_RDONLY | (binary ? PG_BINARY : 0), + .err = err); + if ($haserr(*err)) + { + $iresult(*err); + return res; + } + + amount = pioReadFull($reduce(pioRead, fl), res, err); + if ($haserr(*err)) + { + ft_bytes_free(&res); + $iresult(*err); + return res; + } + + if (amount != st.pst_size) + { + ft_bytes_free(&res); + *err = $err(RT, "File {path:q} is truncated while reading", + path(path), errNo(EBUSY)); + $iresult(*err); + return res; + } + + if (binary) + res.len = amount; + else + { + res.len = amount + 1; + res.ptr[amount] = 0; + } + + $i(pioClose, fl); + return res; +} + /* LOCAL FILE */ static void pioLocalFile_fobjDispose(VSelf) @@ -4576,6 +4681,43 @@ pioRemoteDrive_pioRemoveDir(VSelf, const char *root, bool root_as_well) { elog(ERROR, "couldn't remove remote dir"); } +static ft_bytes_t +pioRemoteDrive_pioReadFile(VSelf, path_t path, bool binary, err_i* err) +{ + FOBJ_FUNC_ARP(); + Self(pioLocalDrive); + ft_bytes_t res; + + fobj_reset_err(err); + + fio_header hdr = { + .cop = FIO_READ_FILE_AT_ONCE, + .handle = -1, + .size = strlen(path)+1, + .arg = binary, + }; + + IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); + + /* get the response */ + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_READ_FILE_AT_ONCE); + + res = ft_bytes_alloc(hdr.size); + IO_CHECK(fio_read_all(fio_stdin, res.ptr, hdr.size), hdr.size); + + if (hdr.arg != 0) + { + *err = $syserr((int)hdr.arg, "Could not read remote file {path:q}: {causeStr}", + path(path), causeStr(res.ptr)); + $iresult(*err); + ft_bytes_free(&res); + } + + return res; +} + /* REMOTE FILE */ static err_i @@ -5540,6 +5682,25 @@ pioCopyWithFilters(pioWriteFlush_i dest, pioRead_i src, return $noerr(); } +size_t +pioReadFull(pioRead_i src, ft_bytes_t bytes, err_i* err) +{ + ft_bytes_t b; + size_t r; + fobj_reset_err(err); + + b = bytes; + while (b.len) + { + r = $i(pioRead, src, b, err); + Assert(r <= b.len); + ft_bytes_consume(&b, r); + if ($haserr(*err)) + break; + } + return bytes.len - b.len; +} + fobj_klass_handle(pioFile); fobj_klass_handle(pioLocalDrive); fobj_klass_handle(pioRemoteDrive); diff --git a/src/utils/file.h b/src/utils/file.h index 5fb0bb3da..440be0210 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -69,6 +69,7 @@ typedef enum FIO_SEND_FILE_CONTENT, FIO_PAGE_ZERO, FIO_FILES_ARE_SAME, + FIO_READ_FILE_AT_ONCE, } fio_operations; typedef struct @@ -228,6 +229,8 @@ typedef const char* path_t; fobj_error_cstr_key(remotemsg); fobj_error_int_key(writtenSz); fobj_error_int_key(wantedSz); +fobj_error_int_key(size); +fobj_error_cstr_key(kind); #ifdef HAVE_LIBZ fobj_error_kind(GZ); @@ -276,6 +279,9 @@ fobj_iface(pioReadCloser); (bool, handle_tablespaces), (bool, symlink_and_hidden), \ (bool, backup_logs), (bool, skip_hidden), (int, external_dir_num) #define mth__pioRemoveDir void, (const char *, root), (bool, root_as_well) +#define mth__pioReadFile ft_bytes_t, (path_t, path), (bool, binary), \ + (err_i *, err) +#define mth__pioReadFile__optional() (binary, true) fobj_method(pioOpen); fobj_method(pioStat); @@ -288,11 +294,12 @@ fobj_method(pioMakeDir); fobj_method(pioFilesAreSame); fobj_method(pioListDir); fobj_method(pioRemoveDir); +fobj_method(pioReadFile); #define iface__pioDrive mth(pioOpen, pioStat, pioRemove, pioRename), \ mth(pioExists, pioGetCRC32, pioIsRemote), \ mth(pioMakeDir, pioListDir, pioRemoveDir), \ - mth(pioFilesAreSame) + mth(pioFilesAreSame), mth(pioReadFile) fobj_iface(pioDrive); extern pioDrive_i pioDriveForLocation(fio_location location); @@ -341,4 +348,6 @@ extern err_i pioCopyWithFilters(pioWriteFlush_i dest, pioRead_i src, pioFilter_i _fltrs_[] = {__VA_ARGS__}; \ pioCopyWithFilters((dest), (src), _fltrs_, ft_arrsz(_fltrs_), NULL); \ }) + +extern size_t pioReadFull(pioRead_i src, ft_bytes_t bytes, err_i* err); #endif