diff --git a/src/archive.c b/src/archive.c index 0f32d9345..01ea15a1f 100644 --- a/src/archive.c +++ b/src/archive.c @@ -3,7 +3,7 @@ * archive.c: - pg_probackup specific archive commands for archive backups. * * - * Portions Copyright (c) 2018-2021, Postgres Professional + * Portions Copyright (c) 2018-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -364,7 +364,7 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir, elog(VERBOSE, "Rename \"%s\" to \"%s\"", wal_file_ready, wal_file_done); /* do not error out, if rename failed */ - if (fio_rename(wal_file_ready, wal_file_done, FIO_DB_HOST) < 0) + if (fio_rename(FIO_DB_HOST, wal_file_ready, wal_file_done) < 0) elog(WARNING, "Cannot rename ready file \"%s\" to \"%s\": %s", wal_file_ready, wal_file_done, strerror(errno)); } @@ -418,7 +418,7 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d snprintf(to_fullpath_part, sizeof(to_fullpath_part), "%s.part", to_fullpath); /* Grab lock by creating temp file in exclusive mode */ - out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); + out = fio_open(FIO_BACKUP_HOST, to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY); if (out < 0) { if (errno != EEXIST) @@ -444,12 +444,12 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d while (partial_try_count < archive_timeout) { - if (fio_stat(to_fullpath_part, &st, false, FIO_BACKUP_HOST) < 0) + if (fio_stat(FIO_BACKUP_HOST, to_fullpath_part, &st, false) < 0) { if (errno == ENOENT) { //part file is gone, lets try to grab it - out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); + out = fio_open(FIO_BACKUP_HOST, to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY); if (out < 0) { if (errno != EEXIST) @@ -497,9 +497,10 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d /* Partial segment is considered stale, so reuse it */ elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_part); - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_part, false) != 0) + elog(ERROR, "Cannot remove stale temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); - out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); + out = fio_open(FIO_BACKUP_HOST, to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY); if (out < 0) elog(ERROR, "Cannot open temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); } @@ -512,8 +513,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d pg_crc32 crc32_src; pg_crc32 crc32_dst; - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); - crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false); + crc32_src = fio_get_crc32(FIO_DB_HOST, from_fullpath, false); + crc32_dst = fio_get_crc32(FIO_BACKUP_HOST, to_fullpath, false); if (crc32_src == crc32_dst) { @@ -522,7 +523,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d /* cleanup */ fclose(in); fio_close(out); - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_part, false) != 0) + elog(WARNING, "Cannot remove temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); return 1; } else @@ -535,7 +537,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d /* Overwriting is forbidden, * so we must unlink partial file and exit with error. */ - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_part, false) != 0) + elog(WARNING, "Cannot remove temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); elog(ERROR, "WAL file already exists in archive with " "different checksum: \"%s\"", to_fullpath); } @@ -552,16 +555,20 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d if (ferror(in)) { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + int save_errno = errno; + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_part, false) != 0) + elog(WARNING, "Cannot remove temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); elog(ERROR, "Cannot read source file \"%s\": %s", - from_fullpath, strerror(errno)); + from_fullpath, strerror(save_errno)); } if (read_len > 0 && fio_write_async(out, buf, read_len) != read_len) { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + int save_errno = errno; + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_part, false) != 0) + elog(WARNING, "Cannot cleanup temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); elog(ERROR, "Cannot write to destination temp file \"%s\": %s", - to_fullpath_part, strerror(errno)); + to_fullpath_part, strerror(save_errno)); } if (feof(in)) @@ -574,7 +581,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d /* Writing is asynchronous in case of push in remote mode, so check agent status */ if (fio_check_error_fd(out, &errmsg)) { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_part, false) != 0) + elog(WARNING, "Cannot cleanup temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); elog(ERROR, "Cannot write to the remote file \"%s\": %s", to_fullpath_part, errmsg); } @@ -582,15 +590,17 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d /* close temp file */ if (fio_close(out) != 0) { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + int save_errno = errno; + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_part, false) != 0) + elog(WARNING, "Cannot cleanup temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); elog(ERROR, "Cannot close temp WAL file \"%s\": %s", - to_fullpath_part, strerror(errno)); + to_fullpath_part, strerror(save_errno)); } /* sync temp file to disk */ if (!no_sync) { - if (fio_sync(to_fullpath_part, FIO_BACKUP_HOST) != 0) + if (fio_sync(FIO_BACKUP_HOST, to_fullpath_part) != 0) elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath_part, strerror(errno)); } @@ -600,11 +610,13 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); /* Rename temp file to destination file */ - if (fio_rename(to_fullpath_part, to_fullpath, FIO_BACKUP_HOST) < 0) + if (fio_rename(FIO_BACKUP_HOST, to_fullpath_part, to_fullpath) < 0) { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); + int save_errno = errno; + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_part, false) != 0) + elog(WARNING, "Cannot cleanup temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - to_fullpath_part, to_fullpath, strerror(errno)); + to_fullpath_part, to_fullpath, strerror(save_errno)); } pg_free(buf); @@ -663,7 +675,7 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, setvbuf(in, NULL, _IONBF, BUFSIZ); /* Grab lock by creating temp file in exclusive mode */ - out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); + out = fio_gzopen(FIO_BACKUP_HOST, to_fullpath_gz_part, PG_BINARY_W, compress_level); if (out == NULL) { if (errno != EEXIST) @@ -689,12 +701,12 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, while (partial_try_count < archive_timeout) { - if (fio_stat(to_fullpath_gz_part, &st, false, FIO_BACKUP_HOST) < 0) + if (fio_stat(FIO_BACKUP_HOST, to_fullpath_gz_part, &st, false) < 0) { if (errno == ENOENT) { //part file is gone, lets try to grab it - out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); + out = fio_gzopen(FIO_BACKUP_HOST, to_fullpath_gz_part, PG_BINARY_W, compress_level); if (out == NULL) { if (errno != EEXIST) @@ -743,9 +755,10 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* Partial segment is considered stale, so reuse it */ elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_gz_part); - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_gz_part, false) != 0) + elog(ERROR, "Cannot remove stale compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); - out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); + out = fio_gzopen(FIO_BACKUP_HOST, to_fullpath_gz_part, PG_BINARY_W, compress_level); if (out == NULL) elog(ERROR, "Cannot open temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); @@ -761,8 +774,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, pg_crc32 crc32_dst; /* TODO: what if one of them goes missing? */ - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); - crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true); + crc32_src = fio_get_crc32(FIO_DB_HOST, from_fullpath, false); + crc32_dst = fio_get_crc32(FIO_BACKUP_HOST, to_fullpath_gz, true); if (crc32_src == crc32_dst) { @@ -771,7 +784,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* cleanup */ fclose(in); fio_gzclose(out); - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_gz_part, false) != 0) + elog(WARNING, "Cannot remove compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); return 1; } else @@ -784,7 +798,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* Overwriting is forbidden, * so we must unlink partial file and exit with error. */ - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_gz_part, false) != 0) + elog(WARNING, "Cannot remove compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); elog(ERROR, "WAL file already exists in archive with " "different checksum: \"%s\"", to_fullpath_gz); } @@ -801,16 +816,20 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, if (ferror(in)) { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + int save_errno = errno; + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_gz_part, false) != 0) + elog(WARNING, "Cannot remove compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); elog(ERROR, "Cannot read from source file \"%s\": %s", - from_fullpath, strerror(errno)); + from_fullpath, strerror(save_errno)); } if (read_len > 0 && fio_gzwrite(out, buf, read_len) != read_len) { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + int save_errno = errno; + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_gz_part, false) != 0) + elog(WARNING, "Cannot cleanup compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); elog(ERROR, "Cannot write to compressed temp WAL file \"%s\": %s", - to_fullpath_gz_part, get_gz_error(out, errno)); + to_fullpath_gz_part, get_gz_error(out, save_errno)); } if (feof(in)) @@ -823,7 +842,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* Writing is asynchronous in case of push in remote mode, so check agent status */ if (fio_check_error_fd_gz(out, &errmsg)) { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_gz_part, false) != 0) + elog(WARNING, "Cannot cleanup remote compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); elog(ERROR, "Cannot write to the remote compressed file \"%s\": %s", to_fullpath_gz_part, errmsg); } @@ -831,15 +851,17 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, /* close temp file, TODO: make it synchronous */ if (fio_gzclose(out) != 0) { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + int save_errno = errno; + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_gz_part, false) != 0) + elog(WARNING, "Cannot cleanup compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); elog(ERROR, "Cannot close compressed temp WAL file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); + to_fullpath_gz_part, strerror(save_errno)); } /* sync temp file to disk */ if (!no_sync) { - if (fio_sync(to_fullpath_gz_part, FIO_BACKUP_HOST) != 0) + if (fio_sync(FIO_BACKUP_HOST, to_fullpath_gz_part) != 0) elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); } @@ -850,11 +872,13 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir, //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); /* Rename temp file to destination file */ - if (fio_rename(to_fullpath_gz_part, to_fullpath_gz, FIO_BACKUP_HOST) < 0) + if (fio_rename(FIO_BACKUP_HOST, to_fullpath_gz_part, to_fullpath_gz) < 0) { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); + int save_errno = errno; + if (fio_remove(FIO_BACKUP_HOST, to_fullpath_gz_part, false) != 0) + elog(WARNING, "Cannot cleanup compressed temp WAL file \"%s\": %s", to_fullpath_gz_part, strerror(errno)); elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - to_fullpath_gz_part, to_fullpath_gz, strerror(errno)); + to_fullpath_gz_part, to_fullpath_gz, strerror(save_errno)); } pg_free(buf); @@ -889,7 +913,7 @@ get_gz_error(gzFile gzf, int errnum) //{ // struct stat st; // -// if (fio_stat(from_path, &st, true, from_location) == -1) +// if (fio_stat(from_location, from_path, &st, true) == -1) // { // if (unlink_on_error) // fio_unlink(to_path, to_location); @@ -897,7 +921,7 @@ get_gz_error(gzFile gzf, int errnum) // from_path, strerror(errno)); // } // -// if (fio_chmod(to_path, st.st_mode, to_location) == -1) +// if (fio_chmod(to_location, to_path, st.st_mode) == -1) // { // if (unlink_on_error) // fio_unlink(to_path, to_location); diff --git a/src/backup.c b/src/backup.c index c575865c4..84b503245 100644 --- a/src/backup.c +++ b/src/backup.c @@ -3,7 +3,7 @@ * backup.c: backup DB cluster, archived WAL * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -136,7 +136,8 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, #if PG_VERSION_NUM >= 90600 current.tli = get_current_timeline(backup_conn); #else - current.tli = get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); + /* PG-9.5 */ + current.tli = get_current_timeline_from_control(FIO_DB_HOST, instance_config.pgdata, false); #endif /* @@ -242,7 +243,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || !current.stream) { /* Check that archive_dir can be reached */ - if (fio_access(instanceState->instance_wal_subdir_path, F_OK, FIO_BACKUP_HOST) != 0) + if (fio_access(FIO_BACKUP_HOST, instanceState->instance_wal_subdir_path, F_OK) != 0) elog(ERROR, "WAL archive directory is not accessible \"%s\": %s", instanceState->instance_wal_subdir_path, strerror(errno)); @@ -260,7 +261,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, char stream_xlog_path[MAXPGPATH]; join_path_components(stream_xlog_path, current.database_dir, PG_XLOG_DIR); - fio_mkdir(stream_xlog_path, DIR_PERMISSION, FIO_BACKUP_HOST); + fio_mkdir(FIO_BACKUP_HOST, stream_xlog_path, DIR_PERMISSION, false); start_WAL_streaming(backup_conn, stream_xlog_path, &instance_config.conn_opt, current.start_lsn, current.tli, true); @@ -413,7 +414,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, join_path_components(dirpath, current.database_dir, file->rel_path); elog(VERBOSE, "Create directory '%s'", dirpath); - fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); + fio_mkdir(FIO_BACKUP_HOST, dirpath, DIR_PERMISSION, false); } } @@ -528,7 +529,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, { cleanup_header_map(&(current.hdr_map)); - if (fio_sync(current.hdr_map.path, FIO_BACKUP_HOST) != 0) + if (fio_sync(FIO_BACKUP_HOST, current.hdr_map.path) != 0) elog(ERROR, "Cannot sync file \"%s\": %s", current.hdr_map.path, strerror(errno)); } @@ -587,7 +588,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, join_path_components(to_fullpath, external_dst, file->rel_path); } - if (fio_sync(to_fullpath, FIO_BACKUP_HOST) != 0) + if (fio_sync(FIO_BACKUP_HOST, to_fullpath) != 0) elog(ERROR, "Cannot sync file \"%s\": %s", to_fullpath, strerror(errno)); } @@ -943,7 +944,7 @@ check_system_identifiers(PGconn *conn, const char *pgdata) uint64 system_id_conn; uint64 system_id_pgdata; - system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false); + system_id_pgdata = get_system_identifier(FIO_DB_HOST, pgdata, false); system_id_conn = get_remote_system_identifier(conn); /* for checkdb check only system_id_pgdata and system_id_conn */ @@ -1788,7 +1789,7 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c char full_filename[MAXPGPATH]; join_path_components(full_filename, path, filename); - fp = fio_fopen(full_filename, PG_BINARY_W, FIO_BACKUP_HOST); + fp = fio_fopen(FIO_BACKUP_HOST, full_filename, PG_BINARY_W); if (fp == NULL) elog(ERROR, "can't open %s file \"%s\": %s", error_msg_filename, full_filename, strerror(errno)); diff --git a/src/catalog.c b/src/catalog.c index b4ed8c189..c4c392d5c 100644 --- a/src/catalog.c +++ b/src/catalog.c @@ -3,7 +3,7 @@ * catalog.c: backup catalog operation * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -326,7 +326,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) * Think not to make the file protection weaker than 0600. See * comments below. */ - fd = fio_open(lock_file, O_RDWR | O_CREAT | O_EXCL, FIO_BACKUP_HOST); + fd = fio_open(FIO_BACKUP_HOST, lock_file, O_RDWR | O_CREAT | O_EXCL); if (fd >= 0) break; /* Success; exit the retry loop */ @@ -451,7 +451,7 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) * it. Need a loop because of possible race condition against other * would-be creators. */ - if (fio_unlink(lock_file, FIO_BACKUP_HOST) < 0) + if (fio_remove(FIO_BACKUP_HOST, lock_file, false) < 0) { if (errno == ENOENT) continue; /* race condition, again */ @@ -476,7 +476,8 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) int save_errno = errno; fio_close(fd); - fio_unlink(lock_file, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, lock_file, false) != 0) + elog(WARNING, "Cannot remove lock file \"%s\": %s", lock_file, strerror(errno)); /* In lax mode if we failed to grab lock because of 'out of space error', * then treat backup as locked. @@ -494,7 +495,8 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) int save_errno = errno; fio_close(fd); - fio_unlink(lock_file, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, lock_file, false) != 0) + elog(WARNING, "Cannot remove lock file \"%s\": %s", lock_file, strerror(errno)); /* In lax mode if we failed to grab lock because of 'out of space error', * then treat backup as locked. @@ -511,9 +513,10 @@ grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) { int save_errno = errno; - fio_unlink(lock_file, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, lock_file, false) != 0) + elog(WARNING, "Cannot remove lock file \"%s\": %s", lock_file, strerror(errno)); - if (!strict && errno == ENOSPC) + if (!strict && save_errno == ENOSPC) return LOCK_FAIL_ENOSPC; else elog(ERROR, "Could not close lock file \"%s\": %s", @@ -608,8 +611,9 @@ wait_shared_owners(pgBackup *backup) return 1; } - /* unlink shared lock file */ - fio_unlink(lock_file, FIO_BACKUP_HOST); + /* remove shared lock file */ + if (fio_remove(FIO_BACKUP_HOST, lock_file, true) != 0) + elog(ERROR, "Cannot remove shared lock file \"%s\": %s", lock_file, strerror(errno)); return 0; } @@ -663,7 +667,7 @@ grab_shared_lock_file(pgBackup *backup) else if (errno != ESRCH) elog(ERROR, "Failed to send signal 0 to a process %d: %s", encoded_pid, strerror(errno)); - } + } if (fp_in) { @@ -727,8 +731,10 @@ release_excl_lock_file(const char *backup_dir) /* TODO Sanity check: maybe we should check, that pid in lock file is my_pid */ - /* unlink pid file */ - fio_unlink(lock_file, FIO_BACKUP_HOST); + /* remove pid file */ + /* exclusive locks releasing multiple times -> missing_ok = true */ + if (fio_remove(FIO_BACKUP_HOST, lock_file, true) != 0) + elog(ERROR, "Cannot remove exclusive lock file \"%s\": %s", lock_file, strerror(errno)); } void @@ -792,7 +798,8 @@ release_shared_lock_file(const char *backup_dir) /* if there is no active pid left, then there is nothing to do */ if (buffer_len == 0) { - fio_unlink(lock_file, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, lock_file, false) != 0) + elog(ERROR, "Cannot remove shared lock file \"%s\": %s", lock_file, strerror(errno)); return; } @@ -846,7 +853,7 @@ IsDir(const char *dirpath, const char *entry, fio_location location) join_path_components(path, dirpath, entry); - return fio_stat(path, &st, false, location) == 0 && S_ISDIR(st.st_mode); + return fio_stat(location, path, &st, false) == 0 && S_ISDIR(st.st_mode); } /* @@ -933,7 +940,7 @@ catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id int i; /* open backup instance backups directory */ - data_dir = fio_opendir(instanceState->instance_backup_subdir_path, FIO_BACKUP_HOST); + data_dir = fio_opendir(FIO_BACKUP_HOST, instanceState->instance_backup_subdir_path); if (data_dir == NULL) { elog(WARNING, "cannot open directory \"%s\": %s", instanceState->instance_backup_subdir_path, @@ -1050,7 +1057,7 @@ get_backup_filelist(pgBackup *backup, bool strict) join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST); - fp = fio_open_stream(backup_filelist_path, FIO_BACKUP_HOST); + fp = fio_open_stream(FIO_BACKUP_HOST, backup_filelist_path); if (fp == NULL) elog(ERROR, "cannot open \"%s\": %s", backup_filelist_path, strerror(errno)); @@ -1418,7 +1425,6 @@ get_multi_timeline_parent(parray *backup_list, parray *tli_list, * It may be ok or maybe not, so it's up to the caller * to fix it or let it be. */ - void pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path) { @@ -1461,7 +1467,7 @@ pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path) char path[MAXPGPATH]; join_path_components(path, backup->root_dir, parray_get(subdirs, i)); - fio_mkdir(path, DIR_PERMISSION, FIO_BACKUP_HOST); + fio_mkdir(FIO_BACKUP_HOST, path, DIR_PERMISSION, false); } free_dir_list(subdirs); @@ -1484,8 +1490,7 @@ create_backup_dir(pgBackup *backup, const char *backup_instance_path) join_path_components(path, backup_instance_path, base36enc(backup_id)); - /* TODO: add wrapper for remote mode */ - rc = dir_create_dir(path, DIR_PERMISSION, true); + rc = fio_mkdir(FIO_BACKUP_HOST, path, DIR_PERMISSION, true); if (rc == 0) { @@ -2462,7 +2467,8 @@ write_backup(pgBackup *backup, bool strict) if (!strict && (save_errno == ENOSPC)) { fclose(fp); - fio_unlink(path_temp, FIO_BACKUP_HOST); + if (fio_remove(FIO_BACKUP_HOST, path_temp, false) != 0) + elog(elevel, "Additionally cannot remove file \"%s\": %s", path_temp, strerror(errno)); return; } } @@ -2471,7 +2477,7 @@ write_backup(pgBackup *backup, bool strict) elog(ERROR, "Cannot close control file \"%s\": %s", path_temp, strerror(errno)); - if (fio_sync(path_temp, FIO_BACKUP_HOST) < 0) + if (fio_sync(FIO_BACKUP_HOST, path_temp) < 0) elog(ERROR, "Cannot sync control file \"%s\": %s", path_temp, strerror(errno)); @@ -2671,7 +2677,7 @@ readBackupControlFile(const char *path) }; pgBackupInit(backup); - if (fio_access(path, F_OK, FIO_BACKUP_HOST) != 0) + if (fio_access(FIO_BACKUP_HOST, path, F_OK) != 0) { elog(WARNING, "Control file \"%s\" doesn't exist", path); pgBackupFree(backup); diff --git a/src/catchup.c b/src/catchup.c index 1b8f8084d..3b6a0fe6a 100644 --- a/src/catchup.c +++ b/src/catchup.c @@ -2,7 +2,7 @@ * * catchup.c: sync DB cluster * - * Copyright (c) 2021, Postgres Professional + * Copyright (c) 2021-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -48,7 +48,7 @@ catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, cons /* Get WAL segments size and system ID of source PG instance */ instance_config.xlog_seg_size = get_xlog_seg_size(source_pgdata); - instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST, false); + instance_config.system_identifier = get_system_identifier(FIO_DB_HOST, source_pgdata, false); current.start_time = time(NULL); strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); @@ -69,8 +69,9 @@ catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, cons #if PG_VERSION_NUM >= 90600 current.tli = get_current_timeline(source_conn); #else + /* PG-9.5 */ instance_config.pgdata = source_pgdata; - current.tli = get_current_timeline_from_control(source_pgdata, FIO_DB_HOST, false); + current.tli = get_current_timeline_from_control(FIO_DB_HOST, source_pgdata, false); #endif elog(INFO, "Catchup start, pg_probackup version: %s, " @@ -133,7 +134,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, if (current.backup_mode != BACKUP_MODE_FULL) { pid_t pid; - pid = fio_check_postmaster(dest_pgdata, FIO_LOCAL_HOST); + pid = fio_check_postmaster(FIO_LOCAL_HOST, dest_pgdata); if (pid == 1) /* postmaster.pid is mangled */ { char pid_filename[MAXPGPATH]; @@ -154,7 +155,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, char backup_label_filename[MAXPGPATH]; join_path_components(backup_label_filename, dest_pgdata, PG_BACKUP_LABEL_FILE); - if (fio_access(backup_label_filename, F_OK, FIO_LOCAL_HOST) == 0) + if (fio_access(FIO_LOCAL_HOST, backup_label_filename, F_OK) == 0) elog(ERROR, "Destination directory contains \"" PG_BACKUP_LABEL_FILE "\" file"); } @@ -163,7 +164,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, uint64 source_conn_id, source_id, dest_id; source_conn_id = get_remote_system_identifier(source_conn); - source_id = get_system_identifier(source_pgdata, FIO_DB_HOST, false); /* same as instance_config.system_identifier */ + source_id = get_system_identifier(FIO_DB_HOST, source_pgdata, false); /* same as instance_config.system_identifier */ if (source_conn_id != source_id) elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", @@ -171,7 +172,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, if (current.backup_mode != BACKUP_MODE_FULL) { - dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST, false); + dest_id = get_system_identifier(FIO_LOCAL_HOST, dest_pgdata, false); if (source_conn_id != dest_id) elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", source_conn_id, dest_pgdata, dest_id); @@ -202,7 +203,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, RedoParams dest_redo = { 0, InvalidXLogRecPtr, 0 }; /* fill dest_redo.lsn and dest_redo.tli */ - get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); + get_redo(FIO_LOCAL_HOST, dest_pgdata, &dest_redo); elog(VERBOSE, "source.tli = %X, dest_redo.lsn = %X/%X, dest_redo.tli = %X", current.tli, (uint32) (dest_redo.lsn >> 32), (uint32) dest_redo.lsn, dest_redo.tli); @@ -555,7 +556,7 @@ catchup_sync_destination_files(const char* pgdata_path, fio_location location, p Assert(file->external_dir_num == 0); join_path_components(fullpath, pgdata_path, file->rel_path); - if (fio_sync(fullpath, location) != 0) + if (fio_sync(location, fullpath) != 0) elog(ERROR, "Cannot sync file \"%s\": %s", fullpath, strerror(errno)); } @@ -563,7 +564,7 @@ catchup_sync_destination_files(const char* pgdata_path, fio_location location, p * sync pg_control file */ join_path_components(fullpath, pgdata_path, pg_control_file->rel_path); - if (fio_sync(fullpath, location) != 0) + if (fio_sync(location, fullpath) != 0) elog(ERROR, "Cannot sync file \"%s\": %s", fullpath, strerror(errno)); time(&end_time); @@ -656,7 +657,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, filter_filelist(dest_filelist, dest_pgdata, exclude_absolute_paths_list, exclude_relative_paths_list, "Destination"); // fill dest_redo.lsn and dest_redo.tli - get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); + get_redo(FIO_LOCAL_HOST, dest_pgdata, &dest_redo); elog(INFO, "syncLSN = %X/%X", (uint32) (dest_redo.lsn >> 32), (uint32) dest_redo.lsn); /* @@ -706,7 +707,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, /* Start stream replication */ join_path_components(dest_xlog_path, dest_pgdata, PG_XLOG_DIR); - fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST); + fio_mkdir(FIO_LOCAL_HOST, dest_xlog_path, DIR_PERMISSION, false); start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt, current.start_lsn, current.tli, false); @@ -822,7 +823,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, join_path_components(dirpath, dest_pgdata, file->rel_path); elog(VERBOSE, "Create directory '%s'", dirpath); - fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST); + fio_mkdir(FIO_LOCAL_HOST, dirpath, DIR_PERMISSION, false); } else { @@ -835,7 +836,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, char source_full_path[MAXPGPATH]; char symlink_content[MAXPGPATH]; join_path_components(source_full_path, source_pgdata, file->rel_path); - fio_readlink(source_full_path, symlink_content, sizeof(symlink_content), FIO_DB_HOST); + fio_readlink(FIO_DB_HOST, source_full_path, symlink_content, sizeof(symlink_content)); /* we checked that mapping exists in preflight_checks for local catchup */ linked_path = get_tablespace_mapping(symlink_content); elog(INFO, "Map tablespace full_path: \"%s\" old_symlink_content: \"%s\" new_symlink_content: \"%s\"\n", @@ -854,14 +855,14 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, linked_path, to_path); /* create tablespace directory */ - if (fio_mkdir(linked_path, file->mode, FIO_LOCAL_HOST) != 0) + if (fio_mkdir(FIO_LOCAL_HOST, linked_path, file->mode, false) != 0) elog(ERROR, "Could not create tablespace directory \"%s\": %s", linked_path, strerror(errno)); /* create link to linked_path */ - if (fio_symlink(linked_path, to_path, true, FIO_LOCAL_HOST) < 0) + if (fio_symlink(FIO_LOCAL_HOST, linked_path, to_path, true) < 0) elog(ERROR, "Could not create symbolic link \"%s\" -> \"%s\": %s", - linked_path, to_path, strerror(errno)); + to_path, linked_path, strerror(errno)); } } @@ -930,8 +931,10 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, char fullpath[MAXPGPATH]; join_path_components(fullpath, dest_pgdata, file->rel_path); - fio_delete(file->mode, fullpath, FIO_LOCAL_HOST); - elog(VERBOSE, "Deleted file \"%s\"", fullpath); + if (fio_remove(FIO_LOCAL_HOST, fullpath, false) == 0) + elog(VERBOSE, "Deleted file \"%s\"", fullpath); + else + elog(ERROR, "Cannot delete redundant file in destination \"%s\": %s", fullpath, strerror(errno)); /* shrink dest pgdata list */ pgFileFree(file); @@ -967,8 +970,8 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, char to_fullpath[MAXPGPATH]; join_path_components(from_fullpath, source_pgdata, source_pg_control_file->rel_path); join_path_components(to_fullpath, dest_pgdata, source_pg_control_file->rel_path); - copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, - to_fullpath, FIO_LOCAL_HOST, source_pg_control_file); + copy_pgcontrol_file(FIO_DB_HOST, from_fullpath, + FIO_LOCAL_HOST, to_fullpath, source_pg_control_file); transfered_datafiles_bytes += source_pg_control_file->size; } diff --git a/src/configure.c b/src/configure.c index 9ffe2d7a7..3871aa8b9 100644 --- a/src/configure.c +++ b/src/configure.c @@ -2,7 +2,7 @@ * * configure.c: - manage backup catalog. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -334,7 +334,7 @@ do_set_config(InstanceState *instanceState, bool missing_ok) if (fclose(fp)) elog(ERROR, "Cannot close configuration file: \"%s\"", path_temp); - if (fio_sync(path_temp, FIO_LOCAL_HOST) != 0) + if (fio_sync(FIO_LOCAL_HOST, path_temp) != 0) elog(ERROR, "Failed to sync temp configuration file \"%s\": %s", path_temp, strerror(errno)); @@ -602,7 +602,7 @@ readInstanceConfigFile(InstanceState *instanceState) init_config(instance, instanceState->instance_name); - if (fio_access(instanceState->instance_config_path, F_OK, FIO_BACKUP_HOST) != 0) + if (fio_access(FIO_BACKUP_HOST, instanceState->instance_config_path, F_OK) != 0) { elog(WARNING, "Control file \"%s\" doesn't exist", instanceState->instance_config_path); pfree(instance); diff --git a/src/data.c b/src/data.c index f02e3fd14..9b95a0308 100644 --- a/src/data.c +++ b/src/data.c @@ -3,7 +3,7 @@ * data.c: utils to parse and backup data pages * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -787,8 +787,8 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, /* special treatment for global/pg_control */ if (file->external_dir_num == 0 && strcmp(file->rel_path, XLOG_CONTROL_FILE) == 0) { - copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, - to_fullpath, FIO_BACKUP_HOST, file); + copy_pgcontrol_file(FIO_DB_HOST, from_fullpath, + FIO_BACKUP_HOST, to_fullpath, file); return; } @@ -800,7 +800,7 @@ backup_non_data_file(pgFile *file, pgFile *prev_file, file->mtime <= parent_backup_time)) { - file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false); + file->crc = fio_get_crc32(FIO_DB_HOST, from_fullpath, false); /* ...and checksum is the same... */ if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) @@ -1325,7 +1325,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, if (already_exists) { /* compare checksums of already existing file and backup file */ - pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false); + pg_crc32 file_crc = fio_get_crc32(FIO_DB_HOST, to_fullpath, false); if (file_crc == tmp_file->crc) { @@ -1522,14 +1522,14 @@ create_empty_file(fio_location from_location, const char *to_root, /* open file for write */ join_path_components(to_path, to_root, file->rel_path); - out = fio_fopen(to_path, PG_BINARY_W, to_location); + out = fio_fopen(to_location, to_path, PG_BINARY_W); if (out == NULL) elog(ERROR, "Cannot open destination file \"%s\": %s", to_path, strerror(errno)); /* update file permission */ - if (fio_chmod(to_path, file->mode, to_location) == -1) + if (fio_chmod(to_location, to_path, file->mode) == -1) elog(ERROR, "Cannot change mode of \"%s\": %s", to_path, strerror(errno)); @@ -2268,7 +2268,7 @@ copy_pages(const char *to_fullpath, const char *from_fullpath, setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); } - out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST); + out = fio_fopen(FIO_BACKUP_HOST, to_fullpath, PG_BINARY_R "+"); if (out == NULL) elog(ERROR, "Cannot open destination file \"%s\": %s", to_fullpath, strerror(errno)); diff --git a/src/delete.c b/src/delete.c index 6c70ff81e..867f6ba11 100644 --- a/src/delete.c +++ b/src/delete.c @@ -3,7 +3,7 @@ * delete.c: delete backup files. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -782,7 +782,8 @@ delete_backup_files(pgBackup *backup) elog(INFO, "Progress: (%zd/%zd). Delete file \"%s\"", i + 1, num_files, full_path); - pgFileDelete(file->mode, full_path); + if (fio_remove(FIO_BACKUP_HOST, full_path, false) != 0) + elog(ERROR, "Cannot remove file or directory \"%s\": %s", full_path, strerror(errno)); } parray_walk(files, pgFileFree); @@ -948,13 +949,11 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli continue; } - /* unlink segment */ - if (fio_unlink(wal_fullpath, FIO_BACKUP_HOST) < 0) + /* remove segment, missing file is not considered as error condition */ + if (fio_remove(FIO_BACKUP_HOST, wal_fullpath, true) < 0) { - /* Missing file is not considered as error condition */ - if (errno != ENOENT) - elog(ERROR, "Could not remove file \"%s\": %s", - wal_fullpath, strerror(errno)); + elog(ERROR, "Could not remove file \"%s\": %s", + wal_fullpath, strerror(errno)); } else { diff --git a/src/dir.c b/src/dir.c index 4ebe0939b..ab678dac7 100644 --- a/src/dir.c +++ b/src/dir.c @@ -3,7 +3,7 @@ * dir.c: directory operation utility. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -135,36 +135,6 @@ static TablespaceList tablespace_dirs = {NULL, NULL}; /* Extra directories mapping */ static TablespaceList external_remap_list = {NULL, NULL}; -/* - * Create directory, also create parent directories if necessary. - * In strict mode treat already existing directory as error. - * Return values: - * 0 - ok - * -1 - error (check errno) - */ -int -dir_create_dir(const char *dir, mode_t mode, bool strict) -{ - char parent[MAXPGPATH]; - - strncpy(parent, dir, MAXPGPATH); - get_parent_directory(parent); - - /* Create parent first */ - if (access(parent, F_OK) == -1) - dir_create_dir(parent, mode, false); - - /* Create directory */ - if (mkdir(dir, mode) == -1) - { - if (errno == EEXIST && !strict) /* already exist */ - return 0; - return -1; - } - - return 0; -} - pgFile * pgFileNew(const char *path, const char *rel_path, bool follow_symlink, int external_dir_num, fio_location location) @@ -173,7 +143,7 @@ pgFileNew(const char *path, const char *rel_path, bool follow_symlink, pgFile *file; /* stat the file */ - if (fio_stat(path, &st, follow_symlink, location) < 0) + if (fio_stat(location, path, &st, follow_symlink) < 0) { /* file not found is not an error case */ if (errno == ENOENT) @@ -226,38 +196,6 @@ pgFileInit(const char *rel_path) return file; } -/* - * Delete file pointed by the pgFile. - * If the pgFile points directory, the directory must be empty. - */ -void -pgFileDelete(mode_t mode, const char *full_path) -{ - if (S_ISDIR(mode)) - { - if (rmdir(full_path) == -1) - { - if (errno == ENOENT) - return; - else if (errno == ENOTDIR) /* could be symbolic link */ - goto delete_file; - - elog(ERROR, "Cannot remove directory \"%s\": %s", - full_path, strerror(errno)); - } - return; - } - -delete_file: - if (remove(full_path) == -1) - { - if (errno == ENOENT) - return; - elog(ERROR, "Cannot remove file \"%s\": %s", full_path, - strerror(errno)); - } -} - /* * Read the local file to compute its CRC. * We cannot make decision about file decompression because @@ -832,7 +770,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, elog(ERROR, "\"%s\" is not a directory", parent_dir); /* Open directory and list contents */ - dir = fio_opendir(parent_dir, location); + dir = fio_opendir(location, parent_dir); if (dir == NULL) { if (errno == ENOENT) @@ -1138,10 +1076,10 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba linked_path, to_path); /* create tablespace directory */ - fio_mkdir(linked_path, pg_tablespace_mode, location); + fio_mkdir(location, linked_path, pg_tablespace_mode, false); /* create link to linked_path */ - if (fio_symlink(linked_path, to_path, incremental, location) < 0) + if (fio_symlink(location, linked_path, to_path, incremental) < 0) elog(ERROR, "Could not create symbolic link \"%s\": %s", to_path, strerror(errno)); @@ -1156,7 +1094,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba join_path_components(to_path, data_dir, dir->rel_path); // TODO check exit code - fio_mkdir(to_path, dir->mode, location); + fio_mkdir(location, to_path, dir->mode, false); } if (extract_tablespaces) @@ -1181,7 +1119,7 @@ read_tablespace_map(parray *links, const char *backup_dir) join_path_components(db_path, backup_dir, DATABASE_DIR); join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); - fp = fio_open_stream(map_path, FIO_BACKUP_HOST); + fp = fio_open_stream(FIO_BACKUP_HOST, map_path); if (fp == NULL) elog(ERROR, "Cannot open tablespace map file \"%s\": %s", map_path, strerror(errno)); @@ -1619,7 +1557,7 @@ dir_is_empty(const char *path, fio_location location) DIR *dir; struct dirent *dir_ent; - dir = fio_opendir(path, location); + dir = fio_opendir(location, path); if (dir == NULL) { /* Directory in path doesn't exist */ @@ -1656,7 +1594,7 @@ fileExists(const char *path, fio_location location) { struct stat buf; - if (fio_stat(path, &buf, true, location) == -1 && errno == ENOENT) + if (fio_stat(location, path, &buf, true) == -1 && errno == ENOENT) return false; else if (!S_ISREG(buf.st_mode)) return false; @@ -1664,17 +1602,6 @@ fileExists(const char *path, fio_location location) return true; } -size_t -pgFileSize(const char *path) -{ - struct stat buf; - - if (stat(path, &buf) == -1) - elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); - - return buf.st_size; -} - /* * Construct parray containing remapped external directories paths * from string like /path1:/path2 @@ -1784,7 +1711,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ join_path_components(database_dir, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, database_dir, DATABASE_MAP); - fp = fio_fopen(database_map_path, PG_BINARY_W, FIO_BACKUP_HOST); + fp = fio_fopen(FIO_BACKUP_HOST, database_map_path, PG_BINARY_W); if (fp == NULL) elog(ERROR, "Cannot open database map \"%s\": %s", database_map_path, strerror(errno)); @@ -1792,9 +1719,11 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_ print_database_map(fp, database_map); if (fio_fflush(fp) || fio_fclose(fp)) { - fio_unlink(database_map_path, FIO_BACKUP_HOST); + int save_errno = errno; + if (fio_remove(FIO_BACKUP_HOST, database_map_path, false) != 0) + elog(WARNING, "Cannot cleanup database map \"%s\": %s", database_map_path, strerror(errno)); elog(ERROR, "Cannot write database map \"%s\": %s", - database_map_path, strerror(errno)); + database_map_path, strerror(save_errno)); } /* Add metadata to backup_content.control */ @@ -1822,7 +1751,7 @@ read_database_map(pgBackup *backup) join_path_components(path, backup->root_dir, DATABASE_DIR); join_path_components(database_map_path, path, DATABASE_MAP); - fp = fio_open_stream(database_map_path, FIO_BACKUP_HOST); + fp = fio_open_stream(FIO_BACKUP_HOST, database_map_path); if (fp == NULL) { /* It is NOT ok for database_map to be missing at this point, so @@ -1888,8 +1817,10 @@ cleanup_tablespace(const char *path) join_path_components(fullpath, path, file->rel_path); - fio_delete(file->mode, fullpath, FIO_DB_HOST); - elog(VERBOSE, "Deleted file \"%s\"", fullpath); + if (fio_remove(FIO_DB_HOST, fullpath, true) == 0) + elog(VERBOSE, "Deleted file \"%s\"", fullpath); + else + elog(ERROR, "Cannot delete file or directory \"%s\": %s", fullpath, strerror(errno)); } parray_walk(files, pgFileFree); diff --git a/src/fetch.c b/src/fetch.c index bef30dac6..bbea7bffe 100644 --- a/src/fetch.c +++ b/src/fetch.c @@ -3,7 +3,7 @@ * fetch.c * Functions for fetching files from PostgreSQL data directory * - * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group * *------------------------------------------------------------------------- */ @@ -25,7 +25,7 @@ * */ char * -slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fio_location location) +slurpFile(fio_location location, const char *datadir, const char *path, size_t *filesize, bool safe) { int fd; char *buffer; @@ -35,7 +35,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fi join_path_components(fullpath, datadir, path); - if ((fd = fio_open(fullpath, O_RDONLY | PG_BINARY, location)) == -1) + if ((fd = fio_open(location, fullpath, O_RDONLY | PG_BINARY)) == -1) { if (safe) return NULL; @@ -44,7 +44,7 @@ slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fi fullpath, strerror(errno)); } - if (fio_stat(fullpath, &statbuf, true, location) < 0) + if (fio_stat(location, fullpath, &statbuf, true) < 0) { if (safe) return NULL; diff --git a/src/init.c b/src/init.c index 8773016b5..41ee2e3c9 100644 --- a/src/init.c +++ b/src/init.c @@ -33,13 +33,13 @@ do_init(CatalogState *catalogState) } /* create backup catalog root directory */ - dir_create_dir(catalogState->catalog_path, DIR_PERMISSION, false); + fio_mkdir(FIO_BACKUP_HOST, catalogState->catalog_path, DIR_PERMISSION, false); /* create backup catalog data directory */ - dir_create_dir(catalogState->backup_subdir_path, DIR_PERMISSION, false); + fio_mkdir(FIO_BACKUP_HOST, catalogState->backup_subdir_path, DIR_PERMISSION, false); /* create backup catalog wal directory */ - dir_create_dir(catalogState->wal_subdir_path, DIR_PERMISSION, false); + fio_mkdir(FIO_BACKUP_HOST, catalogState->wal_subdir_path, DIR_PERMISSION, false); elog(INFO, "Backup catalog '%s' successfully inited", catalogState->catalog_path); return 0; @@ -57,7 +57,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance) "(-D, --pgdata)"); /* Read system_identifier from PGDATA */ - instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST, false); + instance->system_identifier = get_system_identifier(FIO_DB_HOST, instance->pgdata, false); /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata); @@ -86,8 +86,8 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance) instanceState->instance_name, instanceState->instance_wal_subdir_path); /* Create directory for data files of this specific instance */ - dir_create_dir(instanceState->instance_backup_subdir_path, DIR_PERMISSION, false); - dir_create_dir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, false); + fio_mkdir(FIO_BACKUP_HOST, instanceState->instance_backup_subdir_path, DIR_PERMISSION, false); + fio_mkdir(FIO_BACKUP_HOST, instanceState->instance_wal_subdir_path, DIR_PERMISSION, false); /* * Write initial configuration file. diff --git a/src/merge.c b/src/merge.c index ff39c2510..fb04a7a6b 100644 --- a/src/merge.c +++ b/src/merge.c @@ -2,7 +2,7 @@ * * merge.c: merge FULL and incremental backups * - * Copyright (c) 2018-2019, Postgres Professional + * Copyright (c) 2018-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -646,7 +646,7 @@ merge_chain(InstanceState *instanceState, makeExternalDirPathByNum(new_container, full_external_prefix, file->external_dir_num); join_path_components(dirpath, new_container, file->rel_path); - dir_create_dir(dirpath, DIR_PERMISSION, false); + fio_mkdir(FIO_BACKUP_HOST, dirpath, DIR_PERMISSION, false); } pg_atomic_init_flag(&file->lock); @@ -714,7 +714,7 @@ merge_chain(InstanceState *instanceState, cleanup_header_map(&(full_backup->hdr_map)); /* sync new header map to disk */ - if (fio_sync(full_backup->hdr_map.path_tmp, FIO_BACKUP_HOST) != 0) + if (fio_sync(FIO_BACKUP_HOST, full_backup->hdr_map.path_tmp) != 0) elog(ERROR, "Cannot sync temp header map \"%s\": %s", full_backup->hdr_map.path_tmp, strerror(errno)); @@ -809,8 +809,10 @@ merge_chain(InstanceState *instanceState, /* We need full path, file object has relative path */ join_path_components(full_file_path, full_database_dir, full_file->rel_path); - pgFileDelete(full_file->mode, full_file_path); - elog(VERBOSE, "Deleted \"%s\"", full_file_path); + if (fio_remove(FIO_BACKUP_HOST, full_file_path, true) == 0) + elog(VERBOSE, "Deleted \"%s\"", full_file_path); + else + elog(ERROR, "Cannot delete file or directory \"%s\": %s", full_file_path, strerror(errno)); } } @@ -1143,8 +1145,10 @@ remove_dir_with_files(const char *path) join_path_components(full_path, path, file->rel_path); - pgFileDelete(file->mode, full_path); - elog(VERBOSE, "Deleted \"%s\"", full_path); + if (fio_remove(FIO_LOCAL_HOST, full_path, true) == 0) + elog(VERBOSE, "Deleted \"%s\"", full_path); + else + elog(ERROR, "Cannot delete file or directory \"%s\": %s", full_path, strerror(errno)); } /* cleanup */ @@ -1280,7 +1284,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup, return; /* sync second temp file to disk */ - if (!no_sync && fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) + if (!no_sync && fio_sync(FIO_BACKUP_HOST, to_fullpath_tmp2) != 0) elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp2, strerror(errno)); @@ -1386,7 +1390,7 @@ merge_non_data_file(parray *parent_chain, pgBackup *full_backup, to_fullpath_tmp, BACKUP_MODE_FULL, 0, false); /* sync temp file to disk */ - if (!no_sync && fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0) + if (!no_sync && fio_sync(FIO_BACKUP_HOST, to_fullpath_tmp) != 0) elog(ERROR, "Cannot sync merge temp file \"%s\": %s", to_fullpath_tmp, strerror(errno)); @@ -1450,4 +1454,4 @@ is_forward_compatible(parray *parent_chain) } return true; -} \ No newline at end of file +} diff --git a/src/parsexlog.c b/src/parsexlog.c index 7f1ca9c75..df9b96fb3 100644 --- a/src/parsexlog.c +++ b/src/parsexlog.c @@ -5,7 +5,7 @@ * * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -1040,8 +1040,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, reader_data->thread_num, reader_data->xlogpath); reader_data->xlogexists = true; - reader_data->xlogfile = fio_open(reader_data->xlogpath, - O_RDONLY | PG_BINARY, FIO_LOCAL_HOST); + reader_data->xlogfile = fio_open(FIO_LOCAL_HOST, reader_data->xlogpath, + O_RDONLY | PG_BINARY); if (reader_data->xlogfile < 0) { @@ -1059,8 +1059,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, reader_data->thread_num, reader_data->gz_xlogpath); reader_data->xlogexists = true; - reader_data->gz_xlogfile = fio_gzopen(reader_data->gz_xlogpath, - "rb", -1, FIO_LOCAL_HOST); + reader_data->gz_xlogfile = fio_gzopen(FIO_LOCAL_HOST, reader_data->gz_xlogpath, + "rb", -1); if (reader_data->gz_xlogfile == NULL) { elog(WARNING, "Thread [%d]: Could not open compressed WAL segment \"%s\": %s", @@ -1946,4 +1946,4 @@ static XLogReaderState* WalReaderAllocate(uint32 wal_seg_size, XLogReaderData *r #else return XLogReaderAllocate(&SimpleXLogPageRead, reader_data); #endif -} \ No newline at end of file +} diff --git a/src/pg_probackup.c b/src/pg_probackup.c index 49e226ace..d819e1fa7 100644 --- a/src/pg_probackup.c +++ b/src/pg_probackup.c @@ -35,7 +35,7 @@ * which includes info about pgdata directory and connection. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2021, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -516,8 +516,8 @@ main(int argc, char *argv[]) { struct stat st; - if (fio_stat(instanceState->instance_backup_subdir_path, - &st, true, FIO_BACKUP_HOST) != 0) + if (fio_stat(FIO_BACKUP_HOST, instanceState->instance_backup_subdir_path, + &st, true) != 0) { elog(WARNING, "Failed to access directory \"%s\": %s", instanceState->instance_backup_subdir_path, strerror(errno)); @@ -833,7 +833,7 @@ main(int argc, char *argv[]) if (wal_file_path == NULL) { /* 1st case */ - system_id = get_system_identifier(current_dir, FIO_DB_HOST, false); + system_id = get_system_identifier(FIO_DB_HOST, current_dir, false); join_path_components(archive_push_xlog_dir, current_dir, XLOGDIR); } else @@ -849,10 +849,10 @@ main(int argc, char *argv[]) */ char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name); join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR); - if (fio_is_same_file(stripped_wal_file_path, archive_push_xlog_dir, true, FIO_DB_HOST)) + if (fio_is_same_file(FIO_DB_HOST, stripped_wal_file_path, archive_push_xlog_dir, true)) { /* 2nd case */ - system_id = get_system_identifier(instance_config.pgdata, FIO_DB_HOST, false); + system_id = get_system_identifier(FIO_DB_HOST, instance_config.pgdata, false); /* archive_push_xlog_dir already have right value */ } else @@ -862,7 +862,7 @@ main(int argc, char *argv[]) else elog(ERROR, "Value specified to --wal_file_path is too long"); - system_id = get_system_identifier(current_dir, FIO_DB_HOST, true); + system_id = get_system_identifier(FIO_DB_HOST, current_dir, true); /* 3rd case if control file present -- i.e. system_id != 0 */ if (system_id == 0) diff --git a/src/pg_probackup.h b/src/pg_probackup.h index b202b6152..b312bd77f 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -3,7 +3,7 @@ * pg_probackup.h: Backup/Recovery manager for PostgreSQL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2021, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -341,8 +341,8 @@ typedef enum ShowFormat #define PROGRAM_VERSION "2.5.4" /* update when remote agent API or behaviour changes */ -#define AGENT_PROTOCOL_VERSION 20501 -#define AGENT_PROTOCOL_VERSION_STR "2.5.1" +#define AGENT_PROTOCOL_VERSION 20600 +#define AGENT_PROTOCOL_VERSION_STR "2.6.0" /* update only when changing storage format */ #define STORAGE_FORMAT_VERSION "2.4.4" @@ -914,11 +914,11 @@ extern void do_delete_status(InstanceState *instanceState, InstanceConfig *instance_config, const char *status); /* in fetch.c */ -extern char *slurpFile(const char *datadir, +extern char *slurpFile(fio_location location, + const char *datadir, const char *path, size_t *filesize, - bool safe, - fio_location location); + bool safe); extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); /* in help.c */ @@ -1044,7 +1044,6 @@ extern void makeExternalDirPathByNum(char *ret_path, const char *pattern_path, const int dir_num); extern bool backup_contains_external(const char *dir, parray *dirs_list); -extern int dir_create_dir(const char *path, mode_t mode, bool strict); extern bool dir_is_empty(const char *path, fio_location location); extern bool fileExists(const char *path, fio_location location); @@ -1054,9 +1053,6 @@ extern pgFile *pgFileNew(const char *path, const char *rel_path, bool follow_symlink, int external_dir_num, fio_location location); extern pgFile *pgFileInit(const char *rel_path); -extern void pgFileDelete(mode_t mode, const char *full_path); -extern void fio_pgFileDelete(pgFile *file, const char *full_path); - extern void pgFileFree(void *file); extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok); @@ -1151,19 +1147,19 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, T /* in util.c */ extern TimeLineID get_current_timeline(PGconn *conn); -extern TimeLineID get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe); +extern TimeLineID get_current_timeline_from_control(fio_location location, const char *pgdata_path, bool safe); extern XLogRecPtr get_checkpoint_location(PGconn *conn); -extern uint64 get_system_identifier(const char *pgdata_path, fio_location location, bool safe); +extern uint64 get_system_identifier(fio_location location, const char *pgdata_path, bool safe); extern uint64 get_remote_system_identifier(PGconn *conn); extern uint32 get_data_checksum_version(bool safe); extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); -extern DBState get_system_dbstate(const char *pgdata_path, fio_location location); +extern DBState get_system_dbstate(fio_location location, const char *pgdata_path); extern uint32 get_xlog_seg_size(const char *pgdata_path); -extern void get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo); +extern void get_redo(fio_location location, const char *pgdata_path, RedoParams *redo); extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn); -extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, - const char *to_fullpath, fio_location to_location, pgFile *file); +extern void copy_pgcontrol_file(fio_location from_location, const char *from_fullpath, + fio_location to_location, const char *to_fullpath, pgFile *file); extern void time2iso(char *buf, size_t len, time_t time, bool utc); extern const char *status2str(BackupStatus status); @@ -1212,8 +1208,6 @@ extern int copy_pages(const char *to_fullpath, const char *from_fullpath, BackupMode backup_mode); /* FIO */ -extern void setMyLocation(ProbackupSubcmd const subcmd); -extern void fio_delete(mode_t mode, const char *fullpath, fio_location location); extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, bool use_pagemap, BlockNumber *err_blknum, char **errormsg, @@ -1221,29 +1215,15 @@ extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pg extern int fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, bool use_pagemap, BlockNumber *err_blknum, char **errormsg); -/* return codes for fio_send_pages */ extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath, FILE* out, char **errormsg); extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, pgFile *file, char **errormsg); -extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink, - bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num); - extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict); extern void pgut_setenv(const char *key, const char *val); extern void pgut_unsetenv(const char *key); -extern PageState *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, - XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location); - -extern datapagemap_t *fio_get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno, - fio_location location); -extern pid_t fio_check_postmaster(const char *pgdata, fio_location location); - -extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg, char **errormsg); - /* return codes for fio_send_pages() and fio_send_file() */ #define SEND_OK (0) #define FILE_MISSING (-1) @@ -1254,10 +1234,6 @@ extern int32 fio_decompress(void* dst, void const* src, size_t size, int compres #define REMOTE_ERROR (-6) #define PAGE_CORRUPTION (-8) -/* Check if specified location is local for current node */ -extern bool fio_is_remote(fio_location location); -extern bool fio_is_remote_simple(fio_location location); - extern void get_header_errormsg(Page page, char **errormsg); extern void get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno); diff --git a/src/restore.c b/src/restore.c index d8d808a4e..0e92e4935 100644 --- a/src/restore.c +++ b/src/restore.c @@ -3,7 +3,7 @@ * restore.c: restore DB cluster and archived WAL. * * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -485,7 +485,7 @@ do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pg { RedoParams redo; parray *timelines = NULL; - get_redo(instance_config.pgdata, FIO_DB_HOST, &redo); + get_redo(FIO_DB_HOST, instance_config.pgdata, &redo); if (redo.checksum_version == 0) elog(ERROR, "Incremental restore in 'lsn' mode require " @@ -817,8 +817,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, elog(LOG, "Restore external directories"); for (i = 0; i < parray_num(external_dirs); i++) - fio_mkdir(parray_get(external_dirs, i), - DIR_PERMISSION, FIO_DB_HOST); + fio_mkdir(FIO_DB_HOST, parray_get(external_dirs, i), + DIR_PERMISSION, false); } /* @@ -844,7 +844,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, join_path_components(dirpath, external_path, file->rel_path); elog(VERBOSE, "Create external directory \"%s\"", dirpath); - fio_mkdir(dirpath, file->mode, FIO_DB_HOST); + fio_mkdir(FIO_DB_HOST, dirpath, file->mode, false); } } @@ -922,8 +922,10 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, join_path_components(fullpath, pgdata_path, file->rel_path); - fio_delete(file->mode, fullpath, FIO_DB_HOST); - elog(VERBOSE, "Deleted file \"%s\"", fullpath); + if (fio_remove(FIO_DB_HOST, fullpath, false) == 0) + elog(VERBOSE, "Deleted file \"%s\"", fullpath); + else + elog(ERROR, "Cannot delete redundant file \"%s\": %s", fullpath, strerror(errno)); /* shrink pgdata list */ pgFileFree(file); @@ -1064,7 +1066,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain, } /* TODO: write test for case: file to be synced is missing */ - if (fio_sync(to_fullpath, FIO_DB_HOST) != 0) + if (fio_sync(FIO_DB_HOST, to_fullpath) != 0) elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath, strerror(errno)); } @@ -1203,15 +1205,17 @@ restore_files(void *arg) { if (arguments->incremental_mode == INCR_LSN) { - lsn_map = fio_get_lsn_map(to_fullpath, arguments->dest_backup->checksum_version, + lsn_map = fio_get_lsn_map(FIO_DB_HOST, to_fullpath, + arguments->dest_backup->checksum_version, dest_file->n_blocks, arguments->shift_lsn, - dest_file->segno * RELSEG_SIZE, FIO_DB_HOST); + dest_file->segno * RELSEG_SIZE); } else if (arguments->incremental_mode == INCR_CHECKSUM) { - checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version, + checksum_map = fio_get_checksum_map(FIO_DB_HOST, to_fullpath, + arguments->dest_backup->checksum_version, dest_file->n_blocks, arguments->dest_backup->stop_lsn, - dest_file->segno * RELSEG_SIZE, FIO_DB_HOST); + dest_file->segno * RELSEG_SIZE); } } @@ -1221,20 +1225,20 @@ restore_files(void *arg) * if file do not exist */ if ((already_exists && dest_file->write_size == 0) || !already_exists) - out = fio_fopen(to_fullpath, PG_BINARY_W, FIO_DB_HOST); + out = fio_fopen(FIO_DB_HOST, to_fullpath, PG_BINARY_W); /* * If file already exists and dest size is not zero, * then open it for reading and writing. */ else - out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_DB_HOST); + out = fio_fopen(FIO_DB_HOST, to_fullpath, PG_BINARY_R "+"); if (out == NULL) elog(ERROR, "Cannot open restore target file \"%s\": %s", to_fullpath, strerror(errno)); /* update file permission */ - if (fio_chmod(to_fullpath, dest_file->mode, FIO_DB_HOST) == -1) + if (fio_chmod(FIO_DB_HOST, to_fullpath, dest_file->mode) == -1) elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, strerror(errno)); @@ -1483,12 +1487,12 @@ update_recovery_options_before_v12(InstanceState *instanceState, pgBackup *backu elog(LOG, "update recovery settings in recovery.conf"); join_path_components(path, instance_config.pgdata, "recovery.conf"); - fp = fio_fopen(path, "w", FIO_DB_HOST); + fp = fio_fopen(FIO_DB_HOST, path, "w"); if (fp == NULL) elog(ERROR, "cannot open file \"%s\": %s", path, strerror(errno)); - if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1) + if (fio_chmod(FIO_DB_HOST, path, FILE_PERMISSION) == -1) elog(ERROR, "Cannot change mode of \"%s\": %s", path, strerror(errno)); fio_fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", @@ -1540,7 +1544,7 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, join_path_components(postgres_auto_path, instance_config.pgdata, "postgresql.auto.conf"); - if (fio_stat(postgres_auto_path, &st, false, FIO_DB_HOST) < 0) + if (fio_stat(FIO_DB_HOST, postgres_auto_path, &st, false) < 0) { /* file not found is not an error case */ if (errno != ENOENT) @@ -1552,13 +1556,13 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, /* Kludge for 0-sized postgresql.auto.conf file. TODO: make something more intelligent */ if (st.st_size > 0) { - fp = fio_open_stream(postgres_auto_path, FIO_DB_HOST); + fp = fio_open_stream(FIO_DB_HOST, postgres_auto_path); if (fp == NULL) elog(ERROR, "cannot open \"%s\": %s", postgres_auto_path, strerror(errno)); } sprintf(postgres_auto_path_tmp, "%s.tmp", postgres_auto_path); - fp_tmp = fio_fopen(postgres_auto_path_tmp, "w", FIO_DB_HOST); + fp_tmp = fio_fopen(FIO_DB_HOST, postgres_auto_path_tmp, "w"); if (fp_tmp == NULL) elog(ERROR, "cannot open \"%s\": %s", postgres_auto_path_tmp, strerror(errno)); @@ -1607,16 +1611,16 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, strerror(errno)); pg_free(buf); - if (fio_rename(postgres_auto_path_tmp, postgres_auto_path, FIO_DB_HOST) < 0) + if (fio_rename(FIO_DB_HOST, postgres_auto_path_tmp, postgres_auto_path) < 0) elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", postgres_auto_path_tmp, postgres_auto_path, strerror(errno)); - if (fio_chmod(postgres_auto_path, FILE_PERMISSION, FIO_DB_HOST) == -1) + if (fio_chmod(FIO_DB_HOST, postgres_auto_path, FILE_PERMISSION) == -1) elog(ERROR, "Cannot change mode of \"%s\": %s", postgres_auto_path, strerror(errno)); if (params) { - fp = fio_fopen(postgres_auto_path, "a", FIO_DB_HOST); + fp = fio_fopen(FIO_DB_HOST, postgres_auto_path, "a"); if (fp == NULL) elog(ERROR, "cannot open file \"%s\": %s", postgres_auto_path, strerror(errno)); @@ -1650,7 +1654,7 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, elog(LOG, "creating recovery.signal file"); join_path_components(path, instance_config.pgdata, "recovery.signal"); - fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); + fp = fio_fopen(FIO_DB_HOST, path, PG_BINARY_W); if (fp == NULL) elog(ERROR, "cannot open file \"%s\": %s", path, strerror(errno)); @@ -1666,7 +1670,7 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup, elog(LOG, "creating standby.signal file"); join_path_components(path, instance_config.pgdata, "standby.signal"); - fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); + fp = fio_fopen(FIO_DB_HOST, path, PG_BINARY_W); if (fp == NULL) elog(ERROR, "cannot open file \"%s\": %s", path, strerror(errno)); @@ -2158,7 +2162,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, char backup_label[MAXPGPATH]; /* check postmaster pid */ - pid = fio_check_postmaster(pgdata, FIO_DB_HOST); + pid = fio_check_postmaster(FIO_DB_HOST, pgdata); if (pid == 1) /* postmaster.pid is mangled */ { @@ -2190,7 +2194,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, */ elog(INFO, "Trying to read pg_control file in destination directory"); - system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false); + system_id_pgdata = get_system_identifier(FIO_DB_HOST, pgdata, false); if (system_id_pgdata == instance_config.system_identifier) system_id_match = true; @@ -2206,7 +2210,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier, if (incremental_mode == INCR_LSN) { join_path_components(backup_label, pgdata, "backup_label"); - if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0) + if (fio_access(FIO_DB_HOST, backup_label, F_OK) == 0) { elog(WARNING, "Destination directory contains \"backup_control\" file. " "This does NOT mean that you should delete this file and retry, only that " diff --git a/src/util.c b/src/util.c index fb33fd046..e89f5776b 100644 --- a/src/util.c +++ b/src/util.c @@ -3,7 +3,7 @@ * util.c: log messages to log file or stderr, and misc code. * * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional + * Portions Copyright (c) 2015-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -122,7 +122,7 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) * Write ControlFile to pg_control */ static void -writeControlFile(ControlFileData *ControlFile, const char *path, fio_location location) +writeControlFile(fio_location location, const char *path, ControlFileData *ControlFile) { int fd; char *buffer = NULL; @@ -138,8 +138,8 @@ writeControlFile(ControlFileData *ControlFile, const char *path, fio_location lo memcpy(buffer, ControlFile, sizeof(ControlFileData)); /* Write pg_control */ - fd = fio_open(path, - O_RDWR | O_CREAT | O_TRUNC | PG_BINARY, location); + fd = fio_open(location, path, + O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); if (fd < 0) elog(ERROR, "Failed to open file: %s", path); @@ -172,7 +172,7 @@ get_current_timeline(PGconn *conn) if (PQresultStatus(res) == PGRES_TUPLES_OK) val = PQgetvalue(res, 0, 0); else - return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); + return get_current_timeline_from_control(FIO_DB_HOST, instance_config.pgdata, false); if (!parse_uint32(val, &tli, 0)) { @@ -180,7 +180,7 @@ get_current_timeline(PGconn *conn) elog(WARNING, "Invalid value of timeline_id %s", val); /* TODO 3.0 remove it and just error out */ - return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); + return get_current_timeline_from_control(FIO_DB_HOST, instance_config.pgdata, false); } return tli; @@ -188,15 +188,15 @@ get_current_timeline(PGconn *conn) /* Get timeline from pg_control file */ TimeLineID -get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe) +get_current_timeline_from_control(fio_location location, const char *pgdata_path, bool safe) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, - safe, location); + buffer = slurpFile(location, pgdata_path, XLOG_CONTROL_FILE, + &size, safe); if (safe && buffer == NULL) return 0; @@ -234,11 +234,12 @@ get_checkpoint_location(PGconn *conn) return lsn; #else + /* PG-9.5 */ char *buffer; size_t size; ControlFileData ControlFile; - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); + buffer = slurpFile(FIO_DB_HOST, instance_config.pgdata, XLOG_CONTROL_FILE, &size, false); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); @@ -247,14 +248,14 @@ get_checkpoint_location(PGconn *conn) } uint64 -get_system_identifier(const char *pgdata_path, fio_location location, bool safe) +get_system_identifier(fio_location location, const char *pgdata_path, bool safe) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, safe, location); + buffer = slurpFile(location, pgdata_path, XLOG_CONTROL_FILE, &size, safe); if (safe && buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); @@ -284,11 +285,12 @@ get_remote_system_identifier(PGconn *conn) return system_id_conn; #else + /* PG-9.5 */ char *buffer; size_t size; ControlFileData ControlFile; - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); + buffer = slurpFile(FIO_DB_HOST, instance_config.pgdata, XLOG_CONTROL_FILE, &size, false); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); @@ -305,7 +307,7 @@ get_xlog_seg_size(const char *pgdata_path) size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); + buffer = slurpFile(FIO_DB_HOST, pgdata_path, XLOG_CONTROL_FILE, &size, false); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); @@ -323,8 +325,8 @@ get_data_checksum_version(bool safe) size_t size; /* First fetch file... */ - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, - safe, FIO_DB_HOST); + buffer = slurpFile(FIO_DB_HOST, instance_config.pgdata, XLOG_CONTROL_FILE, + &size, safe); if (buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); @@ -341,7 +343,7 @@ get_pgcontrol_checksum(const char *pgdata_path) size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_BACKUP_HOST); + buffer = slurpFile(FIO_BACKUP_HOST, pgdata_path, XLOG_CONTROL_FILE, &size, false); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); @@ -349,14 +351,15 @@ get_pgcontrol_checksum(const char *pgdata_path) return ControlFile.crc; } +/* unused function */ DBState -get_system_dbstate(const char *pgdata_path, fio_location location) +get_system_dbstate(fio_location location, const char *pgdata_path) { ControlFileData ControlFile; char *buffer; size_t size; - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, location); + buffer = slurpFile(location, pgdata_path, XLOG_CONTROL_FILE, &size, false); if (buffer == NULL) return 0; digestControlFile(&ControlFile, buffer, size); @@ -366,14 +369,14 @@ get_system_dbstate(const char *pgdata_path, fio_location location) } void -get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo) +get_redo(fio_location location, const char *pgdata_path, RedoParams *redo) { ControlFileData ControlFile; char *buffer; size_t size; /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, pgdata_location); + buffer = slurpFile(location, pgdata_path, XLOG_CONTROL_FILE, &size, false); digestControlFile(&ControlFile, buffer, size); pg_free(buffer); @@ -412,7 +415,7 @@ set_min_recovery_point(pgFile *file, const char *backup_path, char fullpath[MAXPGPATH]; /* First fetch file content */ - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); + buffer = slurpFile(FIO_DB_HOST, instance_config.pgdata, XLOG_CONTROL_FILE, &size, false); digestControlFile(&ControlFile, buffer, size); elog(LOG, "Current minRecPoint %X/%X", @@ -433,7 +436,7 @@ set_min_recovery_point(pgFile *file, const char *backup_path, /* overwrite pg_control */ join_path_components(fullpath, backup_path, XLOG_CONTROL_FILE); - writeControlFile(&ControlFile, fullpath, FIO_LOCAL_HOST); + writeControlFile(FIO_LOCAL_HOST, fullpath, &ControlFile); /* Update pg_control checksum in backup_list */ file->crc = ControlFile.crc; @@ -445,14 +448,14 @@ set_min_recovery_point(pgFile *file, const char *backup_path, * Copy pg_control file to backup. We do not apply compression to this file. */ void -copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, - const char *to_fullpath, fio_location to_location, pgFile *file) +copy_pgcontrol_file(fio_location from_location, const char *from_fullpath, + fio_location to_location, const char *to_fullpath, pgFile *file) { ControlFileData ControlFile; char *buffer; size_t size; - buffer = slurpFile(from_fullpath, "", &size, false, from_location); + buffer = slurpFile(from_location, from_fullpath, "", &size, false); digestControlFile(&ControlFile, buffer, size); @@ -461,7 +464,7 @@ copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, file->write_size = size; file->uncompressed_size = size; - writeControlFile(&ControlFile, to_fullpath, to_location); + writeControlFile(to_location, to_fullpath, &ControlFile); pg_free(buffer); } diff --git a/src/utils/configuration.c b/src/utils/configuration.c index 04bfbbe3b..fa3f444ce 100644 --- a/src/utils/configuration.c +++ b/src/utils/configuration.c @@ -3,7 +3,7 @@ * configuration.c: - function implementations to work with pg_probackup * configurations. * - * Copyright (c) 2017-2019, Postgres Professional + * Copyright (c) 2017-2022, Postgres Professional * *------------------------------------------------------------------------- */ @@ -88,7 +88,7 @@ static const unit_conversion time_unit_conversion_table[] = }; /* Order is important, keep it in sync with utils/configuration.h:enum ProbackupSubcmd declaration */ -static char const * const subcmd_names[] = +static const char * const subcmd_names[] = { "NO_CMD", "init", @@ -114,7 +114,7 @@ static char const * const subcmd_names[] = }; ProbackupSubcmd -parse_subcmd(char const * const subcmd_str) +parse_subcmd(const char * const subcmd_str) { struct { ProbackupSubcmd id; @@ -137,7 +137,7 @@ parse_subcmd(char const * const subcmd_str) return NO_CMD; } -char const * +const char * get_subcmd_name(ProbackupSubcmd const subcmd) { Assert((int)subcmd < sizeof(subcmd_names) / sizeof(subcmd_names[0])); @@ -555,8 +555,14 @@ config_read_opt(const char *path, ConfigOption options[], int elevel, if (!options) return parsed_options; - if ((fp = pgut_fopen(path, "rt", missing_ok)) == NULL) - return parsed_options; + if ((fp = fio_open_stream(FIO_BACKUP_HOST, path)) == NULL) + { + if (missing_ok && errno == ENOENT) + return parsed_options; + + elog(ERROR, "could not open file \"%s\": %s", + path, strerror(errno)); + } while (fgets(buf, lengthof(buf), fp)) { diff --git a/src/utils/configuration.h b/src/utils/configuration.h index 2c6ea3eec..e42544466 100644 --- a/src/utils/configuration.h +++ b/src/utils/configuration.h @@ -101,8 +101,8 @@ struct ConfigOption #define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) -extern ProbackupSubcmd parse_subcmd(char const * const subcmd_str); -extern char const *get_subcmd_name(ProbackupSubcmd const subcmd); +extern ProbackupSubcmd parse_subcmd(const char * const subcmd_str); +extern const char *get_subcmd_name(ProbackupSubcmd const subcmd); extern int config_get_opt(int argc, char **argv, ConfigOption cmd_options[], ConfigOption options[]); extern int config_read_opt(const char *path, ConfigOption options[], int elevel, diff --git a/src/utils/file.c b/src/utils/file.c index 7d1df554b..7851648a6 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -112,7 +112,7 @@ fio_redirect(int in, int out, int err) } void -fio_error(int rc, int size, char const* file, int line) +fio_error(int rc, int size, const char* file, int line) { if (remote_agent) { @@ -176,7 +176,9 @@ fio_safestat(const char *path, struct stat *buf) } #define stat(x, y) fio_safestat(x, y) +#endif /* WIN32 */ +#ifdef WIN32 /* TODO: use real pread on Linux */ static ssize_t pread(int fd, void* buf, size_t size, off_t off) @@ -186,15 +188,16 @@ pread(int fd, void* buf, size_t size, off_t off) return -1; return read(fd, buf, size); } +#endif /* WIN32 */ +#ifdef WIN32 static int -remove_file_or_dir(char const* path) +remove_file_or_dir(const char* path) { int rc = remove(path); -#ifdef WIN32 + if (rc < 0 && errno == EACCESS) rc = rmdir(path); -#endif return rc; } #else @@ -284,7 +287,7 @@ fio_get_agent_version(void) /* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */ FILE* -fio_open_stream(char const* path, fio_location location) +fio_open_stream(fio_location location, const char* path) { FILE* f; if (fio_is_remote(location)) @@ -337,7 +340,7 @@ fio_close_stream(FILE* f) /* Open directory */ DIR* -fio_opendir(char const* path, fio_location location) +fio_opendir(fio_location location, const char* path) { DIR* dir; if (fio_is_remote(location)) @@ -429,7 +432,7 @@ fio_closedir(DIR *dir) /* Open file */ int -fio_open(char const* path, int mode, fio_location location) +fio_open(fio_location location, const char* path, int mode) { int fd; if (fio_is_remote(location)) @@ -497,7 +500,7 @@ fio_disconnect(void) /* Open stdio file */ FILE* -fio_fopen(char const* path, char const* mode, fio_location location) +fio_fopen(fio_location location, const char* path, const char* mode) { FILE *f = NULL; @@ -528,7 +531,7 @@ fio_fopen(char const* path, char const* mode, fio_location location) } else { Assert(false); } - fd = fio_open(path, flags, location); + fd = fio_open(location, path, flags); if (fd >= 0) f = (FILE*)(size_t)((fd + 1) & ~FIO_PIPE_MARKER); } @@ -543,11 +546,11 @@ fio_fopen(char const* path, char const* mode, fio_location location) /* Format output to file stream */ int -fio_fprintf(FILE* f, char const* format, ...) +fio_fprintf(FILE* f, const char* format, ...) { int rc; - va_list args; - va_start (args, format); + va_list args; + va_start (args, format); if (fio_is_remote_file(f)) { char buf[PRINTF_BUF_SIZE]; @@ -564,7 +567,7 @@ fio_fprintf(FILE* f, char const* format, ...) { rc = vfprintf(f, format, args); } - va_end (args); + va_end (args); return rc; } @@ -600,17 +603,19 @@ fio_close(int fd) { if (fio_is_remote_fd(fd)) { - fio_header hdr; + fio_header hdr = { + .cop = FIO_CLOSE, + .handle = fd & ~FIO_PIPE_MARKER, + .size = 0, + .arg = 0, + }; - hdr.cop = FIO_CLOSE; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; fio_fdset &= ~(1 << hdr.handle); - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); /* Wait for response */ IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_CLOSE); if (hdr.arg != 0) { @@ -630,10 +635,12 @@ fio_close(int fd) static void fio_close_impl(int fd, int out) { - fio_header hdr; - - hdr.cop = FIO_CLOSE; - hdr.arg = 0; + fio_header hdr = { + .cop = FIO_CLOSE, + .handle = -1, + .size = 0, + .arg = 0, + }; if (close(fd) != 0) hdr.arg = errno; @@ -659,12 +666,12 @@ fio_truncate(int fd, off_t size) { if (fio_is_remote_fd(fd)) { - fio_header hdr; - - hdr.cop = FIO_TRUNCATE; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - hdr.arg = size; + fio_header hdr = { + .cop = FIO_TRUNCATE, + .handle = fd & ~FIO_PIPE_MARKER, + .size = 0, + .arg = size, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); @@ -812,17 +819,19 @@ fio_write(int fd, void const* buf, size_t size) { if (fio_is_remote_fd(fd)) { - fio_header hdr; - - hdr.cop = FIO_WRITE; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = size; + fio_header hdr = { + .cop = FIO_WRITE, + .handle = fd & ~FIO_PIPE_MARKER, + .size = size, + .arg = 0, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, buf, size), size); /* check results */ IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_WRITE); /* set errno */ if (hdr.arg > 0) @@ -842,14 +851,16 @@ fio_write(int fd, void const* buf, size_t size) static void fio_write_impl(int fd, void const* buf, size_t size, int out) { + fio_header hdr = { + .cop = FIO_WRITE, + .handle = -1, + .size = 0, + .arg = 0, + }; int rc; - fio_header hdr; rc = durable_write(fd, buf, size); - hdr.arg = 0; - hdr.size = 0; - if (rc < 0) hdr.arg = errno; @@ -877,19 +888,19 @@ fio_write_async(int fd, void const* buf, size_t size) if (fio_is_remote_fd(fd)) { - fio_header hdr; - - hdr.cop = FIO_WRITE_ASYNC; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = size; + fio_header hdr = { + .cop = FIO_WRITE_ASYNC, + .handle = fd & ~FIO_PIPE_MARKER, + .size = size, + .arg = 0, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, buf, size), size); + return size; } else return durable_write(fd, buf, size); - - return size; } static void @@ -906,7 +917,7 @@ fio_write_async_impl(int fd, void const* buf, size_t size, int out) } } -int32 +static int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg, char **errormsg) { const char *internal_errormsg = NULL; @@ -1086,12 +1097,12 @@ fio_read(int fd, void* buf, size_t size) { if (fio_is_remote_fd(fd)) { - fio_header hdr; - - hdr.cop = FIO_READ; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - hdr.arg = size; + fio_header hdr = { + .cop = FIO_READ, + .handle = fd & ~FIO_PIPE_MARKER, + .size = 0, + .arg = size, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); @@ -1109,20 +1120,19 @@ fio_read(int fd, void* buf, size_t size) /* Get information about file */ int -fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location location) +fio_stat(fio_location location, const char* path, struct stat* st, bool follow_symlink) { if (fio_is_remote(location)) { - fio_header hdr; - size_t path_len = strlen(path) + 1; - - hdr.cop = FIO_STAT; - hdr.handle = -1; - hdr.arg = follow_symlink; - hdr.size = path_len; + fio_header hdr = { + .cop = FIO_STAT, + .handle = -1, + .size = strlen(path) + 1, + .arg = follow_symlink, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_STAT); @@ -1146,15 +1156,15 @@ fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location lo * in windows compare only filenames */ bool -fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location) +fio_is_same_file(fio_location location, const char* filename1, const char* filename2, bool follow_symlink) { #ifndef WIN32 struct stat stat1, stat2; - if (fio_stat(filename1, &stat1, follow_symlink, location) < 0) + if (fio_stat(location, filename1, &stat1, follow_symlink) < 0) elog(ERROR, "Can't stat file \"%s\": %s", filename1, strerror(errno)); - if (fio_stat(filename2, &stat2, follow_symlink, location) < 0) + if (fio_stat(location, filename2, &stat2, follow_symlink) < 0) elog(ERROR, "Can't stat file \"%s\": %s", filename2, strerror(errno)); return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev; @@ -1176,7 +1186,7 @@ fio_is_same_file(char const* filename1, char const* filename2, bool follow_symli * returned value >= valsiz) */ ssize_t -fio_readlink(const char *path, char *value, size_t valsiz, fio_location location) +fio_readlink(fio_location location, const char *path, char *value, size_t valsiz) { if (!fio_is_remote(location)) { @@ -1210,19 +1220,19 @@ fio_readlink(const char *path, char *value, size_t valsiz, fio_location location /* Check presence of the file */ int -fio_access(char const* path, int mode, fio_location location) +fio_access(fio_location location, const char* path, int mode) { if (fio_is_remote(location)) { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_ACCESS; - hdr.handle = -1; - hdr.size = path_len; - hdr.arg = mode; + fio_header hdr = { + .cop = FIO_ACCESS, + .handle = -1, + .size = strlen(path) + 1, + .arg = mode, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_ACCESS); @@ -1242,22 +1252,31 @@ fio_access(char const* path, int mode, fio_location location) /* Create symbolic link */ int -fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location) +fio_symlink(fio_location location, const char* target, const char* link_path, bool overwrite) { if (fio_is_remote(location)) { - fio_header hdr; size_t target_len = strlen(target) + 1; size_t link_path_len = strlen(link_path) + 1; - hdr.cop = FIO_SYMLINK; - hdr.handle = -1; - hdr.size = target_len + link_path_len; - hdr.arg = overwrite ? 1 : 0; + fio_header hdr = { + .cop = FIO_SYMLINK, + .handle = -1, + .size = target_len + link_path_len, + .arg = overwrite ? 1 : 0, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, target, target_len), target_len); IO_CHECK(fio_write_all(fio_stdout, link_path, link_path_len), link_path_len); + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_SYMLINK); + + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } return 0; } else @@ -1270,38 +1289,51 @@ fio_symlink(char const* target, char const* link_path, bool overwrite, fio_locat } static void -fio_symlink_impl(int out, char *buf, bool overwrite) +fio_symlink_impl(const char* target, const char* link_path, bool overwrite, int out) { - char *linked_path = buf; - char *link_path = buf + strlen(buf) + 1; + fio_header hdr = { + .cop = FIO_SYMLINK, + .handle = -1, + .size = 0, + .arg = 0, + }; if (overwrite) remove_file_or_dir(link_path); - if (symlink(linked_path, link_path)) - elog(ERROR, "Could not create symbolic link \"%s\": %s", - link_path, strerror(errno)); + if (symlink(target, link_path) != 0) + hdr.arg = errno; + + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } /* Rename file */ int -fio_rename(char const* old_path, char const* new_path, fio_location location) +fio_rename(fio_location location, const char* old_path, const char* new_path) { if (fio_is_remote(location)) { - fio_header hdr; size_t old_path_len = strlen(old_path) + 1; size_t new_path_len = strlen(new_path) + 1; - hdr.cop = FIO_RENAME; - hdr.handle = -1; - hdr.size = old_path_len + new_path_len; + fio_header hdr = { + .cop = FIO_RENAME, + .handle = -1, + .size = old_path_len + new_path_len, + .arg = 0, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, old_path, old_path_len), old_path_len); IO_CHECK(fio_write_all(fio_stdout, new_path, new_path_len), new_path_len); - //TODO: wait for confirmation. + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_RENAME); + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } return 0; } else @@ -1310,9 +1342,25 @@ fio_rename(char const* old_path, char const* new_path, fio_location location) } } +static void +fio_rename_impl(char const* old_path, const char* new_path, int out) +{ + fio_header hdr = { + .cop = FIO_RENAME, + .handle = -1, + .size = 0, + .arg = 0, + }; + + if (rename(old_path, new_path) != 0) + hdr.arg = errno; + + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); +} + /* Sync file to disk */ int -fio_sync(char const* path, fio_location location) +fio_sync(fio_location location, const char* path) { if (fio_is_remote(location)) { @@ -1355,7 +1403,7 @@ fio_sync(char const* path, fio_location location) /* Get crc32 of file */ pg_crc32 -fio_get_crc32(const char *file_path, fio_location location, bool decompress) +fio_get_crc32(fio_location location, const char *file_path, bool decompress) { if (fio_is_remote(location)) { @@ -1385,62 +1433,149 @@ fio_get_crc32(const char *file_path, fio_location location, bool decompress) } } -/* Remove file */ +/* + * Remove file or directory + * if missing_ok, then ignore ENOENT error + */ int -fio_unlink(char const* path, fio_location location) +fio_remove(fio_location location, const char* path, bool missing_ok) { + int result = 0; + if (fio_is_remote(location)) { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_UNLINK; - hdr.handle = -1; - hdr.size = path_len; + fio_header hdr = { + .cop = FIO_REMOVE, + .handle = -1, + .size = strlen(path) + 1, + .arg = missing_ok ? 1 : 0, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - // TODO: error is swallowed ? - return 0; + IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_REMOVE); + + if (hdr.arg != 0) + { + errno = hdr.arg; + result = -1; + } } else { - return remove_file_or_dir(path); + if (remove_file_or_dir(path) != 0) + { + if (!missing_ok || errno != ENOENT) + result = -1; + } + } + return result; +} + +static void +fio_remove_impl(const char* path, bool missing_ok, int out) +{ + fio_header hdr = { + .cop = FIO_REMOVE, + .handle = -1, + .size = 0, + .arg = 0, + }; + + if (remove_file_or_dir(path) != 0) + { + if (!missing_ok || errno != ENOENT) + hdr.arg = errno; } + + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } -/* Create directory - * TODO: add strict flag +/* + * Create directory, also create parent directories if necessary. + * In strict mode treat already existing directory as error. + * Return values: + * 0 - ok + * -1 - error (check errno) + */ +static int +dir_create_dir(const char *dir, mode_t mode, bool strict) +{ + char parent[MAXPGPATH]; + + strncpy(parent, dir, MAXPGPATH); + get_parent_directory(parent); + + /* Create parent first */ + if (access(parent, F_OK) == -1) + dir_create_dir(parent, mode, false); + + /* Create directory */ + if (mkdir(dir, mode) == -1) + { + if (errno == EEXIST && !strict) /* already exist */ + return 0; + return -1; + } + + return 0; +} + +/* + * Create directory */ int -fio_mkdir(char const* path, int mode, fio_location location) +fio_mkdir(fio_location location, const char* path, int mode, bool strict) { if (fio_is_remote(location)) { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_MKDIR; - hdr.handle = -1; - hdr.size = path_len; - hdr.arg = mode; + fio_header hdr = { + .cop = FIO_MKDIR, + .handle = strict ? 1 : 0, /* ugly "hack" to pass more params*/ + .size = strlen(path) + 1, + .arg = mode, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); + IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); Assert(hdr.cop == FIO_MKDIR); - return hdr.arg; + if (hdr.arg != 0) + { + errno = hdr.arg; + return -1; + } + return 0; } else { - return dir_create_dir(path, mode, false); + return dir_create_dir(path, mode, strict); } } +static void +fio_mkdir_impl(const char* path, int mode, bool strict, int out) +{ + fio_header hdr = { + .cop = FIO_MKDIR, + .handle = -1, + .size = 0, + .arg = 0, + }; + + if (dir_create_dir(path, mode, strict) != 0) + hdr.arg = errno; + + IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); +} + /* Change file mode */ int -fio_chmod(char const* path, int mode, fio_location location) +fio_chmod(fio_location location, const char* path, int mode) { if (fio_is_remote(location)) { @@ -1512,7 +1647,7 @@ fio_check_error_fd_gz(gzFile f, char **errmsg) /* On error returns NULL and errno should be checked */ gzFile -fio_gzopen(char const* path, char const* mode, int level, fio_location location) +fio_gzopen(fio_location location, const char* path, const char* mode, int level) { int rc; if (fio_is_remote(location)) @@ -1534,7 +1669,7 @@ fio_gzopen(char const* path, char const* mode, int level, fio_location location) if (rc == Z_OK) { gz->compress = 1; - gz->fd = fio_open(path, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY, location); + gz->fd = fio_open(location, path, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY); if (gz->fd < 0) { free(gz); @@ -1551,7 +1686,7 @@ fio_gzopen(char const* path, char const* mode, int level, fio_location location) if (rc == Z_OK) { gz->compress = 0; - gz->fd = fio_open(path, O_RDONLY | PG_BINARY, location); + gz->fd = fio_open(location, path, O_RDONLY | PG_BINARY); if (gz->fd < 0) { free(gz); @@ -1779,7 +1914,7 @@ fio_gzseek(gzFile f, z_off_t offset, int whence) * Note: it should not be used for large files. */ static void -fio_load_file(int out, char const* path) +fio_load_file(int out, const char* path) { int fd = open(path, O_RDONLY); fio_header hdr; @@ -2058,12 +2193,12 @@ fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, if (use_pagemap) IO_CHECK(fio_write_all(fio_stdout, (*file).pagemap.bitmap, (*file).pagemap.bitmapsize), (*file).pagemap.bitmapsize); - out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST); + out = fio_fopen(FIO_BACKUP_HOST, to_fullpath, PG_BINARY_R "+"); if (out == NULL) elog(ERROR, "Cannot open restore target file \"%s\": %s", to_fullpath, strerror(errno)); /* update file permission */ - if (fio_chmod(to_fullpath, file->mode, FIO_BACKUP_HOST) == -1) + if (fio_chmod(FIO_BACKUP_HOST, to_fullpath, file->mode) == -1) elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, strerror(errno)); @@ -2688,7 +2823,7 @@ fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out, * */ static void -fio_send_file_impl(int out, char const* path) +fio_send_file_impl(int out, const char* path) { FILE *fp; fio_header hdr; @@ -2952,8 +3087,8 @@ fio_list_dir(parray *files, const char *root, bool exclude, } PageState * -fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, - XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location) +fio_get_checksum_map(fio_location location, const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) { if (fio_is_remote(location)) { @@ -2995,7 +3130,7 @@ fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks } static void -fio_get_checksum_map_impl(int out, char *buf) +fio_get_checksum_map_impl(char *buf, int out) { fio_header hdr; PageState *checksum_map = NULL; @@ -3015,9 +3150,9 @@ fio_get_checksum_map_impl(int out, char *buf) } datapagemap_t * -fio_get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno, - fio_location location) +fio_get_lsn_map(fio_location location, const char *fullpath, + uint32 checksum_version, int n_blocks, + XLogRecPtr shift_lsn, BlockNumber segmentno) { datapagemap_t* lsn_map = NULL; @@ -3063,7 +3198,7 @@ fio_get_lsn_map(const char *fullpath, uint32 checksum_version, } static void -fio_get_lsn_map_impl(int out, char *buf) +fio_get_lsn_map_impl(char *buf, int out) { fio_header hdr; datapagemap_t *lsn_map = NULL; @@ -3142,20 +3277,24 @@ local_check_postmaster(const char *pgdata) * and check that process is running, if process is running, return its pid number. */ pid_t -fio_check_postmaster(const char *pgdata, fio_location location) +fio_check_postmaster(fio_location location, const char *pgdata) { if (fio_is_remote(location)) { - fio_header hdr; - - hdr.cop = FIO_CHECK_POSTMASTER; - hdr.size = strlen(pgdata) + 1; + fio_header hdr = { + .cop = FIO_CHECK_POSTMASTER, + .handle = -1, + .size = strlen(pgdata) + 1, + .arg = 0, + }; IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); IO_CHECK(fio_write_all(fio_stdout, pgdata, hdr.size), hdr.size); /* receive result */ IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); + Assert(hdr.cop == FIO_CHECK_POSTMASTER); + return hdr.arg; } else @@ -3163,50 +3302,21 @@ fio_check_postmaster(const char *pgdata, fio_location location) } static void -fio_check_postmaster_impl(int out, char *buf) +fio_check_postmaster_impl(const char *pgdata, int out) { - fio_header hdr; - pid_t postmaster_pid; - char *pgdata = (char*) buf; + fio_header hdr = { + .cop = FIO_CHECK_POSTMASTER, + .handle = -1, + .size = 0, + .arg = 0, + }; - postmaster_pid = local_check_postmaster(pgdata); + hdr.arg = local_check_postmaster(pgdata); /* send arrays of checksums to main process */ - hdr.arg = postmaster_pid; IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } -/* - * Delete file pointed by the pgFile. - * If the pgFile points directory, the directory must be empty. - */ -void -fio_delete(mode_t mode, const char *fullpath, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - - hdr.cop = FIO_DELETE; - hdr.size = strlen(fullpath) + 1; - hdr.arg = mode; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, fullpath, hdr.size), hdr.size); - - } - else - pgFileDelete(mode, fullpath); -} - -static void -fio_delete_impl(mode_t mode, char *buf) -{ - char *fullpath = (char*) buf; - - pgFileDelete(mode, fullpath); -} - /* Execute commands at remote host */ void fio_communicate(int in, int out) @@ -3228,11 +3338,11 @@ fio_communicate(int in, int out) pg_crc32 crc; #ifdef WIN32 - SYS_CHECK(setmode(in, _O_BINARY)); - SYS_CHECK(setmode(out, _O_BINARY)); + SYS_CHECK(setmode(in, _O_BINARY)); + SYS_CHECK(setmode(out, _O_BINARY)); #endif - /* Main loop until end of processing all master commands */ + /* Main loop until end of processing all master commands */ while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) { if (hdr.size != 0) { if (hdr.size > buf_size) { @@ -3328,18 +3438,17 @@ fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); break; case FIO_RENAME: /* Rename file */ - SYS_CHECK(rename(buf, buf + strlen(buf) + 1)); + /* possible buffer overflow */ + fio_rename_impl(buf, buf + strlen(buf) + 1, out); break; case FIO_SYMLINK: /* Create symbolic link */ - fio_symlink_impl(out, buf, hdr.arg > 0 ? true : false); + fio_symlink_impl(buf, buf + strlen(buf) + 1, hdr.arg == 1, out); break; - case FIO_UNLINK: /* Remove file or directory (TODO: Win32) */ - SYS_CHECK(remove_file_or_dir(buf)); + case FIO_REMOVE: /* Remove file or directory (TODO: Win32) */ + fio_remove_impl(buf, hdr.arg == 1, out); break; case FIO_MKDIR: /* Create directory */ - hdr.size = 0; - hdr.arg = dir_create_dir(buf, hdr.arg, false); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); + fio_mkdir_impl(buf, hdr.arg, hdr.handle == 1, out); break; case FIO_CHMOD: /* Change file mode */ SYS_CHECK(chmod(buf, hdr.arg)); @@ -3354,7 +3463,7 @@ fio_communicate(int in, int out) fio_list_dir_impl(out, buf); break; case FIO_SEND_PAGES: - // buf contain fio_send_request header and bitmap. + /* buf contain fio_send_request header and bitmap. */ fio_send_pages_impl(out, buf); break; case FIO_SEND_FILE: @@ -3385,20 +3494,13 @@ fio_communicate(int in, int out) IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); break; case FIO_GET_CHECKSUM_MAP: - /* calculate crc32 for a file */ - fio_get_checksum_map_impl(out, buf); + fio_get_checksum_map_impl(buf, out); break; case FIO_GET_LSN_MAP: - /* calculate crc32 for a file */ - fio_get_lsn_map_impl(out, buf); + fio_get_lsn_map_impl(buf, out); break; case FIO_CHECK_POSTMASTER: - /* calculate crc32 for a file */ - fio_check_postmaster_impl(out, buf); - break; - case FIO_DELETE: - /* delete file */ - fio_delete_impl(hdr.arg, buf); + fio_check_postmaster_impl(buf, out); break; case FIO_DISCONNECT: hdr.cop = FIO_DISCONNECTED; diff --git a/src/utils/file.h b/src/utils/file.h index a554b4ab0..5639a3e4c 100644 --- a/src/utils/file.h +++ b/src/utils/file.h @@ -20,12 +20,11 @@ typedef enum FIO_SYNC, FIO_RENAME, FIO_SYMLINK, - FIO_UNLINK, + FIO_REMOVE, FIO_MKDIR, FIO_CHMOD, FIO_SEEK, FIO_TRUNCATE, - FIO_DELETE, FIO_PREAD, FIO_READ, FIO_LOAD, @@ -41,7 +40,7 @@ typedef enum /* used for incremental restore */ FIO_GET_CHECKSUM_MAP, FIO_GET_LSN_MAP, - /* used in fio_send_pages */ + /* used in fio_send_pages */ FIO_SEND_PAGES, FIO_ERROR, FIO_SEND_FILE, @@ -59,86 +58,86 @@ typedef enum FIO_READLINK } fio_operations; -typedef enum -{ - FIO_LOCAL_HOST, /* data is locate at local host */ - FIO_DB_HOST, /* data is located at Postgres server host */ - FIO_BACKUP_HOST, /* data is located at backup host */ - FIO_REMOTE_HOST /* date is located at remote host */ -} fio_location; - -#define FIO_FDMAX 64 -#define FIO_PIPE_MARKER 0x40000000 - -#define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) -#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) fio_error(_rc, size, __FILE__, __LINE__); } while (0) - typedef struct { // fio_operations cop; // 16 + /* fio operation, see fio_operations enum */ unsigned cop : 32; + /* */ unsigned handle : 32; + /* size of additional data sent after this header */ unsigned size : 32; + /* additional small parameter for requests (varies between operations) or a result code for response */ unsigned arg; } fio_header; +typedef enum +{ + FIO_LOCAL_HOST, /* data is locate at local host */ + FIO_DB_HOST, /* data is located at Postgres server host */ + FIO_BACKUP_HOST, /* data is located at backup host */ + FIO_REMOTE_HOST /* date is located at remote host */ +} fio_location; + extern fio_location MyLocation; +extern void setMyLocation(ProbackupSubcmd const subcmd); +/* Check if specified location is local for current node */ +extern bool fio_is_remote(fio_location location); +extern bool fio_is_remote_simple(fio_location location); + +extern void fio_communicate(int in, int out); +extern void fio_disconnect(void); +extern int fio_get_agent_version(void); + +#define FIO_FDMAX 64 +#define FIO_PIPE_MARKER 0x40000000 + /* Check if FILE handle is local or remote (created by FIO) */ #define fio_is_remote_file(file) ((size_t)(file) <= FIO_FDMAX) extern void fio_redirect(int in, int out, int err); -extern void fio_communicate(int in, int out); +extern void fio_error(int rc, int size, const char* file, int line); -extern int fio_get_agent_version(void); -extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); +#define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) +#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) fio_error(_rc, size, __FILE__, __LINE__); } while (0) + + +/* fd-style functions */ +extern int fio_open(fio_location location, const char* name, int mode); +extern ssize_t fio_write(int fd, void const* buf, size_t size); +extern ssize_t fio_write_async(int fd, void const* buf, size_t size); +extern int fio_check_error_fd(int fd, char **errmsg); +extern int fio_check_error_fd_gz(gzFile f, char **errmsg); +extern ssize_t fio_read(int fd, void* buf, size_t size); +extern int fio_flush(int fd); +extern int fio_seek(int fd, off_t offs); +extern int fio_fstat(int fd, struct stat* st); +extern int fio_truncate(int fd, off_t size); +extern int fio_close(int fd); + +/* FILE-style functions */ +extern FILE* fio_fopen(fio_location location, const char* name, const char* mode); extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); extern ssize_t fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg); extern size_t fio_fwrite_async(FILE* f, void const* buf, size_t size); extern int fio_check_error_file(FILE* f, char **errmsg); -extern int fio_check_error_fd(int fd, char **errmsg); -extern int fio_check_error_fd_gz(gzFile f, char **errmsg); extern ssize_t fio_fread(FILE* f, void* buf, size_t size); extern int fio_pread(FILE* f, void* buf, off_t offs); -extern int fio_fprintf(FILE* f, char const* arg, ...) pg_attribute_printf(2, 3); +extern int fio_fprintf(FILE* f, const char* arg, ...) pg_attribute_printf(2, 3); extern int fio_fflush(FILE* f); extern int fio_fseek(FILE* f, off_t offs); extern int fio_ftruncate(FILE* f, off_t size); extern int fio_fclose(FILE* f); extern int fio_ffstat(FILE* f, struct stat* st); -extern void fio_error(int rc, int size, char const* file, int line); -extern int fio_open(char const* name, int mode, fio_location location); -extern ssize_t fio_write(int fd, void const* buf, size_t size); -extern ssize_t fio_write_async(int fd, void const* buf, size_t size); -extern ssize_t fio_read(int fd, void* buf, size_t size); -extern int fio_flush(int fd); -extern int fio_seek(int fd, off_t offs); -extern int fio_fstat(int fd, struct stat* st); -extern int fio_truncate(int fd, off_t size); -extern int fio_close(int fd); -extern void fio_disconnect(void); -extern int fio_sync(char const* path, fio_location location); -extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress); - -extern int fio_rename(char const* old_path, char const* new_path, fio_location location); -extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location); -extern int fio_unlink(char const* path, fio_location location); -extern int fio_mkdir(char const* path, int mode, fio_location location); -extern int fio_chmod(char const* path, int mode, fio_location location); -extern int fio_access(char const* path, int mode, fio_location location); -extern int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location); -extern bool fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location); -extern ssize_t fio_readlink(const char *path, char *value, size_t valsiz, fio_location location); -extern DIR* fio_opendir(char const* path, fio_location location); -extern struct dirent * fio_readdir(DIR *dirp); -extern int fio_closedir(DIR *dirp); -extern FILE* fio_open_stream(char const* name, fio_location location); +extern FILE* fio_open_stream(fio_location location, const char* name); extern int fio_close_stream(FILE* f); +/* gzFile-style functions */ #ifdef HAVE_LIBZ -extern gzFile fio_gzopen(char const* path, char const* mode, int level, fio_location location); +extern gzFile fio_gzopen(fio_location location, const char* path, const char* mode, int level); extern int fio_gzclose(gzFile file); extern int fio_gzread(gzFile f, void *buf, unsigned size); extern int fio_gzwrite(gzFile f, void const* buf, unsigned size); @@ -147,4 +146,34 @@ extern z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence); extern const char* fio_gzerror(gzFile file, int *errnum); #endif +/* DIR-style functions */ +extern DIR* fio_opendir(fio_location location, const char* path); +extern struct dirent * fio_readdir(DIR *dirp); +extern int fio_closedir(DIR *dirp); + +/* pathname-style functions */ +extern int fio_sync(fio_location location, const char* path); +extern pg_crc32 fio_get_crc32(fio_location location, const char *file_path, bool decompress); + +extern int fio_rename(fio_location location, const char* old_path, const char* new_path); +extern int fio_symlink(fio_location location, const char* target, const char* link_path, bool overwrite); +extern int fio_remove(fio_location location, const char* path, bool missing_ok); +extern int fio_mkdir(fio_location location, const char* path, int mode, bool strict); +extern int fio_chmod(fio_location location, const char* path, int mode); +extern int fio_access(fio_location location, const char* path, int mode); +extern int fio_stat(fio_location location, const char* path, struct stat* st, bool follow_symlinks); +extern bool fio_is_same_file(fio_location location, const char* filename1, const char* filename2, bool follow_symlink); +extern ssize_t fio_readlink(fio_location location, const char *path, char *value, size_t valsiz); +extern pid_t fio_check_postmaster(fio_location location, const char *pgdata); + +extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink, + bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num); + +struct PageState; /* defined in pg_probackup.h */ +extern struct PageState *fio_get_checksum_map(fio_location location, const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); +struct datapagemap; /* defined in datapagemap.h */ +extern struct datapagemap *fio_get_lsn_map(fio_location location, const char *fullpath, uint32 checksum_version, + int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno); + #endif diff --git a/src/utils/pgut.c b/src/utils/pgut.c index 2cf0ccbe7..c220b807d 100644 --- a/src/utils/pgut.c +++ b/src/utils/pgut.c @@ -980,7 +980,7 @@ pgut_strndup(const char *str, size_t n) /* * Allocates new string, that contains part of filepath string minus trailing filename string * If trailing filename string not found, returns copy of filepath. - * Result must be free by caller. + * Result must be freed by caller. */ char * pgut_str_strip_trailing_filename(const char *filepath, const char *filename) @@ -993,23 +993,6 @@ pgut_str_strip_trailing_filename(const char *filepath, const char *filename) return pgut_strndup(filepath, fp_len); } -FILE * -pgut_fopen(const char *path, const char *mode, bool missing_ok) -{ - FILE *fp; - - if ((fp = fio_open_stream(path, FIO_BACKUP_HOST)) == NULL) - { - if (missing_ok && errno == ENOENT) - return NULL; - - elog(ERROR, "could not open file \"%s\": %s", - path, strerror(errno)); - } - - return fp; -} - #ifdef WIN32 static int select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout); #define select select_win32 diff --git a/src/utils/pgut.h b/src/utils/pgut.h index fa0efe816..638259a3c 100644 --- a/src/utils/pgut.h +++ b/src/utils/pgut.h @@ -69,11 +69,6 @@ extern char *pgut_str_strip_trailing_filename(const char *filepath, const char * #define pgut_new0(type) ((type *) pgut_malloc0(sizeof(type))) #define pgut_newarray(type, n) ((type *) pgut_malloc(sizeof(type) * (n))) -/* - * file operations - */ -extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); - /* * Assert */ diff --git a/src/utils/remote.c b/src/utils/remote.c index 2bfd24d1e..ec0110db4 100644 --- a/src/utils/remote.c +++ b/src/utils/remote.c @@ -103,7 +103,7 @@ void launch_ssh(char* argv[]) } #endif -static bool needs_quotes(char const* path) +static bool needs_quotes(const char* path) { return strchr(path, ' ') != NULL; } @@ -156,7 +156,7 @@ bool launch_agent(void) if (instance_config.remote.path) { - char const* probackup = PROGRAM_NAME_FULL; + const char* probackup = PROGRAM_NAME_FULL; char* sep = strrchr(probackup, '/'); if (sep != NULL) { probackup = sep + 1; diff --git a/src/validate.c b/src/validate.c index 4044ac158..d88de5583 100644 --- a/src/validate.c +++ b/src/validate.c @@ -200,10 +200,14 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params) (parse_program_version(backup->program_version) == 20201))) { char path[MAXPGPATH]; + struct stat st; join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); - if (pgFileSize(path) >= (BLCKSZ*500)) + if (fio_stat(FIO_BACKUP_HOST, path, &st, true) < 0) + elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); + + if (st.st_size >= (BLCKSZ*500)) { elog(WARNING, "Backup %s is a victim of metadata corruption. " "Additional information can be found here: " diff --git a/tests/merge.py b/tests/merge.py index fe0927f49..5b04e4704 100644 --- a/tests/merge.py +++ b/tests/merge.py @@ -1191,7 +1191,7 @@ def test_continue_failed_merge_2(self): gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('fio_remove') gdb.run_until_break() @@ -1683,7 +1683,7 @@ def test_failed_merge_after_delete(self): gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('fio_remove') gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL') @@ -1767,7 +1767,7 @@ def test_failed_merge_after_delete_1(self): # gdb.set_breakpoint('parray_bsearch') # gdb.continue_execution_until_break() - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('fio_remove') gdb.continue_execution_until_break(30) gdb._execute('signal SIGKILL') @@ -1829,7 +1829,9 @@ def test_failed_merge_after_delete_2(self): backup_dir, 'node', page_2, gdb=True, options=['--log-level-console=VERBOSE']) - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('delete_backup_files') + gdb.run_until_break() + gdb.set_breakpoint('fio_remove') gdb.run_until_break() gdb.continue_execution_until_break(2) gdb._execute('signal SIGKILL') @@ -1907,7 +1909,7 @@ def test_failed_merge_after_delete_3(self): gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('fio_remove') gdb.continue_execution_until_break(20) gdb._execute('signal SIGKILL')