Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 30 additions & 24 deletions src/scraper/scraper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ mTeamIDs TeamIDMap;
typedef std::map<std::string, std::string> mProjectTeamETags;
mProjectTeamETags ProjTeamETags;

std::vector<std::string> GetTeamWhiteList();

std::string urlsanity(const std::string& s, const std::string& type);
std::string lowercase(std::string s);
std::string ExtractXML(const std::string& XMLdata, const std::string& key, const std::string& key_end);
Expand Down Expand Up @@ -473,6 +475,24 @@ int64_t SuperblockAge()
return NN::Quorum::CurrentSuperblock()->Age();
}

std::vector<std::string> GetTeamWhiteList()
{
std::string delimiter;

// Due to a delimiter changeout from "|" to "<>" we must check to see if "<>" is in use
// in the protocol string.
if (TEAM_WHITELIST.find("<>") != std::string::npos)
{
delimiter = "<>";
}
else
{
delimiter = "|";
}

return split(TEAM_WHITELIST, delimiter);
}

/*********************
* Userpass Data *
*********************/
Expand Down Expand Up @@ -1425,7 +1445,7 @@ bool DownloadProjectTeamFiles(const NN::WhitelistSnapshot& projectWhitelist)
const auto iter = TeamIDMap.find(prjs.m_name);
bool fProjTeamIDsMissing = false;

if (iter == TeamIDMap.end() || iter->second.size() != split(TEAM_WHITELIST, "|").size()) fProjTeamIDsMissing = true;
if (iter == TeamIDMap.end() || iter->second.size() != GetTeamWhiteList().size()) fProjTeamIDsMissing = true;

// If fExplorer is false, which means we do not need to retain team files, and there are no TeamID entries missing,
// then skip processing altogether.
Expand Down Expand Up @@ -1601,7 +1621,7 @@ bool ProcessProjectTeamFile(const std::string& project, const fs::path& file, co

_log(logattribute::INFO, "ProcessProjectTeamFile", "Started processing " + file.filename().string());

std::vector<std::string> vTeamWhiteList = split(TEAM_WHITELIST, "|");
std::vector<std::string> vTeamWhiteList = GetTeamWhiteList();

std::string line;
stringbuilder builder;
Expand Down Expand Up @@ -2137,23 +2157,14 @@ bool LoadTeamIDList(const fs::path& file)
in.push(ingzfile);

std::string line;
std::string separator;
std::string separator = "<>";

// Header. This is used to construct the team names vector, since the team IDs were stored in the same order.
std::getline(in, line);

// This is to detect and handle the loading of a legacy existing TeamID.csv.gz file that contains commas rather than pipes.
// The file will be rewritten with pipe separators when the team files are processed.
if (line.find("|") != std::string::npos)
{
separator = "|";
}
else
{
_log(logattribute::INFO, "LoadTeamIDList", "Loading from legacy TeamID.csv.gz file with comma separator. This will be converted to pipe separator.");

separator = ",";
}
// This is to detect an existing TeamID.csv.gz file that contains the wrong separators. The load will be aborted,
// This will cause an overwrite of the file with the correct separators when the team files are processed fresh.
if (line.find(separator) == std::string::npos) return false;

// This is in the form Project, Gridcoin, ...."
std::vector<std::string> vTeamNames = split(line, separator);
Expand Down Expand Up @@ -2303,15 +2314,15 @@ bool StoreTeamIDList(const fs::path& file)
// Header
stream << "Project";

std::vector<std::string> vTeamWhiteList = split(TEAM_WHITELIST, "|");
std::vector<std::string> vTeamWhiteList = GetTeamWhiteList();
std::set<std::string> setTeamWhiteList;

// Ensure that the team names are in the correct order.
for (auto const& iTeam: vTeamWhiteList)
setTeamWhiteList.insert(iTeam);

for (auto const& iTeam: setTeamWhiteList)
stream << "|" << iTeam;
stream << "<>" << iTeam;

stream << std::endl;

Expand All @@ -2332,19 +2343,14 @@ bool StoreTeamIDList(const fs::path& file)

if (iter != iProject.second.end())
{
sProjectEntry += "|" + std::to_string(iter->second);
sProjectEntry += "<>" + std::to_string(iter->second);
}
else
{
sProjectEntry += "|-1";
sProjectEntry += "<>-1";
}
}

/*
for (auto const& iTeam : iProject.second)
sProjectEntry += "|" + std::to_string(iTeam.second);
*/

stream << sProjectEntry << std::endl;
}

Expand Down