diff --git a/server/proto/testgen.proto b/server/proto/testgen.proto index d66357c8..b44fbda5 100644 --- a/server/proto/testgen.proto +++ b/server/proto/testgen.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package testsgen; import "util.proto"; +import "google/protobuf/duration.proto"; service TestsGenService { rpc Handshake(DummyRequest) returns(DummyResponse) {} @@ -188,6 +189,7 @@ message TestResultObject { string testname = 2; TestStatus status = 3; string output = 4; + google.protobuf.Duration executionTime = 5; } message CoverageAndResultsResponse { diff --git a/server/src/KleeGenerator.cpp b/server/src/KleeGenerator.cpp index a4486861..92a5e5fd 100644 --- a/server/src/KleeGenerator.cpp +++ b/server/src/KleeGenerator.cpp @@ -57,7 +57,7 @@ KleeGenerator::buildByCDb(const CollectionUtils::MapFileTo &filesToBui fs::path makefile = projectTmpPath / "GenerationCompileMakefile.mk"; FileSystemUtils::writeToFile(makefile, makefilePrinter.ss.str()); - auto command = MakefileUtils::makefileCommand(projectContext, makefile, "all"); + auto command = MakefileUtils::MakefileCommand(projectContext, makefile, "all"); ExecUtils::ExecutionResult res = command.run(); if (res.status != 0) { LOG_S(ERROR) << StringUtils::stringFormat("Make for \"%s\" failed.\nCommand: \"%s\"\n%s\n", @@ -205,7 +205,7 @@ Result KleeGenerator::defaultBuild(const fs::path &hintPath, fs::path makefile = projectTmpPath / "BCForKLEE.mk"; FileSystemUtils::writeToFile(makefile, makefilePrinter.ss.str()); - auto makefileCommand = MakefileUtils::makefileCommand(projectContext, makefile, "build"); + auto makefileCommand = MakefileUtils::MakefileCommand(projectContext, makefile, "build"); auto [out, status, _] = makefileCommand.run(); if (status != 0) { LOG_S(ERROR) << "Compilation for " << sourceFilePath << " failed.\n" diff --git a/server/src/Paths.cpp b/server/src/Paths.cpp index 9eaa3cfb..058bc1b2 100644 --- a/server/src/Paths.cpp +++ b/server/src/Paths.cpp @@ -183,6 +183,9 @@ namespace Paths { fs::path getArtifactsRootDir(const utbot::ProjectContext &projectContext) { return projectContext.buildDir / "utbot"; } + fs::path getGTestResultsJsonPath(const utbot::ProjectContext &projectContext) { + return getArtifactsRootDir(projectContext) / "gtest-results.json"; + } fs::path getFlagsDir(const utbot::ProjectContext &projectContext) { return getArtifactsRootDir(projectContext) / "flags"; } diff --git a/server/src/Paths.h b/server/src/Paths.h index be610495..02eeef4b 100644 --- a/server/src/Paths.h +++ b/server/src/Paths.h @@ -273,6 +273,8 @@ namespace Paths { fs::path getArtifactsRootDir(const utbot::ProjectContext &projectContext); + fs::path getGTestResultsJsonPath(const utbot::ProjectContext &projectContext); + fs::path getFlagsDir(const utbot::ProjectContext &projectContext); fs::path getTestExecDir(const utbot::ProjectContext &projectContext); diff --git a/server/src/building/Linker.cpp b/server/src/building/Linker.cpp index c0287668..ee8b366a 100644 --- a/server/src/building/Linker.cpp +++ b/server/src/building/Linker.cpp @@ -364,7 +364,7 @@ Result Linker::link(const CollectionUtils::MapFileTo Linker::linkWithStubsIfNeeded(const fs::path &linkMakefile, return errorMessage; } - auto command = MakefileUtils::makefileCommand(testGen.projectContext, linkMakefile, "all"); + auto command = MakefileUtils::MakefileCommand(testGen.projectContext, linkMakefile, "all"); auto [out, status, _] = command.run(testGen.serverBuildDir); if (status != 0) { std::string errorMessage = diff --git a/server/src/coverage/Coverage.cpp b/server/src/coverage/Coverage.cpp index 940eebf2..05b01dd3 100644 --- a/server/src/coverage/Coverage.cpp +++ b/server/src/coverage/Coverage.cpp @@ -1,9 +1,9 @@ #include "Coverage.h" -int Coverage::TestStatusMap::getNumberOfTests() { +int Coverage::TestResultMap::getNumberOfTests() { int cnt = 0; - for (auto const &[fileName, testsStatus] : *this) { - cnt += testsStatus.size(); + for (auto const &[fileName, testsResult] : *this) { + cnt += testsResult.size(); } return cnt; } diff --git a/server/src/coverage/Coverage.h b/server/src/coverage/Coverage.h index 0580f699..2e6a5801 100644 --- a/server/src/coverage/Coverage.h +++ b/server/src/coverage/Coverage.h @@ -4,6 +4,7 @@ #include "utils/CollectionUtils.h" #include +#include #include #include @@ -37,11 +38,11 @@ namespace Coverage { class FileTestsStatus : public std::unordered_map { }; - class TestStatusMap : public CollectionUtils::MapFileTo { + using FileTestsResult = std::unordered_map; + class TestResultMap : public CollectionUtils::MapFileTo { public: int getNumberOfTests(); }; - } #endif //UNITTESTBOT_COVERAGE_H diff --git a/server/src/coverage/CoverageAndResultsGenerator.cpp b/server/src/coverage/CoverageAndResultsGenerator.cpp index 90a74c67..e592afa4 100644 --- a/server/src/coverage/CoverageAndResultsGenerator.cpp +++ b/server/src/coverage/CoverageAndResultsGenerator.cpp @@ -37,15 +37,17 @@ grpc::Status CoverageAndResultsGenerator::generate(bool withCoverage, } } catch (CoverageGenerationException &e) { showErrors(); + fs::remove(Paths::getGTestResultsJsonPath(projectContext)); return Status(StatusCode::FAILED_PRECONDITION, e.what()); } catch (ExecutionProcessException &e) { exceptions.emplace_back(e); showErrors(); + fs::remove(Paths::getGTestResultsJsonPath(projectContext)); return Status(StatusCode::FAILED_PRECONDITION, e.what()); } catch (CancellationException &e) { + fs::remove(Paths::getGTestResultsJsonPath(projectContext)); return Status::CANCELLED; } - showErrors(); return Status::OK; } @@ -65,7 +67,7 @@ void CoverageAndResultsGenerator::showErrors() const { errorMessage = message; } - coverageAndResultsWriter->writeResponse(testStatusMap, coverageMap, totals, errorMessage); + coverageAndResultsWriter->writeResponse(projectContext, testResultMap, coverageMap, totals, errorMessage); } Coverage::CoverageMap const &CoverageAndResultsGenerator::getCoverageMap() { @@ -83,7 +85,7 @@ void CoverageAndResultsGenerator::collectCoverage() { } std::vector coverageCommands = coverageTool->getCoverageCommands( CollectionUtils::filterToVector(testsToLaunch, [this](const UnitTest &testToLaunch) { - return testStatusMap[testToLaunch.testFilePath][testToLaunch.testname] != + return testResultMap[testToLaunch.testFilePath][testToLaunch.testname].status() != testsgen::TEST_INTERRUPTED; })); if (coverageCommands.empty()) { diff --git a/server/src/coverage/CoverageTool.cpp b/server/src/coverage/CoverageTool.cpp index 2807a4fb..74015419 100644 --- a/server/src/coverage/CoverageTool.cpp +++ b/server/src/coverage/CoverageTool.cpp @@ -8,7 +8,8 @@ using namespace CompilationUtils; -CoverageTool::CoverageTool(ProgressWriter const *progressWriter) : progressWriter(progressWriter) { +CoverageTool::CoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter) : + projectContext(std::move(projectContext)), progressWriter(progressWriter) { } std::unique_ptr getCoverageTool(const std::string &compileCommandsJsonPath, @@ -29,6 +30,10 @@ std::unique_ptr getCoverageTool(const std::string &compileCommands } } -std::string CoverageTool::getTestFilter(const UnitTest &unitTest) const { - return StringUtils::stringFormat("--gtest_filter=*.%s", unitTest.testname); +std::string CoverageTool::getGTestFlags(const UnitTest &unitTest) const { + std::string gtestFilterFlag = StringUtils::stringFormat("\"--gtest_filter=*.%s\"", unitTest.testname); + std::string gtestOutputFlag = StringUtils::stringFormat("\"--gtest_output=json:%s\"", + Paths::getGTestResultsJsonPath(projectContext)); + std::vector gtestFlagsList = { gtestFilterFlag, gtestOutputFlag }; + return StringUtils::joinWith(gtestFlagsList, " "); } diff --git a/server/src/coverage/CoverageTool.h b/server/src/coverage/CoverageTool.h index 4df350ff..67c97760 100644 --- a/server/src/coverage/CoverageTool.h +++ b/server/src/coverage/CoverageTool.h @@ -20,11 +20,12 @@ struct BuildRunCommand { class CoverageTool { protected: ProgressWriter const *progressWriter; + const utbot::ProjectContext projectContext; - [[nodiscard]] std::string getTestFilter(UnitTest const &unitTest) const; + [[nodiscard]] std::string getGTestFlags(const UnitTest &unitTest) const; public: - explicit CoverageTool(ProgressWriter const *progressWriter); + CoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter); [[nodiscard]] virtual std::vector getBuildRunCommands(const std::vector &testsToLaunch, bool withCoverage) = 0; diff --git a/server/src/coverage/GcovCoverageTool.cpp b/server/src/coverage/GcovCoverageTool.cpp index 3566d212..98793050 100644 --- a/server/src/coverage/GcovCoverageTool.cpp +++ b/server/src/coverage/GcovCoverageTool.cpp @@ -24,7 +24,7 @@ using Coverage::FileCoverage; GcovCoverageTool::GcovCoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter) - : CoverageTool(progressWriter), projectContext(std::move(projectContext)) { + : CoverageTool(std::move(projectContext), progressWriter) { } std::vector @@ -38,11 +38,11 @@ GcovCoverageTool::getBuildRunCommands(const std::vector &testsToLaunch auto makefile = Paths::getMakefilePathFromSourceFilePath( projectContext, Paths::testPathToSourcePath(projectContext, testToLaunch.testFilePath)); - auto gtestFlags = getTestFilter(testToLaunch); + auto gtestFlags = getGTestFlags(testToLaunch); auto buildCommand = - MakefileUtils::makefileCommand(projectContext, makefile, "build", gtestFlags); + MakefileUtils::MakefileCommand(projectContext, makefile, "build", gtestFlags); auto runCommand = - MakefileUtils::makefileCommand(projectContext, makefile, "run", gtestFlags); + MakefileUtils::MakefileCommand(projectContext, makefile, "run", gtestFlags); result.push_back({ testToLaunch, buildCommand, runCommand }); }); return result; diff --git a/server/src/coverage/GcovCoverageTool.h b/server/src/coverage/GcovCoverageTool.h index 55ad1896..f2e85bd0 100644 --- a/server/src/coverage/GcovCoverageTool.h +++ b/server/src/coverage/GcovCoverageTool.h @@ -23,8 +23,6 @@ class GcovCoverageTool : public CoverageTool { void cleanCoverage() const override; private: - const utbot::ProjectContext projectContext; - std::vector getGcdaFiles() const; }; diff --git a/server/src/coverage/LlvmCoverageTool.cpp b/server/src/coverage/LlvmCoverageTool.cpp index 533c64bf..bbe3888a 100644 --- a/server/src/coverage/LlvmCoverageTool.cpp +++ b/server/src/coverage/LlvmCoverageTool.cpp @@ -1,5 +1,7 @@ #include "LlvmCoverageTool.h" +#include + #include "Coverage.h" #include "Paths.h" #include "TimeExecStatistics.h" @@ -20,7 +22,7 @@ using Coverage::FileCoverage; LlvmCoverageTool::LlvmCoverageTool(utbot::ProjectContext projectContext, ProgressWriter const *progressWriter) - : CoverageTool(progressWriter), projectContext(projectContext) { + : CoverageTool(std::move(projectContext), progressWriter) { } std::vector @@ -30,15 +32,15 @@ LlvmCoverageTool::getBuildRunCommands(const std::vector &testsToLaunch Paths::testPathToSourcePath(projectContext, testToLaunch.testFilePath); auto makefilePath = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath); auto testName = testToLaunch.testname; - auto gtestFlags = getTestFilter(testToLaunch); + auto gtestFlags = getGTestFlags(testToLaunch); std::vector profileEnv; if (withCoverage) { auto profrawFilePath = Paths::getProfrawFilePath(projectContext, testName); profileEnv = { StringUtils::stringFormat("LLVM_PROFILE_FILE=%s", profrawFilePath) }; } - auto buildCommand = MakefileUtils::makefileCommand(projectContext, makefilePath, "build", + auto buildCommand = MakefileUtils::MakefileCommand(projectContext, makefilePath, "build", gtestFlags, profileEnv); - auto runCommand = MakefileUtils::makefileCommand(projectContext, makefilePath, "run", + auto runCommand = MakefileUtils::MakefileCommand(projectContext, makefilePath, "run", gtestFlags, profileEnv); return BuildRunCommand{ testToLaunch, buildCommand, runCommand }; }); @@ -73,7 +75,7 @@ LlvmCoverageTool::getCoverageCommands(const std::vector &testsToLaunch fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testFilePath); fs::path makefile = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath); - auto makefileCommand = MakefileUtils::makefileCommand(projectContext, makefile, "bin"); + auto makefileCommand = MakefileUtils::MakefileCommand(projectContext, makefile, "bin"); auto res = makefileCommand.run(); if (res.status == 0) { if (res.output.empty()) { diff --git a/server/src/coverage/LlvmCoverageTool.h b/server/src/coverage/LlvmCoverageTool.h index 69534dbc..de48be7d 100644 --- a/server/src/coverage/LlvmCoverageTool.h +++ b/server/src/coverage/LlvmCoverageTool.h @@ -17,7 +17,6 @@ class LlvmCoverageTool : public CoverageTool { [[nodiscard]] nlohmann::json getTotals() const override; void cleanCoverage() const override; private: - const utbot::ProjectContext projectContext; void countLineCoverage(Coverage::CoverageMap& coverageMap, const std::string& filename) const; void checkLineForPartial(Coverage::FileCoverage::SourceLine line, Coverage::FileCoverage& fileCoverage) const; }; diff --git a/server/src/coverage/TestRunner.cpp b/server/src/coverage/TestRunner.cpp index e5e37ef7..fc01427c 100644 --- a/server/src/coverage/TestRunner.cpp +++ b/server/src/coverage/TestRunner.cpp @@ -4,6 +4,7 @@ #include "Paths.h" #include "TimeExecStatistics.h" #include "utils/FileSystemUtils.h" +#include "utils/JsonUtils.h" #include "utils/StringUtils.h" #include "loguru.h" @@ -38,7 +39,7 @@ TestRunner::TestRunner( std::vector TestRunner::getTestsFromMakefile(const fs::path &makefile, const fs::path &testFilePath) { - auto cmdGetAllTests = MakefileUtils::makefileCommand(projectContext, makefile, "run", "--gtest_list_tests", {"GTEST_FILTER=*"}); + auto cmdGetAllTests = MakefileUtils::MakefileCommand(projectContext, makefile, "run", "--gtest_list_tests", {"GTEST_FILTER=*"}); auto [out, status, _] = cmdGetAllTests.run(projectContext.buildDir, false); if (status != 0) { auto [err, _, logFilePath] = cmdGetAllTests.run(projectContext.buildDir, true); @@ -129,21 +130,25 @@ grpc::Status TestRunner::runTests(bool withCoverage, const std::optionalgetBuildRunCommands(testsToLaunch, withCoverage); ExecUtils::doWorkWithProgress(buildRunCommands, progressWriter, "Running tests", [this, testTimeout] (BuildRunCommand const &buildRunCommand) { auto const &[unitTest, buildCommand, runCommand] = buildRunCommand; try { - auto status = runTest(runCommand, testTimeout); - testStatusMap[unitTest.testFilePath][unitTest.testname] = status; + auto status = runTest(buildRunCommand, testTimeout); + testResultMap[unitTest.testFilePath][unitTest.testname] = status; ExecUtils::throwIfCancelled(); } catch (ExecutionProcessException const &e) { - testStatusMap[unitTest.testFilePath][unitTest.testname] = testsgen::TEST_FAILED; + testsgen::TestResultObject testRes; + testRes.set_testfilepath(unitTest.testFilePath); + testRes.set_testname(unitTest.testname); + testRes.set_status(testsgen::TEST_FAILED); + testResultMap[unitTest.testFilePath][unitTest.testname] = testRes; exceptions.emplace_back(e); } }); - LOG_S(DEBUG) << "All run commands were executed"; return Status::OK; } @@ -166,7 +171,7 @@ bool TestRunner::buildTest(const utbot::ProjectContext& projectContext, const fs ExecUtils::throwIfCancelled(); fs::path makefile = Paths::getMakefilePathFromSourceFilePath(projectContext, sourcePath); if (fs::exists(makefile)) { - auto command = MakefileUtils::makefileCommand(projectContext, makefile, "build", "", {}); + auto command = MakefileUtils::MakefileCommand(projectContext, makefile, "build", "", {}); LOG_S(DEBUG) << "Try compile tests for: " << sourcePath.string(); auto[out, status, logFilePath] = command.run(projectContext.buildDir, true); if (status != 0) { @@ -187,23 +192,38 @@ size_t TestRunner::buildTests(const utbot::ProjectContext& projectContext, const return fail_count; } -testsgen::TestStatus TestRunner::runTest(const MakefileUtils::MakefileCommand &command, const std::optional &testTimeout) { - auto res = command.run(projectContext.buildDir, true, true, testTimeout); +testsgen::TestResultObject TestRunner::runTest(const BuildRunCommand &command, const std::optional &testTimeout) { + auto res = command.runCommand.run(projectContext.buildDir, true, true, testTimeout); GTestLogger::log(res.output); - if (StringUtils::contains(res.output, "[ PASSED ] 1 test")) { - return testsgen::TEST_PASSED; + + testsgen::TestResultObject testRes; + testRes.set_testfilepath(command.unitTest.testFilePath); + testRes.set_testname(command.unitTest.testname); + *testRes.mutable_executiontime() = google::protobuf::util::TimeUtil::NanosecondsToDuration(0); + + if (BaseForkTask::wasInterrupted(res.status)) { + testRes.set_status(testsgen::TEST_INTERRUPTED); + return testRes; } - if (StringUtils::contains(res.output, "[ FAILED ] 1 test")) { - return testsgen::TEST_FAILED; + if (!fs::exists(Paths::getGTestResultsJsonPath(projectContext))) { + testRes.set_status(testsgen::TEST_DEATH); + return testRes; } - if (BaseForkTask::wasInterrupted(res.status)) { - return testsgen::TEST_INTERRUPTED; + nlohmann::json gtestResultsJson = JsonUtils::getJsonFromFile(Paths::getGTestResultsJsonPath(projectContext)); + if (!google::protobuf::util::TimeUtil::FromString(gtestResultsJson["time"], testRes.mutable_executiontime())) { + LOG_S(WARNING) << "Cannot parse duration of test execution"; + } + if (gtestResultsJson["failures"] != 0) { + testRes.set_status(testsgen::TEST_FAILED); + } else { + testRes.set_status(testsgen::TEST_PASSED); } - return testsgen::TEST_DEATH; + fs::remove(Paths::getGTestResultsJsonPath(projectContext)); + return testRes; } -const Coverage::TestStatusMap &TestRunner::getTestStatusMap() const { - return testStatusMap; +const Coverage::TestResultMap &TestRunner::getTestResultMap() const { + return testResultMap; } bool TestRunner::hasExceptions() const { diff --git a/server/src/coverage/TestRunner.h b/server/src/coverage/TestRunner.h index 35ffdc1f..88790540 100644 --- a/server/src/coverage/TestRunner.h +++ b/server/src/coverage/TestRunner.h @@ -24,7 +24,7 @@ class TestRunner { std::unique_ptr coverageTool{}; std::vector testsToLaunch{}; - Coverage::TestStatusMap testStatusMap{}; + Coverage::TestResultMap testResultMap{}; std::vector exceptions; @@ -48,7 +48,7 @@ class TestRunner { std::vector getTestsToLaunch(); - const Coverage::TestStatusMap &getTestStatusMap() const; + const Coverage::TestResultMap &getTestResultMap() const; bool hasExceptions() const; @@ -68,8 +68,8 @@ class TestRunner { std::vector getTestsFromMakefile(const fs::path &makefile, const fs::path &testFilePath); - testsgen::TestStatus runTest(const MakefileUtils::MakefileCommand &command, - const std::optional &testTimeout); + testsgen::TestResultObject runTest(const BuildRunCommand &command, + const std::optional &testTimeout); ServerCoverageAndResultsWriter writer{ nullptr }; diff --git a/server/src/printers/CoverageAndResultsStatisticsPrinter.cpp b/server/src/printers/CoverageAndResultsStatisticsPrinter.cpp new file mode 100644 index 00000000..00f44eeb --- /dev/null +++ b/server/src/printers/CoverageAndResultsStatisticsPrinter.cpp @@ -0,0 +1,106 @@ +#include + +#include "CoverageAndResultsStatisticsPrinter.h" +#include "utils/StringUtils.h" +#include "utils/CollectionUtils.h" +#include "Paths.h" +#include + +namespace printer { + std::ostream &operator<<(std::ostream &os, const FileCoverageAndResultsStatistics &statistics) { + // total tests execution file + std::string totalExecutionTimeStr = google::protobuf::util::TimeUtil::ToString(statistics.totalExecutionTime); + os << totalExecutionTimeStr.substr(0, totalExecutionTimeStr.size() - 1) << ','; + // total number of tests + os << statistics.totalTestsNum << ","; + // number of passed tests + os << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_PASSED, 0u) + << ','; + // number of failed tests + os << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_FAILED, 0u) + << ','; + // number of death tests + os << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_DEATH, 0u) << ','; + // number of interrupted tests + os << CollectionUtils::getOrDefault(statistics.testsWithStatusNum, testsgen::TestStatus::TEST_INTERRUPTED, 0u) + << ','; + + uint32_t coveredLinesNum = statistics.fullCoverageLinesNum + statistics.partialCoverageLinesNum; + uint32_t totalLinesNum = coveredLinesNum + statistics.noCoverageLinesNum; + double lineCoverageRatio = 0; + if (totalLinesNum != 0) { + lineCoverageRatio = 100.0 * coveredLinesNum / totalLinesNum; + } + // total number of lines and number of covered lines + os << totalLinesNum << ',' << coveredLinesNum << ','; + // line coverage ratio + os << std::fixed << lineCoverageRatio; + return os; + } + + void CoverageAndResultsStatisticsPrinter::write(const utbot::ProjectContext &projectContext, + const Coverage::TestResultMap &testsResultMap, + const Coverage::CoverageMap &coverageMap) { + for (auto const &[testPath, testsResult]: testsResultMap) { + fs::path sourcePath = Paths::testPathToSourcePath(projectContext, testPath); + Coverage::FileCoverage fileCoverage = CollectionUtils::getOrDefault(coverageMap, + sourcePath, + Coverage::FileCoverage()); + insert({sourcePath, FileCoverageAndResultsStatistics(testsResult, fileCoverage)}); + } + std::vector metricNames = { + "Filename", "Google Test Execution Time (s)", + "Total Tests Number", "Passed Tests Number", "Failed Tests Number", "Death Tests Number", + "Interrupted Tests Number", + "Total Lines Number", "Covered Lines Number", "Line Coverage Ratio (%)" + }; + std::string header = StringUtils::joinWith(metricNames, ","); + std::stringstream ss; + ss << header << '\n'; + + FileCoverageAndResultsStatistics total; + for (auto const &[sourcePath, statistics]: *this) { + total += statistics; + ss << fs::relative(sourcePath, projectContext.projectPath).string() << ","; + ss << statistics << '\n'; + } + ss << "Total," << total << '\n'; + fs::path resultsFilePath = resultsDirectory / "coverage-and-results.csv"; + FileSystemUtils::writeToFile(resultsFilePath, ss.str()); + LOG_S(INFO) << StringUtils::stringFormat("See statistics info here: %s", resultsFilePath); + } + + FileCoverageAndResultsStatistics::FileCoverageAndResultsStatistics( + const Coverage::FileTestsResult &testsResult, + const Coverage::FileCoverage &fileCoverage) { + totalTestsNum = 0; + totalExecutionTime = google::protobuf::Duration(); + for (const auto &[_, testResult]: testsResult) { + totalTestsNum++; + testsWithStatusNum[testResult.status()]++; + totalExecutionTime += testResult.executiontime(); + } + fullCoverageLinesNum = fileCoverage.fullCoverageLines.size(); + partialCoverageLinesNum = fileCoverage.partialCoverageLines.size(); + noCoverageLinesNum = fileCoverage.noCoverageLines.size(); + } + + FileCoverageAndResultsStatistics & + FileCoverageAndResultsStatistics::operator+=(const FileCoverageAndResultsStatistics &other) { + totalExecutionTime += other.totalExecutionTime; + totalTestsNum += other.totalTestsNum; + for (const auto &[status, testsNum] : other.testsWithStatusNum) { + testsWithStatusNum[status] += testsNum; + } + fullCoverageLinesNum += other.fullCoverageLinesNum; + partialCoverageLinesNum += other.partialCoverageLinesNum; + noCoverageLinesNum += other.noCoverageLinesNum; + return *this; + } + + FileCoverageAndResultsStatistics + FileCoverageAndResultsStatistics::operator+(FileCoverageAndResultsStatistics other) const { + other += *this; + return other; + } +} diff --git a/server/src/printers/CoverageAndResultsStatisticsPrinter.h b/server/src/printers/CoverageAndResultsStatisticsPrinter.h new file mode 100644 index 00000000..b392aef1 --- /dev/null +++ b/server/src/printers/CoverageAndResultsStatisticsPrinter.h @@ -0,0 +1,48 @@ +#ifndef UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H +#define UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H + +#include +#include +#include +#include +#include +#include "coverage/Coverage.h" +#include "loguru.h" + +namespace printer { + class FileCoverageAndResultsStatistics { + public: + FileCoverageAndResultsStatistics() = default; + FileCoverageAndResultsStatistics(const Coverage::FileTestsResult &testsResult, const Coverage::FileCoverage &coverage); + + // Time statistics + google::protobuf::Duration totalExecutionTime; + + // Test runs statistics + uint32_t totalTestsNum = 0; + std::unordered_map testsWithStatusNum = {}; + + // Coverage + uint32_t fullCoverageLinesNum = 0; + uint32_t partialCoverageLinesNum = 0; + uint32_t noCoverageLinesNum = 0; + + friend std::ostream& operator<<(std::ostream &os, const FileCoverageAndResultsStatistics &statistics); + FileCoverageAndResultsStatistics& operator+=(const FileCoverageAndResultsStatistics &other); + FileCoverageAndResultsStatistics operator+(FileCoverageAndResultsStatistics other) const; + }; + + std::ostream& operator<<(std::ostream &os, const FileCoverageAndResultsStatistics &statistics); + + class CoverageAndResultsStatisticsPrinter : CollectionUtils::MapFileTo { + public: + explicit CoverageAndResultsStatisticsPrinter(fs::path resultsDirectory) : + resultsDirectory(std::move(resultsDirectory)) {}; + void write(const utbot::ProjectContext &, const Coverage::TestResultMap &, const Coverage::CoverageMap &); + + private: + fs::path resultsDirectory; + }; +} + +#endif //UNITTESTBOT_COVERAGEANDRESULTSSTATISTICSPRINTER_H diff --git a/server/src/streams/coverage/CLICoverageAndResultsWriter.cpp b/server/src/streams/coverage/CLICoverageAndResultsWriter.cpp index 71a7060b..2e02182e 100644 --- a/server/src/streams/coverage/CLICoverageAndResultsWriter.cpp +++ b/server/src/streams/coverage/CLICoverageAndResultsWriter.cpp @@ -2,6 +2,7 @@ #include "utils/FileSystemUtils.h" #include "utils/TimeUtils.h" +#include #include "loguru.h" @@ -22,17 +23,18 @@ std::string statusToString(testsgen::TestStatus status) { return it == description.end() ? "UNKNOWN" : it->second; } -void CLICoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &testsStatusMap, +void CLICoverageAndResultsWriter::writeResponse(const utbot::ProjectContext &projectContext, + const Coverage::TestResultMap &testsResultMap, const Coverage::CoverageMap &coverageMap, const nlohmann::json &totals, std::optional errorMessage) { std::stringstream ss; ss << "Test results summary." << std::endl; - for (const auto &[filepath, fileTestsStatus] : testsStatusMap) { + for (const auto &[filepath, fileTestsResult] : testsResultMap) { ss << "==== Tests in " << filepath << std::endl; - for (const auto &[testName, status] : fileTestsStatus) { - ss << "======== " << testName << " -> " << statusToString(status) << std::endl; + for (const auto &[testName, result] : fileTestsResult) { + ss << "======== " << testName << " -> " << statusToString(result.status()) << std::endl; } } @@ -52,7 +54,9 @@ void CLICoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &t } ss << "Totals:\n"; ss << totals; - fs::path resultsFilePath = resultsDirectory / (TimeUtils::getDate() + ".log"); + fs::path resultsFilePath = resultsDirectory / "tests-result.log"; FileSystemUtils::writeToFile(resultsFilePath, ss.str()); LOG_S(INFO) << ss.str(); + printer::CoverageAndResultsStatisticsPrinter statsPrinter = printer::CoverageAndResultsStatisticsPrinter(resultsDirectory); + statsPrinter.write(projectContext, testsResultMap, coverageMap); } diff --git a/server/src/streams/coverage/CLICoverageAndResultsWriter.h b/server/src/streams/coverage/CLICoverageAndResultsWriter.h index 098136cf..97bad378 100644 --- a/server/src/streams/coverage/CLICoverageAndResultsWriter.h +++ b/server/src/streams/coverage/CLICoverageAndResultsWriter.h @@ -7,7 +7,8 @@ class CLICoverageAndResultsWriter : public CoverageAndResultsWriter { public: explicit CLICoverageAndResultsWriter(const fs::path &resultsDirectory); - virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap, + virtual void writeResponse(const utbot::ProjectContext &projectContext, + const Coverage::TestResultMap &testsResultMap, const Coverage::CoverageMap &coverageMap, const nlohmann::json &totals, std::optional errorMessage) override; diff --git a/server/src/streams/coverage/CoverageAndResultsWriter.h b/server/src/streams/coverage/CoverageAndResultsWriter.h index b20d3283..f7c797be 100644 --- a/server/src/streams/coverage/CoverageAndResultsWriter.h +++ b/server/src/streams/coverage/CoverageAndResultsWriter.h @@ -15,7 +15,8 @@ class CoverageAndResultsWriter : public utbot::ServerWriter *writer); - virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap, + virtual void writeResponse(const utbot::ProjectContext &projectContext, + const Coverage::TestResultMap &testsResultMap, const Coverage::CoverageMap &coverageMap, const nlohmann::json &totals, std::optional errorMessage) = 0; diff --git a/server/src/streams/coverage/ServerCoverageAndResultsWriter.cpp b/server/src/streams/coverage/ServerCoverageAndResultsWriter.cpp index 96d01f90..aac461a1 100644 --- a/server/src/streams/coverage/ServerCoverageAndResultsWriter.cpp +++ b/server/src/streams/coverage/ServerCoverageAndResultsWriter.cpp @@ -9,9 +9,10 @@ ServerCoverageAndResultsWriter::ServerCoverageAndResultsWriter( : CoverageAndResultsWriter(writer) { } -void ServerCoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap &testsStatusMap, +void ServerCoverageAndResultsWriter::writeResponse(const utbot::ProjectContext &projectContext, + const Coverage::TestResultMap &testsResultMap, const Coverage::CoverageMap &coverageMap, - const nlohmann::json &totals, + const nlohmann::json &totals, std::optional errorMessage) { if (!hasStream()) { return; @@ -20,13 +21,10 @@ void ServerCoverageAndResultsWriter::writeResponse(const Coverage::TestStatusMap testsgen::CoverageAndResultsResponse response; - for (const auto &[filepath, fileTestsStatus] : testsStatusMap) { - for (const auto &[testname, status] : fileTestsStatus) { + for (const auto &[filepath, fileTestsResult] : testsResultMap) { + for (const auto &[testname, result] : fileTestsResult) { auto testResultsGrpc = response.add_testrunresults(); - testResultsGrpc->set_testfilepath(filepath); - testResultsGrpc->set_testname(testname); - testResultsGrpc->set_status(status); - testResultsGrpc->set_output(""); + *testResultsGrpc = result; } } diff --git a/server/src/streams/coverage/ServerCoverageAndResultsWriter.h b/server/src/streams/coverage/ServerCoverageAndResultsWriter.h index a9efc530..801a8c40 100644 --- a/server/src/streams/coverage/ServerCoverageAndResultsWriter.h +++ b/server/src/streams/coverage/ServerCoverageAndResultsWriter.h @@ -15,7 +15,8 @@ class ServerCoverageAndResultsWriter : public CoverageAndResultsWriter { explicit ServerCoverageAndResultsWriter( grpc::ServerWriter *writer); - virtual void writeResponse(const Coverage::TestStatusMap &testsStatusMap, + virtual void writeResponse(const utbot::ProjectContext &projectContext, + const Coverage::TestResultMap &testResultMap, const Coverage::CoverageMap &coverageMap, const nlohmann::json &totals, std::optional errorMessage) override; diff --git a/server/src/utils/MakefileUtils.cpp b/server/src/utils/MakefileUtils.cpp index 4e28c60b..72374931 100644 --- a/server/src/utils/MakefileUtils.cpp +++ b/server/src/utils/MakefileUtils.cpp @@ -37,11 +37,12 @@ namespace MakefileUtils { : makefile(std::move(makefile)), target(std::move(target)), projectName(projectContext.projectName) { this->makefile = this->makefile.lexically_normal(); + this->makefile = this->makefile.lexically_normal(); fs::path logDir = Paths::getLogDir(projectContext.projectName); logFile = logDir / "makefile.log"; fs::create_directories(logDir); std::vector argv = std::move(env); - argv.emplace_back(std::string("GTEST_FLAGS=\"") + gtestFlags + "\""); + argv.emplace_back(std::string("GTEST_FLAGS=") + gtestFlags); std::vector makeCommand = getMakeCommand(this->makefile, this->target, false); argv.insert(argv.begin(), makeCommand.begin(), makeCommand.end()); runCommand = ShellExecTask::ExecutionParameters("env", argv); @@ -83,15 +84,6 @@ namespace MakefileUtils { } } - - MakefileCommand makefileCommand(utbot::ProjectContext const &projectContext, - const fs::path &makefile, - const std::string &target, - const std::string >estFlags, - const std::vector &env) { - return MakefileCommand(projectContext, makefile, target, gtestFlags, env); - } - std::string threadFlag() { if (Commands::threadsPerUser != 0) { return "-j" + std::to_string(Commands::threadsPerUser); diff --git a/server/src/utils/MakefileUtils.h b/server/src/utils/MakefileUtils.h index 5c9126d6..22f9e3d7 100644 --- a/server/src/utils/MakefileUtils.h +++ b/server/src/utils/MakefileUtils.h @@ -24,8 +24,8 @@ namespace MakefileUtils { MakefileCommand(const utbot::ProjectContext &projectContext, fs::path makefile, std::string target, - const std::string >estFlags, - std::vector env); + const std::string >estFlags = "", + std::vector env = {}); [[nodiscard]] ExecUtils::ExecutionResult run(const fs::path &buildPath = "", bool redirectStderr = true, @@ -35,12 +35,6 @@ namespace MakefileUtils { [[nodiscard]] std::string getFailedCommand() const; }; - MakefileCommand makefileCommand(utbot::ProjectContext const &projectContext, - const fs::path &makefile, - const std::string &target, - const std::string >estFlags = "", - const std::vector &env = {}); - std::vector getMakeCommand(std::string makefile, std::string target, bool nested); std::string threadFlag(); diff --git a/server/test/framework/CLI_Tests.cpp b/server/test/framework/CLI_Tests.cpp index 1b45c341..59a8ac97 100644 --- a/server/test/framework/CLI_Tests.cpp +++ b/server/test/framework/CLI_Tests.cpp @@ -91,7 +91,7 @@ namespace { void checkCoverageDirectory() { FileSystemUtils::RecursiveDirectoryIterator directoryIterator(suitePath / resultsDirectoryName); - EXPECT_EQ(directoryIterator.size(), 1); + EXPECT_EQ(directoryIterator.size(), 2); for (auto &&it : directoryIterator) { EXPECT_TRUE(it.is_regular_file()); } diff --git a/server/test/framework/Server_Tests.cpp b/server/test/framework/Server_Tests.cpp index 7173db1f..372fb173 100644 --- a/server/test/framework/Server_Tests.cpp +++ b/server/test/framework/Server_Tests.cpp @@ -1331,15 +1331,15 @@ namespace { ASSERT_TRUE(coverageGenerator.getCoverageMap().empty()); - auto statusMap = coverageGenerator.getTestStatusMap(); + auto resultMap = coverageGenerator.getTestResultMap(); auto tests = coverageGenerator.getTestsToLaunch(); - ASSERT_FALSE(statusMap.empty()); + ASSERT_FALSE(resultMap.empty()); - testUtils::checkStatuses(statusMap, tests); + testUtils::checkStatuses(resultMap, tests); StatusCountMap expectedStatusCountMap{{testsgen::TEST_PASSED, 25}}; - testUtils::checkStatusesCount(statusMap, tests, expectedStatusCountMap); + testUtils::checkStatusesCount(resultMap, tests, expectedStatusCountMap); } @@ -1479,16 +1479,16 @@ namespace { ASSERT_TRUE(coverageGenerator.getCoverageMap().empty()); - auto statusMap = coverageGenerator.getTestStatusMap(); + auto resultMap = coverageGenerator.getTestResultMap(); auto tests = coverageGenerator.getTestsToLaunch(); - ASSERT_FALSE(statusMap.empty()); - EXPECT_GT(statusMap.getNumberOfTests(), 2); + ASSERT_FALSE(resultMap.empty()); + EXPECT_GT(resultMap.getNumberOfTests(), 2); - testUtils::checkStatuses(statusMap, tests); + testUtils::checkStatuses(resultMap, tests); StatusCountMap expectedStatusCountMap{{testsgen::TEST_PASSED, 7}}; - testUtils::checkStatusesCount(statusMap, tests, expectedStatusCountMap); + testUtils::checkStatusesCount(resultMap, tests, expectedStatusCountMap); } struct ProjectInfo { @@ -1552,20 +1552,20 @@ namespace { ASSERT_TRUE(coverageGenerator.getCoverageMap().empty()); - auto statusMap = coverageGenerator.getTestStatusMap(); + auto resultMap = coverageGenerator.getTestResultMap(); auto tests = coverageGenerator.getTestsToLaunch(); - ASSERT_FALSE(statusMap.empty()); - EXPECT_EQ(this->numberOfTests, statusMap.getNumberOfTests()); + ASSERT_FALSE(resultMap.empty()); + EXPECT_EQ(this->numberOfTests, resultMap.getNumberOfTests()); if (timeout) { EXPECT_EQ(testsgen::TestStatus::TEST_INTERRUPTED, - statusMap.begin()->second.begin()->second); + resultMap.begin()->second.begin()->second.status()); } else { - testUtils::checkStatuses(statusMap, tests); + testUtils::checkStatuses(resultMap, tests); StatusCountMap expectedStatusCountMap{ {testsgen::TEST_PASSED, 3}}; - testUtils::checkStatusesCount(statusMap, tests, expectedStatusCountMap); + testUtils::checkStatusesCount(resultMap, tests, expectedStatusCountMap); } } diff --git a/server/test/framework/Syntax_Tests.cpp b/server/test/framework/Syntax_Tests.cpp index 3ffd5139..0aa2d2ce 100644 --- a/server/test/framework/Syntax_Tests.cpp +++ b/server/test/framework/Syntax_Tests.cpp @@ -2181,11 +2181,11 @@ namespace { EXPECT_FALSE(coverageGenerator.hasExceptions()); ASSERT_TRUE(coverageGenerator.getCoverageMap().empty()); - auto statusMap = coverageGenerator.getTestStatusMap(); + auto resultsMap = coverageGenerator.getTestResultMap(); auto tests = coverageGenerator.getTestsToLaunch(); // TODO: add checkStatusesCount after linkedlist fix - testUtils::checkStatuses(statusMap, tests); + testUtils::checkStatuses(resultsMap, tests); } TEST_F(Syntax_Test, Run_Tests_For_Tree) { @@ -2214,15 +2214,15 @@ namespace { EXPECT_FALSE(coverageGenerator.hasExceptions()); ASSERT_TRUE(coverageGenerator.getCoverageMap().empty()); - auto statusMap = coverageGenerator.getTestStatusMap(); + auto resultMap = coverageGenerator.getTestResultMap(); auto tests = coverageGenerator.getTestsToLaunch(); - testUtils::checkStatuses(statusMap, tests); + testUtils::checkStatuses(resultMap, tests); StatusCountMap expectedStatusCountMap{ {testsgen::TEST_DEATH, 4}, {testsgen::TEST_PASSED, 6}}; - testUtils::checkStatusesCount(statusMap, tests, expectedStatusCountMap); + testUtils::checkStatusesCount(resultMap, tests, expectedStatusCountMap); } TEST_F(Syntax_Test, Simple_parameter_cpp) { diff --git a/server/test/framework/TestUtils.cpp b/server/test/framework/TestUtils.cpp index 5424967f..9cdb0882 100644 --- a/server/test/framework/TestUtils.cpp +++ b/server/test/framework/TestUtils.cpp @@ -130,20 +130,20 @@ namespace testUtils { } } - void checkStatuses(const Coverage::TestStatusMap &testStatusMap, + void checkStatuses(const Coverage::TestResultMap &testResultMap, const std::vector &tests) { for (auto const &[filename, suitename, testname] : tests) { if (suitename == tests::Tests::ERROR_SUITE_NAME) { continue; } - const auto status = testStatusMap.at(filename).at(testname); + const auto status = testResultMap.at(filename).at(testname).status(); EXPECT_TRUE((testsgen::TestStatus::TEST_PASSED == status) || (testsgen::TestStatus::TEST_DEATH == status)); } } - void checkStatusesCount(const Coverage::TestStatusMap &testStatusMap, + void checkStatusesCount(const Coverage::TestResultMap &testResultMap, const std::vector &tests, const StatusCountMap &expectedStatusCountMap) { StatusCountMap actualStatusCountMap; @@ -151,7 +151,7 @@ namespace testUtils { if (suitename == tests::Tests::ERROR_SUITE_NAME) { continue; } - const auto status = testStatusMap.at(filename).at(testname); + const auto status = testResultMap.at(filename).at(testname).status(); actualStatusCountMap[status]++; } for (const auto& [status, count] : actualStatusCountMap) { diff --git a/server/test/framework/TestUtils.h b/server/test/framework/TestUtils.h index 438b7fd6..c5998089 100644 --- a/server/test/framework/TestUtils.h +++ b/server/test/framework/TestUtils.h @@ -41,9 +41,9 @@ namespace testUtils { const CoverageLines &expectedLinesUncovered, const CoverageLines &expectedLinesNone); - void checkStatuses(const Coverage::TestStatusMap &testStatusMap, const std::vector &tests); + void checkStatuses(const Coverage::TestResultMap &testResultMap, const std::vector &tests); - void checkStatusesCount(const Coverage::TestStatusMap &testStatusMap, + void checkStatusesCount(const Coverage::TestResultMap &testResultsMap, const std::vector &tests, const StatusCountMap &expectedStatusCountMap);