diff --git a/dependencies/spork.jar b/dependencies/spork.jar new file mode 100644 index 000000000..d309c1278 Binary files /dev/null and b/dependencies/spork.jar differ diff --git a/src/main/app/MiningWorker.groovy b/src/main/app/MiningWorker.groovy index 47c5831e3..3a98accaa 100644 --- a/src/main/app/MiningWorker.groovy +++ b/src/main/app/MiningWorker.groovy @@ -1,27 +1,37 @@ package app +import exception.UnstagedChangesException +import interfaces.CommitFilter +import interfaces.DataCollector +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import project.MergeCommit +import project.Project +import project.commitHashesExtraction.CommitHashesExtractor +import services.util.Utils +import util.FileManager +import util.MergeHelper +import util.ProcessRunner import java.util.Collections import java.util.Random import java.text.SimpleDateFormat -import static app.MiningFramework.arguments +import java.text.SimpleDateFormat import java.util.concurrent.BlockingQueue -import project.* -import interfaces.* -import exception.UnstagedChangesException -import util.* - -import services.util.Utils; +import static app.MiningFramework.arguments class MiningWorker implements Runnable { + private static Logger LOG = LogManager.getLogger(MiningWorker.class) private Set dataCollectors private CommitFilter commitFilter private BlockingQueue projectList private String baseDir + private CommitHashesExtractor commitHashesExtractor MiningWorker(Set dataCollectors, CommitFilter commitFilter, BlockingQueue projectList, String baseDir) { + this.commitHashesExtractor = CommitHashesExtractor.Factory.build() this.dataCollectors = dataCollectors this.commitFilter = commitFilter this.projectList = projectList @@ -41,7 +51,7 @@ class MiningWorker implements Runnable { checkForUnstagedChanges(project); } - def (mergeCommits, skipped) = project.getMergeCommits(arguments.getSinceDate(), arguments.getUntilDate()) + def (mergeCommits, skipped) = project.getMergeCommits(commitHashesExtractor) Random random = new Random(arguments.getRandomSeed()) Collections.shuffle(mergeCommits, random) @@ -97,7 +107,7 @@ class MiningWorker implements Runnable { } private void cloneRepository(Project project, String target) { - println "Cloning repository ${project.getName()} into ${target}" + LOG.info("Cloning repository ${project.getName()} into ${target}") File projectDirectory = new File(target) if (projectDirectory.exists()) { @@ -114,10 +124,11 @@ class MiningWorker implements Runnable { } ProcessBuilder builder = ProcessRunner.buildProcess('./', 'git', 'clone', url, target) - builder.redirectOutput(ProcessBuilder.Redirect.INHERIT) - Process process = ProcessRunner.startProcess(builder) + process.getInputStream().eachLine(LOG::trace) + process.getErrorStream().eachLine(LOG::warn) process.waitFor() + LOG.info("Finished cloning repository ${project.getName()} into ${target}") project.setPath(target) } diff --git a/src/main/arguments/ArgsParser.groovy b/src/main/arguments/ArgsParser.groovy index 822170b3a..924397df1 100644 --- a/src/main/arguments/ArgsParser.groovy +++ b/src/main/arguments/ArgsParser.groovy @@ -39,6 +39,7 @@ class ArgsParser { this.cli.e(longOpt: 'extension', args: 1, argName: 'file extenson', 'Specify the file extension that should be used in the analysis (e.g. .rb, .ts, .java, .cpp. Default: .java)') this.cli.l(longOpt: 'language-separators', args: 1, argName: 'language syntactic separators', 'Specify the language separators that should be used in the analysis. Required for (and only considered when) running studies with the CSDiff tool. Default: \"{ } ( ) ; ,\"') this.cli.log(longOpt: 'log-level', args: 1, argName: 'log level', 'Specify the minimum log level: (OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE, ALL). Default: \"INFO\"') + this.cli.pchf(longOpt: 'project-commit-hashes-file', args: 1, argName: 'project commit hashes file', 'Path to a file with projects and commits to be analyzed') } Arguments parse(args) { @@ -142,6 +143,10 @@ class ArgsParser { if(this.options.log) { args.setLogLevel(Level.toLevel(this.options.log)) } + + if (this.options.pchf) { + args.setProjectCommitHashesFile(this.options.pchf) + } } private boolean repositoryExists(String repositoryURL) { diff --git a/src/main/arguments/Arguments.groovy b/src/main/arguments/Arguments.groovy index fed1b55c2..ffc3fcce8 100644 --- a/src/main/arguments/Arguments.groovy +++ b/src/main/arguments/Arguments.groovy @@ -21,6 +21,7 @@ class Arguments { private String syntacticSeparators private String fileExtension private Level logLevel + private String projectCommitHashesFile Arguments() { // set the default values for all parameters randomSeed = 1 @@ -37,6 +38,7 @@ class Arguments { syntacticSeparators = '{ } ( ) ; ,' fileExtension = '.java' logLevel = Level.INFO + projectCommitHashesFile = '' } void setRandomSeed(int randomSeed) { @@ -167,4 +169,12 @@ class Arguments { this.logLevel = logLevel Configurator.setRootLevel(logLevel) } + + String getProjectCommitHashesFile() { + return projectCommitHashesFile + } + + void setProjectCommitHashesFile(String projectCommitHashesFile) { + this.projectCommitHashesFile = projectCommitHashesFile + } } \ No newline at end of file diff --git a/src/main/injectors/GenericMergeModule.groovy b/src/main/injectors/GenericMergeModule.groovy index 9ce457a4a..0c87f018a 100644 --- a/src/main/injectors/GenericMergeModule.groovy +++ b/src/main/injectors/GenericMergeModule.groovy @@ -6,51 +6,77 @@ import interfaces.CommitFilter import interfaces.DataCollector import interfaces.OutputProcessor import interfaces.ProjectProcessor -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger import services.commitFilters.MutuallyModifiedFilesCommitFilter -import services.dataCollectors.GenericMerge.GenericMergeConfig -import services.dataCollectors.GenericMerge.GenericMergeDataCollector -import services.dataCollectors.GenericMerge.MergeConflictsComparator -import services.dataCollectors.GenericMerge.MergeToolsComparator -import services.dataCollectors.GenericMerge.ScenarioLOCsCounter -import services.dataCollectors.GenericMerge.UnstructuredMergeCollector -import services.outputProcessors.GenericMergeDataOutputProcessor +import services.dataCollectors.buildRequester.RequestBuildForRevisionWithFilesDataCollector +import services.dataCollectors.common.CompareScenarioMergeConflictsDataCollector +import services.dataCollectors.common.RunDataCollectorsInParallel +import services.dataCollectors.common.SyntacticallyCompareScenarioFilesDataCollector +import services.dataCollectors.fileSyntacticNormalization.FormatFileSyntacticNormalizationDataCollector +import services.dataCollectors.fileSyntacticNormalization.JDimeFileSyntacticNormalizationDataCollector +import services.dataCollectors.fileSyntacticNormalization.SporkFileSyntacticNormalizationDataCollector +import services.dataCollectors.mergeToolExecutors.JDimeMergeToolExecutorDataCollector +import services.dataCollectors.mergeToolExecutors.LastMergeMergeToolExecutorDataCollector +import services.dataCollectors.mergeToolExecutors.MergirafMergeToolExecutorDataCollector +import services.dataCollectors.mergeToolExecutors.SporkMergeToolExecutorDataCollector +import services.outputProcessors.EmptyOutputProcessor import services.projectProcessors.DummyProjectProcessor -import services.util.ci.CIPlatform -import services.util.ci.TravisPlatform - -import java.nio.file.Files -import java.nio.file.Paths class GenericMergeModule extends AbstractModule { - private static Logger LOG = LogManager.getLogger(GenericMergeModule.class) - @Override protected void configure() { Multibinder projectProcessorBinder = Multibinder.newSetBinder(binder(), ProjectProcessor.class) projectProcessorBinder.addBinding().to(DummyProjectProcessor.class) Multibinder dataCollectorBinder = Multibinder.newSetBinder(binder(), DataCollector.class) - dataCollectorBinder.addBinding().to(ScenarioLOCsCounter.class) - dataCollectorBinder.addBinding().to(GenericMergeDataCollector.class) - dataCollectorBinder.addBinding().to(MergeToolsComparator.class) - dataCollectorBinder.addBinding().to(MergeConflictsComparator.class) - dataCollectorBinder.addBinding().to(UnstructuredMergeCollector.class) - Multibinder outputProcessorBinder = Multibinder.newSetBinder(binder(), OutputProcessor.class) - outputProcessorBinder.addBinding().to(GenericMergeDataOutputProcessor.class) + // Run the merge tools on the scenarios + dataCollectorBinder.addBinding().to(JDimeMergeToolExecutorDataCollector.class) + dataCollectorBinder.addBinding().to(LastMergeMergeToolExecutorDataCollector.class) + dataCollectorBinder.addBinding().to(SporkMergeToolExecutorDataCollector.class) + dataCollectorBinder.addBinding().to(MergirafMergeToolExecutorDataCollector.class) - bind(CommitFilter.class).to(MutuallyModifiedFilesCommitFilter.class) - bind(CIPlatform.class).to(TravisPlatform.class) + // Run, in parallel, normalizations on resulting files + dataCollectorBinder.addBinding().toInstance(new RunDataCollectorsInParallel([ + new JDimeFileSyntacticNormalizationDataCollector("merge.java", "merge.jdime_normalized.java"), + new JDimeFileSyntacticNormalizationDataCollector("merge.last_merge.java", "merge.last_merge.jdime_normalized.java"), + new SporkFileSyntacticNormalizationDataCollector("merge.java", "merge.spork_normalized.java"), + new SporkFileSyntacticNormalizationDataCollector("merge.spork.java", "merge.spork.spork_normalized.java"), + new SporkFileSyntacticNormalizationDataCollector("merge.mergiraf.java", "merge.mergiraf.spork_normalized.java"), + ])) - createExecutionReportsFile() - } + // Run, in parallel, syntactical comparisons between files and textual comparison between commits + dataCollectorBinder.addBinding().toInstance(new RunDataCollectorsInParallel([ + // Syntactic comparison with merge commits + new SyntacticallyCompareScenarioFilesDataCollector("merge.java", "merge.last_merge.java"), + new SyntacticallyCompareScenarioFilesDataCollector("merge.jdime_normalized.java", "merge.jdime.java"), + new SyntacticallyCompareScenarioFilesDataCollector("merge.spork_normalized.java", "merge.spork.spork_normalized.java"), + new SyntacticallyCompareScenarioFilesDataCollector("merge.java", "merge.mergiraf.java"), + + // Syntactic comparison between tools themselves + new SyntacticallyCompareScenarioFilesDataCollector("merge.jdime.java", "merge.last_merge.jdime_normalized.java"), + new SyntacticallyCompareScenarioFilesDataCollector("merge.mergiraf.spork_normalized.java", "merge.spork.spork_normalized.java"), + new SyntacticallyCompareScenarioFilesDataCollector("merge.mergiraf.java", "merge.last_merge.java"), + + // Conflicts comparison between tools themselves + new CompareScenarioMergeConflictsDataCollector("merge.jdime.java", "merge.last_merge.java"), + new CompareScenarioMergeConflictsDataCollector("merge.mergiraf.java", "merge.spork.java"), + new CompareScenarioMergeConflictsDataCollector("merge.mergiraf.java", "merge.last_merge.java"), + ])) - private static void createExecutionReportsFile() { - LOG.info("Creating Generic Merge report file") - Files.createDirectories(Paths.get(GenericMergeConfig.GENERIC_MERGE_REPORT_PATH)) - def reportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_FILE_NAME) - reportFile.createNewFile() + dataCollectorBinder.addBinding().toInstance(new RunDataCollectorsInParallel([new FormatFileSyntacticNormalizationDataCollector("merge.last_merge.java", "merge.last_merge.format_normalized.java"), + new FormatFileSyntacticNormalizationDataCollector("merge.mergiraf.java", "merge.mergiraf.format_normalized.java")])) + + dataCollectorBinder.addBinding().toInstance(new SyntacticallyCompareScenarioFilesDataCollector("merge.mergiraf.format_normalized.java", "merge.last_merge.format_normalized.java")) + + + dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.last_merge.java")) + dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.jdime.java")) + dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.mergiraf.java")) + dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.spork.java")) + + Multibinder outputProcessorBinder = Multibinder.newSetBinder(binder(), OutputProcessor.class) + outputProcessorBinder.addBinding().to(EmptyOutputProcessor.class) + + bind(CommitFilter.class).to(MutuallyModifiedFilesCommitFilter.class) } } diff --git a/src/main/project/Project.groovy b/src/main/project/Project.groovy index 857aea875..b80f9b8e6 100644 --- a/src/main/project/Project.groovy +++ b/src/main/project/Project.groovy @@ -1,14 +1,15 @@ package project -import util.ProcessRunner import exception.UnexpectedOutputException +import org.apache.logging.log4j.LogManager +import project.commitHashesExtraction.CommitHashesExtractor +import util.ProcessRunner -import java.util.Collections -import java.util.Random -import java.util.regex.Pattern import java.util.regex.Matcher +import java.util.regex.Pattern class Project { + private static LOG = LogManager.getLogger(Project.class) private String name private String path @@ -42,49 +43,41 @@ class Project { return matcher.find() } - List getMergeCommits(String sinceDate, String untilDate) { + List getMergeCommits(CommitHashesExtractor commitHashesExtractor) { ArrayList skipped = new ArrayList() ArrayList mergeCommits = new ArrayList() - - Process gitLog = constructAndRunGitLog(sinceDate, untilDate) - def expectedOutput = ~/.*-(.* .*)+/ - gitLog.getInputStream().eachLine { - // Each line contains the hash of the commit followed by the hashes of the parents. - if(it ==~ expectedOutput) { - - String[] informations = it.split('-') // - - String SHA = getSHA(informations) - String[] parentsSHA = getParentsSHA(informations) + commitHashesExtractor.extractCommitHashes(this).each(commitHashes -> { + def SHA = commitHashes.mergeSha + def parentsSHA = commitHashes.parents + try { + String ancestorSHA = getCommonAncestor(SHA, parentsSHA) + MergeCommit mergeCommit = new MergeCommit(SHA, parentsSHA, ancestorSHA) + mergeCommits.add(mergeCommit) + } catch (UnexpectedOutputException e) { + println e.message try { + println "Trying to fetch it" + // Let's try fetching it before skipping it + constructAndRunGitFetch(SHA) String ancestorSHA = getCommonAncestor(SHA, parentsSHA) MergeCommit mergeCommit = new MergeCommit(SHA, parentsSHA, ancestorSHA) mergeCommits.add(mergeCommit) - } catch (UnexpectedOutputException e) { + } catch (UnexpectedOutputException ex) { println "Skipping merge commit ${SHA}" - println e.message + println ex.message skipped.add(SHA) } - } else { - throw new UnexpectedOutputException('Git log returned an unexpected output. Could not retrieve merge commits.', '-', it) } - } - + }) + if(mergeCommits.isEmpty()) println "No merge commits." return [mergeCommits, skipped] } - private String getSHA(String[] informations) { - return informations[0] - } - - private String[] getParentsSHA(String[] informations) { - return informations[1].split(' ') - } - private String getCommonAncestor(mergeCommitSHA, parentsSHA) { Process gitMergeBase = constructAndRunGitMergeBase(mergeCommitSHA, parentsSHA) def expectedOutput = ~/[0-9a-z]{7,}/ @@ -96,6 +89,14 @@ class Project { throw new UnexpectedOutputException('Git merge-base returned an unexpected output. Could not retrieve the ancestor commit.', '', actualOutput) } + private void constructAndRunGitFetch(String mergeCommitSHA) { + ProcessBuilder gitMergeBaseBuilder = ProcessRunner.buildProcess(path, 'git', 'fetch', 'origin', mergeCommitSHA) + def exitCode = ProcessRunner.startProcess(gitMergeBaseBuilder).waitFor() + if (exitCode > 0) { + throw new UnexpectedOutputException("Could not fetch merge commit ${mergeCommitSHA} from origin", "Exit code 0", "Exit code ${exitCode}") + } + } + private Process constructAndRunGitMergeBase(String mergeCommitSHA, String[] parentsSHA) { ProcessBuilder gitMergeBaseBuilder = ProcessRunner.buildProcess(path, 'git', 'merge-base') if (parentsSHA.length > 2) @@ -105,15 +106,6 @@ class Project { return ProcessRunner.startProcess(gitMergeBaseBuilder) } - private Process constructAndRunGitLog(String sinceDate, String untilDate) { - ProcessBuilder gitLogBuilder = ProcessRunner.buildProcess(path, 'git', '--no-pager', 'log', '--merges', '--pretty=%H-%p') - if(!sinceDate.equals('')) - ProcessRunner.addCommand(gitLogBuilder, "--since=\"${sinceDate}\"") - if(!untilDate.equals('')) - ProcessRunner.addCommand(gitLogBuilder, "--until=\"${untilDate}\"") - return ProcessRunner.startProcess(gitLogBuilder) - } - String getName() { return name } diff --git a/src/main/project/commitHashesExtraction/CommitHashesExtractor.groovy b/src/main/project/commitHashesExtraction/CommitHashesExtractor.groovy new file mode 100644 index 000000000..b3d47809b --- /dev/null +++ b/src/main/project/commitHashesExtraction/CommitHashesExtractor.groovy @@ -0,0 +1,33 @@ +package project.commitHashesExtraction + +import project.Project + +import static app.MiningFramework.arguments + +interface CommitHashesExtractor { + class CommitHashes { + public String mergeSha + public String[] parents + + CommitHashes(String mergeSha, String... parents) { + this.mergeSha = mergeSha + this.parents = parents + } + } + + class Factory { + static CommitHashesExtractor build() { + if (arguments.projectCommitHashesFile != '') { + def file = new File(arguments.projectCommitHashesFile) + if (!file.exists()) { + throw new RuntimeException("File ${file.path} not found") + } + return new ProjectCommitListCommitHashesExtractor(file) + } + + return new GitLogCommitHashesExtractor(arguments.sinceDate, arguments.untilDate) + } + } + + List extractCommitHashes(Project project) +} \ No newline at end of file diff --git a/src/main/project/commitHashesExtraction/GitLogCommitHashesExtractor.groovy b/src/main/project/commitHashesExtraction/GitLogCommitHashesExtractor.groovy new file mode 100644 index 000000000..325df7fd3 --- /dev/null +++ b/src/main/project/commitHashesExtraction/GitLogCommitHashesExtractor.groovy @@ -0,0 +1,46 @@ +package project.commitHashesExtraction + +import exception.UnexpectedOutputException +import project.Project +import util.ProcessRunner + +class GitLogCommitHashesExtractor implements CommitHashesExtractor { + private String sinceDate + private String untilDate + + private static final EXPECTED_OUTPUT = ~/.*-(.* .*)+/ + + GitLogCommitHashesExtractor(String sinceDate, String untilDate) { + this.sinceDate = sinceDate + this.untilDate = untilDate + } + + @Override + List extractCommitHashes(Project project) { + List result = new ArrayList() + + def gitLog = constructAndRunGitLog(project) + gitLog.getInputStream().eachLine { + // Each line contains the hash of the commit followed by the hashes of the parents. + if (it ==~ EXPECTED_OUTPUT) { + def informations = it.split('-') // - + def mergeSHA = informations[0] + def parentsSHA = informations[1].split(' ') + result.add(new CommitHashes(mergeSHA, parentsSHA)) + } else { + throw new UnexpectedOutputException('Git log returned an unexpected output. Could not retrieve merge commits.', '-', it) + } + } + + return result + } + + private Process constructAndRunGitLog(String sinceDate, String untilDate) { + ProcessBuilder gitLogBuilder = ProcessRunner.buildProcess(path, 'git', '--no-pager', 'log', '--merges', '--pretty=%H-%p') + if(!sinceDate.equals('')) + ProcessRunner.addCommand(gitLogBuilder, "--since=\"${sinceDate}\"") + if(!untilDate.equals('')) + ProcessRunner.addCommand(gitLogBuilder, "--until=\"${untilDate}\"") + return ProcessRunner.startProcess(gitLogBuilder) + } +} diff --git a/src/main/project/commitHashesExtraction/ProjectCommitListCommitHashesExtractor.groovy b/src/main/project/commitHashesExtraction/ProjectCommitListCommitHashesExtractor.groovy new file mode 100644 index 000000000..622a149c9 --- /dev/null +++ b/src/main/project/commitHashesExtraction/ProjectCommitListCommitHashesExtractor.groovy @@ -0,0 +1,31 @@ +package project.commitHashesExtraction + +import project.Project + +import static com.xlson.groovycsv.CsvParser.parseCsv + +class ProjectCommitListCommitHashesExtractor implements CommitHashesExtractor { + private Map> projectCommitHashes = new HashMap<>() + + ProjectCommitListCommitHashesExtractor(File projectCommitHashesFile) { + for (line in parseCsv(projectCommitHashesFile.getText())) { + String project = line['project'] + String merge = line['merge'] + String left = line['left'] + String right = line['right'] + + def commitHashes = new CommitHashes(merge, left, right) + + if (projectCommitHashes.containsKey(project)) { + projectCommitHashes.get(project).add(commitHashes) + } else { + projectCommitHashes.put(project, new ArrayList([commitHashes])) + } + } + } + + @Override + List extractCommitHashes(Project project) { + return projectCommitHashes.get(project.getName()) + } +} diff --git a/src/main/services/commitFilters/IsInCommitListFilter.groovy b/src/main/services/commitFilters/IsInCommitListFilter.groovy index 635add3d4..044d0cb71 100644 --- a/src/main/services/commitFilters/IsInCommitListFilter.groovy +++ b/src/main/services/commitFilters/IsInCommitListFilter.groovy @@ -1,6 +1,8 @@ package services.commitFilters import interfaces.CommitFilter +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger import project.MergeCommit import project.Project @@ -11,43 +13,33 @@ import static com.xlson.groovycsv.CsvParser.parseCsv * @provides: returns true if the passed mergeCommit SHA is in the commits.csv file */ class IsInCommitListFilter implements CommitFilter { + private static Logger LOG = LogManager.getLogger(IsInCommitListFilter.class) List commitList - @Override - boolean applyFilter(Project project, MergeCommit mergeCommit) { - boolean result = true - File commitsFile = new File("./commits.csv") - - if (commitList != null || commitsFile.exists()) { - // using this check to cache the commitList between multiple executions - // avoiding unnecessary IO operations - if (commitList == null) { - commitList = parseCommitList(commitsFile) - } + IsInCommitListFilter() { + this.commitList = initializeCommitList(new File("./commits.csv")) + } - result = isInCommitList(mergeCommit) + private static List initializeCommitList(File commitsFile) { + if (!commitsFile.exists()) { + LOG.trace("Skipping initialization because the commits.csv file do not exist") + return new ArrayList<>() } - return result - } - private List parseCommitList (File commitsFile) { ArrayList commitList = new ArrayList() def iterator = parseCsv(commitsFile.getText()) for (line in iterator) { + LOG.trace("Registering ${line["commitSHA"]} as a valid commit SHA in list") commitList.add(line["commitSHA"]) } return commitList } - private boolean isInCommitList (MergeCommit mergeCommit) { - for (commit in commitList) { - if (mergeCommit.getSHA() == commit) { - return true; - } - } - return false; + @Override + boolean applyFilter(Project project, MergeCommit mergeCommit) { + return commitList.isEmpty() || commitList.contains(mergeCommit.getSHA()) } } diff --git a/src/main/services/commitFilters/IsInProjectCommitListFilter.groovy b/src/main/services/commitFilters/IsInProjectCommitListFilter.groovy new file mode 100644 index 000000000..5fb8650c1 --- /dev/null +++ b/src/main/services/commitFilters/IsInProjectCommitListFilter.groovy @@ -0,0 +1,51 @@ +package services.commitFilters + +import interfaces.CommitFilter +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import project.MergeCommit +import project.Project + +import static com.xlson.groovycsv.CsvParser.parseCsv + +/** + * @requires: that a commits.csv file exists on the project's root, otherwise it will always return true + * @provides: returns true if the passed mergeCommit SHA is in the commits.csv file + */ +class IsInProjectCommitListFilter implements CommitFilter { + private static Logger LOG = LogManager.getLogger(IsInProjectCommitListFilter.class) + + Map> commitList + + IsInProjectCommitListFilter() { + this.commitList = initializeCommitList(new File("./commits.csv")) + } + + private static Map> initializeCommitList(File commitsFile) { + if (!commitsFile.exists()) { + LOG.trace("Skipping initialization because the commits.csv file do not exist") + return new HashMap>() + } + + def commitList = new HashMap>() + def iterator = parseCsv(commitsFile.getText()) + + for (line in iterator) { + LOG.trace("Registering ${line["commitSHA"]} as a valid commit SHA in list") + String project = line['project'] + String commitSHA = line['commitSHA'] + if (commitList.containsKey(project)) { + commitList.get(project).add(commitSHA) + } else { + commitList.put(project, new HashSet<>([commitSHA])) + } + } + + return commitList + } + + @Override + boolean applyFilter(Project project, MergeCommit mergeCommit) { + return commitList.isEmpty() || commitList.containsKey(project.getName()) && commitList.get(project.getName()).contains(mergeCommit.getSHA()) + } +} diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy deleted file mode 100644 index 2d4567cbd..000000000 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ /dev/null @@ -1,164 +0,0 @@ -package services.dataCollectors.GenericMerge - -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import project.MergeCommit -import project.Project -import services.util.Utils - -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.nio.file.StandardCopyOption - -class BuildRequester { - private static Logger LOG = LogManager.getLogger(BuildRequester.class) - - static requestBuildWithRevision(Project project, MergeCommit mergeCommit, List mergeScenarios, String mergeTool) { - String toReplaceFile = "merge.${mergeTool.toLowerCase()}.java" - - String branchName = "${mergeCommit.getSHA().take(7)}-${mergeTool}" - - createBranchFromCommit(project, mergeCommit, branchName) - replaceFilesInProject(project, mergeCommit, mergeScenarios, toReplaceFile) - createOrReplaceGithubActionsFile(project) - def commitSha = stageAndPushChanges(project, branchName, "Mining Framework Analysis") - - def reportFile = new File(GenericMergeConfig.BUILD_REQUESTER_REPORT_PATH) - reportFile.createNewFile() - reportFile.append("${project.getName()},${branchName},${mergeTool},${commitSha}\n") - } - - private static void createBranchFromCommit(Project project, MergeCommit mergeCommit, String branchName) { - Path projectPath = Paths.get(project.getPath()) - - // Checkout to new branch - Utils.runGitCommand(projectPath, 'checkout', '-b', branchName, mergeCommit.getSHA()) - } - - private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { - mergeScenarios.stream() - .forEach(mergeScenario -> { - LOG.debug("Trying to copy " + getSource(mergeScenario, toReplaceFile) + " into " + getTarget(project, mergeCommit, mergeScenario)) - Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING) - }) - } - - private static void createOrReplaceGithubActionsFile(Project project) { - LOG.debug("Starting creation of github actions file") - def githubActionsFilePath = "${Paths.get(project.getPath()).toAbsolutePath().toString()}/.github/workflows" - LOG.debug("Location of github actions folder ${githubActionsFilePath}") - - def buildSystem = getBuildSystemForProject(project) - LOG.debug("Using ${buildSystem.class.getSimpleName()} as build system for project ${project.getName()}") - - def githubActionsContent = """ -name: Mining Framework Check -on: [push] -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-java@v1 - with: - java-version: 11 - - run: ${buildSystem.getBuildCommand()} - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-java@v1 - with: - java-version: 11 - - run: ${buildSystem.getTestCommand()} -""" - Files.createDirectories(Paths.get(githubActionsFilePath)) - def file = new File("${githubActionsFilePath}/mining_framework.yml") - file.createNewFile() - file.write(githubActionsContent) - LOG.debug("Finished creation of github actions file") - } - - private static Path getSource(Path mergeScenario, String toReplaceFile) { - return mergeScenario.resolve(toReplaceFile) - } - - private static Path getTarget(Project project, MergeCommit mergeCommit, Path mergeScenario) { - Path projectPath = Paths.get(project.getPath()) - Path filePath = Utils.commitFilesPath(project, mergeCommit).relativize(mergeScenario) - return projectPath.resolve(filePath) - } - - private static String stageAndPushChanges(Project project, String branchName, String commitMessage) { - Path projectPath = Paths.get(project.getPath()) - - // Stage changes - Utils.runGitCommand(projectPath, 'add', '.') - - // Commit changes - Utils.runGitCommand(projectPath, 'commit', '-m', commitMessage) - def commitSha = Utils.runGitCommand(projectPath, 'rev-parse', 'HEAD') - LOG.debug("Created commit with hash ${commitSha}") - - // Push changes - Utils.runGitCommand(projectPath, 'push', '--set-upstream', 'origin', branchName, '--force-with-lease') - - return commitSha.get(0) - } - - private static interface BuildSystem { - String getBuildCommand() - - String getTestCommand() - } - - private static class MavenBuildSystem implements BuildSystem { - @Override - String getBuildCommand() { - return "mvn package -Dskiptests" - } - - @Override - String getTestCommand() { - return "mvn test" - } - } - - private static class GradleBuildSystem implements BuildSystem { - @Override - String getBuildCommand() { - return "./gradlew assemble" - } - - @Override - String getTestCommand() { - return "./gradlew test" - } - } - - private static class NoopBuildSystem implements BuildSystem { - @Override - String getBuildCommand() { - return "echo no build available" - } - - @Override - String getTestCommand() { - return "echo no test available" - } - } - - private static BuildSystem getBuildSystemForProject(Project project) { - File mavenFile = new File("${project.getPath()}/pom.xml") - File gradleFile = new File("${project.getPath()}/build.gradle") - - if (mavenFile.exists()) { - return new MavenBuildSystem() - } else if (gradleFile.exists()) { - return new GradleBuildSystem() - } else { - return new NoopBuildSystem() - } - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy b/src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy deleted file mode 100644 index c2a3d2554..000000000 --- a/src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy +++ /dev/null @@ -1,27 +0,0 @@ -package services.dataCollectors.GenericMerge - -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import util.ProcessRunner - -import java.nio.file.Path - -class FileFormatNormalizer { - private static Logger LOG = LogManager.getLogger(FileFormatNormalizer.class) - - static void normalizeFileInPlace(Path file) { - def processBuilder = ProcessRunner.buildProcess(GenericMergeConfig.JDIME_BINARY_PATH, - "./JDime", - "-f", - "--mode=structured", - "--output=${file.toAbsolutePath().toString()}".toString(), - file.toAbsolutePath().toString(), - file.toAbsolutePath().toString(), - file.toAbsolutePath().toString()) - - def exitCode = ProcessRunner.startProcess(processBuilder).waitFor() - if (exitCode != 0) { - LOG.warn("File normalization failed with exit code ${exitCode}") - } - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy b/src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy deleted file mode 100644 index f6471494b..000000000 --- a/src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy +++ /dev/null @@ -1,38 +0,0 @@ -package services.dataCollectors.GenericMerge - -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import util.ProcessRunner - -import java.nio.file.Files -import java.nio.file.Path - -class FileSyntacticDiff { - private static Logger LOG = LogManager.getLogger(FileSyntacticDiff.class) - - static boolean areFilesSyntacticallyEquivalent(Path fileA, Path fileB) { - if (!Files.exists(fileA) || !Files.exists(fileB)) { - LOG.trace("Early returning because one of the files ${} do not exist") - return false - } - - def process = ProcessRunner.buildProcess("./") - - def list = new ArrayList() - list.add(GenericMergeConfig.GENERIC_MERGE_BINARY_PATH) - list.add("diff") - list.add("--left-path=${fileA.toAbsolutePath().toString()}".toString()) - list.add("--right-path=${fileB.toAbsolutePath().toString()}".toString()) - list.add("--language=java") - process.command().addAll(list) - - def output = ProcessRunner.startProcess(process) - output.waitFor() - - if (output.exitValue() > 1) { - LOG.warn("Error while running comparison between ${fileA.toString()} and ${fileB.toString()}: ${output.getInputStream().readLines()}") - } - - return output.exitValue() == 0 - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy deleted file mode 100644 index c503ab2ba..000000000 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy +++ /dev/null @@ -1,20 +0,0 @@ -package services.dataCollectors.GenericMerge - -class GenericMergeConfig { - public static final BASE_EXPERIMENT_PATH = System.getProperty("user.dir") - - public static final BUILD_REQUESTER_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports/generic-merge-execution-build-requests.csv" - - public static final GENERIC_MERGE_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports" - public static final GENERIC_MERGE_REPORT_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution.csv" - public static final GENERIC_MERGE_REPORT_FILES_EQUIVALENT = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-output-equivalent.csv" - public static final GENERIC_MERGE_REPORT_MERGE_CONFLICTS = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-merge-conflicts.csv" - public static final GENERIC_MERGE_REPORT_COMMITS_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution-commits.csv" - public static final GENERIC_MERGE_REPORT_SCENARIO_LOCS_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-scenario-locs.csv" - public static final GENERIC_MERGE_REPORT_UNSTRUCTURED_TIMES_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-unstructured-times.csv" - - public static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" - public static final String JDIME_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/jdime/install/JDime/bin" - - public static final int NUMBER_OF_EXECUTIONS = 5 -} diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy deleted file mode 100644 index 91a04f44c..000000000 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ /dev/null @@ -1,139 +0,0 @@ -package services.dataCollectors.GenericMerge - -import interfaces.DataCollector -import org.apache.commons.io.FileUtils -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import project.MergeCommit -import project.Project -import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor -import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor -import services.dataCollectors.GenericMerge.executors.MergeToolExecutor -import services.dataCollectors.GenericMerge.model.MergeCommitExecutionSummary -import services.dataCollectors.GenericMerge.model.MergeScenarioExecutionSummary -import services.dataCollectors.GenericMerge.model.MergeScenarioResult -import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector - -import java.nio.file.Path -import java.util.stream.Collectors - -class GenericMergeDataCollector implements DataCollector { - private static Logger LOG = LogManager.getLogger(GenericMergeDataCollector.class) - - private final List mergeToolExecutors = Arrays.asList( - new GenericMergeToolExecutor(), - new JDimeMergeToolExecutor() - ) - - @Override - void collectData(Project project, MergeCommit mergeCommit) { - def scenarios = filterScenariosForExecution(MergeScenarioCollector.collectMergeScenarios(project, mergeCommit)).collect(Collectors.toList()) - - LOG.trace("Starting normalization of merge files on scenario") - scenarios.parallelStream().forEach { scenario -> FileFormatNormalizer.normalizeFileInPlace(scenario.resolve("merge.java")) - } - LOG.trace("Finished normalization of merge files on scenario") - - LOG.trace("Starting execution of merge tools on scenario") - def mergeToolsExecutionResults = scenarios - .stream() - .flatMap(scenario -> { - return mergeToolExecutors - .parallelStream() - .map(executor -> executor.runToolForMergeScenario(scenario)) - }).collect(Collectors.toList()) - LOG.trace("Finished execution of merge tools on scenario") - - // Aggregate scenario results by tool - def toolsCommitSummary = mergeToolsExecutionResults - .parallelStream() - .collect(Collectors.groupingBy(MergeScenarioExecutionSummary::getTool, - Collectors.collectingAndThen(Collectors.toList(), - MergeCommitExecutionSummary::fromFileResultsList))) - - - // Check which tools successfully integrated the scenario - def toolsInWhichIntegrationSucceeded = toolsCommitSummary - .entrySet() - .stream() - .filter { it -> it.value.result == MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS } - .map { x -> x.key } - .collect(Collectors.toList()) - - if (toolsInWhichIntegrationSucceeded.size() == 0) { - LOG.info("Integration failed in all tools") - } else if (toolsInWhichIntegrationSucceeded.size() != mergeToolExecutors.size()) { - LOG.info("At least one of the tools either reported a conflict or failed on the commit while the other did not") - toolsInWhichIntegrationSucceeded.forEach { tool -> - { - def toolCommitSummary = toolsCommitSummary.get(tool) - if (toolCommitSummary.allScenariosMatch) { - LOG.info("Output of the tool " + tool + " fully matched the commit merge. Skipping build analysis") - } else { - LOG.info("Output of the tool " + tool + " did not fully matched the commit merge. Starting build analysis") - BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, tool) - } - } - } - } else { - LOG.info("All the tools reported the same response") - } - - LOG.trace("Starting write of files results to report file") - def lines = mergeToolsExecutionResults.parallelStream().map(result -> getReportLine(project, mergeCommit, result)) - def reportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_FILE_NAME) - reportFile << lines.collect(Collectors.joining(System.lineSeparator())) << "\n" - LOG.trace("Finished write of files results to report file") - - LOG.trace("Starting write of commit report") - def commitLines = toolsCommitSummary.entrySet().parallelStream().map { it -> - def list = new ArrayList() - list.add(project.getName()) - list.add(mergeCommit.getSHA()) - list.add(it.key) - list.add(it.value.result.toString()) - list.add(it.value.allScenariosMatch.toString()) - return list.join(",").replaceAll('\\\\', '/') - } - def commitReportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_COMMITS_FILE_NAME) - commitReportFile << commitLines.collect(Collectors.joining(System.lineSeparator())) << "\n" - LOG.trace("Finished write of commit report") - } - - private static getReportLine(Project project, MergeCommit mergeCommit, MergeScenarioExecutionSummary result) { - def list = new ArrayList() - list.add(project.getName()) - list.add(mergeCommit.getSHA()) - list.add(result.tool) - list.add(result.scenario.toAbsolutePath().toString()) - list.add(result.output.toAbsolutePath().toString()) - list.add(result.result.toString()) - list.add(result.time.toString()) - list.add(result.isEquivalentToOracle().toString()) - list.join(",").replaceAll('\\\\', '/') - } - - private static filterScenariosForExecution(List scenarios) { - return scenarios - .parallelStream() - .filter(GenericMergeDataCollector::eitherParentDiffersFromBase) - } - - private static boolean eitherParentDiffersFromBase(Path scenario) { - def leftEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/base.java"), - new File("${scenario.toAbsolutePath()}/left.java")) - - def rightEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/base.java"), - new File("${scenario.toAbsolutePath()}/right.java")) - - if (leftEqualsBase) { - LOG.trace("In scenario ${scenario.toString()} left equals base") - } - - if (rightEqualsBase) { - LOG.trace("In scenario ${scenario.toString()} right equals base") - } - - return !leftEqualsBase && !rightEqualsBase - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/MergeConflictsComparator.groovy b/src/main/services/dataCollectors/GenericMerge/MergeConflictsComparator.groovy deleted file mode 100644 index efad0da4b..000000000 --- a/src/main/services/dataCollectors/GenericMerge/MergeConflictsComparator.groovy +++ /dev/null @@ -1,90 +0,0 @@ -package services.dataCollectors.GenericMerge - - -import interfaces.DataCollector -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import project.MergeCommit -import project.Project -import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector -import services.mergeScenariosFilters.NonFastForwardMergeScenarioFilter -import services.util.MergeConflict -import util.CsvUtils - -import java.nio.file.Files -import java.nio.file.Path -import java.util.stream.Stream - -class MergeConflictsComparator implements DataCollector { - private static Logger LOG = LogManager.getLogger(MergeConflictsComparator.class) - - private static final String GENERIC_MERGE_FILE_NAME = "merge.generic_merge.java" - private static final String JDIME_FILE_NAME = "merge.jdime.java" - - @Override - void collectData(Project project, MergeCommit mergeCommit) { - LOG.trace("Starting execution of Merge Conflicts Comparator on project ${project.getName()} and merge commit ${mergeCommit.getSHA()}") - - def conflictsComparisons = MergeScenarioCollector.collectMergeScenarios(project, mergeCommit) - .parallelStream() - .filter(NonFastForwardMergeScenarioFilter::isNonFastForwardMergeScenario) - .filter(MergeConflictsComparator::hasResponseFromBothTools) - .filter(MergeConflictsComparator::hasConflictsInBothTools) - .map(MergeConflictsComparator::extractConflictsFromFiles) - .flatMap(MergeConflictsComparator::compareMergeConflicts(project, mergeCommit)) - .map(CsvUtils::toCsvRepresentation) - - def reportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_MERGE_CONFLICTS) - def fileContent = conflictsComparisons.collect(CsvUtils.asLines()) - if (fileContent.isBlank() || fileContent.isEmpty()) { - LOG.trace("Finished execution of Merge Conflicts Comparator on project ${project.getName()} and merge commit ${mergeCommit.getSHA()} without conflicts") - return - } - reportFile << fileContent << "\n" - - LOG.trace("Finished execution of Merge Conflicts Comparator on project ${project.getName()} and merge commit ${mergeCommit.getSHA()}") - } - - private static boolean hasResponseFromBothTools(Path scenario) { - LOG.trace("Checking if has response from both tools for ${scenario.toString()}") - return Files.exists(scenario.resolve(GENERIC_MERGE_FILE_NAME)) && Files.exists(scenario.resolve(JDIME_FILE_NAME)) - } - - private static boolean hasConflictsInBothTools(Path scenario) { - LOG.trace("Checking if both files have conflicts") - return MergeConflict.getConflictsNumber(scenario.resolve(GENERIC_MERGE_FILE_NAME)) > 0 && MergeConflict.getConflictsNumber(scenario.resolve(JDIME_FILE_NAME)) > 0 - } - - private static Tuple3, Set> extractConflictsFromFiles(Path scenario) { - LOG.trace("Extracting conflicts from files in ${scenario.toString()}") - return new Tuple3(scenario, MergeConflict.extractMergeConflicts(scenario.resolve(GENERIC_MERGE_FILE_NAME)), - MergeConflict.extractMergeConflicts(scenario.resolve(JDIME_FILE_NAME))) - } - - private static Closure>> compareMergeConflicts(Project project, MergeCommit mergeCommit) { - return (Tuple3, Set> mergeConflicts) -> { - def scenario = mergeConflicts.getV1() - def genericMergeConflicts = mergeConflicts.getV2() - def jDimeConflicts = mergeConflicts.getV3() - - return genericMergeConflicts.withIndex().parallelStream().flatMap(genericMergeTuple -> { - def genericMergeConflict = genericMergeTuple.getV1() - def i = genericMergeTuple.getV2() - - return jDimeConflicts.withIndex().parallelStream().map(jDimeTuple -> { - def jDimeConflict = jDimeTuple.getV1() - def j = jDimeTuple.getV2() - - LOG.trace("Checking if conflicts generic_merge_conflict_${i} and jdime_conflict_${j} are equal") - - return [project.getName(), - mergeCommit.getSHA(), - scenario.toString(), - "generic_merge_conflict_${i}", - "jdime_conflict_${j}", - genericMergeConflict.equals(jDimeConflict).toString()] - }) - }) - } - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/MergeToolsComparator.groovy b/src/main/services/dataCollectors/GenericMerge/MergeToolsComparator.groovy deleted file mode 100644 index d159047d7..000000000 --- a/src/main/services/dataCollectors/GenericMerge/MergeToolsComparator.groovy +++ /dev/null @@ -1,47 +0,0 @@ -package services.dataCollectors.GenericMerge - -import interfaces.DataCollector -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import project.MergeCommit -import project.Project -import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector -import services.mergeScenariosFilters.NonFastForwardMergeScenarioFilter -import util.CsvUtils - -import java.nio.file.Path - -class MergeToolsComparator implements DataCollector { - private static Logger LOG = LogManager.getLogger(MergeToolsComparator.class) - - @Override - void collectData(Project project, MergeCommit mergeCommit) { - LOG.trace("Starting execution of Merge Tools Comparator on project ${project.getName()} and merge commit ${mergeCommit.getSHA()}") - - def results = MergeScenarioCollector.collectMergeScenarios(project, mergeCommit) - .parallelStream() - .filter(NonFastForwardMergeScenarioFilter::isNonFastForwardMergeScenario) - .map(scenario -> checkIfOutputsAreEquivalent(project, mergeCommit, scenario)) - .map(CsvUtils::toCsvRepresentation) - - def reportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_FILES_EQUIVALENT) - reportFile << results.collect(CsvUtils.asLines()) << "\n" - - LOG.trace("Finished execution of Merge Tools Comparator on project ${project.getName()} and merge commit ${mergeCommit.getSHA()}") - } - - private static List checkIfOutputsAreEquivalent(Project project, MergeCommit mergeCommit, Path scenario) { - LOG.trace("Starting to check if output for ${scenario.toString()} are equivalents") - - def genericMergePath = scenario.resolve("merge.generic_merge.java") - def jDimePath = scenario.resolve("merge.jdime.java") - - def result = [project.getName(), - mergeCommit.getSHA(), - scenario.toString(), - FileSyntacticDiff.areFilesSyntacticallyEquivalent(genericMergePath, jDimePath).toString()] - - LOG.trace("Finished checking if output for ${scenario.toString()} are equivalents") - return result - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/ScenarioLOCsCounter.groovy b/src/main/services/dataCollectors/GenericMerge/ScenarioLOCsCounter.groovy deleted file mode 100644 index a41cc4ef2..000000000 --- a/src/main/services/dataCollectors/GenericMerge/ScenarioLOCsCounter.groovy +++ /dev/null @@ -1,56 +0,0 @@ -package services.dataCollectors.GenericMerge - -import interfaces.DataCollector -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import org.json.JSONObject -import project.MergeCommit -import project.Project -import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector -import services.mergeScenariosFilters.NonFastForwardMergeScenarioFilter -import util.CsvUtils -import util.ProcessRunner - -import java.nio.file.Path - -/** - * Counts the number of LOCs (ignoring comments and blank lines) in base, left and right. - * It assumes that there's an executable for cloc in ./dependencies (symbolic links are allowed).*/ -class ScenarioLOCsCounter implements DataCollector { - private static Logger LOG = LogManager.getLogger(MergeToolsComparator.class) - - @Override - void collectData(Project project, MergeCommit mergeCommit) { - def scenarios = MergeScenarioCollector.collectMergeScenarios(project, mergeCommit) - .parallelStream() - .filter(NonFastForwardMergeScenarioFilter::isNonFastForwardMergeScenario) - .map((scenario) -> { - LOG.trace("Starting to count LOCs in ${scenario.toString()}") - def base = countLinesOfCodeInFile(scenario.resolve("base.java")) - def left = countLinesOfCodeInFile(scenario.resolve("left.java")) - def right = countLinesOfCodeInFile(scenario.resolve("right.java")) - def total = base + left + right - - return [project.getName(), mergeCommit.getSHA(), scenario.toString(), base.toString(), left.toString(), right.toString(), total.toString()] - }) - .map(CsvUtils::toCsvRepresentation) - - def reportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_SCENARIO_LOCS_FILE_NAME) - reportFile << scenarios.collect(CsvUtils.asLines()) << "\n" - } - - private static int countLinesOfCodeInFile(Path file) { - def clocProcessBuilder = ProcessRunner.buildProcess("./dependencies", - "./cloc", - file.toAbsolutePath().toString(), - "--json") - - def output = ProcessRunner.startProcess(clocProcessBuilder) - output.waitFor() - - def jsonOutput = new JSONObject(output.getInputStream().readLines().join('\n')) - int sumCode = jsonOutput.getJSONObject("SUM").getInt("code") - - return sumCode - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/UnstructuredMergeCollector.groovy b/src/main/services/dataCollectors/GenericMerge/UnstructuredMergeCollector.groovy deleted file mode 100644 index 6b92bb655..000000000 --- a/src/main/services/dataCollectors/GenericMerge/UnstructuredMergeCollector.groovy +++ /dev/null @@ -1,56 +0,0 @@ -package services.dataCollectors.GenericMerge - -import interfaces.DataCollector -import project.MergeCommit -import project.Project -import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector -import services.mergeScenariosFilters.NonFastForwardMergeScenarioFilter -import util.CsvUtils -import util.ProcessRunner - -import java.nio.file.Files -import java.nio.file.Path - -class UnstructuredMergeCollector implements DataCollector { - @Override - void collectData(Project project, MergeCommit mergeCommit) { - def scenarios = MergeScenarioCollector.collectMergeScenarios(project, mergeCommit) - .stream() - .filter(NonFastForwardMergeScenarioFilter::isNonFastForwardMergeScenario) - .map(scenario -> { - def executionTime = runGitMergeFile(scenario) - return [project.getName(), mergeCommit.getSHA(), scenario.toString(), executionTime] - }) - .map(CsvUtils::toCsvRepresentation) - - def reportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_UNSTRUCTURED_TIMES_FILE_NAME) - reportFile << scenarios.collect(CsvUtils.asLines()) << "\n" - } - - private static long runGitMergeFile(Path scenario) { - def executionTimes = new ArrayList() - - for (int i = 0; i < GenericMergeConfig.NUMBER_OF_EXECUTIONS; i++) { - // We copy the left file, because git merge-file runs in place, replacing the contents of left file - Files.copy(scenario.resolve("left.java"), scenario.resolve("merge.unstructured.java")) - - long startTime = System.nanoTime() - - def processBuilder = ProcessRunner.buildProcess(GenericMergeConfig.BASE_EXPERIMENT_PATH, - "git", - "merge-file", - scenario.resolve("merge.unstructured.java").toString(), - scenario.resolve("base.java").toString(), - scenario.resolve("right.java").toString()) - ProcessRunner.startProcess(processBuilder).waitFor() - - long endTime = System.nanoTime() - // If we're running more than one execution, we use the first one as a warm up - if (GenericMergeConfig.NUMBER_OF_EXECUTIONS == 1 || i > 0) { - executionTimes.add(endTime - startTime) - } - } - - return (long) (executionTimes.stream().reduce(0, (prev, cur) -> prev + cur) / executionTimes.size()) - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy deleted file mode 100644 index 08be54737..000000000 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ /dev/null @@ -1,53 +0,0 @@ -package services.dataCollectors.GenericMerge.executors - -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import services.dataCollectors.GenericMerge.GenericMergeConfig -import services.dataCollectors.GenericMerge.model.MergeScenarioResult -import util.ProcessRunner - -import java.nio.file.Path - -class GenericMergeToolExecutor extends MergeToolExecutor { - private static Logger LOG = LogManager.getLogger(GenericMergeToolExecutor.class) - - @Override - protected MergeScenarioResult executeTool(Path scenario, Path outputFile) { - def working_directory_path = scenario.toAbsolutePath().toString() - - def processBuilder = ProcessRunner.buildProcess(working_directory_path) - processBuilder.command().addAll(getBuildParameters(outputFile)) - - def output = ProcessRunner.startProcess(processBuilder) - output.waitFor() - - if (output.exitValue() > 1) { - LOG.warn("Error while merging ${scenario.toAbsolutePath()}. Generic Merge exited with exitCode ${output.exitValue()}") - LOG.debug("Generic Merge output: ${output.getInputStream().readLines()}") - } - - return output.exitValue() == 0 ? MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? MergeScenarioResult.SUCCESS_WITH_CONFLICTS : MergeScenarioResult.TOOL_ERROR - } - - @Override - protected boolean shouldSkipFileNormalization() { - return false - } - - @Override - String getToolName() { - return "GENERIC_MERGE" - } - - private static List getBuildParameters(Path outputFile) { - def list = new ArrayList() - list.add(GenericMergeConfig.GENERIC_MERGE_BINARY_PATH) - list.add("merge") - list.add("--base-path=base.java") - list.add("--left-path=left.java") - list.add("--right-path=right.java") - list.add("--merge-path=${outputFile.toAbsolutePath().toString()}".toString()) - list.add("--language=java") - return list - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy deleted file mode 100644 index 233740824..000000000 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ /dev/null @@ -1,54 +0,0 @@ -package services.dataCollectors.GenericMerge.executors - -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import services.dataCollectors.GenericMerge.GenericMergeConfig -import services.dataCollectors.GenericMerge.model.MergeScenarioResult -import services.util.MergeConflict -import util.ProcessRunner - -import java.nio.file.Path - -class JDimeMergeToolExecutor extends MergeToolExecutor { - private static Logger LOG = LogManager.getLogger(JDimeMergeToolExecutor.class) - - @Override - protected MergeScenarioResult executeTool(Path scenario, Path outputFile) { - def working_directory_path = scenario.toAbsolutePath().toString() - - def processBuilder = ProcessRunner.buildProcess(GenericMergeConfig.JDIME_BINARY_PATH, - "./JDime", - "-f", - "--mode=structured", - "--output=${outputFile.toAbsolutePath().toString()}".toString(), - "${working_directory_path}/left.java", - "${working_directory_path}/base.java", - "${working_directory_path}/right.java") - - def output = ProcessRunner.startProcess(processBuilder) - output.waitFor() - - if (output.exitValue() >= 200) { - LOG.warn("Error while merging ${scenario.toAbsolutePath()}. jDime exited with exitCode ${output.exitValue()}") - LOG.debug("jDime output: ${output.getInputStream().readLines()}") - return MergeScenarioResult.TOOL_ERROR - } - - def mergeConflictsCount = MergeConflict.getConflictsNumber(outputFile) - if (mergeConflictsCount > 0) { - return MergeScenarioResult.SUCCESS_WITH_CONFLICTS - } - - return MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS - } - - @Override - protected boolean shouldSkipFileNormalization() { - return true - } - - @Override - String getToolName() { - return "JDIME" - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy deleted file mode 100644 index a05475591..000000000 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ /dev/null @@ -1,55 +0,0 @@ -package services.dataCollectors.GenericMerge.executors - -import org.apache.logging.log4j.LogManager -import org.apache.logging.log4j.Logger -import services.dataCollectors.GenericMerge.FileFormatNormalizer -import services.dataCollectors.GenericMerge.GenericMergeConfig -import services.dataCollectors.GenericMerge.model.MergeScenarioExecutionSummary -import services.dataCollectors.GenericMerge.model.MergeScenarioResult - -import java.nio.file.Path - -abstract class MergeToolExecutor { - private static Logger LOG = LogManager.getLogger(MergeToolExecutor.class) - - MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { - LOG.trace("Starting execution of merge scenario with tool ${getToolName()}") - - List executionTimes = new ArrayList<>() - def outputFilePath = scenario.resolve("merge." + getToolName().toLowerCase() + ".java") - MergeScenarioResult result = null - - for (int i = 0; i < GenericMergeConfig.NUMBER_OF_EXECUTIONS; i++) { - LOG.trace("Starting execution ${i + 1} of ${GenericMergeConfig.NUMBER_OF_EXECUTIONS}") - long startTime = System.nanoTime() - result = executeTool(scenario, outputFilePath) - long endTime = System.nanoTime() - LOG.trace("Finished execution ${i + 1} of ${GenericMergeConfig.NUMBER_OF_EXECUTIONS} IN ${endTime - startTime} ns") - // If we're running more than one execution, we use the first one as a warm up - if (GenericMergeConfig.NUMBER_OF_EXECUTIONS == 1 || i > 0) { - executionTimes.add(endTime - startTime) - } - } - - long averageTime = (long) (executionTimes.stream().reduce(0, (prev, cur) -> prev + cur) / executionTimes.size()) - - if (result == MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS && !shouldSkipFileNormalization()) { - FileFormatNormalizer.normalizeFileInPlace(outputFilePath) - } - - def summary = new MergeScenarioExecutionSummary(scenario, - outputFilePath, - result, - averageTime, - this.getToolName()) - - LOG.trace("Finished execution of merge scenario with tool ${summary.tool} in ${summary.time}ns with ${summary.result.toString()}") - return summary - } - - protected abstract MergeScenarioResult executeTool(Path scenario, Path outputFile); - - protected abstract boolean shouldSkipFileNormalization(); - - abstract String getToolName(); -} diff --git a/src/main/services/dataCollectors/GenericMerge/model/MergeCommitExecutionSummary.groovy b/src/main/services/dataCollectors/GenericMerge/model/MergeCommitExecutionSummary.groovy deleted file mode 100644 index bc1f11d05..000000000 --- a/src/main/services/dataCollectors/GenericMerge/model/MergeCommitExecutionSummary.groovy +++ /dev/null @@ -1,23 +0,0 @@ -package services.dataCollectors.GenericMerge.model - -class MergeCommitExecutionSummary { - public final MergeScenarioResult result - public final boolean allScenariosMatch - - private MergeCommitExecutionSummary(MergeScenarioResult result, boolean allScenariosMatch) { - this.result = result - this.allScenariosMatch = allScenariosMatch - } - - static MergeCommitExecutionSummary fromFileResultsList(List results) { - def result = results.stream() - .map { it -> it.result } - .reduce(MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS, MergeCommitExecutionSummary::validateResult) - def allMatchesOracle = results.stream().every { it -> it.isEquivalentToOracle() } - return new MergeCommitExecutionSummary(result, allMatchesOracle) - } - - private static validateResult(MergeScenarioResult prev, MergeScenarioResult cur) { - return cur != MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS ? cur : prev - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioExecutionSummary.groovy b/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioExecutionSummary.groovy deleted file mode 100644 index f18deb00c..000000000 --- a/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioExecutionSummary.groovy +++ /dev/null @@ -1,32 +0,0 @@ -package services.dataCollectors.GenericMerge.model - - -import services.dataCollectors.GenericMerge.FileSyntacticDiff - -import java.nio.file.Path - -class MergeScenarioExecutionSummary { - public final String tool - public final Path scenario - public final Path output - public final MergeScenarioResult result - public final long time - public final boolean equivalentToOracle - - MergeScenarioExecutionSummary(Path scenario, Path output, MergeScenarioResult result, long time, String tool) { - this.scenario = scenario - this.output = output - this.result = result - this.time = time - this.tool = tool - this.equivalentToOracle = FileSyntacticDiff.areFilesSyntacticallyEquivalent(scenario.resolve("merge.java"), output) - } - - String getTool() { - return tool - } - - boolean isEquivalentToOracle() { - return equivalentToOracle - } -} diff --git a/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioResult.groovy b/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioResult.groovy deleted file mode 100644 index 3090aba50..000000000 --- a/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioResult.groovy +++ /dev/null @@ -1,7 +0,0 @@ -package services.dataCollectors.GenericMerge.model - -enum MergeScenarioResult { - SUCCESS_WITHOUT_CONFLICTS, - SUCCESS_WITH_CONFLICTS, - TOOL_ERROR -} diff --git a/src/main/services/dataCollectors/S3MMergesCollector/MergeScenarioCollector.groovy b/src/main/services/dataCollectors/S3MMergesCollector/MergeScenarioCollector.groovy index 256364a00..ba9e32492 100644 --- a/src/main/services/dataCollectors/S3MMergesCollector/MergeScenarioCollector.groovy +++ b/src/main/services/dataCollectors/S3MMergesCollector/MergeScenarioCollector.groovy @@ -2,6 +2,7 @@ package services.dataCollectors.S3MMergesCollector import project.MergeCommit import project.Project +import services.mergeScenariosFilters.NonFastForwardMergeScenarioFilter import util.ProcessRunner import services.util.Utils @@ -29,6 +30,13 @@ class MergeScenarioCollector { .collect(Collectors.toList()) } + static List collectNonFastForwardMergeScenarios(Project project, MergeCommit mergeCommit) { + return collectMergeScenarios(project, mergeCommit) + .parallelStream() + .filter(NonFastForwardMergeScenarioFilter::isNonFastForwardMergeScenario) + .collect(Collectors.toList()) + } + private static Tuple4 storeAndRetrieveMergeQuadruple(Project project, MergeCommit mergeCommit, String modifiedFile) { Path leftFile = storeFile(project, mergeCommit, modifiedFile, mergeCommit.getLeftSHA(), 'left') Path baseFile = storeFile(project, mergeCommit, modifiedFile, mergeCommit.getAncestorSHA(), 'base') @@ -42,6 +50,8 @@ class MergeScenarioCollector { createDirectories(mergeScenarioDirectory) Path filePath = mergeScenarioDirectory.resolve(Utils.getfileNameWithExtension(fileName)) + if (Files.exists(filePath)) return filePath + Files.deleteIfExists(filePath) filePath.toFile() << getFileContent(project, modifiedFile, commitSHA) return filePath diff --git a/src/main/services/dataCollectors/buildRequester/GithubActionsHelper.groovy b/src/main/services/dataCollectors/buildRequester/GithubActionsHelper.groovy new file mode 100644 index 000000000..1d36f3e77 --- /dev/null +++ b/src/main/services/dataCollectors/buildRequester/GithubActionsHelper.groovy @@ -0,0 +1,44 @@ +package services.dataCollectors.buildRequester + +import project.Project + +import java.nio.charset.Charset +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.StandardOpenOption + +class GithubActionsHelper { + static void createGitHubActionsFile(Project project) { + def contents = """ +name: Mining Framework Check +on: [push] +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + java-version: [8, 11, 17] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: Set up Java \${{ matrix.java-version }} + uses: actions/setup-java@v1 + with: + java-version: \${{ matrix.java-version }} + distribution: 'corretto' + - name: Run tests + uses: nick-fields/retry@v3 + with: + timeout_minutes: 30 + max_attempts: 5 + command: mvn clean test -Dcheckstyle.skip=true -Dlicense.skipDownloadLicenses -Dlicense.skip=true + """ + + def githubActionsDirectory = Paths.get(project.getPath()).resolve(".github/workflows") + Files.createDirectories(githubActionsDirectory) + Files.write(githubActionsDirectory.resolve("mining_framework.yaml"), contents.getBytes(Charset.defaultCharset()), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING) + } +} diff --git a/src/main/services/dataCollectors/buildRequester/RequestBuildForRevisionWithFilesDataCollector.groovy b/src/main/services/dataCollectors/buildRequester/RequestBuildForRevisionWithFilesDataCollector.groovy new file mode 100644 index 000000000..ca5b70177 --- /dev/null +++ b/src/main/services/dataCollectors/buildRequester/RequestBuildForRevisionWithFilesDataCollector.groovy @@ -0,0 +1,113 @@ +package services.dataCollectors.buildRequester + +import interfaces.DataCollector +import org.apache.logging.log4j.LogManager +import project.MergeCommit +import project.Project +import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector +import util.ProcessRunner + +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + +import static app.MiningFramework.arguments + +class RequestBuildForRevisionWithFilesDataCollector implements DataCollector { + private static LOG = LogManager.getLogger(RequestBuildForRevisionWithFilesDataCollector.class) + + private String fileName + + RequestBuildForRevisionWithFilesDataCollector(String fileName) { + this.fileName = fileName + } + + @Override + void collectData(Project project, MergeCommit mergeCommit) { + def branchName = "mining-framework-analysis_${project.getName()}_${mergeCommit.getSHA()}_${fileName}" + LOG.debug("Attaching origin to project") + attachOrigin(project) + LOG.debug("Setting up credentials") + setupCredentials(project) + LOG.debug("Deleting and creating branch") + deleteBranch(project, branchName) + LOG.debug("Checking out branch") + checkoutCommitAndCreateBranch(project, branchName, mergeCommit.getSHA()) + LOG.debug("Copying files") + copyFilesIntoRevision(project, mergeCommit) + GithubActionsHelper.createGitHubActionsFile(project) + LOG.debug("Comitting files") + commitChanges(project, "Mining Framework Analysis") + LOG.debug("Pushing analysis") + pushBranch(project, branchName) + } + + static private void attachOrigin(Project project) { + def token = arguments.getAccessKey() + def origin = "https://${token}@github.com/jpedroh/mining-framework-analysis" + def process = ProcessRunner.runProcess(project.getPath(), 'git', 'remote', 'add', 'analysis', origin) + process.getInputStream().eachLine(LOG::trace) + process.getErrorStream().eachLine(LOG::warn) + process.waitFor() + } + + static private void setupCredentials(Project project) { + def configEmail = ProcessRunner.runProcess(project.getPath(), 'git', 'config', 'user.email', '"joao.pedro.hsd@gmail.com"') + configEmail.getInputStream().eachLine(LOG::trace) + configEmail.getErrorStream().eachLine(LOG::warn) + configEmail.waitFor() + def configName = ProcessRunner.runProcess(project.getPath(), 'git', 'config', 'user.name', '"Joao Pedro"') + configName.getInputStream().eachLine(LOG::trace) + configName.getErrorStream().eachLine(LOG::warn) + configName.waitFor() + } + + static private void deleteBranch(Project project, String branchName) { + def process = ProcessRunner.runProcess(project.getPath(), 'git', 'branch', '-D', branchName) + process.getInputStream().eachLine(LOG::trace) + process.getErrorStream().eachLine(LOG::warn) + process.waitFor() + } + + static private void checkoutCommitAndCreateBranch(Project project, String branchName, String commitSha) { + def process = ProcessRunner.runProcess(project.getPath(), 'git', 'checkout', '-b', branchName, commitSha) + process.getInputStream().eachLine(LOG::trace) + process.getErrorStream().eachLine(LOG::warn) + process.waitFor() + } + + private copyFilesIntoRevision(Project project, MergeCommit mergeCommit) { + def scenarioFiles = MergeScenarioCollector.collectNonFastForwardMergeScenarios(project, mergeCommit) + scenarioFiles.stream() + .filter(file -> { + if (Files.notExists(file.resolve(this.fileName))) { + LOG.debug("Skipping copy of file ${file.resolve(this.fileName).toAbsolutePath().toString()} because it does not exist") + return false + } + return true + }) + .forEach(file -> { + def destination = Paths.get(project.getPath()).resolve(file.toString().substring(file.toString().indexOf(mergeCommit.getSHA()) + 1 + mergeCommit.getSHA().length())) + LOG.debug("Copying file ${file.resolve(this.fileName)} to ${destination}") + Files.copy(file.resolve(this.fileName), destination, StandardCopyOption.REPLACE_EXISTING) + }) + } + + static private void pushBranch(Project project, String branchName) { + def process = ProcessRunner.runProcess(project.getPath(), 'git', 'push', 'analysis', branchName, "-f") + process.getInputStream().eachLine(LOG::trace) + process.getErrorStream().eachLine(LOG::warn) + process.waitFor() + } + + static protected void commitChanges(Project project, String message) { + def process = ProcessRunner.runProcess(project.getPath(), "git", "add", ".") + process.getInputStream().eachLine(LOG::trace) + process.getErrorStream().eachLine(LOG::warn) + process.waitFor() + def commit = ProcessRunner.runProcess(project.getPath(), "git", "commit", "-a", "-m", "${message}") + commit.getInputStream().eachLine(LOG::trace) + commit.getErrorStream().eachLine(LOG::warn) + commit.waitFor() + } +} diff --git a/src/main/services/dataCollectors/common/CompareScenarioMergeConflictsDataCollector.groovy b/src/main/services/dataCollectors/common/CompareScenarioMergeConflictsDataCollector.groovy new file mode 100644 index 000000000..140ca0095 --- /dev/null +++ b/src/main/services/dataCollectors/common/CompareScenarioMergeConflictsDataCollector.groovy @@ -0,0 +1,107 @@ +package services.dataCollectors.common + +import interfaces.DataCollector +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import project.MergeCommit +import project.Project +import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector +import services.util.MergeConflict +import util.CsvUtils + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.stream.Collectors +import java.util.stream.Stream + +class CompareScenarioMergeConflictsDataCollector implements DataCollector { + private static Logger LOG = LogManager.getLogger(CompareScenarioMergeConflictsDataCollector.class) + + private static final REPORT_DIRECTORY = "${System.getProperty("user.dir")}/output/reports/conflicts-comparison" + + private String fileA + private String fileB + + CompareScenarioMergeConflictsDataCollector(String fileA, String fileB) { + this.fileA = fileA + this.fileB = fileB + } + + @Override + void collectData(Project project, MergeCommit mergeCommit) { + LOG.trace("Starting execution of Merge Conflicts Comparator on project ${project.getName()} and merge commit ${mergeCommit.getSHA()}") + + def conflictsComparisons = MergeScenarioCollector.collectNonFastForwardMergeScenarios(project, mergeCommit) + .parallelStream() + .filter(this::hasResponseFromBothTools) + .filter(this::hasConflictsInBothTools) + .map(this::extractConflictsFromFiles) + .flatMap(CompareScenarioMergeConflictsDataCollector::compareMergeConflicts(project, mergeCommit)) + .map(CsvUtils::toCsvRepresentation) + .collect(Collectors.toList()) + + if (conflictsComparisons.isEmpty()) { + LOG.trace("Finished execution of Merge Conflicts Comparator on project ${project.getName()} and merge commit ${mergeCommit.getSHA()} without conflicts") + return + } + + LOG.trace("Finished execution of Merge Conflicts Comparator on project ${project.getName()} and merge commit ${mergeCommit.getSHA()}") + writeReportToFile(getReportFileName(), conflictsComparisons.collect()) + } + + protected static synchronized writeReportToFile(Path filePath, List lines) { + def reportFile = filePath.toFile() + Files.createDirectories(filePath.getParent()) + reportFile.createNewFile() + reportFile << lines.stream().collect(CsvUtils.asLines()) << System.lineSeparator() + } + + private Path getReportFileName() { + return Paths.get("${REPORT_DIRECTORY}/${fileA.replace('.', "_")}-${fileB.replace('.', "_")}.csv") + } + + private boolean hasResponseFromBothTools(Path scenario) { + LOG.trace("Checking if has response from both tools for ${scenario.toString()}") + return Files.exists(scenario.resolve(fileA)) && Files.exists(scenario.resolve(fileB)) + } + + private boolean hasConflictsInBothTools(Path scenario) { + LOG.trace("Checking if both files have conflicts for scenario ${scenario}") + def conflictsCountFileA = MergeConflict.getConflictsNumber(scenario.resolve(fileA)) + def conflictsCountFileB = MergeConflict.getConflictsNumber(scenario.resolve(fileB)) + LOG.trace("Found ${conflictsCountFileA} conflicts on file A and ${conflictsCountFileB} on file B for scenario ${scenario}") + return conflictsCountFileA > 0 && conflictsCountFileB > 0 + } + + private Tuple3, Set> extractConflictsFromFiles(Path scenario) { + LOG.trace("Extracting conflicts from files in ${scenario.toString()}") + return new Tuple3(scenario, MergeConflict.extractMergeConflicts(scenario.resolve(fileA)), + MergeConflict.extractMergeConflicts(scenario.resolve(fileB))) + } + + private static Closure>> compareMergeConflicts(Project project, MergeCommit mergeCommit) { + return (Tuple3, Set> mergeConflicts) -> { + def scenario = mergeConflicts.getV1() + def fileAMergeConflicts = mergeConflicts.getV2() + def fileBMergeConflicts = mergeConflicts.getV3() + + return fileAMergeConflicts.withIndex().parallelStream().flatMap(fileATuple -> { + def fileAConflict = fileATuple.getV1() + def i = fileATuple.getV2() + + return fileBMergeConflicts.withIndex().parallelStream().map(fileBTuple -> { + def fileBConflict = fileBTuple.getV1() + def j = fileBTuple.getV2() + + return [project.getName(), + mergeCommit.getSHA(), + scenario.toString(), + "file_a_conflict_${i}", + "file_b_conflict_${j}", + fileAConflict.equalsOrSubstring(fileBConflict).toString()] + }) + }) + } + } +} diff --git a/src/main/services/dataCollectors/common/RunDataCollectorsInParallel.groovy b/src/main/services/dataCollectors/common/RunDataCollectorsInParallel.groovy new file mode 100644 index 000000000..505103bda --- /dev/null +++ b/src/main/services/dataCollectors/common/RunDataCollectorsInParallel.groovy @@ -0,0 +1,18 @@ +package services.dataCollectors.common + +import interfaces.DataCollector +import project.MergeCommit +import project.Project + +class RunDataCollectorsInParallel implements DataCollector { + private List dataCollectors + + RunDataCollectorsInParallel(List dataCollectors) { + this.dataCollectors = dataCollectors + } + + @Override + void collectData(Project project, MergeCommit mergeCommit) { + this.dataCollectors.parallelStream().forEach(collector -> collector.collectData(project, mergeCommit)) + } +} diff --git a/src/main/services/dataCollectors/common/SyntacticallyCompareScenarioFilesDataCollector.groovy b/src/main/services/dataCollectors/common/SyntacticallyCompareScenarioFilesDataCollector.groovy new file mode 100644 index 000000000..2d365dda1 --- /dev/null +++ b/src/main/services/dataCollectors/common/SyntacticallyCompareScenarioFilesDataCollector.groovy @@ -0,0 +1,91 @@ +package services.dataCollectors.common + +import interfaces.DataCollector +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import project.MergeCommit +import project.Project +import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector +import util.CsvUtils +import util.ProcessRunner + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.stream.Collectors + +class SyntacticallyCompareScenarioFilesDataCollector implements DataCollector { + private static Logger LOG = LogManager.getLogger(SyntacticallyCompareScenarioFilesDataCollector.class) + + private static String LAST_MERGE_BINARY_PATH = "${System.getProperty("user.dir")}/dependencies/last-merge" + + private static final REPORT_DIRECTORY = "${System.getProperty("user.dir")}/output/reports/syntactic-comparison" + + private String _fileA + private String _fileB + + SyntacticallyCompareScenarioFilesDataCollector(String fileA, String fileB) { + this._fileA = fileA + this._fileB = fileB + } + + @Override + void collectData(Project project, MergeCommit mergeCommit) { + def results = MergeScenarioCollector.collectNonFastForwardMergeScenarios(project, mergeCommit) + .parallelStream() + .map(file -> { + def fileA = file.resolve(_fileA) + def fileB = file.resolve(_fileB) + LOG.trace("Starting syntactic comparison between ${fileA} and ${fileB}") + def areFilesSyntacticallyEquivalent = areFilesSyntacticallyEquivalent(fileA, fileB) + return [project.getName(), mergeCommit.getSHA(), file, fileA, fileB, areFilesSyntacticallyEquivalent] + }) + .map(CsvUtils::toCsvRepresentation) + .collect(Collectors.toList()) + + writeToReportFile(getReportFileName(), results) + } + + protected synchronized static writeToReportFile(String reportFileName, List lines) { + def reportFile = new File(reportFileName) + Files.createDirectories(Paths.get(REPORT_DIRECTORY)) + reportFile.createNewFile() + reportFile << lines.stream().collect(CsvUtils.asLines()) << System.lineSeparator() + } + + private String getReportFileName() { + return "${REPORT_DIRECTORY}/${_fileA.replace('.', "_")}-${_fileB.replace('.', "_")}.csv" + } + + private static boolean areFilesSyntacticallyEquivalent(Path fileA, Path fileB) { + if (!Files.exists(fileA)) { + LOG.trace("Early returning because the file ${fileA} do not exist") + return false + } + if (!Files.exists(fileB)) { + LOG.trace("Early returning because the file ${fileB} do not exist") + return false + } + + def process = ProcessRunner.buildProcess("./") + + def list = new ArrayList() + list.add(LAST_MERGE_BINARY_PATH) + list.add("diff") + list.add("--left-path=${fileA.toAbsolutePath().toString()}".toString()) + list.add("--right-path=${fileB.toAbsolutePath().toString()}".toString()) + list.add("--language=java") + process.command().addAll(list) + + LOG.trace("Calling generic merge with command \"${process.command().join(' ')}\"") + + def output = ProcessRunner.startProcess(process) + output.waitFor() + + if (output.exitValue() > 1) { + LOG.warn("Error while running comparison between ${fileA.toString()} and ${fileB.toString()}: ${output.getInputStream().readLines()}") + } + + return output.exitValue() == 0 + } +} diff --git a/src/main/services/dataCollectors/fileSyntacticNormalization/BaseFileSyntacticNormalizationDataCollector.groovy b/src/main/services/dataCollectors/fileSyntacticNormalization/BaseFileSyntacticNormalizationDataCollector.groovy new file mode 100644 index 000000000..691a046c9 --- /dev/null +++ b/src/main/services/dataCollectors/fileSyntacticNormalization/BaseFileSyntacticNormalizationDataCollector.groovy @@ -0,0 +1,46 @@ +package services.dataCollectors.fileSyntacticNormalization + +import interfaces.DataCollector +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import project.MergeCommit +import project.Project +import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardCopyOption + +abstract class BaseFileSyntacticNormalizationDataCollector implements DataCollector { + protected static Logger LOG = LogManager.getLogger(BaseFileSyntacticNormalizationDataCollector.class) + + protected String inputFile + protected String outputFile + + BaseFileSyntacticNormalizationDataCollector(String inputFile, String outputFile) { + this.inputFile = inputFile + this.outputFile = outputFile + } + + @Override + void collectData(Project project, MergeCommit mergeCommit) { + def files = MergeScenarioCollector.collectNonFastForwardMergeScenarios(project, mergeCommit) + files.parallelStream().forEach(file -> { + LOG.debug("Starting to run file normalization in file ${inputFile}") + + if (!Files.exists(file.resolve(inputFile))) { + LOG.debug("Skipping normalization because file ${file.resolve(inputFile)} do not exist") + return + } + + def isSuccess = runNormalizationOnFile(file.resolve(inputFile), file.resolve(outputFile)) + if (!isSuccess) { + LOG.debug("Failed to run file normalization in file ${inputFile}, falling back to copy the file") + Files.copy(file.resolve(inputFile), file.resolve(outputFile), StandardCopyOption.REPLACE_EXISTING) + } + LOG.debug("Finished to run file normalization in file ${inputFile}") + }) + } + + protected abstract boolean runNormalizationOnFile(Path inputFile, Path outputFile); +} diff --git a/src/main/services/dataCollectors/fileSyntacticNormalization/FormatFileSyntacticNormalizationDataCollector.groovy b/src/main/services/dataCollectors/fileSyntacticNormalization/FormatFileSyntacticNormalizationDataCollector.groovy new file mode 100644 index 000000000..b15daca56 --- /dev/null +++ b/src/main/services/dataCollectors/fileSyntacticNormalization/FormatFileSyntacticNormalizationDataCollector.groovy @@ -0,0 +1,31 @@ +package services.dataCollectors.fileSyntacticNormalization + + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import util.ProcessRunner + +import java.nio.file.Path + +class FormatFileSyntacticNormalizationDataCollector extends BaseFileSyntacticNormalizationDataCollector { + private static Logger LOG = LogManager.getLogger(FormatFileSyntacticNormalizationDataCollector.class) + + private static final String FORMAT_PATH = "${System.getProperty("user.dir")}/dependencies/format" + + FormatFileSyntacticNormalizationDataCollector(String inputFile, String outputFile) { + super(inputFile, outputFile) + } + + @Override + protected boolean runNormalizationOnFile(Path inputFile, Path outputFile) { + def processBuilder = ProcessRunner.buildProcess(System.getProperty("user.dir")) + processBuilder.command().add(FORMAT_PATH) + processBuilder.command().add(inputFile.toString()) + processBuilder.redirectOutput(outputFile.toFile()) + + LOG.trace("Calling format with command \"${processBuilder.command().join(' ')}\"") + + def exitCode = ProcessRunner.startProcess(processBuilder).waitFor() + return exitCode != 0 + } +} diff --git a/src/main/services/dataCollectors/fileSyntacticNormalization/JDimeFileSyntacticNormalizationDataCollector.groovy b/src/main/services/dataCollectors/fileSyntacticNormalization/JDimeFileSyntacticNormalizationDataCollector.groovy new file mode 100644 index 000000000..6ce5d09f9 --- /dev/null +++ b/src/main/services/dataCollectors/fileSyntacticNormalization/JDimeFileSyntacticNormalizationDataCollector.groovy @@ -0,0 +1,31 @@ +package services.dataCollectors.fileSyntacticNormalization + + +import util.ProcessRunner + +import java.nio.file.Path +import java.util.concurrent.TimeUnit + +class JDimeFileSyntacticNormalizationDataCollector extends BaseFileSyntacticNormalizationDataCollector { + private static final String JDIME_BINARY_PATH = "${System.getProperty("user.dir")}/dependencies/jdime/install/JDime/bin" + + JDimeFileSyntacticNormalizationDataCollector(String inputFile, String outputFile) { + super(inputFile, outputFile) + } + + @Override + boolean runNormalizationOnFile(Path inputFile, Path outputFile) { + def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH, + "./JDime", + "-f", + "--mode=structured", + "--output=${outputFile.toAbsolutePath().toString()}".toString(), + inputFile.toAbsolutePath().toString(), + inputFile.toAbsolutePath().toString(), + inputFile.toAbsolutePath().toString()) + + def output = ProcessRunner.startProcess(processBuilder) + def hasCompleted = output.waitFor(1, TimeUnit.HOURS) + return hasCompleted && output.exitValue() == 0 + } +} diff --git a/src/main/services/dataCollectors/fileSyntacticNormalization/JavaParserFormatFileSyntacticNormalizationDataCollector.groovy b/src/main/services/dataCollectors/fileSyntacticNormalization/JavaParserFormatFileSyntacticNormalizationDataCollector.groovy new file mode 100644 index 000000000..b0a8e9946 --- /dev/null +++ b/src/main/services/dataCollectors/fileSyntacticNormalization/JavaParserFormatFileSyntacticNormalizationDataCollector.groovy @@ -0,0 +1,37 @@ +package services.dataCollectors.fileSyntacticNormalization + +import com.github.javaparser.JavaParser +import com.github.javaparser.printer.DefaultPrettyPrinter +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger + +import java.nio.charset.Charset +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption + +class JavaParserFormatFileSyntacticNormalizationDataCollector extends BaseFileSyntacticNormalizationDataCollector { + private static Logger LOG = LogManager.getLogger(JavaParserFormatFileSyntacticNormalizationDataCollector.class) + + JavaParserFormatFileSyntacticNormalizationDataCollector(String inputFile, String outputFile) { + super(inputFile, outputFile) + } + + @Override + protected boolean runNormalizationOnFile(Path inputFile, Path outputFile) { + def cu = new JavaParser().parse(inputFile.toFile()) + if (!cu.isSuccessful()) { + return false + } + + def prettyPrinter = new DefaultPrettyPrinter() + def result = prettyPrinter.print(cu.getResult().get()) + + Files.write(outputFile, result.getBytes(Charset.defaultCharset()), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING) + + return true + } +} diff --git a/src/main/services/dataCollectors/fileSyntacticNormalization/SporkFileSyntacticNormalizationDataCollector.groovy b/src/main/services/dataCollectors/fileSyntacticNormalization/SporkFileSyntacticNormalizationDataCollector.groovy new file mode 100644 index 000000000..da605153f --- /dev/null +++ b/src/main/services/dataCollectors/fileSyntacticNormalization/SporkFileSyntacticNormalizationDataCollector.groovy @@ -0,0 +1,43 @@ +package services.dataCollectors.fileSyntacticNormalization + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import util.ProcessRunner + +import java.nio.file.Path +import java.util.concurrent.TimeUnit + +class SporkFileSyntacticNormalizationDataCollector extends BaseFileSyntacticNormalizationDataCollector { + private static Logger LOG = LogManager.getLogger(SporkFileSyntacticNormalizationDataCollector.class) + + private static final String SPORK_JAR_PATH = "${System.getProperty("user.dir")}/dependencies/spork.jar" + + SporkFileSyntacticNormalizationDataCollector(String inputFile, String outputFile) { + super(inputFile, outputFile) + } + + @Override + protected boolean runNormalizationOnFile(Path inputFile, Path outputFile) { + def processBuilder = ProcessRunner.buildProcess(System.getProperty("user.dir")) + processBuilder.command().addAll(getBuildParameters(inputFile, outputFile)) + + LOG.trace("Calling spork with command \"${processBuilder.command().join(' ')}\"") + def process = ProcessRunner.startProcess(processBuilder) + process.getInputStream().eachLine(LOG::trace) + process.getErrorStream().eachLine(LOG::warn) + def hasCompleted = process.waitFor(1, TimeUnit.HOURS) + return hasCompleted && process.exitValue() == 0 + } + + private static List getBuildParameters(Path inputFile, Path outputFile) { + def list = new ArrayList() + list.add("java") + list.add("-jar") + list.add(SPORK_JAR_PATH) + list.add(inputFile.toString()) + list.add(inputFile.toString()) + list.add(inputFile.toString()) + list.add("--output=${outputFile.toString()}".toString()) + return list + } +} diff --git a/src/main/services/dataCollectors/mergeToolExecutors/BaseMergeToolExecutorDataCollector.groovy b/src/main/services/dataCollectors/mergeToolExecutors/BaseMergeToolExecutorDataCollector.groovy new file mode 100644 index 000000000..c538f53c3 --- /dev/null +++ b/src/main/services/dataCollectors/mergeToolExecutors/BaseMergeToolExecutorDataCollector.groovy @@ -0,0 +1,106 @@ +package services.dataCollectors.mergeToolExecutors + +import interfaces.DataCollector +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import project.MergeCommit +import project.Project +import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector +import services.dataCollectors.mergeToolExecutors.model.MergeExecutionResult +import services.dataCollectors.mergeToolExecutors.model.MergeExecutionSummary +import services.util.MergeConflict +import util.CsvUtils +import util.ProcessRunner + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.concurrent.TimeUnit +import java.util.stream.Collectors + +import static app.MiningFramework.arguments + +abstract class BaseMergeToolExecutorDataCollector implements DataCollector { + private static Logger LOG = LogManager.getLogger(BaseMergeToolExecutorDataCollector.class) + + protected static PERF_SAMPLING_TOTAL_NUMBER_OF_EXECUTIONS = 1 + protected static TIMEOUT_IN_HOURS = 1 + + @Override + void collectData(Project project, MergeCommit mergeCommit) { + def scenarioFiles = MergeScenarioCollector.collectNonFastForwardMergeScenarios(project, mergeCommit) + if (scenarioFiles.isEmpty()) { + LOG.debug("Early returning because there are no mutually modified files") + return + } + + def summaries = scenarioFiles.stream() + .map(this::runMergeForFile) + .map(summary -> [project.getName(), mergeCommit.getSHA(), summary.file, summary.output, summary.result, summary.time]) + .map(CsvUtils::toCsvRepresentation) + .collect(Collectors.toList()) + + writeReportToFile(arguments.getOutputPath() + "/reports/merge-tools/${getToolName()}.csv", summaries) + } + + protected static synchronized writeReportToFile(String reportFilePath, List lines) { + def reportFile = new File(reportFilePath) + Files.createDirectories(Paths.get(arguments.getOutputPath() + "/reports/merge-tools/")) + reportFile.createNewFile() + reportFile << lines.stream().collect(CsvUtils::asLines()) << System.lineSeparator() + } + + MergeExecutionSummary runMergeForFile(Path file) { + LOG.trace("Starting execution of tool ${getToolName()} in ${file}") + List executionTimes = new ArrayList<>() + def outputFilePath = file.resolve("merge." + getToolName().toLowerCase() + ".java") + + for (int i = 0; i < PERF_SAMPLING_TOTAL_NUMBER_OF_EXECUTIONS; i++) { + LOG.trace("Starting execution ${i + 1} of ${PERF_SAMPLING_TOTAL_NUMBER_OF_EXECUTIONS}") + long startTime = System.nanoTime() + executeTool(file, outputFilePath) + long endTime = System.nanoTime() + LOG.trace("Finished execution ${i + 1} of ${PERF_SAMPLING_TOTAL_NUMBER_OF_EXECUTIONS} IN ${endTime - startTime} ns") + // If we're running more than one execution, we use the first one as a warm up + if (PERF_SAMPLING_TOTAL_NUMBER_OF_EXECUTIONS == 1 || i > 0) { + executionTimes.add(endTime - startTime) + } + } + + def result = decideResult(outputFilePath) + long averageTime = (long) (executionTimes.stream().reduce(0, (prev, cur) -> prev + cur) / executionTimes.size()) + + def summary = new MergeExecutionSummary(file, outputFilePath, result, averageTime) + + LOG.trace("Finished execution of tool ${getToolName()} in ${file}. Execution took ${summary.time}ns and finished with ${summary.result.toString()} status") + return summary + } + + protected String getExecutionDirectory() { + return System.getProperty("user.dir") + } + + private void executeTool(Path file, Path outputFile) { + def processBuilder = ProcessRunner.buildProcess(getExecutionDirectory()) + processBuilder.command().addAll(getArgumentsForTool(file, outputFile)) + + LOG.trace("Calling tool ${getToolName()} with command \"${processBuilder.command().join(' ')}\"") + def process = ProcessRunner.startProcess(processBuilder) + process.getInputStream().eachLine(LOG::trace) + process.getErrorStream().eachLine(LOG::warn) + process.waitFor(TIMEOUT_IN_HOURS, TimeUnit.HOURS) + } + + private static MergeExecutionResult decideResult(Path outputFile) { + if (!Files.exists(outputFile)) { + return MergeExecutionResult.TOOL_ERROR + } else if (MergeConflict.getConflictsNumber(outputFile) > 0) { + return MergeExecutionResult.SUCCESS_WITH_CONFLICTS + } + return MergeExecutionResult.SUCCESS_WITHOUT_CONFLICTS + } + + protected abstract List getArgumentsForTool(Path file, Path outputFile); + + abstract String getToolName(); +} diff --git a/src/main/services/dataCollectors/mergeToolExecutors/JDimeMergeToolExecutorDataCollector.groovy b/src/main/services/dataCollectors/mergeToolExecutors/JDimeMergeToolExecutorDataCollector.groovy new file mode 100644 index 000000000..76780e5f5 --- /dev/null +++ b/src/main/services/dataCollectors/mergeToolExecutors/JDimeMergeToolExecutorDataCollector.groovy @@ -0,0 +1,27 @@ +package services.dataCollectors.mergeToolExecutors + + +import java.nio.file.Path + +class JDimeMergeToolExecutorDataCollector extends BaseMergeToolExecutorDataCollector { + @Override + protected String getExecutionDirectory() { + return "${System.getProperty("user.dir")}/dependencies/jdime/install/JDime/bin" + } + + @Override + protected List getArgumentsForTool(Path file, Path outputFile) { + return Arrays.asList("./JDime", + "-f", + "--mode=structured", + "--output=${outputFile.toAbsolutePath().toString()}".toString(), + file.resolve("left.java").toAbsolutePath().toString(), + file.resolve("base.java").toAbsolutePath().toString(), + file.resolve("right.java").toAbsolutePath().toString()) + } + + @Override + String getToolName() { + return "jdime" + } +} diff --git a/src/main/services/dataCollectors/mergeToolExecutors/LastMergeMergeToolExecutorDataCollector.groovy b/src/main/services/dataCollectors/mergeToolExecutors/LastMergeMergeToolExecutorDataCollector.groovy new file mode 100644 index 000000000..a7b8a84c7 --- /dev/null +++ b/src/main/services/dataCollectors/mergeToolExecutors/LastMergeMergeToolExecutorDataCollector.groovy @@ -0,0 +1,25 @@ +package services.dataCollectors.mergeToolExecutors + + +import java.nio.file.Path + +class LastMergeMergeToolExecutorDataCollector extends BaseMergeToolExecutorDataCollector { + + private static final String LAST_MERGE_BINARY_PATH = "${System.getProperty("user.dir")}/dependencies/last-merge" + + @Override + protected List getArgumentsForTool(Path file, Path outputFile) { + return Arrays.asList(LAST_MERGE_BINARY_PATH, + "merge", + "--base-path=${file.resolve("base.java").toAbsolutePath().toString()}".toString(), + "--left-path=${file.resolve("left.java").toAbsolutePath().toString()}".toString(), + "--right-path=${file.resolve("right.java").toAbsolutePath().toString()}".toString(), + "--merge-path=${outputFile.toAbsolutePath().toString()}".toString(), + "--language=java") + } + + @Override + String getToolName() { + return "last_merge" + } +} diff --git a/src/main/services/dataCollectors/mergeToolExecutors/MergirafMergeToolExecutorDataCollector.groovy b/src/main/services/dataCollectors/mergeToolExecutors/MergirafMergeToolExecutorDataCollector.groovy new file mode 100644 index 000000000..989a37fb7 --- /dev/null +++ b/src/main/services/dataCollectors/mergeToolExecutors/MergirafMergeToolExecutorDataCollector.groovy @@ -0,0 +1,22 @@ +package services.dataCollectors.mergeToolExecutors + +import java.nio.file.Path + +class MergirafMergeToolExecutorDataCollector extends BaseMergeToolExecutorDataCollector { + private static String MERGIRAF_PATH = "./dependencies/mergiraf" + + @Override + protected List getArgumentsForTool(Path file, Path outputFile) { + return Arrays.asList(MERGIRAF_PATH, + "merge", + file.resolve("base.java").toAbsolutePath().toString(), + file.resolve("left.java").toAbsolutePath().toString(), + file.resolve("right.java").toAbsolutePath().toString(), + "--output=${outputFile.toAbsolutePath().toString()}".toString()) + } + + @Override + String getToolName() { + return "mergiraf" + } +} diff --git a/src/main/services/dataCollectors/mergeToolExecutors/SporkMergeToolExecutorDataCollector.groovy b/src/main/services/dataCollectors/mergeToolExecutors/SporkMergeToolExecutorDataCollector.groovy new file mode 100644 index 000000000..dc7967b1f --- /dev/null +++ b/src/main/services/dataCollectors/mergeToolExecutors/SporkMergeToolExecutorDataCollector.groovy @@ -0,0 +1,27 @@ +package services.dataCollectors.mergeToolExecutors + + +import java.nio.file.Path + +class SporkMergeToolExecutorDataCollector extends BaseMergeToolExecutorDataCollector { + private static final String SPORK_JAR_PATH = "${System.getProperty("user.dir")}/dependencies/spork.jar" + + @Override + protected List getArgumentsForTool(Path file, Path outputFile) { + def list = new ArrayList() + list.add("ls") + list.add("-jar") + list.add(SPORK_JAR_PATH) + list.add("-e") + list.add(file.resolve("left.java").toString()) + list.add(file.resolve("base.java").toString()) + list.add(file.resolve("right.java").toString()) + list.add("--output=${outputFile.toAbsolutePath().toString()}".toString()) + return list + } + + @Override + String getToolName() { + return 'spork' + } +} diff --git a/src/main/services/dataCollectors/mergeToolExecutors/model/MergeExecutionResult.groovy b/src/main/services/dataCollectors/mergeToolExecutors/model/MergeExecutionResult.groovy new file mode 100644 index 000000000..438644422 --- /dev/null +++ b/src/main/services/dataCollectors/mergeToolExecutors/model/MergeExecutionResult.groovy @@ -0,0 +1,8 @@ +package services.dataCollectors.mergeToolExecutors.model + +enum MergeExecutionResult { + SUCCESS_WITHOUT_CONFLICTS, + SUCCESS_WITH_CONFLICTS, + TOOL_ERROR, + TIMEOUT +} diff --git a/src/main/services/dataCollectors/mergeToolExecutors/model/MergeExecutionSummary.groovy b/src/main/services/dataCollectors/mergeToolExecutors/model/MergeExecutionSummary.groovy new file mode 100644 index 000000000..94f698f44 --- /dev/null +++ b/src/main/services/dataCollectors/mergeToolExecutors/model/MergeExecutionSummary.groovy @@ -0,0 +1,18 @@ +package services.dataCollectors.mergeToolExecutors.model + + +import java.nio.file.Path + +class MergeExecutionSummary { + public final Path file + public final Path output + public final MergeExecutionResult result + public final long time + + MergeExecutionSummary(Path file, Path output, MergeExecutionResult result, long time) { + this.file = file + this.output = output + this.result = result + this.time = time + } +} diff --git a/src/main/services/outputProcessors/GenericMergeDataOutputProcessor.groovy b/src/main/services/outputProcessors/GenericMergeDataOutputProcessor.groovy deleted file mode 100644 index 5b4b39b45..000000000 --- a/src/main/services/outputProcessors/GenericMergeDataOutputProcessor.groovy +++ /dev/null @@ -1,10 +0,0 @@ -package services.outputProcessors - -import interfaces.OutputProcessor - -class GenericMergeDataOutputProcessor implements OutputProcessor{ - @Override - void processOutput() { - println "Processing output" - } -} diff --git a/src/main/services/util/MergeConflict.groovy b/src/main/services/util/MergeConflict.groovy index b521deb2d..6afbf689b 100644 --- a/src/main/services/util/MergeConflict.groovy +++ b/src/main/services/util/MergeConflict.groovy @@ -36,6 +36,17 @@ class MergeConflict { && StringUtils.deleteWhitespace(right) == StringUtils.deleteWhitespace(((MergeConflict) o).right) } + boolean equalsOrSubstring(MergeConflict other) { + return equalsOrSubstringIgnoreWhitespaces(left, other.left) && equalsOrSubstringIgnoreWhitespaces(right, other.right) + } + + private static boolean equalsOrSubstringIgnoreWhitespaces(String a, String b) { + def trimmedA = StringUtils.deleteWhitespace(a) + def trimmedB = StringUtils.deleteWhitespace(b) + + return trimmedA == trimmedB || trimmedA.contains(trimmedB) || trimmedB.contains(trimmedA) + } + /** * @param file * @return the set of merge conflicts present in the given file diff --git a/src/main/services/util/Utils.groovy b/src/main/services/util/Utils.groovy index 574019d65..5fd31fe97 100644 --- a/src/main/services/util/Utils.groovy +++ b/src/main/services/util/Utils.groovy @@ -1,6 +1,7 @@ package services.util import app.MiningFramework +import org.apache.commons.lang3.tuple.Pair import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import project.MergeCommit @@ -18,7 +19,7 @@ final class Utils { * @param repositoryPath * @param arguments */ - static List runGitCommand(Path repositoryPath, String... arguments) { + static int runGitCommand(Path repositoryPath, String... arguments) { Process gitCommand = ProcessRunner.startProcess(buildGitCommand(repositoryPath, arguments)) def exitCode = gitCommand.waitFor() def commandOutput = gitCommand.getInputStream().readLines(); @@ -27,7 +28,7 @@ final class Utils { LOG.warn("Git command exited with error code ${exitCode}.\n Error stream: ${gitCommand.getErrorStream().readLines()}\n Input stream: ${commandOutput}") } - return commandOutput + return exitCode } private static ProcessBuilder buildGitCommand(Path repositoryPath, String... arguments) {