Implemention of ratchetFrom for plugin-gradle

gradle.properties

@@ -22,6 +22,7 @@ VER_SLF4J=[1.6,2.0[
 
 # Used in multiple places
 VER_DURIAN=1.2.0
+VER_JGIT=5.7.0.202003110725-r
 VER_JUNIT=4.13
 VER_ASSERTJ=3.15.0
 VER_MOCKITO=3.3.3

lib-extra/build.gradle

@@ -12,7 +12,7 @@ dependencies {
 	implementation "com.diffplug.durian:durian-core:${VER_DURIAN}"
 	implementation "com.diffplug.durian:durian-collect:${VER_DURIAN}"
 	// needed by GitAttributesLineEndings
-	implementation "org.eclipse.jgit:org.eclipse.jgit:5.7.0.202003110725-r"
+	implementation "org.eclipse.jgit:org.eclipse.jgit:${VER_JGIT}"
 	implementation "com.googlecode.concurrent-trees:concurrent-trees:2.6.1"
 	// used for xml parsing in EclipseFormatter
 	implementation "org.codehaus.groovy:groovy-xml:3.0.3"

lib/src/main/java/com/diffplug/spotless/PaddedCell.java

@@ -270,6 +270,11 @@ public void writeCanonicalTo(OutputStream out) throws IOException {
 		}
 	}
 
+	/** Returns the DirtyState which corresponds to `isClean()`. */
+	public static DirtyState isClean() {
+		return isClean;
+	}
+
 	private static final DirtyState didNotConverge = new DirtyState(null);
 	private static final DirtyState isClean = new DirtyState(null);
 }

plugin-gradle/CHANGES.md

@@ -4,6 +4,7 @@ We adhere to the [keepachangelog](https://keepachangelog.com/en/1.0.0/) format (
 
 ## [Unreleased]
 ### Added
+* You can now ratchet a project's style by limiting Spotless only to files which have changed since a given [git reference](https://javadoc.io/static/org.eclipse.jgit/org.eclipse.jgit/5.6.1.202002131546-r/org/eclipse/jgit/lib/Repository.html#resolve-java.lang.String-), e.g. `ratchetFrom 'origin/master'`. ([#590](https://github.com/diffplug/spotless/pull/590))
 * Support for ktfmt in KotlinGradleExtension. ([#583](https://github.com/diffplug/spotless/pull/583))
 ### Fixed
 * Users can now run `spotlessCheck` and `spotlessApply` in the same build. ([#584](https://github.com/diffplug/spotless/pull/584))

plugin-gradle/README.md

@@ -543,20 +543,9 @@ to true.
 
 ## License header options
 
-If the license header (specified with `licenseHeader` or `licenseHeaderFile`) contains `$YEAR` or `$today.year`, then that token will be replaced with the current 4-digit year.
+If the license header (specified with `licenseHeader` or `licenseHeaderFile`) contains `$YEAR` or `$today.year`, then that token will be replaced with the current 4-digit year.  For example, if Spotless is launched in 2017, then `/* Licensed under Apache-2.0 $YEAR. */` will produce `/* Licensed under Apache-2.0 2017. */`
 
-For example:
-```
-/* Licensed under Apache-2.0 $YEAR. */
-```
-will produce
-```
-/* Licensed under Apache-2.0 2017. */
-```
-if Spotless is launched in 2017
-
-
-The `licenseHeader` and `licenseHeaderFile` steps will generate license headers with automatic years from the base license header according to the following rules:
+The `licenseHeader` and `licenseHeaderFile` steps will generate license headers with automatic years according to the following rules:
 * A generated license header will be updated with the current year when
   * the generated license header is missing
   * the generated license header is not formatted correctly
@@ -579,6 +568,8 @@ spotless {
 }
 ```
 
+To update the copyright notice only for changed files, use the [`ratchetFrom` functionality](#ratchet).
+
 <a name="custom"></a>
 
 ## Custom rules
@@ -694,10 +685,29 @@ spotless {
 - If you don't like what spotless did, `git reset --hard`
 - If you'd like to remove the "checkpoint" commit, `git reset --soft head~1` will make the checkpoint commit "disappear" from history, but keeps the changes in your working directory.
 
-<a name="examples"></a>
+<a name="ratchet"></a>
+
+## How can I enforce formatting gradually?
+
+If your project is not currently enforcing formatting, then it can be a noisy transition.  Having a giant commit where every single file gets changed makes the history harder to read.  To address this, you can use the `ratchet` feature:
+
+```gradle
+spotless {
+  ratchetFrom 'origin/master'
+  ...
+}
+```
+
+In this mode, Spotless will apply only to files which have changed since `origin/master`.  You can ratchet from [any point you want](https://javadoc.io/static/org.eclipse.jgit/org.eclipse.jgit/5.6.1.202002131546-r/org/eclipse/jgit/lib/Repository.html#resolve-java.lang.String-), even `HEAD`.
+
+However, we strongly recommend that you use a non-local branch, such as a tag or `origin/master`.  The problem with `HEAD` or any local branch is that as soon as you commit a file, that is now the canonical formatting, even if it was formatted incorrectly.  By instead specifying `origin/master` or a tag, your CI server will fail unless every changed file is at least as good or better than it was before the change.
+
+This is especially helpful for injecting accurate copyright dates using the [license step](#license-header).
 
 ## Can I apply Spotless to specific files?
 
+**DEPRECATED: use [`ratchetFrom`]($ratchet) instead.  The regex API below is difficult to use correctly, especially for cross-platform (win/unix) builds.**
+
 You can target specific files by setting the `spotlessFiles` project property to a comma-separated list of file patterns:
 
 ```
@@ -706,6 +716,8 @@ cmd> gradlew spotlessApply -PspotlessFiles=my/file/pattern.java,more/generic/.*-
 
 The patterns are matched using `String#matches(String)` against the absolute file path.
 
+<a name="examples"></a>
+
 ## Example configurations (from real-world projects)
 
 Spotless is hosted on jcenter and at plugins.gradle.org. [Go here](https://plugins.gradle.org/plugin/com.diffplug.gradle.spotless) if you're not sure how to import the plugin.

plugin-gradle/build.gradle

@@ -19,6 +19,7 @@ dependencies {
 	implementation "com.diffplug.durian:durian-core:${VER_DURIAN}"
 	implementation "com.diffplug.durian:durian-io:${VER_DURIAN}"
 	implementation "com.diffplug.durian:durian-collect:${VER_DURIAN}"
+	implementation "org.eclipse.jgit:org.eclipse.jgit:${VER_JGIT}"
 
 	testImplementation project(':testlib')
 	testImplementation "junit:junit:${VER_JUNIT}"

plugin-gradle/src/main/java/com/diffplug/gradle/spotless/FormatExtension.java

@@ -605,6 +605,9 @@ protected void setupTask(SpotlessTask task) {
 		if (root.project != root.project.getRootProject()) {
 			root.registerDependenciesTask.hookSubprojectTask(task);
 		}
+		if (root.getRatchetFrom() != null) {
+			task.treeSha = GitRatchet.treeShaOf(root.project, root.getRatchetFrom());
+		}
 	}
 
 	/** Returns the project that this extension is attached to. */

plugin-gradle/src/main/java/com/diffplug/gradle/spotless/GitRatchet.java

@@ -0,0 +1,209 @@
+/*
+ * Copyright 2016 DiffPlug
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.diffplug.gradle.spotless;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Objects;
+import java.util.TreeMap;
+
+import javax.annotation.Nullable;
+
+import org.eclipse.jgit.dircache.DirCache;
+import org.eclipse.jgit.dircache.DirCacheIterator;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.RepositoryCache;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
+import org.eclipse.jgit.treewalk.AbstractTreeIterator;
+import org.eclipse.jgit.treewalk.FileTreeIterator;
+import org.eclipse.jgit.treewalk.TreeWalk;
+import org.eclipse.jgit.treewalk.WorkingTreeIterator;
+import org.eclipse.jgit.treewalk.filter.AndTreeFilter;
+import org.eclipse.jgit.treewalk.filter.IndexDiffFilter;
+import org.eclipse.jgit.treewalk.filter.PathFilter;
+import org.eclipse.jgit.util.FS;
+import org.gradle.api.Project;
+
+import com.diffplug.common.base.Errors;
+import com.diffplug.common.collect.HashBasedTable;
+import com.diffplug.common.collect.Table;
+
+class GitRatchet implements AutoCloseable {
+	/** There is a single GitRatchet instance shared across the entire Gradle build, this method helps you get it. */
+	private static GitRatchet instance(Project project) {
+		return project.getPlugins().getPlugin(SpotlessPlugin.class).spotlessExtension.registerDependenciesTask.gitRatchet;
+	}
+
+	/**
+	 * This is the highest-level method, which all the others serve.  Given the sha
+	 * of a git tree (not a commit!), and the file in question, this method returns
+	 * true if that file is clean relative to that tree.  A naive implementation of this
+	 * could be verrrry slow, so the rest of this is about speeding this up.
+	 */
+	public static boolean isClean(Project project, ObjectId treeSha, File file) throws IOException {
+		GitRatchet instance = instance(project);
+		Repository repo = instance.repositoryFor(project);
+		String path = repo.getWorkTree().toPath().relativize(file.toPath()).toString();
+
+		// TODO: should be cached-per-repo if it is thread-safe, or per-repo-per-thread if it is not
+		DirCache dirCache = repo.readDirCache();
+
+		try (TreeWalk treeWalk = new TreeWalk(repo)) {
+			treeWalk.addTree(treeSha);
+			treeWalk.addTree(new DirCacheIterator(dirCache));
+			treeWalk.addTree(new FileTreeIterator(repo));
+			treeWalk.setFilter(AndTreeFilter.create(
+					PathFilter.create(path),
+					new IndexDiffFilter(INDEX, WORKDIR)));
+
+			if (!treeWalk.next()) {
+				// the file we care about is git clean
+				return true;
+			} else {
+				AbstractTreeIterator treeIterator = treeWalk.getTree(TREE, AbstractTreeIterator.class);
+				DirCacheIterator dirCacheIterator = treeWalk.getTree(INDEX, DirCacheIterator.class);
+				WorkingTreeIterator workingTreeIterator = treeWalk.getTree(WORKDIR, WorkingTreeIterator.class);
+
+				boolean hasTree = treeIterator != null;
+				boolean hasDirCache = dirCacheIterator != null;
+
+				if (!hasTree) {
+					// it's not in the tree, so it was added
+					return false;
+				} else {
+					if (hasDirCache) {
+						boolean treeEqualsIndex = treeIterator.idEqual(dirCacheIterator) && treeIterator.getEntryRawMode() == dirCacheIterator.getEntryRawMode();
+						boolean indexEqualsWC = !workingTreeIterator.isModified(dirCacheIterator.getDirCacheEntry(), true, treeWalk.getObjectReader());
+						if (treeEqualsIndex != indexEqualsWC) {
+							// if one is equal and the other isn't, then it has definitely changed
+							return false;
+						} else if (treeEqualsIndex) {
+							// this means they are all equal to each other, which should never happen
+							// the IndexDiffFilter should keep those out of the TreeWalk entirely
+							throw new IllegalStateException("Index status for " + file + " against treeSha " + treeSha + " is invalid.");
+						} else {
+							// they are all unique
+							// we have to check manually
+							return worktreeIsCleanCheckout(treeWalk);
+						}
+					} else {
+						// no dirCache, so we will compare the tree to the workdir manually
+						return worktreeIsCleanCheckout(treeWalk);
+					}
+				}
+			}
+		}
+	}
+
+	/** Returns true if the worktree file is a clean checkout of head (possibly smudged). */
+	private static boolean worktreeIsCleanCheckout(TreeWalk treeWalk) {
+		return treeWalk.idEqual(TREE, WORKDIR);
+	}
+
+	private final static int TREE = 0;
+	private final static int INDEX = 1;
+	private final static int WORKDIR = 2;
+
+	TreeMap<Project, Repository> gitRoots = new TreeMap<>();
+	Table<Repository, String, ObjectId> shaCache = HashBasedTable.create();
+
+	/**
+	 * The first part of making this fast is finding the appropriate git repository quickly.  Because of composite
+	 * builds and submodules, it's quite possible that a single Gradle project will span across multiple git repositories.
+	 * We cache the Repository for every Project in `gitRoots`, and use dynamic programming to populate it.
+	 */
+	private Repository repositoryFor(Project project) throws IOException {
+		Repository repo = gitRoots.get(project);
+		if (repo == null) {
+			if (isGitRoot(project.getProjectDir())) {
+				repo = createRepo(project.getProjectDir());
+			} else {
+				Project parentProj = project.getParent();
+				if (parentProj == null) {
+					repo = traverseParentsUntil(project.getProjectDir().getParentFile(), null);
+					if (repo == null) {
+						throw new IllegalArgumentException("Cannot find git repository in any parent directory");
+					}
+				} else {
+					repo = traverseParentsUntil(project.getProjectDir().getParentFile(), parentProj.getProjectDir());
+					if (repo == null) {
+						repo = repositoryFor(parentProj);
+					}
+				}
+			}
+			gitRoots.put(project, repo);
+		}
+		return repo;
+	}
+
+	private static @Nullable Repository traverseParentsUntil(File startWith, File file) throws IOException {
+		do {
+			if (isGitRoot(startWith)) {
+				return createRepo(startWith);
+			} else {
+				startWith = startWith.getParentFile();
+			}
+		} while (!Objects.equals(startWith, file));
+		return null;
+	}
+
+	private static boolean isGitRoot(File dir) {
+		File dotGit = new File(dir, Constants.DOT_GIT);
+		return dotGit.isDirectory() && RepositoryCache.FileKey.isGitRepository(dotGit, FS.DETECTED);
+	}
+
+	static Repository createRepo(File dir) throws IOException {
+		return FileRepositoryBuilder.create(new File(dir, Constants.DOT_GIT));
+	}
+
+	/**
+	 * Fast way to return treeSha of the given ref against the git repository which stores the given project.
+	 * Because of parallel project evaluation, there may be races here, so we synchronize on ourselves.  However, this method
+	 * is the only method which can trigger any changes, and it is only called during project evaluation.  That means our state
+	 * is final/read-only during task execution, so we don't need any locks during the heavy lifting.
+	 */
+	public static ObjectId treeShaOf(Project project, String reference) {
+		GitRatchet instance = instance(project);
+		synchronized (instance) {
+			try {
+				Repository repo = instance.repositoryFor(project);
+				ObjectId treeSha = instance.shaCache.get(repo, reference);
+				if (treeSha == null) {
+					ObjectId commitSha = repo.resolve(reference);
+					try (RevWalk revWalk = new RevWalk(repo)) {
+						RevCommit revCommit = revWalk.parseCommit(commitSha);
+						treeSha = revCommit.getTree();
+					}
+					instance.shaCache.put(repo, reference, treeSha);
+				}
+				return treeSha;
+			} catch (Exception e) {
+				throw Errors.asRuntime(e);
+			}
+		}
+	}
+
+	@Override
+	public void close() {
+		gitRoots.values().stream()
+				.distinct()
+				.forEach(Repository::close);
+	}
+}

plugin-gradle/src/main/java/com/diffplug/gradle/spotless/IdeHook.java

@@ -19,6 +19,8 @@
 import java.io.IOException;
 import java.nio.file.Files;
 
+import org.eclipse.jgit.lib.ObjectId;
+
 import com.diffplug.common.base.Errors;
 import com.diffplug.common.io.ByteStreams;
 import com.diffplug.spotless.Formatter;
@@ -29,6 +31,10 @@ class IdeHook {
 	final static String USE_STD_IN = "spotlessIdeHookUseStdIn";
 	final static String USE_STD_OUT = "spotlessIdeHookUseStdOut";
 
+	private static void dumpIsClean() {
+		System.err.println("IS CLEAN");
+	}
+
 	static void performHook(SpotlessTask spotlessTask) {
 		String path = (String) spotlessTask.getProject().property(PROPERTY);
 		File file = new File(path);
@@ -38,6 +44,12 @@ static void performHook(SpotlessTask spotlessTask) {
 		}
 		if (spotlessTask.getTarget().contains(file)) {
 			try (Formatter formatter = spotlessTask.buildFormatter()) {
+				if (!spotlessTask.getRatchetSha().equals(ObjectId.zeroId())) {
+					if (GitRatchet.isClean(spotlessTask.getProject(), spotlessTask.treeSha, file)) {
+						dumpIsClean();
+						return;
+					}
+				}
 				byte[] bytes;
 				if (spotlessTask.getProject().hasProperty(USE_STD_IN)) {
 					bytes = ByteStreams.toByteArray(System.in);
@@ -46,7 +58,7 @@ static void performHook(SpotlessTask spotlessTask) {
 				}
 				PaddedCell.DirtyState dirty = PaddedCell.calculateDirtyState(formatter, file, bytes);
 				if (dirty.isClean()) {
-					System.err.println("IS CLEAN");
+					dumpIsClean();
 				} else if (dirty.didNotConverge()) {
 					System.err.println("DID NOT CONVERGE");
 					System.err.println("Run 'spotlessDiagnose' for details https://github.com/diffplug/spotless/blob/master/PADDEDCELL.md");
@@ -58,11 +70,12 @@ static void performHook(SpotlessTask spotlessTask) {
 						dirty.writeCanonicalTo(file);
 					}
 				}
-				System.err.close();
-				System.out.close();
 			} catch (IOException e) {
 				e.printStackTrace(System.err);
 				throw Errors.asRuntime(e);
+			} finally {
+				System.err.close();
+				System.out.close();
 			}
 		}
 	}