Fix some fine-grained cache/fswatcher problems (#4560)

In fswatcher, update the cache if mtime changes; otherwise an mtime
change without an md5 change will cause the file to be rechecked every
time. This was triggered in a painful way by fswatcher's cache being
populated with mtimes from the cache files.

Flush the fscache after an initial fine-grained upgrade, which could
cause certain changes to be missed temporarily.

Don't compute hashes for the whole source tree redundantly in caching
mode, which saves a bunch of time on initial load.

Author: msullivan

mypy/dmypy_server.py

@@ -251,8 +251,9 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
         self.fscache = FileSystemCache(self.options.python_version)
         self.fswatcher = FileSystemWatcher(self.fscache)
         self.update_sources(sources)
-        # Stores the initial state of sources as a side effect.
-        self.fswatcher.find_changed()
+        if not self.options.use_fine_grained_cache:
+            # Stores the initial state of sources as a side effect.
+            self.fswatcher.find_changed()
         try:
             # TODO: alt_lib_path
             result = mypy.build.build(sources=sources,
@@ -288,19 +289,26 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
             changed = self.find_changed(sources)
             if changed:
                 messages = self.fine_grained_manager.update(changed)
+            self.fscache.flush()
 
         status = 1 if messages else 0
         self.previous_messages = messages[:]
         return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
 
     def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
+        t0 = time.time()
         self.update_sources(sources)
         changed = self.find_changed(sources)
+        t1 = time.time()
         if not changed:
             # Nothing changed -- just produce the same result as before.
             messages = self.previous_messages
         else:
             messages = self.fine_grained_manager.update(changed)
+        t2 = time.time()
+        self.fine_grained_manager.manager.log(
+            "fine-grained increment: find_changed: {:.3f}s, update: {:.3f}s".format(
+                t1 - t0, t2 - t1))
         status = 1 if messages else 0
         self.previous_messages = messages[:]
         self.previous_sources = sources

mypy/fswatcher.py

@@ -79,8 +79,8 @@ def find_changed(self) -> Set[str]:
                     # Only look for changes if size or mtime has changed as an
                     # optimization, since calculating md5 is expensive.
                     new_md5 = self.fs.md5(path)
+                    self._update(path)
                     if st.st_size != old.st_size or new_md5 != old.md5:
                         # Changed file.
                         changed.add(path)
-                        self._update(path)
         return changed