Bug 1428000 - Migrate: only annotate affected files (#34)

The speed-up is massive: running the bug 1411012 migration on the Italian repo used to take ~25 seconds on my machine. After this patch it's down to 0.7 seconds. Running the migration on all locales (without pull/push) now takes less than 2 minutes, compared to 30 minutes before the patch.

In `hg annotate` data, `path` is the path relative to CWD and `abspath` is the path relative to the root of the repo. With CWD==root, they're the same and we can remove `abspath` to reduce the number of different types of paths used in the code.

Author: stasm

tools/migrate/blame.py

@@ -1,19 +1,22 @@
 import argparse
 import json
+import os
+
+from compare_locales.parser import getParser, Junk
 import hglib
 from hglib.util import b, cmdbuilder
-from compare_locales.parser import getParser, Junk
 
 
 class Blame(object):
-    def __init__(self, repopath):
-        self.client = hglib.open(repopath)
+    def __init__(self, client):
+        self.client = client
         self.users = []
         self.blame = {}
 
-    def main(self):
+    def attribution(self, file_paths):
         args = cmdbuilder(
-            b('annotate'), self.client.root(), d=True, u=True, T='json')
+            b('annotate'), template='json', date=True, user=True,
+            cwd=self.client.root(), file=map(b, file_paths))
         blame_json = ''.join(self.client.rawcommand(args))
         file_blames = json.loads(blame_json)
 
@@ -24,16 +27,16 @@ def main(self):
                 'blame': self.blame}
 
     def handleFile(self, file_blame):
-        abspath = file_blame['abspath']
+        path = file_blame['path']
 
         try:
-            parser = getParser(abspath)
+            parser = getParser(path)
         except UserWarning:
             return
 
-        self.blame[abspath] = {}
+        self.blame[path] = {}
 
-        parser.readFile(file_blame['path'])
+        parser.readFile(os.path.join(self.client.root(), path))
         entities, emap = parser.parse()
         for e in entities:
             if isinstance(e, Junk):
@@ -49,12 +52,13 @@ def handleFile(self, file_blame):
             if user not in self.users:
                 self.users.append(user)
             userid = self.users.index(user)
-            self.blame[abspath][e.key] = [userid, timestamp]
+            self.blame[path][e.key] = [userid, timestamp]
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument("repopath")
+    parser.add_argument('repo_path')
+    parser.add_argument('file_path', nargs='+')
     args = parser.parse_args()
-    blame = Blame(args.repopath)
-    blimey = blame.main()
-    print(json.dumps(blimey, indent=4, separators=(',', ': ')))
+    blame = Blame(hglib.open(args.repo_path))
+    attrib = blame.attribution(args.file_path)
+    print(json.dumps(attrib, indent=4, separators=(',', ': ')))

tools/migrate/migrate-l10n.py

@@ -17,9 +17,8 @@
 from blame import Blame
 
 
-def main(lang, reference_dir, localization_dir, blame, migrations, dry_run):
+def main(lang, reference_dir, localization_dir, migrations, dry_run):
     """Run migrations and commit files with the result."""
-    changesets = convert_blame_to_changesets(blame)
     client = hglib.open(localization_dir)
 
     for migration in migrations:
@@ -38,6 +37,12 @@ def main(lang, reference_dir, localization_dir, blame, migrations, dry_run):
         # Keep track of how many changesets we're committing.
         index = 0
 
+        # Annotate legacy localization files used as sources by this migration
+        # to preserve attribution of translations.
+        files = ctx.localization_resources.keys()
+        blame = Blame(client).attribution(files)
+        changesets = convert_blame_to_changesets(blame)
+
         for changeset in changesets:
             # Run the migration for the changeset.
             snapshot = ctx.serialize_changeset(changeset['changes'])
@@ -91,10 +96,6 @@ def main(lang, reference_dir, localization_dir, blame, migrations, dry_run):
         '--localization-dir', type=str,
         help='directory for localization files'
     )
-    parser.add_argument(
-        '--blame', type=argparse.FileType(), default=None,
-        help='path to a JSON with blame information'
-    )
     parser.add_argument(
         '--dry-run', action='store_true',
         help='do not write to disk nor commit any changes'
@@ -106,19 +107,10 @@ def main(lang, reference_dir, localization_dir, blame, migrations, dry_run):
 
     args = parser.parse_args()
 
-    if args.blame:
-        # Load pre-computed blame from a JSON file.
-        blame = json.load(args.blame)
-    else:
-        # Compute blame right now.
-        print('Annotating {}'.format(args.localization_dir))
-        blame = Blame(args.localization_dir).main()
-
     main(
         lang=args.lang,
         reference_dir=args.reference_dir,
         localization_dir=args.localization_dir,
-        blame=blame,
         migrations=map(importlib.import_module, args.migrations),
         dry_run=args.dry_run
     )