diff --git a/fluent/migrate/__init__.py b/fluent/migrate/__init__.py
index 2a92dd5e..3582468d 100644
--- a/fluent/migrate/__init__.py
+++ b/fluent/migrate/__init__.py
@@ -1,6 +1,9 @@
# coding=utf8
from .context import MergeContext # noqa: F401
+from .errors import ( # noqa: F401
+ MigrationError, NotSupportedError, UnreadableReferenceError
+)
from .transforms import ( # noqa: F401
Source, COPY, REPLACE_IN_TEXT, REPLACE, PLURALS, CONCAT
)
diff --git a/fluent/migrate/context.py b/fluent/migrate/context.py
index 4c8a8c44..974f2cbe 100644
--- a/fluent/migrate/context.py
+++ b/fluent/migrate/context.py
@@ -2,6 +2,7 @@
from __future__ import unicode_literals
import os
+import sys
import codecs
import logging
@@ -18,7 +19,7 @@ def getParser(path):
from .cldr import get_plural_categories
from .transforms import Source
from .merge import merge_resource
-from .util import get_message
+from .errors import NotSupportedError, UnreadableReferenceError
class MergeContext(object):
@@ -79,6 +80,10 @@ def read_ftl_resource(self, path):
f = codecs.open(path, 'r', 'utf8')
try:
contents = f.read()
+ except UnicodeDecodeError as err:
+ logger = logging.getLogger('migrate')
+ logger.warn('Unable to read file {}: {}'.format(path, err))
+ raise err
finally:
f.close()
@@ -94,7 +99,7 @@ def read_ftl_resource(self, path):
logger = logging.getLogger('migrate')
for annot in annots:
msg = annot.message
- logger.warn(u'Syntax error in {}: {}'.format(path, msg))
+ logger.warn('Syntax error in {}: {}'.format(path, msg))
return ast
@@ -105,52 +110,33 @@ def read_legacy_resource(self, path):
# Transform the parsed result which is an iterator into a dict.
return {entity.key: entity.val for entity in parser}
- def add_reference(self, path, realpath=None):
- """Add an FTL AST to this context's reference resources."""
- fullpath = os.path.join(self.reference_dir, realpath or path)
- try:
- ast = self.read_ftl_resource(fullpath)
- except IOError as err:
- logger = logging.getLogger('migrate')
- logger.error(u'Missing reference file: {}'.format(path))
- raise err
- except UnicodeDecodeError as err:
- logger = logging.getLogger('migrate')
- logger.error(u'Error reading file {}: {}'.format(path, err))
- raise err
- else:
- self.reference_resources[path] = ast
+ def maybe_add_localization(self, path):
+ """Add a localization resource to migrate translations from.
- def add_localization(self, path):
- """Add an existing localization resource.
+ Only legacy resources can be added as migration sources. The resource
+ may be missing on disk.
- If it's an FTL resource, add an FTL AST. Otherwise, it's a legacy
- resource. Use a compare-locales parser to create a dict of (key,
- string value) tuples.
+ Uses a compare-locales parser to create a dict of (key, string value)
+ tuples.
"""
- fullpath = os.path.join(self.localization_dir, path)
- if fullpath.endswith('.ftl'):
- try:
- ast = self.read_ftl_resource(fullpath)
- except IOError:
- logger = logging.getLogger('migrate')
- logger.warn(u'Missing localization file: {}'.format(path))
- except UnicodeDecodeError as err:
- logger = logging.getLogger('migrate')
- logger.warn(u'Error reading file {}: {}'.format(path, err))
- else:
- self.localization_resources[path] = ast
+ if path.endswith('.ftl'):
+ error_message = (
+ 'Migrating translations from Fluent files is not supported '
+ '({})'.format(path))
+ logging.getLogger('migrate').error(error_message)
+ raise NotSupportedError(error_message)
+
+ try:
+ fullpath = os.path.join(self.localization_dir, path)
+ collection = self.read_legacy_resource(fullpath)
+ except IOError:
+ logger = logging.getLogger('migrate')
+ logger.warn('Missing localization file: {}'.format(path))
else:
- try:
- collection = self.read_legacy_resource(fullpath)
- except IOError:
- logger = logging.getLogger('migrate')
- logger.warn(u'Missing localization file: {}'.format(path))
- else:
- self.localization_resources[path] = collection
+ self.localization_resources[path] = collection
- def add_transforms(self, path, transforms):
- """Define transforms for path.
+ def add_transforms(self, path, reference, transforms):
+ """Define transforms for path using reference as template.
Each transform is an extended FTL node with `Transform` nodes as some
values. Transforms are stored in their lazy AST form until
@@ -165,6 +151,22 @@ def get_sources(acc, cur):
acc.add((cur.path, cur.key))
return acc
+ refpath = os.path.join(self.reference_dir, reference)
+ try:
+ ast = self.read_ftl_resource(refpath)
+ except IOError as err:
+ error_message = 'Missing reference file: {}'.format(refpath)
+ logging.getLogger('migrate').error(error_message)
+ raise UnreadableReferenceError(error_message)
+ except UnicodeDecodeError as err:
+ error_message = 'Error reading file {}: {}'.format(refpath, err)
+ logging.getLogger('migrate').error(error_message)
+ raise UnreadableReferenceError(error_message)
+ else:
+ # The reference file will be used by the merge function as
+ # a template for serializing the merge results.
+ self.reference_resources[path] = ast
+
for node in transforms:
# Scan `node` for `Source` nodes and collect the information they
# store into a set of dependencies.
@@ -175,17 +177,30 @@ def get_sources(acc, cur):
path_transforms = self.transforms.setdefault(path, [])
path_transforms += transforms
+ if path not in self.localization_resources:
+ fullpath = os.path.join(self.localization_dir, path)
+ try:
+ ast = self.read_ftl_resource(fullpath)
+ except IOError:
+ logger = logging.getLogger('migrate')
+ logger.info(
+ 'Localization file {} does not exist and '
+ 'it will be created'.format(path))
+ except UnicodeDecodeError:
+ logger = logging.getLogger('migrate')
+ logger.warn(
+ 'Localization file {} will be re-created and some '
+ 'translations might be lost'.format(path))
+ else:
+ self.localization_resources[path] = ast
+
def get_source(self, path, key):
- """Get an entity value from the localized source.
+ """Get an entity value from a localized legacy source.
Used by the `Source` transform.
"""
- if path.endswith('.ftl'):
- resource = self.localization_resources[path]
- return get_message(resource.body, key)
- else:
- resource = self.localization_resources[path]
- return resource.get(key, None)
+ resource = self.localization_resources[path]
+ return resource.get(key, None)
def merge_changeset(self, changeset=None):
"""Return a generator of FTL ASTs for the changeset.
@@ -200,10 +215,11 @@ def merge_changeset(self, changeset=None):
"""
if changeset is None:
- # Merge all known legacy translations.
+ # Merge all known legacy translations. Used in tests.
changeset = {
(path, key)
for path, strings in self.localization_resources.iteritems()
+ if not path.endswith('.ftl')
for key in strings.iterkeys()
}
@@ -240,10 +256,15 @@ def in_changeset(ident):
self, reference, current, transforms, in_changeset
)
- # If none of the transforms is in the given changeset, the merged
- # snapshot is identical to the current translation. We compare
- # JSON trees rather then use filtering by `in_changeset` to account
- # for translations removed from `reference`.
+ # Skip this path if the merged snapshot is identical to the current
+ # state of the localization file. This may happen when:
+ #
+ # - none of the transforms is in the changset, or
+ # - all messages which would be migrated by the context's
+ # transforms already exist in the current state.
+ #
+ # We compare JSON trees rather then use filtering by `in_changeset`
+ # to account for translations removed from `reference`.
if snapshot.to_json() == current.to_json():
continue
diff --git a/fluent/migrate/errors.py b/fluent/migrate/errors.py
new file mode 100644
index 00000000..e52984f0
--- /dev/null
+++ b/fluent/migrate/errors.py
@@ -0,0 +1,10 @@
+class MigrationError(ValueError):
+ pass
+
+
+class NotSupportedError(MigrationError):
+ pass
+
+
+class UnreadableReferenceError(MigrationError):
+ pass
diff --git a/fluent/migrate/merge.py b/fluent/migrate/merge.py
index 3112cb54..586393e5 100644
--- a/fluent/migrate/merge.py
+++ b/fluent/migrate/merge.py
@@ -11,9 +11,10 @@ def merge_resource(ctx, reference, current, transforms, in_changeset):
"""Transform legacy translations into FTL.
Use the `reference` FTL AST as a template. For each en-US string in the
- reference, first check if it's in the currently processed changeset with
- `in_changeset`; then check for an existing translation in the current FTL
- `localization` or for a migration specification in `transforms`.
+ reference, first check for an existing translation in the current FTL
+ `localization` and use it if it's present; then if the string has
+ a transform defined in the migration specification and if it's in the
+ currently processed changeset, evaluate the transform.
"""
def merge_body(body):
diff --git a/fluent/migrate/transforms.py b/fluent/migrate/transforms.py
index 942fbd90..b747a128 100644
--- a/fluent/migrate/transforms.py
+++ b/fluent/migrate/transforms.py
@@ -66,6 +66,7 @@
from __future__ import unicode_literals
import fluent.syntax.ast as FTL
+from .errors import NotSupportedError
def pattern_from_text(value):
@@ -113,6 +114,11 @@ class Source(Transform):
# not be replaced?
def __init__(self, path, key):
+ if path.endswith('.ftl'):
+ raise NotSupportedError(
+ 'Migrating translations from Fluent files is not supported '
+ '({})'.format(path))
+
self.path = path
self.key = key
diff --git a/tests/migrate/fixtures/en-US/privacy.ftl b/tests/migrate/fixtures/en-US/privacy.ftl
new file mode 100644
index 00000000..bedd5a7c
--- /dev/null
+++ b/tests/migrate/fixtures/en-US/privacy.ftl
@@ -0,0 +1,9 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+dnt-description =
+ Send websites a “Do Not Track” signal
+ that you don’t want to be tracked
+dnt-learn-more = Learn more
+dnt-always = Always
diff --git a/tests/migrate/fixtures/pl/privacy.dtd b/tests/migrate/fixtures/pl/privacy.dtd
new file mode 100644
index 00000000..36878d2f
--- /dev/null
+++ b/tests/migrate/fixtures/pl/privacy.dtd
@@ -0,0 +1,7 @@
+
+
+
+
+
diff --git a/tests/migrate/fixtures/pl/privacy.ftl b/tests/migrate/fixtures/pl/privacy.ftl
new file mode 100644
index 00000000..772da74e
--- /dev/null
+++ b/tests/migrate/fixtures/pl/privacy.ftl
@@ -0,0 +1,5 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+dnt-description = New Description in Polish
diff --git a/tests/migrate/test_concat.py b/tests/migrate/test_concat.py
index e63f41c2..48a4626e 100644
--- a/tests/migrate/test_concat.py
+++ b/tests/migrate/test_concat.py
@@ -16,6 +16,8 @@
class MockContext(unittest.TestCase):
def get_source(self, path, key):
+ # Ignore path (test.properties) and get translations from self.strings
+ # defined in setUp.
return self.strings.get(key, None).val
@@ -36,7 +38,7 @@ def test_concat_one(self):
msg = FTL.Message(
FTL.Identifier('hello'),
value=CONCAT(
- COPY(self.strings, 'hello'),
+ COPY('test.properties', 'hello'),
)
)
@@ -51,8 +53,8 @@ def test_concat_two(self):
msg = FTL.Message(
FTL.Identifier('hello'),
value=CONCAT(
- COPY(self.strings, 'hello.start'),
- COPY(self.strings, 'hello.end'),
+ COPY('test.properties', 'hello.start'),
+ COPY('test.properties', 'hello.end'),
)
)
@@ -86,8 +88,8 @@ def test_concat_whitespace_begin(self):
msg = FTL.Message(
FTL.Identifier('hello'),
value=CONCAT(
- COPY(self.strings, 'whitespace.begin.start'),
- COPY(self.strings, 'whitespace.begin.end'),
+ COPY('test.properties', 'whitespace.begin.start'),
+ COPY('test.properties', 'whitespace.begin.end'),
)
)
@@ -103,8 +105,8 @@ def test_concat_whitespace_end(self):
msg = FTL.Message(
FTL.Identifier('hello'),
value=CONCAT(
- COPY(self.strings, 'whitespace.end.start'),
- COPY(self.strings, 'whitespace.end.end'),
+ COPY('test.properties', 'whitespace.end.start'),
+ COPY('test.properties', 'whitespace.end.end'),
)
)
@@ -129,11 +131,11 @@ def test_concat_literal(self):
msg = FTL.Message(
FTL.Identifier('update-failed'),
value=CONCAT(
- COPY(self.strings, 'update.failed.start'),
+ COPY('test.properties', 'update.failed.start'),
FTL.TextElement(''),
- COPY(self.strings, 'update.failed.linkText'),
+ COPY('test.properties', 'update.failed.linkText'),
FTL.TextElement(''),
- COPY(self.strings, 'update.failed.end'),
+ COPY('test.properties', 'update.failed.end'),
)
)
@@ -157,9 +159,9 @@ def test_concat_replace(self):
msg = FTL.Message(
FTL.Identifier('channel-desc'),
value=CONCAT(
- COPY(self.strings, 'channel.description.start'),
+ COPY('test.properties', 'channel.description.start'),
FTL.Placeable(EXTERNAL_ARGUMENT('channelname')),
- COPY(self.strings, 'channel.description.end'),
+ COPY('test.properties', 'channel.description.end'),
)
)
@@ -187,7 +189,7 @@ def test_concat_replace(self):
FTL.Identifier('community'),
value=CONCAT(
REPLACE(
- self.strings,
+ 'test.properties',
'community.start',
{
'&brandShortName;': MESSAGE_REFERENCE(
@@ -197,7 +199,7 @@ def test_concat_replace(self):
),
FTL.TextElement(''),
REPLACE(
- self.strings,
+ 'test.properties',
'community.mozillaLink',
{
'&vendorShortName;': MESSAGE_REFERENCE(
@@ -206,11 +208,11 @@ def test_concat_replace(self):
}
),
FTL.TextElement(''),
- COPY(self.strings, 'community.middle'),
+ COPY('test.properties', 'community.middle'),
FTL.TextElement(''),
- COPY(self.strings, 'community.creditsLink'),
+ COPY('test.properties', 'community.creditsLink'),
FTL.TextElement(''),
- COPY(self.strings, 'community.end')
+ COPY('test.properties', 'community.end')
)
)
diff --git a/tests/migrate/test_context.py b/tests/migrate/test_context.py
index 775db8f2..4b7eef32 100644
--- a/tests/migrate/test_context.py
+++ b/tests/migrate/test_context.py
@@ -7,6 +7,7 @@
import fluent.syntax.ast as FTL
+from fluent.migrate.errors import NotSupportedError, UnreadableReferenceError
from fluent.migrate.util import ftl, ftl_resource_to_json, to_json
from fluent.migrate.context import MergeContext
from fluent.migrate.transforms import COPY
@@ -25,15 +26,14 @@ def setUp(self):
localization_dir=here('fixtures/pl')
)
- self.ctx.add_reference('aboutDownloads.ftl')
try:
- self.ctx.add_localization('aboutDownloads.dtd')
- self.ctx.add_localization('aboutDownloads.properties')
+ self.ctx.maybe_add_localization('aboutDownloads.dtd')
+ self.ctx.maybe_add_localization('aboutDownloads.properties')
except RuntimeError:
self.skipTest('compare-locales required')
def test_hardcoded_node(self):
- self.ctx.add_transforms('aboutDownloads.ftl', [
+ self.ctx.add_transforms('aboutDownloads.ftl', 'aboutDownloads.ftl', [
FTL.Message(
id=FTL.Identifier('about'),
value=FTL.Pattern([
@@ -58,7 +58,7 @@ def test_hardcoded_node(self):
)
def test_merge_single_message(self):
- self.ctx.add_transforms('aboutDownloads.ftl', [
+ self.ctx.add_transforms('aboutDownloads.ftl', 'aboutDownloads.ftl', [
FTL.Message(
id=FTL.Identifier('title'),
value=COPY(
@@ -84,7 +84,7 @@ def test_merge_single_message(self):
)
def test_merge_one_changeset(self):
- self.ctx.add_transforms('aboutDownloads.ftl', [
+ self.ctx.add_transforms('aboutDownloads.ftl', 'aboutDownloads.ftl', [
FTL.Message(
id=FTL.Identifier('title'),
value=COPY(
@@ -123,7 +123,7 @@ def test_merge_one_changeset(self):
)
def test_merge_two_changesets(self):
- self.ctx.add_transforms('aboutDownloads.ftl', [
+ self.ctx.add_transforms('aboutDownloads.ftl', 'aboutDownloads.ftl', [
FTL.Message(
id=FTL.Identifier('title'),
value=COPY(
@@ -176,7 +176,7 @@ def test_merge_two_changesets(self):
self.assertDictEqual(merged_b, expected_b)
def test_serialize_changeset(self):
- self.ctx.add_transforms('aboutDownloads.ftl', [
+ self.ctx.add_transforms('aboutDownloads.ftl', 'aboutDownloads.ftl', [
FTL.Message(
id=FTL.Identifier('title'),
value=COPY(
@@ -245,8 +245,8 @@ def tearDown(self):
logging.disable(logging.NOTSET)
def test_missing_reference_file(self):
- with self.assertRaises(IOError):
- self.ctx.add_reference('missing.ftl')
+ with self.assertRaises(UnreadableReferenceError):
+ self.ctx.add_transforms('some.ftl', 'missing.ftl', [])
class TestIncompleteLocalization(unittest.TestCase):
@@ -260,13 +260,12 @@ def setUp(self):
localization_dir=here('fixtures/pl')
)
- self.ctx.add_reference('toolbar.ftl')
try:
- self.ctx.add_localization('browser.dtd')
+ self.ctx.maybe_add_localization('browser.dtd')
except RuntimeError:
self.skipTest('compare-locales required')
- self.ctx.add_transforms('toolbar.ftl', [
+ self.ctx.add_transforms('toolbar.ftl', 'toolbar.ftl', [
FTL.Message(
id=FTL.Identifier('urlbar-textbox'),
attributes=[
@@ -309,3 +308,122 @@ def test_missing_localization_file(self):
to_json(self.ctx.merge_changeset()),
expected
)
+
+
+class TestExistingTarget(unittest.TestCase):
+ def setUp(self):
+ # Silence all logging.
+ logging.disable(logging.CRITICAL)
+
+ self.ctx = MergeContext(
+ lang='pl',
+ reference_dir=here('fixtures/en-US'),
+ localization_dir=here('fixtures/pl')
+ )
+
+ try:
+ self.ctx.maybe_add_localization('privacy.dtd')
+ except RuntimeError:
+ self.skipTest('compare-locales required')
+
+ def tearDown(self):
+ # Resume logging.
+ logging.disable(logging.NOTSET)
+
+ def test_existing_target_ftl_missing_string(self):
+ self.ctx.add_transforms('privacy.ftl', 'privacy.ftl', [
+ FTL.Message(
+ id=FTL.Identifier('dnt-learn-more'),
+ value=COPY(
+ 'privacy.dtd',
+ 'doNotTrack.learnMore.label'
+ )
+ ),
+ ])
+
+ expected = {
+ 'privacy.ftl': ftl_resource_to_json('''
+ // This Source Code Form is subject to the terms of the Mozilla Public
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
+ // file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+ dnt-description = New Description in Polish
+ dnt-learn-more = Więcej informacji
+ ''')
+ }
+
+ self.maxDiff = None
+ self.assertDictEqual(
+ to_json(self.ctx.merge_changeset()),
+ expected
+ )
+
+ def test_existing_target_ftl_existing_string(self):
+ self.ctx.add_transforms('privacy.ftl', 'privacy.ftl', [
+ FTL.Message(
+ id=FTL.Identifier('dnt-description'),
+ value=COPY(
+ 'privacy.dtd',
+ 'doNotTrack.description'
+ )
+ ),
+
+ # Migrate an extra string to populate the iterator returned by
+ # ctx.merge_changeset(). Otherwise it won't yield anything if the
+ # merged contents are the same as the existing file.
+ FTL.Message(
+ id=FTL.Identifier('dnt-always'),
+ value=COPY(
+ 'privacy.dtd',
+ 'doNotTrack.always.label'
+ )
+ ),
+ ])
+
+ expected = {
+ 'privacy.ftl': ftl_resource_to_json('''
+ // This Source Code Form is subject to the terms of the Mozilla Public
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
+ // file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+ dnt-description = New Description in Polish
+ dnt-always = Zawsze
+ ''')
+ }
+
+ self.maxDiff = None
+ self.assertDictEqual(
+ to_json(self.ctx.merge_changeset()),
+ expected
+ )
+
+ def test_existing_target_ftl_with_all_messages(self):
+ self.ctx.add_transforms('privacy.ftl', 'privacy.ftl', [
+ FTL.Message(
+ id=FTL.Identifier('dnt-description'),
+ value=COPY(
+ 'privacy.dtd',
+ 'doNotTrack.description'
+ )
+ ),
+ ])
+
+ # All migrated messages are already in the target FTL and the result of
+ # merge_changeset is an empty iterator.
+ self.assertDictEqual(
+ to_json(self.ctx.merge_changeset()),
+ {}
+ )
+
+
+class TestNotSupportedError(unittest.TestCase):
+ def test_add_ftl(self):
+ pattern = ('Migrating translations from Fluent files is not supported')
+ with self.assertRaisesRegexp(NotSupportedError, pattern):
+ ctx = MergeContext(
+ lang='pl',
+ reference_dir=here('fixtures/en-US'),
+ localization_dir=here('fixtures/pl')
+ )
+
+ ctx.maybe_add_localization('privacy.ftl')
diff --git a/tests/migrate/test_context_real_examples.py b/tests/migrate/test_context_real_examples.py
index 7953885e..e62f47ed 100644
--- a/tests/migrate/test_context_real_examples.py
+++ b/tests/migrate/test_context_real_examples.py
@@ -27,14 +27,13 @@ def setUp(self):
localization_dir=here('fixtures/pl')
)
- self.ctx.add_reference('aboutDownloads.ftl')
try:
- self.ctx.add_localization('aboutDownloads.dtd')
- self.ctx.add_localization('aboutDownloads.properties')
+ self.ctx.maybe_add_localization('aboutDownloads.dtd')
+ self.ctx.maybe_add_localization('aboutDownloads.properties')
except RuntimeError:
self.skipTest('compare-locales required')
- self.ctx.add_transforms('aboutDownloads.ftl', [
+ self.ctx.add_transforms('aboutDownloads.ftl', 'aboutDownloads.ftl', [
FTL.Message(
id=FTL.Identifier('title'),
value=COPY(
@@ -289,12 +288,11 @@ def setUp(self):
)
try:
- self.ctx.add_reference('aboutDialog.ftl')
- self.ctx.add_localization('aboutDialog.dtd')
+ self.ctx.maybe_add_localization('aboutDialog.dtd')
except RuntimeError:
self.skipTest('compare-locales required')
- self.ctx.add_transforms('aboutDialog.ftl', [
+ self.ctx.add_transforms('aboutDialog.ftl', 'aboutDialog.ftl', [
FTL.Message(
id=FTL.Identifier('update-failed'),
value=CONCAT(
diff --git a/tests/migrate/test_copy.py b/tests/migrate/test_copy.py
index 55e78a20..66eb8c85 100644
--- a/tests/migrate/test_copy.py
+++ b/tests/migrate/test_copy.py
@@ -15,6 +15,8 @@
class MockContext(unittest.TestCase):
def get_source(self, path, key):
+ # Ignore path (test.properties) and get translations from self.strings
+ # defined in setUp.
return self.strings.get(key, None).val
@@ -33,7 +35,7 @@ def setUp(self):
def test_copy(self):
msg = FTL.Message(
FTL.Identifier('foo'),
- value=COPY(self.strings, 'foo')
+ value=COPY('test.properties', 'foo')
)
self.assertEqual(
@@ -46,7 +48,7 @@ def test_copy(self):
def test_copy_escape_unicode_middle(self):
msg = FTL.Message(
FTL.Identifier('foo-unicode-middle'),
- value=COPY(self.strings, 'foo.unicode.middle')
+ value=COPY('test.properties', 'foo.unicode.middle')
)
self.assertEqual(
@@ -60,7 +62,7 @@ def test_copy_escape_unicode_middle(self):
def test_copy_escape_unicode_begin(self):
msg = FTL.Message(
FTL.Identifier('foo-unicode-begin'),
- value=COPY(self.strings, 'foo.unicode.begin')
+ value=COPY('test.properties', 'foo.unicode.begin')
)
self.assertEqual(
@@ -74,7 +76,7 @@ def test_copy_escape_unicode_begin(self):
def test_copy_escape_unicode_end(self):
msg = FTL.Message(
FTL.Identifier('foo-unicode-end'),
- value=COPY(self.strings, 'foo.unicode.end')
+ value=COPY('test.properties', 'foo.unicode.end')
)
self.assertEqual(
@@ -87,7 +89,7 @@ def test_copy_escape_unicode_end(self):
def test_copy_html_entity(self):
msg = FTL.Message(
FTL.Identifier('foo-html-entity'),
- value=COPY(self.strings, 'foo.html.entity')
+ value=COPY('test.properties', 'foo.html.entity')
)
self.assertEqual(
@@ -99,7 +101,7 @@ def test_copy_html_entity(self):
@unittest.skipUnless(DTDParser, 'compare-locales required')
-class TestCopyTraits(MockContext):
+class TestCopyAttributes(MockContext):
def setUp(self):
self.strings = parse(DTDParser, '''
@@ -112,12 +114,12 @@ def test_copy_accesskey(self):
attributes=[
FTL.Attribute(
FTL.Identifier('label'),
- COPY(self.strings, 'checkForUpdatesButton.label')
+ COPY('test.properties', 'checkForUpdatesButton.label')
),
FTL.Attribute(
FTL.Identifier('accesskey'),
COPY(
- self.strings, 'checkForUpdatesButton.accesskey'
+ 'test.properties', 'checkForUpdatesButton.accesskey'
)
),
]
diff --git a/tests/migrate/test_merge.py b/tests/migrate/test_merge.py
index 4da4af44..cdf9ed58 100644
--- a/tests/migrate/test_merge.py
+++ b/tests/migrate/test_merge.py
@@ -17,6 +17,8 @@
class MockContext(unittest.TestCase):
def get_source(self, path, key):
+ # Ignore path (test.properties) and get translations from
+ # self.ab_cd_legacy defined in setUp.
return self.ab_cd_legacy.get(key, None).val
@@ -59,7 +61,7 @@ def setUp(self):
self.transforms = [
FTL.Message(
FTL.Identifier('title'),
- value=COPY(None, 'aboutDownloads.title')
+ value=COPY('test.properties', 'aboutDownloads.title')
),
FTL.Message(
FTL.Identifier('about'),
@@ -72,13 +74,13 @@ def setUp(self):
attributes=[
FTL.Attribute(
FTL.Identifier('label'),
- COPY(None, 'aboutDownloads.open')
+ COPY('test.properties', 'aboutDownloads.open')
),
]
),
FTL.Message(
FTL.Identifier('download-state-downloading'),
- value=COPY(None, 'downloadState.downloading')
+ value=COPY('test.properties', 'downloadState.downloading')
)
]
@@ -169,20 +171,20 @@ def setUp(self):
self.transforms = [
FTL.Message(
FTL.Identifier('title'),
- value=COPY(None, 'aboutDownloads.title')
+ value=COPY('test.properties', 'aboutDownloads.title')
),
FTL.Message(
FTL.Identifier('open-menuitem'),
attributes=[
FTL.Attribute(
FTL.Identifier('label'),
- COPY(None, 'aboutDownloads.open')
+ COPY('test.properties', 'aboutDownloads.open')
),
]
),
FTL.Message(
FTL.Identifier('download-state-downloading'),
- value=COPY(None, 'downloadState.downloading')
+ value=COPY('test.properties', 'downloadState.downloading')
)
]
@@ -282,11 +284,11 @@ def setUp(self):
self.transforms = [
FTL.Message(
FTL.Identifier('title'),
- value=COPY(None, 'aboutDownloads.title')
+ value=COPY('test.properties', 'aboutDownloads.title')
),
FTL.Message(
FTL.Identifier('download-state-downloading'),
- value=COPY(None, 'downloadState.downloading')
+ value=COPY('test.properties', 'downloadState.downloading')
)
]
diff --git a/tests/migrate/test_plural.py b/tests/migrate/test_plural.py
index 095f232d..0fb21726 100644
--- a/tests/migrate/test_plural.py
+++ b/tests/migrate/test_plural.py
@@ -19,6 +19,8 @@ class MockContext(unittest.TestCase):
plural_categories = ('one', 'other')
def get_source(self, path, key):
+ # Ignore path (test.properties) and get translations from self.strings
+ # defined in setUp.
return self.strings.get(key, None).val
@@ -32,7 +34,7 @@ def setUp(self):
self.message = FTL.Message(
FTL.Identifier('delete-all'),
value=PLURALS(
- self.strings,
+ 'test.properties',
'deleteAll',
EXTERNAL_ARGUMENT('num')
)
@@ -86,7 +88,7 @@ def setUp(self):
self.message = FTL.Message(
FTL.Identifier('delete-all'),
value=PLURALS(
- self.strings,
+ 'test.properties',
'deleteAll',
EXTERNAL_ARGUMENT('num')
)
@@ -116,7 +118,7 @@ def test_plural_replace(self):
msg = FTL.Message(
FTL.Identifier('delete-all'),
value=PLURALS(
- self.strings,
+ 'test.properties',
'deleteAll',
EXTERNAL_ARGUMENT('num'),
lambda text: REPLACE_IN_TEXT(
diff --git a/tests/migrate/test_replace.py b/tests/migrate/test_replace.py
index 1157daf4..65576d00 100644
--- a/tests/migrate/test_replace.py
+++ b/tests/migrate/test_replace.py
@@ -16,6 +16,8 @@
class MockContext(unittest.TestCase):
def get_source(self, path, key):
+ # Ignore path (test.properties) and get translations from self.strings
+ # defined in setUp.
return self.strings.get(key, None).val
@@ -33,7 +35,7 @@ def test_replace_one(self):
msg = FTL.Message(
FTL.Identifier(u'hello'),
value=REPLACE(
- self.strings,
+ 'test.properties',
'hello',
{
'#1': EXTERNAL_ARGUMENT('username')
@@ -52,7 +54,7 @@ def test_replace_two(self):
msg = FTL.Message(
FTL.Identifier(u'welcome'),
value=REPLACE(
- self.strings,
+ 'test.properties',
'welcome',
{
'#1': EXTERNAL_ARGUMENT('username'),
@@ -72,7 +74,7 @@ def test_replace_too_many(self):
msg = FTL.Message(
FTL.Identifier(u'welcome'),
value=REPLACE(
- self.strings,
+ 'test.properties',
'welcome',
{
'#1': EXTERNAL_ARGUMENT('username'),
@@ -93,7 +95,7 @@ def test_replace_too_few(self):
msg = FTL.Message(
FTL.Identifier(u'welcome'),
value=REPLACE(
- self.strings,
+ 'test.properties',
'welcome',
{
'#1': EXTERNAL_ARGUMENT('username')
@@ -112,7 +114,7 @@ def test_replace_first(self):
msg = FTL.Message(
FTL.Identifier(u'first'),
value=REPLACE(
- self.strings,
+ 'test.properties',
'first',
{
'#1': EXTERNAL_ARGUMENT('foo')
@@ -131,7 +133,7 @@ def test_replace_last(self):
msg = FTL.Message(
FTL.Identifier(u'last'),
value=REPLACE(
- self.strings,
+ 'test.properties',
'last',
{
'#1': EXTERNAL_ARGUMENT('bar')
diff --git a/tests/migrate/test_source.py b/tests/migrate/test_source.py
new file mode 100644
index 00000000..1da96ac4
--- /dev/null
+++ b/tests/migrate/test_source.py
@@ -0,0 +1,51 @@
+# coding=utf8
+from __future__ import unicode_literals
+
+import unittest
+
+import fluent.syntax.ast as FTL
+
+from fluent.migrate.errors import NotSupportedError
+from fluent.migrate.transforms import Source, COPY, PLURALS, REPLACE
+from fluent.migrate.helpers import EXTERNAL_ARGUMENT
+
+
+class TestNotSupportedError(unittest.TestCase):
+ def test_source(self):
+ pattern = ('Migrating translations from Fluent files is not supported')
+ with self.assertRaisesRegexp(NotSupportedError, pattern):
+ Source('test.ftl', 'foo')
+
+ def test_copy(self):
+ pattern = ('Migrating translations from Fluent files is not supported')
+ with self.assertRaisesRegexp(NotSupportedError, pattern):
+ FTL.Message(
+ FTL.Identifier('foo'),
+ value=COPY('test.ftl', 'foo')
+ )
+
+ def test_plurals(self):
+ pattern = ('Migrating translations from Fluent files is not supported')
+ with self.assertRaisesRegexp(NotSupportedError, pattern):
+ FTL.Message(
+ FTL.Identifier('delete-all'),
+ value=PLURALS(
+ 'test.ftl',
+ 'deleteAll',
+ EXTERNAL_ARGUMENT('num')
+ )
+ )
+
+ def test_replace(self):
+ pattern = ('Migrating translations from Fluent files is not supported')
+ with self.assertRaisesRegexp(NotSupportedError, pattern):
+ FTL.Message(
+ FTL.Identifier(u'hello'),
+ value=REPLACE(
+ 'test.ftl',
+ 'hello',
+ {
+ '#1': EXTERNAL_ARGUMENT('username')
+ }
+ )
+ )
diff --git a/tools/migrate/examples/about_dialog.py b/tools/migrate/examples/about_dialog.py
index 84dae067..17ed610e 100644
--- a/tools/migrate/examples/about_dialog.py
+++ b/tools/migrate/examples/about_dialog.py
@@ -9,10 +9,9 @@
def migrate(ctx):
"""Migrate about:dialog, part {index}"""
- ctx.add_reference('browser/about_dialog.ftl', realpath='about_dialog.ftl')
- ctx.add_localization('browser/chrome/browser/aboutDialog.dtd')
+ ctx.maybe_add_localization('browser/chrome/browser/aboutDialog.dtd')
- ctx.add_transforms('browser/about_dialog.ftl', [
+ ctx.add_transforms('browser/about_dialog.ftl', 'about_dialog.ftl', [
FTL.Message(
id=FTL.Identifier('update-failed'),
value=CONCAT(
diff --git a/tools/migrate/examples/about_downloads.py b/tools/migrate/examples/about_downloads.py
index 4d8ba1b2..35cd2beb 100644
--- a/tools/migrate/examples/about_downloads.py
+++ b/tools/migrate/examples/about_downloads.py
@@ -7,14 +7,12 @@
def migrate(ctx):
"""Migrate about:download in Firefox for Android, part {index}"""
- ctx.add_reference(
- 'mobile/about_downloads.ftl',
- realpath='about_downloads.ftl'
- )
- ctx.add_localization('mobile/android/chrome/aboutDownloads.dtd')
- ctx.add_localization('mobile/android/chrome/aboutDownloads.properties')
+ ctx.maybe_add_localization(
+ 'mobile/android/chrome/aboutDownloads.dtd')
+ ctx.maybe_add_localization(
+ 'mobile/android/chrome/aboutDownloads.properties')
- ctx.add_transforms('mobile/about_downloads.ftl', [
+ ctx.add_transforms('mobile/about_downloads.ftl', 'about_downloads.ftl', [
FTL.Message(
id=FTL.Identifier('title'),
value=COPY(
diff --git a/tools/migrate/examples/bug_1291693.py b/tools/migrate/examples/bug_1291693.py
index eebce64d..e745c9a5 100644
--- a/tools/migrate/examples/bug_1291693.py
+++ b/tools/migrate/examples/bug_1291693.py
@@ -7,25 +7,12 @@
def migrate(ctx):
"""Bug 1291693 - Migrate the menubar to FTL, part {index}"""
- ctx.add_reference(
- 'browser/menubar.ftl',
- realpath='menubar.ftl'
- )
- ctx.add_reference(
- 'browser/toolbar.ftl',
- realpath='toolbar.ftl'
- )
- ctx.add_reference(
- 'browser/branding/official/brand.ftl',
- realpath='brand.ftl'
- )
+ ctx.maybe_add_localization('browser/chrome/browser/browser.dtd')
+ ctx.maybe_add_localization('browser/chrome/browser/browser.properties')
+ ctx.maybe_add_localization('browser/branding/official/brand.dtd')
+ ctx.maybe_add_localization('browser/branding/official/brand.properties')
- ctx.add_localization('browser/chrome/browser/browser.dtd')
- ctx.add_localization('browser/chrome/browser/browser.properties')
- ctx.add_localization('browser/branding/official/brand.dtd')
- ctx.add_localization('browser/branding/official/brand.properties')
-
- ctx.add_transforms('browser/menubar.ftl', [
+ ctx.add_transforms('browser/menubar.ftl', 'menubar.ftl', [
FTL.Message(
id=FTL.Identifier('file-menu'),
attributes=[
@@ -1795,7 +1782,7 @@ def migrate(ctx):
),
])
- ctx.add_transforms('browser/toolbar.ftl', [
+ ctx.add_transforms('browser/toolbar.ftl', 'toolbar.ftl', [
FTL.Message(
id=FTL.Identifier('urlbar-textbox'),
attributes=[
@@ -1889,7 +1876,7 @@ def migrate(ctx):
),
])
- ctx.add_transforms('browser/branding/official/brand.ftl', [
+ ctx.add_transforms('browser/branding/official/brand.ftl', 'brand.ftl', [
FTL.Message(
id=FTL.Identifier('brand-shorter-name'),
value=COPY(
diff --git a/tools/migrate/migrate-l10n.py b/tools/migrate/migrate-l10n.py
index 6420991a..d8cf5cb1 100755
--- a/tools/migrate/migrate-l10n.py
+++ b/tools/migrate/migrate-l10n.py
@@ -2,14 +2,18 @@
# coding=utf8
import os
+import sys
import json
+import logging
import argparse
import importlib
import hglib
from hglib.util import b
-from fluent.migrate import MergeContext, convert_blame_to_changesets
+from fluent.migrate import (
+ MergeContext, MigrationError, convert_blame_to_changesets
+)
from blame import Blame
@@ -25,18 +29,20 @@ def main(lang, reference_dir, localization_dir, blame, migrations, dry_run):
# For each migration create a new context.
ctx = MergeContext(lang, reference_dir, localization_dir)
- # Add the migration spec.
- migration.migrate(ctx)
+ try:
+ # Add the migration spec.
+ migration.migrate(ctx)
+ except MigrationError as err:
+ sys.exit(err.message)
# Keep track of how many changesets we're committing.
index = 0
for changeset in changesets:
- # Run the migration.
+ # Run the migration for the changeset.
snapshot = ctx.serialize_changeset(changeset['changes'])
- # The current changeset didn't touch any of the translations
- # affected by the migration.
+ # Did it change any files?
if not snapshot:
continue
@@ -95,6 +101,9 @@ def main(lang, reference_dir, localization_dir, blame, migrations, dry_run):
)
parser.set_defaults(dry_run=False)
+ logger = logging.getLogger('migrate')
+ logger.setLevel(logging.INFO)
+
args = parser.parse_args()
if args.blame: