diff --git a/fluent.runtime/fluent/runtime/__init__.py b/fluent.runtime/fluent/runtime/__init__.py index 55df3368..befc1bb0 100644 --- a/fluent.runtime/fluent/runtime/__init__.py +++ b/fluent.runtime/fluent/runtime/__init__.py @@ -9,6 +9,7 @@ from .builtins import BUILTINS from .resolver import resolve +from .utils import ATTRIBUTE_SEPARATOR, TERM_SIGIL, add_message_and_attrs_to_store, ast_to_id class FluentBundle(object): @@ -41,33 +42,25 @@ def add_messages(self, source): # TODO - warn/error about duplicates for item in resource.body: if isinstance(item, (Message, Term)): - if item.id.name not in self._messages_and_terms: - self._messages_and_terms[item.id.name] = item + full_id = ast_to_id(item) + if full_id not in self._messages_and_terms: + # We add attributes to the store to enable faster looker + # later, and more direct code in some instances. + add_message_and_attrs_to_store(self._messages_and_terms, full_id, item) def has_message(self, message_id): - if message_id.startswith('-'): + if message_id.startswith(TERM_SIGIL) or ATTRIBUTE_SEPARATOR in message_id: return False return message_id in self._messages_and_terms def format(self, message_id, args=None): - message = self._get_message(message_id) + if message_id.startswith(TERM_SIGIL): + raise LookupError(message_id) + message = self._messages_and_terms[message_id] if args is None: args = {} return resolve(self, message, args) - def _get_message(self, message_id): - if message_id.startswith('-'): - raise LookupError(message_id) - if '.' in message_id: - name, attr_name = message_id.split('.', 1) - msg = self._messages_and_terms[name] - for attribute in msg.attributes: - if attribute.id.name == attr_name: - return attribute.value - raise LookupError(message_id) - else: - return self._messages_and_terms[message_id] - def _get_babel_locale(self): for l in self.locales: try: diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index 19adc241..07e03e40 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -1,20 +1,19 @@ from __future__ import absolute_import, unicode_literals +import contextlib from datetime import date, datetime from decimal import Decimal import attr import six -from fluent.syntax.ast import (AttributeExpression, CallExpression, Message, - MessageReference, NumberLiteral, Pattern, - Placeable, SelectExpression, StringLiteral, Term, - TermReference, TextElement, VariableReference, - VariantExpression, VariantList, Identifier) +from fluent.syntax.ast import (Attribute, AttributeExpression, CallExpression, Identifier, Message, MessageReference, + NumberLiteral, Pattern, Placeable, SelectExpression, StringLiteral, Term, TermReference, + TextElement, VariableReference, VariantExpression, VariantList) -from .errors import FluentCyclicReferenceError, FluentReferenceError +from .errors import FluentCyclicReferenceError, FluentFormatError, FluentReferenceError from .types import FluentDateType, FluentNone, FluentNumber, fluent_date, fluent_number -from .utils import numeric_to_native +from .utils import numeric_to_native, reference_to_id, unknown_reference_error_obj try: from functools import singledispatch @@ -37,13 +36,47 @@ PDI = "\u2069" +@attr.s +class CurrentEnvironment(object): + # The parts of ResolverEnvironment that we want to mutate (and restore) + # temporarily for some parts of a call chain. + + # The values of attributes here must not be mutated, they must only be + # swapped out for different objects using `modified` (see below). + + # For Messages, VariableReference nodes are interpreted as external args, + # but for Terms they are the values explicitly passed using CallExpression + # syntax. So we have to be able to change 'args' for this purpose. + args = attr.ib() + # This controls whether we need to report an error if a VariableReference + # refers to an arg that is not present in the args dict. + error_for_missing_arg = attr.ib(default=True) + + @attr.s class ResolverEnvironment(object): context = attr.ib() - args = attr.ib() errors = attr.ib() dirty = attr.ib(factory=set) part_count = attr.ib(default=0) + current = attr.ib(factory=CurrentEnvironment) + + @contextlib.contextmanager + def modified(self, **replacements): + """ + Context manager that modifies the 'current' attribute of the + environment, restoring the old data at the end. + """ + # CurrentEnvironment only has args that we never mutate, so the shallow + # copy returned by attr.evolve is fine (at least for now). + old_current = self.current + self.current = attr.evolve(old_current, **replacements) + yield self + self.current = old_current + + def modified_for_term_reference(self, args=None): + return self.modified(args=args if args is not None else {}, + error_for_missing_arg=False) def resolve(context, message, args): @@ -55,7 +88,7 @@ def resolve(context, message, args): """ errors = [] env = ResolverEnvironment(context=context, - args=args, + current=CurrentEnvironment(args=args), errors=errors) return fully_resolve(message, env), errors @@ -71,8 +104,8 @@ def fully_resolve(expr, env): retval = handle(expr, env) if isinstance(retval, text_type): return retval - else: - return fully_resolve(retval, env) + + return fully_resolve(retval, env) @singledispatch @@ -156,33 +189,38 @@ def handle_number_expression(number_expression, env): @handle.register(MessageReference) def handle_message_reference(message_reference, env): - name = message_reference.id.name - return handle(lookup_reference(name, env), env) + return handle(lookup_reference(message_reference, env), env) @handle.register(TermReference) def handle_term_reference(term_reference, env): - name = term_reference.id.name - return handle(lookup_reference(name, env), env) + with env.modified_for_term_reference(): + return handle(lookup_reference(term_reference, env), env) + +def lookup_reference(ref, env): + """ + Given a MessageReference, TermReference or AttributeExpression, returns the + AST node, or FluentNone if not found, including fallback logic + """ + ref_id = reference_to_id(ref) -def lookup_reference(name, env): - message = None try: - message = env.context._messages_and_terms[name] + return env.context._messages_and_terms[ref_id] except LookupError: - if name.startswith("-"): - env.errors.append( - FluentReferenceError("Unknown term: {0}" - .format(name))) - else: - env.errors.append( - FluentReferenceError("Unknown message: {0}" - .format(name))) - if message is None: - message = FluentNone(name) + env.errors.append(unknown_reference_error_obj(ref_id)) + + if isinstance(ref, AttributeExpression): + # Fallback + parent_id = reference_to_id(ref.ref) + try: + return env.context._messages_and_terms[parent_id] + except LookupError: + # Don't add error here, because we already added error for the + # actual thing we were looking for. + pass - return message + return FluentNone(ref_id) @handle.register(FluentNone) @@ -200,10 +238,11 @@ def handle_none(none, env): def handle_variable_reference(argument, env): name = argument.id.name try: - arg_val = env.args[name] + arg_val = env.current.args[name] except LookupError: - env.errors.append( - FluentReferenceError("Unknown external: {0}".format(name))) + if env.current.error_for_missing_arg: + env.errors.append( + FluentReferenceError("Unknown external: {0}".format(name))) return FluentNone(name) if isinstance(arg_val, @@ -217,21 +256,13 @@ def handle_variable_reference(argument, env): @handle.register(AttributeExpression) -def handle_attribute_expression(attribute, env): - parent_id = attribute.ref.id.name - attr_name = attribute.name.name - message = lookup_reference(parent_id, env) - if isinstance(message, FluentNone): - return message +def handle_attribute_expression(attribute_ref, env): + return handle(lookup_reference(attribute_ref, env), env) - for message_attr in message.attributes: - if message_attr.id.name == attr_name: - return handle(message_attr.value, env) - env.errors.append( - FluentReferenceError("Unknown attribute: {0}.{1}" - .format(parent_id, attr_name))) - return handle(message, env) +@handle.register(Attribute) +def handle_attribute(attribute, env): + return handle(attribute.value, env) @handle.register(VariantList) @@ -260,8 +291,8 @@ def select_from_variant_list(variant_list, env, key): found = default if found is None: return FluentNone() - else: - return handle(found.value, env) + + return handle(found.value, env) @handle.register(SelectExpression) @@ -287,8 +318,7 @@ def select_from_select_expression(expression, env, key): found = default if found is None: return FluentNone() - else: - return handle(found.value, env) + return handle(found.value, env) def is_number(val): @@ -304,9 +334,8 @@ def match(val1, val2, env): if not is_number(val2): # Could be plural rule match return env.context._plural_form(val1) == val2 - else: - if is_number(val2): - return match(val2, val1, env) + elif is_number(val2): + return match(val2, val1, env) return val1 == val2 @@ -318,7 +347,7 @@ def handle_indentifier(identifier, env): @handle.register(VariantExpression) def handle_variant_expression(expression, env): - message = lookup_reference(expression.ref.id.name, env) + message = lookup_reference(expression.ref, env) if isinstance(message, FluentNone): return message @@ -334,7 +363,19 @@ def handle_variant_expression(expression, env): @handle.register(CallExpression) def handle_call_expression(expression, env): - function_name = expression.callee.name + args = [handle(arg, env) for arg in expression.positional] + kwargs = {kwarg.name.name: handle(kwarg.value, env) for kwarg in expression.named} + + if isinstance(expression.callee, (TermReference, AttributeExpression)): + term = lookup_reference(expression.callee, env) + if args: + env.errors.append(FluentFormatError("Ignored positional arguments passed to term '{0}'" + .format(reference_to_id(expression.callee)))) + with env.modified_for_term_reference(args=kwargs): + return handle(term, env) + + # builtin or custom function call + function_name = expression.callee.id.name try: function = env.context._functions[function_name] except LookupError: @@ -342,8 +383,6 @@ def handle_call_expression(expression, env): .format(function_name))) return FluentNone(function_name + "()") - args = [handle(arg, env) for arg in expression.positional] - kwargs = {kwarg.name.name: handle(kwarg.value, env) for kwarg in expression.named} try: return function(*args, **kwargs) except Exception as e: diff --git a/fluent.runtime/fluent/runtime/utils.py b/fluent.runtime/fluent/runtime/utils.py index 1f67dd26..e6f793bd 100644 --- a/fluent.runtime/fluent/runtime/utils.py +++ b/fluent.runtime/fluent/runtime/utils.py @@ -1,3 +1,30 @@ +from fluent.syntax.ast import AttributeExpression, Term, TermReference + +from .errors import FluentReferenceError + +TERM_SIGIL = '-' +ATTRIBUTE_SEPARATOR = '.' + + +def ast_to_id(ast): + """ + Returns a string reference for a Term or Message + """ + if isinstance(ast, Term): + return TERM_SIGIL + ast.id.name + return ast.id.name + + +def add_message_and_attrs_to_store(store, ref_id, item, is_parent=True): + store[ref_id] = item + if is_parent: + for attr in item.attributes: + add_message_and_attrs_to_store(store, + _make_attr_id(ref_id, attr.id.name), + attr, + is_parent=False) + + def numeric_to_native(val): """ Given a numeric string (as defined by fluent spec), @@ -7,5 +34,38 @@ def numeric_to_native(val): # '-'? [0-9]+ ('.' [0-9]+)? if '.' in val: return float(val) - else: - return int(val) + return int(val) + + +def reference_to_id(ref): + """ + Returns a string reference for a MessageReference, TermReference or AttributeExpression + AST node. + + e.g. + message + message.attr + -term + -term.attr + """ + if isinstance(ref, AttributeExpression): + return _make_attr_id(reference_to_id(ref.ref), + ref.name.name) + if isinstance(ref, TermReference): + return TERM_SIGIL + ref.id.name + return ref.id.name + + +def unknown_reference_error_obj(ref_id): + if ATTRIBUTE_SEPARATOR in ref_id: + return FluentReferenceError("Unknown attribute: {0}".format(ref_id)) + if ref_id.startswith(TERM_SIGIL): + return FluentReferenceError("Unknown term: {0}".format(ref_id)) + return FluentReferenceError("Unknown message: {0}".format(ref_id)) + + +def _make_attr_id(parent_ref_id, attr_name): + """ + Given a parent id and the attribute name, return the attribute id + """ + return ''.join([parent_ref_id, ATTRIBUTE_SEPARATOR, attr_name]) diff --git a/fluent.runtime/setup.py b/fluent.runtime/setup.py index fe7e6b8e..7acc0802 100755 --- a/fluent.runtime/setup.py +++ b/fluent.runtime/setup.py @@ -26,7 +26,7 @@ ], packages=['fluent', 'fluent.runtime'], install_requires=[ - 'fluent>=0.9,<0.10', + 'fluent.syntax>=0.10,<=0.11', 'attrs', 'babel', 'pytz', diff --git a/fluent.runtime/tests/format/test_attributes.py b/fluent.runtime/tests/format/test_attributes.py index 6e5b63fb..e2eb4e8f 100644 --- a/fluent.runtime/tests/format/test_attributes.py +++ b/fluent.runtime/tests/format/test_attributes.py @@ -106,6 +106,7 @@ def setUp(self): ref-qux = { qux.missing } attr-only = .attr = Attr Only Attribute + ref-double-missing = { missing.attr } """)) def test_falls_back_for_msg_with_string_value_and_no_attributes(self): @@ -147,3 +148,8 @@ def test_attr_only_attribute(self): val, errs = self.ctx.format('attr-only.attr', {}) self.assertEqual(val, 'Attr Only Attribute') self.assertEqual(len(errs), 0) + + def test_missing_message_and_attribute(self): + val, errs = self.ctx.format('ref-double-missing', {}) + self.assertEqual(val, 'missing.attr') + self.assertEqual(errs, [FluentReferenceError('Unknown attribute: missing.attr')]) diff --git a/fluent.runtime/tests/format/test_parameterized_terms.py b/fluent.runtime/tests/format/test_parameterized_terms.py new file mode 100644 index 00000000..3ff31786 --- /dev/null +++ b/fluent.runtime/tests/format/test_parameterized_terms.py @@ -0,0 +1,196 @@ +from __future__ import absolute_import, unicode_literals + +import unittest + +from fluent.runtime import FluentBundle +from fluent.runtime.errors import FluentFormatError, FluentReferenceError + +from ..utils import dedent_ftl + + +class TestParameterizedTerms(unittest.TestCase): + + def setUp(self): + self.ctx = FluentBundle(['en-US'], use_isolating=False) + self.ctx.add_messages(dedent_ftl(""" + -thing = { $article -> + *[definite] the thing + [indefinite] a thing + [none] thing + } + thing-no-arg = { -thing } + thing-no-arg-alt = { -thing() } + thing-with-arg = { -thing(article: "indefinite") } + thing-positional-arg = { -thing("foo") } + thing-fallback = { -thing(article: "somethingelse") } + bad-term = { -missing() } + """)) + + def test_argument_omitted(self): + val, errs = self.ctx.format('thing-no-arg', {}) + self.assertEqual(val, 'the thing') + self.assertEqual(errs, []) + + def test_argument_omitted_alt(self): + val, errs = self.ctx.format('thing-no-arg-alt', {}) + self.assertEqual(val, 'the thing') + self.assertEqual(errs, []) + + def test_with_argument(self): + val, errs = self.ctx.format('thing-with-arg', {}) + self.assertEqual(val, 'a thing') + self.assertEqual(errs, []) + + def test_positional_arg(self): + val, errs = self.ctx.format('thing-positional-arg', {}) + self.assertEqual(val, 'the thing') + self.assertEqual(errs, [FluentFormatError("Ignored positional arguments passed to term '-thing'")]) + + def test_fallback(self): + val, errs = self.ctx.format('thing-fallback', {}) + self.assertEqual(val, 'the thing') + self.assertEqual(errs, []) + + def test_no_implicit_access_to_external_args(self): + # The '-thing' term should not get passed article="indefinite" + val, errs = self.ctx.format('thing-no-arg', {'article': 'indefinite'}) + self.assertEqual(val, 'the thing') + self.assertEqual(errs, []) + + def test_no_implicit_access_to_external_args_but_term_args_still_passed(self): + val, errs = self.ctx.format('thing-with-arg', {'article': 'none'}) + self.assertEqual(val, 'a thing') + self.assertEqual(errs, []) + + def test_bad_term(self): + val, errs = self.ctx.format('bad-term', {}) + self.assertEqual(val, '-missing') + self.assertEqual(errs, [FluentReferenceError('Unknown term: -missing')]) + + +class TestParameterizedTermAttributes(unittest.TestCase): + + def setUp(self): + self.ctx = FluentBundle(['en-US'], use_isolating=False) + self.ctx.add_messages(dedent_ftl(""" + -brand = Cool Thing + .status = { $version -> + [v2] available + *[v1] deprecated + } + + attr-with-arg = { -brand } is { -brand.status(version: "v2") -> + [available] available, yay! + *[deprecated] deprecated, sorry + } + + -other = { $arg -> + [a] ABC + *[d] DEF + } + + missing-attr-ref = { -other.missing(arg: "a") -> + [ABC] ABC option + *[DEF] DEF option + } + """)) + + def test_with_argument(self): + val, errs = self.ctx.format('attr-with-arg', {}) + self.assertEqual(val, 'Cool Thing is available, yay!') + self.assertEqual(errs, []) + + def test_missing_attr(self): + # We should fall back to the parent, and still pass the args. + val, errs = self.ctx.format('missing-attr-ref', {}) + self.assertEqual(val, 'ABC option') + self.assertEqual(errs, [FluentReferenceError('Unknown attribute: -other.missing')]) + + +class TestNestedParameterizedTerms(unittest.TestCase): + + def setUp(self): + self.ctx = FluentBundle(['en-US'], use_isolating=False) + self.ctx.add_messages(dedent_ftl(""" + -thing = { $article -> + *[definite] { $first-letter -> + *[lower] the thing + [upper] The thing + } + [indefinite] { $first-letter -> + *[lower] a thing + [upper] A thing + } + } + + both-args = { -thing(first-letter: "upper", article: "indefinite") }. + outer-arg = This is { -thing(article: "indefinite") }. + inner-arg = { -thing(first-letter: "upper") }. + neither-arg = { -thing() }. + """)) + + def test_both_args(self): + val, errs = self.ctx.format('both-args', {}) + self.assertEqual(val, 'A thing.') + self.assertEqual(errs, []) + + def test_outer_arg(self): + val, errs = self.ctx.format('outer-arg', {}) + self.assertEqual(val, 'This is a thing.') + self.assertEqual(errs, []) + + def test_inner_arg(self): + val, errs = self.ctx.format('inner-arg', {}) + self.assertEqual(val, 'The thing.') + self.assertEqual(errs, []) + + def test_inner_arg_with_external_args(self): + val, errs = self.ctx.format('inner-arg', {'article': 'indefinite'}) + self.assertEqual(val, 'The thing.') + self.assertEqual(errs, []) + + def test_neither_arg(self): + val, errs = self.ctx.format('neither-arg', {}) + self.assertEqual(val, 'the thing.') + self.assertEqual(errs, []) + + +class TestTermsCalledFromTerms(unittest.TestCase): + + def setUp(self): + self.ctx = FluentBundle(['en-US'], use_isolating=False) + self.ctx.add_messages(dedent_ftl(""" + -foo = {$a} {$b} + -bar = {-foo(b: 2)} + -baz = {-foo} + ref-bar = {-bar(a: 1)} + ref-baz = {-baz(a: 1)} + """)) + + def test_term_args_isolated_with_call_syntax(self): + val, errs = self.ctx.format('ref-bar', {}) + self.assertEqual(val, 'a 2') + self.assertEqual(errs, []) + + def test_term_args_isolated_without_call_syntax(self): + val, errs = self.ctx.format('ref-baz', {}) + self.assertEqual(val, 'a b') + self.assertEqual(errs, []) + + +class TestMessagesCalledFromTerms(unittest.TestCase): + + def setUp(self): + self.ctx = FluentBundle(['en-US'], use_isolating=False) + self.ctx.add_messages(dedent_ftl(""" + msg = Msg is {$arg} + -foo = {msg} + ref-foo = {-foo(arg: 1)} + """)) + + def test_messages_inherit_term_args(self): + # This behaviour may change in future, message calls might be + # disallowed from inside terms + val, errs = self.ctx.format('ref-foo', {'arg': 2}) + self.assertEqual(val, 'Msg is 1') + self.assertEqual(errs, []) diff --git a/fluent.runtime/tests/format/test_select_expression.py b/fluent.runtime/tests/format/test_select_expression.py index bc0af933..8a1b77dc 100644 --- a/fluent.runtime/tests/format/test_select_expression.py +++ b/fluent.runtime/tests/format/test_select_expression.py @@ -173,3 +173,45 @@ def test_with_argument_float(self): val, errs = self.ctx.format('qux', {'num': 1.0}) self.assertEqual(val, "A") self.assertEqual(len(errs), 0) + + +class TestSelectExpressionWithTerms(unittest.TestCase): + + def setUp(self): + self.ctx = FluentBundle(['en-US'], use_isolating=False) + self.ctx.add_messages(dedent_ftl(""" + -my-term = term + .attr = termattribute + + ref-term-attr = { -my-term.attr -> + [termattribute] Term Attribute + *[other] Other + } + + ref-term-attr-other = { -my-term.attr -> + [x] Term Attribute + *[other] Other + } + + ref-term-attr-missing = { -my-term.missing -> + [x] Term Attribute + *[other] Other + } + """)) + + def test_ref_term_attribute(self): + val, errs = self.ctx.format('ref-term-attr') + self.assertEqual(val, "Term Attribute") + self.assertEqual(len(errs), 0) + + def test_ref_term_attribute_fallback(self): + val, errs = self.ctx.format('ref-term-attr-other') + self.assertEqual(val, "Other") + self.assertEqual(len(errs), 0) + + def test_ref_term_attribute_missing(self): + val, errs = self.ctx.format('ref-term-attr-missing') + self.assertEqual(val, "Other") + self.assertEqual(len(errs), 1) + self.assertEqual(errs, + [FluentReferenceError('Unknown attribute: -my-term.missing')]) diff --git a/fluent.runtime/tests/test_bundle.py b/fluent.runtime/tests/test_bundle.py index d11a4819..63710e8a 100644 --- a/fluent.runtime/tests/test_bundle.py +++ b/fluent.runtime/tests/test_bundle.py @@ -98,3 +98,15 @@ def test_format_term(self): self.assertRaises(LookupError, self.ctx.format, '-foo') + self.assertRaises(LookupError, + self.ctx.format, + 'foo') + + def test_message_and_term_separate(self): + self.ctx.add_messages(dedent_ftl(""" + foo = Refers to { -foo } + -foo = Foo + """)) + val, errs = self.ctx.format('foo', {}) + self.assertEqual(val, 'Refers to \u2068Foo\u2069') + self.assertEqual(errs, []) diff --git a/fluent.runtime/tox.ini b/fluent.runtime/tox.ini index 960d76d9..c32eb4d2 100644 --- a/fluent.runtime/tox.ini +++ b/fluent.runtime/tox.ini @@ -6,7 +6,7 @@ skipsdist=True setenv = PYTHONPATH = {toxinidir} deps = - fluent>=0.9,<0.10 + fluent.syntax>=0.10,<=0.11 six attrs Babel