diff --git a/executors/cpp/number_fmt.cpp b/executors/cpp/number_fmt.cpp index 2b11201b..f97b2c87 100644 --- a/executors/cpp/number_fmt.cpp +++ b/executors/cpp/number_fmt.cpp @@ -30,6 +30,7 @@ #include #include +#include #include #include @@ -220,6 +221,42 @@ auto TestNumfmt(json_object *json_in) -> string { locale_string = json_object_get_string(locale_label_obj); } + // JSON for the results + json_object *return_json = json_object_new_object(); + json_object_object_add(return_json, "label", label_obj); + + json_object *pattern_obj = json_object_object_get(json_in, "pattern"); + string pattern = ""; + if (pattern_obj) { + pattern = json_object_get_string(pattern_obj); + + // Check for unsupported patterns. + std::regex check1("(0+0\\.#+E)"); + std::regex check2("(^\\.0#*E)"); + std::smatch m; + + if (std::regex_search(pattern, m, check1) || std::regex_search(pattern, m, check2)) { + // No, not a supported pattern + // Report a failure + json_object_object_add( + return_json, + "error_type", + json_object_new_string("unsupported")); + json_object_object_add( + return_json, + "unsupported", + json_object_new_string("unsupported pattern")); + json_object_object_add( + return_json, + "error_detail", + json_object_new_string(pattern.c_str())); + + // Nothing more to do here. + string return_string = json_object_to_json_string(return_json); + return return_string; + } + } + const Locale displayLocale(locale_string.c_str()); // Get options @@ -244,6 +281,7 @@ auto TestNumfmt(json_object *json_in) -> string { string unitDisplay_string; string style_string; string compactDisplay_string; + string roundingMode_string; // Defaults for settings. CurrencyUnit currency_unit_setting = CurrencyUnit(); @@ -338,13 +376,15 @@ auto TestNumfmt(json_object *json_in) -> string { // TODO: Make this a function rather than inline. roundingMode_obj = json_object_object_get(options_obj, "roundingMode"); if (roundingMode_obj != nullptr) { - string roundingMode_string = json_object_get_string(roundingMode_obj); + roundingMode_string = json_object_get_string(roundingMode_obj); if (roundingMode_string == "floor") { rounding_setting = UNUM_ROUND_FLOOR; } else if (roundingMode_string == "ceil") { rounding_setting = UNUM_ROUND_CEILING; } else if (roundingMode_string == "halfEven") { rounding_setting = UNUM_ROUND_HALFEVEN; + } else if (roundingMode_string == "halfOdd") { + rounding_setting = UNUM_ROUND_HALF_ODD; } else if (roundingMode_string == "halfTrunc") { rounding_setting = UNUM_ROUND_HALFDOWN; } else if (roundingMode_string == "halfExpand") { @@ -353,26 +393,28 @@ auto TestNumfmt(json_object *json_in) -> string { rounding_setting = UNUM_ROUND_DOWN; } else if (roundingMode_string == "expand") { rounding_setting = UNUM_ROUND_UP; + } else if (roundingMode_string == "unnecessary") { + rounding_setting = UNUM_ROUND_UNNECESSARY; } - // TODO: Finish this - // UNUM_ROUND_HALFEVEN , UNUM_FOUND_HALFEVEN = UNUM_ROUND_HALFEVEN , - // UNUM_ROUND_HALFDOWN = UNUM_ROUND_HALFEVEN + 1 , UNUM_ROUND_HALFUP , - // UNUM_ROUND_UNNECESSARY , UNUM_ROUND_HALF_ODD , - // UNUM_ROUND_HALF_CEILING , UNUM_ROUND_HALF_FLOOR } - // TODO: make a function + // TODO: make a function for group_setting group_obj = json_object_object_get(options_obj, "useGrouping"); if (group_obj != nullptr) { string group_string = json_object_get_string(group_obj); - if (group_string == "false") { + if (group_string == "false" || group_string == "off") { grouping_setting = UNUM_GROUPING_OFF; - } else if (group_string == "true") { + } else if (group_string == "true" || group_string == "auto") { grouping_setting = UNUM_GROUPING_AUTO; } else if (group_string == "on_aligned") { grouping_setting = UNUM_GROUPING_ON_ALIGNED; + } else if (group_string == "mim2") { + grouping_setting = UNUM_GROUPING_MIN2; + } else if (group_string == "thousands") { + grouping_setting = UNUM_GROUPING_THOUSANDS; + } else if (group_string == "count") { + grouping_setting = UNUM_GROUPING_COUNT; } - // TODO: FINISH - could be OFF, MIN2, AUTO, ON_ALIGNED, THOUSANDS } // Need to avoid resetting when not options are specifierd. @@ -422,10 +464,6 @@ auto TestNumfmt(json_object *json_in) -> string { // Start using these things - // JSON for the results - json_object *return_json = json_object_new_object(); - json_object_object_add(return_json, "label", label_obj); - int32_t chars_out; // Results of extracting characters from Unicode string bool no_error = true; @@ -504,12 +542,27 @@ auto TestNumfmt(json_object *json_in) -> string { if (U_FAILURE(status) != 0) { // Report a failure - const char* error_name = u_errorName(status); - json_object_object_add( - return_json, "error", - json_object_new_string("error in string extract")); - json_object_object_add( - return_json, "error_detail", json_object_new_string(error_name)); + if (status == U_FORMAT_INEXACT_ERROR) { + const char* error_name = u_errorName(status); + // Inexact result is unsupported. + json_object_object_add( + return_json, + "error_type", + json_object_new_string("unsupported")); + json_object_object_add( + return_json, + "unsupported", + json_object_new_string(error_name)); + } + else { + const char* error_name = u_errorName(status); + + json_object_object_add( + return_json, "error", + json_object_new_string("error getting result string")); + json_object_object_add( + return_json, "error_detail", json_object_new_string(error_name)); + } } else { // It worked! json_object_object_add(return_json, diff --git a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/numberformatter/NumberFormatterOutputJson.java b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/numberformatter/NumberFormatterOutputJson.java index 04a32d15..a0e219f9 100644 --- a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/numberformatter/NumberFormatterOutputJson.java +++ b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/numberformatter/NumberFormatterOutputJson.java @@ -8,7 +8,10 @@ public class NumberFormatterOutputJson implements ITestTypeOutputJson { public String result; + public String unsupported; public String error; - + public String error_detail; + public String error_type; public String error_message; } + diff --git a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/numberformatter/NumberFormatterTester.java b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/numberformatter/NumberFormatterTester.java index 81700582..9ecf1aef 100644 --- a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/numberformatter/NumberFormatterTester.java +++ b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/numberformatter/NumberFormatterTester.java @@ -22,6 +22,8 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.util.HashMap; +import java.util.regex.Pattern; +import java.util.regex.Matcher; import org.unicode.conformance.ExecutorUtils; import org.unicode.conformance.testtype.ITestType; import org.unicode.conformance.testtype.ITestTypeInputJson; @@ -108,13 +110,6 @@ public ITestTypeInputJson inputMapToJson(Map inputMapData) { NumberFormatterTestOptionKey.roundingMode, RoundingModeVal.getFromString( (String) parsedOptionsMap.get("roundingMode"))); - int roundingIncrement = - (int) parsedOptionsMap.getOrDefault("roundingIncrement", RoundingIncrementUtil.DEFAULT); - assert RoundingIncrementUtil.isValidVal(roundingIncrement); - options.put( - NumberFormatterTestOptionKey.roundingIncrement, - roundingIncrement - ); options.put( NumberFormatterTestOptionKey.trailingZeroDisplay, TrailingZeroDispalyVal.getFromString( @@ -136,11 +131,37 @@ public ITestTypeOutputJson execute(ITestTypeInputJson inputJson) { NumberFormatterOutputJson output = new NumberFormatterOutputJson(); output.label = input.label; + // Check for unsupported patterns + if (input.pattern != null && input.pattern != "") { + Pattern check1 = Pattern.compile("0+0\\.#+E"); + Matcher matcher1 = check1.matcher(input.pattern); + Pattern check2 = Pattern.compile("^\\.0#*E"); + Matcher matcher2 = check2.matcher(input.pattern); + if (matcher1.find() || matcher2.find()) { + output.error_type = "unsupported"; + output.unsupported = "unsupported pattern"; + output.error_detail = input.pattern; + return output; + } + } + + // Check unsupport options in skeleton + if (input.skeleton != null) { + Pattern check_half_odd = Pattern.compile("rounding-mode-(half-odd)"); + Matcher matcher1 = check_half_odd.matcher(input.skeleton); + if (matcher1.find()) { + output.error_type = "unsupported"; + output.unsupported = "skeleton option"; + output.error_detail = matcher1.group(); + return output; + } + + } try { output.result = getFormattedNumber(input); } catch (Exception e) { output.error = e.getMessage(); - output.error_message = e.getMessage(); + output.error_type = "formatting number"; return output; } @@ -354,13 +375,6 @@ public String getFormattedNumber(NumberFormatterInputJson input) { precision = fractionPrecision; } - if (precision == null - && input.options.containsKey(NumberFormatterTestOptionKey.roundingIncrement)) { - int roundingIncrement = - (int) input.options.get(NumberFormatterTestOptionKey.roundingIncrement); - precision = Precision.increment(new BigDecimal(roundingIncrement)); - } - if (precision != null && input.options.containsKey(NumberFormatterTestOptionKey.trailingZeroDisplay)) { TrailingZeroDispalyVal trailingZeroDisplayVal = @@ -421,10 +435,12 @@ public String getFormattedNumber(NumberFormatterInputJson input) { break; case halfEven: roundingMode = RoundingMode.HALF_EVEN; - case NONE: // default = halfEven - default: break; case unnecessary: + roundingMode = RoundingMode.UNNECESSARY; + break; + case NONE: // default = halfEven + default: break; } if (roundingMode != null) { diff --git a/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/numberformatter/icu74/NumberFormatterTest.java b/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/numberformatter/icu74/NumberFormatterTest.java index 271e8dcd..e3a69f27 100644 --- a/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/numberformatter/icu74/NumberFormatterTest.java +++ b/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/numberformatter/icu74/NumberFormatterTest.java @@ -2,6 +2,9 @@ import static org.junit.Assert.assertEquals; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.Ignore; import org.junit.Test; import org.unicode.conformance.testtype.numberformatter.NumberFormatterOutputJson; import org.unicode.conformance.testtype.numberformatter.NumberFormatterTester; @@ -41,4 +44,47 @@ public void testDebug2() { assertEquals("-0.2", output.result); } + @Test + public void testDebugBadPattern() { + String testInput = "{\"test_type\": \"number_fmt\", \"label\":\"5865\",\"op\":\"format\",\"pattern\":\"00.##E0\",\"input\":\"1234567\",\"options\":{\"roundingMode\":\"halfEven\",\"notation\":\"scientific\",\"minimumIntegerDigits\":2,\"minimumFractionDigits\":1,\"maximumFractionDigits\":3,\"useGrouping\":false},\"hexhash\":\"bbaf139c38c5be2b1028124e8da7f1497b19d0a3\"}"; + + NumberFormatterOutputJson output = + (NumberFormatterOutputJson) NumberFormatterTester.INSTANCE.getStructuredOutputFromInputStr(testInput); + + assertEquals("unsupported pattern", output.unsupported); + } + + @Test + public void testRoundingUnnecesary() { + String testInput = "{\"test_type\": \"number_fmt\", \"label\":\"5919\",\"op\":\"format\",\"pattern\":\"0.00\",\"skeleton\":\".00 rounding-mode-unnecessary\",\"input\":\"-32.045\",\"options\":{\"roundingMode\":\"unnecessary\",\"minimumIntegerDigits\":1,\"minimumFractionDigits\":2,\"maximumFractionDigits\":2,\"useGrouping\":false},\"hexhash\":\"bbbd9b3299dab2d42ea10ea0b068680b99aef9b1\"}"; + NumberFormatterOutputJson output = + (NumberFormatterOutputJson) NumberFormatterTester.INSTANCE.getStructuredOutputFromInputStr(testInput); + + Pattern check_error = Pattern.compile("Rounding is required"); + Matcher matcher_error = check_error.matcher(output.error); + assertEquals(matcher_error.find(), true); + } + + @Test + public void testRoundingHalfOdd() { + String testInput = +"{\"test_type\": \"number_fmt\", \"label\":\"5927\",\"op\":\"format\",\"pattern\":\"0.00\",\"skeleton\":\".00 rounding-mode-half-odd\",\"input\":\"1.235\",\"options\":{\"roundingMode\":\"halfOdd\",\"minimumIntegerDigits\":1,\"minimumFractionDigits\":2,\"maximumFractionDigits\":2,\"useGrouping\":false},\"hexhash\":\"0f81db6894d53d8bb8973e658acc50726ae11295\"}"; + + NumberFormatterOutputJson output = + (NumberFormatterOutputJson) NumberFormatterTester.INSTANCE.getStructuredOutputFromInputStr(testInput); + + assertEquals("rounding-mode-half-odd", output.error_detail); + } + + @Test + public void testPrecisionIncrement() { + String testInput = + "{\"test_type\": \"number_fmt\", \"label\":\"5868\",\"op\":\"format\",\"pattern\":\"0005\",\"skeleton\":\"integer-width/0000 precision-increment/0005 group-off rounding-mode-half-even\",\"input\":\"1234\",\"options\":{\"roundingMode\":\"halfEven\",\"minimumIntegerDigits\":4,\"roundingIncrement\":5,\"maximumFractionDigits\":0,\"roundingPriority\":\"auto\",\"useGrouping\":false},\"hexhash\":\"1fd49a0510450f4c3cc4dd7a33072488ac9e12ad\"}"; + + NumberFormatterOutputJson output = + (NumberFormatterOutputJson) NumberFormatterTester.INSTANCE.getStructuredOutputFromInputStr(testInput); + + assertEquals("1235", output.result); + } + } diff --git a/executors/node/numberformat.js b/executors/node/numberformat.js index 66d7b208..076bf9d8 100644 --- a/executors/node/numberformat.js +++ b/executors/node/numberformat.js @@ -9,6 +9,12 @@ maximumSignificantDigits: 5}); patternsToOptions.set("0.0000E0", {notation: "scientific", minimunFractionDigits: 4}); + patternsToOptions.set("0005", {useGrouping: false, + minimumIntegerDigits: 4, + roundingIncrement: 5, + maximumFractionDigits: 0, + roundingPriority: "auto", + }); let debug = 1; @@ -103,28 +109,20 @@ const unsupported_combinations = [ {"unit": "furlong"} ]; - const unsupported_rounding_modes = [ + "halfOdd", "unnecessary" ]; + +const unsupported_pattern_regex = [ + /0+0\.#+E/, // More than on signficant integer digit with scientific + /^\.0#*E/, // Zero signficant integer digits with scientific +]; + // TODO: supported options and allowed values should be indexed by Node version module.exports = { - decimalPatternToOptions: function(pattern, rounding) { - let options = {}; - - if (patternsToOptions.has(pattern)) { - options = patternsToOptions.get(pattern); - } - if (rounding) { - options['roundingMode'] = rounding; - } else { - // Default expected by the data - options['roundingMode'] = 'halfEven'; - } - return options; - }, testDecimalFormat: function(json, doLogInput) { const node_version = process.version; @@ -132,119 +130,103 @@ module.exports = { const skeleton = json['skeleton']; const pattern = json['pattern']; - const rounding = json['roundingMode']; - let input = parseFloat(json['input']); // May be changed with some options + + // The formatter can take a string as input, which is needed for very long + // sets of digits. In some cases, the input data may be adjusted, e.g., + // for "percent" and "scale" options (see below). + let input_as_string = json['input']; let options; let error = "unimplemented pattern"; let unsupported_options = []; let return_json = {}; - // If options are in the JSON, use them... - options = json['options']; - if (!options) { - try { - options = this.decimalPatternToOptions(pattern, rounding); - } catch (error) { - // Some error - to return this message - return_json['error'] = "Can't convert pattern"; - return_json['label'] = label; - options = none; - } - } else { - // Default maximumFractionDigits and rounding modes are set in test generation - let roundingMode = options['roundingMode']; - if (! roundingMode) { - // Tests assume halfEven. - roundingMode = options['roundingMode'] = 'halfEven'; + if (pattern) { + // only for checking unsupported patterns + for (let item of unsupported_pattern_regex) { + if (item.test(pattern)) { + unsupported_options.push('pattern: ' + pattern); + return {'label': label, + "unsupported": "unsupported_options", + "error_detail": {'unsupported_options': unsupported_options} + } + } } + } + + // Use options instead of pattern + options = json['options']; + const rounding = options['roundingMode']; + // Default maximumFractionDigits and rounding modes are set in test generation + if (! rounding) { + options['roundingMode'] = 'halfEven'; + } + // Check each option for implementation. - // Check each option for implementation. + // Handle percent - input value is the basis of the actual percent + // expected, e.g., input='0.25' should be interpreted '0.25%' + if (options['style'] && options['style'] === 'percent') { + const input = parseFloat(input_as_string) / 100; + input_as_string = input.toString(); + } - // Check each option for implementation. - // Handle percent - input value is the basis of the actual percent - // expected, e.g., input='0.25' should be interpreted '0.25%' - if (options['style'] && options['style'] === 'percent') { - input = input / 100.0; + // Handle scale in the skeleton + let skeleton_terms; + if (skeleton) { + skeleton_terms = skeleton.split(" "); // all the components + + const scale_regex = /scale\/(\d+\.\d*)/; + const match_scale = skeleton.match(scale_regex); + if (match_scale) { + // Get the value and use it + const scale_value = parseFloat(match_scale[1]); + const input = parseFloat(input_as_string) * scale_value;; + input_as_string = input.toString(); } + } - // Handle scale in the skeleton - let skeleton_terms; - if (skeleton) { - skeleton_terms = skeleton.split(" "); // all the components - if (doLogInput > 0) { - console.log("# SKEL: " + skeleton_terms); - } - const scale_regex = /scale\/(\d+\.\d*)/; - const match_scale = skeleton.match(scale_regex); - if (match_scale) { - // Get the value and use it - const scale_value = parseFloat(match_scale[1]); - input = input * scale_value; - } + // Check for "code":. Change to "currency": + if (options['code']) { + options['currency'] = options['code']; + delete options['code']; + } - // - } + // Supported options depends on the nodejs version + let version_supported_options; + if (node_version >= first_v3_version) { + version_supported_options = + supported_options_by_version['v3']; + } else { + version_supported_options = + supported_options_by_version['pre_v3']; + } - // Check for "code":. Change to "currency": - if (options['code']) { - options['currency'] = options['code']; - delete options['code']; + // Check for option items that are not supported + for (let key in options) { + if (!version_supported_options.includes(key)) { + unsupported_options.push((key + ":" + options[key])); } + } - // Supported options depends on the nodejs version - if (doLogInput > 0) { - console.log("#NNNN " + node_version); - } - let version_supported_options; - if (node_version >= first_v3_version) { - if (doLogInput > 0) { - console.log("#V3 !!!! " + node_version); - } - version_supported_options = - supported_options_by_version['v3']; - } else { - if (doLogInput > 0) { - console.log("#pre_v3 !!!! " + node_version); - } - version_supported_options = - supported_options_by_version['pre_v3']; - } - if (doLogInput > 0) { - console.log("#NNNN " + version_supported_options); - } - // Check for option items that are not supported - for (let key in options) { - if (!version_supported_options.includes(key)) { - unsupported_options.push((key + ":" + options[key])); - } - } + // Check for skeleton terms that are not supported + for (let skel_index in skeleton_terms) { + const skel_term = skeleton_terms[skel_index]; - // Check for skelection terms that are not supported - for (let skel_index in skeleton_terms) { - const skel_term = skeleton_terms[skel_index]; - if (doLogInput > 0) { - console.log("# SKEL_TERM: " + skel_term); - } - if (unsupported_skeleton_terms.includes(skel_term)) { - unsupported_options.push(skel_term); - if (doLogInput > 0) { - console.log("# UNSUPPORTED SKEL_TERM: " + skel_term); - } - } + if (unsupported_skeleton_terms.includes(skel_term)) { + unsupported_options.push(skel_term); } + } - if (unsupported_rounding_modes.includes(roundingMode)) { - unsupported_options.push(roundingMode); - } - if (unsupported_options.length > 0) { - return {'label': label, - "unsupported": "unsupported_options", - "error_detail": {'unsupported_options': unsupported_options} - }; - } + if (unsupported_rounding_modes.includes(options['roundingMode'])) { + unsupported_options.push(options['roundingMode']); } + if (unsupported_options.length > 0) { + return {'label': label, + "unsupported": "unsupported_options", + "error_detail": {'unsupported_options': unsupported_options} + }; + } if (!options) { // Don't test, but return an error return {'label': label, @@ -262,13 +244,12 @@ module.exports = { } let result = 'NOT IMPLEMENTED'; - result = nf.format(input); + // Use the string form, possibly adjusted. + result = nf.format(input_as_string); // TODO: Catch unsupported units, e.g., furlongs. // Formatting as JSON - resultString = result ? result : 'None' - - + resultString = result ? result : 'None'; outputLine = {"label": json['label'], "result": resultString, diff --git a/schema/number_format/result_schema.json b/schema/number_format/result_schema.json index 45ffd4f0..a5101de2 100644 --- a/schema/number_format/result_schema.json +++ b/schema/number_format/result_schema.json @@ -122,7 +122,7 @@ "description": "rounding mode to be used", "enum": ["ceil", "floor", "expand", "trunc", "halfCeil", "halfFloor", "halfExpand", - "halfTrunc", "halfEven", + "halfTrunc", "halfEven", "halfOdd", "unnecessary"], "default": "halfEven" }, diff --git a/schema/number_format/test_schema.json b/schema/number_format/test_schema.json index ed3407e4..5d325760 100644 --- a/schema/number_format/test_schema.json +++ b/schema/number_format/test_schema.json @@ -159,7 +159,7 @@ "description": "rounding mode to be used", "enum": ["ceil", "floor", "expand", "trunc", "halfCeil", "halfFloor", "halfExpand", - "halfTrunc", "halfEven", + "halfTrunc", "halfEven", "halfOdd", "unnecessary"], "default": "halfEven" }, diff --git a/testgen/generators/collation.py b/testgen/generators/collation.py index d48ec21d..a8593d6e 100644 --- a/testgen/generators/collation.py +++ b/testgen/generators/collation.py @@ -4,7 +4,7 @@ import logging from generators.base import DataGenerator -reblankline = re.compile("^\s*$") +reblankline = re.compile(r"^\s*$") class ParseResults(Enum): NO_RESULT = 0 @@ -13,20 +13,20 @@ class ParseResults(Enum): class CollationGenerator(DataGenerator): def set_patterns(self): - self.root_locale = re.compile("@ root") - self.locale_string = re.compile("@ locale (\S+)") + self.root_locale = re.compile(r"@ root") + self.locale_string = re.compile(r"@ locale (\S+)") - self.test_line = re.compile("^\*\* test:(.*)") - self.rule_header_pattern = re.compile("^@ rules") - self.compare_pattern = re.compile("^\* compare(.*)") + self.test_line = re.compile(r"^\*\* test:(.*)") + self.rule_header_pattern = re.compile(r"^@ rules") + self.compare_pattern = re.compile(r"^\* compare(.*)") # A comparison line begins with the type of compare function. - self.comparison_line = re.compile("^([<=]\S*)(\s*)(\S*)(\s*)#?(.*)") + self.comparison_line = re.compile(r"^([<=]\S*)(\s*)(\S*)(\s*)#?(.*)") - self.input_pattern_with_comment = re.compile("^([^#]+)#?(.*)") + self.input_pattern_with_comment = re.compile(r"^([^#]+)#?(.*)") - self.attribute_test = re.compile("^% (\S+)\s*=\s*(.+)") - self.reorder_test = re.compile("^% (reorder)\s+(.+)") + self.attribute_test = re.compile(r"^% (\S+)\s*=\s*(.+)") + self.reorder_test = re.compile(r"^% (reorder)\s+(.+)") def process_test_data(self): # Get each kind of collation tests and create a unified data set @@ -238,7 +238,7 @@ def generateCollTestData2(self, filename, icu_version, start_count=0): max_digits = 1 + self.computeMaxDigitsForCount( len(raw_testdata_list) ) # Approximate - recommentline = re.compile("^[\ufeff\s]*#(.*)") + recommentline = re.compile(r"^[\ufeff\s]*#(.*)") rules = None @@ -398,7 +398,7 @@ def generateCollTestDataObjects(self, filename, icu_version, ignorePunctuation, # Handles lines of strings to be compared with collation. # Adds field for ignoring punctuation as needed. - recommentline = re.compile("^\s*#") + recommentline = re.compile(r"^\s*#") max_digits = 1 + self.computeMaxDigitsForCount( len(raw_testdata_list) @@ -477,9 +477,6 @@ def stringifyCode(self, cp): def check_unpaired_surrogate_in_string(self, text): # Look for unmatched high/low surrogates in the text - # high_surrogate_pattern = re.compile(r'([\ud800-\udbff])') - # low_surrogate_pattern = re.compile(r'([\udc00-\udfff])') - match_high = self.high_surrogate_pattern.findall(text) match_low = self.low_surrogate_pattern.findall(text) diff --git a/testgen/generators/number_fmt.py b/testgen/generators/number_fmt.py index f2d799fc..c96ba742 100644 --- a/testgen/generators/number_fmt.py +++ b/testgen/generators/number_fmt.py @@ -174,15 +174,29 @@ def generateDcmlFmtTestDataObjects(self, rawtestdata, count=0): # Transforming patterns to skeltons pattern_to_skeleton = { - "0.0000E0": "scientific .0000/@", - "00": "integer-width/##00 group-off", - # '0.00': '.##/@@@', # TODO: Fix this skeleton + "0.0000E0": "scientific .0000", + "0.000E0": "scientific .000", + "0.00E0": "scientific .00", + "0.0##E0": "scientific .0##", # ?? + "00.##E0": "scientific .##", # ?? + "0.#E0": "scientific .0", # ?? + "0.##E0": "scientific .00", # ??? + ".0E0": "scientific .0", # ?? + ".0#E0": "scientific .0#", # ?? + ".0##E0": "scientific .0##", # ?? + "00": "integer-width/##00 precision-integer group-off", + "0.0": ".0", + '0.00': '.00', + '0.0##': '.0##', + '0.00##': '.00##', "@@@": "@@@ group-off", "@@###": "@@### group-off", - "#": "@ group-off", - "@@@@E0": "scientific/+e .0000/@@+", + "#": "precision-integer group-off", + "#.#": ".# group-off", + "@@@@E0": "scientific/+e @@@@", "0.0##@E0": "scientific/+e .##/@@+", - "0005": "integer-width/0000 precision-increment/0005", + "0005": "integer-width/0000 precision-increment/0005 group-off", + "@@@@@@@@@@@@@@@@@@@@@@@@@": "@@@@@@@@@@@@@@@@@@@@@@@@@ group-off" } expected = len(test_list) + count @@ -200,13 +214,15 @@ def generateDcmlFmtTestDataObjects(self, rawtestdata, count=0): if pattern == None: continue - rounding_mode = self.mapRoundingToECMA402(round_mode) label = str(count).rjust(max_digits, "0") # TODO!!: Look up the patterns to make skeletons if pattern in pattern_to_skeleton: skeleton = pattern_to_skeleton[pattern] + if round_mode: + skeleton += ' ' + self.mapRoundingToSkeleton(round_mode) else: + logging.error('Pattern %s not converted to skelection', pattern) skeleton = None if skeleton: @@ -234,8 +250,9 @@ def generateDcmlFmtTestDataObjects(self, rawtestdata, count=0): # None of these old patterns use groupings resolved_options_dict["useGrouping"] = False - if rounding_mode: - entry["options"]["roundingMode"] = rounding_mode + if round_mode: + ecma_rounding_mode = self.mapRoundingToECMA402(round_mode) + entry["options"]["roundingMode"] = ecma_rounding_mode else: # Default if not specified entry["options"]["roundingMode"] = self.mapRoundingToECMA402( @@ -255,7 +272,7 @@ def generateDcmlFmtTestDataObjects(self, rawtestdata, count=0): def parseDcmlFmtTestData(self, rawtestdata): reformat = re.compile( - r"format +([\d.E@\#]+) +(default|ceiling|floor|down|up|halfeven|halfdown|halfup|unnecessary) +\"(-?[\d.E]+)\" +\"(-?[\d.E]+|Inexact)\"" + r"format +([\d.E@\#]+) +(default|ceiling|floor|down|up|halfeven|halfdown|halfup|halfodd|halfceiling|halffloor|unnecessary) +\"(-?[\d.E]+)\" +\"(-?[\d.E]+|Inexact)\"" ) # TODO: ignore 'parse' line try: @@ -347,7 +364,12 @@ def mapFmtSkeletonToECMA402(self, options): "minimumFractionDigits": 1, "maximumFractionDigits": 3, }, - "0005": {"minimumIntegerDigits": 2}, + "0005": { + "minimumIntegerDigits": 4, + "roundingIncrement": 5, + "maximumFractionDigits": 0, + "roundingPriority": "auto", + "roundingIncrement": 5}, "0.00": { "minimumIntegerDigits": 1, "minimumFractionDigits": 2, @@ -401,13 +423,13 @@ def mapFmtSkeletonToECMA402(self, options): options_dict = {} # Which combinatins of skeleton entries need modificiation? # Look at the expected output... - for o in options: - if o != "scale/0.5" and o != "decimal-always": - option_detail = ecma402_map[o] + for option in options: + if option != "scale/0.5" and option != "decimal-always": + option_detail = ecma402_map[option] options_dict = options_dict | option_detail - if o[0:5] == "scale": - options_dict = options_dict | {"conformanceScale": o[6:]} - if o == "decimal-always": + if option[0:5] == "scale": + options_dict = options_dict | {"conformanceScale": option[6:]} + if option == "decimal-always": options_dict = options_dict | {"conformanceDecimalAlways": True} # TODO: resolve some combinations of entries that are in conflict @@ -417,7 +439,7 @@ def mapRoundingToECMA402(self, rounding): ecma402_rounding_map = { "default": "halfEven", "halfeven": "halfEven", - "halfodd": "none", + "halfodd": "halfOdd", "halfdown": "halfTrunc", "halfup": "halfExpand", "down": "trunc", @@ -430,6 +452,24 @@ def mapRoundingToECMA402(self, rounding): } return ecma402_rounding_map[rounding] + def mapRoundingToSkeleton(self, rounding): + ecma402_rounding_map = { + "default": "rounding-mode-half-even", + "halfeven": "rounding-mode-half-even", + "halfodd": "rounding-mode-half-odd", # valid?? + "halfdown": "rounding-mode-half-down", + "halfup": "rounding-mode-half-up", + "down": "rounding-mode-down", + "up": "rounding-mode-up", + "halfceiling": "rounding-mode-half-up", # correct? + "halffloor": "rounding-mode-half-down", # correct? + "ceiling": "rounding-mode-ceiling", + "floor": "rounding-mode-floor", + "unnecessary": "rounding-mode-unnecessary", + } + return ecma402_rounding_map[rounding] + + def resolveOptions(self, raw_options, skeleton_list): # Resolve conflicts with options before putting them into the test's options. # TODO: fix all the potential conflicts diff --git a/verifier/check_known_issues.py b/verifier/check_known_issues.py index a4f77a42..278c4556 100644 --- a/verifier/check_known_issues.py +++ b/verifier/check_known_issues.py @@ -63,6 +63,10 @@ class knownIssueType(Enum): langnames_tag_option = 'unsupported option in locale' langnames_bracket_parens = 'brackets_vs_parentheses' + # Number format + # Support expected errors https://github.com/unicode-org/conformance/issues/242 + number_fmt_inexact_rounding = 'Rounding unnecessary' + # Plural rules plural_rules_floating_point_sample = 'limited floating point support' plural_rules_java_4_1_sample = 'ICU4J sample 4.1' @@ -271,6 +275,7 @@ def langname_fonipa(test): else: return None + def langname_tag_option(test): # TODO: Add other unsupported tags input_data = test['input_data'] @@ -295,6 +300,22 @@ def langname_brackets(test): return None +# Number format known issues +def check_number_fmt_issues(test, platform_info): + input_data = test['input_data'] + if 'expected' in test: + expected = test['expected'] + if expected == 'Inexact' and input_data['options']['roundingMode'] == 'unnecessary': + return knownIssueType.number_fmt_inexact_rounding + + if 'result' not in test: + # This must be an error + if 'error' in test and re.match(r'Rounding is required', test['error']): + return knownIssueType.number_fmt_inexact_rounding + # No known issue for this case + return None + + def check_plural_rules_issues(test): try: input_data = test['input_data'] @@ -310,7 +331,7 @@ def check_plural_rules_issues(test): return None -def compute_known_issues_for_single_test(test_type, test): +def compute_known_issues_for_single_test(test_type, test, platform_info): # Based on the type of test, check known issues against the expected vs. actual # results @@ -326,12 +347,13 @@ def compute_known_issues_for_single_test(test_type, test): known_issue_found = check_langnames_issues(test) elif test_type == ddt_data.testType.plural_rules.value: known_issue_found = check_plural_rules_issues(test) - + elif test_type == ddt_data.testType.number_fmt.value: + known_issue_found = check_number_fmt_issues(test, platform_info) # TODO: Add checks here for known issues in other test types return known_issue_found -def check_issues(test_type, test_results_to_check): +def check_issues(test_type, test_results_to_check, platform_info): # Look at the array of test result types, failure, error, unsupported # Extract any tests from these that are known issues # Return the list of tests that are known issues @@ -343,7 +365,7 @@ def check_issues(test_type, test_results_to_check): index = 0 for test in category: - is_known_issue = compute_known_issues_for_single_test(test_type, test) + is_known_issue = compute_known_issues_for_single_test(test_type, test, platform_info) if is_known_issue: known_issues_list.append(test) test_indices_with_known_issues.add(index) diff --git a/verifier/testreport.py b/verifier/testreport.py index 2c900e08..aa4644e3 100644 --- a/verifier/testreport.py +++ b/verifier/testreport.py @@ -385,7 +385,8 @@ def add_known_issues(self): new_known_issues = check_issues( self.test_type, # Don't look at tests labeled as "unsupported" - [self.failing_tests, self.test_errors]) + [self.failing_tests, self.test_errors], + self.platform_info) if new_known_issues: self.known_issues.extend(new_known_issues) @@ -424,27 +425,28 @@ def create_html_report(self): fail_lines = [] max_fail_length = 0 for fail in self.failing_tests: - fail_result = fail['result'] - # if len(fail_result) > max_fail_length: - # max_fail_length = len(fail_result) - - # if len(fail_result) > 30: - # # Make the actual text shorter so it doesn't distort the table column - # fail['result'] = fail_result[0:15] + ' ... ' + fail_result[-14:] - if fail_result is str: - line = self.fail_line_template.safe_substitute(fail_result) - fail_lines.append(line) + if 'result' in fail: + fail_result = fail['result'] + # if len(fail_result) > max_fail_length: + # max_fail_length = len(fail_result) + + # if len(fail_result) > 30: + # # Make the actual text shorter so it doesn't distort the table column + # fail['result'] = fail_result[0:15] + ' ... ' + fail_result[-14:] + if fail_result is str: + line = self.fail_line_template.safe_substitute(fail_result) + fail_lines.append(line) # Call functions to identify known issues, moving things from fail, error, and unsupported # to known_issues as needed new_known_issues = check_issues( self.test_type, - [self.failing_tests, self.test_errors, self.unsupported_cases]) + [self.failing_tests, self.test_errors, self.unsupported_cases], + platform_info) if new_known_issues: self.known_issues.extend(new_known_issues) - # Characterize successes, too. pass_characterized = self.characterize_results_by_options(self.passing_tests, 'pass') flat_combined_passing = self.flatten_and_combine(pass_characterized, None) @@ -821,7 +823,7 @@ def check_text_diffs(self, test_list, category): for fail in test_list: label = fail['label'] - actual = fail['result'] + actual = fail.get('result', None) expected = fail['expected'] if type(actual) != type(expected): # This is a type mismatch. Note this and skip the string-specific characterizations. @@ -942,7 +944,7 @@ def check_list_differences(self, test, results): # results[check] = set() label = test['label'] - actual = test['result'] + actual = test.get('result', None) expected = test.get('expected', None) if len(actual) != len(expected): diff --git a/verifier/verify_plan.py b/verifier/verify_plan.py index 446ae227..408453ac 100644 --- a/verifier/verify_plan.py +++ b/verifier/verify_plan.py @@ -177,7 +177,10 @@ def compare_test_to_expected(self): # Get the result try: - actual_result = test['result'] + if 'result' in test: + actual_result = test['result'] + else: + actual_result = None verification_data = self.find_expected_with_label(test_label)