String token end location off-by-one error (google#139)

sparkprime · web-flow · commit b6ee2c2f5133 · 2017-11-03T21:36:31.000-04:00
* String token end location off-by-one error
diff --git a/main_test.go b/main_test.go
@@ -205,19 +205,19 @@ func TestOneLineError(t *testing.T) {
 // TODO(sbarzowski) checking if the whitespace is right is quite unpleasant, what can we do about it?
 var minimalErrorTests = []errorFormattingTest{
 	{"error", `error "x"`, "RUNTIME ERROR: x\n" +
-		"	error:1:1-9	$\n" + // TODO(sbarzowski) if seems we have off-by-one in location
+		"	error:1:1-10	$\n" + // TODO(sbarzowski) if seems we have off-by-one in location
 		"	During evaluation	\n" +
 		""},
 	{"error_in_func", `local x(n) = if n == 0 then error "x" else x(n - 1); x(3)`, "RUNTIME ERROR: x\n" +
-		"	error_in_func:1:29-37	function <x>\n" +
+		"	error_in_func:1:29-38	function <x>\n" +
 		"	error_in_func:1:44-52	function <x>\n" +
 		"	error_in_func:1:44-52	function <x>\n" +
 		"	error_in_func:1:44-52	function <x>\n" +
 		"	error_in_func:1:54-58	$\n" +
 		"	During evaluation	\n" +
 		""},
 	{"error_in_error", `error (error "x")`, "RUNTIME ERROR: x\n" +
-		"	error_in_error:1:8-16	$\n" +
+		"	error_in_error:1:8-17	$\n" +
 		"	During evaluation	\n" +
 		""},
 }
diff --git a/parser/lexer.go b/parser/lexer.go
@@ -711,15 +711,13 @@ func Lex(fn string, input string) (tokens, error) {
 			// String literals
 		case '"':
 			stringStartLoc := l.prevLocation()
-			l.resetTokenStart() // Don't include the quotes in the token data
 			for r = l.next(); ; r = l.next() {
 				if r == lexEOF {
 					return nil, l.makeStaticErrorPoint("Unterminated String", stringStartLoc)
 				}
 				if r == '"' {
-					l.backup()
-					l.emitToken(tokenStringDouble)
-					_ = l.next()
+					// Don't include the quotes in the token data
+					l.emitFullToken(tokenStringDouble, l.input[l.tokenStart+1:l.pos.byteNo-1], "", "")
 					l.resetTokenStart()
 					break
 				}
@@ -729,15 +727,13 @@ func Lex(fn string, input string) (tokens, error) {
 			}
 		case '\'':
 			stringStartLoc := l.prevLocation()
-			l.resetTokenStart() // Don't include the quotes in the token data
 			for r = l.next(); ; r = l.next() {
 				if r == lexEOF {
 					return nil, l.makeStaticErrorPoint("Unterminated String", stringStartLoc)
 				}
 				if r == '\'' {
-					l.backup()
-					l.emitToken(tokenStringSingle)
-					r = l.next()
+					// Don't include the quotes in the token data
+					l.emitFullToken(tokenStringSingle, l.input[l.tokenStart+1:l.pos.byteNo-1], "", "")
 					l.resetTokenStart()
 					break
 				}
diff --git a/testdata/arrcomp_if6.golden b/testdata/arrcomp_if6.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: x
 -------------------------------------------------
-	testdata/arrcomp_if6:1:20-28	$
+	testdata/arrcomp_if6:1:20-29	$
 
 [x for x in [1] if error "x"]
 
diff --git a/testdata/bad_index_string.golden b/testdata/bad_index_string.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Unexpected type string, expected number
 -------------------------------------------------
-	testdata/bad_index_string:1:2-13	$
+	testdata/bad_index_string:1:1-13	$
 
 "xxx"["xxx"]
 
diff --git a/testdata/binaryNot2.golden b/testdata/binaryNot2.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Unexpected type string, expected number
 -------------------------------------------------
-	testdata/binaryNot2:1:1-6	$
+	testdata/binaryNot2:1:1-7	$
 
 ~"xxx"
 
diff --git a/testdata/bitwise_and4.golden b/testdata/bitwise_and4.golden
@@ -1,11 +1,11 @@
 RUNTIME ERROR: x
 -------------------------------------------------
-	testdata/bitwise_and4:1:5-13	$
+	testdata/bitwise_and4:1:5-14	$
 
 1 & error "x"
 
 -------------------------------------------------
-	testdata/bitwise_and4:1:1-13	$
+	testdata/bitwise_and4:1:1-14	$
 
 1 & error "x"
 
diff --git a/testdata/bitwise_or10.golden b/testdata/bitwise_or10.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Unexpected type string, expected number
 -------------------------------------------------
-	testdata/bitwise_or10:1:2-11	$
+	testdata/bitwise_or10:1:1-11	$
 
 "xxx" | 42
 
diff --git a/testdata/bitwise_xor7.golden b/testdata/bitwise_xor7.golden
@@ -1,11 +1,11 @@
 RUNTIME ERROR: x
 -------------------------------------------------
-	testdata/bitwise_xor7:1:5-13	$
+	testdata/bitwise_xor7:1:5-14	$
 
 1 ^ error "x"
 
 -------------------------------------------------
-	testdata/bitwise_xor7:1:1-13	$
+	testdata/bitwise_xor7:1:1-14	$
 
 1 ^ error "x"
 
diff --git a/testdata/double_thunk.golden b/testdata/double_thunk.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: xxx
 -------------------------------------------------
-	testdata/double_thunk:1:21-31	thunk <y> from <thunk <x> from <$>>
+	testdata/double_thunk:1:21-32	thunk <y> from <thunk <x> from <$>>
 
 local x = local y = error "xxx"; y; x
 
diff --git a/testdata/error.golden b/testdata/error.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: 42
 -------------------------------------------------
-	testdata/error:1:1-10	$
+	testdata/error:1:1-11	$
 
 error "42"
 
diff --git a/testdata/error_from_array.golden b/testdata/error_from_array.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: xxx
 -------------------------------------------------
-	testdata/error_from_array:1:2-12	thunk from <$>
+	testdata/error_from_array:1:2-13	thunk from <$>
 
 [error "xxx"][0]
 
diff --git a/testdata/extvar_error.golden b/testdata/extvar_error.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: xxx
 -------------------------------------------------
-	<extvar:errorVar>:1:1-11	$
+	<extvar:errorVar>:1:1-12	$
 
 error 'xxx'
 
diff --git a/testdata/function_plus_string.golden b/testdata/function_plus_string.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Couldn't manifest function in JSON output.
 -------------------------------------------------
-	testdata/function_plus_string:1:1-23	$
+	testdata/function_plus_string:1:1-24	$
 
 (function() 42) + "xxx"
 
diff --git a/testdata/import_computed.golden b/testdata/import_computed.golden
@@ -1,4 +1,4 @@
-testdata/import_computed:1:9-16 Computed imports are not allowed
+testdata/import_computed:1:8-17 Computed imports are not allowed
 
 import "a" + "b"
 
diff --git a/testdata/import_failure_directory.golden b/testdata/import_failure_directory.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: read testdata: is a directory
 -------------------------------------------------
-	testdata/import_failure_directory:3:1-10	$
+	testdata/import_failure_directory:3:1-11	$
 
 import '.'
 
diff --git a/testdata/importstr_computed.golden b/testdata/importstr_computed.golden
@@ -1,4 +1,4 @@
-testdata/importstr_computed:1:12-19 Computed imports are not allowed
+testdata/importstr_computed:1:11-20 Computed imports are not allowed
 
 importstr "a" + "b"
 
diff --git a/testdata/insuper4.golden b/testdata/insuper4.golden
@@ -1,4 +1,4 @@
-testdata/insuper4:1:2-13 Can't use super outside of an object.
+testdata/insuper4:1:1-13 Can't use super outside of an object.
 
 "x" in super
 
diff --git a/testdata/lazy_operator2.golden b/testdata/lazy_operator2.golden
@@ -1,11 +1,11 @@
 RUNTIME ERROR: should happen
 -------------------------------------------------
-	testdata/lazy_operator2:1:9-29	$
+	testdata/lazy_operator2:1:9-30	$
 
 true && error "should happen"
 
 -------------------------------------------------
-	testdata/lazy_operator2:1:1-29	$
+	testdata/lazy_operator2:1:1-30	$
 
 true && error "should happen"
 
diff --git a/testdata/native4.golden b/testdata/native4.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: xxx
 -------------------------------------------------
-	testdata/native4:1:28-38	thunk from <$>
+	testdata/native4:1:28-39	thunk from <$>
 
 std.native("jsonToString")(error "xxx")
 
diff --git a/testdata/nonexistent_import.golden b/testdata/nonexistent_import.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Couldn't open import "no chance a file with this name exists": No match locally or in the Jsonnet library paths.
 -------------------------------------------------
-	testdata/nonexistent_import:1:1-50	$
+	testdata/nonexistent_import:1:1-51	$
 
 importstr 'no chance a file with this name exists'
 
diff --git a/testdata/nonexistent_import_crazy.golden b/testdata/nonexistent_import_crazy.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Couldn't open import "ąęółńśćźż \" ' \n\n\t\t": No match locally or in the Jsonnet library paths.
 -------------------------------------------------
-	testdata/nonexistent_import_crazy:1:1-45	$
+	testdata/nonexistent_import_crazy:1:1-46	$
 
 importstr "ąęółńśćźż \" \' \n\n\t\t"
 
diff --git a/testdata/number_divided_by_string.golden b/testdata/number_divided_by_string.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Unexpected type string, expected number
 -------------------------------------------------
-	testdata/number_divided_by_string:1:1-10	$
+	testdata/number_divided_by_string:1:1-11	$
 
 42 / "xxx"
 
diff --git a/testdata/number_times_string.golden b/testdata/number_times_string.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Unexpected type string, expected number
 -------------------------------------------------
-	testdata/number_times_string:1:1-10	$
+	testdata/number_times_string:1:1-11	$
 
 42 * "xxx"
 
diff --git a/testdata/object_comp_err_elem.golden b/testdata/object_comp_err_elem.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: xxx
 -------------------------------------------------
-	testdata/object_comp_err_elem:1:11-21	object <anonymous>
+	testdata/object_comp_err_elem:1:11-22	object <anonymous>
 
 { ["x"]:  error "xxx" for x in [1] }
 
diff --git a/testdata/object_comp_err_index.golden b/testdata/object_comp_err_index.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: xxx
 -------------------------------------------------
-	testdata/object_comp_err_index:1:4-14	$
+	testdata/object_comp_err_index:1:4-15	$
 
 { [error "xxx"]: 42 for x in [1] }
 
diff --git a/testdata/object_invariant13.golden b/testdata/object_invariant13.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: x
 -------------------------------------------------
-	testdata/object_invariant13:1:10-18	object <anonymous>
+	testdata/object_invariant13:1:10-19	object <anonymous>
 
 { assert error "x" }
 
diff --git a/testdata/object_invariant7.golden b/testdata/object_invariant7.golden
@@ -5,23 +5,23 @@ RUNTIME ERROR: Attempt to use super when there is no super class.
 { x: 5, assert super.x == 5 } 
 
 -------------------------------------------------
-	<std>:969:29-30	thunk from <thunk <ta> from <function <anonymous>>>
+	<std>:979:29-30	thunk from <thunk <ta> from <function <anonymous>>>
 
         local ta = std.type(a);
 
 -------------------------------------------------
 	<builtin>	builtin function <type>
 
 -------------------------------------------------
-	<std>:971:33-35	thunk from <function <anonymous>>
+	<std>:981:33-35	thunk from <function <anonymous>>
 
         if !std.primitiveEquals(ta, tb) then
 
 -------------------------------------------------
 	<builtin>	builtin function <primitiveEquals>
 
 -------------------------------------------------
-	<std>:971:12-40	function <anonymous>
+	<std>:981:12-40	function <anonymous>
 
         if !std.primitiveEquals(ta, tb) then
 
diff --git a/testdata/or4.golden b/testdata/or4.golden
@@ -1,11 +1,11 @@
 RUNTIME ERROR: xxx
 -------------------------------------------------
-	testdata/or4:1:10-20	$
+	testdata/or4:1:10-21	$
 
 false || error "xxx"
 
 -------------------------------------------------
-	testdata/or4:1:1-20	$
+	testdata/or4:1:1-21	$
 
 false || error "xxx"
 
diff --git a/testdata/or5.golden b/testdata/or5.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Unexpected type string, expected boolean
 -------------------------------------------------
-	testdata/or5:1:2-14	$
+	testdata/or5:1:1-14	$
 
 "xxx" || true
 
diff --git a/testdata/or6.golden b/testdata/or6.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Unexpected type string, expected boolean
 -------------------------------------------------
-	testdata/or6:1:1-14	$
+	testdata/or6:1:1-15	$
 
 false || "xxx"
 
diff --git a/testdata/percent_bad.golden b/testdata/percent_bad.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Operator % cannot be used on types number and string.
 -------------------------------------------------
-	<std>:150:13-99	function <anonymous>
+	<std>:150:13-100	function <anonymous>
 
             error "Operator % cannot be used on types " + std.type(a) + " and " + std.type(b) + ".",
 
diff --git a/testdata/percent_bad3.golden b/testdata/percent_bad3.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Operator % cannot be used on types function and number.
 -------------------------------------------------
-	<std>:150:13-99	function <anonymous>
+	<std>:150:13-100	function <anonymous>
 
             error "Operator % cannot be used on types " + std.type(a) + " and " + std.type(b) + ".",
 
diff --git a/testdata/recursive_thunk.golden b/testdata/recursive_thunk.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: xxx
 -------------------------------------------------
-	testdata/recursive_thunk:1:35-45	function <bar>
+	testdata/recursive_thunk:1:35-46	function <bar>
 
 local bar(th, x) = if x == 0 then error "xxx" else th;
 
diff --git a/testdata/std.filter2.golden b/testdata/std.filter2.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: x
 -------------------------------------------------
-	testdata/std.filter2:1:12-20	thunk from <$>
+	testdata/std.filter2:1:12-21	thunk from <$>
 
 std.filter(error "x", [])
 
diff --git a/testdata/std.primitiveEquals10.golden b/testdata/std.primitiveEquals10.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: x
 -------------------------------------------------
-	testdata/std.primitiveEquals10:1:21-29	thunk from <$>
+	testdata/std.primitiveEquals10:1:21-30	thunk from <$>
 
 std.primitiveEquals(error "x", 42)
 
diff --git a/testdata/std.primitiveEquals9.golden b/testdata/std.primitiveEquals9.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: x
 -------------------------------------------------
-	testdata/std.primitiveEquals9:1:25-33	thunk from <$>
+	testdata/std.primitiveEquals9:1:25-34	thunk from <$>
 
 std.primitiveEquals(42, error "x")
 
diff --git a/testdata/std.toString5.golden b/testdata/std.toString5.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: x
 -------------------------------------------------
-	testdata/std.toString5:1:14-22	thunk from <$>
+	testdata/std.toString5:1:14-23	thunk from <$>
 
 std.toString(error "x")
 
diff --git a/testdata/string_divided_by_number.golden b/testdata/string_divided_by_number.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Unexpected type string, expected number
 -------------------------------------------------
-	testdata/string_divided_by_number:1:2-11	$
+	testdata/string_divided_by_number:1:1-11	$
 
 "xxx" / 42
 
diff --git a/testdata/string_index_negative.golden b/testdata/string_index_negative.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Index -1 out of bounds, not within [0, 4)
 -------------------------------------------------
-	testdata/string_index_negative:1:2-11	$
+	testdata/string_index_negative:1:1-11	$
 
 "abcd"[-1]
 
diff --git a/testdata/string_index_out_of_bounds.golden b/testdata/string_index_out_of_bounds.golden
@@ -1,6 +1,6 @@
 RUNTIME ERROR: Index 4 out of bounds, not within [0, 4)
 -------------------------------------------------
-	testdata/string_index_out_of_bounds:1:2-10	$
+	testdata/string_index_out_of_bounds:1:1-10	$
 
 "abcd"[4]
 
diff --git a/testdata/string_minus_number.golden b/testdata/string_minus_number.golden
diff --git a/testdata/string_plus_function.golden b/testdata/string_plus_function.golden
diff --git a/testdata/string_times_number.golden b/testdata/string_times_number.golden
diff --git a/testdata/tailstrict3.golden b/testdata/tailstrict3.golden
diff --git a/testdata/type_error.golden b/testdata/type_error.golden
diff --git a/testdata/unary_minus4.golden b/testdata/unary_minus4.golden

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-testdata/import_computed:1:9-16 Computed imports are not allowed`
	`1`	`+testdata/import_computed:1:8-17 Computed imports are not allowed`
`2`	`2`
`3`	`3`	`import "a" + "b"`
`4`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-testdata/importstr_computed:1:12-19 Computed imports are not allowed`
	`1`	`+testdata/importstr_computed:1:11-20 Computed imports are not allowed`
`2`	`2`
`3`	`3`	`importstr "a" + "b"`
`4`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-testdata/insuper4:1:2-13 Can't use super outside of an object.`
	`1`	`+testdata/insuper4:1:1-13 Can't use super outside of an object.`
`2`	`2`
`3`	`3`	`"x" in super`
`4`	`4`