diff --git a/gqlerrors/syntax.go b/gqlerrors/syntax.go index 4235a040..e3a3cc5a 100644 --- a/gqlerrors/syntax.go +++ b/gqlerrors/syntax.go @@ -4,10 +4,11 @@ import ( "fmt" "regexp" + "strings" + "github.com/graphql-go/graphql/language/ast" "github.com/graphql-go/graphql/language/location" "github.com/graphql-go/graphql/language/source" - "strings" ) func NewSyntaxError(s *source.Source, position int, description string) *Error { @@ -44,7 +45,7 @@ func highlightSourceAtLocation(s *source.Source, l location.SourceLocation) stri lineNum := fmt.Sprintf("%d", line) nextLineNum := fmt.Sprintf("%d", (line + 1)) padLen := len(nextLineNum) - lines := regexp.MustCompile("\r\n|[\n\r]").Split(s.Body, -1) + lines := regexp.MustCompile("\r\n|[\n\r]").Split(string(s.Body), -1) var highlight string if line >= 2 { highlight += fmt.Sprintf("%s: %s\n", lpad(padLen, prevLineNum), printLine(lines[line-2])) diff --git a/graphql.go b/graphql.go index 047535b4..af9dd65a 100644 --- a/graphql.go +++ b/graphql.go @@ -34,7 +34,7 @@ type Params struct { func Do(p Params) *Result { source := source.NewSource(&source.Source{ - Body: p.RequestString, + Body: []byte(p.RequestString), Name: "GraphQL request", }) AST, err := parser.Parse(parser.ParseParams{Source: source}) diff --git a/language/lexer/lexer.go b/language/lexer/lexer.go index 4c149c34..865c9d6e 100644 --- a/language/lexer/lexer.go +++ b/language/lexer/lexer.go @@ -1,7 +1,9 @@ package lexer import ( + "bytes" "fmt" + "unicode/utf8" "github.com/graphql-go/graphql/gqlerrors" "github.com/graphql-go/graphql/language/source" @@ -81,10 +83,6 @@ type Token struct { Value string } -func (t *Token) String() string { - return fmt.Sprintf("%s", tokenDescription[t.Kind]) -} - type Lexer func(resetPosition int) (Token, error) func Lex(s *source.Source) Lexer { @@ -102,82 +100,80 @@ func Lex(s *source.Source) Lexer { } } -func runeStringValueAt(body string, start, end int) string { - // convert body string to runes, to handle unicode - bodyRunes := []rune(body) - return string(bodyRunes[start:end]) -} - // Reads an alphanumeric + underscore name from the source. // [_A-Za-z][_0-9A-Za-z]* -func readName(source *source.Source, position int) Token { +// position: Points to the byte position in the byte array +// runePosition: Points to the rune position in the byte array +func readName(source *source.Source, position, runePosition int) Token { body := source.Body bodyLength := len(body) - end := position + 1 + endByte := position + 1 + endRune := runePosition + 1 for { - code := charCodeAt(body, end) - if (end != bodyLength) && code != 0 && - (code == 95 || // _ - code >= 48 && code <= 57 || // 0-9 - code >= 65 && code <= 90 || // A-Z - code >= 97 && code <= 122) { // a-z - end++ + code, _ := runeAt(body, endByte) + if (endByte != bodyLength) && + (code == '_' || // _ + code >= '0' && code <= '9' || // 0-9 + code >= 'A' && code <= 'Z' || // A-Z + code >= 'a' && code <= 'z') { // a-z + endByte++ + endRune++ continue } else { break } } - return makeToken(TokenKind[NAME], position, end, runeStringValueAt(body, position, end)) + return makeToken(TokenKind[NAME], runePosition, endRune, string(body[position:endByte])) } // Reads a number token from the source file, either a float // or an int depending on whether a decimal point appears. // Int: -?(0|[1-9][0-9]*) // Float: -?(0|[1-9][0-9]*)(\.[0-9]+)?((E|e)(+|-)?[0-9]+)? -func readNumber(s *source.Source, start int, firstCode rune) (Token, error) { +func readNumber(s *source.Source, start int, firstCode rune, codeLength int) (Token, error) { code := firstCode body := s.Body position := start isFloat := false - if code == 45 { // - - position++ - code = charCodeAt(body, position) + if code == '-' { // - + position += codeLength + code, codeLength = runeAt(body, position) } - if code == 48 { // 0 - position++ - code = charCodeAt(body, position) - if code >= 48 && code <= 57 { + if code == '0' { // 0 + position += codeLength + code, codeLength = runeAt(body, position) + if code >= '0' && code <= '9' { description := fmt.Sprintf("Invalid number, unexpected digit after 0: %v.", printCharCode(code)) return Token{}, gqlerrors.NewSyntaxError(s, position, description) } } else { - p, err := readDigits(s, position, code) + p, err := readDigits(s, position, code, codeLength) if err != nil { return Token{}, err } position = p - code = charCodeAt(body, position) + code, codeLength = runeAt(body, position) } - if code == 46 { // . + if code == '.' { // . isFloat = true - position++ - code = charCodeAt(body, position) - p, err := readDigits(s, position, code) + position += codeLength + code, codeLength = runeAt(body, position) + p, err := readDigits(s, position, code, codeLength) if err != nil { return Token{}, err } position = p - code = charCodeAt(body, position) + code, codeLength = runeAt(body, position) } - if code == 69 || code == 101 { // E e + if code == 'E' || code == 'e' { // E e isFloat = true - position++ - code = charCodeAt(body, position) - if code == 43 || code == 45 { // + - - position++ - code = charCodeAt(body, position) + position += codeLength + code, codeLength = runeAt(body, position) + if code == '+' || code == '-' { // + - + position += codeLength + code, codeLength = runeAt(body, position) } - p, err := readDigits(s, position, code) + p, err := readDigits(s, position, code, codeLength) if err != nil { return Token{}, err } @@ -187,19 +183,20 @@ func readNumber(s *source.Source, start int, firstCode rune) (Token, error) { if isFloat { kind = TokenKind[FLOAT] } - return makeToken(kind, start, position, runeStringValueAt(body, start, position)), nil + + return makeToken(kind, start, position, string(body[start:position])), nil } // Returns the new position in the source after reading digits. -func readDigits(s *source.Source, start int, firstCode rune) (int, error) { +func readDigits(s *source.Source, start int, firstCode rune, codeLength int) (int, error) { body := s.Body position := start code := firstCode - if code >= 48 && code <= 57 { // 0 - 9 + if code >= '0' && code <= '9' { // 0 - 9 for { - if code >= 48 && code <= 57 { // 0 - 9 - position++ - code = charCodeAt(body, position) + if code >= '0' && code <= '9' { // 0 - 9 + position += codeLength + code, codeLength = runeAt(body, position) continue } else { break @@ -215,70 +212,81 @@ func readDigits(s *source.Source, start int, firstCode rune) (int, error) { func readString(s *source.Source, start int) (Token, error) { body := s.Body position := start + 1 + runePosition := start + 1 chunkStart := position var code rune - var value string + var n int + var valueBuffer bytes.Buffer for { - code = charCodeAt(body, position) + code, n = runeAt(body, position) if position < len(body) && // not LineTerminator code != 0x000A && code != 0x000D && // not Quote (") - code != 34 { + code != '"' { // SourceCharacter if code < 0x0020 && code != 0x0009 { - return Token{}, gqlerrors.NewSyntaxError(s, position, fmt.Sprintf(`Invalid character within String: %v.`, printCharCode(code))) + return Token{}, gqlerrors.NewSyntaxError(s, runePosition, fmt.Sprintf(`Invalid character within String: %v.`, printCharCode(code))) } - position++ - if code == 92 { // \ - value += body[chunkStart : position-1] - code = charCodeAt(body, position) + position += n + runePosition++ + if code == '\\' { // \ + valueBuffer.Write(body[chunkStart : position-1]) + code, n = runeAt(body, position) switch code { - case 34: - value += "\"" + case '"': + valueBuffer.WriteRune('"') break - case 47: - value += "\\/" + case '/': + valueBuffer.WriteRune('/') break - case 92: - value += "\\" + case '\\': + valueBuffer.WriteRune('\\') break - case 98: - value += "\b" + case 'b': + valueBuffer.WriteRune('\b') break - case 102: - value += "\f" + case 'f': + valueBuffer.WriteRune('\f') break - case 110: - value += "\n" + case 'n': + valueBuffer.WriteRune('\n') break - case 114: - value += "\r" + case 'r': + valueBuffer.WriteRune('\r') break - case 116: - value += "\t" + case 't': + valueBuffer.WriteRune('\t') break - case 117: // u + case 'u': + // Check if there are at least 4 bytes available + if len(body) <= position+4 { + return Token{}, gqlerrors.NewSyntaxError(s, runePosition, + fmt.Sprintf("Invalid character escape sequence: "+ + "\\u%v", string(body[position+1:]))) + } charCode := uniCharCode( - charCodeAt(body, position+1), - charCodeAt(body, position+2), - charCodeAt(body, position+3), - charCodeAt(body, position+4), + rune(body[position+1]), + rune(body[position+2]), + rune(body[position+3]), + rune(body[position+4]), ) if charCode < 0 { - return Token{}, gqlerrors.NewSyntaxError(s, position, + return Token{}, gqlerrors.NewSyntaxError(s, runePosition, fmt.Sprintf("Invalid character escape sequence: "+ - "\\u%v", body[position+1:position+5])) + "\\u%v", string(body[position+1:position+5]))) } - value += fmt.Sprintf("%c", charCode) + valueBuffer.WriteRune(charCode) position += 4 + runePosition += 4 break default: - return Token{}, gqlerrors.NewSyntaxError(s, position, + return Token{}, gqlerrors.NewSyntaxError(s, runePosition, fmt.Sprintf(`Invalid character escape sequence: \\%c.`, code)) } - position++ + position += n + runePosition++ chunkStart = position } continue @@ -286,10 +294,12 @@ func readString(s *source.Source, start int) (Token, error) { break } } - if code != 34 { // quote (") - return Token{}, gqlerrors.NewSyntaxError(s, position, "Unterminated string.") + if code != '"' { // quote (") + return Token{}, gqlerrors.NewSyntaxError(s, runePosition, "Unterminated string.") } - value += runeStringValueAt(body, chunkStart, position) + stringContent := body[chunkStart:position] + valueBuffer.Write(stringContent) + value := valueBuffer.String() return makeToken(TokenKind[STRING], start, position+1, value), nil } @@ -340,79 +350,81 @@ func printCharCode(code rune) string { func readToken(s *source.Source, fromPosition int) (Token, error) { body := s.Body bodyLength := len(body) - position := positionAfterWhitespace(body, fromPosition) + position, runePosition := positionAfterWhitespace(body, fromPosition) if position >= bodyLength { return makeToken(TokenKind[EOF], position, position, ""), nil } - code := charCodeAt(body, position) + code, codeLength := runeAt(body, position) // SourceCharacter if code < 0x0020 && code != 0x0009 && code != 0x000A && code != 0x000D { - return Token{}, gqlerrors.NewSyntaxError(s, position, fmt.Sprintf(`Invalid character %v`, printCharCode(code))) + return Token{}, gqlerrors.NewSyntaxError(s, runePosition, fmt.Sprintf(`Invalid character %v`, printCharCode(code))) } switch code { // ! - case 33: + case '!': return makeToken(TokenKind[BANG], position, position+1, ""), nil // $ - case 36: + case '$': return makeToken(TokenKind[DOLLAR], position, position+1, ""), nil // ( - case 40: + case '(': return makeToken(TokenKind[PAREN_L], position, position+1, ""), nil // ) - case 41: + case ')': return makeToken(TokenKind[PAREN_R], position, position+1, ""), nil // . - case 46: - if charCodeAt(body, position+1) == 46 && charCodeAt(body, position+2) == 46 { + case '.': + next1, _ := runeAt(body, position+1) + next2, _ := runeAt(body, position+2) + if next1 == '.' && next2 == '.' { return makeToken(TokenKind[SPREAD], position, position+3, ""), nil } break // : - case 58: + case ':': return makeToken(TokenKind[COLON], position, position+1, ""), nil // = - case 61: + case '=': return makeToken(TokenKind[EQUALS], position, position+1, ""), nil // @ - case 64: + case '@': return makeToken(TokenKind[AT], position, position+1, ""), nil // [ - case 91: + case '[': return makeToken(TokenKind[BRACKET_L], position, position+1, ""), nil // ] - case 93: + case ']': return makeToken(TokenKind[BRACKET_R], position, position+1, ""), nil // { - case 123: + case '{': return makeToken(TokenKind[BRACE_L], position, position+1, ""), nil // | - case 124: + case '|': return makeToken(TokenKind[PIPE], position, position+1, ""), nil // } - case 125: + case '}': return makeToken(TokenKind[BRACE_R], position, position+1, ""), nil // A-Z - case 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - 82, 83, 84, 85, 86, 87, 88, 89, 90: - return readName(s, position), nil + case 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', + 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z': + return readName(s, position, runePosition), nil // _ // a-z - case 95, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, - 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122: - return readName(s, position), nil + case '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z': + return readName(s, position, runePosition), nil // - // 0-9 - case 45, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57: - token, err := readNumber(s, position, code) + case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + token, err := readNumber(s, position, code, codeLength) if err != nil { return token, err } return token, nil // " - case 34: + case '"': token, err := readString(s, position) if err != nil { return token, err @@ -420,27 +432,36 @@ func readToken(s *source.Source, fromPosition int) (Token, error) { return token, nil } description := fmt.Sprintf("Unexpected character %v.", printCharCode(code)) - return Token{}, gqlerrors.NewSyntaxError(s, position, description) + return Token{}, gqlerrors.NewSyntaxError(s, runePosition, description) } -func charCodeAt(body string, position int) rune { - r := []rune(body) - if len(r) > position { - return r[position] +// Gets the rune from the byte array at given byte position and it's width in bytes +func runeAt(body []byte, position int) (code rune, charWidth int) { + if len(body) <= position { + // + return -1, utf8.RuneError + } + + c := body[position] + if c < utf8.RuneSelf { + return rune(c), 1 } - return -1 + r, n := utf8.DecodeRune(body[position:]) + return r, n } // Reads from body starting at startPosition until it finds a non-whitespace // or commented character, then returns the position of that character for lexing. // lexing. -func positionAfterWhitespace(body string, startPosition int) int { +// Returns both byte positions and rune position +func positionAfterWhitespace(body []byte, startPosition int) (position int, runePosition int) { bodyLength := len(body) - position := startPosition + position = startPosition + runePosition = startPosition for { if position < bodyLength { - code := charCodeAt(body, position) + code, n := runeAt(body, position) // Skip Ignored if code == 0xFEFF || // BOM @@ -452,16 +473,19 @@ func positionAfterWhitespace(body string, startPosition int) int { code == 0x000D || // carriage return // Comma code == 0x002C { - position++ + position += n + runePosition++ } else if code == 35 { // # - position++ + position += n + runePosition++ for { - code := charCodeAt(body, position) + code, n := runeAt(body, position) if position < bodyLength && code != 0 && // SourceCharacter but not LineTerminator (code > 0x001F || code == 0x0009) && code != 0x000A && code != 0x000D { - position++ + position += n + runePosition++ continue } else { break @@ -475,7 +499,7 @@ func positionAfterWhitespace(body string, startPosition int) int { break } } - return position + return position, runePosition } func GetTokenDesc(token Token) string { diff --git a/language/lexer/lexer_test.go b/language/lexer/lexer_test.go index f18a8b66..efeeff96 100644 --- a/language/lexer/lexer_test.go +++ b/language/lexer/lexer_test.go @@ -13,10 +13,57 @@ type Test struct { } func createSource(body string) *source.Source { - return source.NewSource(&source.Source{Body: body}) + return source.NewSource(&source.Source{Body: []byte(body)}) } -func TestDisallowsUncommonControlCharacters(t *testing.T) { +func TestLexer_GetTokenDesc(t *testing.T) { + expected := `Name "foo"` + tokenDescription := GetTokenDesc(Token{ + Kind: TokenKind[NAME], + Start: 2, + End: 5, + Value: "foo", + }) + if expected != tokenDescription { + t.Errorf("Expected %v, got %v", expected, tokenDescription) + } + + expected = `Name` + tokenDescription = GetTokenDesc(Token{ + Kind: TokenKind[NAME], + Start: 0, + End: 0, + Value: "", + }) + if expected != tokenDescription { + t.Errorf("Expected %v, got %v", expected, tokenDescription) + } + + expected = `String "foo"` + tokenDescription = GetTokenDesc(Token{ + Kind: TokenKind[STRING], + Start: 2, + End: 5, + Value: "foo", + }) + if expected != tokenDescription { + t.Errorf("Expected %v, got %v", expected, tokenDescription) + } + + expected = `String` + tokenDescription = GetTokenDesc(Token{ + Kind: TokenKind[STRING], + Start: 0, + End: 0, + Value: "", + }) + if expected != tokenDescription { + t.Errorf("Expected %v, got %v", expected, tokenDescription) + } + +} + +func TestLexer_DisallowsUncommonControlCharacters(t *testing.T) { tests := []Test{ { Body: "\u0007", @@ -30,15 +77,15 @@ func TestDisallowsUncommonControlCharacters(t *testing.T) { for _, test := range tests { _, err := Lex(createSource(test.Body))(0) if err == nil { - t.Fatalf("unexpected nil error\nexpected:\n%v\n\ngot:\n%v", test.Expected, err) + t.Errorf("unexpected nil error\nexpected:\n%v\n\ngot:\n%v", test.Expected, err) } if err.Error() != test.Expected { - t.Fatalf("unexpected error.\nexpected:\n%v\n\ngot:\n%v", test.Expected, err.Error()) + t.Errorf("unexpected error.\nexpected:\n%v\n\ngot:\n%v", test.Expected, err.Error()) } } } -func TestAcceptsBOMHeader(t *testing.T) { +func TestLexer_AcceptsBOMHeader(t *testing.T) { tests := []Test{ { Body: "\uFEFF foo", @@ -51,17 +98,17 @@ func TestAcceptsBOMHeader(t *testing.T) { }, } for _, test := range tests { - token, err := Lex(&source.Source{Body: test.Body})(0) + token, err := Lex(&source.Source{Body: []byte(test.Body)})(0) if err != nil { - t.Fatalf("unexpected error: %v", err) + t.Errorf("unexpected error: %v", err) } if !reflect.DeepEqual(token, test.Expected) { - t.Fatalf("unexpected token, expected: %v, got: %v", test.Expected, token) + t.Errorf("unexpected token, expected: %v, got: %v", test.Expected, token) } } } -func TestSkipsWhiteSpace(t *testing.T) { +func TestLexer_SkipsWhiteSpace(t *testing.T) { tests := []Test{ { Body: ` @@ -97,19 +144,28 @@ func TestSkipsWhiteSpace(t *testing.T) { Value: "foo", }, }, + { + Body: ``, + Expected: Token{ + Kind: TokenKind[EOF], + Start: 0, + End: 0, + Value: "", + }, + }, } for _, test := range tests { - token, err := Lex(&source.Source{Body: test.Body})(0) + token, err := Lex(&source.Source{Body: []byte(test.Body)})(0) if err != nil { - t.Fatalf("unexpected error: %v", err) + t.Errorf("unexpected error: %v", err) } if !reflect.DeepEqual(token, test.Expected) { - t.Fatalf("unexpected token, expected: %v, got: %v, body: %s", test.Expected, token, test.Body) + t.Errorf("unexpected token, expected: %v, got: %v, body: %s", test.Expected, token, test.Body) } } } -func TestErrorsRespectWhitespace(t *testing.T) { +func TestLexer_ErrorsRespectWhitespace(t *testing.T) { body := ` ? @@ -125,7 +181,39 @@ func TestErrorsRespectWhitespace(t *testing.T) { } } -func TestLexesStrings(t *testing.T) { +func TestLexer_LexesNames(t *testing.T) { + tests := []Test{ + { + Body: "simple", + Expected: Token{ + Kind: TokenKind[NAME], + Start: 0, + End: 6, + Value: "simple", + }, + }, + { + Body: "Capital", + Expected: Token{ + Kind: TokenKind[NAME], + Start: 0, + End: 7, + Value: "Capital", + }, + }, + } + for _, test := range tests { + token, err := Lex(&source.Source{Body: []byte(test.Body)})(0) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !reflect.DeepEqual(token, test.Expected) { + t.Errorf("unexpected token, expected: %v, got: %v", test.Expected, token) + } + } +} + +func TestLexer_LexesStrings(t *testing.T) { tests := []Test{ { Body: "\"simple\"", @@ -169,7 +257,7 @@ func TestLexesStrings(t *testing.T) { Kind: TokenKind[STRING], Start: 0, End: 15, - Value: "slashes \\ \\/", + Value: "slashes \\ /", }, }, { @@ -181,19 +269,46 @@ func TestLexesStrings(t *testing.T) { Value: "unicode \u1234\u5678\u90AB\uCDEF", }, }, + { + Body: "\"unicode фы世界\"", + Expected: Token{ + Kind: TokenKind[STRING], + Start: 0, + End: 20, + Value: "unicode фы世界", + }, + }, + { + Body: "\"фы世界\"", + Expected: Token{ + Kind: TokenKind[STRING], + Start: 0, + End: 12, + Value: "фы世界", + }, + }, + { + Body: "\"Has a фы世界 multi-byte character.\"", + Expected: Token{ + Kind: TokenKind[STRING], + Start: 0, + End: 40, + Value: "Has a фы世界 multi-byte character.", + }, + }, } for _, test := range tests { - token, err := Lex(&source.Source{Body: test.Body})(0) + token, err := Lex(&source.Source{Body: []byte(test.Body)})(0) if err != nil { - t.Fatalf("unexpected error: %v", err) + t.Errorf("unexpected error: %v", err) } if !reflect.DeepEqual(token, test.Expected) { - t.Fatalf("unexpected token, expected: %v, got: %v", test.Expected, token) + t.Errorf("unexpected token, expected: %v, got: %v", test.Expected, token) } } } -func TestLexReportsUsefulStringErrors(t *testing.T) { +func TestLexer_ReportsUsefulStringErrors(t *testing.T) { tests := []Test{ { Body: "\"", @@ -299,21 +414,40 @@ func TestLexReportsUsefulStringErrors(t *testing.T) { 1: "bad \uXXXF esc" ^ +`, + }, + { + Body: "\"bad \\u123", + Expected: `Syntax Error GraphQL (1:7) Invalid character escape sequence: \u123 + +1: "bad \u123 + ^ +`, + }, + { + // Some unicode chars take more than one column of text + // current implementation does not handle this. + // This results in the `^` pointer not being accurate. + Body: "\"bфы世ыы𠱸d \\uXXXF esc\"", + Expected: `Syntax Error GraphQL (1:12) Invalid character escape sequence: \uXXXF + +1: "bфы世ыы𠱸d \uXXXF esc" + ^ `, }, } for _, test := range tests { _, err := Lex(createSource(test.Body))(0) if err == nil { - t.Fatalf("unexpected nil error\nexpected:\n%v\n\ngot:\n%v", test.Expected, err) + t.Errorf("unexpected nil error\nexpected:\n%v\n\ngot:\n%v", test.Expected, err) } if err.Error() != test.Expected { - t.Fatalf("unexpected error.\nexpected:\n%v\n\ngot:\n%v", test.Expected, err.Error()) + t.Errorf("unexpected error.\nexpected:\n%v\n\ngot:\n%v", test.Expected, err.Error()) } } } -func TestLexesNumbers(t *testing.T) { +func TestLexer_LexesNumbers(t *testing.T) { tests := []Test{ { Body: "4", @@ -463,15 +597,15 @@ func TestLexesNumbers(t *testing.T) { for _, test := range tests { token, err := Lex(createSource(test.Body))(0) if err != nil { - t.Fatalf("unexpected error: %v, test: %s", err, test) + t.Errorf("unexpected error: %v, test: %s", err, test) } if !reflect.DeepEqual(token, test.Expected) { - t.Fatalf("unexpected token, expected: %v, got: %v, test: %v", test.Expected, token, test) + t.Errorf("unexpected token, expected: %v, got: %v, test: %v", test.Expected, token, test) } } } -func TestLexReportsUsefulNumbeErrors(t *testing.T) { +func TestLexer_ReportsUsefulNumberErrors(t *testing.T) { tests := []Test{ { Body: "00", @@ -542,15 +676,15 @@ func TestLexReportsUsefulNumbeErrors(t *testing.T) { for _, test := range tests { _, err := Lex(createSource(test.Body))(0) if err == nil { - t.Fatalf("unexpected nil error\nexpected:\n%v\n\ngot:\n%v", test.Expected, err) + t.Errorf("unexpected nil error\nexpected:\n%v\n\ngot:\n%v", test.Expected, err) } if err.Error() != test.Expected { - t.Fatalf("unexpected error.\nexpected:\n%v\n\ngot:\n%v", test.Expected, err.Error()) + t.Errorf("unexpected error.\nexpected:\n%v\n\ngot:\n%v", test.Expected, err.Error()) } } } -func TestLexesPunctuation(t *testing.T) { +func TestLexer_LexesPunctuation(t *testing.T) { tests := []Test{ { Body: "!", @@ -673,15 +807,15 @@ func TestLexesPunctuation(t *testing.T) { for _, test := range tests { token, err := Lex(createSource(test.Body))(0) if err != nil { - t.Fatalf("unexpected error :%v, test: %v", err, test) + t.Errorf("unexpected error :%v, test: %v", err, test) } if !reflect.DeepEqual(token, test.Expected) { - t.Fatalf("unexpected token, expected: %v, got: %v, test: %v", test.Expected, token, test) + t.Errorf("unexpected token, expected: %v, got: %v, test: %v", test.Expected, token, test) } } } -func TestLexReportsUsefulUnknownCharacterError(t *testing.T) { +func TestLexer_ReportsUsefulUnknownCharacterError(t *testing.T) { tests := []Test{ { Body: "..", @@ -713,21 +847,29 @@ func TestLexReportsUsefulUnknownCharacterError(t *testing.T) { 1: ※ ^ +`, + }, + { + Body: "ф", + Expected: `Syntax Error GraphQL (1:1) Unexpected character "\\u0444". + +1: ф + ^ `, }, } for _, test := range tests { _, err := Lex(createSource(test.Body))(0) if err == nil { - t.Fatalf("unexpected nil error\nexpected:\n%v\n\ngot:\n%v", test.Expected, err) + t.Errorf("unexpected nil error\nexpected:\n%v\n\ngot:\n%v", test.Expected, err) } if err.Error() != test.Expected { - t.Fatalf("unexpected error.\nexpected:\n%v\n\ngot:\n%v", test.Expected, err.Error()) + t.Errorf("unexpected error.\nexpected:\n%v\n\ngot:\n%v", test.Expected, err.Error()) } } } -func TestLexRerportsUsefulInformationForDashesInNames(t *testing.T) { +func TestLexer_ReportsUsefulInformationForDashesInNames(t *testing.T) { q := "a-b" lexer := Lex(createSource(q)) firstToken, err := lexer(0) diff --git a/language/location/location.go b/language/location/location.go index ec667caa..04bbde6e 100644 --- a/language/location/location.go +++ b/language/location/location.go @@ -12,14 +12,14 @@ type SourceLocation struct { } func GetLocation(s *source.Source, position int) SourceLocation { - body := "" + body := []byte{} if s != nil { body = s.Body } line := 1 column := position + 1 lineRegexp := regexp.MustCompile("\r\n|[\n\r]") - matches := lineRegexp.FindAllStringIndex(body, -1) + matches := lineRegexp.FindAllIndex(body, -1) for _, match := range matches { matchIndex := match[0] if matchIndex < position { diff --git a/language/parser/parser.go b/language/parser/parser.go index d1e01c50..46ddf751 100644 --- a/language/parser/parser.go +++ b/language/parser/parser.go @@ -36,7 +36,7 @@ func Parse(p ParseParams) (*ast.Document, error) { sourceObj = p.Source.(*source.Source) default: body, _ := p.Source.(string) - sourceObj = source.NewSource(&source.Source{Body: body}) + sourceObj = source.NewSource(&source.Source{Body: []byte(body)}) } parser, err := makeParser(sourceObj, p.Options) if err != nil { @@ -58,7 +58,7 @@ func parseValue(p ParseParams) (ast.Value, error) { sourceObj = p.Source.(*source.Source) default: body, _ := p.Source.(string) - sourceObj = source.NewSource(&source.Source{Body: body}) + sourceObj = source.NewSource(&source.Source{Body: []byte(body)}) } parser, err := makeParser(sourceObj, p.Options) if err != nil { diff --git a/language/parser/parser_test.go b/language/parser/parser_test.go index f8697310..e2198f22 100644 --- a/language/parser/parser_test.go +++ b/language/parser/parser_test.go @@ -17,7 +17,7 @@ import ( func TestBadToken(t *testing.T) { _, err := Parse(ParseParams{ Source: &source.Source{ - Body: "query _ {\n me {\n id`\n }\n}", + Body: []byte("query _ {\n me {\n id`\n }\n}"), Name: "GraphQL", }, }) @@ -137,7 +137,10 @@ fragment MissingOn Type func TestParseProvidesUsefulErrorsWhenUsingSource(t *testing.T) { test := errorMessageTest{ - source.NewSource(&source.Source{Body: "query", Name: "MyQuery.graphql"}), + source.NewSource(&source.Source{ + Body: []byte("query"), + Name: "MyQuery.graphql", + }), `Syntax Error MyQuery.graphql (1:6) Expected {, found EOF`, false, } @@ -189,7 +192,7 @@ func TestDoesNotAllowNullAsValue(t *testing.T) { testErrorMessage(t, test) } -func TestParsesMultiByteCharacters(t *testing.T) { +func TestParsesMultiByteCharacters_Unicode(t *testing.T) { doc := ` # This comment has a \u0A0A multi-byte character. @@ -266,6 +269,83 @@ func TestParsesMultiByteCharacters(t *testing.T) { } } +func TestParsesMultiByteCharacters_UnicodeText(t *testing.T) { + + doc := ` + # This comment has a фы世界 multi-byte character. + { field(arg: "Has a фы世界 multi-byte character.") } + ` + astDoc := parse(t, doc) + + expectedASTDoc := ast.NewDocument(&ast.Document{ + Loc: ast.NewLocation(&ast.Location{ + Start: 67, + End: 121, + }), + Definitions: []ast.Node{ + ast.NewOperationDefinition(&ast.OperationDefinition{ + Loc: ast.NewLocation(&ast.Location{ + Start: 67, + End: 119, + }), + Operation: "query", + SelectionSet: ast.NewSelectionSet(&ast.SelectionSet{ + Loc: ast.NewLocation(&ast.Location{ + Start: 67, + End: 119, + }), + Selections: []ast.Selection{ + ast.NewField(&ast.Field{ + Loc: ast.NewLocation(&ast.Location{ + Start: 67, + End: 117, + }), + Name: ast.NewName(&ast.Name{ + Loc: ast.NewLocation(&ast.Location{ + Start: 69, + End: 74, + }), + Value: "field", + }), + Arguments: []*ast.Argument{ + ast.NewArgument(&ast.Argument{ + Loc: ast.NewLocation(&ast.Location{ + Start: 75, + End: 116, + }), + Name: ast.NewName(&ast.Name{ + + Loc: ast.NewLocation(&ast.Location{ + Start: 75, + End: 78, + }), + Value: "arg", + }), + Value: ast.NewStringValue(&ast.StringValue{ + + Loc: ast.NewLocation(&ast.Location{ + Start: 80, + End: 116, + }), + Value: "Has a фы世界 multi-byte character.", + }), + }), + }, + }), + }, + }), + }), + }, + }) + + astDocQuery := printer.Print(astDoc) + expectedASTDocQuery := printer.Print(expectedASTDoc) + + if !reflect.DeepEqual(astDocQuery, expectedASTDocQuery) { + t.Fatalf("unexpected document, expected: %v, got: %v", astDocQuery, expectedASTDocQuery) + } +} + func TestParsesKitchenSink(t *testing.T) { b, err := ioutil.ReadFile("../../kitchen-sink.graphql") if err != nil { @@ -309,18 +389,17 @@ func TestAllowsNonKeywordsAnywhereNameIsAllowed(t *testing.T) { } } -// -//func TestParsesExperimentalSubscriptionFeature(t *testing.T) { -// source := ` -// subscription Foo { -// subscriptionField -// } -// ` -// _, err := Parse(ParseParams{Source: source}) -// if err != nil { -// t.Fatalf("unexpected error: %v", err) -// } -//} +func TestParsesExperimentalSubscriptionFeature(t *testing.T) { + source := ` + subscription Foo { + subscriptionField + } + ` + _, err := Parse(ParseParams{Source: source}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} func TestParsesAnonymousMutationOperations(t *testing.T) { source := ` @@ -378,7 +457,9 @@ func TestParseCreatesAst(t *testing.T) { } } ` - source := source.NewSource(&source.Source{Body: body}) + source := source.NewSource(&source.Source{ + Body: []byte(body), + }) document, err := Parse( ParseParams{ Source: source, diff --git a/language/parser/schema_parser_test.go b/language/parser/schema_parser_test.go index ce6e552c..510c7d26 100644 --- a/language/parser/schema_parser_test.go +++ b/language/parser/schema_parser_test.go @@ -739,10 +739,10 @@ input Hello { `, Nodes: []ast.Node{}, Source: &source.Source{ - Body: ` + Body: []byte(` input Hello { world(foo: Int): String -}`, +}`), Name: "GraphQL", }, Positions: []int{22}, diff --git a/language/source/source.go b/language/source/source.go index c75192d4..f14af003 100644 --- a/language/source/source.go +++ b/language/source/source.go @@ -5,7 +5,7 @@ const ( ) type Source struct { - Body string + Body []byte Name string } diff --git a/testutil/rules_test_harness.go b/testutil/rules_test_harness.go index 035d8a61..809a1eff 100644 --- a/testutil/rules_test_harness.go +++ b/testutil/rules_test_harness.go @@ -481,7 +481,7 @@ func init() { } func expectValidRule(t *testing.T, schema *graphql.Schema, rules []graphql.ValidationRuleFn, queryString string) { source := source.NewSource(&source.Source{ - Body: queryString, + Body: []byte(queryString), }) AST, err := parser.Parse(parser.ParseParams{Source: source}) if err != nil { @@ -498,7 +498,7 @@ func expectValidRule(t *testing.T, schema *graphql.Schema, rules []graphql.Valid } func expectInvalidRule(t *testing.T, schema *graphql.Schema, rules []graphql.ValidationRuleFn, queryString string, expectedErrors []gqlerrors.FormattedError) { source := source.NewSource(&source.Source{ - Body: queryString, + Body: []byte(queryString), }) AST, err := parser.Parse(parser.ParseParams{Source: source}) if err != nil { diff --git a/validator_test.go b/validator_test.go index f7f19e57..67b7a3dd 100644 --- a/validator_test.go +++ b/validator_test.go @@ -15,7 +15,7 @@ import ( func expectValid(t *testing.T, schema *graphql.Schema, queryString string) { source := source.NewSource(&source.Source{ - Body: queryString, + Body: []byte(queryString), Name: "GraphQL request", }) AST, err := parser.Parse(parser.ParseParams{Source: source})