Fix incorrect lexing and parsing of multi-byte comments

language/lexer/lexer.go

@@ -625,7 +625,7 @@ func positionAfterWhitespace(body []byte, startPosition int) (position int, rune
 						// SourceCharacter but not LineTerminator
 						(code > 0x001F || code == 0x0009) && code != 0x000A && code != 0x000D {
 						position += n
-						runePosition++
+						runePosition += n
 						continue
 					} else {
 						break

language/lexer/lexer_test.go

@@ -153,6 +153,18 @@ func TestLexer_SkipsWhiteSpace(t *testing.T) {
 				Value: "",
 			},
 		},
+		{
+			Body: `
+    #comment фы世界 
+    foo#comment
+`,
+			Expected: Token{
+				Kind:  NAME,
+				Start: 30,
+				End:   33,
+				Value: "foo",
+			},
+		},
 	}
 	for _, test := range tests {
 		token, err := Lex(&source.Source{Body: []byte(test.Body)})(0)

language/parser/parser_test.go

@@ -346,6 +346,84 @@ func TestParsesMultiByteCharacters_UnicodeText(t *testing.T) {
 	}
 }
 
+func TestMultiByteCharactersAmongstOtherTokens(t *testing.T) {
+	doc := `
+        { 
+        	# This comment has a фы世界 multi-byte character.
+        	field(arg: "Has a фы世界 multi-byte character.")
+        }
+	`
+	astDoc := parse(t, doc)
+
+	expectedASTDoc := ast.NewDocument(&ast.Document{
+		Loc: ast.NewLocation(&ast.Location{
+			Start: 67,
+			End:   121,
+		}),
+		Definitions: []ast.Node{
+			ast.NewOperationDefinition(&ast.OperationDefinition{
+				Loc: ast.NewLocation(&ast.Location{
+					Start: 67,
+					End:   119,
+				}),
+				Operation: "query",
+				SelectionSet: ast.NewSelectionSet(&ast.SelectionSet{
+					Loc: ast.NewLocation(&ast.Location{
+						Start: 67,
+						End:   119,
+					}),
+					Selections: []ast.Selection{
+						ast.NewField(&ast.Field{
+							Loc: ast.NewLocation(&ast.Location{
+								Start: 67,
+								End:   117,
+							}),
+							Name: ast.NewName(&ast.Name{
+								Loc: ast.NewLocation(&ast.Location{
+									Start: 69,
+									End:   74,
+								}),
+								Value: "field",
+							}),
+							Arguments: []*ast.Argument{
+								ast.NewArgument(&ast.Argument{
+									Loc: ast.NewLocation(&ast.Location{
+										Start: 75,
+										End:   116,
+									}),
+									Name: ast.NewName(&ast.Name{
+
+										Loc: ast.NewLocation(&ast.Location{
+											Start: 75,
+											End:   78,
+										}),
+										Value: "arg",
+									}),
+									Value: ast.NewStringValue(&ast.StringValue{
+
+										Loc: ast.NewLocation(&ast.Location{
+											Start: 80,
+											End:   116,
+										}),
+										Value: "Has a фы世界 multi-byte character.",
+									}),
+								}),
+							},
+						}),
+					},
+				}),
+			}),
+		},
+	})
+
+	astDocQuery := printer.Print(astDoc)
+	expectedASTDocQuery := printer.Print(expectedASTDoc)
+
+	if !reflect.DeepEqual(astDocQuery, expectedASTDocQuery) {
+		t.Fatalf("unexpected document, expected: %v, got: %v", expectedASTDocQuery, astDocQuery)
+	}
+}
+
 func TestParsesKitchenSink(t *testing.T) {
 	b, err := ioutil.ReadFile("../../kitchen-sink.graphql")
 	if err != nil {