Merge pull request #465 from aycabta/fix-token-position

Fix seek position

Author: hsbt

lib/rdoc/ruby_lex.rb

@@ -364,7 +364,7 @@ def token
 
     if TkSYMBEG === tk then
       tk1 = token
-      set_token_position tk.line_no, tk.char_no
+      set_token_position tk.seek, tk.line_no, tk.char_no
 
       case tk1
       when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then

lib/rdoc/ruby_token.rb

@@ -26,7 +26,8 @@ module RDoc::RubyToken
     Symbol = Integer
   end
 
-  def set_token_position(line, char)
+  def set_token_position(seek, line, char)
+    @prev_seek = seek
     @prev_line_no = line
     @prev_char_no = char
   end

test/test_rdoc_ruby_lex.rb

@@ -9,6 +9,31 @@ def setup
     @TK = RDoc::RubyToken
   end
 
+  def test_token_position
+    tokens = RDoc::RubyLex.tokenize '[ 1, :a, nil ]', nil
+
+    assert_equal '[', tokens[0].text
+    assert_equal 0, tokens[0].seek
+    assert_equal 1, tokens[0].line_no
+    assert_equal 0, tokens[0].char_no
+    assert_equal '1', tokens[2].text
+    assert_equal 2, tokens[2].seek
+    assert_equal 1, tokens[2].line_no
+    assert_equal 2, tokens[2].char_no
+    assert_equal ':a', tokens[5].text
+    assert_equal 5, tokens[5].seek
+    assert_equal 1, tokens[5].line_no
+    assert_equal 5, tokens[5].char_no
+    assert_equal 'nil', tokens[8].text
+    assert_equal 9, tokens[8].seek
+    assert_equal 1, tokens[8].line_no
+    assert_equal 9, tokens[8].char_no
+    assert_equal ']', tokens[10].text
+    assert_equal 13, tokens[10].seek
+    assert_equal 1, tokens[10].line_no
+    assert_equal 13, tokens[10].char_no
+  end
+
   def test_class_tokenize
     tokens = RDoc::RubyLex.tokenize "def x() end", nil