Use Ripper for lexical analyser

lib/rdoc.rb

@@ -148,7 +148,7 @@ def self.load_yaml
 
   autoload :KNOWN_CLASSES,  'rdoc/known_classes'
 
-  autoload :RubyLex,        'rdoc/ruby_lex'
+  autoload :RipperStateLex, 'rdoc/parser/ripper_state_lex'
   autoload :RubyToken,      'rdoc/ruby_token'
   autoload :TokenStream,    'rdoc/token_stream'
 

lib/rdoc/markup/to_html.rb

@@ -200,10 +200,12 @@ def accept_verbatim verbatim
 
     content = if verbatim.ruby? or parseable? text then
                 begin
-                  tokens = RDoc::RubyLex.tokenize text, @options
+                  tokens = RDoc::RipperStateLex.parse text
                   klass  = ' class="ruby"'
 
-                  RDoc::TokenStream.to_html tokens
+                  result = RDoc::TokenStream.to_html tokens
+                  result = result + "\n" unless "\n" == result[-1]
+                  result
                 rescue RDoc::RubyLex::Error
                   CGI.escapeHTML text
                 end
@@ -212,7 +214,7 @@ def accept_verbatim verbatim
               end
 
     if @options.pipe then
-      @res << "\n<pre><code>#{CGI.escapeHTML text}</code></pre>\n"
+      @res << "\n<pre><code>#{CGI.escapeHTML text}\n</code></pre>\n"
     else
       @res << "\n<pre#{klass}>#{content}</pre>\n"
     end

lib/rdoc/parser/c.rb

@@ -666,8 +666,7 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false
 
       #meth_obj.params = params
       meth_obj.start_collecting_tokens
-      tk = RDoc::RubyToken::Token.new nil, 1, 1
-      tk.set_text body
+      tk = { :line_no => 1, :char_no => 1, :text => body }
       meth_obj.add_token tk
       meth_obj.comment = comment
       meth_obj.line    = file_content[0, offset].count("\n") + 1
@@ -684,8 +683,7 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false
       find_modifiers comment, meth_obj
 
       meth_obj.start_collecting_tokens
-      tk = RDoc::RubyToken::Token.new nil, 1, 1
-      tk.set_text body
+      tk = { :line_no => 1, :char_no => 1, :text => body }
       meth_obj.add_token tk
       meth_obj.comment = comment
       meth_obj.line    = file_content[0, offset].count("\n") + 1