Merge branch 'master' into fix-backtick-handling

Author: aycabta

.travis.yml

@@ -6,9 +6,6 @@ notifications:
   email:
   - [email protected]
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1.10
   - 2.2.7
   - 2.3.4
   - 2.4.1

appveyor.yml

@@ -10,11 +10,6 @@ deploy: off
 environment:
   NOBENCHMARK: 1
   matrix:
-    - ruby_version: "193"
-    - ruby_version: "200"
-    - ruby_version: "200-x64"
-    - ruby_version: "21"
-    - ruby_version: "21-x64"
     - ruby_version: "22"
     - ruby_version: "22-x64"
     - ruby_version: "23"

lib/rdoc/parser/ruby.rb

@@ -2022,31 +2022,25 @@ def scan
   def skip_optional_do_after_expression
     skip_tkspace false
     tk = get_tk
-    end_token = get_end_token tk
 
     b_nest = 0
     nest = 0
     @scanner.continue = false
 
     loop do
       case tk
-      when TkSEMICOLON then
+      when TkSEMICOLON, TkNL then
         break if b_nest.zero?
       when TkLPAREN, TkfLPAREN then
         nest += 1
+      when TkRPAREN then
+        nest -= 1
       when TkBEGIN then
         b_nest += 1
       when TkEND then
         b_nest -= 1
       when TkDO
         break if nest.zero?
-      when end_token then
-        if end_token == TkRPAREN
-          nest -= 1
-          break if @scanner.lex_state == :EXPR_END and nest.zero?
-        else
-          break unless @scanner.continue
-        end
       when nil then
         break
       end

lib/rdoc/ruby_lex.rb

@@ -106,6 +106,8 @@ def initialize(content, options)
     @rests = []
     @seek = 0
 
+    @heredoc_queue = []
+
     @indent = 0
     @indent_stack = []
     @lex_state = :EXPR_BEG
@@ -464,21 +466,43 @@ def lex_init()
 
     @OP.def_rule("\n") do |op, io|
       print "\\n\n" if RDoc::RubyLex.debug?
-      case @lex_state
-      when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
-        @continue = true
-      else
-        @continue = false
-        @lex_state = :EXPR_BEG
-        until (@indent_stack.empty? ||
-               [TkLPAREN, TkLBRACK, TkLBRACE,
-                 TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
-          @indent_stack.pop
+      unless @heredoc_queue.empty?
+        info = @heredoc_queue[0]
+        if !info[:started] # "\n"
+          info[:started] = true
+          ungetc "\n"
+        elsif info[:heredoc_end].nil? # heredoc body
+          tk, heredoc_end = identify_here_document_body(info[:quoted], info[:lt], info[:indent])
+          info[:heredoc_end] = heredoc_end
+          ungetc "\n"
+        else # heredoc end
+          @heredoc_queue.shift
+          @lex_state = :EXPR_BEG
+          tk = Token(TkHEREDOCEND, info[:heredoc_end])
+          if !@heredoc_queue.empty?
+            @heredoc_queue[0][:started] = true
+            ungetc "\n"
+          end
         end
       end
-      @current_readed = @readed
-      @here_readed.clear
-      Token(TkNL)
+      unless tk
+        case @lex_state
+        when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
+          @continue = true
+        else
+          @continue = false
+          @lex_state = :EXPR_BEG
+          until (@indent_stack.empty? ||
+                 [TkLPAREN, TkLBRACK, TkLBRACE,
+                   TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
+            @indent_stack.pop
+          end
+        end
+        @current_readed = @readed
+        @here_readed.clear
+        tk = Token(TkNL)
+      end
+      tk
     end
 
     @OP.def_rules("=") do
@@ -509,6 +533,12 @@ def lex_init()
       tk
     end
 
+    @OP.def_rules("->") do
+      |op, io|
+      @lex_state = :EXPR_ENDFN
+      Token(op)
+    end
+
     @OP.def_rules("!", "!=", "!~") do
       |op, io|
       case @lex_state
@@ -527,8 +557,8 @@ def lex_init()
       if @lex_state != :EXPR_END && @lex_state != :EXPR_CLASS &&
          (@lex_state != :EXPR_ARG || @space_seen)
         c = peek(0)
-        if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-")
-          tk = identify_here_document
+        if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-" || c == "~")
+          tk = identify_here_document(op)
         end
       end
       unless tk
@@ -837,14 +867,11 @@ def lex_int2
 
     @OP.def_rule('\\') do
       |op, io|
-      if getc == "\n"
+      if peek(0) == "\n"
         @space_seen = true
         @continue = true
-        Token(TkSPACE)
-      else
-        ungetc
-        Token("\\")
       end
+      Token("\\")
     end
 
     @OP.def_rule('%') do
@@ -1053,7 +1080,11 @@ def identify_identifier
     end
 
     if token[0, 1] =~ /[A-Z]/
-      return Token(TkCONSTANT, token)
+      if token[-1] =~ /[!?]/
+        return Token(TkIDENTIFIER, token)
+      else
+        return Token(TkCONSTANT, token)
+      end
     elsif token[token.size - 1, 1] =~ /[!?]/
       return Token(TkFID, token)
     else
@@ -1066,77 +1097,63 @@ def identify_identifier
     end
   end
 
-  def identify_here_document
+  def identify_here_document(op)
     ch = getc
+    start_token = op
     #    if lt = PERCENT_LTYPE[ch]
-    if ch == "-"
+    if ch == "-" or ch == "~"
+      start_token.concat ch
       ch = getc
       indent = true
     end
     if /['"`]/ =~ ch
+      start_token.concat ch
       user_quote = lt = ch
       quoted = ""
       while (c = getc) && c != lt
         quoted.concat c
       end
+      start_token.concat quoted
+      start_token.concat lt
     else
       user_quote = nil
       lt = '"'
       quoted = ch.dup
       while (c = getc) && c =~ /\w/
         quoted.concat c
       end
+      start_token.concat quoted
       ungetc
     end
 
-    ltback, @ltype = @ltype, lt
-    reserve = []
-    while ch = getc
-      reserve.push ch
-      if ch == "\\"
-        reserve.push ch = getc
-      elsif ch == "\n"
-        break
-      end
-    end
-
-    output_heredoc = reserve.join =~ /\A\r?\n\z/
+    @heredoc_queue << {
+      quoted: quoted,
+      lt: lt,
+      indent: indent,
+      started: false
+    }
+    @lex_state = :EXPR_BEG
+    Token(RDoc::RubyLex::TkHEREDOCBEG, start_token)
+  end
 
-    if output_heredoc then
-      doc = '<<'
-      doc << '-' if indent
-      doc << "#{user_quote}#{quoted}#{user_quote}\n"
-    else
-      doc = '"'
-    end
+  def identify_here_document_body(quoted, lt, indent)
+    ltback, @ltype = @ltype, lt
 
-    @current_readed = @readed
+    doc = ""
+    heredoc_end = nil
     while l = gets
       l = l.sub(/(:?\r)?\n\z/, "\n")
       if (indent ? l.strip : l.chomp) == quoted
+        heredoc_end = l
         break
       end
       doc << l
     end
+    raise Error, "Missing terminating #{quoted} for string" unless heredoc_end
 
-    if output_heredoc then
-      raise Error, "Missing terminating #{quoted} for string" unless l
-
-      doc << l.chomp
-    else
-      doc << '"'
-    end
-
-    @current_readed = @here_readed
-    @here_readed.concat reserve
-    while ch = reserve.pop
-      ungetc ch
-    end
-
-    token_class = output_heredoc ? RDoc::RubyLex::TkHEREDOC : Ltype2Token[lt]
     @ltype = ltback
-    @lex_state = :EXPR_END
-    Token(token_class, doc)
+    @lex_state = :EXPR_BEG
+    [Token(RDoc::RubyLex::TkHEREDOC, doc), heredoc_end]
   end
 
   def identify_quotation
@@ -1163,7 +1180,7 @@ def identify_number(op = "")
 
     num = op
 
-    if peek(0) == "0" && peek(1) !~ /[.eE]/
+    if peek(0) == "0" && peek(1) !~ /[.eEri]/
       num << getc
 
       case peek(0)
@@ -1292,7 +1309,7 @@ def identify_string(ltype, quoted = ltype, type = nil)
     str = if ltype == quoted and %w[" ' / `].include? ltype then
             ltype.dup
           else
-            "%#{type or PERCENT_LTYPE.key ltype}#{PERCENT_PAREN_REV[quoted]||quoted}"
+            "%#{type}#{PERCENT_PAREN_REV[quoted]||quoted}"
           end
 
     subtype = nil

lib/rdoc/ruby_token.rb

@@ -329,6 +329,8 @@ def Token(token, value = nil)
     [:TkCVAR,       TkId],
     [:TkIVAR,       TkId],
     [:TkCONSTANT,   TkId],
+    [:TkHEREDOCBEG, TkId],
+    [:TkHEREDOCEND, TkId],
 
     [:TkINTEGER,    TkVal],
     [:TkFLOAT,      TkVal],
@@ -371,6 +373,7 @@ def Token(token, value = nil)
     [:TkCOLON3,     TkOp,   '::'],
     #[:OPASGN,       TkOp],               # +=, -=  etc. #
     [:TkASSOC,      TkOp,   "=>"],
+    [:TkLAMBDA,     TkOp,   "->"],
     [:TkQUESTION,   TkOp,   "?"], #?
     [:TkCOLON,      TkOp,   ":"],        #:
 

rdoc.gemspec

@@ -46,7 +46,7 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat
     TODO.rdoc
   ]
 
-  s.required_ruby_version = Gem::Requirement.new(">= 1.9.3")
+  s.required_ruby_version = Gem::Requirement.new(">= 2.2.2")
   s.rubygems_version = "2.5.2"
   s.required_rubygems_version = Gem::Requirement.new(">= 2.2")
 

test/test_rdoc_markup_to_html.rb

@@ -451,6 +451,22 @@ def test_accept_verbatim_parseable_error
     assert_equal expected, @to.res.join
   end
 
+  def test_accept_verbatim_nl_after_backslash
+    verb = @RM::Verbatim.new("a = 1 if first_flag_var and \\\n", "  this_is_flag_var\n")
+
+    @to.start_accepting
+    @to.accept_verbatim verb
+
+    expected = <<-EXPECTED
+
+<pre class="ruby"><span class="ruby-identifier">a</span> = <span class="ruby-value">1</span> <span class="ruby-keyword">if</span> <span class="ruby-identifier">first_flag_var</span> <span class="ruby-keyword">and</span> \\
+  <span class="ruby-identifier">this_is_flag_var</span>
+</pre>
+    EXPECTED
+
+    assert_equal expected, @to.res.join
+  end
+
   def test_accept_verbatim_pipe
     @options.pipe = true
 

test/test_rdoc_parser_ruby.rb

@@ -2467,6 +2467,38 @@ class DateTime < Date
     assert_equal :private, date_time_now.visibility, date_time_now.full_name
   end
 
+  def test_parse_statements_complex_condition_in_for
+    util_parser <<RUBY
+class Foo
+  def blah()
+    for i in (k)...n do
+    end
+    for i in (k)...n
+    end
+  end
+end
+RUBY
+
+    expected = <<EXPTECTED
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">blah</span>()
+  <span class="ruby-keyword">for</span> <span class="ruby-identifier">i</span> <span class="ruby-keyword">in</span> (<span class="ruby-identifier">k</span>)<span class="ruby-operator">...</span><span class="ruby-identifier">n</span> <span class="ruby-keyword">do</span>
+  <span class="ruby-keyword">end</span>
+  <span class="ruby-keyword">for</span> <span class="ruby-identifier">i</span> <span class="ruby-keyword">in</span> (<span class="ruby-identifier">k</span>)<span class="ruby-operator">...</span><span class="ruby-identifier">n</span>
+  <span class="ruby-keyword">end</span>
+<span class="ruby-keyword">end</span>
+EXPTECTED
+    expected = expected.rstrip
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    markup_code = blah.markup_code.sub(/^.*\n/, '')
+    assert_equal markup_code, expected
+  end
+
   def test_parse_require_dynamic_string
     content = <<-RUBY
 prefix = 'path'
@@ -2859,7 +2891,7 @@ def test_sanity_interpolation_crazy
   def test_sanity_interpolation_curly
     util_parser '%{ #{} }'
 
-    assert_equal '%Q{ #{} }', @parser.get_tk.text
+    assert_equal '%{ #{} }', @parser.get_tk.text
     assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class
   end