From 9fe38ca87818b60d6016e42ce332c88259d7bf86 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 6 Apr 2017 12:56:27 +0900 Subject: [PATCH 001/151] Use Ripper instead of IRB's lex --- lib/rdoc/parser/ruby.rb | 13 ++++++++++--- lib/rdoc/parser/ruby_tools.rb | 25 ++++++++++++++++++------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index d81a06c007..8fce4a0674 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -140,11 +140,12 @@ # Note that by default, the :method: directive will be ignored if there is a # standard rdocable item following it. +require 'ripper' + class RDoc::Parser::Ruby < RDoc::Parser parse_files_matching(/\.rbw?$/) - include RDoc::RubyToken include RDoc::TokenStream include RDoc::Parser::RubyTools @@ -166,8 +167,14 @@ def initialize(top_level, file_name, content, options, stats) @size = 0 @token_listeners = nil - @scanner = RDoc::RubyLex.new content, @options - @scanner.exception_on_syntax_error = false + @scanner = Ripper.lex(content).map { |tk| + { + :line_no => tk[0][0], + :char_no => tk[0][1], + :kind => tk[1], + :text => tk[2] + } + } @prev_seek = nil @markup = @options.markup @track_visibility = :nodoc != @options.visibility diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb index e35b420f30..dc6e657bb2 100644 --- a/lib/rdoc/parser/ruby_tools.rb +++ b/lib/rdoc/parser/ruby_tools.rb @@ -22,9 +22,15 @@ def get_tk tk = nil if @tokens.empty? then - tk = @scanner.token - @read.push @scanner.get_readed - puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG + if @scanner_point >= @scanner.size + nil + else + tk = @scanner[@scanner_point] + @scanner_point += 1 + tk = @scanner.token + @read.push tk[:text] + puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG + end else @read.push @unget_read.shift tk = @tokens.shift @@ -102,19 +108,24 @@ def reset @tokens = [] @unget_read = [] @nest = 0 + @scanner_point = 0 + end + + def tk_nl?(tk) + :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind] end ## # Skips whitespace tokens including newlines if +skip_nl+ is true - def skip_tkspace(skip_nl = true) # HACK dup + def skip_tkspace(skip_nl = true) tokens = [] - while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do - tokens.push tk + while (tk = get_tk) and (:on_sp == tk[:kind] or (skip_nl and tk_nl?(tk))) do + tokens.push(tk) end - unget_tk tk + unget_tk(tk) tokens end From ae8e8ce542c22dbcd1c877ed75444033bbbb1b7a Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 6 Apr 2017 13:19:34 +0900 Subject: [PATCH 002/151] Fix collect_first_comment for Ripper --- lib/rdoc/parser/ruby.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 8fce4a0674..74acc23292 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -232,29 +232,29 @@ def get_visibility_information tk, single # :nodoc: def collect_first_comment skip_tkspace comment = '' - comment.force_encoding @encoding if @encoding + #comment.force_encoding @encoding if @encoding first_line = true - first_comment_tk_class = nil + first_comment_tk_kind = nil tk = get_tk - while TkCOMMENT === tk - if first_line and tk.text =~ /\A#!/ then + while :on_comment == tk[:kind] + if first_line and tk[:text] =~ /\A#!/ then skip_tkspace tk = get_tk - elsif first_line and tk.text =~ /\A#\s*-\*-/ then + elsif first_line and tk[:text] =~ /\A#\s*-\*-/ then first_line = false skip_tkspace tk = get_tk else - break if first_comment_tk_class and not first_comment_tk_class === tk - first_comment_tk_class = tk.class + break if first_comment_tk_kind and not first_comment_tk_kind === tk[:kind] + first_comment_tk_kind = tk[:kind] first_line = false - comment << tk.text << "\n" + comment << tk[:text] << "\n" tk = get_tk - if TkNL === tk then + if :on_nl === tk then skip_tkspace false tk = get_tk end From d4d684c298daa721fdc85fe0e31e54be762b03b4 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 6 Apr 2017 13:20:33 +0900 Subject: [PATCH 003/151] Fix get_class_or_module for Ripper --- lib/rdoc/parser/ruby.rb | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 74acc23292..54f64087cd 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -339,24 +339,23 @@ def get_class_or_module container, ignore_constants = false given_name = '' # class ::A -> A is in the top level - case name_t - when TkCOLON2, TkCOLON3 then # bug + if :on_op == name_t[:kind] and '::' == name_t[:text] then # bug name_t = get_tk container = @top_level given_name << '::' end skip_tkspace false - given_name << name_t.name + given_name << name_t[:text] - while TkCOLON2 === peek_tk do + while (tk = peek_tk) and :on_op == tk[:kind] and '::' == tk[:text] do prev_container = container - container = container.find_module_named name_t.name + container = container.find_module_named name_t[:text] container ||= if ignore_constants then RDoc::Context.new else - c = prev_container.add_module RDoc::NormalModule, name_t.name + c = prev_container.add_module RDoc::NormalModule, name_t[:text] c.ignore unless prev_container.document_children @top_level.add_to_classes_or_modules c c @@ -367,7 +366,7 @@ def get_class_or_module container, ignore_constants = false get_tk skip_tkspace false name_t = get_tk - given_name << '::' << name_t.name + given_name << '::' << name_t[:text] end skip_tkspace false From c39f7f019b26bf87c00ecb62dcaf30282cac49e4 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 6 Apr 2017 13:22:18 +0900 Subject: [PATCH 004/151] Fix parse_statements for Ripper --- lib/rdoc/parser/ruby.rb | 111 +++++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 53 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 54f64087cd..49d7efa7a1 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1650,14 +1650,14 @@ def parse_statements(container, single = NORMAL, current_method = nil, keep_comment = false try_parse_comment = false - non_comment_seen = true unless TkCOMMENT === tk + non_comment_seen = true unless :on_comment == tk[:kind] - case tk - when TkNL then + case tk[:kind] + when :on_nl, :on_ignored_nl then skip_tkspace tk = get_tk - if TkCOMMENT === tk then + if tk and :on_comment == tk[:kind] then if non_comment_seen then # Look for RDoc in a comment about to be thrown away non_comment_seen = parse_comment container, tk, comment unless @@ -1667,12 +1667,12 @@ def parse_statements(container, single = NORMAL, current_method = nil, comment.force_encoding @encoding if @encoding end - while TkCOMMENT === tk do - comment << tk.text << "\n" + while :on_comment == tk[:kind] do + comment << tk[:text] << "\n" tk = get_tk - if TkNL === tk then + if :on_nl == tk[:kind] then skip_tkspace false # leading spaces tk = get_tk end @@ -1698,59 +1698,73 @@ def parse_statements(container, single = NORMAL, current_method = nil, keep_comment = true container.current_line_visibility = nil - when TkCLASS then - parse_class container, single, tk, comment + when :on_kw then + case tk[:text] + when 'class' then + parse_class container, single, tk, comment - when TkMODULE then - parse_module container, single, tk, comment + when 'module' then + parse_module container, single, tk, comment - when TkDEF then - parse_method container, single, tk, comment + when 'def' then + parse_method container, single, tk, comment - when TkCONSTANT then - unless parse_constant container, tk, comment, current_method then - try_parse_comment = true - end + when 'alias' then + parse_alias container, single, tk, comment unless current_method - when TkALIAS then - parse_alias container, single, tk, comment unless current_method + when 'yield' then + if current_method.nil? then + warn "Warning: yield outside of method" if container.document_self + else + parse_yield container, single, tk, current_method + end - when TkYIELD then - if current_method.nil? then - warn "Warning: yield outside of method" if container.document_self - else - parse_yield container, single, tk, current_method - end + when 'until', 'while' then + nest += 1 + skip_optional_do_after_expression - # Until and While can have a 'do', which shouldn't increase the nesting. - # We can't solve the general case, but we can handle most occurrences by - # ignoring a do at the end of a line. + # Until and While can have a 'do', which shouldn't increase the nesting. + # We can't solve the general case, but we can handle most occurrences by + # ignoring a do at the end of a line. - when TkUNTIL, TkWHILE then - nest += 1 - skip_optional_do_after_expression + # 'for' is trickier + when 'for' then + nest += 1 + skip_for_variable + skip_optional_do_after_expression - # 'for' is trickier - when TkFOR then - nest += 1 - skip_for_variable - skip_optional_do_after_expression + when 'case', 'do', 'if', 'unless', 'begin' then + nest += 1 - when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN then - nest += 1 + when 'super' then + current_method.calls_super = true if current_method + + when 'rescue' then + parse_rescue + + when 'end' then + nest -= 1 + if nest == 0 then + read_documentation_modifiers container, RDoc::CLASS_MODIFIERS + container.ongoing_visibility = save_visibility - when TkSUPER then - current_method.calls_super = true if current_method + parse_comment container, tk, comment unless comment.empty? - when TkRESCUE then - parse_rescue + return + end + end - when TkIDENTIFIER then + when :on_const then + unless parse_constant container, tk, comment, current_method then + try_parse_comment = true + end + + when :on_ident then if nest == 1 and current_method.nil? then keep_comment = parse_identifier container, single, tk, comment end - case tk.name + case tk[:text] when "require" then parse_require container, comment when "include" then @@ -1759,15 +1773,6 @@ def parse_statements(container, single = NORMAL, current_method = nil, parse_extend_or_include RDoc::Extend, container, comment end - when TkEND then - nest -= 1 - if nest == 0 then - container.ongoing_visibility = save_visibility - - parse_comment container, tk, comment unless comment.empty? - - return - end else try_parse_comment = nest == 1 end From 1e61ab1241533c50f282521e5b0d73e8162bc21f Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 6 Apr 2017 13:23:04 +0900 Subject: [PATCH 005/151] Fix read_directive for Ripper --- lib/rdoc/parser/ruby.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 49d7efa7a1..fd39fb932b 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1977,11 +1977,10 @@ def read_directive allowed while tk = get_tk do tokens << tk - case tk - when TkNL, TkDEF then + if :on_nl == tk[:kind] or (:tk_kw == tk[:kind] && 'def' == tk[:text]) then return - when TkCOMMENT then - return unless tk.text =~ /\s*:?([\w-]+):\s*(.*)/ + elsif :on_comment == tk[:kind] then + return unless tk[:text] =~ /\s*:?([\w-]+):\s*(.*)/ directive = $1.downcase @@ -1991,7 +1990,7 @@ def read_directive allowed end end ensure - unless tokens.length == 1 and TkCOMMENT === tokens.first then + unless tokens.length == 1 and :on_comment == tokens.first[:kind] then tokens.reverse_each do |token| unget_tk token end From 9f59a91a29e72d883827ca45bfc53e76df1fa8db Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 6 Apr 2017 13:25:21 +0900 Subject: [PATCH 006/151] Remove unnecessary method calling --- lib/rdoc/parser/ruby.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index fd39fb932b..99b30edd63 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -270,7 +270,6 @@ def collect_first_comment # Consumes trailing whitespace from the token stream def consume_trailing_spaces # :nodoc: - get_tkread skip_tkspace false end @@ -2043,8 +2042,6 @@ def remove_private_comments comment # Scans this Ruby file for Ruby constructs def scan - reset - catch :eof do begin parse_top_level_statements @top_level From 5dcc45216eb1e6677165c7e9580f957cf8f32711 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 6 Apr 2017 13:29:39 +0900 Subject: [PATCH 007/151] Fix parse_module for Ripper --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 99b30edd63..1e84bada22 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1569,7 +1569,7 @@ def parse_method_parameters method def parse_module container, single, tk, comment container, name_t, = get_class_or_module container - name = name_t.name + name = name_t[:text] mod = container.add_module RDoc::NormalModule, name mod.ignore unless container.document_children From 9338eec522488751d76c9a260e8e21e9f3e97276 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 10 Apr 2017 03:43:26 +0900 Subject: [PATCH 008/151] Fix test_get_class_or_module --- test/test_rdoc_parser_ruby.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index f62f59d8e7..a4821c94ac 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -84,7 +84,7 @@ def test_get_class_or_module cont, name_t, given_name = util_parser('A') .get_class_or_module ctxt assert_equal ctxt, cont - assert_equal 'A', name_t.text + assert_equal 'A', name_t[:text] assert_equal 'A', given_name cont, name_t, given_name = util_parser('B::C') .get_class_or_module ctxt @@ -92,13 +92,13 @@ def test_get_class_or_module b = @store.find_module_named('B') assert_equal b, cont assert_equal [@top_level], b.in_files - assert_equal 'C', name_t.text + assert_equal 'C', name_t[:text] assert_equal 'B::C', given_name cont, name_t, given_name = util_parser('D:: E').get_class_or_module ctxt assert_equal @store.find_module_named('D'), cont - assert_equal 'E', name_t.text + assert_equal 'E', name_t[:text] assert_equal 'D::E', given_name assert_raises NoMethodError do From 7cd8575a1f534fa0aad58516022d203065b071cc Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 10 Apr 2017 04:00:30 +0900 Subject: [PATCH 009/151] Raise RDoc::Error when invalid class or module definition --- lib/rdoc/parser/ruby.rb | 3 +++ test/test_rdoc_parser_ruby.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 1e84bada22..aa3da020d5 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -365,6 +365,9 @@ def get_class_or_module container, ignore_constants = false get_tk skip_tkspace false name_t = get_tk + if name_t[:kind] != :on_const + raise RDoc::Error, "Invalid class or module definition: #{given_name}" + end given_name << '::' << name_t[:text] end diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index a4821c94ac..51b3be7a71 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -101,7 +101,7 @@ def test_get_class_or_module assert_equal 'E', name_t[:text] assert_equal 'D::E', given_name - assert_raises NoMethodError do + assert_raises RDoc::Error do util_parser("A::\nB").get_class_or_module ctxt end end From b5443f97cd83e4ee1318419c361d58c6f0066269 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sat, 29 Apr 2017 20:46:19 +0900 Subject: [PATCH 010/151] Comment out some old code --- lib/rdoc/parser/ruby.rb | 78 ++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index aa3da020d5..fc644f4e2c 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -455,13 +455,19 @@ def get_constant_with_optional_parens # won't catch all cases (such as "a = yield + 1" def get_end_token tk # :nodoc: - case tk - when TkLPAREN, TkfLPAREN - TkRPAREN - when TkRPAREN + case tk[:text] + when :on_lparen + { + :kind => :on_rparen, + :text => ')' + } + when :on_rparen nil else - TkNL + { + :kind => :on_nl, + :text => "\n" + } end end @@ -1311,27 +1317,29 @@ def parse_method(container, single, tk, comment) singleton = nil added_container = false name = nil - column = tk.char_no - line_no = tk.line_no + column = tk[:char_no] + line_no = tk[:line_no] - start_collecting_tokens - add_token tk +# start_collecting_tokens # @token_stream = [] +# add_token tk - token_listener self do +# token_listener self do # @token_listeners << obj prev_container = container name, container, singleton = parse_method_name container added_container = container != prev_container - end +# end return unless name - meth = RDoc::AnyMethod.new get_tkread, name + #meth = RDoc::AnyMethod.new get_tkread, name + meth = RDoc::AnyMethod.new '', name meth.singleton = single == SINGLE ? true : singleton record_location meth meth.line = line_no - meth.start_collecting_tokens +=begin + meth.start_collecting_tokens # @token_stream = [] indent = TkSPACE.new 0, 1, 1 indent.set_text " " * column @@ -1339,6 +1347,7 @@ def parse_method(container, single, tk, comment) token.set_text "# File #{@top_level.relative_name}, line #{line_no}" meth.add_tokens [token, NEWLINE_TOKEN, indent] meth.add_tokens @token_stream +=end parse_method_params_and_body container, single, meth, added_container @@ -1357,8 +1366,8 @@ def parse_method(container, single, tk, comment) # Parses the parameters and body of +meth+ def parse_method_params_and_body container, single, meth, added_container - token_listener meth do - @scanner.continue = false +# token_listener meth do +# @scanner.continue = false parse_method_parameters meth if meth.document_self or not @track_visibility then @@ -1381,7 +1390,7 @@ def parse_method_params_and_body container, single, meth, added_container end parse_statements container, single, meth - end +# end end ## @@ -1401,15 +1410,15 @@ def parse_method_dummy container # it is a singleton or regular method. def parse_method_name container # :nodoc: - @scanner.lex_state = :EXPR_FNAME +# @scanner.lex_state = :EXPR_FNAME skip_tkspace name_t = get_tk back_tk = skip_tkspace singleton = false - case dot = get_tk - when TkDOT, TkCOLON2 then + dot = get_tk + if dot[:kind] == :on_period || (dot[:kind] == :on_op && dot[:text] == '::') then singleton = true name, container = parse_method_name_singleton container, name_t @@ -1430,16 +1439,15 @@ def parse_method_name container # :nodoc: # is parsed from the token stream for a regular method. def parse_method_name_regular container, name_t # :nodoc: - case name_t - when TkSTAR, TkAMPER then - name_t.text + if name_t[:kind] == :on_op && (name_t[:text] == '*' || name_t[:text] == '&') then + name_t[:text] else - unless name_t.respond_to? :name then + unless [:on_kw, :on_const, :on_ident].include?(name_t[:kind]) then warn "expected method name token, . or ::, got #{name_t.inspect}" skip_method container return end - name_t.name + name_t[:text] end end @@ -1506,38 +1514,36 @@ def parse_method_or_yield_parameters(method = nil, nest = 0 - loop do - case tk - when TkSEMICOLON then + while tk != nil do + case tk[:kind] + when :on_semicolon then break if nest == 0 - when TkLBRACE, TkfLBRACE then + when :on_lbrace then nest += 1 - when TkRBRACE then + when :on_rbrace then nest -= 1 if nest <= 0 # we might have a.each { |i| yield i } unget_tk(tk) if nest < 0 break end - when TkLPAREN, TkfLPAREN then + when :on_lparen then nest += 1 when end_token then - if end_token == TkRPAREN + if end_token[:kind] == :on_rparen nest -= 1 break if nest <= 0 else break unless @scanner.continue end - when TkRPAREN then + when :on_rparen then nest -= 1 - when method && method.block_params.nil? && TkCOMMENT then + when method && method.block_params.nil? && :on_comment then unget_tk tk read_documentation_modifiers method, modifiers @read.pop - when TkCOMMENT then + when :on_comment then @read.pop - when nil then - break end tk = get_tk end From e914d68a21d9aeac418c49e40275188fa87828dd Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 12 Jun 2017 01:57:17 +0900 Subject: [PATCH 011/151] Fix test_parse_method for Ripper --- lib/rdoc/parser/ruby.rb | 115 +++++++++++++++++----------------- lib/rdoc/parser/ruby_tools.rb | 38 +++++++++-- test/test_rdoc_parser_ruby.rb | 30 +++++---- 3 files changed, 103 insertions(+), 80 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index fc644f4e2c..5a3941908c 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -175,6 +175,7 @@ def initialize(top_level, file_name, content, options, stats) :text => tk[2] } } + @scanner_point = 0 @prev_seek = nil @markup = @options.markup @track_visibility = :nodoc != @options.visibility @@ -183,6 +184,10 @@ def initialize(top_level, file_name, content, options, stats) reset end + def tk_nl?(tk) + :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind] + end + ## # Retrieves the read token stream and replaces +pattern+ with +replacement+ # using gsub. If the result is only a ";" returns an empty string. @@ -455,7 +460,7 @@ def get_constant_with_optional_parens # won't catch all cases (such as "a = yield + 1" def get_end_token tk # :nodoc: - case tk[:text] + case tk[:kind] when :on_lparen { :kind => :on_rparen, @@ -604,7 +609,7 @@ def new_comment comment # +comment+. def parse_attr(context, single, tk, comment) - line_no = tk.line_no + line_no = tk[:line_no] args = parse_symbol_arg 1 if args.size > 0 then @@ -742,28 +747,26 @@ def parse_call_parameters(tk) # Parses a class in +context+ with +comment+ def parse_class container, single, tk, comment - line_no = tk.line_no + line_no = tk[:line_no] declaration_context = container container, name_t, given_name = get_class_or_module container - cls = - case name_t - when TkCONSTANT - parse_class_regular container, declaration_context, single, - name_t, given_name, comment - when TkLSHFT - case name = get_class_specification - when 'self', container.name - parse_statements container, SINGLE - return # don't update line - else - parse_class_singleton container, name, comment - end + if name_t[:kind] == :on_const + cls = parse_class_regular container, declaration_context, single, + name_t, given_name, comment + elsif name_t[:kind] == :on_op && name_t[:text] == '<<' + case name = get_class_specification + when 'self', container.name + parse_statements container, SINGLE + return # don't update line else - warn "Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}" - return + cls = parse_class_singleton container, name, comment end + else + warn "Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}" + return + end cls.line = line_no @@ -785,7 +788,8 @@ def parse_class_regular container, declaration_context, single, # :nodoc: given_name = $' end - if TkLT === peek_tk then + tk = peek_tk + if tk[:kind] == :on_op && tk[:text] == '<' then get_tk skip_tkspace superclass = get_class_specification @@ -978,8 +982,8 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: def parse_comment container, tk, comment return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc' - column = tk.char_no - line_no = tk.line_no + column = tk[:char_no] + line_no = tk[:line_no] text = comment.text @@ -1115,7 +1119,7 @@ def parse_extend_or_include klass, container, comment # :nodoc: # Returns true if the comment was not consumed. def parse_identifier container, single, tk, comment # :nodoc: - case tk.name + case tk[:text] when 'private', 'protected', 'public', 'private_class_method', 'public_class_method', 'module_function' then parse_visibility container, single, tk @@ -1320,34 +1324,30 @@ def parse_method(container, single, tk, comment) column = tk[:char_no] line_no = tk[:line_no] -# start_collecting_tokens # @token_stream = [] -# add_token tk + start_collecting_tokens + add_token tk -# token_listener self do # @token_listeners << obj + token_listener self do prev_container = container name, container, singleton = parse_method_name container added_container = container != prev_container -# end + end return unless name - #meth = RDoc::AnyMethod.new get_tkread, name - meth = RDoc::AnyMethod.new '', name + meth = RDoc::AnyMethod.new get_tkread, name meth.singleton = single == SINGLE ? true : singleton record_location meth meth.line = line_no -=begin - meth.start_collecting_tokens # @token_stream = [] - indent = TkSPACE.new 0, 1, 1 - indent.set_text " " * column - - token = TkCOMMENT.new 0, line_no, 1 - token.set_text "# File #{@top_level.relative_name}, line #{line_no}" - meth.add_tokens [token, NEWLINE_TOKEN, indent] + meth.start_collecting_tokens + indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column } + token = { :line_no => line_no, :char_no => 1, :kind => :on_comment } + token[:text] = "# File #{@top_level.relative_name}, line #{line_no}" + newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" } + meth.add_tokens [token, newline, indent] meth.add_tokens @token_stream -=end parse_method_params_and_body container, single, meth, added_container @@ -1366,8 +1366,7 @@ def parse_method(container, single, tk, comment) # Parses the parameters and body of +meth+ def parse_method_params_and_body container, single, meth, added_container -# token_listener meth do -# @scanner.continue = false + token_listener meth do parse_method_parameters meth if meth.document_self or not @track_visibility then @@ -1390,7 +1389,7 @@ def parse_method_params_and_body container, single, meth, added_container end parse_statements container, single, meth -# end + end end ## @@ -1529,7 +1528,7 @@ def parse_method_or_yield_parameters(method = nil, end when :on_lparen then nest += 1 - when end_token then + when end_token[:kind] then if end_token[:kind] == :on_rparen nest -= 1 break if nest <= 0 @@ -1676,11 +1675,13 @@ def parse_statements(container, single = NORMAL, current_method = nil, end while :on_comment == tk[:kind] do - comment << tk[:text] << "\n" + comment << tk[:text] + comment << "\n" unless "\n" == tk[:text].chars.last + prev_tk = tk tk = get_tk - if :on_nl == tk[:kind] then + if :on_nl == tk[:kind] || "\n" == prev_tk[:text].chars.last then skip_tkspace false # leading spaces tk = get_tk end @@ -1812,8 +1813,8 @@ def parse_statements(container, single = NORMAL, current_method = nil, def parse_symbol_arg(no = nil) skip_tkspace_comment - case tk = get_tk - when TkLPAREN + tk = get_tk + if tk[:kind] == :on_lparen parse_symbol_arg_paren no else parse_symbol_arg_space no, tk @@ -2051,6 +2052,8 @@ def remove_private_comments comment # Scans this Ruby file for Ruby constructs def scan + reset + catch :eof do begin parse_top_level_statements @top_level @@ -2058,27 +2061,21 @@ def scan rescue StandardError => e bytes = '' - 20.times do @scanner.ungetc end - count = 0 - 60.times do |i| - count = i - byte = @scanner.getc - break unless byte - bytes << byte - end - count -= 20 - count.times do @scanner.ungetc end + now_line_no = peek_tk[:line_no] $stderr.puts <<-EOF -#{self.class} failure around line #{@scanner.line_no} of +#{self.class} failure around line #{now_line_no} of #{@file_name} EOF unless bytes.empty? then $stderr.puts - $stderr.puts bytes.inspect + now_line_no = peek_tk[:line_no] + start_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no } + end_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no + 1 } - 1 + $stderr.puts @scanner[start_index..end_index].join end raise e @@ -2150,8 +2147,8 @@ def skip_method container def skip_tkspace_comment(skip_nl = true) loop do skip_tkspace skip_nl - return unless TkCOMMENT === peek_tk - get_tk + return unless @scanner[@scanner_point][:kind] == :on_comment + @scanner_point += 1 end end diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb index dc6e657bb2..3e6173d91d 100644 --- a/lib/rdoc/parser/ruby_tools.rb +++ b/lib/rdoc/parser/ruby_tools.rb @@ -5,8 +5,6 @@ module RDoc::Parser::RubyTools - include RDoc::RubyToken - ## # Adds a token listener +obj+, but you should probably use token_listener @@ -23,11 +21,10 @@ def get_tk if @tokens.empty? then if @scanner_point >= @scanner.size - nil + return nil else tk = @scanner[@scanner_point] @scanner_point += 1 - tk = @scanner.token @read.push tk[:text] puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG end @@ -37,7 +34,38 @@ def get_tk puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG end - tk = nil if TkEND_OF_SCRIPT === tk + tk = nil if tk[:kind] == :on___end__ + + if tk[:kind] == :on_symbeg then + prev_line_no = tk[:line_no] + prev_char_no = tk[:char_no] + + is_symbol = true + symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol } + case (tk1 = get_tk)[:kind] + when :on_ident + symbol_tk[:text] = ":#{tk1[:text]}" + when :on_tstring_content + symbol_tk[:text] = ":#{tk1[:text]}" + get_tk # skip :on_tstring_end + when :on_tstring_end + symbol_tk[:text] = ":#{tk1[:text]}" + when :on_op + symbol_tk[:text] = ":#{tk1[:text]}" + #when :on_symbols_beg + #when :on_qsymbols_beg + else + is_symbol = false + tk = tk1 + end + if is_symbol + tk = symbol_tk + # remove the identifier we just read to replace it with a symbol + @token_listeners.each do |obj| + obj.pop_token + end if @token_listeners + end + end # inform any listeners of our shiny new token @token_listeners.each do |obj| diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 51b3be7a71..859ede3681 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -1658,22 +1658,20 @@ def test_parse_method assert_equal klass.current_section, foo.section stream = [ - tk(:COMMENT, 0, 1, 1, nil, - "# File #{@top_level.relative_name}, line 1"), - RDoc::Parser::Ruby::NEWLINE_TOKEN, - tk(:SPACE, 0, 1, 1, nil, ''), - tk(:DEF, 0, 1, 0, 'def', 'def'), - tk(:SPACE, 3, 1, 3, nil, ' '), - tk(:IDENTIFIER, 4, 1, 4, 'foo', 'foo'), - tk(:LPAREN, 7, 1, 7, nil, '('), - tk(:RPAREN, 8, 1, 8, nil, ')'), - tk(:SPACE, 9, 1, 9, nil, ' '), - tk(:COLON, 10, 1, 10, nil, ':'), - tk(:IDENTIFIER, 11, 1, 11, 'bar', 'bar'), - tk(:SPACE, 14, 1, 14, nil, ' '), - tk(:END, 15, 1, 15, 'end', 'end'), - ] - + { + :line_no => 1, :char_no => 1, :kind => :on_comment, + :text => "# File #{@top_level.relative_name}, line 1" }, + { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }, + { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' }, + { :line_no => 1, :char_no => 0, :kind => :on_kw, :text => 'def' }, + { :line_no => 1, :char_no => 3, :kind => :on_sp, :text => ' ' }, + { :line_no => 1, :char_no => 4, :kind => :on_ident, :text => 'foo' }, + { :line_no => 1, :char_no => 7, :kind => :on_lparen, :text => '(' }, + { :line_no => 1, :char_no => 8, :kind => :on_rparen, :text => ')' }, + { :line_no => 1, :char_no => 9, :kind => :on_sp, :text => ' ' }, + { :line_no => 1, :char_no => 10, :kind => :on_symbol, :text => ':bar' }, + { :line_no => 1, :char_no => 14, :kind => :on_sp, :text => ' ' }, + { :line_no => 1, :char_no => 15, :kind => :on_kw, :text => 'end' }] assert_equal stream, foo.token_stream end From 9eab3867ee626c1772087acf77e43da668f283a1 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 15 Jun 2017 12:16:14 +0900 Subject: [PATCH 012/151] Fix for test_parse_alias --- lib/rdoc/parser/ruby.rb | 30 ++++++++++++++---------------- lib/rdoc/parser/ruby_tools.rb | 6 ++++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 5a3941908c..913cc4ca31 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -524,23 +524,23 @@ def get_method_container container, name_t # :nodoc: def get_symbol_or_name tk = get_tk - case tk - when TkSYMBOL then - text = tk.text.sub(/^:/, '') + case tk[:kind] + when :on_symbol then + text = tk[:text].sub(/^:/, '') - if TkASSIGN === peek_tk then + next_tk = peek_tk + if next_tk && :on_op == next_tk[:kind] && '=' == next_tk[:text] then get_tk text << '=' end text - when TkId, TkOp then - tk.name - when TkAMPER, - TkDSTRING, - TkSTAR, - TkSTRING then - tk.text + when :on_ident, :on_op then + tk[:text] + when :on_tstring_beg then + tk = get_tk # :on_tstring_content + get_tk # skip :on_tstring_end + tk[:text] else raise RDoc::Error, "Name or symbol expected (got #{tk})" end @@ -667,21 +667,19 @@ def parse_attr_accessor(context, single, tk, comment) # Parses an +alias+ in +context+ with +comment+ def parse_alias(context, single, tk, comment) - line_no = tk.line_no + line_no = tk[:line_no] skip_tkspace - if TkLPAREN === peek_tk then + if :on_lparen === peek_tk[:kind] then get_tk skip_tkspace end new_name = get_symbol_or_name - @scanner.lex_state = :EXPR_FNAME - skip_tkspace - if TkCOMMA === peek_tk then + if :on_comma === peek_tk[:kind] then get_tk skip_tkspace end diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb index 3e6173d91d..a4d0f00436 100644 --- a/lib/rdoc/parser/ruby_tools.rb +++ b/lib/rdoc/parser/ruby_tools.rb @@ -34,9 +34,11 @@ def get_tk puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG end - tk = nil if tk[:kind] == :on___end__ + if tk == nil || :on___end__ == tk[:kind] + tk = nil + end - if tk[:kind] == :on_symbeg then + if tk && :on_symbeg == tk[:kind] then prev_line_no = tk[:line_no] prev_char_no = tk[:char_no] From 32eec2a9e8672656739519ad1f9bdfa2aa1223cf Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 15 Jun 2017 12:16:37 +0900 Subject: [PATCH 013/151] Fix for rdoc command running --- lib/rdoc/parser/ruby.rb | 6 +++++- lib/rdoc/token_stream.rb | 10 +--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 913cc4ca31..701e26816e 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -2059,7 +2059,11 @@ def scan rescue StandardError => e bytes = '' - now_line_no = peek_tk[:line_no] + if @scanner_point >= @scanner.size + now_line_no = @scanner[@scanner.size - 1][:line_no] + else + now_line_no = peek_tk[:line_no] + end $stderr.puts <<-EOF diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index 0cfa2f1384..c0b990b341 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -45,15 +45,7 @@ def self.to_html token_stream when RDoc::RubyToken::TkVal then 'ruby-value' end - comment_with_nl = false - case t - when RDoc::RubyToken::TkRD_COMMENT, RDoc::RubyToken::TkHEREDOCEND - comment_with_nl = true if t.text =~ /\n$/ - text = t.text.rstrip - else - text = t.text - end - text = CGI.escapeHTML text + text = CGI.escapeHTML t[:text] if style then "#{text}#{"\n" if comment_with_nl}" From 5021a1b13c58e3d854becb9cbc5a1f95c703e7b3 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 18 Jun 2017 20:32:42 +0900 Subject: [PATCH 014/151] Fix for test_parse_method_nil --- lib/rdoc/parser/ruby.rb | 60 +++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 701e26816e..119bf07e5e 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1454,47 +1454,43 @@ def parse_method_name_regular container, name_t # :nodoc: # for a singleton method. def parse_method_name_singleton container, name_t # :nodoc: - @scanner.lex_state = :EXPR_FNAME skip_tkspace name_t2 = get_tk - name = - case name_t - when TkSELF, TkMOD then - case name_t2 - # NOTE: work around '[' being consumed early and not being re-tokenized - # as a TkAREF - when TkfLBRACK then - get_tk - '[]' - else - name_t2.name - end - when TkCONSTANT then - name = name_t2.name + if (:on_kw == name_t[:kind] && 'self' == name_t[:text]) || (:on_op == name_t[:kind] && '%' == name_t[:text]) then + # NOTE: work around '[' being consumed early and not being re-tokenized + # as a TkAREF + if :on_lbracket == name_t2[:kind] + get_tk + name = '[]' + else + name = name_t2[:text] + end + elsif :on_const == name_t[:kind] then + name = name_t2[:text] - container = get_method_container container, name_t + container = get_method_container container, name_t - return unless container + return unless container - name - when TkIDENTIFIER, TkIVAR, TkGVAR then - parse_method_dummy container + name + elsif :on_ident == name_t[:kind] || :on_ivar == name_t[:kind] || :on_gvar == name_t[:kind] then + parse_method_dummy container - nil - when TkTRUE, TkFALSE, TkNIL then - klass_name = "#{name_t.name.capitalize}Class" - container = @store.find_class_named klass_name - container ||= @top_level.add_class RDoc::NormalClass, klass_name + name = nil + elsif (:on_kw == name_t[:kind]) && ('true' == name_t[:text] || 'false' == name_t[:text] || 'nil' == name_t[:text]) then + klass_name = "#{name_t[:text].capitalize}Class" + container = @store.find_class_named klass_name + container ||= @top_level.add_class RDoc::NormalClass, klass_name - name_t2.name - else - warn "unexpected method name token #{name_t.inspect}" - # break - skip_method container + name = name_t2[:text] + else + warn "unexpected method name token #{name_t.inspect}" + # break + skip_method container - nil - end + name = nil + end return name, container end From abc30fd03d4eff33662d7125744fe185dcbc3c88 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 18 Jun 2017 23:28:03 +0900 Subject: [PATCH 015/151] Fix for test_parse_multi_ghost_methods --- lib/rdoc/parser/ruby.rb | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 119bf07e5e..988600cbc8 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1026,12 +1026,11 @@ def parse_comment_ghost container, text, name, column, line_no, # :nodoc: record_location meth meth.start_collecting_tokens - indent = TkSPACE.new 0, 1, 1 - indent.set_text " " * column - - position_comment = TkCOMMENT.new 0, line_no, 1 - position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}" - meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] + indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column } + position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment } + position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" + newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" } + meth.add_tokens [position_comment, newline, indent] meth.params = if text.sub!(/^#\s+:?args?:\s*(.*?)\s*$/i, '') then @@ -1672,13 +1671,10 @@ def parse_statements(container, single = NORMAL, current_method = nil, comment << tk[:text] comment << "\n" unless "\n" == tk[:text].chars.last - prev_tk = tk - tk = get_tk - - if :on_nl == tk[:kind] || "\n" == prev_tk[:text].chars.last then + if tk[:text].size > 1 && "\n" == tk[:text].chars.last then skip_tkspace false # leading spaces - tk = get_tk end + tk = get_tk end comment = new_comment comment From 352f5edbd2cc9b07f71ab219a8f711b667616050 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 18 Jun 2017 23:28:21 +0900 Subject: [PATCH 016/151] Fix for test_parse_method_constant --- lib/rdoc/parser/ruby.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 988600cbc8..f48d400f9c 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -481,11 +481,11 @@ def get_end_token tk # :nodoc: def get_method_container container, name_t # :nodoc: prev_container = container - container = container.find_module_named(name_t.name) + container = container.find_module_named(name_t[:text]) unless container then constant = prev_container.constants.find do |const| - const.name == name_t.name + const.name == name_t[:text] end if constant then From 829fdbac0e8097546c5286ae47445f027e4d16b6 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 19 Jun 2017 00:10:23 +0900 Subject: [PATCH 017/151] Fix for test_parse_method_toplevel --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index f48d400f9c..64fb97a5cd 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1526,7 +1526,7 @@ def parse_method_or_yield_parameters(method = nil, nest -= 1 break if nest <= 0 else - break unless @scanner.continue + break end when :on_rparen then nest -= 1 From 1a2b6012c82b519cafdb3188f726bd8e7a2c20f6 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 19 Jun 2017 01:19:03 +0900 Subject: [PATCH 018/151] Fix test_parse_statements_def_percent_string_pound for Ripper --- test/test_rdoc_parser_ruby.rb | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 859ede3681..646978dd07 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -2116,19 +2116,23 @@ def test_parse_statements_def_percent_string_pound assert_equal 2, x.method_list.length a = x.method_list.first + expected = [ - tk(:COMMENT, 0, 2, 1, nil, "# File #{@filename}, line 2"), - tk(:NL, 0, 0, 0, nil, "\n"), - tk(:SPACE, 0, 1, 1, nil, ''), - tk(:DEF, 8, 2, 0, 'def', 'def'), - tk(:SPACE, 11, 2, 3, nil, ' '), - tk(:IDENTIFIER, 12, 2, 4, 'a', 'a'), - tk(:NL, 13, 2, 5, nil, "\n"), - tk(:REGEXP, 14, 3, 0, nil, '%r{#}'), - tk(:NL, 19, 3, 5, nil, "\n"), - tk(:DREGEXP, 20, 4, 0, nil, '%r{#{}}'), - tk(:NL, 27, 4, 7, nil, "\n"), - tk(:END, 28, 5, 0, 'end', 'end'), + { + :line_no => 2, :char_no => 1, :kind => :on_comment, + :text => "# File #{@filename}, line 2" + }, + { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }, + { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' }, + { :line_no => 2, :char_no => 0, :kind => :on_kw, :text => 'def' }, + { :line_no => 2, :char_no => 3, :kind => :on_sp, :text => ' ' }, + { :line_no => 2, :char_no => 4, :kind => :on_ident, :text => 'a' }, + { :line_no => 2, :char_no => 5, :kind => :on_nl, :text => "\n" }, + { :line_no => 3, :char_no => 0, :kind => :on_regexp_beg, :text => '%r{' }, + { :line_no => 3, :char_no => 3, :kind => :on_tstring_content, :text => '#' }, + { :line_no => 3, :char_no => 4, :kind => :on_regexp_end, :text => '}' }, + { :line_no => 3, :char_no => 5, :kind => :on_nl, :text => "\n" }, + { :line_no => 4, :char_no => 0, :kind => :on_kw, :text => 'end' } ] assert_equal expected, a.token_stream From d94a38de2eb4fa9c66004320edecde3326b9a343 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 19 Jun 2017 03:58:38 +0900 Subject: [PATCH 019/151] Fix for test_parse_method_gvar_insane --- lib/rdoc/parser/ruby.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 64fb97a5cd..4814e8c0dc 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -385,9 +385,11 @@ def get_class_or_module container, ignore_constants = false # Return a superclass, which can be either a constant of an expression def get_class_specification - case peek_tk - when TkSELF then return 'self' - when TkGVAR then return '' + tk = peek_tk + if :on_kw == tk[:kind] && 'self' == tk[:text] + return 'self' + elsif :on_gvar == tk[:kind] + return '' end res = get_constant @@ -398,8 +400,8 @@ def get_class_specification tk = get_tk - case tk - when TkNL, TkCOMMENT, TkSEMICOLON then + case tk[:kind] + when :on_nl, :on_comment, :on_semicolon then unget_tk(tk) return res end From e2d63ffeda1f0019f53e45825cc2769f1dd4a5e5 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 19 Jun 2017 04:23:44 +0900 Subject: [PATCH 020/151] Fix for test_parse_method_parameters_comment --- lib/rdoc/parser/ruby.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 4814e8c0dc..4d47f2ea88 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1538,6 +1538,9 @@ def parse_method_or_yield_parameters(method = nil, @read.pop when :on_comment then @read.pop + when :on_kw + unget_tk tk + break end tk = get_tk end From 83b5b7817199a810f71ac07a3a7bce166c5b4a02 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 19 Jun 2017 04:41:47 +0900 Subject: [PATCH 021/151] Fix for test_collect_first_comment --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 4d47f2ea88..d99e4b723e 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -256,7 +256,7 @@ def collect_first_comment first_comment_tk_kind = tk[:kind] first_line = false - comment << tk[:text] << "\n" + comment << tk[:text] tk = get_tk if :on_nl === tk then From f6111fdc50d4617bbbbcd77894e9a909ca0a4022 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 19 Jun 2017 05:59:53 +0900 Subject: [PATCH 022/151] Fix for test_parse_method_toplevel_class --- lib/rdoc/parser/ruby.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index d99e4b723e..e8da432b12 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -498,21 +498,21 @@ def get_method_container container, name_t # :nodoc: unless container then # TODO seems broken, should starting at Object in @store - obj = name_t.name.split("::").inject(Object) do |state, item| + obj = name_t[:text].split("::").inject(Object) do |state, item| state.const_get(item) end rescue nil type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule unless [Class, Module].include?(obj.class) then - warn("Couldn't find #{name_t.name}. Assuming it's a module") + warn("Couldn't find #{name_t[:text]}. Assuming it's a module") end if type == RDoc::NormalClass then sclass = obj.superclass ? obj.superclass.name : nil - container = prev_container.add_class type, name_t.name, sclass + container = prev_container.add_class type, name_t[:text], sclass else - container = prev_container.add_module type, name_t.name + container = prev_container.add_module type, name_t[:text] end record_location container @@ -592,7 +592,8 @@ def look_for_directives_in context, comment def make_message message prefix = "#{@file_name}:" - prefix << "#{@scanner.line_no}:#{@scanner.char_no}:" if @scanner + tk = peek_tk + prefix << "#{tk[:line_no]}:#{tk[:char_no]}:" if tk "#{prefix} #{message}" end From d10dcc5d52316bd134b7b17f7bdf95d1a7310cf4 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 19 Jun 2017 06:10:00 +0900 Subject: [PATCH 023/151] Fix for test_parse_meta_method_name --- lib/rdoc/parser/ruby.rb | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index e8da432b12..84aab0baab 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1215,8 +1215,8 @@ def parse_meta_attr(context, single, tk, comment) # Parses a meta-programmed method def parse_meta_method(container, single, tk, comment) - column = tk.char_no - line_no = tk.line_no + column = tk[:char_no] + line_no = tk[:line_no] start_collecting_tokens add_token tk @@ -1238,12 +1238,11 @@ def parse_meta_method(container, single, tk, comment) remove_token_listener self meth.start_collecting_tokens - indent = TkSPACE.new 0, 1, 1 - indent.set_text " " * column - - position_comment = TkCOMMENT.new 0, line_no, 1 - position_comment.value = "# File #{@top_level.relative_name}, line #{line_no}" - meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] + indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column } + position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment } + position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" + newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" } + meth.add_tokens [position_comment, newline, indent] meth.add_tokens @token_stream parse_meta_method_params container, single, meth, tk, comment @@ -1297,14 +1296,13 @@ def parse_meta_method_params container, single, meth, tk, comment # :nodoc: last_tk = tk while tk = get_tk do - case tk - when TkSEMICOLON then + if :on_semicolon == tk[:kind] then break - when TkNL then - break unless last_tk and TkCOMMA === last_tk - when TkSPACE then + elsif :on_nl == tk[:kind] then + break unless last_tk and :on_comma == last_tk[:kind] + elsif :on_sp == tk[:kind] then # expression continues - when TkDO then + elsif :on_op == tk[:kind] && 'do' == tk[:text] then parse_statements container, single, meth break else From b9d53b2d5a32794b970614dd7a326d1872a90bdd Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 19 Jun 2017 06:24:05 +0900 Subject: [PATCH 024/151] Fix for test_parse_meta_method_string_name --- lib/rdoc/parser/ruby.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 84aab0baab..eef8ac17cc 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1266,17 +1266,18 @@ def parse_meta_method_name comment, tk # :nodoc: name_t = get_tk - case name_t - when TkSYMBOL then - name_t.text[1..-1] - when TkSTRING then - name_t.value[1..-2] - when TkASSIGN then # ignore + if :on_symbol == name_t[:kind] then + name_t[:text][1..-1] + elsif :on_tstring_beg then + name_t = get_tk # :on_tstring_content + get_tk # skip :on_tstring_end + name_t[:text] + elsif :on_op == name_t[:kind] && '=' == name_t[:text] then # ignore remove_token_listener self nil else - warn "unknown name token #{name_t.inspect} for meta-method '#{tk.name}'" + warn "unknown name token #{name_t.inspect} for meta-method '#{tk[:text]}'" 'unknown' end end From a1043818511c23cefae058567f41fc2dcde28e09 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 19 Jun 2017 06:31:11 +0900 Subject: [PATCH 025/151] Fix test_parse_meta_method for Ripper --- test/test_rdoc_parser_ruby.rb | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 646978dd07..504544093a 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -1461,17 +1461,19 @@ def test_parse_meta_method assert_equal klass.current_section, foo.section stream = [ - tk(:COMMENT, 0, 1, 1, nil, - "# File #{@top_level.relative_name}, line 1"), - RDoc::Parser::Ruby::NEWLINE_TOKEN, - tk(:SPACE, 0, 1, 1, nil, ''), - tk(:IDENTIFIER, 0, 1, 0, 'add_my_method', 'add_my_method'), - tk(:SPACE, 0, 1, 13, nil, ' '), - tk(:SYMBOL, 0, 1, 14, nil, ':foo'), - tk(:COMMA, 0, 1, 18, nil, ','), - tk(:SPACE, 0, 1, 19, nil, ' '), - tk(:SYMBOL, 0, 1, 20, nil, ':bar'), - tk(:NL, 0, 1, 24, nil, "\n"), + { + :line_no => 1, :char_no => 1, :kind => :on_comment, + :text => "# File #{@top_level.relative_name}, line 1" + }, + { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }, + { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' }, + { :line_no => 1, :char_no => 0, :kind => :on_ident, :text => 'add_my_method' }, + { :line_no => 1, :char_no => 13, :kind => :on_sp, :text => ' ' }, + { :line_no => 1, :char_no => 14, :kind => :on_symbol, :text => ':foo' }, + { :line_no => 1, :char_no => 18, :kind => :on_comma, :text => ',' }, + { :line_no => 1, :char_no => 19, :kind => :on_sp, :text => ' ' }, + { :line_no => 1, :char_no => 20, :kind => :on_symbol, :text => ':bar' }, + { :line_no => 1, :char_no => 24, :kind => :on_nl, :text => "\n" } ] assert_equal stream, foo.token_stream From 55a18bbcadfd4bc265e812f9840237e068d38b66 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 20 Jun 2017 06:45:35 +0900 Subject: [PATCH 026/151] Fix for test_parse_meta_method_block --- lib/rdoc/parser/ruby.rb | 2 +- test/test_rdoc_parser_ruby.rb | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index eef8ac17cc..23276dc89f 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1303,7 +1303,7 @@ def parse_meta_method_params container, single, meth, tk, comment # :nodoc: break unless last_tk and :on_comma == last_tk[:kind] elsif :on_sp == tk[:kind] then # expression continues - elsif :on_op == tk[:kind] && 'do' == tk[:text] then + elsif :on_kw == tk[:kind] && 'do' == tk[:text] then parse_statements container, single, meth break else diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 504544093a..dedda7a6b9 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -1497,7 +1497,8 @@ def test_parse_meta_method_block @parser.parse_meta_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment - assert_equal tk(:NL, 0, 3, 3, 3, "\n"), @parser.get_tk + rest = { :line_no => 3, :char_no => 3, :kind => :on_nl, :text => "\n" } + assert_equal rest, @parser.get_tk end def test_parse_meta_method_define_method From 6fe9b924a021369733fb91265a1716af49cb5aac Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 20 Jun 2017 06:53:19 +0900 Subject: [PATCH 027/151] Fix for test_parse_meta_method_unknown --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 23276dc89f..f4f55e075d 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1268,7 +1268,7 @@ def parse_meta_method_name comment, tk # :nodoc: if :on_symbol == name_t[:kind] then name_t[:text][1..-1] - elsif :on_tstring_beg then + elsif :on_tstring_beg == name_t[:kind] then name_t = get_tk # :on_tstring_content get_tk # skip :on_tstring_end name_t[:text] From e37e73e3d2be6ed8608e6220a925e48e502f5a77 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 20 Jun 2017 07:44:01 +0900 Subject: [PATCH 028/151] Fix for test_parse_class_single --- lib/rdoc/parser/ruby.rb | 66 ++++++++++++++--------------------- lib/rdoc/parser/ruby_tools.rb | 4 +++ 2 files changed, 31 insertions(+), 39 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index f4f55e075d..25be3fcf42 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -419,8 +419,8 @@ def get_constant skip_tkspace false tk = get_tk - while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do - res += tk.name + while (:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind] do + res += tk[:text] tk = get_tk end @@ -709,34 +709,38 @@ def parse_alias(context, single, tk, comment) # Extracts call parameters from the token stream. def parse_call_parameters(tk) - end_token = case tk - when TkLPAREN, TkfLPAREN - TkRPAREN - when TkRPAREN + end_token = case tk[:kind] + when :on_lparen + :on_rparen + when :on_rparen return "" else - TkNL + :on_nl end nest = 0 loop do - case tk - when TkSEMICOLON + break if tk.nil? + case tk[:kind] + when :on_semicolon break - when TkLPAREN, TkfLPAREN + when :on_lparen nest += 1 when end_token - if end_token == TkRPAREN + if end_token == :on_rparen nest -= 1 - break if @scanner.lex_state == :EXPR_END and nest <= 0 + break if lex_end? and nest <= 0 else - break unless @scanner.continue + break if lex_end? end - when TkCOMMENT, TkASSIGN, TkOPASGN + when :on_comment unget_tk(tk) break - when nil then - break + when :on_op + if tk[:text] =~ /^(.{1,2})?=$/ + unget_tk(tk) + break + end end tk = get_tk end @@ -860,49 +864,33 @@ def parse_class_singleton container, name, comment # :nodoc: # true, no found constants will be added to RDoc. def parse_constant container, tk, comment, ignore_constants = false - line_no = tk.line_no + line_no = tk[:line_no] - name = tk.name + name = tk[:text] skip_tkspace false return unless name =~ /^\w+$/ eq_tk = get_tk - if TkCOLON2 === eq_tk then + if :on_op == eq_tk[:kind] && '::' == eq_tk[:text] then unget_tk eq_tk unget_tk tk container, name_t, = get_class_or_module container, ignore_constants - name = name_t.name + name = name_t[:text] eq_tk = get_tk end - is_array_or_hash = false - if TkfLBRACK === eq_tk - nest = 1 - while bracket_tk = get_tk - case bracket_tk - when TkfLBRACK, TkLBRACK - nest += 1 - when TkRBRACK - nest -= 1 - break if nest == 0 - end - end - skip_tkspace false - eq_tk = get_tk - is_array_or_hash = true - end - - unless TkASSIGN === eq_tk then + unless :on_op == eq_tk[:kind] && '=' == eq_tk[:text] then unget_tk eq_tk return false end - if TkGT === peek_tk then + check_tk = peek_tk + if :on_op == check_tk[:kind] && '>' == check_tk[:text] then unget_tk eq_tk return end diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb index a4d0f00436..02ef4f257a 100644 --- a/lib/rdoc/parser/ruby_tools.rb +++ b/lib/rdoc/parser/ruby_tools.rb @@ -13,6 +13,10 @@ def add_token_listener(obj) @token_listeners << obj end + def lex_end? + @scanner_point >= @scanner.size + end + ## # Fetches the next token from the scanner From 9d677e805de870883d36a3b02e5a87683bc1876c Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 20 Jun 2017 11:23:20 +0900 Subject: [PATCH 029/151] Fix for test_parse_class_single_root --- lib/rdoc/parser/ruby.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 25be3fcf42..89d894e0bf 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -352,7 +352,8 @@ def get_class_or_module container, ignore_constants = false skip_tkspace false given_name << name_t[:text] - while (tk = peek_tk) and :on_op == tk[:kind] and '::' == tk[:text] do + is_self = name_t[:kind] == :on_op && name_t[:text] == '<<' + while !is_self && (tk = peek_tk) and :on_op == tk[:kind] and '::' == tk[:text] do prev_container = container container = container.find_module_named name_t[:text] container ||= From b7ec322711752d23034474f6d903a7a2d50487c5 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 20 Jun 2017 18:08:06 +0900 Subject: [PATCH 030/151] Fix for test_parse_constant.* --- lib/rdoc/parser/ruby.rb | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 89d894e0bf..4182dcb0d6 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -925,15 +925,16 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: tk = get_tk loop do - case tk - when TkSEMICOLON then + break if tk.nil? + if :on_semicolon == tk[:kind] then break if nest <= 0 - when TkLPAREN, TkfLPAREN, TkLBRACE, TkfLBRACE, TkLBRACK, TkfLBRACK, - TkDO, TkIF, TkUNLESS, TkCASE, TkDEF, TkBEGIN then + elsif [:on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) || + (:on_kw == tk[:kind] && %w{do if unless case def begin}.include?(tk[:text])) then nest += 1 - when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then + elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) || + :on_kw == tk[:kind] && 'end' == tk[:text] then nest -= 1 - when TkCOMMENT then + elsif :on_comment == tk[:kind] then if nest <= 0 and stop_at_EXPR_END then unget_tk tk break @@ -941,24 +942,21 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: unget_tk tk read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS end - when TkCONSTANT then - rhs_name << tk.name + elsif :on_const == tk[:kind] then + rhs_name << tk[:text] - if nest <= 0 and TkNL === peek_tk then - create_module_alias container, constant, rhs_name unless is_array_or_hash + next_tk = peek_tk + if nest <= 0 and (next_tk.nil? || :on_nl == next_tk[:kind]) then + create_module_alias container, constant, rhs_name break end - when TkNL then + elsif :on_nl == tk[:kind] then if nest <= 0 and stop_at_EXPR_END then unget_tk tk break end - when TkCOLON2, TkCOLON3 then + elsif :on_op == tk[:kind] && '::' == tk[:text] rhs_name << '::' - when TkBACKSLASH then - get_tk if TkNL === peek_tk - when nil then - break end tk = get_tk end From 7039be07dc7af93c5e5548455470ed6303c3e3b1 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 20 Jun 2017 18:28:02 +0900 Subject: [PATCH 031/151] Fix for test_parse_meta_attr.* --- lib/rdoc/parser/ruby.rb | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 4182dcb0d6..b6aa827c69 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1848,7 +1848,7 @@ def parse_symbol_arg_space no, tk # :nodoc: skip_tkspace false tk1 = get_tk - unless TkCOMMA === tk1 then + if tk1.nil? || :on_comma != tk1[:kind] then unget_tk tk1 break end @@ -1867,12 +1867,16 @@ def parse_symbol_arg_space no, tk # :nodoc: # Returns symbol text from the next token def parse_symbol_in_arg - case tk = get_tk - when TkSYMBOL - tk.text.sub(/^:/, '') - when TkSTRING - eval @read[-1] - when TkDSTRING, TkIDENTIFIER then + tk = get_tk + if :on_symbol == tk[:kind] + tk[:text].sub(/^:/, '') + #elsif TkSTRING + # eval @read[-1] + elsif :on_ident == tk[:kind] then + nil # ignore + elsif :on_tstring_beg == tk[:kind] then + get_tk # skip :on_tstring_content + get_tk # skip :on_tstring_end nil # ignore else warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC @@ -2133,8 +2137,9 @@ def skip_method container def skip_tkspace_comment(skip_nl = true) loop do skip_tkspace skip_nl - return unless @scanner[@scanner_point][:kind] == :on_comment - @scanner_point += 1 + next_tk = peek_tk + return if next_tk.nil? || next_tk[:kind] != :on_comment + get_tk end end From 789b3cfab1fde90b26572a00cbea4e3410da279b Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 20 Jun 2017 18:31:59 +0900 Subject: [PATCH 032/151] Fix for test_parse_attr.* --- lib/rdoc/parser/ruby.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index b6aa827c69..8c640d7a76 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -321,10 +321,9 @@ def error(msg) def get_bool skip_tkspace tk = get_tk - case tk - when TkTRUE + if :on_kw == tk[:kind] && 'true' == tk[:text] true - when TkFALSE, TkNIL + elsif :on_kw == tk[:kind] && ('false' == tk[:text] || 'nil' == tk[:text]) false else unget_tk tk @@ -622,7 +621,7 @@ def parse_attr(context, single, tk, comment) skip_tkspace false tk = get_tk - if TkCOMMA === tk then + if :on_comma == tk[:kind] then rw = "RW" if get_bool else unget_tk tk @@ -642,7 +641,7 @@ def parse_attr(context, single, tk, comment) # comment for each to +comment+. def parse_attr_accessor(context, single, tk, comment) - line_no = tk.line_no + line_no = tk[:line_no] args = parse_symbol_arg rw = "?" @@ -653,7 +652,7 @@ def parse_attr_accessor(context, single, tk, comment) # and add found items appropriately but here we do not. I'm not sure why. return if @track_visibility and not tmp.document_self - case tk.name + case tk[:text] when "attr_reader" then rw = "R" when "attr_writer" then rw = "W" when "attr_accessor" then rw = "RW" From d967b497d9b9266e749c7b098a2fb1398bd6256a Mon Sep 17 00:00:00 2001 From: Code Ass Date: Wed, 21 Jun 2017 23:20:04 +0900 Subject: [PATCH 033/151] Fix test_parse_comment_method for Ripper --- test/test_rdoc_parser_ruby.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index dedda7a6b9..9783a36bb8 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -1194,10 +1194,12 @@ def test_parse_comment_method assert_equal klass.current_section, foo.section stream = [ - tk(:COMMENT, 0, 1, 1, nil, - "# File #{@top_level.relative_name}, line 1"), - RDoc::Parser::Ruby::NEWLINE_TOKEN, - tk(:SPACE, 0, 1, 1, nil, ''), + { + :line_no => 1, :char_no => 1, :kind => :on_comment, + :text => "# File #{@top_level.relative_name}, line 1" + }, + { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }, + { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' } ] assert_equal stream, foo.token_stream From 36f6e014acdbc488196a73d75fb0bbe60cec36dd Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 01:00:03 +0900 Subject: [PATCH 034/151] Fix for test_sanity_integer --- lib/rdoc/parser/ruby.rb | 32 ++++++++++++++++++-------------- test/test_rdoc_parser_ruby.rb | 4 ++-- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 8c640d7a76..d3334687cf 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -207,7 +207,7 @@ def get_tkread_clean pattern, replacement # :nodoc: # methods. def get_visibility_information tk, single # :nodoc: - vis_type = tk.name + vis_type = tk[:text] singleton = single == SINGLE vis = @@ -436,7 +436,7 @@ def get_constant_with_optional_parens nest = 0 - while TkLPAREN === (tk = peek_tk) or TkfLPAREN === tk do + while :on_lparen == (tk = peek_tk)[:kind] do get_tk skip_tkspace nest += 1 @@ -447,7 +447,7 @@ def get_constant_with_optional_parens while nest > 0 skip_tkspace tk = get_tk - nest -= 1 if TkRPAREN === tk + nest -= 1 if :on_rparen == tk[:kind] end name @@ -1093,7 +1093,7 @@ def parse_extend_or_include klass, container, comment # :nodoc: record_location obj end - return unless TkCOMMA === peek_tk + return unless :on_comma == peek_tk[:kind] get_tk end @@ -1584,12 +1584,16 @@ def parse_require(context, comment) skip_tkspace_comment tk = get_tk - if TkLPAREN === tk then + if :on_lparen == tk[:kind] then skip_tkspace_comment tk = get_tk end - name = tk.text if TkSTRING === tk + if :on_tstring_beg == tk[:kind] then + tk = get_tk # :on_tstring_content + get_tk # skip :on_tstring_end + name = tk[:text] + end if name then @top_level.add_require RDoc::Require.new(name, comment) @@ -1909,15 +1913,15 @@ def parse_visibility(container, single, tk) skip_tkspace_comment false - case peek_tk - # Ryan Davis suggested the extension to ignore modifiers, because he - # often writes - # - # protected unless $TESTING - # - when TkNL, TkUNLESS_MOD, TkIF_MOD, TkSEMICOLON then + ptk = peek_tk + # Ryan Davis suggested the extension to ignore modifiers, because he + # often writes + # + # protected unless $TESTING + # + if [:on_nl, :on_semicolon].include?(ptk[:kind]) || (:on_kw == ptk[:kind] && (['if', 'unless'].include?(ptk[:text]))) then container.ongoing_visibility = vis - when TkDEF + elsif :on_kw == ptk[:kind] && 'def' == ptk[:text] container.current_line_visibility = vis else update_visibility container, vis_type, vis, singleton diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 9783a36bb8..070e09555b 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -3016,10 +3016,10 @@ def test_read_documentation_modifiers_not_new def test_sanity_integer util_parser '1' - assert_equal '1', @parser.get_tk.text + assert_equal '1', @parser.get_tk[:text] util_parser '1.0' - assert_equal '1.0', @parser.get_tk.text + assert_equal '1.0', @parser.get_tk[:text] end def test_sanity_interpolation From 11a67a47a7092d8d95ae185cb259ff96637c6477 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 01:02:33 +0900 Subject: [PATCH 035/151] Fix for test_scan_tomdoc_meta --- lib/rdoc/parser/ruby.rb | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index d3334687cf..c888f7e62c 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1048,8 +1048,8 @@ def parse_comment_ghost container, text, name, column, line_no, # :nodoc: def parse_comment_tomdoc container, tk, comment return unless signature = RDoc::TomDoc.signature(comment) - column = tk.char_no - line_no = tk.line_no + column = tk[:char_no] + line_no = tk[:line_no] name, = signature.split %r%[ \(]%, 2 @@ -1058,12 +1058,11 @@ def parse_comment_tomdoc container, tk, comment meth.line = line_no meth.start_collecting_tokens - indent = TkSPACE.new 0, 1, 1 - indent.set_text " " * column - - position_comment = TkCOMMENT.new 0, line_no, 1 - position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}" - meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] + indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column } + position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment } + position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" + newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" } + meth.add_tokens [position_comment, newline, indent] meth.call_seq = signature From 8b98cf7e6afc205936bc14d34575f35616e61dac Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 01:07:57 +0900 Subject: [PATCH 036/151] Fix for test_scan_constant_in_rescue --- lib/rdoc/parser/ruby.rb | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index c888f7e62c..a4689d9541 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1609,18 +1609,13 @@ def parse_rescue nest = 0 while tk = get_tk - case tk - when TkNL, TkSEMICOLON then + case tk[:kind] + when :on_nl, :on_semicolon then break - when TkCOMMA then + when :on_comma then skip_tkspace false - get_tk if TkNL === peek_tk - when TkLPAREN, TkfLPAREN then - nest += 1 - when TkRPAREN then - nest -= 1 - break if nest < 0 + get_tk if :on_nl == peek_tk[:kind] end skip_tkspace false From 8cdad9d9bdb6a13fec08fdd3cabfc5820fc3366b Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 01:57:39 +0900 Subject: [PATCH 037/151] Remove unnecessary condition --- lib/rdoc/parser/ruby_tools.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb index 02ef4f257a..91489ee874 100644 --- a/lib/rdoc/parser/ruby_tools.rb +++ b/lib/rdoc/parser/ruby_tools.rb @@ -42,7 +42,9 @@ def get_tk tk = nil end - if tk && :on_symbeg == tk[:kind] then + return nil unless tk + + if :on_symbeg == tk[:kind] then prev_line_no = tk[:line_no] prev_char_no = tk[:char_no] From f8e44b64ca971bd37207a9f1b51a81e2eaff6688 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 02:28:12 +0900 Subject: [PATCH 038/151] Fix for test_sanity_interpolation_{curly,crazy} --- lib/rdoc/parser/ruby_tools.rb | 14 ++++++++++++++ test/test_rdoc_parser_ruby.rb | 8 ++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb index 91489ee874..5049fa29f3 100644 --- a/lib/rdoc/parser/ruby_tools.rb +++ b/lib/rdoc/parser/ruby_tools.rb @@ -73,6 +73,20 @@ def get_tk obj.pop_token end if @token_listeners end + elsif :on_tstring_beg == tk[:kind] then + string = tk[:text] + loop do + inner_str_tk = get_tk + if inner_str_tk.nil? + break + elsif :on_tstring_end == inner_str_tk[:kind] + string = string + inner_str_tk[:text] + break + else + string = string + inner_str_tk[:text] + end + end + tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_tstring, :text => string } end # inform any listeners of our shiny new token diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 070e09555b..2519046db8 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -3036,15 +3036,15 @@ def test_sanity_interpolation def test_sanity_interpolation_crazy util_parser '"#{"#{"a")}" if b}"' - assert_equal '"#{"#{"a")}" if b}"', @parser.get_tk.text - assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class + assert_equal '"#{"#{"a")}" if b}"', @parser.get_tk[:text] + assert_equal nil, @parser.get_tk end def test_sanity_interpolation_curly util_parser '%{ #{} }' - assert_equal '%{ #{} }', @parser.get_tk.text - assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class + assert_equal '%{ #{} }', @parser.get_tk[:text] + assert_equal nil, @parser.get_tk end def test_sanity_interpolation_format From 2ff8110ed3407758d92f5d1ecc5ac60fa6564410 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 02:32:09 +0900 Subject: [PATCH 039/151] Fix test_read_directive --- test/test_rdoc_parser_ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 2519046db8..531e06379e 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -2930,7 +2930,7 @@ def test_read_directive assert_equal 'category', directive assert_equal 'test', value - assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk + assert_equal nil, parser.get_tk end def test_read_directive_allow From d85909e71c92d72a98858c7494c03fdc7a871121 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 02:42:24 +0900 Subject: [PATCH 040/151] Fix test string to parse for Ripper --- test/test_rdoc_parser_ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 531e06379e..e2a6400c2d 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -2964,7 +2964,7 @@ def test_read_directive_no_comment end def test_read_directive_one_liner - parser = util_parser '; end # :category: test' + parser = util_parser '; def foo; end # :category: test' directive, value = parser.read_directive %w[category] From 9bfac9daada71aae922ba16680e73bad23904673 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 02:44:13 +0900 Subject: [PATCH 041/151] Fix test_read_directive_.* --- test/test_rdoc_parser_ruby.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index e2a6400c2d..c17bd14443 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -2940,7 +2940,7 @@ def test_read_directive_allow assert_nil directive - assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk + assert_equal nil, parser.get_tk end def test_read_directive_empty @@ -2950,7 +2950,7 @@ def test_read_directive_empty assert_nil directive - assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk + assert_equal nil, parser.get_tk end def test_read_directive_no_comment @@ -2960,7 +2960,7 @@ def test_read_directive_no_comment assert_nil directive - assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk + assert_equal nil, parser.get_tk end def test_read_directive_one_liner @@ -2971,7 +2971,7 @@ def test_read_directive_one_liner assert_equal 'category', directive assert_equal 'test', value - assert_kind_of RDoc::RubyToken::TkSEMICOLON, parser.get_tk + assert_equal :on_semicolon, parser.get_tk[:kind] end def test_read_documentation_modifiers From 971096481fdf6f9ece78cebc1a609c9d6f2728f2 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 02:47:34 +0900 Subject: [PATCH 042/151] Fix for test_parse_statements_enddoc.* --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index a4689d9541..5868117e3b 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1656,7 +1656,7 @@ def parse_statements(container, single = NORMAL, current_method = nil, comment.force_encoding @encoding if @encoding end - while :on_comment == tk[:kind] do + while tk and :on_comment == tk[:kind] do comment << tk[:text] comment << "\n" unless "\n" == tk[:text].chars.last From f299898ac27bf3d4aabd98690a7024cad6fccdbb Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 02:50:45 +0900 Subject: [PATCH 043/151] Fix test_sanity_interpolation, Ripper doesn't complement newline at last --- test/test_rdoc_parser_ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index c17bd14443..3f5a365d54 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -3028,7 +3028,7 @@ def test_sanity_interpolation while tk = @parser.get_tk do last_tk = tk end - assert_equal "\n", last_tk.text + assert_equal 'end', last_tk[:text] end # If you're writing code like this you're doing it wrong From 288e3b3541872aacf899d7e938c8e3ad0e1999c0 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 02:59:42 +0900 Subject: [PATCH 044/151] Fix for test_parse_yield_in_braces_with_parens --- lib/rdoc/parser/ruby.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 5868117e3b..e4f1ddd6f1 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -2081,7 +2081,6 @@ def skip_optional_do_after_expression b_nest = 0 nest = 0 - @scanner.continue = false loop do case tk From 7acaf58ef834ef64e17e8280ba3f6097a1d8f4af Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 03:24:19 +0900 Subject: [PATCH 045/151] Fix for test_parse_statements_while_begin --- lib/rdoc/parser/ruby.rb | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index e4f1ddd6f1..f863b9fc26 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1945,7 +1945,6 @@ def parse_yield(context, single, tk, method) return if method.block_params get_tkread - @scanner.continue = false method.block_params = parse_method_or_yield_parameters end @@ -2083,28 +2082,36 @@ def skip_optional_do_after_expression nest = 0 loop do - case tk - when TkSEMICOLON, TkNL then + break unless tk + case tk[:kind] + when :on_semicolon then break if b_nest.zero? - when TkLPAREN, TkfLPAREN then + when :on_lparen then nest += 1 - when TkRPAREN then - nest -= 1 - when TkBEGIN then - b_nest += 1 - when TkEND then - b_nest -= 1 - when TkDO - break if nest.zero? - when nil then - break + when :on_kw then + case tk[:text] + when 'begin' + b_nest += 1 + when 'end' + b_nest -= 1 + when 'do' + break if nest.zero? + end + when end_token[:kind] then + if end_token == :on_rparen + nest -= 1 + break if @scanner.lex_state == :EXPR_END and nest.zero? + else + break + #break unless @scanner.continue + end end tk = get_tk end skip_tkspace false - get_tk if TkDO === peek_tk + get_tk if peek_tk && :on_kw == peek_tk[:kind] && 'do' == peek_tk[:text] end ## From 497126558ad20238ae60382094a1e75f82ec1987 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 03:28:02 +0900 Subject: [PATCH 046/151] Fix for test_parse_extend_or_include_extend --- lib/rdoc/parser/ruby.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index f863b9fc26..51847087fb 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -419,7 +419,7 @@ def get_constant skip_tkspace false tk = get_tk - while (:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind] do + while tk && ((:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind]) do res += tk[:text] tk = get_tk end @@ -1092,7 +1092,7 @@ def parse_extend_or_include klass, container, comment # :nodoc: record_location obj end - return unless :on_comma == peek_tk[:kind] + return if peek_tk.nil? || :on_comma != peek_tk[:kind] get_tk end From e91c8b1d846d7125c105b863e9ca2ceec75ece01 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 03:30:18 +0900 Subject: [PATCH 047/151] Fix for test_parse_top_level_statements_enddoc --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 51847087fb..31ff59910d 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -243,7 +243,7 @@ def collect_first_comment tk = get_tk - while :on_comment == tk[:kind] + while tk && :on_comment == tk[:kind] if first_line and tk[:text] =~ /\A#!/ then skip_tkspace tk = get_tk From 8af6e17a59de3e90ee0538bef4b9754ad6e8e00b Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 03:35:57 +0900 Subject: [PATCH 048/151] Fix for test_get_class_specification --- lib/rdoc/parser/ruby.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 31ff59910d..8754a6687c 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -386,7 +386,9 @@ def get_class_or_module container, ignore_constants = false def get_class_specification tk = peek_tk - if :on_kw == tk[:kind] && 'self' == tk[:text] + if tk.nil? + return '' + elsif :on_kw == tk[:kind] && 'self' == tk[:text] return 'self' elsif :on_gvar == tk[:kind] return '' @@ -399,6 +401,7 @@ def get_class_specification get_tkread # empty out read buffer tk = get_tk + return res unless tk case tk[:kind] when :on_nl, :on_comment, :on_semicolon then From 84ff7c629478e10f638813bd4bfd1c7662a2d838 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 04:13:32 +0900 Subject: [PATCH 049/151] Fix for test_parse_symbol_in_arg --- lib/rdoc/parser/ruby.rb | 8 ++++++-- test/test_rdoc_parser_ruby.rb | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 8754a6687c..b7a0ed73c9 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1870,8 +1870,12 @@ def parse_symbol_in_arg tk = get_tk if :on_symbol == tk[:kind] tk[:text].sub(/^:/, '') - #elsif TkSTRING - # eval @read[-1] + elsif :on_tstring == tk[:kind] + begin + eval tk[:text] + rescue + nil + end elsif :on_ident == tk[:kind] then nil # ignore elsif :on_tstring_beg == tk[:kind] then diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 3f5a365d54..369d625248 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -2812,17 +2812,21 @@ def blah end def test_parse_symbol_in_arg - util_parser ':blah "blah" "#{blah}" blah' + util_parser '[:blah, "blah", "#{blah}", blah]' + @parser.get_tk # skip '[' assert_equal 'blah', @parser.parse_symbol_in_arg + @parser.get_tk # skip ',' @parser.skip_tkspace assert_equal 'blah', @parser.parse_symbol_in_arg + @parser.get_tk # skip ',' @parser.skip_tkspace assert_equal nil, @parser.parse_symbol_in_arg + @parser.get_tk # skip ',' @parser.skip_tkspace From 8cd6e15008cbcc601933932361cdc818bb995f90 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 04:16:43 +0900 Subject: [PATCH 050/151] Fix for test_parse_statements_identifier_require --- lib/rdoc/parser/ruby.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index b7a0ed73c9..c190cb7b8d 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1591,10 +1591,8 @@ def parse_require(context, comment) tk = get_tk end - if :on_tstring_beg == tk[:kind] then - tk = get_tk # :on_tstring_content - get_tk # skip :on_tstring_end - name = tk[:text] + if :on_tstring == tk[:kind] then + name = eval tk[:text] end if name then From 659a359bc3516f14cef41d574200b66c517c4e67 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 22 Jun 2017 11:41:59 +0900 Subject: [PATCH 051/151] Squash embdoc --- lib/rdoc/parser/ruby_tools.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb index 5049fa29f3..b72ab44dff 100644 --- a/lib/rdoc/parser/ruby_tools.rb +++ b/lib/rdoc/parser/ruby_tools.rb @@ -87,6 +87,12 @@ def get_tk end end tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_tstring, :text => string } + elsif :on_embdoc_beg == tk[:kind] then + string = '' + until :on_embdoc_end == (embdoc_tk = get_tk)[:kind] do + string = string + embdoc_tk[:text] + end + tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_comment, :text => string } end # inform any listeners of our shiny new token From 8d12a1d59e945f07cf35a7601143904a3416bcf2 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 26 Jun 2017 00:29:28 +0900 Subject: [PATCH 052/151] Ripper treats '=>' as one token --- lib/rdoc/parser/ruby.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index c190cb7b8d..1bb2b9514b 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -892,12 +892,6 @@ def parse_constant container, tk, comment, ignore_constants = false return false end - check_tk = peek_tk - if :on_op == check_tk[:kind] && '>' == check_tk[:text] then - unget_tk eq_tk - return - end - value = '' con = RDoc::Constant.new name, value, comment From 6569d871b730124788455ab3e1bd0a1d33b8ae42 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 2 Jul 2017 03:08:26 +0900 Subject: [PATCH 053/151] Add RipperStateLex for analysis expr state --- lib/rdoc/parser/ripper_state_lex.rb | 192 ++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 lib/rdoc/parser/ripper_state_lex.rb diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb new file mode 100644 index 0000000000..c519c0a346 --- /dev/null +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -0,0 +1,192 @@ +require 'ripper' + +class RipperStateLex < Ripper::Filter + EXPR_NONE = 0 + EXPR_BEG = 1 + EXPR_END = 2 + EXPR_ENDARG = 4 + EXPR_ENDFN = 8 + EXPR_ARG = 16 + EXPR_CMDARG = 32 + EXPR_MID = 64 + EXPR_FNAME = 128 + EXPR_DOT = 256 + EXPR_CLASS = 512 + EXPR_LABEL = 1024 + EXPR_LABELED = 2048 + EXPR_FITEM = 4096 + EXPR_VALUE = EXPR_BEG + EXPR_BEG_ANY = (EXPR_BEG | EXPR_MID | EXPR_CLASS) + EXPR_ARG_ANY = (EXPR_ARG | EXPR_CMDARG) + EXPR_END_ANY = (EXPR_END | EXPR_ENDARG | EXPR_ENDFN) + + def initialize(code) + @lex_state = EXPR_BEG + @in_fname = false + @continue = false + reset + super(code) + end + + def reset + @command_start = false + @cmd_state = @command_start + end + + def on_nl(tok, data) + case @lex_state + when EXPR_FNAME, EXPR_DOT + @continue = true + else + @continue = false + @lex_state = EXPR_BEG + end + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_ignored_nl(tok, data) + case @lex_state + when EXPR_FNAME, EXPR_DOT + @continue = true + else + @continue = false + @lex_state = EXPR_BEG + end + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_op(tok, data) + case tok + when '!', '!=', '!~' + @lex_state = EXPR_BEG + when '<<' + # TODO next token? + case @lex_state + when EXPR_FNAME, EXPR_DOT + @lex_state = EXPR_ARG + else + @lex_state = EXPR_BEG + end + when '?' + @lex_state = :EXPR_BEG + when '&', '&&', + '|', '||', '+=', '-=', '*=', '**=', + '&=', '|=', '^=', '<<=', '>>=', '||=', '&&=' + @lex_state = EXPR_BEG + else + case @lex_state + when EXPR_FNAME, EXPR_DOT + @lex_state = EXPR_ARG + else + @lex_state = EXPR_BEG + end + end + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_kw(tok, data) + case tok + when 'class' + @lex_state = EXPR_CLASS + @in_fname = true + when 'def' + @lex_state = EXPR_FNAME + @continue = true + @in_fname = true + when 'if' + @lex_state = EXPR_BEG + else + if @lex_state == EXPR_FNAME + @lex_state = EXPR_END + else + @lex_state = EXPR_END + end + end + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_tstring_beg(tok, data) + @lex_state = EXPR_BEG + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_tstring_end(tok, data) + @lex_state = EXPR_END + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_CHAR(tok, data) + @lex_state = EXPR_END + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_period(tok, data) + @lex_state = EXPR_DOT + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_int(tok, data) + @lex_state = EXPR_END | EXPR_ENDARG + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_symbeg(tok, data) + @lex_state = EXPR_FNAME + @continue = true + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_ident(tok, data) + if @in_fname + @lex_state = EXPR_ENDFN + @in_fname = false + @continue = false + elsif @continue + case @lex_state + when EXPR_DOT + @lex_state = EXPR_ARG + else + @lex_state = EXPR_ENDFN + @continue = false + end + else + @lex_state = EXPR_CMDARG + end + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_lparen(tok, data) + @lex_state = EXPR_LABEL | EXPR_BEG + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_rparen(tok, data) + @lex_state = EXPR_ENDFN + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_const(tok, data) + case @lex_state + when EXPR_FNAME + @lex_state = EXPR_ENDFN + when EXPR_CLASS + @lex_state = EXPR_ARG + else + @lex_state = EXPR_CMDARG + end + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_sp(tok, data) + data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_default(event, tok, data) + reset + data.push({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state}) + end + + def self.parse(code) + self.new(code).parse([]) + end +end From 692a25aa5071e27ad702d7bb020f428ab5a1e46d Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 2 Jul 2017 03:13:49 +0900 Subject: [PATCH 054/151] Use RipperStateLex --- lib/rdoc/parser/ruby.rb | 22 ++++++--------------- lib/rdoc/parser/ruby_tools.rb | 4 ---- test/test_rdoc_parser_ruby.rb | 37 ++++++++++++++++++++++++++++++----- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 1bb2b9514b..093c903891 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -141,6 +141,7 @@ # standard rdocable item following it. require 'ripper' +require_relative 'ripper_state_lex' class RDoc::Parser::Ruby < RDoc::Parser @@ -167,14 +168,7 @@ def initialize(top_level, file_name, content, options, stats) @size = 0 @token_listeners = nil - @scanner = Ripper.lex(content).map { |tk| - { - :line_no => tk[0][0], - :char_no => tk[0][1], - :kind => tk[1], - :text => tk[2] - } - } + @scanner = RipperStateLex.parse(content) @scanner_point = 0 @prev_seek = nil @markup = @options.markup @@ -551,10 +545,6 @@ def get_symbol_or_name end end - def stop_at_EXPR_END # :nodoc: - @scanner.lex_state == :EXPR_END || !@scanner.continue - end - ## # Marks containers between +container+ and +ancestor+ as ignored @@ -732,9 +722,9 @@ def parse_call_parameters(tk) when end_token if end_token == :on_rparen nest -= 1 - break if lex_end? and nest <= 0 + break if (tk[:state] & RipperStateLex::EXPR_END) and nest <= 0 else - break if lex_end? + break if (tk[:state] & RipperStateLex::EXPR_END) end when :on_comment unget_tk(tk) @@ -931,7 +921,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: :on_kw == tk[:kind] && 'end' == tk[:text] then nest -= 1 elsif :on_comment == tk[:kind] then - if nest <= 0 and stop_at_EXPR_END then + if nest <= 0 and (tk[:state] & RipperStateLex::EXPR_END) then unget_tk tk break else @@ -947,7 +937,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: break end elsif :on_nl == tk[:kind] then - if nest <= 0 and stop_at_EXPR_END then + if nest <= 0 and (tk[:state] & RipperStateLex::EXPR_END) then unget_tk tk break end diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb index b72ab44dff..b6cf081ad9 100644 --- a/lib/rdoc/parser/ruby_tools.rb +++ b/lib/rdoc/parser/ruby_tools.rb @@ -13,10 +13,6 @@ def add_token_listener(obj) @token_listeners << obj end - def lex_end? - @scanner_point >= @scanner.size - end - ## # Fetches the next token from the scanner diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 369d625248..1c7b29eb02 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -1477,8 +1477,16 @@ def test_parse_meta_method { :line_no => 1, :char_no => 20, :kind => :on_symbol, :text => ':bar' }, { :line_no => 1, :char_no => 24, :kind => :on_nl, :text => "\n" } ] + parsed_stream = foo.token_stream.map { |tk| + { + :line_no => tk[:line_no], + :char_no => tk[:char_no], + :kind => tk[:kind], + :text => tk[:text] + } + } - assert_equal stream, foo.token_stream + assert_equal stream, parsed_stream end def test_parse_meta_method_block @@ -1500,7 +1508,9 @@ def test_parse_meta_method_block @parser.parse_meta_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment rest = { :line_no => 3, :char_no => 3, :kind => :on_nl, :text => "\n" } - assert_equal rest, @parser.get_tk + tk = @parser.get_tk + tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => tk[:kind], :text => tk[:text] } + assert_equal rest, tk end def test_parse_meta_method_define_method @@ -1676,8 +1686,17 @@ def test_parse_method { :line_no => 1, :char_no => 9, :kind => :on_sp, :text => ' ' }, { :line_no => 1, :char_no => 10, :kind => :on_symbol, :text => ':bar' }, { :line_no => 1, :char_no => 14, :kind => :on_sp, :text => ' ' }, - { :line_no => 1, :char_no => 15, :kind => :on_kw, :text => 'end' }] - assert_equal stream, foo.token_stream + { :line_no => 1, :char_no => 15, :kind => :on_kw, :text => 'end' } + ] + parsed_stream = foo.token_stream.map { |tk| + { + :line_no => tk[:line_no], + :char_no => tk[:char_no], + :kind => tk[:kind], + :text => tk[:text] + } + } + assert_equal stream, parsed_stream end def test_parse_redefinable_methods @@ -2139,8 +2158,16 @@ def test_parse_statements_def_percent_string_pound { :line_no => 3, :char_no => 5, :kind => :on_nl, :text => "\n" }, { :line_no => 4, :char_no => 0, :kind => :on_kw, :text => 'end' } ] + parsed_stream = a.token_stream.map { |tk| + { + :line_no => tk[:line_no], + :char_no => tk[:char_no], + :kind => tk[:kind], + :text => tk[:text] + } + } - assert_equal expected, a.token_stream + assert_equal expected, parsed_stream end def test_parse_statements_encoding From 1bf4264f17958b9e09f079908e40fdf57c9a278c Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 2 Jul 2017 03:15:48 +0900 Subject: [PATCH 055/151] Use :on_embdoc token --- lib/rdoc/parser/ruby.rb | 38 +++++++++++++++++------------------ lib/rdoc/parser/ruby_tools.rb | 2 +- test/test_rdoc_parser_ruby.rb | 2 +- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 093c903891..c7bf93461c 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -231,13 +231,13 @@ def get_visibility_information tk, single # :nodoc: def collect_first_comment skip_tkspace comment = '' - #comment.force_encoding @encoding if @encoding + comment.force_encoding @encoding if @encoding first_line = true first_comment_tk_kind = nil tk = get_tk - while tk && :on_comment == tk[:kind] + while tk && (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) if first_line and tk[:text] =~ /\A#!/ then skip_tkspace tk = get_tk @@ -398,7 +398,7 @@ def get_class_specification return res unless tk case tk[:kind] - when :on_nl, :on_comment, :on_semicolon then + when :on_nl, :on_comment, :on_embdoc, :on_semicolon then unget_tk(tk) return res end @@ -726,7 +726,7 @@ def parse_call_parameters(tk) else break if (tk[:state] & RipperStateLex::EXPR_END) end - when :on_comment + when :on_comment, :on_embdoc unget_tk(tk) break when :on_op @@ -920,7 +920,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) || :on_kw == tk[:kind] && 'end' == tk[:text] then nest -= 1 - elsif :on_comment == tk[:kind] then + elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then if nest <= 0 and (tk[:state] & RipperStateLex::EXPR_END) then unget_tk tk break @@ -1240,10 +1240,8 @@ def parse_meta_method_name comment, tk # :nodoc: if :on_symbol == name_t[:kind] then name_t[:text][1..-1] - elsif :on_tstring_beg == name_t[:kind] then - name_t = get_tk # :on_tstring_content - get_tk # skip :on_tstring_end - name_t[:text] + elsif :on_tstring == name_t[:kind] then + name_t[:text][1..-2] elsif :on_op == name_t[:kind] && '=' == name_t[:text] then # ignore remove_token_listener self @@ -1504,11 +1502,11 @@ def parse_method_or_yield_parameters(method = nil, end when :on_rparen then nest -= 1 - when method && method.block_params.nil? && :on_comment then - unget_tk tk - read_documentation_modifiers method, modifiers - @read.pop - when :on_comment then + when :on_comment, :on_embdoc then + if method && method.block_params.nil? then + unget_tk tk + read_documentation_modifiers method, modifiers + end @read.pop when :on_kw unget_tk tk @@ -1624,14 +1622,14 @@ def parse_statements(container, single = NORMAL, current_method = nil, keep_comment = false try_parse_comment = false - non_comment_seen = true unless :on_comment == tk[:kind] + non_comment_seen = true unless (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) case tk[:kind] when :on_nl, :on_ignored_nl then skip_tkspace tk = get_tk - if tk and :on_comment == tk[:kind] then + if tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then if non_comment_seen then # Look for RDoc in a comment about to be thrown away non_comment_seen = parse_comment container, tk, comment unless @@ -1641,7 +1639,7 @@ def parse_statements(container, single = NORMAL, current_method = nil, comment.force_encoding @encoding if @encoding end - while tk and :on_comment == tk[:kind] do + while tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) do comment << tk[:text] comment << "\n" unless "\n" == tk[:text].chars.last @@ -1959,7 +1957,7 @@ def read_directive allowed if :on_nl == tk[:kind] or (:tk_kw == tk[:kind] && 'def' == tk[:text]) then return - elsif :on_comment == tk[:kind] then + elsif :on_comment == tk[:kind] or :on_embdoc == tk[:kind] then return unless tk[:text] =~ /\s*:?([\w-]+):\s*(.*)/ directive = $1.downcase @@ -1970,7 +1968,7 @@ def read_directive allowed end end ensure - unless tokens.length == 1 and :on_comment == tokens.first[:kind] then + unless tokens.length == 1 and (:on_comment == tokens.first[:kind] or :on_embdoc == tokens.first[:kind]) then tokens.reverse_each do |token| unget_tk token end @@ -2130,7 +2128,7 @@ def skip_tkspace_comment(skip_nl = true) loop do skip_tkspace skip_nl next_tk = peek_tk - return if next_tk.nil? || next_tk[:kind] != :on_comment + return if next_tk.nil? || (:on_comment != next_tk[:kind] and :on_embdoc != next_tk[:kind]) get_tk end end diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb index b6cf081ad9..b765c2156c 100644 --- a/lib/rdoc/parser/ruby_tools.rb +++ b/lib/rdoc/parser/ruby_tools.rb @@ -88,7 +88,7 @@ def get_tk until :on_embdoc_end == (embdoc_tk = get_tk)[:kind] do string = string + embdoc_tk[:text] end - tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_comment, :text => string } + tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_embdoc, :text => string } end # inform any listeners of our shiny new token diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 1c7b29eb02..8018150ee2 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -74,7 +74,7 @@ class C; end comment = parser.collect_first_comment - assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n\n", @top_level), comment + assert_equal RDoc::Comment.new("first\n", @top_level), comment end def test_get_class_or_module From 5eef8de36734aa2ba8ab051d981fb2678d123ea7 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 2 Jul 2017 03:16:18 +0900 Subject: [PATCH 056/151] Remove unnecessary commented out code --- lib/rdoc/parser/ruby.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index c7bf93461c..370f2733d4 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1378,8 +1378,6 @@ def parse_method_dummy container # it is a singleton or regular method. def parse_method_name container # :nodoc: -# @scanner.lex_state = :EXPR_FNAME - skip_tkspace name_t = get_tk back_tk = skip_tkspace From bfcf3acc1b493d3a0cd37dcd637c0758e7acbfa2 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 2 Jul 2017 14:11:13 +0900 Subject: [PATCH 057/151] Move squashed token method to RipperStateLex --- lib/rdoc/parser/ripper_state_lex.rb | 346 ++++++++++++++++++---------- lib/rdoc/parser/ruby_tools.rb | 51 ---- 2 files changed, 222 insertions(+), 175 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index c519c0a346..50273f48df 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -1,6 +1,6 @@ require 'ripper' -class RipperStateLex < Ripper::Filter +class RipperStateLex EXPR_NONE = 0 EXPR_BEG = 1 EXPR_END = 2 @@ -20,173 +20,271 @@ class RipperStateLex < Ripper::Filter EXPR_ARG_ANY = (EXPR_ARG | EXPR_CMDARG) EXPR_END_ANY = (EXPR_END | EXPR_ENDARG | EXPR_ENDFN) - def initialize(code) - @lex_state = EXPR_BEG - @in_fname = false - @continue = false - reset - super(code) - end - - def reset - @command_start = false - @cmd_state = @command_start - end + class InnerStateLex < Ripper::Filter + include Enumerable - def on_nl(tok, data) - case @lex_state - when EXPR_FNAME, EXPR_DOT - @continue = true - else - @continue = false + def initialize(code) @lex_state = EXPR_BEG + @in_fname = false + @continue = false + reset + super(code) end - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end - def on_ignored_nl(tok, data) - case @lex_state - when EXPR_FNAME, EXPR_DOT - @continue = true - else - @continue = false - @lex_state = EXPR_BEG + def reset + @command_start = false + @cmd_state = @command_start end - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end - def on_op(tok, data) - case tok - when '!', '!=', '!~' - @lex_state = EXPR_BEG - when '<<' - # TODO next token? + def on_nl(tok, data) case @lex_state when EXPR_FNAME, EXPR_DOT - @lex_state = EXPR_ARG + @continue = true else + @continue = false @lex_state = EXPR_BEG end - when '?' - @lex_state = :EXPR_BEG - when '&', '&&', - '|', '||', '+=', '-=', '*=', '**=', - '&=', '|=', '^=', '<<=', '>>=', '||=', '&&=' - @lex_state = EXPR_BEG - else + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_ignored_nl(tok, data) case @lex_state when EXPR_FNAME, EXPR_DOT - @lex_state = EXPR_ARG + @continue = true else + @continue = false @lex_state = EXPR_BEG end + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end - def on_kw(tok, data) - case tok - when 'class' - @lex_state = EXPR_CLASS - @in_fname = true - when 'def' - @lex_state = EXPR_FNAME - @continue = true - @in_fname = true - when 'if' - @lex_state = EXPR_BEG - else - if @lex_state == EXPR_FNAME - @lex_state = EXPR_END + def on_op(tok, data) + case tok + when '!', '!=', '!~' + @lex_state = EXPR_BEG + when '<<' + # TODO next token? + case @lex_state + when EXPR_FNAME, EXPR_DOT + @lex_state = EXPR_ARG + else + @lex_state = EXPR_BEG + end + when '?' + @lex_state = :EXPR_BEG + when '&', '&&', + '|', '||', '+=', '-=', '*=', '**=', + '&=', '|=', '^=', '<<=', '>>=', '||=', '&&=' + @lex_state = EXPR_BEG else - @lex_state = EXPR_END + case @lex_state + when EXPR_FNAME, EXPR_DOT + @lex_state = EXPR_ARG + else + @lex_state = EXPR_BEG + end end + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end - def on_tstring_beg(tok, data) - @lex_state = EXPR_BEG - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end + def on_kw(tok, data) + case tok + when 'class' + @lex_state = EXPR_CLASS + @in_fname = true + when 'def' + @lex_state = EXPR_FNAME + @continue = true + @in_fname = true + when 'if' + @lex_state = EXPR_BEG + else + if @lex_state == EXPR_FNAME + @lex_state = EXPR_END + else + @lex_state = EXPR_END + end + end + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end - def on_tstring_end(tok, data) - @lex_state = EXPR_END - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end + def on_tstring_beg(tok, data) + @lex_state = EXPR_BEG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end - def on_CHAR(tok, data) - @lex_state = EXPR_END - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end + def on_tstring_end(tok, data) + @lex_state = EXPR_END + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end - def on_period(tok, data) - @lex_state = EXPR_DOT - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end + def on_CHAR(tok, data) + @lex_state = EXPR_END + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end - def on_int(tok, data) - @lex_state = EXPR_END | EXPR_ENDARG - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end + def on_period(tok, data) + @lex_state = EXPR_DOT + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end - def on_symbeg(tok, data) - @lex_state = EXPR_FNAME - @continue = true - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end + def on_int(tok, data) + @lex_state = EXPR_END | EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end - def on_ident(tok, data) - if @in_fname + def on_symbeg(tok, data) + @lex_state = EXPR_FNAME + @continue = true + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_ident(tok, data) + if @in_fname + @lex_state = EXPR_ENDFN + @in_fname = false + @continue = false + elsif @continue + case @lex_state + when EXPR_DOT + @lex_state = EXPR_ARG + else + @lex_state = EXPR_ENDFN + @continue = false + end + else + @lex_state = EXPR_CMDARG + end + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_lparen(tok, data) + @lex_state = EXPR_LABEL | EXPR_BEG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_rparen(tok, data) @lex_state = EXPR_ENDFN - @in_fname = false - @continue = false - elsif @continue + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_const(tok, data) case @lex_state - when EXPR_DOT + when EXPR_FNAME + @lex_state = EXPR_ENDFN + when EXPR_CLASS @lex_state = EXPR_ARG else - @lex_state = EXPR_ENDFN - @continue = false + @lex_state = EXPR_CMDARG end - else - @lex_state = EXPR_CMDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end - def on_lparen(tok, data) - @lex_state = EXPR_LABEL | EXPR_BEG - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end + def on_sp(tok, data) + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end - def on_rparen(tok, data) - @lex_state = EXPR_ENDFN - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) - end + def on_default(event, tok, data) + reset + @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state}) + end - def on_const(tok, data) - case @lex_state - when EXPR_FNAME - @lex_state = EXPR_ENDFN - when EXPR_CLASS - @lex_state = EXPR_ARG - else - @lex_state = EXPR_CMDARG + def each(&block) + @callback = block + parse end - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end - def on_sp(tok, data) - data.push({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + def get_squashed_tk + tk = @inner_lex.next + case tk[:kind] + when :on_symbeg then + prev_line_no = tk[:line_no] + prev_char_no = tk[:char_no] + + is_symbol = true + symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol } + case (tk1 = get_squashed_tk)[:kind] + when :on_ident + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_tstring_content + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end + when :on_tstring_end + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_op + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + #when :on_symbols_beg + #when :on_qsymbols_beg + else + is_symbol = false + tk = tk1 + end + if is_symbol + tk = symbol_tk + end + when :on_tstring_beg then + string = tk[:text] + state = nil + loop do + inner_str_tk = get_squashed_tk + if inner_str_tk.nil? + break + elsif :on_tstring_end == inner_str_tk[:kind] + string = string + inner_str_tk[:text] + state = inner_str_tk[:state] + break + else + string = string + inner_str_tk[:text] + end + end + tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_tstring, :text => string, :state => state } + when :on_regexp_beg then + string = tk[:text] + state = nil + loop do + inner_str_tk = get_squashed_tk + if inner_str_tk.nil? + break + elsif :on_regexp_end == inner_str_tk[:kind] + string = string + inner_str_tk[:text] + state = inner_str_tk[:state] + break + else + string = string + inner_str_tk[:text] + end + end + tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_regexp, :text => string, :state => state } + when :on_embdoc_beg then + string = '' + until :on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do + string = string + embdoc_tk[:text] + end + tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_embdoc, :text => string, :state => embdoc_tk[:state] } + end + tk end - def on_default(event, tok, data) - reset - data.push({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state}) + def initialize(code) + @inner_lex = Enumerator.new do |y| + InnerStateLex.new(code).each do |tk| + y << tk + end + end end def self.parse(code) - self.new(code).parse([]) + lex = RipperStateLex.new(code) + tokens = [] + begin + while tk = lex.get_squashed_tk + tokens.push tk + end + rescue StopIteration + end + tokens end end diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb index b765c2156c..0a566827a5 100644 --- a/lib/rdoc/parser/ruby_tools.rb +++ b/lib/rdoc/parser/ruby_tools.rb @@ -40,57 +40,6 @@ def get_tk return nil unless tk - if :on_symbeg == tk[:kind] then - prev_line_no = tk[:line_no] - prev_char_no = tk[:char_no] - - is_symbol = true - symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol } - case (tk1 = get_tk)[:kind] - when :on_ident - symbol_tk[:text] = ":#{tk1[:text]}" - when :on_tstring_content - symbol_tk[:text] = ":#{tk1[:text]}" - get_tk # skip :on_tstring_end - when :on_tstring_end - symbol_tk[:text] = ":#{tk1[:text]}" - when :on_op - symbol_tk[:text] = ":#{tk1[:text]}" - #when :on_symbols_beg - #when :on_qsymbols_beg - else - is_symbol = false - tk = tk1 - end - if is_symbol - tk = symbol_tk - # remove the identifier we just read to replace it with a symbol - @token_listeners.each do |obj| - obj.pop_token - end if @token_listeners - end - elsif :on_tstring_beg == tk[:kind] then - string = tk[:text] - loop do - inner_str_tk = get_tk - if inner_str_tk.nil? - break - elsif :on_tstring_end == inner_str_tk[:kind] - string = string + inner_str_tk[:text] - break - else - string = string + inner_str_tk[:text] - end - end - tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_tstring, :text => string } - elsif :on_embdoc_beg == tk[:kind] then - string = '' - until :on_embdoc_end == (embdoc_tk = get_tk)[:kind] do - string = string + embdoc_tk[:text] - end - tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_embdoc, :text => string } - end - # inform any listeners of our shiny new token @token_listeners.each do |obj| obj.add_token(tk) From 375b2f9c02d7efe7f84e4a77ced6d050ba65aa0e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 2 Jul 2017 14:15:23 +0900 Subject: [PATCH 058/151] Use RipperStateLex for markup/to_html.rb and token_stream.rb --- lib/rdoc/markup/to_html.rb | 8 +++++--- lib/rdoc/token_stream.rb | 31 ++++++++++++++++++------------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/lib/rdoc/markup/to_html.rb b/lib/rdoc/markup/to_html.rb index aa5de7bf66..3980b9341e 100644 --- a/lib/rdoc/markup/to_html.rb +++ b/lib/rdoc/markup/to_html.rb @@ -1,5 +1,6 @@ # frozen_string_literal: false require 'cgi' +require 'rdoc/parser/ripper_state_lex' ## # Outputs RDoc markup as HTML. @@ -200,10 +201,11 @@ def accept_verbatim verbatim content = if verbatim.ruby? or parseable? text then begin - tokens = RDoc::RubyLex.tokenize text, @options + tokens = RipperStateLex.parse text klass = ' class="ruby"' - RDoc::TokenStream.to_html tokens + result = RDoc::TokenStream.to_html tokens + result + "\n" rescue RDoc::RubyLex::Error CGI.escapeHTML text end @@ -212,7 +214,7 @@ def accept_verbatim verbatim end if @options.pipe then - @res << "\n
#{CGI.escapeHTML text}
\n" + @res << "\n
#{CGI.escapeHTML text}\n
\n" else @res << "\n#{content}\n" end diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index c0b990b341..5f7854603a 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -30,21 +30,26 @@ def self.to_html token_stream token_stream.map do |t| next unless t - style = case t - when RDoc::RubyToken::TkCONSTANT then 'ruby-constant' - when RDoc::RubyToken::TkKW then 'ruby-keyword' - when RDoc::RubyToken::TkIVAR then 'ruby-ivar' - when RDoc::RubyToken::TkOp then 'ruby-operator' - when RDoc::RubyToken::TkId then 'ruby-identifier' - when RDoc::RubyToken::TkREGEXP then 'ruby-regexp' - when RDoc::RubyToken::TkDREGEXP then 'ruby-regexp' - when RDoc::RubyToken::TkNode then 'ruby-node' - when RDoc::RubyToken::TkCOMMENT then 'ruby-comment' - when RDoc::RubyToken::TkXSTRING then 'ruby-string' - when RDoc::RubyToken::TkSTRING then 'ruby-string' - when RDoc::RubyToken::TkVal then 'ruby-value' + style = case t[:kind] + when :on_const then 'ruby-constant' + when :on_kw then 'ruby-keyword' + when :on_ivar then 'ruby-ivar' + when :on_op then 'ruby-operator' + when :on_ident then 'ruby-identifier' + when :on_backref then 'ruby-node' + when :on_comment then 'ruby-comment' + when :on_regexp then 'ruby-regexp' + when :on_tstring then 'ruby-string' + when :on_int, :on_float, + :on_embdoc, + :on_symbol, :on_CHAR then 'ruby-value' end + comment_with_nl = false + if :on_comment == t[:kind] + comment_with_nl = true if t[:text] =~ /\n$/ + t[:text] = t[:text].rstrip + end text = CGI.escapeHTML t[:text] if style then From b71ce89ac46dffe2c3d1110212ca41eaa8b5c6dd Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 2 Jul 2017 14:17:34 +0900 Subject: [PATCH 059/151] Fix test_rdoc_any_method.rb, test_rdoc_parser_ruby.rb and test_rdoc_token_stream.rb for RipperStateLex --- test/test_rdoc_any_method.rb | 2 +- test/test_rdoc_parser_ruby.rb | 4 +--- test/test_rdoc_token_stream.rb | 22 +++++++++++----------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/test/test_rdoc_any_method.rb b/test/test_rdoc_any_method.rb index fdbb62efa6..fdfb0f3995 100644 --- a/test/test_rdoc_any_method.rb +++ b/test/test_rdoc_any_method.rb @@ -74,7 +74,7 @@ def test_is_alias_for def test_markup_code tokens = [ - RDoc::RubyToken::TkCONSTANT. new(0, 0, 0, 'CONSTANT'), + { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' }, ] @c2_a.collect_tokens diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 8018150ee2..f403e71a73 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -2152,9 +2152,7 @@ def test_parse_statements_def_percent_string_pound { :line_no => 2, :char_no => 3, :kind => :on_sp, :text => ' ' }, { :line_no => 2, :char_no => 4, :kind => :on_ident, :text => 'a' }, { :line_no => 2, :char_no => 5, :kind => :on_nl, :text => "\n" }, - { :line_no => 3, :char_no => 0, :kind => :on_regexp_beg, :text => '%r{' }, - { :line_no => 3, :char_no => 3, :kind => :on_tstring_content, :text => '#' }, - { :line_no => 3, :char_no => 4, :kind => :on_regexp_end, :text => '}' }, + { :line_no => 3, :char_no => 0, :kind => :on_regexp, :text => '%r{#}' }, { :line_no => 3, :char_no => 5, :kind => :on_nl, :text => "\n" }, { :line_no => 4, :char_no => 0, :kind => :on_kw, :text => 'end' } ] diff --git a/test/test_rdoc_token_stream.rb b/test/test_rdoc_token_stream.rb index 5ed7b1d1a9..95c3ffd13e 100644 --- a/test/test_rdoc_token_stream.rb +++ b/test/test_rdoc_token_stream.rb @@ -5,17 +5,17 @@ class TestRDocTokenStream < RDoc::TestCase def test_class_to_html tokens = [ - RDoc::RubyToken::TkCONSTANT. new(0, 0, 0, 'CONSTANT'), - RDoc::RubyToken::TkDEF. new(0, 0, 0, 'KW'), - RDoc::RubyToken::TkIVAR. new(0, 0, 0, 'IVAR'), - RDoc::RubyToken::TkOp. new(0, 0, 0, 'Op'), - RDoc::RubyToken::TkId. new(0, 0, 0, 'Id'), - RDoc::RubyToken::TkNode. new(0, 0, 0, 'Node'), - RDoc::RubyToken::TkCOMMENT. new(0, 0, 0, 'COMMENT'), - RDoc::RubyToken::TkREGEXP. new(0, 0, 0, 'REGEXP'), - RDoc::RubyToken::TkSTRING. new(0, 0, 0, 'STRING'), - RDoc::RubyToken::TkVal. new(0, 0, 0, 'Val'), - RDoc::RubyToken::TkBACKSLASH.new(0, 0, 0, '\\'), + { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' }, + { :line_no => 0, :char_no => 0, :kind => :on_kw, :text => 'KW' }, + { :line_no => 0, :char_no => 0, :kind => :on_ivar, :text => 'IVAR' }, + { :line_no => 0, :char_no => 0, :kind => :on_op, :text => 'Op' }, + { :line_no => 0, :char_no => 0, :kind => :on_ident, :text => 'Id' }, + { :line_no => 0, :char_no => 0, :kind => :on_backref, :text => 'Node' }, + { :line_no => 0, :char_no => 0, :kind => :on_comment, :text => 'COMMENT' }, + { :line_no => 0, :char_no => 0, :kind => :on_regexp, :text => 'REGEXP' }, + { :line_no => 0, :char_no => 0, :kind => :on_tstring, :text => 'STRING' }, + { :line_no => 0, :char_no => 0, :kind => :on_int, :text => 'Val' }, + { :line_no => 0, :char_no => 0, :kind => :on_unknown, :text => '\\' } ] expected = [ From d6066f14903e2a996264b8b06c3e892eb8b8729d Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 2 Jul 2017 16:03:43 +0900 Subject: [PATCH 060/151] Add closing ')', ']' and '}' token role to RipperStateLex --- lib/rdoc/parser/ripper_state_lex.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 50273f48df..c705805c14 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -76,6 +76,8 @@ def on_op(tok, data) '|', '||', '+=', '-=', '*=', '**=', '&=', '|=', '^=', '<<=', '>>=', '||=', '&&=' @lex_state = EXPR_BEG + when ')', ']', '}' + @lex_state = EXPR_END else case @lex_state when EXPR_FNAME, EXPR_DOT From cb883034c4355916c0e773f22f13f0e46860458e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 2 Jul 2017 16:05:18 +0900 Subject: [PATCH 061/151] Add RipperStateLex.end? for checking EXPR_END --- lib/rdoc/parser/ripper_state_lex.rb | 4 ++++ lib/rdoc/parser/ruby.rb | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index c705805c14..51cae440d9 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -289,4 +289,8 @@ def self.parse(code) end tokens end + + def self.end?(token) + (token[:state] & EXPR_END) + end end diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 370f2733d4..fba7f7baec 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -722,9 +722,9 @@ def parse_call_parameters(tk) when end_token if end_token == :on_rparen nest -= 1 - break if (tk[:state] & RipperStateLex::EXPR_END) and nest <= 0 + break if RipperStateLex.end?(tk) and nest <= 0 else - break if (tk[:state] & RipperStateLex::EXPR_END) + break if RipperStateLex.end?(tk) end when :on_comment, :on_embdoc unget_tk(tk) @@ -921,7 +921,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: :on_kw == tk[:kind] && 'end' == tk[:text] then nest -= 1 elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then - if nest <= 0 and (tk[:state] & RipperStateLex::EXPR_END) then + if nest <= 0 and RipperStateLex.end?(tk) then unget_tk tk break else @@ -937,7 +937,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: break end elsif :on_nl == tk[:kind] then - if nest <= 0 and (tk[:state] & RipperStateLex::EXPR_END) then + if nest <= 0 and RipperStateLex.end?(tk) then unget_tk tk break end From fb6e70cadc0c698db85a97fc8bf8c95bea67c0b7 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 2 Jul 2017 16:20:17 +0900 Subject: [PATCH 062/151] Fix for variable warnings --- lib/rdoc/parser/ripper_state_lex.rb | 3 --- test/test_rdoc_parser_ruby.rb | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 51cae440d9..2a9441c5c6 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -201,9 +201,6 @@ def get_squashed_tk tk = @inner_lex.next case tk[:kind] when :on_symbeg then - prev_line_no = tk[:line_no] - prev_char_no = tk[:char_no] - is_symbol = true symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol } case (tk1 = get_squashed_tk)[:kind] diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index f403e71a73..fbd5bd470c 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -1477,12 +1477,12 @@ def test_parse_meta_method { :line_no => 1, :char_no => 20, :kind => :on_symbol, :text => ':bar' }, { :line_no => 1, :char_no => 24, :kind => :on_nl, :text => "\n" } ] - parsed_stream = foo.token_stream.map { |tk| + parsed_stream = foo.token_stream.map { |t| { - :line_no => tk[:line_no], - :char_no => tk[:char_no], - :kind => tk[:kind], - :text => tk[:text] + :line_no => t[:line_no], + :char_no => t[:char_no], + :kind => t[:kind], + :text => t[:text] } } @@ -1688,12 +1688,12 @@ def test_parse_method { :line_no => 1, :char_no => 14, :kind => :on_sp, :text => ' ' }, { :line_no => 1, :char_no => 15, :kind => :on_kw, :text => 'end' } ] - parsed_stream = foo.token_stream.map { |tk| + parsed_stream = foo.token_stream.map { |t| { - :line_no => tk[:line_no], - :char_no => tk[:char_no], - :kind => tk[:kind], - :text => tk[:text] + :line_no => t[:line_no], + :char_no => t[:char_no], + :kind => t[:kind], + :text => t[:text] } } assert_equal stream, parsed_stream From 1afc442a5aaaf63b55e7d41afaabd3ac1845cf07 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 2 Jul 2017 17:35:43 +0900 Subject: [PATCH 063/151] In Ruby 1.9 String#chars gives Enumerator and Enumerator#last doesn't exist --- lib/rdoc/parser/ruby.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index fba7f7baec..5802bdc1bb 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1639,9 +1639,9 @@ def parse_statements(container, single = NORMAL, current_method = nil, while tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) do comment << tk[:text] - comment << "\n" unless "\n" == tk[:text].chars.last + comment << "\n" unless "\n" == tk[:text].chars.to_a.last - if tk[:text].size > 1 && "\n" == tk[:text].chars.last then + if tk[:text].size > 1 && "\n" == tk[:text].chars.to_a.last then skip_tkspace false # leading spaces end tk = get_tk From b5ee10294427902cf5dd8832a54df20354cad1ea Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 4 Jul 2017 21:56:09 +0900 Subject: [PATCH 064/151] Delete ruby_lex.rb and all tests are passed at last! --- lib/rdoc/ruby_lex.rb | 1523 ------------------------------------------ rdoc.gemspec | 2 +- 2 files changed, 1 insertion(+), 1524 deletions(-) delete mode 100644 lib/rdoc/ruby_lex.rb diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb deleted file mode 100644 index 965a1ffda6..0000000000 --- a/lib/rdoc/ruby_lex.rb +++ /dev/null @@ -1,1523 +0,0 @@ -# coding: US-ASCII -# frozen_string_literal: false - -#-- -# irb/ruby-lex.rb - ruby lexcal analyzer -# $Release Version: 0.9.5$ -# $Revision: 17979 $ -# $Date: 2008-07-09 10:17:05 -0700 (Wed, 09 Jul 2008) $ -# by Keiju ISHITSUKA(keiju@ruby-lang.org) -# -#++ - -require "e2mmap" -require "irb/slex" -require "stringio" - -## -# Ruby lexer adapted from irb. -# -# The internals are not documented because they are scary. - -class RDoc::RubyLex - - ## - # Raised upon invalid input - - class Error < RDoc::Error - end - - # :stopdoc: - - extend Exception2MessageMapper - - def_exception(:AlreadyDefinedToken, "Already defined token(%s)") - def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')") - def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')") - def_exception(:TkReading2TokenDuplicateError, - "key duplicate(token_n='%s', key='%s')") - def_exception(:SyntaxError, "%s") - - def_exception(:TerminateLineInput, "Terminate Line Input") - - include RDoc::RubyToken - include IRB - - attr_accessor :continue - attr_accessor :lex_state - attr_accessor :first_in_method_statement - attr_reader :reader - - class << self - attr_accessor :debug_level - end - - def self.debug? - @debug_level > 0 - end - - self.debug_level = 0 - - # :startdoc: - - ## - # Returns an Array of +ruby+ tokens. See ::new for a description of - # +options+. - - def self.tokenize ruby, options - tokens = [] - - scanner = RDoc::RubyLex.new ruby, options - scanner.exception_on_syntax_error = true - - while token = scanner.token do - tokens << token - end - - tokens - end - - ## - # Creates a new lexer for +content+. +options+ is an RDoc::Options, only - # +tab_width is used. - - def initialize(content, options) - lex_init - - if /\t/ =~ content then - tab_width = options.tab_width - content = content.split(/\n/).map do |line| - 1 while line.gsub!(/\t+/) { - ' ' * (tab_width*$&.length - $`.length % tab_width) - } && $~ - line - end.join("\n") - end - - content << "\n" unless content[-1, 1] == "\n" - - set_input StringIO.new content - - @base_char_no = 0 - @char_no = 0 - @exp_line_no = @line_no = 1 - @here_readed = [] - @readed = [] - @current_readed = @readed - @rests = [] - @seek = 0 - - @heredoc_queue = [] - - @indent = 0 - @indent_stack = [] - @lex_state = :EXPR_BEG - @space_seen = false - @escaped_nl = false - @first_in_method_statement = false - @after_question = false - - @continue = false - @line = "" - - @skip_space = false - @readed_auto_clean_up = false - @exception_on_syntax_error = true - - @prompt = nil - @prev_seek = nil - @ltype = nil - end - - # :stopdoc: - - def inspect # :nodoc: - "#<%s:0x%x pos %d lex_state %p space_seen %p>" % [ - self.class, object_id, - @io.pos, @lex_state, @space_seen, - ] - end - - attr_accessor :skip_space - attr_accessor :readed_auto_clean_up - attr_accessor :exception_on_syntax_error - - attr_reader :seek - attr_reader :char_no - attr_reader :line_no - attr_reader :indent - - # io functions - def set_input(io, p = nil, &block) - @io = io - if p.respond_to?(:call) - @input = p - elsif block_given? - @input = block - else - @input = Proc.new{@io.gets} - end - end - - def get_readed - if idx = @readed.rindex("\n") - @base_char_no = @readed.size - (idx + 1) - else - @base_char_no += @readed.size - end - - readed = @readed.join("") - @readed.clear - readed - end - - def getc - while @rests.empty? - # return nil unless buf_input - @rests.push nil unless buf_input - end - c = @rests.shift - @current_readed.push c - @seek += 1 - if c == "\n".freeze - @line_no += 1 - @char_no = 0 - else - @char_no += 1 - end - - c - end - - def gets - l = "" - while c = getc - l.concat(c) - break if c == "\n" - end - return nil if l == "" and c.nil? - l - end - - def eof? - @io.eof? - end - - def getc_of_rests - if @rests.empty? - nil - else - getc - end - end - - def ungetc(c = nil) - if @here_readed.empty? - c2 = @readed.pop - else - c2 = @here_readed.pop - end - c = c2 unless c - @rests.unshift c #c = - @seek -= 1 - if c == "\n" - @line_no -= 1 - if idx = @readed.rindex("\n") - @char_no = idx + 1 - else - @char_no = @base_char_no + @readed.size - end - else - @char_no -= 1 - end - end - - def peek_equal?(str) - chrs = str.split(//) - until @rests.size >= chrs.size - return false unless buf_input - end - @rests[0, chrs.size] == chrs - end - - def peek_match?(regexp) - while @rests.empty? - return false unless buf_input - end - regexp =~ @rests.join("") - end - - def peek(i = 0) - while @rests.size <= i - return nil unless buf_input - end - @rests[i] - end - - def buf_input - prompt - line = @input.call - return nil unless line - @rests.concat line.split(//) - true - end - private :buf_input - - def set_prompt(p = nil, &block) - p = block if block_given? - if p.respond_to?(:call) - @prompt = p - else - @prompt = Proc.new{print p} - end - end - - def prompt - if @prompt - @prompt.call(@ltype, @indent, @continue, @line_no) - end - end - - def initialize_input - @ltype = nil - @quoted = nil - @indent = 0 - @indent_stack = [] - @lex_state = :EXPR_BEG - @space_seen = false - @current_readed = @readed - - @continue = false - prompt - - @line = "" - @exp_line_no = @line_no - end - - def each_top_level_statement - initialize_input - catch(:TERM_INPUT) do - loop do - begin - @continue = false - prompt - unless l = lex - throw :TERM_INPUT if @line == '' - else - #p l - @line.concat l - if @ltype or @continue or @indent > 0 - next - end - end - if @line != "\n" - yield @line, @exp_line_no - end - break unless l - @line = '' - @exp_line_no = @line_no - - @indent = 0 - @indent_stack = [] - prompt - rescue TerminateLineInput - initialize_input - prompt - get_readed - end - end - end - end - - def lex - until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) && - !@continue or - tk.nil?) - #p tk - #p @lex_state - #p self - end - line = get_readed - # print self.inspect - if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil? - nil - else - line - end - end - - def token - # require "tracer" - # Tracer.on - @prev_seek = @seek - @prev_line_no = @line_no - @prev_char_no = @char_no - begin - begin - tk = @OP.match(self) - @space_seen = tk.kind_of?(TkSPACE) - @first_in_method_statement = false if !@space_seen && @first_in_method_statement - rescue SyntaxError => e - raise Error, "syntax error: #{e.message}" if - @exception_on_syntax_error - - tk = TkError.new(@seek, @line_no, @char_no) - end - end while @skip_space and tk.kind_of?(TkSPACE) - - if @readed_auto_clean_up - get_readed - end - - if TkSYMBEG === tk then - tk1 = token - set_token_position tk.seek, tk.line_no, tk.char_no - - case tk1 - when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then - if tk1.respond_to?(:name) then - tk = Token(TkSYMBOL, ":" + tk1.name) - else - tk = Token(TkSYMBOL, ":" + tk1.text) - end - else - tk = tk1 - end - elsif (TkPLUS === tk or TkMINUS === tk) and peek(0) =~ /\d/ then - tk1 = token - set_token_position tk.seek, tk.line_no, tk.char_no - tk = Token(tk1.class, tk.text + tk1.text) - end - @after_question = false if @after_question and !(TkQUESTION === tk) - - # Tracer.off - tk - end - - ENINDENT_CLAUSE = [ - "case", "class", "def", "do", "for", "if", - "module", "unless", "until", "while", "begin" #, "when" - ] - - DEINDENT_CLAUSE = ["end" #, "when" - ] - - PERCENT_LTYPE = { - "q" => "\'", - "Q" => "\"", - "x" => "\`", - "r" => "/", - "w" => "]", - "W" => "]", - "s" => ":", - "i" => "]", - "I" => "]" - } - - PERCENT_PAREN = { - "{" => "}", - "[" => "]", - "<" => ">", - "(" => ")" - } - - PERCENT_PAREN_REV = PERCENT_PAREN.invert - - Ltype2Token = { - "\'" => TkSTRING, - "\"" => TkSTRING, - "\`" => TkXSTRING, - "/" => TkREGEXP, - "]" => TkDSTRING, - ":" => TkSYMBOL - } - DLtype2Token = { - "\"" => TkDSTRING, - "\`" => TkDXSTRING, - "/" => TkDREGEXP, - } - - def lex_init() - @OP = IRB::SLex.new - @OP.def_rules("\0", "\004", "\032") do |op, io| - Token(TkEND_OF_SCRIPT, '') - end - - @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io| - @space_seen = true - str = op - while (ch = getc) =~ /[ \t\f\r\13]/ do - str << ch - end - ungetc - Token TkSPACE, str - end - - @OP.def_rule("#") do |op, io| - identify_comment - end - - @OP.def_rule("=begin", - proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do - |op, io| - @ltype = "=" - res = op - until (ch = getc) == "\n" do - res << ch - end - res << ch - - until ( peek_equal?("=end") && peek(4) =~ /\s/ ) do - (ch = getc) - res << ch - end - - res << gets # consume =end - - @ltype = nil - Token(TkRD_COMMENT, res) - end - - @OP.def_rule("\n") do |op, io| - print "\\n\n" if RDoc::RubyLex.debug? - unless @heredoc_queue.empty? - info = @heredoc_queue[0] - if !info[:started] # "\n" - info[:started] = true - ungetc "\n" - elsif info[:heredoc_end].nil? # heredoc body - tk, heredoc_end = identify_here_document_body(info[:quoted], info[:lt], info[:indent]) - info[:heredoc_end] = heredoc_end - ungetc "\n" - else # heredoc end - @heredoc_queue.shift - @lex_state = :EXPR_BEG - tk = Token(TkHEREDOCEND, info[:heredoc_end]) - if !@heredoc_queue.empty? - @heredoc_queue[0][:started] = true - ungetc "\n" - end - end - end - unless tk - case @lex_state - when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT - @continue = true - else - @continue = false - @lex_state = :EXPR_BEG unless @escaped_nl - until (@indent_stack.empty? || - [TkLPAREN, TkLBRACK, TkLBRACE, - TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last)) - @indent_stack.pop - end - end - @current_readed = @readed - @here_readed.clear - tk = Token(TkNL) - end - @escaped_nl = false - tk - end - - @OP.def_rules("=") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - @lex_state = :EXPR_ARG - else - @lex_state = :EXPR_BEG - end - Token(op) - end - - @OP.def_rules("*", "**", - "==", "===", - "=~", "<=>", - "<", "<=", - ">", ">=", ">>", "=>") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - tk = Token(TkId, op) - @lex_state = :EXPR_ARG - else - tk = Token(op) - @lex_state = :EXPR_BEG - end - tk - end - - @OP.def_rules("->") do - |op, io| - @lex_state = :EXPR_ENDFN - Token(op) - end - - @OP.def_rules("!", "!=", "!~") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - @lex_state = :EXPR_ARG - Token(TkId, op) - else - @lex_state = :EXPR_BEG - Token(op) - end - end - - @OP.def_rules("<<") do - |op, io| - tk = nil - if @lex_state != :EXPR_END && @lex_state != :EXPR_CLASS && - (@lex_state != :EXPR_ARG || @space_seen) - c = peek(0) - if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-" || c == "~") - tk = identify_here_document(op) - end - end - unless tk - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - tk = Token(TkId, op) - @lex_state = :EXPR_ARG - else - tk = Token(op) - @lex_state = :EXPR_BEG - end - end - tk - end - - @OP.def_rules("'", '"') do - |op, io| - identify_string(op) - end - - @OP.def_rules("`") do - |op, io| - if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state - @lex_state = :EXPR_ARG - Token(TkId, op) - else - identify_string(op) - end - end - - @OP.def_rules('?') do - |op, io| - if @lex_state == :EXPR_END - @lex_state = :EXPR_BEG - @after_question = true - Token(TkQUESTION) - else - ch = getc - if @lex_state == :EXPR_ARG && ch =~ /\s/ - ungetc - @lex_state = :EXPR_BEG; - Token(TkQUESTION) - else - @lex_state = :EXPR_END - ch << getc if "\\" == ch - Token(TkCHAR, "?#{ch}") - end - end - end - - @OP.def_rules("&&", "||") do - |op, io| - @lex_state = :EXPR_BEG - Token(op) - end - - @OP.def_rules("&", "|") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - tk = Token(TkId, op) - @lex_state = :EXPR_ARG - else - tk = Token(op) - @lex_state = :EXPR_BEG - end - tk - end - - @OP.def_rules("+=", "-=", "*=", "**=", - "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do - |op, io| - @lex_state = :EXPR_BEG - op =~ /^(.*)=$/ - Token(TkOPASGN, $1) - end - - @OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do - |op, io| - @lex_state = :EXPR_ARG - Token(TkId, op) - end - - @OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do - |op, io| - @lex_state = :EXPR_ARG - Token(TkId, op) - end - - @OP.def_rules("+", "-") do - |op, io| - catch(:RET) do - if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state - tk = Token(TkId, op) - @lex_state = :EXPR_ARG - elsif @lex_state == :EXPR_ARG - if @space_seen and peek(0) =~ /[0-9]/ - throw :RET, identify_number(op) - else - @lex_state = :EXPR_BEG - end - elsif @lex_state != :EXPR_END and peek(0) =~ /[0-9]/ - throw :RET, identify_number(op) - else - @lex_state = :EXPR_BEG - end - tk = Token(op) unless tk - tk - end - end - - @OP.def_rules(".", "&.") do - |op, io| - @lex_state = :EXPR_BEG - if peek(0) =~ /[0-9]/ - ungetc - identify_number - else - # for "obj.if" or "obj&.if" etc. - @lex_state = :EXPR_DOT - Token(op) - end - end - - @OP.def_rules("..", "...") do - |op, io| - @lex_state = :EXPR_BEG - Token(op) - end - - lex_int2 - end - - def lex_int2 - @OP.def_rules("]", "}", ")") do - |op, io| - @lex_state = :EXPR_END - @indent -= 1 - @indent_stack.pop - Token(op) - end - - @OP.def_rule(":") do - |op, io| - if @lex_state == :EXPR_END || peek(0) =~ /\s/ - @lex_state = :EXPR_BEG - Token(TkCOLON) - else - @lex_state = :EXPR_FNAME; - Token(TkSYMBEG) - end - end - - @OP.def_rule("::") do - |op, io| - # p @lex_state.id2name, @space_seen - if @lex_state == :EXPR_BEG or @lex_state == :EXPR_ARG && @space_seen - @lex_state = :EXPR_BEG - Token(TkCOLON3) - else - @lex_state = :EXPR_DOT - Token(TkCOLON2) - end - end - - @OP.def_rule("/") do - |op, io| - if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state - @lex_state = :EXPR_ARG - Token(TkId, op) - elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID || @first_in_method_statement - identify_string(op) - elsif peek(0) == '=' - getc - @lex_state = :EXPR_BEG - Token(TkOPASGN, "/") #/) - elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/ - identify_string(op) - else - @lex_state = :EXPR_BEG - Token("/") #/) - end - end - - @OP.def_rules("^") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - tk = Token(TkId, op) - @lex_state = :EXPR_ARG - else - tk = Token(op) - @lex_state = :EXPR_BEG - end - tk - end - - # @OP.def_rules("^=") do - # @lex_state = :EXPR_BEG - # Token(OP_ASGN, :^) - # end - - @OP.def_rules(",") do - |op, io| - @lex_state = :EXPR_BEG - Token(op) - end - - @OP.def_rules(";") do - |op, io| - @lex_state = :EXPR_BEG - until (@indent_stack.empty? || - [TkLPAREN, TkLBRACK, TkLBRACE, - TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last)) - @indent_stack.pop - end - Token(op) - end - - @OP.def_rule("~") do - |op, io| - case @lex_state - when :EXPR_FNAME, :EXPR_DOT - @lex_state = :EXPR_ARG - Token(TkId, op) - else - @lex_state = :EXPR_BEG - Token(op) - end - end - - @OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do - |op, io| - @lex_state = :EXPR_BEG - Token("~") - end - - @OP.def_rule("(") do - |op, io| - @indent += 1 - if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID - @lex_state = :EXPR_BEG - tk_c = TkfLPAREN - else - @lex_state = :EXPR_BEG - tk_c = TkLPAREN - end - @indent_stack.push tk_c - Token tk_c - end - - @OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do - |op, io| - @lex_state = :EXPR_ARG - Token(TkId, op) - end - - @OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do - |op, io| - @lex_state = :EXPR_ARG - Token(TkId, op) - end - - @OP.def_rule("[") do - |op, io| - text = nil - @indent += 1 - if @lex_state == :EXPR_FNAME - tk_c = TkfLBRACK - else - if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID - tk_c = TkLBRACK - elsif @lex_state == :EXPR_ARG && @space_seen - tk_c = TkLBRACK - elsif @lex_state == :EXPR_DOT - if peek(0) == "]" - tk_c = TkIDENTIFIER - getc - if peek(0) == "=" - text = "[]=" - else - text = "[]" - end - else - tk_c = TkOp - end - else - tk_c = TkfLBRACK - end - @lex_state = :EXPR_BEG - end - @indent_stack.push tk_c - Token(tk_c, text) - end - - @OP.def_rule("{") do - |op, io| - @indent += 1 - if @lex_state != :EXPR_END && @lex_state != :EXPR_ARG - tk_c = TkLBRACE - else - tk_c = TkfLBRACE - end - @lex_state = :EXPR_BEG - @indent_stack.push tk_c - Token(tk_c) - end - - @OP.def_rule('\\') do - |op, io| - if peek(0) == "\n" - @space_seen = true - @continue = true - @escaped_nl = true - end - Token("\\") - end - - @OP.def_rule('%') do - |op, io| - if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state - @lex_state = :EXPR_ARG - Token(TkId, op) - elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID - identify_quotation - elsif peek(0) == '=' - getc - @lex_state = :EXPR_BEG - Token(TkOPASGN, '%') - elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/ - identify_quotation - else - @lex_state = :EXPR_BEG - Token("%") #)) - end - end - - @OP.def_rule('$') do - |op, io| - identify_gvar - end - - @OP.def_rule('@') do - |op, io| - if peek(0) =~ /[\w@]/ - ungetc - identify_identifier - else - Token("@") - end - end - - # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do - # |op, io| - # @indent += 1 - # @lex_state = :EXPR_FNAME - # # @lex_state = :EXPR_END - # # until @rests[0] == "\n" or @rests[0] == ";" - # # rests.shift - # # end - # end - - @OP.def_rule("_") do - if peek_match?(/_END__/) and @lex_state == :EXPR_BEG then - 6.times { getc } - Token(TkEND_OF_SCRIPT, '__END__') - else - ungetc - identify_identifier - end - end - - @OP.def_rule("") do - |op, io| - printf "MATCH: start %s: %s\n", op, io.inspect if RDoc::RubyLex.debug? - if peek(0) =~ /[0-9]/ - t = identify_number - else - t = identify_identifier - end - printf "MATCH: end %s: %s\n", op, io.inspect if RDoc::RubyLex.debug? - t - end - - p @OP if RDoc::RubyLex.debug? - end - - def identify_gvar - @lex_state = :EXPR_END - - case ch = getc - when /[~_*$?!@\/\\;,=:<>".]/ #" - Token(TkGVAR, "$" + ch) - when "-" - Token(TkGVAR, "$-" + getc) - when "&", "`", "'", "+" - Token(TkBACK_REF, "$"+ch) - when /[1-9]/ - ref = ch - while (ch = getc) =~ /[0-9]/ do ref << ch end - ungetc - Token(TkNTH_REF, "$#{ref}") - when /\w/ - ungetc - ungetc - identify_identifier - else - ungetc - Token("$") - end - end - - IDENT_RE = eval '/[\w\u{0080}-\u{FFFFF}]/u' - - def identify_identifier - token = "" - if peek(0) =~ /[$@]/ - token.concat(c = getc) - if c == "@" and peek(0) == "@" - token.concat getc - end - end - - while (ch = getc) =~ IDENT_RE do - print " :#{ch}: " if RDoc::RubyLex.debug? - token.concat ch - end - - ungetc - - if ((ch == "!" && peek(1) != "=") || ch == "?") && token[0,1] =~ /\w/ - token.concat getc - end - - # almost fix token - - case token - when /^\$/ - return Token(TkGVAR, token) - when /^\@\@/ - @lex_state = :EXPR_END - # p Token(TkCVAR, token) - return Token(TkCVAR, token) - when /^\@/ - @lex_state = :EXPR_END - return Token(TkIVAR, token) - end - - if @lex_state != :EXPR_DOT - print token, "\n" if RDoc::RubyLex.debug? - - token_c, *trans = TkReading2Token[token] - if token_c - # reserved word? - - if (@lex_state != :EXPR_BEG && - @lex_state != :EXPR_FNAME && - trans[1]) - # modifiers - token_c = TkSymbol2Token[trans[1]] - @lex_state = trans[0] - else - if @lex_state != :EXPR_FNAME - if ENINDENT_CLAUSE.include?(token) - valid = peek(0) != ':' - - # check for ``class = val'' etc. - case token - when "class" - valid = false unless peek_match?(/^\s*(<<|\w|::)/) - when "def" - valid = false if peek_match?(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/) - when "do" - valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&)/) - when *ENINDENT_CLAUSE - valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&|\|)/) - else - # no nothing - end if valid - - if valid - if token == "do" - if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last) - @indent += 1 - @indent_stack.push token_c - end - else - @indent += 1 - @indent_stack.push token_c - end - else - token_c = TkIDENTIFIER - end - - elsif DEINDENT_CLAUSE.include?(token) - @indent -= 1 - @indent_stack.pop - end - @lex_state = trans[0] - else - @lex_state = :EXPR_END - end - end - if token_c.ancestors.include?(TkId) and peek(0) == ':' and !peek_match?(/^::/) - token.concat getc - token_c = TkSYMBOL - end - return Token(token_c, token) - end - end - - if @lex_state == :EXPR_FNAME - @lex_state = :EXPR_END - if peek(0) == '=' and peek(1) != '>' - token.concat getc - end - elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_DOT || - @lex_state == :EXPR_ARG || @lex_state == :EXPR_MID - @lex_state = :EXPR_ARG - else - @lex_state = :EXPR_END - end - - if token[0, 1] =~ /[A-Z]/ - if token[-1] =~ /[!?]/ - token_c = TkIDENTIFIER - else - token_c = TkCONSTANT - end - elsif token[token.size - 1, 1] =~ /[!?]/ - token_c = TkFID - else - token_c = TkIDENTIFIER - end - if peek(0) == ':' and !peek_match?(/^::/) - token.concat getc - return Token(TkSYMBOL, token) - else - return Token(token_c, token) - end - end - - def identify_here_document(op) - ch = getc - start_token = op - # if lt = PERCENT_LTYPE[ch] - if ch == "-" or ch == "~" - start_token.concat ch - ch = getc - indent = true - end - if /['"`]/ =~ ch - start_token.concat ch - user_quote = lt = ch - quoted = "" - while (c = getc) && c != lt - quoted.concat c - end - start_token.concat quoted - start_token.concat lt - else - user_quote = nil - lt = '"' - quoted = ch.dup - while (c = getc) && c =~ /\w/ - quoted.concat c - end - start_token.concat quoted - ungetc - end - - @heredoc_queue << { - quoted: quoted, - lt: lt, - indent: indent, - started: false - } - @lex_state = :EXPR_END - Token(RDoc::RubyLex::TkHEREDOCBEG, start_token) - end - - def identify_here_document_body(quoted, lt, indent) - ltback, @ltype = @ltype, lt - - doc = "" - heredoc_end = nil - while l = gets - l = l.sub(/(:?\r)?\n\z/, "\n") - if (indent ? l.strip : l.chomp) == quoted - heredoc_end = l - break - end - doc << l - end - raise Error, "Missing terminating #{quoted} for string" unless heredoc_end - - @ltype = ltback - @lex_state = :EXPR_BEG - [Token(RDoc::RubyLex::TkHEREDOC, doc), heredoc_end] - end - - def identify_quotation - type = ch = getc - if lt = PERCENT_LTYPE[type] - ch = getc - elsif type =~ /\W/ - type = nil - lt = "\"" - else - return Token(TkMOD, '%') - end - # if ch !~ /\W/ - # ungetc - # next - # end - #@ltype = lt - @quoted = ch unless @quoted = PERCENT_PAREN[ch] - identify_string(lt, @quoted, type) - end - - def identify_number(op = "") - @lex_state = :EXPR_END - - num = op - - if peek(0) == "0" && peek(1) !~ /[.eEri]/ - num << getc - - case peek(0) - when /[xX]/ - ch = getc - match = /[0-9a-fA-F_]/ - when /[bB]/ - ch = getc - match = /[01_]/ - when /[oO]/ - ch = getc - match = /[0-7_]/ - when /[dD]/ - ch = getc - match = /[0-9_]/ - when /[0-7]/ - match = /[0-7_]/ - when /[89]/ - raise Error, "Illegal octal digit" - else - return Token(TkINTEGER, num) - end - - num << ch if ch - - len0 = true - non_digit = false - while ch = getc - num << ch - if match =~ ch - if ch == "_" - if non_digit - raise Error, "trailing `#{ch}' in number" - else - non_digit = ch - end - else - non_digit = false - len0 = false - end - else - ungetc - num[-1, 1] = '' - if len0 - raise Error, "numeric literal without digits" - end - if non_digit - raise Error, "trailing `#{non_digit}' in number" - end - break - end - end - return Token(TkINTEGER, num) - end - - type = TkINTEGER - allow_point = true - allow_e = true - allow_ri = true - non_digit = false - while ch = getc - num << ch - case ch - when /[0-9]/ - non_digit = false - when "_" - non_digit = ch - when allow_point && "." - if non_digit - raise Error, "trailing `#{non_digit}' in number" - end - type = TkFLOAT - if peek(0) !~ /[0-9]/ - type = TkINTEGER - ungetc - num[-1, 1] = '' - break - end - allow_point = false - when allow_e && "e", allow_e && "E" - if non_digit - raise Error, "trailing `#{non_digit}' in number" - end - type = TkFLOAT - if peek(0) =~ /[+-]/ - num << getc - end - allow_e = false - allow_ri = false - allow_point = false - non_digit = ch - when allow_ri && "r" - if non_digit - raise Error, "trailing `#{non_digit}' in number" - end - type = TkRATIONAL - if peek(0) == 'i' - type = TkIMAGINARY - num << getc - end - break - when allow_ri && "i" - if non_digit && non_digit != "r" - raise Error, "trailing `#{non_digit}' in number" - end - type = TkIMAGINARY - break - else - if non_digit - raise Error, "trailing `#{non_digit}' in number" - end - ungetc - num[-1, 1] = '' - break - end - end - - Token(type, num) - end - - def identify_string(ltype, quoted = ltype, type = nil) - close = PERCENT_PAREN.values.include?(quoted) - @ltype = ltype - @quoted = quoted - - str = if ltype == quoted and %w[" ' / `].include? ltype and type.nil? then - ltype.dup - else - "%#{type}#{PERCENT_PAREN_REV[quoted]||quoted}" - end - - subtype = nil - begin - nest = 0 - - while ch = getc - str << ch - - if @quoted == ch and nest <= 0 - break - elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#" - ch = getc - if ch == "{" then - subtype = true - str << ch << skip_inner_expression - next - else - ungetc - end - elsif ch == '\\' - case @ltype - when "'" then - case ch = getc - when "'", '\\' then - str << ch - else - str << ch - end - else - str << read_escape - end - end - - if close then - if PERCENT_PAREN[ch] == @quoted - nest += 1 - elsif ch == @quoted - nest -= 1 - end - end - end - - if @ltype == "/" - while peek(0) =~ /i|m|x|o|e|s|u|n/ - str << getc - end - end - - if peek(0) == ':' and !peek_match?(/^::/) and :EXPR_BEG == @lex_state and !@after_question - str.concat getc - @lex_state = :EXPR_ARG if peek_match?(/\s*:/) - Token(TkSYMBOL, str) - elsif subtype - @lex_state = :EXPR_END - Token(DLtype2Token[ltype], str) - else - @lex_state = :EXPR_END - Token(Ltype2Token[ltype], str) - end - ensure - @ltype = nil - @quoted = nil - end - end - - def skip_inner_expression - res = "" - nest = 0 - while ch = getc - res << ch - if ch == '}' - break if nest.zero? - nest -= 1 - elsif ch == '{' - nest += 1 - end - end - res - end - - def identify_comment - @ltype = "#" - - comment = '#' - - while ch = getc - # if ch == "\\" #" - # read_escape - # end - if ch == "\n" - @ltype = nil - ungetc - break - end - - comment << ch - end - - return Token(TkCOMMENT, comment) - end - - def read_escape - escape = '' - ch = getc - - case ch - when "\n", "\r", "\f" - escape << ch - when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #" - escape << ch - when /[0-7]/ - ungetc ch - 3.times do - ch = getc - case ch - when /[0-7]/ - escape << ch - when nil - break - else - ungetc - break - end - end - - when "x" - escape << ch - - 2.times do - ch = getc - case ch - when /[0-9a-fA-F]/ - escape << ch - when nil - break - else - ungetc - break - end - end - - when "M" - escape << ch - - ch = getc - if ch != '-' - ungetc - else - escape << ch - - ch = getc - if ch == "\\" #" - ungetc - escape << read_escape - else - escape << ch - end - end - - when "C", "c" #, "^" - escape << ch - - if ch == "C" - ch = getc - - if ch == "-" - escape << ch - ch = getc - escape << ch - - escape << read_escape if ch == "\\" - else - ungetc - end - elsif (ch = getc) == "\\" #" - escape << ch << read_escape - end - else - escape << ch - - # other characters - end - - escape - end - - # :startdoc: - -end - -#RDoc::RubyLex.debug_level = 1 diff --git a/rdoc.gemspec b/rdoc.gemspec index 3c86d49444..4909d695b3 100644 --- a/rdoc.gemspec +++ b/rdoc.gemspec @@ -26,7 +26,7 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat s.executables = ["rdoc", "ri"] s.require_paths = ["lib"] # for ruby core repository. It was generated by `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } - s.files = [".document", ".gitignore", ".travis.yml", "CONTRIBUTING.rdoc", "CVE-2013-0256.rdoc", "ExampleMarkdown.md", "ExampleRDoc.rdoc", "Gemfile", "History.rdoc", "LEGAL.rdoc", "LICENSE.rdoc", "README.rdoc", "RI.rdoc", "Rakefile", "TODO.rdoc", "bin/console", "bin/setup", "exe/rdoc", "exe/ri", "lib/gauntlet_rdoc.rb", "lib/rdoc.rb", "lib/rdoc/alias.rb", "lib/rdoc/anon_class.rb", "lib/rdoc/any_method.rb", "lib/rdoc/attr.rb", "lib/rdoc/class_module.rb", "lib/rdoc/code_object.rb", "lib/rdoc/code_objects.rb", "lib/rdoc/comment.rb", "lib/rdoc/constant.rb", "lib/rdoc/context.rb", "lib/rdoc/context/section.rb", "lib/rdoc/cross_reference.rb", "lib/rdoc/encoding.rb", "lib/rdoc/erb_partial.rb", "lib/rdoc/erbio.rb", "lib/rdoc/extend.rb", "lib/rdoc/generator.rb", "lib/rdoc/generator/darkfish.rb", "lib/rdoc/generator/json_index.rb", "lib/rdoc/generator/markup.rb", "lib/rdoc/generator/pot.rb", "lib/rdoc/generator/pot/message_extractor.rb", "lib/rdoc/generator/pot/po.rb", "lib/rdoc/generator/pot/po_entry.rb", "lib/rdoc/generator/ri.rb", "lib/rdoc/generator/template/darkfish/.document", "lib/rdoc/generator/template/darkfish/_footer.rhtml", "lib/rdoc/generator/template/darkfish/_head.rhtml","lib/rdoc/generator/template/darkfish/_sidebar_VCS_info.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_classes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_extends.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_in_files.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_includes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_installed.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_methods.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_navigation.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_pages.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_parent.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_sections.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_table_of_contents.rhtml", "lib/rdoc/generator/template/darkfish/class.rhtml", "lib/rdoc/generator/template/darkfish/css/fonts.css", "lib/rdoc/generator/template/darkfish/css/rdoc.css", "lib/rdoc/generator/template/darkfish/fonts/Lato-Light.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-LightItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-Regular.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-RegularItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Bold.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Regular.ttf", "lib/rdoc/generator/template/darkfish/images/add.png", "lib/rdoc/generator/template/darkfish/images/arrow_up.png", "lib/rdoc/generator/template/darkfish/images/brick.png", "lib/rdoc/generator/template/darkfish/images/brick_link.png", "lib/rdoc/generator/template/darkfish/images/bug.png", "lib/rdoc/generator/template/darkfish/images/bullet_black.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_minus.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_plus.png", "lib/rdoc/generator/template/darkfish/images/date.png", "lib/rdoc/generator/template/darkfish/images/delete.png", "lib/rdoc/generator/template/darkfish/images/find.png", "lib/rdoc/generator/template/darkfish/images/loadingAnimation.gif", "lib/rdoc/generator/template/darkfish/images/macFFBgHack.png", "lib/rdoc/generator/template/darkfish/images/package.png", "lib/rdoc/generator/template/darkfish/images/page_green.png", "lib/rdoc/generator/template/darkfish/images/page_white_text.png", "lib/rdoc/generator/template/darkfish/images/page_white_width.png", "lib/rdoc/generator/template/darkfish/images/plugin.png", "lib/rdoc/generator/template/darkfish/images/ruby.png", "lib/rdoc/generator/template/darkfish/images/tag_blue.png", "lib/rdoc/generator/template/darkfish/images/tag_green.png", "lib/rdoc/generator/template/darkfish/images/transparent.png", "lib/rdoc/generator/template/darkfish/images/wrench.png", "lib/rdoc/generator/template/darkfish/images/wrench_orange.png", "lib/rdoc/generator/template/darkfish/images/zoom.png", "lib/rdoc/generator/template/darkfish/index.rhtml", "lib/rdoc/generator/template/darkfish/js/darkfish.js", "lib/rdoc/generator/template/darkfish/js/jquery.js", "lib/rdoc/generator/template/darkfish/js/search.js", "lib/rdoc/generator/template/darkfish/page.rhtml", "lib/rdoc/generator/template/darkfish/servlet_not_found.rhtml", "lib/rdoc/generator/template/darkfish/servlet_root.rhtml", "lib/rdoc/generator/template/darkfish/table_of_contents.rhtml", "lib/rdoc/generator/template/json_index/.document", "lib/rdoc/generator/template/json_index/js/navigation.js", "lib/rdoc/generator/template/json_index/js/searcher.js", "lib/rdoc/ghost_method.rb", "lib/rdoc/i18n.rb", "lib/rdoc/i18n/locale.rb", "lib/rdoc/i18n/text.rb", "lib/rdoc/include.rb", "lib/rdoc/known_classes.rb", "lib/rdoc/markdown.kpeg", "lib/rdoc/markdown/entities.rb", "lib/rdoc/markdown/literals.kpeg", "lib/rdoc/markdown/literals.rb", "lib/rdoc/markup.rb", "lib/rdoc/markup/attr_changer.rb", "lib/rdoc/markup/attr_span.rb", "lib/rdoc/markup/attribute_manager.rb", "lib/rdoc/markup/attributes.rb", "lib/rdoc/markup/blank_line.rb", "lib/rdoc/markup/block_quote.rb", "lib/rdoc/markup/document.rb", "lib/rdoc/markup/formatter.rb", "lib/rdoc/markup/formatter_test_case.rb", "lib/rdoc/markup/hard_break.rb", "lib/rdoc/markup/heading.rb", "lib/rdoc/markup/include.rb", "lib/rdoc/markup/indented_paragraph.rb", "lib/rdoc/markup/inline.rb", "lib/rdoc/markup/list.rb", "lib/rdoc/markup/list_item.rb", "lib/rdoc/markup/paragraph.rb", "lib/rdoc/markup/parser.rb", "lib/rdoc/markup/pre_process.rb", "lib/rdoc/markup/raw.rb", "lib/rdoc/markup/rule.rb", "lib/rdoc/markup/special.rb", "lib/rdoc/markup/text_formatter_test_case.rb", "lib/rdoc/markup/to_ansi.rb", "lib/rdoc/markup/to_bs.rb", "lib/rdoc/markup/to_html.rb", "lib/rdoc/markup/to_html_crossref.rb", "lib/rdoc/markup/to_html_snippet.rb", "lib/rdoc/markup/to_joined_paragraph.rb", "lib/rdoc/markup/to_label.rb", "lib/rdoc/markup/to_markdown.rb", "lib/rdoc/markup/to_rdoc.rb", "lib/rdoc/markup/to_table_of_contents.rb", "lib/rdoc/markup/to_test.rb", "lib/rdoc/markup/to_tt_only.rb", "lib/rdoc/markup/verbatim.rb", "lib/rdoc/meta_method.rb", "lib/rdoc/method_attr.rb", "lib/rdoc/mixin.rb", "lib/rdoc/normal_class.rb", "lib/rdoc/normal_module.rb", "lib/rdoc/options.rb", "lib/rdoc/parser.rb", "lib/rdoc/parser/c.rb", "lib/rdoc/parser/changelog.rb", "lib/rdoc/parser/markdown.rb", "lib/rdoc/parser/rd.rb", "lib/rdoc/parser/ruby.rb", "lib/rdoc/parser/ruby_tools.rb", "lib/rdoc/parser/simple.rb", "lib/rdoc/parser/text.rb", "lib/rdoc/rd.rb", "lib/rdoc/rd/block_parser.ry", "lib/rdoc/rd/inline.rb", "lib/rdoc/rd/inline_parser.ry", "lib/rdoc/rdoc.rb", "lib/rdoc/require.rb", "lib/rdoc/ri.rb", "lib/rdoc/ri/driver.rb", "lib/rdoc/ri/formatter.rb", "lib/rdoc/ri/paths.rb", "lib/rdoc/ri/store.rb", "lib/rdoc/ri/task.rb", "lib/rdoc/ruby_lex.rb", "lib/rdoc/ruby_token.rb", "lib/rdoc/rubygems_hook.rb", "lib/rdoc/servlet.rb", "lib/rdoc/single_class.rb", "lib/rdoc/stats.rb", "lib/rdoc/stats/normal.rb", "lib/rdoc/stats/quiet.rb", "lib/rdoc/stats/verbose.rb", "lib/rdoc/store.rb", "lib/rdoc/task.rb", "lib/rdoc/test_case.rb", "lib/rdoc/text.rb", "lib/rdoc/token_stream.rb", "lib/rdoc/tom_doc.rb", "lib/rdoc/top_level.rb", "rdoc.gemspec"] + s.files = [".document", ".gitignore", ".travis.yml", "CONTRIBUTING.rdoc", "CVE-2013-0256.rdoc", "ExampleMarkdown.md", "ExampleRDoc.rdoc", "Gemfile", "History.rdoc", "LEGAL.rdoc", "LICENSE.rdoc", "README.rdoc", "RI.rdoc", "Rakefile", "TODO.rdoc", "bin/console", "bin/setup", "exe/rdoc", "exe/ri", "lib/gauntlet_rdoc.rb", "lib/rdoc.rb", "lib/rdoc/alias.rb", "lib/rdoc/anon_class.rb", "lib/rdoc/any_method.rb", "lib/rdoc/attr.rb", "lib/rdoc/class_module.rb", "lib/rdoc/code_object.rb", "lib/rdoc/code_objects.rb", "lib/rdoc/comment.rb", "lib/rdoc/constant.rb", "lib/rdoc/context.rb", "lib/rdoc/context/section.rb", "lib/rdoc/cross_reference.rb", "lib/rdoc/encoding.rb", "lib/rdoc/erb_partial.rb", "lib/rdoc/erbio.rb", "lib/rdoc/extend.rb", "lib/rdoc/generator.rb", "lib/rdoc/generator/darkfish.rb", "lib/rdoc/generator/json_index.rb", "lib/rdoc/generator/markup.rb", "lib/rdoc/generator/pot.rb", "lib/rdoc/generator/pot/message_extractor.rb", "lib/rdoc/generator/pot/po.rb", "lib/rdoc/generator/pot/po_entry.rb", "lib/rdoc/generator/ri.rb", "lib/rdoc/generator/template/darkfish/.document", "lib/rdoc/generator/template/darkfish/_footer.rhtml", "lib/rdoc/generator/template/darkfish/_head.rhtml","lib/rdoc/generator/template/darkfish/_sidebar_VCS_info.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_classes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_extends.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_in_files.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_includes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_installed.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_methods.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_navigation.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_pages.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_parent.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_sections.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_table_of_contents.rhtml", "lib/rdoc/generator/template/darkfish/class.rhtml", "lib/rdoc/generator/template/darkfish/css/fonts.css", "lib/rdoc/generator/template/darkfish/css/rdoc.css", "lib/rdoc/generator/template/darkfish/fonts/Lato-Light.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-LightItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-Regular.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-RegularItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Bold.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Regular.ttf", "lib/rdoc/generator/template/darkfish/images/add.png", "lib/rdoc/generator/template/darkfish/images/arrow_up.png", "lib/rdoc/generator/template/darkfish/images/brick.png", "lib/rdoc/generator/template/darkfish/images/brick_link.png", "lib/rdoc/generator/template/darkfish/images/bug.png", "lib/rdoc/generator/template/darkfish/images/bullet_black.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_minus.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_plus.png", "lib/rdoc/generator/template/darkfish/images/date.png", "lib/rdoc/generator/template/darkfish/images/delete.png", "lib/rdoc/generator/template/darkfish/images/find.png", "lib/rdoc/generator/template/darkfish/images/loadingAnimation.gif", "lib/rdoc/generator/template/darkfish/images/macFFBgHack.png", "lib/rdoc/generator/template/darkfish/images/package.png", "lib/rdoc/generator/template/darkfish/images/page_green.png", "lib/rdoc/generator/template/darkfish/images/page_white_text.png", "lib/rdoc/generator/template/darkfish/images/page_white_width.png", "lib/rdoc/generator/template/darkfish/images/plugin.png", "lib/rdoc/generator/template/darkfish/images/ruby.png", "lib/rdoc/generator/template/darkfish/images/tag_blue.png", "lib/rdoc/generator/template/darkfish/images/tag_green.png", "lib/rdoc/generator/template/darkfish/images/transparent.png", "lib/rdoc/generator/template/darkfish/images/wrench.png", "lib/rdoc/generator/template/darkfish/images/wrench_orange.png", "lib/rdoc/generator/template/darkfish/images/zoom.png", "lib/rdoc/generator/template/darkfish/index.rhtml", "lib/rdoc/generator/template/darkfish/js/darkfish.js", "lib/rdoc/generator/template/darkfish/js/jquery.js", "lib/rdoc/generator/template/darkfish/js/search.js", "lib/rdoc/generator/template/darkfish/page.rhtml", "lib/rdoc/generator/template/darkfish/servlet_not_found.rhtml", "lib/rdoc/generator/template/darkfish/servlet_root.rhtml", "lib/rdoc/generator/template/darkfish/table_of_contents.rhtml", "lib/rdoc/generator/template/json_index/.document", "lib/rdoc/generator/template/json_index/js/navigation.js", "lib/rdoc/generator/template/json_index/js/searcher.js", "lib/rdoc/ghost_method.rb", "lib/rdoc/i18n.rb", "lib/rdoc/i18n/locale.rb", "lib/rdoc/i18n/text.rb", "lib/rdoc/include.rb", "lib/rdoc/known_classes.rb", "lib/rdoc/markdown.kpeg", "lib/rdoc/markdown/entities.rb", "lib/rdoc/markdown/literals.kpeg", "lib/rdoc/markdown/literals.rb", "lib/rdoc/markup.rb", "lib/rdoc/markup/attr_changer.rb", "lib/rdoc/markup/attr_span.rb", "lib/rdoc/markup/attribute_manager.rb", "lib/rdoc/markup/attributes.rb", "lib/rdoc/markup/blank_line.rb", "lib/rdoc/markup/block_quote.rb", "lib/rdoc/markup/document.rb", "lib/rdoc/markup/formatter.rb", "lib/rdoc/markup/formatter_test_case.rb", "lib/rdoc/markup/hard_break.rb", "lib/rdoc/markup/heading.rb", "lib/rdoc/markup/include.rb", "lib/rdoc/markup/indented_paragraph.rb", "lib/rdoc/markup/inline.rb", "lib/rdoc/markup/list.rb", "lib/rdoc/markup/list_item.rb", "lib/rdoc/markup/paragraph.rb", "lib/rdoc/markup/parser.rb", "lib/rdoc/markup/pre_process.rb", "lib/rdoc/markup/raw.rb", "lib/rdoc/markup/rule.rb", "lib/rdoc/markup/special.rb", "lib/rdoc/markup/text_formatter_test_case.rb", "lib/rdoc/markup/to_ansi.rb", "lib/rdoc/markup/to_bs.rb", "lib/rdoc/markup/to_html.rb", "lib/rdoc/markup/to_html_crossref.rb", "lib/rdoc/markup/to_html_snippet.rb", "lib/rdoc/markup/to_joined_paragraph.rb", "lib/rdoc/markup/to_label.rb", "lib/rdoc/markup/to_markdown.rb", "lib/rdoc/markup/to_rdoc.rb", "lib/rdoc/markup/to_table_of_contents.rb", "lib/rdoc/markup/to_test.rb", "lib/rdoc/markup/to_tt_only.rb", "lib/rdoc/markup/verbatim.rb", "lib/rdoc/meta_method.rb", "lib/rdoc/method_attr.rb", "lib/rdoc/mixin.rb", "lib/rdoc/normal_class.rb", "lib/rdoc/normal_module.rb", "lib/rdoc/options.rb", "lib/rdoc/parser.rb", "lib/rdoc/parser/c.rb", "lib/rdoc/parser/changelog.rb", "lib/rdoc/parser/markdown.rb", "lib/rdoc/parser/rd.rb", "lib/rdoc/parser/ruby.rb", "lib/rdoc/parser/ruby_tools.rb", "lib/rdoc/parser/simple.rb", "lib/rdoc/parser/text.rb", "lib/rdoc/rd.rb", "lib/rdoc/rd/block_parser.ry", "lib/rdoc/rd/inline.rb", "lib/rdoc/rd/inline_parser.ry", "lib/rdoc/rdoc.rb", "lib/rdoc/require.rb", "lib/rdoc/ri.rb", "lib/rdoc/ri/driver.rb", "lib/rdoc/ri/formatter.rb", "lib/rdoc/ri/paths.rb", "lib/rdoc/ri/store.rb", "lib/rdoc/ri/task.rb", "lib/rdoc/parser/ripper_state_lex.rb", "lib/rdoc/ruby_token.rb", "lib/rdoc/rubygems_hook.rb", "lib/rdoc/servlet.rb", "lib/rdoc/single_class.rb", "lib/rdoc/stats.rb", "lib/rdoc/stats/normal.rb", "lib/rdoc/stats/quiet.rb", "lib/rdoc/stats/verbose.rb", "lib/rdoc/store.rb", "lib/rdoc/task.rb", "lib/rdoc/test_case.rb", "lib/rdoc/text.rb", "lib/rdoc/token_stream.rb", "lib/rdoc/tom_doc.rb", "lib/rdoc/top_level.rb", "rdoc.gemspec"] # files from .gitignore s.files << "lib/rdoc/rd/block_parser.rb" << "lib/rdoc/rd/inline_parser.rb" << "lib/rdoc/markdown.rb" From 9bae9dcb83f9bce338767610301302ea8aa8708e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 6 Jul 2017 10:07:12 +0900 Subject: [PATCH 065/151] Fix for test_parse_symbol_in_paren_arg --- lib/rdoc/parser/ruby.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 5802bdc1bb..f4bba58779 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1796,10 +1796,10 @@ def parse_symbol_arg_paren no # :nodoc: end skip_tkspace_comment - case tk2 = get_tk - when TkRPAREN + case (tk2 = get_tk)[:kind] + when :on_rparen break - when TkCOMMA + when :on_comma else warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC break From 9d54fa31484416d9205fe142444d12e7d2119210 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 6 Jul 2017 10:07:27 +0900 Subject: [PATCH 066/151] Fix for test_parse_for_in --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index f4bba58779..a5e74f4351 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -2107,7 +2107,7 @@ def skip_for_variable get_tk skip_tkspace false tk = get_tk - unget_tk(tk) unless TkIN === tk + unget_tk(tk) unless :on_kw == tk[:kind] and 'in' == tk[:text] end ## From 298e6984fb9d103598bd553962da07c14acfc8bb Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 9 Jul 2017 15:55:16 +0900 Subject: [PATCH 067/151] Fix postfix if/unless --- lib/rdoc/parser/ripper_state_lex.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 2a9441c5c6..fc7436fe0e 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -98,8 +98,12 @@ def on_kw(tok, data) @lex_state = EXPR_FNAME @continue = true @in_fname = true - when 'if' - @lex_state = EXPR_BEG + when 'if', 'unless' + if ((EXPR_ENDARG | EXPR_ENDFN | EXPR_CMDARG) & @lex_state) != 0 # postfix if + @lex_state = EXPR_BEG | EXPR_LABEL + else + @lex_state = EXPR_BEG + end else if @lex_state == EXPR_FNAME @lex_state = EXPR_END @@ -116,7 +120,7 @@ def on_tstring_beg(tok, data) end def on_tstring_end(tok, data) - @lex_state = EXPR_END + @lex_state = EXPR_END | EXPR_ENDARG @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end @@ -138,6 +142,7 @@ def on_int(tok, data) def on_symbeg(tok, data) @lex_state = EXPR_FNAME @continue = true + @in_fname = true @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end From c658e09f4e6b770701fc15fe7907bc3810114add Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 9 Jul 2017 15:56:31 +0900 Subject: [PATCH 068/151] Detect postfix if/unless --- lib/rdoc/parser/ruby.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index a5e74f4351..db4cef5b02 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1703,7 +1703,9 @@ def parse_statements(container, single = NORMAL, current_method = nil, skip_optional_do_after_expression when 'case', 'do', 'if', 'unless', 'begin' then - nest += 1 + if (RipperStateLex::EXPR_LABEL & tk[:state]) == 0 + nest += 1 + end when 'super' then current_method.calls_super = true if current_method From 4d8fc86670188de7f45138c4f8b08268e8be22ff Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 9 Jul 2017 15:57:17 +0900 Subject: [PATCH 069/151] Fix condition for invalid class exception --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index db4cef5b02..71b15cf201 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -364,7 +364,7 @@ def get_class_or_module container, ignore_constants = false get_tk skip_tkspace false name_t = get_tk - if name_t[:kind] != :on_const + unless :on_const == name_t[:kind] || :on_ident == name_t[:kind] raise RDoc::Error, "Invalid class or module definition: #{given_name}" end given_name << '::' << name_t[:text] From 905b5cf5956f66e8221c5a7ad477e41d758250b0 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 11 Jul 2017 14:57:22 +0900 Subject: [PATCH 070/151] Split off dynamic string token from tstring --- lib/rdoc/parser/ripper_state_lex.rb | 6 +++++- lib/rdoc/parser/ruby.rb | 26 +++++++------------------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index fc7436fe0e..e37b06aaa5 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -233,6 +233,7 @@ def get_squashed_tk when :on_tstring_beg then string = tk[:text] state = nil + expanded = false loop do inner_str_tk = get_squashed_tk if inner_str_tk.nil? @@ -243,9 +244,12 @@ def get_squashed_tk break else string = string + inner_str_tk[:text] + if :on_tstring_content != inner_str_tk[:kind] then + expanded = true + end end end - tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_tstring, :text => string, :state => state } + tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => expanded ? :on_dstring : :on_tstring, :text => string, :state => state } when :on_regexp_beg then string = tk[:text] state = nil diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 71b15cf201..b3f85fa47a 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -536,10 +536,8 @@ def get_symbol_or_name text when :on_ident, :on_op then tk[:text] - when :on_tstring_beg then - tk = get_tk # :on_tstring_content - get_tk # skip :on_tstring_end - tk[:text] + when :on_tstring, :on_dstring then + tk[:text][1..-2] else raise RDoc::Error, "Name or symbol expected (got #{tk})" end @@ -1571,9 +1569,7 @@ def parse_require(context, comment) tk = get_tk end - if :on_tstring == tk[:kind] then - name = eval tk[:text] - end + name = tk[:text][1..-2] if :on_tstring == tk[:kind] if name then @top_level.add_require RDoc::Require.new(name, comment) @@ -1848,19 +1844,11 @@ def parse_symbol_arg_space no, tk # :nodoc: def parse_symbol_in_arg tk = get_tk - if :on_symbol == tk[:kind] + if :on_symbol == tk[:kind] then tk[:text].sub(/^:/, '') - elsif :on_tstring == tk[:kind] - begin - eval tk[:text] - rescue - nil - end - elsif :on_ident == tk[:kind] then - nil # ignore - elsif :on_tstring_beg == tk[:kind] then - get_tk # skip :on_tstring_content - get_tk # skip :on_tstring_end + elsif :on_tstring == tk[:kind] then + tk[:text][1..-2] + elsif :on_dstring == tk[:kind] or :on_ident == tk[:kind] then nil # ignore else warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC From d7dc98974a4a3e357d100073f17086f9ac3f319e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 11 Jul 2017 14:59:35 +0900 Subject: [PATCH 071/151] Fix get_symbol_or_name with supporting global, class and instance variable --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index b3f85fa47a..3ea13d24ba 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -534,7 +534,7 @@ def get_symbol_or_name end text - when :on_ident, :on_op then + when :on_ident, :on_const, :on_gvar, :on_cvar, :on_ivar, :on_op then tk[:text] when :on_tstring, :on_dstring then tk[:text][1..-2] From e4e33fefd96e66aaa136f71ce093558f3585fa4e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 11 Jul 2017 15:06:52 +0900 Subject: [PATCH 072/151] Fix get_symbol_or_name with supporting keyword --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 3ea13d24ba..11fc12604c 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -534,7 +534,7 @@ def get_symbol_or_name end text - when :on_ident, :on_const, :on_gvar, :on_cvar, :on_ivar, :on_op then + when :on_ident, :on_const, :on_gvar, :on_cvar, :on_ivar, :on_op, :on_kw then tk[:text] when :on_tstring, :on_dstring then tk[:text][1..-2] From d645082f134e479317d98ab9a39377033ba35362 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Wed, 12 Jul 2017 14:53:16 +0900 Subject: [PATCH 073/151] Remove object of RubyToken::Token in parser/c.rb --- lib/rdoc/parser/c.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb index 0e72df2ecd..5c940ab28e 100644 --- a/lib/rdoc/parser/c.rb +++ b/lib/rdoc/parser/c.rb @@ -666,8 +666,7 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false #meth_obj.params = params meth_obj.start_collecting_tokens - tk = RDoc::RubyToken::Token.new nil, 1, 1 - tk.set_text body + tk = { :line_no => 1, :char_no => 1, :text => body } meth_obj.add_token tk meth_obj.comment = comment meth_obj.line = file_content[0, offset].count("\n") + 1 @@ -684,8 +683,7 @@ def find_body class_name, meth_name, meth_obj, file_content, quiet = false find_modifiers comment, meth_obj meth_obj.start_collecting_tokens - tk = RDoc::RubyToken::Token.new nil, 1, 1 - tk.set_text body + tk = { :line_no => 1, :char_no => 1, :text => body } meth_obj.add_token tk meth_obj.comment = comment meth_obj.line = file_content[0, offset].count("\n") + 1 From 262c595962cee9c9431d820c3f9777ff800cfca6 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 13 Jul 2017 21:28:57 +0900 Subject: [PATCH 074/151] Fix for test_parse_class_lower_name_warning --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 11fc12604c..0401e196ac 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -760,7 +760,7 @@ def parse_class container, single, tk, comment cls = parse_class_singleton container, name, comment end else - warn "Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}" + warn "Expected class name or '<<'. Got #{name_t[:kind]}: #{name_t[:text].inspect}" return end From 07ee8b0e1705327efdfe73d94257e4c9d47d0a94 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 13 Jul 2017 21:29:34 +0900 Subject: [PATCH 075/151] Fix rescue with comment in the same line --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 0401e196ac..af148ea109 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1587,7 +1587,7 @@ def parse_rescue nest = 0 while tk = get_tk case tk[:kind] - when :on_nl, :on_semicolon then + when :on_nl, :on_semicolon, :on_comment then break when :on_comma then skip_tkspace false From 030d9cd4150e86156e0f51aa2fc417acbbfbf55e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 18 Jul 2017 09:02:46 +0900 Subject: [PATCH 076/151] Add on_variables for on_{i,c,g}var --- lib/rdoc/parser/ripper_state_lex.rb | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index e37b06aaa5..06b52bbedb 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -146,7 +146,7 @@ def on_symbeg(tok, data) @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end - def on_ident(tok, data) + private def on_variables(event, tok, data) if @in_fname @lex_state = EXPR_ENDFN @in_fname = false @@ -162,7 +162,23 @@ def on_ident(tok, data) else @lex_state = EXPR_CMDARG end - @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state}) + end + + def on_ident(tok, data) + on_variables(__method__, tok, data) + end + + def on_ivar(tok, data) + on_variables(__method__, tok, data) + end + + def on_cvar(tok, data) + on_variables(__method__, tok, data) + end + + def on_gvar(tok, data) + on_variables(__method__, tok, data) end def on_lparen(tok, data) From 3382e3977ad68bcc23b2ef95fb85ed6201659b62 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 18 Jul 2017 09:03:38 +0900 Subject: [PATCH 077/151] Add on_comment to RipperStateLex --- lib/rdoc/parser/ripper_state_lex.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 06b52bbedb..113bcecd3b 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -207,6 +207,11 @@ def on_sp(tok, data) @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end + def on_comment(tok, data) + @lex_state = EXPR_BEG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + def on_default(event, tok, data) reset @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state}) From acc048660c2b66e8f9510e4c7b53351224bfa27b Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 18 Jul 2017 09:06:56 +0900 Subject: [PATCH 078/151] Collect up number with +/- --- lib/rdoc/parser/ripper_state_lex.rb | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 113bcecd3b..2f967c69f2 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -224,7 +224,11 @@ def each(&block) end def get_squashed_tk - tk = @inner_lex.next + if @buf.empty? + tk = @inner_lex.next + else + tk = @buf.shift + end case tk[:kind] when :on_symbeg then is_symbol = true @@ -293,11 +297,24 @@ def get_squashed_tk string = string + embdoc_tk[:text] end tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_embdoc, :text => string, :state => embdoc_tk[:state] } + when :on_op then + if tk[:text] =~ /^[-+]$/ then + tk_ahead = get_squashed_tk + case tk_ahead[:kind] + when :on_int, :on_float, :on_rational, :on_imaginary + tk[:text] += tk_ahead[:text] + tk[:kind] = tk_ahead[:kind] + tk[:state] = tk_ahead[:state] + else + @buf.unshift tk_ahead + end + end end tk end def initialize(code) + @buf = [] @inner_lex = Enumerator.new do |y| InnerStateLex.new(code).each do |tk| y << tk From f711f2313e6523908f4028321beb42f3bcf4bba7 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 18 Jul 2017 09:18:40 +0900 Subject: [PATCH 079/151] Fix string treatment --- lib/rdoc/parser/ripper_state_lex.rb | 8 ++++++-- lib/rdoc/token_stream.rb | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 2f967c69f2..b6a3159a3a 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -255,10 +255,14 @@ def get_squashed_tk if is_symbol tk = symbol_tk end - when :on_tstring_beg then + when :on_tstring_beg, :on_backtick then string = tk[:text] state = nil - expanded = false + if :on_backtick == tk[:kind] + expanded = true + else + expanded = false + end loop do inner_str_tk = get_squashed_tk if inner_str_tk.nil? diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index 5f7854603a..2bb3b398f6 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -36,7 +36,8 @@ def self.to_html token_stream when :on_ivar then 'ruby-ivar' when :on_op then 'ruby-operator' when :on_ident then 'ruby-identifier' - when :on_backref then 'ruby-node' + when :on_backref, :on_dstring + then 'ruby-node' when :on_comment then 'ruby-comment' when :on_regexp then 'ruby-regexp' when :on_tstring then 'ruby-string' From 010a7f5ad14d2d0b8c89b190c110981762b6d087 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 18 Jul 2017 09:19:52 +0900 Subject: [PATCH 080/151] Omit '=' from operater markup --- lib/rdoc/token_stream.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index 2bb3b398f6..fd83e0c161 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -34,7 +34,8 @@ def self.to_html token_stream when :on_const then 'ruby-constant' when :on_kw then 'ruby-keyword' when :on_ivar then 'ruby-ivar' - when :on_op then 'ruby-operator' + when '=' != t[:text] && :on_op + then 'ruby-operator' when :on_ident then 'ruby-identifier' when :on_backref, :on_dstring then 'ruby-node' From 1633735bfed45b77f3d2b8ce63021a57b3808f35 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 18 Jul 2017 09:21:16 +0900 Subject: [PATCH 081/151] Add EXPR_END to postfix 'if' checking --- lib/rdoc/parser/ripper_state_lex.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index b6a3159a3a..724dbe3543 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -99,7 +99,7 @@ def on_kw(tok, data) @continue = true @in_fname = true when 'if', 'unless' - if ((EXPR_ENDARG | EXPR_ENDFN | EXPR_CMDARG) & @lex_state) != 0 # postfix if + if ((EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_CMDARG) & @lex_state) != 0 # postfix if @lex_state = EXPR_BEG | EXPR_LABEL else @lex_state = EXPR_BEG From 6959f7278f0ea21e1f08971602c5e2f5bf735a5b Mon Sep 17 00:00:00 2001 From: Code Ass Date: Wed, 19 Jul 2017 19:17:27 +0900 Subject: [PATCH 082/151] Use autoload for RipperStateLex --- lib/rdoc.rb | 2 +- lib/rdoc/markup/to_html.rb | 3 +-- lib/rdoc/parser/ripper_state_lex.rb | 4 ++-- lib/rdoc/parser/ruby.rb | 13 ++++++------- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/rdoc.rb b/lib/rdoc.rb index 8482e910f5..3bb24e749f 100644 --- a/lib/rdoc.rb +++ b/lib/rdoc.rb @@ -148,7 +148,7 @@ def self.load_yaml autoload :KNOWN_CLASSES, 'rdoc/known_classes' - autoload :RubyLex, 'rdoc/ruby_lex' + autoload :RipperStateLex, 'rdoc/parser/ripper_state_lex' autoload :RubyToken, 'rdoc/ruby_token' autoload :TokenStream, 'rdoc/token_stream' diff --git a/lib/rdoc/markup/to_html.rb b/lib/rdoc/markup/to_html.rb index 3980b9341e..245d43e3b9 100644 --- a/lib/rdoc/markup/to_html.rb +++ b/lib/rdoc/markup/to_html.rb @@ -1,6 +1,5 @@ # frozen_string_literal: false require 'cgi' -require 'rdoc/parser/ripper_state_lex' ## # Outputs RDoc markup as HTML. @@ -201,7 +200,7 @@ def accept_verbatim verbatim content = if verbatim.ruby? or parseable? text then begin - tokens = RipperStateLex.parse text + tokens = RDoc::RipperStateLex.parse text klass = ' class="ruby"' result = RDoc::TokenStream.to_html tokens diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 724dbe3543..e6ef739ed9 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -1,6 +1,6 @@ require 'ripper' -class RipperStateLex +class RDoc::RipperStateLex EXPR_NONE = 0 EXPR_BEG = 1 EXPR_END = 2 @@ -327,7 +327,7 @@ def initialize(code) end def self.parse(code) - lex = RipperStateLex.new(code) + lex = self.new(code) tokens = [] begin while tk = lex.get_squashed_tk diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index af148ea109..ccf2af2e40 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -141,7 +141,6 @@ # standard rdocable item following it. require 'ripper' -require_relative 'ripper_state_lex' class RDoc::Parser::Ruby < RDoc::Parser @@ -168,7 +167,7 @@ def initialize(top_level, file_name, content, options, stats) @size = 0 @token_listeners = nil - @scanner = RipperStateLex.parse(content) + @scanner = RDoc::RipperStateLex.parse(content) @scanner_point = 0 @prev_seek = nil @markup = @options.markup @@ -720,9 +719,9 @@ def parse_call_parameters(tk) when end_token if end_token == :on_rparen nest -= 1 - break if RipperStateLex.end?(tk) and nest <= 0 + break if RDoc::RipperStateLex.end?(tk) and nest <= 0 else - break if RipperStateLex.end?(tk) + break if RDoc::RipperStateLex.end?(tk) end when :on_comment, :on_embdoc unget_tk(tk) @@ -919,7 +918,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: :on_kw == tk[:kind] && 'end' == tk[:text] then nest -= 1 elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then - if nest <= 0 and RipperStateLex.end?(tk) then + if nest <= 0 and RDoc::RipperStateLex.end?(tk) then unget_tk tk break else @@ -935,7 +934,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: break end elsif :on_nl == tk[:kind] then - if nest <= 0 and RipperStateLex.end?(tk) then + if nest <= 0 and RDoc::RipperStateLex.end?(tk) then unget_tk tk break end @@ -1699,7 +1698,7 @@ def parse_statements(container, single = NORMAL, current_method = nil, skip_optional_do_after_expression when 'case', 'do', 'if', 'unless', 'begin' then - if (RipperStateLex::EXPR_LABEL & tk[:state]) == 0 + if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0 nest += 1 end From 4f62d1c3df1883f51d54f991727ebd553cbd6d40 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Wed, 19 Jul 2017 19:38:45 +0900 Subject: [PATCH 083/151] Fix for test_parse_method_bracket --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index ccf2af2e40..da15b3e54e 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1402,7 +1402,7 @@ def parse_method_name container # :nodoc: # is parsed from the token stream for a regular method. def parse_method_name_regular container, name_t # :nodoc: - if name_t[:kind] == :on_op && (name_t[:text] == '*' || name_t[:text] == '&') then + if :on_op == name_t[:kind] && (%w{* & [] []= <<}.include?(name_t[:text])) then name_t[:text] else unless [:on_kw, :on_const, :on_ident].include?(name_t[:kind]) then From bdd07e5cf162816ea91415431af116e50d965e53 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Wed, 19 Jul 2017 19:39:22 +0900 Subject: [PATCH 084/151] Support rational and imaginary --- lib/rdoc/token_stream.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index fd83e0c161..8b83e428ff 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -43,6 +43,7 @@ def self.to_html token_stream when :on_regexp then 'ruby-regexp' when :on_tstring then 'ruby-string' when :on_int, :on_float, + :on_rational, :on_imaginary, :on_embdoc, :on_symbol, :on_CHAR then 'ruby-value' end From 9c038833526eee6e174e8e032be7837e578e892d Mon Sep 17 00:00:00 2001 From: Code Ass Date: Wed, 19 Jul 2017 19:39:48 +0900 Subject: [PATCH 085/151] Fix for Ripper token access --- test/test_rdoc_parser_c.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_rdoc_parser_c.rb b/test/test_rdoc_parser_c.rb index d0732597dc..5341da16e2 100644 --- a/test/test_rdoc_parser_c.rb +++ b/test/test_rdoc_parser_c.rb @@ -1037,7 +1037,7 @@ def test_find_body other_function.comment.text assert_equal '()', other_function.params - code = other_function.token_stream.first.text + code = other_function.token_stream.first[:text] assert_equal "VALUE\nother_function() {\n}", code end @@ -1107,7 +1107,7 @@ def test_find_body_cast other_function.comment.text assert_equal '()', other_function.params - code = other_function.token_stream.first.text + code = other_function.token_stream.first[:text] assert_equal "VALUE\nother_function() {\n}", code end @@ -1141,7 +1141,7 @@ def test_find_body_define assert_equal '()', other_function.params assert_equal 8, other_function.line - code = other_function.token_stream.first.text + code = other_function.token_stream.first[:text] assert_equal "VALUE\nrb_other_function() {\n}", code end @@ -1174,7 +1174,7 @@ def test_find_body_define_comment assert_equal '()', other_function.params assert_equal 4, other_function.line - code = other_function.token_stream.first.text + code = other_function.token_stream.first[:text] assert_equal "#define other_function rb_other_function", code end @@ -1314,7 +1314,7 @@ def test_find_body_macro other_function.comment.text assert_equal '()', other_function.params - code = other_function.token_stream.first.text + code = other_function.token_stream.first[:text] assert_equal "DLL_LOCAL VALUE\nother_function() {\n}", code end From 4a3210ae254a3938ec821af1334a2292edb0a24b Mon Sep 17 00:00:00 2001 From: Code Ass Date: Wed, 19 Jul 2017 19:40:00 +0900 Subject: [PATCH 086/151] Skip all tests for RubyLex --- test/test_rdoc_ruby_lex.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb index 130ee84d84..cf517c9d4e 100644 --- a/test/test_rdoc_ruby_lex.rb +++ b/test/test_rdoc_ruby_lex.rb @@ -6,6 +6,7 @@ class TestRDocRubyLex < RDoc::TestCase def setup + raise MiniTest::Skip @TK = RDoc::RubyToken end From f2a0b082326376208520b2c9459a36a6a142de15 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sat, 22 Jul 2017 14:12:30 +0900 Subject: [PATCH 087/151] Squash dwrods with dstring --- lib/rdoc/parser/ripper_state_lex.rb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index e6ef739ed9..226ed81bd1 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -301,6 +301,33 @@ def get_squashed_tk string = string + embdoc_tk[:text] end tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_embdoc, :text => string, :state => embdoc_tk[:state] } + when :on_qwords_beg then + string_array = [] + start_token = tk[:text] + start_quote = tk[:text][-1] + line_no = tk[:line_no] + char_no = tk[:char_no] + state = tk[:state] + end_quote = + case start_quote + when ?( then ?) + when ?[ then ?] + when ?{ then ?} + when ?< then ?> + else start_quote + end + loop do + tk = get_squashed_tk + if tk.nil? + break + elsif :on_tstring_content == tk[:kind] then + string_array << tk[:text] + elsif :on_words_sep == tk[:kind] and end_quote == tk[:text] then + break + end + end + text = "#{start_token}#{string_array.join(' ')}#{end_quote}" + tk = { :line_no => line_no, :char_no => char_no, :kind => :on_dstring, :text => text, :state => state } when :on_op then if tk[:text] =~ /^[-+]$/ then tk_ahead = get_squashed_tk From 012ccea63674dd7fb3c869c0a6cb30e463d07ea9 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sat, 22 Jul 2017 14:29:24 +0900 Subject: [PATCH 088/151] Supports % notation with unnecessary spaces --- lib/rdoc/parser/ripper_state_lex.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 226ed81bd1..2b789c1f62 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -304,7 +304,7 @@ def get_squashed_tk when :on_qwords_beg then string_array = [] start_token = tk[:text] - start_quote = tk[:text][-1] + start_quote = tk[:text].rstrip[-1] line_no = tk[:line_no] char_no = tk[:char_no] state = tk[:state] @@ -316,17 +316,20 @@ def get_squashed_tk when ?< then ?> else start_quote end + end_token = nil loop do tk = get_squashed_tk if tk.nil? + end_token = end_quote break elsif :on_tstring_content == tk[:kind] then string_array << tk[:text] - elsif :on_words_sep == tk[:kind] and end_quote == tk[:text] then + elsif :on_words_sep == tk[:kind] and end_quote == tk[:text].strip then + end_token = tk[:text] break end end - text = "#{start_token}#{string_array.join(' ')}#{end_quote}" + text = "#{start_token}#{string_array.join(' ')}#{end_token}" tk = { :line_no => line_no, :char_no => char_no, :kind => :on_dstring, :text => text, :state => state } when :on_op then if tk[:text] =~ /^[-+]$/ then From 7df38a8ed031acb4ffa596bdc305afe1fe2797fe Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 23 Jul 2017 00:46:25 +0900 Subject: [PATCH 089/151] Treat operation after dot as method --- lib/rdoc/token_stream.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index 8b83e428ff..88548e4a60 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -34,8 +34,12 @@ def self.to_html token_stream when :on_const then 'ruby-constant' when :on_kw then 'ruby-keyword' when :on_ivar then 'ruby-ivar' - when '=' != t[:text] && :on_op - then 'ruby-operator' + when '=' != t[:text] && :on_op then + if RDoc::RipperStateLex::EXPR_ARG == t[:state] then + 'ruby-identifier' + else + 'ruby-operator' + end when :on_ident then 'ruby-identifier' when :on_backref, :on_dstring then 'ruby-node' From 80f68418240923ccf3a4fe79e0d06696c89efd82 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 01:08:07 +0900 Subject: [PATCH 090/151] Handle JSON-style hash key as a value in HTML --- lib/rdoc/token_stream.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index 88548e4a60..601d35cb3d 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -41,6 +41,7 @@ def self.to_html token_stream 'ruby-operator' end when :on_ident then 'ruby-identifier' + when :on_label then 'ruby-value' when :on_backref, :on_dstring then 'ruby-node' when :on_comment then 'ruby-comment' From 93e787d1b6f766cfbb6e024cf3ac8e09fa7682cd Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 01:10:57 +0900 Subject: [PATCH 091/151] Fix for argument default value with reserved word --- lib/rdoc/parser/ruby.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index da15b3e54e..8796891f23 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1503,9 +1503,6 @@ def parse_method_or_yield_parameters(method = nil, read_documentation_modifiers method, modifiers end @read.pop - when :on_kw - unget_tk tk - break end tk = get_tk end From b318cf47d49b1722939733523c531c5f2f754f39 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 01:54:12 +0900 Subject: [PATCH 092/151] Treat global variable as identifier in HTML --- lib/rdoc/token_stream.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index 601d35cb3d..c7612a07e5 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -34,6 +34,7 @@ def self.to_html token_stream when :on_const then 'ruby-constant' when :on_kw then 'ruby-keyword' when :on_ivar then 'ruby-ivar' + when :on_gvar then 'ruby-identifier' when '=' != t[:text] && :on_op then if RDoc::RipperStateLex::EXPR_ARG == t[:state] then 'ruby-identifier' From 7309e8397833c4048d2522d987cc90cae8bbeb1e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 06:11:52 +0900 Subject: [PATCH 093/151] Fix %w[] treatment --- lib/rdoc/parser/ripper_state_lex.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 2b789c1f62..1b3c289316 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -302,7 +302,7 @@ def get_squashed_tk end tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_embdoc, :text => string, :state => embdoc_tk[:state] } when :on_qwords_beg then - string_array = [] + string = '' start_token = tk[:text] start_quote = tk[:text].rstrip[-1] line_no = tk[:line_no] @@ -323,13 +323,17 @@ def get_squashed_tk end_token = end_quote break elsif :on_tstring_content == tk[:kind] then - string_array << tk[:text] - elsif :on_words_sep == tk[:kind] and end_quote == tk[:text].strip then - end_token = tk[:text] - break + string += tk[:text] + elsif :on_words_sep == tk[:kind] then + if end_quote == tk[:text].strip then + end_token = tk[:text] + break + else + string += tk[:text] + end end end - text = "#{start_token}#{string_array.join(' ')}#{end_token}" + text = "#{start_token}#{string}#{end_token}" tk = { :line_no => line_no, :char_no => char_no, :kind => :on_dstring, :text => text, :state => state } when :on_op then if tk[:text] =~ /^[-+]$/ then From 3e036cfd49dcd70ae772ec923cd5ee257ff0d27b Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 06:14:34 +0900 Subject: [PATCH 094/151] Fix for on_nl ending with comment --- lib/rdoc/parser/ruby.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 8796891f23..a66bf93fc1 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1498,7 +1498,9 @@ def parse_method_or_yield_parameters(method = nil, when :on_rparen then nest -= 1 when :on_comment, :on_embdoc then - if method && method.block_params.nil? then + if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] then + break + elsif method && method.block_params.nil? then unget_tk tk read_documentation_modifiers method, modifiers end @@ -2076,6 +2078,10 @@ def skip_optional_do_after_expression break #break unless @scanner.continue end + when :on_comment, :on_embdoc then + if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] then + break + end end tk = get_tk end From 54ee3291e2f05e92b375adab3bc1d698195fc29b Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 06:15:10 +0900 Subject: [PATCH 095/151] Treat class variable as identifier --- lib/rdoc/token_stream.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index c7612a07e5..5a3ee10e0b 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -34,6 +34,7 @@ def self.to_html token_stream when :on_const then 'ruby-constant' when :on_kw then 'ruby-keyword' when :on_ivar then 'ruby-ivar' + when :on_cvar then 'ruby-identifier' when :on_gvar then 'ruby-identifier' when '=' != t[:text] && :on_op then if RDoc::RipperStateLex::EXPR_ARG == t[:state] then From bb4cf2910144254bd0322ec98114ab3f53096323 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 09:43:42 +0900 Subject: [PATCH 096/151] Split get_squashed_tk --- lib/rdoc/parser/ripper_state_lex.rb | 266 +++++++++++++++++----------- 1 file changed, 161 insertions(+), 105 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 1b3c289316..ac98127dfd 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -231,121 +231,177 @@ def get_squashed_tk end case tk[:kind] when :on_symbeg then - is_symbol = true - symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol } - case (tk1 = get_squashed_tk)[:kind] - when :on_ident - symbol_tk[:text] = ":#{tk1[:text]}" - symbol_tk[:state] = tk1[:state] - when :on_tstring_content - symbol_tk[:text] = ":#{tk1[:text]}" - symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end - when :on_tstring_end - symbol_tk[:text] = ":#{tk1[:text]}" - symbol_tk[:state] = tk1[:state] - when :on_op - symbol_tk[:text] = ":#{tk1[:text]}" - symbol_tk[:state] = tk1[:state] - #when :on_symbols_beg - #when :on_qsymbols_beg - else - is_symbol = false - tk = tk1 - end - if is_symbol - tk = symbol_tk - end + tk = get_symbol_tk(tk) when :on_tstring_beg, :on_backtick then - string = tk[:text] - state = nil - if :on_backtick == tk[:kind] - expanded = true + tk = get_string_tk(tk) + tk = get_string_tk(tk) + when :on_regexp_beg then + tk = get_regexp_tk(tk) + when :on_embdoc_beg then + tk = get_embdoc_tk(tk) + when :on_qwords_beg then + tk = get_qwords_tk(tk) + when :on_op then + tk = get_op_tk(tk) + end + tk + end + + private def get_symbol_tk(tk) + is_symbol = true + symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol } + case (tk1 = get_squashed_tk)[:kind] + when :on_ident + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_tstring_content + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end + when :on_tstring_end + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_op + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + #when :on_symbols_beg + #when :on_qsymbols_beg + else + is_symbol = false + tk = tk1 + end + if is_symbol + tk = symbol_tk + end + tk + end + + private def get_string_tk(tk) + string = tk[:text] + state = nil + if :on_backtick == tk[:kind] + expanded = true + else + expanded = false + end + loop do + inner_str_tk = get_squashed_tk + if inner_str_tk.nil? + break + elsif :on_tstring_end == inner_str_tk[:kind] + string = string + inner_str_tk[:text] + state = inner_str_tk[:state] + break else - expanded = false - end - loop do - inner_str_tk = get_squashed_tk - if inner_str_tk.nil? - break - elsif :on_tstring_end == inner_str_tk[:kind] - string = string + inner_str_tk[:text] - state = inner_str_tk[:state] - break - else - string = string + inner_str_tk[:text] - if :on_tstring_content != inner_str_tk[:kind] then - expanded = true - end + string = string + inner_str_tk[:text] + if :on_tstring_content != inner_str_tk[:kind] then + expanded = true end end - tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => expanded ? :on_dstring : :on_tstring, :text => string, :state => state } - when :on_regexp_beg then - string = tk[:text] - state = nil - loop do - inner_str_tk = get_squashed_tk - if inner_str_tk.nil? - break - elsif :on_regexp_end == inner_str_tk[:kind] - string = string + inner_str_tk[:text] - state = inner_str_tk[:state] - break - else - string = string + inner_str_tk[:text] - end + end + { + :line_no => tk[:line_no], + :char_no => tk[:char_no], + :kind => expanded ? :on_dstring : :on_tstring, + :text => string, + :state => state + } + end + + private def get_regexp_tk(tk) + string = tk[:text] + state = nil + loop do + inner_str_tk = get_squashed_tk + if inner_str_tk.nil? + break + elsif :on_regexp_end == inner_str_tk[:kind] + string = string + inner_str_tk[:text] + state = inner_str_tk[:state] + break + else + string = string + inner_str_tk[:text] end - tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_regexp, :text => string, :state => state } - when :on_embdoc_beg then - string = '' - until :on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do - string = string + embdoc_tk[:text] + end + { + :line_no => tk[:line_no], + :char_no => tk[:char_no], + :kind => :on_regexp, + :text => string, + :state => state + } + end + + private def get_embdoc_tk(tk) + string = '' + until :on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do + string = string + embdoc_tk[:text] + end + { + :line_no => tk[:line_no], + :char_no => tk[:char_no], + :kind => :on_embdoc, + :text => string, + :state => embdoc_tk[:state] + } + end + + private def get_qwords_tk(tk) + string = '' + start_token = tk[:text] + start_quote = tk[:text].rstrip[-1] + line_no = tk[:line_no] + char_no = tk[:char_no] + state = tk[:state] + end_quote = + case start_quote + when ?( then ?) + when ?[ then ?] + when ?{ then ?} + when ?< then ?> + else start_quote end - tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_embdoc, :text => string, :state => embdoc_tk[:state] } - when :on_qwords_beg then - string = '' - start_token = tk[:text] - start_quote = tk[:text].rstrip[-1] - line_no = tk[:line_no] - char_no = tk[:char_no] - state = tk[:state] - end_quote = - case start_quote - when ?( then ?) - when ?[ then ?] - when ?{ then ?} - when ?< then ?> - else start_quote - end - end_token = nil - loop do - tk = get_squashed_tk - if tk.nil? - end_token = end_quote + end_token = nil + loop do + tk = get_squashed_tk + if tk.nil? + end_token = end_quote + break + elsif :on_tstring_content == tk[:kind] then + string += tk[:text] + elsif :on_words_sep == tk[:kind] then + if end_quote == tk[:text].strip then + end_token = tk[:text] break - elsif :on_tstring_content == tk[:kind] then + else string += tk[:text] - elsif :on_words_sep == tk[:kind] then - if end_quote == tk[:text].strip then - end_token = tk[:text] - break - else - string += tk[:text] - end end end - text = "#{start_token}#{string}#{end_token}" - tk = { :line_no => line_no, :char_no => char_no, :kind => :on_dstring, :text => text, :state => state } - when :on_op then - if tk[:text] =~ /^[-+]$/ then - tk_ahead = get_squashed_tk - case tk_ahead[:kind] - when :on_int, :on_float, :on_rational, :on_imaginary - tk[:text] += tk_ahead[:text] - tk[:kind] = tk_ahead[:kind] - tk[:state] = tk_ahead[:state] - else - @buf.unshift tk_ahead - end + end + text = "#{start_token}#{string}#{end_token}" + { + :line_no => line_no, + :char_no => char_no, + :kind => :on_dstring, + :text => text, + :state => state + } + end + + private def get_op_tk(tk) + redefinable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~] + if redefinable_operators.include?(tk[:text]) and EXPR_ARG == tk[:state] then + @lex_state = EXPR_ARG + tk[:kind] = :on_ident + tk[:state] = @lex_state + elsif tk[:text] =~ /^[-+]$/ then + tk_ahead = get_squashed_tk + case tk_ahead[:kind] + when :on_int, :on_float, :on_rational, :on_imaginary then + tk[:text] += tk_ahead[:text] + tk[:kind] = tk_ahead[:kind] + tk[:state] = tk_ahead[:state] + else + @buf.unshift tk_ahead end end tk From e27c741243950de6fd0aab86397136d5c93c5b5e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 09:45:38 +0900 Subject: [PATCH 097/151] Support redefinable operators --- lib/rdoc/parser/ripper_state_lex.rb | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index ac98127dfd..91f49bb2a6 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -60,8 +60,13 @@ def on_ignored_nl(tok, data) def on_op(tok, data) case tok - when '!', '!=', '!~' - @lex_state = EXPR_BEG + when '&', '|', '!', '!=', '!~' + case @lex_state + when EXPR_FNAME, EXPR_DOT + @lex_state = EXPR_ARG + else + @lex_state = EXPR_BEG + end when '<<' # TODO next token? case @lex_state @@ -71,9 +76,8 @@ def on_op(tok, data) @lex_state = EXPR_BEG end when '?' - @lex_state = :EXPR_BEG - when '&', '&&', - '|', '||', '+=', '-=', '*=', '**=', + @lex_state = EXPR_BEG + when '&&', '||', '+=', '-=', '*=', '**=', '&=', '|=', '^=', '<<=', '>>=', '||=', '&&=' @lex_state = EXPR_BEG when ')', ']', '}' @@ -232,9 +236,16 @@ def get_squashed_tk case tk[:kind] when :on_symbeg then tk = get_symbol_tk(tk) - when :on_tstring_beg, :on_backtick then - tk = get_string_tk(tk) + when :on_tstring_beg then tk = get_string_tk(tk) + when :on_backtick then + if (EXPR_FNAME & tk[:state]) != 0 + @lex_state = EXPR_ARG + tk[:kind] = :on_ident + tk[:state] = @lex_state + else + tk = get_string_tk(tk) + end when :on_regexp_beg then tk = get_regexp_tk(tk) when :on_embdoc_beg then From 6dc557b1522caa4b534a87053512cef69df16606 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 24 Jul 2017 21:22:40 +0900 Subject: [PATCH 098/151] Support instance, class, global variable as symbol --- lib/rdoc/parser/ripper_state_lex.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 91f49bb2a6..4c026b51a2 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -274,6 +274,15 @@ def get_squashed_tk when :on_op symbol_tk[:text] = ":#{tk1[:text]}" symbol_tk[:state] = tk1[:state] + when :on_ivar + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_cvar + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_gvar + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] #when :on_symbols_beg #when :on_qsymbols_beg else From 47b431b2469ef2316d2f9f6df980c19be2c57640 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 27 Jul 2017 09:14:58 +0900 Subject: [PATCH 099/151] Support postfix if/unless/do/case/begin in parse_constant_body --- lib/rdoc/parser/ruby.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index a66bf93fc1..c0d5028007 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -911,9 +911,14 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: break if tk.nil? if :on_semicolon == tk[:kind] then break if nest <= 0 - elsif [:on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) || - (:on_kw == tk[:kind] && %w{do if unless case def begin}.include?(tk[:text])) then + elsif [:on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) then nest += 1 + elsif (:on_kw == tk[:kind] && 'def' == tk[:text]) then + nest += 1 + elsif (:on_kw == tk[:kind] && %w{do if unless case begin}.include?(tk[:text])) then + if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0 + nest += 1 + end elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) || :on_kw == tk[:kind] && 'end' == tk[:text] then nest -= 1 From 21bb9caeb5178a81509d2a886cc235dce9992bda Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 27 Jul 2017 09:17:24 +0900 Subject: [PATCH 100/151] Support %i, %I and %W --- lib/rdoc/parser/ripper_state_lex.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 4c026b51a2..b0afe45531 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -250,8 +250,14 @@ def get_squashed_tk tk = get_regexp_tk(tk) when :on_embdoc_beg then tk = get_embdoc_tk(tk) + when :on_words_beg then + tk = get_words_tk(tk) when :on_qwords_beg then - tk = get_qwords_tk(tk) + tk = get_words_tk(tk) + when :on_symbols_beg then + tk = get_words_tk(tk) + when :on_qsymbols_beg then + tk = get_words_tk(tk) when :on_op then tk = get_op_tk(tk) end @@ -283,8 +289,6 @@ def get_squashed_tk when :on_gvar symbol_tk[:text] = ":#{tk1[:text]}" symbol_tk[:state] = tk1[:state] - #when :on_symbols_beg - #when :on_qsymbols_beg else is_symbol = false tk = tk1 @@ -365,7 +369,7 @@ def get_squashed_tk } end - private def get_qwords_tk(tk) + private def get_words_tk(tk) string = '' start_token = tk[:text] start_quote = tk[:text].rstrip[-1] From 099472cbc66fae2460372602cd5f66bce04d84cc Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 27 Jul 2017 09:18:19 +0900 Subject: [PATCH 101/151] Support symbol with const --- lib/rdoc/parser/ripper_state_lex.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index b0afe45531..78dee10995 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -289,6 +289,9 @@ def get_squashed_tk when :on_gvar symbol_tk[:text] = ":#{tk1[:text]}" symbol_tk[:state] = tk1[:state] + when :on_const + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] else is_symbol = false tk = tk1 From c1e3b839b9ad03db2c97829981671a2755419574 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 27 Jul 2017 09:19:04 +0900 Subject: [PATCH 102/151] Support % literal end token with on_tstring_end --- lib/rdoc/parser/ripper_state_lex.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 78dee10995..2eafb11840 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -395,7 +395,7 @@ def get_squashed_tk break elsif :on_tstring_content == tk[:kind] then string += tk[:text] - elsif :on_words_sep == tk[:kind] then + elsif :on_words_sep == tk[:kind] or :on_tstring_end == tk[:kind] then if end_quote == tk[:text].strip then end_token = tk[:text] break From 5a94c0a242d83ea09ba77576a633538915648074 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 27 Jul 2017 17:37:04 +0900 Subject: [PATCH 103/151] Fix treating comment at the same line of def --- lib/rdoc/parser/ruby.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index c0d5028007..87bc9d0935 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1503,13 +1503,14 @@ def parse_method_or_yield_parameters(method = nil, when :on_rparen then nest -= 1 when :on_comment, :on_embdoc then + @read.pop if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] then - break - elsif method && method.block_params.nil? then - unget_tk tk - read_documentation_modifiers method, modifiers + if method && method.block_params.nil? then + unget_tk tk + read_documentation_modifiers method, modifiers + end + break if nest <= 0 end - @read.pop end tk = get_tk end From b26c08d9e19d584005c1517dfe74e05f60456c3e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Fri, 28 Jul 2017 03:51:34 +0900 Subject: [PATCH 104/151] Fix ')', ']' and '}' token handling --- lib/rdoc/parser/ripper_state_lex.rb | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 2eafb11840..bb18ae74f8 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -80,8 +80,6 @@ def on_op(tok, data) when '&&', '||', '+=', '-=', '*=', '**=', '&=', '|=', '^=', '<<=', '>>=', '||=', '&&=' @lex_state = EXPR_BEG - when ')', ']', '}' - @lex_state = EXPR_END else case @lex_state when EXPR_FNAME, EXPR_DOT @@ -195,6 +193,26 @@ def on_rparen(tok, data) @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end + def on_lbrace(tok, data) + @lex_state = EXPR_LABEL | EXPR_BEG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_rbrace(tok, data) + @lex_state = EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_lbracket(tok, data) + @lex_state = EXPR_LABEL | EXPR_BEG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_rbracket(tok, data) + @lex_state = EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + def on_const(tok, data) case @lex_state when EXPR_FNAME From e3525fd5033ea6b6db35c25820b641ae8ada194b Mon Sep 17 00:00:00 2001 From: Code Ass Date: Fri, 28 Jul 2017 22:06:16 +0900 Subject: [PATCH 105/151] Squash heredoc as one token --- lib/rdoc/parser/ripper_state_lex.rb | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index bb18ae74f8..a4b33d341d 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -268,6 +268,19 @@ def get_squashed_tk tk = get_regexp_tk(tk) when :on_embdoc_beg then tk = get_embdoc_tk(tk) + when :on_heredoc_beg then + tk = get_heredoc_tk(tk) + if "\n" == tk[:text][-1] + tk[:text] = tk[:text][0..-2] + nl_tk = { + :line_no => tk[:line_no] + tk[:text].count("\n"), + :char_no => tk[:text].split("\n")[-1].size, + :kind => :on_nl, + :text => "\n", + :state => tk[:state] + } + @buf.unshift nl_tk + end when :on_words_beg then tk = get_words_tk(tk) when :on_qwords_beg then @@ -390,6 +403,21 @@ def get_squashed_tk } end + private def get_heredoc_tk(tk) + string = tk[:text] + until :on_heredoc_end == (heredoc_tk = get_squashed_tk)[:kind] do + string = string + heredoc_tk[:text] + end + string = string + heredoc_tk[:text] + { + :line_no => tk[:line_no], + :char_no => tk[:char_no], + :kind => :on_embdoc, + :text => string, + :state => heredoc_tk[:state] + } + end + private def get_words_tk(tk) string = '' start_token = tk[:text] From fe2ee654b6bd7092d565b1d0d1637e9afb6162ce Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sat, 29 Jul 2017 05:28:18 +0900 Subject: [PATCH 106/151] Fix postfix while/until handling --- lib/rdoc/parser/ripper_state_lex.rb | 2 +- lib/rdoc/parser/ruby.rb | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index a4b33d341d..1aabb65c5a 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -100,7 +100,7 @@ def on_kw(tok, data) @lex_state = EXPR_FNAME @continue = true @in_fname = true - when 'if', 'unless' + when 'if', 'unless', 'while', 'until' if ((EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_CMDARG) & @lex_state) != 0 # postfix if @lex_state = EXPR_BEG | EXPR_LABEL else diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 87bc9d0935..e9baae83d4 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1689,8 +1689,10 @@ def parse_statements(container, single = NORMAL, current_method = nil, end when 'until', 'while' then - nest += 1 - skip_optional_do_after_expression + if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0 + nest += 1 + skip_optional_do_after_expression + end # Until and While can have a 'do', which shouldn't increase the nesting. # We can't solve the general case, but we can handle most occurrences by From 0ec3a8e02b876e2a608214d90ab86a25bd7084d9 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sat, 29 Jul 2017 06:39:43 +0900 Subject: [PATCH 107/151] Fix for test_parse_method_parameters_comment_continue --- lib/rdoc/parser/ruby.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index e9baae83d4..6ba98ad8fa 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1477,6 +1477,7 @@ def parse_method_or_yield_parameters(method = nil, return '' unless end_token nest = 0 + continue = false while tk != nil do case tk[:kind] @@ -1504,13 +1505,18 @@ def parse_method_or_yield_parameters(method = nil, nest -= 1 when :on_comment, :on_embdoc then @read.pop - if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] then + if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and + (!continue or (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) != 0) then if method && method.block_params.nil? then unget_tk tk read_documentation_modifiers method, modifiers end break if nest <= 0 end + when :on_comma then + continue = true + when :on_ident then + continue = false if continue end tk = get_tk end From 7c1fa223d4deff143713ad59298eafeb754979ee Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 30 Jul 2017 12:58:38 +0900 Subject: [PATCH 108/151] Homologize backtick handling behavior for previous --- lib/rdoc/parser/ripper_state_lex.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 1aabb65c5a..89dd695a1c 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -336,11 +336,7 @@ def get_squashed_tk private def get_string_tk(tk) string = tk[:text] state = nil - if :on_backtick == tk[:kind] - expanded = true - else - expanded = false - end + expanded = false loop do inner_str_tk = get_squashed_tk if inner_str_tk.nil? From e75ddc4d9c10db30d36e0ecf75e49f145613c687 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 30 Jul 2017 13:11:24 +0900 Subject: [PATCH 109/151] Treat ivar, cvar and gvar as EXPR_END --- lib/rdoc/parser/ripper_state_lex.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 89dd695a1c..59fa431edb 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -172,14 +172,17 @@ def on_ident(tok, data) end def on_ivar(tok, data) + @lex_state = EXPR_END on_variables(__method__, tok, data) end def on_cvar(tok, data) + @lex_state = EXPR_END on_variables(__method__, tok, data) end def on_gvar(tok, data) + @lex_state = EXPR_END on_variables(__method__, tok, data) end From 666bb0dc5e94cff5299d393126d10f7489392131 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 30 Jul 2017 13:12:08 +0900 Subject: [PATCH 110/151] Add to treat backref with EXPR_END --- lib/rdoc/parser/ripper_state_lex.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 59fa431edb..bbf5b9c033 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -186,6 +186,11 @@ def on_gvar(tok, data) on_variables(__method__, tok, data) end + def on_backref(tok, data) + @lex_state = EXPR_END + on_variables(__method__, tok, data) + end + def on_lparen(tok, data) @lex_state = EXPR_LABEL | EXPR_BEG @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) From 55f2f6fd2a6303f634b0b343ee16dadae3d0b57e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 30 Jul 2017 13:12:40 +0900 Subject: [PATCH 111/151] Support lambda -> operator --- lib/rdoc/token_stream.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index 5a3ee10e0b..71e58ad7a9 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -42,6 +42,7 @@ def self.to_html token_stream else 'ruby-operator' end + when :on_tlambda then 'ruby-operator' when :on_ident then 'ruby-identifier' when :on_label then 'ruby-value' when :on_backref, :on_dstring From e5195ee0b3c361fc657e9f8708d0e8588ac02e01 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 3 Aug 2017 15:02:56 +0900 Subject: [PATCH 112/151] Fix condition of detecting dynamic string --- lib/rdoc/parser/ripper_state_lex.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index bbf5b9c033..6feecaa106 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -355,7 +355,7 @@ def get_squashed_tk break else string = string + inner_str_tk[:text] - if :on_tstring_content != inner_str_tk[:kind] then + if :on_embexpr_beg == inner_str_tk[:kind] then expanded = true end end From 783c59e36a293b68ae4c14314a52ea6999a83fc4 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Thu, 3 Aug 2017 15:04:42 +0900 Subject: [PATCH 113/151] Fix for complex heredoc cases --- lib/rdoc/parser/ripper_state_lex.rb | 70 ++++++++++++++++++++--------- lib/rdoc/token_stream.rb | 4 +- 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 6feecaa106..a93153f53f 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -242,6 +242,11 @@ def on_comment(tok, data) @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end + def on_ignored_sp(tok, data) + @lex_state = EXPR_BEG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + def on_default(event, tok, data) reset @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state}) @@ -277,17 +282,10 @@ def get_squashed_tk when :on_embdoc_beg then tk = get_embdoc_tk(tk) when :on_heredoc_beg then - tk = get_heredoc_tk(tk) - if "\n" == tk[:text][-1] - tk[:text] = tk[:text][0..-2] - nl_tk = { - :line_no => tk[:line_no] + tk[:text].count("\n"), - :char_no => tk[:text].split("\n")[-1].size, - :kind => :on_nl, - :text => "\n", - :state => tk[:state] - } - @buf.unshift nl_tk + @heredoc_queue << retrieve_heredoc_info(tk) + when :on_nl, :on_ignored_nl, :on_comment then + unless @heredoc_queue.empty? + get_heredoc_tk(*@heredoc_queue.shift) end when :on_words_beg then tk = get_words_tk(tk) @@ -407,19 +405,46 @@ def get_squashed_tk } end - private def get_heredoc_tk(tk) - string = tk[:text] - until :on_heredoc_end == (heredoc_tk = get_squashed_tk)[:kind] do - string = string + heredoc_tk[:text] - end - string = string + heredoc_tk[:text] - { - :line_no => tk[:line_no], - :char_no => tk[:char_no], - :kind => :on_embdoc, + private def get_heredoc_tk(heredoc_name, indent) + string = '' + start_tk = nil + prev_tk = nil + until heredoc_end?(heredoc_name, indent, tk = @inner_lex.next) do + start_tk = tk unless start_tk + if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no] + string = string + (' ' * tk[:char_no]) + end + string = string + tk[:text] + prev_tk = tk + end + start_tk = tk unless start_tk + prev_tk = tk unless prev_tk + @buf.unshift tk # closing heredoc + heredoc_tk = { + :line_no => start_tk[:line_no], + :char_no => start_tk[:char_no], + :kind => :on_heredoc, :text => string, - :state => heredoc_tk[:state] + :state => prev_tk[:state] } + @buf.unshift heredoc_tk + end + + private def retrieve_heredoc_info(tk) + name = tk[:text].gsub(/\A<<[-~]?['"`]?(\w+)['"`]?\z/, '\1') + indent = tk[:text].match?(/\A<<[-~]/) + [name, indent] + end + + private def heredoc_end?(name, indent, tk) + result = false + if :on_heredoc_end == tk[:kind] then + tk_name = (indent ? tk[:text].gsub(/\A *(\w+)\n\z/, '\1') : tk[:text].gsub(/\n\z/, '')) + if name == tk_name + result = true + end + end + result end private def get_words_tk(tk) @@ -486,6 +511,7 @@ def get_squashed_tk def initialize(code) @buf = [] + @heredoc_queue = [] @inner_lex = Enumerator.new do |y| InnerStateLex.new(code).each do |tk| y << tk diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index 71e58ad7a9..b7d9ef40da 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -52,8 +52,10 @@ def self.to_html token_stream when :on_tstring then 'ruby-string' when :on_int, :on_float, :on_rational, :on_imaginary, - :on_embdoc, + :on_embdoc, :on_heredoc, :on_symbol, :on_CHAR then 'ruby-value' + when :on_heredoc_beg, :on_heredoc_end + then 'ruby-identifier' end comment_with_nl = false From 10949c8dcecab1d03a51b0a9358c2e4bc635d50f Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sat, 5 Aug 2017 23:02:21 +0900 Subject: [PATCH 114/151] Remove hard tab from source code --- lib/rdoc/parser/ruby.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 6ba98ad8fa..f4718640d1 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -165,6 +165,16 @@ class RDoc::Parser::Ruby < RDoc::Parser def initialize(top_level, file_name, content, options, stats) super + if /\t/ =~ content then + tab_width = @options.tab_width + content = content.split(/\n/).map do |line| + 1 while line.gsub!(/\t+/) { + ' ' * (tab_width*$&.length - $`.length % tab_width) + } && $~ + line + end.join("\n").gsub(/(? Date: Sat, 5 Aug 2017 23:10:51 +0900 Subject: [PATCH 115/151] Fix complex condition for for/while/until from 7d99dc7 --- lib/rdoc/parser/ruby.rb | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index f4718640d1..8a9fc40a84 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -2081,10 +2081,12 @@ def skip_optional_do_after_expression loop do break unless tk case tk[:kind] - when :on_semicolon then + when :on_semicolon, :on_nl, :on_ignored_nl then break if b_nest.zero? when :on_lparen then nest += 1 + when :on_rparen then + nest -= 1 when :on_kw then case tk[:text] when 'begin' @@ -2094,16 +2096,8 @@ def skip_optional_do_after_expression when 'do' break if nest.zero? end - when end_token[:kind] then - if end_token == :on_rparen - nest -= 1 - break if @scanner.lex_state == :EXPR_END and nest.zero? - else - break - #break unless @scanner.continue - end when :on_comment, :on_embdoc then - if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] then + if b_nest.zero? and "\n" == tk[:text][-1] then break end end From 5b65747d5a53a4d4680f1911c9ec489f54de8276 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sat, 5 Aug 2017 23:14:50 +0900 Subject: [PATCH 116/151] Set EXPR_END to @lex_state when on_heredoc_beg --- lib/rdoc/parser/ripper_state_lex.rb | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index a93153f53f..f1cc261c58 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -21,7 +21,7 @@ class RDoc::RipperStateLex EXPR_END_ANY = (EXPR_END | EXPR_ENDARG | EXPR_ENDFN) class InnerStateLex < Ripper::Filter - include Enumerable + attr_accessor :lex_state def initialize(code) @lex_state = EXPR_BEG @@ -260,7 +260,7 @@ def each(&block) def get_squashed_tk if @buf.empty? - tk = @inner_lex.next + tk = @inner_lex_enumerator.next else tk = @buf.shift end @@ -271,9 +271,9 @@ def get_squashed_tk tk = get_string_tk(tk) when :on_backtick then if (EXPR_FNAME & tk[:state]) != 0 - @lex_state = EXPR_ARG + @inner_lex.lex_state = EXPR_ARG tk[:kind] = :on_ident - tk[:state] = @lex_state + tk[:state] = @inner_lex.lex_state else tk = get_string_tk(tk) end @@ -283,6 +283,7 @@ def get_squashed_tk tk = get_embdoc_tk(tk) when :on_heredoc_beg then @heredoc_queue << retrieve_heredoc_info(tk) + @inner_lex.lex_state = EXPR_END when :on_nl, :on_ignored_nl, :on_comment then unless @heredoc_queue.empty? get_heredoc_tk(*@heredoc_queue.shift) @@ -409,7 +410,7 @@ def get_squashed_tk string = '' start_tk = nil prev_tk = nil - until heredoc_end?(heredoc_name, indent, tk = @inner_lex.next) do + until heredoc_end?(heredoc_name, indent, tk = @inner_lex_enumerator.next) do start_tk = tk unless start_tk if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no] string = string + (' ' * tk[:char_no]) @@ -492,9 +493,9 @@ def get_squashed_tk private def get_op_tk(tk) redefinable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~] if redefinable_operators.include?(tk[:text]) and EXPR_ARG == tk[:state] then - @lex_state = EXPR_ARG + @inner_lex.lex_state = EXPR_ARG tk[:kind] = :on_ident - tk[:state] = @lex_state + tk[:state] = @inner_lex.lex_state elsif tk[:text] =~ /^[-+]$/ then tk_ahead = get_squashed_tk case tk_ahead[:kind] @@ -512,8 +513,9 @@ def get_squashed_tk def initialize(code) @buf = [] @heredoc_queue = [] - @inner_lex = Enumerator.new do |y| - InnerStateLex.new(code).each do |tk| + @inner_lex = InnerStateLex.new(code) + @inner_lex_enumerator = Enumerator.new do |y| + @inner_lex.each do |tk| y << tk end end From 00e88d290a7baf856fe8ecd0a9b779b9535360aa Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 6 Aug 2017 17:49:37 +0900 Subject: [PATCH 117/151] Support symbol literal with quote --- lib/rdoc/parser/ripper_state_lex.rb | 58 ++++++++++++++++------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index f1cc261c58..ab60ff4cd9 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -305,34 +305,40 @@ def get_squashed_tk private def get_symbol_tk(tk) is_symbol = true symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol } - case (tk1 = get_squashed_tk)[:kind] - when :on_ident - symbol_tk[:text] = ":#{tk1[:text]}" - symbol_tk[:state] = tk1[:state] - when :on_tstring_content - symbol_tk[:text] = ":#{tk1[:text]}" - symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end - when :on_tstring_end - symbol_tk[:text] = ":#{tk1[:text]}" - symbol_tk[:state] = tk1[:state] - when :on_op - symbol_tk[:text] = ":#{tk1[:text]}" - symbol_tk[:state] = tk1[:state] - when :on_ivar - symbol_tk[:text] = ":#{tk1[:text]}" - symbol_tk[:state] = tk1[:state] - when :on_cvar - symbol_tk[:text] = ":#{tk1[:text]}" - symbol_tk[:state] = tk1[:state] - when :on_gvar - symbol_tk[:text] = ":#{tk1[:text]}" - symbol_tk[:state] = tk1[:state] - when :on_const - symbol_tk[:text] = ":#{tk1[:text]}" + if ":'" == tk[:text] or ':"' == tk[:text] + tk1 = get_string_tk(tk) + symbol_tk[:text] = tk1[:text] symbol_tk[:state] = tk1[:state] else - is_symbol = false - tk = tk1 + case (tk1 = get_squashed_tk)[:kind] + when :on_ident + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_tstring_content + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end + when :on_tstring_end + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_op + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_ivar + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_cvar + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_gvar + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + when :on_const + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] + else + is_symbol = false + tk = tk1 + end end if is_symbol tk = symbol_tk From d2b01e0747f097de143bbb821f1e262f4ef93466 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 7 Aug 2017 04:56:32 +0900 Subject: [PATCH 118/151] Fix typo --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 8a9fc40a84..a816756faf 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1965,7 +1965,7 @@ def read_directive allowed while tk = get_tk do tokens << tk - if :on_nl == tk[:kind] or (:tk_kw == tk[:kind] && 'def' == tk[:text]) then + if :on_nl == tk[:kind] or (:on_kw == tk[:kind] && 'def' == tk[:text]) then return elsif :on_comment == tk[:kind] or :on_embdoc == tk[:kind] then return unless tk[:text] =~ /\s*:?([\w-]+):\s*(.*)/ From 3c55a3910edcf8c7a9c73d6ac4bfe90c96eb1f8e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 7 Aug 2017 04:56:45 +0900 Subject: [PATCH 119/151] Skip unnecessary :on_comment The read_directive method sets back tokens and the parse_statements method read comment token after newline token, but Ripper doesn't separate comment and newline unlike IRB scanner. So previous directive at end of line improperly connects to next comment tokens for documentation. This commit fixes it. --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index a816756faf..682fda8133 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1639,7 +1639,7 @@ def parse_statements(container, single = NORMAL, current_method = nil, non_comment_seen = true unless (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) case tk[:kind] - when :on_nl, :on_ignored_nl then + when :on_nl, :on_ignored_nl, :on_comment, :on_embdoc then skip_tkspace tk = get_tk From 34fb3ba71b8a9e489f20d0f58e4796c50b1ec3bd Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 8 Aug 2017 14:15:47 +0900 Subject: [PATCH 120/151] Support symbol with keyword --- lib/rdoc/parser/ripper_state_lex.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index ab60ff4cd9..57214fe62a 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -335,6 +335,9 @@ def get_squashed_tk when :on_const symbol_tk[:text] = ":#{tk1[:text]}" symbol_tk[:state] = tk1[:state] + when :on_kw + symbol_tk[:text] = ":#{tk1[:text]}" + symbol_tk[:state] = tk1[:state] else is_symbol = false tk = tk1 From d1cfa707063b14a6a0fa3242ad31153283aff64c Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 8 Aug 2017 14:17:26 +0900 Subject: [PATCH 121/151] If newline comes, it's end of method name --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 682fda8133..eb869b3b42 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1392,7 +1392,7 @@ def parse_method_dummy container def parse_method_name container # :nodoc: skip_tkspace name_t = get_tk - back_tk = skip_tkspace + back_tk = skip_tkspace(false) singleton = false dot = get_tk From b7e39ce48e9f2a77246567eebf5ff2c964792d4d Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sat, 12 Aug 2017 06:54:16 +0900 Subject: [PATCH 122/151] Support safe navigation operator --- lib/rdoc/parser/ripper_state_lex.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 57214fe62a..f24950ba9f 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -297,7 +297,11 @@ def get_squashed_tk when :on_qsymbols_beg then tk = get_words_tk(tk) when :on_op then - tk = get_op_tk(tk) + if '&.' == tk[:text] + tk[:kind] = :on_period + else + tk = get_op_tk(tk) + end end tk end From 0c5ec7ddf2b006a0e006637e893e550e605d1351 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 13 Aug 2017 00:27:21 +0900 Subject: [PATCH 123/151] Add EXPR_LABEL when inside args --- lib/rdoc/parser/ripper_state_lex.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index f24950ba9f..269330802c 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -42,7 +42,7 @@ def on_nl(tok, data) @continue = true else @continue = false - @lex_state = EXPR_BEG + @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0 end @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end @@ -53,7 +53,7 @@ def on_ignored_nl(tok, data) @continue = true else @continue = false - @lex_state = EXPR_BEG + @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0 end @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end @@ -237,13 +237,18 @@ def on_sp(tok, data) @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end + def on_comma(tok, data) + @lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0 + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + def on_comment(tok, data) - @lex_state = EXPR_BEG + @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0 @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end def on_ignored_sp(tok, data) - @lex_state = EXPR_BEG + @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0 @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end From 1955b68f39af1582994d31375e758e1b84cfd19b Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 13 Aug 2017 07:00:30 +0900 Subject: [PATCH 124/151] Add condition of "begin" keyword with EXPR_BEG --- lib/rdoc/parser/ripper_state_lex.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 269330802c..af1944d348 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -106,6 +106,8 @@ def on_kw(tok, data) else @lex_state = EXPR_BEG end + when 'begin' + @lex_state = EXPR_BEG else if @lex_state == EXPR_FNAME @lex_state = EXPR_END From bfda75eb87b001282750e14d8bf8defae9ffe147 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 13 Aug 2017 07:00:58 +0900 Subject: [PATCH 125/151] Fix % words syntax --- lib/rdoc/parser/ripper_state_lex.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index af1944d348..e6575115f6 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -498,6 +498,8 @@ def get_squashed_tk else string += tk[:text] end + else + string += tk[:text] end end text = "#{start_token}#{string}#{end_token}" From 4fa207c18832c3dd0de11c62bcf262ccd286db95 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 14 Aug 2017 23:03:52 +0900 Subject: [PATCH 126/151] Fix around newline after embdoc --- lib/rdoc/markup/to_html.rb | 3 ++- lib/rdoc/parser/ripper_state_lex.rb | 2 +- lib/rdoc/parser/ruby.rb | 2 +- lib/rdoc/token_stream.rb | 10 ++++++---- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/rdoc/markup/to_html.rb b/lib/rdoc/markup/to_html.rb index 245d43e3b9..98ed7926fd 100644 --- a/lib/rdoc/markup/to_html.rb +++ b/lib/rdoc/markup/to_html.rb @@ -204,7 +204,8 @@ def accept_verbatim verbatim klass = ' class="ruby"' result = RDoc::TokenStream.to_html tokens - result + "\n" + result = result + "\n" unless "\n" == result[-1] + result rescue RDoc::RubyLex::Error CGI.escapeHTML text end diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index e6575115f6..f4265bb000 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -291,7 +291,7 @@ def get_squashed_tk when :on_heredoc_beg then @heredoc_queue << retrieve_heredoc_info(tk) @inner_lex.lex_state = EXPR_END - when :on_nl, :on_ignored_nl, :on_comment then + when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then unless @heredoc_queue.empty? get_heredoc_tk(*@heredoc_queue.shift) end diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index eb869b3b42..0220de4fde 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -172,7 +172,7 @@ def initialize(top_level, file_name, content, options, stats) ' ' * (tab_width*$&.length - $`.length % tab_width) } && $~ line - end.join("\n").gsub(/(?#{text}#{"\n" if comment_with_nl}" From cc208fe696269e43bdac0e2fc2fbe5f6e641c7e1 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 14 Aug 2017 23:04:25 +0900 Subject: [PATCH 127/151] Fix embdoc token's actual data --- lib/rdoc/parser/ripper_state_lex.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index f4265bb000..e5deb5790a 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -413,10 +413,11 @@ def get_squashed_tk end private def get_embdoc_tk(tk) - string = '' + string = tk[:text] until :on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do string = string + embdoc_tk[:text] end + string = string + embdoc_tk[:text] { :line_no => tk[:line_no], :char_no => tk[:char_no], From debdaee4a71077aeb649be6f935a15f38c0a60ec Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 14 Aug 2017 23:06:08 +0900 Subject: [PATCH 128/151] Fix checking correspondence of heredoc tokens --- lib/rdoc/parser/ripper_state_lex.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index e5deb5790a..0392bd4e93 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -453,7 +453,7 @@ def get_squashed_tk end private def retrieve_heredoc_info(tk) - name = tk[:text].gsub(/\A<<[-~]?['"`]?(\w+)['"`]?\z/, '\1') + name = tk[:text].gsub(/\A<<[-~]?(['"`]?)(.+)\1\z/, '\2') indent = tk[:text].match?(/\A<<[-~]/) [name, indent] end @@ -461,7 +461,7 @@ def get_squashed_tk private def heredoc_end?(name, indent, tk) result = false if :on_heredoc_end == tk[:kind] then - tk_name = (indent ? tk[:text].gsub(/\A *(\w+)\n\z/, '\1') : tk[:text].gsub(/\n\z/, '')) + tk_name = (indent ? tk[:text].gsub(/\A *(.+)\n\z/, '\1') : tk[:text].gsub(/\n\z/, '')) if name == tk_name result = true end From b06882afcfd589c862d2a1b6e7c23509522b2bbf Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 14 Aug 2017 23:06:26 +0900 Subject: [PATCH 129/151] Check nil --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 0220de4fde..100ebb2154 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -884,7 +884,7 @@ def parse_constant container, tk, comment, ignore_constants = false eq_tk = get_tk end - unless :on_op == eq_tk[:kind] && '=' == eq_tk[:text] then + unless eq_tk && :on_op == eq_tk[:kind] && '=' == eq_tk[:text] then unget_tk eq_tk return false end From 184352809b42b2e960ac712ea78fd28df521a26e Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 14 Aug 2017 23:06:56 +0900 Subject: [PATCH 130/151] Treat :on_embdoc as comment in highlighting --- lib/rdoc/token_stream.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index b9984fe1dc..072fb3f32e 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -48,11 +48,12 @@ def self.to_html token_stream when :on_backref, :on_dstring then 'ruby-node' when :on_comment then 'ruby-comment' + when :on_embdoc then 'ruby-comment' when :on_regexp then 'ruby-regexp' when :on_tstring then 'ruby-string' when :on_int, :on_float, :on_rational, :on_imaginary, - :on_embdoc, :on_heredoc, + :on_heredoc, :on_symbol, :on_CHAR then 'ruby-value' when :on_heredoc_beg, :on_heredoc_end then 'ruby-identifier' From 8dbd3fb4b17e1173ad22ead05905bb52528a6386 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 21 Aug 2017 14:58:09 +0900 Subject: [PATCH 131/151] Skip create alias for module when array --- lib/rdoc/parser/ruby.rb | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 100ebb2154..2afa21172c 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -884,6 +884,23 @@ def parse_constant container, tk, comment, ignore_constants = false eq_tk = get_tk end + is_array_or_hash = false + if eq_tk && :on_lbracket == eq_tk[:kind] + nest = 1 + while bracket_tk = get_tk + case bracket_tk[:kind] + when :on_lbracket + nest += 1 + when :on_rbracket + nest -= 1 + break if nest == 0 + end + end + skip_tkspace false + eq_tk = get_tk + is_array_or_hash = true + end + unless eq_tk && :on_op == eq_tk[:kind] && '=' == eq_tk[:text] then unget_tk eq_tk return false @@ -945,7 +962,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: next_tk = peek_tk if nest <= 0 and (next_tk.nil? || :on_nl == next_tk[:kind]) then - create_module_alias container, constant, rhs_name + create_module_alias container, constant, rhs_name unless is_array_or_hash break end elsif :on_nl == tk[:kind] then From 8193399fe257eee8c5ddc9b4c36b3d9bdc1fefd6 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 21 Aug 2017 15:43:45 +0900 Subject: [PATCH 132/151] Fix comment handling after :nodoc: --- lib/rdoc/parser/ruby.rb | 22 ++++++++++++++++++++-- lib/rdoc/token_stream.rb | 2 +- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 2afa21172c..3e93b52844 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1657,8 +1657,26 @@ def parse_statements(container, single = NORMAL, current_method = nil, case tk[:kind] when :on_nl, :on_ignored_nl, :on_comment, :on_embdoc then - skip_tkspace - tk = get_tk + if :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind] + skip_tkspace + tk = get_tk + else + past_tokens = @read.size > 1 ? @read[0..-2] : [] + nl_position = 0 + past_tokens.reverse.each_with_index do |read_tk, i| + if read_tk.match?(/^\n$/) then + nl_position = (past_tokens.size - 1) - i + break + elsif read_tk.match?(/^#.*\n$/) then + nl_position = ((past_tokens.size - 1) - i) + 1 + break + end + end + comment_only_line = past_tokens[nl_position..-1].all?{ |c| c.match?(/^\s+$/) } + unless comment_only_line then + tk = get_tk + end + end if tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then if non_comment_seen then diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index 072fb3f32e..df5232f7ea 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -60,7 +60,7 @@ def self.to_html token_stream end comment_with_nl = false - if :on_comment == t[:kind] or :on_embdoc == t[:kind] + if :on_comment == t[:kind] or :on_embdoc == t[:kind] or :on_heredoc_end == t[:kind] comment_with_nl = true if "\n" == t[:text][-1] text = t[:text].rstrip else From a11018967e83d085e02d43176c2266fb9276b15c Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 21 Aug 2017 15:44:10 +0900 Subject: [PATCH 133/151] Fix postfix :nodoc: handling --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 3e93b52844..3d8f5e54de 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1769,7 +1769,6 @@ def parse_statements(container, single = NORMAL, current_method = nil, when 'end' then nest -= 1 if nest == 0 then - read_documentation_modifiers container, RDoc::CLASS_MODIFIERS container.ongoing_visibility = save_visibility parse_comment container, tk, comment unless comment.empty? @@ -2027,6 +2026,7 @@ def read_directive allowed # See also RDoc::Markup::PreProcess#handle_directive def read_documentation_modifiers context, allowed + skip_tkspace(false) directive, value = read_directive allowed return unless directive From 6aec6b04bf2283a2f540b89ff4696fa2d33022a1 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 22 Aug 2017 07:53:23 +0900 Subject: [PATCH 134/151] Read directives for singleton class definition --- lib/rdoc/parser/ruby.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 3d8f5e54de..e5a8320cf8 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -763,6 +763,7 @@ def parse_class container, single, tk, comment elsif name_t[:kind] == :on_op && name_t[:text] == '<<' case name = get_class_specification when 'self', container.name + read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS parse_statements container, SINGLE return # don't update line else From e7a504c82838b3be74f33bde77394ee6989a9da1 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Wed, 23 Aug 2017 03:50:15 +0900 Subject: [PATCH 135/151] Support string literal ends as symbol --- lib/rdoc/parser/ripper_state_lex.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 0392bd4e93..717d27cd0e 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -363,7 +363,7 @@ def get_squashed_tk private def get_string_tk(tk) string = tk[:text] state = nil - expanded = false + kind = :on_tstring loop do inner_str_tk = get_squashed_tk if inner_str_tk.nil? @@ -372,17 +372,22 @@ def get_squashed_tk string = string + inner_str_tk[:text] state = inner_str_tk[:state] break + elsif :on_label_end == inner_str_tk[:kind] + string = string + inner_str_tk[:text] + state = inner_str_tk[:state] + kind = :on_symbol + break else string = string + inner_str_tk[:text] if :on_embexpr_beg == inner_str_tk[:kind] then - expanded = true + kind = :on_dstring if :on_tstring == kind end end end { :line_no => tk[:line_no], :char_no => tk[:char_no], - :kind => expanded ? :on_dstring : :on_tstring, + :kind => kind, :text => string, :state => state } From 050de61d9454f0d8bcfa3d6ec9b5a8936392bc22 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Wed, 23 Aug 2017 03:50:36 +0900 Subject: [PATCH 136/151] Fix heredoc end checking --- lib/rdoc/parser/ripper_state_lex.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 717d27cd0e..00738e9f16 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -466,7 +466,7 @@ def get_squashed_tk private def heredoc_end?(name, indent, tk) result = false if :on_heredoc_end == tk[:kind] then - tk_name = (indent ? tk[:text].gsub(/\A *(.+)\n\z/, '\1') : tk[:text].gsub(/\n\z/, '')) + tk_name = (indent ? tk[:text].gsub(/^ *(.+)\n?$/, '\1') : tk[:text].gsub(/\n\z/, '')) if name == tk_name result = true end From 042227579563e615b1cb6b98da589b652486ace4 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 27 Aug 2017 18:59:11 +0900 Subject: [PATCH 137/151] Add :on_tlambeg for nesting in parse_constant_body --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index e5a8320cf8..248645d17d 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -939,7 +939,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: break if tk.nil? if :on_semicolon == tk[:kind] then break if nest <= 0 - elsif [:on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) then + elsif [:on_tlambeg, :on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) then nest += 1 elsif (:on_kw == tk[:kind] && 'def' == tk[:text]) then nest += 1 From 7b470609d6473e040e9fa7f91250e80a3aa64367 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 27 Aug 2017 18:59:53 +0900 Subject: [PATCH 138/151] Fix condition with paren --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 248645d17d..b2ef518148 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -948,7 +948,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: nest += 1 end elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) || - :on_kw == tk[:kind] && 'end' == tk[:text] then + (:on_kw == tk[:kind] && 'end' == tk[:text]) then nest -= 1 elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then if nest <= 0 and RDoc::RipperStateLex.end?(tk) then From 415a40df3796b0e8b6390aeb97942a25310bd495 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Sun, 27 Aug 2017 19:00:27 +0900 Subject: [PATCH 139/151] Check directive in constant always --- lib/rdoc/parser/ruby.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index b2ef518148..57af252080 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -951,12 +951,10 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: (:on_kw == tk[:kind] && 'end' == tk[:text]) then nest -= 1 elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then + unget_tk tk + read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS if nest <= 0 and RDoc::RipperStateLex.end?(tk) then - unget_tk tk break - else - unget_tk tk - read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS end elsif :on_const == tk[:kind] then rhs_name << tk[:text] From 6d808d36dbc9f243e67aef4bca90e546acc0ddaf Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 4 Sep 2017 18:01:31 +0900 Subject: [PATCH 140/151] Treat float, rational and imaginary as numbers --- lib/rdoc/parser/ripper_state_lex.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 00738e9f16..cac9c397e1 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -143,6 +143,21 @@ def on_int(tok, data) @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end + def on_float(tok, data) + @lex_state = EXPR_END | EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_rational(tok, data) + @lex_state = EXPR_END | EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + + def on_imaginary(tok, data) + @lex_state = EXPR_END | EXPR_ENDARG + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + end + def on_symbeg(tok, data) @lex_state = EXPR_FNAME @continue = true From 1ffde6c02b9fb797ef1705fb2531a07fae127dd1 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 4 Sep 2017 18:02:29 +0900 Subject: [PATCH 141/151] Set EXPR_BEG to :on_heredoc_end --- lib/rdoc/parser/ripper_state_lex.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index cac9c397e1..aa491f9328 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -269,6 +269,11 @@ def on_ignored_sp(tok, data) @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) end + def on_heredoc_end(tok, data) + @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state}) + @lex_state = EXPR_BEG + end + def on_default(event, tok, data) reset @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state}) From 4eb4651a80af09b190f9d275a8c7958569765266 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 4 Sep 2017 18:02:47 +0900 Subject: [PATCH 142/151] Fix parse_constant_visibility for Ripper --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 57af252080..c5cdde7c08 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1957,7 +1957,7 @@ def parse_visibility(container, single, tk) def parse_constant_visibility(container, single, tk) args = parse_symbol_arg - case tk.name + case tk[:text] when 'private_constant' vis = :private when 'public_constant' From d881b6baa14f6de4ce8075627c117c64cafcbbe4 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Mon, 4 Sep 2017 18:08:19 +0900 Subject: [PATCH 143/151] Add EXPR_ARG to condition of postfix if --- lib/rdoc/parser/ripper_state_lex.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index aa491f9328..8508738cab 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -101,7 +101,7 @@ def on_kw(tok, data) @continue = true @in_fname = true when 'if', 'unless', 'while', 'until' - if ((EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_CMDARG) & @lex_state) != 0 # postfix if + if ((EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if @lex_state = EXPR_BEG | EXPR_LABEL else @lex_state = EXPR_BEG From e26fdfcc1cff54e4f35fdb08e5d9ea5f1d8cc99d Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 5 Sep 2017 12:29:54 +0900 Subject: [PATCH 144/151] Get constant body without last comment --- lib/rdoc/parser/ruby.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index c5cdde7c08..b26359aca4 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -935,6 +935,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: tk = get_tk + body = nil loop do break if tk.nil? if :on_semicolon == tk[:kind] then @@ -952,9 +953,12 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: nest -= 1 elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then unget_tk tk - read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS if nest <= 0 and RDoc::RipperStateLex.end?(tk) then + body = get_tkread_clean(/^[ \t]+/, '') + read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS break + else + read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS end elsif :on_const == tk[:kind] then rhs_name << tk[:text] @@ -975,7 +979,7 @@ def parse_constant_body container, constant, is_array_or_hash # :nodoc: tk = get_tk end - get_tkread_clean(/^[ \t]+/, '') + body ? body : get_tkread_clean(/^[ \t]+/, '') end ## From a8ea9550b11f9881f86c1471dbc8be2366811156 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 5 Sep 2017 15:11:45 +0900 Subject: [PATCH 145/151] Fix test_read_directive_one_liner --- test/test_rdoc_parser_ruby.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index fbd5bd470c..70f0f7f99c 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -2993,14 +2993,14 @@ def test_read_directive_no_comment end def test_read_directive_one_liner - parser = util_parser '; def foo; end # :category: test' + parser = util_parser 'AAA = 1 # :category: test' directive, value = parser.read_directive %w[category] assert_equal 'category', directive assert_equal 'test', value - assert_equal :on_semicolon, parser.get_tk[:kind] + assert_equal :on_const, parser.get_tk[:kind] end def test_read_documentation_modifiers From a2b511c79f2bef178891eb90f2a9e58611543605 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 5 Sep 2017 15:14:09 +0900 Subject: [PATCH 146/151] Remove unnecessary variable --- lib/rdoc/parser/ruby.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index b26359aca4..b77603f0cb 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1550,7 +1550,6 @@ def parse_method_or_yield_parameters(method = nil, end tk = get_tk end - @scanner.first_in_method_statement = true get_tkread_clean(/\s+/, ' ') end From 2725175d3738e325bdca9e617c61aeeef11112b3 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 5 Sep 2017 15:17:17 +0900 Subject: [PATCH 147/151] Fix test_parse_statements_def_percent_string_pound --- test/test_rdoc_parser_ruby.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 70f0f7f99c..0f0ffd1e4f 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -74,7 +74,7 @@ class C; end comment = parser.collect_first_comment - assert_equal RDoc::Comment.new("first\n", @top_level), comment + assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n", @top_level), comment end def test_get_class_or_module @@ -2154,7 +2154,9 @@ def test_parse_statements_def_percent_string_pound { :line_no => 2, :char_no => 5, :kind => :on_nl, :text => "\n" }, { :line_no => 3, :char_no => 0, :kind => :on_regexp, :text => '%r{#}' }, { :line_no => 3, :char_no => 5, :kind => :on_nl, :text => "\n" }, - { :line_no => 4, :char_no => 0, :kind => :on_kw, :text => 'end' } + { :line_no => 4, :char_no => 0, :kind => :on_regexp, :text => '%r{#{}}' }, + { :line_no => 4, :char_no => 7, :kind => :on_nl, :text => "\n" }, + { :line_no => 5, :char_no => 0, :kind => :on_kw, :text => 'end' } ] parsed_stream = a.token_stream.map { |tk| { From 80b7445e0dea565f46120e933ab02ddebb035d9f Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 5 Sep 2017 15:19:00 +0900 Subject: [PATCH 148/151] Fix test_parse_redefinable_methods --- test/test_rdoc_parser_ruby.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 0f0ffd1e4f..30058d2473 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -1713,8 +1713,8 @@ def test_parse_redefinable_methods end klass.method_list.each do |method| - assert_kind_of RDoc::RubyToken::TkId, method.token_stream[5] - assert_includes redefinable_ops, method.token_stream[5].text + assert_equal :on_ident, method.token_stream[5][:kind] + assert_includes redefinable_ops, method.token_stream[5][:text] end end From f3807d72b37f006b96b4d485463c34dc4b564fd8 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 5 Sep 2017 15:47:29 +0900 Subject: [PATCH 149/151] Fix handling method parmeter def a, b, # comment c end This is valid definition of method with a comment inside args. This commit fixes behavior for it. --- lib/rdoc/parser/ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index b77603f0cb..ed8ca9fb12 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1541,7 +1541,7 @@ def parse_method_or_yield_parameters(method = nil, unget_tk tk read_documentation_modifiers method, modifiers end - break if nest <= 0 + break if !continue and nest <= 0 end when :on_comma then continue = true From 6ff176600060f54dd100a4ddc4e544aa5fe71200 Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 5 Sep 2017 15:50:14 +0900 Subject: [PATCH 150/151] Add test_parse_method_parameters_with_paren_comment_continue --- test/test_rdoc_parser_ruby.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 30058d2473..69553d4791 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -1958,6 +1958,20 @@ def test_parse_method_parameters_comment_continue assert_equal '(arg1, arg2, arg3)', foo.params end + def test_parse_method_parameters_with_paren_comment_continue + klass = RDoc::NormalClass.new 'Foo' + klass.parent = @top_level + + util_parser "def foo(arg1, arg2, # some useful comment\narg3)\nend" + + tk = @parser.get_tk + + @parser.parse_method klass, RDoc::Parser::Ruby::NORMAL, tk, @comment + + foo = klass.method_list.first + assert_equal '(arg1, arg2, arg3)', foo.params + end + def test_parse_method_star klass = RDoc::NormalClass.new 'Foo' klass.parent = @top_level From 1946c08402d27a7167451f78d34511946570b14f Mon Sep 17 00:00:00 2001 From: Code Ass Date: Tue, 5 Sep 2017 18:03:06 +0900 Subject: [PATCH 151/151] Support earlier than Ruby 2.4 Remove String#match? and heredoc with ">>~". --- lib/rdoc/parser/ripper_state_lex.rb | 2 +- lib/rdoc/parser/ruby.rb | 6 +++--- test/test_rdoc_parser_ruby.rb | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 8508738cab..c9a0f5a21e 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -479,7 +479,7 @@ def get_squashed_tk private def retrieve_heredoc_info(tk) name = tk[:text].gsub(/\A<<[-~]?(['"`]?)(.+)\1\z/, '\2') - indent = tk[:text].match?(/\A<<[-~]/) + indent = tk[:text] =~ /\A<<[-~]/ [name, indent] end diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index ed8ca9fb12..130eca89c7 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -1666,15 +1666,15 @@ def parse_statements(container, single = NORMAL, current_method = nil, past_tokens = @read.size > 1 ? @read[0..-2] : [] nl_position = 0 past_tokens.reverse.each_with_index do |read_tk, i| - if read_tk.match?(/^\n$/) then + if read_tk =~ /^\n$/ then nl_position = (past_tokens.size - 1) - i break - elsif read_tk.match?(/^#.*\n$/) then + elsif read_tk =~ /^#.*\n$/ then nl_position = ((past_tokens.size - 1) - i) + 1 break end end - comment_only_line = past_tokens[nl_position..-1].all?{ |c| c.match?(/^\s+$/) } + comment_only_line = past_tokens[nl_position..-1].all?{ |c| c =~ /^\s+$/ } unless comment_only_line then tk = get_tk end diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb index 69553d4791..cda407ed51 100644 --- a/test/test_rdoc_parser_ruby.rb +++ b/test/test_rdoc_parser_ruby.rb @@ -2587,7 +2587,7 @@ def test_parse_statements_postfix_if_after_heredocbeg util_parser <def blah() - <<~EOM if true + <<-EOM if true EOM end EXPTECTED @@ -2608,7 +2608,7 @@ def blah() blah = foo.method_list.first markup_code = blah.markup_code.sub(/^.*\n/, '') - assert_equal markup_code, expected + assert_equal expected, markup_code end def test_parse_statements_method_oneliner_with_regexp