ok
Direktori : /opt/alt/ruby26/lib64/ruby/2.6.0/rdoc/parser/ |
Current File : //opt/alt/ruby26/lib64/ruby/2.6.0/rdoc/parser/ruby.rb |
# frozen_string_literal: true ## # This file contains stuff stolen outright from: # # rtags.rb - # ruby-lex.rb - ruby lexcal analyzer # ruby-token.rb - ruby tokens # by Keiju ISHITSUKA (Nippon Rational Inc.) # ## # Extracts code elements from a source file returning a TopLevel object # containing the constituent file elements. # # This file is based on rtags # # RubyParser understands how to document: # * classes # * modules # * methods # * constants # * aliases # * private, public, protected # * private_class_function, public_class_function # * private_constant, public_constant # * module_function # * attr, attr_reader, attr_writer, attr_accessor # * extra accessors given on the command line # * metaprogrammed methods # * require # * include # # == Method Arguments # #-- # NOTE: I don't think this works, needs tests, remove the paragraph following # this block when known to work # # The parser extracts the arguments from the method definition. You can # override this with a custom argument definition using the :args: directive: # # ## # # This method tries over and over until it is tired # # def go_go_go(thing_to_try, tries = 10) # :args: thing_to_try # puts thing_to_try # go_go_go thing_to_try, tries - 1 # end # # If you have a more-complex set of overrides you can use the :call-seq: # directive: #++ # # The parser extracts the arguments from the method definition. You can # override this with a custom argument definition using the :call-seq: # directive: # # ## # # This method can be called with a range or an offset and length # # # # :call-seq: # # my_method(Range) # # my_method(offset, length) # # def my_method(*args) # end # # The parser extracts +yield+ expressions from method bodies to gather the # yielded argument names. If your method manually calls a block instead of # yielding or you want to override the discovered argument names use # the :yields: directive: # # ## # # My method is awesome # # def my_method(&block) # :yields: happy, times # block.call 1, 2 # end # # == Metaprogrammed Methods # # To pick up a metaprogrammed method, the parser looks for a comment starting # with '##' before an identifier: # # ## # # This is a meta-programmed method! # # add_my_method :meta_method, :arg1, :arg2 # # The parser looks at the token after the identifier to determine the name, in # this example, :meta_method. If a name cannot be found, a warning is printed # and 'unknown is used. # # You can force the name of a method using the :method: directive: # # ## # # :method: some_method! # # By default, meta-methods are instance methods. To indicate that a method is # a singleton method instead use the :singleton-method: directive: # # ## # # :singleton-method: # # You can also use the :singleton-method: directive with a name: # # ## # # :singleton-method: some_method! # # You can define arguments for metaprogrammed methods via either the # :call-seq:, :arg: or :args: directives. # # Additionally you can mark a method as an attribute by # using :attr:, :attr_reader:, :attr_writer: or :attr_accessor:. Just like # for :method:, the name is optional. # # ## # # :attr_reader: my_attr_name # # == Hidden methods and attributes # # You can provide documentation for methods that don't appear using # the :method:, :singleton-method: and :attr: directives: # # ## # # :attr_writer: ghost_writer # # There is an attribute here, but you can't see it! # # ## # # :method: ghost_method # # There is a method here, but you can't see it! # # ## # # this is a comment for a regular method # # def regular_method() end # # Note that by default, the :method: directive will be ignored if there is a # standard rdocable item following it. require 'ripper' require_relative 'ripper_state_lex' class RDoc::Parser::Ruby < RDoc::Parser parse_files_matching(/\.rbw?$/) include RDoc::TokenStream include RDoc::Parser::RubyTools ## # RDoc::NormalClass type NORMAL = "::" ## # RDoc::SingleClass type SINGLE = "<<" ## # Creates a new Ruby parser. def initialize(top_level, file_name, content, options, stats) super if /\t/ =~ content then tab_width = @options.tab_width content = content.split(/\n/).map do |line| 1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width) } && $~ line end.join("\n") end @size = 0 @token_listeners = nil content = RDoc::Encoding.remove_magic_comment content @scanner = RDoc::Parser::RipperStateLex.parse(content) @content = content @scanner_point = 0 @prev_seek = nil @markup = @options.markup @track_visibility = :nodoc != @options.visibility @encoding = @options.encoding reset end def tk_nl?(tk) :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind] end ## # Retrieves the read token stream and replaces +pattern+ with +replacement+ # using gsub. If the result is only a ";" returns an empty string. def get_tkread_clean pattern, replacement # :nodoc: read = get_tkread.gsub(pattern, replacement).strip return '' if read == ';' read end ## # Extracts the visibility information for the visibility token +tk+ # and +single+ class type identifier. # # Returns the visibility type (a string), the visibility (a symbol) and # +singleton+ if the methods following should be converted to singleton # methods. def get_visibility_information tk, single # :nodoc: vis_type = tk[:text] singleton = single == SINGLE vis = case vis_type when 'private' then :private when 'protected' then :protected when 'public' then :public when 'private_class_method' then singleton = true :private when 'public_class_method' then singleton = true :public when 'module_function' then singleton = true :public else raise RDoc::Error, "Invalid visibility: #{tk.name}" end return vis_type, vis, singleton end ## # Look for the first comment in a file that isn't a shebang line. def collect_first_comment skip_tkspace comment = ''.dup comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding first_line = true first_comment_tk_kind = nil tk = get_tk while tk && (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) comment_body = retrieve_comment_body(tk) if first_line and comment_body =~ /\A#!/ then skip_tkspace tk = get_tk elsif first_line and comment_body =~ /\A#\s*-\*-/ then first_line = false skip_tkspace tk = get_tk else break if first_comment_tk_kind and not first_comment_tk_kind === tk[:kind] first_comment_tk_kind = tk[:kind] first_line = false comment << comment_body tk = get_tk if :on_nl === tk then skip_tkspace_without_nl tk = get_tk end end end unget_tk tk new_comment comment end ## # Consumes trailing whitespace from the token stream def consume_trailing_spaces # :nodoc: skip_tkspace_without_nl end ## # Creates a new attribute in +container+ with +name+. def create_attr container, single, name, rw, comment # :nodoc: att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE record_location att container.add_attribute att @stats.add_attribute att att end ## # Creates a module alias in +container+ at +rhs_name+ (or at the top-level # for "::") with the name from +constant+. def create_module_alias container, constant, rhs_name # :nodoc: mod = if rhs_name =~ /^::/ then @store.find_class_or_module rhs_name else container.find_module_named rhs_name end container.add_module_alias mod, rhs_name, constant, @top_level end ## # Aborts with +msg+ def error(msg) msg = make_message msg abort msg end ## # Looks for a true or false token. def get_bool skip_tkspace tk = get_tk if :on_kw == tk[:kind] && 'true' == tk[:text] true elsif :on_kw == tk[:kind] && ('false' == tk[:text] || 'nil' == tk[:text]) false else unget_tk tk true end end ## # Look for the name of a class of module (optionally with a leading :: or # with :: separated named) and return the ultimate name, the associated # container, and the given name (with the ::). def get_class_or_module container, ignore_constants = false skip_tkspace name_t = get_tk given_name = ''.dup # class ::A -> A is in the top level if :on_op == name_t[:kind] and '::' == name_t[:text] then # bug name_t = get_tk container = @top_level given_name << '::' end skip_tkspace_without_nl given_name << name_t[:text] is_self = name_t[:kind] == :on_op && name_t[:text] == '<<' new_modules = [] while !is_self && (tk = peek_tk) and :on_op == tk[:kind] and '::' == tk[:text] do prev_container = container container = container.find_module_named name_t[:text] container ||= if ignore_constants then c = RDoc::NormalModule.new name_t[:text] c.store = @store new_modules << [prev_container, c] c else c = prev_container.add_module RDoc::NormalModule, name_t[:text] c.ignore unless prev_container.document_children @top_level.add_to_classes_or_modules c c end record_location container get_tk skip_tkspace_without_nl name_t = get_tk unless :on_const == name_t[:kind] || :on_ident == name_t[:kind] raise RDoc::Error, "Invalid class or module definition: #{given_name}" end if prev_container == container and !ignore_constants given_name = name_t[:text] else given_name << '::' + name_t[:text] end end skip_tkspace_without_nl return [container, name_t, given_name, new_modules] end ## # Return a superclass, which can be either a constant of an expression def get_class_specification tk = peek_tk if tk.nil? return '' elsif :on_kw == tk[:kind] && 'self' == tk[:text] return 'self' elsif :on_gvar == tk[:kind] return '' end res = get_constant skip_tkspace_without_nl get_tkread # empty out read buffer tk = get_tk return res unless tk case tk[:kind] when :on_nl, :on_comment, :on_embdoc, :on_semicolon then unget_tk(tk) return res end res += parse_call_parameters(tk) res end ## # Parse a constant, which might be qualified by one or more class or module # names def get_constant res = "" skip_tkspace_without_nl tk = get_tk while tk && ((:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind]) do res += tk[:text] tk = get_tk end unget_tk(tk) res end ## # Get an included module that may be surrounded by parens def get_included_module_with_optional_parens skip_tkspace_without_nl get_tkread tk = get_tk end_token = get_end_token tk return '' unless end_token nest = 0 continue = false only_constant = true while tk != nil do is_element_of_constant = false case tk[:kind] when :on_semicolon then break if nest == 0 when :on_lbracket then nest += 1 when :on_rbracket then nest -= 1 when :on_lbrace then nest += 1 when :on_rbrace then nest -= 1 if nest <= 0 # we might have a.each { |i| yield i } unget_tk(tk) if nest < 0 break end when :on_lparen then nest += 1 when end_token[:kind] then if end_token[:kind] == :on_rparen nest -= 1 break if nest <= 0 else break if nest <= 0 end when :on_rparen then nest -= 1 when :on_comment, :on_embdoc then @read.pop if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and (!continue or (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0) then break if !continue and nest <= 0 end when :on_comma then continue = true when :on_ident then continue = false if continue when :on_kw then case tk[:text] when 'def', 'do', 'case', 'for', 'begin', 'class', 'module' nest += 1 when 'if', 'unless', 'while', 'until', 'rescue' # postfix if/unless/while/until/rescue must be EXPR_LABEL nest += 1 unless (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0 when 'end' nest -= 1 break if nest == 0 end when :on_const then is_element_of_constant = true when :on_op then is_element_of_constant = true if '::' == tk[:text] end only_constant = false unless is_element_of_constant tk = get_tk end if only_constant get_tkread_clean(/\s+/, ' ') else '' end end ## # Little hack going on here. In the statement: # # f = 2*(1+yield) # # We see the RPAREN as the next token, so we need to exit early. This still # won't catch all cases (such as "a = yield + 1" def get_end_token tk # :nodoc: case tk[:kind] when :on_lparen token = RDoc::Parser::RipperStateLex::Token.new token[:kind] = :on_rparen token[:text] = ')' token when :on_rparen nil else token = RDoc::Parser::RipperStateLex::Token.new token[:kind] = :on_nl token[:text] = "\n" token end end ## # Retrieves the method container for a singleton method. def get_method_container container, name_t # :nodoc: prev_container = container container = container.find_module_named(name_t[:text]) unless container then constant = prev_container.constants.find do |const| const.name == name_t[:text] end if constant then parse_method_dummy prev_container return end end unless container then # TODO seems broken, should starting at Object in @store obj = name_t[:text].split("::").inject(Object) do |state, item| state.const_get(item) end rescue nil type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule unless [Class, Module].include?(obj.class) then warn("Couldn't find #{name_t[:text]}. Assuming it's a module") end if type == RDoc::NormalClass then sclass = obj.superclass ? obj.superclass.name : nil container = prev_container.add_class type, name_t[:text], sclass else container = prev_container.add_module type, name_t[:text] end record_location container end container end ## # Extracts a name or symbol from the token stream. def get_symbol_or_name tk = get_tk case tk[:kind] when :on_symbol then text = tk[:text].sub(/^:/, '') next_tk = peek_tk if next_tk && :on_op == next_tk[:kind] && '=' == next_tk[:text] then get_tk text << '=' end text when :on_ident, :on_const, :on_gvar, :on_cvar, :on_ivar, :on_op, :on_kw then tk[:text] when :on_tstring, :on_dstring then tk[:text][1..-2] else raise RDoc::Error, "Name or symbol expected (got #{tk})" end end ## # Marks containers between +container+ and +ancestor+ as ignored def suppress_parents container, ancestor # :nodoc: while container and container != ancestor do container.suppress unless container.documented? container = container.parent end end ## # Look for directives in a normal comment block: # # # :stopdoc: # # Don't display comment from this point forward # # This routine modifies its +comment+ parameter. def look_for_directives_in container, comment @preprocess.handle comment, container do |directive, param| case directive when 'method', 'singleton-method', 'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then false # handled elsewhere when 'section' then break unless container.kind_of?(RDoc::Context) container.set_current_section param, comment.dup comment.text = '' break end end comment.remove_private end ## # Adds useful info about the parser to +message+ def make_message message prefix = "#{@file_name}:".dup tk = peek_tk prefix << "#{tk[:line_no]}:#{tk[:char_no]}:" if tk "#{prefix} #{message}" end ## # Creates a comment with the correct format def new_comment comment c = RDoc::Comment.new comment, @top_level c.format = @markup c end ## # Creates an RDoc::Attr for the name following +tk+, setting the comment to # +comment+. def parse_attr(context, single, tk, comment) line_no = tk[:line_no] args = parse_symbol_arg 1 if args.size > 0 then name = args[0] rw = "R" skip_tkspace_without_nl tk = get_tk if :on_comma == tk[:kind] then rw = "RW" if get_bool else unget_tk tk end att = create_attr context, single, name, rw, comment att.line = line_no read_documentation_modifiers att, RDoc::ATTR_MODIFIERS else warn "'attr' ignored - looks like a variable" end end ## # Creates an RDoc::Attr for each attribute listed after +tk+, setting the # comment for each to +comment+. def parse_attr_accessor(context, single, tk, comment) line_no = tk[:line_no] args = parse_symbol_arg rw = "?" tmp = RDoc::CodeObject.new read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS # TODO In most other places we let the context keep track of document_self # and add found items appropriately but here we do not. I'm not sure why. return if @track_visibility and not tmp.document_self case tk[:text] when "attr_reader" then rw = "R" when "attr_writer" then rw = "W" when "attr_accessor" then rw = "RW" else rw = '?' end for name in args att = create_attr context, single, name, rw, comment att.line = line_no end end ## # Parses an +alias+ in +context+ with +comment+ def parse_alias(context, single, tk, comment) line_no = tk[:line_no] skip_tkspace if :on_lparen === peek_tk[:kind] then get_tk skip_tkspace end new_name = get_symbol_or_name skip_tkspace if :on_comma === peek_tk[:kind] then get_tk skip_tkspace end begin old_name = get_symbol_or_name rescue RDoc::Error return end al = RDoc::Alias.new(get_tkread, old_name, new_name, comment, single == SINGLE) record_location al al.line = line_no read_documentation_modifiers al, RDoc::ATTR_MODIFIERS context.add_alias al @stats.add_alias al al end ## # Extracts call parameters from the token stream. def parse_call_parameters(tk) end_token = case tk[:kind] when :on_lparen :on_rparen when :on_rparen return "" else :on_nl end nest = 0 loop do break if tk.nil? case tk[:kind] when :on_semicolon break when :on_lparen nest += 1 when end_token if end_token == :on_rparen nest -= 1 break if RDoc::Parser::RipperStateLex.end?(tk) and nest <= 0 else break if RDoc::Parser::RipperStateLex.end?(tk) end when :on_comment, :on_embdoc unget_tk(tk) break when :on_op if tk[:text] =~ /^(.{1,2})?=$/ unget_tk(tk) break end end tk = get_tk end get_tkread_clean "\n", " " end ## # Parses a class in +context+ with +comment+ def parse_class container, single, tk, comment line_no = tk[:line_no] declaration_context = container container, name_t, given_name, = get_class_or_module container if name_t[:kind] == :on_const cls = parse_class_regular container, declaration_context, single, name_t, given_name, comment elsif name_t[:kind] == :on_op && name_t[:text] == '<<' case name = get_class_specification when 'self', container.name read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS parse_statements container, SINGLE return # don't update line else cls = parse_class_singleton container, name, comment end else warn "Expected class name or '<<'. Got #{name_t[:kind]}: #{name_t[:text].inspect}" return end cls.line = line_no # after end modifiers read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS cls end ## # Parses and creates a regular class def parse_class_regular container, declaration_context, single, # :nodoc: name_t, given_name, comment superclass = '::Object' if given_name =~ /^::/ then declaration_context = @top_level given_name = $' end tk = peek_tk if tk[:kind] == :on_op && tk[:text] == '<' then get_tk skip_tkspace superclass = get_class_specification superclass = '(unknown)' if superclass.empty? end cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass cls = declaration_context.add_class cls_type, given_name, superclass cls.ignore unless container.document_children read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS record_location cls cls.add_comment comment, @top_level @top_level.add_to_classes_or_modules cls @stats.add_class cls suppress_parents container, declaration_context unless cls.document_self parse_statements cls cls end ## # Parses a singleton class in +container+ with the given +name+ and # +comment+. def parse_class_singleton container, name, comment # :nodoc: other = @store.find_class_named name unless other then if name =~ /^::/ then name = $' container = @top_level end other = container.add_module RDoc::NormalModule, name record_location other # class << $gvar other.ignore if name.empty? other.add_comment comment, @top_level end # notify :nodoc: all if not a constant-named class/module # (and remove any comment) unless name =~ /\A(::)?[A-Z]/ then other.document_self = nil other.document_children = false other.clear_comment end @top_level.add_to_classes_or_modules other @stats.add_class other read_documentation_modifiers other, RDoc::CLASS_MODIFIERS parse_statements(other, SINGLE) other end ## # Parses a constant in +context+ with +comment+. If +ignore_constants+ is # true, no found constants will be added to RDoc. def parse_constant container, tk, comment, ignore_constants = false line_no = tk[:line_no] name = tk[:text] skip_tkspace_without_nl return unless name =~ /^\w+$/ new_modules = [] if :on_op == peek_tk[:kind] && '::' == peek_tk[:text] then unget_tk tk container, name_t, _, new_modules = get_class_or_module container, true name = name_t[:text] end is_array_or_hash = false if peek_tk && :on_lbracket == peek_tk[:kind] get_tk nest = 1 while bracket_tk = get_tk case bracket_tk[:kind] when :on_lbracket nest += 1 when :on_rbracket nest -= 1 break if nest == 0 end end skip_tkspace_without_nl is_array_or_hash = true end unless peek_tk && :on_op == peek_tk[:kind] && '=' == peek_tk[:text] then return false end get_tk unless ignore_constants new_modules.each do |prev_c, new_module| prev_c.add_module_by_normal_module new_module new_module.ignore unless prev_c.document_children @top_level.add_to_classes_or_modules new_module end end value = '' con = RDoc::Constant.new name, value, comment body = parse_constant_body container, con, is_array_or_hash return unless body con.value = body record_location con con.line = line_no read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS return if is_array_or_hash @stats.add_constant con container.add_constant con true end def parse_constant_body container, constant, is_array_or_hash # :nodoc: nest = 0 rhs_name = ''.dup get_tkread tk = get_tk body = nil loop do break if tk.nil? if :on_semicolon == tk[:kind] then break if nest <= 0 elsif [:on_tlambeg, :on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) then nest += 1 elsif (:on_kw == tk[:kind] && 'def' == tk[:text]) then nest += 1 elsif (:on_kw == tk[:kind] && %w{do if unless case begin}.include?(tk[:text])) then if (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) == 0 nest += 1 end elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) || (:on_kw == tk[:kind] && 'end' == tk[:text]) then nest -= 1 elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then unget_tk tk if nest <= 0 and RDoc::Parser::RipperStateLex.end?(tk) then body = get_tkread_clean(/^[ \t]+/, '') read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS break else read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS end elsif :on_const == tk[:kind] then rhs_name << tk[:text] next_tk = peek_tk if nest <= 0 and (next_tk.nil? || :on_nl == next_tk[:kind]) then create_module_alias container, constant, rhs_name unless is_array_or_hash break end elsif :on_nl == tk[:kind] then if nest <= 0 and RDoc::Parser::RipperStateLex.end?(tk) then unget_tk tk break end elsif :on_op == tk[:kind] && '::' == tk[:text] rhs_name << '::' end tk = get_tk end body ? body : get_tkread_clean(/^[ \t]+/, '') end ## # Generates an RDoc::Method or RDoc::Attr from +comment+ by looking for # :method: or :attr: directives in +comment+. def parse_comment container, tk, comment return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc' column = tk[:char_no] line_no = tk[:line_no] comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3') singleton = !!$~ co = if (comment.text = comment.text.sub(/^# +:?method: *(\S*).*?\n/i, '')) && !!$~ then parse_comment_ghost container, comment.text, $1, column, line_no, comment elsif (comment.text = comment.text.sub(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '')) && !!$~ then parse_comment_attr container, $1, $3, comment end if co then co.singleton = singleton co.line = line_no end true end ## # Parse a comment that is describing an attribute in +container+ with the # given +name+ and +comment+. def parse_comment_attr container, type, name, comment # :nodoc: return if name.empty? rw = case type when 'attr_reader' then 'R' when 'attr_writer' then 'W' else 'RW' end create_attr container, NORMAL, name, rw, comment end def parse_comment_ghost container, text, name, column, line_no, # :nodoc: comment name = nil if name.empty? meth = RDoc::GhostMethod.new get_tkread, name record_location meth meth.start_collecting_tokens indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column) position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment) position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n") meth.add_tokens [position_comment, newline, indent] meth.params = if text.sub!(/^#\s+:?args?:\s*(.*?)\s*$/i, '') then $1 else '' end comment.normalize comment.extract_call_seq meth return unless meth.name container.add_method meth meth.comment = comment @stats.add_method meth meth end ## # Creates an RDoc::Method on +container+ from +comment+ if there is a # Signature section in the comment def parse_comment_tomdoc container, tk, comment return unless signature = RDoc::TomDoc.signature(comment) column = tk[:char_no] line_no = tk[:line_no] name, = signature.split %r%[ \(]%, 2 meth = RDoc::GhostMethod.new get_tkread, name record_location meth meth.line = line_no meth.start_collecting_tokens indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column) position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment) position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n") meth.add_tokens [position_comment, newline, indent] meth.call_seq = signature comment.normalize return unless meth.name container.add_method meth meth.comment = comment @stats.add_method meth end ## # Parses an +include+ or +extend+, indicated by the +klass+ and adds it to # +container+ # with +comment+ def parse_extend_or_include klass, container, comment # :nodoc: loop do skip_tkspace_comment name = get_included_module_with_optional_parens unless name.empty? then obj = container.add klass, name, comment record_location obj end return if peek_tk.nil? || :on_comma != peek_tk[:kind] get_tk end end ## # Parses identifiers that can create new methods or change visibility. # # Returns true if the comment was not consumed. def parse_identifier container, single, tk, comment # :nodoc: case tk[:text] when 'private', 'protected', 'public', 'private_class_method', 'public_class_method', 'module_function' then parse_visibility container, single, tk return true when 'private_constant', 'public_constant' parse_constant_visibility container, single, tk return true when 'attr' then parse_attr container, single, tk, comment when /^attr_(reader|writer|accessor)$/ then parse_attr_accessor container, single, tk, comment when 'alias_method' then parse_alias container, single, tk, comment when 'require', 'include' then # ignore else if comment.text =~ /\A#\#$/ then case comment.text when /^# +:?attr(_reader|_writer|_accessor)?:/ then parse_meta_attr container, single, tk, comment else method = parse_meta_method container, single, tk, comment method.params = container.params if container.params method.block_params = container.block_params if container.block_params end end end false end ## # Parses a meta-programmed attribute and creates an RDoc::Attr. # # To create foo and bar attributes on class C with comment "My attributes": # # class C # # ## # # :attr: # # # # My attributes # # my_attr :foo, :bar # # end # # To create a foo attribute on class C with comment "My attribute": # # class C # # ## # # :attr: foo # # # # My attribute # # my_attr :foo, :bar # # end def parse_meta_attr(context, single, tk, comment) args = parse_symbol_arg rw = "?" # If nodoc is given, don't document any of them tmp = RDoc::CodeObject.new read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS regexp = /^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i if regexp =~ comment.text then comment.text = comment.text.sub(regexp, '') rw = case $1 when 'attr_reader' then 'R' when 'attr_writer' then 'W' else 'RW' end name = $3 unless $3.empty? end if name then att = create_attr context, single, name, rw, comment else args.each do |attr_name| att = create_attr context, single, attr_name, rw, comment end end att end ## # Parses a meta-programmed method def parse_meta_method(container, single, tk, comment) column = tk[:char_no] line_no = tk[:line_no] start_collecting_tokens add_token tk add_token_listener self skip_tkspace_without_nl comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3') singleton = !!$~ name = parse_meta_method_name comment, tk return unless name meth = RDoc::MetaMethod.new get_tkread, name record_location meth meth.line = line_no meth.singleton = singleton remove_token_listener self meth.start_collecting_tokens indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column) position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment) position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}" newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n") meth.add_tokens [position_comment, newline, indent] meth.add_tokens @token_stream parse_meta_method_params container, single, meth, tk, comment meth.comment = comment @stats.add_method meth meth end ## # Parses the name of a metaprogrammed method. +comment+ is used to # determine the name while +tk+ is used in an error message if the name # cannot be determined. def parse_meta_method_name comment, tk # :nodoc: if comment.text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then return $1 unless $1.empty? end name_t = get_tk if :on_symbol == name_t[:kind] then name_t[:text][1..-1] elsif :on_tstring == name_t[:kind] then name_t[:text][1..-2] elsif :on_op == name_t[:kind] && '=' == name_t[:text] then # ignore remove_token_listener self nil else warn "unknown name token #{name_t.inspect} for meta-method '#{tk[:text]}'" 'unknown' end end ## # Parses the parameters and block for a meta-programmed method. def parse_meta_method_params container, single, meth, tk, comment # :nodoc: token_listener meth do meth.params = '' look_for_directives_in meth, comment comment.normalize comment.extract_call_seq meth container.add_method meth last_tk = tk while tk = get_tk do if :on_semicolon == tk[:kind] then break elsif :on_nl == tk[:kind] then break unless last_tk and :on_comma == last_tk[:kind] elsif :on_sp == tk[:kind] then # expression continues elsif :on_kw == tk[:kind] && 'do' == tk[:text] then parse_statements container, single, meth break else last_tk = tk end end end end ## # Parses a normal method defined by +def+ def parse_method(container, single, tk, comment) singleton = nil added_container = false name = nil column = tk[:char_no] line_no = tk[:line_no] start_collecting_tokens add_token tk token_listener self do prev_container = container name, container, singleton = parse_method_name container added_container = container != prev_container end return unless name meth = RDoc::AnyMethod.new get_tkread, name look_for_directives_in meth, comment meth.singleton = single == SINGLE ? true : singleton record_location meth meth.line = line_no meth.start_collecting_tokens indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column) token = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment) token[:text] = "# File #{@top_level.relative_name}, line #{line_no}" newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n") meth.add_tokens [token, newline, indent] meth.add_tokens @token_stream parse_method_params_and_body container, single, meth, added_container comment.normalize comment.extract_call_seq meth meth.comment = comment # after end modifiers read_documentation_modifiers meth, RDoc::METHOD_MODIFIERS @stats.add_method meth end ## # Parses the parameters and body of +meth+ def parse_method_params_and_body container, single, meth, added_container token_listener meth do parse_method_parameters meth if meth.document_self or not @track_visibility then container.add_method meth elsif added_container then container.document_self = false end # Having now read the method parameters and documentation modifiers, we # now know whether we have to rename #initialize to ::new if meth.name == "initialize" && !meth.singleton then if meth.dont_rename_initialize then meth.visibility = :protected else meth.singleton = true meth.name = "new" meth.visibility = :public end end parse_statements container, single, meth end end ## # Parses a method that needs to be ignored. def parse_method_dummy container dummy = RDoc::Context.new dummy.parent = container dummy.store = container.store skip_method dummy end ## # Parses the name of a method in +container+. # # Returns the method name, the container it is in (for def Foo.name) and if # it is a singleton or regular method. def parse_method_name container # :nodoc: skip_tkspace name_t = get_tk back_tk = skip_tkspace_without_nl singleton = false dot = get_tk if dot[:kind] == :on_period || (dot[:kind] == :on_op && dot[:text] == '::') then singleton = true name, container = parse_method_name_singleton container, name_t else unget_tk dot back_tk.reverse_each do |token| unget_tk token end name = parse_method_name_regular container, name_t end return name, container, singleton end ## # For the given +container+ and initial name token +name_t+ the method name # is parsed from the token stream for a regular method. def parse_method_name_regular container, name_t # :nodoc: if :on_op == name_t[:kind] && (%w{* & [] []= <<}.include?(name_t[:text])) then name_t[:text] else unless [:on_kw, :on_const, :on_ident].include?(name_t[:kind]) then warn "expected method name token, . or ::, got #{name_t.inspect}" skip_method container return end name_t[:text] end end ## # For the given +container+ and initial name token +name_t+ the method name # and the new +container+ (if necessary) are parsed from the token stream # for a singleton method. def parse_method_name_singleton container, name_t # :nodoc: skip_tkspace name_t2 = get_tk if (:on_kw == name_t[:kind] && 'self' == name_t[:text]) || (:on_op == name_t[:kind] && '%' == name_t[:text]) then # NOTE: work around '[' being consumed early if :on_lbracket == name_t2[:kind] get_tk name = '[]' else name = name_t2[:text] end elsif :on_const == name_t[:kind] then name = name_t2[:text] container = get_method_container container, name_t return unless container name elsif :on_ident == name_t[:kind] || :on_ivar == name_t[:kind] || :on_gvar == name_t[:kind] then parse_method_dummy container name = nil elsif (:on_kw == name_t[:kind]) && ('true' == name_t[:text] || 'false' == name_t[:text] || 'nil' == name_t[:text]) then klass_name = "#{name_t[:text].capitalize}Class" container = @store.find_class_named klass_name container ||= @top_level.add_class RDoc::NormalClass, klass_name name = name_t2[:text] else warn "unexpected method name token #{name_t.inspect}" # break skip_method container name = nil end return name, container end ## # Extracts +yield+ parameters from +method+ def parse_method_or_yield_parameters(method = nil, modifiers = RDoc::METHOD_MODIFIERS) skip_tkspace_without_nl tk = get_tk end_token = get_end_token tk return '' unless end_token nest = 0 continue = false while tk != nil do case tk[:kind] when :on_semicolon then break if nest == 0 when :on_lbracket then nest += 1 when :on_rbracket then nest -= 1 when :on_lbrace then nest += 1 when :on_rbrace then nest -= 1 if nest <= 0 # we might have a.each { |i| yield i } unget_tk(tk) if nest < 0 break end when :on_lparen then nest += 1 when end_token[:kind] then if end_token[:kind] == :on_rparen nest -= 1 break if nest <= 0 else break end when :on_rparen then nest -= 1 when :on_comment, :on_embdoc then @read.pop if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and (!continue or (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0) then if method && method.block_params.nil? then unget_tk tk read_documentation_modifiers method, modifiers end break if !continue and nest <= 0 end when :on_comma then continue = true when :on_ident then continue = false if continue end tk = get_tk end get_tkread_clean(/\s+/, ' ') end ## # Capture the method's parameters. Along the way, look for a comment # containing: # # # yields: .... # # and add this as the block_params for the method def parse_method_parameters method res = parse_method_or_yield_parameters method res = "(#{res})" unless res =~ /\A\(/ method.params = res unless method.params return if method.block_params skip_tkspace_without_nl read_documentation_modifiers method, RDoc::METHOD_MODIFIERS end ## # Parses an RDoc::NormalModule in +container+ with +comment+ def parse_module container, single, tk, comment container, name_t, = get_class_or_module container name = name_t[:text] mod = container.add_module RDoc::NormalModule, name mod.ignore unless container.document_children record_location mod read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS mod.add_comment comment, @top_level parse_statements mod # after end modifiers read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS @stats.add_module mod end ## # Parses an RDoc::Require in +context+ containing +comment+ def parse_require(context, comment) skip_tkspace_comment tk = get_tk if :on_lparen == tk[:kind] then skip_tkspace_comment tk = get_tk end name = tk[:text][1..-2] if :on_tstring == tk[:kind] if name then @top_level.add_require RDoc::Require.new(name, comment) else unget_tk tk end end ## # Parses a rescue def parse_rescue skip_tkspace_without_nl while tk = get_tk case tk[:kind] when :on_nl, :on_semicolon, :on_comment then break when :on_comma then skip_tkspace_without_nl get_tk if :on_nl == peek_tk[:kind] end skip_tkspace_without_nl end end ## # Retrieve comment body without =begin/=end def retrieve_comment_body(tk) if :on_embdoc == tk[:kind] tk[:text].gsub(/\A=begin.*\n/, '').gsub(/=end\n?\z/, '') else tk[:text] end end ## # The core of the Ruby parser. def parse_statements(container, single = NORMAL, current_method = nil, comment = new_comment('')) raise 'no' unless RDoc::Comment === comment comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding nest = 1 save_visibility = container.visibility non_comment_seen = true while tk = get_tk do keep_comment = false try_parse_comment = false non_comment_seen = true unless (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) case tk[:kind] when :on_nl, :on_ignored_nl, :on_comment, :on_embdoc then if :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind] skip_tkspace tk = get_tk else past_tokens = @read.size > 1 ? @read[0..-2] : [] nl_position = 0 past_tokens.reverse.each_with_index do |read_tk, i| if read_tk =~ /^\n$/ then nl_position = (past_tokens.size - 1) - i break elsif read_tk =~ /^#.*\n$/ then nl_position = ((past_tokens.size - 1) - i) + 1 break end end comment_only_line = past_tokens[nl_position..-1].all?{ |c| c =~ /^\s+$/ } unless comment_only_line then tk = get_tk end end if tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then if non_comment_seen then # Look for RDoc in a comment about to be thrown away non_comment_seen = parse_comment container, tk, comment unless comment.empty? comment = '' comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding end while tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) do comment_body = retrieve_comment_body(tk) comment += comment_body comment += "\n" unless "\n" == comment_body.chars.to_a.last if comment_body.size > 1 && "\n" == comment_body.chars.to_a.last then skip_tkspace_without_nl # leading spaces end tk = get_tk end comment = new_comment comment unless comment.empty? then look_for_directives_in container, comment if container.done_documenting then throw :eof if RDoc::TopLevel === container container.ongoing_visibility = save_visibility end end keep_comment = true else non_comment_seen = true end unget_tk tk keep_comment = true container.current_line_visibility = nil when :on_kw then case tk[:text] when 'class' then parse_class container, single, tk, comment when 'module' then parse_module container, single, tk, comment when 'def' then parse_method container, single, tk, comment when 'alias' then parse_alias container, single, tk, comment unless current_method when 'yield' then if current_method.nil? then warn "Warning: yield outside of method" if container.document_self else parse_yield container, single, tk, current_method end when 'until', 'while' then if (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) == 0 nest += 1 skip_optional_do_after_expression end # Until and While can have a 'do', which shouldn't increase the nesting. # We can't solve the general case, but we can handle most occurrences by # ignoring a do at the end of a line. # 'for' is trickier when 'for' then nest += 1 skip_for_variable skip_optional_do_after_expression when 'case', 'do', 'if', 'unless', 'begin' then if (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) == 0 nest += 1 end when 'super' then current_method.calls_super = true if current_method when 'rescue' then parse_rescue when 'end' then nest -= 1 if nest == 0 then container.ongoing_visibility = save_visibility parse_comment container, tk, comment unless comment.empty? return end end when :on_const then unless parse_constant container, tk, comment, current_method then try_parse_comment = true end when :on_ident then if nest == 1 and current_method.nil? then keep_comment = parse_identifier container, single, tk, comment end case tk[:text] when "require" then parse_require container, comment when "include" then parse_extend_or_include RDoc::Include, container, comment when "extend" then parse_extend_or_include RDoc::Extend, container, comment end else try_parse_comment = nest == 1 end if try_parse_comment then non_comment_seen = parse_comment container, tk, comment unless comment.empty? keep_comment = false end unless keep_comment then comment = new_comment '' comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding container.params = nil container.block_params = nil end consume_trailing_spaces end container.params = nil container.block_params = nil end ## # Parse up to +no+ symbol arguments def parse_symbol_arg(no = nil) skip_tkspace_comment tk = get_tk if tk[:kind] == :on_lparen parse_symbol_arg_paren no else parse_symbol_arg_space no, tk end end ## # Parses up to +no+ symbol arguments surrounded by () and places them in # +args+. def parse_symbol_arg_paren no # :nodoc: args = [] loop do skip_tkspace_comment if tk1 = parse_symbol_in_arg args.push tk1 break if no and args.size >= no end skip_tkspace_comment case (tk2 = get_tk)[:kind] when :on_rparen break when :on_comma else warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC break end end args end ## # Parses up to +no+ symbol arguments separated by spaces and places them in # +args+. def parse_symbol_arg_space no, tk # :nodoc: args = [] unget_tk tk if tk = parse_symbol_in_arg args.push tk return args if no and args.size >= no end loop do skip_tkspace_without_nl tk1 = get_tk if tk1.nil? || :on_comma != tk1[:kind] then unget_tk tk1 break end skip_tkspace_comment if tk = parse_symbol_in_arg args.push tk break if no and args.size >= no end end args end ## # Returns symbol text from the next token def parse_symbol_in_arg tk = get_tk if :on_symbol == tk[:kind] then tk[:text].sub(/^:/, '') elsif :on_tstring == tk[:kind] then tk[:text][1..-2] elsif :on_dstring == tk[:kind] or :on_ident == tk[:kind] then nil # ignore else warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC nil end end ## # Parses statements in the top-level +container+ def parse_top_level_statements container comment = collect_first_comment look_for_directives_in container, comment throw :eof if container.done_documenting @markup = comment.format # HACK move if to RDoc::Context#comment= container.comment = comment if container.document_self unless comment.empty? parse_statements container, NORMAL, nil, comment end ## # Determines the visibility in +container+ from +tk+ def parse_visibility(container, single, tk) vis_type, vis, singleton = get_visibility_information tk, single skip_tkspace_comment false ptk = peek_tk # Ryan Davis suggested the extension to ignore modifiers, because he # often writes # # protected unless $TESTING # if [:on_nl, :on_semicolon].include?(ptk[:kind]) || (:on_kw == ptk[:kind] && (['if', 'unless'].include?(ptk[:text]))) then container.ongoing_visibility = vis elsif :on_kw == ptk[:kind] && 'def' == ptk[:text] container.current_line_visibility = vis else update_visibility container, vis_type, vis, singleton end end ## # Parses a Module#private_constant or Module#public_constant call from +tk+. def parse_constant_visibility(container, single, tk) args = parse_symbol_arg case tk[:text] when 'private_constant' vis = :private when 'public_constant' vis = :public else raise RDoc::Error, 'Unreachable' end container.set_constant_visibility_for args, vis end ## # Determines the block parameter for +context+ def parse_yield(context, single, tk, method) return if method.block_params get_tkread method.block_params = parse_method_or_yield_parameters end ## # Directives are modifier comments that can appear after class, module, or # method names. For example: # # def fred # :yields: a, b # # or: # # class MyClass # :nodoc: # # We return the directive name and any parameters as a two element array if # the name is in +allowed+. A directive can be found anywhere up to the end # of the current line. def read_directive allowed tokens = [] while tk = get_tk do tokens << tk if :on_nl == tk[:kind] or (:on_kw == tk[:kind] && 'def' == tk[:text]) then return elsif :on_comment == tk[:kind] or :on_embdoc == tk[:kind] then return unless tk[:text] =~ /\s*:?([\w-]+):\s*(.*)/ directive = $1.downcase return [directive, $2] if allowed.include? directive return end end ensure unless tokens.length == 1 and (:on_comment == tokens.first[:kind] or :on_embdoc == tokens.first[:kind]) then tokens.reverse_each do |token| unget_tk token end end end ## # Handles directives following the definition for +context+ (any # RDoc::CodeObject) if the directives are +allowed+ at this point. # # See also RDoc::Markup::PreProcess#handle_directive def read_documentation_modifiers context, allowed skip_tkspace_without_nl directive, value = read_directive allowed return unless directive @preprocess.handle_directive '', directive, value, context do |dir, param| if %w[notnew not_new not-new].include? dir then context.dont_rename_initialize = true true end end end ## # Records the location of this +container+ in the file for this parser and # adds it to the list of classes and modules in the file. def record_location container # :nodoc: case container when RDoc::ClassModule then @top_level.add_to_classes_or_modules container end container.record_location @top_level end ## # Scans this Ruby file for Ruby constructs def scan reset catch :eof do begin parse_top_level_statements @top_level rescue StandardError => e if @content.include?('<%') and @content.include?('%>') then # Maybe, this is ERB. $stderr.puts "\033[2KRDoc detects ERB file. Skips it for compatibility:" $stderr.puts @file_name return end if @scanner_point >= @scanner.size now_line_no = @scanner[@scanner.size - 1][:line_no] else now_line_no = peek_tk[:line_no] end first_tk_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no } last_tk_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no + 1 } last_tk_index = last_tk_index ? last_tk_index - 1 : @scanner.size - 1 code = @scanner[first_tk_index..last_tk_index].map{ |t| t[:text] }.join $stderr.puts <<-EOF #{self.class} failure around line #{now_line_no} of #{@file_name} EOF unless code.empty? then $stderr.puts code $stderr.puts end raise e end end @top_level end ## # while, until, and for have an optional do def skip_optional_do_after_expression skip_tkspace_without_nl tk = get_tk b_nest = 0 nest = 0 loop do break unless tk case tk[:kind] when :on_semicolon, :on_nl, :on_ignored_nl then break if b_nest.zero? when :on_lparen then nest += 1 when :on_rparen then nest -= 1 when :on_kw then case tk[:text] when 'begin' b_nest += 1 when 'end' b_nest -= 1 when 'do' break if nest.zero? end when :on_comment, :on_embdoc then if b_nest.zero? and "\n" == tk[:text][-1] then break end end tk = get_tk end skip_tkspace_without_nl get_tk if peek_tk && :on_kw == peek_tk[:kind] && 'do' == peek_tk[:text] end ## # skip the var [in] part of a 'for' statement def skip_for_variable skip_tkspace_without_nl get_tk skip_tkspace_without_nl tk = get_tk unget_tk(tk) unless :on_kw == tk[:kind] and 'in' == tk[:text] end ## # Skips the next method in +container+ def skip_method container meth = RDoc::AnyMethod.new "", "anon" parse_method_parameters meth parse_statements container, false, meth end ## # Skip spaces until a comment is found def skip_tkspace_comment(skip_nl = true) loop do skip_nl ? skip_tkspace : skip_tkspace_without_nl next_tk = peek_tk return if next_tk.nil? || (:on_comment != next_tk[:kind] and :on_embdoc != next_tk[:kind]) get_tk end end ## # Updates visibility in +container+ from +vis_type+ and +vis+. def update_visibility container, vis_type, vis, singleton # :nodoc: new_methods = [] case vis_type when 'module_function' then args = parse_symbol_arg container.set_visibility_for args, :private, false container.methods_matching args do |m| s_m = m.dup record_location s_m s_m.singleton = true new_methods << s_m end when 'public_class_method', 'private_class_method' then args = parse_symbol_arg container.methods_matching args, true do |m| if m.parent != container then m = m.dup record_location m new_methods << m end m.visibility = vis end else args = parse_symbol_arg container.set_visibility_for args, vis, singleton end new_methods.each do |method| case method when RDoc::AnyMethod then container.add_method method when RDoc::Attr then container.add_attribute method end method.visibility = vis end end ## # Prints +message+ to +$stderr+ unless we're being quiet def warn message @options.warn make_message message end end