U:RDoc::AnyMethod[iI"lex:ETI"Ripper::lex;TT:publico:RDoc::Markup::Document:@parts[o:RDoc::Markup::Paragraph;	[I"BTokenizes the Ruby program and returns an array of an array, ;TI"which is formatted like ;TI"9<code>[[lineno, column], type, token, state]</code>.;To:RDoc::Markup::BlankLineo:RDoc::Markup::Verbatim;	[I"require 'ripper'
;TI"require 'pp'
;TI"
;TI"'pp Ripper.lex("def m(a) nil end")
;TI"3#=> [[[1,  0], :on_kw,     "def", FNAME    ],
;TI"3     [[1,  3], :on_sp,     " ",   FNAME    ],
;TI"3     [[1,  4], :on_ident,  "m",   ENDFN    ],
;TI"3     [[1,  5], :on_lparen, "(",   BEG|LABEL],
;TI"3     [[1,  6], :on_ident,  "a",   ARG      ],
;TI"3     [[1,  7], :on_rparen, ")",   ENDFN    ],
;TI"3     [[1,  8], :on_sp,     " ",   BEG      ],
;TI"3     [[1,  9], :on_kw,     "nil", END      ],
;TI"3     [[1, 12], :on_sp,     " ",   END      ],
;TI"2     [[1, 13], :on_kw,     "end", END      ]];T:@format0:
@fileI"#ext/ripper/lib/ripper/lexer.rb;T:0@omit_headings_from_table_of_contents_below000[I"&(src, filename = '-', lineno = 1);T@!FI"Ripper;TcRDoc::NormalClass00