Refactor RipperStateLex

I'm using Stackprof for a performance check of Rails documents
generation.

I checked methods below:

- RDoc::Parser::RipperStateLex#get_squashed_tk
- RDoc::Parser::RipperStateLex#initialize
- RDoc::Parser::RipperStateLex::InnerStateLex#on_default

Before this commit, these use processing time about 19.6%.

aycabta@x270:~/rdoc$ stackprof stackprof-rails-before.dump --limit 1
==================================
  Mode: cpu(1000)
  Samples: 66894 (0.83% miss rate)
  GC: 15700 (23.47%)
==================================
TOTAL   (pct) SAMPLES   (pct) FRAME
15700 (23.5%)   15700 (23.5%) (garbage collection)
aycabta@x270:~/rdoc$ stackprof stackprof-rails-before.dump | \
        grep 'RipperStateLex'
 1727  (2.6%)    1472  (2.2%) (snip)RipperStateLex#get_squashed_tk
 9352 (14.0%)    1281  (1.9%) (snip)RipperStateLex#initialize
 2018  (3.0%)     475  (0.7%) (snip)RipperStateLex::InnerStateLex#on_default

After this commit, processing time is reduced to 15.0%.

aycabta@x270:~/rdoc$ stackprof stackprof-rails-after.dump --limit 1
==================================
  Mode: cpu(1000)
  Samples: 61020 (1.01% miss rate)
  GC: 13824 (22.65%)
==================================
TOTAL   (pct) SAMPLES   (pct) FRAME
13824 (22.7%)   13824 (22.7%) (garbage collection)
aycabta@x270:~/rdoc$ stackprof stackprof-rails-after.dump --limit 0 | \
        grep '#get_squashed_tk\|RipperStateLex#initialize\|#on_default'
  735  (1.2%)     533  (0.9%) (snip)RipperStateLex::InnerStateLex#on_default
  760  (1.2%)     517  (0.8%) (snip)RipperStateLex#get_squashed_tk
 7663 (12.6%)       1  (0.0%) (snip)RipperStateLex#initialize

lib/rdoc/parser/ripper_state_lex.rb

@@ -50,7 +50,7 @@ def on_nl(tok, data)
         @continue = false
         @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
       end
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_ignored_nl(tok, data)
@@ -61,7 +61,7 @@ def on_ignored_nl(tok, data)
         @continue = false
         @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
       end
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_op(tok, data)
@@ -103,7 +103,7 @@ def on_op(tok, data)
           @lex_state = EXPR_BEG
         end
       end
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_kw(tok, data)
@@ -132,54 +132,54 @@ def on_kw(tok, data)
           @lex_state = EXPR_END
         end
       end
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_tstring_beg(tok, data)
       @lex_state = EXPR_BEG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_tstring_end(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_CHAR(tok, data)
       @lex_state = EXPR_END
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_period(tok, data)
       @lex_state = EXPR_DOT
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_int(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_float(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rational(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_imaginary(tok, data)
       @lex_state = EXPR_END | EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_symbeg(tok, data)
       @lex_state = EXPR_FNAME
       @continue = true
       @in_fname = true
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     private def on_variables(event, tok, data)
@@ -198,7 +198,7 @@ def on_symbeg(tok, data)
       else
         @lex_state = EXPR_CMDARG
       end
-      @callback.call(Token.new(lineno, column, event, tok, @lex_state))
+      data << Token.new(lineno, column, event, tok, @lex_state)
     end
 
     def on_ident(tok, data)
@@ -227,32 +227,32 @@ def on_backref(tok, data)
 
     def on_lparen(tok, data)
       @lex_state = EXPR_LABEL | EXPR_BEG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rparen(tok, data)
       @lex_state = EXPR_ENDFN
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_lbrace(tok, data)
       @lex_state = EXPR_LABEL | EXPR_BEG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rbrace(tok, data)
       @lex_state = EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_lbracket(tok, data)
       @lex_state = EXPR_LABEL | EXPR_BEG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_rbracket(tok, data)
       @lex_state = EXPR_ENDARG
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_const(tok, data)
@@ -264,41 +264,43 @@ def on_const(tok, data)
       else
         @lex_state = EXPR_CMDARG
       end
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_sp(tok, data)
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_comma(tok, data)
       @lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_comment(tok, data)
       @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
     end
 
     def on_ignored_sp(tok, data)
       @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
+    end
+
+    def on_heredoc_beg(tok, data)
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
+      @lex_state = EXPR_END
+      data
     end
 
     def on_heredoc_end(tok, data)
-      @callback.call(Token.new(lineno, column, __method__, tok, @lex_state))
+      data << Token.new(lineno, column, __method__, tok, @lex_state)
       @lex_state = EXPR_BEG
+      data
     end
 
     def on_default(event, tok, data)
       reset
-      @callback.call(Token.new(lineno, column, event, tok, @lex_state))
-    end
-
-    def each(&block)
-      @callback = block
-      parse
+      data << Token.new(lineno, column, event, tok, @lex_state)
     end
   end unless RIPPER_HAS_LEX_STATE
 
@@ -308,21 +310,17 @@ def initialize(code)
     end
 
     def on_default(event, tok, data)
-      @callback.call(Token.new(lineno, column, event, tok, state))
-    end
-
-    def each(&block)
-      @callback = block
-      parse
+      data << Token.new(lineno, column, event, tok, state)
     end
   end if RIPPER_HAS_LEX_STATE
 
   def get_squashed_tk
     if @buf.empty?
-      tk = @inner_lex_enumerator.next
+      tk = @tokens.shift
     else
       tk = @buf.shift
     end
+    return nil if tk.nil?
     case tk[:kind]
     when :on_symbeg then
       tk = get_symbol_tk(tk)
@@ -472,7 +470,7 @@ def get_squashed_tk
     string = ''
     start_tk = nil
     prev_tk = nil
-    until heredoc_end?(heredoc_name, indent, tk = @inner_lex_enumerator.next) do
+    until heredoc_end?(heredoc_name, indent, tk = @tokens.shift) do
       start_tk = tk unless start_tk
       if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no]
         string = string + (' ' * tk[:char_no])
@@ -566,11 +564,7 @@ def initialize(code)
     @buf = []
     @heredoc_queue = []
     @inner_lex = InnerStateLex.new(code)
-    @inner_lex_enumerator = Enumerator.new do |y|
-      @inner_lex.each do |tk|
-        y << tk
-      end
-    end
+    @tokens = @inner_lex.parse([])
   end
 
   def self.parse(code)