Import RDoc 3

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@30249 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

lib/rdoc/token_stream.rb

@@ -0,0 +1,50 @@
+##
+# A TokenStream is a list of tokens, gathered during the parse of some entity
+# (say a method). Entities populate these streams by being registered with the
+# lexer. Any class can collect tokens by including TokenStream. From the
+# outside, you use such an object by calling the start_collecting_tokens
+# method, followed by calls to add_token and pop_token.
+
+module RDoc::TokenStream
+
+  ##
+  # Adds +tokens+ to the collected tokens
+
+  def add_tokens(*tokens)
+    tokens.flatten.each { |token| @token_stream << token }
+  end
+
+  alias add_token add_tokens
+
+  ##
+  # Starts collecting tokens
+
+  def collect_tokens
+    @token_stream = []
+  end
+
+  alias start_collecting_tokens collect_tokens
+
+  ##
+  # Remove the last token from the collected tokens
+
+  def pop_token
+    @token_stream.pop
+  end
+
+  ##
+  # Current token stream
+
+  def token_stream
+    @token_stream
+  end
+
+  ##
+  # Returns a string representation of the token stream
+
+  def tokens_to_s
+    token_stream.map { |token| token.text }.join ''
+  end
+
+end
+