Added grammar files

bin/setup

@@ -5,4 +5,4 @@ set -vx
 
 bundle install
 
-# Do any other automated setup that you need to do here
+# Do any other automated setup that you need to do here

lib/craftbook/nbt/snbt/lexer.rb

@@ -0,0 +1,161 @@
+#--
+# DO NOT MODIFY!!!!
+# This file is automatically generated by rex 1.0.7
+# from lexical definition file "lib/craftbook/nbt/snbt/snbt.rex".
+#++
+
+require 'racc/parser'
+module CraftBook
+module NBT
+class Tokenizer < Racc::Parser
+      require 'strscan'
+
+      class ScanError < StandardError ; end
+
+      attr_reader   :lineno
+      attr_reader   :filename
+      attr_accessor :state
+
+      def scan_setup(str)
+        @ss = StringScanner.new(str)
+        @lineno =  1
+        @state  = nil
+      end
+
+      def action
+        yield
+      end
+
+      def scan_str(str)
+        scan_setup(str)
+        do_parse
+      end
+      alias :scan :scan_str
+
+      def load_file( filename )
+        @filename = filename
+        File.open(filename, "r") do |f|
+          scan_setup(f.read)
+        end
+      end
+
+      def scan_file( filename )
+        load_file(filename)
+        do_parse
+      end
+
+
+        def next_token
+          return if @ss.eos?
+
+          # skips empty actions
+          until token = _next_token or @ss.eos?; end
+          token
+        end
+
+        def _next_token
+          text = @ss.peek(1)
+          @lineno  +=  1  if text == "\n"
+          token = case @state
+            when nil
+          case
+                  when (text = @ss.scan(/\{[\s]*/))
+                     action { [:COMPOUND_BEGIN] }
+
+                  when (text = @ss.scan(/[\s]*\}/))
+                     action { [:COMPOUND_END] }
+
+                  when (text = @ss.scan(/".+?"(?=:)/))
+                     action { [:IDENTIFIER, text.gsub!(/\A"|"\Z/, '') ] }
+
+                  when (text = @ss.scan(/'.+?'(?=:)/))
+                     action { [:IDENTIFIER, text.gsub!(/\A'|'\Z/, '') ] }
+
+                  when (text = @ss.scan(/[A-Za-z0-9_-]+?(?=:)/))
+                     action { [:IDENTIFIER, text] }
+
+                  when (text = @ss.scan(/".*?"/))
+                     action { [:STRING, text.gsub!(/\A"|"\Z/, '') ] }
+
+                  when (text = @ss.scan(/'.*?'/))
+                     action { [:STRING, text.gsub!(/\A'|'\Z/, '') ] }
+
+                  when (text = @ss.scan(/[\s]*:[\s]*/))
+                     action { [:SEPARATOR, text] }
+
+                  when (text = @ss.scan(/[\s]*,[\s]*/))
+                     action { [:COMMA, text] }
+
+                  when (text = @ss.scan(/\[B;[\s]*?/))
+                     action { [:BYTE_ARRAY, text] }
+
+                  when (text = @ss.scan(/\[I;[\s]*?/))
+                     action { [:INT_ARRAY, text]  }
+
+                  when (text = @ss.scan(/\[L;[\s]*?/))
+                     action { [:LONG_ARRAY, text] }
+
+                  when (text = @ss.scan(/\[[\s]*?/))
+                     action { [:LIST_ARRAY, text] }
+
+                  when (text = @ss.scan(/[\s]*\]/))
+                     action { [:END_ARRAY, text]  }
+
+                  when (text = @ss.scan(/-?[0-9]*\.[0-9]+[Ff]/))
+                     action { [:FLOAT,  text.chop.to_f         ] }
+
+                  when (text = @ss.scan(/-?[0-9]*\.[0-9]+[Dd]?/))
+                     action { [:DOUBLE, text.tr('Dd', '').to_f ] }
+
+                  when (text = @ss.scan(/-?([0-9]+)[Bb]/))
+                     action { [:BYTE,   text.chop.to_i         ] }
+
+                  when (text = @ss.scan(/-?([0-9]+)[Ss]/))
+                     action { [:SHORT,  text.chop.to_i         ] }
+
+                  when (text = @ss.scan(/-?([0-9]+)[Ll]/))
+                     action { [:LONG,   text.chop.to_i         ] }
+
+                  when (text = @ss.scan(/-?([0-9]+)/))
+                     action { [:INT,    text.to_i              ] }
+
+                  when (text = @ss.scan(/[\s]+/))
+                     action { [:WHITESPACE, text] }
+
+                  when (text = @ss.scan(/[\S]+/))
+                     action { [:STRING, text] }
+
+                  when (text = @ss.scan(/./))
+                     action { [:CHAR, text] }
+
+
+          else
+            text = @ss.string[@ss.pos .. -1]
+            raise  ScanError, "can not match: '" + text + "'"
+          end  # if
+
+        else
+          raise  ScanError, "undefined state: '" + state.to_s + "'"
+        end  # case state
+          token
+        end  # def _next_token
+
+  Token = Struct.new(:type, :value)
+  def tokenize(code)
+    scan_setup(code)
+    if block_given?
+      while token = next_token
+        yield Token.new(*token)
+      end
+      return self
+    end
+    tokens = []
+    while token = next_token
+      tokens << Token.new(*token)
+    end
+    tokens
+  end
+end # class
+
+end
+end

lib/craftbook/nbt/snbt/snbt.rex

@@ -0,0 +1,74 @@
+module CraftBook
+module NBT
+class Tokenizer
+
+macro
+  nl          \n|\r\n|\r|\f
+  w           [\s]*
+  num         -?([0-9]+|[0-9]*\.[0-9]+)
+  l_bracket   \[
+  r_bracket   \]
+  l_brace     \{
+  r_brace     \}
+
+  integer     -?([0-9]+)
+  decimal     -?[0-9]*\.[0-9]+
+  escape    {unicode}|\\[^\n\r\f0-9A-Fa-f]
+  id [A-Za-z0-9-_]
+rule
+    \{{w}                { [:COMPOUND_BEGIN] }
+    {w}\}                { [:COMPOUND_END] }
+
+    ".+?"(?=:)      { [:IDENTIFIER, text.gsub!(/\A"|"\Z/, '') ] }
+    '.+?'(?=:)      { [:IDENTIFIER, text.gsub!(/\A'|'\Z/, '') ] }
+    [A-Za-z0-9_-]+?(?=:)        { [:IDENTIFIER, text] }
+    ".*?"        { [:STRING, text.gsub!(/\A"|"\Z/, '') ] }
+    '.*?'        { [:STRING, text.gsub!(/\A'|'\Z/, '') ] }
+
+    # Control Characters
+    {w}:{w}                { [:SEPARATOR, text] }
+    {w},{w}                { [:COMMA, text] }
+
+    # Collection Types
+
+    {l_bracket}B;{w}?    { [:BYTE_ARRAY, text] }
+    {l_bracket}I;{w}?    { [:INT_ARRAY, text]  }
+    {l_bracket}L;{w}?    { [:LONG_ARRAY, text] }
+    \[{w}?               { [:LIST_ARRAY, text] }
+    {w}\]                { [:END_ARRAY, text]  }
+
+    # Numeric Types
+    {decimal}[Ff]    { [:FLOAT,  text.chop.to_f         ] }
+    {decimal}[Dd]?   { [:DOUBLE, text.tr('Dd', '').to_f ] }
+    {integer}[Bb]    { [:BYTE,   text.chop.to_i         ] }
+    {integer}[Ss]    { [:SHORT,  text.chop.to_i         ] }
+    {integer}[Ll]    { [:LONG,   text.chop.to_i         ] }
+    {integer}        { [:INT,    text.to_i              ] }
+
+    [\s]+            { [:WHITESPACE, text] }
+    [\S]+            { [:STRING, text] }
+    .                { [:CHAR, text] }
+
+inner
+
+  Token = Struct.new(:type, :value)
+
+  def tokenize(code)
+    scan_setup(code)
+    if block_given?
+      while token = next_token
+        yield Token.new(*token)
+      end
+      return self
+    end
+    tokens = []
+    while token = next_token
+      tokens << Token.new(*token)
+    end
+    tokens
+  end
+end
+
+end
+end
+end