Improve set handling

ChangeLog

@@ -1,13 +1,26 @@
 UPCOMING
 
+	* Breaking changes to character set and property handling:
+	* Changed parsing of sets (a.k.a. character classes or "bracket expressions")
+	  - see PR #55 / issue #47 for details
+	  - sets are now parsed to expression trees like other nestable expressions
+	  - #scan now emits the same tokens as outside sets (no longer :set, :member)
+	  - new Range and Intersection classes represent corresponding syntax features
+	  - a new PosixClass expression class represents e.g. [[:ascii:]]
+	    - PosixClass instances behave like Property ones, e.g. support #negative?
+	    - #scan emits :(non)posixclass, :<type> instead of :set, :char_(non)<type>
+	* Changed Expression emissions for some escape sequences
+	  - EscapeSequence::Codepoint, CodepointList, Hex and Octal are now all used
+	  - they already existed, but were all parsed as EscapeSequence::Literal
+	  - e.g. \x97 is now EscapeSequence::Hex instead of EscapeSequence::Literal 
 	* Changed naming of many property tokens (emitted for \p{...})
 	  - if you work with these tokens, see PR #56 for details
-	* Added support for all previously missing properties (about 250)
-	* Added Expression::UnicodeProperty#shortcut (e.g. returns 'm' for '\p{mark}')
+	  - e.g. :punct_dash is now :dash_punctuation
 	* Fixed ruby version mapping of some properties
 	* Fixed scanning of some property spellings, e.g. with dashes
 	* Fixed some incorrect property alias normalizations
-	* Improved the speed of the properties machine
+	* Added support for all previously missing properties (about 250)
+	* Added Expression::UnicodeProperty#shortcut (e.g. returns 'm' for '\p{mark}')
 	* Bumped version to XXX
 
 Sun Apr 29 2018 Janosch Müller <janosch84@gmail.com>

lib/regexp_parser/expression.rb

@@ -3,7 +3,7 @@ module Regexp::Expression
   class Base
     attr_accessor :type, :token
     attr_accessor :text, :ts
-    attr_accessor :level, :set_level, :conditional_level
+    attr_accessor :level, :set_level, :conditional_level, :nesting_level
 
     attr_accessor :quantifier
     attr_accessor :options
@@ -16,6 +16,7 @@ def initialize(token, options = {})
       self.level             = token.level
       self.set_level         = token.set_level
       self.conditional_level = token.conditional_level
+      self.nesting_level     = 0
       self.quantifier        = nil
       self.options           = options
     end
@@ -169,6 +170,7 @@ def self.parsed(exp)
 require 'regexp_parser/expression/quantifier'
 require 'regexp_parser/expression/subexpression'
 require 'regexp_parser/expression/sequence'
+require 'regexp_parser/expression/sequence_operation'
 
 require 'regexp_parser/expression/classes/alternation'
 require 'regexp_parser/expression/classes/anchor'
@@ -179,7 +181,10 @@ def self.parsed(exp)
 require 'regexp_parser/expression/classes/group'
 require 'regexp_parser/expression/classes/keep'
 require 'regexp_parser/expression/classes/literal'
+require 'regexp_parser/expression/classes/posix_class'
 require 'regexp_parser/expression/classes/property'
 require 'regexp_parser/expression/classes/root'
 require 'regexp_parser/expression/classes/set'
+require 'regexp_parser/expression/classes/set/intersection'
+require 'regexp_parser/expression/classes/set/range'
 require 'regexp_parser/expression/classes/type'

lib/regexp_parser/expression/classes/alternation.rb

@@ -1,33 +1,10 @@
 module Regexp::Expression
-
-  # This is not a subexpression really, but considering it one simplifies
-  # the API when it comes to handling the alternatives.
-  class Alternation < Regexp::Expression::Subexpression
-    alias :alternatives :expressions
-
-    def starts_at
-      expressions.first.starts_at
-    end
-    alias :ts :starts_at
-
-    def <<(exp)
-      expressions.last << exp
-    end
-
-    def alternative(exp = nil)
-      expressions << (exp ? exp : Alternative.new(level, set_level, conditional_level))
-    end
-
-    def quantify(token, text, min = nil, max = nil, mode = :greedy)
-      alternatives.last.last.quantify(token, text, min, max, mode)
-    end
-
-    def to_s(format = :full)
-      alternatives.map{|e| e.to_s(format)}.join('|')
-    end
-  end
-
   # A sequence of expressions, used by Alternation as one of its alternative.
   class Alternative < Regexp::Expression::Sequence; end
 
+  class Alternation < Regexp::Expression::SequenceOperation
+    OPERAND = Alternative
+
+    alias :alternatives :expressions
+  end
 end

lib/regexp_parser/expression/classes/escape.rb

@@ -11,13 +11,13 @@ class Bell          < EscapeSequence::Base; end
     class FormFeed      < EscapeSequence::Base; end
     class Newline       < EscapeSequence::Base; end
     class Return        < EscapeSequence::Base; end
-    class Space         < EscapeSequence::Base; end
     class Tab           < EscapeSequence::Base; end
     class VerticalTab   < EscapeSequence::Base; end
 
+    class Codepoint     < EscapeSequence::Base; end
+    class CodepointList < EscapeSequence::Base; end
     class Octal         < EscapeSequence::Base; end
     class Hex           < EscapeSequence::Base; end
-    class HexWide       < EscapeSequence::Base; end
 
     class Control       < EscapeSequence::Base; end
     class Meta          < EscapeSequence::Base; end

lib/regexp_parser/expression/classes/posix_class.rb

@@ -0,0 +1,11 @@
+module Regexp::Expression
+  class PosixClass < Regexp::Expression::Base
+    def negative?
+      type == :nonposixclass
+    end
+
+    def name
+      token.to_s
+    end
+  end
+end

lib/regexp_parser/expression/classes/set.rb

@@ -1,110 +1,73 @@
 module Regexp::Expression
 
-  class CharacterSet < Regexp::Expression::Base
-    attr_accessor :members
+  class CharacterSet < Regexp::Expression::Subexpression
+    attr_accessor :closed, :negative
+
+    alias :negative? :negative
+    alias :negated?  :negative
+    alias :closed?   :closed
 
     def initialize(token, options = {})
-      @members  = []
-      @negative = false
-      @closed   = false
+      self.negative = false
+      self.closed   = false
       super
     end
 
-    # Override base method to clone set members as well.
-    def clone
-      copy = super
-      copy.members = @members.map {|m| m.clone }
-      copy
-    end
-
-    def <<(member)
-      if @members.last.is_a?(CharacterSubSet) and not @members.last.closed?
-        @members.last << member
-      else
-        @members << member
-      end
-    end
-
-    def include?(member, directly = false)
-      @members.each do |m|
-        if m.is_a?(CharacterSubSet) and not directly
-          return true if m.include?(member)
-        else
-          return true if member == m.to_s
-        end
-      end; false
-    end
-
-    def each(&block)
-      @members.each {|m| yield m}
+    def negate
+      self.negative = true
     end
 
-    def each_with_index(&block)
-      @members.each_with_index {|m, i| yield m, i}
+    def close
+      self.closed = true
     end
 
-    def length
-      @members.length
+    def to_s(format = :full)
+      "#{text}#{'^' if negated?}#{expressions.join}]#{quantifier_affix(format)}"
     end
 
-    def negate
-      if @members.last.is_a?(CharacterSubSet)
-        @members.last.negate
-      else
-        @negative = true
+    # TODO: these made more sense with string members. remove/replace in v1.0.0?
+    module LegacyCompatibilityMethods
+      def members
+        expressions.map { |exp| exp.is_a?(CharacterSet) ? exp : exp.to_s }
       end
-    end
 
-    def negative?
-      @negative
-    end
-    alias :negated? :negative?
-
-    def close
-      if @members.last.is_a?(CharacterSubSet) and not @members.last.closed?
-        @members.last.close
-      else
-        @closed = true
+      # Returns an array of the members with any shorthand members like \d and \W
+      # expanded to either traditional form or unicode properties.
+      def expand_members(use_properties = false)
+        members.map do |member|
+          case member
+          when "\\d"
+            use_properties ? '\p{Digit}'  : '0-9'
+          when "\\D"
+            use_properties ? '\P{Digit}'  : '^0-9'
+          when "\\w"
+            use_properties ? '\p{Word}'   : 'A-Za-z0-9_'
+          when "\\W"
+            use_properties ? '\P{Word}'   : '^A-Za-z0-9_'
+          when "\\s"
+            use_properties ? '\p{Space}'  : ' \t\f\v\n\r'
+          when "\\S"
+            use_properties ? '\P{Space}'  : '^ \t\f\v\n\r'
+          when "\\h"
+            use_properties ? '\p{Xdigit}' : '0-9A-Fa-f'
+          when "\\H"
+            use_properties ? '\P{Xdigit}' : '^0-9A-Fa-f'
+          else
+            member
+          end
+        end
       end
-    end
-
-    def closed?
-      @closed
-    end
 
-    # Returns an array of the members with any shorthand members like \d and \W
-    # expanded to either traditional form or unicode properties.
-    def expand_members(use_properties = false)
-      @members.map do |member|
-        case member
-        when "\\d"
-          use_properties ? '\p{Digit}'  : '0-9'
-        when "\\D"
-          use_properties ? '\P{Digit}'  : '^0-9'
-        when "\\w"
-          use_properties ? '\p{Word}'   : 'A-Za-z0-9_'
-        when "\\W"
-          use_properties ? '\P{Word}'   : '^A-Za-z0-9_'
-        when "\\s"
-          use_properties ? '\p{Space}'  : ' \t\f\v\n\r'
-        when "\\S"
-          use_properties ? '\P{Space}'  : '^ \t\f\v\n\r'
-        when "\\h"
-          use_properties ? '\p{Xdigit}' : '0-9A-Fa-f'
-        when "\\H"
-          use_properties ? '\P{Xdigit}' : '^0-9A-Fa-f'
-        else
-          member
+      def include?(member, directly = false)
+        members.any? do |m|
+          if m.is_a?(CharacterSet)
+            !directly && m.include?(member)
+          else
+            m == member
+          end
         end
       end
     end
-
-    def to_s(format = :full)
-      "#{text}#{'^' if negative?}#{members.join}]#{quantifier_affix(format)}"
-    end
+    include LegacyCompatibilityMethods
   end
-
-  class CharacterSubSet < CharacterSet
-  end
-
 end # module Regexp::Expression

lib/regexp_parser/expression/classes/set/intersection.rb

@@ -0,0 +1,9 @@
+module Regexp::Expression
+  class CharacterSet < Regexp::Expression::Subexpression
+    class IntersectedSequence < Regexp::Expression::Sequence; end
+
+    class Intersection < Regexp::Expression::SequenceOperation
+      OPERAND = IntersectedSequence
+    end
+  end
+end

lib/regexp_parser/expression/classes/set/range.rb

@@ -0,0 +1,23 @@
+module Regexp::Expression
+  class CharacterSet < Regexp::Expression::Subexpression
+    class Range < Regexp::Expression::Subexpression
+      def starts_at
+        expressions.first.starts_at
+      end
+      alias :ts :starts_at
+
+      def <<(exp)
+        complete? && raise("Can't add more than 2 expressions to a Range")
+        super
+      end
+
+      def complete?
+        count == 2
+      end
+
+      def to_s(_format = :full)
+        expressions.join(text)
+      end
+    end
+  end
+end

lib/regexp_parser/expression/methods/strfregexp.rb

@@ -40,14 +40,16 @@ def strfregexp(format = '%a', indent_offset = 0, index = nil)
 
       part = {}
 
+      print_level = nesting_level > 0 ? nesting_level - 1 : nil
+
       # Order is important! Fields that use other fields in their
       # definition must appear before the fields they use.
       part_keys = %w{a m b o i l x s e S y k c q Q z Z t ~t T >}
       part.keys.each {|k| part[k] = "<?#{k}?>"}
 
-      part['>'] = level ? ('  ' * (level + indent_offset)) : ''
+      part['>'] = print_level ? ('  ' * (print_level + indent_offset)) : ''
 
-      part['l'] = level ? "#{'%d' % level}" : 'root'
+      part['l'] = print_level ? "#{'%d' % print_level}" : 'root'
       part['x'] = "#{'%d' % index}" if have_index
 
       part['s'] = starts_at
@@ -101,9 +103,9 @@ class Subexpression < Regexp::Expression::Base
     def strfregexp_tree(format = '%a', include_self = true, separator = "\n")
       output = include_self ? [self.strfregexp(format)] : []
 
-      output += map {|exp, index|
+      output += map do |exp, index|
         exp.strfregexp(format, (include_self ? 1 : 0), index)
-      }
+      end
 
       output.join(separator)
     end

lib/regexp_parser/expression/methods/tests.rb

@@ -7,22 +7,12 @@ class Base
     #   # is it a :group expression
     #   exp.type? :group
     #
-    #   # is it a :set, :subset, or :meta
-    #   exp.type? [:set, :subset, :meta]
+    #   # is it a :set, or :meta
+    #   exp.type? [:set, :meta]
     #
     def type?(test_type)
-      case test_type
-      when Array
-        if test_type.include?(:*)
-          return (test_type.include?(type) or test_type.include?(:*))
-        else
-          return test_type.include?(type)
-        end
-      when Symbol
-        return (type == test_type or test_type == :*)
-      else
-        raise "Array or Symbol expected, #{test_type.class.name} given"
-      end
+      test_types = Array(test_type).map(&:to_sym)
+      test_types.include?(:*) || test_types.include?(type)
     end
 
     # Test if this expression has the given test_token, and optionally a given