Merge pull request #73 from CityOfZion/sequence-slicing

Sequence slicing

boa3/analyser/typeanalyser.py

@@ -360,7 +360,7 @@ def validate_get_or_set(self, subscript: ast.Subscript, index_node: ast.Index) -
         if isinstance(index, ast.Name):
             index = self.get_symbol(index.id)
         if not isinstance(index, tuple):
-            index = (index, )
+            index = (index,)
 
         # if it is a type hint, returns the outer type
         if isinstance(value, IType) and all(isinstance(i, IType) for i in index):
@@ -434,18 +434,7 @@ def validate_slice(self, subscript: ast.Subscript, slice_node: ast.Slice) -> ITy
         upper = upper if upper is not Type.none else symbol_type.valid_key
 
         # TODO: remove when slices of other sequence types are implemented
-        if symbol_type is not Type.str:
-            expected: IType = symbol_type.valid_key
-            actual: Tuple[IType, ...] = (lower, upper) if step is Type.none else (lower, upper, step)
-            self._log_error(
-                CompilerError.MismatchedTypes(
-                    subscript.lineno, subscript.col_offset,
-                    expected_type_id=expected.identifier,
-                    actual_type_id=[value.identifier for value in actual]
-                )
-            )
-        elif (
-            not symbol_type.is_valid_key(lower)
+        if (not symbol_type.is_valid_key(lower)
             or not symbol_type.is_valid_key(upper)
             or (step is not Type.none and not symbol_type.is_valid_key(step))
         ):

boa3/compiler/codegenerator.py

@@ -11,6 +11,7 @@
 from boa3.model.operation.binaryop import BinaryOp
 from boa3.model.operation.operation import IOperation
 from boa3.model.symbol import ISymbol
+from boa3.model.type.collection.sequence.sequencetype import SequenceType
 from boa3.model.type.type import IType, Type
 from boa3.model.variable import Variable
 from boa3.neo.vm.VMCode import VMCode
@@ -76,6 +77,20 @@ def last_code(self) -> Optional[VMCode]:
         else:
             return None
 
+    @property
+    def last_code_start_address(self) -> int:
+        """
+        Gets the first address from last code in the bytecode
+
+        :return: the last code. If the bytecode is empty, returns None
+        :rtype: VMCode or None
+        """
+        instance = VMCodeMapping.instance()
+        if len(instance.codes) > 0:
+            return instance.get_start_address(instance.codes[-1])
+        else:
+            return 0
+
     @property
     def _args(self) -> List[str]:
         """
@@ -174,7 +189,7 @@ def convert_begin_while(self) -> int:
         """
         # it will be updated when the while ends
         self._insert_jump(OpcodeInfo.JMP, 0)
-        return VMCodeMapping.instance().get_start_address(self.last_code)
+        return self.last_code_start_address
 
     def convert_end_while(self, start_address: int, test_address: int):
         """
@@ -250,7 +265,7 @@ def convert_begin_else(self, start_address: int) -> int:
         # updates the begin jmp with the target address
         self._update_jump(start_address, VMCodeMapping.instance().bytecode_size)
 
-        return VMCodeMapping.instance().get_start_address(self.last_code)
+        return self.last_code_start_address
 
     def convert_end_if(self, start_address: int):
         """
@@ -431,7 +446,7 @@ def _set_array_item(self, value_start_address: int):
         """
         Converts the end of setting af a value in an array
         """
-        value_code = VMCodeMapping.instance().get_start_address(self.last_code)
+        value_code = self.last_code_start_address
         index_type: IType = self._stack[-2]  # top: index
         if index_type is Type.int:
             self.fix_negative_index(value_start_address)
@@ -482,29 +497,98 @@ def convert_get_substring(self):
         self._stack.append(Type.bytes)  # substr returns a buffer instead of a bytestring
         self.convert_cast(Type.str)
 
+    def convert_get_array_slice(self, array: SequenceType):
+        """
+        Converts the end of get a substring
+        """
+        self.convert_new_empty_array(0, array)      # slice = []
+        self.duplicate_stack_item(3)                # index = slice_start
+
+        start_jump = self.convert_begin_while()  # while index < slice_end
+        self.duplicate_stack_top_item()             # if index >= slice_start
+        self.duplicate_stack_item(5)
+        self.convert_operation(BinaryOp.GtE)
+        is_valid_index = self.convert_begin_if()
+
+        self.duplicate_stack_item(2)                    # slice.append(array[index])
+        self.duplicate_stack_item(6)
+        self.duplicate_stack_item(3)
+        self.convert_get_item()
+        self.convert_builtin_method_call(Builtin.SequenceAppend)
+        self.convert_end_if(is_valid_index)
+
+        self.__insert1(OpcodeInfo.INC)              # index += 1
+
+        condition_address = VMCodeMapping.instance().bytecode_size
+        self.duplicate_stack_top_item()         # end while index < slice_end
+        self.duplicate_stack_item(4)
+        self.convert_operation(BinaryOp.Lt)
+        self.convert_end_while(start_jump, condition_address)
+
+        self.remove_stack_top_item()        # removes from the stack the arguments and the index
+        self.swap_reverse_stack_items(4)    # doesn't use CLEAR opcode because this would delete
+        self.remove_stack_top_item()        # data from external scopes
+        self.remove_stack_top_item()
+        self.remove_stack_top_item()
+
     def convert_get_sub_array(self):
         """
         Converts the end of get a slice in the beginning of an array
         """
         # top: length, index, array
-        if len(self._stack) > 2 and self._stack[-3] is Type.str:
-            self.convert_get_substring()
+        if len(self._stack) > 2 and isinstance(self._stack[-3], SequenceType):
+            if self._stack[-3].stack_item in (StackItemType.ByteString,
+                                              StackItemType.Buffer):
+                self.duplicate_stack_item(2)
+                self.convert_operation(BinaryOp.Sub)
+                self.convert_get_substring()
+            else:
+                array = self._stack[-3]
+                self.duplicate_stack_item(3)        # if slice end is greater than the array size, fixes them
+                self.convert_builtin_method_call(Builtin.Len)
+
+                # TODO: change to convert_builtin_method_call(Builtin.Min) when min(a, b) is implemented
+                self.__insert1(OpcodeInfo.MIN)
+                self._stack.pop()
+                self.convert_get_array_slice(array)
 
     def convert_get_array_beginning(self):
         """
         Converts the end of get a slice in the beginning of an array
         """
-        self.__insert1(OpcodeInfo.LEFT)
-        self._stack.pop()  # length
-        self._stack.pop()  # original array
+        if len(self._stack) > 1 and isinstance(self._stack[-2], SequenceType):
+            if self._stack[-2].stack_item in (StackItemType.ByteString,
+                                              StackItemType.Buffer):
+                self.__insert1(OpcodeInfo.LEFT)
+                self._stack.pop()  # length
+                self._stack.pop()  # original array
+            else:
+                array = self._stack[-2]
+                self.convert_literal(0)
+                self.swap_reverse_stack_items(2)
+                self.convert_get_array_slice(array)
 
     def convert_get_array_ending(self):
         """
         Converts the end of get a slice in the ending of an array
         """
-        self.__insert1(OpcodeInfo.RIGHT)
-        self._stack.pop()  # length
-        self._stack.pop()  # original array
+        # top: start_slice, array_length, array
+        if len(self._stack) > 2 and isinstance(self._stack[-3], SequenceType):
+            if self._stack[-3].stack_item in (StackItemType.ByteString,
+                                              StackItemType.Buffer):
+                self.convert_operation(BinaryOp.Sub)
+                self.__insert1(OpcodeInfo.RIGHT)
+                self._stack.pop()  # length
+                self._stack.pop()  # original array
+            else:
+                array = self._stack[-3]
+                self.swap_reverse_stack_items(2)
+                self.convert_get_array_slice(array)
+
+    def convert_copy(self):
+        if self._stack[-1].stack_item is StackItemType.Array:
+            self.__insert1(OpcodeInfo.UNPACK)
+            self.__insert1(OpcodeInfo.PACK)    # creates a new array with the values
 
     def convert_load_symbol(self, symbol_id: str, params_addresses: List[int] = None):
         """
@@ -615,11 +699,15 @@ def convert_builtin_method_call(self, function: IBuiltinMethod, args_address: Li
             self.__insert1(op_info, data)
 
         if store_opcode is not None:
+            self._insert_jump(OpcodeInfo.JMP, 0)
+            jump = self.last_code_start_address
             self.__insert1(store_opcode, store_data)
+            self._update_jump(jump, VMCodeMapping.instance().bytecode_size)
 
         for arg in function.args:
             self._stack.pop()
-        self._stack.append(function.return_type)
+        if function.return_type is not None:
+            self._stack.append(function.return_type)
 
     def convert_method_call(self, function: Method):
         """
@@ -750,27 +838,29 @@ def _get_jump_data(self, op_info: OpcodeInformation, jump_to: int) -> bytes:
         return Integer(jump_to).to_byte_array(min_length=op_info.data_len, signed=True)
 
     def duplicate_stack_top_item(self):
-        self.duplicate_stack_item()
+        self.duplicate_stack_item(1)
 
-    def duplicate_stack_item(self, pos: int = 1):
+    def duplicate_stack_item(self, pos: int = 0):
         """
         Duplicates the item n back in the stack
 
         :param pos: index of the variable
         """
         # n = 1 -> duplicates stack top item
-        if pos > 0:
+        # n = 0 -> value varies in runtime
+        if pos >= 0:
             opcode: Opcode = Opcode.get_dup(pos)
-            if opcode is Opcode.PICK:
+            if opcode is Opcode.PICK and pos > 0:
                 self.convert_literal(pos - 1)
+                self._stack.pop()
             op_info = OpcodeInfo.get_info(opcode)
             self.__insert1(op_info)
             self._stack.append(self._stack[-pos])
 
     def remove_stack_top_item(self):
-        self.remove_stack_item()
+        self.remove_stack_item(1)
 
-    def remove_stack_item(self, pos: int = 1):
+    def remove_stack_item(self, pos: int = 0):
         """
         Removes the item n from the stack
 
@@ -781,17 +871,21 @@ def remove_stack_item(self, pos: int = 1):
             opcode: Opcode = Opcode.get_drop(pos)
             if opcode is Opcode.XDROP:
                 self.convert_literal(pos - 1)
+                self._stack.pop()
             op_info = OpcodeInfo.get_info(opcode)
             self.__insert1(op_info)
-            self._stack.append(self._stack[-pos])
+            if pos > 0:
+                self._stack.pop(-pos)
 
-    def swap_reverse_stack_items(self, no_items: int):
-        if no_items > 1:
+    def swap_reverse_stack_items(self, no_items: int = 0):
+        # n = 0 -> value varies in runtime
+        if 0 <= no_items != 1:
             opcode: Opcode = Opcode.get_reverse(no_items)
-            if opcode is Opcode.REVERSEN:
+            if opcode is Opcode.REVERSEN and no_items > 0:
                 self.convert_literal(no_items)
             op_info = OpcodeInfo.get_info(opcode)
             self.__insert1(op_info)
-            reverse = list(reversed(self._stack[-no_items:]))
-            self._stack = self._stack[:-no_items]
-            self._stack.extend(reverse)
+            if no_items > 0:
+                reverse = list(reversed(self._stack[-no_items:]))
+                self._stack = self._stack[:-no_items]
+                self._stack.extend(reverse)

boa3/compiler/codegeneratorvisitor.py

@@ -215,8 +215,6 @@ def visit_Subscript_Slice(self, subscript: ast.Subscript):
             self.visit_to_generate(subscript.slice.lower)
             # length of slice
             self.visit_to_generate(subscript.slice.upper)
-            self.visit_to_generate(subscript.slice.lower)
-            self.generator.convert_operation(BinaryOp.Sub)
             self.generator.convert_get_sub_array()
         # only one of them is omitted
         elif lower_omitted != upper_omitted:
@@ -226,12 +224,13 @@ def visit_Subscript_Slice(self, subscript: ast.Subscript):
                 self.generator.convert_get_array_beginning()
             # start position is omitted
             else:
-                self.visit_to_generate(subscript.value)
+                self.generator.duplicate_stack_top_item()
                 # length of slice
                 self.generator.convert_builtin_method_call(Builtin.Len)
                 self.visit_to_generate(subscript.slice.lower)
-                self.generator.convert_operation(BinaryOp.Sub)
                 self.generator.convert_get_array_ending()
+        else:
+            self.generator.convert_copy()
 
     def _convert_unary_operation(self, operand, op):
         self.visit_to_generate(operand)

boa3/model/builtin/classmethod/appendmethod.py

@@ -53,8 +53,7 @@ def opcode(self) -> List[Tuple[Opcode, bytes]]:
             (Opcode.JMPIFNOT, Integer(8).to_byte_array(min_length=1)),  # when it's bytearray, concatenates the value
             (Opcode.CAT, b''),
             (Opcode.JMP, Integer(5).to_byte_array(min_length=1)),
-            (Opcode.APPEND, b''),
-            (Opcode.JMP, Integer(2).to_byte_array(min_length=1))
+            (Opcode.APPEND, b'')
         ]
 
     def push_self_first(self) -> bool:

boa3/neo/vm/opcode/Opcode.py

@@ -277,7 +277,7 @@ def get_dup(position: int):
             2: Opcode.OVER
         }
 
-        if position > 0:
+        if position >= 0:
             if position in duplicate_item:
                 return duplicate_item[position]
             else:
@@ -313,7 +313,7 @@ def get_reverse(no_stack_items: int):
             4: Opcode.REVERSE4
         }
 
-        if no_stack_items > 1:
+        if no_stack_items >= 0:
             if no_stack_items in reverse_stack:
                 return reverse_stack[no_stack_items]
             else:

boa3_test/example/list_test/ListSlicingEndOmitted.py

@@ -0,0 +1,3 @@
+def Main() -> list:
+    a = [0, 1, 2, 3, 4, 5]
+    return a[2:]   # expect [2, 3, 4, 5]

boa3_test/example/list_test/ListSlicing.py → boa3_test/example/list_test/ListSlicingLiteralValues.py

@@ -1,3 +1,3 @@
 def Main() -> list:
     a = [0, 1, 2, 3, 4, 5]
-    return a[2:3]   # expect 1
+    return a[2:3]   # expect [2]

boa3_test/example/list_test/ListSlicingOmitted.py

@@ -0,0 +1,3 @@
+def Main() -> list:
+    a = [0, 1, 2, 3, 4, 5]
+    return a[:]   # expect [0, 1, 2, 3, 4, 5]

boa3_test/example/list_test/ListSlicingOmittedWithStride.py

@@ -0,0 +1,3 @@
+def Main() -> list:
+    a = [0, 1, 2, 3, 4, 5]
+    return a[::2]   # expect [0, 2, 4]

boa3_test/example/list_test/ListSlicingStartOmitted.py

@@ -0,0 +1,3 @@
+def Main() -> list:
+    a = [0, 1, 2, 3, 4, 5]
+    return a[:3]   # expect [0, 1, 2]

boa3_test/example/list_test/ListSlicingVariableValues.py

@@ -0,0 +1,5 @@
+def Main() -> list:
+    a1 = 2
+    a2 = 3
+    a = [0, 1, 2, 3, 4, 5]
+    return a[a1:a2]   # expect [2,]

boa3_test/example/list_test/ListSlicingWithStride.py

@@ -0,0 +1,3 @@
+def Main() -> list:
+    a = [0, 1, 2, 3, 4, 5]
+    return a[2:5:2]   # expect [2, 4]

boa3_test/example/tuple_test/TupleSlicingEndOmitted.py

@@ -0,0 +1,3 @@
+def Main() -> tuple:
+    a = (0, 1, 2, 3, 4, 5)
+    return a[2:]   # expect (2, 3, 4, 5)

boa3_test/example/tuple_test/TupleSlicingLiteralValues.py

@@ -0,0 +1,3 @@
+def Main() -> tuple:
+    a = (0, 1, 2, 3, 4, 5)
+    return a[2:3]   # expect (2,)

boa3_test/example/tuple_test/TupleSlicingOmitted.py

@@ -0,0 +1,3 @@
+def Main() -> tuple:
+    a = (0, 1, 2, 3, 4, 5)
+    return a[:]   # expect (0, 1, 2, 3, 4, 5)

boa3_test/example/tuple_test/TupleSlicingOmittedWithStride.py

@@ -0,0 +1,3 @@
+def Main() -> tuple:
+    a = (0, 1, 2, 3, 4, 5)
+    return a[::2]   # expect (0, 2, 4)

boa3_test/example/tuple_test/TupleSlicing.py → boa3_test/example/tuple_test/TupleSlicingStartOmitted.py

@@ -1,3 +1,3 @@
 def Main() -> tuple:
     a = (0, 1, 2, 3, 4, 5)
-    return a[2:3]   # expect 1
+    return a[:3]   # (0, 1, 2)

boa3_test/example/tuple_test/TupleSlicingVariableValues.py

@@ -0,0 +1,5 @@
+def Main() -> tuple:
+    a1 = 2
+    a2 = 3
+    a = (0, 1, 2, 3, 4, 5)
+    return a[a1:a2]   # expect (2,)

boa3_test/example/tuple_test/TupleSlicingWithStride.py

@@ -0,0 +1,3 @@
+def Main() -> tuple:
+    a = (0, 1, 2, 3, 4, 5)
+    return a[2:5:2]   # expect (2, 4)