Port to Python 3 (#250)

* Initial py3 support

This is work in progress

* Use intelhex pkg, fix integer division

* Use intelhex module

* Update pyyaml to latest version

* Fix typo

* Update module reqs

* ecdsa module do not support bytearray, use bytes

* Fix binary write to file and hex conversion

* Use intelhex pkg

* Use bytes and not decode

* Convert hci.slip to Python 3

* Py3ize part of bl_dfu_sett

* Fix assertion to expect bytes

* Gitignore tests run artefacts

* Fix str used as bytes

* Py3ize Signing

There is a rather arbitrary decision make in this change that PEMs are
represented in bytes. It was selected in the interest of keeping
conversions to a minimum. Both bytes and str are good candidated for
representing PEM, which consists only of ascii chars.

* Upgrade pc_ble_driver_py to 0.12 for py3 support

* Py3ize zigbee/prod_config

* Remove unneeded NoneType import that breaks py3

Python's types.NoneType is no longer available, and it was not used
anyway.

* Add None guard before comparing as needed for py3

* Add some py3 support in TestDfuTransportSerial

The test will still not complete due to changes in interal apis.

* Adjust requirements for py3

* Evaluate map object of serial ports to list

* Py3ize dfu_transport_ble

* Port dfu_transport_ble to SDv5

The new pc_ble_driver_py module with py3 support is SDv5 based.

* Change type warning string to bytes

The alternative is to instead .decode('ascii') the result from get_sk,
but since everyting is ascii, bytes is a suitable (as intended) type
that can actually be safer in case PEM files cannot contain non-ascii
values.

* Remove map(ord, ...) on struct.pack in Ant

struct.pack in Python 3 returns bytes which gives ints when iterated
over.

* Update requirements.txt and add requirements-dev.txt

* Change readme to run package instead of __main__.py

This way, importing nordicsemi works without changes to PYTHONPATH.

* Remove Python 2 specific note in README

* Add tests artefacts to gitignore

* Remove vestigal debug print from test

* Use ABC instead of ABCMeta

* Fixup copyright year to a range

.gitignore

@@ -1,4 +1,5 @@
 *.pyc
+*.py.bak
 
 # Virtualenv
 venv/
@@ -7,3 +8,7 @@ venv/
 build/
 dist/
 nrfutil.egg-info/
+
+# Test artefacts
+/nordicsemi/dfu/tests/mypackage.zip
+/tests/test.zip

README.md

@@ -61,7 +61,7 @@ You will need to clone the present repository first to run or install nrfutil fr
 
 To install nrfutil from source the following prerequisites must be satisfied:
 
-* [Python 2.7 (2.7.10 or newer, not Python 3)](https://www.python.org/downloads/)
+* [Python 3.7](https://www.python.org/downloads/)
 * [pip](https://pip.pypa.io/en/stable/installing.html)
 * setuptools (upgrade to latest version): `pip install -U setuptools`
 
@@ -81,7 +81,7 @@ pip install -r requirements.txt
 
 You can run the program directly without installing it by executing:
 ```
-python nordicsemi/__main__.py
+python ./nordicsemi
 ```
 
 ### Installing from source
@@ -101,8 +101,6 @@ pyinstaller /full/path/to/nrfutil.spec
 
 **Note**: Some anti-virus programs will stop PyInstaller from executing correctly when it modifies the executable file.
 
-**Note**: PyInstaller on macOS may have issues with newer versions of Python 2.7.x., e.g. error message `Failed to execute script pyi_rth_pkgres` at runtime. Try rolling back to Python v2.7.10 if you experience such problems.
-
 ## Usage
 
 To get info on usage of nrfutil:

nordicsemi/__main__.py

@@ -485,7 +485,7 @@ def display(key_file, key, format, out_file):
     if key == "pk":
         kstr = signer.get_vk(format, dbg)
     elif key == "sk":
-        kstr = "\nWARNING: Security risk! Do not share the private key.\n\n"
+        kstr = b"\nWARNING: Security risk! Do not share the private key.\n\n"
         kstr = kstr + signer.get_sk(format, dbg)
 
     if not out_file:
@@ -806,7 +806,7 @@ def generate(zipfile,
         try:
             # This will parse any string starting with 0x as base 16.
             sd_req_list = sd_req.split(',')
-            sd_req_list = map(int_as_text_to_int, sd_req_list)
+            sd_req_list = list(map(int_as_text_to_int, sd_req_list))
         except ValueError:
             raise NordicSemiException("Could not parse value for --sd-req. "
                                       "Hex values should be prefixed with 0x.")
@@ -816,7 +816,7 @@ def generate(zipfile,
         try:
             # This will parse any string starting with 0x as base 16.
             sd_id_list = sd_id.split(',')
-            sd_id_list = map(int_as_text_to_int, sd_id_list)
+            sd_id_list = list(map(int_as_text_to_int, sd_id_list))
 
             # Copy all IDs from sd_id_list to sd_req_list, without duplicates.
             # This ensures that the softdevice update can be repeated in case
@@ -867,11 +867,11 @@ def generate(zipfile,
     if zigbee:
         inner_external_app = False
 
-    if zigbee_ota_min_hw_version > 0xFFFF:
+    if zigbee_ota_min_hw_version is not None and zigbee_ota_min_hw_version > 0xFFFF:
         click.echo('Error: zigbee-ota-min-hw-version exceeds 2-byte long integer.')
         return
 
-    if zigbee_ota_max_hw_version > 0xFFFF:
+    if zigbee_ota_max_hw_version is not None and zigbee_ota_max_hw_version > 0xFFFF:
         click.echo('Error: zigbee-ota-max-hw-version exceeds 2-byte long integer.')
         return
 
@@ -1092,7 +1092,7 @@ def serial(package, port, connect_delay, flow_control, packet_receipt_notificati
 
 
 def enumerate_ports():
-    descs   = BLEDriver.enum_serial_ports()
+    descs   = list(BLEDriver.enum_serial_ports())
     if len(descs) == 0:
         return None
     click.echo('Please select connectivity serial port:')
@@ -1364,7 +1364,7 @@ def thread(package, port, address, server_port, panid, channel, jlink_snr, flash
     mcast_dfu = False
 
     if address is None:
-        address = ipaddress.ip_address(u"ff03::1")
+        address = ipaddress.ip_address("ff03::1")
         click.echo("Address not specified. Using ff03::1 (all Thread nodes)")
     else:
         try:

nordicsemi/bluetooth/hci/slip.py

@@ -1,5 +1,4 @@
-#
-# Copyright (c) 2016 Nordic Semiconductor ASA
+# Copyright (c) 2016 - 2019 Nordic Semiconductor ASA
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -33,26 +32,25 @@
 # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
 
 import logging
 
 logger = logging.getLogger(__name__)
 
 
-class Slip(object):
-    def __init__(self):
-        self.SLIP_END = '\xc0'
-        self.SLIP_ESC = '\xdb'
-        self.SLIP_ESC_END = '\xdc'
-        self.SLIP_ESC_ESC = '\xdd'
+class Slip:
+    SLIP_END = 0xc0
+    SLIP_ESC = 0xdb
+    SLIP_ESC_END = 0xdc
+    SLIP_ESC_ESC = 0xdd
 
+    def __init__(self):
         self.started = False
         self.escaped = False
-        self.stream = ''
-        self.packet = ''
+        self.stream = bytearray()
+        self.packet = bytearray()
 
-    def append(self, data):
+    def append(self, data: bytes):
         """
         Append a new
         :param data: Append a new block of data to do decoding on when calling decode.
@@ -66,59 +64,59 @@ def decode(self):
         Decodes a package according to http://en.wikipedia.org/wiki/Serial_Line_Internet_Protocol
         :return Slip: A list of decoded slip packets
         """
-        packet_list = list()
+        packet_list = []
 
         for char in self.stream:
             if char == self.SLIP_END:
                 if self.started:
                     if len(self.packet) > 0:
                         self.started = False
                         packet_list.append(self.packet)
-                        self.packet = ''
+                        self.packet = bytearray()
                 else:
                     self.started = True
-                    self.packet = ''
+                    self.packet = bytearray()
             elif char == self.SLIP_ESC:
                 self.escaped = True
             elif char == self.SLIP_ESC_END:
                 if self.escaped:
-                    self.packet += self.SLIP_END
+                    self.packet.append(self.SLIP_END)
                     self.escaped = False
                 else:
                     self.packet += char
             elif char == self.SLIP_ESC_ESC:
                 if self.escaped:
-                    self.packet += self.SLIP_ESC
+                    self.packet.append(self.SLIP_ESC)
                     self.escaped = False
                 else:
-                    self.packet += char
+                    self.packet.append(char)
             else:
                 if self.escaped:
                     logging.error("Error in SLIP packet, ignoring error.")
-                    self.packet = ''
+                    self.packet = bytearray()
                     self.escaped = False
                 else:
-                    self.packet += char
+                    self.packet.append(char)
 
-        self.stream = ''
+        self.stream.clear()
 
         return packet_list
 
-    def encode(self, packet):
+    def encode(self, packet: bytes):
         """
         Encode a packet according to SLIP.
         :param packet: A str array that represents the package
         :return: str array with an encoded SLIP packet
         """
-        encoded = self.SLIP_END
+        encoded = bytearray([self.SLIP_END])
 
         for char in packet:
             if char == self.SLIP_END:
-                encoded += self.SLIP_ESC + self.SLIP_ESC_END
+                encoded.extend([self.SLIP_ESC, self.SLIP_ESC_END])
             elif char == self.SLIP_ESC:
-                encoded += self.SLIP_ESC + self.SLIP_ESC_ESC
+                encoded.extend([self.SLIP_ESC, self.SLIP_ESC_ESC])
             else:
-                encoded += char
-        encoded += self.SLIP_END
+                encoded.append(char)
+        encoded.append(self.SLIP_END)
 
         return encoded

nordicsemi/bluetooth/hci/tests/test_codec.py

@@ -64,7 +64,7 @@ def test_decode_packet(self):
 
         for uart_packet in read_packets:
             hex_string = uart_packet.replace(" ", "")
-            hex_data = hex_string.decode("hex")
+            hex_data = bytes.fromhex(hex_string)
             slip.append(hex_data)
 
         packets = slip.decode()

nordicsemi/dfu/bl_dfu_sett.py

@@ -47,7 +47,6 @@
 
 # 3rd party libraries
 import intelhex
-from intelhex import IntelHexError
 
 # Nordic libraries
 from nordicsemi.dfu.nrfhex import *
@@ -112,8 +111,7 @@ class BLDFUSettings(object):
     bl_sett_52840_addr    = 0x000FF000
     bl_sett_backup_offset = 0x1000
 
-
-    def __init__(self, ):
+    def __init__(self):
         """
         """
         # instantiate a hex object
@@ -168,7 +166,7 @@ def _calculate_crc32_from_hex(self, ih_object, start_addr=None, end_addr=None):
         list = []
         if start_addr == None and end_addr == None:
             hex_dict = ih_object.todict()
-            for addr, byte in hex_dict.items():
+            for addr, byte in list(hex_dict.items()):
                 list.append(byte)
         else:
             for addr in range(start_addr, end_addr + 1):
@@ -215,21 +213,22 @@ def generate(self, arch, app_file, app_ver, bl_ver, bl_sett_ver, custom_bl_sett_
                 self.app_boot_validation_bytes = struct.pack('<I', self.app_crc)
             elif app_boot_validation_type == 'VALIDATE_GENERATED_SHA256':
                 self.app_boot_validation_type = 2 & 0xffffffff
-                sha256 = Package.calculate_sha256_hash(self.app_bin)
-                self.app_boot_validation_bytes = bytearray([int(binascii.hexlify(i), 16) for i in list(sha256)][31::-1])
+                # Package.calculate_sha256_hash gives a reversed
+                # digest. It need to be reversed back to a normal
+                # sha256 digest.
+                self.app_boot_validation_bytes = Package.calculate_sha256_hash(self.app_bin)[::-1]
             elif app_boot_validation_type == 'VALIDATE_ECDSA_P256_SHA256':
                 self.app_boot_validation_type = 3 & 0xffffffff
-                ecdsa = Package.sign_firmware(key_file, self.app_bin)
-                self.app_boot_validation_bytes = bytearray([int(binascii.hexlify(i), 16) for i in list(ecdsa)])
+                self.app_boot_validation_bytes = Package.sign_firmware(key_file, self.app_bin)
             else:  # This also covers 'NO_VALIDATION' case
                 self.app_boot_validation_type = 0 & 0xffffffff
-                self.app_boot_validation_bytes = bytearray(0)
+                self.app_boot_validation_bytes = bytes(0)
         else:
             self.app_sz = 0x0 & 0xffffffff
             self.app_crc = 0x0 & 0xffffffff
             self.bank0_bank_code = 0x0 & 0xffffffff
             self.app_boot_validation_type = 0x0 & 0xffffffff
-            self.app_boot_validation_bytes = bytearray(0)
+            self.app_boot_validation_bytes = bytes(0)
 
         if sd_file is not None:
             # Load SD to calculate CRC
@@ -254,19 +253,20 @@ def generate(self, arch, app_file, app_ver, bl_ver, bl_sett_ver, custom_bl_sett_
                 self.sd_boot_validation_bytes = struct.pack('<I', sd_crc)
             elif sd_boot_validation_type == 'VALIDATE_GENERATED_SHA256':
                 self.sd_boot_validation_type = 2 & 0xffffffff
-                sha256 = Package.calculate_sha256_hash(self.sd_bin)
-                self.sd_boot_validation_bytes = bytearray([int(binascii.hexlify(i), 16) for i in list(sha256)][31::-1])
+                # Package.calculate_sha256_hash gives a reversed
+                # digest. It need to be reversed back to a normal
+                # sha256 digest.
+                self.sd_boot_validation_bytes = Package.calculate_sha256_hash(self.sd_bin)[::-1]
             elif sd_boot_validation_type == 'VALIDATE_ECDSA_P256_SHA256':
                 self.sd_boot_validation_type = 3 & 0xffffffff
-                ecdsa = Package.sign_firmware(key_file, self.sd_bin)
-                self.sd_boot_validation_bytes = bytearray([int(binascii.hexlify(i), 16) for i in list(ecdsa)])
+                self.sd_boot_validation_bytes = Package.sign_firmware(key_file, self.sd_bin)
             else:  # This also covers 'NO_VALIDATION_CASE'
                 self.sd_boot_validation_type = 0 & 0xffffffff
-                self.sd_boot_validation_bytes = bytearray(0)
+                self.sd_boot_validation_bytes = bytes(0)
         else:
             self.sd_sz = 0x0 & 0xffffffff
             self.sd_boot_validation_type = 0 & 0xffffffff
-            self.sd_boot_validation_bytes = bytearray(0)
+            self.sd_boot_validation_bytes = bytes(0)
 
         # additional harcoded values
         self.bank_layout = 0x0 & 0xffffffff

nordicsemi/dfu/crc16.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2016 Nordic Semiconductor ASA
+# Copyright (c) 2019 Nordic Semiconductor ASA
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
@@ -35,7 +35,7 @@
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
-def calc_crc16(binary_data, crc=0xffff):
+def calc_crc16(binary_data: bytes, crc=0xffff):
     """
     Calculates CRC16 on binary_data
 
@@ -46,7 +46,6 @@ def calc_crc16(binary_data, crc=0xffff):
 
     for b in binary_data:
         crc = (crc >> 8 & 0x00FF) | (crc << 8 & 0xFF00)
-        crc ^= ord(b)
         crc ^= (crc & 0x00FF) >> 4
         crc ^= (crc << 8) << 4
         crc ^= ((crc & 0x00FF) << 4) << 1