From 96cdbc7ade42cc24fdefe5f4823e116d0c0d4dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ken=20A=2E=20Rederg=C3=A5rd?= <64542+kenr@users.noreply.github.com> Date: Wed, 19 Jun 2019 15:39:20 +0200 Subject: [PATCH 01/34] Initial py3 support This is work in progress --- .gitignore | 1 + README.md | 2 +- nordicsemi/__main__.py | 8 +-- nordicsemi/dfu/bl_dfu_sett.py | 2 +- nordicsemi/dfu/dfu_transport.py | 5 +- nordicsemi/dfu/dfu_transport_ant.py | 14 ++-- nordicsemi/dfu/dfu_transport_ble.py | 16 ++--- nordicsemi/dfu/dfu_transport_serial.py | 8 +-- nordicsemi/dfu/init_packet_pb.py | 2 +- nordicsemi/dfu/intelhex/__init__.py | 66 +++++++++---------- nordicsemi/dfu/manifest.py | 2 +- nordicsemi/dfu/nrfhex.py | 2 +- nordicsemi/dfu/package.py | 4 +- nordicsemi/dfu/signing.py | 12 ++-- nordicsemi/dfu/tests/test_bl_dfu_sett.py | 20 +++--- nordicsemi/dfu/tests/test_package.py | 20 +++--- nordicsemi/dfu/util.py | 6 +- nordicsemi/lister/lister_backend.py | 4 +- nordicsemi/lister/unix/unix_lister.py | 2 +- nordicsemi/lister/windows/constants.py | 2 +- nordicsemi/lister/windows/lister_win32.py | 4 +- nordicsemi/thread/dfu_server.py | 4 +- nordicsemi/thread/tncp.py | 6 +- nordicsemi/utility/target_registry.py | 6 +- nordicsemi/version.py | 2 +- setup.py | 2 +- tests/bdd/steps/common_steps.py | 2 +- tests/bdd/steps/dfu_steps.py | 14 ++-- .../genpkg_generate_dfu_package_steps.py | 44 ++++++------- tests/bdd/steps/help_information_steps.py | 8 +-- tests/bdd/steps/util.py | 4 +- tests/protobuf/__main__.py | 2 +- tests/test_cli.py | 2 +- 33 files changed, 147 insertions(+), 151 deletions(-) diff --git a/.gitignore b/.gitignore index 4b84051..f56cd43 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +*.py.bak # Virtualenv venv/ diff --git a/README.md b/README.md index 4b65f54..cc26fd4 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ You will need to clone the present repository first to run or install nrfutil fr To install nrfutil from source the following prerequisites must be satisfied: -* [Python 2.7 (2.7.10 or newer, not Python 3)](https://www.python.org/downloads/) +* [Python 3.7](https://www.python.org/downloads/) * [pip](https://pip.pypa.io/en/stable/installing.html) * setuptools (upgrade to latest version): `pip install -U setuptools` diff --git a/nordicsemi/__main__.py b/nordicsemi/__main__.py index d1cea27..849e3c1 100755 --- a/nordicsemi/__main__.py +++ b/nordicsemi/__main__.py @@ -146,7 +146,7 @@ def int_as_text_to_int(value): def pause(): while True: try: - raw_input() + input() except (KeyboardInterrupt, EOFError): break @@ -793,7 +793,7 @@ def generate(zipfile, try: # This will parse any string starting with 0x as base 16. sd_req_list = sd_req.split(',') - sd_req_list = map(int_as_text_to_int, sd_req_list) + sd_req_list = list(map(int_as_text_to_int, sd_req_list)) except ValueError: raise NordicSemiException("Could not parse value for --sd-req. " "Hex values should be prefixed with 0x.") @@ -803,7 +803,7 @@ def generate(zipfile, try: # This will parse any string starting with 0x as base 16. sd_id_list = sd_id.split(',') - sd_id_list = map(int_as_text_to_int, sd_id_list) + sd_id_list = list(map(int_as_text_to_int, sd_id_list)) # Copy all IDs from sd_id_list to sd_req_list, without duplicates. # This ensures that the softdevice update can be repeated in case @@ -1351,7 +1351,7 @@ def thread(package, port, address, server_port, panid, channel, jlink_snr, flash mcast_dfu = False if address is None: - address = ipaddress.ip_address(u"ff03::1") + address = ipaddress.ip_address("ff03::1") click.echo("Address not specified. Using ff03::1 (all Thread nodes)") else: try: diff --git a/nordicsemi/dfu/bl_dfu_sett.py b/nordicsemi/dfu/bl_dfu_sett.py index fa8b708..e646661 100755 --- a/nordicsemi/dfu/bl_dfu_sett.py +++ b/nordicsemi/dfu/bl_dfu_sett.py @@ -166,7 +166,7 @@ def _calculate_crc32_from_hex(self, ih_object, start_addr=None, end_addr=None): list = [] if start_addr == None and end_addr == None: hex_dict = ih_object.todict() - for addr, byte in hex_dict.items(): + for addr, byte in list(hex_dict.items()): list.append(byte) else: for addr in range(start_addr, end_addr + 1): diff --git a/nordicsemi/dfu/dfu_transport.py b/nordicsemi/dfu/dfu_transport.py index 939dd88..628ebbe 100644 --- a/nordicsemi/dfu/dfu_transport.py +++ b/nordicsemi/dfu/dfu_transport.py @@ -54,7 +54,7 @@ class DfuEvent: PROGRESS_EVENT = 1 -class DfuTransport(object): +class DfuTransport(object, metaclass=abc.ABCMeta): """ This class as an abstract base class inherited from when implementing transports. @@ -62,7 +62,6 @@ class DfuTransport(object): than this class describes. But the intent is that the implementer shall follow the semantic as best as she can. """ - __metaclass__ = abc.ABCMeta OP_CODE = { 'CreateObject' : 0x01, @@ -176,6 +175,6 @@ def _send_event(self, event_type, **kwargs): :param kwargs: Arguments to callback function :return: """ - if event_type in self.callbacks.keys(): + if event_type in list(self.callbacks.keys()): for callback in self.callbacks[event_type]: callback(**kwargs) diff --git a/nordicsemi/dfu/dfu_transport_ant.py b/nordicsemi/dfu/dfu_transport_ant.py index ceead23..86b3b06 100755 --- a/nordicsemi/dfu/dfu_transport_ant.py +++ b/nordicsemi/dfu/dfu_transport_ant.py @@ -39,7 +39,7 @@ import binascii from datetime import datetime, timedelta import logging -import Queue +import queue import struct import sys @@ -132,7 +132,7 @@ def __init__(self, ant_dev, timeout, search_timeout, ant_config): self.tx_seq = None self.rx_seq = None self.rx_data = None - self.resp_queue = Queue.Queue() + self.resp_queue = queue.Queue() def open(self): # TODO: use constant from antlib when it exists. @@ -187,7 +187,7 @@ def send_message(self, req): logger.log(TRANSPORT_LOGGING_LEVEL, "ANT: --> {}".format(req)) self.tx_seq = (self.tx_seq + 1) & 0xFF - data = map(ord, struct.pack('= 0.') return self._buf.get(addr, self.padding) elif t == slice: - addresses = self._buf.keys() + addresses = list(self._buf.keys()) ih = IntelHex() if addresses: addresses.sort() start = addr.start or addresses[0] stop = addr.stop or (addresses[-1]+1) step = addr.step or 1 - for i in xrange(start, stop, step): + for i in range(start, stop, step): x = self._buf.get(i) if x is not None: ih[i] = x @@ -474,7 +474,7 @@ def __getitem__(self, addr): def __setitem__(self, addr, byte): """Set byte at address.""" t = type(addr) - if t in (int, long): + if t in (int, int): if addr < 0: raise TypeError('Address should be >= 0.') self._buf[addr] = byte @@ -485,7 +485,7 @@ def __setitem__(self, addr, byte): stop = addr.stop step = addr.step or 1 if None not in (start, stop): - ra = range(start, stop, step) + ra = list(range(start, stop, step)) if len(ra) != len(byte): raise ValueError('Length of bytes sequence does not match ' 'address range') @@ -500,7 +500,7 @@ def __setitem__(self, addr, byte): if stop < 0: raise TypeError('stop address cannot be negative') j = 0 - for i in xrange(start, stop, step): + for i in range(start, stop, step): self._buf[i] = byte[j] j += 1 else: @@ -509,18 +509,18 @@ def __setitem__(self, addr, byte): def __delitem__(self, addr): """Delete byte at address.""" t = type(addr) - if t in (int, long): + if t in (int, int): if addr < 0: raise TypeError('Address should be >= 0.') del self._buf[addr] elif t == slice: - addresses = self._buf.keys() + addresses = list(self._buf.keys()) if addresses: addresses.sort() start = addr.start or addresses[0] stop = addr.stop or (addresses[-1]+1) step = addr.step or 1 - for i in xrange(start, stop, step): + for i in range(start, stop, step): x = self._buf.get(i) if x is not None: del self._buf[i] @@ -529,7 +529,7 @@ def __delitem__(self, addr): def __len__(self): """Return count of bytes with real values.""" - return len(self._buf.keys()) + return len(list(self._buf.keys())) def write_hex_file(self, f, write_start_addr=True): """Write data to file f in HEX format. @@ -554,7 +554,7 @@ def write_hex_file(self, f, write_start_addr=True): # is faster than hexstr.upper(): # 0.452ms vs. 0.652ms (translate vs. upper) if sys.version_info[0] >= 3: - table = bytes(range(256)).upper() + table = bytes(list(range(256))).upper() else: table = ''.join(chr(i).upper() for i in range(256)) @@ -562,7 +562,7 @@ def write_hex_file(self, f, write_start_addr=True): # start address record if any if self.start_addr and write_start_addr: - keys = self.start_addr.keys() + keys = list(self.start_addr.keys()) keys.sort() bin = array('B', asbytes('\0'*9)) if keys == ['CS','IP']: @@ -602,7 +602,7 @@ def write_hex_file(self, f, write_start_addr=True): raise InvalidStartAddressValueError(start_addr=self.start_addr) # data - addresses = self._buf.keys() + addresses = list(self._buf.keys()) addresses.sort() addr_len = len(addresses) if addr_len: @@ -708,7 +708,7 @@ def gets(self, addr, length): be raised. Padding is not used.""" a = array('B', asbytes('\0'*length)) try: - for i in xrange(length): + for i in range(length): a[i] = self._buf[addr+i] except KeyError: raise NotEnoughDataError(address=addr, length=length) @@ -719,7 +719,7 @@ def puts(self, addr, s): entries. """ a = array('B', asbytes(s)) - for i in xrange(len(a)): + for i in range(len(a)): self._buf[addr+i] = a[i] def getsz(self, addr): @@ -764,7 +764,7 @@ def dump(self, tofile=None): else: tofile.write('start_addr = %r\n' % start_addr) # actual data - addresses = self._buf.keys() + addresses = list(self._buf.keys()) if addresses: addresses.sort() minaddr = addresses[0] @@ -773,8 +773,8 @@ def dump(self, tofile=None): endaddr = int((maxaddr>>4)+1)*16 maxdigits = max(len(str(endaddr)), 4) templa = '%%0%dX' % maxdigits - range16 = range(16) - for i in xrange(startaddr, endaddr, 16): + range16 = list(range(16)) + for i in range(startaddr, endaddr, 16): tofile.write(templa % i) tofile.write(' ') s = [] @@ -907,7 +907,7 @@ def minaddr(self): @return minimal address used in this object ''' - aa = self._buf.keys() + aa = list(self._buf.keys()) if aa == []: return 0 else: @@ -918,7 +918,7 @@ def maxaddr(self): @return maximal address used in this object ''' - aa = self._buf.keys() + aa = list(self._buf.keys()) if aa == []: return 0 else: @@ -943,7 +943,7 @@ def tobinarray(self, start=None, end=None, size=None): start, end = self._get_start_end(start, end, size) - for addr in xrange(start, end+1): + for addr in range(start, end+1): bin.append(self[addr]) return bin @@ -1035,7 +1035,7 @@ def diff_dumps(ih1, ih2, tofile=None, name1="a", name2="b", n_context=3): @param n_context number of context lines in the unidiff output """ def prepare_lines(ih): - from cStringIO import StringIO + from io import StringIO sio = StringIO() ih.dump(sio) dump = sio.getvalue() @@ -1208,7 +1208,7 @@ def __init__(self, msg=None, **kw): """Initialize the Exception with the given message. """ self.msg = msg - for key, value in kw.items(): + for key, value in list(kw.items()): setattr(self, key, value) def __str__(self): diff --git a/nordicsemi/dfu/manifest.py b/nordicsemi/dfu/manifest.py index e2480c7..2d3d7e5 100644 --- a/nordicsemi/dfu/manifest.py +++ b/nordicsemi/dfu/manifest.py @@ -92,7 +92,7 @@ def remove_none_entries(d): if not isinstance(d, dict): return d - return dict((k, remove_none_entries(v)) for k, v in d.iteritems() if v is not None) + return dict((k, remove_none_entries(v)) for k, v in d.items() if v is not None) return json.dumps({'manifest': self.manifest}, default=lambda o: remove_none_entries(o.__dict__), diff --git a/nordicsemi/dfu/nrfhex.py b/nordicsemi/dfu/nrfhex.py index 2b52510..ec94a68 100644 --- a/nordicsemi/dfu/nrfhex.py +++ b/nordicsemi/dfu/nrfhex.py @@ -116,7 +116,7 @@ def get_softdevice_variant(self): if self.address_has_magic_number(potential_magic_number_address): return "s1x0" - for i in xrange(4): + for i in range(4): potential_magic_number_address += nRFHex.info_struct_address_offset if self.address_has_magic_number(potential_magic_number_address): diff --git a/nordicsemi/dfu/package.py b/nordicsemi/dfu/package.py index 5a84f4b..1b4d8ef 100644 --- a/nordicsemi/dfu/package.py +++ b/nordicsemi/dfu/package.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import + # # Copyright (c) 2016 Nordic Semiconductor ASA # All rights reserved. @@ -406,7 +406,7 @@ def generate_package(self, filename, preserve_work_dir=False): sd_bin_path = os.path.join(self.work_dir, sd_bin) sd_bin_created = True - for key, firmware_data in self.firmwares_data.iteritems(): + for key, firmware_data in self.firmwares_data.items(): # Normalize the firmware file and store it in the work directory firmware_data[FirmwareKeys.BIN_FILENAME] = \ diff --git a/nordicsemi/dfu/signing.py b/nordicsemi/dfu/signing.py index 700fcb7..74c423e 100644 --- a/nordicsemi/dfu/signing.py +++ b/nordicsemi/dfu/signing.py @@ -1,4 +1,4 @@ -from __future__ import print_function + # # Copyright (c) 2016 Nordic Semiconductor ASA # All rights reserved. @@ -169,7 +169,7 @@ def get_sk_hex(self): sk_hexlify = binascii.hexlify(self.sk.to_string()) sk_hexlify_list = [] - for i in xrange(len(sk_hexlify)-2, -2, -2): + for i in range(len(sk_hexlify)-2, -2, -2): sk_hexlify_list.append(sk_hexlify[i:i+2]) sk_hexlify_list_str = ''.join(sk_hexlify_list) @@ -189,10 +189,10 @@ def get_vk_hex(self): vk_hexlify = binascii.hexlify(vk.to_string()) vk_hexlify_list = [] - for i in xrange(len(vk_hexlify[0:64])-2, -2, -2): + for i in range(len(vk_hexlify[0:64])-2, -2, -2): vk_hexlify_list.append(vk_hexlify[i:i+2]) - for i in xrange(len(vk_hexlify)-2, 62, -2): + for i in range(len(vk_hexlify)-2, 62, -2): vk_hexlify_list.append(vk_hexlify[i:i+2]) vk_hexlify_list_str = ''.join(vk_hexlify_list) @@ -240,12 +240,12 @@ def get_vk_code(self, dbg): vk_x_separated = "" vk_x_str = vk_hex[0:64] - for i in xrange(0, len(vk_x_str), 2): + for i in range(0, len(vk_x_str), 2): vk_x_separated = "0x" + vk_x_str[i:i+2] + ", " + vk_x_separated vk_y_separated = "" vk_y_str = vk_hex[64:128] - for i in xrange(0, len(vk_y_str), 2): + for i in range(0, len(vk_y_str), 2): vk_y_separated = "0x" + vk_y_str[i:i+2] + ", " + vk_y_separated vk_y_separated = vk_y_separated[:-2] diff --git a/nordicsemi/dfu/tests/test_bl_dfu_sett.py b/nordicsemi/dfu/tests/test_bl_dfu_sett.py index 3c76c8d..905b414 100644 --- a/nordicsemi/dfu/tests/test_bl_dfu_sett.py +++ b/nordicsemi/dfu/tests/test_bl_dfu_sett.py @@ -169,7 +169,7 @@ def test_generate_with_backup_page_check_size(self): sd_file=None, key_file=None) - self.assertEqual(len(settings.ihex.todict().keys()), len(settings_raw.ihex.todict().keys()) * 2) + self.assertEqual(len(list(settings.ihex.todict().keys())), len(list(settings_raw.ihex.todict().keys())) * 2) def test_generate_with_backup_page_check_values(self): settings = BLDFUSettings() @@ -186,8 +186,8 @@ def test_generate_with_backup_page_check_values(self): sd_file=None, key_file=None) - backup_dict = {(k + settings.bl_sett_backup_offset): v for k, v in settings.ihex.todict().items() if k < settings.bl_sett_addr} - settings_dict = {k: v for k, v in settings.ihex.todict().items() if k >= settings.bl_sett_addr} + backup_dict = {(k + settings.bl_sett_backup_offset): v for k, v in list(settings.ihex.todict().items()) if k < settings.bl_sett_addr} + settings_dict = {k: v for k, v in list(settings.ihex.todict().items()) if k >= settings.bl_sett_addr} self.assertEqual(backup_dict, settings_dict) def test_generate_with_backup_page_custom_address(self): @@ -206,7 +206,7 @@ def test_generate_with_backup_page_custom_address(self): key_file=None) self.assertEqual(settings.backup_address, 0x0006F000) - self.assertTrue(0x0006F000 in settings.ihex.todict().keys()) + self.assertTrue(0x0006F000 in list(settings.ihex.todict().keys())) def test_generate_with_backup_page_default_address(self): settings = BLDFUSettings() @@ -224,7 +224,7 @@ def test_generate_with_backup_page_default_address(self): key_file=None) self.assertEqual(settings.backup_address, (0x0006F000 - settings.bl_sett_backup_offset)) - self.assertTrue((0x0006F000 - settings.bl_sett_backup_offset) in settings.ihex.todict().keys()) + self.assertTrue((0x0006F000 - settings.bl_sett_backup_offset) in list(settings.ihex.todict().keys())) class TestBLDFUSettingsV2(unittest.TestCase): def setUp(self): @@ -369,7 +369,7 @@ def test_generate_with_backup_page_check_size(self): sd_file=None, key_file=None) - self.assertEqual(len(settings.ihex.todict().keys()), len(settings_raw.ihex.todict().keys()) * 2) + self.assertEqual(len(list(settings.ihex.todict().keys())), len(list(settings_raw.ihex.todict().keys())) * 2) def test_generate_with_backup_page_check_values(self): settings = BLDFUSettings() @@ -386,8 +386,8 @@ def test_generate_with_backup_page_check_values(self): sd_file=None, key_file=None) - backup_dict = {(k + settings.bl_sett_backup_offset): v for k, v in settings.ihex.todict().items() if k < settings.bl_sett_addr} - settings_dict = {k: v for k, v in settings.ihex.todict().items() if k >= settings.bl_sett_addr} + backup_dict = {(k + settings.bl_sett_backup_offset): v for k, v in list(settings.ihex.todict().items()) if k < settings.bl_sett_addr} + settings_dict = {k: v for k, v in list(settings.ihex.todict().items()) if k >= settings.bl_sett_addr} self.assertEqual(backup_dict, settings_dict) def test_generate_with_backup_page_custom_address(self): @@ -406,7 +406,7 @@ def test_generate_with_backup_page_custom_address(self): key_file=None) self.assertEqual(settings.backup_address, 0x0006F000) - self.assertTrue(0x0006F000 in settings.ihex.todict().keys()) + self.assertTrue(0x0006F000 in list(settings.ihex.todict().keys())) def test_generate_with_backup_page_default_address(self): settings = BLDFUSettings() @@ -424,7 +424,7 @@ def test_generate_with_backup_page_default_address(self): key_file=None) self.assertEqual(settings.backup_address, (0x0006F000 - settings.bl_sett_backup_offset)) - self.assertTrue((0x0006F000 - settings.bl_sett_backup_offset) in settings.ihex.todict().keys()) + self.assertTrue((0x0006F000 - settings.bl_sett_backup_offset) in list(settings.ihex.todict().keys())) def test_generate_with_app_boot_validation_crc(self): settings = BLDFUSettings() diff --git a/nordicsemi/dfu/tests/test_package.py b/nordicsemi/dfu/tests/test_package.py index 6203a9d..18340a8 100644 --- a/nordicsemi/dfu/tests/test_package.py +++ b/nordicsemi/dfu/tests/test_package.py @@ -76,11 +76,11 @@ def test_generate_package_application(self): with open(os.path.join(self.work_directory, 'manifest.json'), 'r') as f: _json = json.load(f) - self.assertEqual(u'bar.bin', _json['manifest']['application']['bin_file']) - self.assertEqual(u'bar.dat', _json['manifest']['application']['dat_file']) - self.assertTrue(u'softdevice' not in _json['manifest']) - self.assertTrue(u'softdevice_bootloader' not in _json['manifest']) - self.assertTrue(u'bootloader' not in _json['manifest']) + self.assertEqual('bar.bin', _json['manifest']['application']['bin_file']) + self.assertEqual('bar.dat', _json['manifest']['application']['dat_file']) + self.assertTrue('softdevice' not in _json['manifest']) + self.assertTrue('softdevice_bootloader' not in _json['manifest']) + self.assertTrue('bootloader' not in _json['manifest']) def test_generate_package_sd_bl(self): self.p = Package(app_version=100, @@ -108,8 +108,8 @@ def test_generate_package_sd_bl(self): with open(os.path.join(self.work_directory, 'manifest.json'), 'r') as f: _json = json.load(f) - self.assertEqual(u'sd_bl.bin', _json['manifest']['softdevice_bootloader']['bin_file']) - self.assertEqual(u'sd_bl.dat', _json['manifest']['softdevice_bootloader']['dat_file']) + self.assertEqual('sd_bl.bin', _json['manifest']['softdevice_bootloader']['bin_file']) + self.assertEqual('sd_bl.dat', _json['manifest']['softdevice_bootloader']['dat_file']) def test_unpack_package_a(self): self.p = Package(app_version=100, @@ -122,7 +122,7 @@ def test_unpack_package_a(self): unpacked_dir = os.path.join(self.work_directory, "unpacked") manifest = self.p.unpack_package(os.path.join(self.work_directory, pkg_name), unpacked_dir) self.assertIsNotNone(manifest) - self.assertEqual(u'bar.bin', manifest.softdevice.bin_file) + self.assertEqual('bar.bin', manifest.softdevice.bin_file) # self.assertEqual(0, manifest.softdevice.init_packet_data.ext_packet_id) # self.assertIsNotNone(manifest.softdevice.init_packet_data.firmware_crc16) @@ -137,7 +137,7 @@ def test_unpack_package_b(self): unpacked_dir = os.path.join(self.work_directory, "unpacked") manifest = self.p.unpack_package(os.path.join(self.work_directory, pkg_name), unpacked_dir) self.assertIsNotNone(manifest) - self.assertEqual(u'bar.bin', manifest.softdevice.bin_file) + self.assertEqual('bar.bin', manifest.softdevice.bin_file) def test_unpack_package_c(self): self.p = Package(app_version=100, @@ -150,7 +150,7 @@ def test_unpack_package_c(self): unpacked_dir = os.path.join(self.work_directory, "unpacked") manifest = self.p.unpack_package(os.path.join(self.work_directory, pkg_name), unpacked_dir) self.assertIsNotNone(manifest) - self.assertEqual(u'bar.bin', manifest.softdevice.bin_file) + self.assertEqual('bar.bin', manifest.softdevice.bin_file) if __name__ == '__main__': diff --git a/nordicsemi/dfu/util.py b/nordicsemi/dfu/util.py index bd9674d..8f85c5b 100644 --- a/nordicsemi/dfu/util.py +++ b/nordicsemi/dfu/util.py @@ -1,4 +1,4 @@ -from __future__ import print_function + # # Copyright (c) 2016 Nordic Semiconductor ASA # All rights reserved. @@ -44,7 +44,7 @@ class NordicEnum(Enum): @classmethod def tostring(cls, val): - for k,v in vars(cls).iteritems(): + for k,v in vars(cls).items(): if v==val: return k @@ -66,7 +66,7 @@ def query_func(question, default=False): while True: print("%s %s" % (question, prompt)) - choice = raw_input().lower() + choice = input().lower() if choice == '': return default elif choice in valid: diff --git a/nordicsemi/lister/lister_backend.py b/nordicsemi/lister/lister_backend.py index 5194419..50dc7f8 100644 --- a/nordicsemi/lister/lister_backend.py +++ b/nordicsemi/lister/lister_backend.py @@ -37,9 +37,7 @@ import abc -class AbstractLister(object): - __metaclass__ = abc.ABCMeta - +class AbstractLister(object, metaclass=abc.ABCMeta): @abc.abstractmethod def enumerate(self): """ diff --git a/nordicsemi/lister/unix/unix_lister.py b/nordicsemi/lister/unix/unix_lister.py index 6716d06..946d1a1 100644 --- a/nordicsemi/lister/unix/unix_lister.py +++ b/nordicsemi/lister/unix/unix_lister.py @@ -69,4 +69,4 @@ def enumerate(self): else: device_identities[id] = EnumeratedDevice(vendor_id, product_id, serial_number, [com_port]) - return [device for device in device_identities.values()] + return [device for device in list(device_identities.values())] diff --git a/nordicsemi/lister/windows/constants.py b/nordicsemi/lister/windows/constants.py index c63cf1b..62cf05d 100644 --- a/nordicsemi/lister/windows/constants.py +++ b/nordicsemi/lister/windows/constants.py @@ -23,7 +23,7 @@ """ import enum -from structures import DevicePropertyKey +from .structures import DevicePropertyKey # noinspection SpellCheckingInspection diff --git a/nordicsemi/lister/windows/lister_win32.py b/nordicsemi/lister/windows/lister_win32.py index bf03da1..e1d9aef 100644 --- a/nordicsemi/lister/windows/lister_win32.py +++ b/nordicsemi/lister/windows/lister_win32.py @@ -40,8 +40,8 @@ from nordicsemi.lister.enumerated_device import EnumeratedDevice if sys.platform == 'win32': - from constants import DIGCF_PRESENT, DEVPKEY, DIGCF_DEVICEINTERFACE - from structures import GUID, DeviceInfoData, ctypesInternalGUID + from .constants import DIGCF_PRESENT, DEVPKEY, DIGCF_DEVICEINTERFACE + from .structures import GUID, DeviceInfoData, ctypesInternalGUID import ctypes from ctypes.wintypes import * diff --git a/nordicsemi/thread/dfu_server.py b/nordicsemi/thread/dfu_server.py index daa0729..215119b 100644 --- a/nordicsemi/thread/dfu_server.py +++ b/nordicsemi/thread/dfu_server.py @@ -122,7 +122,7 @@ def __init__(self): Resource = namedtuple('Resource', ['path', 'data']) class ThreadDfuServer(): - REALM_LOCAL_ADDR = ip_address(u'FF03::1') + REALM_LOCAL_ADDR = ip_address('FF03::1') SPBLK_SIZE = 64 # number of CoAP blocks of BLOCK_SZX size each SPBLK_UPLOAD_RATE = 1 # in blocks / seconds @@ -370,7 +370,7 @@ def receive_request(self, request): ThreadDfuServer.BITMAP_URI : self._handle_bitmap_request, } - for uri, handler in handlers.items(): + for uri, handler in list(handlers.items()): if '/'.join(request.opt.uri_path).startswith(uri): return handler(request) diff --git a/nordicsemi/thread/tncp.py b/nordicsemi/thread/tncp.py index 196da0c..814bf5a 100644 --- a/nordicsemi/thread/tncp.py +++ b/nordicsemi/thread/tncp.py @@ -78,7 +78,7 @@ def __init__(self, port, stream_descriptor, config = None): @staticmethod def _propid_to_str(propid): - for name, value in SPINEL.__dict__.iteritems(): + for name, value in SPINEL.__dict__.items(): if (name.startswith('PROP_') and value == propid): return name @@ -160,7 +160,7 @@ def add_ip_address(self, ipaddr): flags = 0 prefix_len = 64 - prefix = ipaddress.IPv6Interface(unicode(ipaddr)) + prefix = ipaddress.IPv6Interface(str(ipaddr)) arr = prefix.ip.packed arr += self._wpan.encode_fields('CLLC', prefix_len, @@ -174,7 +174,7 @@ def add_ip_address(self, ipaddr): def print_addresses(self): logger.info("NCP Thread IPv6 addresses:") for addr in self._wpan.get_ipaddrs(): - logger.info(unicode(addr)) + logger.info(str(addr)) def send(self, payload, dest): if (dest.addr.is_multicast): diff --git a/nordicsemi/utility/target_registry.py b/nordicsemi/utility/target_registry.py index 6d2ea05..cc991de 100644 --- a/nordicsemi/utility/target_registry.py +++ b/nordicsemi/utility/target_registry.py @@ -41,9 +41,7 @@ from abc import ABCMeta, abstractmethod -class TargetDatabase(object): - __metaclass__ = ABCMeta - +class TargetDatabase(object, metaclass=ABCMeta): @abstractmethod def get_targets(self): pass @@ -73,7 +71,7 @@ def get_targets(self): if self.targets is None: self.targets = [] - for key, value in os.environ.iteritems(): + for key, value in os.environ.items(): match = re.match("NORDICSEMI_TARGET_(?P\d+)_(?P[a-zA-Z_]+)", key) if match: diff --git a/nordicsemi/version.py b/nordicsemi/version.py index 08a20e5..6716769 100644 --- a/nordicsemi/version.py +++ b/nordicsemi/version.py @@ -37,4 +37,4 @@ """ Version definition for nrfutil. """ -NRFUTIL_VERSION = "5.1.0" +NRFUTIL_VERSION = "6.0.0a0" diff --git a/setup.py b/setup.py index d14f653..7fd0688 100644 --- a/setup.py +++ b/setup.py @@ -167,7 +167,7 @@ def run_tests(self): 'License :: Other/Proprietary License', - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.7', ], keywords = 'nordic nrf51 nrf52 ble bluetooth dfu ota softdevice serialization nrfutil pc-nrfutil', cmdclass={ diff --git a/tests/bdd/steps/common_steps.py b/tests/bdd/steps/common_steps.py index 0aa8d7f..116c64b 100644 --- a/tests/bdd/steps/common_steps.py +++ b/tests/bdd/steps/common_steps.py @@ -35,7 +35,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -from Queue import Queue +from queue import Queue import logging import os import subprocess diff --git a/tests/bdd/steps/dfu_steps.py b/tests/bdd/steps/dfu_steps.py index b90ceb0..c5ec833 100644 --- a/tests/bdd/steps/dfu_steps.py +++ b/tests/bdd/steps/dfu_steps.py @@ -35,7 +35,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -from Queue import Empty +from queue import Empty import logging import os import time @@ -188,7 +188,7 @@ def program_image_ble(nrfjprog, full_image_path, snr): STDOUT_TEXT_WAIT_TIME = 50 # Number of seconds to wait for expected output from stdout -@given(u'the user wants to perform dfu {dfu_type}') +@given('the user wants to perform dfu {dfu_type}') def step_impl(context, dfu_type): runner = CliRunner() context.runner = runner @@ -196,17 +196,17 @@ def step_impl(context, dfu_type): context.args = args -@given(u'using package {package}') +@given('using package {package}') def step_impl(context, package): full_package_path = resolve_hex_path(package) context.args.extend(['-pkg', full_package_path]) context.pkg = full_package_path -@given(u'option {args}') +@given('option {args}') def step_impl(context, args): context.args.extend(args.split(" ")) -@given(u'-snr {device}') +@given('-snr {device}') def step_impl(context, device): assert device in os.environ, \ "Environment variable '{}' must be exported with device serial number".format(device) @@ -214,7 +214,7 @@ def step_impl(context, device): snr = str(os.environ[device]) context.args.extend(["-snr", snr]) -@given(u'nrfjprog {image} for {image_type} {board}') +@given('nrfjprog {image} for {image_type} {board}') def step_impl(context, image, image_type, board): full_image_path = resolve_hex_path(image) @@ -246,7 +246,7 @@ def step_impl(context, image, image_type, board): -@then(u'perform dfu') +@then('perform dfu') def step_impl(context): result = context.runner.invoke(cli, context.args) logger.debug("exit_code: %s, output: \'%s\'", result.exit_code, result.output) diff --git a/tests/bdd/steps/genpkg_generate_dfu_package_steps.py b/tests/bdd/steps/genpkg_generate_dfu_package_steps.py index b777dc9..bcc1c3a 100644 --- a/tests/bdd/steps/genpkg_generate_dfu_package_steps.py +++ b/tests/bdd/steps/genpkg_generate_dfu_package_steps.py @@ -48,25 +48,25 @@ logger = logging.getLogger(__file__) -@given(u'the user wants to generate a DFU package with application {application}, bootloader {bootloader} and SoftDevice {softdevice} with name {package}') +@given('the user wants to generate a DFU package with application {application}, bootloader {bootloader} and SoftDevice {softdevice} with name {package}') def step_impl(context, application, bootloader, softdevice, package): runner = CliRunner() context.runner = runner args = ['dfu', 'genpkg'] - if application != u'not_set': + if application != 'not_set': args.extend(['--application', os.path.join(get_resources_path(), application)]) context.application = application else: context.application = None - if bootloader != u'not_set': + if bootloader != 'not_set': args.extend(['--bootloader', os.path.join(get_resources_path(), bootloader)]) context.bootloader = bootloader else: context.bootloader = None - if softdevice != u'not_set': + if softdevice != 'not_set': args.extend(['--softdevice', os.path.join(get_resources_path(), softdevice)]) context.softdevice = softdevice else: @@ -77,52 +77,52 @@ def step_impl(context, application, bootloader, softdevice, package): context.args = args -@given(u'with option --application-version {app_ver}') +@given('with option --application-version {app_ver}') def step_impl(context, app_ver): context.application_version = None - if app_ver == u'not_set': + if app_ver == 'not_set': context.application_version = 0xFFFFFFFF - elif app_ver == u'none': + elif app_ver == 'none': context.args.extend(['--application-version', 'None']) else: context.args.extend(['--application-version', app_ver]) context.application_version = int_as_text_to_int(app_ver) -@given(u'with option --dev-revision {dev_rev}') +@given('with option --dev-revision {dev_rev}') def step_impl(context, dev_rev): context.dev_revision = None - if dev_rev == u'not_set': + if dev_rev == 'not_set': context.dev_revision = 0xFFFF - elif dev_rev == u'none': + elif dev_rev == 'none': context.args.extend(['--dev-revision', 'None']) else: context.args.extend(['--dev-revision', dev_rev]) context.dev_revision = int_as_text_to_int(dev_rev) -@given(u'with option --dev-type {dev_type}') +@given('with option --dev-type {dev_type}') def step_impl(context, dev_type): context.dev_type = None - if dev_type == u'not_set': + if dev_type == 'not_set': context.dev_type = 0xFFFF - elif dev_type == u'none': + elif dev_type == 'none': context.args.extend(['--dev-type', 'None']) else: context.args.extend(['--dev-type', dev_type]) context.dev_type = int_as_text_to_int(dev_type) -@given(u'with option --dfu-ver {dfu_ver}') +@given('with option --dfu-ver {dfu_ver}') def step_impl(context, dfu_ver): context.firmware_hash = None context.ext_packet_id = None context.init_packet_ecds = None - if dfu_ver == u'not_set': + if dfu_ver == 'not_set': context.dfu_ver = 0.5 context.ext_packet_id = 0 else: @@ -142,13 +142,13 @@ def step_impl(context, dfu_ver): context.dfu_ver = float(dfu_ver) -@given(u'with option --sd-req {sd_reqs}') +@given('with option --sd-req {sd_reqs}') def step_impl(context, sd_reqs): context.sd_req = None - if sd_reqs == u'not_set': + if sd_reqs == 'not_set': context.sd_req = [0xFFFE] - elif sd_reqs == u'none': + elif sd_reqs == 'none': context.args.extend(['--sd-req', 'None']) else: context.args.extend(['--sd-req', sd_reqs]) @@ -162,19 +162,19 @@ def step_impl(context, sd_reqs): context.sd_req = sd_reqs_value -@given(u'with option --key-file {pem_file}') +@given('with option --key-file {pem_file}') def step_impl(context, pem_file): - if pem_file != u'not_set': + if pem_file != 'not_set': context.args.extend(['--key-file', os.path.join(get_resources_path(), pem_file)]) context.dfu_ver = 0.8 -@when(u'user press enter') +@when('user press enter') def step_impl(context): pass -@then(u'the generated DFU package {package} contains correct data') +@then('the generated DFU package {package} contains correct data') def step_impl(context, package): with context.runner.isolated_filesystem(): pkg_full_name = os.path.join(os.getcwd(), package) diff --git a/tests/bdd/steps/help_information_steps.py b/tests/bdd/steps/help_information_steps.py index a34da48..6cedc8a 100644 --- a/tests/bdd/steps/help_information_steps.py +++ b/tests/bdd/steps/help_information_steps.py @@ -35,7 +35,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -from Queue import Empty +from queue import Empty import logging import os import time @@ -52,7 +52,7 @@ STDOUT_TEXT_WAIT_TIME = 50 # Number of seconds to wait for expected output from stdout -@given(u'user types \'{command}\'') +@given('user types \'{command}\'') def step_impl(context, command): args = command.split(' ') assert args[0] == 'nrfutil' @@ -64,7 +64,7 @@ def step_impl(context, command): context.args = exec_args -@then(u'output contains \'{stdout_text}\' and exit code is {exit_code}') +@then('output contains \'{stdout_text}\' and exit code is {exit_code}') def step_impl(context, stdout_text, exit_code): result = context.runner.invoke(cli, context.args) logger.debug("exit_code: %s, output: \'%s\'", result.exit_code, result.output) @@ -72,7 +72,7 @@ def step_impl(context, stdout_text, exit_code): assert result.output != None assert result.output.find(stdout_text) >= 0 -@then(u'output version is correct') +@then('output version is correct') def step_impl(context): assert "nrfutil_version" in os.environ, \ "Environment variable 'nrfutil_version' must be exported" diff --git a/tests/bdd/steps/util.py b/tests/bdd/steps/util.py index 3a67e9e..eb5eff4 100644 --- a/tests/bdd/steps/util.py +++ b/tests/bdd/steps/util.py @@ -79,7 +79,7 @@ def generate_options_table_for_cucumber(): number_of_optional_option_permutations *= int(math.pow(3, number_of_3_option_options)) number_of_optional_option_permutations *= int(math.pow(4, number_of_4_option_options)) - for x in xrange(0, number_of_optional_option_permutations): + for x in range(0, number_of_optional_option_permutations): retval += "{0:<8}".format(" ") retval += "| {0:<12}| {1:<29}| {2:<29}|".format("blinky.bin", "not_set", "not_set") @@ -144,7 +144,7 @@ def generate_options_table_for_cucumber(): if option == 2: sd_reqs = [] - for i in xrange(0, randint(1, 4)): + for i in range(0, randint(1, 4)): sd_reqs.append("0x{0:04x}".format(randint(0, 65535))) retval += " {0:<28}|".format(",".join(sd_reqs)) diff --git a/tests/protobuf/__main__.py b/tests/protobuf/__main__.py index 98932b4..74b03bb 100644 --- a/tests/protobuf/__main__.py +++ b/tests/protobuf/__main__.py @@ -20,7 +20,7 @@ def test_construct_from_empty_string(self): empty buffer """ - self.assertRaisesRegexp( + self.assertRaisesRegex( RuntimeError, "app_size is not set. It must be set when type is APPLICATION", InitPacketPB, diff --git a/tests/test_cli.py b/tests/test_cli.py index bef6134..d308e63 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -41,7 +41,7 @@ def test_dfu_ble_address(self): address = 'AABBCC11223' result = self.runner.invoke(self.cli, argumentList + [address]) self.assertTrue('Invalid address' in result.output) - print(result.exception) + print((result.exception)) self.assertIsNone(result.exception) address = 'AABBCC1122334' From 448e437c78453adaaee924b3179a796e799cf9d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ken=20A=2E=20Rederg=C3=A5rd?= <64542+kenr@users.noreply.github.com> Date: Thu, 20 Jun 2019 15:16:22 +0200 Subject: [PATCH 02/34] Use intelhex pkg, fix integer division --- nordicsemi/dfu/intelhex/__init__.py | 1288 --------------------------- nordicsemi/dfu/intelhex/compat.py | 57 -- nordicsemi/dfu/nrfhex.py | 4 +- 3 files changed, 2 insertions(+), 1347 deletions(-) delete mode 100644 nordicsemi/dfu/intelhex/__init__.py delete mode 100644 nordicsemi/dfu/intelhex/compat.py diff --git a/nordicsemi/dfu/intelhex/__init__.py b/nordicsemi/dfu/intelhex/__init__.py deleted file mode 100644 index b0cfcda..0000000 --- a/nordicsemi/dfu/intelhex/__init__.py +++ /dev/null @@ -1,1288 +0,0 @@ -# Copyright (c) 2005-2013, Alexander Belchenko -# All rights reserved. -# -# Redistribution and use in source and binary forms, -# with or without modification, are permitted provided -# that the following conditions are met: -# -# * Redistributions of source code must retain -# the above copyright notice, this list of conditions -# and the following disclaimer. -# * Redistributions in binary form must reproduce -# the above copyright notice, this list of conditions -# and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# * Neither the name of the author nor the names -# of its contributors may be used to endorse -# or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, -# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, -# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED -# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -'''Intel HEX file format reader and converter. - -@author Alexander Belchenko (alexander dot belchenko at gmail dot com) -@version 1.5 -''' - - - - -__docformat__ = "javadoc" - - -from array import array -from binascii import hexlify, unhexlify -from bisect import bisect_right -import os -import sys - -from .compat import asbytes, asstr - - -class _DeprecatedParam(object): - pass - -_DEPRECATED = _DeprecatedParam() - - -class IntelHex(object): - ''' Intel HEX file reader. ''' - - def __init__(self, source=None): - ''' Constructor. If source specified, object will be initialized - with the contents of source. Otherwise the object will be empty. - - @param source source for initialization - (file name of HEX file, file object, addr dict or - other IntelHex object) - ''' - # public members - self.padding = 0x0FF - # Start Address - self.start_addr = None - - # private members - self._buf = {} - self._offset = 0 - - if source is not None: - if isinstance(source, str) or getattr(source, "read", None): - # load hex file - self.loadhex(source) - elif isinstance(source, dict): - self.fromdict(source) - elif isinstance(source, IntelHex): - self.padding = source.padding - if source.start_addr: - self.start_addr = source.start_addr.copy() - self._buf = source._buf.copy() - else: - raise ValueError("source: bad initializer type") - - def _decode_record(self, s, line=0): - '''Decode one record of HEX file. - - @param s line with HEX record. - @param line line number (for error messages). - - @raise EndOfFile if EOF record encountered. - ''' - s = s.rstrip('\r\n') - if not s: - return # empty line - - if s[0] == ':': - try: - bin = array('B', unhexlify(asbytes(s[1:]))) - except (TypeError, ValueError): - # this might be raised by unhexlify when odd hexascii digits - raise HexRecordError(line=line) - length = len(bin) - if length < 5: - raise HexRecordError(line=line) - else: - raise HexRecordError(line=line) - - record_length = bin[0] - if length != (5 + record_length): - raise RecordLengthError(line=line) - - addr = bin[1]*256 + bin[2] - - record_type = bin[3] - if not (0 <= record_type <= 5): - raise RecordTypeError(line=line) - - crc = sum(bin) - crc &= 0x0FF - if crc != 0: - raise RecordChecksumError(line=line) - - if record_type == 0: - # data record - addr += self._offset - for i in range(4, 4+record_length): - if not self._buf.get(addr, None) is None: - raise AddressOverlapError(address=addr, line=line) - self._buf[addr] = bin[i] - addr += 1 # FIXME: addr should be wrapped - # BUT after 02 record (at 64K boundary) - # and after 04 record (at 4G boundary) - - elif record_type == 1: - # end of file record - if record_length != 0: - raise EOFRecordError(line=line) - raise _EndOfFile - - elif record_type == 2: - # Extended 8086 Segment Record - if record_length != 2 or addr != 0: - raise ExtendedSegmentAddressRecordError(line=line) - self._offset = (bin[4]*256 + bin[5]) * 16 - - elif record_type == 4: - # Extended Linear Address Record - if record_length != 2 or addr != 0: - raise ExtendedLinearAddressRecordError(line=line) - self._offset = (bin[4]*256 + bin[5]) * 65536 - - elif record_type == 3: - # Start Segment Address Record - if record_length != 4 or addr != 0: - raise StartSegmentAddressRecordError(line=line) - if self.start_addr: - raise DuplicateStartAddressRecordError(line=line) - self.start_addr = {'CS': bin[4]*256 + bin[5], - 'IP': bin[6]*256 + bin[7], - } - - elif record_type == 5: - # Start Linear Address Record - if record_length != 4 or addr != 0: - raise StartLinearAddressRecordError(line=line) - if self.start_addr: - raise DuplicateStartAddressRecordError(line=line) - self.start_addr = {'EIP': (bin[4]*16777216 + - bin[5]*65536 + - bin[6]*256 + - bin[7]), - } - - def loadhex(self, fobj): - """Load hex file into internal buffer. This is not necessary - if object was initialized with source set. This will overwrite - addresses if object was already initialized. - - @param fobj file name or file-like object - """ - if getattr(fobj, "read", None) is None: - fobj = open(fobj, "r") - fclose = fobj.close - else: - fclose = None - - self._offset = 0 - line = 0 - - try: - decode = self._decode_record - try: - for s in fobj: - line += 1 - decode(s, line) - except _EndOfFile: - pass - finally: - if fclose: - fclose() - - def loadbin(self, fobj, offset=0): - """Load bin file into internal buffer. Not needed if source set in - constructor. This will overwrite addresses without warning - if object was already initialized. - - @param fobj file name or file-like object - @param offset starting address offset - """ - fread = getattr(fobj, "read", None) - if fread is None: - f = open(fobj, "rb") - fread = f.read - fclose = f.close - else: - fclose = None - - try: - self.frombytes(array('B', asbytes(fread())), offset=offset) - finally: - if fclose: - fclose() - - def loadfile(self, fobj, format): - """Load data file into internal buffer. Preferred wrapper over - loadbin or loadhex. - - @param fobj file name or file-like object - @param format file format ("hex" or "bin") - """ - if format == "hex": - self.loadhex(fobj) - elif format == "bin": - self.loadbin(fobj) - else: - raise ValueError('format should be either "hex" or "bin";' - ' got %r instead' % format) - - # alias (to be consistent with method tofile) - fromfile = loadfile - - def fromdict(self, dikt): - """Load data from dictionary. Dictionary should contain int keys - representing addresses. Values should be the data to be stored in - those addresses in unsigned char form (i.e. not strings). - The dictionary may contain the key, ``start_addr`` - to indicate the starting address of the data as described in README. - - The contents of the dict will be merged with this object and will - overwrite any conflicts. This function is not necessary if the - object was initialized with source specified. - """ - s = dikt.copy() - start_addr = s.get('start_addr') - if start_addr is not None: - del s['start_addr'] - for k in list(s.keys()): - if type(k) not in (int, int) or k < 0: - raise ValueError('Source dictionary should have only int keys') - self._buf.update(s) - if start_addr is not None: - self.start_addr = start_addr - - def frombytes(self, bytes, offset=0): - """Load data from array or list of bytes. - Similar to loadbin() method but works directly with iterable bytes. - """ - for b in bytes: - self._buf[offset] = b - offset += 1 - - def _get_start_end(self, start=None, end=None, size=None): - """Return default values for start and end if they are None. - If this IntelHex object is empty then it's error to - invoke this method with both start and end as None. - """ - if (start,end) == (None,None) and self._buf == {}: - raise EmptyIntelHexError - if size is not None: - if None not in (start, end): - raise ValueError("tobinarray: you can't use start,end and size" - " arguments in the same time") - if (start, end) == (None, None): - start = self.minaddr() - if start is not None: - end = start + size - 1 - else: - start = end - size + 1 - if start < 0: - raise ValueError("tobinarray: invalid size (%d) " - "for given end address (%d)" % (size,end)) - else: - if start is None: - start = self.minaddr() - if end is None: - end = self.maxaddr() - if start > end: - start, end = end, start - return start, end - - def tobinarray(self, start=None, end=None, pad=_DEPRECATED, size=None): - ''' Convert this object to binary form as array. If start and end - unspecified, they will be inferred from the data. - @param start start address of output bytes. - @param end end address of output bytes (inclusive). - @param pad [DEPRECATED PARAMETER, please use self.padding instead] - fill empty spaces with this value - (if pad is None then this method uses self.padding). - @param size size of the block, used with start or end parameter. - @return array of unsigned char data. - ''' - if not isinstance(pad, _DeprecatedParam): - print("IntelHex.tobinarray: 'pad' parameter is deprecated.") - if pad is not None: - print("Please, use IntelHex.padding attribute instead.") - else: - print("Please, don't pass it explicitly.") - print("Use syntax like this: ih.tobinarray(start=xxx, end=yyy, size=zzz)") - else: - pad = None - return self._tobinarray_really(start, end, pad, size) - - def _tobinarray_really(self, start, end, pad, size): - if pad is None: - pad = self.padding - - bin = array('B') - - if self._buf == {} and None in (start, end): - return bin - - if size is not None and size <= 0: - raise ValueError("tobinarray: wrong value for size") - - start, end = self._get_start_end(start, end, size) - - for i in range(start, end+1): - bin.append(self._buf.get(i, pad)) - - return bin - - def tobinstr(self, start=None, end=None, pad=_DEPRECATED, size=None): - ''' Convert to binary form and return as a string. - @param start start address of output bytes. - @param end end address of output bytes (inclusive). - @param pad [DEPRECATED PARAMETER, please use self.padding instead] - fill empty spaces with this value - (if pad is None then this method uses self.padding). - @param size size of the block, used with start or end parameter. - @return string of binary data. - ''' - if not isinstance(pad, _DeprecatedParam): - print("IntelHex.tobinstr: 'pad' parameter is deprecated.") - if pad is not None: - print("Please, use IntelHex.padding attribute instead.") - else: - print("Please, don't pass it explicitly.") - print("Use syntax like this: ih.tobinstr(start=xxx, end=yyy, size=zzz)") - else: - pad = None - return self._tobinstr_really(start, end, pad, size) - - def _tobinstr_really(self, start, end, pad, size): - return asstr(self._tobinarray_really(start, end, pad, size).tostring()) - - def tobinfile(self, fobj, start=None, end=None, pad=_DEPRECATED, size=None): - '''Convert to binary and write to file. - - @param fobj file name or file object for writing output bytes. - @param start start address of output bytes. - @param end end address of output bytes (inclusive). - @param pad [DEPRECATED PARAMETER, please use self.padding instead] - fill empty spaces with this value - (if pad is None then this method uses self.padding). - @param size size of the block, used with start or end parameter. - ''' - if not isinstance(pad, _DeprecatedParam): - print("IntelHex.tobinfile: 'pad' parameter is deprecated.") - if pad is not None: - print("Please, use IntelHex.padding attribute instead.") - else: - print("Please, don't pass it explicitly.") - print("Use syntax like this: ih.tobinfile(start=xxx, end=yyy, size=zzz)") - else: - pad = None - if getattr(fobj, "write", None) is None: - fobj = open(fobj, "wb") - close_fd = True - else: - close_fd = False - - fobj.write(self._tobinstr_really(start, end, pad, size)) - - if close_fd: - fobj.close() - - def todict(self): - '''Convert to python dictionary. - - @return dict suitable for initializing another IntelHex object. - ''' - r = {} - r.update(self._buf) - if self.start_addr: - r['start_addr'] = self.start_addr - return r - - def addresses(self): - '''Returns all used addresses in sorted order. - @return list of occupied data addresses in sorted order. - ''' - aa = list(self._buf.keys()) - aa.sort() - return aa - - def minaddr(self): - '''Get minimal address of HEX content. - @return minimal address or None if no data - ''' - aa = list(self._buf.keys()) - if aa == []: - return None - else: - return min(aa) - - def maxaddr(self): - '''Get maximal address of HEX content. - @return maximal address or None if no data - ''' - aa = list(self._buf.keys()) - if aa == []: - return None - else: - return max(aa) - - def __getitem__(self, addr): - ''' Get requested byte from address. - @param addr address of byte. - @return byte if address exists in HEX file, or self.padding - if no data found. - ''' - t = type(addr) - if t in (int, int): - if addr < 0: - raise TypeError('Address should be >= 0.') - return self._buf.get(addr, self.padding) - elif t == slice: - addresses = list(self._buf.keys()) - ih = IntelHex() - if addresses: - addresses.sort() - start = addr.start or addresses[0] - stop = addr.stop or (addresses[-1]+1) - step = addr.step or 1 - for i in range(start, stop, step): - x = self._buf.get(i) - if x is not None: - ih[i] = x - return ih - else: - raise TypeError('Address has unsupported type: %s' % t) - - def __setitem__(self, addr, byte): - """Set byte at address.""" - t = type(addr) - if t in (int, int): - if addr < 0: - raise TypeError('Address should be >= 0.') - self._buf[addr] = byte - elif t == slice: - if not isinstance(byte, (list, tuple)): - raise ValueError('Slice operation expects sequence of bytes') - start = addr.start - stop = addr.stop - step = addr.step or 1 - if None not in (start, stop): - ra = list(range(start, stop, step)) - if len(ra) != len(byte): - raise ValueError('Length of bytes sequence does not match ' - 'address range') - elif (start, stop) == (None, None): - raise TypeError('Unsupported address range') - elif start is None: - start = stop - len(byte) - elif stop is None: - stop = start + len(byte) - if start < 0: - raise TypeError('start address cannot be negative') - if stop < 0: - raise TypeError('stop address cannot be negative') - j = 0 - for i in range(start, stop, step): - self._buf[i] = byte[j] - j += 1 - else: - raise TypeError('Address has unsupported type: %s' % t) - - def __delitem__(self, addr): - """Delete byte at address.""" - t = type(addr) - if t in (int, int): - if addr < 0: - raise TypeError('Address should be >= 0.') - del self._buf[addr] - elif t == slice: - addresses = list(self._buf.keys()) - if addresses: - addresses.sort() - start = addr.start or addresses[0] - stop = addr.stop or (addresses[-1]+1) - step = addr.step or 1 - for i in range(start, stop, step): - x = self._buf.get(i) - if x is not None: - del self._buf[i] - else: - raise TypeError('Address has unsupported type: %s' % t) - - def __len__(self): - """Return count of bytes with real values.""" - return len(list(self._buf.keys())) - - def write_hex_file(self, f, write_start_addr=True): - """Write data to file f in HEX format. - - @param f filename or file-like object for writing - @param write_start_addr enable or disable writing start address - record to file (enabled by default). - If there is no start address in obj, nothing - will be written regardless of this setting. - """ - fwrite = getattr(f, "write", None) - if fwrite: - fobj = f - fclose = None - else: - fobj = open(f, 'w') - fwrite = fobj.write - fclose = fobj.close - - # Translation table for uppercasing hex ascii string. - # timeit shows that using hexstr.translate(table) - # is faster than hexstr.upper(): - # 0.452ms vs. 0.652ms (translate vs. upper) - if sys.version_info[0] >= 3: - table = bytes(list(range(256))).upper() - else: - table = ''.join(chr(i).upper() for i in range(256)) - - - - # start address record if any - if self.start_addr and write_start_addr: - keys = list(self.start_addr.keys()) - keys.sort() - bin = array('B', asbytes('\0'*9)) - if keys == ['CS','IP']: - # Start Segment Address Record - bin[0] = 4 # reclen - bin[1] = 0 # offset msb - bin[2] = 0 # offset lsb - bin[3] = 3 # rectyp - cs = self.start_addr['CS'] - bin[4] = (cs >> 8) & 0x0FF - bin[5] = cs & 0x0FF - ip = self.start_addr['IP'] - bin[6] = (ip >> 8) & 0x0FF - bin[7] = ip & 0x0FF - bin[8] = (-sum(bin)) & 0x0FF # chksum - fwrite(':' + - asstr(hexlify(bin.tostring()).translate(table)) + - '\n') - elif keys == ['EIP']: - # Start Linear Address Record - bin[0] = 4 # reclen - bin[1] = 0 # offset msb - bin[2] = 0 # offset lsb - bin[3] = 5 # rectyp - eip = self.start_addr['EIP'] - bin[4] = (eip >> 24) & 0x0FF - bin[5] = (eip >> 16) & 0x0FF - bin[6] = (eip >> 8) & 0x0FF - bin[7] = eip & 0x0FF - bin[8] = (-sum(bin)) & 0x0FF # chksum - fwrite(':' + - asstr(hexlify(bin.tostring()).translate(table)) + - '\n') - else: - if fclose: - fclose() - raise InvalidStartAddressValueError(start_addr=self.start_addr) - - # data - addresses = list(self._buf.keys()) - addresses.sort() - addr_len = len(addresses) - if addr_len: - minaddr = addresses[0] - maxaddr = addresses[-1] - - if maxaddr > 65535: - need_offset_record = True - else: - need_offset_record = False - high_ofs = 0 - - cur_addr = minaddr - cur_ix = 0 - - while cur_addr <= maxaddr: - if need_offset_record: - bin = array('B', asbytes('\0'*7)) - bin[0] = 2 # reclen - bin[1] = 0 # offset msb - bin[2] = 0 # offset lsb - bin[3] = 4 # rectyp - high_ofs = int(cur_addr>>16) - b = divmod(high_ofs, 256) - bin[4] = b[0] # msb of high_ofs - bin[5] = b[1] # lsb of high_ofs - bin[6] = (-sum(bin)) & 0x0FF # chksum - fwrite(':' + - asstr(hexlify(bin.tostring()).translate(table)) + - '\n') - - while True: - # produce one record - low_addr = cur_addr & 0x0FFFF - # chain_len off by 1 - chain_len = min(15, 65535-low_addr, maxaddr-cur_addr) - - # search continuous chain - stop_addr = cur_addr + chain_len - if chain_len: - ix = bisect_right(addresses, stop_addr, - cur_ix, - min(cur_ix+chain_len+1, addr_len)) - chain_len = ix - cur_ix # real chain_len - # there could be small holes in the chain - # but we will catch them by try-except later - # so for big continuous files we will work - # at maximum possible speed - else: - chain_len = 1 # real chain_len - - bin = array('B', asbytes('\0'*(5+chain_len))) - b = divmod(low_addr, 256) - bin[1] = b[0] # msb of low_addr - bin[2] = b[1] # lsb of low_addr - bin[3] = 0 # rectype - try: # if there is small holes we'll catch them - for i in range(chain_len): - bin[4+i] = self._buf[cur_addr+i] - except KeyError: - # we catch a hole so we should shrink the chain - chain_len = i - bin = bin[:5+i] - bin[0] = chain_len - bin[4+chain_len] = (-sum(bin)) & 0x0FF # chksum - fwrite(':' + - asstr(hexlify(bin.tostring()).translate(table)) + - '\n') - - # adjust cur_addr/cur_ix - cur_ix += chain_len - if cur_ix < addr_len: - cur_addr = addresses[cur_ix] - else: - cur_addr = maxaddr + 1 - break - high_addr = int(cur_addr>>16) - if high_addr > high_ofs: - break - - # end-of-file record - fwrite(":00000001FF\n") - if fclose: - fclose() - - def tofile(self, fobj, format): - """Write data to hex or bin file. Preferred method over tobin or tohex. - - @param fobj file name or file-like object - @param format file format ("hex" or "bin") - """ - if format == 'hex': - self.write_hex_file(fobj) - elif format == 'bin': - self.tobinfile(fobj) - else: - raise ValueError('format should be either "hex" or "bin";' - ' got %r instead' % format) - - def gets(self, addr, length): - """Get string of bytes from given address. If any entries are blank - from addr through addr+length, a NotEnoughDataError exception will - be raised. Padding is not used.""" - a = array('B', asbytes('\0'*length)) - try: - for i in range(length): - a[i] = self._buf[addr+i] - except KeyError: - raise NotEnoughDataError(address=addr, length=length) - return asstr(a.tostring()) - - def puts(self, addr, s): - """Put string of bytes at given address. Will overwrite any previous - entries. - """ - a = array('B', asbytes(s)) - for i in range(len(a)): - self._buf[addr+i] = a[i] - - def getsz(self, addr): - """Get zero-terminated string from given address. Will raise - NotEnoughDataError exception if a hole is encountered before a 0. - """ - i = 0 - try: - while True: - if self._buf[addr+i] == 0: - break - i += 1 - except KeyError: - raise NotEnoughDataError(msg=('Bad access at 0x%X: ' - 'not enough data to read zero-terminated string') % addr) - return self.gets(addr, i) - - def putsz(self, addr, s): - """Put string in object at addr and append terminating zero at end.""" - self.puts(addr, s) - self._buf[addr+len(s)] = 0 - - def dump(self, tofile=None): - """Dump object content to specified file object or to stdout if None. - Format is a hexdump with some header information at the beginning, - addresses on the left, and data on right. - - @param tofile file-like object to dump to - """ - - if tofile is None: - tofile = sys.stdout - # start addr possibly - if self.start_addr is not None: - cs = self.start_addr.get('CS') - ip = self.start_addr.get('IP') - eip = self.start_addr.get('EIP') - if eip is not None and cs is None and ip is None: - tofile.write('EIP = 0x%08X\n' % eip) - elif eip is None and cs is not None and ip is not None: - tofile.write('CS = 0x%04X, IP = 0x%04X\n' % (cs, ip)) - else: - tofile.write('start_addr = %r\n' % start_addr) - # actual data - addresses = list(self._buf.keys()) - if addresses: - addresses.sort() - minaddr = addresses[0] - maxaddr = addresses[-1] - startaddr = int(minaddr>>4)*16 - endaddr = int((maxaddr>>4)+1)*16 - maxdigits = max(len(str(endaddr)), 4) - templa = '%%0%dX' % maxdigits - range16 = list(range(16)) - for i in range(startaddr, endaddr, 16): - tofile.write(templa % i) - tofile.write(' ') - s = [] - for j in range16: - x = self._buf.get(i+j) - if x is not None: - tofile.write(' %02X' % x) - if 32 <= x < 127: # GNU less does not like 0x7F (128 decimal) so we'd better show it as dot - s.append(chr(x)) - else: - s.append('.') - else: - tofile.write(' --') - s.append(' ') - tofile.write(' |' + ''.join(s) + '|\n') - - def merge(self, other, overlap='error'): - """Merge content of other IntelHex object into current object (self). - @param other other IntelHex object. - @param overlap action on overlap of data or starting addr: - - error: raising OverlapError; - - ignore: ignore other data and keep current data - in overlapping region; - - replace: replace data with other data - in overlapping region. - - @raise TypeError if other is not instance of IntelHex - @raise ValueError if other is the same object as self - (it can't merge itself) - @raise ValueError if overlap argument has incorrect value - @raise AddressOverlapError on overlapped data - """ - # check args - if not isinstance(other, IntelHex): - raise TypeError('other should be IntelHex object') - if other is self: - raise ValueError("Can't merge itself") - if overlap not in ('error', 'ignore', 'replace'): - raise ValueError("overlap argument should be either " - "'error', 'ignore' or 'replace'") - # merge data - this_buf = self._buf - other_buf = other._buf - for i in other_buf: - if i in this_buf: - if overlap == 'error': - raise AddressOverlapError( - 'Data overlapped at address 0x%X' % i) - elif overlap == 'ignore': - continue - this_buf[i] = other_buf[i] - # merge start_addr - if self.start_addr != other.start_addr: - if self.start_addr is None: # set start addr from other - self.start_addr = other.start_addr - elif other.start_addr is None: # keep existing start addr - pass - else: # conflict - if overlap == 'error': - raise AddressOverlapError( - 'Starting addresses are different') - elif overlap == 'replace': - self.start_addr = other.start_addr -#/IntelHex - - -class IntelHex16bit(IntelHex): - """Access to data as 16-bit words. Intended to use with Microchip HEX files.""" - - def __init__(self, source=None): - """Construct class from HEX file - or from instance of ordinary IntelHex class. If IntelHex object - is passed as source, the original IntelHex object should not be used - again because this class will alter it. This class leaves padding - alone unless it was precisely 0xFF. In that instance it is sign - extended to 0xFFFF. - - @param source file name of HEX file or file object - or instance of ordinary IntelHex class. - Will also accept dictionary from todict method. - """ - if isinstance(source, IntelHex): - # from ihex8 - self.padding = source.padding - self.start_addr = source.start_addr - # private members - self._buf = source._buf - self._offset = source._offset - elif isinstance(source, dict): - raise IntelHexError("IntelHex16bit does not support initialization from dictionary yet.\n" - "Patches are welcome.") - else: - IntelHex.__init__(self, source) - - if self.padding == 0x0FF: - self.padding = 0x0FFFF - - def __getitem__(self, addr16): - """Get 16-bit word from address. - Raise error if only one byte from the pair is set. - We assume a Little Endian interpretation of the hex file. - - @param addr16 address of word (addr8 = 2 * addr16). - @return word if bytes exists in HEX file, or self.padding - if no data found. - """ - addr1 = addr16 * 2 - addr2 = addr1 + 1 - byte1 = self._buf.get(addr1, None) - byte2 = self._buf.get(addr2, None) - - if byte1 != None and byte2 != None: - return byte1 | (byte2 << 8) # low endian - - if byte1 == None and byte2 == None: - return self.padding - - raise BadAccess16bit(address=addr16) - - def __setitem__(self, addr16, word): - """Sets the address at addr16 to word assuming Little Endian mode. - """ - addr_byte = addr16 * 2 - b = divmod(word, 256) - self._buf[addr_byte] = b[1] - self._buf[addr_byte+1] = b[0] - - def minaddr(self): - '''Get minimal address of HEX content in 16-bit mode. - - @return minimal address used in this object - ''' - aa = list(self._buf.keys()) - if aa == []: - return 0 - else: - return min(aa)>>1 - - def maxaddr(self): - '''Get maximal address of HEX content in 16-bit mode. - - @return maximal address used in this object - ''' - aa = list(self._buf.keys()) - if aa == []: - return 0 - else: - return max(aa)>>1 - - def tobinarray(self, start=None, end=None, size=None): - '''Convert this object to binary form as array (of 2-bytes word data). - If start and end unspecified, they will be inferred from the data. - @param start start address of output data. - @param end end address of output data (inclusive). - @param size size of the block (number of words), - used with start or end parameter. - @return array of unsigned short (uint16_t) data. - ''' - bin = array('H') - - if self._buf == {} and None in (start, end): - return bin - - if size is not None and size <= 0: - raise ValueError("tobinarray: wrong value for size") - - start, end = self._get_start_end(start, end, size) - - for addr in range(start, end+1): - bin.append(self[addr]) - - return bin - - -#/class IntelHex16bit - - -def hex2bin(fin, fout, start=None, end=None, size=None, pad=None): - """Hex-to-Bin convertor engine. - @return 0 if all OK - - @param fin input hex file (filename or file-like object) - @param fout output bin file (filename or file-like object) - @param start start of address range (optional) - @param end end of address range (inclusive; optional) - @param size size of resulting file (in bytes) (optional) - @param pad padding byte (optional) - """ - try: - h = IntelHex(fin) - except HexReaderError as e: - txt = "ERROR: bad HEX file: %s" % str(e) - print(txt) - return 1 - - # start, end, size - if size != None and size != 0: - if end == None: - if start == None: - start = h.minaddr() - end = start + size - 1 - else: - if (end+1) >= size: - start = end + 1 - size - else: - start = 0 - - try: - if pad is not None: - # using .padding attribute rather than pad argument to function call - h.padding = pad - h.tobinfile(fout, start, end) - except IOError as e: - txt = "ERROR: Could not write to file: %s: %s" % (fout, str(e)) - print(txt) - return 1 - - return 0 -#/def hex2bin - - -def bin2hex(fin, fout, offset=0): - """Simple bin-to-hex convertor. - @return 0 if all OK - - @param fin input bin file (filename or file-like object) - @param fout output hex file (filename or file-like object) - @param offset starting address offset for loading bin - """ - h = IntelHex() - try: - h.loadbin(fin, offset) - except IOError as e: - txt = 'ERROR: unable to load bin file:', str(e) - print(txt) - return 1 - - try: - h.tofile(fout, format='hex') - except IOError as e: - txt = "ERROR: Could not write to file: %s: %s" % (fout, str(e)) - print(txt) - return 1 - - return 0 -#/def bin2hex - - -def diff_dumps(ih1, ih2, tofile=None, name1="a", name2="b", n_context=3): - """Diff 2 IntelHex objects and produce unified diff output for their - hex dumps. - - @param ih1 first IntelHex object to compare - @param ih2 second IntelHex object to compare - @param tofile file-like object to write output - @param name1 name of the first hex file to show in the diff header - @param name2 name of the first hex file to show in the diff header - @param n_context number of context lines in the unidiff output - """ - def prepare_lines(ih): - from io import StringIO - sio = StringIO() - ih.dump(sio) - dump = sio.getvalue() - lines = dump.splitlines() - return lines - a = prepare_lines(ih1) - b = prepare_lines(ih2) - import difflib - result = list(difflib.unified_diff(a, b, fromfile=name1, tofile=name2, n=n_context, lineterm='')) - if tofile is None: - tofile = sys.stdout - output = '\n'.join(result)+'\n' - tofile.write(output) - - -class Record(object): - """Helper methods to build valid ihex records.""" - - def _from_bytes(bytes): - """Takes a list of bytes, computes the checksum, and outputs the entire - record as a string. bytes should be the hex record without the colon - or final checksum. - - @param bytes list of byte values so far to pack into record. - @return String representation of one HEX record - """ - assert len(bytes) >= 4 - # calculate checksum - s = (-sum(bytes)) & 0x0FF - bin = array('B', bytes + [s]) - return ':' + asstr(hexlify(bin.tostring())).upper() - _from_bytes = staticmethod(_from_bytes) - - def data(offset, bytes): - """Return Data record. This constructs the full record, including - the length information, the record type (0x00), the - checksum, and the offset. - - @param offset load offset of first byte. - @param bytes list of byte values to pack into record. - - @return String representation of one HEX record - """ - assert 0 <= offset < 65536 - assert 0 < len(bytes) < 256 - b = [len(bytes), (offset>>8)&0x0FF, offset&0x0FF, 0x00] + bytes - return Record._from_bytes(b) - data = staticmethod(data) - - def eof(): - """Return End of File record as a string. - @return String representation of Intel Hex EOF record - """ - return ':00000001FF' - eof = staticmethod(eof) - - def extended_segment_address(usba): - """Return Extended Segment Address Record. - @param usba Upper Segment Base Address. - - @return String representation of Intel Hex USBA record. - """ - b = [2, 0, 0, 0x02, (usba>>8)&0x0FF, usba&0x0FF] - return Record._from_bytes(b) - extended_segment_address = staticmethod(extended_segment_address) - - def start_segment_address(cs, ip): - """Return Start Segment Address Record. - @param cs 16-bit value for CS register. - @param ip 16-bit value for IP register. - - @return String representation of Intel Hex SSA record. - """ - b = [4, 0, 0, 0x03, (cs>>8)&0x0FF, cs&0x0FF, - (ip>>8)&0x0FF, ip&0x0FF] - return Record._from_bytes(b) - start_segment_address = staticmethod(start_segment_address) - - def extended_linear_address(ulba): - """Return Extended Linear Address Record. - @param ulba Upper Linear Base Address. - - @return String representation of Intel Hex ELA record. - """ - b = [2, 0, 0, 0x04, (ulba>>8)&0x0FF, ulba&0x0FF] - return Record._from_bytes(b) - extended_linear_address = staticmethod(extended_linear_address) - - def start_linear_address(eip): - """Return Start Linear Address Record. - @param eip 32-bit linear address for the EIP register. - - @return String representation of Intel Hex SLA record. - """ - b = [4, 0, 0, 0x05, (eip>>24)&0x0FF, (eip>>16)&0x0FF, - (eip>>8)&0x0FF, eip&0x0FF] - return Record._from_bytes(b) - start_linear_address = staticmethod(start_linear_address) - - -class _BadFileNotation(Exception): - """Special error class to use with _get_file_and_addr_range.""" - pass - -def _get_file_and_addr_range(s, _support_drive_letter=None): - """Special method for hexmerge.py script to split file notation - into 3 parts: (filename, start, end) - - @raise _BadFileNotation when string cannot be safely split. - """ - if _support_drive_letter is None: - _support_drive_letter = (os.name == 'nt') - drive = '' - if _support_drive_letter: - if s[1:2] == ':' and s[0].upper() in ''.join([chr(i) for i in range(ord('A'), ord('Z')+1)]): - drive = s[:2] - s = s[2:] - parts = s.split(':') - n = len(parts) - if n == 1: - fname = parts[0] - fstart = None - fend = None - elif n != 3: - raise _BadFileNotation - else: - fname = parts[0] - def ascii_hex_to_int(ascii): - if ascii is not None: - try: - return int(ascii, 16) - except ValueError: - raise _BadFileNotation - return ascii - fstart = ascii_hex_to_int(parts[1] or None) - fend = ascii_hex_to_int(parts[2] or None) - return drive+fname, fstart, fend - - -## -# IntelHex Errors Hierarchy: -# -# IntelHexError - basic error -# HexReaderError - general hex reader error -# AddressOverlapError - data for the same address overlap -# HexRecordError - hex record decoder base error -# RecordLengthError - record has invalid length -# RecordTypeError - record has invalid type (RECTYP) -# RecordChecksumError - record checksum mismatch -# EOFRecordError - invalid EOF record (type 01) -# ExtendedAddressRecordError - extended address record base error -# ExtendedSegmentAddressRecordError - invalid extended segment address record (type 02) -# ExtendedLinearAddressRecordError - invalid extended linear address record (type 04) -# StartAddressRecordError - start address record base error -# StartSegmentAddressRecordError - invalid start segment address record (type 03) -# StartLinearAddressRecordError - invalid start linear address record (type 05) -# DuplicateStartAddressRecordError - start address record appears twice -# InvalidStartAddressValueError - invalid value of start addr record -# _EndOfFile - it's not real error, used internally by hex reader as signal that EOF record found -# BadAccess16bit - not enough data to read 16 bit value (deprecated, see NotEnoughDataError) -# NotEnoughDataError - not enough data to read N contiguous bytes -# EmptyIntelHexError - requested operation cannot be performed with empty object - -class IntelHexError(Exception): - '''Base Exception class for IntelHex module''' - - _fmt = 'IntelHex base error' #: format string - - def __init__(self, msg=None, **kw): - """Initialize the Exception with the given message. - """ - self.msg = msg - for key, value in list(kw.items()): - setattr(self, key, value) - - def __str__(self): - """Return the message in this Exception.""" - if self.msg: - return self.msg - try: - return self._fmt % self.__dict__ - except (NameError, ValueError, KeyError) as e: - return 'Unprintable exception %s: %s' \ - % (repr(e), str(e)) - -class _EndOfFile(IntelHexError): - """Used for internal needs only.""" - _fmt = 'EOF record reached -- signal to stop read file' - -class HexReaderError(IntelHexError): - _fmt = 'Hex reader base error' - -class AddressOverlapError(HexReaderError): - _fmt = 'Hex file has data overlap at address 0x%(address)X on line %(line)d' - -# class NotAHexFileError was removed in trunk.revno.54 because it's not used - - -class HexRecordError(HexReaderError): - _fmt = 'Hex file contains invalid record at line %(line)d' - - -class RecordLengthError(HexRecordError): - _fmt = 'Record at line %(line)d has invalid length' - -class RecordTypeError(HexRecordError): - _fmt = 'Record at line %(line)d has invalid record type' - -class RecordChecksumError(HexRecordError): - _fmt = 'Record at line %(line)d has invalid checksum' - -class EOFRecordError(HexRecordError): - _fmt = 'File has invalid End-of-File record' - - -class ExtendedAddressRecordError(HexRecordError): - _fmt = 'Base class for extended address exceptions' - -class ExtendedSegmentAddressRecordError(ExtendedAddressRecordError): - _fmt = 'Invalid Extended Segment Address Record at line %(line)d' - -class ExtendedLinearAddressRecordError(ExtendedAddressRecordError): - _fmt = 'Invalid Extended Linear Address Record at line %(line)d' - - -class StartAddressRecordError(HexRecordError): - _fmt = 'Base class for start address exceptions' - -class StartSegmentAddressRecordError(StartAddressRecordError): - _fmt = 'Invalid Start Segment Address Record at line %(line)d' - -class StartLinearAddressRecordError(StartAddressRecordError): - _fmt = 'Invalid Start Linear Address Record at line %(line)d' - -class DuplicateStartAddressRecordError(StartAddressRecordError): - _fmt = 'Start Address Record appears twice at line %(line)d' - -class InvalidStartAddressValueError(StartAddressRecordError): - _fmt = 'Invalid start address value: %(start_addr)s' - - -class NotEnoughDataError(IntelHexError): - _fmt = ('Bad access at 0x%(address)X: ' - 'not enough data to read %(length)d contiguous bytes') - -class BadAccess16bit(NotEnoughDataError): - _fmt = 'Bad access at 0x%(address)X: not enough data to read 16 bit value' - -class EmptyIntelHexError(IntelHexError): - _fmt = "Requested operation cannot be executed with empty object" diff --git a/nordicsemi/dfu/intelhex/compat.py b/nordicsemi/dfu/intelhex/compat.py deleted file mode 100644 index 3f48350..0000000 --- a/nordicsemi/dfu/intelhex/compat.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2011, Bernhard Leiner -# All rights reserved. -# -# Redistribution and use in source and binary forms, -# with or without modification, are permitted provided -# that the following conditions are met: -# -# * Redistributions of source code must retain -# the above copyright notice, this list of conditions -# and the following disclaimer. -# * Redistributions in binary form must reproduce -# the above copyright notice, this list of conditions -# and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# * Neither the name of the author nor the names -# of its contributors may be used to endorse -# or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, -# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, -# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED -# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -'''Compatibility functions for python 2 and 3. - -@author Bernhard Leiner (bleiner AT gmail com) -@version 1.0 -''' - -__docformat__ = "javadoc" - - -import sys - -if sys.version_info[0] >= 3: - def asbytes(s): - if isinstance(s, bytes): - return s - return s.encode('latin1') - def asstr(s): - if isinstance(s, str): - return s - return s.decode('latin1') -else: - asbytes = str - asstr = str - diff --git a/nordicsemi/dfu/nrfhex.py b/nordicsemi/dfu/nrfhex.py index ec94a68..8bf8e45 100644 --- a/nordicsemi/dfu/nrfhex.py +++ b/nordicsemi/dfu/nrfhex.py @@ -35,7 +35,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -from nordicsemi.dfu import intelhex +import intelhex from struct import unpack from enum import Enum @@ -153,7 +153,7 @@ def size(self): # Round up to nearest word word_size = 4 - number_of_words = (size + (word_size - 1)) / word_size + number_of_words = (size + (word_size - 1)) // word_size size = number_of_words * word_size return size From 6bb1ff8cb1cce9404396610bc01d388ad62d6cd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ken=20A=2E=20Rederg=C3=A5rd?= <64542+kenr@users.noreply.github.com> Date: Thu, 20 Jun 2019 15:17:25 +0200 Subject: [PATCH 03/34] Use intelhex module --- nordicsemi/dfu/bl_dfu_sett.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nordicsemi/dfu/bl_dfu_sett.py b/nordicsemi/dfu/bl_dfu_sett.py index e646661..a1937c0 100755 --- a/nordicsemi/dfu/bl_dfu_sett.py +++ b/nordicsemi/dfu/bl_dfu_sett.py @@ -46,8 +46,8 @@ from enum import Enum # Nordic libraries -from nordicsemi.dfu import intelhex -from nordicsemi.dfu.intelhex import IntelHexError +import intelhex +from intelhex import IntelHexError from nordicsemi.dfu.nrfhex import * from nordicsemi.dfu.package import Package from pc_ble_driver_py.exceptions import NordicSemiException From 6c7f84ce62257766e8e76c3affe71ba28f9125a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ken=20A=2E=20Rederg=C3=A5rd?= <64542+kenr@users.noreply.github.com> Date: Thu, 20 Jun 2019 15:17:55 +0200 Subject: [PATCH 04/34] Update pyyaml to latest version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f7f82b3..25f5ec4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,6 @@ tqdm ~= 4.25 piccata ~= 1.0 pyspinel == 1.0.0a3 intelhex ~= 2.2 -pyyaml ~= 4.2b1 +pyyaml ~= 5.1 crcmod ~= 1.7 libusb1 ~= 1.7 From 5aa4b8c299168229565b84b6b420a9d5821434e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ken=20A=2E=20Rederg=C3=A5rd?= <64542+kenr@users.noreply.github.com> Date: Thu, 20 Jun 2019 15:18:11 +0200 Subject: [PATCH 05/34] Fix typo --- nordicsemi/dfu/tests/test_signing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nordicsemi/dfu/tests/test_signing.py b/nordicsemi/dfu/tests/test_signing.py index c056513..8dd740c 100644 --- a/nordicsemi/dfu/tests/test_signing.py +++ b/nordicsemi/dfu/tests/test_signing.py @@ -44,7 +44,7 @@ from nordicsemi.dfu.signing import Signing -class TestSinging(unittest.TestCase): +class TestSigning(unittest.TestCase): def setUp(self): script_abspath = os.path.abspath(__file__) script_dirname = os.path.dirname(script_abspath) From 7ef6b4e868e4c3b99d8157c0c434e2692157a433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ken=20A=2E=20Rederg=C3=A5rd?= <64542+kenr@users.noreply.github.com> Date: Thu, 20 Jun 2019 16:19:37 +0200 Subject: [PATCH 06/34] Update module reqs --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 25f5ec4..464171b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,9 @@ six ~= 1.9 pyserial ~= 3.0 enum34 ~= 1.0 click ~= 7.0 -ecdsa ~= 0.13.0 +ecdsa ~= 0.13.2 behave ~= 1.0 -protobuf ~= 3.6 +protobuf ~= 3.8 pc_ble_driver_py ~= 0.11.4 tqdm ~= 4.25 piccata ~= 1.0 From 8a67f6d795212f9e1ab1d68c1f605e73c4ec4faa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ken=20A=2E=20Rederg=C3=A5rd?= <64542+kenr@users.noreply.github.com> Date: Thu, 20 Jun 2019 16:20:44 +0200 Subject: [PATCH 07/34] ecdsa module do not support bytearray, use bytes --- nordicsemi/dfu/bl_dfu_sett.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/nordicsemi/dfu/bl_dfu_sett.py b/nordicsemi/dfu/bl_dfu_sett.py index a1937c0..aafe079 100755 --- a/nordicsemi/dfu/bl_dfu_sett.py +++ b/nordicsemi/dfu/bl_dfu_sett.py @@ -110,8 +110,7 @@ class BLDFUSettings(object): bl_sett_52840_addr = 0x000FF000 bl_sett_backup_offset = 0x1000 - - def __init__(self, ): + def __init__(self): """ """ # instantiate a hex object @@ -214,20 +213,20 @@ def generate(self, arch, app_file, app_ver, bl_ver, bl_sett_ver, custom_bl_sett_ elif app_boot_validation_type == 'VALIDATE_GENERATED_SHA256': self.app_boot_validation_type = 2 & 0xffffffff sha256 = Package.calculate_sha256_hash(self.app_bin) - self.app_boot_validation_bytes = bytearray([int(binascii.hexlify(i), 16) for i in list(sha256)][31::-1]) + self.app_boot_validation_bytes = bytes([int(binascii.hexlify(i), 16) for i in list(sha256)][31::-1]) elif app_boot_validation_type == 'VALIDATE_ECDSA_P256_SHA256': self.app_boot_validation_type = 3 & 0xffffffff ecdsa = Package.sign_firmware(key_file, self.app_bin) - self.app_boot_validation_bytes = bytearray([int(binascii.hexlify(i), 16) for i in list(ecdsa)]) + self.app_boot_validation_bytes = bytes([int(binascii.hexlify(i), 16) for i in list(ecdsa)]) else: # This also covers 'NO_VALIDATION' case self.app_boot_validation_type = 0 & 0xffffffff - self.app_boot_validation_bytes = bytearray(0) + self.app_boot_validation_bytes = bytes(0) else: self.app_sz = 0x0 & 0xffffffff self.app_crc = 0x0 & 0xffffffff self.bank0_bank_code = 0x0 & 0xffffffff self.app_boot_validation_type = 0x0 & 0xffffffff - self.app_boot_validation_bytes = bytearray(0) + self.app_boot_validation_bytes = bytes(0) if sd_file is not None: # Load SD to calculate CRC @@ -253,18 +252,18 @@ def generate(self, arch, app_file, app_ver, bl_ver, bl_sett_ver, custom_bl_sett_ elif sd_boot_validation_type == 'VALIDATE_GENERATED_SHA256': self.sd_boot_validation_type = 2 & 0xffffffff sha256 = Package.calculate_sha256_hash(self.sd_bin) - self.sd_boot_validation_bytes = bytearray([int(binascii.hexlify(i), 16) for i in list(sha256)][31::-1]) + self.sd_boot_validation_bytes = bytes([int(binascii.hexlify(i), 16) for i in list(sha256)][31::-1]) elif sd_boot_validation_type == 'VALIDATE_ECDSA_P256_SHA256': self.sd_boot_validation_type = 3 & 0xffffffff ecdsa = Package.sign_firmware(key_file, self.sd_bin) - self.sd_boot_validation_bytes = bytearray([int(binascii.hexlify(i), 16) for i in list(ecdsa)]) + self.sd_boot_validation_bytes = bytes([int(binascii.hexlify(i), 16) for i in list(ecdsa)]) else: # This also covers 'NO_VALIDATION_CASE' self.sd_boot_validation_type = 0 & 0xffffffff - self.sd_boot_validation_bytes = bytearray(0) + self.sd_boot_validation_bytes = bytes(0) else: self.sd_sz = 0x0 & 0xffffffff self.sd_boot_validation_type = 0 & 0xffffffff - self.sd_boot_validation_bytes = bytearray(0) + self.sd_boot_validation_bytes = bytes(0) # additional harcoded values self.bank_layout = 0x0 & 0xffffffff From 96f1cb923ce0b960a3efff67c20d90a09e919f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ken=20A=2E=20Rederg=C3=A5rd?= <64542+kenr@users.noreply.github.com> Date: Thu, 20 Jun 2019 16:21:27 +0200 Subject: [PATCH 08/34] Fix binary write to file and hex conversion --- nordicsemi/dfu/signing.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nordicsemi/dfu/signing.py b/nordicsemi/dfu/signing.py index 74c423e..e4b8830 100644 --- a/nordicsemi/dfu/signing.py +++ b/nordicsemi/dfu/signing.py @@ -75,7 +75,7 @@ def gen_key(self, filename): """ self.sk = SigningKey.generate(curve=NIST256p) - with open(filename, "w") as sk_file: + with open(filename, "wb") as sk_file: sk_file.write(self.sk.to_pem()) def load_key(self, filename): @@ -88,8 +88,6 @@ def load_key(self, filename): sk_pem = sk_file.read() self.sk = SigningKey.from_pem(sk_pem) - - sk_hex = "".join(c.encode('hex') for c in self.sk.to_string()) return default_sk.to_string() == self.sk.to_string() def sign(self, init_packet_data): From 33000b8198c5a873d0f9c0b1965d8532a05d8cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ken=20A=2E=20Rederg=C3=A5rd?= <64542+kenr@users.noreply.github.com> Date: Thu, 20 Jun 2019 16:22:00 +0200 Subject: [PATCH 09/34] Use intelhex pkg --- nordicsemi/dfu/tests/test_nrfhex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nordicsemi/dfu/tests/test_nrfhex.py b/nordicsemi/dfu/tests/test_nrfhex.py index 9207c10..d9ff91d 100644 --- a/nordicsemi/dfu/tests/test_nrfhex.py +++ b/nordicsemi/dfu/tests/test_nrfhex.py @@ -39,7 +39,7 @@ import unittest import nordicsemi.dfu.nrfhex as nrfhex -import nordicsemi.dfu.intelhex as intelhex +import intelhex class TestnRFHex(unittest.TestCase): From 851ff500837a426de746835c8c073d907c29d9b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ken=20A=2E=20Rederg=C3=A5rd?= <64542+kenr@users.noreply.github.com> Date: Thu, 20 Jun 2019 16:22:17 +0200 Subject: [PATCH 10/34] Use bytes and not decode --- nordicsemi/bluetooth/hci/tests/test_codec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nordicsemi/bluetooth/hci/tests/test_codec.py b/nordicsemi/bluetooth/hci/tests/test_codec.py index b0d2a62..8967d2e 100644 --- a/nordicsemi/bluetooth/hci/tests/test_codec.py +++ b/nordicsemi/bluetooth/hci/tests/test_codec.py @@ -64,7 +64,7 @@ def test_decode_packet(self): for uart_packet in read_packets: hex_string = uart_packet.replace(" ", "") - hex_data = hex_string.decode("hex") + hex_data = bytes.fromhex(hex_string) slip.append(hex_data) packets = slip.decode() From 0ea72ec017054ebdf2a912b50549790c12eae243 Mon Sep 17 00:00:00 2001 From: Wasaznik Date: Tue, 3 Sep 2019 14:33:22 +0200 Subject: [PATCH 11/34] Convert hci.slip to Python 3 --- nordicsemi/bluetooth/hci/slip.py | 50 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/nordicsemi/bluetooth/hci/slip.py b/nordicsemi/bluetooth/hci/slip.py index b95d97b..20df683 100644 --- a/nordicsemi/bluetooth/hci/slip.py +++ b/nordicsemi/bluetooth/hci/slip.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016 Nordic Semiconductor ASA +# Copyright (c) 2019 Nordic Semiconductor ASA # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -40,19 +40,19 @@ logger = logging.getLogger(__name__) -class Slip(object): - def __init__(self): - self.SLIP_END = '\xc0' - self.SLIP_ESC = '\xdb' - self.SLIP_ESC_END = '\xdc' - self.SLIP_ESC_ESC = '\xdd' +class Slip: + SLIP_END = 0xc0 + SLIP_ESC = 0xdb + SLIP_ESC_END = 0xdc + SLIP_ESC_ESC = 0xdd + def __init__(self): self.started = False self.escaped = False - self.stream = '' - self.packet = '' + self.stream = bytearray() + self.packet = bytearray() - def append(self, data): + def append(self, data: bytes): """ Append a new :param data: Append a new block of data to do decoding on when calling decode. @@ -66,7 +66,7 @@ def decode(self): Decodes a package according to http://en.wikipedia.org/wiki/Serial_Line_Internet_Protocol :return Slip: A list of decoded slip packets """ - packet_list = list() + packet_list = [] for char in self.stream: if char == self.SLIP_END: @@ -74,51 +74,51 @@ def decode(self): if len(self.packet) > 0: self.started = False packet_list.append(self.packet) - self.packet = '' + self.packet = bytearray() else: self.started = True - self.packet = '' + self.packet = bytearray() elif char == self.SLIP_ESC: self.escaped = True elif char == self.SLIP_ESC_END: if self.escaped: - self.packet += self.SLIP_END + self.packet.append(self.SLIP_END) self.escaped = False else: self.packet += char elif char == self.SLIP_ESC_ESC: if self.escaped: - self.packet += self.SLIP_ESC + self.packet.append(self.SLIP_ESC) self.escaped = False else: - self.packet += char + self.packet.append(char) else: if self.escaped: logging.error("Error in SLIP packet, ignoring error.") - self.packet = '' + self.packet = bytearray() self.escaped = False else: - self.packet += char + self.packet.append(char) - self.stream = '' + self.stream.clear() return packet_list - def encode(self, packet): + def encode(self, packet: bytes): """ Encode a packet according to SLIP. :param packet: A str array that represents the package :return: str array with an encoded SLIP packet """ - encoded = self.SLIP_END + encoded = bytearray([self.SLIP_END]) for char in packet: if char == self.SLIP_END: - encoded += self.SLIP_ESC + self.SLIP_ESC_END + encoded.extend([self.SLIP_ESC, self.SLIP_ESC_END]) elif char == self.SLIP_ESC: - encoded += self.SLIP_ESC + self.SLIP_ESC_ESC + encoded.extend([self.SLIP_ESC, self.SLIP_ESC_ESC]) else: - encoded += char - encoded += self.SLIP_END + encoded.append(char) + encoded.append(self.SLIP_END) return encoded From 539a9ca51d3d0cf86aee475c82cb0a68c893af1d Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Wed, 4 Sep 2019 10:11:58 +0200 Subject: [PATCH 12/34] Py3ize part of bl_dfu_sett --- nordicsemi/dfu/bl_dfu_sett.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/nordicsemi/dfu/bl_dfu_sett.py b/nordicsemi/dfu/bl_dfu_sett.py index aafe079..dfad301 100755 --- a/nordicsemi/dfu/bl_dfu_sett.py +++ b/nordicsemi/dfu/bl_dfu_sett.py @@ -212,12 +212,13 @@ def generate(self, arch, app_file, app_ver, bl_ver, bl_sett_ver, custom_bl_sett_ self.app_boot_validation_bytes = struct.pack(' Date: Wed, 4 Sep 2019 11:28:13 +0200 Subject: [PATCH 13/34] Fix assertion to expect bytes --- nordicsemi/dfu/tests/test_bl_dfu_sett.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nordicsemi/dfu/tests/test_bl_dfu_sett.py b/nordicsemi/dfu/tests/test_bl_dfu_sett.py index 905b414..182e9ff 100644 --- a/nordicsemi/dfu/tests/test_bl_dfu_sett.py +++ b/nordicsemi/dfu/tests/test_bl_dfu_sett.py @@ -464,8 +464,8 @@ def test_generate_with_app_boot_validation_sha256(self): self.assertEqual(0x1316CFD0, settings.crc) self.assertEqual(0xF78E451E, settings.boot_validation_crc) self.assertEqual(0x02, settings.app_boot_validation_type) - self.assertEqual('036F52C9EBB53819D6E2B6FB57803823E864783B04D7331B46C0B5897CA9F1C7', - binascii.hexlify(settings.app_boot_validation_bytes).upper()) + self.assertEqual(bytes.fromhex('036F52C9EBB53819D6E2B6FB57803823E864783B04D7331B46C0B5897CA9F1C7'), + settings.app_boot_validation_bytes) def test_generate_with_app_boot_validation_ecdsa(self): settings = BLDFUSettings() @@ -525,8 +525,8 @@ def test_generate_with_sd_boot_validation_sha256(self): self.assertEqual(0x4637019F, settings.crc) self.assertEqual(0x9C761426, settings.boot_validation_crc) self.assertEqual(0x02, settings.sd_boot_validation_type) - self.assertEqual('036F52C9EBB53819D6E2B6FB57803823E864783B04D7331B46C0B5897CA9F1C7', - binascii.hexlify(settings.sd_boot_validation_bytes).upper()) + self.assertEqual(bytes.fromhex('036F52C9EBB53819D6E2B6FB57803823E864783B04D7331B46C0B5897CA9F1C7'), + settings.sd_boot_validation_bytes) def test_generate_with_sd_boot_validation_ecdsa(self): settings = BLDFUSettings() From 89fbcbbe747212a023b0891b4a961ef5d6c616f5 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Wed, 4 Sep 2019 12:55:21 +0200 Subject: [PATCH 14/34] Gitignore tests run artefacts --- nordicsemi/dfu/tests/firmwares/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 nordicsemi/dfu/tests/firmwares/.gitignore diff --git a/nordicsemi/dfu/tests/firmwares/.gitignore b/nordicsemi/dfu/tests/firmwares/.gitignore new file mode 100644 index 0000000..e9a333a --- /dev/null +++ b/nordicsemi/dfu/tests/firmwares/.gitignore @@ -0,0 +1,4 @@ +# Artefacts generated during test run +/bar.bin +/foobar.bin +/foo.bin From abf6f8522d23172c9a71a65b0d4b76c0f3a3da16 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Wed, 4 Sep 2019 13:10:56 +0200 Subject: [PATCH 15/34] Fix str used as bytes --- nordicsemi/dfu/package.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nordicsemi/dfu/package.py b/nordicsemi/dfu/package.py index 1b4d8ef..b6627fe 100644 --- a/nordicsemi/dfu/package.py +++ b/nordicsemi/dfu/package.py @@ -241,11 +241,11 @@ def parse_package(self, filename, preserve_work_dir=False): self.zip_file = filename self.zip_dir = os.path.join(self.work_dir, 'unpacked_zip') self.manifest = Package.unpack_package(filename, self.zip_dir) - + self.rm_work_dir(preserve_work_dir) def image_str(self, index, hex_type, img): - type_strs = {HexType.SD_BL : "sd_bl", + type_strs = {HexType.SD_BL : "sd_bl", HexType.SOFTDEVICE : "softdevice", HexType.BOOTLOADER : "bootloader", HexType.APPLICATION : "application", @@ -329,7 +329,7 @@ def image_str(self, index, hex_type, img): return s def __str__(self): - + imgs = "" i = 0 if self.manifest.softdevice_bootloader: @@ -439,7 +439,7 @@ def generate_package(self, filename, preserve_work_dir=False): else: boot_validation_bytes_array.append(Package.sign_firmware(self.key_file, bin_file_path)) else: - boot_validation_bytes_array.append('') + boot_validation_bytes_array.append(b'') init_packet = InitPacketPB( @@ -599,7 +599,7 @@ def __add_firmware_info(self, firmware_type, firmware_version, filename, init_pa if firmware_type == HexType.SD_BL: self.firmwares_data[firmware_type][FirmwareKeys.SD_SIZE] = sd_size self.firmwares_data[firmware_type][FirmwareKeys.BL_SIZE] = bl_size - + if firmware_version is not None: self.firmwares_data[firmware_type][FirmwareKeys.INIT_PACKET_DATA][PacketField.FW_VERSION] = firmware_version From 5b08d1815de8a16eaeda0ceb1a7aee701e99745e Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Wed, 4 Sep 2019 13:38:21 +0200 Subject: [PATCH 16/34] Py3ize Signing There is a rather arbitrary decision make in this change that PEMs are represented in bytes. It was selected in the interest of keeping conversions to a minimum. Both bytes and str are good candidated for representing PEM, which consists only of ascii chars. --- nordicsemi/dfu/signing.py | 58 +++++++++------------------- nordicsemi/dfu/tests/test_signing.py | 8 ++-- 2 files changed, 23 insertions(+), 43 deletions(-) diff --git a/nordicsemi/dfu/signing.py b/nordicsemi/dfu/signing.py index e4b8830..e9c67ac 100644 --- a/nordicsemi/dfu/signing.py +++ b/nordicsemi/dfu/signing.py @@ -164,17 +164,13 @@ def get_sk_hex(self): if self.sk is None: raise IllegalStateException("Can't get key. No key created/loaded") - sk_hexlify = binascii.hexlify(self.sk.to_string()) + # Reverse the key for display. This emulates a memory + # dump of the key interpreted a 256bit litte endian + # integer. + key = self.sk.to_string() + displayed_key = key[::-1].hex() - sk_hexlify_list = [] - for i in range(len(sk_hexlify)-2, -2, -2): - sk_hexlify_list.append(sk_hexlify[i:i+2]) - - sk_hexlify_list_str = ''.join(sk_hexlify_list) - - vk_hex = "Private (signing) key sk:\n{0}".format(sk_hexlify_list_str) - - return vk_hex + return f"Private (signing) key sk:\n{displayed_key}" def get_vk_hex(self): """ @@ -183,21 +179,13 @@ def get_vk_hex(self): if self.sk is None: raise IllegalStateException("Can't get key. No key created/loaded") - vk = self.sk.get_verifying_key() - vk_hexlify = binascii.hexlify(vk.to_string()) - - vk_hexlify_list = [] - for i in range(len(vk_hexlify[0:64])-2, -2, -2): - vk_hexlify_list.append(vk_hexlify[i:i+2]) - - for i in range(len(vk_hexlify)-2, 62, -2): - vk_hexlify_list.append(vk_hexlify[i:i+2]) + # Reverse the two halves of key for display. This + # emulates a memory dump of the key interpreted as two + # 256bit litte endian integers. + key = self.sk.get_verifying_key().to_string() + displayed_key = (key[:32][::-1] + key[32:][::-1]).hex() - vk_hexlify_list_str = ''.join(vk_hexlify_list) - - vk_hex = "Public (verification) key pk:\n{0}".format(vk_hexlify_list_str) - - return vk_hex + return f"Public (verification) key pk:\n{displayed_key}" def wrap_code(self, key_code, dbg): @@ -212,7 +200,7 @@ def wrap_code(self, key_code, dbg): /* This file was generated with a throwaway private key, that is only inteded for a debug version of the DFU project. Please see https://github.com/NordicSemiconductor/pc-nrfutil/blob/master/README.md to generate a valid public key. */ -#ifdef NRF_DFU_DEBUG_VERSION +#ifdef NRF_DFU_DEBUG_VERSION """ dbg_footer=""" #else @@ -233,20 +221,12 @@ def get_vk_code(self, dbg): if self.sk is None: raise IllegalStateException("Can't get key. No key created/loaded") - vk = self.sk.get_verifying_key() - vk_hex = binascii.hexlify(vk.to_string()) - - vk_x_separated = "" - vk_x_str = vk_hex[0:64] - for i in range(0, len(vk_x_str), 2): - vk_x_separated = "0x" + vk_x_str[i:i+2] + ", " + vk_x_separated - - vk_y_separated = "" - vk_y_str = vk_hex[64:128] - for i in range(0, len(vk_y_str), 2): - vk_y_separated = "0x" + vk_y_str[i:i+2] + ", " + vk_y_separated - vk_y_separated = vk_y_separated[:-2] - + to_two_digit_hex_with_0x = '0x{0:02x}'.format + + key = self.sk.get_verifying_key().to_string() + vk_x_separated = ', '.join(map(to_two_digit_hex_with_0x, key[:64])) + vk_y_separated = ', '.join(map(to_two_digit_hex_with_0x, key[64:])) + key_code =""" /** @brief Public key used to verify DFU images */ __ALIGN(4) const uint8_t pk[64] = diff --git a/nordicsemi/dfu/tests/test_signing.py b/nordicsemi/dfu/tests/test_signing.py index 8dd740c..e74af27 100644 --- a/nordicsemi/dfu/tests/test_signing.py +++ b/nordicsemi/dfu/tests/test_signing.py @@ -114,10 +114,10 @@ def test_get_sk_hex(self): def test_get_vk_pem(self): key_file_name = 'key.pem' - expected_vk_pem = "-----BEGIN PUBLIC KEY-----\n" \ - "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEZY2i7duYH2l9rnIg1oIXq+0/uHAF\n" \ - "7IoFubVru6oX9GCQm67NrXImwgS2ErZi/0/MvRsMkIQQkNg6Wc2tbJgdTA==\n" \ - "-----END PUBLIC KEY-----\n" + expected_vk_pem = b"-----BEGIN PUBLIC KEY-----\n" \ + b"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEZY2i7duYH2l9rnIg1oIXq+0/uHAF\n" \ + b"7IoFubVru6oX9GCQm67NrXImwgS2ErZi/0/MvRsMkIQQkNg6Wc2tbJgdTA==\n" \ + b"-----END PUBLIC KEY-----\n" signing = Signing() signing.load_key(key_file_name) From df86fdc13aded1834a82a17fdb70212f1bd9b934 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Wed, 4 Sep 2019 16:37:11 +0200 Subject: [PATCH 17/34] Upgrade pc_ble_driver_py to 0.12 for py3 support --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 464171b..a3684e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ click ~= 7.0 ecdsa ~= 0.13.2 behave ~= 1.0 protobuf ~= 3.8 -pc_ble_driver_py ~= 0.11.4 +pc_ble_driver_py ~= 0.12 tqdm ~= 4.25 piccata ~= 1.0 pyspinel == 1.0.0a3 From 6d55772e591892509cfec17f4a0c06e10834082d Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Mon, 16 Sep 2019 11:24:10 +0200 Subject: [PATCH 18/34] Py3ize zigbee/prod_config --- nordicsemi/zigbee/prod_config.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/nordicsemi/zigbee/prod_config.py b/nordicsemi/zigbee/prod_config.py index 46f3b8d..2374fe3 100644 --- a/nordicsemi/zigbee/prod_config.py +++ b/nordicsemi/zigbee/prod_config.py @@ -63,7 +63,7 @@ def __init__(self, path): # Open the YAML file with open(path, 'r') as f: self._yaml = yaml.load(f) - + except yaml.YAMLError as e: raise ProductionConfigWrongException @@ -85,21 +85,21 @@ def __init__(self, path): self._parsed_values["install_code"] = str(b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') self._ic_crc = 0 else: - self._parsed_values["install_code"] = self._yaml["install_code"].decode('hex') + self._parsed_values["install_code"] = bytes.fromhex(self._yaml["install_code"]) self._ic_crc = self._crc16(self._parsed_values["install_code"]) - + # Handle the Transmission Power if "tx_power" not in self._yaml: - self._parsed_values["tx_power"] = str(bytearray(16)) + self._parsed_values["tx_power"] = bytes(16) else: - self._parsed_values["tx_power"] = str(bytearray([self._yaml["tx_power"]] * 16)) + self._parsed_values["tx_power"] = bytes([self._yaml["tx_power"]] * 16) # Handle Application Data (optional) if "app_data" not in self._yaml: - self._parsed_values["app_data"] = '' + self._parsed_values["app_data"] = b'' self._ad_len = 0 else: - self._parsed_values["app_data"] = self._yaml["app_data"].decode('hex') + self._parsed_values["app_data"] = bytes.fromhex(self._yaml["app_data"]) self._ad_len = len(self._parsed_values["app_data"]) except (TypeError, ValueError) as e: @@ -112,7 +112,6 @@ def _custom_crc32(self, data): ZB_CRC32_POLY = 0x04C11DB7 crc = 0 for d in data: - d = ord(d) c = ((( crc ^ d ) & 0xff) << 24); for j in range(8): if c & 0x80000000: From 34c07154d865993e448cb32b91793c1d084aadb1 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Mon, 16 Sep 2019 11:24:45 +0200 Subject: [PATCH 19/34] Remove unneeded NoneType import that breaks py3 Python's types.NoneType is no longer available, and it was not used anyway. --- nordicsemi/dfu/dfu_transport_ble.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nordicsemi/dfu/dfu_transport_ble.py b/nordicsemi/dfu/dfu_transport_ble.py index 23562ac..18105fe 100644 --- a/nordicsemi/dfu/dfu_transport_ble.py +++ b/nordicsemi/dfu/dfu_transport_ble.py @@ -45,7 +45,6 @@ import logging import binascii -from types import NoneType from nordicsemi.dfu.dfu_transport import DfuTransport, DfuEvent from pc_ble_driver_py.exceptions import NordicSemiException, IllegalStateException from pc_ble_driver_py.ble_driver import BLEDriver, BLEDriverObserver, BLEEnableParams, BLEUUIDBase, BLEGapSecKDist, BLEGapSecParams, \ From 78c51c29c5a924540cf0e5848a9da71309c8d650 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Mon, 16 Sep 2019 12:23:51 +0200 Subject: [PATCH 20/34] Add None guard before comparing as needed for py3 --- nordicsemi/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nordicsemi/__main__.py b/nordicsemi/__main__.py index 849e3c1..70688a9 100755 --- a/nordicsemi/__main__.py +++ b/nordicsemi/__main__.py @@ -854,11 +854,11 @@ def generate(zipfile, if zigbee: inner_external_app = False - if zigbee_ota_min_hw_version > 0xFFFF: + if zigbee_ota_min_hw_version is not None and zigbee_ota_min_hw_version > 0xFFFF: click.echo('Error: zigbee-ota-min-hw-version exceeds 2-byte long integer.') return - if zigbee_ota_max_hw_version > 0xFFFF: + if zigbee_ota_max_hw_version is not None and zigbee_ota_max_hw_version > 0xFFFF: click.echo('Error: zigbee-ota-max-hw-version exceeds 2-byte long integer.') return From acddb27ff8de3837d4462bb0e56b5c50be56e437 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Mon, 16 Sep 2019 13:06:46 +0200 Subject: [PATCH 21/34] Add some py3 support in TestDfuTransportSerial The test will still not complete due to changes in interal apis. --- nordicsemi/dfu/crc16.py | 5 ++--- nordicsemi/dfu/tests/test_dfu_transport_serial.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/nordicsemi/dfu/crc16.py b/nordicsemi/dfu/crc16.py index aa7bc80..11de1a7 100644 --- a/nordicsemi/dfu/crc16.py +++ b/nordicsemi/dfu/crc16.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2016 Nordic Semiconductor ASA +# Copyright (c) 2019 Nordic Semiconductor ASA # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -35,7 +35,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -def calc_crc16(binary_data, crc=0xffff): +def calc_crc16(binary_data: bytes, crc=0xffff): """ Calculates CRC16 on binary_data @@ -46,7 +46,6 @@ def calc_crc16(binary_data, crc=0xffff): for b in binary_data: crc = (crc >> 8 & 0x00FF) | (crc << 8 & 0xFF00) - crc ^= ord(b) crc ^= (crc & 0x00FF) >> 4 crc ^= (crc << 8) << 4 crc ^= ((crc & 0x00FF) << 4) << 1 diff --git a/nordicsemi/dfu/tests/test_dfu_transport_serial.py b/nordicsemi/dfu/tests/test_dfu_transport_serial.py index 33be969..fa049cf 100644 --- a/nordicsemi/dfu/tests/test_dfu_transport_serial.py +++ b/nordicsemi/dfu/tests/test_dfu_transport_serial.py @@ -103,7 +103,7 @@ def error_callback(log_message=""): self.transport.register_events_callback(DfuEvent.PROGRESS_EVENT, progress_callback) self.transport.register_events_callback(DfuEvent.ERROR_EVENT, error_callback()) - firmware = '' + firmware = b'' test_firmware_path = os.path.join("firmwares", "pca10028_nrf51422_xxac_blinky.bin") with open(test_firmware_path, 'rb') as f: From 48e0e59d5c2bd42e80f07efd0ef372d1010a7146 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Tue, 17 Sep 2019 12:24:46 +0200 Subject: [PATCH 22/34] Adjust requirements for py3 --- requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a3684e9..ec7e607 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ -six ~= 1.9 +pywin32; sys_platform == 'win32' +antlib; sys_platform == 'win32' +pyinstaller pyserial ~= 3.0 -enum34 ~= 1.0 click ~= 7.0 ecdsa ~= 0.13.2 behave ~= 1.0 From 438fc6fecf0364e588b114159b371334dccb02a6 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Fri, 20 Sep 2019 12:47:34 +0200 Subject: [PATCH 23/34] Evaluate map object of serial ports to list --- nordicsemi/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nordicsemi/__main__.py b/nordicsemi/__main__.py index 70688a9..1f2de9f 100755 --- a/nordicsemi/__main__.py +++ b/nordicsemi/__main__.py @@ -1079,7 +1079,7 @@ def serial(package, port, connect_delay, flow_control, packet_receipt_notificati def enumerate_ports(): - descs = BLEDriver.enum_serial_ports() + descs = list(BLEDriver.enum_serial_ports()) if len(descs) == 0: return None click.echo('Please select connectivity serial port:') From 403c294d77377687dad864849b0915ac42ea2d3c Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Fri, 20 Sep 2019 12:49:14 +0200 Subject: [PATCH 24/34] Py3ize dfu_transport_ble --- nordicsemi/dfu/dfu_transport_ble.py | 6 +++--- nordicsemi/dfu/dfu_transport_serial.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nordicsemi/dfu/dfu_transport_ble.py b/nordicsemi/dfu/dfu_transport_ble.py index 18105fe..4fb8438 100644 --- a/nordicsemi/dfu/dfu_transport_ble.py +++ b/nordicsemi/dfu/dfu_transport_ble.py @@ -559,7 +559,7 @@ def try_to_recover(): def __set_prn(self): logger.debug("BLE: Set Packet Receipt Notification {}".format(self.prn)) - self.dfu_adapter.write_control_point([DfuTransportBle.OP_CODE['SetPRN']] + list(map(ord, struct.pack(' Date: Fri, 20 Sep 2019 12:49:42 +0200 Subject: [PATCH 25/34] Port dfu_transport_ble to SDv5 The new pc_ble_driver_py module with py3 support is SDv5 based. --- nordicsemi/dfu/dfu_transport_ble.py | 42 ++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/nordicsemi/dfu/dfu_transport_ble.py b/nordicsemi/dfu/dfu_transport_ble.py index 4fb8438..6bf71aa 100644 --- a/nordicsemi/dfu/dfu_transport_ble.py +++ b/nordicsemi/dfu/dfu_transport_ble.py @@ -49,7 +49,7 @@ from pc_ble_driver_py.exceptions import NordicSemiException, IllegalStateException from pc_ble_driver_py.ble_driver import BLEDriver, BLEDriverObserver, BLEEnableParams, BLEUUIDBase, BLEGapSecKDist, BLEGapSecParams, \ BLEGapIOCaps, BLEUUID, BLEAdvData, BLEGapConnParams, BLEEvtID, BLEGattHVXType, NordicSemiErrorCheck, BLEGapSecStatus, driver -from pc_ble_driver_py.ble_driver import ATT_MTU_DEFAULT +from pc_ble_driver_py.ble_driver import ATT_MTU_DEFAULT, BLEConfig, BLEConfigConnGatt from pc_ble_driver_py.ble_adapter import BLEAdapter, BLEAdapterObserver, EvtSync logger = logging.getLogger(__name__) @@ -103,16 +103,27 @@ def __init__(self, adapter, bonded=False, keyset=None): def open(self): self.adapter.driver.open() - ble_enable_params = BLEEnableParams(vs_uuid_count = 10, - service_changed = True, - periph_conn_count = 0, - central_conn_count = 1, - central_sec_count = 1) - if nrf_sd_ble_api_ver >= 3: - logger.info("\nBLE: ble_enable with local ATT MTU: {}".format(DFUAdapter.LOCAL_ATT_MTU)) - ble_enable_params.att_mtu = DFUAdapter.LOCAL_ATT_MTU - self.adapter.driver.ble_enable(ble_enable_params) + assert nrf_sd_ble_api_ver in [2, 5] + + if nrf_sd_ble_api_ver == 2: + self.adapter.driver.ble_enable( + BLEEnableParams( + vs_uuid_count = 10, + service_changed = True, + periph_conn_count = 0, + central_conn_count = 1, + central_sec_count = 1, + ) + ) + + if nrf_sd_ble_api_ver == 5: + self.adapter.driver.ble_cfg_set( + BLEConfig.conn_gatt, + BLEConfigConnGatt(att_mtu=DFUAdapter.LOCAL_ATT_MTU), + ) + self.adapter.driver.ble_enable() + self.adapter.driver.ble_vs_uuid_add(DFUAdapter.BASE_UUID) def close(self): @@ -158,7 +169,7 @@ def connect(self, target_device_name, target_device_addr): if nrf_sd_ble_api_ver >= 3: if DFUAdapter.LOCAL_ATT_MTU > ATT_MTU_DEFAULT: logger.info('BLE: Enabling longer ATT MTUs') - self.att_mtu = self.adapter.att_mtu_exchange(self.conn_handle) + self.att_mtu = self.adapter.att_mtu_exchange(self.conn_handle, DFUAdapter.LOCAL_ATT_MTU) else: logger.info('BLE: Using default ATT MTU') @@ -370,7 +381,12 @@ def on_gap_evt_adv_report(self, ble_driver, conn_handle, peer_addr, rssi, adv_ty slave_latency = 0) logger.info('BLE: Found target advertiser, address: 0x{}, name: {}'.format(address_string, dev_name)) logger.info('BLE: Connecting to 0x{}'.format(address_string)) - self.adapter.connect(address = peer_addr, conn_params = self.conn_params) + # Connect must specify tag=1 to enable the settings + # set with BLEConfigConnGatt (that implictly operates + # on connections with tag 1) to allow for larger MTU. + self.adapter.connect(address=peer_addr, + conn_params=self.conn_params, + tag=1) # store the address for subsequent connections self.target_device_addr = address_string self.target_device_addr_type = peer_addr @@ -425,7 +441,7 @@ def __init__(self, att_mtu, target_device_name=None, target_device_addr=None, - baud_rate=115200, + baud_rate=1000000, prn=0): super(DfuTransportBle, self).__init__() DFUAdapter.LOCAL_ATT_MTU = att_mtu From b02ab548294c06d0b11411910c2bfa5ee4a394a9 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Fri, 20 Sep 2019 14:27:54 +0200 Subject: [PATCH 26/34] Change type warning string to bytes The alternative is to instead .decode('ascii') the result from get_sk, but since everyting is ascii, bytes is a suitable (as intended) type that can actually be safer in case PEM files cannot contain non-ascii values. --- nordicsemi/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nordicsemi/__main__.py b/nordicsemi/__main__.py index 1f2de9f..f16f1b9 100755 --- a/nordicsemi/__main__.py +++ b/nordicsemi/__main__.py @@ -483,7 +483,7 @@ def display(key_file, key, format, out_file): if key == "pk": kstr = signer.get_vk(format, dbg) elif key == "sk": - kstr = "\nWARNING: Security risk! Do not share the private key.\n\n" + kstr = b"\nWARNING: Security risk! Do not share the private key.\n\n" kstr = kstr + signer.get_sk(format, dbg) if not out_file: From ecb7683a613aaa94c087510b5335c426ae3e07d8 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Fri, 20 Sep 2019 14:55:06 +0200 Subject: [PATCH 27/34] Remove map(ord, ...) on struct.pack in Ant struct.pack in Python 3 returns bytes which gives ints when iterated over. --- nordicsemi/dfu/dfu_transport_ant.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nordicsemi/dfu/dfu_transport_ant.py b/nordicsemi/dfu/dfu_transport_ant.py index 86b3b06..03f4655 100755 --- a/nordicsemi/dfu/dfu_transport_ant.py +++ b/nordicsemi/dfu/dfu_transport_ant.py @@ -187,7 +187,7 @@ def send_message(self, req): logger.log(TRANSPORT_LOGGING_LEVEL, "ANT: --> {}".format(req)) self.tx_seq = (self.tx_seq + 1) & 0xFF - data = list(map(ord, struct.pack(' Date: Tue, 24 Sep 2019 09:45:19 +0200 Subject: [PATCH 28/34] Update requirements.txt and add requirements-dev.txt --- requirements-dev.txt | 4 ++++ requirements.txt | 20 +++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..533fe0c --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +-r requirements.txt +behave +nose +pyinstaller diff --git a/requirements.txt b/requirements.txt index ec7e607..10d7f02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,14 @@ -pywin32; sys_platform == 'win32' -antlib; sys_platform == 'win32' -pyinstaller -pyserial ~= 3.0 +antlib == 1.1b0.post0; sys_platform == 'win32' click ~= 7.0 +crcmod ~= 1.7 ecdsa ~= 0.13.2 -behave ~= 1.0 +intelhex ~= 2.2 +libusb1 ~= 1.7 +pc_ble_driver_py == 0.12.0 +piccata # FIXME: Specify version constraint when piccata with py3 support becomes available. protobuf ~= 3.8 -pc_ble_driver_py ~= 0.12 -tqdm ~= 4.25 -piccata ~= 1.0 +pyserial ~= 3.0 pyspinel == 1.0.0a3 -intelhex ~= 2.2 +pywin32 == 225; sys_platform == 'win32' pyyaml ~= 5.1 -crcmod ~= 1.7 -libusb1 ~= 1.7 +tqdm ~= 4.36 From ba835b815c89dbb7e4bce034bf37026bbf828973 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Wed, 25 Sep 2019 08:56:06 +0200 Subject: [PATCH 29/34] Change readme to run package instead of __main__.py This way, importing nordicsemi works without changes to PYTHONPATH. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cc26fd4..1538c90 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ pip install -r requirements.txt You can run the program directly without installing it by executing: ``` -python nordicsemi/__main__.py +python ./nordicsemi ``` ### Installing from source From f0532dd2c3e733f6b60e5a6efe811f5979b2bbfe Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Wed, 25 Sep 2019 09:09:50 +0200 Subject: [PATCH 30/34] Remove Python 2 specific note in README --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 1538c90..e7288e9 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,6 @@ pyinstaller /full/path/to/nrfutil.spec **Note**: Some anti-virus programs will stop PyInstaller from executing correctly when it modifies the executable file. -**Note**: PyInstaller on macOS may have issues with newer versions of Python 2.7.x., e.g. error message `Failed to execute script pyi_rth_pkgres` at runtime. Try rolling back to Python v2.7.10 if you experience such problems. - ## Usage To get info on usage of nrfutil: From dec0c716e8a0b0e4a265c9d8c24f9bbbe4172afd Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Thu, 26 Sep 2019 10:11:16 +0200 Subject: [PATCH 31/34] Add tests artefacts to gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index f56cd43..62b9af2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ venv/ build/ dist/ nrfutil.egg-info/ + +# Test artefacts +/nordicsemi/dfu/tests/mypackage.zip +/tests/test.zip From 99350e6f68fd9e8ec85c35615b28fd861fb631b6 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Thu, 26 Sep 2019 12:25:41 +0200 Subject: [PATCH 32/34] Remove vestigal debug print from test --- tests/test_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index d308e63..bee4220 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -41,7 +41,6 @@ def test_dfu_ble_address(self): address = 'AABBCC11223' result = self.runner.invoke(self.cli, argumentList + [address]) self.assertTrue('Invalid address' in result.output) - print((result.exception)) self.assertIsNone(result.exception) address = 'AABBCC1122334' From a766ffa6a0f872cc207d01d71515f630772e2112 Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Thu, 26 Sep 2019 12:37:16 +0200 Subject: [PATCH 33/34] Use ABC instead of ABCMeta --- nordicsemi/dfu/dfu_transport.py | 20 +++++++++----------- nordicsemi/lister/lister_backend.py | 9 ++++----- nordicsemi/utility/target_registry.py | 9 ++++----- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/nordicsemi/dfu/dfu_transport.py b/nordicsemi/dfu/dfu_transport.py index 628ebbe..1910f78 100644 --- a/nordicsemi/dfu/dfu_transport.py +++ b/nordicsemi/dfu/dfu_transport.py @@ -1,5 +1,4 @@ -# -# Copyright (c) 2016 Nordic Semiconductor ASA +# Copyright (c) 2016 - 2019 Nordic Semiconductor ASA # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -33,12 +32,11 @@ # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Python specific imports -import abc import logging +from abc import ABC, abstractmethod + # Nordic Semiconductor imports logger = logging.getLogger(__name__) @@ -54,7 +52,7 @@ class DfuEvent: PROGRESS_EVENT = 1 -class DfuTransport(object, metaclass=abc.ABCMeta): +class DfuTransport(ABC): """ This class as an abstract base class inherited from when implementing transports. @@ -104,12 +102,12 @@ class DfuTransport(object, metaclass=abc.ABCMeta): "The requested firmware to update was already present on the system.", ] - @abc.abstractmethod + @abstractmethod def __init__(self): self.callbacks = {} - @abc.abstractmethod + @abstractmethod def open(self): """ Open a port if appropriate for the transport. @@ -118,7 +116,7 @@ def open(self): pass - @abc.abstractmethod + @abstractmethod def close(self): """ Close a port if appropriate for the transport. @@ -126,7 +124,7 @@ def close(self): """ pass - @abc.abstractmethod + @abstractmethod def send_init_packet(self, init_packet): """ Send init_packet to device. @@ -139,7 +137,7 @@ def send_init_packet(self, init_packet): pass - @abc.abstractmethod + @abstractmethod def send_firmware(self, firmware): """ Start sending firmware to device. diff --git a/nordicsemi/lister/lister_backend.py b/nordicsemi/lister/lister_backend.py index 50dc7f8..7d6be80 100644 --- a/nordicsemi/lister/lister_backend.py +++ b/nordicsemi/lister/lister_backend.py @@ -1,4 +1,3 @@ -# # Copyright (c) 2019 Nordic Semiconductor ASA # All rights reserved. # @@ -33,12 +32,12 @@ # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -import abc + +from abc import ABC, abstractmethod -class AbstractLister(object, metaclass=abc.ABCMeta): - @abc.abstractmethod +class AbstractLister(ABC): + @abstractmethod def enumerate(self): """ Enumerate all usb devices diff --git a/nordicsemi/utility/target_registry.py b/nordicsemi/utility/target_registry.py index cc991de..f567546 100644 --- a/nordicsemi/utility/target_registry.py +++ b/nordicsemi/utility/target_registry.py @@ -1,5 +1,4 @@ -# -# Copyright (c) 2016 Nordic Semiconductor ASA +# Copyright (c) 2016 - 2019 Nordic Semiconductor ASA # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -33,15 +32,15 @@ # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# import re import os import json -from abc import ABCMeta, abstractmethod + +from abc import ABC, abstractmethod -class TargetDatabase(object, metaclass=ABCMeta): +class TargetDatabase(ABC): @abstractmethod def get_targets(self): pass From 24bc33b8af750fe45b85044289526fb2b640bdca Mon Sep 17 00:00:00 2001 From: Aleksander Wasaznik Date: Thu, 26 Sep 2019 12:43:26 +0200 Subject: [PATCH 34/34] Fixup copyright year to a range --- nordicsemi/bluetooth/hci/slip.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nordicsemi/bluetooth/hci/slip.py b/nordicsemi/bluetooth/hci/slip.py index 20df683..33a8e08 100644 --- a/nordicsemi/bluetooth/hci/slip.py +++ b/nordicsemi/bluetooth/hci/slip.py @@ -1,5 +1,4 @@ -# -# Copyright (c) 2019 Nordic Semiconductor ASA +# Copyright (c) 2016 - 2019 Nordic Semiconductor ASA # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, @@ -33,7 +32,6 @@ # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# import logging