Add Support for Github Actions workflows (#798)

* Move all tests file tests folder in root path
* Update tox.ini
* Create ci.yml
* Update test file to support workflows
* Move all tests file tests folder in root path
* Adapted arxml.py to satisefied Test File
* Update Test & be able to Encode with phys data
* Update ci.yml
* Update setup.py to satify workflows
* revert some change & Correct phys2raw & Revert to use Pathlib in cli.compare cli.convert
* Update Arxml Ethernet Endpoints -Add TTL

.github/workflows/ci.yml

@@ -0,0 +1,152 @@
+name: Tests
+
+on:
+  release:
+    types: [ published ]
+  pull_request:
+  push:
+
+env:
+  PY_COLORS: "1"
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }} # See: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontinue-on-error
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        # os: [ubuntu-latest]
+        experimental: [false]
+        # python-version: ["2.7","3.4","3.5","3.6","3.7","3.8","3.9","3.10","3.11","3.12"]
+        python-version: ["3.7","3.8","3.9","3.10","3.11","3.12"]
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+    - name: Test with pytest via tox
+      run: |
+        tox -e gh
+    - name: Coveralls
+      uses: coverallsapp/github-action@v2.3.0
+      with:
+        github-token: ${{ secrets.github_token }}
+        flag-name: Test_${{ matrix.os }}_${{ matrix.python-version }}
+        parallel: true
+        path-to-lcov: ./coverage.lcov
+
+  coveralls:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Coveralls Finished
+        uses: coverallsapp/github-action@v2.3.0
+        with:
+          github-token: ${{ secrets.github_token }}
+          parallel-finished: true
+
+  # static-code-analysis:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #   - uses: actions/checkout@v3
+  #   - name: Set up Python
+  #     uses: actions/setup-python@v4
+  #     with:
+  #       python-version: "3.10"
+  #   - name: Install dependencies
+  #     run: |
+  #       python -m pip install --upgrade pip
+  #       pip install -e .[lint]
+  #   - name: ruff
+  #     run: |
+  #       ruff check can
+  #   - name: pylint
+  #     run: |
+  #       pylint \
+  #       src/**.py \
+  #       can/io \
+  #       doc/conf.py \
+  #       examples/**.py \
+
+  # format:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #   - uses: actions/checkout@v3
+  #   - name: Set up Python
+  #     uses: actions/setup-python@v4
+  #     with:
+  #       python-version: "3.10"
+  #   - name: Install dependencies
+  #     run: |
+  #       python -m pip install --upgrade pip
+  #       pip install -e .[lint]
+  #   - name: Code Format Check with Black
+  #     run: |
+  #       black --check --verbose .
+
+  # docs:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #   - uses: actions/checkout@v3
+  #   - name: Set up Python
+  #     uses: actions/setup-python@v4
+  #     with:
+  #       python-version: "3.10"
+  #   - name: Install dependencies
+  #     run: |
+  #       python -m pip install --upgrade pip
+  #       pip install -e .[canalystii,gs_usb,mf4]
+  #       pip install -r doc/doc-requirements.txt
+  #   - name: Build documentation
+  #     run: |
+  #       python -m sphinx -Wan --keep-going doc build
+  #   - name: Run doctest
+  #     run: |
+  #       python -m sphinx -b doctest -W --keep-going doc build
+  #   - uses: actions/upload-artifact@v3
+  #     with:
+  #       name: sphinx-out
+  #       path: ./build/
+  #       retention-days: 5
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Build wheel and sdist
+        run: pipx run build
+      - name: Check build artifacts
+        run: pipx run twine check --strict dist/*
+      - name: Save artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: python-can-dist
+          path: ./dist
+
+  upload_pypi:
+    needs: [build]
+    runs-on: ubuntu-latest
+
+    # upload to PyPI only on release
+    if: github.event.release && github.event.action == 'published'
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: python-can-dist
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@v1.4.2
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

setup.py

@@ -81,6 +81,7 @@
     name = "canmatrix",
     version = versioneer.get_version(),
     cmdclass = versioneer.get_cmdclass(),
+    long_description_content_type='text/x-rst',
     maintainer = "Eduard Broecker",
     maintainer_email = "eduard@gmx.de",
     url = "http://github.com/ebroecker/canmatrix",

src/canmatrix/canmatrix.py

@@ -436,25 +436,28 @@ def phys2raw(self, value=None):
             if not (self.min <= value <= self.max):
                 value = self.min
 
-        if isinstance(value, str):
+        if isinstance(value, str) and self.values:
             for value_key, value_string in self.values.items():
                 if value_string == value:
                     value = value_key
                     return value
-            else:
-                raise ValueError(
-                        "{} is invalid value choice for {}".format(value, self)
-                )
 
+        try:
+            value = decimal.Decimal(value)
+        except Exception as e:
+            raise e
+
+        # if not (0 <= value <= 10):
         if not (self.min <= value <= self.max):
-            logger.info(
+            logger.warning(
                 "Value {} is not valid for {}. Min={} and Max={}".format(
                     value, self, self.min, self.max)
                 )
         raw_value = (self.float_factory(value) - self.float_factory(self.offset)) / self.float_factory(self.factor)
 
         if not self.is_float:
             raw_value = int(round(raw_value))
+
         return raw_value
 
     def raw2phys(self, value, decode_to_str=False):
@@ -793,6 +796,7 @@ class Endpoint(object):
     server_port = attr.ib(default=0)  # type: int
     client_ip = attr.ib(default="")  # type: str
     client_port = attr.ib(default=0)  # type: int
+    ttl = attr.ib(default=0)  # type: int
 
 
 @attr.s(eq=False)
@@ -1335,6 +1339,11 @@ def signals_to_bytes(self, data):
         for signal in self.signals:
             if signal.name in data:
                 value = data.get(signal.name)
+                if isinstance(value, str):
+                    value = signal.phys2raw(value)
+                    if value is None:
+                        # TODO Error Handling
+                        value = 0
                 bits = pack_bitstring(signal.size, signal.is_float, value, signal.is_signed)
 
                 if signal.is_little_endian:
@@ -1369,7 +1378,6 @@ def encode(self, data=None):
         """
 
         data = dict() if data is None else data
-
         if self.is_complex_multiplexed:
             raise EncodingComplexMultiplexed
         elif self.is_pdu_container:

src/canmatrix/cli/compare.py

@@ -105,4 +105,4 @@ def cli_compare(matrix1, matrix2, verbosity, silent, check_comments, check_attri
 
 # to be run as module `python -m canmatrix.compare`, NOT as script with argument `canmatrix/compare.py`
 if __name__ == '__main__':
-    sys.exit(cli_compare())
+    sys.exit(cli_compare())

src/canmatrix/cli/convert.py

@@ -140,7 +140,7 @@ def cli_convert(infile, outfile, silent, verbosity, **options):
     import-file: *.dbc|*.dbf|*.kcd|*.arxml|*.json|*.xls(x)|*.sym
     export-file: *.dbc|*.dbf|*.kcd|*.arxml|*.json|*.xls(x)|*.sym|*.py
 
-    \n"""
+    """
 
     root_logger = canmatrix.log.setup_logger()
 
@@ -160,4 +160,4 @@ def cli_convert(infile, outfile, silent, verbosity, **options):
 
 
 if __name__ == '__main__':
-    sys.exit(cli_convert())
+    sys.exit(cli_convert())

src/canmatrix/convert.py

@@ -64,12 +64,11 @@ def convert_pdu_container_to_multiplexed(frame):  # type: (canmatrix.Frame) -> c
 
 
 def convert(infile, out_file_name, **options):  # type: (str, str, **str) -> None
-    logger.info("Importing " + infile + " ... ")
+    logger.info(f"Importing " + infile + " ...")
     dbs = canmatrix.formats.loadp(infile, **options)
-    logger.info("done\n")
-
-    logger.info("Exporting " + out_file_name + " ... ")
+    logger.info("Import Done")
 
+    logger.info("Exporting " + out_file_name + " ...")
     out_dbs = {}  # type: typing.Dict[str, canmatrix.CanMatrix]
     for name in dbs:
         db = None
@@ -104,7 +103,7 @@ def convert(infile, out_file_name, **options):  # type: (str, str, **str) -> Non
                 db_temp_list = canmatrix.formats.loadp(merge_string[0])
                 for dbTemp in db_temp_list:
                     if merge_string.__len__() == 1:
-                        print("merge complete: " + merge_string[0])
+                        # logger.debug("merge complete: " + merge_string[0])
                         db.merge([db_temp_list[dbTemp]])
                         # for frame in db_temp_list[dbTemp].frames:
                         #    copyResult = canmatrix.copy.copy_frame(frame.id, db_temp_list[dbTemp], db)
@@ -337,9 +336,7 @@ def convert(infile, out_file_name, **options):  # type: (str, str, **str) -> Non
                 frame.is_j1939=True
             db.add_attribute("ProtocolType", "J1939")
 
-
-
-        logger.info(name)
+        logger.debug(f"{name}")
         logger.info("%d Frames found" % (db.frames.__len__()))
 
         out_dbs[name] = db
@@ -349,4 +346,4 @@ def convert(infile, out_file_name, **options):  # type: (str, str, **str) -> Non
                                 'force_output'], **options)
     else:
         canmatrix.formats.dumpp(out_dbs, out_file_name, **options)
-    logger.info("done")
+    logger.info("Export Done")

src/canmatrix/copy.py

@@ -70,7 +70,7 @@ def copy_ecu(ecu_or_glob, source_db, target_db):
 
 
 def copy_ecu_with_frames(ecu_or_glob, source_db, target_db, rx=True, tx=True, direct_ecu_only=True):
-    # type: (typing.Union[canmatrix.Ecu, str], canmatrix.CanMatrix, canmatrix.CanMatrix) -> None
+    # type: (typing.Union[canmatrix.Ecu, str], canmatrix.CanMatrix, canmatrix.CanMatrix, bool, bool, bool) -> None
     """
     Copy ECU(s) identified by Name or as Object from source CAN matrix to target CAN matrix.
     This function additionally copy all relevant Frames and Defines.

src/canmatrix/formats/arxml.py

@@ -1048,9 +1048,8 @@ def decode_compu_method(compu_method, ea, float_factory):
 
         rational = ea.get_child(compu_scale, "COMPU-RATIONAL-COEFFS")
 
-        if rational is None and ll is not None and desc is not None and canmatrix.utils.decode_number(ul.text,
-                                                                                 float_factory) == canmatrix.utils.decode_number(
-            ll.text, float_factory):
+        if rational is None and ll is not None and desc is not None and \
+            canmatrix.utils.decode_number(ul.text,float_factory) == canmatrix.utils.decode_number(ll.text, float_factory):
             #####################################################################################################
             #####################################################################################################
             values[ll.text] = desc
@@ -1600,23 +1599,26 @@ def get_frame(frame_triggering, ea, multiplex_translation, float_factory, header
 
         secOC_properties = None
         if pdu is not None and 'SECURED-I-PDU' in pdu.tag:
-            payload_length = ea.get_child(pdu, "LENGTH").text
+            try:
+                payload_length = ea.get_child(pdu, "LENGTH").text
 
-            secured_ipdu_SecoC = ea.get_child(pdu, "SECURE-COMMUNICATION-PROPS")
+                secured_ipdu_SecoC = ea.get_child(pdu, "SECURE-COMMUNICATION-PROPS")
 
-            auth_algorithm = ea.get_child(secured_ipdu_SecoC, "AUTH-ALGORITHM").text
-            auth_tx_length = ea.get_child(secured_ipdu_SecoC, "AUTH-INFO-TX-LENGTH").text
-            data_id = ea.get_child(secured_ipdu_SecoC, "DATA-ID").text
-            freshness_bit_length = ea.get_child(secured_ipdu_SecoC, "FRESHNESS-VALUE-LENGTH").text
-            freshness_tx_length = ea.get_child(secured_ipdu_SecoC, "FRESHNESS-VALUE-TX-LENGTH").text
-
-            secOC_properties = canmatrix.AutosarSecOCProperties(auth_algorithm, 
-                                                               int(payload_length, 0),
-                                                               int(auth_tx_length, 0),
-                                                               int(data_id, 0),
-                                                               int(freshness_bit_length, 0),
-                                                               int(freshness_tx_length, 0)
-                                                               )
+                auth_algorithm = ea.get_child(secured_ipdu_SecoC, "AUTH-ALGORITHM").text
+                auth_tx_length = ea.get_child(secured_ipdu_SecoC, "AUTH-INFO-TX-LENGTH").text
+                data_id = ea.get_child(secured_ipdu_SecoC, "DATA-ID").text
+                freshness_bit_length = ea.get_child(secured_ipdu_SecoC, "FRESHNESS-VALUE-LENGTH").text
+                freshness_tx_length = ea.get_child(secured_ipdu_SecoC, "FRESHNESS-VALUE-TX-LENGTH").text
+
+                secOC_properties = canmatrix.AutosarSecOCProperties(auth_algorithm, 
+                                                                   int(payload_length, 0),
+                                                                   int(auth_tx_length, 0),
+                                                                   int(data_id, 0),
+                                                                   int(freshness_bit_length, 0),
+                                                                   int(freshness_tx_length, 0)
+                                                                   )
+            except Exception as e:
+                logger.warning(f"{e}")
 
             ipdu = ea.selector(pdu, ">PAYLOAD-REF>I-PDU-REF")
             if not ipdu:
@@ -1898,9 +1900,13 @@ def decode_ethernet_helper(ea, float_factory):
                     client_app_endpoint = ea.get_child(client_port_ref, "APPLICATION-ENDPOINT")
                     client_endpoint_ref = ea.follow_ref(client_app_endpoint, "NETWORK-ENDPOINT-REF")
                     client_ipv4 = ea.find("IPV-4-ADDRESS", client_endpoint_ref)
+                    ttl = ea.find("TTL", client_endpoint_ref)
 
-                    endpoint = canmatrix.Endpoint(server_ipv4.text, int(server_port.text, 0), 
-                                                client_ipv4.text, int(client_port.text, 0))
+                    endpoint = canmatrix.Endpoint(server_ipv4.text, 
+                                                  int(server_port.text, 0),
+                                                  client_ipv4.text, 
+                                                  int(client_port.text, 0),
+                                                  ttl)
 
                     for scii in ea.findall("SOCKET-CONNECTION-IPDU-IDENTIFIER", socket_connection):
 
@@ -2076,8 +2082,9 @@ def decode_can_helper(ea, float_factory, ignore_cluster_info):
         multiplex_translation = {}  # type: typing.Dict[str, str]
         for frameTrig in can_frame_trig:  # type: _Element
             frame = get_frame(frameTrig, ea, multiplex_translation, float_factory, headers_are_littleendian)
-            frame.is_j1939 = "J-1939" in cc.tag
             if frame is not None:
+                frame.is_j1939 = "J-1939" in cc.tag
+
                 comm_directions = ea.selector(frameTrig, ">>FRAME-PORT-REF/COMMUNICATION-DIRECTION")
                 for comm_direction in comm_directions:
                     ecu_elem = ea.get_ecu_instance(element=comm_direction)

src/canmatrix/formats/odx.py

@@ -168,7 +168,7 @@ def get_odx_info(eo, element_type):
     for request in eo.findall(element_type):
         short_name = eo.get_short_name(request)
         service_id_param = find_param(eo, request, "SERVICE-ID")
-        if service_id_param == None:
+        if service_id_param is None:
             continue
 
         service_id_value = eo.find("CODED-VALUE", service_id_param)
@@ -178,7 +178,7 @@ def get_odx_info(eo, element_type):
             continue
 
         id_param = find_param(eo, request, "ID")
-        if id_param == None:
+        if id_param is None:
             continue
         did_value = eo.find("CODED-VALUE", id_param)
         did = int(did_value.text)