Merge pull request #772 from eblis/feature/load-progress

Fix for issue #741.

asammdf/blocks/mdf_v4.py

@@ -12,7 +12,7 @@
 from hashlib import md5
 from io import BufferedReader, BytesIO
 import logging
-from math import ceil
+from math import ceil, floor
 import mmap
 import os
 from pathlib import Path
@@ -160,6 +160,9 @@
 
 EMPTY_TUPLE = tuple()
 
+# 100 extra steps for the sorting, 1 step after sorting and 1 step at finish
+SORT_STEPS = 102
+
 
 logger = logging.getLogger("asammdf")
 
@@ -475,7 +478,7 @@ def _read(self, mapped: bool = False) -> None:
         stream.seek(0)
 
         cg_count, _ = count_channel_groups(stream)
-        progress_steps = cg_count + 2
+        progress_steps = cg_count + SORT_STEPS
         if self._callback:
             self._callback(0, progress_steps)
         current_cg_index = 0
@@ -775,9 +778,13 @@ def _read(self, mapped: bool = False) -> None:
                     else:
                         break
 
-        self._sort()
+        self._sort(
+            current_progress_index=current_cg_index, max_progress_count=progress_steps
+        )
         if self._callback:
-            self._callback(cg_count + 1, progress_steps)
+            self._callback(
+                progress_steps - 1, progress_steps
+            )  # second to last step now
 
         for grp in self.groups:
             channels = grp.channels
@@ -832,7 +839,9 @@ def _read(self, mapped: bool = False) -> None:
         self._attachments_map.clear()
 
         if self._callback:
-            self._callback(progress_steps, progress_steps)
+            self._callback(
+                progress_steps, progress_steps
+            )  # last step, we've completely loaded the file for sure
 
         self.progress = cg_count, cg_count
 
@@ -10191,7 +10200,12 @@ def _finalize(self) -> None:
             )
         self.identification.file_identification = b"MDF     "
 
-    def _sort(self, compress: bool = True) -> None:
+    def _sort(
+        self,
+        compress: bool = True,
+        current_progress_index: int = 0,
+        max_progress_count: int = 0,
+    ) -> None:
         if self._file is None:
             return
 
@@ -10247,7 +10261,13 @@ def _sort(self, compress: bool = True) -> None:
                 raise MdfException(message)
 
             rem = b""
-            for info in group.get_data_blocks():
+            blocks = list(group.get_data_blocks())  # might be expensive ?
+            # most of the steps are for sorting, but the last 2 are after we've done sorting
+            # so remove the 2 steps that are not related to sorting from the count
+            step = float(SORT_STEPS - 2) / len(blocks) / len(common)
+            index = float(current_progress_index)
+            previous = index
+            for info in blocks:
                 dtblock_address, dtblock_raw_size, dtblock_size, block_type, param = (
                     info.address,
                     info.original_size,
@@ -10256,6 +10276,20 @@ def _sort(self, compress: bool = True) -> None:
                     info.param,
                 )
 
+                index += step
+
+                # if we've been told to notify about progress
+                # and we've been given a max progress count (only way we can do progress updates)
+                # and there's a tick update (at least 1 integer between the last update and the current index)
+                # then we can notify about the callback progress
+                if (
+                    self._callback
+                    and max_progress_count
+                    and floor(previous) < floor(index)
+                ):
+                    self._callback(floor(index), max_progress_count)
+                    previous = index
+
                 seek(dtblock_address)
 
                 if block_type != v4c.DT_BLOCK: