Skip to content

Commit

Permalink
Merge branch '336-219-353-installer-do_update' into dev:
Browse files Browse the repository at this point in the history 
This started light-heartedly inspired by the remark in
955a15b about obsoleting AINIInfo -
we can't (more on that later) but I started looking at #336 again -
this led to extending the AFile API to cover installers, then to
reworking refresh and to some serious performance improvements:

__init__.py 4271 RefreshData boot:
d1ad84                             : 14636191 function calls (14609410 primitive calls) in 13.283 seconds
Nit and small fixes/opts           : 13706608 function calls (13679827 primitive calls) in 12.229 seconds
Rename Installer.modified attribute: 13379068 function calls (13352287 primitive calls) in 11.719 seconds
Installer.do_update                : 13179209 function calls (13152391 primitive calls) in 11.780 seconds
InstellersData.new_info            : 12290347 function calls (12263529 primitive calls) in 11.400 seconds
scandir IProject stat_tuple        : 12304196 function calls (12283979 primitive calls) in 10.239 seconds (Functions slightly up but times lower always)
Absorb _process_data_dir           : 11092826 function calls (11086518 primitive calls) in 6.727 seconds
Empties handling                   : 11092798 function calls (11086490 primitive calls) in 6.680 seconds
Drop pending_size handling         : 11085395 function calls (11079087 primitive calls) in 6.579 seconds

Same for tab in refresh:

__init__.py 4271 RefreshData tab in:
d1ad84                             : 284138 function calls (275354 primitive calls) in 0.737 seconds
Nit and small fixes/opts           : 283127 function calls (274343 primitive calls) in 0.689 seconds
Rename Installer.modified attribute: 283719 function calls (274935 primitive calls) in 0.697 seconds
Installer.do_update                : 496795 function calls (487975 primitive calls) in 0.580 seconds
InstellersData.new_info            : 485745 function calls (476926 primitive calls) in 0.885 seconds
scandir IProject stat_tuple        : 436784 function calls (434428 primitive calls) in 0.293 seconds (!)

In the second case further optimizations had no effect. As a bonus:

__init__.py 3455 ShowPanel full refresh:
d1ad84                             : 27182371 function calls (27129707 primitive calls) in 505.038 seconds
Rename Installer.modified attribute: 25924831 function calls (25872167 primitive calls) in 499.038 seconds
Installer.do_update                : 25923394 function calls (25870692 primitive calls) in 543.284 seconds
Drop pending_size handling         : 19720110 function calls (19714293 primitive calls) in 521.993 seconds

Don't mind the timings, as seen they fluctuate - and don't forget that
cProfile has an overhead. Dropping pending_size handling as expected
took down millions of calls - we should have a progress bar on the work
on progress. Note that I threw in some more opts before the final merge
(see 223f5e5) so timings are slightly
better even.

The main contribution here however was the great simplification of the
Installer/InstellersData refresh APIs. Performance was a result of this
rather than the goal. A lot of peculiarFunctions were absorbed by AFile
or were inlined - there are some more things to be done and some dusting
(see "cleanup after merge" commit) but I believe that refresh is not
anywhere near the opaque blob of nested function calls it used to be.

Best part of #336, ghost of #219, harbinger of #353
  • Loading branch information
@Utumno
Utumno committed Jul 16, 2023
2 parents d1ad842 + 3addb79 commit fbb1925
Show file tree
Hide file tree
Showing 10 changed files with 706 additions and 702 deletions.
119 changes: 58 additions & 61 deletions Mopy/bash/basher/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2593,7 +2593,7 @@ class InstallersList(UIList):
_sort_keys = {
u'Package' : None,
u'Order' : lambda self, x: self.data_store[x].order,
u'Modified': lambda self, x: self.data_store[x].modified,
u'Modified': lambda self, x: self.data_store[x].file_mod_time,
u'Size' : lambda self, x: self.data_store[x].fsize,
u'Files' : lambda self, x: self.data_store[x].num_of_files,
}
Expand All @@ -2612,7 +2612,7 @@ def _sortProjects(self, items):
labels = {
'Package': lambda self, p: p,
'Order': lambda self, p: f'{self.data_store[p].order}',
'Modified': lambda self, p: format_date(self.data_store[p].modified),
'Modified': lambda self, p: format_date(self.data_store[p].file_mod_time),
'Size': lambda self, p: self.data_store[p].size_string(),
'Files': lambda self, p: self.data_store[p].number_string(
self.data_store[p].num_of_files),
Expand Down Expand Up @@ -2959,8 +2959,9 @@ def addMarker(self):
new_marker = FName('====')
try:
index = self._get_uil_index(new_marker)
except KeyError: # u'====' not found in the internal dictionary
self.data_store.add_marker(new_marker, max_order)
except KeyError: # '====' not found in the internal dictionary
self.data_store.new_info(new_marker, install_order=max_order,
is_mark=True)
self.RefreshUI() # need to redraw all items cause order changed
index = self._get_uil_index(new_marker)
if index != -1:
Expand All @@ -2976,24 +2977,24 @@ def rescanInstallers(self, toRefresh, abort, update_from_data=True,
install, in case a refresh is requested because those files were
modified/deleted (BAIN only scans Data/ once or boot). If 'shallow' is
True (only the configurations of the installers changed) it will run
refreshDataSizeCrc of the installers, otherwise a full refreshBasic."""
toRefresh = list(self.data_store.ipackages(toRefresh))
refreshDataSizeCrc of the installers, otherwise a full _reset_cache."""
toRefresh = self.data_store.sorted_values(
self.data_store.ipackages(toRefresh))
if not toRefresh: return
try:
with balt.Progress(_('Refreshing Packages...'),
abort=abort) as progress:
progress.setFull(len(toRefresh))
dest = set() # installer's destination paths rel to Data/
for index, installer in enumerate(
self.data_store.sorted_values(toRefresh)):
for index, installer in enumerate(toRefresh):
progress(index,
_('Refreshing Packages...') + f'\n{installer}')
if shallow:
op = installer.refreshDataSizeCrc
else:
op = partial(installer.refreshBasic,
SubProgress(progress, index, index + 1),
calculate_projects_crc)
op = partial(installer._reset_cache,
progress=SubProgress(progress, index, index + 1),
recalculate_project_crc=calculate_projects_crc)
dest.update(op())
self.data_store.hasChanged = True # is it really needed ?
if update_from_data:
Expand Down Expand Up @@ -3222,7 +3223,7 @@ def _dumpFiles(files, header=u''):
inf_.extend([
_('Modified: %(modified_date)s') % {
'modified_date': 'N/A' if is_mark else
format_date(installer.modified)},
format_date(installer.file_mod_time)},
_('Data CRC: %(data_crc)s') % {
'data_crc': 'N/A' if is_mark else f'{installer.crc:08X}'},
_('Files: %(num_files)s') % {
Expand Down Expand Up @@ -3441,58 +3442,54 @@ def ShowPanel(self, canCancel=True, fullRefresh=False, scan_data_dir=False,
return
try:
self.refreshing = True
self._refresh_installers_if_needed(canCancel, fullRefresh,
scan_data_dir, focus_list)
super(InstallersPanel, self).ShowPanel()
finally:
self.refreshing = False

@balt.conversation
@bosh.bain.projects_walk_cache
def _refresh_installers_if_needed(self, canCancel, fullRefresh,
scan_data_dir, focus_list):
if settings.get('bash.installers.updatedCRCs',True): #only checked here
settings[u'bash.installers.updatedCRCs'] = False
self._data_dir_scanned = False
do_refresh = scan_data_dir = scan_data_dir or not self._data_dir_scanned
refresh_info = None
if self.frameActivated:
folders, files = map(list, top_level_items(bass.dirs[u'installers']))
omds = [fninst for fninst in files if
fninst.fn_ext in archives.omod_exts]
if any(inst_path not in omods.failedOmods for inst_path in omds):
omod_projects = self.__extractOmods(omds) ##: change above to filter?
if omod_projects:
deprint(f'Extending projects: {omod_projects}')
folders.extend(omod_projects)
if not do_refresh:
refresh_info = self.listData.scan_installers_dir(folders,
files, fullRefresh)
do_refresh = refresh_info.refresh_needed()
refreshui = False
if do_refresh:
with balt.Progress(_('Refreshing Installers...'),
abort=canCancel) as progress:
if settings.get('bash.installers.updatedCRCs', True): # only checked here
settings['bash.installers.updatedCRCs'] = False
self._data_dir_scanned = False
do_refresh = scan_data_dir = scan_data_dir or not self._data_dir_scanned
refresh_info = None
if self.frameActivated: # otherwise we are called directly
folders, files = map(list,
top_level_items(bass.dirs['installers']))
omds = [fninst for fninst in files if
fninst.fn_ext in archives.omod_exts]
if any(inst_path not in omods.failedOmods for inst_path in
omds):
omod_projects = self.__extractOmods(omds) ##: change above to filter?
if omod_projects:
deprint(f'Extending projects: {omod_projects}')
folders.extend(omod_projects)
if not do_refresh:
#with balt.Progress(_('Scanning Packages...')) as progress:
refresh_info = self.listData.update_installers(folders,
files, fullRefresh, progress=bolt.Progress())
do_refresh = refresh_info.refresh_needed()
refreshui = False
if do_refresh:
with balt.Progress(_('Refreshing Installers...'),
abort=canCancel) as progress:
try:
what = 'DISC' if scan_data_dir else 'IC'
refreshui |= self.listData.irefresh(progress, what,
fullRefresh,
refresh_info)
self.frameActivated = False
except CancelError:
self._user_cancelled = True # User canceled the refresh
finally:
self._data_dir_scanned = True
elif self.frameActivated:
try:
what = u'DISC' if scan_data_dir else u'IC'
refreshui |= self.listData.irefresh(progress, what,
fullRefresh,
refresh_info)
refreshui |= self.listData.irefresh(what='C',
fullRefresh=fullRefresh)
self.frameActivated = False
except CancelError:
self._user_cancelled = True # User canceled the refresh
finally:
self._data_dir_scanned = True
elif self.frameActivated:
try:
refreshui |= self.listData.irefresh(what='C',
fullRefresh=fullRefresh)
self.frameActivated = False
except CancelError:
pass # User canceled the refresh
do_refresh = self.listData.refreshTracked()
refreshui |= do_refresh and self.listData.refreshInstallersStatus()
if refreshui: self.uiList.RefreshUI(focus_list=focus_list)
pass # User canceled the refresh
do_refresh = self.listData.refreshTracked()
refreshui |= do_refresh and self.listData.refreshInstallersStatus()
if refreshui: self.uiList.RefreshUI(focus_list=focus_list)
super(InstallersPanel, self).ShowPanel()
finally:
self.refreshing = False

def __extractOmods(self, omds):
omod_projects = []
Expand Down
6 changes: 2 additions & 4 deletions Mopy/bash/basher/dialogs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,10 +278,8 @@ def OnClose(self):
new_installer_order = sel_installers[-1].order + 1
##: This is mostly copy-pasted from InstallerArchive_Unpack
with balt.Progress(_('Creating Project...')) as prog:
InstallerProject.refresh_installer(
fn_result_proj, self._parent.data_store, progress=prog,
install_order=new_installer_order, do_refresh=False)
self._parent.data_store.refresh_ns()
self._parent.data_store.new_info(fn_result_proj, progress=prog,
install_order=new_installer_order)
self._parent.RefreshUI(detail_item=fn_result_proj)
self._parent.SelectItemsNoCallback([fn_result_proj])

Expand Down
37 changes: 14 additions & 23 deletions Mopy/bash/basher/installer_links.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,7 @@
UIList_Hide
from ..belt import InstallerWizard, generateTweakLines
from ..bolt import FName, LogFile, SubProgress, deprint, round_size
from ..bosh import InstallerArchive, InstallerConverter, InstallerProject, \
converters
from ..bosh import InstallerConverter, InstallerProject, converters
from ..exception import CancelError, SkipError, StateError, XMLParsingError
from ..gui import BusyCursor, copy_text_to_clipboard

Expand Down Expand Up @@ -106,9 +105,8 @@ def _pack(self, archive_path, installer, project, release=False):
SubProgress(progress, 0, 0.8),
release=release)
#--Add the new archive to Bash
iArchive = InstallerArchive.refresh_installer(
archive_path, self.idata, progress=progress,
install_order=installer.order + 1, do_refresh=True)
iArchive = self.idata.new_info(archive_path, progress,
is_proj=False, install_order=installer.order + 1)
iArchive.blockSize = blockSize
self.window.RefreshUI(detail_item=archive_path)

Expand Down Expand Up @@ -287,11 +285,8 @@ def _execute_action(self, sel_package, ret, ui_refresh):
# to the final project, so clean it up
bass.rmTempDir()
with balt.Progress(_('Creating Project...')) as prog:
InstallerProject.refresh_installer(pr_path, self.idata,
progress=prog, install_order=sel_package.order + 1,
do_refresh=False)
##: Copied from InstallerArchive_Unpack - needed?
self.idata.refresh_ns()
self.idata.new_info(pr_path, prog,
install_order=sel_package.order + 1)

class Installer_EditWizard(_Installer_AViewOrEditFile):
"""View or edit the wizard.txt associated with this package."""
Expand Down Expand Up @@ -494,8 +489,7 @@ def Execute(self):
if not result: return
#--Duplicate
with BusyCursor():
self.idata.copy_installer(fn_inst, result)
self.idata.refresh_n()
self.idata.copy_installer(self._selected_info, result)
self.window.RefreshUI(detail_item=result)

class Installer_Hide(_InstallerLink, UIList_Hide):
Expand Down Expand Up @@ -547,9 +541,7 @@ def Execute(self):
skipRefresh is set to False."""
installer = self._selected_info
installer.skipRefresh ^= True
if not installer.skipRefresh:
installer.refreshBasic(progress=None,
recalculate_project_crc=False)
if installer.do_update(): # will return False if skipRefresh == True
installer.refreshStatus(self.idata)
self.idata.refresh_n()
self.window.RefreshUI()
Expand Down Expand Up @@ -970,8 +962,7 @@ def _copy_conflicts(curFile):
g_path = f'{order if order < src_order else order + 1:03d} ' \
f'- {package}'
curFile = _copy_conflicts(curFile)
InstallerProject.refresh_installer(fn_conflicts_dir, self.idata,
progress=None, install_order=src_order + 1, do_refresh=True)
self.idata.new_info(fn_conflicts_dir, install_order=src_order + 1)
self.window.RefreshUI(detail_item=fn_conflicts_dir)

#------------------------------------------------------------------------------
Expand Down Expand Up @@ -1171,8 +1162,7 @@ def Execute(self):
SubProgress(progress, 0, 0.8))
if not count_unpacked:
continue # no files were unpacked - stat would fail below
InstallerProject.refresh_installer(project, self.idata,
progress=SubProgress(progress, 0.8, 0.99),
self.idata.new_info(project, SubProgress(progress, 0.8, 0.99),
install_order=installer.order + 1, do_refresh=False)
projects.append(project)
if not projects: return
Expand Down Expand Up @@ -1240,13 +1230,14 @@ def Execute(self):
'act_deleted': actual_del, 'exp_deleted': len(ed_missing),
'act_updated': actual_upd,
'exp_updated': len(ed_mismatched)})
self._selected_info.refreshBasic(SubProgress(progress, 0.7, 0.8))
self._selected_info.do_update(force_update=True,
recalculate_project_crc=True,
progress=SubProgress(progress, 0.7, 0.8))
if was_rar:
final_package = self._selected_info.writable_archive_name()
# Move the new archive directly underneath the old archive
InstallerArchive.refresh_installer(
final_package, self.idata, do_refresh=False,
progress=SubProgress(progress, 0.8, 0.9),
self.idata.new_info(final_package, progress, is_proj=False,
do_refresh=False, _index=0.8,
install_order=self._selected_info.order + 1)
created_package = self.idata[final_package]
created_package.is_active = self._selected_info.is_active
Expand Down
Loading

0 comments on commit fbb1925

Please sign in to comment.