Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

make outer_stack a parameter #148

Merged
merged 2 commits into from
Mar 4, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,20 @@ Read
>>>
>>> db.close()

Efficiency
----------

By default, sqlitedict's exception handling favors verbosity over efficiency.
It extracts and outputs the outer exception stack to the error logs.
If you favor efficiency, then initialize the DB with outer_stack=False.

.. code-block:: python

>>> from sqlitedict import SqliteDict
>>> db = SqliteDict("example.sqlite", outer_stack=False) # True is the default
>>> db[1]
{'name': 'first item'}

Context Manager
---------------

Expand Down
60 changes: 40 additions & 20 deletions sqlitedict.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,8 @@ class SqliteDict(DictClass):
VALID_FLAGS = ['c', 'r', 'w', 'n']

def __init__(self, filename=None, tablename='unnamed', flag='c',
autocommit=False, journal_mode="DELETE", encode=encode, decode=decode, timeout=5):
autocommit=False, journal_mode="DELETE", encode=encode,
decode=decode, timeout=5, outer_stack=True):
"""
Initialize a thread-safe sqlite-backed dictionary. The dictionary will
be a table `tablename` in database file `filename`. A single file (=database)
Expand All @@ -125,6 +126,10 @@ def __init__(self, filename=None, tablename='unnamed', flag='c',
Set `journal_mode` to 'OFF' if you're experiencing sqlite I/O problems
or if you need performance and don't care about crash-consistency.

Set `outer_stack` to False to disable the output of the outer exception
to the error logs. This may improve the efficiency of sqlitedict
operation at the expense of a detailed exception trace.

The `flag` parameter. Exactly one of:
'c': default mode, open for read/write, creating the db/table if necessary.
'w': open for r/w, but drop `tablename` contents first (start with empty table)
Expand Down Expand Up @@ -172,6 +177,7 @@ def __init__(self, filename=None, tablename='unnamed', flag='c',
self.encode = encode
self.decode = decode
self.timeout = timeout
self._outer_stack = outer_stack

logger.info("opening Sqlite table %r in %r" % (tablename, filename))
self.conn = self._new_conn()
Expand All @@ -187,8 +193,13 @@ def __init__(self, filename=None, tablename='unnamed', flag='c',
self.clear()

def _new_conn(self):
return SqliteMultithread(self.filename, autocommit=self.autocommit, journal_mode=self.journal_mode,
timeout=self.timeout)
return SqliteMultithread(
self.filename,
autocommit=self.autocommit,
journal_mode=self.journal_mode,
timeout=self.timeout,
outer_stack=self._outer_stack,
)

def __enter__(self):
if not hasattr(self, 'conn') or self.conn is None:
Expand Down Expand Up @@ -375,12 +386,6 @@ def __del__(self):
pass


# Adding extra methods for python 2 compatibility (at import time)
if major_version == 2:
SqliteDict.__nonzero__ = SqliteDict.__bool__
del SqliteDict.__bool__ # not needed and confusing


class SqliteMultithread(Thread):
"""
Wrap sqlite connection in a way that allows concurrent requests from multiple threads.
Expand All @@ -389,7 +394,7 @@ class SqliteMultithread(Thread):
in a separate thread (in the same order they arrived).

"""
def __init__(self, filename, autocommit, journal_mode, timeout):
def __init__(self, filename, autocommit, journal_mode, timeout, outer_stack=True):
super(SqliteMultithread, self).__init__()
self.filename = filename
self.autocommit = autocommit
Expand All @@ -399,6 +404,7 @@ def __init__(self, filename, autocommit, journal_mode, timeout):
self.setDaemon(True) # python2.5-compatible
self.exception = None
self._sqlitedict_thread_initialized = None
self._outer_stack = outer_stack
self.timeout = timeout
self.log = logging.getLogger('sqlitedict.SqliteMultithread')
self.start()
Expand Down Expand Up @@ -462,10 +468,18 @@ def run(self):
self.log.error(item)

self.log.error('') # exception & outer stack w/blank line
self.log.error('Outer stack:')
for item in traceback.format_list(outer_stack):
self.log.error(item)
self.log.error('Exception will be re-raised at next call.')

if self._outer_stack:
self.log.error('Outer stack:')
for item in traceback.format_list(outer_stack):
self.log.error(item)
self.log.error('Exception will be re-raised at next call.')
else:
self.log.error(
'Unable to show the outer stack. Pass '
'outer_stack=True when initializing the '
'SqliteDict instance to show the outer stack.'
)

if res:
for rec in cursor:
Expand Down Expand Up @@ -513,12 +527,15 @@ def execute(self, req, arg=None, res=None):
"""
self._wait_for_initialization()
self.check_raise_error()
stack = None

if self._outer_stack:
# NOTE: This might be a lot of information to pump into an input
# queue, affecting performance. I've also seen earlier versions of
# jython take a severe performance impact for throwing exceptions
# so often.
stack = traceback.extract_stack()[:-1]

# NOTE: This might be a lot of information to pump into an input
# queue, affecting performance. I've also seen earlier versions of
# jython take a severe performance impact for throwing exceptions
# so often.
stack = traceback.extract_stack()[:-1]
self.reqs.put((req, arg or tuple(), res, stack))

def executemany(self, req, items):
Expand Down Expand Up @@ -594,8 +611,11 @@ def _wait_for_initialization(self):
if self._sqlitedict_thread_initialized or self.exception:
return
time.sleep(0.1)
raise TimeoutError("SqliteMultithread failed to flag initialization withing %0.0f seconds." % self.timeout)
raise TimeoutError("SqliteMultithread failed to flag initialization within %0.0f seconds." % self.timeout)


#
# This is here for .github/workflows/release.yml
#
if __name__ == '__main__':
print(__version__)