make SA engine configuration more flexible

docs/config.rst

@@ -1,3 +1,5 @@
+.. currentmodule:: flask_sqlalchemy
+
 Configuration
 =============
 
@@ -37,10 +39,16 @@ A list of configuration keys currently understood by the extension:
                                    on some Ubuntu versions) when used with
                                    improper database defaults that specify
                                    encoding-less databases.
+
+                                   **Deprecated** as of v2.4 and will be removed in v3.0.
 ``SQLALCHEMY_POOL_SIZE``           The size of the database pool.  Defaults
-                                   to the engine's default (usually 5)
+                                   to the engine's default (usually 5).
+
+                                   **Deprecated** as of v2.4 and will be removed in v3.0.
 ``SQLALCHEMY_POOL_TIMEOUT``        Specifies the connection timeout in seconds
                                    for the pool.
+
+                                   **Deprecated** as of v2.4 and will be removed in v3.0.
 ``SQLALCHEMY_POOL_RECYCLE``        Number of seconds after which a
                                    connection is automatically recycled.
                                    This is required for MySQL, which removes
@@ -51,18 +59,25 @@ A list of configuration keys currently understood by the extension:
                                    different default timeout value. For more
                                    information about timeouts see
                                    :ref:`timeouts`.
+
+                                   **Deprecated** as of v2.4 and will be removed in v3.0.
 ``SQLALCHEMY_MAX_OVERFLOW``        Controls the number of connections that
                                    can be created after the pool reached
                                    its maximum size.  When those additional
                                    connections are returned to the pool,
                                    they are disconnected and discarded.
+
+                                   **Deprecated** as of v2.4 and will be removed in v3.0.
 ``SQLALCHEMY_TRACK_MODIFICATIONS`` If set to ``True``, Flask-SQLAlchemy will
                                    track modifications of objects and emit
                                    signals.  The default is ``None``, which
                                    enables tracking but issues a warning
                                    that it will be disabled by default in
                                    the future.  This requires extra memory
                                    and should be disabled if not needed.
+``SQLALCHEMY_ENGINE_OPTIONS``      A dictionary of keyword args to send to
+                                   :func:`~sqlalchemy.create_engine`.  See
+                                   also ``engine_options`` to :class:`SQLAlchemy`.
 ================================== =========================================
 
 .. versionadded:: 0.8
@@ -78,9 +93,22 @@ A list of configuration keys currently understood by the extension:
 
 .. versionadded:: 2.0
    The ``SQLALCHEMY_TRACK_MODIFICATIONS`` configuration key was added.
+
 .. versionchanged:: 2.1
    ``SQLALCHEMY_TRACK_MODIFICATIONS`` will warn if unset.
 
+.. versionchanged:: 2.4
+
+* ``SQLALCHEMY_ENGINE_OPTIONS`` configuration key was added.
+* Deprecated keys
+
+  * ``SQLALCHEMY_NATIVE_UNICODE``
+  * ``SQLALCHEMY_POOL_SIZE``
+  * ``SQLALCHEMY_POOL_TIMEOUT``
+  * ``SQLALCHEMY_POOL_RECYCLE``
+  * ``SQLALCHEMY_MAX_OVERFLOW``
+
+
 Connection URI Format
 ---------------------
 

flask_sqlalchemy/__init__.py

@@ -31,6 +31,7 @@
 from flask_sqlalchemy.model import Model
 from ._compat import itervalues, string_types, xrange
 from .model import DefaultMeta
+from . import utils
 
 __version__ = '2.3.2'
 
@@ -157,7 +158,14 @@ def __init__(self, db, autocommit=False, autoflush=True, **options):
     def get_bind(self, mapper=None, clause=None):
         # mapper is None if someone tries to just get a connection
         if mapper is not None:
-            info = getattr(mapper.mapped_table, 'info', {})
+            try:
+                # SA >= 1.3
+                persist_selectable = mapper.persist_selectable
+            except AttributeError:
+                # SA < 1.3
+                persist_selectable = mapper.mapped_table
+
+            info = getattr(persist_selectable, 'info', {})
             bind_key = info.get('bind_key')
             if bind_key is not None:
                 state = get_state(self.app)
@@ -553,19 +561,36 @@ def get_engine(self):
             echo = self._app.config['SQLALCHEMY_ECHO']
             if (uri, echo) == self._connected_for:
                 return self._engine
-            info = make_url(uri)
-            options = {'convert_unicode': True}
-            self._sa.apply_pool_defaults(self._app, options)
-            self._sa.apply_driver_hacks(self._app, info, options)
-            if echo:
-                options['echo'] = echo
-            self._engine = rv = sqlalchemy.create_engine(info, **options)
+
+            sa_url = make_url(uri)
+            options = self.get_options(sa_url, echo)
+            self._engine = rv = self._sa.create_engine(sa_url, options)
+
             if _record_queries(self._app):
                 _EngineDebuggingSignalEvents(self._engine,
                                              self._app.import_name).register()
+
             self._connected_for = (uri, echo)
+
             return rv
 
+    def get_options(self, sa_url, echo):
+        options = {}
+
+        self._sa.apply_pool_defaults(self._app, options)
+        self._sa.apply_driver_hacks(self._app, sa_url, options)
+        if echo:
+            options['echo'] = echo
+
+        # Give the config options set by a developer explicitly priority
+        # over decisions FSA makes.
+        options.update(self._app.config['SQLALCHEMY_ENGINE_OPTIONS'])
+
+        # Give options set in SQLAlchemy.__init__() ultimate priority
+        options.update(self._sa._engine_options)
+
+        return options
+
 
 def get_state(app):
     """Gets the state for the application"""
@@ -610,13 +635,18 @@ def create_app():
     the second case a :meth:`flask.Flask.app_context` has to exist.
 
     By default Flask-SQLAlchemy will apply some backend-specific settings
-    to improve your experience with them.  As of SQLAlchemy 0.6 SQLAlchemy
+    to improve your experience with them.
+
+    As of SQLAlchemy 0.6 SQLAlchemy
     will probe the library for native unicode support.  If it detects
     unicode it will let the library handle that, otherwise do that itself.
     Sometimes this detection can fail in which case you might want to set
     ``use_native_unicode`` (or the ``SQLALCHEMY_NATIVE_UNICODE`` configuration
     key) to ``False``.  Note that the configuration key overrides the
-    value you pass to the constructor.
+    value you pass to the constructor.  Direct support for ``use_native_unicode``
+    and SQLALCHEMY_NATIVE_UNICODE are deprecated as of v2.4 and will be removed
+    in v3.0.  ``engine_options`` and ``SQLALCHEMY_ENGINE_OPTIONS`` may be used
+    instead.
 
     This class also provides access to all the SQLAlchemy functions and classes
     from the :mod:`sqlalchemy` and :mod:`sqlalchemy.orm` modules.  So you can
@@ -644,6 +674,12 @@ class User(db.Model):
     to be passed to the session constructor.  See :class:`~sqlalchemy.orm.session.Session`
     for the standard options.
 
+    The ``engine_options`` parameter, if provided, is a dict of parameters
+    to be passed to create engine.  See :func:`~sqlalchemy.create_engine`
+    for the standard options.  The values given here will be merged with and
+    override anything set in the ``'SQLALCHEMY_ENGINE_OPTIONS'`` config
+    variable or othewise set by this library.
+
     .. versionadded:: 0.10
        The `session_options` parameter was added.
 
@@ -663,6 +699,12 @@ class to be used in place of :class:`Model`.
 
     .. versionchanged:: 2.1
        Utilise the same query class across `session`, `Model.query` and `Query`.
+
+    .. versionadded:: 2.4
+       The `engine_options` parameter was added.
+
+    .. versionchanged:: 2.4
+       The `use_native_unicode` parameter was deprecated.
     """
 
     #: Default query class used by :attr:`Model.query` and other queries.
@@ -671,14 +713,16 @@ class to be used in place of :class:`Model`.
     Query = None
 
     def __init__(self, app=None, use_native_unicode=True, session_options=None,
-                 metadata=None, query_class=BaseQuery, model_class=Model):
+                 metadata=None, query_class=BaseQuery, model_class=Model,
+                 engine_options=None):
 
         self.use_native_unicode = use_native_unicode
         self.Query = query_class
         self.session = self.create_scoped_session(session_options)
         self.Model = self.make_declarative_base(model_class, metadata)
         self._engine_lock = Lock()
         self.app = app
+        self._engine_options = engine_options or {}
         _include_sqlalchemy(self, query_class)
 
         if app is not None:
@@ -790,6 +834,7 @@ def init_app(self, app):
         track_modifications = app.config.setdefault(
             'SQLALCHEMY_TRACK_MODIFICATIONS', None
         )
+        app.config.setdefault('SQLALCHEMY_ENGINE_OPTIONS', {})
 
         if track_modifications is None:
             warnings.warn(FSADeprecationWarning(
@@ -798,6 +843,12 @@ def init_app(self, app):
                 'or False to suppress this warning.'
             ))
 
+        # Deprecation warnings for config keys that should be replaced by SQLALCHEMY_ENGINE_OPTIONS.
+        utils.engine_config_warning(app.config, '3.0', 'SQLALCHEMY_POOL_SIZE', 'pool_size')
+        utils.engine_config_warning(app.config, '3.0', 'SQLALCHEMY_POOL_TIMEOUT', 'pool_timeout')
+        utils.engine_config_warning(app.config, '3.0', 'SQLALCHEMY_POOL_RECYCLE', 'pool_recycle')
+        utils.engine_config_warning(app.config, '3.0', 'SQLALCHEMY_MAX_OVERFLOW', 'max_overflow')
+
         app.extensions['sqlalchemy'] = _SQLAlchemyState(self)
 
         @app.teardown_appcontext
@@ -819,7 +870,7 @@ def _setdefault(optionkey, configkey):
         _setdefault('pool_recycle', 'SQLALCHEMY_POOL_RECYCLE')
         _setdefault('max_overflow', 'SQLALCHEMY_MAX_OVERFLOW')
 
-    def apply_driver_hacks(self, app, info, options):
+    def apply_driver_hacks(self, app, sa_url, options):
         """This method is called before engine creation and used to inject
         driver specific hacks into the options.  The `options` parameter is
         a dictionary of keyword arguments that will then be used to call
@@ -829,15 +880,15 @@ def apply_driver_hacks(self, app, info, options):
         like pool sizes for MySQL and sqlite.  Also it injects the setting of
         `SQLALCHEMY_NATIVE_UNICODE`.
         """
-        if info.drivername.startswith('mysql'):
-            info.query.setdefault('charset', 'utf8')
-            if info.drivername != 'mysql+gaerdbms':
+        if sa_url.drivername.startswith('mysql'):
+            sa_url.query.setdefault('charset', 'utf8')
+            if sa_url.drivername != 'mysql+gaerdbms':
                 options.setdefault('pool_size', 10)
                 options.setdefault('pool_recycle', 7200)
-        elif info.drivername == 'sqlite':
+        elif sa_url.drivername == 'sqlite':
             pool_size = options.get('pool_size')
             detected_in_memory = False
-            if info.database in (None, '', ':memory:'):
+            if sa_url.database in (None, '', ':memory:'):
                 detected_in_memory = True
                 from sqlalchemy.pool import StaticPool
                 options['poolclass'] = StaticPool
@@ -860,14 +911,27 @@ def apply_driver_hacks(self, app, info, options):
 
             # if it's not an in memory database we make the path absolute.
             if not detected_in_memory:
-                info.database = os.path.join(app.root_path, info.database)
+                sa_url.database = os.path.join(app.root_path, sa_url.database)
 
         unu = app.config['SQLALCHEMY_NATIVE_UNICODE']
         if unu is None:
             unu = self.use_native_unicode
         if not unu:
             options['use_native_unicode'] = False
 
+        if app.config['SQLALCHEMY_NATIVE_UNICODE'] is not None:
+            warnings.warn(
+                "The 'SQLALCHEMY_NATIVE_UNICODE' config option is deprecated and will be removed in"
+                " v3.0.  Use 'SQLALCHEMY_ENGINE_OPTIONS' instead.",
+                DeprecationWarning
+            )
+        if not self.use_native_unicode:
+            warnings.warn(
+                "'use_native_unicode' is deprecated and will be removed in v3.0."
+                "  Use the 'engine_options' parameter instead.",
+                DeprecationWarning
+            )
+
     @property
     def engine(self):
         """Gives access to the engine.  If the database configuration is bound
@@ -897,6 +961,16 @@ def get_engine(self, app=None, bind=None):
 
             return connector.get_engine()
 
+    def create_engine(self, sa_url, engine_opts):
+        """
+            Override this method to have final say over how the SQLAlchemy engine
+            is created.
+
+            In most cases, you will want to use ``'SQLALCHEMY_ENGINE_OPTIONS'``
+            config variable or set ``engine_options`` for :func:`SQLAlchemy`.
+        """
+        return sqlalchemy.create_engine(sa_url, **engine_opts)
+
     def get_app(self, reference_app=None):
         """Helper method that implements the logic to look up an
         application."""

flask_sqlalchemy/utils.py

@@ -0,0 +1,45 @@
+import warnings
+
+import sqlalchemy
+
+
+def parse_version(v):
+    """
+        Take a string version and conver it to a tuple (for easier comparison), e.g.:
+
+            "1.2.3" --> (1, 2, 3)
+            "1.2" --> (1, 2, 0)
+            "1" --> (1, 0, 0)
+    """
+    parts = v.split(".")
+    # Pad the list to make sure there is three elements so that we get major, minor, point
+    # comparisons that default to "0" if not given.  I.e. "1.2" --> (1, 2, 0)
+    parts = (parts + 3 * ['0'])[:3]
+    return tuple(int(x) for x in parts)
+
+
+def sqlalchemy_version(op, val):
+    sa_ver = parse_version(sqlalchemy.__version__)
+    target_ver = parse_version(val)
+
+    assert op in ('<', '>', '<=', '>=', '=='), 'op {} not supported'.format(op)
+
+    if op == '<':
+        return sa_ver < target_ver
+    if op == '>':
+        return sa_ver > target_ver
+    if op == '<=':
+        return sa_ver <= target_ver
+    if op == '>=':
+        return sa_ver >= target_ver
+    return sa_ver == target_ver
+
+
+def engine_config_warning(config, version, deprecated_config_key, engine_option):
+    if config[deprecated_config_key] is not None:
+        warnings.warn(
+            'The `{}` config option is deprecated and will be removed in'
+            ' v{}.  Use `SQLALCHEMY_ENGINE_OPTIONS[\'{}\']` instead.'
+            .format(deprecated_config_key, version, engine_option),
+            DeprecationWarning
+        )

tests/test_basic_app.py

@@ -49,3 +49,13 @@ def test_query_recording(app, db, Todo):
 
 def test_helper_api(db):
     assert db.metadata == db.Model.metadata
+
+
+def test_persist_selectable(app, db, Todo, recwarn):
+    """ In SA 1.3, mapper.mapped_table should be replaced with mapper.persist_selectable """
+    with app.test_request_context():
+        todo = Todo('Test 1', 'test')
+        db.session.add(todo)
+        db.session.commit()
+
+    assert len(recwarn) == 0