Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Fix synapse failing to start when python module sqlite3 is not installed #12974

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/12974.bugfix
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix a bug introduced in v1.60.0 that causes synapse to fail to start when python module sqlite3 is not installed.
2 changes: 1 addition & 1 deletion synapse/replication/slave/storage/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

from synapse.storage.database import DatabasePool, LoggingDatabaseConnection
from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.util.id_generators import MultiWriterIdGenerator

if TYPE_CHECKING:
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/background_updates.py
Original file line number Diff line number Diff line change
Expand Up @@ -628,7 +628,7 @@ def create_index_sqlite(conn: Connection) -> None:
logger.debug("[SQL] %s", sql)
c.execute(sql)

if isinstance(self.db_pool.engine, engines.PostgresEngine):
if isinstance(self.db_pool.engine, engines.postgres.PostgresEngine):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh bugger. I forgot that we use isinstance everywhere, which means that PostgresEngine and SqliteEngine need to be importable everywhere.

Argh. Maybe my advice in #12962 was wrong.

runner: Optional[Callable[[Connection], None]] = create_index_psql
elif psql_only:
runner = None
Expand Down
4 changes: 3 additions & 1 deletion synapse/storage/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@
from synapse.metrics import register_threadpool
from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.storage.background_updates import BackgroundUpdater
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.engines.sqlite import Sqlite3Engine
from synapse.storage.types import Connection, Cursor
from synapse.util.async_helpers import delay_cancellation
from synapse.util.iterutils import batch_iter
Expand Down
3 changes: 2 additions & 1 deletion synapse/storage/databases/main/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@
LoggingTransaction,
)
from synapse.storage.databases.main.stats import UserSortOrder
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.types import Cursor
from synapse.storage.util.id_generators import MultiWriterIdGenerator, StreamIdGenerator
from synapse.types import JsonDict, get_domain_from_id
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/databases/main/account_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
)
from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
from synapse.storage.databases.main.push_rule import PushRulesWorkerStore
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.util.id_generators import (
AbstractStreamIdGenerator,
AbstractStreamIdTracker,
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/databases/main/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
LoggingDatabaseConnection,
LoggingTransaction,
)
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.util.caches.descriptors import _CachedFunction
from synapse.util.iterutils import batch_iter

Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/databases/main/deviceinbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
LoggingTransaction,
make_in_list_sql_clause,
)
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.util.id_generators import (
AbstractStreamIdGenerator,
MultiWriterIdGenerator,
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/databases/main/end_to_end_keys.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
make_in_list_sql_clause,
)
from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.util.id_generators import StreamIdGenerator
from synapse.types import JsonDict
from synapse.util import json_encoder
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/databases/main/event_federation.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
)
from synapse.storage.databases.main.events_worker import EventsWorkerStore
from synapse.storage.databases.main.signatures import SignatureWorkerStore
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.types import JsonDict
from synapse.util import json_encoder
from synapse.util.caches.descriptors import cached
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/databases/main/events_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@
LoggingDatabaseConnection,
LoggingTransaction,
)
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.types import Cursor
from synapse.storage.util.id_generators import (
AbstractStreamIdTracker,
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/databases/main/presence.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
LoggingTransaction,
)
from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.types import Connection
from synapse.storage.util.id_generators import (
AbstractStreamIdGenerator,
Expand Down
3 changes: 2 additions & 1 deletion synapse/storage/databases/main/push_rule.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@
from synapse.storage.databases.main.pusher import PusherWorkerStore
from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
from synapse.storage.engines import PostgresEngine, Sqlite3Engine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.engines.sqlite import Sqlite3Engine
from synapse.storage.push_rule import InconsistentRuleException, RuleNotFoundException
from synapse.storage.util.id_generators import (
AbstractStreamIdGenerator,
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/databases/main/receipts.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
LoggingDatabaseConnection,
LoggingTransaction,
)
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.util.id_generators import (
AbstractStreamIdTracker,
MultiWriterIdGenerator,
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/databases/main/relations.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
from synapse.storage._base import SQLBaseStore
from synapse.storage.database import LoggingTransaction, make_in_list_sql_clause
from synapse.storage.databases.main.stream import generate_pagination_where_clause
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.types import JsonDict, RoomStreamToken, StreamKeyType, StreamToken
from synapse.util.caches.descriptors import cached, cachedList

Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/databases/main/roommember.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
)
from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore
from synapse.storage.databases.main.events_worker import EventsWorkerStore
from synapse.storage.engines import Sqlite3Engine
from synapse.storage.engines.sqlite import Sqlite3Engine
from synapse.storage.roommember import (
GetRoomsForUserWithStreamOrdering,
MemberSummary,
Expand Down
4 changes: 3 additions & 1 deletion synapse/storage/databases/main/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@
LoggingTransaction,
)
from synapse.storage.databases.main.events_worker import EventRedactBehaviour
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.engines.sqlite import Sqlite3Engine
from synapse.types import JsonDict

if TYPE_CHECKING:
Expand Down
3 changes: 2 additions & 1 deletion synapse/storage/databases/main/stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@
make_in_list_sql_clause,
)
from synapse.storage.databases.main.events_worker import EventsWorkerStore
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.util.id_generators import MultiWriterIdGenerator
from synapse.types import PersistedEventPosition, RoomStreamToken
from synapse.util.caches.descriptors import cached
Expand Down
3 changes: 2 additions & 1 deletion synapse/storage/databases/main/user_directory.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@
)
from synapse.storage.databases.main.state import StateFilter
from synapse.storage.databases.main.state_deltas import StateDeltasStore
from synapse.storage.engines import PostgresEngine, Sqlite3Engine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.engines.sqlite import Sqlite3Engine
from synapse.types import (
JsonDict,
UserProfile,
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/databases/state/bg_updates.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
LoggingDatabaseConnection,
LoggingTransaction,
)
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.state import StateFilter
from synapse.types import MutableStateMap, StateMap

Expand Down
6 changes: 4 additions & 2 deletions synapse/storage/engines/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,19 @@
from typing import Any, Mapping

from ._base import BaseDatabaseEngine, IncorrectDatabaseSetup
from .postgres import PostgresEngine
from .sqlite import Sqlite3Engine


def create_engine(database_config: Mapping[str, Any]) -> BaseDatabaseEngine:
name = database_config["name"]

if name == "sqlite3":
from .sqlite import Sqlite3Engine

return Sqlite3Engine(database_config)

if name == "psycopg2":
from .postgres import PostgresEngine

return PostgresEngine(database_config)

raise RuntimeError("Unsupported database engine '%s'" % (name,))
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/engines/postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from synapse.storage.types import Cursor

if TYPE_CHECKING:
import psycopg2 # noqa: F401
import psycopg2

from synapse.storage.database import LoggingDatabaseConnection

Expand Down
3 changes: 2 additions & 1 deletion synapse/storage/schema/main/delta/25/fts.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
import json
import logging

from synapse.storage.engines import PostgresEngine, Sqlite3Engine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.engines.sqlite import Sqlite3Engine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/schema/main/delta/31/search_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import json
import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/schema/main/delta/34/cache_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine

logger = logging.getLogger(__name__)

Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/schema/main/delta/37/remove_auth_idx.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand Down
3 changes: 2 additions & 1 deletion synapse/storage/schema/main/delta/42/user_dir.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@

import logging

from synapse.storage.engines import PostgresEngine, Sqlite3Engine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.engines.sqlite import Sqlite3Engine
from synapse.storage.prepare_database import get_statements

logger = logging.getLogger(__name__)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.prepare_database import get_statements

FIX_INDEXES = """
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@

import logging

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine

logger = logging.getLogger(__name__)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import logging
from io import StringIO

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.prepare_database import execute_statements_from_stream

logger = logging.getLogger(__name__)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
import logging
from io import StringIO

from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.prepare_database import execute_statements_from_stream
from synapse.storage.types import Cursor

Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/schema/main/delta/58/11user_id_seq.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from synapse.storage.databases.main.registration import (
find_max_generated_user_id_localpart,
)
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine


def run_create(cur, database_engine, *args, **kwargs):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
This migration handles the process of changing the type of `room_depth.min_depth` to
a BIGINT.
"""
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.types import Cursor


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@

Triggers cannot be expressed in .sql files, so we have to use a separate file.
"""
from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine
from synapse.storage.engines import BaseDatabaseEngine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.engines.sqlite import Sqlite3Engine
from synapse.storage.types import Cursor


Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/schema/main/delta/69/01as_txn_seq.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
Adds a postgres SEQUENCE for generating application service transaction IDs.
"""

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine


def run_create(cur, database_engine, *args, **kwargs):
Expand Down
2 changes: 1 addition & 1 deletion synapse/storage/schema/state/delta/47/state_group_seq.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine


def run_create(cur, database_engine, *args, **kwargs):
Expand Down
7 changes: 2 additions & 5 deletions synapse/storage/util/sequence.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,8 @@
import threading
from typing import TYPE_CHECKING, Callable, List, Optional

from synapse.storage.engines import (
BaseDatabaseEngine,
IncorrectDatabaseSetup,
PostgresEngine,
)
from synapse.storage.engines import BaseDatabaseEngine, IncorrectDatabaseSetup
from synapse.storage.engines.postgres import PostgresEngine
from synapse.storage.types import Connection, Cursor

if TYPE_CHECKING:
Expand Down
3 changes: 2 additions & 1 deletion tests/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@
from synapse.logging.context import ContextResourceUsage
from synapse.server import HomeServer
from synapse.storage import DataStore
from synapse.storage.engines import PostgresEngine, create_engine
from synapse.storage.engines import create_engine
from synapse.storage.engines.postgres import PostgresEngine
from synapse.types import JsonDict
from synapse.util import Clock

Expand Down
2 changes: 1 addition & 1 deletion tests/storage/test_room_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from synapse.api.constants import EventTypes
from synapse.api.errors import StoreError
from synapse.rest.client import login, room
from synapse.storage.engines import PostgresEngine
from synapse.storage.engines.postgres import PostgresEngine

from tests.unittest import HomeserverTestCase, skip_unless
from tests.utils import USE_POSTGRES_FOR_TESTS
Expand Down