Skip to content

Commit

Permalink
REGR: fix roundtripping datetimes with sqlite type detection (pandas-…
Browse files Browse the repository at this point in the history
…dev#55690)

Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com>
  • Loading branch information
jorisvandenbossche and mroeschke authored Oct 26, 2023
1 parent 54814c3 commit aeb3644
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 4 deletions.
1 change: 1 addition & 0 deletions doc/source/whatsnew/v2.1.2.rst
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ Fixed regressions
- Fixed regression in :meth:`DataFrameGroupBy.agg` and :meth:`SeriesGroupBy.agg` where if the option ``compute.use_numba`` was set to True, groupby methods not supported by the numba engine would raise a ``TypeError`` (:issue:`55520`)
- Fixed performance regression with wide DataFrames, typically involving methods where all columns were accessed individually (:issue:`55256`, :issue:`55245`)
- Fixed regression in :func:`merge_asof` raising ``TypeError`` for ``by`` with datetime and timedelta dtypes (:issue:`55453`)
- Fixed regression in :meth:`DataFrame.to_sql` not roundtripping datetime columns correctly for sqlite when using ``detect_types`` (:issue:`55554`)

.. ---------------------------------------------------------------------------
.. _whatsnew_212.bug_fixes:
Expand Down
6 changes: 2 additions & 4 deletions pandas/io/sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -2094,19 +2094,17 @@ def _adapt_time(t) -> str:
# Python 3.12+ doesn't auto-register adapters for us anymore

adapt_date_iso = lambda val: val.isoformat()
adapt_datetime_iso = lambda val: val.isoformat()
adapt_datetime_iso = lambda val: val.isoformat(" ")

sqlite3.register_adapter(time, _adapt_time)

sqlite3.register_adapter(date, adapt_date_iso)
sqlite3.register_adapter(datetime, adapt_datetime_iso)

convert_date = lambda val: date.fromisoformat(val.decode())
convert_datetime = lambda val: datetime.fromisoformat(val.decode())
convert_timestamp = lambda val: datetime.fromtimestamp(int(val))
convert_timestamp = lambda val: datetime.fromisoformat(val.decode())

sqlite3.register_converter("date", convert_date)
sqlite3.register_converter("datetime", convert_datetime)
sqlite3.register_converter("timestamp", convert_timestamp)

def sql_schema(self) -> str:
Expand Down
18 changes: 18 additions & 0 deletions pandas/tests/io/test_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -3368,6 +3368,24 @@ def test_roundtripping_datetimes(sqlite_engine):
assert result == "2020-12-31 12:00:00.000000"


@pytest.fixture
def sqlite_builtin_detect_types():
with contextlib.closing(
sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_DECLTYPES)
) as closing_conn:
with closing_conn as conn:
yield conn


def test_roundtripping_datetimes_detect_types(sqlite_builtin_detect_types):
# https://github.com/pandas-dev/pandas/issues/55554
conn = sqlite_builtin_detect_types
df = DataFrame({"t": [datetime(2020, 12, 31, 12)]}, dtype="datetime64[ns]")
df.to_sql("test", conn, if_exists="replace", index=False)
result = pd.read_sql("select * from test", conn).iloc[0, 0]
assert result == Timestamp("2020-12-31 12:00:00.000000")


@pytest.mark.db
def test_psycopg2_schema_support(postgresql_psycopg2_engine):
conn = postgresql_psycopg2_engine
Expand Down

0 comments on commit aeb3644

Please sign in to comment.