From 9bf9eb7b5b2a9f174da7073826c9ca98d6c21b50 Mon Sep 17 00:00:00 2001 From: joel Date: Fri, 28 Sep 2018 14:56:53 +0200 Subject: [PATCH 01/10] bpo-34828 sqlite3.iterdump now correctly handles tables with autoincrement The iterdump command in Lib/sqlite3/dump.py assumed that the table "sqlite_sequence" already exists. When this was not the case and exception was thrown. --- Lib/sqlite3/dump.py | 9 +++++- Lib/sqlite3/test/dump.py | 61 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index de9c368be3014e..96798aa73960af 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -28,9 +28,14 @@ def _iterdump(connection): ORDER BY "name" """ schema_res = cu.execute(q) + sqlite_sequence = [] for table_name, type, sql in schema_res.fetchall(): if table_name == 'sqlite_sequence': - yield('DELETE FROM "sqlite_sequence";') + sqlite_sequence = ['DELETE FROM "sqlite_sequence"'] + rows = cu.execute('SELECT * FROM "sqlite_sequence";').fetchall() + sqlite_sequence.extend(['INSERT INTO "sqlite_sequence" VALUES(\'{0}\',{1})' + .format(row[0], row[1]) for row in rows]) + continue elif table_name == 'sqlite_stat1': yield('ANALYZE "sqlite_master";') elif table_name.startswith('sqlite_'): @@ -67,4 +72,6 @@ def _iterdump(connection): for name, type, sql in schema_res.fetchall(): yield('{0};'.format(sql)) + for row in sqlite_sequence: + yield("{0};".format(row)) yield('COMMIT;') diff --git a/Lib/sqlite3/test/dump.py b/Lib/sqlite3/test/dump.py index a1f45a46dc474a..a362356ff1d7e8 100644 --- a/Lib/sqlite3/test/dump.py +++ b/Lib/sqlite3/test/dump.py @@ -49,6 +49,67 @@ def CheckTableDump(self): [self.assertEqual(expected_sqls[i], actual_sqls[i]) for i in range(len(expected_sqls))] + def CheckIterdumpAutoincrement(self): + expected_sqls = [ + """CREATE TABLE "posts" (id int primary key);""" + , + """INSERT INTO "posts" VALUES(0);""" + , + "CREATE TABLE \"tags\" ( " \ + "id integer primary key autoincrement," \ + "tag varchar(256)," \ + "post int references posts);" + , + """INSERT INTO "tags" VALUES(NULL,'test',0);""" + , + "CREATE TABLE \"tags2\" ( " \ + "id integer primary key autoincrement," \ + "tag varchar(256)," \ + "post int references posts);" + ] + [self.cu.execute(s) for s in expected_sqls] + i = self.cx.iterdump() + actual_sqls = [s for s in i] + expected_sqls[3] = expected_sqls[3].replace("NULL", "1") + expected_sqls = ['BEGIN TRANSACTION;'] + expected_sqls + \ + ['DELETE FROM "sqlite_sequence";'] + \ + ["""INSERT INTO "sqlite_sequence" VALUES('tags',1);"""] + \ + ['COMMIT;'] + [self.assertEqual(expected_sqls[i], actual_sqls[i]) + for i in range(len(expected_sqls))] + + def CheckIterdumpAutoincrementCreateNewDB(self): + old_db = [ + "BEGIN TRANSACTION ;" + , + """CREATE TABLE "posts" (id int primary key);""" + , + """INSERT INTO "posts" VALUES(0);""" + , + "CREATE TABLE \"tags\" ( " \ + "id integer primary key autoincrement," \ + "tag varchar(256)," \ + "post int references posts);" + , + "CREATE TABLE \"tags2\" ( " \ + "id integer primary key autoincrement," \ + "tag varchar(256)," \ + "post int references posts);" + ] + [old_db.append("""INSERT INTO "tags" VALUES(NULL,'test{0}',0);""".format(i)) for i in range(1, 10)] + [old_db.append("""INSERT INTO "tags2" VALUES(NULL,'test{0}',0);""".format(i)) for i in range(1, 5)] + old_db.append("COMMIT;") + [self.cu.execute(s) for s in old_db] + i = self.cx.iterdump() + cx2 = sqlite.connect(":memory:") + query = "".join(line for line in self.cx.iterdump()) + cx2.executescript(query) + cu2 = cx2.cursor() + self.assertEqual(cu2.execute('SELECT "seq" FROM "sqlite_sequence" WHERE "name" == "tags"').fetchall()[0][0], 9) + self.assertEqual(cu2.execute('SELECT "seq" FROM "sqlite_sequence" WHERE "name" == "tags2"').fetchall()[0][0], 4) + cu2.close() + cx2.close() + def CheckUnorderableRow(self): # iterdump() should be able to cope with unorderable row types (issue #15545) class UnorderableRow: From 5d7b0aede8fe635f6e1782f1cefde08cde303278 Mon Sep 17 00:00:00 2001 From: joel Date: Fri, 28 Sep 2018 22:24:19 +0200 Subject: [PATCH 02/10] bpo-34828: added entry in NEWS.d/next/Library and in ACKS --- Misc/ACKS | 1 + .../next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst diff --git a/Misc/ACKS b/Misc/ACKS index 272130f4e643ec..0a24c325aa7d14 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -836,6 +836,7 @@ Akira Kitada Ron Klatchko Reid Kleckner Bastian Kleineidam +Joel Klimont Bob Kline Matthias Klose Jeremy Kloth diff --git a/Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst b/Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst new file mode 100644 index 00000000000000..4bd81f1c122215 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst @@ -0,0 +1,3 @@ +Sqlite3.iterdump now correctly deletes the "sqlite_sequence" table and +inserts the rows from the dumped db. Just like the command ".dump" in +sqlite3 does. From 365a2f1a683aaf0dce16a951e75506bd1f5cde1d Mon Sep 17 00:00:00 2001 From: Joel Klimont Date: Sat, 3 Jul 2021 20:38:42 +0200 Subject: [PATCH 03/10] bpo-34828 Removed redundant code and made some small formatting changes according to PEP 8 --- Lib/sqlite3/dump.py | 2 ++ Lib/sqlite3/test/dump.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 96798aa73960af..40033e757b89ec 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -72,6 +72,8 @@ def _iterdump(connection): for name, type, sql in schema_res.fetchall(): yield('{0};'.format(sql)) + # Yield statements concerning the sqlite_sequence table at the end of the transaction: (bpo-34828) for row in sqlite_sequence: yield("{0};".format(row)) + yield('COMMIT;') diff --git a/Lib/sqlite3/test/dump.py b/Lib/sqlite3/test/dump.py index bfdb85ed01028f..477a5f71c60640 100644 --- a/Lib/sqlite3/test/dump.py +++ b/Lib/sqlite3/test/dump.py @@ -1,7 +1,8 @@ # Author: Paul Kippes -import unittest import sqlite3 as sqlite +import unittest + class DumpTests(unittest.TestCase): def setUp(self): @@ -70,6 +71,7 @@ def test_dump_autoincrement(self): [self.cu.execute(s) for s in expected_sqls] i = self.cx.iterdump() actual_sqls = [s for s in i] + # the NULL value should now be automatically be set to 1 expected_sqls[3] = expected_sqls[3].replace("NULL", "1") expected_sqls = ['BEGIN TRANSACTION;'] + expected_sqls + \ ['DELETE FROM "sqlite_sequence";'] + \ @@ -100,7 +102,6 @@ def test_dump_autoincrement_create_new_db(self): [old_db.append("""INSERT INTO "tags2" VALUES(NULL,'test{0}',0);""".format(i)) for i in range(1, 5)] old_db.append("COMMIT;") [self.cu.execute(s) for s in old_db] - i = self.cx.iterdump() cx2 = sqlite.connect(":memory:") query = "".join(line for line in self.cx.iterdump()) cx2.executescript(query) @@ -131,6 +132,7 @@ def __getitem__(self, index): got = list(self.cx.iterdump()) self.assertEqual(expected, got) + def suite(): tests = [ DumpTests, @@ -139,9 +141,11 @@ def suite(): [unittest.TestLoader().loadTestsFromTestCase(t) for t in tests] ) + def test(): runner = unittest.TextTestRunner() runner.run(suite()) + if __name__ == "__main__": test() From b2592197a23fe0c47faa4811eafe397401cb52e2 Mon Sep 17 00:00:00 2001 From: Joel Klimont Date: Sat, 16 Oct 2021 01:49:39 +0200 Subject: [PATCH 04/10] bpo-34828 formatting changes as well as readability improvements in sqlite3 dump tests --- Lib/sqlite3/test/test_dump.py | 89 ++++++++++--------- .../2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst | 4 +- 2 files changed, 50 insertions(+), 43 deletions(-) diff --git a/Lib/sqlite3/test/test_dump.py b/Lib/sqlite3/test/test_dump.py index 74e1f6167d0e5b..ddaf3e3d7d3952 100644 --- a/Lib/sqlite3/test/test_dump.py +++ b/Lib/sqlite3/test/test_dump.py @@ -51,62 +51,71 @@ def test_table_dump(self): def test_dump_autoincrement(self): expected_sqls = [ - """CREATE TABLE "posts" (id int primary key);""" - , - """INSERT INTO "posts" VALUES(0);""" - , - "CREATE TABLE \"tags\" ( " \ - "id integer primary key autoincrement," \ - "tag varchar(256)," \ - "post int references posts);" - , - """INSERT INTO "tags" VALUES(NULL,'test',0);""" - , - "CREATE TABLE \"tags2\" ( " \ - "id integer primary key autoincrement," \ - "tag varchar(256)," \ - "post int references posts);" + """CREATE TABLE "posts" (id int primary key);""", + """INSERT INTO "posts" VALUES(0);""", + """CREATE TABLE "tags" ( """ + """id integer primary key autoincrement,""" + """tag varchar(256),""" + """post int references posts);""", + """INSERT INTO "tags" VALUES(NULL,'test',0);""", + """CREATE TABLE "tags2" ( """ + """id integer primary key autoincrement,""" + """tag varchar(256),""" + """post int references posts);""" ] - [self.cu.execute(s) for s in expected_sqls] - i = self.cx.iterdump() - actual_sqls = [s for s in i] + + for sql_statement in expected_sqls: + self.cu.execute(sql_statement) + iterdump = self.cx.iterdump() + actual_sqls = [sql_statement for sql_statement in iterdump] + # the NULL value should now be automatically be set to 1 expected_sqls[3] = expected_sqls[3].replace("NULL", "1") expected_sqls = ['BEGIN TRANSACTION;'] + expected_sqls + \ ['DELETE FROM "sqlite_sequence";'] + \ ["""INSERT INTO "sqlite_sequence" VALUES('tags',1);"""] + \ ['COMMIT;'] - [self.assertEqual(expected_sqls[i], actual_sqls[i]) - for i in range(len(expected_sqls))] + + for i in range(len(expected_sqls)): + self.assertEqual(expected_sqls[i], actual_sqls[i]) def test_dump_autoincrement_create_new_db(self): old_db = [ - "BEGIN TRANSACTION ;" - , - """CREATE TABLE "posts" (id int primary key);""" - , - """INSERT INTO "posts" VALUES(0);""" - , - "CREATE TABLE \"tags\" ( " \ - "id integer primary key autoincrement," \ - "tag varchar(256)," \ - "post int references posts);" - , - "CREATE TABLE \"tags2\" ( " \ - "id integer primary key autoincrement," \ - "tag varchar(256)," \ - "post int references posts);" + """BEGIN TRANSACTION ;""", + """CREATE TABLE "posts" (id int primary key);""", + """INSERT INTO "posts" VALUES(0);""", + """CREATE TABLE "tags" ( """ + """id integer primary key autoincrement,""" + """tag varchar(256),""" + """post int references posts);""", + """CREATE TABLE "tags2" ( """ + """id integer primary key autoincrement,""" + """tag varchar(256),""" + """post int references posts);""" ] - [old_db.append("""INSERT INTO "tags" VALUES(NULL,'test{0}',0);""".format(i)) for i in range(1, 10)] - [old_db.append("""INSERT INTO "tags2" VALUES(NULL,'test{0}',0);""".format(i)) for i in range(1, 5)] + for i in range(1, 10): + old_db.append("""INSERT INTO "tags" + VALUES(NULL,'test{0}',0);""".format(i)) + for i in range(1, 5): + old_db.append("""INSERT INTO "tags2" + VALUES(NULL,'test{0}',0);""".format(i)) old_db.append("COMMIT;") - [self.cu.execute(s) for s in old_db] + + for sql_statement in old_db: + self.cu.execute(sql_statement) + cx2 = sqlite.connect(":memory:") query = "".join(line for line in self.cx.iterdump()) cx2.executescript(query) cu2 = cx2.cursor() - self.assertEqual(cu2.execute('SELECT "seq" FROM "sqlite_sequence" WHERE "name" == "tags"').fetchall()[0][0], 9) - self.assertEqual(cu2.execute('SELECT "seq" FROM "sqlite_sequence" WHERE "name" == "tags2"').fetchall()[0][0], 4) + + self.assertEqual(cu2.execute('SELECT "seq"' + 'FROM "sqlite_sequence"' + 'WHERE "name" == "tags"').fetchall()[0][0], 9) + self.assertEqual(cu2.execute('SELECT "seq" FROM' + '"sqlite_sequence"' + 'WHERE "name" == "tags2"').fetchall()[0][0], 4) + cu2.close() cx2.close() diff --git a/Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst b/Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst index 4bd81f1c122215..0ae563ba123a75 100644 --- a/Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst +++ b/Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst @@ -1,3 +1 @@ -Sqlite3.iterdump now correctly deletes the "sqlite_sequence" table and -inserts the rows from the dumped db. Just like the command ".dump" in -sqlite3 does. +:meth:`~sqlite3.Connection.iterdump` now handles databases that use ``AUTOINCREMENT`` in one or more tables. From daed6e50fc65ef27a64d1be6fc3440e1ec884bfb Mon Sep 17 00:00:00 2001 From: Joel Klimont Date: Sun, 22 May 2022 22:32:50 +0200 Subject: [PATCH 05/10] bpo-34828: reformatted some tests in test_sqlite3/test.dump.py as well as small formatting changes in sqlite3/dump.py --- Lib/sqlite3/dump.py | 8 +-- Lib/test/test_sqlite3/test_dump.py | 79 ++++++++++++------------------ 2 files changed, 36 insertions(+), 51 deletions(-) diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 40033e757b89ec..911275a39f87a1 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -30,10 +30,10 @@ def _iterdump(connection): schema_res = cu.execute(q) sqlite_sequence = [] for table_name, type, sql in schema_res.fetchall(): - if table_name == 'sqlite_sequence': - sqlite_sequence = ['DELETE FROM "sqlite_sequence"'] - rows = cu.execute('SELECT * FROM "sqlite_sequence";').fetchall() - sqlite_sequence.extend(['INSERT INTO "sqlite_sequence" VALUES(\'{0}\',{1})' + if table_name == "sqlite_sequence": + sqlite_sequence = ["DELETE FROM \"sqlite_sequence\""] + rows = cu.execute("SELECT * FROM \"sqlite_sequence\";").fetchall() + sqlite_sequence.extend(["INSERT INTO \"sqlite_sequence\" VALUES('{0}',{1})" .format(row[0], row[1]) for row in rows]) continue elif table_name == 'sqlite_stat1': diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index ddaf3e3d7d3952..579bc0ed469aac 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -2,6 +2,8 @@ import unittest import sqlite3 as sqlite +from .test_dbapi import memory_database + class DumpTests(unittest.TestCase): def setUp(self): @@ -51,17 +53,11 @@ def test_table_dump(self): def test_dump_autoincrement(self): expected_sqls = [ - """CREATE TABLE "posts" (id int primary key);""", - """INSERT INTO "posts" VALUES(0);""", - """CREATE TABLE "tags" ( """ - """id integer primary key autoincrement,""" - """tag varchar(256),""" - """post int references posts);""", - """INSERT INTO "tags" VALUES(NULL,'test',0);""", - """CREATE TABLE "tags2" ( """ - """id integer primary key autoincrement,""" - """tag varchar(256),""" - """post int references posts);""" + "CREATE TABLE \"posts\" (id int primary key);", + "INSERT INTO \"posts\" VALUES(0);", + "CREATE TABLE \"tags\" (id integer primary key autoincrement, tag varchar(256), post int references posts);", + "INSERT INTO \"tags\" VALUES(NULL,'test',0);", + "CREATE TABLE \"tags2\" (id integer primary key autoincrement, tag varchar(256), post int references posts);" ] for sql_statement in expected_sqls: @@ -71,53 +67,42 @@ def test_dump_autoincrement(self): # the NULL value should now be automatically be set to 1 expected_sqls[3] = expected_sqls[3].replace("NULL", "1") - expected_sqls = ['BEGIN TRANSACTION;'] + expected_sqls + \ - ['DELETE FROM "sqlite_sequence";'] + \ - ["""INSERT INTO "sqlite_sequence" VALUES('tags',1);"""] + \ - ['COMMIT;'] + expected_sqls.insert(0, "BEGIN TRANSACTION;") + expected_sqls.extend([ + "DELETE FROM \"sqlite_sequence\";", + "INSERT INTO \"sqlite_sequence\" VALUES('tags',1);", + "COMMIT;" + ]) - for i in range(len(expected_sqls)): - self.assertEqual(expected_sqls[i], actual_sqls[i]) + self.assertEqual(expected_sqls, actual_sqls) def test_dump_autoincrement_create_new_db(self): old_db = [ - """BEGIN TRANSACTION ;""", - """CREATE TABLE "posts" (id int primary key);""", - """INSERT INTO "posts" VALUES(0);""", - """CREATE TABLE "tags" ( """ - """id integer primary key autoincrement,""" - """tag varchar(256),""" - """post int references posts);""", - """CREATE TABLE "tags2" ( """ - """id integer primary key autoincrement,""" - """tag varchar(256),""" - """post int references posts);""" + "BEGIN TRANSACTION ;", + "CREATE TABLE \"posts\" (id int primary key);", + "INSERT INTO \"posts\" VALUES(0);", + "CREATE TABLE \"tags\" (id integer primary key autoincrement, tag varchar(256), post int references posts);", + "CREATE TABLE \"tags2\" (id integer primary key autoincrement, tag varchar(256), post int references posts);" ] for i in range(1, 10): - old_db.append("""INSERT INTO "tags" - VALUES(NULL,'test{0}',0);""".format(i)) + old_db.append("INSERT INTO \"tags\" VALUES(NULL,'test{0}',0);".format(i)) for i in range(1, 5): - old_db.append("""INSERT INTO "tags2" - VALUES(NULL,'test{0}',0);""".format(i)) + old_db.append("INSERT INTO \"tags2\" VALUES(NULL,'test{0}',0);".format(i)) old_db.append("COMMIT;") - for sql_statement in old_db: - self.cu.execute(sql_statement) - - cx2 = sqlite.connect(":memory:") - query = "".join(line for line in self.cx.iterdump()) - cx2.executescript(query) - cu2 = cx2.cursor() + self.cu.executescript("".join(old_db)) + query = "".join(self.cx.iterdump()) - self.assertEqual(cu2.execute('SELECT "seq"' - 'FROM "sqlite_sequence"' - 'WHERE "name" == "tags"').fetchall()[0][0], 9) - self.assertEqual(cu2.execute('SELECT "seq" FROM' - '"sqlite_sequence"' - 'WHERE "name" == "tags2"').fetchall()[0][0], 4) + with memory_database() as cx2: + cx2.executescript(query) + cu2 = cx2.cursor() - cu2.close() - cx2.close() + self.assertEqual(cu2.execute("SELECT \"seq\"" + "FROM \"sqlite_sequence\"" + "WHERE \"name\" == \"tags\"").fetchall()[0][0], 9) + self.assertEqual(cu2.execute("SELECT \"seq\" FROM" + "\"sqlite_sequence\"" + "WHERE \"name\" == \"tags2\"").fetchall()[0][0], 4) def test_unorderable_row(self): # iterdump() should be able to cope with unorderable row types (issue #15545) From 4e573e9c12c32cedbe64a2be5cfdc9a4d1e97a36 Mon Sep 17 00:00:00 2001 From: Joel Klimont Date: Sat, 18 Jun 2022 00:10:48 +0200 Subject: [PATCH 06/10] bpo-34828: minified test case for some sqlite3 dump command tests, fixed wrong use of double quotes and single quotes in sqlite3 statements --- Lib/test/test_sqlite3/test_dump.py | 32 +++++++++++++----------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 579bc0ed469aac..3f02901c9ec06f 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -53,11 +53,9 @@ def test_table_dump(self): def test_dump_autoincrement(self): expected_sqls = [ - "CREATE TABLE \"posts\" (id int primary key);", - "INSERT INTO \"posts\" VALUES(0);", - "CREATE TABLE \"tags\" (id integer primary key autoincrement, tag varchar(256), post int references posts);", - "INSERT INTO \"tags\" VALUES(NULL,'test',0);", - "CREATE TABLE \"tags2\" (id integer primary key autoincrement, tag varchar(256), post int references posts);" + "CREATE TABLE \"t1\" (id integer primary key autoincrement);", + "INSERT INTO \"t1\" VALUES(NULL);", + "CREATE TABLE \"t2\" (id integer primary key autoincrement);" ] for sql_statement in expected_sqls: @@ -66,11 +64,11 @@ def test_dump_autoincrement(self): actual_sqls = [sql_statement for sql_statement in iterdump] # the NULL value should now be automatically be set to 1 - expected_sqls[3] = expected_sqls[3].replace("NULL", "1") + expected_sqls[1] = expected_sqls[1].replace("NULL", "1") expected_sqls.insert(0, "BEGIN TRANSACTION;") expected_sqls.extend([ "DELETE FROM \"sqlite_sequence\";", - "INSERT INTO \"sqlite_sequence\" VALUES('tags',1);", + "INSERT INTO \"sqlite_sequence\" VALUES('t1',1);", "COMMIT;" ]) @@ -79,15 +77,13 @@ def test_dump_autoincrement(self): def test_dump_autoincrement_create_new_db(self): old_db = [ "BEGIN TRANSACTION ;", - "CREATE TABLE \"posts\" (id int primary key);", - "INSERT INTO \"posts\" VALUES(0);", - "CREATE TABLE \"tags\" (id integer primary key autoincrement, tag varchar(256), post int references posts);", - "CREATE TABLE \"tags2\" (id integer primary key autoincrement, tag varchar(256), post int references posts);" + "CREATE TABLE \"t1\" (id integer primary key autoincrement);", + "CREATE TABLE \"t2\" (id integer primary key autoincrement);" ] for i in range(1, 10): - old_db.append("INSERT INTO \"tags\" VALUES(NULL,'test{0}',0);".format(i)) + old_db.append("INSERT INTO \"t1\" VALUES(NULL);".format(i)) for i in range(1, 5): - old_db.append("INSERT INTO \"tags2\" VALUES(NULL,'test{0}',0);".format(i)) + old_db.append("INSERT INTO \"t2\" VALUES(NULL);".format(i)) old_db.append("COMMIT;") self.cu.executescript("".join(old_db)) @@ -97,12 +93,12 @@ def test_dump_autoincrement_create_new_db(self): cx2.executescript(query) cu2 = cx2.cursor() - self.assertEqual(cu2.execute("SELECT \"seq\"" + self.assertEqual(cu2.execute("SELECT \"seq\" " "FROM \"sqlite_sequence\"" - "WHERE \"name\" == \"tags\"").fetchall()[0][0], 9) - self.assertEqual(cu2.execute("SELECT \"seq\" FROM" - "\"sqlite_sequence\"" - "WHERE \"name\" == \"tags2\"").fetchall()[0][0], 4) + "WHERE \"name\" == 't1'").fetchall()[0][0], 9) + self.assertEqual(cu2.execute("SELECT \"seq\" " + "FROM \"sqlite_sequence\"" + "WHERE \"name\" == 't2'").fetchall()[0][0], 4) def test_unorderable_row(self): # iterdump() should be able to cope with unorderable row types (issue #15545) From 1abc7d05b7a605224611a09eaa74e50ef1cc38e1 Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Mon, 20 Jun 2022 00:12:36 +0200 Subject: [PATCH 07/10] Clean up dump.py - consolidate construction of sqlite_sequence - use existing quoting style --- Lib/sqlite3/dump.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 911275a39f87a1..540166a915c621 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -30,12 +30,13 @@ def _iterdump(connection): schema_res = cu.execute(q) sqlite_sequence = [] for table_name, type, sql in schema_res.fetchall(): - if table_name == "sqlite_sequence": - sqlite_sequence = ["DELETE FROM \"sqlite_sequence\""] - rows = cu.execute("SELECT * FROM \"sqlite_sequence\";").fetchall() - sqlite_sequence.extend(["INSERT INTO \"sqlite_sequence\" VALUES('{0}',{1})" - .format(row[0], row[1]) for row in rows]) - continue + if table_name == 'sqlite_sequence': + rows = cu.execute('SELECT * FROM "sqlite_sequence";').fetchall() + sqlite_sequence = ['DELETE FROM "sqlite_sequence"'] + sqlite_sequence += [ + f'INSERT INTO "sqlite_sequence" VALUES(\'{row[0]}\',{row[1]})' + for row in rows + ] elif table_name == 'sqlite_stat1': yield('ANALYZE "sqlite_master";') elif table_name.startswith('sqlite_'): @@ -72,8 +73,9 @@ def _iterdump(connection): for name, type, sql in schema_res.fetchall(): yield('{0};'.format(sql)) - # Yield statements concerning the sqlite_sequence table at the end of the transaction: (bpo-34828) + # gh-79009: Yield statements concerning the sqlite_sequence table at the + # end of the transaction. for row in sqlite_sequence: - yield("{0};".format(row)) + yield('{0};'.format(row)) yield('COMMIT;') From 6bfac7e03aee3104c49c0c45e32933ad042faf30 Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Mon, 20 Jun 2022 00:16:13 +0200 Subject: [PATCH 08/10] Cleanup: don't use abbreviated link in NEWS entry --- .../next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst b/Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst index 0ae563ba123a75..b0e10a158b5b19 100644 --- a/Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst +++ b/Misc/NEWS.d/next/Library/2018-09-28-22-18-03.bpo-34828.5Zyi_S.rst @@ -1 +1 @@ -:meth:`~sqlite3.Connection.iterdump` now handles databases that use ``AUTOINCREMENT`` in one or more tables. +:meth:`sqlite3.Connection.iterdump` now handles databases that use ``AUTOINCREMENT`` in one or more tables. From 103c50ffdcfe454a89fed09586524244cde68bdd Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Mon, 20 Jun 2022 00:27:10 +0200 Subject: [PATCH 09/10] Cleanup: tests - reduce number of escape chars in string literals - normalise variable names - in test_dump_autoincrement_create_new_db, simplify setup - use subTest in order to make it easier to assert more thoroughly --- Lib/test/test_sqlite3/test_dump.py | 66 ++++++++++++++---------------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index 3f02901c9ec06f..d0c24b9c60e613 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -52,53 +52,49 @@ def test_table_dump(self): for i in range(len(expected_sqls))] def test_dump_autoincrement(self): - expected_sqls = [ - "CREATE TABLE \"t1\" (id integer primary key autoincrement);", - "INSERT INTO \"t1\" VALUES(NULL);", - "CREATE TABLE \"t2\" (id integer primary key autoincrement);" + expected = [ + 'CREATE TABLE "t1" (id integer primary key autoincrement);', + 'INSERT INTO "t1" VALUES(NULL);', + 'CREATE TABLE "t2" (id integer primary key autoincrement);', ] - - for sql_statement in expected_sqls: - self.cu.execute(sql_statement) - iterdump = self.cx.iterdump() - actual_sqls = [sql_statement for sql_statement in iterdump] + self.cu.executescript("".join(expected)) # the NULL value should now be automatically be set to 1 - expected_sqls[1] = expected_sqls[1].replace("NULL", "1") - expected_sqls.insert(0, "BEGIN TRANSACTION;") - expected_sqls.extend([ - "DELETE FROM \"sqlite_sequence\";", - "INSERT INTO \"sqlite_sequence\" VALUES('t1',1);", - "COMMIT;" + expected[1] = expected[1].replace("NULL", "1") + expected.insert(0, "BEGIN TRANSACTION;") + expected.extend([ + 'DELETE FROM "sqlite_sequence";', + 'INSERT INTO "sqlite_sequence" VALUES(\'t1\',1);', + 'COMMIT;', ]) - self.assertEqual(expected_sqls, actual_sqls) + actual = [stmt for stmt in self.cx.iterdump()] + self.assertEqual(expected, actual) def test_dump_autoincrement_create_new_db(self): - old_db = [ - "BEGIN TRANSACTION ;", - "CREATE TABLE \"t1\" (id integer primary key autoincrement);", - "CREATE TABLE \"t2\" (id integer primary key autoincrement);" - ] - for i in range(1, 10): - old_db.append("INSERT INTO \"t1\" VALUES(NULL);".format(i)) - for i in range(1, 5): - old_db.append("INSERT INTO \"t2\" VALUES(NULL);".format(i)) - old_db.append("COMMIT;") - - self.cu.executescript("".join(old_db)) - query = "".join(self.cx.iterdump()) + self.cu.execute("BEGIN TRANSACTION") + self.cu.execute("CREATE TABLE t1 (id integer primary key autoincrement)") + self.cu.execute("CREATE TABLE t2 (id integer primary key autoincrement)") + self.cu.executemany("INSERT INTO t1 VALUES(?)", ((None,) for _ in range(9))) + self.cu.executemany("INSERT INTO t2 VALUES(?)", ((None,) for _ in range(4))) + self.cx.commit() with memory_database() as cx2: + query = "".join(self.cx.iterdump()) cx2.executescript(query) cu2 = cx2.cursor() - self.assertEqual(cu2.execute("SELECT \"seq\" " - "FROM \"sqlite_sequence\"" - "WHERE \"name\" == 't1'").fetchall()[0][0], 9) - self.assertEqual(cu2.execute("SELECT \"seq\" " - "FROM \"sqlite_sequence\"" - "WHERE \"name\" == 't2'").fetchall()[0][0], 4) + dataset = ( + ("t1", 9), + ("t2", 4), + ) + for table, seq in dataset: + with self.subTest(table=table, seq=seq): + res = cu2.execute(""" + SELECT "seq" FROM "sqlite_sequence" WHERE "name" == ? + """, (table,)) + rows = res.fetchall() + self.assertEqual(rows[0][0], seq) def test_unorderable_row(self): # iterdump() should be able to cope with unorderable row types (issue #15545) From 53076e91daee6639c9d6400741c2ee581997d677 Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Mon, 20 Jun 2022 00:34:49 +0200 Subject: [PATCH 10/10] Fix 1abc7d05b7a605224611a09eaa74e50ef1cc38e1 --- Lib/sqlite3/dump.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 540166a915c621..07b9da10b920f9 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -37,6 +37,7 @@ def _iterdump(connection): f'INSERT INTO "sqlite_sequence" VALUES(\'{row[0]}\',{row[1]})' for row in rows ] + continue elif table_name == 'sqlite_stat1': yield('ANALYZE "sqlite_master";') elif table_name.startswith('sqlite_'):