diff --git a/.driver/environments.yaml.dist b/.driver/environments.yaml.dist index a24ee19..05c7df7 100644 --- a/.driver/environments.yaml.dist +++ b/.driver/environments.yaml.dist @@ -10,5 +10,7 @@ environments: # For example, if your `TABLE_NAME` is `core_config_data`, Driver will search for a table in the database that # ends with `core_config_data`. Thus, `core_config_data` and `sample_core_config_data` would all match. ignored_tables: # OPTIONAL, tables listed here will be ignored in the final dump with: - # mysqldump ... --ignored-tables=DATABASE.table_1 + # mysqldump ... --ignored-tables=TABLE_NAME + - TABLE_NAME + empty_tables: # OPTIONAL, tables listed here will be dumped without data - TABLE_NAME diff --git a/.driver/environments.yaml.example b/.driver/environments.yaml.example index 833d7dd..1098bc7 100644 --- a/.driver/environments.yaml.example +++ b/.driver/environments.yaml.example @@ -29,48 +29,7 @@ environments: - "UPDATE {{table_name}} SET value = 'store.local' WHERE path LIKE 'web/cookie/cookie_domain' AND scope_id = 0;" - "UPDATE {{table_name}} SET value = 'localhost' WHERE path LIKE 'catalog/search/elasticsearch%_server_hostname' AND scope_id = 0;" ignored_tables: - - setup_module - - customer_address_entity - - customer_address_entity_datetime - - customer_address_entity_decimal - - customer_address_entity_int - - customer_address_entity_text - - customer_address_entity_varchar - - customer_entity - - customer_entity_datetime - - customer_entity_decimal - - customer_entity_int - - customer_entity_text - - customer_entity_varchar - - sales_creditmemo - - sales_credimemo_comment - - sales_creditmemo_grid - - sales_creditmemo_item - - sales_invoice - - sales_invoice_comment - - sales_invoice_grid - - sales_invoice_item - - sales_order - - sales_order_address - - sales_order_grid - - sales_order_item - - sales_order_payment - - sales_order_status_history - - sales_shipment - - sales_shipment_comment - - sales_shipment_grid - - sales_shipment_item - - sales_shipment_track - - sales_invoiced_aggregated - - sales_invoiced_aggregated_order - - sales_payment_transaction - - sales_order_aggregated_created - - sales_order_tax - - sales_order_tax_item - - sales_quote - - sales_quote_address - - sales_quote_address_item - - sales_quote_item - - sales_quote_item_option - - sales_quote_payment - - sales_quote_shipping_rate + - some_non_magento_table + empty_tables: + - customer_log + - customer_visitor diff --git a/src/Engines/MySql/Export/CommandAssembler.php b/src/Engines/MySql/Export/CommandAssembler.php index dcba6db..dd0f0c1 100644 --- a/src/Engines/MySql/Export/CommandAssembler.php +++ b/src/Engines/MySql/Export/CommandAssembler.php @@ -7,6 +7,7 @@ use Driver\Engines\ConnectionInterface; use Driver\Pipeline\Environment\EnvironmentInterface; +use function array_diff; use function array_unshift; use function implode; use function in_array; @@ -26,54 +27,101 @@ public function __construct(TablesProvider $tablesProvider) public function execute( ConnectionInterface $connection, EnvironmentInterface $environment, - string $dumpFile + string $dumpFile, + string $triggersDumpFile ): array { + $allTables = $this->tablesProvider->getAllTables($connection); $ignoredTables = $this->tablesProvider->getIgnoredTables($environment); + if (array_diff($allTables, $ignoredTables) === []) { + return []; + } $emptyTables = $this->tablesProvider->getEmptyTables($environment); - foreach ($this->tablesProvider->getAllTables($connection) as $table) { + $commands = [$this->getStructureCommand($connection, $ignoredTables, $dumpFile)]; + foreach ($allTables as $table) { if (in_array($table, $ignoredTables) || in_array($table, $emptyTables)) { continue; } - $commands[] = $this->getSingleCommand($connection, [$table], $dumpFile); - } - if (!empty($emptyTables)) { - $commands[] = $this->getSingleCommand($connection, $emptyTables, $dumpFile, false); - } - if (empty($commands)) { - return []; + $commands[] = $this->getDataCommand($connection, [$table], $dumpFile); } array_unshift( $commands, "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'" - . ">> $dumpFile" + . " | gzip >> $dumpFile" ); - $commands[] = "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> $dumpFile"; - $commands[] = "cat $dumpFile | " - . "sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > $dumpFile.gz"; + $commands[] = "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' | gzip >> $dumpFile"; + $commands[] = $this->getTriggersCommand($connection, $ignoredTables, $triggersDumpFile); return $commands; } /** - * @param string[] $tables + * @param string[] $ignoredTables */ - private function getSingleCommand( + private function getStructureCommand( ConnectionInterface $connection, - array $tables, - string $dumpFile, - bool $withData = true + array $ignoredTables, + string $dumpFile ): string { $parts = [ "mysqldump --user=\"{$connection->getUser()}\"", "--password=\"{$connection->getPassword()}\"", "--single-transaction", "--no-tablespaces", + "--no-data", + "--skip-triggers", + "--host={$connection->getHost()}", + $connection->getDatabase() + ]; + foreach ($ignoredTables as $table) { + $parts[] = "--ignore-table={$connection->getDatabase()}.{$table}"; + } + $parts[] = "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'"; + $parts[] = "| gzip"; + $parts[] = ">> $dumpFile"; + return implode(' ', $parts); + } + + /** + * @param string[] $tables + */ + private function getDataCommand(ConnectionInterface $connection, array $tables, string $dumpFile): string + { + $parts = [ + "mysqldump --user=\"{$connection->getUser()}\"", + "--password=\"{$connection->getPassword()}\"", + "--single-transaction", + "--no-tablespaces", + "--no-create-info", + "--skip-triggers", "--host={$connection->getHost()}", $connection->getDatabase(), implode(' ', $tables) ]; - if (!$withData) { - $parts[] = '--no-data'; + $parts[] = "| gzip"; + $parts[] = ">> $dumpFile"; + return implode(' ', $parts); + } + + /** + * @param string[] $ignoredTables + */ + private function getTriggersCommand(ConnectionInterface $connection, array $ignoredTables, string $dumpFile): string + { + $parts = [ + "mysqldump --user=\"{$connection->getUser()}\"", + "--password=\"{$connection->getPassword()}\"", + "--single-transaction", + "--no-tablespaces", + "--no-data", + "--no-create-info", + "--triggers", + "--host={$connection->getHost()}", + $connection->getDatabase() + ]; + foreach ($ignoredTables as $table) { + $parts[] = "--ignore-table={$connection->getDatabase()}.{$table}"; } + $parts[] = "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'"; + $parts[] = "| gzip"; $parts[] = ">> $dumpFile"; return implode(' ', $parts); } diff --git a/src/Engines/MySql/Export/Primary.php b/src/Engines/MySql/Export/Primary.php index 0c35fff..3444675 100755 --- a/src/Engines/MySql/Export/Primary.php +++ b/src/Engines/MySql/Export/Primary.php @@ -27,7 +27,8 @@ class Primary extends Command implements CommandInterface, CleanupInterface private array $properties; private LoggerInterface $logger; private Random $random; - private ?string $path = null; + /** @var array */ + private array $filePaths = []; private Configuration $configuration; private ConsoleOutput $output; private CommandAssembler $commandAssembler; @@ -58,7 +59,8 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $this->output->writeln("Exporting database from local MySql"); try { - $commands = $this->commandAssembler->execute($this->localConnection, $environment, $this->getDumpFile()); + $commands = $this->commandAssembler + ->execute($this->localConnection, $environment, $this->getDumpFile(), $this->getDumpFile('triggers')); if (empty($commands)) { throw new RuntimeException('Nothing to import'); } @@ -70,7 +72,7 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $result = system($command, $resultCode); if ($result === false || $resultCode !== 0) { $message = sprintf('Error (%s) when executing command: %s', $resultCode, $strippedCommand); - $this->output->writeln("${$message}"); + $this->output->writeln("$message"); throw new RuntimeException($message); } } @@ -82,14 +84,17 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $this->logger->notice("Database dump has completed."); $this->output->writeln("Database dump has completed."); return $transport->withStatus(new Status('sandbox_init', 'success')) - ->withNewData('dump-file', $this->getDumpFile()); + ->withNewData('dump-file', $this->getDumpFile()) + ->withNewData('triggers-dump-file', $this->getDumpFile('triggers')); } public function cleanup(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface { - if ($this->getDumpFile() && file_exists($this->getDumpFile())) { - @unlink($this->getDumpFile()); - } + array_walk($this->filePaths, function (string $filePath): void { + if ($filePath && file_exists($filePath)) { + @unlink($filePath); + } + }); return $transport; } @@ -99,18 +104,18 @@ public function getProperties(): array return $this->properties; } - private function getDumpFile(): string + private function getDumpFile(string $code = 'default'): string { - if (!$this->path) { + if (!\array_key_exists($code, $this->filePaths)) { $path = $this->configuration->getNode('connections/mysql/dump-path'); if (!$path) { $path = self::DEFAULT_DUMP_PATH; } - $filename = 'driver-' . $this->random->getRandomString(6) . '.sql'; + $filename = 'driver-' . $this->random->getRandomString(6) . '.gz'; - $this->path = $path . '/' . $filename; + $this->filePaths[$code] = $path . '/' . $filename; } - return $this->path; + return $this->filePaths[$code]; } } diff --git a/src/Engines/MySql/Sandbox/Export.php b/src/Engines/MySql/Sandbox/Export.php index 800bc60..29098f4 100755 --- a/src/Engines/MySql/Sandbox/Export.php +++ b/src/Engines/MySql/Sandbox/Export.php @@ -68,7 +68,11 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm ); $environmentName = $environment->getName(); - $command = $this->assembleCommand($environmentName, $environment->getIgnoredTables()); + $command = $this->assembleCommand( + $environmentName, + $environment->getIgnoredTables(), + $transport->getData('triggers-dump-file') + ); $this->files[] = $this->getFilename($environmentName); @@ -98,8 +102,9 @@ public function cleanup(TransportInterface $transport, EnvironmentInterface $env /** * @param string[] $ignoredTables */ - private function assembleCommand(string $environmentName, array $ignoredTables): string + private function assembleCommand(string $environmentName, array $ignoredTables, string $triggersDumpFile): string { + $filename = $this->getFilename($environmentName); $command = implode(' ', array_merge([ "mysqldump --user={$this->connection->getUser()}", "--password={$this->connection->getPassword()}", @@ -107,21 +112,13 @@ private function assembleCommand(string $environmentName, array $ignoredTables): "--port={$this->connection->getPort()}", "--no-tablespaces" ], $this->getIgnoredTables($ignoredTables))); - $command .= " {$this->connection->getDatabase()} "; $command .= "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' "; - if ($this->compressOutput()) { - $command .= ' ' . implode(' ', [ - '|', - 'gzip --best' - ]); + $command .= "| gzip --best "; } - - $command .= ' ' . implode(' ', [ - '>', - $this->getFilename($environmentName) - ]); + $command .= "> $filename;"; + $command .= ($this->compressOutput() ? "cat" : "gunzip < ") . " $triggersDumpFile >> $filename;"; return $command; } diff --git a/src/Engines/MySql/Sandbox/Import.php b/src/Engines/MySql/Sandbox/Import.php index 771489a..698cfaf 100755 --- a/src/Engines/MySql/Sandbox/Import.php +++ b/src/Engines/MySql/Sandbox/Import.php @@ -74,14 +74,13 @@ public function getProperties(): array public function assembleCommand(string $path): string { $command = implode(' ', [ + "gunzip < $path |", "mysql --user={$this->remoteConnection->getUser()}", - "--password={$this->remoteConnection->getPassword()}", - "--host={$this->remoteConnection->getHost()}", - "--port={$this->remoteConnection->getPort()}", - $this->remoteConnection->useSsl() ? "--ssl-ca={$this->ssl->getPath()}" : "", - "{$this->remoteConnection->getDatabase()}", - "<", - $path + "--password={$this->remoteConnection->getPassword()}", + "--host={$this->remoteConnection->getHost()}", + "--port={$this->remoteConnection->getPort()}", + $this->remoteConnection->useSsl() ? "--ssl-ca={$this->ssl->getPath()}" : "", + "{$this->remoteConnection->getDatabase()}" ]); if ( diff --git a/src/Engines/MySql/Transformation.php b/src/Engines/MySql/Transformation.php index cb0a7f3..a5427d1 100644 --- a/src/Engines/MySql/Transformation.php +++ b/src/Engines/MySql/Transformation.php @@ -78,10 +78,7 @@ private function applyTransformationsTo(ReconnectingPDO $connection, array $tran $ex->getMessage(), $ex->getTraceAsString() ]); - $this->output->writeln("Query transformation failed: " . $query, [ - $ex->getMessage(), - $ex->getTraceAsString() - ] . ''); + $this->output->writeln("Query transformation failed: " . $query . ''); } }); } diff --git a/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php index 9b7a8fe..595df08 100644 --- a/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php +++ b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php @@ -42,7 +42,7 @@ public function testReturnsEmptyArrayIfNoTables(): void $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn([]); $this->assertSame( [], - $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.gz', 'triggers.gz') ); } @@ -52,7 +52,7 @@ public function testReturnsEmptyArrayIfAllTablesAreIgnored(): void $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['a', 'b']); $this->assertSame( [], - $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.gz', 'triggers.gz') ); } @@ -62,15 +62,14 @@ public function testReturnsCommandsForNormalTables(): void $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]); $this->assertSame( [ - "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db a >> dump.sql', - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db b >> dump.sql', - "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", - "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --skip-triggers --host=host db | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> dump.gz", + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --no-create-info --skip-triggers --host=host db a | gzip >> dump.gz', + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --no-create-info --skip-triggers --host=host db b | gzip >> dump.gz', + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --no-create-info --triggers --host=host db | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> triggers.gz" ], - $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.gz', 'triggers.gz') ); } @@ -81,13 +80,12 @@ public function testReturnsCommandsForEmptyTables(): void $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['a', 'b']); $this->assertSame( [ - "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db a b --no-data >> dump.sql', - "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", - "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --skip-triggers --host=host db | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> dump.gz", + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --no-create-info --triggers --host=host db | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> triggers.gz" ], - $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.gz', 'triggers.gz') ); } @@ -99,17 +97,14 @@ public function testReturnsCommandsForMixedTables(): void $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['b', 'e']); $this->assertSame( [ - "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;'>> dump.sql", - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db a >> dump.sql', - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db d >> dump.sql', - 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --host=host ' - . 'db b e --no-data >> dump.sql', - "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' >> dump.sql", - "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz" + "echo '/*!40014 SET @ORG_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --skip-triggers --host=host db --ignore-table=db.c --ignore-table=db.f | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> dump.gz", + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --no-create-info --skip-triggers --host=host db a | gzip >> dump.gz', + 'mysqldump --user="user" --password="password" --single-transaction --no-tablespaces --no-create-info --skip-triggers --host=host db d | gzip >> dump.gz', + "echo '/*!40014 SET FOREIGN_KEY_CHECKS=@ORG_FOREIGN_KEY_CHECKS */;' | gzip >> dump.gz", + "mysqldump --user=\"user\" --password=\"password\" --single-transaction --no-tablespaces --no-data --no-create-info --triggers --host=host db --ignore-table=db.c --ignore-table=db.f | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip >> triggers.gz" ], - $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.gz', 'triggers.gz') ); } }