From ec39c6e2b4cf875cfcd4487419d6a704a477111a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Biarda?= <1135380+michalbiarda@users.noreply.github.com> Date: Thu, 17 Nov 2022 18:02:36 +0100 Subject: [PATCH] #46: Split local export mysqldump command to multiple commands (one per table) --- src/Engines/MySql/Export/CommandAssembler.php | 70 +++++++++ src/Engines/MySql/Export/Primary.php | 133 +++++------------- src/Engines/MySql/Export/TablesProvider.php | 46 ++++++ .../MySql/Export/CommandAssemblerTest.php | 99 +++++++++++++ 4 files changed, 250 insertions(+), 98 deletions(-) create mode 100644 src/Engines/MySql/Export/CommandAssembler.php create mode 100644 src/Engines/MySql/Export/TablesProvider.php create mode 100644 src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php diff --git a/src/Engines/MySql/Export/CommandAssembler.php b/src/Engines/MySql/Export/CommandAssembler.php new file mode 100644 index 0000000..56ac06d --- /dev/null +++ b/src/Engines/MySql/Export/CommandAssembler.php @@ -0,0 +1,70 @@ +tablesProvider = $tablesProvider; + } + + public function execute( + ConnectionInterface $connection, + EnvironmentInterface $environment, + string $dumpFile + ): string { + $commands = []; + $ignoredTables = $this->tablesProvider->getIgnoredTables($environment); + $emptyTables = $this->tablesProvider->getEmptyTables($environment); + foreach ($this->tablesProvider->getAllTables($connection) as $table) { + if (in_array($table, $ignoredTables) || in_array($table, $emptyTables)) { + continue; + } + $commands[] = $this->getSingleCommand($connection, [$table], $dumpFile); + } + if (!empty($emptyTables)) { + $commands[] = $this->getSingleCommand($connection, $emptyTables, $dumpFile, false); + } + if (empty($commands)) { + return ''; + } + $commands[] = "cat $dumpFile | " + . "sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > $dumpFile.gz"; + return implode(';', $commands); + } + + /** + * @param string[] $tables + */ + private function getSingleCommand( + ConnectionInterface $connection, + array $tables, + string $dumpFile, + bool $withData = true + ): string { + $parts = [ + "mysqldump --user=\"{$connection->getUser()}\"", + "--password=\"{$connection->getPassword()}\"", + "--single-transaction", + "--host={$connection->getHost()}", + $connection->getDatabase(), + implode(' ', $tables) + ]; + if (!$withData) { + $parts[] = '--no-data'; + } + $parts[] = ">> $dumpFile"; + return implode(' ', $parts); + } +} diff --git a/src/Engines/MySql/Export/Primary.php b/src/Engines/MySql/Export/Primary.php index 7de4c05..d837749 100755 --- a/src/Engines/MySql/Export/Primary.php +++ b/src/Engines/MySql/Export/Primary.php @@ -13,6 +13,8 @@ use Driver\System\Configuration; use Driver\System\Logs\LoggerInterface; use Driver\System\Random; +use Exception; +use RuntimeException; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Output\ConsoleOutput; @@ -28,6 +30,7 @@ class Primary extends Command implements CommandInterface, CleanupInterface private ?string $path = null; private Configuration $configuration; private ConsoleOutput $output; + private CommandAssembler $commandAssembler; // phpcs:ignore SlevomatCodingStandard.TypeHints.ParameterTypeHint.MissingTraversableTypeHintSpecification public function __construct( @@ -36,6 +39,7 @@ public function __construct( LoggerInterface $logger, Random $random, ConsoleOutput $output, + CommandAssembler $commandAssembler, array $properties = [] ) { $this->localConnection = $localConnection; @@ -44,6 +48,7 @@ public function __construct( $this->random = $random; $this->configuration = $configuration; $this->output = $output; + $this->commandAssembler = $commandAssembler; return parent::__construct('mysql-default-export'); } @@ -52,35 +57,39 @@ public function go(TransportInterface $transport, EnvironmentInterface $environm $transport->getLogger()->notice("Exporting database from local MySql"); $this->output->writeln("Exporting database from local MySql"); - $transport->getLogger()->debug( - "Local connection string: " . str_replace( + try { + $command = $this->commandAssembler->execute($this->localConnection, $environment, $this->getDumpFile()); + if (empty($command)) { + throw new RuntimeException('Nothing to import'); + } + + $transport->getLogger()->debug( + "Local connection string: " . str_replace( + $this->localConnection->getPassword(), + '', + $command + ) + ); + $this->output->writeln("Local connection string: " . str_replace( $this->localConnection->getPassword(), '', - $this->assembleCommand($environment) - ) - ); - $this->output->writeln("Local connection string: " . str_replace( - $this->localConnection->getPassword(), - '', - $this->assembleCommand($environment) - )); - - $command = implode(';', array_filter([ - $this->assembleCommand($environment), - $this->assembleEmptyCommand($environment) - ])); - - $results = system($command); - - if ($results) { - $this->output->writeln('Import to RDS instance failed: ' . $results . ''); - throw new \Exception('Import to RDS instance failed: ' . $results); - } else { - $this->logger->notice("Database dump has completed."); - $this->output->writeln("Database dump has completed."); - return $transport->withStatus(new Status('sandbox_init', 'success')) - ->withNewData('dump-file', $this->getDumpFile()); + $command + )); + + $results = system($command); + + if ($results) { + throw new RuntimeException($results); + } + } catch (Exception $e) { + $this->output->writeln('Import to RDS instance failed: ' . $e->getMessage() . ''); + throw new Exception('Import to RDS instance failed: ' . $e->getMessage()); } + + $this->logger->notice("Database dump has completed."); + $this->output->writeln("Database dump has completed."); + return $transport->withStatus(new Status('sandbox_init', 'success')) + ->withNewData('dump-file', $this->getDumpFile()); } public function cleanup(TransportInterface $transport, EnvironmentInterface $environment): TransportInterface @@ -97,78 +106,6 @@ public function getProperties(): array return $this->properties; } - public function assembleEmptyCommand(EnvironmentInterface $environment): string - { - $tables = implode(' ', $environment->getEmptyTables()); - - if (!$tables) { - return ''; - } - - return implode(' ', array_merge( - $this->getDumpCommand(), - [ - "--no-data", - $tables, - "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'", - ">>", - $this->getDumpFile() - ] - )); - } - - public function assembleCommand(EnvironmentInterface $environment): string - { - return implode(' ', array_merge( - $this->getDumpCommand(), - [ - $this->assembleEmptyTables($environment), - $this->assembleIgnoredTables($environment), - "| sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g'", - ">", - $this->getDumpFile() - ] - )); - } - - /** - * @return string[] - */ - private function getDumpCommand(): array - { - return [ - "mysqldump --user=\"{$this->localConnection->getUser()}\"", - "--password=\"{$this->localConnection->getPassword()}\"", - "--single-transaction", - "--compress", - "--order-by-primary", - "--host={$this->localConnection->getHost()}", - "{$this->localConnection->getDatabase()}" - ]; - } - - private function assembleEmptyTables(EnvironmentInterface $environment): string - { - $tables = $environment->getEmptyTables(); - $output = []; - - foreach ($tables as $table) { - $output[] = '--ignore-table=' . $this->localConnection->getDatabase() . '.' . $table; - } - - return implode(' ', $output); - } - - private function assembleIgnoredTables(EnvironmentInterface $environment): string - { - $tables = $environment->getIgnoredTables(); - $output = implode(' | ', array_map(function ($table) { - return "awk '!/^INSERT INTO `{$table}` VALUES/'"; - }, $tables)); - - return $output ? ' | ' . $output : ''; - } - private function getDumpFile(): string { if (!$this->path) { diff --git a/src/Engines/MySql/Export/TablesProvider.php b/src/Engines/MySql/Export/TablesProvider.php new file mode 100644 index 0000000..9dd2de4 --- /dev/null +++ b/src/Engines/MySql/Export/TablesProvider.php @@ -0,0 +1,46 @@ +getUser()}\" --password=\"{$connection->getPassword()}\" " + . "--host=\"{$connection->getHost()}\" --skip-column-names " + . "-e \"SELECT GROUP_CONCAT(table_name SEPARATOR ',') FROM information_schema.tables " + . "WHERE table_schema = '{$connection->getDatabase()}';\""; + $result = exec($command); + if (!$result) { + throw new \RuntimeException('Unable to get table names'); + } + return explode(",", $result); + } + + /** + * @return string[] + */ + public function getEmptyTables(EnvironmentInterface $environment): array + { + return $environment->getEmptyTables(); + } + + /** + * @return string[] + */ + public function getIgnoredTables(EnvironmentInterface $environment): array + { + return $environment->getIgnoredTables(); + } +} diff --git a/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php new file mode 100644 index 0000000..d3ca759 --- /dev/null +++ b/src/Tests/Unit/Engines/MySql/Export/CommandAssemblerTest.php @@ -0,0 +1,99 @@ +tablesProviderMock = $this->getMockBuilder(TablesProvider::class) + ->disableOriginalConstructor()->getMock(); + $this->connectionMock = $this->getMockBuilder(ConnectionInterface::class)->getMockForAbstractClass(); + $this->connectionMock->expects($this->any())->method('getUser')->willReturn('user'); + $this->connectionMock->expects($this->any())->method('getPassword')->willReturn('password'); + $this->connectionMock->expects($this->any())->method('getHost')->willReturn('host'); + $this->connectionMock->expects($this->any())->method('getDatabase')->willReturn('db'); + $this->environmentMock = $this->getMockBuilder(EnvironmentInterface::class)->getMockForAbstractClass(); + $this->commandAssembler = new CommandAssembler($this->tablesProviderMock); + } + + public function testReturnsEmptyStringIfNoTables(): void + { + $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn([]); + $this->assertSame( + '', + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + ); + } + + public function testReturnsEmptyStringIfAllTablesAreIgnored(): void + { + $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); + $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['a', 'b']); + $this->assertSame( + '', + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + ); + } + + public function testReturnsCommandForNormalTables(): void + { + $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); + $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]); + $this->assertSame( + 'mysqldump --user="user" --password="password" --single-transaction --host=host db a >> dump.sql;' + . 'mysqldump --user="user" --password="password" --single-transaction --host=host db b >> dump.sql;' + . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + ); + } + + public function testReturnsCommandForEmptyTables(): void + { + $this->tablesProviderMock->expects($this->any())->method('getAllTables')->willReturn(['a', 'b']); + $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn([]); + $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['a', 'b']); + $this->assertSame( + 'mysqldump --user="user" --password="password" --single-transaction --host=host ' + . 'db a b --no-data >> dump.sql;' + . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + ); + } + + public function testReturnsCommandForMixedTables(): void + { + $this->tablesProviderMock->expects($this->any())->method('getAllTables') + ->willReturn(['a', 'b', 'c', 'd', 'e', 'f']); + $this->tablesProviderMock->expects($this->any())->method('getIgnoredTables')->willReturn(['c', 'f']); + $this->tablesProviderMock->expects($this->any())->method('getEmptyTables')->willReturn(['b', 'e']); + $this->assertSame( + 'mysqldump --user="user" --password="password" --single-transaction --host=host db a >> dump.sql;' + . 'mysqldump --user="user" --password="password" --single-transaction --host=host db d >> dump.sql;' + . 'mysqldump --user="user" --password="password" --single-transaction --host=host ' + . 'db b e --no-data >> dump.sql;' + . "cat dump.sql | sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' | gzip > dump.sql.gz", + $this->commandAssembler->execute($this->connectionMock, $this->environmentMock, 'dump.sql') + ); + } +}