diff --git a/apps/files/appinfo/app.php b/apps/files/appinfo/app.php index 26352795800e..9fbbc0a4dfd3 100644 --- a/apps/files/appinfo/app.php +++ b/apps/files/appinfo/app.php @@ -29,6 +29,9 @@ \OC::$server->getSearch()->registerProvider('OC\Search\Provider\File', ['apps' => ['files']]); +// instantiate to make sure services get registered +$app = new \OCA\Files\AppInfo\Application(); + $templateManager = \OC_Helper::getFileTemplateManager(); $templateManager->registerTemplate('text/html', 'core/templates/filetemplates/template.html'); $templateManager->registerTemplate('application/vnd.oasis.opendocument.presentation', 'core/templates/filetemplates/template.odp'); diff --git a/apps/files/lib/AppInfo/Application.php b/apps/files/lib/AppInfo/Application.php index 020c64178955..7953afc68d0d 100644 --- a/apps/files/lib/AppInfo/Application.php +++ b/apps/files/lib/AppInfo/Application.php @@ -87,6 +87,13 @@ public function __construct(array $urlParams= []) { ); }); + $container->registerService('OCP\Lock\ILockingProvider', function(IContainer $c) { + return $c->query('ServerContainer')->getLockingProvider(); + }); + $container->registerService('OCP\Files\IMimeTypeLoader', function(IContainer $c) { + return $c->query('ServerContainer')->getMimeTypeLoader(); + }); + /* * Register capabilities */ diff --git a/apps/files/lib/Command/Scan.php b/apps/files/lib/Command/Scan.php index 5693817d3fe4..511d744439a7 100644 --- a/apps/files/lib/Command/Scan.php +++ b/apps/files/lib/Command/Scan.php @@ -39,11 +39,23 @@ use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Helper\Table; +use OC\Repair\RepairMismatchFileCachePath; +use OC\Migration\ConsoleOutput; +use OCP\Lock\ILockingProvider; +use OCP\Lock\LockedException; +use OCP\Files\IMimeTypeLoader; +use OCP\IConfig; class Scan extends Base { /** @var IUserManager $userManager */ private $userManager; + /** @var ILockingProvider */ + private $lockingProvider; + /** @var IMimeTypeLoader */ + private $mimeTypeLoader; + /** @var IConfig */ + private $config; /** @var float */ protected $execTime = 0; /** @var int */ @@ -51,8 +63,16 @@ class Scan extends Base { /** @var int */ protected $filesCounter = 0; - public function __construct(IUserManager $userManager) { + public function __construct( + IUserManager $userManager, + ILockingProvider $lockingProvider, + IMimeTypeLoader $mimeTypeLoader, + IConfig $config + ) { $this->userManager = $userManager; + $this->lockingProvider = $lockingProvider; + $this->mimeTypeLoader = $mimeTypeLoader; + $this->config = $config; parent::__construct(); } @@ -90,6 +110,12 @@ protected function configure() { null, InputOption::VALUE_NONE, 'will rescan all files of all known users' + ) + ->addOption( + 'repair', + null, + InputOption::VALUE_NONE, + 'will repair detached filecache entries (slow)' )->addOption( 'unscanned', null, @@ -107,9 +133,48 @@ public function checkScanWarning($fullPath, OutputInterface $output) { } } - protected function scanFiles($user, $path, $verbose, OutputInterface $output, $backgroundScan = false) { + /** + * Repair all storages at once + * + * @param OutputInterface $output + */ + protected function repairAll(OutputInterface $output) { + $connection = $this->reconnectToDatabase($output); + $repairStep = new RepairMismatchFileCachePath( + $connection, + $this->mimeTypeLoader + ); + $repairStep->setStorageNumericId(null); + $repairStep->setCountOnly(false); + $repairStep->run(new ConsoleOutput($output)); + } + + protected function scanFiles($user, $path, $verbose, OutputInterface $output, $backgroundScan = false, $shouldRepair = false) { $connection = $this->reconnectToDatabase($output); $scanner = new \OC\Files\Utils\Scanner($user, $connection, \OC::$server->getLogger()); + if ($shouldRepair) { + $scanner->listen('\OC\Files\Utils\Scanner', 'beforeScanStorage', function ($storage) use ($output, $connection) { + try { + // FIXME: this will lock the storage even if there is nothing to repair + $storage->acquireLock('', ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); + } catch (OCP\Lock\LockedException $e) { + $output->writeln("\tStorage \"" . $storage->getCache()->getNumericStorageId() . '" cannot be repaired as it is currently in use, please try again later'); + return; + } + try { + $repairStep = new RepairMismatchFileCachePath( + $connection, + $this->mimeTypeLoader + ); + $repairStep->setStorageNumericId($storage->getCache()->getNumericStorageId()); + $repairStep->setCountOnly(false); + $repairStep->run(new ConsoleOutput($output)); + } finally { + $storage->releaseLock('', ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider); + } + }); + } + # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception # printout and count if ($verbose) { @@ -156,7 +221,7 @@ protected function scanFiles($user, $path, $verbose, OutputInterface $output, $b if ($backgroundScan) { $scanner->backgroundScan($path); }else { - $scanner->scan($path); + $scanner->scan($path, $shouldRepair); } } catch (ForbiddenException $e) { $output->writeln("Home storage for user $user not writable"); @@ -166,18 +231,32 @@ protected function scanFiles($user, $path, $verbose, OutputInterface $output, $b $output->writeln('Interrupted by user'); return; } catch (\Exception $e) { - $output->writeln('Exception during scan: ' . $e->getMessage() . "\n" . $e->getTraceAsString() . ''); + $output->writeln('Exception during scan: ' . get_class($e) . ': ' . $e->getMessage() . "\n" . $e->getTraceAsString() . ''); } } protected function execute(InputInterface $input, OutputInterface $output) { $inputPath = $input->getOption('path'); + $shouldRepairStoragesIndividually = (bool) $input->getOption('repair'); + if ($inputPath) { $inputPath = '/' . trim($inputPath, '/'); list (, $user,) = explode('/', $inputPath, 3); $users = [$user]; } else if ($input->getOption('all')) { + // we can only repair all storages in bulk (more efficient) if singleuser or maintenance mode + // is enabled to prevent concurrent user access + if ($input->getOption('repair') && + ($this->config->getSystemValue('singleuser', false) || $this->config->getSystemValue('maintenance', false))) { + // repair all storages at once + $this->repairAll($output); + // don't fix individually + $shouldRepairStoragesIndividually = false; + } else { + $output->writeln("Repairing every storage individually is slower than repairing in bulk"); + $output->writeln("To repair in bulk, please switch to single user mode first: occ maintenance:singleuser --on"); + } $users = $this->userManager->search(''); } else { $users = $input->getArgument('user_id'); @@ -223,9 +302,10 @@ protected function execute(InputInterface $input, OutputInterface $output) { if ($this->userManager->userExists($user)) { # add an extra line when verbose is set to optical separate users if ($verbose) {$output->writeln(""); } - $output->writeln("Starting scan for user $user_count out of $users_total ($user)"); + $r = $shouldRepairStoragesIndividually ? ' (and repair)' : ''; + $output->writeln("Starting scan$r for user $user_count out of $users_total ($user)"); # full: printout data if $verbose was set - $this->scanFiles($user, $path, $verbose, $output, $input->getOption('unscanned')); + $this->scanFiles($user, $path, $verbose, $output, $input->getOption('unscanned'), $shouldRepairStoragesIndividually); } else { $output->writeln("Unknown user $user_count $user"); } diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php index b0bd2038590a..5314cddcc8a5 100644 --- a/lib/private/Files/Utils/Scanner.php +++ b/lib/private/Files/Utils/Scanner.php @@ -211,6 +211,9 @@ public function scan($dir = '') { if ($storage->instanceOfStorage('OCA\Files_Sharing\ISharedStorage')) { continue; } + + $this->emit('\OC\Files\Utils\Scanner', 'beforeScanStorage', [$storage]); + $relativePath = $mount->getInternalPath($dir); $scanner = $storage->getScanner(); $scanner->setUseTransactions(false); @@ -247,6 +250,7 @@ public function scan($dir = '') { if ($this->useTransaction) { $this->db->commit(); } + $this->emit('\OC\Files\Utils\Scanner', 'afterScanStorage', [$storage]); } } diff --git a/lib/private/Repair.php b/lib/private/Repair.php index fe9e6ba756a6..968db2ce8d04 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -53,6 +53,7 @@ use Symfony\Component\EventDispatcher\EventDispatcher; use Symfony\Component\EventDispatcher\GenericEvent; use OC\Repair\MoveAvatarOutsideHome; +use OC\Repair\RepairMismatchFileCachePath; class Repair implements IOutput{ /* @var IRepairStep[] */ @@ -126,6 +127,7 @@ public function addStep($repairStep) { public static function getRepairSteps() { return [ new RepairMimeTypes(\OC::$server->getConfig()), + new RepairMismatchFileCachePath(\OC::$server->getDatabaseConnection(), \OC::$server->getMimeTypeLoader()), new FillETags(\OC::$server->getDatabaseConnection()), new CleanTags(\OC::$server->getDatabaseConnection(), \OC::$server->getUserManager()), new DropOldTables(\OC::$server->getDatabaseConnection()), diff --git a/lib/private/Repair/RepairMismatchFileCachePath.php b/lib/private/Repair/RepairMismatchFileCachePath.php new file mode 100644 index 000000000000..57473c4b6899 --- /dev/null +++ b/lib/private/Repair/RepairMismatchFileCachePath.php @@ -0,0 +1,563 @@ + + * + * @copyright Copyright (c) 2017, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\Repair; + +use Doctrine\DBAL\Platforms\PostgreSqlPlatform; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; +use Doctrine\DBAL\Platforms\MySqlPlatform; +use Doctrine\DBAL\Platforms\OraclePlatform; +use OCP\Files\IMimeTypeLoader; +use OCP\IDBConnection; + +/** + * Repairs file cache entry which path do not match the parent-child relationship + */ +class RepairMismatchFileCachePath implements IRepairStep { + + const CHUNK_SIZE = 10000; + + /** @var IDBConnection */ + protected $connection; + + /** @var IMimeTypeLoader */ + protected $mimeLoader; + + /** @var int */ + protected $dirMimeTypeId; + + /** @var int */ + protected $dirMimePartId; + + /** @var int|null */ + protected $storageNumericId = null; + + /** @var bool */ + protected $countOnly = true; + + /** + * @param \OCP\IDBConnection $connection + */ + public function __construct(IDBConnection $connection, IMimeTypeLoader $mimeLoader) { + $this->connection = $connection; + $this->mimeLoader = $mimeLoader; + } + + public function getName() { + if ($this->countOnly) { + return 'Detect file cache entries with path that does not match parent-child relationships'; + } else { + return 'Repair file cache entries with path that does not match parent-child relationships'; + } + } + + /** + * Sets the numeric id of the storage to process or null to process all. + * + * @param int $storageNumericId numeric id of the storage + */ + public function setStorageNumericId($storageNumericId) { + $this->storageNumericId = $storageNumericId; + } + + /** + * Sets whether to actually repair or only count entries + * + * @param bool $countOnly count only + */ + public function setCountOnly($countOnly) { + $this->countOnly = $countOnly; + } + + /** + * Fixes the broken entry's path. + * + * @param IOutput $out repair output + * @param int $fileId file id of the entry to fix + * @param string $wrongPath wrong path of the entry to fix + * @param int $correctStorageNumericId numeric idea of the correct storage + * @param string $correctPath value to which to set the path of the entry + * @return bool true for success + */ + private function fixEntryPath(IOutput $out, $fileId, $wrongPath, $correctStorageNumericId, $correctPath) { + // delete target if exists + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('storage', $qb->createNamedParameter($correctStorageNumericId))); + + if ($correctPath === '' && $this->connection->getDatabasePlatform() instanceof OraclePlatform) { + $qb->andWhere($qb->expr()->isNull('path')); + } else { + $qb->andWhere($qb->expr()->eq('path', $qb->createNamedParameter($correctPath))); + } + $entryExisted = $qb->execute() > 0; + + $qb = $this->connection->getQueryBuilder(); + $qb->update('filecache') + ->set('path', $qb->createNamedParameter($correctPath)) + ->set('path_hash', $qb->createNamedParameter(md5($correctPath))) + ->set('storage', $qb->createNamedParameter($correctStorageNumericId)) + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))); + $qb->execute(); + + $text = "Fixed file cache entry with fileid $fileId, set wrong path \"$wrongPath\" to \"$correctPath\""; + if ($entryExisted) { + $text = " (replaced an existing entry)"; + } + $out->advance(1, $text); + } + + private function addQueryConditionsParentIdWrongPath($qb) { + // thanks, VicDeo! + if ($this->connection->getDatabasePlatform() instanceof MySqlPlatform) { + $concatFunction = $qb->createFunction("CONCAT(fcp.path, '/', fc.name)"); + } else { + $concatFunction = $qb->createFunction("(fcp.`path` || '/' || fc.`name`)"); + } + + if ($this->connection->getDatabasePlatform() instanceof OraclePlatform) { + $emptyPathExpr = $qb->expr()->isNotNull('fcp.path'); + } else { + $emptyPathExpr = $qb->expr()->neq('fcp.path', $qb->expr()->literal('')); + } + + $qb + ->from('filecache', 'fc') + ->from('filecache', 'fcp') + ->where($qb->expr()->eq('fc.parent', 'fcp.fileid')) + ->andWhere( + $qb->expr()->orX( + $qb->expr()->neq( + $qb->createFunction($concatFunction), + 'fc.path' + ), + $qb->expr()->neq('fc.storage', 'fcp.storage') + ) + ) + ->andWhere($emptyPathExpr) + // yes, this was observed in the wild... + ->andWhere($qb->expr()->neq('fc.fileid', 'fcp.fileid')); + + if ($this->storageNumericId !== null) { + // use the target storage of the failed move when filtering + $qb->andWhere( + $qb->expr()->eq('fc.storage', $qb->createNamedParameter($this->storageNumericId)) + ); + } + } + + private function addQueryConditionsNonExistingParentIdEntry($qb, $storageNumericId = null) { + // Subquery for parent existence + $qbe = $this->connection->getQueryBuilder(); + $qbe->select($qbe->expr()->literal('1')) + ->from('filecache', 'fce') + ->where($qbe->expr()->eq('fce.fileid', 'fc.parent')); + + // Find entries to repair + // select fc.storage,fc.fileid,fc.parent as "wrongparent",fc.path,fc.etag + // and not exists (select 1 from oc_filecache fc2 where fc2.fileid = fc.parent) + $qb->select('storage', 'fileid', 'path', 'parent') + // from oc_filecache fc + ->from('filecache', 'fc') + // where fc.parent <> -1 + ->where($qb->expr()->neq('fc.parent', $qb->createNamedParameter(-1))) + // and not exists (select 1 from oc_filecache fc2 where fc2.fileid = fc.parent) + ->andWhere( + $qb->expr()->orX( + $qb->expr()->eq('fc.fileid', 'fc.parent'), + $qb->createFunction('NOT EXISTS (' . $qbe->getSQL() . ')') + ) + ); + + if ($storageNumericId !== null) { + // filter on destination storage of a failed move + $qb->andWhere($qb->expr()->eq('fc.storage', $qb->createNamedParameter($storageNumericId))); + } + } + + private function countResultsToProcessParentIdWrongPath($storageNumericId = null) { + $qb = $this->connection->getQueryBuilder(); + $qb->select($qb->createFunction('COUNT(*)')); + $this->addQueryConditionsParentIdWrongPath($qb, $storageNumericId); + $results = $qb->execute(); + $count = $results->fetchColumn(0); + $results->closeCursor(); + return $count; + } + + private function countResultsToProcessNonExistingParentIdEntry($storageNumericId = null) { + $qb = $this->connection->getQueryBuilder(); + $qb->select($qb->createFunction('COUNT(*)')); + $this->addQueryConditionsNonExistingParentIdEntry($qb, $storageNumericId); + $results = $qb->execute(); + $count = $results->fetchColumn(0); + $results->closeCursor(); + return $count; + } + + + /** + * Outputs a report about storages with wrong path that need repairing in the file cache + */ + private function reportAffectedStoragesParentIdWrongPath(IOutput $out) { + $qb = $this->connection->getQueryBuilder(); + $qb->selectDistinct('fc.storage'); + $this->addQueryConditionsParentIdWrongPath($qb); + + // TODO: max results + paginate ? + // TODO: join with oc_storages / oc_mounts to deliver user id ? + + $results = $qb->execute(); + $rows = $results->fetchAll(); + $results->closeCursor(); + + $storageIds = []; + foreach ($rows as $row) { + $storageIds[] = $row['storage']; + } + + if (!empty($storageIds)) { + $out->warning('The file cache contains entries with invalid path values for the following storage numeric ids: ' . implode(' ', $storageIds)); + $out->warning('Please run `occ files:scan --all --repair` to repair' + .'all affected storages or run `occ files:scan userid --repair for ' + .'each user with affected storages'); + } + } + + /** + * Outputs a report about storages with non existing parents that need repairing in the file cache + */ + private function reportAffectedStoragesNonExistingParentIdEntry(IOutput $out) { + $qb = $this->connection->getQueryBuilder(); + $qb->selectDistinct('fc.storage'); + $this->addQueryConditionsNonExistingParentIdEntry($qb); + + // TODO: max results + paginate ? + // TODO: join with oc_storages / oc_mounts to deliver user id ? + + $results = $qb->execute(); + $rows = $results->fetchAll(); + $results->closeCursor(); + + $storageIds = []; + foreach ($rows as $row) { + $storageIds[] = $row['storage']; + } + + if (!empty($storageIds)) { + $out->warning('The file cache contains entries where the parent id does not point to any existing entry for the following storage numeric ids: ' . implode(' ', $storageIds)); + $out->warning('Please run `occ files:scan --all --repair` to repair all affected storages'); + } + } + + /** + * Repair all entries for which the parent entry exists but the path + * value doesn't match the parent's path. + * + * @param IOutput $out + * @param int|null $storageNumericId storage to fix or null for all + * @return int[] storage numeric ids that were targets to a move and needs further fixing + */ + private function fixEntriesWithCorrectParentIdButWrongPath(IOutput $out, $storageNumericId = null) { + $totalResultsCount = 0; + $affectedStorages = [$storageNumericId => true]; + + // find all entries where the path entry doesn't match the path value that would + // be expected when following the parent-child relationship, basically + // concatenating the parent's "path" value with the name of the child + $qb = $this->connection->getQueryBuilder(); + $qb->select('fc.storage', 'fc.fileid', 'fc.name') + ->selectAlias('fc.path', 'path') + ->selectAlias('fc.parent', 'wrongparentid') + ->selectAlias('fcp.storage', 'parentstorage') + ->selectAlias('fcp.path', 'parentpath'); + $this->addQueryConditionsParentIdWrongPath($qb, $storageNumericId); + $qb->setMaxResults(self::CHUNK_SIZE); + + do { + $results = $qb->execute(); + // since we're going to operate on fetched entry, better cache them + // to avoid DB lock ups + $rows = $results->fetchAll(); + $results->closeCursor(); + + $this->connection->beginTransaction(); + $lastResultsCount = 0; + foreach ($rows as $row) { + $wrongPath = $row['path']; + $correctPath = $row['parentpath'] . '/' . $row['name']; + // make sure the target is on a different subtree + if (substr($correctPath, 0, strlen($wrongPath)) === $wrongPath) { + // the path based parent entry is referencing one of its own children, + // fix the entry's parent id instead + // note: fixEntryParent cannot fail to find the parent entry by path + // here because the reason we reached this code is because we already + // found it + $this->fixEntryParent( + $out, + $row['storage'], + $row['fileid'], + $row['path'], + $row['wrongparentid'], + true + ); + } else { + $this->fixEntryPath( + $out, + $row['fileid'], + $wrongPath, + $row['parentstorage'], + $correctPath + ); + // we also need to fix the target storage + $affectedStorages[$row['parentstorage']] = true; + } + $lastResultsCount++; + } + $this->connection->commit(); + + $totalResultsCount += $lastResultsCount; + + // note: this is not pagination but repeating the query over and over again + // until all possible entries were fixed + } while ($lastResultsCount > 0); + + if ($totalResultsCount > 0) { + $out->info("Fixed $totalResultsCount file cache entries with wrong path"); + } + + return array_keys($affectedStorages); + } + + /** + * Gets the file id of the entry. If none exists, create it + * up to the root if needed. + * + * @param int $storageId storage id + * @param string $path path for which to create the parent entry + * @return int file id of the newly created parent + */ + private function getOrCreateEntry($storageId, $path, $reuseFileId = null) { + if ($path === '.') { + $path = ''; + } + // find the correct parent + $qb = $this->connection->getQueryBuilder(); + // select fileid as "correctparentid" + $qb->select('fileid') + // from oc_filecache + ->from('filecache') + // where storage=$storage and path='$parentPath' + ->where($qb->expr()->eq('storage', $qb->createNamedParameter($storageId))); + + if ($path === '' && $this->connection->getDatabasePlatform() instanceof OraclePlatform) { + $qb->andWhere($qb->expr()->isNull('path')); + } else { + $qb->andWhere($qb->expr()->eq('path', $qb->createNamedParameter($path))); + } + $results = $qb->execute(); + $rows = $results->fetchAll(); + $results->closeCursor(); + + if (!empty($rows)) { + return $rows[0]['fileid']; + } + + if ($path !== '') { + $parentId = $this->getOrCreateEntry($storageId, dirname($path)); + } else { + // root entry missing, create it + $parentId = -1; + } + + $qb = $this->connection->getQueryBuilder(); + $values = [ + 'storage' => $qb->createNamedParameter($storageId), + 'path' => $qb->createNamedParameter($path), + 'path_hash' => $qb->createNamedParameter(md5($path)), + 'name' => $qb->createNamedParameter(basename($path)), + 'parent' => $qb->createNamedParameter($parentId), + 'size' => $qb->createNamedParameter(-1), + 'etag' => $qb->createNamedParameter('zombie'), + 'mimetype' => $qb->createNamedParameter($this->dirMimeTypeId), + 'mimepart' => $qb->createNamedParameter($this->dirMimePartId), + ]; + + if ($reuseFileId !== null) { + // purpose of reusing the fileid of the parent is to salvage potential + // metadata that might have previously been linked to this file id + $values['fileid'] = $qb->createNamedParameter($reuseFileId); + } + $qb->insert('filecache')->values($values); + $qb->execute(); + + // If we reused the fileid then this is the id to return + if($reuseFileId !== null) { + // with Oracle, the trigger gets in the way and does not let us specify + // a fileid value on insert + if ($this->connection->getDatabasePlatform() instanceof OraclePlatform) { + $lastFileId = $this->connection->lastInsertId('*PREFIX*filecache'); + if ($reuseFileId !== $lastFileId) { + // use update to set it directly + $qb = $this->connection->getQueryBuilder(); + $qb->update('filecache') + ->set('fileid', $qb->createNamedParameter($reuseFileId)) + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($lastFileId))); + $qb->execute(); + } + } + + return $reuseFileId; + } else { + // Else we inserted a new row with auto generated id, use that + return $this->connection->lastInsertId('*PREFIX*filecache'); + } + } + + /** + * Fixes the broken entry's path. + * + * @param IOutput $out repair output + * @param int $storageId storage id of the entry to fix + * @param int $fileId file id of the entry to fix + * @param string $path path from the entry to fix + * @param int $wrongParentId wrong parent id + * @param bool $parentIdExists true if the entry from the $wrongParentId exists (but is the wrong one), + * false if it doesn't + * @return bool true if the entry was fixed, false otherwise + */ + private function fixEntryParent(IOutput $out, $storageId, $fileId, $path, $wrongParentId, $parentIdExists = false) { + if (!$parentIdExists) { + // if the parent doesn't exist, let us reuse its id in case there is metadata to salvage + $correctParentId = $this->getOrCreateEntry($storageId, dirname($path), $wrongParentId); + } else { + // parent exists and is the wrong one, so recreating would need a new fileid + $correctParentId = $this->getOrCreateEntry($storageId, dirname($path)); + } + + $this->connection->beginTransaction(); + + $qb = $this->connection->getQueryBuilder(); + $qb->update('filecache') + ->set('parent', $qb->createNamedParameter($correctParentId)) + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))); + $qb->execute(); + + $text = "Fixed file cache entry with fileid $fileId, set wrong parent \"$wrongParentId\" to \"$correctParentId\""; + $out->advance(1, $text); + + $this->connection->commit(); + + return true; + } + + /** + * Repair entries where the parent id doesn't point to any existing entry + * by finding the actual parent entry matching the entry's path dirname. + * + * @param IOutput $out output + * @param int|null $storageNumericId storage to fix or null for all + * @return int number of results that were fixed + */ + private function fixEntriesWithNonExistingParentIdEntry(IOutput $out, $storageNumericId = null) { + $qb = $this->connection->getQueryBuilder(); + $this->addQueryConditionsNonExistingParentIdEntry($qb, $storageNumericId); + $qb->setMaxResults(self::CHUNK_SIZE); + + $totalResultsCount = 0; + do { + $results = $qb->execute(); + // since we're going to operate on fetched entry, better cache them + // to avoid DB lock ups + $rows = $results->fetchAll(); + $results->closeCursor(); + + $lastResultsCount = 0; + foreach ($rows as $row) { + $this->fixEntryParent( + $out, + $row['storage'], + $row['fileid'], + $row['path'], + $row['parent'], + // in general the parent doesn't exist except + // for the one condition where parent=fileid + $row['parent'] === $row['fileid'] + ); + $lastResultsCount++; + } + + $totalResultsCount += $lastResultsCount; + + // note: this is not pagination but repeating the query over and over again + // until all possible entries were fixed + } while ($lastResultsCount > 0); + + if ($totalResultsCount > 0) { + $out->info("Fixed $totalResultsCount file cache entries with wrong path"); + } + + return $totalResultsCount; + } + + /** + * Run the repair step + * + * @param IOutput $out output + */ + public function run(IOutput $out) { + + $this->dirMimeTypeId = $this->mimeLoader->getId('httpd/unix-directory'); + $this->dirMimePartId = $this->mimeLoader->getId('httpd'); + + if ($this->countOnly) { + $this->reportAffectedStoragesParentIdWrongPath($out); + $this->reportAffectedStoragesNonExistingParentIdEntry($out); + } else { + $brokenPathEntries = $this->countResultsToProcessParentIdWrongPath($this->storageNumericId); + $brokenParentIdEntries = $this->countResultsToProcessNonExistingParentIdEntry($this->storageNumericId); + $out->startProgress($brokenPathEntries + $brokenParentIdEntries); + + $totalFixed = 0; + + /* + * This repair itself might overwrite existing target parent entries and create + * orphans where the parent entry of the parent id doesn't exist but the path matches. + * This needs to be repaired by fixEntriesWithNonExistingParentIdEntry(), this is why + * we need to keep this specific order of repair. + */ + $affectedStorages = $this->fixEntriesWithCorrectParentIdButWrongPath($out, $this->storageNumericId); + + if ($this->storageNumericId !== null) { + foreach ($affectedStorages as $storageNumericId) { + $this->fixEntriesWithNonExistingParentIdEntry($out, $storageNumericId); + } + } else { + // just fix all + $this->fixEntriesWithNonExistingParentIdEntry($out); + } + $out->finishProgress(); + $out->info(''); + } + } +} diff --git a/lib/private/Server.php b/lib/private/Server.php index 96da071a583a..59fa05491c4e 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -652,6 +652,7 @@ public function __construct($webRoot, \OC\Config $config) { } return new NoopLockingProvider(); }); + $this->registerAlias('OCP\Lock\ILockingProvider', 'LockingProvider'); $this->registerService('MountManager', function () { return new \OC\Files\Mount\Manager(); }); @@ -667,6 +668,7 @@ public function __construct($webRoot, \OC\Config $config) { $c->getDatabaseConnection() ); }); + $this->registerAlias('OCP\Files\IMimeTypeLoader', 'MimeTypeLoader'); $this->registerService('NotificationManager', function () { return new Manager(); }); diff --git a/lib/public/IDBConnection.php b/lib/public/IDBConnection.php index ac09a680f22d..788aba37e331 100644 --- a/lib/public/IDBConnection.php +++ b/lib/public/IDBConnection.php @@ -92,6 +92,9 @@ public function executeUpdate($query, array $params = [], array $types = []); /** * Used to get the id of the just inserted element + * Note: On postgres platform, this will return the last sequence id which + * may not be the id last inserted if you were reinserting a previously + * used auto_increment id. * @param string $table the name of the table where we inserted the item * @return int the id of the inserted element * @since 6.0.0 diff --git a/tests/lib/Repair/RepairMismatchFileCachePathTest.php b/tests/lib/Repair/RepairMismatchFileCachePathTest.php new file mode 100644 index 000000000000..7c4497a8ed15 --- /dev/null +++ b/tests/lib/Repair/RepairMismatchFileCachePathTest.php @@ -0,0 +1,734 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace Test\Repair; + + +use OC\Repair\RepairMismatchFileCachePath; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; +use Test\TestCase; +use OCP\Files\IMimeTypeLoader; + +/** + * Tests for repairing mismatch file cache paths + * + * @group DB + * + * @see \OC\Repair\RepairMismatchFileCachePath + */ +class RepairMismatchFileCachePathTest extends TestCase { + + /** @var IRepairStep */ + private $repair; + + /** @var \OCP\IDBConnection */ + private $connection; + + protected function setUp() { + parent::setUp(); + + $this->connection = \OC::$server->getDatabaseConnection(); + + $mimeLoader = $this->createMock(IMimeTypeLoader::class); + $mimeLoader->method('getId') + ->will($this->returnValueMap([ + ['httpd', 1], + ['httpd/unix-directory', 2], + ])); + $this->repair = new RepairMismatchFileCachePath($this->connection, $mimeLoader); + $this->repair->setCountOnly(false); + } + + protected function tearDown() { + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache')->execute(); + parent::tearDown(); + } + + private function createFileCacheEntry($storage, $path, $parent = -1) { + $qb = $this->connection->getQueryBuilder(); + $qb->insert('filecache') + ->values([ + 'storage' => $qb->createNamedParameter($storage), + 'path' => $qb->createNamedParameter($path), + 'path_hash' => $qb->createNamedParameter(md5($path)), + 'name' => $qb->createNamedParameter(basename($path)), + 'parent' => $qb->createNamedParameter($parent), + ]); + $qb->execute(); + return $this->connection->lastInsertId('*PREFIX*filecache'); + } + + /** + * Returns autoincrement compliant fileid for an entry that might + * have existed + * + * @return int fileid + */ + private function createNonExistingId() { + // why are we doing this ? well, if we just pick an arbitrary + // value ahead of the autoincrement, this will not reflect real scenarios + // and also will likely cause potential collisions as some newly inserted entries + // might receive said arbitrary id through autoincrement + // + // So instead, we insert a dummy entry and delete it afterwards so we have + // "reserved" the fileid and also somehow simulated whatever might have happened + // on a real system when a file cache entry suddenly disappeared for whatever + // mysterious reasons + $entryId = $this->createFileCacheEntry(1, 'goodbye-entry'); + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($entryId))); + $qb->execute(); + return $entryId; + } + + private function getFileCacheEntry($fileId) { + $qb = $this->connection->getQueryBuilder(); + $qb->select('*') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))); + $results = $qb->execute(); + $result = $results->fetch(); + $results->closeCursor(); + return $result; + } + + /** + * Sets the parent of the given file id to the given parent id + * + * @param int $fileId file id of the entry to adjust + * @param int $parentId parent id to set to + */ + private function setFileCacheEntryParent($fileId, $parentId) { + $qb = $this->connection->getQueryBuilder(); + $qb->update('filecache') + ->set('parent', $qb->createNamedParameter($parentId)) + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))); + $qb->execute(); + } + + public function repairCasesProvider() { + return [ + // same storage, different target dir + [1, 1, 'target', false, null], + [1, 1, 'target', false, [1]], + // different storage, same target dir name + [1, 2, 'source', false, null], + [1, 2, 'source', false, [1, 2]], + [1, 2, 'source', false, [2, 1]], + // different storage, different target dir + [1, 2, 'target', false, null], + [1, 2, 'target', false, [1, 2]], + [1, 2, 'target', false, [2, 1]], + + // same storage, different target dir, target exists + [1, 1, 'target', true, null], + [1, 1, 'target', true, [1, 2]], + [1, 1, 'target', true, [2, 1]], + // different storage, same target dir name, target exists + [1, 2, 'source', true, null], + [1, 2, 'source', true, [1, 2]], + [1, 2, 'source', true, [2, 1]], + // different storage, different target dir, target exists + [1, 2, 'target', true, null], + [1, 2, 'target', true, [1, 2]], + [1, 2, 'target', true, [2, 1]], + ]; + } + + /** + * Test repair + * + * @dataProvider repairCasesProvider + */ + public function testRepairEntry($sourceStorageId, $targetStorageId, $targetDir, $targetExists, $repairStoragesOrder) { + /* + * Tree: + * + * source storage: + * - files/ + * - files/source/ + * - files/source/to_move (not created as we simulate that it was already moved) + * - files/source/to_move/content_to_update (bogus entry to fix) + * - files/source/to_move/content_to_update/sub (bogus subentry to fix) + * - files/source/do_not_touch (regular entry outside of the repair scope) + * + * target storage: + * - files/ + * - files/target/ + * - files/target/moved_renamed (already moved target) + * - files/target/moved_renamed/content_to_update (missing until repair) + * + * if $targetExists: pre-create these additional entries: + * - files/target/moved_renamed/content_to_update (will be overwritten) + * - files/target/moved_renamed/content_to_update/sub (will be overwritten) + * - files/target/moved_renamed/content_to_update/unrelated (will be reparented) + * + */ + + // source storage entries + $rootId1 = $this->createFileCacheEntry($sourceStorageId, ''); + $baseId1 = $this->createFileCacheEntry($sourceStorageId, 'files', $rootId1); + if ($sourceStorageId !== $targetStorageId) { + $rootId2 = $this->createFileCacheEntry($targetStorageId, ''); + $baseId2 = $this->createFileCacheEntry($targetStorageId, 'files', $rootId2); + } else { + $rootId2 = $rootId1; + $baseId2 = $baseId1; + } + $sourceId = $this->createFileCacheEntry($sourceStorageId, 'files/source', $baseId1); + + // target storage entries + $targetParentId = $this->createFileCacheEntry($targetStorageId, 'files/' . $targetDir, $baseId2); + + // the move does create the parent in the target + $targetId = $this->createFileCacheEntry($targetStorageId, 'files/' . $targetDir . '/moved_renamed', $targetParentId); + + // bogus entry: any children of the source are not properly updated + $movedId = $this->createFileCacheEntry($sourceStorageId, 'files/source/to_move/content_to_update', $targetId); + $movedSubId = $this->createFileCacheEntry($sourceStorageId, 'files/source/to_move/content_to_update/sub', $movedId); + + if ($targetExists) { + // after the bogus move happened, some code path recreated the parent under a + // different file id + $existingTargetId = $this->createFileCacheEntry($targetStorageId, 'files/' . $targetDir . '/moved_renamed/content_to_update', $targetId); + $existingTargetSubId = $this->createFileCacheEntry($targetStorageId, 'files/' . $targetDir . '/moved_renamed/content_to_update/sub', $existingTargetId); + $existingTargetUnrelatedId = $this->createFileCacheEntry($targetStorageId, 'files/' . $targetDir . '/moved_renamed/content_to_update/unrelated', $existingTargetId); + } + + $doNotTouchId = $this->createFileCacheEntry($sourceStorageId, 'files/source/do_not_touch', $sourceId); + + $outputMock = $this->createMock(IOutput::class); + if (is_null($repairStoragesOrder)) { + // no storage selected, full repair + $this->repair->setStorageNumericId(null); + $this->repair->run($outputMock); + } else { + foreach ($repairStoragesOrder as $storageId) { + $this->repair->setStorageNumericId($storageId); + $this->repair->run($outputMock); + } + } + + $entry = $this->getFileCacheEntry($movedId); + $this->assertEquals($targetId, $entry['parent']); + $this->assertEquals((string)$targetStorageId, $entry['storage']); + $this->assertEquals('files/' . $targetDir . '/moved_renamed/content_to_update', $entry['path']); + $this->assertEquals(md5('files/' . $targetDir . '/moved_renamed/content_to_update'), $entry['path_hash']); + + $entry = $this->getFileCacheEntry($movedSubId); + $this->assertEquals($movedId, $entry['parent']); + $this->assertEquals((string)$targetStorageId, $entry['storage']); + $this->assertEquals('files/' . $targetDir . '/moved_renamed/content_to_update/sub', $entry['path']); + $this->assertEquals(md5('files/' . $targetDir . '/moved_renamed/content_to_update/sub'), $entry['path_hash']); + + if ($targetExists) { + $this->assertFalse($this->getFileCacheEntry($existingTargetId)); + $this->assertFalse($this->getFileCacheEntry($existingTargetSubId)); + + // unrelated folder has been reparented + $entry = $this->getFileCacheEntry($existingTargetUnrelatedId); + $this->assertEquals($movedId, $entry['parent']); + $this->assertEquals((string)$targetStorageId, $entry['storage']); + $this->assertEquals('files/' . $targetDir . '/moved_renamed/content_to_update/unrelated', $entry['path']); + $this->assertEquals(md5('files/' . $targetDir . '/moved_renamed/content_to_update/unrelated'), $entry['path_hash']); + } + + // root entries left alone + $entry = $this->getFileCacheEntry($rootId1); + $this->assertEquals(-1, $entry['parent']); + $this->assertEquals((string)$sourceStorageId, $entry['storage']); + $this->assertEquals('', $entry['path']); + $this->assertEquals(md5(''), $entry['path_hash']); + + $entry = $this->getFileCacheEntry($rootId2); + $this->assertEquals(-1, $entry['parent']); + $this->assertEquals((string)$targetStorageId, $entry['storage']); + $this->assertEquals('', $entry['path']); + $this->assertEquals(md5(''), $entry['path_hash']); + + // "do not touch" entry left untouched + $entry = $this->getFileCacheEntry($doNotTouchId); + $this->assertEquals($sourceId, $entry['parent']); + $this->assertEquals((string)$sourceStorageId, $entry['storage']); + $this->assertEquals('files/source/do_not_touch', $entry['path']); + $this->assertEquals(md5('files/source/do_not_touch'), $entry['path_hash']); + + } + + /** + * Test repair self referencing entries + */ + public function testRepairSelfReferencing() { + /** + * This is the storage tree that is created + * (alongside a normal storage, without corruption, but same structure) + * + * + * Self-referencing: + * - files/all_your_zombies (parent=fileid must be reparented) + * + * Referencing child one level: + * - files/ref_child1 (parent=fileid of the child) + * - files/ref_child1/child + * + * Referencing child two levels: + * - files/ref_child2/ (parent=fileid of the child's child) + * - files/ref_child2/child + * - files/ref_child2/child/child + * + * Referencing child two levels detached: + * - detached/ref_child3/ (parent=fileid of the child, "detached" has no entry) + * - detached/ref_child3/child + * + * Normal files that should be untouched + * - files/untouched_folder + * - files/untouched.file + */ + + // Test, corrupt storage + $storageId = 1; + $rootId1 = $this->createFileCacheEntry($storageId, ''); + $baseId1 = $this->createFileCacheEntry($storageId, 'files', $rootId1); + + $selfRefId = $this->createFileCacheEntry($storageId, 'files/all_your_zombies', $baseId1); + $this->setFileCacheEntryParent($selfRefId, $selfRefId); + + $refChild1Id = $this->createFileCacheEntry($storageId, 'files/ref_child1', $baseId1); + $refChild1ChildId = $this->createFileCacheEntry($storageId, 'files/ref_child1/child', $refChild1Id); + // make it reference its own child + $this->setFileCacheEntryParent($refChild1Id, $refChild1ChildId); + + $refChild2Id = $this->createFileCacheEntry($storageId, 'files/ref_child2', $baseId1); + $refChild2ChildId = $this->createFileCacheEntry($storageId, 'files/ref_child2/child', $refChild2Id); + $refChild2ChildChildId = $this->createFileCacheEntry($storageId, 'files/ref_child2/child/child', $refChild2ChildId); + // make it reference its own sub child + $this->setFileCacheEntryParent($refChild2Id, $refChild2ChildChildId); + + $willBeOverwritten = -1; + $refChild3Id = $this->createFileCacheEntry($storageId, 'detached/ref_child3', $willBeOverwritten); + $refChild3ChildId = $this->createFileCacheEntry($storageId, 'detached/ref_child3/child', $refChild3Id); + // make it reference its own child + $this->setFileCacheEntryParent($refChild3Id, $refChild3ChildId); + + $untouchedFileId = $this->createFileCacheEntry($storageId, 'files/untouched.file', $baseId1); + $untouchedFolderId = $this->createFileCacheEntry($storageId, 'files/untouched_folder', $baseId1); + // End correct storage + + // Parallel, correct, but identical storage - used to check for storage isolation and query scope + $storageId2 = 2; + $rootId2 = $this->createFileCacheEntry($storageId2, ''); + $baseId2 = $this->createFileCacheEntry($storageId2, 'files', $rootId2); + + $selfRefId_parallel = $this->createFileCacheEntry($storageId2, 'files/all_your_zombies', $baseId2); + + $refChild1Id_parallel = $this->createFileCacheEntry($storageId2, 'files/ref_child1', $baseId2); + $refChild1ChildId_parallel = $this->createFileCacheEntry($storageId2, 'files/ref_child1/child', $refChild1Id_parallel); + + $refChild2Id_parallel = $this->createFileCacheEntry($storageId2, 'files/ref_child2', $baseId2); + $refChild2ChildId_parallel = $this->createFileCacheEntry($storageId2, 'files/ref_child2/child', $refChild2Id_parallel); + $refChild2ChildChildId_parallel = $this->createFileCacheEntry($storageId2, 'files/ref_child2/child/child', $refChild2ChildId_parallel); + + $refChild3DetachedId_parallel = $this->createFileCacheEntry($storageId2, 'detached', $rootId2); + $refChild3Id_parallel = $this->createFileCacheEntry($storageId2, 'detached/ref_child3', $refChild3DetachedId_parallel); + $refChild3ChildId_parallel = $this->createFileCacheEntry($storageId2, 'detached/ref_child3/child', $refChild3Id_parallel); + + $untouchedFileId_parallel = $this->createFileCacheEntry($storageId2, 'files/untouched.file', $baseId2); + $untouchedFolderId_parallel = $this->createFileCacheEntry($storageId2, 'files/untouched_folder', $baseId2); + // End parallel storage + + + $outputMock = $this->createMock(IOutput::class); + $this->repair->setStorageNumericId($storageId); + $this->repair->run($outputMock); + + // self-referencing updated + $entry = $this->getFileCacheEntry($selfRefId); + $this->assertEquals($baseId1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('files/all_your_zombies', $entry['path']); + $this->assertEquals(md5('files/all_your_zombies'), $entry['path_hash']); + + // ref child 1 case was reparented to "files" + $entry = $this->getFileCacheEntry($refChild1Id); + $this->assertEquals($baseId1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('files/ref_child1', $entry['path']); + $this->assertEquals(md5('files/ref_child1'), $entry['path_hash']); + + // ref child 1 child left alone + $entry = $this->getFileCacheEntry($refChild1ChildId); + $this->assertEquals($refChild1Id, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('files/ref_child1/child', $entry['path']); + $this->assertEquals(md5('files/ref_child1/child'), $entry['path_hash']); + + // ref child 2 case was reparented to "files" + $entry = $this->getFileCacheEntry($refChild2Id); + $this->assertEquals($baseId1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('files/ref_child2', $entry['path']); + $this->assertEquals(md5('files/ref_child2'), $entry['path_hash']); + + // ref child 2 child left alone + $entry = $this->getFileCacheEntry($refChild2ChildId); + $this->assertEquals($refChild2Id, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('files/ref_child2/child', $entry['path']); + $this->assertEquals(md5('files/ref_child2/child'), $entry['path_hash']); + + // ref child 2 child child left alone + $entry = $this->getFileCacheEntry($refChild2ChildChildId); + $this->assertEquals($refChild2ChildId, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('files/ref_child2/child/child', $entry['path']); + $this->assertEquals(md5('files/ref_child2/child/child'), $entry['path_hash']); + + // root entry left alone + $entry = $this->getFileCacheEntry($rootId1); + $this->assertEquals(-1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('', $entry['path']); + $this->assertEquals(md5(''), $entry['path_hash']); + + // ref child 3 child left alone + $entry = $this->getFileCacheEntry($refChild3ChildId); + $this->assertEquals($refChild3Id, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('detached/ref_child3/child', $entry['path']); + $this->assertEquals(md5('detached/ref_child3/child'), $entry['path_hash']); + + // ref child 3 case was reparented to a new "detached" entry + $entry = $this->getFileCacheEntry($refChild3Id); + $this->assertTrue(isset($entry['parent'])); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('detached/ref_child3', $entry['path']); + $this->assertEquals(md5('detached/ref_child3'), $entry['path_hash']); + + // entry "detached" was restored + $entry = $this->getFileCacheEntry($entry['parent']); + $this->assertEquals($rootId1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('detached', $entry['path']); + $this->assertEquals(md5('detached'), $entry['path_hash']); + + // untouched file and folder are untouched + $entry = $this->getFileCacheEntry($untouchedFileId); + $this->assertEquals($baseId1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('files/untouched.file', $entry['path']); + $this->assertEquals(md5('files/untouched.file'), $entry['path_hash']); + $entry = $this->getFileCacheEntry($untouchedFolderId); + $this->assertEquals($baseId1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('files/untouched_folder', $entry['path']); + $this->assertEquals(md5('files/untouched_folder'), $entry['path_hash']); + + // check that parallel storage is untouched + // self-referencing updated + $entry = $this->getFileCacheEntry($selfRefId_parallel); + $this->assertEquals($baseId2, $entry['parent']); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('files/all_your_zombies', $entry['path']); + $this->assertEquals(md5('files/all_your_zombies'), $entry['path_hash']); + + // ref child 1 case was reparented to "files" + $entry = $this->getFileCacheEntry($refChild1Id_parallel); + $this->assertEquals($baseId2, $entry['parent']); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('files/ref_child1', $entry['path']); + $this->assertEquals(md5('files/ref_child1'), $entry['path_hash']); + + // ref child 1 child left alone + $entry = $this->getFileCacheEntry($refChild1ChildId_parallel); + $this->assertEquals($refChild1Id_parallel, $entry['parent']); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('files/ref_child1/child', $entry['path']); + $this->assertEquals(md5('files/ref_child1/child'), $entry['path_hash']); + + // ref child 2 case was reparented to "files" + $entry = $this->getFileCacheEntry($refChild2Id_parallel); + $this->assertEquals($baseId2, $entry['parent']); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('files/ref_child2', $entry['path']); + $this->assertEquals(md5('files/ref_child2'), $entry['path_hash']); + + // ref child 2 child left alone + $entry = $this->getFileCacheEntry($refChild2ChildId_parallel); + $this->assertEquals($refChild2Id_parallel, $entry['parent']); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('files/ref_child2/child', $entry['path']); + $this->assertEquals(md5('files/ref_child2/child'), $entry['path_hash']); + + // ref child 2 child child left alone + $entry = $this->getFileCacheEntry($refChild2ChildChildId_parallel); + $this->assertEquals($refChild2ChildId_parallel, $entry['parent']); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('files/ref_child2/child/child', $entry['path']); + $this->assertEquals(md5('files/ref_child2/child/child'), $entry['path_hash']); + + // root entry left alone + $entry = $this->getFileCacheEntry($rootId2); + $this->assertEquals(-1, $entry['parent']); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('', $entry['path']); + $this->assertEquals(md5(''), $entry['path_hash']); + + // ref child 3 child left alone + $entry = $this->getFileCacheEntry($refChild3ChildId_parallel); + $this->assertEquals($refChild3Id_parallel, $entry['parent']); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('detached/ref_child3/child', $entry['path']); + $this->assertEquals(md5('detached/ref_child3/child'), $entry['path_hash']); + + // ref child 3 case was reparented to a new "detached" entry + $entry = $this->getFileCacheEntry($refChild3Id_parallel); + $this->assertTrue(isset($entry['parent'])); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('detached/ref_child3', $entry['path']); + $this->assertEquals(md5('detached/ref_child3'), $entry['path_hash']); + + // entry "detached" was untouched + $entry = $this->getFileCacheEntry($entry['parent']); + $this->assertEquals($rootId2, $entry['parent']); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('detached', $entry['path']); + $this->assertEquals(md5('detached'), $entry['path_hash']); + + // untouched file and folder are untouched + $entry = $this->getFileCacheEntry($untouchedFileId_parallel); + $this->assertEquals($baseId2, $entry['parent']); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('files/untouched.file', $entry['path']); + $this->assertEquals(md5('files/untouched.file'), $entry['path_hash']); + $entry = $this->getFileCacheEntry($untouchedFolderId_parallel); + $this->assertEquals($baseId2, $entry['parent']); + $this->assertEquals((string)$storageId2, $entry['storage']); + $this->assertEquals('files/untouched_folder', $entry['path']); + $this->assertEquals(md5('files/untouched_folder'), $entry['path_hash']); + + // end testing parallel storage + } + + + /** + * Test repair wrong parent id + */ + public function testRepairParentIdPointingNowhere() { + /** + * Wrong parent id + * - wrongparentroot + * - files/wrongparent + */ + $storageId = 1; + $rootId1 = $this->createFileCacheEntry($storageId, ''); + $baseId1 = $this->createFileCacheEntry($storageId, 'files', $rootId1); + + $nonExistingParentId = $this->createNonExistingId(); + $wrongParentRootId = $this->createFileCacheEntry($storageId, 'wrongparentroot', $nonExistingParentId); + $wrongParentId = $this->createFileCacheEntry($storageId, 'files/wrongparent', $nonExistingParentId); + + $outputMock = $this->createMock(IOutput::class); + $this->repair->setStorageNumericId($storageId); + $this->repair->run($outputMock); + + // wrong parent root reparented to actual root + $entry = $this->getFileCacheEntry($wrongParentRootId); + $this->assertEquals($rootId1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('wrongparentroot', $entry['path']); + $this->assertEquals(md5('wrongparentroot'), $entry['path_hash']); + + // wrong parent subdir reparented to "files" + $entry = $this->getFileCacheEntry($wrongParentId); + $this->assertEquals($baseId1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('files/wrongparent', $entry['path']); + $this->assertEquals(md5('files/wrongparent'), $entry['path_hash']); + + // root entry left alone + $entry = $this->getFileCacheEntry($rootId1); + $this->assertEquals(-1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('', $entry['path']); + $this->assertEquals(md5(''), $entry['path_hash']); + } + + /** + * Test repair detached subtree + */ + public function testRepairDetachedSubtree() { + /** + * - files/missingdir/orphaned1 (orphaned entry as "missingdir" is missing) + * - missingdir/missingdir1/orphaned2 (orphaned entry two levels up to root) + */ + + // Corrupt storage + $storageId = 1; + $rootId1 = $this->createFileCacheEntry($storageId, ''); + $baseId1 = $this->createFileCacheEntry($storageId, 'files', $rootId1); + + $nonExistingParentId = $this->createNonExistingId(); + $orphanedId1 = $this->createFileCacheEntry($storageId, 'files/missingdir/orphaned1', $nonExistingParentId); + + $nonExistingParentId2 = $this->createNonExistingId(); + $orphanedId2 = $this->createFileCacheEntry($storageId, 'missingdir/missingdir1/orphaned2', $nonExistingParentId2); + // end corrupt storage + + // Parallel test storage + $storageId_parallel = 2; + $rootId1_parallel = $this->createFileCacheEntry($storageId_parallel, ''); + $baseId1_parallel = $this->createFileCacheEntry($storageId_parallel, 'files', $rootId1_parallel); + $notOrphanedFolder_parallel = $this->createFileCacheEntry($storageId_parallel, 'files/missingdir', $baseId1_parallel); + $notOrphanedId1_parallel = $this->createFileCacheEntry($storageId_parallel, 'files/missingdir/orphaned1', $notOrphanedFolder_parallel); + $notOrphanedFolder2_parallel = $this->createFileCacheEntry($storageId_parallel, 'missingdir', $rootId1_parallel); + $notOrphanedFolderChild2_parallel = $this->createFileCacheEntry($storageId_parallel, 'missingdir/missingdir1', $notOrphanedFolder2_parallel); + $notOrphanedId2_parallel = $this->createFileCacheEntry($storageId_parallel, 'missingdir/missingdir1/orphaned2', $notOrphanedFolder2_parallel); + // end parallel test storage + + $outputMock = $this->createMock(IOutput::class); + $this->repair->setStorageNumericId($storageId); + $this->repair->run($outputMock); + + // orphaned entry reattached + $entry = $this->getFileCacheEntry($orphanedId1); + $this->assertEquals($nonExistingParentId, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('files/missingdir/orphaned1', $entry['path']); + $this->assertEquals(md5('files/missingdir/orphaned1'), $entry['path_hash']); + + // non existing id exists now + $entry = $this->getFileCacheEntry($entry['parent']); + $this->assertEquals($baseId1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('files/missingdir', $entry['path']); + $this->assertEquals(md5('files/missingdir'), $entry['path_hash']); + + // orphaned entry reattached + $entry = $this->getFileCacheEntry($orphanedId2); + $this->assertEquals($nonExistingParentId2, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('missingdir/missingdir1/orphaned2', $entry['path']); + $this->assertEquals(md5('missingdir/missingdir1/orphaned2'), $entry['path_hash']); + + // non existing id exists now + $entry = $this->getFileCacheEntry($entry['parent']); + $this->assertTrue(isset($entry['parent'])); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('missingdir/missingdir1', $entry['path']); + $this->assertEquals(md5('missingdir/missingdir1'), $entry['path_hash']); + + // non existing id parent exists now + $entry = $this->getFileCacheEntry($entry['parent']); + $this->assertEquals($rootId1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('missingdir', $entry['path']); + $this->assertEquals(md5('missingdir'), $entry['path_hash']); + + // root entry left alone + $entry = $this->getFileCacheEntry($rootId1); + $this->assertEquals(-1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('', $entry['path']); + $this->assertEquals(md5(''), $entry['path_hash']); + + // now check the parallel storage is intact + // orphaned entry reattached + $entry = $this->getFileCacheEntry($notOrphanedId1_parallel); + $this->assertEquals($notOrphanedFolder_parallel, $entry['parent']); + $this->assertEquals((string)$storageId_parallel, $entry['storage']); + $this->assertEquals('files/missingdir/orphaned1', $entry['path']); + $this->assertEquals(md5('files/missingdir/orphaned1'), $entry['path_hash']); + + // not orphaned folder still exists + $entry = $this->getFileCacheEntry($notOrphanedFolder_parallel); + $this->assertEquals($baseId1_parallel, $entry['parent']); + $this->assertEquals((string)$storageId_parallel, $entry['storage']); + $this->assertEquals('files/missingdir', $entry['path']); + $this->assertEquals(md5('files/missingdir'), $entry['path_hash']); + + // not orphaned entry still exits + $entry = $this->getFileCacheEntry($notOrphanedId2_parallel); + $this->assertEquals($notOrphanedFolder2_parallel, $entry['parent']); + $this->assertEquals((string)$storageId_parallel, $entry['storage']); + $this->assertEquals('missingdir/missingdir1/orphaned2', $entry['path']); + $this->assertEquals(md5('missingdir/missingdir1/orphaned2'), $entry['path_hash']); + + // non existing id exists now + $entry = $this->getFileCacheEntry($notOrphanedFolderChild2_parallel); + $this->assertEquals($notOrphanedFolder2_parallel, $entry['parent']); + $this->assertEquals((string)$storageId_parallel, $entry['storage']); + $this->assertEquals('missingdir/missingdir1', $entry['path']); + $this->assertEquals(md5('missingdir/missingdir1'), $entry['path_hash']); + + // non existing id parent exists now + $entry = $this->getFileCacheEntry($notOrphanedFolder2_parallel); + $this->assertEquals($rootId1_parallel, $entry['parent']); + $this->assertEquals((string)$storageId_parallel, $entry['storage']); + $this->assertEquals('missingdir', $entry['path']); + $this->assertEquals(md5('missingdir'), $entry['path_hash']); + + // root entry left alone + $entry = $this->getFileCacheEntry($rootId1_parallel); + $this->assertEquals(-1, $entry['parent']); + $this->assertEquals((string)$storageId_parallel, $entry['storage']); + $this->assertEquals('', $entry['path']); + $this->assertEquals(md5(''), $entry['path_hash']); + } + + /** + * Test repair missing root + */ + public function testRepairMissingRoot() { + /** + * - noroot (orphaned entry on a storage that has no root entry) + */ + $storageId = 1; + $nonExistingParentId = $this->createNonExistingId(); + $orphanedId = $this->createFileCacheEntry($storageId, 'noroot', $nonExistingParentId); + + // Test parallel storage which should be untouched by the repair operation + $testStorageId = 2; + $baseId = $this->createFileCacheEntry($testStorageId, ''); + $noRootid = $this->createFileCacheEntry($testStorageId, 'noroot', $baseId); + + + $outputMock = $this->createMock(IOutput::class); + $this->repair->setStorageNumericId($storageId); + $this->repair->run($outputMock); + + // orphaned entry with no root reattached + $entry = $this->getFileCacheEntry($orphanedId); + $this->assertTrue(isset($entry['parent'])); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('noroot', $entry['path']); + $this->assertEquals(md5('noroot'), $entry['path_hash']); + + // recreated root entry + $entry = $this->getFileCacheEntry($entry['parent']); + $this->assertEquals(-1, $entry['parent']); + $this->assertEquals((string)$storageId, $entry['storage']); + $this->assertEquals('', $entry['path']); + $this->assertEquals(md5(''), $entry['path_hash']); + + // Check that the parallel test storage is still intact + $entry = $this->getFileCacheEntry($noRootid); + $this->assertEquals($baseId, $entry['parent']); + $this->assertEquals((string)$testStorageId, $entry['storage']); + $this->assertEquals('noroot', $entry['path']); + $this->assertEquals(md5('noroot'), $entry['path_hash']); + $entry = $this->getFileCacheEntry($baseId); + $this->assertEquals(-1, $entry['parent']); + $this->assertEquals((string)$testStorageId, $entry['storage']); + $this->assertEquals('', $entry['path']); + $this->assertEquals(md5(''), $entry['path_hash']); + } +} + diff --git a/tests/lib/TestCase.php b/tests/lib/TestCase.php index bff39f7ef7a9..e3920bc9009c 100644 --- a/tests/lib/TestCase.php +++ b/tests/lib/TestCase.php @@ -221,7 +221,8 @@ public static function tearDownAfterClass() { } $dataDir = \OC::$server->getConfig()->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data-autotest'); if (self::$wasDatabaseAllowed && \OC::$server->getDatabaseConnection()) { - $queryBuilder = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $connection = \OC::$server->getDatabaseConnection(); + $queryBuilder = $connection->getQueryBuilder(); self::tearDownAfterClassCleanShares($queryBuilder); self::tearDownAfterClassCleanStorages($queryBuilder);