diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 949079dfa224b..dc6ab6dae2212 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -37,6 +37,7 @@ * along with this program. If not, see * */ + namespace OC\Files\Cache; use Doctrine\DBAL\Exception\UniqueConstraintViolationException; @@ -188,6 +189,7 @@ public static function cacheEntryFromData($data, IMimeTypeLoader $mimetypeLoader $data['fileid'] = (int)$data['fileid']; $data['parent'] = (int)$data['parent']; $data['size'] = 0 + $data['size']; + $data['unencrypted_size'] = 0 + ($data['unencrypted_size'] ?? 0); $data['mtime'] = (int)$data['mtime']; $data['storage_mtime'] = (int)$data['storage_mtime']; $data['encryptedVersion'] = (int)$data['encrypted']; @@ -428,7 +430,7 @@ public function update($id, array $data) { protected function normalizeData(array $data): array { $fields = [ 'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted', - 'etag', 'permissions', 'checksum', 'storage']; + 'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size']; $extensionFields = ['metadata_etag', 'creation_time', 'upload_time']; $doNotCopyStorageMTime = false; @@ -873,18 +875,32 @@ public function calculateFolderSize($path, $entry = null) { $id = $entry['fileid']; $query = $this->getQueryBuilder(); - $query->selectAlias($query->func()->sum('size'), 'f1') - ->selectAlias($query->func()->min('size'), 'f2') + $query->select('size', 'unencrypted_size') ->from('filecache') - ->whereStorageId($this->getNumericStorageId()) ->whereParent($id); $result = $query->execute(); - $row = $result->fetch(); + $rows = $result->fetchAll(); $result->closeCursor(); - if ($row) { - [$sum, $min] = array_values($row); + if ($rows) { + $sizes = array_map(function (array $row) { + return (int)$row['size']; + }, $rows); + $unencryptedOnlySizes = array_map(function (array $row) { + return (int)$row['unencrypted_size']; + }, $rows); + $unencryptedSizes = array_map(function (array $row) { + return (int)(($row['unencrypted_size'] > 0) ? $row['unencrypted_size']: $row['size']); + }, $rows); + + $sum = array_sum($sizes); + $min = min($sizes); + + $unencryptedSum = array_sum($unencryptedSizes); + $unencryptedMin = min($unencryptedSizes); + $unencryptedMax = max($unencryptedOnlySizes); + $sum = 0 + $sum; $min = 0 + $min; if ($min === -1) { @@ -892,8 +908,23 @@ public function calculateFolderSize($path, $entry = null) { } else { $totalSize = $sum; } + if ($unencryptedMin === -1 || $min === -1) { + $unencryptedTotal = $unencryptedMin; + } else { + $unencryptedTotal = $unencryptedSum; + } if ($entry['size'] !== $totalSize) { - $this->update($id, ['size' => $totalSize]); + // only set unencrypted size for a folder if any child entries have it set + if ($unencryptedMax > 0) { + $this->update($id, [ + 'size' => $totalSize, + 'unencrypted_size' => $unencryptedTotal, + ]); + } else { + $this->update($id, [ + 'size' => $totalSize, + ]); + } } } } diff --git a/lib/private/Files/Cache/CacheEntry.php b/lib/private/Files/Cache/CacheEntry.php index 12f0273fb6e7e..8ac76acf6d140 100644 --- a/lib/private/Files/Cache/CacheEntry.php +++ b/lib/private/Files/Cache/CacheEntry.php @@ -132,4 +132,12 @@ public function getData() { public function __clone() { $this->data = array_merge([], $this->data); } + + public function getUnencryptedSize(): int { + if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { + return $this->data['unencrypted_size']; + } else { + return $this->data['size']; + } + } } diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index b5a9101877c5b..fbcc9a0555744 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -44,7 +44,7 @@ public function __construct(IDBConnection $connection, SystemConfig $systemConfi public function selectFileCache(string $alias = null) { $name = $alias ? $alias : 'filecache'; $this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", 'name', 'mimetype', 'mimepart', 'size', 'mtime', - 'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time') + 'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time', 'unencrypted_size') ->from('filecache', $name) ->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid')); diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php index 270b2b013f55a..4086701f01129 100644 --- a/lib/private/Files/Cache/Propagator.php +++ b/lib/private/Files/Cache/Propagator.php @@ -24,6 +24,8 @@ namespace OC\Files\Cache; +use OC\DB\QueryBuilder\QueryFunction; +use OC\Files\Storage\Wrapper\Encryption; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\IPropagator; use OCP\Files\Storage\IReliableEtagStorage; @@ -113,6 +115,20 @@ public function propagateChange($internalPath, $time, $sizeDifference = 0) { ->andWhere($builder->expr()->in('path_hash', $hashParams)) ->andWhere($builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT))); + if ($this->storage->instanceOfStorage(Encryption::class)) { + // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size + $eq = $builder->expr()->eq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT)); + $sizeColumn = $builder->getColumnName('size'); + $unencryptedSizeColumn = $builder->getColumnName('unencrypted_size'); + $builder->set('unencrypted_size', $builder->func()->greatest( + $builder->func()->add( + new QueryFunction("CASE WHEN $eq THEN $unencryptedSizeColumn ELSE $sizeColumn END"), + $builder->createNamedParameter($sizeDifference) + ), + $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT) + )); + } + $builder->execute(); } } diff --git a/lib/private/Files/FileInfo.php b/lib/private/Files/FileInfo.php index 6389544184fca..5912eefcf9c8d 100644 --- a/lib/private/Files/FileInfo.php +++ b/lib/private/Files/FileInfo.php @@ -101,7 +101,11 @@ public function __construct($path, $storage, $internalPath, $data, $mount, $owne $this->data = $data; $this->mount = $mount; $this->owner = $owner; - $this->rawSize = $this->data['size'] ?? 0; + if (isset($this->data['unencrypted_size'])) { + $this->rawSize = $this->data['unencrypted_size']; + } else { + $this->rawSize = $this->data['size'] ?? 0; + } } public function offsetSet($offset, $value): void { @@ -208,7 +212,12 @@ public function getEtag() { public function getSize($includeMounts = true) { if ($includeMounts) { $this->updateEntryfromSubMounts(); - return isset($this->data['size']) ? 0 + $this->data['size'] : 0; + + if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) { + return $this->data['unencrypted_size']; + } else { + return isset($this->data['size']) ? 0 + $this->data['size'] : 0; + } } else { return $this->rawSize; } @@ -386,7 +395,19 @@ private function updateEntryfromSubMounts() { * @param string $entryPath full path of the child entry */ public function addSubEntry($data, $entryPath) { - $this->data['size'] += isset($data['size']) ? $data['size'] : 0; + if (!$data) { + return; + } + $hasUnencryptedSize = isset($data['unencrypted_size']) && $data['unencrypted_size'] > 0; + if ($hasUnencryptedSize) { + $subSize = $data['unencrypted_size']; + } else { + $subSize = $data['size'] ?: 0; + } + $this->data['size'] += $subSize; + if ($hasUnencryptedSize) { + $this->data['unencrypted_size'] += $subSize; + } if (isset($data['mtime'])) { $this->data['mtime'] = max($this->data['mtime'], $data['mtime']); } diff --git a/lib/private/Files/Storage/Wrapper/Encryption.php b/lib/private/Files/Storage/Wrapper/Encryption.php index 4cfe932cc9ff7..d5bf929101fd9 100644 --- a/lib/private/Files/Storage/Wrapper/Encryption.php +++ b/lib/private/Files/Storage/Wrapper/Encryption.php @@ -33,6 +33,7 @@ * along with this program. If not, see * */ + namespace OC\Files\Storage\Wrapper; use OC\Encryption\Exceptions\ModuleDoesNotExistsException; @@ -41,6 +42,7 @@ use OC\Files\Cache\CacheEntry; use OC\Files\Filesystem; use OC\Files\Mount\Manager; +use OC\Files\ObjectStore\ObjectStoreStorage; use OC\Files\Storage\LocalTempFileTrait; use OC\Memcache\ArrayCache; use OCP\Encryption\Exceptions\GenericEncryptionException; @@ -139,28 +141,36 @@ public function filesize($path) { $size = $this->unencryptedSize[$fullPath]; // update file cache if ($info instanceof ICacheEntry) { - $info = $info->getData(); $info['encrypted'] = $info['encryptedVersion']; } else { if (!is_array($info)) { $info = []; } $info['encrypted'] = true; + $info = new CacheEntry($info); } - $info['size'] = $size; - $this->getCache()->put($path, $info); + if ($size !== $info->getUnencryptedSize()) { + $this->getCache()->update($info->getId(), [ + 'unencrypted_size' => $size + ]); + } return $size; } if (isset($info['fileid']) && $info['encrypted']) { - return $this->verifyUnencryptedSize($path, $info['size']); + return $this->verifyUnencryptedSize($path, $info->getUnencryptedSize()); } return $this->storage->filesize($path); } + /** + * @param string $path + * @param array $data + * @return array + */ private function modifyMetaData(string $path, array $data): array { $fullPath = $this->getFullPath($path); $info = $this->getCache()->get($path); @@ -170,7 +180,7 @@ private function modifyMetaData(string $path, array $data): array { $data['size'] = $this->unencryptedSize[$fullPath]; } else { if (isset($info['fileid']) && $info['encrypted']) { - $data['size'] = $this->verifyUnencryptedSize($path, $info['size']); + $data['size'] = $this->verifyUnencryptedSize($path, $info->getUnencryptedSize()); $data['encrypted'] = true; } } @@ -478,7 +488,7 @@ public function fopen($path, $mode) { * * @return int unencrypted size */ - protected function verifyUnencryptedSize($path, $unencryptedSize) { + protected function verifyUnencryptedSize(string $path, int $unencryptedSize): int { $size = $this->storage->filesize($path); $result = $unencryptedSize; @@ -510,7 +520,7 @@ protected function verifyUnencryptedSize($path, $unencryptedSize) { * * @return int calculated unencrypted size */ - protected function fixUnencryptedSize($path, $size, $unencryptedSize) { + protected function fixUnencryptedSize(string $path, int $size, int $unencryptedSize): int { $headerSize = $this->getHeaderSize($path); $header = $this->getHeader($path); $encryptionModule = $this->getEncryptionModule($path); @@ -581,7 +591,9 @@ protected function fixUnencryptedSize($path, $size, $unencryptedSize) { $cache = $this->storage->getCache(); if ($cache) { $entry = $cache->get($path); - $cache->update($entry['fileid'], ['size' => $newUnencryptedSize]); + $cache->update($entry['fileid'], [ + 'unencrypted_size' => $newUnencryptedSize + ]); } return $newUnencryptedSize; @@ -621,7 +633,12 @@ private function fread_block($handle, int $blockSize): string { * @param bool $preserveMtime * @return bool */ - public function moveFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = true) { + public function moveFromStorage( + Storage\IStorage $sourceStorage, + $sourceInternalPath, + $targetInternalPath, + $preserveMtime = true + ) { if ($sourceStorage === $this) { return $this->rename($sourceInternalPath, $targetInternalPath); } @@ -656,7 +673,13 @@ public function moveFromStorage(Storage\IStorage $sourceStorage, $sourceInternal * @param bool $isRename * @return bool */ - public function copyFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = false, $isRename = false) { + public function copyFromStorage( + Storage\IStorage $sourceStorage, + $sourceInternalPath, + $targetInternalPath, + $preserveMtime = false, + $isRename = false + ) { // TODO clean this up once the underlying moveFromStorage in OC\Files\Storage\Wrapper\Common is fixed: // - call $this->storage->copyFromStorage() instead of $this->copyBetweenStorage @@ -676,7 +699,13 @@ public function copyFromStorage(Storage\IStorage $sourceStorage, $sourceInternal * @param bool $isRename * @param bool $keepEncryptionVersion */ - private function updateEncryptedVersion(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, $keepEncryptionVersion) { + private function updateEncryptedVersion( + Storage\IStorage $sourceStorage, + $sourceInternalPath, + $targetInternalPath, + $isRename, + $keepEncryptionVersion + ) { $isEncrypted = $this->encryptionManager->isEnabled() && $this->shouldEncrypt($targetInternalPath); $cacheInformation = [ 'encrypted' => $isEncrypted, @@ -725,7 +754,13 @@ private function updateEncryptedVersion(Storage\IStorage $sourceStorage, $source * @return bool * @throws \Exception */ - private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, $isRename) { + private function copyBetweenStorage( + Storage\IStorage $sourceStorage, + $sourceInternalPath, + $targetInternalPath, + $preserveMtime, + $isRename + ) { // for versions we have nothing to do, because versions should always use the // key from the original file. Just create a 1:1 copy and done @@ -743,7 +778,7 @@ private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInte if (isset($info['encrypted']) && $info['encrypted'] === true) { $this->updateUnencryptedSize( $this->getFullPath($targetInternalPath), - $info['size'] + $info->getUnencryptedSize() ); } $this->updateEncryptedVersion($sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, true); @@ -808,13 +843,6 @@ private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInte return (bool)$result; } - /** - * get the path to a local version of the file. - * The local version of the file can be temporary and doesn't have to be persistent across requests - * - * @param string $path - * @return string - */ public function getLocalFile($path) { if ($this->encryptionManager->isEnabled()) { $cachedFile = $this->getCachedFile($path); @@ -825,11 +853,6 @@ public function getLocalFile($path) { return $this->storage->getLocalFile($path); } - /** - * Returns the wrapped storage's value for isLocal() - * - * @return bool wrapped storage's isLocal() value - */ public function isLocal() { if ($this->encryptionManager->isEnabled()) { return false; @@ -837,15 +860,11 @@ public function isLocal() { return $this->storage->isLocal(); } - /** - * see https://www.php.net/manual/en/function.stat.php - * only the following keys are required in the result: size and mtime - * - * @param string $path - * @return array - */ public function stat($path) { $stat = $this->storage->stat($path); + if (!$stat) { + return false; + } $fileSize = $this->filesize($path); $stat['size'] = $fileSize; $stat[7] = $fileSize; @@ -853,14 +872,6 @@ public function stat($path) { return $stat; } - /** - * see https://www.php.net/manual/en/function.hash.php - * - * @param string $type - * @param string $path - * @param bool $raw - * @return string - */ public function hash($type, $path, $raw = false) { $fh = $this->fopen($path, 'rb'); $ctx = hash_init($type); @@ -1068,6 +1079,13 @@ public function writeStream(string $path, $stream, int $size = null): int { [$count, $result] = \OC_Helper::streamCopy($stream, $target); fclose($stream); fclose($target); + + // object store, stores the size after write and doesn't update this during scan + // manually store the unencrypted size + if ($result && $this->getWrapperStorage()->instanceOfStorage(ObjectStoreStorage::class)) { + $this->getCache()->put($path, ['unencrypted_size' => $count]); + } + return $count; } } diff --git a/lib/public/Files/Cache/ICacheEntry.php b/lib/public/Files/Cache/ICacheEntry.php index 17eecf89ddb2f..e1e8129394c71 100644 --- a/lib/public/Files/Cache/ICacheEntry.php +++ b/lib/public/Files/Cache/ICacheEntry.php @@ -162,4 +162,14 @@ public function getCreationTime(): ?int; * @since 18.0.0 */ public function getUploadTime(): ?int; + + /** + * Get the unencrypted size + * + * This might be different from the result of getSize + * + * @return int + * @since 25.0.0 + */ + public function getUnencryptedSize(): int; } diff --git a/tests/lib/Files/ObjectStore/S3Test.php b/tests/lib/Files/ObjectStore/S3Test.php index a7a95d5337576..fd451dc3c016f 100644 --- a/tests/lib/Files/ObjectStore/S3Test.php +++ b/tests/lib/Files/ObjectStore/S3Test.php @@ -176,7 +176,7 @@ public function testFileSizes($size) { // end of file reached fseek($result, $size); - self:self::assertTrue(feof($result)); + self::assertTrue(feof($result)); $this->assertNoUpload('testfilesizes'); } diff --git a/tests/lib/Files/Storage/Wrapper/EncryptionTest.php b/tests/lib/Files/Storage/Wrapper/EncryptionTest.php index d26e5c499e70b..ebb97a25c77e0 100644 --- a/tests/lib/Files/Storage/Wrapper/EncryptionTest.php +++ b/tests/lib/Files/Storage/Wrapper/EncryptionTest.php @@ -5,6 +5,7 @@ use OC\Encryption\Exceptions\ModuleDoesNotExistsException; use OC\Encryption\Update; use OC\Encryption\Util; +use OC\Files\Cache\CacheEntry; use OC\Files\Storage\Temporary; use OC\Files\Storage\Wrapper\Encryption; use OC\Files\View; @@ -259,7 +260,7 @@ public function testGetMetaData($path, $metaData, $encrypted, $unencryptedSizeSe ->method('get') ->willReturnCallback( function ($path) use ($encrypted) { - return ['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1]; + return new CacheEntry(['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1]); } ); @@ -332,7 +333,7 @@ public function testFilesize() { ->disableOriginalConstructor()->getMock(); $cache->expects($this->any()) ->method('get') - ->willReturn(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]); + ->willReturn(new CacheEntry(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1])); $this->instance = $this->getMockBuilder('\OC\Files\Storage\Wrapper\Encryption') ->setConstructorArgs( @@ -910,7 +911,7 @@ public function testCopyBetweenStorageVersions($sourceInternalPath, $targetInter if ($copyResult) { $cache->expects($this->once())->method('get') ->with($sourceInternalPath) - ->willReturn(['encrypted' => $encrypted, 'size' => 42]); + ->willReturn(new CacheEntry(['encrypted' => $encrypted, 'size' => 42])); if ($encrypted) { $instance->expects($this->once())->method('updateUnencryptedSize') ->with($mountPoint . $targetInternalPath, 42); diff --git a/tests/lib/HelperStorageTest.php b/tests/lib/HelperStorageTest.php index 6d7ea513d3f49..d3f480502b2d8 100644 --- a/tests/lib/HelperStorageTest.php +++ b/tests/lib/HelperStorageTest.php @@ -104,6 +104,9 @@ public function testGetStorageInfoExcludingExtStorage() { $extStorage->file_put_contents('extfile.txt', 'abcdefghijklmnopq'); $extStorage->getScanner()->scan(''); // update root size + $config = \OC::$server->getConfig(); + $config->setSystemValue('quota_include_external_storage', false); + \OC\Files\Filesystem::mount($extStorage, [], '/' . $this->user . '/files/ext'); $storageInfo = \OC_Helper::getStorageInfo('');