store unencrypted size in the unencrypted_size column

Signed-off-by: Robin Appelman <robin@icewind.nl>

lib/private/Files/Cache/Cache.php

@@ -428,7 +428,7 @@ public function update($id, array $data) {
 	protected function normalizeData(array $data): array {
 		$fields = [
 			'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted',
-			'etag', 'permissions', 'checksum', 'storage'];
+			'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size'];
 		$extensionFields = ['metadata_etag', 'creation_time', 'upload_time'];
 
 		$doNotCopyStorageMTime = false;

lib/private/Files/Cache/CacheEntry.php

@@ -132,4 +132,12 @@ public function getData() {
 	public function __clone() {
 		$this->data = array_merge([], $this->data);
 	}
+
+	public function getUnencryptedSize(): int {
+		if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
+			return $this->data['unencrypted_size'];
+		} else {
+			return $this->data['size'];
+		}
+	}
 }

lib/private/Files/Cache/CacheQueryBuilder.php

@@ -44,7 +44,7 @@ public function __construct(IDBConnection $connection, SystemConfig $systemConfi
 	public function selectFileCache(string $alias = null) {
 		$name = $alias ? $alias : 'filecache';
 		$this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", 'name', 'mimetype', 'mimepart', 'size', 'mtime',
-			'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time')
+			'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time', 'unencrypted_size')
 			->from('filecache', $name)
 			->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid'));
 

lib/private/Files/FileInfo.php

@@ -101,7 +101,11 @@ public function __construct($path, $storage, $internalPath, $data, $mount, $owne
 		$this->data = $data;
 		$this->mount = $mount;
 		$this->owner = $owner;
-		$this->rawSize = $this->data['size'] ?? 0;
+		if (isset($this->data['unencrypted_size'])) {
+			$this->rawSize = $this->data['unencrypted_size'];
+		} else {
+			$this->rawSize = $this->data['size'] ?? 0;
+		}
 	}
 
 	public function offsetSet($offset, $value): void {
@@ -208,7 +212,12 @@ public function getEtag() {
 	public function getSize($includeMounts = true) {
 		if ($includeMounts) {
 			$this->updateEntryfromSubMounts();
-			return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
+
+			if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
+				return $this->data['unencrypted_size'];
+			} else {
+				return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
+			}
 		} else {
 			return $this->rawSize;
 		}
@@ -386,7 +395,19 @@ private function updateEntryfromSubMounts() {
 	 * @param string $entryPath full path of the child entry
 	 */
 	public function addSubEntry($data, $entryPath) {
-		$this->data['size'] += isset($data['size']) ? $data['size'] : 0;
+		if (!$data) {
+			return;
+		}
+		$hasUnencryptedSize = isset($data['unencrypted_size']) && $data['unencrypted_size'] > 0;
+		if ($hasUnencryptedSize) {
+			$subSize = $data['unencrypted_size'];
+		} else {
+			$subSize = $data['size'] ?: 0;
+		}
+		$this->data['size'] += $subSize;
+		if ($hasUnencryptedSize) {
+			$this->data['unencrypted_size'] += $subSize;
+		}
 		if (isset($data['mtime'])) {
 			$this->data['mtime'] = max($this->data['mtime'], $data['mtime']);
 		}

lib/private/Files/Storage/Wrapper/Encryption.php

@@ -33,6 +33,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>
  *
  */
+
 namespace OC\Files\Storage\Wrapper;
 
 use OC\Encryption\Exceptions\ModuleDoesNotExistsException;
@@ -41,6 +42,7 @@
 use OC\Files\Cache\CacheEntry;
 use OC\Files\Filesystem;
 use OC\Files\Mount\Manager;
+use OC\Files\ObjectStore\ObjectStoreStorage;
 use OC\Files\Storage\LocalTempFileTrait;
 use OC\Memcache\ArrayCache;
 use OCP\Encryption\Exceptions\GenericEncryptionException;
@@ -139,28 +141,36 @@ public function filesize($path) {
 			$size = $this->unencryptedSize[$fullPath];
 			// update file cache
 			if ($info instanceof ICacheEntry) {
-				$info = $info->getData();
 				$info['encrypted'] = $info['encryptedVersion'];
 			} else {
 				if (!is_array($info)) {
 					$info = [];
 				}
 				$info['encrypted'] = true;
+				$info = new CacheEntry($info);
 			}
 
-			$info['size'] = $size;
-			$this->getCache()->put($path, $info);
+			if ($size !== $info->getUnencryptedSize()) {
+				$this->getCache()->update($info->getId(), [
+					'unencrypted_size' => $size
+				]);
+			}
 
 			return $size;
 		}
 
 		if (isset($info['fileid']) && $info['encrypted']) {
-			return $this->verifyUnencryptedSize($path, $info['size']);
+			return $this->verifyUnencryptedSize($path, $info->getUnencryptedSize());
 		}
 
 		return $this->storage->filesize($path);
 	}
 
+	/**
+	 * @param string $path
+	 * @param array $data
+	 * @return array
+	 */
 	private function modifyMetaData(string $path, array $data): array {
 		$fullPath = $this->getFullPath($path);
 		$info = $this->getCache()->get($path);
@@ -170,7 +180,7 @@ private function modifyMetaData(string $path, array $data): array {
 			$data['size'] = $this->unencryptedSize[$fullPath];
 		} else {
 			if (isset($info['fileid']) && $info['encrypted']) {
-				$data['size'] = $this->verifyUnencryptedSize($path, $info['size']);
+				$data['size'] = $this->verifyUnencryptedSize($path, $info->getUnencryptedSize());
 				$data['encrypted'] = true;
 			}
 		}
@@ -478,7 +488,7 @@ public function fopen($path, $mode) {
 	 *
 	 * @return int unencrypted size
 	 */
-	protected function verifyUnencryptedSize($path, $unencryptedSize) {
+	protected function verifyUnencryptedSize(string $path, int $unencryptedSize) {
 		$size = $this->storage->filesize($path);
 		$result = $unencryptedSize;
 
@@ -510,7 +520,7 @@ protected function verifyUnencryptedSize($path, $unencryptedSize) {
 	 *
 	 * @return int calculated unencrypted size
 	 */
-	protected function fixUnencryptedSize($path, $size, $unencryptedSize) {
+	protected function fixUnencryptedSize(string $path, int $size, int $unencryptedSize) {
 		$headerSize = $this->getHeaderSize($path);
 		$header = $this->getHeader($path);
 		$encryptionModule = $this->getEncryptionModule($path);
@@ -581,7 +591,9 @@ protected function fixUnencryptedSize($path, $size, $unencryptedSize) {
 		$cache = $this->storage->getCache();
 		if ($cache) {
 			$entry = $cache->get($path);
-			$cache->update($entry['fileid'], ['size' => $newUnencryptedSize]);
+			$cache->update($entry['fileid'], [
+				'unencrypted_size' => $newUnencryptedSize
+			]);
 		}
 
 		return $newUnencryptedSize;
@@ -621,7 +633,12 @@ private function fread_block($handle, int $blockSize): string {
 	 * @param bool $preserveMtime
 	 * @return bool
 	 */
-	public function moveFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = true) {
+	public function moveFromStorage(
+		Storage\IStorage $sourceStorage,
+		$sourceInternalPath,
+		$targetInternalPath,
+		$preserveMtime = true
+	) {
 		if ($sourceStorage === $this) {
 			return $this->rename($sourceInternalPath, $targetInternalPath);
 		}
@@ -656,7 +673,13 @@ public function moveFromStorage(Storage\IStorage $sourceStorage, $sourceInternal
 	 * @param bool $isRename
 	 * @return bool
 	 */
-	public function copyFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = false, $isRename = false) {
+	public function copyFromStorage(
+		Storage\IStorage $sourceStorage,
+		$sourceInternalPath,
+		$targetInternalPath,
+		$preserveMtime = false,
+		$isRename = false
+	) {
 
 		// TODO clean this up once the underlying moveFromStorage in OC\Files\Storage\Wrapper\Common is fixed:
 		// - call $this->storage->copyFromStorage() instead of $this->copyBetweenStorage
@@ -676,7 +699,13 @@ public function copyFromStorage(Storage\IStorage $sourceStorage, $sourceInternal
 	 * @param bool $isRename
 	 * @param bool $keepEncryptionVersion
 	 */
-	private function updateEncryptedVersion(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, $keepEncryptionVersion) {
+	private function updateEncryptedVersion(
+		Storage\IStorage $sourceStorage,
+		$sourceInternalPath,
+		$targetInternalPath,
+		$isRename,
+		$keepEncryptionVersion
+	) {
 		$isEncrypted = $this->encryptionManager->isEnabled() && $this->shouldEncrypt($targetInternalPath);
 		$cacheInformation = [
 			'encrypted' => $isEncrypted,
@@ -725,7 +754,13 @@ private function updateEncryptedVersion(Storage\IStorage $sourceStorage, $source
 	 * @return bool
 	 * @throws \Exception
 	 */
-	private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, $isRename) {
+	private function copyBetweenStorage(
+		Storage\IStorage $sourceStorage,
+		$sourceInternalPath,
+		$targetInternalPath,
+		$preserveMtime,
+		$isRename
+	) {
 
 		// for versions we have nothing to do, because versions should always use the
 		// key from the original file. Just create a 1:1 copy and done
@@ -743,7 +778,7 @@ private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInte
 				if (isset($info['encrypted']) && $info['encrypted'] === true) {
 					$this->updateUnencryptedSize(
 						$this->getFullPath($targetInternalPath),
-						$info['size']
+						$info->getUnencryptedSize()
 					);
 				}
 				$this->updateEncryptedVersion($sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, true);
@@ -808,13 +843,6 @@ private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInte
 		return (bool)$result;
 	}
 
-	/**
-	 * get the path to a local version of the file.
-	 * The local version of the file can be temporary and doesn't have to be persistent across requests
-	 *
-	 * @param string $path
-	 * @return string
-	 */
 	public function getLocalFile($path) {
 		if ($this->encryptionManager->isEnabled()) {
 			$cachedFile = $this->getCachedFile($path);
@@ -825,42 +853,25 @@ public function getLocalFile($path) {
 		return $this->storage->getLocalFile($path);
 	}
 
-	/**
-	 * Returns the wrapped storage's value for isLocal()
-	 *
-	 * @return bool wrapped storage's isLocal() value
-	 */
 	public function isLocal() {
 		if ($this->encryptionManager->isEnabled()) {
 			return false;
 		}
 		return $this->storage->isLocal();
 	}
 
-	/**
-	 * see https://www.php.net/manual/en/function.stat.php
-	 * only the following keys are required in the result: size and mtime
-	 *
-	 * @param string $path
-	 * @return array
-	 */
 	public function stat($path) {
 		$stat = $this->storage->stat($path);
+		if (!$stat) {
+			return false;
+		}
 		$fileSize = $this->filesize($path);
 		$stat['size'] = $fileSize;
 		$stat[7] = $fileSize;
 		$stat['hasHeader'] = $this->getHeaderSize($path) > 0;
 		return $stat;
 	}
 
-	/**
-	 * see https://www.php.net/manual/en/function.hash.php
-	 *
-	 * @param string $type
-	 * @param string $path
-	 * @param bool $raw
-	 * @return string
-	 */
 	public function hash($type, $path, $raw = false) {
 		$fh = $this->fopen($path, 'rb');
 		$ctx = hash_init($type);
@@ -1068,6 +1079,13 @@ public function writeStream(string $path, $stream, int $size = null): int {
 		[$count, $result] = \OC_Helper::streamCopy($stream, $target);
 		fclose($stream);
 		fclose($target);
+
+		// object store, stores the size after write and doesn't update this during scan
+		// manually store the unencrypted size
+		if ($result && $this->getWrapperStorage()->instanceOfStorage(ObjectStoreStorage::class)) {
+			$this->getCache()->put($path, ['unencrypted_size' => $count]);
+		}
+
 		return $count;
 	}
 }

lib/public/Files/Cache/ICacheEntry.php

@@ -162,4 +162,14 @@ public function getCreationTime(): ?int;
 	 * @since 18.0.0
 	 */
 	public function getUploadTime(): ?int;
+
+	/**
+	 * Get the unencrypted size
+	 *
+	 * this might be different from the result of getSize
+	 *
+	 * @return int
+	 * @since 24.0.0
+	 */
+	public function getUnencryptedSize(): int;
 }

tests/lib/Files/Storage/Wrapper/EncryptionTest.php

@@ -5,6 +5,7 @@
 use OC\Encryption\Exceptions\ModuleDoesNotExistsException;
 use OC\Encryption\Update;
 use OC\Encryption\Util;
+use OC\Files\Cache\CacheEntry;
 use OC\Files\Storage\Temporary;
 use OC\Files\Storage\Wrapper\Encryption;
 use OC\Files\View;
@@ -259,7 +260,7 @@ public function testGetMetaData($path, $metaData, $encrypted, $unencryptedSizeSe
 			->method('get')
 			->willReturnCallback(
 				function ($path) use ($encrypted) {
-					return ['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1];
+					return new CacheEntry(['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1]);
 				}
 			);
 
@@ -332,7 +333,7 @@ public function testFilesize() {
 			->disableOriginalConstructor()->getMock();
 		$cache->expects($this->any())
 			->method('get')
-			->willReturn(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]);
+			->willReturn(new CacheEntry(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]));
 
 		$this->instance = $this->getMockBuilder('\OC\Files\Storage\Wrapper\Encryption')
 			->setConstructorArgs(
@@ -910,7 +911,7 @@ public function testCopyBetweenStorageVersions($sourceInternalPath, $targetInter
 		if ($copyResult) {
 			$cache->expects($this->once())->method('get')
 				->with($sourceInternalPath)
-				->willReturn(['encrypted' => $encrypted, 'size' => 42]);
+				->willReturn(new CacheEntry(['encrypted' => $encrypted, 'size' => 42]));
 			if ($encrypted) {
 				$instance->expects($this->once())->method('updateUnencryptedSize')
 					->with($mountPoint . $targetInternalPath, 42);

tests/lib/HelperStorageTest.php

@@ -104,6 +104,9 @@ public function testGetStorageInfoExcludingExtStorage() {
 		$extStorage->file_put_contents('extfile.txt', 'abcdefghijklmnopq');
 		$extStorage->getScanner()->scan(''); // update root size
 
+		$config = \OC::$server->getConfig();
+		$config->setSystemValue('quota_include_external_storage', false);
+
 		\OC\Files\Filesystem::mount($extStorage, [], '/' . $this->user . '/files/ext');
 
 		$storageInfo = \OC_Helper::getStorageInfo('');