Skip to content

Commit

Permalink
Do not set up filesystem on every call
Browse files Browse the repository at this point in the history 
Also remove old Oc_FileChunking logis that produced GC- collectable chunks

Signed-off-by: Anna Larch <anna@nextcloud.com>
  • Loading branch information
@miaulalala
miaulalala committed Feb 7, 2023
1 parent 6077003 commit aadd950
Show file tree
Hide file tree
Showing 15 changed files with 25 additions and 966 deletions.
33 changes: 8 additions & 25 deletions apps/dav/lib/Connector/Sabre/Directory.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,26 +109,13 @@ public function __construct(View $view, FileInfo $info, ?CachingTree $tree = nul
* @throws \Sabre\DAV\Exception
* @throws \Sabre\DAV\Exception\BadRequest
* @throws \Sabre\DAV\Exception\Forbidden
* @throws \Sabre\DAV\Exception\ServiceUnavailable
* @throws ServiceUnavailable
*/
public function createFile($name, $data = null) {
try {
// for chunked upload also updating a existing file is a "createFile"
// because we create all the chunks before re-assemble them to the existing file.
if (isset($_SERVER['HTTP_OC_CHUNKED'])) {

// exit if we can't create a new file and we don't updatable existing file
$chunkInfo = \OC_FileChunking::decodeName($name);
if (!$this->fileView->isCreatable($this->path) &&
!$this->fileView->isUpdatable($this->path . '/' . $chunkInfo['name'])
) {
throw new \Sabre\DAV\Exception\Forbidden();
}
} else {
// For non-chunked upload it is enough to check if we can create a new file
if (!$this->fileView->isCreatable($this->path)) {
throw new \Sabre\DAV\Exception\Forbidden();
}
// For non-chunked upload it is enough to check if we can create a new file
if (!$this->fileView->isCreatable($this->path)) {
throw new \Sabre\DAV\Exception\Forbidden();
}

$this->fileView->verifyPath($this->path, $name);
Expand All @@ -153,12 +140,8 @@ public function createFile($name, $data = null) {
$this->fileView->unlockFile($path . '.upload.part', ILockingProvider::LOCK_EXCLUSIVE);
$node->releaseLock(ILockingProvider::LOCK_SHARED);
return $result;
} catch (\OCP\Files\StorageNotAvailableException $e) {
throw new \Sabre\DAV\Exception\ServiceUnavailable($e->getMessage(), $e->getCode(), $e);
} catch (InvalidPathException $ex) {
throw new InvalidPath($ex->getMessage(), false, $ex);
} catch (ForbiddenException $ex) {
throw new Forbidden($ex->getMessage(), $ex->getRetry(), $ex);
} catch (LockedException $e) {
throw new FileLocked($e->getMessage(), $e->getCode(), $e);
}
Expand All @@ -171,7 +154,7 @@ public function createFile($name, $data = null) {
* @throws FileLocked
* @throws InvalidPath
* @throws \Sabre\DAV\Exception\Forbidden
* @throws \Sabre\DAV\Exception\ServiceUnavailable
* @throws ServiceUnavailable
*/
public function createDirectory($name) {
try {
Expand All @@ -185,7 +168,7 @@ public function createDirectory($name) {
throw new \Sabre\DAV\Exception\Forbidden('Could not create directory ' . $newPath);
}
} catch (\OCP\Files\StorageNotAvailableException $e) {
throw new \Sabre\DAV\Exception\ServiceUnavailable($e->getMessage());
throw new ServiceUnavailable($e->getMessage());
} catch (InvalidPathException $ex) {
throw new InvalidPath($ex->getMessage());
} catch (ForbiddenException $ex) {
Expand All @@ -203,7 +186,7 @@ public function createDirectory($name) {
* @return \Sabre\DAV\INode
* @throws InvalidPath
* @throws \Sabre\DAV\Exception\NotFound
* @throws \Sabre\DAV\Exception\ServiceUnavailable
* @throws ServiceUnavailable
*/
public function getChild($name, $info = null) {
if (!$this->info->isReadable()) {
Expand All @@ -217,7 +200,7 @@ public function getChild($name, $info = null) {
$this->fileView->verifyPath($this->path, $name);
$info = $this->fileView->getFileInfo($path);
} catch (\OCP\Files\StorageNotAvailableException $e) {
throw new \Sabre\DAV\Exception\ServiceUnavailable($e->getMessage());
throw new ServiceUnavailable($e->getMessage());
} catch (InvalidPathException $ex) {
throw new InvalidPath($ex->getMessage());
} catch (ForbiddenException $e) {
Expand Down
135 changes: 0 additions & 135 deletions apps/dav/lib/Connector/Sabre/File.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,15 +148,6 @@ public function put($data) {
// verify path of the target
$this->verifyPath();

// chunked handling
if (isset($_SERVER['HTTP_OC_CHUNKED'])) {
try {
return $this->createFileChunked($data);
} catch (\Exception $e) {
$this->convertToSabreException($e);
}
}

/** @var Storage $partStorage */
[$partStorage] = $this->fileView->resolvePath($this->path);
$needsPartFile = $partStorage->needsPartFile() && (strlen($this->path) > 1);
Expand Down Expand Up @@ -577,132 +568,6 @@ public function getDirectDownload() {
return $storage->getDirectDownload($internalPath);
}

/**
* @param resource $data
* @return null|string
* @throws Exception
* @throws BadRequest
* @throws NotImplemented
* @throws ServiceUnavailable
*/
private function createFileChunked($data) {
[$path, $name] = \Sabre\Uri\split($this->path);

$info = \OC_FileChunking::decodeName($name);
if (empty($info)) {
throw new NotImplemented($this->l10n->t('Invalid chunk name'));
}

$chunk_handler = new \OC_FileChunking($info);
$bytesWritten = $chunk_handler->store($info['index'], $data);

//detect aborted upload
if (isset($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] === 'PUT') {
if (isset($_SERVER['CONTENT_LENGTH'])) {
$expected = (int)$_SERVER['CONTENT_LENGTH'];
if ($bytesWritten !== $expected) {
$chunk_handler->remove($info['index']);
throw new BadRequest(
$this->l10n->t(
'Expected filesize of %1$s but read (from Nextcloud client) and wrote (to Nextcloud storage) %2$s. Could either be a network problem on the sending side or a problem writing to the storage on the server side.',
[
$this->l10n->n('%n byte', '%n bytes', $expected),
$this->l10n->n('%n byte', '%n bytes', $bytesWritten),
],
)
);
}
}
}

if ($chunk_handler->isComplete()) {
/** @var Storage $storage */
[$storage,] = $this->fileView->resolvePath($path);
$needsPartFile = $storage->needsPartFile();
$partFile = null;

$targetPath = $path . '/' . $info['name'];
/** @var \OC\Files\Storage\Storage $targetStorage */
[$targetStorage, $targetInternalPath] = $this->fileView->resolvePath($targetPath);

$exists = $this->fileView->file_exists($targetPath);

try {
$this->fileView->lockFile($targetPath, ILockingProvider::LOCK_SHARED);

$this->emitPreHooks($exists, $targetPath);
$this->fileView->changeLock($targetPath, ILockingProvider::LOCK_EXCLUSIVE);
/** @var \OC\Files\Storage\Storage $targetStorage */
[$targetStorage, $targetInternalPath] = $this->fileView->resolvePath($targetPath);

if ($needsPartFile) {
// we first assembly the target file as a part file
$partFile = $this->getPartFileBasePath($path . '/' . $info['name']) . '.ocTransferId' . $info['transferid'] . '.part';
/** @var \OC\Files\Storage\Storage $targetStorage */
[$partStorage, $partInternalPath] = $this->fileView->resolvePath($partFile);


$chunk_handler->file_assemble($partStorage, $partInternalPath);

// here is the final atomic rename
$renameOkay = $targetStorage->moveFromStorage($partStorage, $partInternalPath, $targetInternalPath);
$fileExists = $targetStorage->file_exists($targetInternalPath);
if ($renameOkay === false || $fileExists === false) {
\OC::$server->get(LoggerInterface::class)->error('\OC\Files\Filesystem::rename() failed', ['app' => 'webdav']);
// only delete if an error occurred and the target file was already created
if ($fileExists) {
// set to null to avoid double-deletion when handling exception
// stray part file
$partFile = null;
$targetStorage->unlink($targetInternalPath);
}
$this->fileView->changeLock($targetPath, ILockingProvider::LOCK_SHARED);
throw new Exception($this->l10n->t('Could not rename part file assembled from chunks'));
}
} else {
// assemble directly into the final file
$chunk_handler->file_assemble($targetStorage, $targetInternalPath);
}

// allow sync clients to send the mtime along in a header
if (isset($this->request->server['HTTP_X_OC_MTIME'])) {
$mtime = $this->sanitizeMtime($this->request->server['HTTP_X_OC_MTIME']);
if ($targetStorage->touch($targetInternalPath, $mtime)) {
$this->header('X-OC-MTime: accepted');
}
}

// since we skipped the view we need to scan and emit the hooks ourselves
$targetStorage->getUpdater()->update($targetInternalPath);

$this->fileView->changeLock($targetPath, ILockingProvider::LOCK_SHARED);

$this->emitPostHooks($exists, $targetPath);

// FIXME: should call refreshInfo but can't because $this->path is not the of the final file
$info = $this->fileView->getFileInfo($targetPath);

if (isset($this->request->server['HTTP_OC_CHECKSUM'])) {
$checksum = trim($this->request->server['HTTP_OC_CHECKSUM']);
$this->fileView->putFileInfo($targetPath, ['checksum' => $checksum]);
} elseif ($info->getChecksum() !== null && $info->getChecksum() !== '') {
$this->fileView->putFileInfo($this->path, ['checksum' => '']);
}

$this->fileView->unlockFile($targetPath, ILockingProvider::LOCK_SHARED);

return $info->getEtag();
} catch (\Exception $e) {
if ($partFile !== null) {
$targetStorage->unlink($targetInternalPath);
}
$this->convertToSabreException($e);
}
}

return null;
}

/**
* Convert the given exception to a SabreException instance
*
Expand Down
9 changes: 0 additions & 9 deletions apps/dav/lib/Connector/Sabre/FilesPlugin.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -577,15 +577,6 @@ public function handleUpdateProperties($path, PropPatch $propPatch) {
* @throws \Sabre\DAV\Exception\BadRequest
*/
public function sendFileIdHeader($filePath, \Sabre\DAV\INode $node = null) {
// chunked upload handling
if (isset($_SERVER['HTTP_OC_CHUNKED'])) {
[$path, $name] = \Sabre\Uri\split($filePath);
$info = \OC_FileChunking::decodeName($name);
if (!empty($info)) {
$filePath = $path . '/' . $info['name'];
}
}

// we get the node for the given $filePath here because in case of afterCreateFile $node is the parent folder
if (!$this->server->tree->nodeExists($filePath)) {
return;
Expand Down
4 changes: 2 additions & 2 deletions apps/dav/lib/Connector/Sabre/LockPlugin.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public function initialize(\Sabre\DAV\Server $server) {
public function getLock(RequestInterface $request) {
// we can't listen on 'beforeMethod:PUT' due to order of operations with setting up the tree
// so instead we limit ourselves to the PUT method manually
if ($request->getMethod() !== 'PUT' || isset($_SERVER['HTTP_OC_CHUNKED'])) {
if ($request->getMethod() !== 'PUT') {
return;
}
try {
Expand All @@ -84,7 +84,7 @@ public function releaseLock(RequestInterface $request) {
if ($this->isLocked === false) {
return;
}
if ($request->getMethod() !== 'PUT' || isset($_SERVER['HTTP_OC_CHUNKED'])) {
if ($request->getMethod() !== 'PUT') {
return;
}
try {
Expand Down
38 changes: 0 additions & 38 deletions apps/dav/lib/Connector/Sabre/ObjectTree.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,35 +67,6 @@ public function init(\Sabre\DAV\INode $rootNode, \OC\Files\View $view, \OCP\File
$this->mountManager = $mountManager;
}

/**
* If the given path is a chunked file name, converts it
* to the real file name. Only applies if the OC-CHUNKED header
* is present.
*
* @param string $path chunk file path to convert
*
* @return string path to real file
*/
private function resolveChunkFile($path) {
if (isset($_SERVER['HTTP_OC_CHUNKED'])) {
// resolve to real file name to find the proper node
[$dir, $name] = \Sabre\Uri\split($path);
if ($dir === '/' || $dir === '.') {
$dir = '';
}

$info = \OC_FileChunking::decodeName($name);
// only replace path if it was really the chunked file
if (isset($info['transferid'])) {
// getNodePath is called for multiple nodes within a chunk
// upload call
$path = $dir . '/' . $info['name'];
$path = ltrim($path, '/');
}
}
return $path;
}

/**
* Returns the INode object for the requested path
*
Expand Down Expand Up @@ -147,9 +118,6 @@ public function getNodeForPath($path) {
$info = null;
}
} else {
// resolve chunk file name to real name, if applicable
$path = $this->resolveChunkFile($path);

// read from cache
try {
$info = $this->fileView->getFileInfo($path);
Expand All @@ -159,12 +127,6 @@ public function getNodeForPath($path) {
}
} catch (StorageNotAvailableException $e) {
throw new \Sabre\DAV\Exception\ServiceUnavailable('Storage is temporarily not available', 0, $e);
} catch (StorageInvalidException $e) {
throw new \Sabre\DAV\Exception\NotFound('Storage ' . $path . ' is invalid');
} catch (LockedException $e) {
throw new \Sabre\DAV\Exception\Locked();
} catch (ForbiddenException $e) {
throw new \Sabre\DAV\Exception\Forbidden();
}
}

Expand Down
17 changes: 0 additions & 17 deletions apps/dav/lib/Connector/Sabre/QuotaPlugin.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,31 +193,14 @@ public function checkQuota($path, $length = null) {
$parentPath = '';
}
$req = $this->server->httpRequest;
if ($req->getHeader('OC-Chunked')) {
$info = \OC_FileChunking::decodeName($newName);
$chunkHandler = $this->getFileChunking($info);
// subtract the already uploaded size to see whether
// there is still enough space for the remaining chunks
$length -= $chunkHandler->getCurrentSize();
// use target file name for free space check in case of shared files
$path = rtrim($parentPath, '/') . '/' . $info['name'];
}
$freeSpace = $this->getFreeSpace($path);
if ($freeSpace >= 0 && $length > $freeSpace) {
if (isset($chunkHandler)) {
$chunkHandler->cleanup();
}
throw new InsufficientStorage("Insufficient space in $path, $length required, $freeSpace available");
}
}
return true;
}

public function getFileChunking($info) {
// FIXME: need a factory for better mocking support
return new \OC_FileChunking($info);
}

public function getLength() {
$req = $this->server->httpRequest;
$length = $req->getHeader('X-Expected-Entity-Length');
Expand Down
Loading

0 comments on commit aadd950

Please sign in to comment.