diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp index 31cd76b4c2c..fde053af0da 100644 --- a/src/libsync/capabilities.cpp +++ b/src/libsync/capabilities.cpp @@ -116,5 +116,12 @@ bool Capabilities::chunkingNg() const return _capabilities["dav"].toMap()["chunking"].toByteArray() >= "1.0"; } +quint64 Capabilities::requestMaxDurationDC() const +{ + QByteArray requestMaxDurationDC = _capabilities["dav"].toMap()["max_single_upload_request_duration_msec"].toByteArray(); + if (!requestMaxDurationDC.isEmpty()) + return requestMaxDurationDC.toLongLong(); + return 0; +} } diff --git a/src/libsync/capabilities.h b/src/libsync/capabilities.h index 861cf18aee0..bc8e97c7e3b 100644 --- a/src/libsync/capabilities.h +++ b/src/libsync/capabilities.h @@ -41,6 +41,7 @@ class OWNCLOUDSYNC_EXPORT Capabilities { int sharePublicLinkExpireDateDays() const; bool shareResharing() const; bool chunkingNg() const; + quint64 requestMaxDurationDC() const; /// returns true if the capabilities report notifications bool notificationsAvailable() const; diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp index 4c8a30bf6f9..ff13ad0f74a 100644 --- a/src/libsync/configfile.cpp +++ b/src/libsync/configfile.cpp @@ -52,6 +52,7 @@ static const char updateCheckIntervalC[] = "updateCheckInterval"; static const char geometryC[] = "geometry"; static const char timeoutC[] = "timeout"; static const char chunkSizeC[] = "chunkSize"; +static const char maxChunkSizeC[] = "maxChunkSizeC"; static const char proxyHostC[] = "Proxy/host"; static const char proxyTypeC[] = "Proxy/type"; @@ -128,6 +129,18 @@ quint64 ConfigFile::chunkSize() const return settings.value(QLatin1String(chunkSizeC), 10*1000*1000).toLongLong(); // default to 10 MB } +quint64 ConfigFile::maxChunkSize() const +{ + QSettings settings(configFile(), QSettings::IniFormat); + return settings.value(QLatin1String(maxChunkSizeC), 50*1000*1000).toLongLong(); // default to 50 MB +} + +quint64 ConfigFile::minChunkSize() const +{ + QSettings settings(configFile(), QSettings::IniFormat); + return settings.value(QLatin1String(maxChunkSizeC), 1000*1000).toLongLong(); // default to 1 MB +} + void ConfigFile::setOptionalDesktopNotifications(bool show) { QSettings settings(configFile(), QSettings::IniFormat); diff --git a/src/libsync/configfile.h b/src/libsync/configfile.h index cd95f1a0bce..e94b94e9b6f 100644 --- a/src/libsync/configfile.h +++ b/src/libsync/configfile.h @@ -113,6 +113,8 @@ class OWNCLOUDSYNC_EXPORT ConfigFile int timeout() const; quint64 chunkSize() const; + quint64 maxChunkSize() const; + quint64 minChunkSize() const; void saveGeometry(QWidget *w); void restoreGeometry(QWidget *w); diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp index e2ce6c5dde9..c6913a1a8e4 100644 --- a/src/libsync/owncloudpropagator.cpp +++ b/src/libsync/owncloudpropagator.cpp @@ -277,7 +277,7 @@ PropagateItemJob* OwncloudPropagator::createJob(const SyncFileItemPtr &item) { } else { PropagateUploadFileCommon *job = 0; if (item->_size > chunkSize() && account()->capabilities().chunkingNg()) { - job = new PropagateUploadFileNG(this, item); + job = new PropagateUploadFileNG(this, item, account()->capabilities().requestMaxDurationDC()); } else { job = new PropagateUploadFileV1(this, item); } @@ -459,6 +459,18 @@ quint64 OwncloudPropagator::chunkSize() return chunkSize; } +quint64 OwncloudPropagator::maxChunkSize() +{ + static uint chunkSize; + if (!chunkSize) { + chunkSize = qgetenv("OWNCLOUD_MAX_CHUNK_SIZE").toUInt(); + if (chunkSize == 0) { + ConfigFile cfg; + chunkSize = cfg.maxChunkSize(); + } + } + return chunkSize; +} bool OwncloudPropagator::localFileNameClash( const QString& relFile ) { diff --git a/src/libsync/owncloudpropagator.h b/src/libsync/owncloudpropagator.h index 1097b7f8747..cc9d18b3f7d 100644 --- a/src/libsync/owncloudpropagator.h +++ b/src/libsync/owncloudpropagator.h @@ -272,7 +272,6 @@ class OwncloudPropagator : public QObject { SyncJournalDb * const _journal; bool _finishedEmited; // used to ensure that finished is only emitted once - public: OwncloudPropagator(AccountPtr account, const QString &localDir, const QString &remoteFolder, SyncJournalDb *progressDb) @@ -327,6 +326,7 @@ class OwncloudPropagator : public QObject { /** returns the size of chunks in bytes */ static quint64 chunkSize(); + static quint64 maxChunkSize(); AccountPtr account() const; diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index bac3e112ba5..d6c292bb432 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -290,12 +290,36 @@ class PropagateUploadFileNG : public PropagateUploadFileCommon { uint _transferId; /// transfer id (part of the url) int _currentChunk; /// Id of the next chunk that will be sent bool _removeJobError; /// If not null, there was an error removing the job + quint64 _lastChunkSize; /// current chunk size + + /* + * This is value in ms obtained from the server. + * + * Dynamic Chunking attribute the maximum number of miliseconds that single request below chunk size can take + * This value should be based on heuristics with default value 10000ms, time it takes to transfer 10MB chunk on 1MB/s upload link. + * + * Suggested solution will be to evaluate max(SNR, MORD) where: + * > SNR - Slow network request, so time it will take to transmit default chunking sized request at specific low upload bandwidth + * > MORD - Maximum observed request time, so double the time of maximum observed RTT of the very small PUT request (e.g. 1kB) to the system + * + * Exemplary, syncing 100MB files, with chunking size 10MB, will cause sync of 10 PUT requests which max evaluation was set to + * + * Dynamic chunking client algorithm is specified in the ownCloud documentation and uses to estimate if given + * bandwidth allows higher chunk sizes (because of high goodput) + */ + quint64 _requestMaxDuration; // Map chunk number with its size from the PROPFIND on resume. // (Only used from slotPropfindIterate/slotPropfindFinished because the LsColJob use signals to report data.) QMap _serverChunks; quint64 chunkSize() const { return _propagator->chunkSize(); } + quint64 maxChunkSize() const { return _propagator->maxChunkSize(); } + + quint64 getRequestMaxDurationDC(){ + return _requestMaxDuration; + } + /** * Return the URL of a chunk. * If chunk == -1, returns the URL of the parent folder containing the chunks @@ -303,10 +327,11 @@ class PropagateUploadFileNG : public PropagateUploadFileCommon { QUrl chunkUrl(int chunk = -1); public: - PropagateUploadFileNG(OwncloudPropagator* propagator,const SyncFileItemPtr& item) : - PropagateUploadFileCommon(propagator,item) {} + PropagateUploadFileNG(OwncloudPropagator* propagator,const SyncFileItemPtr& item, const quint64& requestMaxDuration) : + PropagateUploadFileCommon(propagator,item), _lastChunkSize(0), _requestMaxDuration(requestMaxDuration) {} void doStartUpload() Q_DECL_OVERRIDE; + private: void startNewUpload(); void startNextChunk(); diff --git a/src/libsync/propagateuploadng.cpp b/src/libsync/propagateuploadng.cpp index 106b0dcdb2f..cf988a1276c 100644 --- a/src/libsync/propagateuploadng.cpp +++ b/src/libsync/propagateuploadng.cpp @@ -32,7 +32,7 @@ #include #include #include - +#include namespace OCC { QUrl PropagateUploadFileNG::chunkUrl(int chunk) @@ -262,7 +262,42 @@ void PropagateUploadFileNG::startNextChunk() quint64 fileSize = _item->_size; Q_ASSERT(fileSize >= _sent); - quint64 currentChunkSize = qMin(chunkSize(), fileSize - _sent); + + quint64 currentChunkSize = chunkSize(); + + // this will check if getRequestMaxDurationDC is set to 0 or not + double requestMaxDurationDC = (double) getRequestMaxDurationDC(); + if (requestMaxDurationDC != 0) { + // this if first chunked file request, so it can start with default size of chunkSize() + // if _lastChunkSize != 0 it means that we already have send one request + if(_lastChunkSize != 0){ + //TODO: this is done step by step for debugging purposes + + //get last request timestamp + double lastChunkLap = (double) _stopWatch.durationOfLap(QLatin1String("ChunkDuration")); + + //get duration of the request + double requestDuration = (double) _stopWatch.addLapTime(QLatin1String("ChunkDuration")) - lastChunkLap; + + // calculate natural logarithm + double correctionParameter = log(requestMaxDurationDC / requestDuration) - 1; + + // If logarithm is smaller or equal zero, it means that we exceeded max request duration + // If exceeded it will use currentChunkSize = chunkSize() + // If did not exceeded, we will increase the chunk size + // motivation for logarithm is specified in the dynamic chunking documentation + // TODO: give link to documentation + if (correctionParameter>0){ + currentChunkSize = qMin(_lastChunkSize + (qint64) correctionParameter*chunkSize(), maxChunkSize()); + } + } + + //remember the value of last chunk size + _lastChunkSize = currentChunkSize; + } + + // prevent situation that chunk size is bigger then required one to send + currentChunkSize = qMin(currentChunkSize, fileSize - _sent); if (currentChunkSize == 0) { Q_ASSERT(_jobs.isEmpty()); // There should be no running job anymore