dynamic chunking prototype

src/libsync/capabilities.cpp

@@ -116,5 +116,12 @@ bool Capabilities::chunkingNg() const
     return _capabilities["dav"].toMap()["chunking"].toByteArray() >= "1.0";
 }
 
+quint64 Capabilities::requestMaxDurationDC() const
+{
+    QByteArray requestMaxDurationDC = _capabilities["dav"].toMap()["max_single_upload_request_duration_msec"].toByteArray();
+    if (!requestMaxDurationDC.isEmpty())
+        return requestMaxDurationDC.toLongLong();
+    return 0;
+}
 
 }

src/libsync/capabilities.h

@@ -41,6 +41,7 @@ class OWNCLOUDSYNC_EXPORT Capabilities {
     int  sharePublicLinkExpireDateDays() const;
     bool shareResharing() const;
     bool chunkingNg() const;
+    quint64 requestMaxDurationDC() const;
 
     /// returns true if the capabilities report notifications
     bool notificationsAvailable() const;

src/libsync/configfile.cpp

@@ -52,6 +52,7 @@ static const char updateCheckIntervalC[] = "updateCheckInterval";
 static const char geometryC[] = "geometry";
 static const char timeoutC[] = "timeout";
 static const char chunkSizeC[] = "chunkSize";
+static const char maxChunkSizeC[] = "maxChunkSizeC";
 
 static const char proxyHostC[] = "Proxy/host";
 static const char proxyTypeC[] = "Proxy/type";
@@ -128,6 +129,18 @@ quint64 ConfigFile::chunkSize() const
     return settings.value(QLatin1String(chunkSizeC), 10*1000*1000).toLongLong(); // default to 10 MB
 }
 
+quint64 ConfigFile::maxChunkSize() const
+{
+    QSettings settings(configFile(), QSettings::IniFormat);
+    return settings.value(QLatin1String(maxChunkSizeC), 50*1000*1000).toLongLong(); // default to 50 MB
+}
+
+quint64 ConfigFile::minChunkSize() const
+{
+    QSettings settings(configFile(), QSettings::IniFormat);
+    return settings.value(QLatin1String(maxChunkSizeC), 1000*1000).toLongLong(); // default to 1 MB
+}
+
 void ConfigFile::setOptionalDesktopNotifications(bool show)
 {
     QSettings settings(configFile(), QSettings::IniFormat);

src/libsync/configfile.h

@@ -113,6 +113,8 @@ class OWNCLOUDSYNC_EXPORT ConfigFile
 
     int timeout() const;
     quint64 chunkSize() const;
+    quint64 maxChunkSize() const;
+    quint64 minChunkSize() const;
 
     void saveGeometry(QWidget *w);
     void restoreGeometry(QWidget *w);

src/libsync/owncloudpropagator.cpp

@@ -277,7 +277,7 @@ PropagateItemJob* OwncloudPropagator::createJob(const SyncFileItemPtr &item) {
             } else {
                 PropagateUploadFileCommon *job = 0;
                 if (item->_size > chunkSize() && account()->capabilities().chunkingNg()) {
-                    job = new PropagateUploadFileNG(this, item);
+                    job = new PropagateUploadFileNG(this, item, account()->capabilities().requestMaxDurationDC());
                 } else {
                     job = new PropagateUploadFileV1(this, item);
                 }
@@ -459,6 +459,18 @@ quint64 OwncloudPropagator::chunkSize()
     return chunkSize;
 }
 
+quint64 OwncloudPropagator::maxChunkSize()
+{
+    static uint chunkSize;
+    if (!chunkSize) {
+        chunkSize = qgetenv("OWNCLOUD_MAX_CHUNK_SIZE").toUInt();
+        if (chunkSize == 0) {
+            ConfigFile cfg;
+            chunkSize = cfg.maxChunkSize();
+        }
+    }
+    return chunkSize;
+}
 
 bool OwncloudPropagator::localFileNameClash( const QString& relFile )
 {

src/libsync/owncloudpropagator.h

@@ -272,7 +272,6 @@ class OwncloudPropagator : public QObject {
     SyncJournalDb * const _journal;
     bool _finishedEmited; // used to ensure that finished is only emitted once
 
-
 public:
     OwncloudPropagator(AccountPtr account, const QString &localDir,
                        const QString &remoteFolder, SyncJournalDb *progressDb)
@@ -327,6 +326,7 @@ class OwncloudPropagator : public QObject {
 
     /** returns the size of chunks in bytes  */
     static quint64 chunkSize();
+    static quint64 maxChunkSize();
 
     AccountPtr account() const;
 

src/libsync/propagateupload.h

@@ -290,23 +290,48 @@ class PropagateUploadFileNG : public PropagateUploadFileCommon {
     uint _transferId; /// transfer id (part of the url)
     int _currentChunk; /// Id of the next chunk that will be sent
     bool _removeJobError; /// If not null, there was an error removing the job
+    quint64 _lastChunkSize; /// current chunk size
+
+    /*
+    * This is value in ms obtained from the server.
+    *
+    * Dynamic Chunking attribute the maximum number of miliseconds that single request below chunk size can take
+    * This value should be based on heuristics with default value 10000ms, time it takes to transfer 10MB chunk on 1MB/s upload link.
+    *
+    * Suggested solution will be to evaluate max(SNR, MORD) where:
+    * > SNR - Slow network request, so time it will take to transmit default chunking sized request at specific low upload bandwidth
+    * > MORD - Maximum observed request time, so double the time of maximum observed RTT of the very small PUT request (e.g. 1kB) to the system
+    *
+    * Exemplary, syncing 100MB files, with chunking size 10MB, will cause sync of 10 PUT requests which max evaluation was set to <max_single_upload_request_duration_msec>
+    *
+    * Dynamic chunking client algorithm is specified in the ownCloud documentation and uses <max_single_upload_request_duration_msec> to estimate if given
+    * bandwidth allows higher chunk sizes (because of high goodput)
+    */
+    quint64 _requestMaxDuration;
 
     // Map chunk number with its size  from the PROPFIND on resume.
     // (Only used from slotPropfindIterate/slotPropfindFinished because the LsColJob use signals to report data.)
     QMap<int, quint64> _serverChunks;
 
     quint64 chunkSize() const { return _propagator->chunkSize(); }
+    quint64 maxChunkSize() const { return _propagator->maxChunkSize(); }
+
+    quint64 getRequestMaxDurationDC(){
+        return _requestMaxDuration;
+    }
+
     /**
      * Return the URL of a chunk.
      * If chunk == -1, returns the URL of the parent folder containing the chunks
      */
     QUrl chunkUrl(int chunk = -1);
 
 public:
-    PropagateUploadFileNG(OwncloudPropagator* propagator,const SyncFileItemPtr& item) :
-        PropagateUploadFileCommon(propagator,item) {}
+    PropagateUploadFileNG(OwncloudPropagator* propagator,const SyncFileItemPtr& item, const quint64& requestMaxDuration) :
+        PropagateUploadFileCommon(propagator,item), _lastChunkSize(0), _requestMaxDuration(requestMaxDuration) {}
 
     void doStartUpload() Q_DECL_OVERRIDE;
+
 private:
     void startNewUpload();
     void startNextChunk();

src/libsync/propagateuploadng.cpp

@@ -32,7 +32,7 @@
 #include <QDir>
 #include <cmath>
 #include <cstring>
-
+#include <cmath>
 namespace OCC {
 
 QUrl PropagateUploadFileNG::chunkUrl(int chunk)
@@ -262,7 +262,42 @@ void PropagateUploadFileNG::startNextChunk()
 
     quint64 fileSize = _item->_size;
     Q_ASSERT(fileSize >= _sent);
-    quint64 currentChunkSize = qMin(chunkSize(), fileSize - _sent);
+
+    quint64 currentChunkSize = chunkSize();
+
+    // this will check if getRequestMaxDurationDC is set to 0 or not
+    double requestMaxDurationDC = (double) getRequestMaxDurationDC();
+    if (requestMaxDurationDC != 0) {
+        // this if first chunked file request, so it can start with default size of chunkSize()
+        // if _lastChunkSize != 0 it means that we already have send one request
+        if(_lastChunkSize != 0){
+            //TODO: this is done step by step for debugging purposes
+
+            //get last request timestamp
+            double lastChunkLap = (double) _stopWatch.durationOfLap(QLatin1String("ChunkDuration"));
+
+            //get duration of the request
+            double requestDuration = (double) _stopWatch.addLapTime(QLatin1String("ChunkDuration")) - lastChunkLap;
+
+            // calculate natural logarithm
+            double correctionParameter = log(requestMaxDurationDC / requestDuration) - 1;
+
+            // If logarithm is smaller or equal zero, it means that we exceeded max request duration
+            // If exceeded it will use currentChunkSize = chunkSize()
+            // If did not exceeded, we will increase the chunk size
+            // motivation for logarithm is specified in the dynamic chunking documentation
+            // TODO: give link to documentation
+            if (log>0){
+                currentChunkSize = qMin(_lastChunkSize + (qint64) correctionParameter*chunkSize(), maxChunkSize());
+            }
+        }
+
+        //remember the value of last chunk size
+        _lastChunkSize = currentChunkSize;
+    }
+
+    // prevent situation that chunk size is bigger then required one to send
+    currentChunkSize = qMin(currentChunkSize, fileSize - _sent);
 
     if (currentChunkSize == 0) {
         Q_ASSERT(_jobs.isEmpty()); // There should be no running job anymore