-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: report ipfs.add progress over http (#3310)
The browser fetch api doesn't allow reading of any data until the whole request has been sent which means progress events only fire after the upload is complete which rather defeats the purpose of reporting upload progress. Here we switch to XHR for uploads with progress that does allow reading response data before the request is complete. Co-authored-by: achingbrain <alex@achingbrain.net>
- Loading branch information
1 parent
c281053
commit 16f754d
Showing
5 changed files
with
155 additions
and
87 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,87 +1,10 @@ | ||
| 'use strict' | ||
|
|
||
| const normaliseInput = require('ipfs-core-utils/src/files/normalise-input/index') | ||
| const { nanoid } = require('nanoid') | ||
| const modeToString = require('../lib/mode-to-string') | ||
| const mtimeToObject = require('../lib/mtime-to-object') | ||
| const merge = require('merge-options').bind({ ignoreUndefined: true }) | ||
| const toStream = require('it-to-stream') | ||
| const { isElectronRenderer } = require('ipfs-utils/src/env') | ||
|
|
||
| /** | ||
| * | ||
| * @param {Object} source | ||
| * @param {AbortController} abortController | ||
| * @param {Headers|Record<string, string>} [headers] | ||
| * @param {string} [boundary] | ||
| */ | ||
| async function multipartRequest (source = '', abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { | ||
| async function * streamFiles (source) { | ||
| try { | ||
| let index = 0 | ||
|
|
||
| for await (const { content, path, mode, mtime } of normaliseInput(source)) { | ||
| let fileSuffix = '' | ||
| const type = content ? 'file' : 'dir' | ||
|
|
||
| if (index > 0) { | ||
| yield '\r\n' | ||
|
|
||
| fileSuffix = `-${index}` | ||
| } | ||
|
|
||
| let fieldName = type + fileSuffix | ||
| const qs = [] | ||
|
|
||
| if (mode !== null && mode !== undefined) { | ||
| qs.push(`mode=${modeToString(mode)}`) | ||
| } | ||
|
|
||
| const time = mtimeToObject(mtime) | ||
| if (time != null) { | ||
| const { secs, nsecs } = time | ||
|
|
||
| qs.push(`mtime=${secs}`) | ||
|
|
||
| if (nsecs != null) { | ||
| qs.push(`mtime-nsecs=${nsecs}`) | ||
| } | ||
| } | ||
|
|
||
| if (qs.length) { | ||
| fieldName = `${fieldName}?${qs.join('&')}` | ||
| } | ||
|
|
||
| yield `--${boundary}\r\n` | ||
| yield `Content-Disposition: form-data; name="${fieldName}"; filename="${encodeURIComponent(path)}"\r\n` | ||
| yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n` | ||
| yield '\r\n' | ||
|
|
||
| if (content) { | ||
| yield * content | ||
| } | ||
|
|
||
| index++ | ||
| } | ||
| } catch (err) { | ||
| // workaround for https://github.com/node-fetch/node-fetch/issues/753 | ||
| // @ts-ignore - abort does not expect an arguments | ||
| abortController.abort(err) | ||
| } finally { | ||
| yield `\r\n--${boundary}--\r\n` | ||
| } | ||
| } | ||
|
|
||
| return { | ||
| headers: merge(headers, { | ||
| 'Content-Type': `multipart/form-data; boundary=${boundary}` | ||
| }), | ||
| body: await toStream(streamFiles(source)) | ||
| } | ||
| } | ||
|
|
||
| module.exports = multipartRequest | ||
|
|
||
| // In electron-renderer we use native fetch and should encode body using native | ||
| // form data. | ||
| if (isElectronRenderer) { | ||
| module.exports = require('./multipart-request.browser') | ||
| } else { | ||
| module.exports = require('./multipart-request.node') | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| 'use strict' | ||
|
|
||
| const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') | ||
| const { nanoid } = require('nanoid') | ||
| const modeToString = require('./mode-to-string') | ||
| const mtimeToObject = require('./mtime-to-object') | ||
| const merge = require('merge-options').bind({ ignoreUndefined: true }) | ||
| const toStream = require('it-to-stream') | ||
|
|
||
| /** | ||
| * | ||
| * @param {Object} source | ||
| * @param {AbortController} abortController | ||
| * @param {Headers|Record<string, string>} [headers] | ||
| * @param {string} [boundary] | ||
| */ | ||
| async function multipartRequest (source = '', abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { | ||
| async function * streamFiles (source) { | ||
| try { | ||
| let index = 0 | ||
|
|
||
| for await (const { content, path, mode, mtime } of normaliseInput(source)) { | ||
| let fileSuffix = '' | ||
| const type = content ? 'file' : 'dir' | ||
|
|
||
| if (index > 0) { | ||
| yield '\r\n' | ||
|
|
||
| fileSuffix = `-${index}` | ||
| } | ||
|
|
||
| let fieldName = type + fileSuffix | ||
| const qs = [] | ||
|
|
||
| if (mode !== null && mode !== undefined) { | ||
| qs.push(`mode=${modeToString(mode)}`) | ||
| } | ||
|
|
||
| const time = mtimeToObject(mtime) | ||
| if (time != null) { | ||
| const { secs, nsecs } = time | ||
|
|
||
| qs.push(`mtime=${secs}`) | ||
|
|
||
| if (nsecs != null) { | ||
| qs.push(`mtime-nsecs=${nsecs}`) | ||
| } | ||
| } | ||
|
|
||
| if (qs.length) { | ||
| fieldName = `${fieldName}?${qs.join('&')}` | ||
| } | ||
|
|
||
| yield `--${boundary}\r\n` | ||
| yield `Content-Disposition: form-data; name="${fieldName}"; filename="${encodeURIComponent(path)}"\r\n` | ||
| yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n` | ||
| yield '\r\n' | ||
|
|
||
| if (content) { | ||
| yield * content | ||
| } | ||
|
|
||
| index++ | ||
| } | ||
| } catch (err) { | ||
| // workaround for https://github.com/node-fetch/node-fetch/issues/753 | ||
| // @ts-ignore - abort does not expect an arguments | ||
| abortController.abort(err) | ||
| } finally { | ||
| yield `\r\n--${boundary}--\r\n` | ||
| } | ||
| } | ||
|
|
||
| return { | ||
| parts: null, | ||
| total: -1, | ||
| headers: merge(headers, { | ||
| 'Content-Type': `multipart/form-data; boundary=${boundary}` | ||
| }), | ||
| body: await toStream(streamFiles(source)) | ||
| } | ||
| } | ||
|
|
||
| module.exports = multipartRequest |