From 5c542492ad883b811078b602b4499a6b91fdc296 Mon Sep 17 00:00:00 2001 From: Harry Chen Date: Sat, 21 Sep 2024 20:35:17 +0800 Subject: [PATCH] feat: busboy support async generator mode (#4074) * feat: support async generator mode * fix: limit case * fix: limit case * fix: limit case * chore: test case * fix: EPIPE error * fix: test * fix: test * chore: hidden init * docs: update * fix: file stream delay * fix: file stream delay --- .gitignore | 1 + packages/busboy/src/error.ts | 6 +- packages/busboy/src/interface.ts | 22 +- packages/busboy/src/middleware.ts | 506 +++++++++++------- packages/busboy/src/utils.ts | 9 + packages/busboy/test/clean.test.ts | 30 +- packages/busboy/test/express.test.ts | 23 +- .../fixtures/koa-file/src/configuration.ts | 8 + .../fixtures/koa-stream/src/configuration.ts | 31 +- packages/busboy/test/index.test.ts | 19 +- packages/busboy/test/koa.test.ts | 478 +++++++++++++++-- packages/core/src/util/webRouterParam.ts | 6 +- site/blog/2024-08-29-release-3.17.md | 1 - site/docs/extensions/busboy.md | 209 ++++++-- .../current/extensions/busboy.md | 204 +++++-- 15 files changed, 1219 insertions(+), 334 deletions(-) diff --git a/.gitignore b/.gitignore index 459a927009b7..772788faca99 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ site/changelog .changelog **/test/*.txt .audit +packages/**/test/tmp diff --git a/packages/busboy/src/error.ts b/packages/busboy/src/error.ts index 2a916b5f814c..d1abd9108566 100644 --- a/packages/busboy/src/error.ts +++ b/packages/busboy/src/error.ts @@ -2,16 +2,16 @@ import { httpError } from '@midwayjs/core'; export class MultipartInvalidFilenameError extends httpError.BadRequestError { constructor(filename: string) { - super(`Invalid upload file name "${filename}", please check it`); + super(`Invalid file name or suffix "${filename}".`); } } export class MultipartInvalidFileTypeError extends httpError.BadRequestError { constructor(filename: string, currentType: string, type: string) { super( - `Invalid upload file type, "${filename}" type(${ + `Invalid file type, "${filename}" type(${ currentType || 'unknown' - }) is not ${type} , please check it` + }) is not ${type}.` ); } } diff --git a/packages/busboy/src/interface.ts b/packages/busboy/src/interface.ts index 1d5a083470c1..79db94dc11d5 100644 --- a/packages/busboy/src/interface.ts +++ b/packages/busboy/src/interface.ts @@ -2,7 +2,7 @@ import { Readable } from 'stream'; import { IgnoreMatcher, IMidwayContext } from '@midwayjs/core'; import { BusboyConfig } from 'busboy'; -export type UploadMode = 'stream' | 'file'; +export type UploadMode = 'stream' | 'file' | 'asyncIterator'; export interface UploadOptions extends BusboyConfig { /** @@ -52,6 +52,10 @@ export interface UploadFileInfo { * file data, a string of path */ data: string; + /** + * field name + */ + fieldName: string; } export interface UploadStreamFileInfo { @@ -66,7 +70,21 @@ export interface UploadStreamFileInfo { /** * file data, Readable stream */ - data: Readable ; + data: Readable; + /** + * field name + */ + fieldName: string; } +export interface UploadStreamFieldInfo { + /** + * field name + */ + name: string; + /** + * field value + */ + value: any; +} diff --git a/packages/busboy/src/middleware.ts b/packages/busboy/src/middleware.ts index f9cf1346616a..091b2fe319e1 100644 --- a/packages/busboy/src/middleware.ts +++ b/packages/busboy/src/middleware.ts @@ -13,7 +13,6 @@ import { resolve } from 'path'; import { createWriteStream, promises } from 'fs'; import { Readable, Stream } from 'stream'; import { - EXT_KEY, MultipartError, MultipartFieldsLimitError, MultipartFileLimitError, @@ -21,15 +20,20 @@ import { MultipartInvalidFilenameError, MultipartInvalidFileTypeError, MultipartPartsLimitError, +} from './error'; +import { + UploadStreamFieldInfo, UploadFileInfo, UploadOptions, UploadStreamFileInfo, -} from '.'; + UploadMode, +} from './interface'; import { parseMultipart } from './parse'; import { fromBuffer } from 'file-type'; -import { formatExt } from './utils'; +import { formatExt, streamToAsyncIterator } from './utils'; import * as busboy from 'busboy'; import { BusboyConfig } from 'busboy'; +import { EXT_KEY } from './constants'; const { unlink, writeFile } = promises; @@ -55,7 +59,7 @@ export class UploadMiddleware implements IMiddleware { ignore: IgnoreMatcher[]; @Init() - async init() { + protected async init() { if (this.uploadConfig.match) { this.match = [].concat(this.uploadConfig.match || []); } else { @@ -81,7 +85,7 @@ export class UploadMiddleware implements IMiddleware { resolve( app: IMidwayApplication, options?: { - mode?: 'file' | 'stream'; + mode?: UploadMode; } & BusboyConfig ) { const isExpress = app.getNamespace() === 'express'; @@ -97,7 +101,7 @@ export class UploadMiddleware implements IMiddleware { return next(); } - const { mode, tmpdir } = uploadConfig; + const useAsyncIterator = uploadConfig.mode === 'asyncIterator'; // create new map include custom white list const currentContextWhiteListMap = new Map([ @@ -144,208 +148,346 @@ export class UploadMiddleware implements IMiddleware { }; if (this.isReadableStream(req, isExpress)) { - let isStreamResolve = false; - const { files = [], fields = [] } = await new Promise( - (resolveP, reject) => { - const bb = busboy({ - headers: req.headers, - ...uploadConfig, - }); - const fields: Promise[] = []; - const files: Promise[] = []; - let fileModeCount = 0; - bb.on('file', async (name, file, info) => { - const { filename, encoding, mimeType } = info; - const ext = this.checkAndGetExt( - filename, - currentContextWhiteListMap - ); - if (!ext) { - reject(new MultipartInvalidFilenameError(filename)); - } + if (useAsyncIterator) { + await this.processAsyncIterator( + req, + uploadConfig, + currentContextWhiteListMap, + currentContextMimeTypeWhiteListMap, + ctxOrReq + ); + } else { + await this.processFileOrStream( + req, + uploadConfig, + currentContextWhiteListMap, + currentContextMimeTypeWhiteListMap, + ctxOrReq + ); + } + } else { + await this.processServerlessUpload( + req, + boundary, + uploadConfig, + next, + currentContextWhiteListMap, + currentContextMimeTypeWhiteListMap, + ctxOrReq + ); + } - file.on('limit', () => { - reject(new MultipartFileSizeLimitError(filename)); - }); + await next(); + }; + } - if (mode === 'stream') { - if (isStreamResolve) { - // will be skip - file.resume(); - return; - } - files.push( - new Promise(resolve => { - resolve({ - filename, - mimeType, - encoding, - data: file, - }); - }) - ); - isStreamResolve = true; - return resolveP({ - fields: await Promise.all(fields), - files: await Promise.all(files), - }); - } else { - fileModeCount++; - // file mode - const requireId = `upload_${Date.now()}.${Math.random()}`; - // read stream pipe to temp file - const tempFile = resolve(tmpdir, `${requireId}${ext}`); - // get buffer from stream, and check file type - file.once('data', async chunk => { - const { passed, mime, current } = await this.checkFileType( - ext as string, - chunk, - currentContextMimeTypeWhiteListMap - ); - if (!passed) { - file.pause(); - reject( - new MultipartInvalidFileTypeError(filename, current, mime) - ); - } - }); - const writeStream = file.pipe(createWriteStream(tempFile)); - file.on('end', () => { - fileModeCount--; - }); - writeStream.on('error', reject); - - writeStream.on('finish', async () => { - files.push( - new Promise(resolve => { - resolve({ - filename, - mimeType, - encoding, - data: tempFile, - }); - }) - ); - if (fileModeCount === 0) { - return resolveP({ - fields: await Promise.all(fields), - files: await Promise.all(files), - }); - } + private async processFileOrStream( + req, + uploadConfig, + currentContextWhiteListMap, + currentContextMimeTypeWhiteListMap, + ctxOrReq + ) { + let isStreamResolve = false; + const { mode, tmpdir } = uploadConfig; + const { files = [], fields = [] } = await new Promise( + (resolveP, reject) => { + const bb = busboy({ + headers: req.headers, + ...uploadConfig, + }); + const fields: Array = []; + const files: Array = []; + bb.on('file', async (name, file, info) => { + this.logger.debug('[busboy]: busboy file event'); + const { filename, encoding, mimeType } = info; + const ext = this.checkAndGetExt(filename, currentContextWhiteListMap); + if (!ext) { + reject(new MultipartInvalidFilenameError(filename)); + } + + file.once('limit', () => { + reject(new MultipartFileSizeLimitError(filename)); + }); + + if (mode === 'stream') { + if (isStreamResolve) { + // will be skip + file.resume(); + return; + } + (files as Array).push( + new Promise(resolve => { + resolve({ + fieldName: name, + filename, + encoding, + mimeType, + data: file, }); - } + }) + ); + isStreamResolve = true; + // busboy 这里无法触发 close 事件，所以这里直接 resolve + return resolveP({ + fields, + files: await Promise.all(files), }); - bb.on('field', (name, value, info) => { - fields.push( - new Promise(resolve => { - resolve({ - name, - value, - }); - }) + } else { + this.logger.debug('[busboy]: file mode, file data event'); + // file mode + const requireId = `upload_${Date.now()}.${Math.random()}`; + // read stream pipe to temp file + const tempFile = resolve(tmpdir, `${requireId}${ext}`); + // get buffer from stream, and check file type + file.once('data', async chunk => { + const { passed, mime, current } = await this.checkFileType( + ext as string, + chunk, + currentContextMimeTypeWhiteListMap ); + if (!passed) { + file.pause(); + reject( + new MultipartInvalidFileTypeError(filename, current, mime) + ); + } }); - bb.on('error', (err: Error) => { - reject(new MultipartError(err)); + const fp = new Promise(resolve => { + const writeStream = file.pipe(createWriteStream(tempFile)); + writeStream.on('error', reject); + writeStream.on('finish', () => { + resolve({ + filename, + mimeType, + encoding, + fieldName: name, + data: tempFile, + }); + }); }); - bb.on('partsLimit', () => { - reject(new MultipartPartsLimitError()); - }); + (files as Array).push(fp); + } + }); - bb.on('filesLimit', () => { - reject(new MultipartFileLimitError()); - }); + bb.on('close', async () => { + this.logger.debug('[busboy]: busboy close'); + // close 事件会在 busboy 解析完所有数据后触发，但是这个时候有可能没有写完文件，所以使用 Promise.all 等待所有文件写入完成 + resolveP({ + fields, + files: await Promise.all(files), + }); + }); - bb.on('fieldsLimit', () => { - reject(new MultipartFieldsLimitError()); - }); + bb.on('field', (name, value, info) => { + (fields as Array).push({ + name, + value, + info, + }); + }); - req.pipe(bb); - } - ); - ctxOrReq.files = files; - ctxOrReq.fields = fields.reduce((accumulator, current) => { - accumulator[current.name] = current.value; - return accumulator; - }, {}); - Object.defineProperty(ctxOrReq, 'file', { - get() { - return ctxOrReq.files[0]; - }, - set(v) { - ctxOrReq.files = [v]; - }, + bb.on('error', (err: Error) => { + reject(new MultipartError(err)); }); - } else { - let body; - if ( - req?.originEvent?.body && - (typeof req.originEvent.body === 'string' || - Buffer.isBuffer(req.originEvent.body)) - ) { - body = req.originEvent.body; - } else { - body = req.body; - } + bb.on('partsLimit', () => { + reject(new MultipartPartsLimitError()); + }); + bb.on('filesLimit', () => { + reject(new MultipartFileLimitError()); + }); + bb.on('fieldsLimit', () => { + reject(new MultipartFieldsLimitError()); + }); + req.pipe(bb); + } + ); - const data = await parseMultipart(body, boundary, uploadConfig); - if (!data) { - return next(); - } + ctxOrReq.files = files; + ctxOrReq.fields = fields.reduce((accumulator, current) => { + accumulator[current.name] = current.value; + return accumulator; + }, {}); + } - ctxOrReq.fields = data.fields; - const requireId = `upload_${Date.now()}.${Math.random()}`; - const files = data.files; - for (const fileInfo of files) { - const ext = this.checkAndGetExt( - fileInfo.filename, - currentContextWhiteListMap - ); - if (!ext) { - throw new MultipartInvalidFilenameError(fileInfo.filename); - } - const { passed, mime, current } = await this.checkFileType( - ext as string, - fileInfo.data, + private async processAsyncIterator( + req, + uploadConfig, + currentContextWhiteListMap, + currentContextMimeTypeWhiteListMap, + ctxOrReq + ) { + const bb = busboy({ + headers: req.headers, + ...uploadConfig, + }); + + const fileReadable = new Readable({ + objectMode: true, + read() {}, + autoDestroy: true, + }); + const fieldReadable = new Readable({ + objectMode: true, + read() {}, + autoDestroy: true, + }); + + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + async function* streamToAsyncIteratorWithError( + stream: Readable + ): AsyncGenerator { + for await (const chunk of stream) { + chunk.data.once('data', async chunk => { + const { passed, mime, current } = await self.checkFileType( + chunk.ext, + chunk, currentContextMimeTypeWhiteListMap ); if (!passed) { throw new MultipartInvalidFileTypeError( - fileInfo.filename, + chunk.filename, current, mime ); } + }); - fileInfo[EXT_KEY] = ext; - } - ctxOrReq.files = await Promise.all( - files.map(async (file, index) => { - const { data } = file; - if (mode === 'file') { - const ext = file[EXT_KEY]; - const tmpFileName = resolve( - tmpdir, - `${requireId}.${index}${ext}` - ); - await writeFile(tmpFileName, data, 'binary'); - file.data = tmpFileName; - } else if (mode === 'stream') { - file.data = new Readable({ - read() { - this.push(data); - this.push(null); - }, - }); - } - return file; - }) + yield chunk; + } + } + + const files = streamToAsyncIteratorWithError(fileReadable); + const fields = streamToAsyncIterator(fieldReadable); + + bb.on('file', (name, file, info) => { + const { filename, encoding, mimeType } = info; + const ext = this.checkAndGetExt(filename, currentContextWhiteListMap); + if (!ext) { + fileReadable.destroy(new MultipartInvalidFilenameError(filename)); + return; + } + + file.once('limit', () => { + fileReadable.destroy(new MultipartFileSizeLimitError(filename)); + }); + + fileReadable.push({ + fieldName: name, + filename, + mimeType, + encoding, + data: file, + ext, + }); + }); + + bb.on('field', (name, value, info) => { + fieldReadable.push({ + name, + value, + info, + }); + }); + + bb.on('close', () => { + fileReadable.push(null); + fieldReadable.push(null); + }); + + bb.on('error', (err: Error) => { + fileReadable.destroy(new MultipartError(err)); + }); + + bb.on('partsLimit', () => { + fileReadable.destroy(new MultipartPartsLimitError()); + }); + bb.on('filesLimit', () => { + fileReadable.destroy(new MultipartFileLimitError()); + }); + bb.on('fieldsLimit', () => { + fieldReadable.destroy(new MultipartFieldsLimitError()); + }); + req.pipe(bb); + + ctxOrReq.fields = fields; + ctxOrReq.files = files; + } + + private async processServerlessUpload( + req, + boundary, + uploadConfig, + next, + currentContextWhiteListMap, + currentContextMimeTypeWhiteListMap, + ctxOrReq + ) { + let body; + if ( + req?.originEvent?.body && + (typeof req.originEvent.body === 'string' || + Buffer.isBuffer(req.originEvent.body)) + ) { + body = req.originEvent.body; + } else { + body = req.body; + } + + const { mode, tmpdir } = uploadConfig; + const data = await parseMultipart(body, boundary, uploadConfig); + if (!data) { + return next(); + } + + const requireId = `upload_${Date.now()}.${Math.random()}`; + for (const fileInfo of data.files) { + const ext = this.checkAndGetExt( + fileInfo.filename, + currentContextWhiteListMap + ); + if (!ext) { + throw new MultipartInvalidFilenameError(fileInfo.filename); + } + const { passed, mime, current } = await this.checkFileType( + ext as string, + fileInfo.data, + currentContextMimeTypeWhiteListMap + ); + if (!passed) { + throw new MultipartInvalidFileTypeError( + fileInfo.filename, + current, + mime ); } - await next(); - }; + fileInfo[EXT_KEY] = ext; + } + const files = await Promise.all( + data.files.map(async (file, index) => { + const { data } = file; + if (mode === 'file') { + const ext = file[EXT_KEY]; + const tmpFileName = resolve(tmpdir, `${requireId}.${index}${ext}`); + await writeFile(tmpFileName, data, 'binary'); + file.data = tmpFileName; + } else if (mode === 'stream') { + file.data = new Readable({ + read() { + this.push(data); + this.push(null); + }, + }); + } + return file; + }) + ); + + ctxOrReq.fields = data.fields; + ctxOrReq.files = files; } static getName() { diff --git a/packages/busboy/src/utils.ts b/packages/busboy/src/utils.ts index a25a2aba122c..8dac31fc3e11 100644 --- a/packages/busboy/src/utils.ts +++ b/packages/busboy/src/utils.ts @@ -1,5 +1,6 @@ import { promises, constants } from 'fs'; import { join } from 'path'; +import { Readable } from 'stream'; const { readdir, access, stat, unlink, mkdir } = promises; let autoRemoveUploadTmpFileTimeoutHandler; let autoRemoveUploadTmpFilePromise; @@ -91,3 +92,11 @@ export const formatExt = (ext: string): string => { }) .toString(); }; + +export async function* streamToAsyncIterator( + stream: Readable +): AsyncGenerator { + for await (const chunk of stream) { + yield chunk; + } +} diff --git a/packages/busboy/test/clean.test.ts b/packages/busboy/test/clean.test.ts index 402439b49acf..236bb9de952d 100644 --- a/packages/busboy/test/clean.test.ts +++ b/packages/busboy/test/clean.test.ts @@ -1,11 +1,10 @@ import { createHttpRequest, close, createFunctionApp } from '@midwayjs/mock'; import { join } from 'path'; -import * as assert from 'assert'; import * as ServerlessApp from '../../../packages-legacy/serverless-app/src'; import { existsSync, statSync } from 'fs'; import { sleep } from '@midwayjs/core'; -describe('test/clan.test.ts', function () { +describe('test/clean.test.ts', function () { it('upload file auto clean', async () => { const appDir = join(__dirname, 'fixtures/clean'); @@ -13,20 +12,21 @@ describe('test/clan.test.ts', function () { const app = await createFunctionApp(appDir, {}, ServerlessApp); const request = await createHttpRequest(app); const stat = statSync(imagePath); - await request.post('/upload') + const response = await request.post('/upload') .field('name', 'form') - .attach('file', imagePath) - .expect(200) - .then(async response => { - assert(response.body.files.length === 1); - assert(response.body.files[0].filename === '1.jpg'); - assert(response.body.fields.name === 'form'); - const file1Stat = statSync(response.body.files[0].data); - assert(file1Stat.size && file1Stat.size === stat.size); - await sleep(2000); - const exists = existsSync(response.body.files[0].data); - assert(!exists); - }); + .attach('file', imagePath); + + expect(response.status).toBe(200); + expect(response.body.files.length).toBe(1); + expect(response.body.files[0].filename).toBe('1.jpg'); + expect(response.body.fields.name).toBe('form'); + const file1Stat = statSync(response.body.files[0].data); + expect(file1Stat.size).toBe(stat.size); + + await sleep(2000); + + const exists = existsSync(response.body.files[0].data); + expect(exists).toBe(false); await close(app); }); }); diff --git a/packages/busboy/test/express.test.ts b/packages/busboy/test/express.test.ts index 77ffc215f164..ba12ee894f90 100644 --- a/packages/busboy/test/express.test.ts +++ b/packages/busboy/test/express.test.ts @@ -61,22 +61,19 @@ describe('test/express.test.ts', function () { const pdfPath = join(__dirname, 'fixtures/test.pdf'); const request = await createHttpRequest(app); const stat = statSync(pdfPath); - await request.post('/upload') + const response = await request.post('/upload') .field('name', 'form') .field('name2', 'form2') .attach('file', pdfPath) - .attach('file2', pdfPath) - .expect(200) - .then(async response => { - assert(response.body.files.length === 2); - // assert(response.body.files[0].fieldName === 'file'); - // assert(response.body.files[1].fieldName === 'file2'); - assert(response.body.files[1].mimeType === 'application/pdf'); - assert(response.body.fields.name === 'form'); - assert(response.body.fields.name2 === 'form2'); - const file1Stat = statSync(response.body.files[0].data); - assert(file1Stat.size && file1Stat.size === stat.size); - }); + .attach('file2', pdfPath); + + expect(response.status).toBe(200); + expect(response.body.files.length).toBe(2); + expect(response.body.files[1].mimeType).toBe('application/pdf'); + expect(response.body.fields.name).toBe('form'); + expect(response.body.fields.name2).toBe('form2'); + const file1Stat = statSync(response.body.files[0].data); + expect(file1Stat.size).toBe(stat.size); }); it('upload file ignore path', async () => { diff --git a/packages/busboy/test/fixtures/koa-file/src/configuration.ts b/packages/busboy/test/fixtures/koa-file/src/configuration.ts index c0de13e87e1a..3a0d6e317a6e 100644 --- a/packages/busboy/test/fixtures/koa-file/src/configuration.ts +++ b/packages/busboy/test/fixtures/koa-file/src/configuration.ts @@ -18,6 +18,13 @@ import * as upload from '../../../../src'; whitelist: uploadWhiteList.filter(ext => { return ext !== '.gz'; }).concat('.tar.gz') + }, + midwayLogger: { + clients: { + appLogger: { + level: 'debug', + } + } } } } @@ -41,6 +48,7 @@ export class HomeController { @Post('/upload') async upload(@Fields() fields, @Files() files: UploadFileInfo[]) { const stat = statSync(files[0].data); + return { size: stat.size, files, diff --git a/packages/busboy/test/fixtures/koa-stream/src/configuration.ts b/packages/busboy/test/fixtures/koa-stream/src/configuration.ts index 82ee55ebff11..55487e7b0139 100644 --- a/packages/busboy/test/fixtures/koa-stream/src/configuration.ts +++ b/packages/busboy/test/fixtures/koa-stream/src/configuration.ts @@ -1,4 +1,4 @@ -import { Configuration, Controller, Fields, Inject, Post, File, Get, App } from '@midwayjs/core'; +import { Configuration, Controller, Fields, Inject, Post, Get, App, Files } from "@midwayjs/core"; import * as koa from '@midwayjs/koa'; import { createWriteStream, statSync } from 'fs' import { join } from 'path'; @@ -40,21 +40,26 @@ export class HomeController { return 'home' } @Post('/upload') - async upload(@File() file: upload.UploadStreamFileInfo, @Fields() fields) { - const path = join(__dirname, '../logs/test.pdf'); - const stream = createWriteStream(path); - const end = new Promise(resolve => { - stream.on('close', () => { - resolve(void 0) + async upload(@Files() files: Array, @Fields() fields) { + for(let file of files) { + const path = join(__dirname, `../logs/${file.fieldName}.pdf`); + const stream = createWriteStream(path); + const end = new Promise(resolve => { + stream.on('close', () => { + resolve(void 0) + }); }); - }); - file.data.pipe(stream); - await end; - const stat = statSync(path); + + file.data.pipe(stream); + await end; + } + + const stat = statSync(join(__dirname, `../logs/${files[0].fieldName}.pdf`)); return { size: stat.size, - files: [file], - fields + files, + fields, + fieldName: files[0].fieldName, } } } diff --git a/packages/busboy/test/index.test.ts b/packages/busboy/test/index.test.ts index d3605b67eb8e..f02f931c01ce 100644 --- a/packages/busboy/test/index.test.ts +++ b/packages/busboy/test/index.test.ts @@ -18,6 +18,7 @@ describe('/test/index.test.ts', () => { const fields = ctx.fields; const fileName = join(tmpdir(), Date.now() + '_' + files[0].filename); const fsWriteStream = createWriteStream(fileName); + const fieldName = files[0].fieldName; await new Promise(resolve => { fsWriteStream.on('close', resolve); @@ -28,7 +29,8 @@ describe('/test/index.test.ts', () => { return { size: stat.size, files, - fields + fields, + fieldName, } } } @@ -57,7 +59,7 @@ describe('/test/index.test.ts', () => { for (let i = 0; i < 10; i++) { await request.post('/upload') .field('name', 'form') - .attach('file', filePath) + .attach('file12', filePath) .expect(200) .then(async response => { const stat = statSync(filePath); @@ -65,6 +67,7 @@ describe('/test/index.test.ts', () => { assert(response.body.files.length === 1); assert(response.body.files[0].filename === 'default.txt'); assert(response.body.fields.name === 'form'); + assert(response.body.fieldName === 'file12'); }); } @@ -115,7 +118,13 @@ describe('/test/index.test.ts', () => { class APIController { @Post('/upload', { middleware: [UploadMiddleware]}) async upload(ctx) { - // TODO + // throw error + // const fs = createWriteStream(join(tmpdir(), ctx.files[0].filename)); + // ctx.files[0].data.pipe(fs); + // + // await new Promise(resolve => { + // fs.on('finish', resolve); + // }); } } @@ -198,12 +207,12 @@ describe('/test/index.test.ts', () => { const request = await createHttpRequest(app); await request.post('/upload') .field('name', 'form') - .attach('file', filePath) + .attach('file1', filePath) .expect(200); await request.post('/upload1') .field('name', 'form') - .attach('file', filePath) + .attach('file2', filePath) .expect(200); await close(app); diff --git a/packages/busboy/test/koa.test.ts b/packages/busboy/test/koa.test.ts index 43a3f8a79275..1a4249b7e177 100644 --- a/packages/busboy/test/koa.test.ts +++ b/packages/busboy/test/koa.test.ts @@ -1,7 +1,13 @@ -import { createHttpRequest, close, createApp } from '@midwayjs/mock'; +import { createHttpRequest, close, createApp, createLightApp } from "@midwayjs/mock"; import { join } from 'path'; +import { createWriteStream, existsSync, statSync } from "fs"; +import * as koa from '@midwayjs/koa'; +import * as busboy from '../src'; +import { Controller, createMiddleware, Fields, Files, Post, File } from "@midwayjs/core"; +import { ensureDir, removeSync } from "fs-extra"; import * as assert from 'assert'; -import { statSync } from 'fs'; +import { tmpdir } from "os"; +import { randomUUID } from "node:crypto"; describe('test/koa.test.ts', function () { @@ -18,21 +24,20 @@ describe('test/koa.test.ts', function () { it('upload stream mode', async () => { const pdfPath = join(__dirname, 'fixtures/test.pdf'); const request = createHttpRequest(app); - await request.post('/upload') + const response = await request.post('/upload') .field('name', 'form') .field('name2', 'form2') - .attach('file', pdfPath) - .attach('file2', pdfPath) - .expect(200) - .then(async response => { - const stat = statSync(pdfPath); - assert(response.body.size === stat.size); - assert(response.body.files.length === 1); - assert(response.body.files[0].filename === 'test.pdf'); - assert(response.body.fields.name === 'form'); - assert(response.body.fields.name2 === 'form2'); - }).catch(err => console.log) - }) + .attach('file123', pdfPath) + .expect(200); + + const stat = statSync(pdfPath); + expect(response.body.size).toBe(stat.size); + expect(response.body.files.length).toBe(1); + expect(response.body.files[0].filename).toBe('test.pdf'); + expect(response.body.fields.name).toBe('form'); + expect(response.body.fields.name2).toBe('form2'); + expect(response.body.fieldName).toBe('file123'); + }); it('upload stream mode 3kb', async () => { // fix: issue#2309 @@ -45,9 +50,9 @@ describe('test/koa.test.ts', function () { .expect(200) .then(async response => { const stat = statSync(pdfPath); - assert(response.body.size === stat.size); - assert(response.body.files.length === 1); - assert(response.body.files[0].filename === '3kb.png'); + expect(response.body.size).toBe(stat.size); + expect(response.body.files.length).toBe(1); + expect(response.body.files[0].filename).toBe('3kb.png'); }); }); @@ -76,20 +81,19 @@ describe('test/koa.test.ts', function () { it('upload file mode', async () => { const pdfPath = join(__dirname, 'fixtures/test.pdf'); const request = createHttpRequest(app); - await request.post('/upload') + const response = await request.post('/upload') .field('name', 'form') .field('name2', 'form2') .attach('file', pdfPath) .attach('file2', pdfPath) - .expect(200) - .then(async response => { - assert(response.body.files.length === 2); - // assert(response.body.files[0].fieldName === 'file'); - // assert(response.body.files[1].fieldName === 'file2'); - assert(response.body.files[1].mimeType === 'application/pdf'); - assert(response.body.fields.name === 'form'); - assert(response.body.fields.name2 === 'form2'); - }); + .expect(200); + + expect(response.body.files.length).toBe(2); + expect(response.body.files[0].fieldName).toBe('file'); + expect(response.body.files[1].fieldName).toBe('file2'); + expect(response.body.files[1].mimeType).toBe('application/pdf'); + expect(response.body.fields.name).toBe('form'); + expect(response.body.fields.name2).toBe('form2'); }); it('upload file type .tar.gz', async () => { @@ -102,8 +106,8 @@ describe('test/koa.test.ts', function () { .expect(200) .then(async response => { const stat = statSync(path); - assert(response.body.size === stat.size); - assert(response.body.files[0].data.endsWith('.tar.gz')); + expect(response.body.size).toBe(stat.size); + expect(response.body.files[0].data.endsWith('.tar.gz')).toBe(true); }); }); @@ -207,11 +211,417 @@ describe('test/koa.test.ts', function () { .attach('file', filePath) .expect(200) .then(async response => { - assert(response.body.files.length === 1); - assert(response.body.files[0].filename === 'test.pdf'); - assert(response.body.fields.name === 'form'); - assert(response.body.fields.name2 === 'form2'); + expect(response.body.files.length).toBe(1); + expect(response.body.files[0].filename).toBe('test.pdf'); + expect(response.body.fields.name).toBe('form'); + expect(response.body.fields.name2).toBe('form2'); }); }); }); + + describe('koa iterator', () => { + + let resourceDir: string; + + beforeEach(() => { + resourceDir = join(tmpdir(), randomUUID()); + if (existsSync(resourceDir)) { + removeSync(resourceDir); + } + ensureDir(resourceDir); + }) + + it('upload stream mode and multi file', async () => { + @Controller('/') + class HomeController { + @Post('/upload-multi', { middleware: [ createMiddleware(busboy.UploadMiddleware, { mode: 'asyncIterator' }) ] }) + async uploadMore( + @Files() fileIterator: AsyncGenerator, + @Fields() fieldIterator: AsyncGenerator, + @File() singleFileIterator: AsyncGenerator) + { + assert(singleFileIterator === fileIterator); + const files = [], fields = []; + for await (const { data, fieldName, filename } of fileIterator) { + const path = join(resourceDir, `${fieldName}.pdf`); + const stream = createWriteStream(path); + const end = new Promise(resolve => { + stream.on('close', () => { + resolve(void 0) + }); + }); + + data.pipe(stream); + await end; + + files.push({ + fieldName, + filename, + }); + } + + for await (const field of fieldIterator) { + fields.push(field); + } + + const stat = statSync(join(resourceDir, `${files[0].fieldName}.pdf`)); + return { + size: stat.size, + files, + fields + } + } + } + const app = await createLightApp({ + imports: [ + koa, + busboy, + HomeController + ], + globalConfig: { + keys: '123', + busboy: {} + }, + }); + + const pdfPath = join(__dirname, 'fixtures/test.pdf'); + const request = createHttpRequest(app); + const response = await request.post('/upload-multi') + .field('name', 'form') + .field('name2', 'form2') + .attach('file123', pdfPath) + .attach('file2', pdfPath) + .expect(200); + + const stat = statSync(pdfPath); + expect(response.body.size).toBe(stat.size); + expect(response.body.files.length).toBe(2); + expect(response.body.files[0].filename).toBe('test.pdf'); + expect(response.body.files[0].fieldName).toBe('file123'); + expect(response.body.files[1].filename).toBe('test.pdf'); + expect(response.body.files[1].fieldName).toBe('file2'); + expect(response.body.fields[0].value).toBe('form'); + expect(response.body.fields[1].value).toBe('form2'); + + await close(app); + }); + + it('upload stream mode and multi file with read fields', async () => { + @Controller('/') + class HomeController { + @Post('/upload-multi', { middleware: [ createMiddleware(busboy.UploadMiddleware, { mode: 'asyncIterator' }) ] }) + async uploadMore(@Files() fileIterator: AsyncGenerator) { + const files = []; + for await (const file of fileIterator) { + const path = join(resourceDir, `${file.fieldName}.pdf`); + const stream = createWriteStream(path); + const end = new Promise(resolve => { + stream.on('close', () => { + resolve(void 0) + }); + }); + + file.data.pipe(stream); + await end; + + files.push(file); + } + + const stat = statSync(join(resourceDir, `${files[0].fieldName}.pdf`)); + return { + size: stat.size, + files, + } + } + } + const app = await createLightApp({ + imports: [ + koa, + busboy, + HomeController + ], + globalConfig: { + keys: '123', + busboy: {} + }, + }); + + const pdfPath = join(__dirname, 'fixtures/test.pdf'); + const request = createHttpRequest(app); + const response = await request.post('/upload-multi') + .field('name', 'form') + .field('name2', 'form2') + .attach('file123', pdfPath) + .attach('file2', pdfPath) + .expect(200); + + const stat = statSync(pdfPath); + expect(response.body.size).toBe(stat.size); + expect(response.body.files.length).toBe(2); + expect(response.body.files[0].filename).toBe('test.pdf'); + expect(response.body.files[0].fieldName).toBe('file123'); + expect(response.body.files[1].filename).toBe('test.pdf'); + expect(response.body.files[1].fieldName).toBe('file2'); + + await close(app); + }); + + it('upload stream mode and multi file and trigger limit error', async () => { + @Controller('/') + class HomeController { + @Post("/upload-multi", { + middleware: [createMiddleware(busboy.UploadMiddleware, { + mode: "asyncIterator", + limits: { + fileSize: 1 + } + })] + }) + async uploadMore(@Files() fileIterator: AsyncGenerator) { + const files = []; + for await (const file of fileIterator) { + const path = join(resourceDir, `${file.fieldName}.pdf`); + const stream = createWriteStream(path); + const end = new Promise(resolve => { + stream.on('close', () => { + resolve(void 0) + }); + }); + + file.data.pipe(stream); + await end; + files.push(file); + } + + const stat = statSync(join(resourceDir, `${files[0].fieldName}.pdf`)); + return { + size: stat.size, + files, + } + } + } + const app = await createLightApp({ + imports: [ + koa, + busboy, + HomeController + ], + globalConfig: { + keys: '123', + busboy: {} + }, + }); + + const pdfPath = join(__dirname, 'fixtures/test.pdf'); + const request = createHttpRequest(app); + const response = await request.post('/upload-multi') + .field('name', 'form') + .field('name2', 'form2') + .attach('file123', pdfPath) + .attach('file2', pdfPath); + + expect(response.status).toBe(400); + await close(app); + }); + + it('upload stream mode trigger limit error and catch it', async () => { + @Controller('/') + class HomeController { + @Post("/upload-multi", { + middleware: [createMiddleware(busboy.UploadMiddleware, { + mode: "asyncIterator", + limits: { + fileSize: 1 + } + })] + }) + async uploadMore(@Files() fileIterator: AsyncGenerator) { + const files = []; + try { + for await (const file of fileIterator) { + const path = join(resourceDir, `${file.fieldName}.pdf`); + const stream = createWriteStream(path); + const end = new Promise(resolve => { + stream.on('close', () => { + resolve(void 0) + }); + }); + + file.data.pipe(stream); + await end; + files.push(file); + } + } catch (err) { + console.error(err.message); + } + return 'ok'; + } + } + const app = await createLightApp({ + imports: [ + koa, + busboy, + HomeController + ], + globalConfig: { + keys: '123', + busboy: {} + }, + }); + + const pdfPath = join(__dirname, 'fixtures/test.pdf'); + const request = createHttpRequest(app); + const response = await request.post('/upload-multi') + .field('name', 'form') + .field('name2', 'form2') + .attach('file123', pdfPath) + .attach('file2', pdfPath); + + expect(response.status).toBe(200); + await close(app); + }); + + it.skip("should got 204 when iterator not use", async () => { + @Controller("/") + class HomeController { + @Post("/upload-multi", { + middleware: [createMiddleware(busboy.UploadMiddleware, { + mode: "asyncIterator", + limits: {} + })] + }) + async uploadMore(@Files() fileIterator: AsyncGenerator) { + // 这里如果不处理迭代器，会出现 unhandled error + } + } + + const app = await createLightApp({ + imports: [ + koa, + busboy, + HomeController + ], + globalConfig: { + keys: "123", + busboy: {} + } + }); + + const txtPath = join(__dirname, "fixtures/1.test"); + const request = createHttpRequest(app); + const response = await request.post("/upload-multi") + .attach("file", txtPath); + + expect(response.status).toBe(204); + await close(app); + }); + + it("should check type", async () => { + @Controller("/") + class HomeController { + @Post("/upload-multi", { + middleware: [createMiddleware(busboy.UploadMiddleware, { + mode: "asyncIterator", + limits: {} + })] + }) + async uploadMore(@Files() fileIterator: AsyncGenerator) { + const files = []; + for await (const file of fileIterator) { + const path = join(resourceDir, `${file.fieldName}.pdf`); + const stream = createWriteStream(path); + const end = new Promise(resolve => { + stream.on("close", () => { + resolve(void 0); + }); + }); + + file.data.pipe(stream); + await end; + files.push(file); + } + } + } + + const app = await createLightApp({ + imports: [ + koa, + busboy, + HomeController + ], + globalConfig: { + keys: "123", + busboy: {} + } + }); + + const txtPath = join(__dirname, 'fixtures/1.test'); + const request = createHttpRequest(app); + const response = await request.post('/upload-multi') + .attach('file', txtPath); + + expect(response.status).toBe(400); + await close(app); + }); + + it("should check size of result", async () => { + @Controller("/") + class HomeController { + @Post("/upload", { + middleware: [createMiddleware(busboy.UploadMiddleware, { + mode: "file" + })] + }) + async uploadFile(@File() file) { + return statSync(file.data).size; + } + + @Post("/upload-multi", { + middleware: [createMiddleware(busboy.UploadMiddleware, { + mode: "asyncIterator" + })] + }) + async uploadMore(@Files() fileIterator: AsyncGenerator) { + const files = []; + for await (const file of fileIterator) { + const path = join(resourceDir, `${file.fieldName}.pdf`); + const stream = createWriteStream(path); + const end = new Promise(resolve => { + stream.on("close", () => { + resolve(void 0); + }); + }); + + file.data.pipe(stream); + await end; + files.push(statSync(path).size); + } + + return files[0]; + } + } + const app = await createLightApp({ + imports: [ + koa, + busboy, + HomeController + ], + globalConfig: { + keys: '123', + busboy: { + whitelist: ['.test'] + } + }, + }); + + const txtPath = join(__dirname, 'fixtures/1.test'); + const request = createHttpRequest(app); + const response = await request.post('/upload-multi') + .attach('file', txtPath); + + expect(response.status).toBe(200); + expect(response.body).toBe(6); + await close(app); + }); + }); }); diff --git a/packages/core/src/util/webRouterParam.ts b/packages/core/src/util/webRouterParam.ts index 7fd92ca51ca1..9550356e3dd1 100644 --- a/packages/core/src/util/webRouterParam.ts +++ b/packages/core/src/util/webRouterParam.ts @@ -39,7 +39,11 @@ export const extractKoaLikeValue = (key, data, paramType?) => { if (ctx.getFileStream) { return ctx.getFileStream(data); } else if (ctx.files) { - return ctx.files[0]; + if (Array.isArray(ctx.files)) { + return ctx.files[0]; + } else { + return ctx.files; + } } else { return undefined; } diff --git a/site/blog/2024-08-29-release-3.17.md b/site/blog/2024-08-29-release-3.17.md index 4bbedb3ff20b..be25a81748e2 100644 --- a/site/blog/2024-08-29-release-3.17.md +++ b/site/blog/2024-08-29-release-3.17.md @@ -95,7 +95,6 @@ return new HttpServerResponse(this.ctx).fail().json('hello world'); * 1、不再默认加载中间件，因为上传只是少部分接口的特殊逻辑，不需要全局加载 * 2、配置的 key 为了避免冲突，从 `upload` 变为 `busboy` -* 3、原有上传的数据中的 `filedName`，在流式模式下不再提供 其余的使用方式和原有的上传组件一致， diff --git a/site/docs/extensions/busboy.md b/site/docs/extensions/busboy.md index d36a9a32d9c1..d26eeabfe3ea 100644 --- a/site/docs/extensions/busboy.md +++ b/site/docs/extensions/busboy.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; * 1、配置的 key 从 `upload` 调整为 `busboy` * 2、中间件不再默认加载，手动可配置到全局或者路由 -* 3、流式上传时不再提供 fieldName 字段，入参定义类型调整为 `UploadStreamFileInfo` +* 3、入参定义类型调整为 `UploadStreamFileInfo` * 4、`fileSize` 的配置有调整 ::: @@ -198,11 +198,18 @@ export class MainConfiguration { 组件使用 `busboy` 作为配置的 key。 -### 上传模式 - file -`file` 为默认值，也是框架的推荐值。 -配置 mode 为 `file` 字符串。 +### 上传模式 + +上传分为三种模式，文件模式，流式模式以及新增的异步迭代器模式。 + +代码中使用 `@Files()` 装饰器获取上传的文件， `@Fields` 装饰器获取其他上传表单字段。 + + + + +`file` 为默认值，配置 mode 为 `file` 字符串。 ```typescript // src/config/config.default.ts @@ -212,10 +219,9 @@ export default { mode: 'file', }, } - ``` -在代码中获取上传的文件。 +在代码中获取上传的文件，支持同时上传多个文件。 ```typescript import { Controller, Post, Files, Fields } from '@midwayjs/core'; @@ -225,61 +231,178 @@ import { UploadFileInfo } from '@midwayjs/busboy'; export class HomeController { @Post('/upload', /*...*/) - async upload(@Files() files: Array, @Fields() fields: Record, @Fields() fields: Record) { /* files = [ { filename: 'test.pdf', // 文件原名 data: '/var/tmp/xxx.pdf', // 服务器临时文件地址 mimeType: 'application/pdf', // mime + fieldName: 'file' // field name }, - // ...file 下支持同时上传多个文件 ] */ - return { - files, - fields - } } } ``` -使用 file 模式时，通过 `this.ctx.files` 中获取的 `data` 为上传的文件在服务器的 `临时文件地址`，后续可以再通过 `fs.createReadStream` 等方式来处理此文件内容。 +使用 file 模式时， 获取的 `data` 为上传的文件在服务器的 `临时文件地址`，后续可以再通过 `fs.createReadStream` 等方式来处理此文件内容，支持同时上传多个文件，多个文件会以数组的形式存放。 -使用 file 模式时，支持同时上传多个文件，多个文件会以数组的形式存放在 `this.ctx.files` 中。 +每个数组内的对象包含以下几个字段 +```typescript +export interface UploadFileInfo { + /** + * 上传的文件名 + */ + filename: string; + /** + * 上传文件 mime 类型 + */ + mimeType: string; + /** + * 上传服务端保存的路径 + */ + data: string; + /** + * 上传的表单字段名 + */ + fieldName: string; +} +``` -:::caution -当采取 `file` 模式时，由于上传组件会在接收到请求时，会根据请求的 `method` 和 `headers` 中的部分标志性内容进行匹配，如果认为是一个文件上传请求，就会对请求进行解析，将其中的文件 `写入` 到服务器的临时缓存目录，您可以通过本组件的 `match` 或 `ignore` 配置来设置允许解析文件的路径。 + -配置 `match` 或 `ignore`后，则可以保证您的普通 post 等请求接口，不会被用户非法用作上传，可以 `避免` 服务器缓存被充满的风险。 + - 您可以查看下面的 `配置 允许(match) 或 忽略(ignore)的上传路径` 章节，来进行配置。 +从 `v3.18.0` 提供，替代原有的 `stream` 模式，该模式支持多个文件流式上传。 -::: +配置 upload 的 mode 为 `asyncIterator` 字符串。 +```typescript +// src/config/config.default.ts +export default { + // ... + busboy: { + mode: 'asyncIterator', + }, +} +``` +在代码中获取上传的文件。 -:::caution +```typescript +import { Controller, Post, Files, Fields } from '@midwayjs/core'; +import { UploadStreamFileInfo, UploadStreamFieldInfo } from '@midwayjs/busboy'; -如果同时开启了 swagger 组件，请务必添加上传参数的类型（装饰器对应的类型，以及 @ApiBody 中的 type），否则会报错，更多请参考 swagger 的文件上传章节。 +@Controller('/') +export class HomeController { -::: + @Post('/upload', /*...*/) + async upload( + @Files() fileIterator: AsyncGenerator, + @Fields() fieldIterator: AsyncGenerator + ) { + // ... + } +} +``` +在该模式下，`@Files` 和 `@File` 装饰器会提供同一个 `AsyncGenerator` ，而 `@Fields` 会也同样会提供一个 `AsyncGenerator`。 +通过循环 `AsyncGenerator` ，可以针对每个上传文件的 `ReadStream` 做处理。 +```typescript +import { Controller, Post, Files, Fields } from '@midwayjs/core'; +import { UploadStreamFileInfo, UploadStreamFieldInfo } from '@midwayjs/busboy'; +import { tmpdir } from 'os'; +import { createWriteStream } from 'fs'; -### 上传模式 - stream +@Controller('/') +export class HomeController { -配置 upload 的 mode 为 `stream` 字符串。 + @Post('/upload', /*...*/) + async upload( + @Files() fileIterator: AsyncGenerator, + @Fields() fieldIterator: AsyncGenerator + ) { + for await (const file of fileIterator) { + const { filename, data } = file; + const p = join(tmpdir, filename); + const stream = createWriteStream(p); + data.pipe(stream); + } + + for await (const { name, value } of fieldIterator) { + // ... + } + + // ... + } +} +``` + +注意，如果一次上传中任意一个文件抛出了错误，本次上传流会直接关闭，所有未传输完成的文件都会异常。 + +异步迭代器中的上传对象包含以下几个字段。 +```typescript +export interface UploadStreamFieldInfo { + /** + * 上传的文件名 + */ + filename: string; + /** + * 上传文件 mime 类型 + */ + mimeType: string; + /** + * 上传文件的文件流 + */ + data: Readable; + /** + * 上传的表单字段名 + */ + fieldName: string; +} +``` + +异步迭代器中的 `@Fields` 的对象略有不同，返回的数据会包含 `name` 和 `value` 字段。 + +```typescript +export interface UploadStreamFieldInfo { + /** + * 表单名 + */ + name: string; + /** + * 表单值 + */ + value: any; +} +``` -使用 stream 模式时，通过 `this.ctx.files` 中获取的 `data` 为 `ReadStream`，后续可以再通过 `pipe` 等方式继续将数据流转至其他 `WriteStream` 或 `TransformStream`。 -使用 stream 模式时，仅同时上传一个文件，即 `this.ctx.files` 数组中只有一个文件数据对象。 + + + + +:::caution + +不再推荐使用。 + +::: + +配置 mode 为 `stream` 字符串。 + + +使用 stream 模式时，通过 `@Files` 中获取的 `data` 为 `ReadStream`，后续可以再通过 `pipe` 等方式继续将数据流转至其他 `WriteStream` 或 `TransformStream`。 + + +使用 stream 模式时，仅同时上传一个文件，即 `@Files` 数组中只有一个文件数据对象。 另外，stream 模式 `不会` 在服务器上产生临时文件，所以获取到上传的内容后无需手动清理临时文件缓存。 @@ -304,25 +427,43 @@ export class HomeController { files = [ { filename: 'test.pdf', // 文件原名 - data: ReadStream, // 文件流 + data: ReadStream, // 文件流 mimeType: 'application/pdf', // mime + fieldName: 'file' // field name }, ] */ - return { - files, - fields - } } } ``` -:::caution +流式模式中的对象包含以下几个字段。 -如果同时开启了 swagger 组件，请务必添加上传参数的类型（装饰器对应的类型，以及 @ApiBody 中的 type），否则会报错，更多请参考 swagger 的文件上传章节。 +```typescript +export interface UploadStreamFieldInfo { + /** + * 上传的文件名 + */ + filename: string; + /** + * 上传文件 mime 类型 + */ + mimeType: string; + /** + * 上传文件的文件流 + */ + data: Readable; + /** + * 上传的表单字段名 + */ + fieldName: string; +} +``` -::: + + + diff --git a/site/i18n/en/docusaurus-plugin-content-docs/current/extensions/busboy.md b/site/i18n/en/docusaurus-plugin-content-docs/current/extensions/busboy.md index 6a335be9004e..6a00fb0dd4b8 100644 --- a/site/i18n/en/docusaurus-plugin-content-docs/current/extensions/busboy.md +++ b/site/i18n/en/docusaurus-plugin-content-docs/current/extensions/busboy.md @@ -30,7 +30,7 @@ The differences from the upload component are: * 2. The middleware is no longer loaded by default, and can be manually configured to the global or route -* 3. The fieldName field is no longer provided for streaming upload, and the input parameter definition type is adjusted to `UploadStreamFileInfo` +* 3. the input parameter definition type is adjusted to `UploadStreamFileInfo` * 4. The configuration of `fileSize` has been adjusted @@ -197,11 +197,19 @@ export class MainConfiguration { The component uses `busboy` as the configuration key. -### Upload mode - file +### Upload mode -`file` is the default value and the recommended value of the framework. +### Upload Modes -Configure mode as a `file` string. +There are three upload modes: file mode, stream mode, and the newly added async iterator mode. + +In the code, the `@Files()` decorator is used to obtain the uploaded files, and the `@Fields` decorator is used to get other upload form fields. + + + + + +`file` is the default value, with `mode` configured as the string `file`. ```typescript // src/config/config.default.ts @@ -214,7 +222,8 @@ export default { ``` -Get the uploaded file in the code. +In the code, the uploaded files can be retrieved, and multiple files can be uploaded simultaneously. + ```typescript import { Controller, Post, Files, Fields } from '@midwayjs/core'; @@ -224,58 +233,173 @@ import { UploadFileInfo } from '@midwayjs/busboy'; export class HomeController { @Post('/upload', /*...*/) - async upload(@Files() files: Array, @Fields() fields: Record, @Fields() fields: Record) { /* files = [ { filename: 'test.pdf', // file name data: '/var/tmp/xxx.pdf', // Server temporary file address mimeType: 'application/pdf', // mime + fieldName: 'file' // field name }, // ...Support uploading multiple files at the same time under file ] */ - return { - files, - fields - } } } ``` -When using file mode, the `data` obtained from `this.ctx.files` is the `temporary file address` of the uploaded file on the server. The content of this file can be processed later by `fs.createReadStream` and other methods. +When using the file mode, the retrieved `data` represents the `temporary file path` of the uploaded file on the server. You can later handle the file contents using methods like `fs.createReadStream`. Multiple files can be uploaded at once, and they will be stored in an array. -When using file mode, it supports uploading multiple files at the same time, and multiple files will be stored in `this.ctx.files` in the form of an array. +Each object in the array contains the following fields: +```typescript +export interface UploadFileInfo { + /** + * The name of the uploaded file + */ + filename: string; + /** + * The MIME type of the uploaded file + */ + mimeType: string; + /** + * The path where the file is saved on the server + */ + data: string; + /** + * The form field name of the uploaded file + */ + fieldName: string; +} +``` + -:::caution + -When using the `file` mode, the upload component will match the `method` of the request and some of the iconic contents in the `headers` when receiving the request. If it is a file upload request, it will parse the request and `write` the file in it to the temporary cache directory of the server. You can set the path that allows parsing files through the `match` or `ignore` configuration of this component. +Available since `v3.18.0`, this mode replaces the previous `stream` mode and supports streaming uploads of multiple files. -After configuring `match` or `ignore`, you can ensure that your ordinary post and other request interfaces will not be illegally used by users for upload, and you can `avoid` the risk of the server cache being full. +Configure the upload `mode` as the string `asyncIterator`. -You can view the `Configure the upload path allowed (match) or ignored (ignore)` section below to configure it. +```typescript +// src/config/config.default.ts +export default { + // ... + busboy: { + mode: 'asyncIterator', + }, +} +``` -::: +Retrieve the uploaded files in the code. +```typescript +import { Controller, Post, Files, Fields } from '@midwayjs/core'; +import { UploadStreamFileInfo, UploadStreamFieldInfo } from '@midwayjs/busboy'; +@Controller('/') +export class HomeController { -:::caution + @Post('/upload', /*...*/) + async upload( + @Files() fileIterator: AsyncGenerator, + @Fields() fieldIterator: AsyncGenerator + ) { + // ... + } +} +``` -If the swagger component is enabled at the same time, be sure to add the type of the upload parameter (the type corresponding to the decorator and the type in @ApiBody), otherwise an error will be reported. For more information, please refer to the file upload section of swagger. +In this mode, both `@Files` and `@File` decorators provide the same `AsyncGenerator`, and `@Fields` also provides an `AsyncGenerator`. -::: +By looping through the `AsyncGenerator`, you can handle the `ReadStream` of each uploaded file. +```typescript +import { Controller, Post, Files, Fields } from '@midwayjs/core'; +import { UploadStreamFileInfo, UploadStreamFieldInfo } from '@midwayjs/busboy'; +import { tmpdir } from 'os'; +import { createWriteStream } from 'fs'; +@Controller('/') +export class HomeController { -### Upload mode - stream + @Post('/upload', /*...*/) + async upload( + @Files() fileIterator: AsyncGenerator, + @Fields() fieldIterator: AsyncGenerator + ) { + for await (const file of fileIterator) { + const { filename, data } = file; + const p = join(tmpdir, filename); + const stream = createWriteStream(p); + data.pipe(stream); + } + + for await (const { name, value } of fieldIterator) { + // ... + } + + // ... + } +} +``` + +Note that if any file throws an error during the upload process, the entire upload stream will close, and all incomplete file uploads will fail. + +The upload object in the async iterator contains the following fields. + +```typescript +export interface UploadStreamFieldInfo { + /** + * The name of the uploaded file + */ + filename: string; + /** + * The MIME type of the uploaded file + */ + mimeType: string; + /** + * The file stream of the uploaded file + */ + data: Readable; + /** + * The form field name of the uploaded file + */ + fieldName: string; +} +``` + +The object for `@Fields` in the async iterator is slightly different, with the returned data containing `name` and `value` fields. + +```typescript +export interface UploadStreamFieldInfo { + /** + * Form name + */ + name: string; + /** + * Form value + */ + value: any; +} +``` + + + + + +:::caution + +No longer recommended for use. + +::: -Configure the upload mode to be a `stream` string. +Configure `mode` as the string `stream`. -When using stream mode, the `data` obtained from `this.ctx.files` is a `ReadStream`, and the data can be transferred to other `WriteStream` or `TransformStream` through `pipe` and other methods. +When using stream mode, the `data` obtained from `@Files` is a `ReadStream`, and the data can be transferred to other `WriteStream` or `TransformStream` through `pipe` and other methods. -When using stream mode, only one file is uploaded at a time, that is, there is only one file data object in the `this.ctx.files` array. +When using stream mode, only one file is uploaded at a time, that is, there is only one file data object in the `@Files` array. In addition, stream mode `does not` generate temporary files on the server, so there is no need to manually clean up the temporary file cache after obtaining the uploaded content. @@ -302,23 +426,41 @@ export class HomeController { filename: 'test.pdf', // file name data: ReadStream, // file stream mimeType: 'application/pdf', // mime + fieldName: 'file' // field name }, ] */ - return { - files, - fields - } } } ``` -:::caution +The object in stream mode contains the following fields: -If the swagger component is enabled at the same time, be sure to add the type of the upload parameter (the type corresponding to the decorator and the type in @ApiBody), otherwise an error will be reported. For more information, please refer to the file upload section of swagger. +```typescript +export interface UploadStreamFieldInfo { + /** + * The name of the uploaded file + */ + filename: string; + /** + * The MIME type of the uploaded file + */ + mimeType: string; + /** + * The file stream of the uploaded file + */ + data: Readable; + /** + * The form field name of the uploaded file + */ + fieldName: string; +} +``` -::: + + +