diff --git a/app-config.yaml b/app-config.yaml index 914af624..572032d5 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -1,5 +1,6 @@ httpApi: host: 0.0.0.0 + fileSizeLimit: 1000000 port: 1337 allowedOrigins: '*' cookieSecret: 'secret' @@ -27,6 +28,11 @@ logging: appServer: info database: info +s3: + accessKeyId: 'secret' + secretAccessKey: 'secret' + endpoint: 'http://localhost:9000' + database: dsn: 'postgres://codex:postgres@postgres:5432/notes' diff --git a/migrations/tenant/0028-files@add-column-size.sql b/migrations/tenant/0028-files@add-column-size.sql new file mode 100644 index 00000000..836d2255 --- /dev/null +++ b/migrations/tenant/0028-files@add-column-size.sql @@ -0,0 +1,72 @@ +DO $$ +BEGIN + -- Add sixe column if it not exists + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'files' + AND column_name = 'size' + ) + THEN + ALTER TABLE files ADD COLUMN size integer NOT NULL; + END IF; + + -- Make id column autoincrementing + CREATE SEQUENCE IF NOT EXISTS public.files_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- Make identifier default value incrementing + ALTER TABLE ONLY public.files ALTER COLUMN id SET DEFAULT nextval('public.files_id_seq'::regclass); + + -- Remove column note_id + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'files' + AND column_name = 'note_id' + ) THEN + -- Drop the column if it exists + ALTER TABLE files + DROP COLUMN note_id; + END IF; + + -- Add column location + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'files' + AND column_name = 'location' + ) THEN + ALTER TABLE files + ADD COLUMN location JSONB; + END IF; + + -- Remove column user_id + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'files' + AND column_name = 'user_id' + ) THEN + -- Drop the column if it exists + ALTER TABLE files + DROP COLUMN user_id; + END IF; + + -- Add column metadata + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'files' + AND column_name = 'metadata' + ) THEN + ALTER TABLE files + ADD COLUMN metadata JSONB; + END IF; +END $$; diff --git a/package.json b/package.json index 3a4b881c..6617dea3 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "@codex-team/config-loader": "^1.0.0", "@fastify/cookie": "^8.3.0", "@fastify/cors": "^8.3.0", + "@fastify/multipart": "^8.1.0", "@fastify/oauth2": "^7.2.1", "@fastify/swagger": "^8.8.0", "@fastify/swagger-ui": "^1.9.3", @@ -48,6 +49,7 @@ "fastify": "^4.17.0", "http-status-codes": "^2.2.0", "jsonwebtoken": "^9.0.0", + "mime": "^4.0.1", "nanoid": "^4.0.2", "pg": "^8.11.0", "pg-hstore": "^2.3.4", diff --git a/src/domain/entities/file.ts b/src/domain/entities/file.ts index 9d18b251..d524817e 100644 --- a/src/domain/entities/file.ts +++ b/src/domain/entities/file.ts @@ -1,15 +1,55 @@ import type { NoteInternalId } from './note.js'; -import type User from './user.js'; import type { Buffer } from 'buffer'; +import type User from './user.js'; /** * File types for storing in object storage */ -export enum FileTypes { +export enum FileType { + /** + * Type for testing uploads + */ + Test = 0, + /** - * @todo define real types + * File is a part of note */ - test = 'test', + NoteAttachment = 1, +} + +/** + * Additional data about uploaded file, ex. user id, who uploaded it + */ +export interface FileMetadata { + /** + * User who uploaded file + */ + userId: User['id']; +} + +/** + * File location for testing uploads, there is no defined location + */ +export type TestFileLocation = Record; + +/** + * File location, when it is a part of note + */ +export type NoteAttachmentFileLocation = { + noteId: NoteInternalId, +}; + +/** + * Possible file location + */ +export type FileLocation = TestFileLocation | NoteAttachmentFileLocation; + +/** + * File location type, wich depends on file type + */ +export interface FileLocationByType { + [FileType.Test]: TestFileLocation, + [FileType.NoteAttachment]: NoteAttachmentFileLocation, } /** @@ -26,11 +66,6 @@ export default interface UploadedFile { */ key: string; - /** - * User who uploaded the file - */ - userId?: User['id']; - /** * File name */ @@ -44,7 +79,7 @@ export default interface UploadedFile { /** * File type, using to store in object storage */ - type: FileTypes; + type: FileType; /** * File size in bytes @@ -57,9 +92,14 @@ export default interface UploadedFile { createdAt: Date; /** - * In case if file is a part of note, note id to identify permissions to access + * Object, which stores information about file location + */ + location: FileLocation; + + /** + * File metadata */ - noteId?: NoteInternalId; + metadata: FileMetadata; } /** diff --git a/src/domain/service/fileUploader.service.ts b/src/domain/service/fileUploader.service.ts index 97fc83f2..280dd977 100644 --- a/src/domain/service/fileUploader.service.ts +++ b/src/domain/service/fileUploader.service.ts @@ -1,16 +1,16 @@ -import type { FileData } from '@domain/entities/file.js'; -import { FileTypes } from '@domain/entities/file.js'; -import type { NoteInternalId } from '@domain/entities/note.js'; -import type User from '@domain/entities/user.js'; +import type { FileData, NoteAttachmentFileLocation, FileLocationByType, FileLocation, FileMetadata } from '@domain/entities/file.js'; +import type UploadedFile from '@domain/entities/file.js'; +import { FileType } from '@domain/entities/file.js'; import { createFileId } from '@infrastructure/utils/id.js'; import type FileRepository from '@repository/file.repository.js'; import type ObjectRepository from '@repository/object.repository.js'; import { DomainError } from '@domain/entities/DomainError.js'; +import mime from 'mime'; /** * File data for upload */ -interface FileToUpload { +interface UploadFileData { /** * File data */ @@ -23,25 +23,6 @@ interface FileToUpload { * Mimetype of the file */ mimetype: string; - /** - * File type - */ - type: FileTypes; -} - -/** - * File upload details, contains user id, who uploaded the file and note id, in case if file is a part of note - */ -interface FileUploadDetails { - /** - * User who uploaded the file - */ - userId?: User['id']; - - /** - * In case if file is a part of note, note id to identify permissions to access - */ - noteId?: NoteInternalId; } /** @@ -72,13 +53,30 @@ export default class FileUploaderService { /** * Upload file * - * @param fileData - file data to upload (e.g. buffer, name, mimetype) - * @param details - file upload details (e.g. user id, note id) + * @param fileData - file data, including file data, name and mimetype + * @param location - file location depending on type + * @param metadata - file metadata, including user id who uploaded the file */ - public async uploadFile(fileData: FileToUpload, details?: FileUploadDetails): Promise { - const key = createFileId(); + public async uploadFile(fileData: UploadFileData, location: FileLocation, metadata: FileMetadata): Promise { + const type = this.defineFileType(location); + + const fileHash = createFileId(); + + /** + * Extension can be null if file mime type is unknown or not supported + */ + const fileExtension = mime.getExtension(fileData.mimetype); + + if (fileExtension === null) { + throw new DomainError('Unknown file extension'); + } - const bucket = this.defineBucketByFileType(fileData.type); + /** + * Key is a combination of file hash and file extension, separated by dot, e.g. `HgduSDGmsdrs.png` + */ + const key = `${fileHash}.${fileExtension}`; + + const bucket = this.defineBucketByFileType(type); const uploaded = await this.objectRepository.insert(fileData.data, key, bucket); @@ -88,25 +86,50 @@ export default class FileUploaderService { const file = await this.fileRepository.insert({ ...fileData, + type, key, - userId: details?.userId, - noteId: details?.noteId, + metadata, + location: location, size: fileData.data.length, }); if (file === null) { - throw new Error('File was not uploaded'); + throw new DomainError('File was not uploaded'); } return file.key; } + /** + * Get file location by key and type + * Returns null if where is no such file + * + * @param type - file type + * @param key - file unique key + */ + public async getFileLocationByKey(type: T, key: UploadedFile['key']): Promise { + return await this.fileRepository.getFileLocationByKey(type, key); + } + /** * Get file data by key * * @param objectKey - unique file key in object storage + * @param location - file location */ - public async getFileDataByKey(objectKey: string): Promise { + public async getFileData(objectKey: string, location: FileLocation): Promise { + /** + * If type of requested file is note attchement, we need to check if saved file location is the same of requested + */ + if (this.isNoteAttachemntFileLocation(location)) { + const fileType = FileType.NoteAttachment; + const fileLocationFromStorage = await this.fileRepository.getFileLocationByKey(fileType, objectKey); + + if (fileLocationFromStorage === null || location.noteId !== fileLocationFromStorage.noteId) { + throw new DomainError('File not found'); + } + } + const file = await this.fileRepository.getByKey(objectKey); if (file === null) { @@ -124,17 +147,42 @@ export default class FileUploaderService { return fileData; } + + /** + * Define file type by location + * + * @param location - file location + */ + private defineFileType(location: FileLocation): FileType { + if (this.isNoteAttachemntFileLocation(location)) { + return FileType.NoteAttachment; + } + + return FileType.Test; + } + + /** + * Check if file location is note attachemnt + * + * @param location - to check + */ + private isNoteAttachemntFileLocation(location: FileLocation): location is NoteAttachmentFileLocation { + return 'noteId' in location; + } + /** * Define bucket name by file type * * @param fileType - file type */ - private defineBucketByFileType(fileType: FileTypes): string { + private defineBucketByFileType(fileType: FileType): string { switch (fileType) { - case FileTypes.test: + case FileType.Test: return 'test'; + case FileType.NoteAttachment: + return 'note-attachment'; default: - throw new Error('Unknown file type'); + throw new DomainError('Unknown file type'); } } } diff --git a/src/infrastructure/config/index.ts b/src/infrastructure/config/index.ts index 6c47a14c..a1d43b00 100644 --- a/src/infrastructure/config/index.ts +++ b/src/infrastructure/config/index.ts @@ -98,6 +98,7 @@ export type LoggingConfig = z.infer; * Http API configuration */ const HttpApiConfig = z.object({ + fileSizeLimit: z.number(), host: z.string(), port: z.number(), cookieSecret: z.string(), @@ -126,6 +127,7 @@ export type AppConfig = z.infer; const defaultConfig: AppConfig = { httpApi: { + fileSizeLimit: 10000000, // 10mb host: '0.0.0.0', port: 3000, cookieSecret: 'cookieSecret', diff --git a/src/presentation/http/http-api.ts b/src/presentation/http/http-api.ts index 928937e9..54e1bd38 100644 --- a/src/presentation/http/http-api.ts +++ b/src/presentation/http/http-api.ts @@ -28,6 +28,7 @@ import { EditorToolSchema } from './schema/EditorTool.js'; import JoinRouter from '@presentation/http/router/join.js'; import { JoinSchemaParams, JoinSchemaResponse } from './schema/Join.js'; import { DomainError } from '@domain/entities/DomainError.js'; +import UploadRouter from './router/upload.js'; const appServerLogger = getLogger('appServer'); @@ -241,6 +242,13 @@ export default class HttpApi implements Api { prefix: '/editor-tools', editorToolsService: domainServices.editorToolsService, }); + + await this.server?.register(UploadRouter, { + prefix: '/upload', + fileUploaderService: domainServices.fileUploaderService, + noteService: domainServices.noteService, + fileSizeLimit: this.config.fileSizeLimit, + }); } /** diff --git a/src/presentation/http/router/upload.ts b/src/presentation/http/router/upload.ts new file mode 100644 index 00000000..448bcec6 --- /dev/null +++ b/src/presentation/http/router/upload.ts @@ -0,0 +1,108 @@ +import type FileUploaderService from '@domain/service/fileUploader.service.js'; +import { fastifyMultipart, type MultipartFile } from '@fastify/multipart'; +import type { FastifyPluginCallback } from 'fastify'; +import type NoteService from '@domain/service/note.js'; +import useNoteResolver from '../middlewares/note/useNoteResolver.js'; +import type { NoteAttachmentFileLocation } from '@domain/entities/file.js'; + +/** + * Interface for upload router options + */ +interface UploadRouterOptions { + /** + * File uploader service + */ + fileUploaderService: FileUploaderService; + + /** + * Note service instance + */ + noteService: NoteService; + + /** + * Limit for uploaded files size + */ + fileSizeLimit: number; +} + +const UploadRouter: FastifyPluginCallback = (fastify, opts, done) => { + const { fileUploaderService } = opts; + + /** + * Prepare note id resolver middleware + * It should be used in routes that accepts note public id + */ + const { noteResolver } = useNoteResolver(opts.noteService); + + void fastify.register(fastifyMultipart, { + limits: { + fieldSize: opts.fileSizeLimit, + }, + attachFieldsToBody: true, + }); + + fastify.post<{ + Body: { + /** + * File to upload + */ + file: MultipartFile; + } + }>('/:notePublicId', { + config: { + policy: [ + 'authRequired', + 'userCanEdit', + ], + }, + preHandler: [ noteResolver ], + }, async (request, reply) => { + const { userId } = request; + + const location: NoteAttachmentFileLocation = { + noteId: request.note!.id as number, + }; + + const uploadedFileKey = await fileUploaderService.uploadFile( + { + data: await request.body.file.toBuffer(), + mimetype: request.body.file.mimetype, + name: request.body.file.filename, + }, + location, + { + userId: userId!, + } + ); + + return reply.send({ + key: uploadedFileKey, + }); + }); + + fastify.get<{ + Params: { + key: string; + } + }>('/:notePublicId/:key', { + config: { + policy: [ + 'notePublicOrUserInTeam', + ], + }, + preHandler: [ noteResolver ], + }, async (request, reply) => { + const fileLocation: NoteAttachmentFileLocation = { + noteId: request.note!.id as number, + }; + + const fileData = await fileUploaderService.getFileData(request.params.key, fileLocation); + + return reply.send(fileData); + }); + + done(); +}; + +export default UploadRouter; + diff --git a/src/repository/file.repository.ts b/src/repository/file.repository.ts index 265be1c2..2d1636f6 100644 --- a/src/repository/file.repository.ts +++ b/src/repository/file.repository.ts @@ -1,5 +1,5 @@ import type UploadedFile from '@domain/entities/file.js'; -import type { FileCreationAttributes } from '@domain/entities/file.js'; +import type { FileCreationAttributes, FileLocationByType, FileType } from '@domain/entities/file.js'; import type FileStorage from './storage/file.storage.js'; /** @@ -34,4 +34,14 @@ export default class FileRepository { public async insert(fileData: FileCreationAttributes): Promise { return await this.storage.insertFile(fileData); } + + /** + * Get file location by key and type, files with different types have different locations + * + * @param type - file type + * @param key - file unique key + */ + public async getFileLocationByKey(type: T, key: UploadedFile['key']): Promise { + return await this.storage.getFileLocationByKey(type, key); + }; } diff --git a/src/repository/storage/postgres/orm/sequelize/file.ts b/src/repository/storage/postgres/orm/sequelize/file.ts index 9c758291..46225a53 100644 --- a/src/repository/storage/postgres/orm/sequelize/file.ts +++ b/src/repository/storage/postgres/orm/sequelize/file.ts @@ -1,11 +1,7 @@ import type { CreationOptional, InferAttributes, InferCreationAttributes, Sequelize } from 'sequelize'; import { DataTypes, Model } from 'sequelize'; -import type User from '@domain/entities/user.js'; -import { UserModel } from './user.js'; import type UploadedFile from '@domain/entities/file.js'; -import type { FileCreationAttributes } from '@domain/entities/file.js'; -import type { Note } from '@domain/entities/note.js'; -import { NoteModel } from './note.js'; +import type { FileCreationAttributes, FileType, FileLocation, FileLocationByType } from '@domain/entities/file.js'; import type Orm from '@repository/storage/postgres/orm/sequelize/index.js'; /** @@ -23,9 +19,9 @@ export class FileModel extends Model, InferCreationAt public declare key: UploadedFile['key']; /** - * User who uploaded the file + * Additional data about uploaded file */ - public declare userId: CreationOptional; + public declare metadata: UploadedFile['metadata']; /** * File uploaded at @@ -45,7 +41,7 @@ export class FileModel extends Model, InferCreationAt /** * File type, using to store in object storage */ - public declare type: UploadedFile['type']; + public declare type: FileType; /** * File size in bytes @@ -53,9 +49,9 @@ export class FileModel extends Model, InferCreationAt public declare size: UploadedFile['size']; /** - * In case if file is a part of note, note id to indetify permissions to access + * Object, which stores information about file location */ - public declare noteId: CreationOptional; + public declare location: FileLocation; } /** @@ -95,13 +91,8 @@ export default class FileSequelizeStorage { type: DataTypes.STRING, allowNull: false, }, - userId: { - type: DataTypes.INTEGER, - allowNull: true, - references: { - model: UserModel, - key: 'id', - }, + metadata: { + type: DataTypes.JSONB, }, createdAt: DataTypes.DATE, mimetype: { @@ -116,13 +107,8 @@ export default class FileSequelizeStorage { type: DataTypes.INTEGER, allowNull: false, }, - noteId: { - type: DataTypes.INTEGER, - references: { - model: NoteModel, - key: 'id', - }, - allowNull: true, + location: { + type: DataTypes.JSONB, }, name: { type: DataTypes.STRING, @@ -130,6 +116,7 @@ export default class FileSequelizeStorage { }, }, { + updatedAt: false, sequelize: this.database, tableName: this.tableName, } @@ -157,4 +144,25 @@ export default class FileSequelizeStorage { }, }); } + + /** + * Get file location by key and type + * + * @param type - file type + * @param key - file unique key + */ + public async getFileLocationByKey(type: T, key: UploadedFile['key']): Promise { + const res = await this.model.findOne({ + where: { + key, + type, + }, + }); + + if (res === null) { + return null; + } + + return res.location as FileLocationByType[T]; + } } diff --git a/yarn.lock b/yarn.lock index 98afa853..fbdf00a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -288,6 +288,15 @@ __metadata: languageName: node linkType: hard +"@fastify/busboy@npm:^1.0.0": + version: 1.2.1 + resolution: "@fastify/busboy@npm:1.2.1" + dependencies: + text-decoding: ^1.0.0 + checksum: 6e773a2929fd7732fd8ba8f9e1c1b9d622c6165b6e0bed9268e1785f8fd5e8b0a35d6adfe86f15a701bf7783d09c629f3437b3578d34c0246eb26f973ede20f0 + languageName: node + linkType: hard + "@fastify/cookie@npm:^8.3.0": version: 8.3.0 resolution: "@fastify/cookie@npm:8.3.0" @@ -325,6 +334,13 @@ __metadata: languageName: node linkType: hard +"@fastify/error@npm:^3.0.0": + version: 3.4.1 + resolution: "@fastify/error@npm:3.4.1" + checksum: 2c2e98c33327884c0927a73e8c3b8f162acbf1e4d058bacb68bca0c3607f36d6fde8c376fde45b2097e724d450266f8bb29134305fa24aabd200f83f087c7321 + languageName: node + linkType: hard + "@fastify/error@npm:^3.4.0": version: 3.4.0 resolution: "@fastify/error@npm:3.4.0" @@ -341,6 +357,20 @@ __metadata: languageName: node linkType: hard +"@fastify/multipart@npm:^8.1.0": + version: 8.1.0 + resolution: "@fastify/multipart@npm:8.1.0" + dependencies: + "@fastify/busboy": ^1.0.0 + "@fastify/deepmerge": ^1.0.0 + "@fastify/error": ^3.0.0 + fastify-plugin: ^4.0.0 + secure-json-parse: ^2.4.0 + stream-wormhole: ^1.1.0 + checksum: 06c13b6669497d2eec55cbf4be9272f67c80947f521b9fb824a9947143fef38c1918badf2b51d67e2f8ba2193ba5a914bfeb7a6164bbb0a63acbaf1f644a6471 + languageName: node + linkType: hard + "@fastify/oauth2@npm:^7.2.1": version: 7.5.0 resolution: "@fastify/oauth2@npm:7.5.0" @@ -4229,6 +4259,15 @@ __metadata: languageName: node linkType: hard +"mime@npm:^4.0.1": + version: 4.0.1 + resolution: "mime@npm:4.0.1" + bin: + mime: bin/cli.js + checksum: a931283bc31570cc9c63fbad24fdf178b4dd545462f93543eff634b24d2b65064585eb347cdf0720316bfa5ca0943115694672f2bc4895f8e2366d280ad481f2 + languageName: node + linkType: hard + "mimic-fn@npm:^4.0.0": version: 4.0.0 resolution: "mimic-fn@npm:4.0.0" @@ -4572,6 +4611,7 @@ __metadata: "@codex-team/config-loader": ^1.0.0 "@fastify/cookie": ^8.3.0 "@fastify/cors": ^8.3.0 + "@fastify/multipart": ^8.1.0 "@fastify/oauth2": ^7.2.1 "@fastify/swagger": ^8.8.0 "@fastify/swagger-ui": ^1.9.3 @@ -4588,6 +4628,7 @@ __metadata: fastify: ^4.17.0 http-status-codes: ^2.2.0 jsonwebtoken: ^9.0.0 + mime: ^4.0.1 nanoid: ^4.0.2 nodemon: ^2.0.22 pg: ^8.11.0 @@ -6034,6 +6075,13 @@ __metadata: languageName: node linkType: hard +"stream-wormhole@npm:^1.1.0": + version: 1.1.0 + resolution: "stream-wormhole@npm:1.1.0" + checksum: cc19e0235c5d031bd530fa83913c807d9525fa4ba33d51691dd822c0726b8b7ef138b34f289d063a3018cddba67d3ba7fd0ecedaa97242a0f1ed2eed3c6a2ab1 + languageName: node + linkType: hard + "streamx@npm:^2.15.0": version: 2.15.1 resolution: "streamx@npm:2.15.1" @@ -6293,6 +6341,13 @@ __metadata: languageName: node linkType: hard +"text-decoding@npm:^1.0.0": + version: 1.0.0 + resolution: "text-decoding@npm:1.0.0" + checksum: 4b2359d8efdabea72ac470304e991913e9b82a55b1c33ab5204f115d11305ac5900add80aee5f7d22b2bcf0faebaf35b193d28a10b74adf175d9ac9d63604445 + languageName: node + linkType: hard + "text-table@npm:^0.2.0": version: 0.2.0 resolution: "text-table@npm:0.2.0"