Server: Add support for content drivers

packages/server/jest.setup.js

@@ -4,7 +4,9 @@ const nodeSqlite = require('sqlite3');
 shimInit({ nodeSqlite });
 
 // We don't want the tests to fail due to timeout, especially on CI, and certain
-// tests can take more time since we do integration testing too.
-jest.setTimeout(30 * 1000);
+// tests can take more time since we do integration testing too. The share tests
+// in particular can take a while.
+
+jest.setTimeout(60 * 1000);
 
 process.env.JOPLIN_IS_TESTING = '1';

packages/server/package.json

@@ -21,6 +21,7 @@
     "watch": "tsc --watch --project tsconfig.json"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.40.0",
     "@fortawesome/fontawesome-free": "^5.15.1",
     "@joplin/lib": "~2.6",
     "@joplin/renderer": "~2.6",

packages/server/src/app.ts

@@ -5,7 +5,7 @@ import * as Koa from 'koa';
 import * as fs from 'fs-extra';
 import Logger, { LoggerWrapper, TargetType } from '@joplin/lib/Logger';
 import config, { initConfig, runningInDocker } from './config';
-import { migrateLatest, waitForConnection, sqliteDefaultDir, latestMigration } from './db';
+import { migrateLatest, waitForConnection, sqliteDefaultDir, latestMigration, DbConnection } from './db';
 import { AppContext, Env, KoaNext } from './utils/types';
 import FsDriverNode from '@joplin/lib/fs-driver-node';
 import routeHandler from './middleware/routeHandler';
@@ -17,10 +17,11 @@ import startServices from './utils/startServices';
 import { credentialFile } from './utils/testing/testUtils';
 import apiVersionHandler from './middleware/apiVersionHandler';
 import clickJackingHandler from './middleware/clickJackingHandler';
-import newModelFactory from './models/factory';
+import newModelFactory, { Options } from './models/factory';
 import setupCommands from './utils/setupCommands';
 import { RouteResponseFormat, routeResponseFormat } from './utils/routeUtils';
 import { parseEnv } from './env';
+import storageDriverFromConfig from './models/items/storage/storageDriverFromConfig';
 
 interface Argv {
 	env?: Env;
@@ -61,6 +62,8 @@ function appLogger(): LoggerWrapper {
 }
 
 function markPasswords(o: Record<string, any>): Record<string, any> {
+	if (!o) return o;
+
 	const output: Record<string, any> = {};
 
 	for (const k of Object.keys(o)) {
@@ -219,6 +222,13 @@ async function main() {
 		fs.writeFileSync(pidFile, `${process.pid}`);
 	}
 
+	const newModelFactoryOptions = async (db: DbConnection): Promise<Options> => {
+		return {
+			storageDriver: await storageDriverFromConfig(config().storageDriver, db, { assignDriverId: env !== 'buildTypes' }),
+			storageDriverFallback: await storageDriverFromConfig(config().storageDriverFallback, db, { assignDriverId: env !== 'buildTypes' }),
+		};
+	};
+
 	let runCommandAndExitApp = true;
 
 	if (selectedCommand) {
@@ -235,7 +245,7 @@ async function main() {
 			});
 		} else {
 			const connectionCheck = await waitForConnection(config().database);
-			const models = newModelFactory(connectionCheck.connection, config());
+			const models = newModelFactory(connectionCheck.connection, config(), await newModelFactoryOptions(connectionCheck.connection));
 
 			await selectedCommand.run(commandArgv, {
 				db: connectionCheck.connection,
@@ -253,6 +263,8 @@ async function main() {
 		appLogger().info('Log dir:', config().logDir);
 		appLogger().info('DB Config:', markPasswords(config().database));
 		appLogger().info('Mailer Config:', markPasswords(config().mailer));
+		appLogger().info('Content driver:', markPasswords(config().storageDriver));
+		appLogger().info('Content driver (fallback):', markPasswords(config().storageDriverFallback));
 
 		appLogger().info('Trying to connect to database...');
 		const connectionCheck = await waitForConnection(config().database);
@@ -263,7 +275,8 @@ async function main() {
 		appLogger().info('Connection check:', connectionCheckLogInfo);
 		const ctx = app.context as AppContext;
 
-		await setupAppContext(ctx, env, connectionCheck.connection, appLogger);
+		await setupAppContext(ctx, env, connectionCheck.connection, appLogger, await newModelFactoryOptions(connectionCheck.connection));
+
 		await initializeJoplinUtils(config(), ctx.joplinBase.models, ctx.joplinBase.services.mustache);
 
 		if (config().database.autoMigration) {

packages/server/src/config.ts

@@ -3,6 +3,7 @@ import { Config, DatabaseConfig, DatabaseConfigClient, Env, MailerConfig, RouteT
 import * as pathUtils from 'path';
 import { loadStripeConfig, StripePublicConfig } from '@joplin/lib/utils/joplinCloud';
 import { EnvVariables } from './env';
+import parseStorageDriverConnectionString from './models/items/storage/parseStorageDriverConnectionString';
 
 interface PackageJson {
 	version: string;
@@ -130,6 +131,8 @@ export async function initConfig(envType: Env, env: EnvVariables, overrides: any
 		supportName: env.SUPPORT_NAME || appName,
 		businessEmail: env.BUSINESS_EMAIL || supportEmail,
 		cookieSecure: env.COOKIES_SECURE,
+		storageDriver: parseStorageDriverConnectionString(env.STORAGE_DRIVER),
+		storageDriverFallback: parseStorageDriverConnectionString(env.STORAGE_DRIVER_FALLBACK),
 		...overrides,
 	};
 }

packages/server/src/env.ts

@@ -1,8 +1,82 @@
-export interface EnvVariables {
+// The possible env variables and their defaults are listed below.
+//
+// The env variables can be of type string, integer or boolean. When the type is
+// boolean, set the variable to "0" or "1" in your env file.
+
+const defaultEnvValues: EnvVariables = {
 	// ==================================================
 	// General config
 	// ==================================================
 
+	APP_NAME: 'Joplin Server',
+	APP_PORT: 22300,
+	SIGNUP_ENABLED: false,
+	TERMS_ENABLED: false,
+	ACCOUNT_TYPES_ENABLED: false,
+	ERROR_STACK_TRACES: false,
+	COOKIES_SECURE: false,
+	RUNNING_IN_DOCKER: false,
+
+	// ==================================================
+	// URL config
+	// ==================================================
+
+	APP_BASE_URL: '',
+	USER_CONTENT_BASE_URL: '',
+	API_BASE_URL: '',
+	JOPLINAPP_BASE_URL: 'https://joplinapp.org',
+
+	// ==================================================
+	// Database config
+	// ==================================================
+
+	DB_CLIENT: 'sqlite3',
+	DB_SLOW_QUERY_LOG_ENABLED: false,
+	DB_SLOW_QUERY_LOG_MIN_DURATION: 1000,
+	DB_AUTO_MIGRATION: true,
+
+	POSTGRES_PASSWORD: 'joplin',
+	POSTGRES_DATABASE: 'joplin',
+	POSTGRES_USER: 'joplin',
+	POSTGRES_HOST: '',
+	POSTGRES_PORT: 5432,
+
+	// This must be the full path to the database file
+	SQLITE_DATABASE: '',
+
+	// ==================================================
+	// Content driver config
+	// ==================================================
+
+	STORAGE_DRIVER: 'Type=Database',
+	STORAGE_DRIVER_FALLBACK: '',
+
+	// ==================================================
+	// Mailer config
+	// ==================================================
+
+	MAILER_ENABLED: false,
+	MAILER_HOST: '',
+	MAILER_PORT: 587,
+	MAILER_SECURE: true,
+	MAILER_AUTH_USER: '',
+	MAILER_AUTH_PASSWORD: '',
+	MAILER_NOREPLY_NAME: '',
+	MAILER_NOREPLY_EMAIL: '',
+
+	SUPPORT_EMAIL: 'SUPPORT_EMAIL', // Defaults to "SUPPORT_EMAIL" so that server admin knows they have to set it.
+	SUPPORT_NAME: '',
+	BUSINESS_EMAIL: '',
+
+	// ==================================================
+	// Stripe config
+	// ==================================================
+
+	STRIPE_SECRET_KEY: '',
+	STRIPE_WEBHOOK_SECRET: '',
+};
+
+export interface EnvVariables {
 	APP_NAME: string;
 	APP_PORT: number;
 	SIGNUP_ENABLED: boolean;
@@ -12,19 +86,11 @@ export interface EnvVariables {
 	COOKIES_SECURE: boolean;
 	RUNNING_IN_DOCKER: boolean;
 
-	// ==================================================
-	// URL config
-	// ==================================================
-
 	APP_BASE_URL: string;
 	USER_CONTENT_BASE_URL: string;
 	API_BASE_URL: string;
 	JOPLINAPP_BASE_URL: string;
 
-	// ==================================================
-	// Database config
-	// ==================================================
-
 	DB_CLIENT: string;
 	DB_SLOW_QUERY_LOG_ENABLED: boolean;
 	DB_SLOW_QUERY_LOG_MIN_DURATION: number;
@@ -36,12 +102,10 @@ export interface EnvVariables {
 	POSTGRES_HOST: string;
 	POSTGRES_PORT: number;
 
-	// This must be the full path to the database file
 	SQLITE_DATABASE: string;
 
-	// ==================================================
-	// Mailer config
-	// ==================================================
+	STORAGE_DRIVER: string;
+	STORAGE_DRIVER_FALLBACK: string;
 
 	MAILER_ENABLED: boolean;
 	MAILER_HOST: string;
@@ -56,59 +120,10 @@ export interface EnvVariables {
 	SUPPORT_NAME: string;
 	BUSINESS_EMAIL: string;
 
-	// ==================================================
-	// Stripe config
-	// ==================================================
-
 	STRIPE_SECRET_KEY: string;
 	STRIPE_WEBHOOK_SECRET: string;
 }
 
-const defaultEnvValues: EnvVariables = {
-	APP_NAME: 'Joplin Server',
-	APP_PORT: 22300,
-	SIGNUP_ENABLED: false,
-	TERMS_ENABLED: false,
-	ACCOUNT_TYPES_ENABLED: false,
-	ERROR_STACK_TRACES: false,
-	COOKIES_SECURE: false,
-	RUNNING_IN_DOCKER: false,
-
-	APP_BASE_URL: '',
-	USER_CONTENT_BASE_URL: '',
-	API_BASE_URL: '',
-	JOPLINAPP_BASE_URL: 'https://joplinapp.org',
-
-	DB_CLIENT: 'sqlite3',
-	DB_SLOW_QUERY_LOG_ENABLED: false,
-	DB_SLOW_QUERY_LOG_MIN_DURATION: 1000,
-	DB_AUTO_MIGRATION: true,
-
-	POSTGRES_PASSWORD: 'joplin',
-	POSTGRES_DATABASE: 'joplin',
-	POSTGRES_USER: 'joplin',
-	POSTGRES_HOST: '',
-	POSTGRES_PORT: 5432,
-
-	SQLITE_DATABASE: '',
-
-	MAILER_ENABLED: false,
-	MAILER_HOST: '',
-	MAILER_PORT: 587,
-	MAILER_SECURE: true,
-	MAILER_AUTH_USER: '',
-	MAILER_AUTH_PASSWORD: '',
-	MAILER_NOREPLY_NAME: '',
-	MAILER_NOREPLY_EMAIL: '',
-
-	SUPPORT_EMAIL: 'SUPPORT_EMAIL', // Defaults to "SUPPORT_EMAIL" so that server admin knows they have to set it.
-	SUPPORT_NAME: '',
-	BUSINESS_EMAIL: '',
-
-	STRIPE_SECRET_KEY: '',
-	STRIPE_WEBHOOK_SECRET: '',
-};
-
 export function parseEnv(rawEnv: any, defaultOverrides: any = null): EnvVariables {
 	const output: EnvVariables = {
 		...defaultEnvValues,
@@ -125,7 +140,7 @@ export function parseEnv(rawEnv: any, defaultOverrides: any = null): EnvVariable
 			if (isNaN(v)) throw new Error(`Invalid number value for env variable ${key} = ${rawEnvValue}`);
 			(output as any)[key] = v;
 		} else if (typeof value === 'boolean') {
-			if (rawEnvValue !== '0' && rawEnvValue !== '1') throw new Error(`Invalid boolean for for env variable ${key}: ${rawEnvValue}`);
+			if (rawEnvValue !== '0' && rawEnvValue !== '1') throw new Error(`Invalid boolean value for env variable ${key}: ${rawEnvValue} (Should be either "0" or "1")`);
 			(output as any)[key] = rawEnvValue === '1';
 		} else if (typeof value === 'string') {
 			(output as any)[key] = `${rawEnvValue}`;

packages/server/src/migrations/20211105183559_storage.ts

@@ -0,0 +1,32 @@
+import { Knex } from 'knex';
+import { DbConnection } from '../db';
+
+export async function up(db: DbConnection): Promise<any> {
+	await db.schema.createTable('storages', (table: Knex.CreateTableBuilder) => {
+		table.increments('id').unique().primary().notNullable();
+		table.text('connection_string').notNullable();
+	});
+
+	await db('storages').insert({
+		connection_string: 'Type=Database',
+	});
+
+	// First we create the column and set a default so as to populate the
+	// content_storage_id field.
+	await db.schema.alterTable('items', (table: Knex.CreateTableBuilder) => {
+		table.integer('content_storage_id').defaultTo(1).notNullable();
+	});
+
+	// Once it's set, we remove the default as that should be explicitly set.
+	await db.schema.alterTable('items', (table: Knex.CreateTableBuilder) => {
+		table.integer('content_storage_id').notNullable().alter();
+	});
+}
+
+export async function down(db: DbConnection): Promise<any> {
+	await db.schema.dropTable('storages');
+
+	await db.schema.alterTable('items', (table: Knex.CreateTableBuilder) => {
+		table.dropColumn('content_storage_id');
+	});
+}

packages/server/src/models/BaseModel.ts

@@ -3,7 +3,7 @@ import { DbConnection } from '../db';
 import TransactionHandler from '../utils/TransactionHandler';
 import uuidgen from '../utils/uuidgen';
 import { ErrorUnprocessableEntity, ErrorBadRequest } from '../utils/errors';
-import { Models } from './factory';
+import { Models, NewModelFactoryHandler } from './factory';
 import * as EventEmitter from 'events';
 import { Config } from '../utils/types';
 import personalizedUserContentBaseUrl from '@joplin/lib/services/joplinServer/personalizedUserContentBaseUrl';
@@ -54,12 +54,12 @@ export default abstract class BaseModel<T> {
 	private defaultFields_: string[] = [];
 	private db_: DbConnection;
 	private transactionHandler_: TransactionHandler;
-	private modelFactory_: Function;
+	private modelFactory_: NewModelFactoryHandler;
 	private static eventEmitter_: EventEmitter = null;
 	private config_: Config;
 	private savePoints_: SavePoint[] = [];
 
-	public constructor(db: DbConnection, modelFactory: Function, config: Config) {
+	public constructor(db: DbConnection, modelFactory: NewModelFactoryHandler, config: Config) {
 		this.db_ = db;
 		this.modelFactory_ = modelFactory;
 		this.config_ = config;
@@ -71,7 +71,7 @@ export default abstract class BaseModel<T> {
 	// connection is passed to it. That connection can be the regular db
 	// connection, or the active transaction.
 	protected models(db: DbConnection = null): Models {
-		return this.modelFactory_(db || this.db, this.config_);
+		return this.modelFactory_(db || this.db);
 	}
 
 	protected get baseUrl(): string {
@@ -90,7 +90,7 @@ export default abstract class BaseModel<T> {
 		return this.config_.appName;
 	}
 
-	protected get db(): DbConnection {
+	public get db(): DbConnection {
 		if (this.transactionHandler_.activeTransaction) return this.transactionHandler_.activeTransaction;
 		return this.db_;
 	}

packages/server/src/models/ChangeModel.test.ts

@@ -38,12 +38,12 @@ describe('ChangeModel', function() {
 		const changeModel = models().change();
 
 		await msleep(1); const item1 = await models().item().makeTestItem(user.id, 1); // [1] CREATE 1
-		await msleep(1); await itemModel.saveForUser(user.id, { id: item1.id, name: '0000000000000000000000000000001A.md' }); // [2] UPDATE 1a
-		await msleep(1); await itemModel.saveForUser(user.id, { id: item1.id, name: '0000000000000000000000000000001B.md' }); // [3] UPDATE 1b
+		await msleep(1); await itemModel.saveForUser(user.id, { id: item1.id, name: '0000000000000000000000000000001A.md', content: Buffer.from('') }); // [2] UPDATE 1a
+		await msleep(1); await itemModel.saveForUser(user.id, { id: item1.id, name: '0000000000000000000000000000001B.md', content: Buffer.from('') }); // [3] UPDATE 1b
 		await msleep(1); const item2 = await models().item().makeTestItem(user.id, 2); // [4] CREATE 2
-		await msleep(1); await itemModel.saveForUser(user.id, { id: item2.id, name: '0000000000000000000000000000002A.md' }); // [5] UPDATE 2a
+		await msleep(1); await itemModel.saveForUser(user.id, { id: item2.id, name: '0000000000000000000000000000002A.md', content: Buffer.from('') }); // [5] UPDATE 2a
 		await msleep(1); await itemModel.delete(item1.id); // [6] DELETE 1
-		await msleep(1); await itemModel.saveForUser(user.id, { id: item2.id, name: '0000000000000000000000000000002B.md' }); // [7] UPDATE 2b
+		await msleep(1); await itemModel.saveForUser(user.id, { id: item2.id, name: '0000000000000000000000000000002B.md', content: Buffer.from('') }); // [7] UPDATE 2b
 		await msleep(1); const item3 = await models().item().makeTestItem(user.id, 3); // [8] CREATE 3
 
 		// Check that the 8 changes were created
@@ -120,7 +120,7 @@ describe('ChangeModel', function() {
 
 		let i = 1;
 		await msleep(1); const item1 = await models().item().makeTestItem(user.id, 1); // CREATE 1
-		await msleep(1); await itemModel.saveForUser(user.id, { id: item1.id, name: `test_mod${i++}` }); // UPDATE 1
+		await msleep(1); await itemModel.saveForUser(user.id, { id: item1.id, name: `test_mod${i++}`, content: Buffer.from('') }); // UPDATE 1
 
 		await expectThrow(async () => changeModel.delta(user.id, { limit: 1, cursor: 'invalid' }), 'resyncRequired');
 	});