From 9e80cd7b3f33c87a9a82f6db0cf9cafd8069fb62 Mon Sep 17 00:00:00 2001 From: Yang Jun Date: Tue, 9 Jul 2024 21:36:47 +0800 Subject: [PATCH] feat: DoS prevention, #250 --- docs/source/_data/sidebar.yml | 1 + docs/source/tutorials/dos.md | 58 +++++++++++++ docs/source/zh-cn/tutorials/dos.md | 56 ++++++++++++ docs/themes/navy/languages/en.yml | 1 + docs/themes/navy/languages/zh-cn.yml | 1 + src/context/context.ts | 20 ++++- src/filters/array.ts | 101 +++++++++++++++------- src/filters/date.ts | 2 + src/filters/html.ts | 33 ++++--- src/filters/string.ts | 125 ++++++++++++++++++--------- src/fs/map-fs.ts | 3 +- src/liquid-options.ts | 23 ++++- src/liquid.ts | 17 ++-- src/parser/parser.ts | 9 +- src/render/render.ts | 1 + src/tags/block.ts | 5 +- src/tags/capture.ts | 5 +- src/tags/case.ts | 5 +- src/tags/for.ts | 5 +- src/tags/if.ts | 5 +- src/tags/include.ts | 5 +- src/tags/layout.ts | 7 +- src/tags/liquid.ts | 5 +- src/tags/render.ts | 13 +-- src/tags/tablerow.ts | 5 +- src/tags/unless.ts | 5 +- src/template/tag.ts | 4 +- src/util/index.ts | 1 + src/util/limiter.ts | 18 ++++ src/util/underscore.ts | 4 +- test/integration/liquid/dos.spec.ts | 61 +++++++++++++ 31 files changed, 471 insertions(+), 133 deletions(-) create mode 100644 docs/source/tutorials/dos.md create mode 100644 docs/source/zh-cn/tutorials/dos.md create mode 100644 src/util/limiter.ts create mode 100644 test/integration/liquid/dos.spec.ts diff --git a/docs/source/_data/sidebar.yml b/docs/source/_data/sidebar.yml index 4063e44b39..c7294e804d 100644 --- a/docs/source/_data/sidebar.yml +++ b/docs/source/_data/sidebar.yml @@ -19,6 +19,7 @@ tutorials: plugins: plugins.html operators: operators.html truth: truthy-and-falsy.html + dos: dos.html miscellaneous: migration9: migrate-to-9.html changelog: changelog.html diff --git a/docs/source/tutorials/dos.md b/docs/source/tutorials/dos.md new file mode 100644 index 0000000000..3cde6f9f80 --- /dev/null +++ b/docs/source/tutorials/dos.md @@ -0,0 +1,58 @@ +--- +title: DoS Prevention +--- + +When the template or data context cannot be trusted, enabling DoS prevention options is crucial. LiquidJS provides 3 options for this purpose: `parseLimit`, `renderLimit`, and `memoryLimit`. + +## TL;DR + +Setting these options can largely ensure that your LiquidJS instance won't hang for extended periods or consume excessive memory. These limits are based on the available JavaScript APIs, so they are not precise hard limits but thresholds to help prevent your process from failing or hanging. + +```typescript +const liquid = new Liquid({ + parseLimit: 1e8, // typical size of your templates in each render + renderLimit: 1000, // limit each render to be completed in 1s + memoryLimit: 1e9, // memory available for LiquidJS (1e9 for 1GB) +}) +``` + +When a `parse()` or `render()` cannot be completed within given resource, it throws. + +## parseLimit + +[parseLimit][parseLimit] restricts the size (character length) of templates parsed in each `.parse()` call, including referenced partials and layouts. Since LiquidJS parses template strings in near O(n) time, limiting total template length is usually sufficient. + +A typical PC handles `1e8` (100M) characters without issues. + +## renderLimit + +Restricting template size alone is insufficient because dynamic loops with large counts can occur in render time. [renderLimit][renderLimit] mitigates this by limiting the time consumed by each `render()` call. + +```liquid +{%- for i in (1..10000000) -%} + order: {{i}} +{%- endfor -%} +``` + +Render time is checked on a per-template basis (before rendering each template). In the above example, there are 2 templates in the loop: `order: ` and `{{i}}`, render time will be checked 10000000x2 times. + +For time-consuming tags and filters within a single template, the process can still hang. For fully controlled rendering, consider using a process manager like [paralleljs][paralleljs]. + +## memoryLimit + +Even with small number of templates and iterations, memory usage can grow exponentially. In the following example, memory doubles with each iteration: + +```liquid +{% assign array = "1,2,3" | split: "," %} +{% for i in (1..32) %} + {% assign array = array | concat: array %} +{% endfor %} +``` + +[memoryLimit][memoryLimit] restricts memory-sensitive filters to prevent excessive memory allocation. As [JavaScript uses GC to manage memory](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Memory_management), `memoryLimit` limits only the total number of objects allocated by memory sensitive filters in LiquidJS thus may not reflect the actual memory footprint. + +[paralleljs]: https://www.npmjs.com/package/paralleljs +[parseLimit]: /api/interfaces/LiquidOptions.html#parseLimit +[renderLimit]: /api/interfaces/LiquidOptions.html#renderLimit +[cpuLimit]: /api/interfaces/LiquidOptions.html#cpuLimit +[memoryLimit]: /api/interfaces/LiquidOptions.html#memoryLimit \ No newline at end of file diff --git a/docs/source/zh-cn/tutorials/dos.md b/docs/source/zh-cn/tutorials/dos.md new file mode 100644 index 0000000000..2c79c16e16 --- /dev/null +++ b/docs/source/zh-cn/tutorials/dos.md @@ -0,0 +1,56 @@ +--- +title: 防止 DoS 攻击 +--- + +当模板或数据上下文不可信时,启用DoS预防选项至关重要。LiquidJS 提供了三个选项用于此目的:`parseLimit`、`renderLimit` 和 `memoryLimit`。 + +## TL;DR + +设置这些选项可以在很大程度上确保你的 LiquidJS 实例不会长时间挂起或消耗过多内存。这些限制基于可用的 JavaScript API,因此它们不是精确的硬性限制,而是确保你的进程不会失败或挂起的阈值。 + +```typescript +const liquid = new Liquid({ + parseLimit: 1e8, // 每次渲染的模板的典型大小 + renderLimit: 1000, // 每次渲染最多 1s + memoryLimit: 1e9, // LiquidJS 可用的内存(1e9 对应 1GB) +}) +``` + +## parseLimit + +[parseLimit][parseLimit] 限制每次 `.parse()` 调用中解析的模板大小(字符长度),包括引用的 partials 和 layouts。由于 LiquidJS 解析模板字符串的时间复杂度接近 O(n),限制模板总长度通常就足够了。 + +普通电脑可以很容易处理 `1e8`(100M)个字符的模板。 + +## renderLimit + +仅限制模板大小是不够的,因为在渲染时可能会出现动态的数组和循环。[renderLimit][renderLimit] 通过限制每次 `render()` 调用的时间来缓解这些问题。 + +```liquid +{%- for i in (1..10000000) -%} + order: {{i}} +{%- endfor -%} +``` + +渲染时间是在渲染每个模板之前检查的。在上面的例子中,循环中有两个模板:`order: ` 和 `{{i}}`,因此会检查 2x10000000 次。 + +单个模板内的标签和过滤器仍然可能把进程挂起。要完全控制渲染过程,建议使用类似 [paralleljs][paralleljs] 的进程管理器。 + +## memoryLimit + +即使模板和迭代次数较少,内存使用量也可能呈指数增长。在下面的示例中,内存会在每次迭代中翻倍: + +```liquid +{% assign array = "1,2,3" | split: "," %} +{% for i in (1..32) %} + {% assign array = array | concat: array %} +{% endfor %} +``` + +[memoryLimit][memoryLimit] 限制内存敏感的过滤器,以防止过度的内存分配。由于 [JavaScript 使用 GC 来管理内存](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Memory_management),`memoryLimit` 仅限制 LiquidJS 中内存敏感过滤器分配的对象总数,因此可能无法反映实际的内存占用。 + +[paralleljs]: https://www.npmjs.com/package/paralleljs +[parseLimit]: /api/interfaces/LiquidOptions.html#parseLimit +[renderLimit]: /api/interfaces/LiquidOptions.html#renderLimit +[cpuLimit]: /api/interfaces/LiquidOptions.html#cpuLimit +[memoryLimit]: /api/interfaces/LiquidOptions.html#memoryLimit \ No newline at end of file diff --git a/docs/themes/navy/languages/en.yml b/docs/themes/navy/languages/en.yml index d47b787f5c..8ad435b18e 100644 --- a/docs/themes/navy/languages/en.yml +++ b/docs/themes/navy/languages/en.yml @@ -51,6 +51,7 @@ sidebar: plugins: Plugins operators: Operators truth: Truthy and Falsy + dos: DoS miscellaneous: Miscellaneous migration9: 'Migrate to LiquidJS 9' diff --git a/docs/themes/navy/languages/zh-cn.yml b/docs/themes/navy/languages/zh-cn.yml index c8281e00c7..8a2ab42bdc 100644 --- a/docs/themes/navy/languages/zh-cn.yml +++ b/docs/themes/navy/languages/zh-cn.yml @@ -51,6 +51,7 @@ sidebar: plugins: 插件 operators: 运算符 truth: 真和假 + dos: DoS miscellaneous: 其他 migration9: '迁移到 LiquidJS 9' diff --git a/src/context/context.ts b/src/context/context.ts index f54b5ed6ea..2c331340c6 100644 --- a/src/context/context.ts +++ b/src/context/context.ts @@ -2,7 +2,7 @@ import { Drop } from '../drop/drop' import { __assign } from 'tslib' import { NormalizedFullOptions, defaultOptions, RenderOptions } from '../liquid-options' import { Scope } from './scope' -import { isArray, isNil, isUndefined, isString, isFunction, toLiquid, InternalUndefinedVariableError, toValueSync, isObject } from '../util' +import { isArray, isNil, isUndefined, isString, isFunction, toLiquid, InternalUndefinedVariableError, toValueSync, isObject, Limiter } from '../util' type PropertyKey = string | number; @@ -33,13 +33,17 @@ export class Context { */ public strictVariables: boolean; public ownPropertyOnly: boolean; - public constructor (env: object = {}, opts: NormalizedFullOptions = defaultOptions, renderOptions: RenderOptions = {}) { + public memoryLimit: Limiter; + public renderLimit: Limiter; + public constructor (env: object = {}, opts: NormalizedFullOptions = defaultOptions, renderOptions: RenderOptions = {}, { memoryLimit, renderLimit }: { [key: string]: Limiter } = {}) { this.sync = !!renderOptions.sync this.opts = opts this.globals = renderOptions.globals ?? opts.globals this.environments = isObject(env) ? env : Object(env) this.strictVariables = renderOptions.strictVariables ?? this.opts.strictVariables this.ownPropertyOnly = renderOptions.ownPropertyOnly ?? opts.ownPropertyOnly + this.memoryLimit = memoryLimit ?? new Limiter('memory alloc', renderOptions.memoryLimit ?? opts.memoryLimit) + this.renderLimit = renderLimit ?? new Limiter('template render', performance.now() + (renderOptions.templateLimit ?? opts.renderLimit)) } public getRegister (key: string) { return (this.registers[key] = this.registers[key] || {}) @@ -95,6 +99,16 @@ export class Context { public bottom () { return this.scopes[0] } + public spawn (scope = {}) { + return new Context(scope, this.opts, { + sync: this.sync, + globals: this.globals, + strictVariables: this.strictVariables + }, { + renderLimit: this.renderLimit, + memoryLimit: this.memoryLimit + }) + } private findScope (key: string | number) { for (let i = this.scopes.length - 1; i >= 0; i--) { const candidate = this.scopes[i] @@ -108,7 +122,7 @@ export class Context { export function readProperty (obj: Scope, key: PropertyKey, ownPropertyOnly: boolean) { obj = toLiquid(obj) if (isNil(obj)) return obj - if (isArray(obj) && key < 0) return obj[obj.length + +key] + if (isArray(obj) && (key as number) < 0) return obj[obj.length + +key] const value = readJSProperty(obj, key, ownPropertyOnly) if (value === undefined && obj instanceof Drop) return obj.liquidMethodMissing(key) if (isFunction(value)) return value.call(obj) diff --git a/src/filters/array.ts b/src/filters/array.ts index e0100f4c62..d9a392733d 100644 --- a/src/filters/array.ts +++ b/src/filters/array.ts @@ -1,17 +1,29 @@ import { toArray, argumentsToValue, toValue, stringify, caseInsensitiveCompare, isArray, isNil, last as arrayLast, hasOwnProperty } from '../util' import { equals, evalToken, isTruthy } from '../render' import { Value, FilterImpl } from '../template' -import { Context, Scope } from '../context' import { Tokenizer } from '../parser' - -export const join = argumentsToValue((v: any[], arg: string) => toArray(v).join(arg === undefined ? ' ' : arg)) +import type { Scope } from '../context' + +export const join = argumentsToValue(function (this: FilterImpl, v: any[], arg: string) { + const array = toArray(v) + const sep = arg === undefined ? ' ' : arg + const complexity = array.length * (1 + sep.length) + this.context.memoryLimit.use(complexity) + return array.join(sep) +}) export const last = argumentsToValue((v: any) => isArray(v) ? arrayLast(v) : '') export const first = argumentsToValue((v: any) => isArray(v) ? v[0] : '') -export const reverse = argumentsToValue((v: any[]) => [...toArray(v)].reverse()) +export const reverse = argumentsToValue(function (this: FilterImpl, v: any[]) { + const array = toArray(v) + this.context.memoryLimit.use(array.length) + return [...array].reverse() +}) export function * sort (this: FilterImpl, arr: T[], property?: string): IterableIterator { const values: [T, string | number][] = [] - for (const item of toArray(arr)) { + const array = toArray(arr) + this.context.memoryLimit.use(array.length) + for (const item of array) { values.push([ item, property ? yield this.context._getFromScope(item, stringify(property).split('.'), false) : item @@ -24,19 +36,23 @@ export function * sort (this: FilterImpl, arr: T[], property?: string): Itera }).map(tuple => tuple[0]) } -export function sort_natural (input: T[], property?: string) { +export function sort_natural (this: FilterImpl, input: T[], property?: string) { const propertyString = stringify(property) const compare = property === undefined ? caseInsensitiveCompare : (lhs: T, rhs: T) => caseInsensitiveCompare(lhs[propertyString], rhs[propertyString]) - return [...toArray(input)].sort(compare) + const array = toArray(input) + this.context.memoryLimit.use(array.length) + return [...array].sort(compare) } export const size = (v: string | any[]) => (v && v.length) || 0 export function * map (this: FilterImpl, arr: Scope[], property: string): IterableIterator { const results = [] - for (const item of toArray(arr)) { + const array = toArray(arr) + this.context.memoryLimit.use(array.length) + for (const item of array) { results.push(yield this.context._getFromScope(item, stringify(property), false)) } return results @@ -44,7 +60,8 @@ export function * map (this: FilterImpl, arr: Scope[], property: string): Iterab export function * sum (this: FilterImpl, arr: Scope[], property?: string): IterableIterator { let sum = 0 - for (const item of toArray(arr)) { + const array = toArray(arr) + for (const item of array) { const data = Number(property ? yield this.context._getFromScope(item, stringify(property), false) : item) sum += Number.isNaN(data) ? 0 : data } @@ -52,19 +69,26 @@ export function * sum (this: FilterImpl, arr: Scope[], property?: string): Itera } export function compact (this: FilterImpl, arr: T[]) { - return toArray(arr).filter(x => !isNil(toValue(x))) + const array = toArray(arr) + this.context.memoryLimit.use(array.length) + return array.filter(x => !isNil(toValue(x))) } -export function concat (v: T1[], arg: T2[] = []): (T1 | T2)[] { - return toArray(v).concat(toArray(arg)) +export function concat (this: FilterImpl, v: T1[], arg: T2[] = []): (T1 | T2)[] { + const lhs = toArray(v) + const rhs = toArray(arg) + this.context.memoryLimit.use(lhs.length + rhs.length) + return lhs.concat(rhs) } -export function push (v: T[], arg: T): T[] { - return concat(v, [arg]) +export function push (this: FilterImpl, v: T[], arg: T): T[] { + return concat.call(this, v, [arg]) as T[] } -export function unshift (v: T[], arg: T): T[] { - const clone = [...toArray(v)] +export function unshift (this: FilterImpl, v: T[], arg: T): T[] { + const array = toArray(v) + this.context.memoryLimit.use(array.length) + const clone = [...array] clone.unshift(arg) return clone } @@ -75,26 +99,30 @@ export function pop (v: T[]): T[] { return clone } -export function shift (v: T[]): T[] { - const clone = [...toArray(v)] +export function shift (this: FilterImpl, v: T[]): T[] { + const array = toArray(v) + this.context.memoryLimit.use(array.length) + const clone = [...array] clone.shift() return clone } -export function slice (v: T[] | string, begin: number, length = 1): T[] | string { +export function slice (this: FilterImpl, v: T[] | string, begin: number, length = 1): T[] | string { v = toValue(v) if (isNil(v)) return [] if (!isArray(v)) v = stringify(v) begin = begin < 0 ? v.length + begin : begin + this.context.memoryLimit.use(length) return v.slice(begin, begin + length) } export function * where (this: FilterImpl, arr: T[], property: string, expected?: any): IterableIterator { const values: unknown[] = [] arr = toArray(arr) + this.context.memoryLimit.use(arr.length) const token = new Tokenizer(stringify(property)).readScopeValue() for (const item of arr) { - values.push(yield evalToken(token, new Context(item))) + values.push(yield evalToken(token, this.context.spawn(item))) } return arr.filter((_, i) => { if (expected === undefined) return isTruthy(values[i], this.context) @@ -105,19 +133,22 @@ export function * where (this: FilterImpl, arr: T[], property: export function * where_exp (this: FilterImpl, arr: T[], itemName: string, exp: string): IterableIterator { const filtered: unknown[] = [] const keyTemplate = new Value(stringify(exp), this.liquid) - for (const item of toArray(arr)) { - const value = yield keyTemplate.value(new Context({ [itemName]: item })) + const array = toArray(arr) + this.context.memoryLimit.use(array.length) + for (const item of array) { + const value = yield keyTemplate.value(this.context.spawn({ [itemName]: item })) if (value) filtered.push(item) } return filtered } -export function * group_by (arr: T[], property: string): IterableIterator { +export function * group_by (this: FilterImpl, arr: T[], property: string): IterableIterator { const map = new Map() arr = toArray(arr) const token = new Tokenizer(stringify(property)).readScopeValue() + this.context.memoryLimit.use(arr.length) for (const item of arr) { - const key = yield evalToken(token, new Context(item)) + const key = yield evalToken(token, this.context.spawn(item)) if (!map.has(key)) map.set(key, []) map.get(key).push(item) } @@ -127,8 +158,10 @@ export function * group_by (arr: T[], property: string): Itera export function * group_by_exp (this: FilterImpl, arr: T[], itemName: string, exp: string): IterableIterator { const map = new Map() const keyTemplate = new Value(stringify(exp), this.liquid) - for (const item of toArray(arr)) { - const key = yield keyTemplate.value(new Context({ [itemName]: item })) + arr = toArray(arr) + this.context.memoryLimit.use(arr.length) + for (const item of arr) { + const key = yield keyTemplate.value(this.context.spawn({ [itemName]: item })) if (!map.has(key)) map.set(key, []) map.get(key).push(item) } @@ -137,8 +170,9 @@ export function * group_by_exp (this: FilterImpl, arr: T[], it export function * find (this: FilterImpl, arr: T[], property: string, expected: string): IterableIterator { const token = new Tokenizer(stringify(property)).readScopeValue() - for (const item of toArray(arr)) { - const value = yield evalToken(token, new Context(item)) + const array = toArray(arr) + for (const item of array) { + const value = yield evalToken(token, this.context.spawn(item)) if (equals(value, expected)) return item } return null @@ -146,15 +180,17 @@ export function * find (this: FilterImpl, arr: T[], property: export function * find_exp (this: FilterImpl, arr: T[], itemName: string, exp: string): IterableIterator { const predicate = new Value(stringify(exp), this.liquid) - for (const item of toArray(arr)) { - const value = yield predicate.value(new Context({ [itemName]: item })) + const array = toArray(arr) + for (const item of array) { + const value = yield predicate.value(this.context.spawn({ [itemName]: item })) if (value) return item } return null } -export function uniq (arr: T[]): T[] { +export function uniq (this: FilterImpl, arr: T[]): T[] { arr = toValue(arr) + this.context.memoryLimit.use(arr.length) const u = {} return (arr || []).filter(val => { if (hasOwnProperty.call(u, String(val))) return false @@ -163,10 +199,11 @@ export function uniq (arr: T[]): T[] { }) } -export function sample (v: T[] | string, count = 1): T | string | (T | string)[] { +export function sample (this: FilterImpl, v: T[] | string, count = 1): T | string | (T | string)[] { v = toValue(v) if (isNil(v)) return [] if (!isArray(v)) v = stringify(v) + this.context.memoryLimit.use(count) const shuffled = [...v].sort(() => Math.random() - 0.5) if (count === 1) return shuffled[0] return shuffled.slice(0, count) diff --git a/src/filters/date.ts b/src/filters/date.ts index eede5fa915..9a346f86d2 100644 --- a/src/filters/date.ts +++ b/src/filters/date.ts @@ -3,6 +3,8 @@ import { FilterImpl } from '../template' import { LiquidOptions } from '../liquid-options' export function date (this: FilterImpl, v: string | Date, format?: string, timezoneOffset?: number | string) { + const size = ((v as string)?.length ?? 0) + (format?.length ?? 0) + ((timezoneOffset as string)?.length ?? 0) + this.context.memoryLimit.use(size) const date = parseDate(v, this.context.opts, timezoneOffset) if (!date) return v format = toValue(format) diff --git a/src/filters/html.ts b/src/filters/html.ts index 0dd14d8845..3121857e3b 100644 --- a/src/filters/html.ts +++ b/src/filters/html.ts @@ -1,3 +1,4 @@ +import { FilterImpl } from '../template' import { stringify } from '../util/underscore' const escapeMap = { @@ -15,26 +16,34 @@ const unescapeMap = { ''': "'" } -export function escape (str: string) { - return stringify(str).replace(/&|<|>|"|'/g, m => escapeMap[m]) +export function escape (this: FilterImpl, str: string) { + str = stringify(str) + this.context.memoryLimit.use(str.length) + return str.replace(/&|<|>|"|'/g, m => escapeMap[m]) } -export function xml_escape (str: string) { - return escape(str) +export function xml_escape (this: FilterImpl, str: string) { + return escape.call(this, str) } -function unescape (str: string) { - return stringify(str).replace(/&(amp|lt|gt|#34|#39);/g, m => unescapeMap[m]) +function unescape (this: FilterImpl, str: string) { + str = stringify(str) + this.context.memoryLimit.use(str.length) + return str.replace(/&(amp|lt|gt|#34|#39);/g, m => unescapeMap[m]) } -export function escape_once (str: string) { - return escape(unescape(stringify(str))) +export function escape_once (this: FilterImpl, str: string) { + return escape.call(this, unescape.call(this, str)) } -export function newline_to_br (v: string) { - return stringify(v).replace(/\r?\n/gm, '
\n') +export function newline_to_br (this: FilterImpl, v: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + return str.replace(/\r?\n/gm, '
\n') } -export function strip_html (v: string) { - return stringify(v).replace(/||<.*?>|/g, '') +export function strip_html (this: FilterImpl, v: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + return str.replace(/||<.*?>|/g, '') } diff --git a/src/filters/string.ts b/src/filters/string.ts index 0a431b352a..ec163d717f 100644 --- a/src/filters/string.ts +++ b/src/filters/string.ts @@ -10,6 +10,7 @@ // Katakana (Japanese): \u30A0-\u30FF // Hiragana (Japanese): \u3040-\u309F // Hangul (Korean): \uAC00-\uD7AF +import { FilterImpl } from '../template' import { assert, escapeRegExp, stringify } from '../util' const rCJKWord = /[\u4E00-\u9FFF\uF900-\uFAFF\u3400-\u4DBF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF]/gu @@ -17,93 +18,124 @@ const rCJKWord = /[\u4E00-\u9FFF\uF900-\uFAFF\u3400-\u4DBF\u3040-\u309F\u30A0-\u // Word boundary followed by word characters (for detecting words) const rNonCJKWord = /[^\u4E00-\u9FFF\uF900-\uFAFF\u3400-\u4DBF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF\s]+/gu -export function append (v: string, arg: string) { +export function append (this: FilterImpl, v: string, arg: string) { assert(arguments.length === 2, 'append expect 2 arguments') - return stringify(v) + stringify(arg) + const lhs = stringify(v) + const rhs = stringify(arg) + this.context.memoryLimit.use(lhs.length + rhs.length) + return lhs + rhs } -export function prepend (v: string, arg: string) { +export function prepend (this: FilterImpl, v: string, arg: string) { assert(arguments.length === 2, 'prepend expect 2 arguments') - return stringify(arg) + stringify(v) + const lhs = stringify(v) + const rhs = stringify(arg) + this.context.memoryLimit.use(lhs.length + rhs.length) + return rhs + lhs } -export function lstrip (v: string, chars?: string) { +export function lstrip (this: FilterImpl, v: string, chars?: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) if (chars) { chars = escapeRegExp(stringify(chars)) - return stringify(v).replace(new RegExp(`^[${chars}]+`, 'g'), '') + return str.replace(new RegExp(`^[${chars}]+`, 'g'), '') } - return stringify(v).replace(/^\s+/, '') + return str.replace(/^\s+/, '') } -export function downcase (v: string) { - return stringify(v).toLowerCase() +export function downcase (this: FilterImpl, v: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + return str.toLowerCase() } -export function upcase (str: string) { +export function upcase (this: FilterImpl, v: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) return stringify(str).toUpperCase() } -export function remove (v: string, arg: string) { - return stringify(v).split(stringify(arg)).join('') +export function remove (this: FilterImpl, v: string, arg: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + return str.split(stringify(arg)).join('') } -export function remove_first (v: string, l: string) { - return stringify(v).replace(stringify(l), '') +export function remove_first (this: FilterImpl, v: string, l: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + return str.replace(stringify(l), '') } -export function remove_last (v: string, l: string) { +export function remove_last (this: FilterImpl, v: string, l: string) { const str = stringify(v) + this.context.memoryLimit.use(str.length) const pattern = stringify(l) const index = str.lastIndexOf(pattern) if (index === -1) return str return str.substring(0, index) + str.substring(index + pattern.length) } -export function rstrip (str: string, chars?: string) { +export function rstrip (this: FilterImpl, str: string, chars?: string) { + str = stringify(str) + this.context.memoryLimit.use(str.length) if (chars) { chars = escapeRegExp(stringify(chars)) - return stringify(str).replace(new RegExp(`[${chars}]+$`, 'g'), '') + return str.replace(new RegExp(`[${chars}]+$`, 'g'), '') } - return stringify(str).replace(/\s+$/, '') + return str.replace(/\s+$/, '') } -export function split (v: string, arg: string) { - const arr = stringify(v).split(stringify(arg)) +export function split (this: FilterImpl, v: string, arg: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + const arr = str.split(stringify(arg)) // align to ruby split, which is the behavior of shopify/liquid // see: https://ruby-doc.org/core-2.4.0/String.html#method-i-split while (arr.length && arr[arr.length - 1] === '') arr.pop() return arr } -export function strip (v: string, chars?: string) { +export function strip (this: FilterImpl, v: string, chars?: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) if (chars) { chars = escapeRegExp(stringify(chars)) - return stringify(v) + return str .replace(new RegExp(`^[${chars}]+`, 'g'), '') .replace(new RegExp(`[${chars}]+$`, 'g'), '') } - return stringify(v).trim() + return str.trim() } -export function strip_newlines (v: string) { - return stringify(v).replace(/\r?\n/gm, '') +export function strip_newlines (this: FilterImpl, v: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + return str.replace(/\r?\n/gm, '') } -export function capitalize (str: string) { +export function capitalize (this: FilterImpl, str: string) { str = stringify(str) + this.context.memoryLimit.use(str.length) return str.charAt(0).toUpperCase() + str.slice(1).toLowerCase() } -export function replace (v: string, pattern: string, replacement: string) { - return stringify(v).split(stringify(pattern)).join(replacement) +export function replace (this: FilterImpl, v: string, pattern: string, replacement: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + return str.split(stringify(pattern)).join(replacement) } -export function replace_first (v: string, arg1: string, arg2: string) { - return stringify(v).replace(stringify(arg1), arg2) +export function replace_first (this: FilterImpl, v: string, arg1: string, arg2: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + return str.replace(stringify(arg1), arg2) } -export function replace_last (v: string, arg1: string, arg2: string) { +export function replace_last (this: FilterImpl, v: string, arg1: string, arg2: string) { const str = stringify(v) + this.context.memoryLimit.use(str.length) const pattern = stringify(arg1) const index = str.lastIndexOf(pattern) if (index === -1) return str @@ -111,27 +143,33 @@ export function replace_last (v: string, arg1: string, arg2: string) { return str.substring(0, index) + replacement + str.substring(index + pattern.length) } -export function truncate (v: string, l = 50, o = '...') { - v = stringify(v) - if (v.length <= l) return v - return v.substring(0, l - o.length) + o +export function truncate (this: FilterImpl, v: string, l = 50, o = '...') { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + if (str.length <= l) return v + return str.substring(0, l - o.length) + o } -export function truncatewords (v: string, words = 15, o = '...') { - const arr = stringify(v).split(/\s+/) +export function truncatewords (this: FilterImpl, v: string, words = 15, o = '...') { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + const arr = str.split(/\s+/) if (words <= 0) words = 1 let ret = arr.slice(0, words).join(' ') if (arr.length >= words) ret += o return ret } -export function normalize_whitespace (v: string) { - v = stringify(v) - return v.replace(/\s+/g, ' ') +export function normalize_whitespace (this: FilterImpl, v: string) { + const str = stringify(v) + this.context.memoryLimit.use(str.length) + return str.replace(/\s+/g, ' ') } -export function number_of_words (input: string, mode?: 'cjk' | 'auto') { - input = stringify(input).trim() +export function number_of_words (this: FilterImpl, input: string, mode?: 'cjk' | 'auto') { + const str = stringify(input) + this.context.memoryLimit.use(str.length) + input = str.trim() if (!input) return 0 switch (mode) { case 'cjk': @@ -148,7 +186,8 @@ export function number_of_words (input: string, mode?: 'cjk' | 'auto') { } } -export function array_to_sentence_string (array: unknown[], connector = 'and') { +export function array_to_sentence_string (this: FilterImpl, array: unknown[], connector = 'and') { + this.context.memoryLimit.use(array.length) switch (array.length) { case 0: return '' diff --git a/src/fs/map-fs.ts b/src/fs/map-fs.ts index 78bc611253..aba5faab24 100644 --- a/src/fs/map-fs.ts +++ b/src/fs/map-fs.ts @@ -37,8 +37,7 @@ export class MapFS { if (segment === '.' || segment === '') continue else if (segment === '..') { if (segments.length > 1 || segments[0] !== '') segments.pop() - } - else segments.push(segment) + } else segments.push(segment) } return segments.join(this.sep) } diff --git a/src/liquid-options.ts b/src/liquid-options.ts index dc90332243..e76e2b3253 100644 --- a/src/liquid-options.ts +++ b/src/liquid-options.ts @@ -77,6 +77,12 @@ export interface LiquidOptions { operators?: Operators; /** Respect parameter order when using filters like "for ... reversed limit", Defaults to `false`. */ orderedFilterParameters?: boolean; + /** For DoS handling, limit total length of templates parsed in one `parse()` call. A typical PC can handle 1e8 (100M) characters without issues. */ + parseLimit?: number; + /** For DoS handling, limit total time (in ms) for each `render()` call. */ + renderLimit?: number; + /** For DoS handling, limit new objects creation, including array concat/join/strftime, etc. A typical PC can handle 1e9 (1G) memory without issue. */ + memoryLimit?: number; } export interface RenderOptions { @@ -96,6 +102,12 @@ export interface RenderOptions { * Same as `ownPropertyOnly` on LiquidOptions, but only for current render() call */ ownPropertyOnly?: boolean; + /** For DoS handling, limit total renders of tag/HTML/output in one `render()` call. A typical PC can handle 1e5 renders of typical templates per second. */ + templateLimit?: number; + /** For DoS handling, limit total time (in ms) for each `render()` call. */ + renderLimit?: number; + /** For DoS handling, limit new objects creation, including array concat/join/strftime, etc. A typical PC can handle 1e9 (1G) memory without issue.. */ + memoryLimit?: number; } export interface RenderFileOptions extends RenderOptions { @@ -139,6 +151,9 @@ export interface NormalizedFullOptions extends NormalizedOptions { globals: object; keepOutputType: boolean; operators: Operators; + parseLimit: number; + renderLimit: number; + memoryLimit: number; } export const defaultOptions: NormalizedFullOptions = { @@ -169,7 +184,10 @@ export const defaultOptions: NormalizedFullOptions = { lenientIf: false, globals: {}, keepOutputType: false, - operators: defaultOperators + operators: defaultOperators, + memoryLimit: Infinity, + parseLimit: Infinity, + renderLimit: Infinity } export function normalize (options: LiquidOptions): NormalizedFullOptions { @@ -193,6 +211,9 @@ export function normalize (options: LiquidOptions): NormalizedFullOptions { options.partials = normalizeDirectoryList(options.partials) options.layouts = normalizeDirectoryList(options.layouts) options.outputEscape = options.outputEscape && getOutputEscapeFunction(options.outputEscape) + options.parseLimit = options.parseLimit || Infinity + options.renderLimit = options.renderLimit || Infinity + options.memoryLimit = options.memoryLimit || Infinity if (options.templates) { options.fs = new MapFS(options.templates) options.relativeReference = true diff --git a/src/liquid.ts b/src/liquid.ts index 0336fa3c48..49c4082ed2 100644 --- a/src/liquid.ts +++ b/src/liquid.ts @@ -11,18 +11,23 @@ import { LiquidOptions, normalizeDirectoryList, NormalizedFullOptions, normalize export class Liquid { public readonly options: NormalizedFullOptions public readonly renderer = new Render() + /** + * @deprecated will be removed. In tags use `this.parser` instead + */ public readonly parser: Parser public readonly filters: Record = {} public readonly tags: Record = {} public constructor (opts: LiquidOptions = {}) { this.options = normalize(opts) + // eslint-disable-next-line deprecation/deprecation this.parser = new Parser(this) forOwn(tags, (conf: TagClass, name: string) => this.registerTag(name, conf)) forOwn(filters, (handler: FilterImplOptions, name: string) => this.registerFilter(name, handler)) } public parse (html: string, filepath?: string): Template[] { - return this.parser.parse(html, filepath) + const parser = new Parser(this) + return parser.parse(html, filepath) } public _render (tpl: Template[], scope: Context | object | undefined, renderOptions: RenderOptions): IterableIterator { @@ -52,19 +57,19 @@ export class Liquid { } public _parsePartialFile (file: string, sync?: boolean, currentFile?: string) { - return this.parser.parseFile(file, sync, LookupType.Partials, currentFile) + return new Parser(this).parseFile(file, sync, LookupType.Partials, currentFile) } public _parseLayoutFile (file: string, sync?: boolean, currentFile?: string) { - return this.parser.parseFile(file, sync, LookupType.Layouts, currentFile) + return new Parser(this).parseFile(file, sync, LookupType.Layouts, currentFile) } public _parseFile (file: string, sync?: boolean, lookupType?: LookupType, currentFile?: string): Generator { - return this.parser.parseFile(file, sync, lookupType, currentFile) + return new Parser(this).parseFile(file, sync, lookupType, currentFile) } public async parseFile (file: string, lookupType?: LookupType): Promise { - return toPromise(this.parser.parseFile(file, false, lookupType)) + return toPromise(new Parser(this).parseFile(file, false, lookupType)) } public parseFileSync (file: string, lookupType?: LookupType): Template[] { - return toValueSync(this.parser.parseFile(file, true, lookupType)) + return toValueSync(new Parser(this).parseFile(file, true, lookupType)) } public * _renderFile (file: string, ctx: Context | object | undefined, renderFileOptions: RenderFileOptions): Generator { const templates = (yield this._parseFile(file, renderFileOptions.sync, renderFileOptions.lookupType)) as Template[] diff --git a/src/parser/parser.ts b/src/parser/parser.ts index e1b30ebbfc..44084c2841 100644 --- a/src/parser/parser.ts +++ b/src/parser/parser.ts @@ -1,4 +1,4 @@ -import { toPromise, assert, isTagToken, isOutputToken, ParseError } from '../util' +import { Limiter, toPromise, assert, isTagToken, isOutputToken, ParseError } from '../util' import { Tokenizer } from './tokenizer' import { ParseStream } from './parse-stream' import { TopLevelToken, OutputToken } from '../tokens' @@ -15,6 +15,7 @@ export class Parser { private fs: FS private cache?: LiquidCache private loader: Loader + private parseLimit: Limiter public constructor (liquid: Liquid) { this.liquid = liquid @@ -22,8 +23,10 @@ export class Parser { this.fs = this.liquid.options.fs this.parseFile = this.cache ? this._parseFileCached : this._parseFile this.loader = new Loader(this.liquid.options) + this.parseLimit = new Limiter('parse length', liquid.options.parseLimit) } public parse (html: string, filepath?: string): Template[] { + this.parseLimit.use(html.length) const tokenizer = new Tokenizer(html, this.liquid.options.operators, filepath) const tokens = tokenizer.readTopLevelTokens(this.liquid.options) return this.parseTokens(tokens) @@ -48,7 +51,7 @@ export class Parser { if (isTagToken(token)) { const TagClass = this.liquid.tags[token.name] assert(TagClass, `tag "${token.name}" not found`) - return new TagClass(token, remainTokens, this.liquid) + return new TagClass(token, remainTokens, this.liquid, this) } if (isOutputToken(token)) { return new Output(token as OutputToken, this.liquid) @@ -78,6 +81,6 @@ export class Parser { } private * _parseFile (file: string, sync?: boolean, type: LookupType = LookupType.Root, currentFile?: string): Generator { const filepath = yield this.loader.lookup(file, type, sync, currentFile) - return this.liquid.parse(sync ? this.fs.readFileSync(filepath) : yield this.fs.readFile(filepath), filepath) + return this.parse(sync ? this.fs.readFileSync(filepath) : yield this.fs.readFile(filepath), filepath) } } diff --git a/src/render/render.ts b/src/render/render.ts index 4949f729f6..2705815781 100644 --- a/src/render/render.ts +++ b/src/render/render.ts @@ -16,6 +16,7 @@ export class Render { } const errors = [] for (const tpl of templates) { + ctx.renderLimit.check(performance.now()) try { // if tpl.render supports emitter, it'll return empty `html` const html = yield tpl.render(ctx, emitter) diff --git a/src/tags/block.ts b/src/tags/block.ts index 450d6c1364..a246ae34fa 100644 --- a/src/tags/block.ts +++ b/src/tags/block.ts @@ -2,18 +2,19 @@ import { BlockMode } from '../context' import { isTagToken } from '../util' import { BlockDrop } from '../drop' import { Liquid, TagToken, TopLevelToken, Template, Context, Emitter, Tag } from '..' +import { Parser } from '../parser' export default class extends Tag { block: string templates: Template[] = [] - constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) { + constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(token, remainTokens, liquid) const match = /\w+/.exec(token.args) this.block = match ? match[0] : '' while (remainTokens.length) { const token = remainTokens.shift()! if (isTagToken(token) && token.name === 'endblock') return - const template = liquid.parser.parseToken(token, remainTokens) + const template = parser.parseToken(token, remainTokens) this.templates.push(template) } throw new Error(`tag ${token.getText()} not closed`) diff --git a/src/tags/capture.ts b/src/tags/capture.ts index deb493c00c..8221bf750b 100644 --- a/src/tags/capture.ts +++ b/src/tags/capture.ts @@ -1,18 +1,19 @@ import { Liquid, Tag, Template, Context, TagToken, TopLevelToken } from '..' +import { Parser } from '../parser' import { evalQuotedToken } from '../render' import { isTagToken } from '../util' export default class extends Tag { variable: string templates: Template[] = [] - constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) { + constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(tagToken, remainTokens, liquid) this.variable = this.readVariableName() while (remainTokens.length) { const token = remainTokens.shift()! if (isTagToken(token) && token.name === 'endcapture') return - this.templates.push(liquid.parser.parseToken(token, remainTokens)) + this.templates.push(parser.parseToken(token, remainTokens)) } throw new Error(`tag ${tagToken.getText()} not closed`) } diff --git a/src/tags/case.ts b/src/tags/case.ts index d8bff9e5a1..0cf99581fd 100644 --- a/src/tags/case.ts +++ b/src/tags/case.ts @@ -1,18 +1,19 @@ import { ValueToken, Liquid, toValue, evalToken, Value, Emitter, TagToken, TopLevelToken, Context, Template, Tag, ParseStream } from '..' +import { Parser } from '../parser' import { equals } from '../render' export default class extends Tag { value: Value branches: { values: ValueToken[], templates: Template[] }[] = [] elseTemplates: Template[] = [] - constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) { + constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(tagToken, remainTokens, liquid) this.value = new Value(this.tokenizer.readFilteredValue(), this.liquid) this.elseTemplates = [] let p: Template[] = [] let elseCount = 0 - const stream: ParseStream = this.liquid.parser.parseStream(remainTokens) + const stream: ParseStream = parser.parseStream(remainTokens) .on('tag:when', (token: TagToken) => { if (elseCount > 0) { return diff --git a/src/tags/for.ts b/src/tags/for.ts index 53f76e7485..9c1581f4c4 100644 --- a/src/tags/for.ts +++ b/src/tags/for.ts @@ -1,6 +1,7 @@ import { Hash, ValueToken, Liquid, Tag, evalToken, Emitter, TagToken, TopLevelToken, Context, Template, ParseStream } from '..' import { assertEmpty, toEnumerable } from '../util' import { ForloopDrop } from '../drop/forloop-drop' +import { Parser } from '../parser' const MODIFIERS = ['offset', 'limit', 'reversed'] @@ -13,7 +14,7 @@ export default class extends Tag { templates: Template[] elseTemplates: Template[] - constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) { + constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(token, remainTokens, liquid) const variable = this.tokenizer.readIdentifier() const inStr = this.tokenizer.readIdentifier() @@ -29,7 +30,7 @@ export default class extends Tag { this.elseTemplates = [] let p - const stream: ParseStream = this.liquid.parser.parseStream(remainTokens) + const stream: ParseStream = parser.parseStream(remainTokens) .on('start', () => (p = this.templates)) .on('tag:else', tag => { assertEmpty(tag.args); p = this.elseTemplates }) .on('tag:endfor', tag => { assertEmpty(tag.args); stream.stop() }) diff --git a/src/tags/if.ts b/src/tags/if.ts index 45114d0f70..702b918b99 100644 --- a/src/tags/if.ts +++ b/src/tags/if.ts @@ -1,14 +1,15 @@ import { Liquid, Tag, Value, Emitter, isTruthy, TagToken, TopLevelToken, Context, Template } from '..' +import { Parser } from '../parser' import { assert, assertEmpty } from '../util' export default class extends Tag { branches: { value: Value, templates: Template[] }[] = [] elseTemplates: Template[] | undefined - constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) { + constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(tagToken, remainTokens, liquid) let p: Template[] = [] - liquid.parser.parseStream(remainTokens) + parser.parseStream(remainTokens) .on('start', () => this.branches.push({ value: new Value(tagToken.args, this.liquid), templates: (p = []) diff --git a/src/tags/include.ts b/src/tags/include.ts index d433199f7f..909e0da7a6 100644 --- a/src/tags/include.ts +++ b/src/tags/include.ts @@ -1,14 +1,15 @@ import { Template, ValueToken, TopLevelToken, Liquid, Tag, assert, evalToken, Hash, Emitter, TagToken, Context } from '..' import { BlockMode, Scope } from '../context' +import { Parser } from '../parser' import { parseFilePath, renderFilePath } from './render' export default class extends Tag { private withVar?: ValueToken private hash: Hash - constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) { + constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(token, remainTokens, liquid) const { tokenizer } = token - this['file'] = parseFilePath(tokenizer, this.liquid) + this['file'] = parseFilePath(tokenizer, this.liquid, parser) this['currentFile'] = token.file const begin = tokenizer.p diff --git a/src/tags/layout.ts b/src/tags/layout.ts index d0b612dd22..e928adf118 100644 --- a/src/tags/layout.ts +++ b/src/tags/layout.ts @@ -2,17 +2,18 @@ import { Scope, Template, Liquid, Tag, assert, Emitter, Hash, TagToken, TopLevel import { BlockMode } from '../context' import { parseFilePath, renderFilePath, ParsedFileName } from './render' import { BlankDrop } from '../drop' +import { Parser } from '../parser' export default class extends Tag { args: Hash templates: Template[] file?: ParsedFileName - constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) { + constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(token, remainTokens, liquid) - this.file = parseFilePath(this.tokenizer, this.liquid) + this.file = parseFilePath(this.tokenizer, this.liquid, parser) this['currentFile'] = token.file this.args = new Hash(this.tokenizer.remaining()) - this.templates = this.liquid.parser.parseTokens(remainTokens) + this.templates = parser.parseTokens(remainTokens) } * render (ctx: Context, emitter: Emitter): Generator { const { liquid, args, file } = this diff --git a/src/tags/liquid.ts b/src/tags/liquid.ts index 7479591464..25ecb383c4 100644 --- a/src/tags/liquid.ts +++ b/src/tags/liquid.ts @@ -1,11 +1,12 @@ import { Template, Emitter, Liquid, TopLevelToken, TagToken, Context, Tag } from '..' +import { Parser } from '../parser' export default class extends Tag { templates: Template[] - constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) { + constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(token, remainTokens, liquid) const tokens = this.tokenizer.readLiquidTagTokens(this.liquid.options) - this.templates = this.liquid.parser.parseTokens(tokens) + this.templates = parser.parseTokens(tokens) } * render (ctx: Context, emitter: Emitter): Generator { yield this.liquid.renderer.renderTemplates(this.templates, ctx, emitter) diff --git a/src/tags/render.ts b/src/tags/render.ts index 395d7dbe3c..1fff29238e 100644 --- a/src/tags/render.ts +++ b/src/tags/render.ts @@ -2,6 +2,7 @@ import { __assign } from 'tslib' import { ForloopDrop } from '../drop' import { toEnumerable } from '../util' import { TopLevelToken, assert, Liquid, Token, Template, evalQuotedToken, TypeGuards, Tokenizer, evalToken, Hash, Emitter, TagToken, Context, Tag } from '..' +import { Parser } from '../parser' export type ParsedFileName = Template[] | Token | string | undefined @@ -9,10 +10,10 @@ export default class extends Tag { private file: ParsedFileName private currentFile?: string private hash: Hash - constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) { + constructor (token: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(token, remainTokens, liquid) const tokenizer = this.tokenizer - this.file = parseFilePath(tokenizer, this.liquid) + this.file = parseFilePath(tokenizer, this.liquid, parser) this.currentFile = token.file while (!tokenizer.end()) { tokenizer.skipBlank() @@ -51,7 +52,7 @@ export default class extends Tag { const filepath = (yield renderFilePath(this['file'], ctx, liquid)) as string assert(filepath, () => `illegal file path "${filepath}"`) - const childCtx = new Context({}, ctx.opts, { sync: ctx.sync, globals: ctx.globals, strictVariables: ctx.strictVariables }) + const childCtx = ctx.spawn() const scope = childCtx.bottom() __assign(scope, yield hash.render(ctx)) if (this['with']) { @@ -82,20 +83,20 @@ export default class extends Tag { * @return Token for expression (not quoted) * @throws TypeError if cannot read next token */ -export function parseFilePath (tokenizer: Tokenizer, liquid: Liquid): ParsedFileName { +export function parseFilePath (tokenizer: Tokenizer, liquid: Liquid, parser: Parser): ParsedFileName { if (liquid.options.dynamicPartials) { const file = tokenizer.readValue() tokenizer.assert(file, 'illegal file path') if (file!.getText() === 'none') return if (TypeGuards.isQuotedToken(file)) { // for filenames like "files/{{file}}", eval as liquid template - const templates = liquid.parse(evalQuotedToken(file)) + const templates = parser.parse(evalQuotedToken(file)) return optimize(templates) } return file } const tokens = [...tokenizer.readFileNameTemplate(liquid.options)] - const templates = optimize(liquid.parser.parseTokens(tokens)) + const templates = optimize(parser.parseTokens(tokens)) return templates === 'none' ? undefined : templates } diff --git a/src/tags/tablerow.ts b/src/tags/tablerow.ts index 59e167a490..454506b778 100644 --- a/src/tags/tablerow.ts +++ b/src/tags/tablerow.ts @@ -1,13 +1,14 @@ import { toEnumerable } from '../util' import { ValueToken, Liquid, Tag, evalToken, Emitter, Hash, TagToken, TopLevelToken, Context, Template, ParseStream } from '..' import { TablerowloopDrop } from '../drop/tablerowloop-drop' +import { Parser } from '../parser' export default class extends Tag { variable: string args: Hash templates: Template[] collection: ValueToken - constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) { + constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(tagToken, remainTokens, liquid) const variable = this.tokenizer.readIdentifier() this.tokenizer.skipBlank() @@ -24,7 +25,7 @@ export default class extends Tag { this.templates = [] let p - const stream: ParseStream = this.liquid.parser.parseStream(remainTokens) + const stream: ParseStream = parser.parseStream(remainTokens) .on('start', () => (p = this.templates)) .on('tag:endtablerow', () => stream.stop()) .on('template', (tpl: Template) => p.push(tpl)) diff --git a/src/tags/unless.ts b/src/tags/unless.ts index 56216aa5c7..cd7cf9f3f9 100644 --- a/src/tags/unless.ts +++ b/src/tags/unless.ts @@ -1,13 +1,14 @@ import { Liquid, Tag, Value, TopLevelToken, Template, Emitter, isTruthy, isFalsy, Context, TagToken } from '..' +import { Parser } from '../parser' export default class extends Tag { branches: { value: Value, test: (val: any, ctx: Context) => boolean, templates: Template[] }[] = [] elseTemplates: Template[] = [] - constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid) { + constructor (tagToken: TagToken, remainTokens: TopLevelToken[], liquid: Liquid, parser: Parser) { super(tagToken, remainTokens, liquid) let p: Template[] = [] let elseCount = 0 - this.liquid.parser.parseStream(remainTokens) + parser.parseStream(remainTokens) .on('start', () => this.branches.push({ value: new Value(tagToken.args, this.liquid), test: isFalsy, diff --git a/src/template/tag.ts b/src/template/tag.ts index 6f51752499..a8b0fa0886 100644 --- a/src/template/tag.ts +++ b/src/template/tag.ts @@ -1,6 +1,6 @@ import { TemplateImpl } from './template-impl' import type { Emitter } from '../emitters/emitter' -import type { Tokenizer } from '../parser' +import type { Parser, Tokenizer } from '../parser' import type { Context } from '../context/context' import type { TopLevelToken, TagToken } from '../tokens' import type { Template } from './template' @@ -23,5 +23,5 @@ export abstract class Tag extends TemplateImpl implements Template { } export interface TagClass { - new(token: TagToken, tokens: TopLevelToken[], liquid: Liquid): Tag + new(token: TagToken, tokens: TopLevelToken[], liquid: Liquid, parser: Parser): Tag } diff --git a/src/util/index.ts b/src/util/index.ts index 8b57690b66..fc5c2c9b13 100644 --- a/src/util/index.ts +++ b/src/util/index.ts @@ -9,3 +9,4 @@ export * from './async' export * from './strftime' export * from './liquid-date' export * from './timezone-date' +export * from './limiter' diff --git a/src/util/limiter.ts b/src/util/limiter.ts new file mode 100644 index 0000000000..593e3b6931 --- /dev/null +++ b/src/util/limiter.ts @@ -0,0 +1,18 @@ +import { assert } from './assert' + +export class Limiter { + private message: string + private base = 0 + private limit: number + constructor (resource: string, limit: number) { + this.message = `${resource} limit exceeded` + this.limit = limit + } + use (count: number) { + assert(this.base + count <= this.limit, this.message) + this.base += count + } + check (count: number) { + assert(count <= this.limit, this.message) + } +} diff --git a/src/util/underscore.ts b/src/util/underscore.ts index c851ef6d0e..297bcb2c90 100644 --- a/src/util/underscore.ts +++ b/src/util/underscore.ts @@ -178,8 +178,8 @@ export function caseInsensitiveCompare (a: any, b: any) { return 0 } -export function argumentsToValue any> (fn: F) { - return (...args: Parameters) => fn(...args.map(toValue)) +export function argumentsToValue any, T> (fn: F) { + return function (this: T, ...args: Parameters) { return fn.call(this, ...args.map(toValue)) } } export function escapeRegExp (text: string) { diff --git a/test/integration/liquid/dos.spec.ts b/test/integration/liquid/dos.spec.ts new file mode 100644 index 0000000000..c3f6641aa8 --- /dev/null +++ b/test/integration/liquid/dos.spec.ts @@ -0,0 +1,61 @@ +import { Liquid } from '../../../src/liquid' +import { mock, restore } from '../../stub/mockfs' + +describe('DoS related', function () { + describe('#parseLimit', function () { + afterEach(restore) + it('should throw when parse limit exceeded', async () => { + const noLimit = new Liquid() + const limit10 = new Liquid({ parseLimit: 10 }) + const limit90 = new Liquid({ parseLimit: 90 }) + const template = '{% capture bar %}{{ foo | bar: 3, a[3] }}{% endcapture %}' + await expect(noLimit.parseAndRender(template)).resolves.toBe('') + await expect(limit10.parseAndRender(template)).rejects.toThrow('parse length limit exceeded') + await expect(limit90.parseAndRender(template)).resolves.toBe('') + }) + it('should take included template into account', async () => { + mock({ + '/small': 'Lorem ipsum', + '/large': 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.' + }) + const liquid = new Liquid({ root: '/', parseLimit: 50 }) + await expect(liquid.parseAndRender('{% include "small" %}')).resolves.toBe('Lorem ipsum') + await expect(liquid.parseAndRender('{% include "large" %}')).rejects.toThrow('parse length limit exceeded') + }) + }) + describe('#renderLimit', () => { + it('should throw when rendering too many templates', async () => { + const src = '{% for i in (1..5) %}{{i}},{% endfor %}' + const noLimit = new Liquid() + const limit10 = new Liquid({ renderLimit: 0.01 }) + const limit20 = new Liquid({ renderLimit: 1000 }) + await expect(noLimit.parseAndRender(src)).resolves.toBe('1,2,3,4,5,') + await expect(limit10.parseAndRender(src)).rejects.toThrow('template render limit exceeded') + await expect(limit20.parseAndRender(src)).resolves.toBe('1,2,3,4,5,') + }) + it('should take partials into account', async () => { + mock({ + '/small': '{% for i in (1..5) %}{{i}}{% endfor %}', + '/large': '{% for i in (1..50000000) %}{{i}}{% endfor %}' + }) + const liquid = new Liquid({ root: '/', renderLimit: 100 }) + await expect(liquid.parseAndRender('{% render "large" %}')).rejects.toThrow('template render limit exceeded') + await expect(liquid.parseAndRender('{% render "small" %}')).resolves.toBe('12345') + }) + }) + describe('#memoryLimit', () => { + it('should throw for too many array creation in filters', async () => { + const array = Array(1e3).fill(0) + const liquid = new Liquid({ memoryLimit: 100 }) + await expect(liquid.parseAndRender('{{ array | slice: 0, 3 | join }}', { array })).resolves.toBe('0 0 0') + await expect(liquid.parseAndRender('{{ array | slice: 0, 300 | join }}', { array })).rejects.toThrow('memory alloc limit exceeded, line:1, col:1') + }) + it('should throw for too many array iteration in tags', async () => { + const array = ['a'] + const liquid = new Liquid({ memoryLimit: 100 }) + const src = '{% for i in (1..count) %}{% assign array = array | concat: array %}{% endfor %}{{ array | join }}' + await expect(liquid.parseAndRender(src, { array, count: 3 })).resolves.toBe('a a a a a a a a') + await expect(liquid.parseAndRender(src, { array, count: 100 })).rejects.toThrow('memory alloc limit exceeded, line:1, col:26') + }) + }) +})