From e8ac65d261ac1d4f78007248b8b04ca43ebba130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Fri, 22 Nov 2024 23:54:02 +0000 Subject: [PATCH 1/5] :sparkles: decode: add throwOnLimitExceeded option qs#517 --- lib/src/extensions/decode.dart | 70 +++++++++++++++--- lib/src/extensions/extensions.dart | 16 ++-- lib/src/models/decode_options.dart | 4 + lib/src/qs.dart | 1 + lib/src/utils.dart | 29 +++++--- test/unit/decode_test.dart | 90 ++++++++++++++++++++++- test/unit/extensions/extensions_test.dart | 20 ++++- test/unit/uri_extension_test.dart | 4 +- 8 files changed, 202 insertions(+), 32 deletions(-) diff --git a/lib/src/extensions/decode.dart b/lib/src/extensions/decode.dart index 16e0131..9e38ca3 100644 --- a/lib/src/extensions/decode.dart +++ b/lib/src/extensions/decode.dart @@ -8,10 +8,25 @@ extension _$Decode on QS { ), ); - static dynamic _parseArrayValue(dynamic val, DecodeOptions options) => - val is String && val.isNotEmpty && options.comma && val.contains(',') - ? val.split(',') - : val; + static dynamic _parseListValue( + dynamic val, + DecodeOptions options, + int currentListLength, + ) { + if (val is String && val.isNotEmpty && options.comma && val.contains(',')) { + return val.split(','); + } + + if (options.throwOnLimitExceeded && + currentListLength >= options.listLimit) { + throw RangeError( + 'List limit exceeded. ' + 'Only ${options.listLimit} element${options.listLimit == 1 ? '' : 's'} allowed in a list.', + ); + } + + return val; + } static Map _parseQueryStringValues( String str, [ @@ -23,12 +38,23 @@ extension _$Decode on QS { (options.ignoreQueryPrefix ? str.replaceFirst('?', '') : str) .replaceAll(RegExp(r'%5B', caseSensitive: false), '[') .replaceAll(RegExp(r'%5D', caseSensitive: false), ']'); - final num? limit = options.parameterLimit == double.infinity + + final int? limit = options.parameterLimit == double.infinity ? null - : options.parameterLimit; + : options.parameterLimit.toInt(); + final Iterable parts = limit != null && limit > 0 - ? cleanStr.split(options.delimiter).take(limit.toInt()) + ? cleanStr + .split(options.delimiter) + .take(options.throwOnLimitExceeded ? limit + 1 : limit) : cleanStr.split(options.delimiter); + + if (options.throwOnLimitExceeded && limit != null && parts.length > limit) { + throw RangeError( + 'Parameter limit exceeded. Only $limit parameter${limit == 1 ? '' : 's'} allowed.', + ); + } + int skipIndex = -1; // Keep track of where the utf8 sentinel was found int i; @@ -65,7 +91,13 @@ extension _$Decode on QS { } else { key = options.decoder(part.slice(0, pos), charset: charset); val = Utils.apply( - _parseArrayValue(part.slice(pos + 1), options), + _parseListValue( + part.slice(pos + 1), + options, + obj.containsKey(key) && obj[key] is List + ? (obj[key] as List).length + : 0, + ), (dynamic val) => options.decoder(val, charset: charset), ); } @@ -102,7 +134,27 @@ extension _$Decode on QS { DecodeOptions options, bool valuesParsed, ) { - dynamic leaf = valuesParsed ? val : _parseArrayValue(val, options); + late final int currentListLength; + + if (chain.isNotEmpty && chain.last == '[]') { + final int? parentKey = int.tryParse(chain.slice(0, -1).join('')); + + currentListLength = parentKey != null && + val is List && + val.firstWhereIndexedOrNull((int i, _) => i == parentKey) != null + ? val.elementAt(parentKey).length + : 0; + } else { + currentListLength = 0; + } + + dynamic leaf = valuesParsed + ? val + : _parseListValue( + val, + options, + currentListLength, + ); for (int i = chain.length - 1; i >= 0; --i) { dynamic obj; diff --git a/lib/src/extensions/extensions.dart b/lib/src/extensions/extensions.dart index c54663f..03f2b2f 100644 --- a/lib/src/extensions/extensions.dart +++ b/lib/src/extensions/extensions.dart @@ -1,14 +1,20 @@ import 'dart:math' show min; -import 'package:qs_dart/src/models/undefined.dart'; extension IterableExtension on Iterable { - /// Returns a new [Iterable] without [Undefined] elements. - Iterable whereNotUndefined() => where((T el) => el is! Undefined); + /// Returns a new [Iterable] without elements of type [Q]. + Iterable whereNotType() => where((T el) => el is! Q); } extension ListExtension on List { - /// Returns a new [List] without [Undefined] elements. - List whereNotUndefined() => where((T el) => el is! Undefined).toList(); + /// Extracts a section of a list and returns a new list. + /// + /// Modeled after JavaScript's `Array.prototype.slice()` method. + /// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/slice + List slice([int start = 0, int? end]) => sublist( + (start < 0 ? length + start : start).clamp(0, length), + (end == null ? length : (end < 0 ? length + end : end)) + .clamp(0, length), + ); } extension StringExtension on String { diff --git a/lib/src/models/decode_options.dart b/lib/src/models/decode_options.dart index 6b09154..7e591f1 100644 --- a/lib/src/models/decode_options.dart +++ b/lib/src/models/decode_options.dart @@ -26,6 +26,7 @@ final class DecodeOptions with EquatableMixin { this.parseLists = true, this.strictDepth = false, this.strictNullHandling = false, + this.throwOnLimitExceeded = false, }) : allowDots = allowDots ?? decodeDotInKeys == true || false, decodeDotInKeys = decodeDotInKeys ?? false, _decoder = decoder, @@ -110,6 +111,9 @@ final class DecodeOptions with EquatableMixin { /// Set to true to decode values without `=` to `null`. final bool strictNullHandling; + /// Set to `true` to throw an error when the limit is exceeded. + final bool throwOnLimitExceeded; + /// Set a [Decoder] to affect the decoding of the input. final Decoder? _decoder; diff --git a/lib/src/qs.dart b/lib/src/qs.dart index bd37331..950dae3 100644 --- a/lib/src/qs.dart +++ b/lib/src/qs.dart @@ -1,6 +1,7 @@ import 'dart:convert' show latin1, utf8, Encoding; import 'dart:typed_data' show ByteBuffer; +import 'package:collection/collection.dart' show IterableExtension; import 'package:qs_dart/src/enums/duplicates.dart'; import 'package:qs_dart/src/enums/format.dart'; import 'package:qs_dart/src/enums/list_format.dart'; diff --git a/lib/src/utils.dart b/lib/src/utils.dart index b6ad780..1c4d228 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -42,11 +42,14 @@ final class Utils { target_[target_.length] = source; } - if (target is Set) { - target = target_.values.whereNotUndefined().toSet(); - } else { - target = target_.values.whereNotUndefined().toList(); - } + target = target_.values.any((el) => el is Undefined) + ? SplayTreeMap.from({ + for (final MapEntry entry in target_.entries) + if (entry.value is! Undefined) entry.key: entry.value, + }) + : target is Set + ? target_.values.toSet() + : target_.values.toList(); } else { if (source is Iterable) { // check if source is a list of maps and target is a list of maps @@ -70,9 +73,11 @@ final class Utils { } } else { if (target is Set) { - target = Set.of(target)..addAll(source.whereNotUndefined()); + target = Set.of(target) + ..addAll(source.whereNotType()); } else { - target = List.of(target)..addAll(source.whereNotUndefined()); + target = List.of(target) + ..addAll(source.whereNotType()); } } } else if (source != null) { @@ -96,7 +101,7 @@ final class Utils { } } else if (source != null) { if (target is! Iterable && source is Iterable) { - return [target, ...source.whereNotUndefined()]; + return [target, ...source.whereNotType()]; } return [target, source]; } @@ -115,11 +120,11 @@ final class Utils { return [ if (target is Iterable) - ...target.whereNotUndefined() + ...target.whereNotType() else if (target != null) target, if (source is Iterable) - ...(source as Iterable).whereNotUndefined() + ...(source as Iterable).whereNotType() else source, ]; @@ -381,9 +386,9 @@ final class Utils { if (obj is Iterable) { if (obj is Set) { - item['obj'][item['prop']] = obj.whereNotUndefined().toSet(); + item['obj'][item['prop']] = obj.whereNotType().toSet(); } else { - item['obj'][item['prop']] = obj.whereNotUndefined().toList(); + item['obj'][item['prop']] = obj.whereNotType().toList(); } } } diff --git a/test/unit/decode_test.dart b/test/unit/decode_test.dart index ef0e3ac..fd9c551 100644 --- a/test/unit/decode_test.dart +++ b/test/unit/decode_test.dart @@ -412,7 +412,7 @@ void main() { expect( QS.decode('a[1]=b&a=c', const DecodeOptions(listLimit: 20)), equals({ - 'a': ['b', 'c'] + 'a': {1: 'b', 2: 'c'} }), ); expect( @@ -818,7 +818,7 @@ void main() { expect( QS.decode('a[10]=1&a[2]=2', const DecodeOptions(listLimit: 20)), equals({ - 'a': ['2', '1'] + 'a': {2: '2', 10: '1'} }), ); expect( @@ -1768,4 +1768,90 @@ void main() { }, ); }); + + group('parameter limit', () { + test('does not throw error when within parameter limit', () { + expect( + QS.decode('a=1&b=2&c=3', + const DecodeOptions(parameterLimit: 5, throwOnLimitExceeded: true)), + equals({'a': '1', 'b': '2', 'c': '3'}), + ); + }); + + test('throws error when parameter limit exceeded', () { + expect( + () => QS.decode( + 'a=1&b=2&c=3&d=4&e=5&f=6', + const DecodeOptions(parameterLimit: 3, throwOnLimitExceeded: true), + ), + throwsA(isA()), + ); + }); + + test('silently truncates when throwOnLimitExceeded is not given', () { + expect( + QS.decode( + 'a=1&b=2&c=3&d=4&e=5', + const DecodeOptions(parameterLimit: 3), + ), + equals({'a': '1', 'b': '2', 'c': '3'}), + ); + }); + + test('silently truncates when parameter limit exceeded without error', () { + expect( + QS.decode( + 'a=1&b=2&c=3&d=4&e=5', + const DecodeOptions(parameterLimit: 3, throwOnLimitExceeded: false), + ), + equals({'a': '1', 'b': '2', 'c': '3'}), + ); + }); + + test('allows unlimited parameters when parameterLimit set to Infinity', () { + expect( + QS.decode( + 'a=1&b=2&c=3&d=4&e=5&f=6', + const DecodeOptions(parameterLimit: double.infinity), + ), + equals({'a': '1', 'b': '2', 'c': '3', 'd': '4', 'e': '5', 'f': '6'}), + ); + }); + }); + + group('list limit tests', () { + test('does not throw error when list is within limit', () { + expect( + QS.decode( + 'a[]=1&a[]=2&a[]=3', + const DecodeOptions(listLimit: 5, throwOnLimitExceeded: true), + ), + equals({ + 'a': ['1', '2', '3'] + }), + ); + }); + + test('throws error when list limit exceeded', () { + expect( + () => QS.decode( + 'a[]=1&a[]=2&a[]=3&a[]=4', + const DecodeOptions(listLimit: 3, throwOnLimitExceeded: true), + ), + throwsA(isA()), + ); + }); + + test('converts list to map if length is greater than limit', () { + expect( + QS.decode( + 'a[1]=1&a[2]=2&a[3]=3&a[4]=4&a[5]=5&a[6]=6', + const DecodeOptions(listLimit: 5), + ), + equals({ + 'a': {'1': '1', '2': '2', '3': '3', '4': '4', '5': '5', '6': '6'} + }), + ); + }); + }); } diff --git a/test/unit/extensions/extensions_test.dart b/test/unit/extensions/extensions_test.dart index e57fefd..d776dcb 100644 --- a/test/unit/extensions/extensions_test.dart +++ b/test/unit/extensions/extensions_test.dart @@ -6,7 +6,7 @@ void main() { group('IterableExtension', () { test('whereNotUndefined', () { const Iterable iterable = [1, 2, Undefined(), 4, 5]; - final Iterable result = iterable.whereNotUndefined(); + final Iterable result = iterable.whereNotType(); expect(result, isA>()); expect(result, [1, 2, 4, 5]); }); @@ -15,10 +15,26 @@ void main() { group('ListExtension', () { test('whereNotUndefined', () { const List list = [1, 2, Undefined(), 4, 5]; - final List result = list.whereNotUndefined(); + final List result = list.whereNotType().toList(); expect(result, isA>()); expect(result, [1, 2, 4, 5]); }); + + test('slice', () { + const List animals = [ + 'ant', + 'bison', + 'camel', + 'duck', + 'elephant', + ]; + expect(animals.slice(2), ['camel', 'duck', 'elephant']); + expect(animals.slice(2, 4), ['camel', 'duck']); + expect(animals.slice(1, 5), ['bison', 'camel', 'duck', 'elephant']); + expect(animals.slice(-2), ['duck', 'elephant']); + expect(animals.slice(2, -1), ['camel', 'duck']); + expect(animals.slice(), ['ant', 'bison', 'camel', 'duck', 'elephant']); + }); }); group('StringExtensions', () { diff --git a/test/unit/uri_extension_test.dart b/test/unit/uri_extension_test.dart index 3c97fa7..956965d 100644 --- a/test/unit/uri_extension_test.dart +++ b/test/unit/uri_extension_test.dart @@ -439,7 +439,7 @@ void main() { Uri.parse('$testUrl?a[1]=b&a=c') .queryParametersQs(const DecodeOptions(listLimit: 20)), equals({ - 'a': ['b', 'c'] + 'a': {1: 'b', 2: 'c'} }), ); expect( @@ -864,7 +864,7 @@ void main() { Uri.parse('$testUrl?a[10]=1&a[2]=2') .queryParametersQs(const DecodeOptions(listLimit: 20)), equals({ - 'a': ['2', '1'] + 'a': {2: '2', 10: '1'} }), ); expect( From 3887d46a9fd3e654bd4913efb25179003e02b04f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 23 Nov 2024 11:12:34 +0000 Subject: [PATCH 2/5] :safety_vest: add additional validation logic --- lib/src/extensions/decode.dart | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/src/extensions/decode.dart b/lib/src/extensions/decode.dart index 9e38ca3..27cb01f 100644 --- a/lib/src/extensions/decode.dart +++ b/lib/src/extensions/decode.dart @@ -14,7 +14,14 @@ extension _$Decode on QS { int currentListLength, ) { if (val is String && val.isNotEmpty && options.comma && val.contains(',')) { - return val.split(','); + final List splitVal = val.split(','); + if (options.throwOnLimitExceeded && splitVal.length > options.listLimit) { + throw RangeError( + 'List limit exceeded. ' + 'Only ${options.listLimit} element${options.listLimit == 1 ? '' : 's'} allowed in a list.', + ); + } + return splitVal; } if (options.throwOnLimitExceeded && @@ -43,6 +50,10 @@ extension _$Decode on QS { ? null : options.parameterLimit.toInt(); + if (limit != null && limit <= 0) { + throw ArgumentError('Parameter limit must be a positive integer.'); + } + final Iterable parts = limit != null && limit > 0 ? cleanStr .split(options.delimiter) From 7215cd11569d3baaf73b8d33fed90bf50678a8c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 23 Nov 2024 11:12:49 +0000 Subject: [PATCH 3/5] :white_check_mark: update tests --- test/unit/decode_test.dart | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/unit/decode_test.dart b/test/unit/decode_test.dart index fd9c551..d2f5d1f 100644 --- a/test/unit/decode_test.dart +++ b/test/unit/decode_test.dart @@ -465,6 +465,17 @@ void main() { ); }); + test('decodes nested lists with parentKey not null', () { + expect( + QS.decode('a[0][]=b'), + equals({ + 'a': [ + ['b'] + ] + }), + ); + }); + test('allows to specify list indices', () { expect( QS.decode('a[1]=c&a[0]=b&a[2]=d'), From 9f6076137e8b1b99864ef1e1e9b14dedd73fc1a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 23 Nov 2024 11:38:49 +0000 Subject: [PATCH 4/5] :fire: remove Utils._compactQueue --- lib/src/utils.dart | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 1c4d228..c685d73 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -372,28 +372,11 @@ final class Utils { } } - _compactQueue(queue); - removeUndefinedFromMap(value); return value; } - static void _compactQueue(List queue) { - while (queue.length > 1) { - final Map item = queue.removeLast(); - final dynamic obj = item['obj'][item['prop']]; - - if (obj is Iterable) { - if (obj is Set) { - item['obj'][item['prop']] = obj.whereNotType().toSet(); - } else { - item['obj'][item['prop']] = obj.whereNotType().toList(); - } - } - } - } - @visibleForTesting static void removeUndefinedFromList(List value) { for (int i = 0; i < value.length; i++) { From 23ca2fcc7633cece26cdf462ff8ca044c179e984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 23 Nov 2024 11:39:03 +0000 Subject: [PATCH 5/5] :white_check_mark: add more tests --- test/unit/decode_test.dart | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/unit/decode_test.dart b/test/unit/decode_test.dart index d2f5d1f..ce4766d 100644 --- a/test/unit/decode_test.dart +++ b/test/unit/decode_test.dart @@ -1864,5 +1864,37 @@ void main() { }), ); }); + + test('handles list limit of zero correctly', () { + expect( + QS.decode( + 'a[]=1&a[]=2', + const DecodeOptions(listLimit: 0), + ), + equals({ + 'a': ['1', '2'] + }), + ); + }); + + test('handles negative list limit correctly', () { + expect( + () => QS.decode( + 'a[]=1&a[]=2', + const DecodeOptions(listLimit: -1, throwOnLimitExceeded: true), + ), + throwsA(isA()), + ); + }); + + test('applies list limit to nested lists', () { + expect( + () => QS.decode( + 'a[0][]=1&a[0][]=2&a[0][]=3&a[0][]=4', + const DecodeOptions(listLimit: 3, throwOnLimitExceeded: true), + ), + throwsA(isA()), + ); + }); }); }