From 7501cb4540999936dca7c8d52dae4d2e2dbe496d Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Sun, 30 Jun 2024 17:06:09 +0100 Subject: [PATCH 01/25] Add function to skip and/or extract comments --- lib/src/utils.dart | 112 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/lib/src/utils.dart b/lib/src/utils.dart index c1e1755..32f279f 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -273,6 +273,118 @@ String getLineEnding(String yaml) { return windowsNewlines > unixNewlines ? '\r\n' : '\n'; } +/// Extracts comments for a node that is replaced within a [YamlMap] or +/// [YamlList] or a top-level [YamlScalar] of the [yaml] string provided. +/// +/// [currentEndOffset] represents the end offset of [YamlScalar] or [YamlList] +/// or [YamlMap] being replaced, that is, `end + 1`. +/// +/// [nextStartOffset] represents the start offset of the next [YamlNode]. +/// May be null if the current [YamlNode] being replaced is the last node +/// in a [YamlScalar] or [YamlList] or if its the only top-level [YamlScalar]. +/// If not sure of the next [YamlNode]'s [nextStartOffset] pass in null and +/// allow this function to handle that manually. +/// +/// Do note that this function has no context of the structure of the [yaml] +/// but assumes the caller does and requires comments based on the offsets +/// provided and thus, may be erroneus since it exclusively scans for `#` +/// delimiter or extracts the comments between the [currentEndOffset] and +/// [nextStartOffset] if both are provided. +/// +/// Returns the `endOffset` of the last comment extracted that is `end + 1` +/// and a `List comments`. It is recommended (but not necessary) that +/// the caller checks the `endOffset` is still within the bounds of the [yaml]. +(int endOffset, List comments) skipAndExtractCommentsInBlock( + String yaml, + int currentEndOffset, + int? nextStartOffset, [ + String lineEnding = '\n', +]) { + /// If [nextStartOffset] is null, this may be the last element in a collection + /// and thus we have to check and extract comments manually. + /// + /// Also, the caller may not be sure where the next node starts. + if (nextStartOffset == null) { + final comments = []; + + /// Skips white-space while extracting comments. + /// + /// Returns [null] if the end of the [yaml] was encountered while + /// skipping any white-space. Otherwise, returns the [index] of the next + /// non-white-space character. + int? skipWhitespace(int index) { + var nextIndex = index; + + while (true) { + if (nextIndex == yaml.length) return null; + if (yaml[nextIndex].trim().isNotEmpty) return nextIndex; + ++nextIndex; + } + } + + var currentOffset = currentEndOffset; + + externalLoop: + while (true) { + if (currentOffset == yaml.length) break; + + var leadingChar = yaml[currentOffset].trim(); + var indexOfCommentStart = -1; + + if (leadingChar.isEmpty) { + switch (skipWhitespace(currentOffset)) { + case final int nextIndex: + currentOffset = nextIndex; + leadingChar = yaml[currentOffset]; + break; + + default: + currentOffset = yaml.length; + break externalLoop; // Exit loop entirely! + } + } + + /// We need comments only, nothing else. This may be pointless but will + /// help us avoid extracting comments when provided random offsets + /// within a string. + if (leadingChar == '#') indexOfCommentStart = currentOffset; + + /// This is a mindless assumption that the last character was either + /// `\n` or [white-space] or the last erroneus offset provided. + if (indexOfCommentStart == -1) break; + + final indexOfLineBreak = yaml.indexOf(lineEnding, currentOffset); + final isEnd = indexOfLineBreak == -1; + + final comment = yaml + .substring(indexOfCommentStart, isEnd ? null : indexOfLineBreak) + .trim(); + + if (comment.isNotEmpty) comments.add(comment); + + if (isEnd) { + currentOffset += comment.length; + break; + } + currentOffset = indexOfLineBreak + 1; // Skip line-break eagerly + } + + return (currentOffset, comments); + } + + return ( + nextStartOffset, + yaml.substring(currentEndOffset, nextStartOffset).split(lineEnding).fold( + [], + (buffer, current) { + final comment = current.trim(); + if (comment.isNotEmpty) buffer.add(comment); + return buffer; + }, + ) + ); +} + extension YamlNodeExtension on YamlNode { /// Returns the [CollectionStyle] of `this` if `this` is [YamlMap] or /// [YamlList]. From 647329c33aa7fce42fb9a7f48ce1ca826ddc1164 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:21:53 +0100 Subject: [PATCH 02/25] Add function to normalize trailing line breaks in encode block --- lib/src/utils.dart | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 32f279f..05451c3 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -385,6 +385,84 @@ String getLineEnding(String yaml) { ); } +/// Normalizes an encoded [YamlNode] encoded as a string by pruning any +/// dangling line-breaks. +/// +/// This function checks the last `YamlNode` of the [update] that is a +/// `YamlScalar` and removes any unwanted line-break within the +/// [updateAsString]. +/// +/// This is achieved by obtaining the chunk of the [yaml] that is after the +/// current node being replaced using its [nodeToReplaceEndOffset]. If: +/// 1. The chunk has any trailing line-break then the it is left untouched. +/// 2. The node being replaced with [update] is not the last node, then it +/// is left untouched. +/// 3. The terminal node in [update] is a `YamlScalar`, that is, +/// the last [YamlNode] within the [update] that is not a collection. +String normalizeEncodedBlock( + String yaml, + String lineEnding, + int nodeToReplaceEndOffset, + YamlNode update, + String updateAsString, +) { + var terminalNode = update; + + if (terminalNode is! YamlScalar) { + loop: + while (terminalNode is! YamlScalar) { + switch (terminalNode) { + case YamlList list: + { + if (list.isEmpty) { + terminalNode = list; + break loop; + } + + terminalNode = list.nodes.last; + } + + case YamlMap map: + { + if (map.isEmpty) { + terminalNode = map; + break loop; + } + + terminalNode = map.nodes.entries.last.value; + } + } + } + } + + /// The node may end up being an empty [YamlMap] or [YamlList] or + /// [YamlScalar]. We never normalize a literal/folded string irrespective of + /// its position + if (terminalNode case YamlScalar(style: var style) + when style == ScalarStyle.LITERAL || style == ScalarStyle.FOLDED) { + return updateAsString; + } + + var normalizedString = updateAsString; + + /// We need to be methodical as we only want to strip it if at the end of the + /// yaml. If not at the end, this `\n` acts as a line break. + final trailing = yaml.substring(nodeToReplaceEndOffset); + + /// We trim it since `package: yaml` only includes an offset with meaningful + /// content. A further check for the trailing `\n` ensures we respect its + /// initial state. + if (trailing.trimRight().isEmpty && !trailing.endsWith(lineEnding)) { + final size = lineEnding == '\r\n' ? 2 : 1; + normalizedString = updateAsString.substring( + 0, + updateAsString.length - size, + ); + } + + return normalizedString; +} + extension YamlNodeExtension on YamlNode { /// Returns the [CollectionStyle] of `this` if `this` is [YamlMap] or /// [YamlList]. From 502fa0af51de4cab831b53313eee0c4cadde7d30 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:22:34 +0100 Subject: [PATCH 03/25] Return index getting key node in map --- lib/src/equality.dart | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/src/equality.dart b/lib/src/equality.dart index 0c6a952..fae0ed3 100644 --- a/lib/src/equality.dart +++ b/lib/src/equality.dart @@ -87,8 +87,10 @@ int deepHashCode(Object? value) { } /// Returns the [YamlNode] corresponding to the provided [key]. -YamlNode getKeyNode(YamlMap map, Object? key) { - return map.nodes.keys.firstWhere((node) => deepEquals(node, key)) as YamlNode; +(int index, YamlNode keyNode) getKeyNode(YamlMap map, Object? key) { + return map.nodes.keys.indexed.firstWhere( + (value) => deepEquals(value.$2, key), + ) as (int, YamlNode); } /// Returns the [YamlNode] after the [YamlNode] corresponding to the provided From c3056c81e2000b794303bdad053b876b25a32881 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:32:03 +0100 Subject: [PATCH 04/25] Apply line-break after each encoded yaml block --- lib/src/strings.dart | 59 ++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/lib/src/strings.dart b/lib/src/strings.dart index dcb1b72..649cc8a 100644 --- a/lib/src/strings.dart +++ b/lib/src/strings.dart @@ -276,64 +276,81 @@ String yamlEncodeFlow(YamlNode value) { } /// Returns [value] with the necessary formatting applied in a block context. +/// +/// It is recommended that callers of this method also make a call to +/// [normalizeEncodedBlock] with this [value] as the `update` and output +/// of this call as the `updateAsString` to prune any dangling line-break. String yamlEncodeBlock( YamlNode value, int indentation, String lineEnding, +) { + return _encodeBlockRecursively(value, indentation, lineEnding).$2; +} + +(bool addedLineBreak, String value) _encodeBlockRecursively( + YamlNode value, + int indentation, + String lineEnding, ) { const additionalIndentation = 2; - if (!isBlockNode(value)) return yamlEncodeFlow(value); + if (!isBlockNode(value)) return (true, yamlEncodeFlow(value) + lineEnding); final newIndentation = indentation + additionalIndentation; if (value is YamlList) { - if (value.isEmpty) return '${' ' * indentation}[]'; + if (value.isEmpty) return (true, '${' ' * indentation}[]$lineEnding'); - Iterable safeValues; - - final children = value.nodes; + final encodedList = value.nodes.fold('', (string, element) { + var (addedLineBreak, valueString) = _encodeBlockRecursively( + element, + newIndentation, + lineEnding, + ); - safeValues = children.map((child) { - var valueString = yamlEncodeBlock(child, newIndentation, lineEnding); - if (isCollection(child) && !isFlowYamlCollectionNode(child)) { + if (isCollection(element) && !isFlowYamlCollectionNode(element)) { valueString = valueString.substring(newIndentation); } - return '${' ' * indentation}- $valueString'; + final appended = '$string${' ' * indentation}- $valueString'; + return addedLineBreak ? appended : '$appended$lineEnding'; }); - return safeValues.join(lineEnding); + return (true, encodedList); } else if (value is YamlMap) { - if (value.isEmpty) return '${' ' * indentation}{}'; + if (value.isEmpty) return (true, '${' ' * indentation}{}$lineEnding'); - return value.nodes.entries.map((entry) { + final encodedMap = value.nodes.entries.fold('', (string, entry) { final MapEntry(:key, :value) = entry; final safeKey = yamlEncodeFlow(key as YamlNode); - final formattedKey = ' ' * indentation + safeKey; + var formattedKey = ' ' * indentation + safeKey; - final formattedValue = yamlEncodeBlock( + final (addedLineBreak, formattedValue) = _encodeBlockRecursively( value, newIndentation, lineEnding, ); /// Empty collections are always encoded in flow-style, so new-line must - /// be avoided - if (isCollection(value) && !isEmpty(value)) { - return '$formattedKey:$lineEnding$formattedValue'; - } + /// be avoided. Otherwise, begin the collection on a new line. + formattedKey = '$formattedKey:' + '${isCollection(value) && !isEmpty(value) ? lineEnding : " "}'; - return '$formattedKey: $formattedValue'; - }).join(lineEnding); + final appended = '$string$formattedKey$formattedValue'; + return addedLineBreak ? appended : '$appended$lineEnding'; + }); + return (true, encodedMap); } - return _yamlEncodeBlockScalar( + final encodedScalar = _yamlEncodeBlockScalar( value as YamlScalar, newIndentation, lineEnding, ); + + return (true, encodedScalar + lineEnding); } /// List of unprintable characters. From e610993314654a3f0ed3a64a61971c65fa5a325b Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:33:44 +0100 Subject: [PATCH 05/25] Encode folded/literal strings based on c3056c8 --- lib/src/strings.dart | 72 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 14 deletions(-) diff --git a/lib/src/strings.dart b/lib/src/strings.dart index 649cc8a..139bc8a 100644 --- a/lib/src/strings.dart +++ b/lib/src/strings.dart @@ -106,7 +106,7 @@ String? _tryYamlEncodeFolded(String string, int indentSize, String lineEnding) { /// Remove trailing `\n` & white-space to ease string folding var trimmed = string.trimRight(); - final stripped = string.substring(trimmed.length); + var stripped = string.substring(trimmed.length); final trimmedSplit = trimmed.replaceAll('\n', lineEnding + indent).split(lineEnding); @@ -137,9 +137,30 @@ String? _tryYamlEncodeFolded(String string, int indentSize, String lineEnding) { return previous + lineEnding + updated; }); - return '>-\n' + stripped = stripped.replaceAll('\n', lineEnding); // Mild paranoia + final ignoreTrailingLineBreak = stripped.endsWith(lineEnding); + + // We ignore it with conviction as explained below. + if (ignoreTrailingLineBreak) { + stripped = stripped.substring(0, stripped.length - 1); + } + + /// If indeed we have a trailing line, we apply a `chomping hack`. We use a + /// `clip indicator` (no chomping indicator) if we need to ignore the `\n` + /// and `strip indicator` if not to remove any trailing indents. + /// + /// The caller of this method, that is, [yamlEncodeBlock] will apply a + /// dangling `\n` that will\should be normalized by + /// [normalizeEncodedBlock] which allows trailing `\n` for [folded] + /// strings such that: + /// * If we had a string `"my string \n"`: + /// 1. This function excludes it and it becomes `>\nmy string ` + /// 2. [yamlEncodeBlock] applies `\n` that we skipped. + /// 2. [normalizeEncodedBlock] ignores the trailing `\n` for folded + /// string by default. + return '>${ignoreTrailingLineBreak ? '' : '-'}\n' '$indent$trimmed' - '${stripped.replaceAll('\n', lineEnding + indent)}'; + '${stripped.replaceAll(lineEnding, lineEnding + indent)}'; } /// Attempts to encode a [string] as a _YAML literal string_ and apply the @@ -170,13 +191,41 @@ String? _tryYamlEncodeLiteral( // encoded in literal mode. if (_hasUnprintableCharacters(string)) return null; + final indent = ' ' * indentSize; + // TODO: Are there other strings we can't encode in literal mode? + final trimmed = string.trimRight(); - final indent = ' ' * indentSize; + // Mild paranoia + var stripped = string + .substring( + trimmed.length, + ) + .replaceAll('\n', lineEnding); + + final ignoreTrailingLineBreak = stripped.endsWith(lineEnding); - /// Simplest block style. - /// * https://yaml.org/spec/1.2.2/#812-literal-style - return '|-\n$indent${string.replaceAll('\n', lineEnding + indent)}'; + // We ignore it with conviction as explained below. + if (ignoreTrailingLineBreak) { + stripped = stripped.substring(0, stripped.length - 1); + } + + /// If indeed we have a trailing line, we apply a `chomping hack`. We use a + /// `clip indicator` (no chomping indicator) if we need to ignore the `\n` + /// and `strip indicator` if not to remove any trailing indents. + /// + /// The caller of this method, that is, [yamlEncodeBlock] will apply a + /// dangling `\n` that will\should be normalized by + /// [normalizeEncodedBlock] which allows trailing `\n` for [literal] + /// strings such that: + /// * If we had a string `"my string \n"`: + /// 1. This function excludes it and it becomes `|\nmy string ` + /// 2. [yamlEncodeBlock] applies `\n` that we skipped. + /// 2. [normalizeEncodedBlock] ignores the trailing `\n` for literal + /// string by default. + return '|${ignoreTrailingLineBreak ? '' : '-'}\n' + '$indent${trimmed.replaceAll('\n', lineEnding + indent)}' + '${stripped.replaceAll(lineEnding, lineEnding + indent)}'; } /// Encodes a flow [YamlScalar] based on the provided [YamlScalar.style]. @@ -280,13 +329,8 @@ String yamlEncodeFlow(YamlNode value) { /// It is recommended that callers of this method also make a call to /// [normalizeEncodedBlock] with this [value] as the `update` and output /// of this call as the `updateAsString` to prune any dangling line-break. -String yamlEncodeBlock( - YamlNode value, - int indentation, - String lineEnding, -) { - return _encodeBlockRecursively(value, indentation, lineEnding).$2; -} +String yamlEncodeBlock(YamlNode value, int indentation, String lineEnding) => + _encodeBlockRecursively(value, indentation, lineEnding).$2; (bool addedLineBreak, String value) _encodeBlockRecursively( YamlNode value, From e659cb91e554ae5d25dad7eb2ad1595d0c6238b0 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:36:25 +0100 Subject: [PATCH 06/25] Skip comments and include `\n` in map mutations --- lib/src/map_mutations.dart | 65 +++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/lib/src/map_mutations.dart b/lib/src/map_mutations.dart index 67665d9..4dac29e 100644 --- a/lib/src/map_mutations.dart +++ b/lib/src/map_mutations.dart @@ -36,7 +36,7 @@ SourceEdit updateInMap( /// removing the element at [key] when re-parsed. SourceEdit removeInMap(YamlEditor yamlEdit, YamlMap map, Object? key) { assert(containsKey(map, key)); - final keyNode = getKeyNode(map, key); + final (_, keyNode) = getKeyNode(map, key); final valueNode = map.nodes[keyNode]!; if (map.style == CollectionStyle.FLOW) { @@ -83,13 +83,14 @@ SourceEdit _addToBlockMap( } } - var valueString = yamlEncodeBlock(newValue, newIndentation, lineEnding); + final valueString = yamlEncodeBlock(newValue, newIndentation, lineEnding); + if (isCollection(newValue) && !isFlowYamlCollectionNode(newValue) && !isEmpty(newValue)) { - formattedValue += '$keyString:$lineEnding$valueString$lineEnding'; + formattedValue += '$keyString:$lineEnding$valueString'; } else { - formattedValue += '$keyString: $valueString$lineEnding'; + formattedValue += '$keyString: $valueString'; } return SourceEdit(offset, 0, formattedValue); @@ -127,12 +128,18 @@ SourceEdit _replaceInBlockMap( YamlEditor yamlEdit, YamlMap map, Object? key, YamlNode newValue) { final yaml = yamlEdit.toString(); final lineEnding = getLineEnding(yaml); - final newIndentation = - getMapIndentation(yaml, map) + getIndentation(yamlEdit); + final mapIndentation = getMapIndentation(yaml, map); + final newIndentation = mapIndentation + getIndentation(yamlEdit); + + // TODO: Compensate for the indent eaten up + final (keyIndex, keyNode) = getKeyNode(map, key); + + var valueAsString = yamlEncodeBlock( + wrapAsYamlNode(newValue), + newIndentation, + lineEnding, + ); - final keyNode = getKeyNode(map, key); - var valueAsString = - yamlEncodeBlock(wrapAsYamlNode(newValue), newIndentation, lineEnding); if (isCollection(newValue) && !isFlowYamlCollectionNode(newValue) && !isEmpty(newValue)) { @@ -150,9 +157,43 @@ SourceEdit _replaceInBlockMap( var end = getContentSensitiveEnd(map.nodes[key]!); /// `package:yaml` parses empty nodes in a way where the start/end of the - /// empty value node is the end of the key node, so we have to adjust for - /// this. - if (end < start) end = start; + /// empty value node is the end of the key node. + /// + /// In our case, we need to ensure that any line-breaks are included in the + /// edit such that: + /// 1. We account for `\n` after a key within other keys or at the start + /// Example.. + /// a: + /// b: value + /// + /// or.. + /// a: value + /// b: + /// c: value + /// + /// 2. We don't suggest edits that are not within the string bounds because + /// of the `\n` we need to account for in Rule 1 above. This could be a + /// key: + /// * At the index `0` but it's the only key + /// * At the end in a map with more than one key + end = start == yaml.length + ? start + : end < start + ? start + 1 + : end; + + // Aggressively skip all comments + final (offsetOfLastComment, _) = + skipAndExtractCommentsInBlock(yaml, end, null, lineEnding); + end = offsetOfLastComment; + + valueAsString = + normalizeEncodedBlock(yaml, lineEnding, end, newValue, valueAsString); + + /// [skipAndExtractCommentsInBlock] is greedy and eats up any whitespace + /// it encounters in search of comments. Compensate indent lost in the + /// current edit + if (keyIndex != map.length - 1) valueAsString += ' ' * mapIndentation; return SourceEdit(start, end - start, valueAsString); } From 53f9637a9671d7fcf2fea220ae26c32466c18b0d Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:37:39 +0100 Subject: [PATCH 07/25] Skip comments and remove additional `\n` added in list mutations --- lib/src/list_mutations.dart | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/lib/src/list_mutations.dart b/lib/src/list_mutations.dart index 17da6dd..de808bc 100644 --- a/lib/src/list_mutations.dart +++ b/lib/src/list_mutations.dart @@ -29,18 +29,21 @@ SourceEdit updateInList( final listIndentation = getListIndentation(yaml, list); final indentation = listIndentation + getIndentation(yamlEdit); final lineEnding = getLineEnding(yaml); - valueString = - yamlEncodeBlock(wrapAsYamlNode(newValue), indentation, lineEnding); + + final encoded = yamlEncodeBlock( + wrapAsYamlNode(newValue), + indentation, + lineEnding, + ); + valueString = encoded; /// We prefer the compact nested notation for collections. /// - /// By virtue of [yamlEncodeBlockString], collections automatically + /// By virtue of [yamlEncodeBlock], collections automatically /// have the necessary line endings. if ((newValue is List && (newValue as List).isNotEmpty) || (newValue is Map && (newValue as Map).isNotEmpty)) { valueString = valueString.substring(indentation); - } else if (currValue.collectionStyle == CollectionStyle.BLOCK) { - valueString += lineEnding; } var end = getContentSensitiveEnd(currValue); @@ -50,6 +53,19 @@ SourceEdit updateInList( valueString = ' $valueString'; } + // Aggressively skip all comments + final (offsetOfLastComment, _) = + skipAndExtractCommentsInBlock(yaml, end, null, lineEnding); + end = offsetOfLastComment; + + valueString = + normalizeEncodedBlock(yaml, lineEnding, end, newValue, valueString); + + /// [skipAndExtractCommentsInBlock] is greedy and eats up any whitespace + /// it encounters in search of comments. Compensate indent lost in the + /// current edit + if (index != list.length - 1) valueString += ' ' * listIndentation; + return SourceEdit(offset, end - offset, valueString); } else { valueString = yamlEncodeFlow(newValue); @@ -146,7 +162,7 @@ SourceEdit _appendToBlockList( valueString = valueString.substring(newIndentation); } - return (listIndentation, '- $valueString$lineEnding'); + return (listIndentation, '- $valueString'); } /// Formats [item] into a new node for flow lists. From edd8d384b4a6b6eb057270cd5c034366940fde42 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:48:10 +0100 Subject: [PATCH 08/25] Normalize top level edits --- lib/src/editor.dart | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/src/editor.dart b/lib/src/editor.dart index 54775cc..c31717d 100644 --- a/lib/src/editor.dart +++ b/lib/src/editor.dart @@ -243,9 +243,10 @@ class YamlEditor { final start = _contents.span.start.offset; final end = getContentSensitiveEnd(_contents); final lineEnding = getLineEnding(_yaml); - final edit = SourceEdit( - start, end - start, yamlEncodeBlock(valueNode, 0, lineEnding)); - + var encoded = yamlEncodeBlock(valueNode, 0, lineEnding); + encoded = + normalizeEncodedBlock(_yaml, lineEnding, end, valueNode, encoded); + final edit = SourceEdit(start, end - start, encoded); return _performEdit(edit, path, valueNode); } @@ -483,7 +484,7 @@ class YamlEditor { if (!containsKey(map, keyOrIndex)) { return _pathErrorOrElse(path, path.take(i + 1), map, orElse); } - final keyNode = getKeyNode(map, keyOrIndex); + final (_, keyNode) = getKeyNode(map, keyOrIndex); if (checkAlias) { if (_aliases.contains(keyNode)) throw AliasException(path, keyNode); From f5a259ba5bf4d8766c11af2455ac1e838b7c9809 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Mon, 1 Jul 2024 11:06:24 +0100 Subject: [PATCH 09/25] Remove defensive encoding function after fix in e659cb9 and 53f9637 --- lib/src/strings.dart | 42 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/lib/src/strings.dart b/lib/src/strings.dart index 139bc8a..c3fe9e7 100644 --- a/lib/src/strings.dart +++ b/lib/src/strings.dart @@ -329,63 +329,43 @@ String yamlEncodeFlow(YamlNode value) { /// It is recommended that callers of this method also make a call to /// [normalizeEncodedBlock] with this [value] as the `update` and output /// of this call as the `updateAsString` to prune any dangling line-break. -String yamlEncodeBlock(YamlNode value, int indentation, String lineEnding) => - _encodeBlockRecursively(value, indentation, lineEnding).$2; - -(bool addedLineBreak, String value) _encodeBlockRecursively( - YamlNode value, - int indentation, - String lineEnding, -) { +String yamlEncodeBlock(YamlNode value, int indentation, String lineEnding) { const additionalIndentation = 2; - if (!isBlockNode(value)) return (true, yamlEncodeFlow(value) + lineEnding); + if (!isBlockNode(value)) return yamlEncodeFlow(value) + lineEnding; final newIndentation = indentation + additionalIndentation; if (value is YamlList) { - if (value.isEmpty) return (true, '${' ' * indentation}[]$lineEnding'); + if (value.isEmpty) return '${' ' * indentation}[]$lineEnding'; - final encodedList = value.nodes.fold('', (string, element) { - var (addedLineBreak, valueString) = _encodeBlockRecursively( - element, - newIndentation, - lineEnding, - ); + return value.nodes.fold('', (string, element) { + var valueString = yamlEncodeBlock(element, newIndentation, lineEnding); if (isCollection(element) && !isFlowYamlCollectionNode(element)) { valueString = valueString.substring(newIndentation); } - final appended = '$string${' ' * indentation}- $valueString'; - return addedLineBreak ? appended : '$appended$lineEnding'; + return '$string${' ' * indentation}- $valueString'; }); - - return (true, encodedList); } else if (value is YamlMap) { - if (value.isEmpty) return (true, '${' ' * indentation}{}$lineEnding'); + if (value.isEmpty) return '${' ' * indentation}{}$lineEnding'; - final encodedMap = value.nodes.entries.fold('', (string, entry) { + return value.nodes.entries.fold('', (string, entry) { final MapEntry(:key, :value) = entry; final safeKey = yamlEncodeFlow(key as YamlNode); var formattedKey = ' ' * indentation + safeKey; - final (addedLineBreak, formattedValue) = _encodeBlockRecursively( - value, - newIndentation, - lineEnding, - ); + final formattedValue = yamlEncodeBlock(value, newIndentation, lineEnding); /// Empty collections are always encoded in flow-style, so new-line must /// be avoided. Otherwise, begin the collection on a new line. formattedKey = '$formattedKey:' '${isCollection(value) && !isEmpty(value) ? lineEnding : " "}'; - final appended = '$string$formattedKey$formattedValue'; - return addedLineBreak ? appended : '$appended$lineEnding'; + return '$string$formattedKey$formattedValue'; }); - return (true, encodedMap); } final encodedScalar = _yamlEncodeBlockScalar( @@ -394,7 +374,7 @@ String yamlEncodeBlock(YamlNode value, int indentation, String lineEnding) => lineEnding, ); - return (true, encodedScalar + lineEnding); + return encodedScalar + lineEnding; } /// List of unprintable characters. From 5a51cbebc7f68fee7ed2d89650491039195276f3 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Mon, 1 Jul 2024 15:43:14 +0100 Subject: [PATCH 10/25] Run dart format --- lib/src/list_mutations.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/list_mutations.dart b/lib/src/list_mutations.dart index de808bc..25d19ad 100644 --- a/lib/src/list_mutations.dart +++ b/lib/src/list_mutations.dart @@ -57,7 +57,7 @@ SourceEdit updateInList( final (offsetOfLastComment, _) = skipAndExtractCommentsInBlock(yaml, end, null, lineEnding); end = offsetOfLastComment; - + valueString = normalizeEncodedBlock(yaml, lineEnding, end, newValue, valueString); From c7aec859e55c184522f9ef28ae0c038ae2521ab5 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:50:30 +0100 Subject: [PATCH 11/25] Skip comments for top-level edits --- lib/src/editor.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/src/editor.dart b/lib/src/editor.dart index c31717d..4a01217 100644 --- a/lib/src/editor.dart +++ b/lib/src/editor.dart @@ -241,8 +241,9 @@ class YamlEditor { if (path.isEmpty) { final start = _contents.span.start.offset; - final end = getContentSensitiveEnd(_contents); + var end = getContentSensitiveEnd(_contents); final lineEnding = getLineEnding(_yaml); + end = skipAndExtractCommentsInBlock(_yaml, end, null, lineEnding).$1; var encoded = yamlEncodeBlock(valueNode, 0, lineEnding); encoded = normalizeEncodedBlock(_yaml, lineEnding, end, valueNode, encoded); From 157063aadd6bbf4eb641e0c290e0f9a2bbf24398 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Tue, 2 Jul 2024 23:22:06 +0100 Subject: [PATCH 12/25] Lazily look ahead for comments --- lib/src/utils.dart | 54 +++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 05451c3..cf02616 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -312,36 +312,54 @@ String getLineEnding(String yaml) { /// Returns [null] if the end of the [yaml] was encountered while /// skipping any white-space. Otherwise, returns the [index] of the next /// non-white-space character. - int? skipWhitespace(int index) { - var nextIndex = index; + (int? firstLineBreakOffset, int? nextIndex) skipWhitespace(int index) { + int? firstLineBreak; + int? nextIndex = index; while (true) { - if (nextIndex == yaml.length) return null; - if (yaml[nextIndex].trim().isNotEmpty) return nextIndex; + if (nextIndex == yaml.length) { + nextIndex = null; + break; + } + + final char = yaml[nextIndex!]; + + if (char == lineEnding && firstLineBreak == null) { + firstLineBreak = nextIndex; + } + + if (char.trim().isNotEmpty) break; ++nextIndex; } + + if (firstLineBreak != null) firstLineBreak += 1; // Skip it if not null + return (firstLineBreak, nextIndex); } var currentOffset = currentEndOffset; - externalLoop: while (true) { if (currentOffset == yaml.length) break; var leadingChar = yaml[currentOffset].trim(); var indexOfCommentStart = -1; + int? firstLineBreak; + if (leadingChar.isEmpty) { - switch (skipWhitespace(currentOffset)) { - case final int nextIndex: - currentOffset = nextIndex; - leadingChar = yaml[currentOffset]; - break; - - default: - currentOffset = yaml.length; - break externalLoop; // Exit loop entirely! + final (firstLE, nextIndex) = skipWhitespace(currentEndOffset); + + /// If the next index is null, it means we reached the end of the + /// string. Since we lazily evaluated the string, attempt to return the + /// first [lineEnding] we encountered only if not null. + if (nextIndex == null) { + currentOffset = firstLE ?? yaml.length; + break; } + + firstLineBreak = firstLE; + currentOffset = nextIndex; + leadingChar = yaml[currentOffset]; } /// We need comments only, nothing else. This may be pointless but will @@ -351,7 +369,13 @@ String getLineEnding(String yaml) { /// This is a mindless assumption that the last character was either /// `\n` or [white-space] or the last erroneus offset provided. - if (indexOfCommentStart == -1) break; + /// + /// Since we lazily evaluated the string, attempt to return the + /// first [lineEnding] we encountered only if not null. + if (indexOfCommentStart == -1) { + currentOffset = firstLineBreak ?? currentOffset; + break; + } final indexOfLineBreak = yaml.indexOf(lineEnding, currentOffset); final isEnd = indexOfLineBreak == -1; From f7fe2d3cb713d649951e16b11f82c99df1c73726 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Tue, 2 Jul 2024 23:25:54 +0100 Subject: [PATCH 13/25] Refactor function to normalize encoded blocks --- lib/src/editor.dart | 11 +++- lib/src/list_mutations.dart | 14 ++--- lib/src/map_mutations.dart | 17 +++--- lib/src/utils.dart | 114 +++++++++++++++++++----------------- 4 files changed, 84 insertions(+), 72 deletions(-) diff --git a/lib/src/editor.dart b/lib/src/editor.dart index 4a01217..313d8ee 100644 --- a/lib/src/editor.dart +++ b/lib/src/editor.dart @@ -245,8 +245,15 @@ class YamlEditor { final lineEnding = getLineEnding(_yaml); end = skipAndExtractCommentsInBlock(_yaml, end, null, lineEnding).$1; var encoded = yamlEncodeBlock(valueNode, 0, lineEnding); - encoded = - normalizeEncodedBlock(_yaml, lineEnding, end, valueNode, encoded); + encoded = normalizeEncodedBlock( + _yaml, + lineEnding: lineEnding, + nodeToReplaceEndOffset: end, + update: valueNode, + updateAsString: encoded, + skipPreservationCheck: true, + isTopLevelScalar: true, + ); final edit = SourceEdit(start, end - start, encoded); return _performEdit(edit, path, valueNode); } diff --git a/lib/src/list_mutations.dart b/lib/src/list_mutations.dart index 25d19ad..0b0df51 100644 --- a/lib/src/list_mutations.dart +++ b/lib/src/list_mutations.dart @@ -58,13 +58,13 @@ SourceEdit updateInList( skipAndExtractCommentsInBlock(yaml, end, null, lineEnding); end = offsetOfLastComment; - valueString = - normalizeEncodedBlock(yaml, lineEnding, end, newValue, valueString); - - /// [skipAndExtractCommentsInBlock] is greedy and eats up any whitespace - /// it encounters in search of comments. Compensate indent lost in the - /// current edit - if (index != list.length - 1) valueString += ' ' * listIndentation; + valueString = normalizeEncodedBlock( + yaml, + lineEnding: lineEnding, + nodeToReplaceEndOffset: end, + update: newValue, + updateAsString: valueString, + ); return SourceEdit(offset, end - offset, valueString); } else { diff --git a/lib/src/map_mutations.dart b/lib/src/map_mutations.dart index 4dac29e..bf15c0c 100644 --- a/lib/src/map_mutations.dart +++ b/lib/src/map_mutations.dart @@ -131,8 +131,7 @@ SourceEdit _replaceInBlockMap( final mapIndentation = getMapIndentation(yaml, map); final newIndentation = mapIndentation + getIndentation(yamlEdit); - // TODO: Compensate for the indent eaten up - final (keyIndex, keyNode) = getKeyNode(map, key); + final (_, keyNode) = getKeyNode(map, key); var valueAsString = yamlEncodeBlock( wrapAsYamlNode(newValue), @@ -187,13 +186,13 @@ SourceEdit _replaceInBlockMap( skipAndExtractCommentsInBlock(yaml, end, null, lineEnding); end = offsetOfLastComment; - valueAsString = - normalizeEncodedBlock(yaml, lineEnding, end, newValue, valueAsString); - - /// [skipAndExtractCommentsInBlock] is greedy and eats up any whitespace - /// it encounters in search of comments. Compensate indent lost in the - /// current edit - if (keyIndex != map.length - 1) valueAsString += ' ' * mapIndentation; + valueAsString = normalizeEncodedBlock( + yaml, + lineEnding: lineEnding, + nodeToReplaceEndOffset: end, + update: newValue, + updateAsString: valueAsString, + ); return SourceEdit(start, end - start, valueAsString); } diff --git a/lib/src/utils.dart b/lib/src/utils.dart index cf02616..f12e014 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -413,78 +413,84 @@ String getLineEnding(String yaml) { /// dangling line-breaks. /// /// This function checks the last `YamlNode` of the [update] that is a -/// `YamlScalar` and removes any unwanted line-break within the +/// `YamlScalar` and removes any dangling line-break within the /// [updateAsString]. /// -/// This is achieved by obtaining the chunk of the [yaml] that is after the -/// current node being replaced using its [nodeToReplaceEndOffset]. If: -/// 1. The chunk has any trailing line-break then the it is left untouched. -/// 2. The node being replaced with [update] is not the last node, then it -/// is left untouched. -/// 3. The terminal node in [update] is a `YamlScalar`, that is, -/// the last [YamlNode] within the [update] that is not a collection. +/// [skipPreservationCheck] and [isTopLevelScalar] should always remain false +/// if updating a value within a [YamlList] or [YamlMap] that isn't an +/// existing top-level [YamlNode]. +/// +/// [isTopLevelScalar] ensures this function doesn't prune raw line breaks +/// present in strings encoded with [ScalarStyle.PLAIN] or [ScalarStyle.ANY]. +/// +/// [skipPreservationCheck] ensures this function prunes any dangling line +/// breaks that fail the [isTopLevelScalar] check and don't need to be included +/// the top-level [YamlScalar] or [YamlList] or [YamlMap]. String normalizeEncodedBlock( - String yaml, - String lineEnding, - int nodeToReplaceEndOffset, - YamlNode update, - String updateAsString, -) { + String yaml, { + required String lineEnding, + required int nodeToReplaceEndOffset, + required YamlNode update, + required String updateAsString, + bool skipPreservationCheck = false, + bool isTopLevelScalar = false, +}) { var terminalNode = update; - if (terminalNode is! YamlScalar) { - loop: - while (terminalNode is! YamlScalar) { - switch (terminalNode) { - case YamlList list: - { - if (list.isEmpty) { - terminalNode = list; - break loop; - } - - terminalNode = list.nodes.last; + loop: + while (terminalNode is! YamlScalar) { + switch (terminalNode) { + case YamlList list: + { + if (list.isEmpty) { + terminalNode = list; + break loop; } - case YamlMap map: - { - if (map.isEmpty) { - terminalNode = map; - break loop; - } + terminalNode = list.nodes.last; + } - terminalNode = map.nodes.entries.last.value; + case YamlMap map: + { + if (map.isEmpty) { + terminalNode = map; + break loop; } - } + + terminalNode = map.nodes.entries.last.value; + } } } /// The node may end up being an empty [YamlMap] or [YamlList] or /// [YamlScalar]. We never normalize a literal/folded string irrespective of - /// its position - if (terminalNode case YamlScalar(style: var style) - when style == ScalarStyle.LITERAL || style == ScalarStyle.FOLDED) { - return updateAsString; + /// its position. Also, in case our value ended with a raw line-break for + /// a top level scalar + if (terminalNode case YamlScalar(style: var style, value: var value)) { + // + if (style == ScalarStyle.LITERAL || + style == ScalarStyle.FOLDED || + (isTopLevelScalar && + value is String && + (value.endsWith('\n') || value.endsWith('\r\n')))) { + return updateAsString; + } } - var normalizedString = updateAsString; - - /// We need to be methodical as we only want to strip it if at the end of the - /// yaml. If not at the end, this `\n` acts as a line break. - final trailing = yaml.substring(nodeToReplaceEndOffset); - - /// We trim it since `package: yaml` only includes an offset with meaningful - /// content. A further check for the trailing `\n` ensures we respect its - /// initial state. - if (trailing.trimRight().isEmpty && !trailing.endsWith(lineEnding)) { - final size = lineEnding == '\r\n' ? 2 : 1; - normalizedString = updateAsString.substring( - 0, - updateAsString.length - size, - ); + if (yaml.isNotEmpty && !skipPreservationCheck) { + // Move it back one position. Offset passed in is/should be exclusive + final offsetBeforeEnd = nodeToReplaceEndOffset > 0 + ? nodeToReplaceEndOffset - 1 + : nodeToReplaceEndOffset; + + /// Leave as is. The [update] is: + /// 1. An element not at the end of [YamlList] or [YamlMap] + /// 2. [YamlNode] replaced had a `\n` + if (yaml[offsetBeforeEnd] == '\n') return updateAsString; } - return normalizedString; + // Remove trailing line-break by default. + return updateAsString.trimRight(); } extension YamlNodeExtension on YamlNode { From 9101d7940ce3392c15ec2ef0b1d532669058758d Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Wed, 3 Jul 2024 09:50:18 +0100 Subject: [PATCH 14/25] Prevent pruning in YamlScalar with ScalarStyles plain, any, folded, literal --- lib/src/editor.dart | 1 - lib/src/utils.dart | 53 ++++++++++++++++++++++++++------------------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/lib/src/editor.dart b/lib/src/editor.dart index 313d8ee..1a1c995 100644 --- a/lib/src/editor.dart +++ b/lib/src/editor.dart @@ -252,7 +252,6 @@ class YamlEditor { update: valueNode, updateAsString: encoded, skipPreservationCheck: true, - isTopLevelScalar: true, ); final edit = SourceEdit(start, end - start, encoded); return _performEdit(edit, path, valueNode); diff --git a/lib/src/utils.dart b/lib/src/utils.dart index f12e014..2d18228 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -416,16 +416,15 @@ String getLineEnding(String yaml) { /// `YamlScalar` and removes any dangling line-break within the /// [updateAsString]. /// -/// [skipPreservationCheck] and [isTopLevelScalar] should always remain false -/// if updating a value within a [YamlList] or [YamlMap] that isn't an -/// existing top-level [YamlNode]. +/// Line breaks are allowed if a: +/// 1. [YamlScalar] has [ScalarStyle.LITERAL] or [ScalarStyle.FOLDED] +/// 2. [YamlScalar] has [ScalarStyle.PLAIN] or [ScalarStyle.ANY] and its +/// raw value is a [String] with a trailing line break. +/// 3. [YamlNode] being replaced has a line break. /// -/// [isTopLevelScalar] ensures this function doesn't prune raw line breaks -/// present in strings encoded with [ScalarStyle.PLAIN] or [ScalarStyle.ANY]. -/// -/// [skipPreservationCheck] ensures this function prunes any dangling line -/// breaks that fail the [isTopLevelScalar] check and don't need to be included -/// the top-level [YamlScalar] or [YamlList] or [YamlMap]. +/// [skipPreservationCheck] should always remain false if updating a value +/// within a [YamlList] or [YamlMap] that isn't an existing top-level +/// [YamlNode]. String normalizeEncodedBlock( String yaml, { required String lineEnding, @@ -433,10 +432,28 @@ String normalizeEncodedBlock( required YamlNode update, required String updateAsString, bool skipPreservationCheck = false, - bool isTopLevelScalar = false, }) { var terminalNode = update; + /// Checks if the dangling line break should be allowed within the deepest + /// [YamlNode] that is a [YamlScalar]. + bool allowInYamlScalar(ScalarStyle style, dynamic value) { + /// We never normalize a literal/folded string irrespective of + /// its position. We allow the block indicators to define how line break + /// will be treated + if (style == ScalarStyle.LITERAL || style == ScalarStyle.FOLDED) { + return true; + } + + // Allow trailing line break if the raw value has a explicit line break. + if (style == ScalarStyle.PLAIN || style == ScalarStyle.ANY) { + return value is String && + (value.endsWith('\n') || value.endsWith('\r\n')); + } + + return false; + } + loop: while (terminalNode is! YamlScalar) { switch (terminalNode) { @@ -463,18 +480,10 @@ String normalizeEncodedBlock( } /// The node may end up being an empty [YamlMap] or [YamlList] or - /// [YamlScalar]. We never normalize a literal/folded string irrespective of - /// its position. Also, in case our value ended with a raw line-break for - /// a top level scalar - if (terminalNode case YamlScalar(style: var style, value: var value)) { - // - if (style == ScalarStyle.LITERAL || - style == ScalarStyle.FOLDED || - (isTopLevelScalar && - value is String && - (value.endsWith('\n') || value.endsWith('\r\n')))) { - return updateAsString; - } + /// [YamlScalar]. + if (terminalNode case YamlScalar(style: var style, value: var value) + when allowInYamlScalar(style, value)) { + return updateAsString; } if (yaml.isNotEmpty && !skipPreservationCheck) { From 3d99caf3f89f58759a6af68f5f840b7edf9b32c4 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Wed, 3 Jul 2024 13:31:00 +0100 Subject: [PATCH 15/25] Allow comments to be skipped greedily or lazily --- lib/src/editor.dart | 8 +++++++- lib/src/map_mutations.dart | 2 +- lib/src/utils.dart | 16 ++++++++++++---- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/src/editor.dart b/lib/src/editor.dart index 1a1c995..d44b171 100644 --- a/lib/src/editor.dart +++ b/lib/src/editor.dart @@ -243,7 +243,13 @@ class YamlEditor { final start = _contents.span.start.offset; var end = getContentSensitiveEnd(_contents); final lineEnding = getLineEnding(_yaml); - end = skipAndExtractCommentsInBlock(_yaml, end, null, lineEnding).$1; + end = skipAndExtractCommentsInBlock( + _yaml, + end, + null, + lineEnding: lineEnding, + greedy: true, + ).$1; var encoded = yamlEncodeBlock(valueNode, 0, lineEnding); encoded = normalizeEncodedBlock( _yaml, diff --git a/lib/src/map_mutations.dart b/lib/src/map_mutations.dart index bf15c0c..234c7ee 100644 --- a/lib/src/map_mutations.dart +++ b/lib/src/map_mutations.dart @@ -183,7 +183,7 @@ SourceEdit _replaceInBlockMap( // Aggressively skip all comments final (offsetOfLastComment, _) = - skipAndExtractCommentsInBlock(yaml, end, null, lineEnding); + skipAndExtractCommentsInBlock(yaml, end, null, lineEnding: lineEnding); end = offsetOfLastComment; valueAsString = normalizeEncodedBlock( diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 2d18228..39430c2 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -297,9 +297,10 @@ String getLineEnding(String yaml) { (int endOffset, List comments) skipAndExtractCommentsInBlock( String yaml, int currentEndOffset, - int? nextStartOffset, [ + int? nextStartOffset, { String lineEnding = '\n', -]) { + bool greedy = false, +}) { /// If [nextStartOffset] is null, this may be the last element in a collection /// and thus we have to check and extract comments manually. /// @@ -336,6 +337,13 @@ String getLineEnding(String yaml) { return (firstLineBreak, nextIndex); } + /// Returns the [currentOffset] if [greedy] is true. Otherwise, attempts + /// returning the [firstLineBreakOffset] if not null if [greedy] is false. + int earlyBreakOffset(int currentOffset, int? firstLineBreakOffset) { + if (greedy) return currentOffset; + return firstLineBreakOffset ?? currentOffset; + } + var currentOffset = currentEndOffset; while (true) { @@ -353,7 +361,7 @@ String getLineEnding(String yaml) { /// string. Since we lazily evaluated the string, attempt to return the /// first [lineEnding] we encountered only if not null. if (nextIndex == null) { - currentOffset = firstLE ?? yaml.length; + currentOffset = earlyBreakOffset(yaml.length, firstLE); break; } @@ -373,7 +381,7 @@ String getLineEnding(String yaml) { /// Since we lazily evaluated the string, attempt to return the /// first [lineEnding] we encountered only if not null. if (indexOfCommentStart == -1) { - currentOffset = firstLineBreak ?? currentOffset; + currentOffset = earlyBreakOffset(currentOffset, firstLineBreak); break; } From 4034652351f1a097904c3ca6540f9d84b9b2dbdd Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Wed, 3 Jul 2024 14:06:37 +0100 Subject: [PATCH 16/25] Ensure `_appendToBlockList` appends after last comment --- lib/src/list_mutations.dart | 48 ++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/lib/src/list_mutations.dart b/lib/src/list_mutations.dart index 0b0df51..45be47a 100644 --- a/lib/src/list_mutations.dart +++ b/lib/src/list_mutations.dart @@ -55,7 +55,7 @@ SourceEdit updateInList( // Aggressively skip all comments final (offsetOfLastComment, _) = - skipAndExtractCommentsInBlock(yaml, end, null, lineEnding); + skipAndExtractCommentsInBlock(yaml, end, null, lineEnding: lineEnding); end = offsetOfLastComment; valueString = normalizeEncodedBlock( @@ -128,22 +128,42 @@ SourceEdit _appendToFlowList( /// block list. SourceEdit _appendToBlockList( YamlEditor yamlEdit, YamlList list, YamlNode item) { - var (indentSize, valueToIndent) = _formatNewBlock(yamlEdit, list, item); - var formattedValue = '${' ' * indentSize}$valueToIndent'; + /// A block list can never be empty since a `-` must be seen for it to be a + /// valid block sequence. + /// + /// See description of: + /// https://yaml.org/spec/1.2.2/#82-block-collection-styles. + assert( + list.isNotEmpty, + 'A YamlList encoded as CollectionStyle.BLOCK must have a value', + ); final yaml = yamlEdit.toString(); - var offset = list.span.end.offset; + final lineEnding = getLineEnding(yaml); - // Adjusts offset to after the trailing newline of the last entry, if it - // exists - if (list.isNotEmpty) { - final lastValueSpanEnd = list.nodes.last.span.end.offset; - final nextNewLineIndex = yaml.indexOf('\n', lastValueSpanEnd - 1); - if (nextNewLineIndex == -1) { - formattedValue = getLineEnding(yaml) + formattedValue; - } else { - offset = nextNewLineIndex + 1; - } + // Lazily skip all comments and white-space at the end. + final (offset, _) = skipAndExtractCommentsInBlock( + yaml, + list.nodes.last.span.end.offset, + null, + lineEnding: lineEnding, + ); + + var (indentSize, formattedValue) = _formatNewBlock(yamlEdit, list, item); + + formattedValue = normalizeEncodedBlock( + yaml, + lineEnding: lineEnding, + nodeToReplaceEndOffset: offset, + update: item, + updateAsString: formattedValue, + ); + + formattedValue = '${' ' * indentSize}$formattedValue'; + + // Apply line ending incase it's missing + if (yaml[offset - 1] != '\n') { + formattedValue = '$lineEnding$formattedValue'; } return SourceEdit(offset, 0, formattedValue); From 7ada07ee44f1072268cd9579c55c89b83c59537b Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Wed, 3 Jul 2024 19:57:26 +0100 Subject: [PATCH 17/25] Fix issue where loop never exits --- lib/src/utils.dart | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 39430c2..4173377 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -276,7 +276,7 @@ String getLineEnding(String yaml) { /// Extracts comments for a node that is replaced within a [YamlMap] or /// [YamlList] or a top-level [YamlScalar] of the [yaml] string provided. /// -/// [currentEndOffset] represents the end offset of [YamlScalar] or [YamlList] +/// [endOfNodeOffset] represents the end offset of [YamlScalar] or [YamlList] /// or [YamlMap] being replaced, that is, `end + 1`. /// /// [nextStartOffset] represents the start offset of the next [YamlNode]. @@ -285,10 +285,14 @@ String getLineEnding(String yaml) { /// If not sure of the next [YamlNode]'s [nextStartOffset] pass in null and /// allow this function to handle that manually. /// +/// If [greedy] is `true`, whitespace and any line breaks are skipped. If +/// `false`, this function looks for comments lazily and returns the offset of +/// the first line break that was encountered if no comments were found. +/// /// Do note that this function has no context of the structure of the [yaml] /// but assumes the caller does and requires comments based on the offsets /// provided and thus, may be erroneus since it exclusively scans for `#` -/// delimiter or extracts the comments between the [currentEndOffset] and +/// delimiter or extracts the comments between the [endOfNodeOffset] and /// [nextStartOffset] if both are provided. /// /// Returns the `endOffset` of the last comment extracted that is `end + 1` @@ -296,7 +300,7 @@ String getLineEnding(String yaml) { /// the caller checks the `endOffset` is still within the bounds of the [yaml]. (int endOffset, List comments) skipAndExtractCommentsInBlock( String yaml, - int currentEndOffset, + int endOfNodeOffset, int? nextStartOffset, { String lineEnding = '\n', bool greedy = false, @@ -344,7 +348,7 @@ String getLineEnding(String yaml) { return firstLineBreakOffset ?? currentOffset; } - var currentOffset = currentEndOffset; + var currentOffset = endOfNodeOffset; while (true) { if (currentOffset == yaml.length) break; @@ -355,7 +359,7 @@ String getLineEnding(String yaml) { int? firstLineBreak; if (leadingChar.isEmpty) { - final (firstLE, nextIndex) = skipWhitespace(currentEndOffset); + final (firstLE, nextIndex) = skipWhitespace(currentOffset); /// If the next index is null, it means we reached the end of the /// string. Since we lazily evaluated the string, attempt to return the @@ -406,7 +410,7 @@ String getLineEnding(String yaml) { return ( nextStartOffset, - yaml.substring(currentEndOffset, nextStartOffset).split(lineEnding).fold( + yaml.substring(endOfNodeOffset, nextStartOffset).split(lineEnding).fold( [], (buffer, current) { final comment = current.trim(); From ee6a29e3be8987ce95588b22f3530c6503987662 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:27:58 +0100 Subject: [PATCH 18/25] Avoid skipping line break eagerly when extracting comments --- lib/src/utils.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 4173377..e310410 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -402,7 +402,7 @@ String getLineEnding(String yaml) { currentOffset += comment.length; break; } - currentOffset = indexOfLineBreak + 1; // Skip line-break eagerly + currentOffset = indexOfLineBreak; } return (currentOffset, comments); From 422731d02ab6d32ad3a2ea097dcc10656eb116ae Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Thu, 4 Jul 2024 12:25:01 +0100 Subject: [PATCH 19/25] Add utility method to reclaim indent skipped --- lib/src/utils.dart | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/src/utils.dart b/lib/src/utils.dart index e310410..70c1510 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -421,6 +421,30 @@ String getLineEnding(String yaml) { ); } +/// Reclaims any indent greedily skipped by [skipAndExtractCommentsInBlock] +/// and returns the start `offset` (inclusive). +/// +/// If [isSingle] is `true`, then the `offset` of the line-break is included. +/// It is excluded if `false`. +/// +/// It is recommended that this function is called when removing the last +/// [YamlNode] in a block [YamlMap] or [YamlList]. +int reclaimIndentAndLinebreak( + String yaml, + int currentOffset, { + required bool isSingle, +}) { + var indexOfLineBreak = yaml.lastIndexOf('\n', currentOffset); + + /// In case, this isn't the only element, we ignore the line-break while + /// reclaiming the indent. As the element that remains, will have a line + /// break the next node needs to indicate start of a new node! + if (!isSingle) indexOfLineBreak += 1; + + final indentDiff = currentOffset - indexOfLineBreak; + return currentOffset - indentDiff; +} + /// Normalizes an encoded [YamlNode] encoded as a string by pruning any /// dangling line-breaks. /// From f3a265e431dcaef3561326bf897d5eaabea2a665 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Thu, 4 Jul 2024 17:35:46 +0100 Subject: [PATCH 20/25] Refactor `_removeFromBlockList` to correctly skip comments --- lib/src/list_mutations.dart | 133 ++++++++++++++++++++---------------- 1 file changed, 74 insertions(+), 59 deletions(-) diff --git a/lib/src/list_mutations.dart b/lib/src/list_mutations.dart index 45be47a..f8d93f9 100644 --- a/lib/src/list_mutations.dart +++ b/lib/src/list_mutations.dart @@ -275,7 +275,7 @@ SourceEdit _insertInBlockList( /// ``` (bool isNested, int offset) _isNestedInBlockList( int currentSequenceOffset, String yaml) { - final startIndex = currentSequenceOffset - 1; + final startOffset = currentSequenceOffset - 1; /// Indicates the element we are inserting before is at index `0` of the list /// at the root of the yaml @@ -284,10 +284,10 @@ SourceEdit _insertInBlockList( /// /// - foo /// ^ Inserting before this - if (startIndex < 0) return (false, 0); + if (startOffset < 0) return (false, 0); - final newLineStart = yaml.lastIndexOf('\n', startIndex); - final seqStart = yaml.lastIndexOf('-', startIndex); + final newLineStart = yaml.lastIndexOf('\n', startOffset); + final seqStart = yaml.lastIndexOf('-', startOffset); /// Indicates that a `\n` is closer to the last `-`. Meaning this list is not /// nested. @@ -344,70 +344,85 @@ SourceEdit _removeFromBlockList( YamlEditor yamlEdit, YamlList list, YamlNode nodeToRemove, int index) { RangeError.checkValueInInterval(index, 0, list.length - 1); - var end = getContentSensitiveEnd(nodeToRemove); - - /// If we are removing the last element in a block list, convert it into a - /// flow empty list. - if (list.length == 1) { - final start = list.span.start.offset; + final yaml = yamlEdit.toString(); + final yamlSize = yaml.length; - return SourceEdit(start, end - start, '[]'); + final lineEnding = getLineEnding(yaml); + final YamlNode(:span) = nodeToRemove; + + var startOffset = span.start.offset; + startOffset = + span.length == 0 ? startOffset : yaml.lastIndexOf('-', startOffset - 1); + + var endOffset = getContentSensitiveEnd(nodeToRemove); + + /// YamlMap may have `null` value for the last key and we need to ensure the + /// correct [endOffset] is provided to [skipAndExtractCommentsInBlock], + /// otherwise [skipAndExtractCommentsInBlock] may prematurely return an + /// incorrect offset because it immediately saw `:` + if (nodeToRemove is YamlMap && + endOffset < yamlSize && + nodeToRemove.nodes.entries.last.value.value == null) { + endOffset += 1; } - final yaml = yamlEdit.toString(); - final span = nodeToRemove.span; + // We remove any content belonging to [nodeToRemove] greedily + endOffset = skipAndExtractCommentsInBlock( + yaml, + endOffset == startOffset ? endOffset + 1 : endOffset, + null, + lineEnding: lineEnding, + greedy: true, + ).$1; - /// Adjust the end to clear the new line after the end too. - /// - /// We do this because we suspect that our users will want the inline - /// comments to disappear too. - final nextNewLine = yaml.indexOf('\n', end); - if (nextNewLine != -1) { - end = nextNewLine + 1; - } + final listSize = list.length; - /// If the value is empty - if (span.length == 0) { - var start = span.start.offset; - return SourceEdit(start, end - start, ''); + final isSingleElement = listSize == 1; + final isLastElementInList = index == listSize - 1; + final isLastInYaml = endOffset == yamlSize; + + final replacement = listSize == 1 ? '[]' : ''; + + /// Adjust [startIndent] to include any indent this element may have had + /// to prevent it from interfering with the indent of the next [YamlNode] + /// which isn't in this list. We move it back if: + /// 1. The [nodeToRemove] is the last element in a [list] with more than + /// one element. + /// 2. It also isn't the first element in the yaml. + /// + /// Doing this only for the last element ensures that any value's indent is + /// automatically given to the next element in the list such that, + /// + /// 1. If nested: + /// - - value + /// ^ This space goes to the next element that ends up here + /// + /// 2. If not nested, then the next element gets the indent if any is present. + if (isLastElementInList && startOffset != 0 && !isSingleElement) { + final index = yaml.lastIndexOf('\n', startOffset); + startOffset = index == -1 ? startOffset : index + 1; } - /// -1 accounts for the fact that the content can start with a dash - var start = yaml.lastIndexOf('-', span.start.offset - 1); - - /// Check if there is a `-` before the node - if (start > 0) { - final lastHyphen = yaml.lastIndexOf('-', start - 1); - final lastNewLine = yaml.lastIndexOf('\n', start - 1); - if (lastHyphen > lastNewLine) { - start = lastHyphen + 2; - - /// If there is a `-` before the node, we need to check if we have - /// to update the indentation of the next node. - if (index < list.length - 1) { - /// Since [end] is currently set to the next new line after the current - /// node, check if we see a possible comment first, or a hyphen first. - /// Note that no actual content can appear here. - /// - /// We check this way because the start of a span in a block list is - /// the start of its value, and checking from the back leaves us - /// easily confused if there are comments that have dashes in them. - final nextHash = yaml.indexOf('#', end); - final nextHyphen = yaml.indexOf('-', end); - final nextNewLine = yaml.indexOf('\n', end); - - /// If [end] is on the same line as the hyphen of the next node - if ((nextHash == -1 || nextHyphen < nextHash) && - nextHyphen < nextNewLine) { - end = nextHyphen; - } - } - } else if (lastNewLine > lastHyphen) { - start = lastNewLine + 1; - } + /// We intentionally [skipAndExtractCommentsInBlock] greedily which also + /// consumes the next [YamlNode]'s indent. + /// + /// For elements at the last index, we need to reclaim the indent belonging + /// to the next node not in the list and optionally include a line break if + /// if it is the only element. See [reclaimIndentAndLinebreak] for more info. + if (isLastElementInList && !isLastInYaml) { + endOffset = reclaimIndentAndLinebreak( + yaml, + endOffset, + isSingle: isSingleElement, + ); + } else if (isLastInYaml && yaml[endOffset - 1] == '\n' && isSingleElement) { + /// Include any trailing line break that may have been part of the yaml: + /// -`\r\n` = 2 + /// - `\n` = 1 + endOffset -= lineEnding == '\n' ? 1 : 2; } - return SourceEdit(start, end - start, ''); + return SourceEdit(startOffset, endOffset - startOffset, replacement); } /// Returns a [SourceEdit] describing the change to be made on [yamlEdit] to From 814672b4458d4ee4a96e01e88389cd63ca767563 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Thu, 4 Jul 2024 17:37:30 +0100 Subject: [PATCH 21/25] Refactor `_removeFromBlockMap` to correctly skip comments --- lib/src/map_mutations.dart | 102 +++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 45 deletions(-) diff --git a/lib/src/map_mutations.dart b/lib/src/map_mutations.dart index 234c7ee..5fa0f17 100644 --- a/lib/src/map_mutations.dart +++ b/lib/src/map_mutations.dart @@ -214,62 +214,74 @@ SourceEdit _replaceInFlowMap( SourceEdit _removeFromBlockMap( YamlEditor yamlEdit, YamlMap map, YamlNode keyNode, YamlNode valueNode) { final keySpan = keyNode.span; - var end = getContentSensitiveEnd(valueNode); + final yaml = yamlEdit.toString(); + final yamlSize = yaml.length; + final lineEnding = getLineEnding(yaml); - if (map.length == 1) { - final start = map.span.start.offset; - final nextNewLine = yaml.indexOf(lineEnding, end); - if (nextNewLine != -1) { - // Remove everything up to the next newline, this strips comments that - // follows on the same line as the value we're removing. - // It also ensures we consume colon when [valueNode.value] is `null` - // because there is no value (e.g. `key: \n`). Because [valueNode.span] in - // such cases point to the colon `:`. - end = nextNewLine; - } else { - // Remove everything until the end of the document, if there is no newline - end = yaml.length; - } - return SourceEdit(start, end - start, '{}'); - } + final (keyIndex, _) = getKeyNode(map, keyNode); - var start = keySpan.start.offset; + var startOffset = keySpan.start.offset; - /// Adjust the end to clear the new line after the end too. + /// Null values have an invalid offset. Include colon. /// - /// We do this because we suspect that our users will want the inline - /// comments to disappear too. - final nextNewLine = yaml.indexOf(lineEnding, end); - if (nextNewLine != -1) { - end = nextNewLine + lineEnding.length; - } else { - // Remove everything until the end of the document, if there is no newline - end = yaml.length; - } + /// See issue open in `package: yaml`. + var endOffset = valueNode.value == null + ? keySpan.end.offset + 2 + : getContentSensitiveEnd(valueNode) + 1; // Overeager to avoid issues - final nextNode = getNextKeyNode(map, keyNode); + if (endOffset > yamlSize) endOffset -= 1; - if (start > 0) { - final lastHyphen = yaml.lastIndexOf('-', start - 1); - final lastNewLine = yaml.lastIndexOf(lineEnding, start - 1); - if (lastHyphen > lastNewLine) { - start = lastHyphen + 2; + endOffset = skipAndExtractCommentsInBlock( + yaml, + endOffset, + null, + lineEnding: lineEnding, + greedy: true, + ).$1; - /// If there is a `-` before the node, and the end is on the same line - /// as the next node, we need to add the necessary offset to the end to - /// make sure the next node has the correct indentation. - if (nextNode != null && - nextNode.span.start.offset - end <= nextNode.span.start.column) { - end += nextNode.span.start.column; - } - } else if (lastNewLine > lastHyphen) { - start = lastNewLine + lineEnding.length; - } + final mapSize = map.length; + + final isSingleEntry = mapSize == 1; + final isLastEntryInMap = keyIndex == mapSize - 1; + final isLastNodeInYaml = endOffset == yamlSize; + + final replacement = isSingleEntry ? '{}' : ''; + + /// Adjust [startIndent] to include any indent this element may have had + /// to prevent it from interfering with the indent of the next [YamlNode] + /// which isn't in this map. We move it back if: + /// 1. The entry is the last entry in a [map] with more than one element. + /// 2. It also isn't the first entry of map in the yaml. + /// + /// Doing this only for the last element ensures that any value's indent is + /// automatically given to the next entry in the map. + if (isLastEntryInMap && startOffset != 0 && !isSingleEntry) { + final index = yaml.lastIndexOf('\n', startOffset); + startOffset = index == -1 ? startOffset : index + 1; } - return SourceEdit(start, end - start, ''); + /// We intentionally [skipAndExtractCommentsInBlock] greedily which also + /// consumes the next [YamlNode]'s indent. + /// + /// For elements at the last index, we need to reclaim the indent belonging + /// to the next node not in the map and optionally include a line break if + /// if it is the only entry. See [reclaimIndentAndLinebreak] for more info. + if (isLastEntryInMap && !isLastNodeInYaml) { + endOffset = reclaimIndentAndLinebreak( + yaml, + endOffset, + isSingle: isSingleEntry, + ); + } else if (isLastNodeInYaml && yaml[endOffset - 1] == '\n' && isSingleEntry) { + /// Include any trailing line break that may have been part of the yaml: + /// -`\r\n` = 2 + /// - `\n` = 1 + endOffset -= lineEnding == '\n' ? 1 : 2; + } + + return SourceEdit(startOffset, endOffset - startOffset, replacement); } /// Performs the string operation on [yamlEdit] to achieve the effect of From 7378e9212575ddcea4627fef0cb28ab7d6b53fec Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Thu, 4 Jul 2024 17:57:56 +0100 Subject: [PATCH 22/25] Use span length to determine true state of `null` --- lib/src/map_mutations.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/map_mutations.dart b/lib/src/map_mutations.dart index 5fa0f17..2a2f906 100644 --- a/lib/src/map_mutations.dart +++ b/lib/src/map_mutations.dart @@ -227,7 +227,7 @@ SourceEdit _removeFromBlockMap( /// Null values have an invalid offset. Include colon. /// /// See issue open in `package: yaml`. - var endOffset = valueNode.value == null + var endOffset = valueNode.span.length == 0 ? keySpan.end.offset + 2 : getContentSensitiveEnd(valueNode) + 1; // Overeager to avoid issues From 7332de617b9a06b235a81f900ebcf341797f4208 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Thu, 24 Oct 2024 20:04:21 +0100 Subject: [PATCH 23/25] Improve comments describing chomping hack --- lib/src/strings.dart | 60 ++++++++++++++++++++++++++----------------- test/string_test.dart | 1 + 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/lib/src/strings.dart b/lib/src/strings.dart index c3fe9e7..66c5891 100644 --- a/lib/src/strings.dart +++ b/lib/src/strings.dart @@ -145,19 +145,25 @@ String? _tryYamlEncodeFolded(String string, int indentSize, String lineEnding) { stripped = stripped.substring(0, stripped.length - 1); } - /// If indeed we have a trailing line, we apply a `chomping hack`. We use a - /// `clip indicator` (no chomping indicator) if we need to ignore the `\n` - /// and `strip indicator` if not to remove any trailing indents. + /// If indeed we have a trailing line-break, we apply a `chomping hack`. /// - /// The caller of this method, that is, [yamlEncodeBlock] will apply a - /// dangling `\n` that will\should be normalized by - /// [normalizeEncodedBlock] which allows trailing `\n` for [folded] - /// strings such that: - /// * If we had a string `"my string \n"`: - /// 1. This function excludes it and it becomes `>\nmy string ` - /// 2. [yamlEncodeBlock] applies `\n` that we skipped. - /// 2. [normalizeEncodedBlock] ignores the trailing `\n` for folded - /// string by default. + /// We use a `clip indicator` (no chomping indicator) if we need to ignore the + /// `\n` and `strip indicator` to remove any trailing line-break and its + /// indent. + /// + /// The caller of this method, that is, [yamlEncodeBlock], will apply a + /// dangling `\n` that must be normalized by [normalizeEncodedBlock] which + /// allows trailing `\n` for [folded] strings such that: + /// * If we had a string "example \n": + /// 1. This function excludes the line-break at the end and it becomes: + /// - ">" + "\n" + + "example " + /// + /// 2. [yamlEncodeBlock] applies a dangling `\n` that we skipped and it + /// becomes: + /// - ">" + "\n" + + "example " + \n` + /// + /// 3. [normalizeEncodedBlock] never prunes the dangling `\n` applied for + /// folded strings by default. return '>${ignoreTrailingLineBreak ? '' : '-'}\n' '$indent$trimmed' '${stripped.replaceAll(lineEnding, lineEnding + indent)}'; @@ -210,19 +216,25 @@ String? _tryYamlEncodeLiteral( stripped = stripped.substring(0, stripped.length - 1); } - /// If indeed we have a trailing line, we apply a `chomping hack`. We use a - /// `clip indicator` (no chomping indicator) if we need to ignore the `\n` - /// and `strip indicator` if not to remove any trailing indents. + /// If indeed we have a trailing line-break, we apply a `chomping hack`. + /// + /// We use a `clip indicator` (no chomping indicator) if we need to ignore the + /// `\n` and `strip indicator` to remove any trailing line-break and its + /// indent. + /// + /// The caller of this method, that is, [yamlEncodeBlock], will apply a + /// dangling `\n` that must be normalized by [normalizeEncodedBlock] which + /// allows trailing `\n` for [literal] strings such that: + /// * If we had a string "example \n": + /// 1. This function excludes the line-break at the end and it becomes: + /// - ">" + "\n" + + "example " + /// + /// 2. [yamlEncodeBlock] applies a dangling `\n` that we skipped and it + /// becomes: + /// - ">" + "\n" + + "example " + \n` /// - /// The caller of this method, that is, [yamlEncodeBlock] will apply a - /// dangling `\n` that will\should be normalized by - /// [normalizeEncodedBlock] which allows trailing `\n` for [literal] - /// strings such that: - /// * If we had a string `"my string \n"`: - /// 1. This function excludes it and it becomes `|\nmy string ` - /// 2. [yamlEncodeBlock] applies `\n` that we skipped. - /// 2. [normalizeEncodedBlock] ignores the trailing `\n` for literal - /// string by default. + /// 3. [normalizeEncodedBlock] never prunes the dangling `\n` applied for + /// literal strings by default. return '|${ignoreTrailingLineBreak ? '' : '-'}\n' '$indent${trimmed.replaceAll('\n', lineEnding + indent)}' '${stripped.replaceAll(lineEnding, lineEnding + indent)}'; diff --git a/test/string_test.dart b/test/string_test.dart index b92c20a..8fb0fb5 100644 --- a/test/string_test.dart +++ b/test/string_test.dart @@ -6,6 +6,7 @@ import 'package:test/test.dart'; import 'package:yaml/yaml.dart'; import 'package:yaml_edit/yaml_edit.dart'; +// TODO: Add test for string with trailing space final _testStrings = [ "this is a fairly' long string with\nline breaks", 'whitespace\n after line breaks', From 6619266d9c64a9e03497a5d08b414c696f64cd20 Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Thu, 24 Oct 2024 21:15:28 +0100 Subject: [PATCH 24/25] Move extraction of terminal YamlScalar to utility function --- lib/src/utils.dart | 64 ++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 70c1510..7b988b9 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -469,14 +469,14 @@ String normalizeEncodedBlock( required String updateAsString, bool skipPreservationCheck = false, }) { - var terminalNode = update; + final terminalNode = _findTerminalScalar(update); - /// Checks if the dangling line break should be allowed within the deepest - /// [YamlNode] that is a [YamlScalar]. + /// Nested function that checks if the dangling line break should be allowed + /// within the deepest [YamlNode] that is a [YamlScalar]. bool allowInYamlScalar(ScalarStyle style, dynamic value) { /// We never normalize a literal/folded string irrespective of - /// its position. We allow the block indicators to define how line break - /// will be treated + /// its position. We allow the block indicators to define how the + /// line-break will be treated if (style == ScalarStyle.LITERAL || style == ScalarStyle.FOLDED) { return true; } @@ -490,31 +490,6 @@ String normalizeEncodedBlock( return false; } - loop: - while (terminalNode is! YamlScalar) { - switch (terminalNode) { - case YamlList list: - { - if (list.isEmpty) { - terminalNode = list; - break loop; - } - - terminalNode = list.nodes.last; - } - - case YamlMap map: - { - if (map.isEmpty) { - terminalNode = map; - break loop; - } - - terminalNode = map.nodes.entries.last.value; - } - } - } - /// The node may end up being an empty [YamlMap] or [YamlList] or /// [YamlScalar]. if (terminalNode case YamlScalar(style: var style, value: var value) @@ -538,6 +513,35 @@ String normalizeEncodedBlock( return updateAsString.trimRight(); } +/// Returns the terminal [YamlNode] that is a [YamlScalar]. +/// +/// If within a [YamlList], then the last value that is a [YamlScalar]. If +/// within a [YamlMap], then the last entry with a value that is a [YamlScalar]. +YamlScalar? _findTerminalScalar(YamlNode node) { + YamlNode? terminalNode = node; + + while (terminalNode is! YamlScalar) { + switch (terminalNode) { + case YamlList list: + { + if (list.isEmpty) return null; + terminalNode = list.nodes.last; + } + + case YamlMap map: + { + if (map.isEmpty) return null; + terminalNode = map.nodes.entries.last.value; + } + + default: + return null; + } + } + + return terminalNode; +} + extension YamlNodeExtension on YamlNode { /// Returns the [CollectionStyle] of `this` if `this` is [YamlMap] or /// [YamlList]. From a87b3fe86ddca12972b25ef7dbaa3f43c2289eac Mon Sep 17 00:00:00 2001 From: Kelvin Kavisi <68240897+kekavc24@users.noreply.github.com> Date: Sat, 26 Oct 2024 18:31:01 +0100 Subject: [PATCH 25/25] Refactor `skipAndExtractComments` function > Update function doc and inline comments > Make function return named record > Prefer named parameters > Refactor existing code referencing function --- lib/src/editor.dart | 7 ++--- lib/src/list_mutations.dart | 24 ++++++++--------- lib/src/map_mutations.dart | 15 ++++++----- lib/src/utils.dart | 53 +++++++++++++++++-------------------- 4 files changed, 49 insertions(+), 50 deletions(-) diff --git a/lib/src/editor.dart b/lib/src/editor.dart index d44b171..03b4f49 100644 --- a/lib/src/editor.dart +++ b/lib/src/editor.dart @@ -243,13 +243,14 @@ class YamlEditor { final start = _contents.span.start.offset; var end = getContentSensitiveEnd(_contents); final lineEnding = getLineEnding(_yaml); + end = skipAndExtractCommentsInBlock( _yaml, - end, - null, + endOfNodeOffset: end, lineEnding: lineEnding, greedy: true, - ).$1; + ).endOffset; + var encoded = yamlEncodeBlock(valueNode, 0, lineEnding); encoded = normalizeEncodedBlock( _yaml, diff --git a/lib/src/list_mutations.dart b/lib/src/list_mutations.dart index f8d93f9..fea58a7 100644 --- a/lib/src/list_mutations.dart +++ b/lib/src/list_mutations.dart @@ -54,9 +54,11 @@ SourceEdit updateInList( } // Aggressively skip all comments - final (offsetOfLastComment, _) = - skipAndExtractCommentsInBlock(yaml, end, null, lineEnding: lineEnding); - end = offsetOfLastComment; + end = skipAndExtractCommentsInBlock( + yaml, + endOfNodeOffset: end, + lineEnding: lineEnding, + ).endOffset; valueString = normalizeEncodedBlock( yaml, @@ -142,12 +144,11 @@ SourceEdit _appendToBlockList( final lineEnding = getLineEnding(yaml); // Lazily skip all comments and white-space at the end. - final (offset, _) = skipAndExtractCommentsInBlock( + final offset = skipAndExtractCommentsInBlock( yaml, - list.nodes.last.span.end.offset, - null, + endOfNodeOffset: list.nodes.last.span.end.offset, lineEnding: lineEnding, - ); + ).endOffset; var (indentSize, formattedValue) = _formatNewBlock(yamlEdit, list, item); @@ -369,11 +370,10 @@ SourceEdit _removeFromBlockList( // We remove any content belonging to [nodeToRemove] greedily endOffset = skipAndExtractCommentsInBlock( yaml, - endOffset == startOffset ? endOffset + 1 : endOffset, - null, + endOfNodeOffset: endOffset == startOffset ? endOffset + 1 : endOffset, lineEnding: lineEnding, greedy: true, - ).$1; + ).endOffset; final listSize = list.length; @@ -381,7 +381,7 @@ SourceEdit _removeFromBlockList( final isLastElementInList = index == listSize - 1; final isLastInYaml = endOffset == yamlSize; - final replacement = listSize == 1 ? '[]' : ''; + final replacement = isSingleElement ? '[]' : ''; /// Adjust [startIndent] to include any indent this element may have had /// to prevent it from interfering with the indent of the next [YamlNode] @@ -416,7 +416,7 @@ SourceEdit _removeFromBlockList( isSingle: isSingleElement, ); } else if (isLastInYaml && yaml[endOffset - 1] == '\n' && isSingleElement) { - /// Include any trailing line break that may have been part of the yaml: + /// Remove any dangling line-break that may have been part of the yaml: /// -`\r\n` = 2 /// - `\n` = 1 endOffset -= lineEnding == '\n' ? 1 : 2; diff --git a/lib/src/map_mutations.dart b/lib/src/map_mutations.dart index 2a2f906..d0a526a 100644 --- a/lib/src/map_mutations.dart +++ b/lib/src/map_mutations.dart @@ -181,10 +181,12 @@ SourceEdit _replaceInBlockMap( ? start + 1 : end; - // Aggressively skip all comments - final (offsetOfLastComment, _) = - skipAndExtractCommentsInBlock(yaml, end, null, lineEnding: lineEnding); - end = offsetOfLastComment; + // Skip comments lazily + end = skipAndExtractCommentsInBlock( + yaml, + endOfNodeOffset: end, + lineEnding: lineEnding, + ).endOffset; valueAsString = normalizeEncodedBlock( yaml, @@ -235,11 +237,10 @@ SourceEdit _removeFromBlockMap( endOffset = skipAndExtractCommentsInBlock( yaml, - endOffset, - null, + endOfNodeOffset: endOffset, lineEnding: lineEnding, greedy: true, - ).$1; + ).endOffset; final mapSize = map.length; diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 7b988b9..60c3f97 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -273,17 +273,19 @@ String getLineEnding(String yaml) { return windowsNewlines > unixNewlines ? '\r\n' : '\n'; } -/// Extracts comments for a node that is replaced within a [YamlMap] or -/// [YamlList] or a top-level [YamlScalar] of the [yaml] string provided. +/// Skips and extracts comments for a replaced/removed [YamlNode]. /// -/// [endOfNodeOffset] represents the end offset of [YamlScalar] or [YamlList] -/// or [YamlMap] being replaced, that is, `end + 1`. +/// [endOfNodeOffset] represents the end offset of the [YamlNode] being +/// replaced, that is, `end + 1`. /// /// [nextStartOffset] represents the start offset of the next [YamlNode]. -/// May be null if the current [YamlNode] being replaced is the last node -/// in a [YamlScalar] or [YamlList] or if its the only top-level [YamlScalar]. -/// If not sure of the next [YamlNode]'s [nextStartOffset] pass in null and -/// allow this function to handle that manually. +/// Should be null if the current [YamlNode] being replaced is: +/// - The terminal node in a top-level [YamlList] +/// - The last entry or value in an entry in a top-level [YamlMap] +/// - The only top-level [YamlScalar]. +/// +/// It is recommended to ignore or pass in `null` for [nextStartOffset] since +/// this function immediately exits once no comments are found. /// /// If [greedy] is `true`, whitespace and any line breaks are skipped. If /// `false`, this function looks for comments lazily and returns the offset of @@ -298,10 +300,10 @@ String getLineEnding(String yaml) { /// Returns the `endOffset` of the last comment extracted that is `end + 1` /// and a `List comments`. It is recommended (but not necessary) that /// the caller checks the `endOffset` is still within the bounds of the [yaml]. -(int endOffset, List comments) skipAndExtractCommentsInBlock( - String yaml, - int endOfNodeOffset, - int? nextStartOffset, { +({int endOffset, List comments}) skipAndExtractCommentsInBlock( + String yaml, { + required int endOfNodeOffset, + int? nextStartOffset, String lineEnding = '\n', bool greedy = false, }) { @@ -312,7 +314,7 @@ String getLineEnding(String yaml) { if (nextStartOffset == null) { final comments = []; - /// Skips white-space while extracting comments. + /// Nested function that skips white-space while extracting comments. /// /// Returns [null] if the end of the [yaml] was encountered while /// skipping any white-space. Otherwise, returns the [index] of the next @@ -341,8 +343,8 @@ String getLineEnding(String yaml) { return (firstLineBreak, nextIndex); } - /// Returns the [currentOffset] if [greedy] is true. Otherwise, attempts - /// returning the [firstLineBreakOffset] if not null if [greedy] is false. + /// Nested function that returns the [currentOffset] if [greedy] is true. + /// Otherwise, attempts to return the [firstLineBreakOffset] if not null. int earlyBreakOffset(int currentOffset, int? firstLineBreakOffset) { if (greedy) return currentOffset; return firstLineBreakOffset ?? currentOffset; @@ -351,7 +353,7 @@ String getLineEnding(String yaml) { var currentOffset = endOfNodeOffset; while (true) { - if (currentOffset == yaml.length) break; + if (currentOffset >= yaml.length) break; var leadingChar = yaml[currentOffset].trim(); var indexOfCommentStart = -1; @@ -361,9 +363,7 @@ String getLineEnding(String yaml) { if (leadingChar.isEmpty) { final (firstLE, nextIndex) = skipWhitespace(currentOffset); - /// If the next index is null, it means we reached the end of the - /// string. Since we lazily evaluated the string, attempt to return the - /// first [lineEnding] we encountered only if not null. + // We skipped everything to the end of the yaml if (nextIndex == null) { currentOffset = earlyBreakOffset(yaml.length, firstLE); break; @@ -374,16 +374,12 @@ String getLineEnding(String yaml) { leadingChar = yaml[currentOffset]; } - /// We need comments only, nothing else. This may be pointless but will - /// help us avoid extracting comments when provided random offsets - /// within a string. + /// We need comments only! This may be pointless but will help us exit + /// early when provided random offsets within a string. if (leadingChar == '#') indexOfCommentStart = currentOffset; /// This is a mindless assumption that the last character was either /// `\n` or [white-space] or the last erroneus offset provided. - /// - /// Since we lazily evaluated the string, attempt to return the - /// first [lineEnding] we encountered only if not null. if (indexOfCommentStart == -1) { currentOffset = earlyBreakOffset(currentOffset, firstLineBreak); break; @@ -405,12 +401,13 @@ String getLineEnding(String yaml) { currentOffset = indexOfLineBreak; } - return (currentOffset, comments); + return (endOffset: currentOffset, comments: comments); } return ( - nextStartOffset, - yaml.substring(endOfNodeOffset, nextStartOffset).split(lineEnding).fold( + endOffset: nextStartOffset, + comments: + yaml.substring(endOfNodeOffset, nextStartOffset).split(lineEnding).fold( [], (buffer, current) { final comment = current.trim();