diff --git a/src/libraries/System.Memory/src/System/Buffers/SequenceReader.Search.cs b/src/libraries/System.Memory/src/System/Buffers/SequenceReader.Search.cs index 27aef7056647d..8fb93907b62e9 100644 --- a/src/libraries/System.Memory/src/System/Buffers/SequenceReader.Search.cs +++ b/src/libraries/System.Memory/src/System/Buffers/SequenceReader.Search.cs @@ -32,7 +32,7 @@ public bool TryReadTo(out ReadOnlySpan span, T delimiter, bool advancePastDel private bool TryReadToSlow(out ReadOnlySpan span, T delimiter, bool advancePastDelimiter) { - if (!TryReadToInternal(out ReadOnlySequence sequence, delimiter, advancePastDelimiter, CurrentSpan.Length - CurrentSpanIndex)) + if (!TryReadToInternal(out ReadOnlySequence sequence, delimiter, advancePastDelimiter, _currentSpan.Length - _currentSpanIndex)) { span = default; return false; @@ -132,7 +132,7 @@ private bool TryReadToSlow(out ReadOnlySequence sequence, T delimiter, T deli // Found the delimiter. Move to it, slice, then move past it. AdvanceCurrentSpan(index); - sequence = Sequence.Slice(copy.Position, Position); + sequence = _sequence.Slice(copy.Position, Position); if (advancePastDelimiter) { Advance(1); @@ -198,7 +198,7 @@ private bool TryReadToInternal(out ReadOnlySequence sequence, T delimiter, bo Advance(skip); ReadOnlySpan remaining = UnreadSpan; - while (_moreData) + while (!End) { int index = remaining.IndexOf(delimiter); if (index != -1) @@ -209,7 +209,7 @@ private bool TryReadToInternal(out ReadOnlySequence sequence, T delimiter, bo AdvanceCurrentSpan(index); } - sequence = Sequence.Slice(copy.Position, Position); + sequence = _sequence.Slice(copy.Position, Position); if (advancePastDelimiter) { Advance(1); @@ -243,7 +243,7 @@ public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, T delimiter ReadOnlySpan remaining = UnreadSpan; bool priorEscape = false; - while (_moreData) + while (!End) { int index = remaining.IndexOf(delimiter); if (index != -1) @@ -286,7 +286,7 @@ public bool TryReadTo(out ReadOnlySequence sequence, T delimiter, T delimiter Advance(index); } - sequence = Sequence.Slice(copy.Position, Position); + sequence = _sequence.Slice(copy.Position, Position); if (advancePastDelimiter) { Advance(1); @@ -344,7 +344,7 @@ public bool TryReadToAny(out ReadOnlySpan span, scoped ReadOnlySpan delimi private bool TryReadToAnySlow(out ReadOnlySpan span, scoped ReadOnlySpan delimiters, bool advancePastDelimiter) { - if (!TryReadToAnyInternal(out ReadOnlySequence sequence, delimiters, advancePastDelimiter, CurrentSpan.Length - CurrentSpanIndex)) + if (!TryReadToAnyInternal(out ReadOnlySequence sequence, delimiters, advancePastDelimiter, _currentSpan.Length - _currentSpanIndex)) { span = default; return false; @@ -387,7 +387,7 @@ private bool TryReadToAnyInternal(out ReadOnlySequence sequence, scoped ReadO AdvanceCurrentSpan(index); } - sequence = Sequence.Slice(copy.Position, Position); + sequence = _sequence.Slice(copy.Position, Position); if (advancePastDelimiter) { Advance(1); @@ -481,7 +481,7 @@ public bool TryReadTo(out ReadOnlySequence sequence, scoped ReadOnlySpan d // Probably a faster way to do this, potentially by avoiding the Advance in the previous TryReadTo call if (advanced) { - sequence = copy.Sequence.Slice(copy.Consumed, Consumed - copy.Consumed); + sequence = copy._sequence.Slice(copy.Consumed, Consumed - copy.Consumed); } if (advancePastDelimiter) @@ -520,7 +520,7 @@ public bool TryReadExact(int count, out ReadOnlySequence sequence) return false; } - sequence = Sequence.Slice(Position, count); + sequence = _sequence.Slice(Position, count); if (count != 0) { Advance(count); @@ -578,22 +578,22 @@ public long AdvancePast(T value) { // Advance past all matches in the current span int i; - for (i = CurrentSpanIndex; i < CurrentSpan.Length && CurrentSpan[i].Equals(value); i++) + ReadOnlySpan unread = UnreadSpan; // eat the slice to avoid bounds checks + for (i = 0; i < unread.Length && unread[i].Equals(value); i++) { } - int advanced = i - CurrentSpanIndex; - if (advanced == 0) + if (i == 0) { // Didn't advance at all in this span, exit. break; } - AdvanceCurrentSpan(advanced); + AdvanceCurrentSpan(i); // If we're at position 0 after advancing and not at the End, // we're in a new span and should continue the loop. - } while (CurrentSpanIndex == 0 && !End); + } while (_currentSpanIndex == 0 && !End); return Consumed - start; } @@ -610,22 +610,22 @@ public long AdvancePastAny(scoped ReadOnlySpan values) { // Advance past all matches in the current span int i; - for (i = CurrentSpanIndex; i < CurrentSpan.Length && values.IndexOf(CurrentSpan[i]) != -1; i++) + ReadOnlySpan unread = UnreadSpan; // eat the slice to avoid bounds checks + for (i = 0; i < unread.Length && values.IndexOf(unread[i]) != -1; i++) { } - int advanced = i - CurrentSpanIndex; - if (advanced == 0) + if (i == 0) { // Didn't advance at all in this span, exit. break; } - AdvanceCurrentSpan(advanced); + AdvanceCurrentSpan(i); // If we're at position 0 after advancing and not at the End, // we're in a new span and should continue the loop. - } while (CurrentSpanIndex == 0 && !End); + } while (_currentSpanIndex == 0 && !End); return Consumed - start; } @@ -642,27 +642,27 @@ public long AdvancePastAny(T value0, T value1, T value2, T value3) { // Advance past all matches in the current span int i; - for (i = CurrentSpanIndex; i < CurrentSpan.Length; i++) + ReadOnlySpan unread = UnreadSpan; // eat the slice to avoid bounds checks + for (i = 0; i < unread.Length; i++) { - T value = CurrentSpan[i]; + ref readonly T value = ref unread[i]; if (!value.Equals(value0) && !value.Equals(value1) && !value.Equals(value2) && !value.Equals(value3)) { break; } } - int advanced = i - CurrentSpanIndex; - if (advanced == 0) + if (i == 0) { // Didn't advance at all in this span, exit. break; } - AdvanceCurrentSpan(advanced); + AdvanceCurrentSpan(i); // If we're at position 0 after advancing and not at the End, // we're in a new span and should continue the loop. - } while (CurrentSpanIndex == 0 && !End); + } while (_currentSpanIndex == 0 && !End); return Consumed - start; } @@ -679,27 +679,27 @@ public long AdvancePastAny(T value0, T value1, T value2) { // Advance past all matches in the current span int i; - for (i = CurrentSpanIndex; i < CurrentSpan.Length; i++) + ReadOnlySpan unread = UnreadSpan; // eat the slice to avoid bounds checks + for (i = 0; i < unread.Length; i++) { - T value = CurrentSpan[i]; + ref readonly T value = ref unread[i]; if (!value.Equals(value0) && !value.Equals(value1) && !value.Equals(value2)) { break; } } - int advanced = i - CurrentSpanIndex; - if (advanced == 0) + if (i == 0) { // Didn't advance at all in this span, exit. break; } - AdvanceCurrentSpan(advanced); + AdvanceCurrentSpan(i); // If we're at position 0 after advancing and not at the End, // we're in a new span and should continue the loop. - } while (CurrentSpanIndex == 0 && !End); + } while (_currentSpanIndex == 0 && !End); return Consumed - start; } @@ -716,27 +716,27 @@ public long AdvancePastAny(T value0, T value1) { // Advance past all matches in the current span int i; - for (i = CurrentSpanIndex; i < CurrentSpan.Length; i++) + ReadOnlySpan unread = UnreadSpan; // eat the slice to avoid bounds checks + for (i = 0; i < unread.Length; i++) { - T value = CurrentSpan[i]; + ref readonly T value = ref unread[i]; if (!value.Equals(value0) && !value.Equals(value1)) { break; } } - int advanced = i - CurrentSpanIndex; - if (advanced == 0) + if (i == 0) { // Didn't advance at all in this span, exit. break; } - AdvanceCurrentSpan(advanced); + AdvanceCurrentSpan(i); // If we're at position 0 after advancing and not at the End, // we're in a new span and should continue the loop. - } while (CurrentSpanIndex == 0 && !End); + } while (_currentSpanIndex == 0 && !End); return Consumed - start; } @@ -746,14 +746,18 @@ public long AdvancePastAny(T value0, T value1) /// public void AdvanceToEnd() { - if (_moreData) + AssertValidPosition(); + if (!End) { - Consumed = Length; - CurrentSpan = default; - CurrentSpanIndex = 0; - _currentPosition = Sequence.End; - _nextPosition = default; - _moreData = false; + _consumedAtStartOfCurrentSpan = Length; + _currentSpanIndex = 0; + _currentSpan = default; + + SequencePosition position = _sequence.End; + _currentPositionObject = position.GetObject(); + _currentPositionInteger = position.GetInteger(); + + AssertValidPosition(); } } @@ -768,7 +772,7 @@ public bool IsNext(T next, bool advancePast = false) if (End) return false; - if (CurrentSpan[CurrentSpanIndex].Equals(next)) + if (_currentSpan[_currentSpanIndex].Equals(next)) { if (advancePast) { @@ -809,7 +813,7 @@ private bool IsNextSlow(scoped ReadOnlySpan next, bool advancePast) Debug.Assert(currentSpan.Length < next.Length); int fullLength = next.Length; - SequencePosition nextPosition = _nextPosition; + object? segment = _currentPositionObject; while (next.StartsWith(currentSpan)) { @@ -823,25 +827,19 @@ private bool IsNextSlow(scoped ReadOnlySpan next, bool advancePast) return true; } + // move past the values we have validated + next = next.Slice(currentSpan.Length); + // Need to check the next segment - while (true) + if (TryGetNextBuffer(in _sequence, ref segment, ref currentSpan) != TryGetNextBufferResult.SuccessHaveData) { - if (!Sequence.TryGet(ref nextPosition, out ReadOnlyMemory nextSegment, advance: true)) - { - // Nothing left - return false; - } + // Nothing left + return false; + } - if (nextSegment.Length > 0) - { - next = next.Slice(currentSpan.Length); - currentSpan = nextSegment.Span; - if (currentSpan.Length > next.Length) - { - currentSpan = currentSpan.Slice(0, next.Length); - } - break; - } + if (currentSpan.Length > next.Length) + { + currentSpan = currentSpan.Slice(0, next.Length); } } diff --git a/src/libraries/System.Memory/src/System/Buffers/SequenceReader.cs b/src/libraries/System.Memory/src/System/Buffers/SequenceReader.cs index b574a7298c6f0..86c8552430160 100644 --- a/src/libraries/System.Memory/src/System/Buffers/SequenceReader.cs +++ b/src/libraries/System.Memory/src/System/Buffers/SequenceReader.cs @@ -8,10 +8,16 @@ namespace System.Buffers { public ref partial struct SequenceReader where T : unmanaged, IEquatable { - private SequencePosition _currentPosition; - private SequencePosition _nextPosition; - private bool _moreData; + // keep all fields explicit to track (and pack) space + + // deconstruct position for packing purposes + private object? _currentPositionObject; + private int _currentPositionInteger, _currentSpanIndex; + private readonly long _length; + private long _consumedAtStartOfCurrentSpan; + private readonly ReadOnlySequence _sequence; + private ReadOnlySpan _currentSpan; /// /// Create a over the given . @@ -19,32 +25,41 @@ namespace System.Buffers [MethodImpl(MethodImplOptions.AggressiveInlining)] public SequenceReader(ReadOnlySequence sequence) { - CurrentSpanIndex = 0; - Consumed = 0; - Sequence = sequence; - _currentPosition = sequence.Start; - _length = -1; + _sequence = sequence; - sequence.GetFirstSpan(out ReadOnlySpan first, out _nextPosition); - CurrentSpan = first; - _moreData = first.Length > 0; + SequencePosition position = sequence.Start; + _currentPositionObject = position.GetObject(); + _currentPositionInteger = position.GetInteger(); + _currentSpan = sequence.FirstSpan; + _currentSpanIndex = 0; + _consumedAtStartOfCurrentSpan = 0; - if (!_moreData && !sequence.IsSingleSegment) + if (sequence.IsSingleSegment) + { + _length = _currentSpan.Length; + } + else { - _moreData = true; - GetNextSpan(); + _length = -1; // computed on-demand + if (_currentSpan.IsEmpty) + { + // edge-case; multi-segment with zero-length as first + TryGetNextSpan(); + } } + + AssertValidPosition(); } /// /// True when there is no more data in the . /// - public readonly bool End => !_moreData; + public readonly bool End => _currentSpanIndex == _currentSpan.Length; // because of eager fetch, only ever true at EOF /// /// The underlying for the reader. /// - public ReadOnlySequence Sequence { get; } + public readonly ReadOnlySequence Sequence => _sequence; /// /// Gets the unread portion of the . @@ -52,23 +67,22 @@ public SequenceReader(ReadOnlySequence sequence) /// /// The unread portion of the . /// - public readonly ReadOnlySequence UnreadSequence => Sequence.Slice(Position); + public readonly ReadOnlySequence UnreadSequence => _sequence.Slice(Position); /// /// The current position in the . /// - public readonly SequencePosition Position - => Sequence.GetPosition(CurrentSpanIndex, _currentPosition); + public readonly SequencePosition Position => new(_currentPositionObject, _currentPositionInteger + _currentSpanIndex); // since index in same segment, this is valid /// /// The current segment in the as a span. /// - public ReadOnlySpan CurrentSpan { get; private set; } + public readonly ReadOnlySpan CurrentSpan => _currentSpan; /// /// The index in the . /// - public int CurrentSpanIndex { get; private set; } + public readonly int CurrentSpanIndex => _currentSpanIndex; /// /// The unread portion of the . @@ -76,13 +90,13 @@ public readonly SequencePosition Position public readonly ReadOnlySpan UnreadSpan { [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => CurrentSpan.Slice(CurrentSpanIndex); + get => _currentSpan.Slice(_currentSpanIndex); } /// /// The total number of 's processed by the reader. /// - public long Consumed { get; private set; } + public readonly long Consumed => _consumedAtStartOfCurrentSpan + _currentSpanIndex; /// /// Remaining 's in the reader's . @@ -99,7 +113,7 @@ public readonly long Length if (_length < 0) { // Cast-away readonly to initialize lazy field - Unsafe.AsRef(in _length) = Sequence.Length; + Unsafe.AsRef(in _length) = _sequence.Length; } return _length; } @@ -113,16 +127,20 @@ public readonly long Length [MethodImpl(MethodImplOptions.AggressiveInlining)] public readonly bool TryPeek(out T value) { - if (_moreData) + AssertValidPosition(); + + // hoisting into locals for the compare allows us to avoid a bounds check + int i = _currentSpanIndex; + ReadOnlySpan currentSpan = _currentSpan; + + if ((uint)i < (uint)currentSpan.Length) { - value = CurrentSpan[CurrentSpanIndex]; + value = currentSpan[i]; return true; } - else - { - value = default; - return false; - } + + value = default; + return false; } /// @@ -133,11 +151,12 @@ public readonly bool TryPeek(out T value) /// true if the reader is not at its end and the peek operation succeeded; false if at the end of the reader. public readonly bool TryPeek(long offset, out T value) { + AssertValidPosition(); if (offset < 0) ThrowHelper.ThrowArgumentOutOfRangeException_OffsetOutOfRange(); // If we've got data and offset is not out of bounds - if (!_moreData || Remaining <= offset) + if (_currentSpanIndex == _currentSpan.Length || Remaining <= offset) { value = default; return false; @@ -145,40 +164,36 @@ public readonly bool TryPeek(long offset, out T value) // Sum CurrentSpanIndex + offset could overflow as is but the value of offset should be very large // because we check Remaining <= offset above so to overflow we should have a ReadOnlySequence close to 8 exabytes - Debug.Assert(CurrentSpanIndex + offset >= 0); + Debug.Assert(_currentSpanIndex + offset >= 0); // If offset doesn't fall inside current segment move to next until we find correct one - if ((CurrentSpanIndex + offset) <= CurrentSpan.Length - 1) + if ((_currentSpanIndex + offset) <= _currentSpan.Length - 1) { Debug.Assert(offset <= int.MaxValue); - value = CurrentSpan[CurrentSpanIndex + (int)offset]; + value = _currentSpan[_currentSpanIndex + (int)offset]; return true; } else { - long remainingOffset = offset - (CurrentSpan.Length - CurrentSpanIndex); - SequencePosition nextPosition = _nextPosition; - ReadOnlyMemory currentMemory; + long remainingOffset = offset - (_currentSpan.Length - _currentSpanIndex); + + object? segment = _currentPositionObject; - while (Sequence.TryGet(ref nextPosition, out currentMemory, advance: true)) + ReadOnlySpan currentSpan = default; + while (TryGetNextBuffer(in _sequence, ref segment, ref currentSpan) == TryGetNextBufferResult.SuccessHaveData) { - // Skip empty segment - if (currentMemory.Length > 0) + if (remainingOffset >= currentSpan.Length) + { + // Subtract current non consumed data + remainingOffset -= currentSpan.Length; + } + else { - if (remainingOffset >= currentMemory.Length) - { - // Subtract current non consumed data - remainingOffset -= currentMemory.Length; - } - else - { - break; - } + break; } } - - value = currentMemory.Span[(int)remainingOffset]; + value = currentSpan[(int)remainingOffset]; return true; } } @@ -191,22 +206,25 @@ public readonly bool TryPeek(long offset, out T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TryRead(out T value) { - if (End) - { - value = default; - return false; - } + AssertValidPosition(); - value = CurrentSpan[CurrentSpanIndex]; - CurrentSpanIndex++; - Consumed++; + // hoisting into locals for the compare allows us to avoid a bounds check + int i = _currentSpanIndex; + ReadOnlySpan currentSpan = _currentSpan; - if (CurrentSpanIndex >= CurrentSpan.Length) + if ((uint)i < (uint)currentSpan.Length) { - GetNextSpan(); - } + value = currentSpan[i]; - return true; + if ((_currentSpanIndex = i + 1) == currentSpan.Length) + { + TryGetNextSpan(); // move ahead eagerly + } + AssertValidPosition(); + return true; + } + value = default; + return false; } /// @@ -228,17 +246,15 @@ public void Rewind(long count) return; } - Consumed -= count; - - if (CurrentSpanIndex >= count) + if (_currentSpanIndex >= count) { - CurrentSpanIndex -= (int)count; - _moreData = true; + _currentSpanIndex -= (int)count; + AssertValidPosition(); } else { // Current segment doesn't have enough data, scan backward through segments - RetreatToPreviousSpan(Consumed); + RetreatToPreviousSpan(Consumed - count); } } @@ -251,60 +267,93 @@ private void RetreatToPreviousSpan(long consumed) private void ResetReader() { - CurrentSpanIndex = 0; - Consumed = 0; - _currentPosition = Sequence.Start; - _nextPosition = _currentPosition; + // preserve the length - it can be relatively expensive to calculate on demand + long length = _length; - if (Sequence.TryGet(ref _nextPosition, out ReadOnlyMemory memory, advance: true)) - { - _moreData = true; + // reset all state + this = new(_sequence); - if (memory.Length == 0) - { - CurrentSpan = default; - // No data in the first span, move to one with data - GetNextSpan(); - } - else - { - CurrentSpan = memory.Span; - } - } - else - { - // No data in any spans and at end of sequence - _moreData = false; - CurrentSpan = default; - } + // reinstate the length + Unsafe.AsRef(in _length) = length; } /// /// Get the next segment with available data, if any. /// - private void GetNextSpan() + [MethodImpl(MethodImplOptions.NoInlining)] + private bool TryGetNextSpan() + { + int lastLength = _currentSpan.Length; + switch (TryGetNextBuffer(in _sequence, ref _currentPositionObject, ref _currentSpan)) + { + case TryGetNextBufferResult.SuccessHaveData: + _currentPositionInteger = _currentSpanIndex = 0; + _consumedAtStartOfCurrentSpan += lastLength; // track consumed length + AssertValidPosition(); + return true; + case TryGetNextBufferResult.FailureAllRemainingSegmentsEmpty: + // means we advanced, so we still need to reset some things + _currentPositionInteger = _currentSpanIndex = 0; + _consumedAtStartOfCurrentSpan += lastLength; // track consumed length + break; + default: // no more segments + // make sure we position at the end of the last (current) segment, + // even if we didn't change segment + _currentSpanIndex = lastLength; + break; + } + AssertValidPosition(); + return false; + } + + [Conditional("DEBUG")] + private readonly void AssertValidPosition() { - if (!Sequence.IsSingleSegment) + Debug.Assert(_currentSpanIndex >= 0 && _currentSpanIndex <= _currentSpan.Length, "span index out of range"); + if (_currentSpanIndex == _currentSpan.Length) // should only be at-length for EOF { - SequencePosition previousNextPosition = _nextPosition; - while (Sequence.TryGet(ref _nextPosition, out ReadOnlyMemory memory, advance: true)) + ReadOnlySpan span = default; + object? segment = _currentPositionObject; + Debug.Assert(TryGetNextBuffer(in _sequence, ref segment, ref span) == TryGetNextBufferResult.FailureNoMoreSegments, "failed to eagerly read-ahead"); + } + } + + private enum TryGetNextBufferResult + { + SuccessHaveData = 0, + FailureNoMoreSegments = 1, + FailureAllRemainingSegmentsEmpty = 2, + } + + private static TryGetNextBufferResult TryGetNextBuffer(in ReadOnlySequence sequence, ref object? @object, ref ReadOnlySpan buffer) + { + SequencePosition end = sequence.End; + object? endObject = end.GetObject(); + TryGetNextBufferResult result = TryGetNextBufferResult.FailureNoMoreSegments; + ReadOnlySequenceSegment? segment = @object as ReadOnlySequenceSegment; + if (segment is not null) + { + while (!ReferenceEquals(segment, endObject) && (@object = segment = segment!.Next) is not null) { - _currentPosition = previousNextPosition; - if (memory.Length > 0) + buffer = segment!.Memory.Span; + + if (ReferenceEquals(segment, endObject)) { - CurrentSpan = memory.Span; - CurrentSpanIndex = 0; - return; + // the last segment ends early + buffer = buffer.Slice(0, end.GetInteger()); } - else + + if (!buffer.IsEmpty) // skip empty segments { - CurrentSpan = default; - CurrentSpanIndex = 0; - previousNextPosition = _nextPosition; + result = TryGetNextBufferResult.SuccessHaveData; + break; } + + // tell the caller that we *had* more segments (and have advanced), even if not useful + result = TryGetNextBufferResult.FailureAllRemainingSegmentsEmpty; } } - _moreData = false; + return result; } /// @@ -313,17 +362,19 @@ private void GetNextSpan() [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Advance(long count) { + AssertValidPosition(); const long TooBigOrNegative = unchecked((long)0xFFFFFFFF80000000); - if ((count & TooBigOrNegative) == 0 && CurrentSpan.Length - CurrentSpanIndex > (int)count) + if ((count & TooBigOrNegative) == 0 & _currentSpan.Length - _currentSpanIndex > (int)count) { - CurrentSpanIndex += (int)count; - Consumed += count; + // ^^^ note & (rather than &&) is intentional; both sides can be safely evaluated + _currentSpanIndex += (int)count; } else { // Can't satisfy from the current span AdvanceToNextSpan(count); } + AssertValidPosition(); } /// @@ -332,57 +383,44 @@ public void Advance(long count) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void AdvanceCurrentSpan(long count) { - Debug.Assert(count >= 0); - - Consumed += count; - CurrentSpanIndex += (int)count; - if (CurrentSpanIndex >= CurrentSpan.Length) - GetNextSpan(); - } - - /// - /// Only call this helper if you know that you are advancing in the current span - /// with valid count and there is no need to fetch the next one. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void AdvanceWithinSpan(long count) - { - Debug.Assert(count >= 0); + // worst case here is we end at the exact end of the current span + Debug.Assert(count >= 0 && _currentSpanIndex + count <= _currentSpan.Length); - Consumed += count; - CurrentSpanIndex += (int)count; - - Debug.Assert(CurrentSpanIndex < CurrentSpan.Length); + _currentSpanIndex += (int)count; + if (_currentSpanIndex == _currentSpan.Length) + { + TryGetNextSpan(); + } + AssertValidPosition(); } + [MethodImpl(MethodImplOptions.NoInlining)] // avoid inlining this too much from Advance private void AdvanceToNextSpan(long count) { + AssertValidPosition(); if (count < 0) { ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); } - Consumed += count; - while (_moreData) + while (true) { - int remaining = CurrentSpan.Length - CurrentSpanIndex; + int remaining = _currentSpan.Length - _currentSpanIndex; if (remaining > count) { - CurrentSpanIndex += (int)count; + _currentSpanIndex += (int)count; count = 0; break; } - // As there may not be any further segments we need to - // push the current index to the end of the span. - CurrentSpanIndex += remaining; count -= remaining; Debug.Assert(count >= 0); - GetNextSpan(); - - if (count == 0) + // always want to move to next span here, even + // if we've consumed everything we want (to enforce + // eager fetch) + if (!TryGetNextSpan() || count == 0) { break; } @@ -391,9 +429,9 @@ private void AdvanceToNextSpan(long count) if (count != 0) { // Not enough data left- adjust for where we actually ended and throw - Consumed -= count; ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count); } + AssertValidPosition(); } /// @@ -430,24 +468,20 @@ internal readonly bool TryCopyMultisegment(Span destination) if (Remaining < destination.Length) return false; - ReadOnlySpan firstSpan = UnreadSpan; - Debug.Assert(firstSpan.Length < destination.Length); - firstSpan.CopyTo(destination); - int copied = firstSpan.Length; + ReadOnlySpan currentSpan = UnreadSpan; + Debug.Assert(currentSpan.Length < destination.Length); + currentSpan.CopyTo(destination); + int copied = currentSpan.Length; - SequencePosition next = _nextPosition; - while (Sequence.TryGet(ref next, out ReadOnlyMemory nextSegment, true)) + object? segment = _currentPositionObject; + while (TryGetNextBuffer(in _sequence, ref segment, ref currentSpan) == TryGetNextBufferResult.SuccessHaveData) { - if (nextSegment.Length > 0) + int toCopy = Math.Min(currentSpan.Length, destination.Length - copied); + currentSpan.Slice(0, toCopy).CopyTo(destination.Slice(copied)); + copied += toCopy; + if (copied >= destination.Length) { - ReadOnlySpan nextSpan = nextSegment.Span; - int toCopy = Math.Min(nextSpan.Length, destination.Length - copied); - nextSpan.Slice(0, toCopy).CopyTo(destination.Slice(copied)); - copied += toCopy; - if (copied >= destination.Length) - { - break; - } + break; } }