Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[OneCollectorExporter] Perf improvements #1361

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/OpenTelemetry.Exporter.OneCollector/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@
extension because it is not currently supported by the OneCollector service.
([#1345](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/1345))

* Added dedicated handling for `IReadOnlyList<KeyValuePair<string, object>>`
types during serialization to improve performance.
([#1361](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/1361))

* Added caching of extension property UTF8 JSON strings to improve performance.
([#1361](https://github.com/open-telemetry/opentelemetry-dotnet-contrib/pull/1361))

## 1.5.1

Released 2023-Aug-07
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// <copyright file="ExtensionFieldInformation.cs" company="OpenTelemetry Authors">
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// </copyright>

using System.Text.Json;

namespace OpenTelemetry.Exporter.OneCollector;

internal sealed class ExtensionFieldInformation
{
public string? ExtensionName;
public JsonEncodedText EncodedExtensionName;
public string? FieldName;
public JsonEncodedText EncodedFieldName;

public bool IsValid => this.ExtensionName != null;
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@

using System.Collections;
using System.Diagnostics;
#if NETSTANDARD2_1_OR_GREATER || NET6_0_OR_GREATER
using System.Diagnostics.CodeAnalysis;
#endif
using System.Text.Json;

namespace OpenTelemetry.Exporter.OneCollector;

Expand All @@ -28,9 +32,14 @@ internal sealed class ExtensionFieldInformationManager

public int CountOfCachedExtensionFields => this.fieldInformationCache.Count;

public bool TryResolveExtensionFieldInformation(string fullFieldName, out (string ExtensionName, string FieldName) resolvedFieldInformation)
public bool TryResolveExtensionFieldInformation(
string fullFieldName,
#if NETSTANDARD2_1_OR_GREATER || NET6_0_OR_GREATER
[NotNullWhen(true)]
#endif
out ExtensionFieldInformation? resolvedFieldInformation)
{
if (this.fieldInformationCache[fullFieldName] is not FieldInformation fieldInformation)
if (this.fieldInformationCache[fullFieldName] is not ExtensionFieldInformation fieldInformation)
{
fieldInformation = this.ResolveExtensionFieldInformationRare(fullFieldName);
}
Expand All @@ -41,39 +50,44 @@ public bool TryResolveExtensionFieldInformation(string fullFieldName, out (strin
return false;
}

resolvedFieldInformation = new(fieldInformation.ExtensionName!, fieldInformation.FieldName!);
resolvedFieldInformation = fieldInformation;
return true;
}

private static FieldInformation BuildFieldInformation(string fullFieldName)
private static ExtensionFieldInformation BuildFieldInformation(string fullFieldName)
{
Debug.Assert(fullFieldName.Length >= 4, "fullFieldName length was invalid");
Debug.Assert(fullFieldName.StartsWith("ext.", StringComparison.OrdinalIgnoreCase), "fullFieldName did not start with 'ext.'");

var extensionName = fullFieldName.AsSpan().Slice(4);
var extensionName = fullFieldName.AsSpan().Slice(4).TrimEnd();
var locationOfDot = extensionName.IndexOf('.');
if (locationOfDot <= 0)
{
return new();
}

var fieldName = extensionName.Slice(locationOfDot + 1);
var fieldName = extensionName.Slice(locationOfDot + 1).TrimStart();
if (fieldName.Length <= 0)
{
return new();
}

extensionName = extensionName.Slice(0, locationOfDot);
extensionName = extensionName.Slice(0, locationOfDot).TrimEnd();
if (extensionName.Length <= 0)
{
return new();
}

return new FieldInformation
return new ExtensionFieldInformation
{
ExtensionName = extensionName.ToString(),
EncodedExtensionName = JsonEncodedText.Encode(extensionName),
FieldName = fieldName.ToString(),
IsValid = true,
EncodedFieldName = JsonEncodedText.Encode(fieldName),
};
}

private FieldInformation ResolveExtensionFieldInformationRare(string fullFieldName)
private ExtensionFieldInformation ResolveExtensionFieldInformationRare(string fullFieldName)
{
if (this.fieldInformationCache.Count >= MaxNumberOfCachedFieldInformations)
{
Expand All @@ -82,7 +96,7 @@ private FieldInformation ResolveExtensionFieldInformationRare(string fullFieldNa

lock (this.fieldInformationCache)
{
if (this.fieldInformationCache[fullFieldName] is not FieldInformation fieldInformation)
if (this.fieldInformationCache[fullFieldName] is not ExtensionFieldInformation fieldInformation)
{
fieldInformation = BuildFieldInformation(fullFieldName);
if (this.fieldInformationCache.Count < MaxNumberOfCachedFieldInformations)
Expand All @@ -94,11 +108,4 @@ private FieldInformation ResolveExtensionFieldInformationRare(string fullFieldNa
return fieldInformation;
}
}

private sealed class FieldInformation
{
public string? ExtensionName;
public string? FieldName;
public bool IsValid;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,10 @@ public static void SerializeValueToJson(object? value, Utf8JsonWriter writer)
SerializeArrayValueToJson(v, writer);
return;

case IReadOnlyList<KeyValuePair<string, object?>> v:
SerializeMapValueToJson(v, writer);
return;

case IEnumerable<KeyValuePair<string, object?>> v:
SerializeMapValueToJson(v, writer);
return;
Expand All @@ -180,6 +184,25 @@ private static void SerializeArrayValueToJson(Array value, Utf8JsonWriter writer
writer.WriteEndArray();
}

private static void SerializeMapValueToJson(IReadOnlyList<KeyValuePair<string, object?>> value, Utf8JsonWriter writer)
{
writer.WriteStartObject();

for (int i = 0; i < value.Count; i++)
{
var element = value[i];

if (string.IsNullOrEmpty(element.Key))
{
continue;
}

SerializeKeyValueToJson(element.Key, element.Value, writer);
}

writer.WriteEndObject();
}

private static void SerializeMapValueToJson(IEnumerable<KeyValuePair<string, object?>> value, Utf8JsonWriter writer)
{
writer.WriteStartObject();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
// limitations under the License.
// </copyright>

using System.Diagnostics;
#if NET6_0_OR_GREATER
using System.Runtime.InteropServices;
#endif
Expand All @@ -26,7 +27,7 @@ internal sealed class CommonSchemaJsonSerializationState
public const int MaxNumberOfExtensionKeys = 64;
public const int MaxNumberOfExtensionValuesPerKey = 16;
private readonly Dictionary<string, int> keys = new(4, StringComparer.OrdinalIgnoreCase);
private readonly List<KeyValuePair<string, object?>> allValues = new(16);
private readonly List<KeyValuePair<ExtensionFieldInformation, object?>> allValues = new(16);
private string itemType;
private int nextKeysToAllValuesLookupIndex;
private KeyValueLookup[] keysToAllValuesLookup = new KeyValueLookup[4];
Expand All @@ -47,23 +48,28 @@ public void AddExtensionAttribute(KeyValuePair<string, object?> attribute)
{
if (!ExtensionFieldInformationManager.SharedCache.TryResolveExtensionFieldInformation(
attribute.Key,
out (string ExtensionName, string FieldName) fieldInformation))
out var fieldInformation))
{
OneCollectorExporterEventSource.Log.AttributeDropped(this.itemType, attribute.Key, "Invalid extension field name");
return;
}

Debug.Assert(fieldInformation?.ExtensionName != null, "fieldInformation.ExtensionName was null");
Debug.Assert(fieldInformation?.EncodedExtensionName.EncodedUtf8Bytes.Length > 0, "fieldInformation.EncodedExtensionName was empty");
Debug.Assert(fieldInformation?.FieldName != null, "fieldInformation.FieldName was null");
Debug.Assert(fieldInformation?.EncodedFieldName.EncodedUtf8Bytes.Length > 0, "fieldInformation.EncodedFieldName was empty");

#if NET6_0_OR_GREATER
ref var lookupIndex = ref CollectionsMarshal.GetValueRefOrAddDefault(this.keys, fieldInformation.ExtensionName, out var existed);
if (!existed)
{
this.AssignNewExtensionToLookupIndex(ref lookupIndex);
}
#else
if (!this.keys.TryGetValue(fieldInformation.ExtensionName, out int lookupIndex))
if (!this.keys.TryGetValue(fieldInformation!.ExtensionName!, out int lookupIndex))
{
this.AssignNewExtensionToLookupIndex(ref lookupIndex);
this.keys[fieldInformation.ExtensionName] = lookupIndex;
this.keys[fieldInformation.ExtensionName!] = lookupIndex;
}
#endif

Expand All @@ -82,7 +88,7 @@ public void AddExtensionAttribute(KeyValuePair<string, object?> attribute)
}

int index = this.allValues.Count;
this.allValues.Add(new KeyValuePair<string, object?>(fieldInformation.FieldName, attribute.Value));
this.allValues.Add(new KeyValuePair<ExtensionFieldInformation, object?>(fieldInformation, attribute.Value));

unsafe
{
Expand All @@ -107,7 +113,7 @@ public void SerializeExtensionPropertiesToJson(bool writeExtensionObjectEnvelope

foreach (var extensionPropertyKey in this.keys)
{
writer.WriteStartObject(extensionPropertyKey.Key);
var wroteStartObject = false;

ref KeyValueLookup keyLookup = ref this.keysToAllValuesLookup[extensionPropertyKey.Value];

Expand All @@ -120,12 +126,23 @@ public void SerializeExtensionPropertiesToJson(bool writeExtensionObjectEnvelope
#else
var attribute = allValues[keyLookup.ValueIndicies[i]];
#endif
var fieldInformation = attribute.Key;

if (!wroteStartObject)
{
writer.WriteStartObject(fieldInformation.EncodedExtensionName);
wroteStartObject = true;
}

CommonSchemaJsonSerializationHelper.SerializeKeyValueToJson(attribute.Key, attribute.Value, writer);
writer.WritePropertyName(fieldInformation.EncodedFieldName);
CommonSchemaJsonSerializationHelper.SerializeValueToJson(attribute.Value, writer);
}
}

writer.WriteEndObject();
if (wroteStartObject)
{
writer.WriteEndObject();
}
}

if (writeExtensionObjectEnvelope)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,11 @@ public void SerializeComplexValueToJsonTest()
var array = new[] { 0, 1, 18 };
this.SerializeValueToJsonTest(array, "[0,1,18]");

var map = new List<KeyValuePair<string, object?>> { new KeyValuePair<string, object?>("key1", "value1") };
this.SerializeValueToJsonTest(map, "{\"key1\":\"value1\"}");
var listMap = new List<KeyValuePair<string, object?>> { new KeyValuePair<string, object?>("key1", "value1") };
this.SerializeValueToJsonTest(listMap, "{\"key1\":\"value1\"}");

var dictMap = new Dictionary<string, object?> { ["key1"] = "value1" };
this.SerializeValueToJsonTest(dictMap, "{\"key1\":\"value1\"}");

var typeWithToString = new TypeWithToString();
this.SerializeValueToJsonTest(typeWithToString, "\"Hello world\"");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public void AddExtensionAttributeTest()
var json = Encoding.UTF8.GetString(stream.ToArray());

Assert.Equal(
"{\"ext\":{\"something\":{\"field1\":1,\"field2\":2,\"field3\":3,\"field4\":6},\"food\":{\"field1\":4,\"field2\":5}}}",
"""{"ext":{"something":{"field1":1,"field2":2,"field3":3,"field4":6},"food":{"field1":4,"field2":5}}}""",
json);

stream.SetLength(0);
Expand All @@ -74,7 +74,7 @@ public void AddExtensionAttributeTest()
json = Encoding.UTF8.GetString(stream.ToArray());

Assert.Equal(
"{\"ext\":{\"something\":{\"field1\":1},\"food\":{\"field1\":1}}}",
"""{"ext":{"something":{"field1":1},"food":{"field1":1}}}""",
json);
}

Expand Down Expand Up @@ -102,7 +102,7 @@ public void AddExtensionAttributeDuplicatesTest()
var json = Encoding.UTF8.GetString(stream.ToArray());

Assert.Equal(
"{\"ext\":{\"something\":{\"field1\":1,\"field1\":2}}}",
"""{"ext":{"something":{"field1":1,"field1":2}}}""",
json);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public void FieldInformationIsCachedTest()
var result = extensionFieldInformationManager.TryResolveExtensionFieldInformation("ext.something.fieldName1", out var fieldInformation);

Assert.True(result);
Assert.NotNull(fieldInformation);
Assert.Equal("something", fieldInformation.ExtensionName);
Assert.Equal("fieldName1", fieldInformation.FieldName);

Expand All @@ -38,6 +39,7 @@ public void FieldInformationIsCachedTest()
Assert.Equal(1, extensionFieldInformationManager.CountOfCachedExtensionFields);

Assert.True(result);
Assert.NotNull(fieldInformation);
Assert.Equal("something", fieldInformation.ExtensionName);
Assert.Equal("fieldName1", fieldInformation.FieldName);

Expand All @@ -46,6 +48,7 @@ public void FieldInformationIsCachedTest()
Assert.Equal(2, extensionFieldInformationManager.CountOfCachedExtensionFields);

Assert.True(result);
Assert.NotNull(fieldInformation);
Assert.Equal("something", fieldInformation.ExtensionName);
Assert.Equal("field.Name2", fieldInformation.FieldName);
}
Expand All @@ -54,19 +57,23 @@ public void FieldInformationIsCachedTest()
public void InvalidFieldNamesIgnoredTest()
{
var extensionFieldInformationManager = new ExtensionFieldInformationManager();

Assert.False(extensionFieldInformationManager.TryResolveExtensionFieldInformation("ext.", out _));

Assert.Equal(1, extensionFieldInformationManager.CountOfCachedExtensionFields);

Assert.False(extensionFieldInformationManager.TryResolveExtensionFieldInformation("ext.", out _));
Assert.False(extensionFieldInformationManager.TryResolveExtensionFieldInformation("EXT.", out _));

Assert.Equal(1, extensionFieldInformationManager.CountOfCachedExtensionFields);

Assert.False(extensionFieldInformationManager.TryResolveExtensionFieldInformation("ext..", out _));
Assert.False(extensionFieldInformationManager.TryResolveExtensionFieldInformation("ext. .", out _));
Assert.False(extensionFieldInformationManager.TryResolveExtensionFieldInformation("ext..field", out _));
Assert.False(extensionFieldInformationManager.TryResolveExtensionFieldInformation("ext.something", out _));
Assert.False(extensionFieldInformationManager.TryResolveExtensionFieldInformation("ext.something.", out _));
Assert.False(extensionFieldInformationManager.TryResolveExtensionFieldInformation("ext.SOMETHING.", out _));
Assert.False(extensionFieldInformationManager.TryResolveExtensionFieldInformation("ext.something. ", out _));

Assert.Equal(3, extensionFieldInformationManager.CountOfCachedExtensionFields);
Assert.Equal(7, extensionFieldInformationManager.CountOfCachedExtensionFields);
}

[Fact]
Expand All @@ -81,6 +88,7 @@ public void FieldInformationCacheLimitTest()
var result = extensionFieldInformationManager.TryResolveExtensionFieldInformation($"ext.something.{fieldName}", out var fieldInformation);

Assert.True(result);
Assert.NotNull(fieldInformation);
Assert.Equal("something", fieldInformation.ExtensionName);
Assert.Equal(fieldName, fieldInformation.FieldName);
}
Expand Down
Loading
Loading