From 25b0940984f5246a775fe4f3766a7aacf3842a54 Mon Sep 17 00:00:00 2001
From: JKamsker <11245306+JKamsker@users.noreply.github.com>
Date: Fri, 7 Jun 2024 00:30:41 +0200
Subject: [PATCH 1/4] Add support for on-the-fly document upgrades
---
.../Database/DocumentUpgrade_Tests.cs | 76 +++++++++++++++++++
LiteDB/Client/Structures/ConnectionString.cs | 6 +-
LiteDB/Document/DataReader/BsonDataReader.cs | 12 +--
LiteDB/Engine/EngineSettings.cs | 8 +-
LiteDB/Engine/EngineState.cs | 16 ++--
5 files changed, 104 insertions(+), 14 deletions(-)
create mode 100644 LiteDB.Tests/Database/DocumentUpgrade_Tests.cs
diff --git a/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs b/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs
new file mode 100644
index 000000000..b265326b0
--- /dev/null
+++ b/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs
@@ -0,0 +1,76 @@
+using FluentAssertions;
+
+using LiteDB.Engine;
+
+using System.IO;
+
+using Xunit;
+
+namespace LiteDB.Tests.Database;
+
+public class DocumentUpgrade_Tests
+{
+ [Fact]
+ public void DocumentUpgrade_Test()
+ {
+ var ms = new MemoryStream();
+ using (var db = new LiteDatabase(ms))
+ {
+ var col = db.GetCollection("col");
+
+ col.Insert(new BsonDocument { ["version"] = 1, ["_id"] = 1, ["name"] = "John" });
+ }
+
+ ms.Position = 0;
+
+ using (var db = new LiteDatabase(ms))
+ {
+ var col = db.GetCollection("col");
+
+ col.Count().Should().Be(1);
+
+ var doc = col.FindById(1);
+
+ doc["version"].AsInt32.Should().Be(1);
+ doc["name"].AsString.Should().Be("John");
+ doc["age"].AsInt32.Should().Be(0);
+ }
+
+ ms.Position = 0;
+
+ var engine = new LiteEngine(new EngineSettings
+ {
+ DataStream = ms,
+ ReadTransform = ReadTransform
+ });
+
+ using (var db = new LiteDatabase(engine))
+ {
+ var col = db.GetCollection("col");
+
+ col.Count().Should().Be(1);
+
+ var doc = col.FindById(1);
+
+ doc["version"].AsInt32.Should().Be(2);
+ doc["name"].AsString.Should().Be("John");
+ doc["age"].AsInt32.Should().Be(30);
+ }
+ }
+
+ private BsonValue ReadTransform(string arg1, BsonValue val)
+ {
+ if (!(val is BsonDocument bdoc))
+ {
+ return val;
+ }
+
+ if (bdoc.TryGetValue("version", out var version) && version.AsInt32 == 1)
+ {
+ bdoc["version"] = 2;
+ bdoc["age"] = 30;
+ }
+
+ return val;
+ }
+}
\ No newline at end of file
diff --git a/LiteDB/Client/Structures/ConnectionString.cs b/LiteDB/Client/Structures/ConnectionString.cs
index 3f44acd76..945649764 100644
--- a/LiteDB/Client/Structures/ConnectionString.cs
+++ b/LiteDB/Client/Structures/ConnectionString.cs
@@ -1,4 +1,4 @@
-using LiteDB.Engine;
+using LiteDB.Engine;
using System;
using System.Collections.Generic;
using System.Globalization;
@@ -107,7 +107,7 @@ public ConnectionString(string connectionString)
///
/// Create ILiteEngine instance according string connection parameters. For now, only Local/Shared are supported
///
- internal ILiteEngine CreateEngine()
+ internal ILiteEngine CreateEngine(Action engineSettingsAction = null)
{
var settings = new EngineSettings
{
@@ -120,6 +120,8 @@ internal ILiteEngine CreateEngine()
AutoRebuild = this.AutoRebuild,
};
+ engineSettingsAction?.Invoke(settings);
+
// create engine implementation as Connection Type
if (this.Connection == ConnectionType.Direct)
{
diff --git a/LiteDB/Document/DataReader/BsonDataReader.cs b/LiteDB/Document/DataReader/BsonDataReader.cs
index c76125085..33131362a 100644
--- a/LiteDB/Document/DataReader/BsonDataReader.cs
+++ b/LiteDB/Document/DataReader/BsonDataReader.cs
@@ -1,4 +1,4 @@
-using LiteDB.Engine;
+using LiteDB.Engine;
using System;
using System.Collections;
using System.Collections.Generic;
@@ -56,10 +56,10 @@ internal BsonDataReader(IEnumerable values, string collection, Engine
if (_source.MoveNext())
{
_hasValues = _isFirst = true;
- _current = _source.Current;
+ _current = _state.ReadTransform(_collection, _source.Current);
}
}
- catch(Exception ex)
+ catch (Exception ex)
{
_state.Handle(ex);
throw;
@@ -102,10 +102,10 @@ public bool Read()
try
{
var read = _source.MoveNext(); // can throw any error here
- _current = _source.Current;
+ _current = _state.ReadTransform(_collection, _source.Current);
return read;
}
- catch(Exception ex)
+ catch (Exception ex)
{
_state.Handle(ex);
throw ex;
@@ -117,7 +117,7 @@ public bool Read()
}
}
}
-
+
public BsonValue this[string field]
{
get
diff --git a/LiteDB/Engine/EngineSettings.cs b/LiteDB/Engine/EngineSettings.cs
index d2bb96372..609dedfad 100644
--- a/LiteDB/Engine/EngineSettings.cs
+++ b/LiteDB/Engine/EngineSettings.cs
@@ -1,4 +1,4 @@
-using System;
+using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
@@ -7,6 +7,7 @@
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
+
using static LiteDB.Constants;
namespace LiteDB.Engine
@@ -66,6 +67,11 @@ public class EngineSettings
///
public bool Upgrade { get; set; } = false;
+ ///
+ /// Is used to transform a from the database on read. This can be used to upgrade data from older versions.
+ ///
+ public Func ReadTransform { get; set; }
+
///
/// Create new IStreamFactory for datafile
///
diff --git a/LiteDB/Engine/EngineState.cs b/LiteDB/Engine/EngineState.cs
index cfdad8a25..bd3dafac7 100644
--- a/LiteDB/Engine/EngineState.cs
+++ b/LiteDB/Engine/EngineState.cs
@@ -1,4 +1,4 @@
-using System;
+using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
@@ -9,7 +9,6 @@
using static LiteDB.Constants;
-
namespace LiteDB.Engine
{
internal class EngineState
@@ -25,7 +24,7 @@ internal class EngineState
#endif
public EngineState(LiteEngine engine, EngineSettings settings)
- {
+ {
_engine = engine;
_settings = settings;
}
@@ -39,7 +38,7 @@ public bool Handle(Exception ex)
{
LOG(ex.Message, "ERROR");
- if (ex is IOException ||
+ if (ex is IOException ||
(ex is LiteException lex && lex.ErrorCode == LiteException.INVALID_DATAFILE_STATE))
{
_exception = ex;
@@ -51,5 +50,12 @@ public bool Handle(Exception ex)
return true;
}
+
+ public BsonValue ReadTransform(string collection, BsonValue value)
+ {
+ if (_settings?.ReadTransform is null) return value;
+
+ return _settings.ReadTransform(collection, value);
+ }
}
-}
+}
\ No newline at end of file
From 02e23d92361a0eb1817f1f11c03a096d5cda5679 Mon Sep 17 00:00:00 2001
From: JKamsker
Date: Fri, 7 Jun 2024 00:52:58 +0200
Subject: [PATCH 2/4] Add deserialization hook to bsonmapper
---
.../Database/DocumentUpgrade_Tests.cs | 77 ++++++++++++++++---
.../Client/Mapper/BsonMapper.Deserialize.cs | 27 +++++++
LiteDB/LiteDB.csproj | 1 +
3 files changed, 95 insertions(+), 10 deletions(-)
diff --git a/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs b/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs
index b265326b0..5bf3ad3a2 100644
--- a/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs
+++ b/LiteDB.Tests/Database/DocumentUpgrade_Tests.cs
@@ -1,4 +1,4 @@
-using FluentAssertions;
+using FluentAssertions;
using LiteDB.Engine;
@@ -38,10 +38,24 @@ public void DocumentUpgrade_Test()
ms.Position = 0;
- var engine = new LiteEngine(new EngineSettings
+ using var engine = new LiteEngine(new EngineSettings
{
DataStream = ms,
- ReadTransform = ReadTransform
+ ReadTransform = (collectionName, val) =>
+ {
+ if (val is not BsonDocument doc)
+ {
+ return val;
+ }
+
+ if (doc.TryGetValue("version", out var version) && version.AsInt32 == 1)
+ {
+ doc["version"] = 2;
+ doc["age"] = 30;
+ }
+
+ return val;
+ }
});
using (var db = new LiteDatabase(engine))
@@ -58,19 +72,62 @@ public void DocumentUpgrade_Test()
}
}
- private BsonValue ReadTransform(string arg1, BsonValue val)
+ [Fact]
+ public void DocumentUpgrade_BsonMapper_Test()
{
- if (!(val is BsonDocument bdoc))
+ var ms = new MemoryStream();
+ using (var db = new LiteDatabase(ms))
{
- return val;
+ var col = db.GetCollection("col");
+
+ col.Insert(new BsonDocument { ["version"] = 1, ["_id"] = 1, ["name"] = "John" });
}
- if (bdoc.TryGetValue("version", out var version) && version.AsInt32 == 1)
+ ms.Position = 0;
+
+ using (var db = new LiteDatabase(ms))
{
- bdoc["version"] = 2;
- bdoc["age"] = 30;
+ var col = db.GetCollection("col");
+
+ col.Count().Should().Be(1);
+
+ var doc = col.FindById(1);
+
+ doc["version"].AsInt32.Should().Be(1);
+ doc["name"].AsString.Should().Be("John");
+ doc["age"].AsInt32.Should().Be(0);
}
- return val;
+ ms.Position = 0;
+
+ var mapper = new BsonMapper();
+ mapper.OnDeserialization = (sender, type, val) =>
+ {
+ if (val is not BsonDocument doc)
+ {
+ return val;
+ }
+
+ if (doc.TryGetValue("version", out var version) && version.AsInt32 == 1)
+ {
+ doc["version"] = 2;
+ doc["age"] = 30;
+ }
+
+ return doc;
+ };
+
+ using (var db = new LiteDatabase(ms, mapper))
+ {
+ var col = db.GetCollection("col");
+
+ col.Count().Should().Be(1);
+
+ var doc = col.FindById(1);
+
+ doc["version"].AsInt32.Should().Be(2);
+ doc["name"].AsString.Should().Be("John");
+ doc["age"].AsInt32.Should().Be(30);
+ }
}
}
\ No newline at end of file
diff --git a/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs b/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs
index 3b24d4b2c..ef7f1b10d 100644
--- a/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs
+++ b/LiteDB/Client/Mapper/BsonMapper.Deserialize.cs
@@ -9,6 +9,24 @@ namespace LiteDB
{
public partial class BsonMapper
{
+ #region Deserialization Hooks
+
+ ///
+ /// Delegate for deserialization callback.
+ ///
+ /// The BsonMapper instance that triggered the deserialization.
+ /// The target type for deserialization.
+ /// The BsonValue to be deserialized.
+ /// The deserialized BsonValue.
+ public delegate BsonValue DeserializationCallback(BsonMapper sender, Type target, BsonValue value);
+
+ ///
+ /// Gets called before deserialization of a value
+ ///
+ public DeserializationCallback? OnDeserialization { get; set; }
+
+ #endregion Deserialization Hooks
+
#region Basic direct .NET convert types
// direct bson types
@@ -78,6 +96,15 @@ public T Deserialize(BsonValue value)
///
public object Deserialize(Type type, BsonValue value)
{
+ if (OnDeserialization is not null)
+ {
+ var result = OnDeserialization(this, type, value);
+ if (result is not null)
+ {
+ value = result;
+ }
+ }
+
// null value - null returns
if (value.IsNull) return null;
diff --git a/LiteDB/LiteDB.csproj b/LiteDB/LiteDB.csproj
index 721395e91..5d26fbd8e 100644
--- a/LiteDB/LiteDB.csproj
+++ b/LiteDB/LiteDB.csproj
@@ -28,6 +28,7 @@
true
LiteDB.snk
true
+ 9.0