Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for SQL Server Vector Search #722

Merged
merged 21 commits into from
Sep 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
563f6ca
Add support for Vector Search
marcominerva Jul 17, 2024
ac6e6ff
Merge branch 'main' into sqlserver-vectorsearch
marcominerva Jul 23, 2024
4190706
Merge branch 'main' into sqlserver-vectorsearch
marcominerva Jul 29, 2024
8c0fd3b
Merge commit
marcominerva Jul 30, 2024
7f030c5
Update code
marcominerva Jul 30, 2024
0d81477
Refactor SQL Server Memory moving all SQL manipulations in external d…
marcominerva Jul 31, 2024
8652676
Move common logic to base class
marcominerva Aug 1, 2024
8dcdc8b
Delete unnecessary check in Delete Index query
marcominerva Aug 2, 2024
4614b02
Add new details on UseVectorSearch property.
marcominerva Aug 2, 2024
d2705f1
Merge branch 'main' into sqlserver-vectorsearch
dluc Aug 19, 2024
8ccb729
Merge branch 'main' into sqlserver-vectorsearch
marcominerva Sep 2, 2024
fddcf90
Merge branch 'main' into sqlserver-vectorsearch
marcominerva Sep 9, 2024
2f7f9fd
Merge branch 'main' into sqlserver-vectorsearch
dluc Sep 10, 2024
7339688
Merge branch 'main' into sqlserver-vectorsearch
dluc Sep 10, 2024
d287dff
Update extensions/SQLServer/SQLServer/DependencyInjection.cs
dluc Sep 10, 2024
bd25d17
Merge branch 'main' into sqlserver-vectorsearch
marcominerva Sep 11, 2024
910e814
Merge branch 'sqlserver-vectorsearch' of https://github.com/marcomine…
marcominerva Sep 11, 2024
8f224a3
Rename config prop
dluc Sep 20, 2024
42b7940
Fix code style
dluc Sep 20, 2024
3806104
Refactor to use interface
dluc Sep 20, 2024
d8b4b1e
Merge branch 'main' into sqlserver-vectorsearch
dluc Sep 20, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 15 additions & 11 deletions extensions/SQLServer/SQLServer/DependencyInjection.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ namespace Microsoft.KernelMemory;
public static partial class KernelMemoryBuilderExtensions
{
/// <summary>
/// Kernel Memory Builder extension method to add SqlServer memory connector.
/// Kernel Memory Builder extension method to add SQL Server memory connector.
/// </summary>
/// <param name="builder">KM builder instance</param>
/// <param name="config">SqlServer configuration</param>
/// <param name="config">SQL Server configuration</param>
public static IKernelMemoryBuilder WithSqlServerMemoryDb(
this IKernelMemoryBuilder builder,
SqlServerConfig config)
Expand All @@ -27,15 +27,17 @@ public static IKernelMemoryBuilder WithSqlServerMemoryDb(
}

/// <summary>
/// Kernel Memory Builder extension method to add SqlServer memory connector.
/// Kernel Memory Builder extension method to add SQL Server memory connector.
/// </summary>
/// <param name="builder">KM builder instance</param>
/// <param name="connString">SqlServer connection string</param>
/// <param name="connString">SQL Server connection string</param>
/// <param name="useNativeVectorSearch">Whether to use native vector search or not</param>
public static IKernelMemoryBuilder WithSqlServerMemoryDb(
this IKernelMemoryBuilder builder,
string connString)
string connString,
bool useNativeVectorSearch = false)
{
builder.Services.AddSqlServerAsMemoryDb(connString);
builder.Services.AddSqlServerAsMemoryDb(connString, useNativeVectorSearch);
return builder;
}
}
Expand All @@ -46,10 +48,10 @@ public static IKernelMemoryBuilder WithSqlServerMemoryDb(
public static partial class DependencyInjection
{
/// <summary>
/// Inject SqlServer as the default implementation of IMemoryDb
/// Inject SQL Server as the default implementation of IMemoryDb
/// </summary>
/// <param name="services">Service collection</param>
/// <param name="config">Postgres configuration</param>
/// <param name="config">SQL Server configuration</param>
public static IServiceCollection AddSqlServerAsMemoryDb(
this IServiceCollection services,
SqlServerConfig config)
Expand All @@ -60,15 +62,17 @@ public static IServiceCollection AddSqlServerAsMemoryDb(
}

/// <summary>
/// Inject SqlServer as the default implementation of IMemoryDb
/// Inject SQL Server as the default implementation of IMemoryDb
/// </summary>
/// <param name="services">Service collection</param>
/// <param name="connString">SQL Server connection string</param>
/// <param name="useNativeVectorSearch">Whether to use native vector search or not</param>
public static IServiceCollection AddSqlServerAsMemoryDb(
this IServiceCollection services,
string connString)
string connString,
bool useNativeVectorSearch = false)
{
var config = new SqlServerConfig { ConnectionString = connString };
var config = new SqlServerConfig { ConnectionString = connString, UseNativeVectorSearch = useNativeVectorSearch };
return services.AddSqlServerAsMemoryDb(config);
}
}

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright (c) Microsoft. All rights reserved.

using Microsoft.Data.SqlClient;

namespace Microsoft.KernelMemory.MemoryDb.SQLServer.QueryProviders;

public interface ISqlServerQueryProvider
{
/// <summary>
/// Return SQL used to create a new index
/// </summary>
public string PrepareCreateIndexQuery(
int sqlServerVersion,
string index,
int vectorSize);

/// <summary>
/// Return SQL used to get a list of indexes
/// </summary>
public string PrepareGetIndexesQuery();

/// <summary>
/// Return SQL used to delete an index
/// </summary>
public string PrepareDeleteIndexQuery(string index);

/// <summary>
/// Return SQL used to delete a memory record
/// </summary>
public string PrepareDeleteRecordQuery(string index);

/// <summary>
/// Return SQL used to get a list of memory records
/// </summary>
public string PrepareGetRecordsListQuery(
string index,
ICollection<MemoryFilter>? filters,
bool withEmbedding,
SqlParameterCollection parameters);

/// <summary>
/// Return SQL used to get a list of similar memory records
/// </summary>
public string PrepareGetSimilarRecordsListQuery(
string index,
ICollection<MemoryFilter>? filters,
bool withEmbedding,
SqlParameterCollection parameters);

/// <summary>
/// Return SQL used to upsert a batch of memory records
/// </summary>
public string PrepareUpsertRecordsBatchQuery(string index);

/// <summary>
/// Return SQL used to create all supporting tables
/// </summary>
public string PrepareCreateAllSupportingTablesQuery();
}
85 changes: 85 additions & 0 deletions extensions/SQLServer/SQLServer/QueryProviders/Utils.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// Copyright (c) Microsoft. All rights reserved.

using System.Globalization;
using System.Text;
using Microsoft.Data.SqlClient;

namespace Microsoft.KernelMemory.MemoryDb.SQLServer.QueryProviders;

internal static class Utils
{
/// <summary>
/// Gets the full table name with schema.
/// </summary>
/// <param name="config">Server settings</param>
/// <param name="tableName">The table name.</param>
internal static string GetFullTableName(SqlServerConfig config, string tableName)
{
return $"[{config.Schema}].[{tableName}]";
}

/// <summary>
/// Generates the filters as SQL commands and sets the SQL parameters
/// </summary>
/// <param name="config">Server settings</param>
/// <param name="index">The index name.</param>
/// <param name="parameters">The SQL parameters to populate.</param>
/// <param name="filters">The filters to apply</param>
internal static string GenerateFilters(
SqlServerConfig config,
string index,
SqlParameterCollection parameters,
ICollection<MemoryFilter>? filters)
{
var filterBuilder = new StringBuilder();

if (filters is null || filters.Count <= 0 || filters.All(f => f.Count <= 0))
{
return string.Empty;
}

filterBuilder.Append("AND ( ");

for (int i = 0; i < filters.Count; i++)
{
var filter = filters.ElementAt(i);

if (i > 0)
{
filterBuilder.Append(" OR ");
}

for (int j = 0; j < filter.Pairs.Count(); j++)
{
var value = filter.Pairs.ElementAt(j);

if (j > 0)
{
filterBuilder.Append(" AND ");
}

filterBuilder.Append(" ( ");

filterBuilder.Append(CultureInfo.CurrentCulture, $@"EXISTS (
SELECT
1
FROM {GetFullTableName(config, $"{config.TagsTableName}_{index}")} AS [tags]
WHERE
[tags].[memory_id] = {GetFullTableName(config, config.MemoryTableName)}.[id]
AND [name] = @filter_{i}_{j}_name
AND [value] = @filter_{i}_{j}_value
)
");

filterBuilder.Append(" ) ");

parameters.AddWithValue($"@filter_{i}_{j}_name", value.Key);
parameters.AddWithValue($"@filter_{i}_{j}_value", value.Value);
}
}

filterBuilder.Append(" )");

return filterBuilder.ToString();
}
}
Loading
Loading