Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/mevd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -155,10 +155,10 @@ jobs:
fail-fast: false
matrix:
provider: ${{ fromJson(needs.discover.outputs.all_providers) }}
# AzureAISearch and CosmosMongoDB require cloud instances and can't run in CI.
# AzureAISearch and AzureDocumentDB require cloud instances and can't run in CI.
exclude:
- provider: AzureAISearch
- provider: CosmosMongoDB
- provider: AzureDocumentDB
steps:
- uses: actions/checkout@v6

Expand Down
6 changes: 3 additions & 3 deletions CommunityToolkit.AI.slnx
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
<File Path="MEVD/src/Directory.Build.props" />
<Project Path="MEVD/src/AzureAISearch/AzureAISearch.csproj" />
<Project Path="MEVD/src/CosmosNoSql/CosmosNoSql.csproj" />
<Project Path="MEVD/src/CosmosMongoDB/CosmosMongoDB.csproj" />
<Project Path="MEVD/src/AzureDocumentDB/AzureDocumentDB.csproj" />
<Project Path="MEVD/src/InMemory/InMemory.csproj" />
<Project Path="MEVD/src/PgVector/PgVector.csproj" />
<Project Path="MEVD/src/Qdrant/Qdrant.csproj" />
Expand All @@ -25,8 +25,8 @@
<Project Path="MEVD/test/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj" />
<Project Path="MEVD/test/CosmosNoSql.UnitTests/CosmosNoSql.UnitTests.csproj" />
<Project Path="MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj" />
<Project Path="MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoDB.UnitTests.csproj" />
<Project Path="MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj" />
<Project Path="MEVD/test/AzureDocumentDB.UnitTests/AzureDocumentDB.UnitTests.csproj" />
<Project Path="MEVD/test/AzureDocumentDB.ConformanceTests/AzureDocumentDB.ConformanceTests.csproj" />
<Project Path="MEVD/test/InMemory.UnitTests/InMemory.UnitTests.csproj" />
<Project Path="MEVD/test/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj" />
<Project Path="MEVD/test/PgVector.UnitTests/PgVector.UnitTests.csproj" />
Expand Down
6 changes: 3 additions & 3 deletions MEVD/MEVD.slnf
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
[
"MEVD/src/AzureAISearch/AzureAISearch.csproj",
"MEVD/src/CosmosNoSql/CosmosNoSql.csproj",
"MEVD/src/CosmosMongoDB/CosmosMongoDB.csproj",
"MEVD/src/AzureDocumentDB/AzureDocumentDB.csproj",
"MEVD/src/InMemory/InMemory.csproj",
"MEVD/src/PgVector/PgVector.csproj",
"MEVD/src/Qdrant/Qdrant.csproj",
Expand All @@ -18,8 +18,8 @@
"MEVD/test/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj",
"MEVD/test/CosmosNoSql.UnitTests/CosmosNoSql.UnitTests.csproj",
"MEVD/test/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj",
"MEVD/test/CosmosMongoDB.UnitTests/CosmosMongoDB.UnitTests.csproj",
"MEVD/test/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj",
"MEVD/test/AzureDocumentDB.UnitTests/AzureDocumentDB.UnitTests.csproj",
"MEVD/test/AzureDocumentDB.ConformanceTests/AzureDocumentDB.ConformanceTests.csproj",
"MEVD/test/InMemory.UnitTests/InMemory.UnitTests.csproj",
"MEVD/test/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj",
"MEVD/test/PgVector.UnitTests/PgVector.UnitTests.csproj",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,18 @@

<PropertyGroup>
<Version>1.0.0</Version>
<AssemblyName>CommunityToolkit.VectorData.CosmosMongoDB</AssemblyName>
<AssemblyName>CommunityToolkit.VectorData.AzureDocumentDB</AssemblyName>
<RootNamespace>$(AssemblyName)</RootNamespace>
<TargetFrameworks>net10.0;net8.0;netstandard2.1;net472</TargetFrameworks>
<IsAotCompatible Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0'))">true</IsAotCompatible>

<Title>Azure CosmosDB MongoDB vCore provider for Microsoft.Extensions.VectorData</Title>
<Description>Azure CosmosDB MongoDB vCore provider for Microsoft.Extensions.VectorData by the .NET Community Toolkit</Description>
<Title>Azure DocumentDB (with MongoDB compatibility) provider for Microsoft.Extensions.VectorData</Title>
<Description>Azure DocumentDB (with MongoDB compatibility) provider for Microsoft.Extensions.VectorData by the .NET Community Toolkit</Description>
<PackageTags>AzureDocumentDB;AzureMongoDB;CosmosMongoDB;VectorData</PackageTags>
</PropertyGroup>

<ItemGroup>
<InternalsVisibleTo Include="CommunityToolkit.VectorData.CosmosMongoDB.UnitTests" />
<InternalsVisibleTo Include="CommunityToolkit.VectorData.AzureDocumentDB.UnitTests" />
<InternalsVisibleTo Include="DynamicProxyGenAssembly2" />
</ItemGroup>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,15 @@
using MEVD = Microsoft.Extensions.VectorData;
using Microsoft.Shared.Diagnostics;

namespace CommunityToolkit.VectorData.CosmosMongoDB;
namespace CommunityToolkit.VectorData.AzureDocumentDB;

/// <summary>
/// Service for storing and retrieving vector records, that uses Azure CosmosDB MongoDB as the underlying storage.
/// Service for storing and retrieving vector records, that uses Azure DocumentDB as the underlying storage.
/// </summary>
/// <typeparam name="TKey">The data type of the record key. Must be either <see cref="string"/>.</typeparam>
/// <typeparam name="TRecord">The data model to use for adding, updating and retrieving data from storage.</typeparam>
#pragma warning disable CA1711 // Identifiers should not have incorrect suffix
public class CosmosMongoCollection<TKey, TRecord> : VectorStoreCollection<TKey, TRecord>
public class DocumentDBCollection<TKey, TRecord> : VectorStoreCollection<TKey, TRecord>
where TKey : notnull
where TRecord : class
#pragma warning restore CA1711 // Identifiers should not have incorrect suffix
Expand All @@ -40,10 +40,10 @@ public class CosmosMongoCollection<TKey, TRecord> : VectorStoreCollection<TKey,
/// <summary>The default options for vector search.</summary>
private static readonly MEVD.VectorSearchOptions<TRecord> s_defaultVectorSearchOptions = new();

/// <summary><see cref="IMongoDatabase"/> that can be used to manage the collections in Azure CosmosDB MongoDB.</summary>
/// <summary><see cref="IMongoDatabase"/> that can be used to manage the collections in Azure DocumentDB.</summary>
private readonly IMongoDatabase _mongoDatabase;

/// <summary>Azure CosmosDB MongoDB collection to perform record operations.</summary>
/// <summary>Azure DocumentDB collection to perform record operations.</summary>
private readonly IMongoCollection<BsonDocument> _mongoCollection;

/// <summary>Interface for mapping between a storage model, and the consumer record data model.</summary>
Expand All @@ -70,28 +70,28 @@ public class CosmosMongoCollection<TKey, TRecord> : VectorStoreCollection<TKey,
private static readonly Type[] s_validKeyTypes = [typeof(string), typeof(Guid), typeof(ObjectId), typeof(int), typeof(long)];

/// <summary>
/// Initializes a new instance of the <see cref="CosmosMongoCollection{TKey, TRecord}"/> class.
/// Initializes a new instance of the <see cref="DocumentDBCollection{TKey, TRecord}"/> class.
/// </summary>
/// <param name="mongoDatabase"><see cref="IMongoDatabase"/> that can be used to manage the collections in Azure CosmosDB MongoDB.</param>
/// <param name="name">The name of the collection that this <see cref="CosmosMongoCollection{TKey, TRecord}"/> will access.</param>
/// <param name="mongoDatabase"><see cref="IMongoDatabase"/> that can be used to manage the collections in Azure DocumentDB.</param>
/// <param name="name">The name of the collection that this <see cref="DocumentDBCollection{TKey, TRecord}"/> will access.</param>
/// <param name="options">Optional configuration options for this class.</param>
[RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary<string, object?>, instantiate CosmosMongoDynamicCollection instead.")]
[RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary<string, object?>, instantiate CosmosMongoDynamicCollection instead.")]
public CosmosMongoCollection(
[RequiresDynamicCode("This constructor is incompatible with NativeAOT. For dynamic mapping via Dictionary<string, object?>, instantiate DocumentDBDynamicCollection instead.")]
[RequiresUnreferencedCode("This constructor is incompatible with trimming. For dynamic mapping via Dictionary<string, object?>, instantiate DocumentDBDynamicCollection instead.")]
public DocumentDBCollection(
IMongoDatabase mongoDatabase,
string name,
CosmosMongoCollectionOptions? options = default)
DocumentDBCollectionOptions? options = default)
: this(
mongoDatabase,
name,
static options => typeof(TRecord) == typeof(Dictionary<string, object?>)
? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(CosmosMongoDynamicCollection)))
? throw new NotSupportedException(VectorDataStrings.NonDynamicCollectionWithDictionaryNotSupported(typeof(DocumentDBDynamicCollection)))
: new MongoModelBuilder().Build(typeof(TRecord), typeof(TKey), options.Definition, options.EmbeddingGenerator),
options)
{
}

internal CosmosMongoCollection(IMongoDatabase mongoDatabase, string name, Func<CosmosMongoCollectionOptions, CollectionModel> modelFactory, CosmosMongoCollectionOptions? options)
internal DocumentDBCollection(IMongoDatabase mongoDatabase, string name, Func<DocumentDBCollectionOptions, CollectionModel> modelFactory, DocumentDBCollectionOptions? options)
{
// Verify.
Throw.IfNull(mongoDatabase);
Expand All @@ -102,7 +102,7 @@ internal CosmosMongoCollection(IMongoDatabase mongoDatabase, string name, Func<C
throw new NotSupportedException("Only ObjectID, string, Guid, int and long keys are supported.");
}

options ??= CosmosMongoCollectionOptions.Default;
options ??= DocumentDBCollectionOptions.Default;

// Assign.
_mongoDatabase = mongoDatabase;
Expand All @@ -119,7 +119,7 @@ internal CosmosMongoCollection(IMongoDatabase mongoDatabase, string name, Func<C

_collectionMetadata = new()
{
VectorStoreSystemName = CosmosMongoConstants.VectorStoreSystemName,
VectorStoreSystemName = DocumentDBConstants.VectorStoreSystemName,
VectorStoreName = mongoDatabase.DatabaseNamespace?.DatabaseName,
CollectionName = name
};
Expand Down Expand Up @@ -370,34 +370,34 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null
};

var filter = options.Filter is not null
? new CosmosMongoFilterTranslator().Translate(options.Filter, _model)
? new DocumentDBFilterTranslator().Translate(options.Filter, _model)
: null;

// Constructing a query to fetch "skip + top" total items
// to perform skip logic locally, since skip option is not part of API.
var itemsAmount = options.Skip + top;

var vectorPropertyIndexKind = CosmosMongoCollectionSearchMapping.GetVectorPropertyIndexKind(vectorProperty.IndexKind);
var vectorPropertyIndexKind = DocumentDBCollectionSearchMapping.GetVectorPropertyIndexKind(vectorProperty.IndexKind);

var searchQuery = vectorPropertyIndexKind switch
{
IndexKind.Hnsw => CosmosMongoCollectionSearchMapping.GetSearchQueryForHnswIndex(
IndexKind.Hnsw => DocumentDBCollectionSearchMapping.GetSearchQueryForHnswIndex(
vector,
vectorProperty.StorageName,
itemsAmount,
_efSearch,
filter),
IndexKind.IvfFlat => CosmosMongoCollectionSearchMapping.GetSearchQueryForIvfIndex(
IndexKind.IvfFlat => DocumentDBCollectionSearchMapping.GetSearchQueryForIvfIndex(
vector,
vectorProperty.StorageName,
itemsAmount,
filter),
_ => throw new InvalidOperationException(
$"Index kind '{vectorProperty.IndexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorProperty.StorageName}' is not supported by the Azure CosmosDB for MongoDB VectorStore. " +
$"Index kind '{vectorProperty.IndexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorProperty.StorageName}' is not supported by the Azure DocumentDB VectorStore. " +
$"Supported index kinds are: {string.Join(", ", [IndexKind.Hnsw, IndexKind.IvfFlat])}")
};

var projectionQuery = CosmosMongoCollectionSearchMapping.GetProjectionQuery(
var projectionQuery = DocumentDBCollectionSearchMapping.GetProjectionQuery(
ScorePropertyName,
DocumentPropertyName);

Expand All @@ -406,7 +406,7 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null
// Add score threshold filter as a $match stage if specified
if (options.ScoreThreshold.HasValue)
{
pipeline.Add(CosmosMongoCollectionSearchMapping.GetScoreThresholdMatchQuery(ScorePropertyName, options.ScoreThreshold.Value));
pipeline.Add(DocumentDBCollectionSearchMapping.GetScoreThresholdMatchQuery(ScorePropertyName, options.ScoreThreshold.Value));
}

const string OperationName = "Aggregate";
Expand Down Expand Up @@ -455,7 +455,7 @@ public override async IAsyncEnumerable<TRecord> GetAsync(
options ??= new();

// Translate the filter now, so if it fails, we throw immediately.
var translatedFilter = new CosmosMongoFilterTranslator().Translate(filter, _model);
var translatedFilter = new DocumentDBFilterTranslator().Translate(filter, _model);

SortDefinition<BsonDocument>? sortDefinition = null;
var orderBy = options.OrderBy?.Invoke(new()).Values;
Expand Down Expand Up @@ -503,13 +503,13 @@ private async Task CreateIndexesAsync(string collectionName, CancellationToken c

var indexArray = new BsonArray();

indexArray.AddRange(CosmosMongoCollectionCreateMapping.GetVectorIndexes(
indexArray.AddRange(DocumentDBCollectionCreateMapping.GetVectorIndexes(
_model.VectorProperties,
uniqueIndexes,
_numLists,
_efConstruction));

indexArray.AddRange(CosmosMongoCollectionCreateMapping.GetFilterableDataIndexes(
indexArray.AddRange(DocumentDBCollectionCreateMapping.GetFilterableDataIndexes(
_model.DataProperties,
uniqueIndexes));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
using Microsoft.Extensions.VectorData.ProviderServices;
using MongoDB.Bson;

namespace CommunityToolkit.VectorData.CosmosMongoDB;
namespace CommunityToolkit.VectorData.AzureDocumentDB;

/// <summary>
/// Contains mapping helpers to use when creating a collection in Azure CosmosDB MongoDB.
/// Contains mapping helpers to use when creating a collection in Azure DocumentDB.
/// </summary>
internal static class CosmosMongoCollectionCreateMapping
internal static class DocumentDBCollectionCreateMapping
{
/// <summary>
/// Returns an array of indexes to create for vector properties.
Expand Down Expand Up @@ -104,25 +104,25 @@ public static BsonArray GetFilterableDataIndexes(
}

/// <summary>
/// More information about Azure CosmosDB for MongoDB index kinds here: <see href="https://learn.microsoft.com/azure/cosmos-db/mongodb/vcore/vector-search" />.
/// More information about Azure DocumentDB index kinds here: <see href="https://learn.microsoft.com/azure/documentdb/vector-search" />.
/// </summary>
private static string GetIndexKind(string? indexKind, string vectorPropertyName)
=> CosmosMongoCollectionSearchMapping.GetVectorPropertyIndexKind(indexKind) switch
=> DocumentDBCollectionSearchMapping.GetVectorPropertyIndexKind(indexKind) switch
{
IndexKind.Hnsw => "vector-hnsw",
IndexKind.IvfFlat => "vector-ivf",
_ => throw new NotSupportedException($"Index kind '{indexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB for MongoDB VectorStore.")
_ => throw new NotSupportedException($"Index kind '{indexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure DocumentDB VectorStore.")
};

/// <summary>
/// More information about Azure CosmosDB for MongoDB distance functions here: <see href="https://learn.microsoft.com/azure/cosmos-db/mongodb/vcore/vector-search" />.
/// More information about Azure DocumentDB distance functions here: <see href="https://learn.microsoft.com/azure/documentdb/vector-search" />.
/// </summary>
private static string GetDistanceFunction(string? distanceFunction, string vectorPropertyName)
=> CosmosMongoCollectionSearchMapping.GetVectorPropertyDistanceFunction(distanceFunction) switch
=> DocumentDBCollectionSearchMapping.GetVectorPropertyDistanceFunction(distanceFunction) switch
{
DistanceFunction.CosineDistance => "COS",
DistanceFunction.DotProductSimilarity => "IP",
DistanceFunction.EuclideanDistance => "L2",
_ => throw new NotSupportedException($"Distance function '{distanceFunction}' for {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB for MongoDB VectorStore.")
_ => throw new NotSupportedException($"Distance function '{distanceFunction}' for {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure DocumentDB VectorStore.")
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,23 @@

using Microsoft.Extensions.VectorData;

namespace CommunityToolkit.VectorData.CosmosMongoDB;
namespace CommunityToolkit.VectorData.AzureDocumentDB;

/// <summary>
/// Options when creating a <see cref="CosmosMongoCollection{TKey, TRecord}"/>.
/// Options when creating a <see cref="DocumentDBCollection{TKey, TRecord}"/>.
/// </summary>
public sealed class CosmosMongoCollectionOptions : VectorStoreCollectionOptions
public sealed class DocumentDBCollectionOptions : VectorStoreCollectionOptions
{
internal static readonly CosmosMongoCollectionOptions Default = new();
internal static readonly DocumentDBCollectionOptions Default = new();

/// <summary>
/// Initializes a new instance of the <see cref="CosmosMongoCollectionOptions"/> class.
/// Initializes a new instance of the <see cref="DocumentDBCollectionOptions"/> class.
/// </summary>
public CosmosMongoCollectionOptions()
public DocumentDBCollectionOptions()
{
}

internal CosmosMongoCollectionOptions(CosmosMongoCollectionOptions? source) : base(source)
internal DocumentDBCollectionOptions(DocumentDBCollectionOptions? source) : base(source)
{
NumLists = source?.NumLists ?? Default.NumLists;
EfConstruction = source?.EfConstruction ?? Default.EfConstruction;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
using MongoDB.Bson;


namespace CommunityToolkit.VectorData.CosmosMongoDB;
namespace CommunityToolkit.VectorData.AzureDocumentDB;

/// <summary>
/// Contains mapping helpers to use when searching for documents using Azure CosmosDB MongoDB.
/// Contains mapping helpers to use when searching for documents using Azure DocumentDB.
/// </summary>
internal static class CosmosMongoCollectionSearchMapping
internal static class DocumentDBCollectionSearchMapping
{
/// <summary>Returns index kind specified on vector property or default <see cref="MongoConstants.DefaultIndexKind"/>.</summary>
public static string GetVectorPropertyIndexKind(string? indexKind) => !string.IsNullOrWhiteSpace(indexKind) ? indexKind! : MongoConstants.DefaultIndexKind;
Expand Down Expand Up @@ -98,7 +98,7 @@ public static BsonDocument GetProjectionQuery(string scorePropertyName, string d

/// <summary>Returns a $match stage to filter results by score threshold.</summary>
/// <remarks>
/// Cosmos MongoDB returns a similarity score where higher values mean more similar,
/// Azure DocumentDB returns a similarity score where higher values mean more similar,
/// so we filter with $gte to keep results at or above the threshold.
/// </remarks>
public static BsonDocument GetScoreThresholdMatchQuery(string scorePropertyName, double scoreThreshold)
Expand Down
Loading
Loading