Skip to content

Optimize indexed read-only collection writes #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
skuirrels merged 1 commit into from
Aug 4, 2026
Merged

Optimize indexed read-only collection writes #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 48 additions & 4 deletions DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ internal sealed unsafe class ListVectorDataWriter : VectorDataWriterBase
typeof(ListVectorDataWriter).GetMethod(nameof(WriteArray), BindingFlags.Static | BindingFlags.NonPublic)!;
private static readonly MethodInfo WriteListMethod =
typeof(ListVectorDataWriter).GetMethod(nameof(WriteList), BindingFlags.Static | BindingFlags.NonPublic)!;
private static readonly MethodInfo WriteIListMethod =
typeof(ListVectorDataWriter).GetMethod(nameof(WriteIList), BindingFlags.Static | BindingFlags.NonPublic)!;
private static readonly MethodInfo WriteReadOnlyListMethod =
typeof(ListVectorDataWriter).GetMethod(nameof(WriteReadOnlyList), BindingFlags.Static | BindingFlags.NonPublic)!;
private static readonly MethodInfo WriteEnumerableMethod =
typeof(ListVectorDataWriter).GetMethod(nameof(WriteEnumerable), BindingFlags.Static | BindingFlags.NonPublic)!;

Expand Down Expand Up @@ -115,8 +119,22 @@ private static CollectionWriterPlan CreateCollectionWriter(Type collectionType)
}
else
{
elementType = GetEnumerableElementType(collectionType);
openMethod = elementType is null ? null : WriteEnumerableMethod;
elementType = GetUniqueGenericElementType(collectionType, typeof(IEnumerable<>));
if (elementType is not null)
{
if (typeof(IList<>).MakeGenericType(elementType).IsAssignableFrom(collectionType))
{
openMethod = WriteIListMethod;
}
else if (typeof(IReadOnlyList<>).MakeGenericType(elementType).IsAssignableFrom(collectionType))
{
openMethod = WriteReadOnlyListMethod;
}
else
{
openMethod = WriteEnumerableMethod;
}
}
}

return new CollectionWriterPlan(
Expand All @@ -125,14 +143,14 @@ openMethod is null || elementType is null
: openMethod.MakeGenericMethod(elementType).CreateDelegate<CollectionWriter>());
}

private static Type? GetEnumerableElementType(Type collectionType)
private static Type? GetUniqueGenericElementType(Type collectionType, Type genericInterfaceType)
{
Type? elementType = null;

foreach (var interfaceType in collectionType.GetInterfaces())
{
if (!interfaceType.IsGenericType ||
interfaceType.GetGenericTypeDefinition() != typeof(IEnumerable<>))
interfaceType.GetGenericTypeDefinition() != genericInterfaceType)
{
continue;
}
Expand Down Expand Up @@ -175,6 +193,32 @@ private static void WriteList<T>(
}
}

private static void WriteIList<T>(
ListVectorDataWriter writer,
ICollection collection,
ulong startIndex)
{
var values = (IList<T>)collection;

for (var index = 0; index < values.Count; index++)
{
writer.listItemWriter.WriteValue(values[index], startIndex + (ulong)index);
}
}

private static void WriteReadOnlyList<T>(
ListVectorDataWriter writer,
ICollection collection,
ulong startIndex)
{
var values = (IReadOnlyList<T>)collection;

for (var index = 0; index < values.Count; index++)
{
writer.listItemWriter.WriteValue(values[index], startIndex + (ulong)index);
}
}

private static void WriteEnumerable<T>(
ListVectorDataWriter writer,
ICollection collection,
Expand Down
92 changes: 86 additions & 6 deletions DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using DuckDB.NET.Data.DataChunk.Writer;
using System.Collections;
using System.Collections.ObjectModel;
using System.Reflection;

Expand Down Expand Up @@ -251,34 +252,38 @@ CREATE TABLE indexed_collection_paths(
}

[Fact]
public void TypedCollectionFallbackSupportsReadOnlyAndDerivedLists()
public void IndexedCollectionInterfacesSupportReadOnlyAndDerivedLists()
{
Command.CommandText = """
CREATE TABLE typed_collection_fallback(
CREATE TABLE indexed_collection_interfaces(
id INTEGER,
read_only_values INTEGER[],
read_only_list_values INTEGER[],
derived_values INTEGER[]);
""";
Command.ExecuteNonQuery();

using (var appender = Connection.CreateAppender("typed_collection_fallback"))
using (var appender = Connection.CreateAppender("indexed_collection_interfaces"))
{
for (var index = 0; index < 3_000; index++)
{
ReadOnlyCollection<int> readOnlyValues =
Array.AsReadOnly(new[] { index, index + 1, index + 2 });
DerivedIntList derivedValues = [index + 3, index + 4, index + 5];
ReadOnlyIntListCollection readOnlyListValues =
new([index + 3, index + 4, index + 5]);
DerivedIntList derivedValues = [index + 6, index + 7, index + 8];

appender.AppendRow(
(index, readOnlyValues, derivedValues),
(index, readOnlyValues, readOnlyListValues, derivedValues),
static (row, values) => row
.AppendValue(values.index)
.AppendValue(values.readOnlyValues)
.AppendValue(values.readOnlyListValues)
.AppendValue(values.derivedValues));
}
}

Command.CommandText = "SELECT * FROM typed_collection_fallback ORDER BY id";
Command.CommandText = "SELECT * FROM indexed_collection_interfaces ORDER BY id";
using var reader = Command.ExecuteReader();

for (var index = 0; index < 3_000; index++)
Expand All @@ -287,11 +292,31 @@ CREATE TABLE typed_collection_fallback(
reader.GetInt32(0).Should().Be(index);
reader.GetFieldValue<List<int>>(1).Should().Equal(index, index + 1, index + 2);
reader.GetFieldValue<List<int>>(2).Should().Equal(index + 3, index + 4, index + 5);
reader.GetFieldValue<List<int>>(3).Should().Equal(index + 6, index + 7, index + 8);
}

reader.Read().Should().BeFalse();
}

[Theory]
[InlineData(typeof(int[]), "WriteArray")]
[InlineData(typeof(List<int>), "WriteList")]
[InlineData(typeof(ReadOnlyCollection<int>), "WriteIList")]
[InlineData(typeof(DerivedIntList), "WriteIList")]
[InlineData(typeof(ReadOnlyIntListCollection), "WriteReadOnlyList")]
public void SelectsIndexedCollectionWriter(Type collectionType, string expectedMethod)
{
var createWriter = typeof(ListVectorDataWriter).GetMethod(
"CreateCollectionWriter",
BindingFlags.Static | BindingFlags.NonPublic);

var plan = createWriter!.Invoke(null, [collectionType]);
var writer = plan!.GetType().GetProperty("Writer")!.GetValue(plan) as Delegate;

writer.Should().NotBeNull();
writer!.Method.Name.Should().Be(expectedMethod);
}

[Fact]
public void NonSzArraysUseTheEnumerableFallback()
{
Expand All @@ -306,6 +331,19 @@ public void NonSzArraysUseTheEnumerableFallback()
writer.Should().BeNull();
}

[Fact]
public void CollectionsWithAmbiguousElementTypesUseTheEnumerableFallback()
{
var createWriter = typeof(ListVectorDataWriter).GetMethod(
"CreateCollectionWriter",
BindingFlags.Static | BindingFlags.NonPublic);

var plan = createWriter!.Invoke(null, [typeof(AmbiguousCollection)]);
var writer = plan!.GetType().GetProperty("Writer")!.GetValue(plan);

writer.Should().BeNull();
}

[Fact]
public void ListValuesEnum()
{
Expand Down Expand Up @@ -466,4 +504,46 @@ private enum TestEnum
}

private sealed class DerivedIntList : List<int>;

private sealed class ReadOnlyIntListCollection(IReadOnlyList<int> values) : IReadOnlyList<int>, ICollection
{
public int Count => values.Count;

public int this[int index] => values[index];

public bool IsSynchronized => false;

public object SyncRoot => this;

public void CopyTo(Array array, int index)
{
for (var valueIndex = 0; valueIndex < values.Count; valueIndex++)
{
array.SetValue(values[valueIndex], index + valueIndex);
}
}

public IEnumerator<int> GetEnumerator() => values.GetEnumerator();

IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
}

private sealed class AmbiguousCollection : ICollection, IEnumerable<int>, IEnumerable<string>
{
public int Count => 0;

public bool IsSynchronized => false;

public object SyncRoot => this;

public void CopyTo(Array array, int index)
{
}

IEnumerator<int> IEnumerable<int>.GetEnumerator() => Enumerable.Empty<int>().GetEnumerator();

IEnumerator<string> IEnumerable<string>.GetEnumerator() => Enumerable.Empty<string>().GetEnumerator();

IEnumerator IEnumerable.GetEnumerator() => Enumerable.Empty<int>().GetEnumerator();
}
}
Loading