From 03053e34b769f714f4ddfe331f141977a9f0702e Mon Sep 17 00:00:00 2001 From: Skuirrels Date: Tue, 4 Aug 2026 10:08:26 +0100 Subject: [PATCH] Optimize indexed read-only collection writes --- .../DataChunk/Writer/ListVectorDataWriter.cs | 52 ++++++++++- .../DuckDBManagedAppenderListTests.cs | 92 +++++++++++++++++-- 2 files changed, 134 insertions(+), 10 deletions(-) diff --git a/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs b/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs index a51269c..0516d5b 100644 --- a/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs +++ b/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs @@ -13,6 +13,10 @@ internal sealed unsafe class ListVectorDataWriter : VectorDataWriterBase typeof(ListVectorDataWriter).GetMethod(nameof(WriteArray), BindingFlags.Static | BindingFlags.NonPublic)!; private static readonly MethodInfo WriteListMethod = typeof(ListVectorDataWriter).GetMethod(nameof(WriteList), BindingFlags.Static | BindingFlags.NonPublic)!; + private static readonly MethodInfo WriteIListMethod = + typeof(ListVectorDataWriter).GetMethod(nameof(WriteIList), BindingFlags.Static | BindingFlags.NonPublic)!; + private static readonly MethodInfo WriteReadOnlyListMethod = + typeof(ListVectorDataWriter).GetMethod(nameof(WriteReadOnlyList), BindingFlags.Static | BindingFlags.NonPublic)!; private static readonly MethodInfo WriteEnumerableMethod = typeof(ListVectorDataWriter).GetMethod(nameof(WriteEnumerable), BindingFlags.Static | BindingFlags.NonPublic)!; @@ -115,8 +119,22 @@ private static CollectionWriterPlan CreateCollectionWriter(Type collectionType) } else { - elementType = GetEnumerableElementType(collectionType); - openMethod = elementType is null ? null : WriteEnumerableMethod; + elementType = GetUniqueGenericElementType(collectionType, typeof(IEnumerable<>)); + if (elementType is not null) + { + if (typeof(IList<>).MakeGenericType(elementType).IsAssignableFrom(collectionType)) + { + openMethod = WriteIListMethod; + } + else if (typeof(IReadOnlyList<>).MakeGenericType(elementType).IsAssignableFrom(collectionType)) + { + openMethod = WriteReadOnlyListMethod; + } + else + { + openMethod = WriteEnumerableMethod; + } + } } return new CollectionWriterPlan( @@ -125,14 +143,14 @@ openMethod is null || elementType is null : openMethod.MakeGenericMethod(elementType).CreateDelegate()); } - private static Type? GetEnumerableElementType(Type collectionType) + private static Type? GetUniqueGenericElementType(Type collectionType, Type genericInterfaceType) { Type? elementType = null; foreach (var interfaceType in collectionType.GetInterfaces()) { if (!interfaceType.IsGenericType || - interfaceType.GetGenericTypeDefinition() != typeof(IEnumerable<>)) + interfaceType.GetGenericTypeDefinition() != genericInterfaceType) { continue; } @@ -175,6 +193,32 @@ private static void WriteList( } } + private static void WriteIList( + ListVectorDataWriter writer, + ICollection collection, + ulong startIndex) + { + var values = (IList)collection; + + for (var index = 0; index < values.Count; index++) + { + writer.listItemWriter.WriteValue(values[index], startIndex + (ulong)index); + } + } + + private static void WriteReadOnlyList( + ListVectorDataWriter writer, + ICollection collection, + ulong startIndex) + { + var values = (IReadOnlyList)collection; + + for (var index = 0; index < values.Count; index++) + { + writer.listItemWriter.WriteValue(values[index], startIndex + (ulong)index); + } + } + private static void WriteEnumerable( ListVectorDataWriter writer, ICollection collection, diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs index ddb1947..2f717f1 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs @@ -1,4 +1,5 @@ using DuckDB.NET.Data.DataChunk.Writer; +using System.Collections; using System.Collections.ObjectModel; using System.Reflection; @@ -251,34 +252,38 @@ CREATE TABLE indexed_collection_paths( } [Fact] - public void TypedCollectionFallbackSupportsReadOnlyAndDerivedLists() + public void IndexedCollectionInterfacesSupportReadOnlyAndDerivedLists() { Command.CommandText = """ - CREATE TABLE typed_collection_fallback( + CREATE TABLE indexed_collection_interfaces( id INTEGER, read_only_values INTEGER[], + read_only_list_values INTEGER[], derived_values INTEGER[]); """; Command.ExecuteNonQuery(); - using (var appender = Connection.CreateAppender("typed_collection_fallback")) + using (var appender = Connection.CreateAppender("indexed_collection_interfaces")) { for (var index = 0; index < 3_000; index++) { ReadOnlyCollection readOnlyValues = Array.AsReadOnly(new[] { index, index + 1, index + 2 }); - DerivedIntList derivedValues = [index + 3, index + 4, index + 5]; + ReadOnlyIntListCollection readOnlyListValues = + new([index + 3, index + 4, index + 5]); + DerivedIntList derivedValues = [index + 6, index + 7, index + 8]; appender.AppendRow( - (index, readOnlyValues, derivedValues), + (index, readOnlyValues, readOnlyListValues, derivedValues), static (row, values) => row .AppendValue(values.index) .AppendValue(values.readOnlyValues) + .AppendValue(values.readOnlyListValues) .AppendValue(values.derivedValues)); } } - Command.CommandText = "SELECT * FROM typed_collection_fallback ORDER BY id"; + Command.CommandText = "SELECT * FROM indexed_collection_interfaces ORDER BY id"; using var reader = Command.ExecuteReader(); for (var index = 0; index < 3_000; index++) @@ -287,11 +292,31 @@ CREATE TABLE typed_collection_fallback( reader.GetInt32(0).Should().Be(index); reader.GetFieldValue>(1).Should().Equal(index, index + 1, index + 2); reader.GetFieldValue>(2).Should().Equal(index + 3, index + 4, index + 5); + reader.GetFieldValue>(3).Should().Equal(index + 6, index + 7, index + 8); } reader.Read().Should().BeFalse(); } + [Theory] + [InlineData(typeof(int[]), "WriteArray")] + [InlineData(typeof(List), "WriteList")] + [InlineData(typeof(ReadOnlyCollection), "WriteIList")] + [InlineData(typeof(DerivedIntList), "WriteIList")] + [InlineData(typeof(ReadOnlyIntListCollection), "WriteReadOnlyList")] + public void SelectsIndexedCollectionWriter(Type collectionType, string expectedMethod) + { + var createWriter = typeof(ListVectorDataWriter).GetMethod( + "CreateCollectionWriter", + BindingFlags.Static | BindingFlags.NonPublic); + + var plan = createWriter!.Invoke(null, [collectionType]); + var writer = plan!.GetType().GetProperty("Writer")!.GetValue(plan) as Delegate; + + writer.Should().NotBeNull(); + writer!.Method.Name.Should().Be(expectedMethod); + } + [Fact] public void NonSzArraysUseTheEnumerableFallback() { @@ -306,6 +331,19 @@ public void NonSzArraysUseTheEnumerableFallback() writer.Should().BeNull(); } + [Fact] + public void CollectionsWithAmbiguousElementTypesUseTheEnumerableFallback() + { + var createWriter = typeof(ListVectorDataWriter).GetMethod( + "CreateCollectionWriter", + BindingFlags.Static | BindingFlags.NonPublic); + + var plan = createWriter!.Invoke(null, [typeof(AmbiguousCollection)]); + var writer = plan!.GetType().GetProperty("Writer")!.GetValue(plan); + + writer.Should().BeNull(); + } + [Fact] public void ListValuesEnum() { @@ -466,4 +504,46 @@ private enum TestEnum } private sealed class DerivedIntList : List; + + private sealed class ReadOnlyIntListCollection(IReadOnlyList values) : IReadOnlyList, ICollection + { + public int Count => values.Count; + + public int this[int index] => values[index]; + + public bool IsSynchronized => false; + + public object SyncRoot => this; + + public void CopyTo(Array array, int index) + { + for (var valueIndex = 0; valueIndex < values.Count; valueIndex++) + { + array.SetValue(values[valueIndex], index + valueIndex); + } + } + + public IEnumerator GetEnumerator() => values.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + private sealed class AmbiguousCollection : ICollection, IEnumerable, IEnumerable + { + public int Count => 0; + + public bool IsSynchronized => false; + + public object SyncRoot => this; + + public void CopyTo(Array array, int index) + { + } + + IEnumerator IEnumerable.GetEnumerator() => Enumerable.Empty().GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => Enumerable.Empty().GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => Enumerable.Empty().GetEnumerator(); + } }