From 8d88ab2522d176bad66759abdaf08dc104486768 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 2 Jun 2026 17:11:02 +0300 Subject: [PATCH 01/28] Add Identifier Quoting Helper --- .../Utilities/IdentifierHelper.cs | 37 ++++++++++++++ .../IdentifierHelperTests.cs | 49 +++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 src/SingleStoreConnector/Utilities/IdentifierHelper.cs create mode 100644 tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs diff --git a/src/SingleStoreConnector/Utilities/IdentifierHelper.cs b/src/SingleStoreConnector/Utilities/IdentifierHelper.cs new file mode 100644 index 000000000..ed0cbd611 --- /dev/null +++ b/src/SingleStoreConnector/Utilities/IdentifierHelper.cs @@ -0,0 +1,37 @@ +using System; +using System.Linq; + +namespace SingleStoreConnector.Utilities +{ + internal static class IdentifierHelper + { + /// + /// Quotes a SQL identifier with backticks, escaping any backticks within the identifier. + /// + /// The identifier to quote. + /// The quoted identifier. + /// If identifier is null, empty, or contains null characters. + public static string QuoteIdentifier(string identifier) + { + if (string.IsNullOrWhiteSpace(identifier)) + throw new ArgumentException("Identifier cannot be null or empty.", nameof(identifier)); + + if (identifier.Contains('\0')) + throw new ArgumentException("Identifier cannot contain null characters.", nameof(identifier)); + + // Backticks inside the identifier must be doubled + return "`" + identifier.Replace("`", "``") + "`"; + } + + /// + /// Quotes a qualified identifier (e.g., "database.table" becomes "`database`.`table`"). + /// + public static string QuoteQualifiedIdentifier(string qualifiedName) + { + if (string.IsNullOrWhiteSpace(qualifiedName)) + throw new ArgumentException("Qualified name cannot be null or empty.", nameof(qualifiedName)); + + return string.Join(".", qualifiedName.Split('.').Select(QuoteIdentifier)); + } + } +} diff --git a/tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs b/tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs new file mode 100644 index 000000000..c010c5011 --- /dev/null +++ b/tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs @@ -0,0 +1,49 @@ +using System; +using SingleStoreConnector.Utilities; +using Xunit; + +namespace SingleStoreConnector.Tests +{ + public class IdentifierHelperTests + { + [Theory] + [InlineData("users", "`users`")] + [InlineData("user_id", "`user_id`")] + [InlineData("select", "`select`")] // Reserved word + [InlineData("order", "`order`")] // Reserved word + [InlineData("my table", "`my table`")] // Space + [InlineData("my-table", "`my-table`")] // Hyphen + [InlineData("my`table", "`my``table`")] // Backtick inside + [InlineData("用户表", "`用户表`")] // Unicode + [InlineData("таблиця", "`таблиця`")] // Unicode + public void QuoteIdentifier_ValidInput_ReturnsQuoted(string input, string expected) + { + var result = IdentifierHelper.QuoteIdentifier(input); + Assert.Equal(expected, result); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void QuoteIdentifier_NullOrEmpty_Throws(string input) + { + Assert.Throws(() => IdentifierHelper.QuoteIdentifier(input)); + } + + [Fact] + public void QuoteIdentifier_NullChar_Throws() + { + Assert.Throws(() => IdentifierHelper.QuoteIdentifier("table\0name")); + } + + [Theory] + [InlineData("db.users", "`db`.`users`")] + [InlineData("my db.my table", "`my db`.`my table`")] + public void QuoteQualifiedIdentifier_ValidInput_ReturnsQuoted(string input, string expected) + { + var result = IdentifierHelper.QuoteQualifiedIdentifier(input); + Assert.Equal(expected, result); + } + } +} From 269c6cf2d39a0faab00903ecd6bc9b9b5693188f Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 3 Jun 2026 15:02:10 +0300 Subject: [PATCH 02/28] add SchemaDetector class --- .../Utilities/IdentifierHelper.cs | 172 +++++++++++++--- .../Utilities/SchemaDetector.cs | 193 ++++++++++++++++++ tests/SideBySide/SchemaDetectorTests.cs | 180 ++++++++++++++++ .../IdentifierHelperTests.cs | 144 +++++++++---- 4 files changed, 623 insertions(+), 66 deletions(-) create mode 100644 src/SingleStoreConnector/Utilities/SchemaDetector.cs create mode 100644 tests/SideBySide/SchemaDetectorTests.cs diff --git a/src/SingleStoreConnector/Utilities/IdentifierHelper.cs b/src/SingleStoreConnector/Utilities/IdentifierHelper.cs index ed0cbd611..82fc79874 100644 --- a/src/SingleStoreConnector/Utilities/IdentifierHelper.cs +++ b/src/SingleStoreConnector/Utilities/IdentifierHelper.cs @@ -1,37 +1,163 @@ using System; +using System.Collections.Generic; using System.Linq; +using System.Text; -namespace SingleStoreConnector.Utilities +namespace SingleStoreConnector.Utilities; + +internal static class IdentifierHelper { - internal static class IdentifierHelper + /// + /// Quotes a SQL identifier with backticks, escaping any backticks within the identifier. + /// + /// The identifier to quote. + /// The quoted identifier. + /// If identifier is null, empty, or contains null characters. + public static string QuoteIdentifier(string identifier) { - /// - /// Quotes a SQL identifier with backticks, escaping any backticks within the identifier. - /// - /// The identifier to quote. - /// The quoted identifier. - /// If identifier is null, empty, or contains null characters. - public static string QuoteIdentifier(string identifier) - { - if (string.IsNullOrWhiteSpace(identifier)) - throw new ArgumentException("Identifier cannot be null or empty.", nameof(identifier)); + if (string.IsNullOrWhiteSpace(identifier)) + throw new ArgumentException("Identifier cannot be null or empty.", nameof(identifier)); - if (identifier.Contains('\0')) - throw new ArgumentException("Identifier cannot contain null characters.", nameof(identifier)); + if (identifier.Contains('\0')) + throw new ArgumentException("Identifier cannot contain null characters.", nameof(identifier)); - // Backticks inside the identifier must be doubled - return "`" + identifier.Replace("`", "``") + "`"; + // Backticks inside the identifier must be doubled. + return "`" + identifier.Replace("`", "``") + "`"; + } + + /// + /// Quotes a qualified identifier (e.g., "database.table" becomes "`database`.`table`"). + /// + public static string QuoteQualifiedIdentifier(string qualifiedName) + { + if (string.IsNullOrWhiteSpace(qualifiedName)) + throw new ArgumentException("Qualified name cannot be null or empty.", nameof(qualifiedName)); + + if (qualifiedName.Contains('\0')) + throw new ArgumentException("Qualified name cannot contain null characters.", nameof(qualifiedName)); + + // Split on dots that are outside backtick-quoted identifier parts. + // This supports names such as: + // database.table + // `database`.`table` + // `database.with.dot`.`table` + // `database`.`table.with.dot` + var parts = SplitQualifiedIdentifier(qualifiedName); + + return string.Join(".", parts.Select(QuoteIdentifier)); + } + + private static List SplitQualifiedIdentifier(string qualifiedName) + { + var parts = new List(); + var current = new StringBuilder(); + var inBackticks = false; + + for (var i = 0; i < qualifiedName.Length; i++) + { + var ch = qualifiedName[i]; + + if (ch == '`') + { + if (inBackticks) + { + if (i + 1 < qualifiedName.Length && qualifiedName[i + 1] == '`') + { + // Escaped backtick inside a quoted identifier. + current.Append("``"); + i++; + } + else + { + // Closing backtick. + current.Append(ch); + inBackticks = false; + } + } + else if (IsOnlyWhitespace(current)) + { + // Opening backtick at the start of an identifier part. + current.Append(ch); + inBackticks = true; + } + else + { + // Treat backticks in unquoted input as literal identifier characters. + // They will be escaped later by QuoteIdentifier. + current.Append(ch); + } + } + else if (ch == '.' && !inBackticks) + { + AddPart(parts, current, qualifiedName); + } + else + { + current.Append(ch); + } } - /// - /// Quotes a qualified identifier (e.g., "database.table" becomes "`database`.`table`"). - /// - public static string QuoteQualifiedIdentifier(string qualifiedName) + if (inBackticks) + throw new ArgumentException("Qualified name contains an unterminated quoted identifier.", nameof(qualifiedName)); + + AddPart(parts, current, qualifiedName); + return parts; + } + + private static void AddPart(List parts, StringBuilder current, string qualifiedName) + { + var part = current.ToString().Trim(); + + if (part.Length == 0) + throw new ArgumentException("Qualified name contains an empty identifier part.", nameof(qualifiedName)); + + parts.Add(UnquoteIdentifierPart(part, qualifiedName)); + current.Clear(); + } + + private static string UnquoteIdentifierPart(string part, string qualifiedName) + { + if (part[0] != '`') + return part; + + var identifier = new StringBuilder(); + + for (var i = 1; i < part.Length; i++) { - if (string.IsNullOrWhiteSpace(qualifiedName)) - throw new ArgumentException("Qualified name cannot be null or empty.", nameof(qualifiedName)); + var ch = part[i]; - return string.Join(".", qualifiedName.Split('.').Select(QuoteIdentifier)); + if (ch == '`') + { + if (i + 1 < part.Length && part[i + 1] == '`') + { + identifier.Append('`'); + i++; + } + else + { + if (i != part.Length - 1) + throw new ArgumentException("Qualified name contains unexpected characters after a quoted identifier.", nameof(qualifiedName)); + + return identifier.ToString(); + } + } + else + { + identifier.Append(ch); + } + } + + throw new ArgumentException("Qualified name contains an unterminated quoted identifier.", nameof(qualifiedName)); + } + + private static bool IsOnlyWhitespace(StringBuilder builder) + { + for (var i = 0; i < builder.Length; i++) + { + if (!char.IsWhiteSpace(builder[i])) + return false; } + + return true; } } diff --git a/src/SingleStoreConnector/Utilities/SchemaDetector.cs b/src/SingleStoreConnector/Utilities/SchemaDetector.cs new file mode 100644 index 000000000..f9949289f --- /dev/null +++ b/src/SingleStoreConnector/Utilities/SchemaDetector.cs @@ -0,0 +1,193 @@ +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; + +namespace SingleStoreConnector.Utilities; + +internal sealed class SchemaDetector(SingleStoreConnection connection) +{ + private static readonly Regex referenceTableRegex = + new(@"CREATE\s+(?:(?:ROWSTORE|COLUMNSTORE)\s+)?REFERENCE\s+TABLE", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex shardKeyRegex = + new(@"SHARD\s+KEY(?:\s+(?:`(?:``|[^`])*`|[^\s(]+))?\s*\((?[^)]*)\)", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private readonly SingleStoreConnection m_connection = + connection ?? throw new ArgumentNullException(nameof(connection)); + + /// + /// Detects if the specified table is a reference table. + /// + public async Task IsReferenceTableAsync(string tableName, CancellationToken cancellationToken = default) + { + EnsureConnectionIsOpen(); + + var createTableSql = await GetCreateTableStatementAsync(tableName, cancellationToken) + .ConfigureAwait(false); + + return referenceTableRegex.IsMatch(createTableSql); + } + + /// + /// Gets the shard key columns for the specified table. + /// + /// List of shard key column names, or empty list if no shard key. + public async Task> GetShardKeyColumnsAsync(string tableName, CancellationToken cancellationToken = default) + { + EnsureConnectionIsOpen(); + + var shardKeys = new List<(int Sequence, string ColumnName)>(); + + // Method 1: Check SHOW INDEXES for __SHARDKEY. + // This is preferred because Seq_in_index preserves composite shard key column order. + using (var cmd = m_connection.CreateCommand()) + { + cmd.CommandText = $"SHOW INDEXES FROM {IdentifierHelper.QuoteQualifiedIdentifier(tableName)}"; + + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken) + .ConfigureAwait(false); + + var keyNameOrdinal = reader.GetOrdinal("Key_name"); + var columnNameOrdinal = reader.GetOrdinal("Column_name"); + var sequenceOrdinal = reader.GetOrdinal("Seq_in_index"); + + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + if (reader.IsDBNull(keyNameOrdinal)) + continue; + + var keyName = reader.GetString(keyNameOrdinal); + if (!string.Equals(keyName, "__SHARDKEY", StringComparison.Ordinal)) + continue; + + if (reader.IsDBNull(columnNameOrdinal)) + continue; + + var sequence = reader.IsDBNull(sequenceOrdinal) + ? shardKeys.Count + 1 + : Convert.ToInt32(reader.GetValue(sequenceOrdinal), CultureInfo.InvariantCulture); + + var columnName = reader.GetString(columnNameOrdinal); + shardKeys.Add((sequence, columnName)); + } + } + + if (shardKeys.Count != 0) + { + return shardKeys + .OrderBy(x => x.Sequence) + .Select(x => x.ColumnName) + .ToList(); + } + + // Method 2: Parse SHOW CREATE TABLE for SHARD KEY. + // This is a fallback for cases where SHOW INDEXES doesn't expose __SHARDKEY. + var createTableSql = await GetCreateTableStatementAsync(tableName, cancellationToken) + .ConfigureAwait(false); + + var match = shardKeyRegex.Match(createTableSql); + if (!match.Success) + return new List(); + + var shardKeyList = match.Groups["columns"].Value; + if (string.IsNullOrWhiteSpace(shardKeyList)) + return new List(); + + return ParseIdentifierList(shardKeyList); + } + + /// + /// Gets the CREATE TABLE statement for the specified table. + /// + private async Task GetCreateTableStatementAsync(string tableName, CancellationToken cancellationToken) + { + EnsureConnectionIsOpen(); + + using var cmd = m_connection.CreateCommand(); + cmd.CommandText = $"SHOW CREATE TABLE {IdentifierHelper.QuoteQualifiedIdentifier(tableName)}"; + + await using var reader = await cmd.ExecuteReaderAsync(cancellationToken) + .ConfigureAwait(false); + + if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + // SHOW CREATE TABLE returns the CREATE TABLE statement in the second column. + return reader.GetString(1); + } + + throw new InvalidOperationException($"Unable to retrieve CREATE TABLE statement for {tableName}."); + } + + /// + /// Gets column metadata for the specified table. + /// + public async Task GetTableSchemaAsync(string tableName, CancellationToken cancellationToken = default) + { + EnsureConnectionIsOpen(); + + using var cmd = m_connection.CreateCommand(); + cmd.CommandText = $"SELECT * FROM {IdentifierHelper.QuoteQualifiedIdentifier(tableName)} LIMIT 0"; + + await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly, cancellationToken) + .ConfigureAwait(false); + + return reader.GetSchemaTable() + ?? throw new InvalidOperationException($"Unable to retrieve schema for {tableName}."); + } + + private void EnsureConnectionIsOpen() + { + if (m_connection.State != ConnectionState.Open) + throw new InvalidOperationException("Connection must be open before detecting schema."); + } + + private static List ParseIdentifierList(string identifierList) + { + var identifiers = new List(); + var current = new StringBuilder(); + var inBackticks = false; + + for (var i = 0; i < identifierList.Length; i++) + { + var ch = identifierList[i]; + + if (ch == '`') + { + if (inBackticks && i + 1 < identifierList.Length && identifierList[i + 1] == '`') + { + current.Append('`'); + i++; + } + else + { + inBackticks = !inBackticks; + } + } + else if (ch == ',' && !inBackticks) + { + AddIdentifier(); + } + else + { + current.Append(ch); + } + } + + if (inBackticks) + throw new InvalidOperationException("Invalid shard key definition: unterminated quoted identifier."); + + AddIdentifier(); + return identifiers; + + void AddIdentifier() + { + var identifier = current.ToString().Trim(); + if (identifier.Length != 0) + identifiers.Add(identifier); + + current.Clear(); + } + } +} diff --git a/tests/SideBySide/SchemaDetectorTests.cs b/tests/SideBySide/SchemaDetectorTests.cs new file mode 100644 index 000000000..a28821b57 --- /dev/null +++ b/tests/SideBySide/SchemaDetectorTests.cs @@ -0,0 +1,180 @@ +using SingleStoreConnector.Utilities; + +namespace SideBySide; + +public class SchemaDetectorTests(DatabaseFixture database) : IClassFixture +{ + [Fact] + public async Task IsReferenceTable_RegularTable_ReturnsFalse() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "test_regular"; + var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + + try + { + await using (var cmd = new SingleStoreCommand($@" + DROP TABLE IF EXISTS {quotedTableName}; + CREATE TABLE {quotedTableName} + ( + id INT PRIMARY KEY + );", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var detector = new SchemaDetector(connection); + var isReference = await detector.IsReferenceTableAsync(tableName); + + Assert.False(isReference); + } + finally + { + await DropTableIfExistsAsync(connection, tableName); + } + } + + [Fact] + public async Task IsReferenceTable_ReferenceTable_ReturnsTrue() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "test_reference"; + var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + + try + { + await using (var cmd = new SingleStoreCommand($@" + DROP TABLE IF EXISTS {quotedTableName}; + CREATE REFERENCE TABLE {quotedTableName} + ( + id INT PRIMARY KEY + );", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var detector = new SchemaDetector(connection); + var isReference = await detector.IsReferenceTableAsync(tableName); + + Assert.True(isReference); + } + finally + { + await DropTableIfExistsAsync(connection, tableName); + } + } + + [Fact] + public async Task GetShardKeyColumns_TableWithShardKey_ReturnsColumnsInOrder() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "test_sharded"; + var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + + try + { + await using (var cmd = new SingleStoreCommand($@" + DROP TABLE IF EXISTS {quotedTableName}; + CREATE TABLE {quotedTableName} + ( + tenant_id INT, + user_id INT, + name VARCHAR(100), + SHARD KEY (tenant_id, user_id) + );", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var detector = new SchemaDetector(connection); + var shardKeys = await detector.GetShardKeyColumnsAsync(tableName); + + Assert.Equal(new[] { "tenant_id", "user_id" }, shardKeys); + } + finally + { + await DropTableIfExistsAsync(connection, tableName); + } + } + + [Fact] + public async Task GetShardKeyColumns_TableWithoutShardKey_ReturnsEmpty() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "test_no_shard"; + var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + + try + { + await using (var cmd = new SingleStoreCommand($@" + DROP TABLE IF EXISTS {quotedTableName}; + CREATE REFERENCE TABLE {quotedTableName} + ( + id INT PRIMARY KEY + );", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var detector = new SchemaDetector(connection); + var shardKeys = await detector.GetShardKeyColumnsAsync(tableName); + + Assert.Empty(shardKeys); + } + finally + { + await DropTableIfExistsAsync(connection, tableName); + } + } + + [Fact] + public async Task GetTableSchema_ExistingTable_ReturnsSchema() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "test_schema"; + var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + + try + { + await using (var cmd = new SingleStoreCommand($@" + DROP TABLE IF EXISTS {quotedTableName}; + CREATE TABLE {quotedTableName} + ( + id INT PRIMARY KEY, + name VARCHAR(100) + );", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var detector = new SchemaDetector(connection); + var schema = await detector.GetTableSchemaAsync(tableName); + + Assert.NotNull(schema); + Assert.Contains(schema.Rows.Cast(), row => row["ColumnName"].ToString() == "id"); + Assert.Contains(schema.Rows.Cast(), row => row["ColumnName"].ToString() == "name"); + } + finally + { + await DropTableIfExistsAsync(connection, tableName); + } + } + + private static async Task DropTableIfExistsAsync(SingleStoreConnection connection, string tableName) + { + var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + + await using var cmd = new SingleStoreCommand($"DROP TABLE IF EXISTS {quotedTableName}", connection); + await cmd.ExecuteNonQueryAsync(); + } +} diff --git a/tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs b/tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs index c010c5011..f563d287c 100644 --- a/tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs +++ b/tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs @@ -1,49 +1,107 @@ -using System; using SingleStoreConnector.Utilities; -using Xunit; -namespace SingleStoreConnector.Tests +namespace SingleStoreConnector.Tests; + +public class IdentifierHelperTests { - public class IdentifierHelperTests + [Theory] + [InlineData("users", "`users`")] + [InlineData("user_id", "`user_id`")] + [InlineData("select", "`select`")] // Reserved word + [InlineData("order", "`order`")] // Reserved word + [InlineData("my table", "`my table`")] // Space + [InlineData("my-table", "`my-table`")] // Hyphen + [InlineData("my`table", "`my``table`")] // Backtick inside + [InlineData("用户表", "`用户表`")] // Unicode + [InlineData("таблиця", "`таблиця`")] // Unicode + public void QuoteIdentifier_ValidInput_ReturnsQuoted(string input, string expected) { - [Theory] - [InlineData("users", "`users`")] - [InlineData("user_id", "`user_id`")] - [InlineData("select", "`select`")] // Reserved word - [InlineData("order", "`order`")] // Reserved word - [InlineData("my table", "`my table`")] // Space - [InlineData("my-table", "`my-table`")] // Hyphen - [InlineData("my`table", "`my``table`")] // Backtick inside - [InlineData("用户表", "`用户表`")] // Unicode - [InlineData("таблиця", "`таблиця`")] // Unicode - public void QuoteIdentifier_ValidInput_ReturnsQuoted(string input, string expected) - { - var result = IdentifierHelper.QuoteIdentifier(input); - Assert.Equal(expected, result); - } - - [Theory] - [InlineData(null)] - [InlineData("")] - [InlineData(" ")] - public void QuoteIdentifier_NullOrEmpty_Throws(string input) - { - Assert.Throws(() => IdentifierHelper.QuoteIdentifier(input)); - } - - [Fact] - public void QuoteIdentifier_NullChar_Throws() - { - Assert.Throws(() => IdentifierHelper.QuoteIdentifier("table\0name")); - } - - [Theory] - [InlineData("db.users", "`db`.`users`")] - [InlineData("my db.my table", "`my db`.`my table`")] - public void QuoteQualifiedIdentifier_ValidInput_ReturnsQuoted(string input, string expected) - { - var result = IdentifierHelper.QuoteQualifiedIdentifier(input); - Assert.Equal(expected, result); - } + var result = IdentifierHelper.QuoteIdentifier(input); + + Assert.Equal(expected, result); } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void QuoteIdentifier_NullOrEmpty_Throws(string input) + => Assert.Throws(() => IdentifierHelper.QuoteIdentifier(input)); + + [Fact] + public void QuoteIdentifier_NullChar_Throws() + => Assert.Throws(() => IdentifierHelper.QuoteIdentifier("table\0name")); + + [Theory] + [InlineData("db.users", "`db`.`users`")] + [InlineData("my db.my table", "`my db`.`my table`")] + [InlineData("select.order", "`select`.`order`")] // Reserved words + [InlineData("db.my-table", "`db`.`my-table`")] // Hyphen + [InlineData("db.my table", "`db`.`my table`")] // Space + [InlineData("db.用户表", "`db`.`用户表`")] // Unicode + [InlineData("db.таблиця", "`db`.`таблиця`")] // Unicode + public void QuoteQualifiedIdentifier_ValidInput_ReturnsQuoted(string input, string expected) + { + var result = IdentifierHelper.QuoteQualifiedIdentifier(input); + + Assert.Equal(expected, result); + } + + [Theory] + [InlineData("`db`.`users`", "`db`.`users`")] + [InlineData("`my db`.`my table`", "`my db`.`my table`")] + [InlineData("`db.with.dot`.`users`", "`db.with.dot`.`users`")] + [InlineData("`db`.`table.with.dot`", "`db`.`table.with.dot`")] + [InlineData("`db.with.dot`.`table.with.dot`", "`db.with.dot`.`table.with.dot`")] + [InlineData("db.`table.with.dot`", "`db`.`table.with.dot`")] + [InlineData("`db.with.dot`.users", "`db.with.dot`.`users`")] + public void QuoteQualifiedIdentifier_AlreadyQuotedInput_ReturnsQuoted(string input, string expected) + { + var result = IdentifierHelper.QuoteQualifiedIdentifier(input); + + Assert.Equal(expected, result); + } + + [Theory] + [InlineData("db.my`table", "`db`.`my``table`")] + [InlineData("my`db.my`table", "`my``db`.`my``table`")] + [InlineData("`my``db`.`my``table`", "`my``db`.`my``table`")] + public void QuoteQualifiedIdentifier_BackticksInsideIdentifiers_ReturnsQuoted(string input, string expected) + { + var result = IdentifierHelper.QuoteQualifiedIdentifier(input); + + Assert.Equal(expected, result); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void QuoteQualifiedIdentifier_NullOrEmpty_Throws(string input) + => Assert.Throws(() => IdentifierHelper.QuoteQualifiedIdentifier(input)); + + [Fact] + public void QuoteQualifiedIdentifier_NullChar_Throws() + => Assert.Throws(() => IdentifierHelper.QuoteQualifiedIdentifier("db.table\0name")); + + [Theory] + [InlineData(".users")] + [InlineData("db.")] + [InlineData("db..users")] + [InlineData("db. .users")] + public void QuoteQualifiedIdentifier_EmptyPart_Throws(string input) + => Assert.Throws(() => IdentifierHelper.QuoteQualifiedIdentifier(input)); + + [Theory] + [InlineData("`db.users")] + [InlineData("db.`users")] + [InlineData("`db`.`users")] + public void QuoteQualifiedIdentifier_UnterminatedQuotedIdentifier_Throws(string input) + => Assert.Throws(() => IdentifierHelper.QuoteQualifiedIdentifier(input)); + + [Theory] + [InlineData("`db`extra.users")] + [InlineData("db.`users`extra")] + public void QuoteQualifiedIdentifier_UnexpectedCharactersAfterQuotedIdentifier_Throws(string input) + => Assert.Throws(() => IdentifierHelper.QuoteQualifiedIdentifier(input)); } From 3015c1fcaa81e2d9c956386b9462be41b68c47c1 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 3 Jun 2026 15:18:00 +0300 Subject: [PATCH 03/28] add logging infra --- src/SingleStoreConnector/Logging/EventIds.cs | 11 ++++++++ src/SingleStoreConnector/Logging/Log.cs | 27 ++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/SingleStoreConnector/Logging/EventIds.cs b/src/SingleStoreConnector/Logging/EventIds.cs index 29fb2a86c..537cd184c 100644 --- a/src/SingleStoreConnector/Logging/EventIds.cs +++ b/src/SingleStoreConnector/Logging/EventIds.cs @@ -152,6 +152,17 @@ internal static class EventIds public const int RollingBackTransaction = 2704; public const int RolledBackTransaction = 2705; + // Bulk update events, 2800-2899 + public const int StartingBulkUpdate = 2800; + public const int CreatedStagingTableForBulkUpdate = 2801; + public const int StagedDataForBulkUpdate = 2802; + public const int QueriedMatchCountForBulkUpdate = 2803; + public const int ExecutedBulkUpdate = 2804; + public const int CompletedBulkUpdate = 2805; + public const int KeyColumnsNotIndexedForBulkUpdate = 2806; + public const int ShardKeyMismatchForBulkUpdate = 2807; + public const int LargeUnmatchedCountForBulkUpdate = 2808; + // Connection pool events, 3000-3099 public const int WaitingForAvailableSession = 3000; public const int CreatingNewConnectionPool = 3001; diff --git a/src/SingleStoreConnector/Logging/Log.cs b/src/SingleStoreConnector/Logging/Log.cs index bd0e66124..4da783a93 100644 --- a/src/SingleStoreConnector/Logging/Log.cs +++ b/src/SingleStoreConnector/Logging/Log.cs @@ -383,6 +383,33 @@ internal static partial class Log [LoggerMessage(EventIds.RolledBackTransaction, LogLevel.Debug, "Session {SessionId} rolled back transaction")] public static partial void RolledBackTransaction(ILogger logger, string sessionId); + [LoggerMessage(EventIds.StartingBulkUpdate, LogLevel.Information, "Starting bulk update: Table={TableName}, Keys=[{KeyColumns}], UpdateColumns=[{UpdateColumns}], RowCount={RowCount}")] + public static partial void StartingBulkUpdate(ILogger logger, string tableName, string keyColumns, string updateColumns, int rowCount); + + [LoggerMessage(EventIds.CreatedStagingTableForBulkUpdate, LogLevel.Debug, "Created staging table for bulk update: {TempTableName}, Columns={ColumnCount}")] + public static partial void CreatedStagingTableForBulkUpdate(ILogger logger, string tempTableName, int columnCount); + + [LoggerMessage(EventIds.StagedDataForBulkUpdate, LogLevel.Debug, "Staged data for bulk update: RowsStaged={RowsStaged}, Warnings={WarningCount}")] + public static partial void StagedDataForBulkUpdate(ILogger logger, int rowsStaged, int warningCount); + + [LoggerMessage(EventIds.QueriedMatchCountForBulkUpdate, LogLevel.Debug, "Queried match count for bulk update: RowsMatched={RowsMatched}")] + public static partial void QueriedMatchCountForBulkUpdate(ILogger logger, int rowsMatched); + + [LoggerMessage(EventIds.ExecutedBulkUpdate, LogLevel.Debug, "Executed bulk update: RowsUpdated={RowsUpdated}")] + public static partial void ExecutedBulkUpdate(ILogger logger, int rowsUpdated); + + [LoggerMessage(EventIds.CompletedBulkUpdate, LogLevel.Information, "Completed bulk update: Staged={Staged}, Matched={Matched}, Updated={Updated}, Duration={Duration}ms")] + public static partial void CompletedBulkUpdate(ILogger logger, int staged, int matched, int updated, long duration); + + [LoggerMessage(EventIds.KeyColumnsNotIndexedForBulkUpdate, LogLevel.Warning, "Key columns not indexed for bulk update, performance may be slow: {Columns}")] + public static partial void KeyColumnsNotIndexedForBulkUpdate(ILogger logger, string columns); + + [LoggerMessage(EventIds.ShardKeyMismatchForBulkUpdate, LogLevel.Warning, "Shard key mismatch for bulk update: Staging=[{StagingKey}], Target=[{TargetKey}]")] + public static partial void ShardKeyMismatchForBulkUpdate(ILogger logger, string stagingKey, string targetKey); + + [LoggerMessage(EventIds.LargeUnmatchedCountForBulkUpdate, LogLevel.Warning, "Large unmatched count for bulk update: Staged={Staged}, Matched={Matched}, Unmatched={Unmatched}")] + public static partial void LargeUnmatchedCountForBulkUpdate(ILogger logger, int staged, int matched, int unmatched); + [LoggerMessage(EventIds.WaitingForAvailableSession, LogLevel.Trace, "Pool {PoolId} waiting for an available session")] public static partial void WaitingForAvailableSession(ILogger logger, int poolId); From 3fb85339ba0c2216abfba599c5ece3985d23d008 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 3 Jun 2026 16:36:46 +0300 Subject: [PATCH 04/28] add the skeleton for BulkUpdate --- .../SingleStoreBulkUpdate.cs | 85 +++++++++++++++++++ .../SingleStoreBulkUpdateResult.cs | 40 +++++++++ .../SingleStoreRowsStagedEventArgs.cs | 25 ++++++ 3 files changed, 150 insertions(+) create mode 100644 src/SingleStoreConnector/SingleStoreBulkUpdate.cs create mode 100644 src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs create mode 100644 src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs new file mode 100644 index 000000000..1b76f2360 --- /dev/null +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -0,0 +1,85 @@ +using System.Data; +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using SingleStoreConnector.Logging; +using SingleStoreConnector.Utilities; + +namespace SingleStoreConnector; + +/// +/// Provides efficient bulk update operations for SingleStore databases. +/// +/// +/// +/// This class stages source rows into a temporary table using , +/// then updates matching rows in the destination table using a single UPDATE ... JOIN statement. +/// +/// This API is experimental and may change in the future. +/// +public sealed class SingleStoreBulkUpdate +{ + /// + /// Initializes a object with the specified connection, and optionally the active transaction. + /// + /// The to use. + /// (Optional) The to use. + public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransaction transaction) + { + m_connection = connection ?? throw new ArgumentNullException(nameof(connection)); + m_transaction = transaction; + m_logger = m_connection.LoggingConfiguration.BulkCopyLogger; + m_warnings = []; + ColumnMappings = []; + KeyColumns = []; + } + + /// + /// Gets or sets the name of the destination table. + /// + public string? DestinationTableName { get; set; } + + /// + /// Gets the list of key columns used for the JOIN condition. + /// These columns identify which rows to update. + /// + public List KeyColumns { get; } + + /// + /// Gets the collection of column mappings between source data and destination table. + /// + public List ColumnMappings { get; } + + /// + /// Gets or sets the timeout in seconds for bulk operations. + /// + public int BulkCopyTimeout { get; set; } = 30; + + /// + /// Gets or sets the number of rows to stage before firing the SingleStoreRowsStaged event. + /// Only applies to the staging phase (LOAD DATA), not the UPDATE execution. + /// Set to 0 to disable progress notifications. + /// + public int NotifyAfter { get; set; } + + /// + /// Gets or sets whether to compute the RowsMatched count via a COUNT query. + /// Default is true. Set to false to skip the COUNT query for better performance. + /// When false, RowsMatched will be null in the result. + /// + public bool ComputeRowsMatched { get; set; } = true; + + /// + /// This event is raised every time that the number of rows specified by the property have been processed. + /// + /// + /// Receipt of a RowsStaged event does not imply that any rows have been sent to the server or committed. + /// The property can be set to true by the event handler to abort the staging. + /// + public event SingleStoreRowsStagedEventHandler? SingleStoreRowsStaged; + + // TODO: Add WriteToServer overloads in next steps + private readonly SingleStoreConnection m_connection; + private readonly SingleStoreTransaction m_transaction; + private readonly ILogger m_logger; + private readonly List m_warnings; +} diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs b/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs new file mode 100644 index 000000000..5e40629ec --- /dev/null +++ b/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs @@ -0,0 +1,40 @@ +namespace SingleStoreConnector; + +/// +/// Represents the result of a operation. +/// +public sealed class SingleStoreBulkUpdateResult +{ + /// + /// The warnings, if any. Users of should check that this collection is empty to avoid + /// potential data loss from failed data type conversions. + /// + public IReadOnlyList Warnings { get; } + + /// + /// The number of rows that were loaded into the staging table during the bulk update operation. + /// + public int RowsStaged { get; } + + /// + /// The number of staged rows that matched rows in the destination table. + /// + public int RowsMatched { get; } + + /// + /// The number of rows that were updated during the bulk update operation. + /// + public int RowsUpdated { get; } + + internal SingleStoreBulkUpdateResult( + IReadOnlyList warnings, + int rowsStaged, + int rowsMatched, + int rowsUpdated) + { + Warnings = warnings; + RowsStaged = rowsStaged; + RowsMatched = rowsMatched; + RowsUpdated = rowsUpdated; + } +} diff --git a/src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs b/src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs new file mode 100644 index 000000000..9e471b897 --- /dev/null +++ b/src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs @@ -0,0 +1,25 @@ +namespace SingleStoreConnector; + +public sealed class SingleStoreRowsStagedEventArgs : EventArgs +{ + /// + /// Gets or sets a value indicating whether the bulk update operation should be aborted. + /// + public bool Abort { get; set; } + + /// + /// Gets a value that returns the number of rows staged during the current bulk update operation. + /// + public int RowsStaged { get; internal set; } + + internal SingleStoreRowsStagedEventArgs() + { + } +} + +/// +/// Represents the method that handles the event of a . +/// +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public delegate void SingleStoreRowsStagedEventHandler(object sender, SingleStoreRowsStagedEventArgs e); +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix From b94ae62cecc675e3e2aaf56bdf3e7e9b40b2a10f Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Thu, 4 Jun 2026 13:25:13 +0300 Subject: [PATCH 05/28] add validator for column mappings --- ...ingleStoreConnectorLoggingConfiguration.cs | 1 + .../SingleStoreBulkUpdate.cs | 74 ++++++++++++++++++- .../SingleStoreRowsStagedEventArgs.cs | 4 +- 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/src/SingleStoreConnector/Logging/SingleStoreConnectorLoggingConfiguration.cs b/src/SingleStoreConnector/Logging/SingleStoreConnectorLoggingConfiguration.cs index 785d20fec..85b8de142 100644 --- a/src/SingleStoreConnector/Logging/SingleStoreConnectorLoggingConfiguration.cs +++ b/src/SingleStoreConnector/Logging/SingleStoreConnectorLoggingConfiguration.cs @@ -10,6 +10,7 @@ internal sealed class SingleStoreConnectorLoggingConfiguration(ILoggerFactory lo public ILogger CommandLogger { get; } = loggerFactory.CreateLogger("SingleStoreConnector.SingleStoreCommand"); public ILogger PoolLogger { get; } = loggerFactory.CreateLogger("SingleStoreConnector.ConnectionPool"); public ILogger BulkCopyLogger { get; } = loggerFactory.CreateLogger("SingleStoreConnector.SingleStoreBulkCopy"); + public ILogger BulkUpdateLogger { get; } = loggerFactory.CreateLogger("SingleStoreConnector.SingleStoreBulkUpdate"); public ILogger TransactionLogger { get; } = loggerFactory.CreateLogger("SingleStoreConnector.Transaction"); public static SingleStoreConnectorLoggingConfiguration NullConfiguration { get; } = new SingleStoreConnectorLoggingConfiguration(NullLoggerFactory.Instance); diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index 1b76f2360..cc740dc81 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -27,7 +27,7 @@ public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransa { m_connection = connection ?? throw new ArgumentNullException(nameof(connection)); m_transaction = transaction; - m_logger = m_connection.LoggingConfiguration.BulkCopyLogger; + m_logger = m_connection.LoggingConfiguration.BulkUpdateLogger; m_warnings = []; ColumnMappings = []; KeyColumns = []; @@ -68,16 +68,84 @@ public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransa /// public bool ComputeRowsMatched { get; set; } = true; - /// + /*/// /// This event is raised every time that the number of rows specified by the property have been processed. /// /// /// Receipt of a RowsStaged event does not imply that any rows have been sent to the server or committed. /// The property can be set to true by the event handler to abort the staging. /// - public event SingleStoreRowsStagedEventHandler? SingleStoreRowsStaged; + public event SingleStoreRowsStagedEventHandler? SingleStoreRowsStaged;*/ // TODO: Add WriteToServer overloads in next steps + private void ValidateColumnMappings() + { + // Ensure the caller specified at least one key column. + // Key columns define the JOIN condition between the destination table and the staging table. + if (KeyColumns.Count == 0) + throw new InvalidOperationException("KeyColumns must contain at least one column. KeyColumns are required in this version."); + + // Ensure the caller explicitly mapped the source data to destination columns. + // Bulk update needs mappings to know which columns should be staged and which non-key columns should be updated. + if (ColumnMappings.Count == 0) + throw new InvalidOperationException("ColumnMappings cannot be empty. Add at least one column mapping."); + + // Validate destination column mappings. + // Each mapping must have a destination column, destination columns must be unique, + // and expression mappings are not supported by bulk update in this version. + var seenColumns = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var mapping in ColumnMappings) + { + // Ensure every mapping points to a real destination column name. + if (string.IsNullOrWhiteSpace(mapping.DestinationColumn)) + throw new InvalidOperationException("ColumnMappings contains a mapping with a null or empty DestinationColumn."); + + // Ensure the same destination column isn't mapped more than once. + // Duplicate mappings would make the staging table and UPDATE SET clause ambiguous. + if (!seenColumns.Add(mapping.DestinationColumn)) + throw new InvalidOperationException($"ColumnMappings contains duplicate destination column '{mapping.DestinationColumn}'."); + + // Reject expression mappings for now. + // Bulk update stages real destination columns into a temporary table, while expression mappings + // may use user variables such as @tmp that are not real staging table columns. + if (mapping.Expression is not null) + throw new NotSupportedException("Expression column mappings are not supported by SingleStoreBulkUpdate in this version."); + } + + // Validate key column names. + // Key columns must be non-empty and unique because they are used to build the JOIN condition. + var keyColumnsSet = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var keyColumn in KeyColumns) + { + // Ensure the key column has a valid name. + if (string.IsNullOrWhiteSpace(keyColumn)) + throw new InvalidOperationException("KeyColumns cannot contain null or empty column names."); + + // Ensure the same key column isn't specified more than once. + if (!keyColumnsSet.Add(keyColumn)) + throw new InvalidOperationException($"KeyColumns contains duplicate column '{keyColumn}'."); + } + + // Ensure every key column is included in ColumnMappings. + // The staging table must contain the key columns so the UPDATE JOIN can match rows. + foreach (var keyColumn in keyColumnsSet) + { + if (!seenColumns.Contains(keyColumn)) + { + throw new InvalidOperationException( + $"Key column '{keyColumn}' not found in ColumnMappings. All key columns must be mapped."); + } + } + + // Ensure there is at least one non-key column to update. + // If all mapped columns are key columns, the UPDATE statement would have an empty SET clause. + var hasUpdateColumn = ColumnMappings.Any(m => !keyColumnsSet.Contains(m.DestinationColumn)); + + if (!hasUpdateColumn) + throw new InvalidOperationException("ColumnMappings must contain at least one non-key column to update."); + } + + private readonly SingleStoreConnection m_connection; private readonly SingleStoreTransaction m_transaction; private readonly ILogger m_logger; diff --git a/src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs b/src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs index 9e471b897..a5ac88d07 100644 --- a/src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs +++ b/src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs @@ -17,9 +17,9 @@ internal SingleStoreRowsStagedEventArgs() } } -/// +/*/// /// Represents the method that handles the event of a . /// #pragma warning disable CA1711 // Identifiers should not have incorrect suffix public delegate void SingleStoreRowsStagedEventHandler(object sender, SingleStoreRowsStagedEventArgs e); -#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix*/ From a9d1ee45668e272626e3be0f21858e8845c2d4b0 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Thu, 4 Jun 2026 14:16:40 +0300 Subject: [PATCH 06/28] add schema validation --- .../SingleStoreBulkUpdate.cs | 52 ++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index cc740dc81..38b4be411 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -6,6 +6,8 @@ namespace SingleStoreConnector; +// TODO: consider upsert support in a future version. + /// /// Provides efficient bulk update operations for SingleStore databases. /// @@ -23,7 +25,7 @@ public sealed class SingleStoreBulkUpdate /// /// The to use. /// (Optional) The to use. - public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransaction transaction) + public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransaction? transaction = null) { m_connection = connection ?? throw new ArgumentNullException(nameof(connection)); m_transaction = transaction; @@ -77,7 +79,6 @@ public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransa /// public event SingleStoreRowsStagedEventHandler? SingleStoreRowsStaged;*/ - // TODO: Add WriteToServer overloads in next steps private void ValidateColumnMappings() { // Ensure the caller specified at least one key column. @@ -139,15 +140,54 @@ private void ValidateColumnMappings() // Ensure there is at least one non-key column to update. // If all mapped columns are key columns, the UPDATE statement would have an empty SET clause. - var hasUpdateColumn = ColumnMappings.Any(m => !keyColumnsSet.Contains(m.DestinationColumn)); - - if (!hasUpdateColumn) + if (GetUpdateColumns().Count == 0) throw new InvalidOperationException("ColumnMappings must contain at least one non-key column to update."); } + private async ValueTask ValidateSchemaAsync(string tableName, CancellationToken cancellationToken) + { + var schemaDetector = new SchemaDetector(m_connection); + + // TODO: make changes to support the solutions described here -- https://docs.singlestore.com/cloud/reference/troubleshooting-reference/query-errors/error-1706-hy-000-feature-multi-table-update-delete-with-a-reference-table-as-target-table-is-not-supported-by-memsql/ + if (await schemaDetector.IsReferenceTableAsync(tableName, cancellationToken).ConfigureAwait(false)) + throw new NotSupportedException($"Target table '{tableName}' is a reference table. Bulk updates on reference tables are not supported in this version."); + + var shardKeyColumns = await schemaDetector.GetShardKeyColumnsAsync(tableName, cancellationToken).ConfigureAwait(false); + var updateColumns = GetUpdateColumns(); + + foreach (var updateColumn in updateColumns) + { + if (shardKeyColumns.Contains(updateColumn, StringComparer.OrdinalIgnoreCase)) + throw new InvalidOperationException($"Column '{updateColumn}' is a shard key. SingleStore does not support updating shard key columns."); + } + + var schema = await schemaDetector.GetTableSchemaAsync(tableName, cancellationToken).ConfigureAwait(false); + + var tableColumns = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (DataRow row in schema.Rows) + { + var columnName = row["ColumnName"]?.ToString(); + if (columnName is not null) + tableColumns.Add(columnName); + } + + foreach (var columnMapping in ColumnMappings) + { + if (!tableColumns.Contains(columnMapping.DestinationColumn)) + throw new InvalidOperationException($"Column '{columnMapping.DestinationColumn}' does not exist in target table '{tableName}'."); + } + } + + private List GetUpdateColumns() => + ColumnMappings + .Select(x => x.DestinationColumn) + .Where(x => !KeyColumns.Contains(x, StringComparer.OrdinalIgnoreCase)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); private readonly SingleStoreConnection m_connection; - private readonly SingleStoreTransaction m_transaction; + private readonly SingleStoreTransaction? m_transaction; private readonly ILogger m_logger; private readonly List m_warnings; } From 13f101e1c6acfb07e40ad4fa204fa5e704eb59e2 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Mon, 15 Jun 2026 23:44:59 +0300 Subject: [PATCH 07/28] add staging table creation logic --- .../SingleStoreBulkUpdate.cs | 132 ++++++++ .../Utilities/SchemaDetector.cs | 298 ++++++++++++++++++ 2 files changed, 430 insertions(+) diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index 38b4be411..0e92639de 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -1,5 +1,6 @@ using System.Data; using System.Diagnostics; +using System.Text; using Microsoft.Extensions.Logging; using SingleStoreConnector.Logging; using SingleStoreConnector.Utilities; @@ -186,6 +187,137 @@ private List GetUpdateColumns() => .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); + /// + /// Creates a session-scoped temporary staging table containing only the mapped (key + update) columns + /// of the destination table, ready to receive the source data via . + /// + /// The destination table whose column types are mirrored. + /// A token to cancel the asynchronous operation. + /// The name of the created temporary table. + /// + /// + /// Column type definitions are copied verbatim from SHOW CREATE TABLE rather than reconstructed from + /// GetSchemaTable(). The schema table is lossy for several SingleStore types (for example + /// VARBINARY is reported as BLOB and BIT(1) as BIGINT, and UNSIGNED, + /// character set, collation and ENUM/SET member lists are not exposed), so copying the exact + /// definition is the only way to guarantee the staging column matches the destination column. Matching the + /// collation in particular keeps the key-column equality used by the UPDATE ... JOIN well defined. + /// + /// + /// The key columns form the staging table's PRIMARY KEY, so they are always declared NOT NULL + /// even when the destination column is nullable (a nullable primary key column is not allowed, and SQL + /// equality on NULL would not match rows in the join anyway). When the destination table's shard key + /// is a subset of the key columns, the staging table is sharded the same way so the join can run locally; + /// otherwise it falls back to the primary key distribution and logs a shard-key mismatch warning. + /// + /// + /// This must run on the same open connection (and transaction) used for staging, counting and updating, + /// because the temporary table is session-scoped. + /// + /// + private async Task CreateStagingTableAsync(string destinationTableName, CancellationToken cancellationToken) + { + // Generate a unique temporary table name. The "g" suffix on the GUID guarantees the identifier + // starts with a letter regardless of the GUID's first hex digit. + var tempTableName = $"_bulk_update_staging_g{Guid.NewGuid():N}"; + + var schemaDetector = new SchemaDetector(m_connection); + + // Pull the exact, server-rendered type definition for every column so the staging columns are + // byte-for-byte type compatible with the destination (see remarks). + var columnTypeDefinitions = await schemaDetector.GetColumnTypeDefinitionsAsync(destinationTableName, cancellationToken).ConfigureAwait(false); + var shardKeyColumns = await schemaDetector.GetShardKeyColumnsAsync(destinationTableName, cancellationToken).ConfigureAwait(false); + + // Emit a column definition for each mapped column, preserving the order in which the columns appear + // in the destination table is unnecessary here: the staging table only needs the columns to exist by + // name. SingleStoreBulkCopy maps source ordinals to these destination column names when staging. + var keyColumnSet = new HashSet(KeyColumns, StringComparer.OrdinalIgnoreCase); + var seenColumns = new HashSet(StringComparer.OrdinalIgnoreCase); + var columnDefinitions = new List(ColumnMappings.Count); + + foreach (var mapping in ColumnMappings) + { + var columnName = mapping.DestinationColumn; + + // ColumnMappings is validated for duplicates earlier, but guard anyway so a duplicate can never + // produce an invalid CREATE TABLE with two columns of the same name. + if (!seenColumns.Add(columnName)) + continue; + + if (!columnTypeDefinitions.TryGetValue(columnName, out var typeDefinition)) + throw new InvalidOperationException($"Column '{columnName}' not found in destination table '{destinationTableName}'."); + + // Key columns become the staging primary key, so they must be NOT NULL even when nullable in the + // destination. Update columns are left nullable so a NULL source value stages successfully; any + // real NOT NULL violation then surfaces against the destination during the UPDATE. + var nullability = keyColumnSet.Contains(columnName) ? "NOT NULL" : "NULL"; + + columnDefinitions.Add($"{IdentifierHelper.QuoteIdentifier(columnName)} {typeDefinition} {nullability}"); + } + + // The key columns identify the rows to update, so they are the natural primary key of the staging + // table. This also rejects duplicate keys in the source data with a clear primary-key violation. + var primaryKey = $"PRIMARY KEY ({string.Join(", ", KeyColumns.Select(IdentifierHelper.QuoteIdentifier))})"; + + var stagingShardKey = ComputeStagingShardKey(shardKeyColumns, keyColumnSet); + + var createTableSql = new StringBuilder(); + createTableSql.Append("CREATE TEMPORARY TABLE "); + createTableSql.Append(IdentifierHelper.QuoteIdentifier(tempTableName)); + createTableSql.Append(" ("); + createTableSql.Append(string.Join(", ", columnDefinitions)); + createTableSql.Append(", "); + createTableSql.Append(primaryKey); + + if (stagingShardKey.Count != 0) + { + createTableSql.Append(", SHARD KEY ("); + createTableSql.Append(string.Join(", ", stagingShardKey.Select(IdentifierHelper.QuoteIdentifier))); + createTableSql.Append(')'); + } + + createTableSql.Append(')'); + + using (var cmd = m_connection.CreateCommand()) + { + cmd.CommandText = createTableSql.ToString(); + cmd.Transaction = m_transaction; + cmd.CommandTimeout = BulkCopyTimeout; + await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + Log.CreatedStagingTableForBulkUpdate(m_logger, tempTableName, columnDefinitions.Count); + + return tempTableName; + } + + /// + /// Determines the shard key to declare on the staging table so that the UPDATE ... JOIN can run as a + /// local (non-reshuffled) join whenever possible. + /// + /// + /// A shard key must be a subset of the primary key, which for the staging table is exactly the key columns. + /// When the destination's shard key is contained in the key columns we reuse it verbatim (preserving its + /// column order) so both tables hash to the same partitions. When it is not — for example the destination is + /// sharded on a column that is not a join key — the staging table cannot be aligned, so we fall back to the + /// primary-key distribution (by returning an empty list, which omits an explicit shard key) and warn. + /// + private List ComputeStagingShardKey(List destinationShardKeyColumns, HashSet keyColumnSet) + { + if (destinationShardKeyColumns.Count == 0) + return []; + + if (destinationShardKeyColumns.All(keyColumnSet.Contains)) + return destinationShardKeyColumns; + + Log.ShardKeyMismatchForBulkUpdate( + m_logger, + string.Join(", ", KeyColumns), + string.Join(", ", destinationShardKeyColumns)); + + return []; + } + private readonly SingleStoreConnection m_connection; private readonly SingleStoreTransaction? m_transaction; private readonly ILogger m_logger; diff --git a/src/SingleStoreConnector/Utilities/SchemaDetector.cs b/src/SingleStoreConnector/Utilities/SchemaDetector.cs index f9949289f..15d29b1ae 100644 --- a/src/SingleStoreConnector/Utilities/SchemaDetector.cs +++ b/src/SingleStoreConnector/Utilities/SchemaDetector.cs @@ -120,6 +120,57 @@ private async Task GetCreateTableStatementAsync(string tableName, Cancel throw new InvalidOperationException($"Unable to retrieve CREATE TABLE statement for {tableName}."); } + /// + /// Gets the exact type definition (including length/precision, character set and collation) for each + /// column of the specified table, as it appears in SHOW CREATE TABLE. + /// + /// + /// + /// The returned type definitions are taken verbatim from the server rather than reconstructed from + /// GetSchemaTable(). The schema table is lossy for several SingleStore types: + /// for example VARBINARY is reported as BLOB, BIT(1) as BIGINT, the + /// UNSIGNED flag is dropped, and character set, collation and ENUM/SET member + /// lists are not exposed at all. Copying the definition verbatim guarantees the staging column has the + /// exact same type as the destination column, which keeps key-column equality (including collation) + /// well defined in the UPDATE ... JOIN. + /// + /// + /// Only the type portion is returned (data type, any parenthesised arguments, UNSIGNED/ZEROFILL, + /// CHARACTER SET and COLLATE). Column options that are inappropriate for a staging table — + /// NOT NULL/NULL, DEFAULT, AUTO_INCREMENT, generated-column expressions and + /// COMMENT — are intentionally excluded so the caller can decide nullability itself. + /// + /// + /// A case-insensitive map of column name to its type definition. + public async Task> GetColumnTypeDefinitionsAsync(string tableName, CancellationToken cancellationToken = default) + { + EnsureConnectionIsOpen(); + + var createTableSql = await GetCreateTableStatementAsync(tableName, cancellationToken) + .ConfigureAwait(false); + + var definitions = new Dictionary(StringComparer.OrdinalIgnoreCase); + + var body = ExtractTableBody(createTableSql, tableName); + foreach (var item in SplitTopLevel(body)) + { + var trimmed = item.TrimStart(); + + // In SHOW CREATE TABLE output every column definition begins with a backtick-quoted column name, + // while table-level constraints (PRIMARY KEY, SHARD KEY, KEY, UNIQUE KEY, CONSTRAINT, ...) begin + // with an unquoted keyword. Skip anything that is not a column definition. + if (trimmed.Length == 0 || trimmed[0] != '`') + continue; + + var (columnName, rest) = ParseQuotedNameAndRest(trimmed); + var typeDefinition = ExtractTypeDefinition(rest); + if (typeDefinition.Length != 0) + definitions[columnName] = typeDefinition; + } + + return definitions; + } + /// /// Gets column metadata for the specified table. /// @@ -190,4 +241,251 @@ void AddIdentifier() current.Clear(); } } + + /// + /// Returns the contents of the column-list parentheses in a CREATE TABLE statement + /// (everything between the first top-level ( and its matching )). + /// + private static string ExtractTableBody(string createTableSql, string tableName) + { + var start = -1; + var depth = 0; + + for (var i = 0; i < createTableSql.Length; i++) + { + var ch = createTableSql[i]; + + if (ch == '`' || ch == '\'' || ch == '"') + { + i = SkipQuoted(createTableSql, i, ch); + continue; + } + + if (ch == '(') + { + if (depth == 0) + start = i + 1; + depth++; + } + else if (ch == ')') + { + depth--; + if (depth == 0) + return createTableSql.Substring(start, i - start); + } + } + + throw new InvalidOperationException($"Unable to parse CREATE TABLE statement for {tableName}."); + } + + /// + /// Splits a column/constraint list on commas that are at the top level + /// (not inside parentheses, backticks or quotes). + /// + private static List SplitTopLevel(string body) + { + var items = new List(); + var depth = 0; + var start = 0; + + for (var i = 0; i < body.Length; i++) + { + var ch = body[i]; + + if (ch == '`' || ch == '\'' || ch == '"') + { + i = SkipQuoted(body, i, ch); + continue; + } + + if (ch == '(') + { + depth++; + } + else if (ch == ')') + { + depth--; + } + else if (ch == ',' && depth == 0) + { + items.Add(body.Substring(start, i - start)); + start = i + 1; + } + } + + items.Add(body.Substring(start)); + return items; + } + + /// + /// Parses a leading backtick-quoted identifier from a column definition and returns the unquoted + /// identifier together with the remainder of the definition (the part after the column name). + /// + private static (string Name, string Remainder) ParseQuotedNameAndRest(string columnDefinition) + { + // columnDefinition is known to start with a backtick. + var name = new StringBuilder(); + + for (var i = 1; i < columnDefinition.Length; i++) + { + var ch = columnDefinition[i]; + + if (ch == '`') + { + if (i + 1 < columnDefinition.Length && columnDefinition[i + 1] == '`') + { + name.Append('`'); + i++; + } + else + { + return (name.ToString(), columnDefinition.Substring(i + 1)); + } + } + else + { + name.Append(ch); + } + } + + throw new InvalidOperationException("Invalid column definition: unterminated quoted column name."); + } + + /// + /// Extracts only the type portion of a column definition (the part after the column name), keeping the + /// data type, any parenthesised arguments, UNSIGNED/ZEROFILL and CHARACTER SET/COLLATE + /// clauses, and dropping column options such as NOT NULL, DEFAULT, AUTO_INCREMENT and COMMENT. + /// + private static string ExtractTypeDefinition(string rest) + { + var tokens = TokenizeTopLevel(rest); + if (tokens.Count == 0) + return string.Empty; + + // The first token is always the data type, including any parenthesised arguments such as + // "varchar(255)", "decimal(18,4)", "bit(1)", "enum('a','b')" or "vector(4, F32)". + var kept = new List { tokens[0] }; + + for (var i = 1; i < tokens.Count;) + { + var keyword = tokens[i].ToUpperInvariant(); + + if (keyword is "UNSIGNED" or "SIGNED" or "ZEROFILL") + { + kept.Add(tokens[i]); + i++; + } + else if (keyword == "CHARACTER" && i + 2 < tokens.Count && string.Equals(tokens[i + 1], "SET", StringComparison.OrdinalIgnoreCase)) + { + kept.Add(tokens[i]); + kept.Add(tokens[i + 1]); + kept.Add(tokens[i + 2]); + i += 3; + } + else if (keyword == "CHARSET" && i + 1 < tokens.Count) + { + kept.Add(tokens[i]); + kept.Add(tokens[i + 1]); + i += 2; + } + else if (keyword == "COLLATE" && i + 1 < tokens.Count) + { + kept.Add(tokens[i]); + kept.Add(tokens[i + 1]); + i += 2; + } + else + { + // Anything else (NOT, NULL, DEFAULT, AUTO_INCREMENT, GENERATED, AS, COMMENT, ...) is a + // column option rather than part of the type, so the type definition ends here. + break; + } + } + + return string.Join(" ", kept); + } + + /// + /// Splits text into whitespace-delimited tokens, treating a parenthesised group as part of the token it + /// is attached to and never splitting inside parentheses, backticks or quotes. + /// + private static List TokenizeTopLevel(string text) + { + var tokens = new List(); + var current = new StringBuilder(); + var depth = 0; + + for (var i = 0; i < text.Length; i++) + { + var ch = text[i]; + + if (ch == '`' || ch == '\'' || ch == '"') + { + var end = SkipQuoted(text, i, ch); + current.Append(text, i, end - i + 1); + i = end; + } + else if (ch == '(') + { + depth++; + current.Append(ch); + } + else if (ch == ')') + { + depth--; + current.Append(ch); + } + else if (char.IsWhiteSpace(ch) && depth == 0) + { + if (current.Length != 0) + { + tokens.Add(current.ToString()); + current.Clear(); + } + } + else + { + current.Append(ch); + } + } + + if (current.Length != 0) + tokens.Add(current.ToString()); + + return tokens; + } + + /// + /// Given the index of an opening quote character (backtick, single quote or double quote), returns the + /// index of the matching closing quote, accounting for doubled-quote escaping and backslash escaping. + /// + private static int SkipQuoted(string text, int openIndex, char quote) + { + for (var i = openIndex + 1; i < text.Length; i++) + { + var ch = text[i]; + + // Backslash escaping applies inside string literals but not inside backtick identifiers. + if (ch == '\\' && quote != '`' && i + 1 < text.Length) + { + i++; + continue; + } + + if (ch == quote) + { + // A doubled quote is an escaped quote, not a terminator. + if (i + 1 < text.Length && text[i + 1] == quote) + { + i++; + continue; + } + + return i; + } + } + + // Unterminated quote: treat the rest of the string as quoted. + return text.Length - 1; + } } From d1a87d1135fb066260bdc6218058376eafd8f133 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 16 Jun 2026 19:58:48 +0300 Subject: [PATCH 08/28] add data staging logic --- .../SingleStoreBulkUpdate.cs | 93 ++++++++++++++++++- .../SingleStoreRowsStagedEventArgs.cs | 6 +- .../packages.lock.json | 24 ++--- 3 files changed, 106 insertions(+), 17 deletions(-) diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index 0e92639de..0397ea524 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -71,14 +71,14 @@ public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransa /// public bool ComputeRowsMatched { get; set; } = true; - /*/// + /// /// This event is raised every time that the number of rows specified by the property have been processed. /// /// /// Receipt of a RowsStaged event does not imply that any rows have been sent to the server or committed. /// The property can be set to true by the event handler to abort the staging. /// - public event SingleStoreRowsStagedEventHandler? SingleStoreRowsStaged;*/ + public event SingleStoreRowsStagedEventHandler? SingleStoreRowsStaged; private void ValidateColumnMappings() { @@ -318,6 +318,95 @@ private List ComputeStagingShardKey(List destinationShardKeyColu return []; } + /// + /// Stages the source rows into the temporary table created by , + /// using (which loads the data via LOAD DATA LOCAL INFILE). + /// + /// The session-scoped temporary staging table to load into. + /// The source data: a , a array, or an . + /// A token to cancel the asynchronous operation. + /// The number of rows staged into the temporary table. + /// + /// + /// This must run on the same open connection (and transaction) as , + /// because the temporary table is session-scoped. The caller is responsible for opening the connection + /// and creating the staging table before calling this method. + /// + /// + /// The bulk-update column mappings are forwarded verbatim to . Each + /// mapping's selects a column from the source + /// data, and its names a column in the + /// staging table (which contains exactly the mapped columns by name). This keeps the source-ordinal / + /// destination-name relationship identical between staging and the later UPDATE ... JOIN. + /// + /// + private async Task StageDataAsync(string tempTableName, object source, CancellationToken cancellationToken) + { + var bulkCopy = new SingleStoreBulkCopy(m_connection, m_transaction) + { + DestinationTableName = tempTableName, + BulkCopyTimeout = BulkCopyTimeout, + NotifyAfter = NotifyAfter, + }; + + // Forward our column mappings unchanged: source ordinal -> staging column name. + foreach (var mapping in ColumnMappings) + bulkCopy.ColumnMappings.Add(mapping); + + // Re-raise SingleStoreBulkCopy's progress event as a bulk-update staging event, and propagate the + // caller's request to abort. Only subscribe when progress notifications are actually requested. + void OnRowsCopied(object sender, SingleStoreRowsCopiedEventArgs e) + { + var args = new SingleStoreRowsStagedEventArgs { RowsStaged = e.RowsCopied }; + SingleStoreRowsStaged?.Invoke(this, args); + if (args.Abort) + e.Abort = true; + } + + var notifyProgress = NotifyAfter > 0 && SingleStoreRowsStaged is not null; + if (notifyProgress) + bulkCopy.SingleStoreRowsCopied += OnRowsCopied; + + try + { + var result = source switch + { + DataTable dataTable => await bulkCopy.WriteToServerAsync(dataTable, cancellationToken).ConfigureAwait(false), + DataRow[] rows => await bulkCopy.WriteToServerAsync(rows, GetDataRowColumnCount(rows), cancellationToken).ConfigureAwait(false), + IDataReader dataReader => await bulkCopy.WriteToServerAsync(dataReader, cancellationToken).ConfigureAwait(false), + _ => throw new ArgumentException($"Unsupported source type '{source.GetType()}'.", nameof(source)), + }; + + m_warnings.AddRange(result.Warnings); + + Log.StagedDataForBulkUpdate(m_logger, result.RowsInserted, result.Warnings.Count); + + return result.RowsInserted; + } + finally + { + if (notifyProgress) + bulkCopy.SingleStoreRowsCopied -= OnRowsCopied; + } + } + + /// + /// Determines the column count to pass to for a array. + /// + /// + /// requires + /// the source column count up front, which is taken from the owning of the first row. + /// Empty input is short-circuited before staging, so the array is expected to be non-empty here; it is still + /// guarded so an unexpected empty array fails clearly rather than dereferencing a missing row. + /// + private static int GetDataRowColumnCount(DataRow[] rows) + { + if (rows.Length == 0) + throw new ArgumentException("Cannot stage an empty DataRow array.", nameof(rows)); + + return rows[0].Table.Columns.Count; + } + private readonly SingleStoreConnection m_connection; private readonly SingleStoreTransaction? m_transaction; private readonly ILogger m_logger; diff --git a/src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs b/src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs index a5ac88d07..aeaaa2152 100644 --- a/src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs +++ b/src/SingleStoreConnector/SingleStoreRowsStagedEventArgs.cs @@ -10,16 +10,16 @@ public sealed class SingleStoreRowsStagedEventArgs : EventArgs /// /// Gets a value that returns the number of rows staged during the current bulk update operation. /// - public int RowsStaged { get; internal set; } + public long RowsStaged { get; internal set; } internal SingleStoreRowsStagedEventArgs() { } } -/*/// +/// /// Represents the method that handles the event of a . /// #pragma warning disable CA1711 // Identifiers should not have incorrect suffix public delegate void SingleStoreRowsStagedEventHandler(object sender, SingleStoreRowsStagedEventArgs e); -#pragma warning restore CA1711 // Identifiers should not have incorrect suffix*/ +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix diff --git a/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json b/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json index e466b627d..6ab93bf5e 100644 --- a/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json +++ b/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json @@ -4,15 +4,15 @@ "net10.0": { "Microsoft.DotNet.ILCompiler": { "type": "Direct", - "requested": "[10.0.7, )", - "resolved": "10.0.7", - "contentHash": "S53itVb3/l5wEyvdOukwxGzRtw7JSKuh2GurW3W5R57w4nxbZZ+7qCu+5jVlowLmgGrQ7yxJLZdpJc1Vpv8aig==" + "requested": "[10.0.8, )", + "resolved": "10.0.8", + "contentHash": "IG4ZS8fP7whR7OeFjC2mxq9n/yd/QsUliHdQ/Kp0OJ+RObeSJx4HLLGs7/Sxghk/U+S3rmpUOsgLheqBcqlMUw==" }, "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[10.0.7, )", - "resolved": "10.0.7", - "contentHash": "Y0O2XrQiNYsW9UM2GhH/8faornu9pYXdd9YBcb/CrBnCOw5QtFqgh8bFd5zjxbrdoSVxGPUn1hWSrKtD0SEPzg==" + "requested": "[10.0.8, )", + "resolved": "10.0.8", + "contentHash": "SyfHfPwnylzDkj7u6KiwiIgVP+gc1Ud0ybomjT/sOGJLrGC6KLWR3zUvnTTcwwGK3YqGXsFWEl0opREH11H8pg==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", @@ -80,15 +80,15 @@ "net8.0": { "Microsoft.DotNet.ILCompiler": { "type": "Direct", - "requested": "[8.0.26, )", - "resolved": "8.0.26", - "contentHash": "TopkZJbGfDfqNRQatno3/UZh8t/mo4tlglGVtcUAZDEplx/DhmBUSoT7UM7M6W0/ah1Wbe6e46FOLDyHL8K4nQ==" + "requested": "[8.0.27, )", + "resolved": "8.0.27", + "contentHash": "2RDzeGQisWc0e4uJpT8WmMQ8Xccf4bZr4bVVV9iGnfnEZCPo//G01MLLCmGHScwLX4zCegNZRNftgUfgWlBb+g==" }, "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.26, )", - "resolved": "8.0.26", - "contentHash": "o7/yVssM2r9Wyln2s9edBd5ANZXqdSdBI+g7JqXkyJmXrhs2WsJp25K5yPnYrTgdKBCjKB8bg+O2oew4sgzFaA==" + "requested": "[8.0.27, )", + "resolved": "8.0.27", + "contentHash": "rQi9TxifHRnXP7lVRZH05DxD2/XGbJp12q0ozcbrlBlBnyyzssFTH/2vLhtKWUp2CT1qVscTrcYTFiwTyKPKRg==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", From 0e6d56c0dbe526d6862f98252ab1aa70263e7fb4 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 16 Jun 2026 21:29:47 +0300 Subject: [PATCH 09/28] add match count logic --- .../SingleStoreBulkUpdate.cs | 61 +++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index 0397ea524..13c0831d8 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -1,5 +1,6 @@ using System.Data; using System.Diagnostics; +using System.Globalization; using System.Text; using Microsoft.Extensions.Logging; using SingleStoreConnector.Logging; @@ -215,7 +216,7 @@ private List GetUpdateColumns() => /// because the temporary table is session-scoped. /// /// - private async Task CreateStagingTableAsync(string destinationTableName, CancellationToken cancellationToken) + private async Task CreateStagingTableAsync(string destinationTableName, CancellationToken cancellationToken) { // Generate a unique temporary table name. The "g" suffix on the GUID guarantees the identifier // starts with a letter regardless of the GUID's first hex digit. @@ -302,7 +303,7 @@ private async Task CreateStagingTableAsync(string destinationTableName, /// sharded on a column that is not a join key — the staging table cannot be aligned, so we fall back to the /// primary-key distribution (by returning an empty list, which omits an explicit shard key) and warn. /// - private List ComputeStagingShardKey(List destinationShardKeyColumns, HashSet keyColumnSet) + private List ComputeStagingShardKey(List destinationShardKeyColumns, HashSet keyColumnSet) { if (destinationShardKeyColumns.Count == 0) return []; @@ -340,7 +341,7 @@ private List ComputeStagingShardKey(List destinationShardKeyColu /// destination-name relationship identical between staging and the later UPDATE ... JOIN. /// /// - private async Task StageDataAsync(string tempTableName, object source, CancellationToken cancellationToken) + private async Task StageDataAsync(string tempTableName, object source, CancellationToken cancellationToken) { var bulkCopy = new SingleStoreBulkCopy(m_connection, m_transaction) { @@ -399,7 +400,7 @@ void OnRowsCopied(object sender, SingleStoreRowsCopiedEventArgs e) /// Empty input is short-circuited before staging, so the array is expected to be non-empty here; it is still /// guarded so an unexpected empty array fails clearly rather than dereferencing a missing row. /// - private static int GetDataRowColumnCount(DataRow[] rows) + private static int GetDataRowColumnCount(DataRow[] rows) { if (rows.Length == 0) throw new ArgumentException("Cannot stage an empty DataRow array.", nameof(rows)); @@ -407,6 +408,58 @@ private static int GetDataRowColumnCount(DataRow[] rows) return rows[0].Table.Columns.Count; } + /// + /// Counts how many staged rows match a row in the destination table, joined on the key columns. + /// + /// The session-scoped staging table populated by . + /// A token to cancel the asynchronous operation. + /// + /// The number of staged rows that match a destination row, or when + /// is (the count was intentionally skipped). + /// + /// + /// + /// This runs an extra SELECT COUNT(*) over the same INNER JOIN that the subsequent + /// UPDATE ... JOIN uses, letting the caller distinguish staged rows that updated a destination row + /// from staged rows that matched nothing. Callers that do not need this distinction can set + /// to to skip the query. + /// + /// + /// The join uses the key columns, which were created in the staging table with the destination's exact + /// type and collation (see ), so this count is consistent with the + /// rows the UPDATE will match. It must run on the same open connection/transaction as the rest of the + /// operation because the staging table is session-scoped. + /// + /// + private async Task ComputeMatchedRowsAsync(string tempTableName, CancellationToken cancellationToken) + { + if (!ComputeRowsMatched) + return null; + + // Build the key-column equi-join: t.`k1` = s.`k1` AND t.`k2` = s.`k2` ... + var joinCondition = string.Join( + " AND ", + KeyColumns.Select(k => $"t.{IdentifierHelper.QuoteIdentifier(k)} = s.{IdentifierHelper.QuoteIdentifier(k)}")); + + var countSql = + $"SELECT COUNT(*) FROM {IdentifierHelper.QuoteQualifiedIdentifier(DestinationTableName!)} AS t " + + $"INNER JOIN {IdentifierHelper.QuoteIdentifier(tempTableName)} AS s ON {joinCondition}"; + + using var cmd = m_connection.CreateCommand(); + cmd.CommandText = countSql; + cmd.Transaction = m_transaction; + cmd.CommandTimeout = BulkCopyTimeout; + + var scalar = await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + + // COUNT(*) comes back as a long; convert rather than cast so the boxed type is handled correctly. + var rowsMatched = scalar is null or DBNull ? 0 : Convert.ToInt32(scalar, CultureInfo.InvariantCulture); + + Log.QueriedMatchCountForBulkUpdate(m_logger, rowsMatched); + + return rowsMatched; + } + private readonly SingleStoreConnection m_connection; private readonly SingleStoreTransaction? m_transaction; private readonly ILogger m_logger; From 4c767751b08cdceb6c682a822e74967bd89bdf36 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 16 Jun 2026 23:21:38 +0300 Subject: [PATCH 10/28] add UPDATE execution logic --- .../SingleStoreBulkUpdate.cs | 78 +++++++++++++++++-- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index 13c0831d8..4b1f83905 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -436,14 +436,9 @@ private static int GetDataRowColumnCount(DataRow[] rows) if (!ComputeRowsMatched) return null; - // Build the key-column equi-join: t.`k1` = s.`k1` AND t.`k2` = s.`k2` ... - var joinCondition = string.Join( - " AND ", - KeyColumns.Select(k => $"t.{IdentifierHelper.QuoteIdentifier(k)} = s.{IdentifierHelper.QuoteIdentifier(k)}")); - var countSql = $"SELECT COUNT(*) FROM {IdentifierHelper.QuoteQualifiedIdentifier(DestinationTableName!)} AS t " + - $"INNER JOIN {IdentifierHelper.QuoteIdentifier(tempTableName)} AS s ON {joinCondition}"; + $"INNER JOIN {IdentifierHelper.QuoteIdentifier(tempTableName)} AS s ON {BuildKeyJoinCondition()}"; using var cmd = m_connection.CreateCommand(); cmd.CommandText = countSql; @@ -460,6 +455,77 @@ private static int GetDataRowColumnCount(DataRow[] rows) return rowsMatched; } + /// + /// Executes the UPDATE ... JOIN that copies the non-key column values from the staging table into + /// the matching rows of the destination table. + /// + /// The session-scoped staging table populated by . + /// A token to cancel the asynchronous operation. + /// The number of rows affected by the update, as reported by the server. + /// + /// + /// Rows are matched on the key columns (the same join used by ) and the + /// non-key mapped columns are assigned from the staging row. The statement runs on the same open + /// connection/transaction as the rest of the operation because the staging table is session-scoped. + /// + /// + /// The returned count reflects the server's affected-row semantics, which depend on the connection's + /// setting. With the default + /// (UseAffectedRows=false, i.e. CLIENT_FOUND_ROWS), the count is the number of rows + /// matched by the join — including rows that already held the target values — so it typically + /// equals 's result. With UseAffectedRows=true, it is the number + /// of rows whose values actually changed. + /// + /// + /// Warnings raised while executing the statement (for example truncation or conversion warnings) are + /// collected via the connection's event and surfaced on the + /// operation result. + /// + /// + private async Task ExecuteUpdateAsync(string tempTableName, CancellationToken cancellationToken) + { + // Assign each non-key mapped column from the staging row: t.`c1` = s.`c1`, t.`c2` = s.`c2` ... + var setClause = string.Join( + ", ", + GetUpdateColumns().Select(c => $"t.{IdentifierHelper.QuoteIdentifier(c)} = s.{IdentifierHelper.QuoteIdentifier(c)}")); + + var updateSql = + $"UPDATE {IdentifierHelper.QuoteQualifiedIdentifier(DestinationTableName!)} AS t " + + $"INNER JOIN {IdentifierHelper.QuoteIdentifier(tempTableName)} AS s ON {BuildKeyJoinCondition()} " + + $"SET {setClause}"; + + using var cmd = m_connection.CreateCommand(); + cmd.CommandText = updateSql; + cmd.Transaction = m_transaction; + cmd.CommandTimeout = BulkCopyTimeout; + + // Collect any warnings raised during the UPDATE. Errors is already IReadOnlyList. + void OnInfoMessage(object sender, SingleStoreInfoMessageEventArgs args) => m_warnings.AddRange(args.Errors); + + m_connection.InfoMessage += OnInfoMessage; + try + { + var rowsUpdated = await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + + Log.ExecutedBulkUpdate(m_logger, rowsUpdated); + + return rowsUpdated; + } + finally + { + m_connection.InfoMessage -= OnInfoMessage; + } + } + + /// + /// Builds the key-column equi-join predicate shared by the match-count query and the update, joining the + /// destination table (alias t) to the staging table (alias s) on every key column. + /// + private string BuildKeyJoinCondition() => + string.Join( + " AND ", + KeyColumns.Select(k => $"t.{IdentifierHelper.QuoteIdentifier(k)} = s.{IdentifierHelper.QuoteIdentifier(k)}")); + private readonly SingleStoreConnection m_connection; private readonly SingleStoreTransaction? m_transaction; private readonly ILogger m_logger; From 991d1e33b1687f13a56e20c1bbc94011a87c1d0c Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 17 Jun 2026 13:20:54 +0300 Subject: [PATCH 11/28] add cleanup (temp staging table) logic --- src/SingleStoreConnector/Logging/EventIds.cs | 1 + src/SingleStoreConnector/Logging/Log.cs | 3 ++ .../SingleStoreBulkUpdate.cs | 43 +++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/src/SingleStoreConnector/Logging/EventIds.cs b/src/SingleStoreConnector/Logging/EventIds.cs index 537cd184c..83ad95277 100644 --- a/src/SingleStoreConnector/Logging/EventIds.cs +++ b/src/SingleStoreConnector/Logging/EventIds.cs @@ -162,6 +162,7 @@ internal static class EventIds public const int KeyColumnsNotIndexedForBulkUpdate = 2806; public const int ShardKeyMismatchForBulkUpdate = 2807; public const int LargeUnmatchedCountForBulkUpdate = 2808; + public const int FailedToDropStagingTableForBulkUpdate = 2809; // Connection pool events, 3000-3099 public const int WaitingForAvailableSession = 3000; diff --git a/src/SingleStoreConnector/Logging/Log.cs b/src/SingleStoreConnector/Logging/Log.cs index 4da783a93..990bb1945 100644 --- a/src/SingleStoreConnector/Logging/Log.cs +++ b/src/SingleStoreConnector/Logging/Log.cs @@ -410,6 +410,9 @@ internal static partial class Log [LoggerMessage(EventIds.LargeUnmatchedCountForBulkUpdate, LogLevel.Warning, "Large unmatched count for bulk update: Staged={Staged}, Matched={Matched}, Unmatched={Unmatched}")] public static partial void LargeUnmatchedCountForBulkUpdate(ILogger logger, int staged, int matched, int unmatched); + [LoggerMessage(EventIds.FailedToDropStagingTableForBulkUpdate, LogLevel.Warning, "Failed to drop staging table {TempTableName} for bulk update: {ExceptionMessage}")] + public static partial void FailedToDropStagingTableForBulkUpdate(ILogger logger, Exception exception, string tempTableName, string exceptionMessage); + [LoggerMessage(EventIds.WaitingForAvailableSession, LogLevel.Trace, "Pool {PoolId} waiting for an available session")] public static partial void WaitingForAvailableSession(ILogger logger, int poolId); diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index 4b1f83905..cb345ad9a 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -517,6 +517,49 @@ private async Task ExecuteUpdateAsync(string tempTableName, CancellationTok } } + /// + /// Drops the temporary staging table created by on a best-effort basis. + /// + /// The name of the staging table, or if none was created. + /// + /// + /// The staging table is session-scoped, so it is discarded automatically when the session ends (for example + /// when a connection opened by the bulk update is closed, or when a pooled connection is reset). This explicit + /// drop matters mainly when the caller supplied an already-open connection that they continue to use, freeing + /// the temporary table promptly rather than leaving it until the session is reset. + /// + /// + /// Cleanup never throws: a failed drop is logged and swallowed so it cannot mask the outcome (or the original + /// exception) of the bulk update. The drop is skipped when the connection is no longer open, because in that + /// case the session — and therefore the temporary table — is already gone. + /// is used deliberately so cleanup still runs after a cancelled or timed-out operation. + /// + /// + private async Task DropStagingTableAsync(string? tempTableName) + { + if (string.IsNullOrEmpty(tempTableName)) + return; + + // A session-scoped temporary table cannot outlive its session, so there is nothing to drop (and no usable + // connection to issue the command on) once the connection is no longer open. + if (m_connection.State != ConnectionState.Open) + return; + + try + { + using var cmd = m_connection.CreateCommand(); + cmd.CommandText = $"DROP TEMPORARY TABLE IF EXISTS {IdentifierHelper.QuoteIdentifier(tempTableName!)}"; + cmd.Transaction = m_transaction; + cmd.CommandTimeout = BulkCopyTimeout; + await cmd.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false); + } + catch (Exception ex) + { + // A failed cleanup is non-fatal: the temporary table will be discarded when the session ends. + Log.FailedToDropStagingTableForBulkUpdate(m_logger, ex, tempTableName!, ex.Message); + } + } + /// /// Builds the key-column equi-join predicate shared by the match-count query and the update, joining the /// destination table (alias t) to the staging table (alias s) on every key column. From 0ebb175579866b06384713ed41e946593e4c7992 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 17 Jun 2026 15:33:32 +0300 Subject: [PATCH 12/28] add WriteToServer wrappers (same as in BulkCopy class) --- .../SingleStoreBulkUpdate.cs | 127 ++++++++++++++++-- 1 file changed, 113 insertions(+), 14 deletions(-) diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index cb345ad9a..9d1961de4 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -4,6 +4,7 @@ using System.Text; using Microsoft.Extensions.Logging; using SingleStoreConnector.Logging; +using SingleStoreConnector.Protocol.Serialization; using SingleStoreConnector.Utilities; namespace SingleStoreConnector; @@ -81,6 +82,99 @@ public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransa /// public event SingleStoreRowsStagedEventHandler? SingleStoreRowsStaged; + /// + /// Updates rows in the destination table using the data in the supplied . + /// + /// The containing the key and update column values. + /// A describing the result of the operation. + public SingleStoreBulkUpdateResult WriteToServer(DataTable dataTable) + { + ArgumentNullException.ThrowIfNull(dataTable); +#pragma warning disable CA2012 // Safe because method completes synchronously + return WriteToServerAsync(IOBehavior.Synchronous, dataTable, CancellationToken.None).GetAwaiter().GetResult(); +#pragma warning restore CA2012 + } + + /// + /// Asynchronously updates rows in the destination table using the data in the supplied . + /// + /// The containing the key and update column values. + /// A token to cancel the asynchronous operation. + /// A describing the result of the operation. + public async ValueTask WriteToServerAsync(DataTable dataTable, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(dataTable); + return await WriteToServerAsync(IOBehavior.Asynchronous, dataTable, cancellationToken).ConfigureAwait(false); + } + + /// + /// Updates rows in the destination table using the data in the supplied sequence of objects. + /// + /// The collection of objects containing the key and update column values. + /// A describing the result of the operation. + public SingleStoreBulkUpdateResult WriteToServer(IEnumerable dataRows) + { + ArgumentNullException.ThrowIfNull(dataRows); +#pragma warning disable CA2012 // Safe because method completes synchronously + return WriteToServerAsync(IOBehavior.Synchronous, dataRows, CancellationToken.None).GetAwaiter().GetResult(); +#pragma warning restore CA2012 + } + + /// + /// Asynchronously updates rows in the destination table using the data in the supplied sequence of objects. + /// + /// The collection of objects containing the key and update column values. + /// A token to cancel the asynchronous operation. + /// A describing the result of the operation. + public async ValueTask WriteToServerAsync(IEnumerable dataRows, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(dataRows); + return await WriteToServerAsync(IOBehavior.Asynchronous, dataRows, cancellationToken).ConfigureAwait(false); + } + + /// + /// Updates rows in the destination table using the data read from the supplied . + /// + /// The to read the key and update column values from. + /// A describing the result of the operation. + public SingleStoreBulkUpdateResult WriteToServer(IDataReader dataReader) + { + ArgumentNullException.ThrowIfNull(dataReader); +#pragma warning disable CA2012 // Safe because method completes synchronously + return WriteToServerAsync(IOBehavior.Synchronous, dataReader, CancellationToken.None).GetAwaiter().GetResult(); +#pragma warning restore CA2012 + } + + /// + /// Asynchronously updates rows in the destination table using the data read from the supplied . + /// + /// The to read the key and update column values from. + /// A token to cancel the asynchronous operation. + /// A describing the result of the operation. + public async ValueTask WriteToServerAsync(IDataReader dataReader, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(dataReader); + return await WriteToServerAsync(IOBehavior.Asynchronous, dataReader, cancellationToken).ConfigureAwait(false); + } + + /// + /// The single implementation behind every WriteToServer/WriteToServerAsync overload. + /// + /// + /// Whether to perform database I/O synchronously or asynchronously. The synchronous public overloads pass + /// , which causes every inner database call to complete inline so the + /// returned task is already finished — making the GetAwaiter().GetResult() in those overloads safe + /// (no blocking wait on outstanding async work). This mirrors . + /// + /// The source data: a , a sequence of , or an . + /// A token to cancel the asynchronous operation. + private ValueTask WriteToServerAsync(IOBehavior ioBehavior, object source, CancellationToken cancellationToken) + { + // The full orchestration (validate, open connection, create staging table, stage, count, update, + // drop staging table) is implemented in step 1.12. This method is intentionally a placeholder until then. + throw new NotImplementedException(); + } + private void ValidateColumnMappings() { // Ensure the caller specified at least one key column. @@ -324,7 +418,7 @@ private List ComputeStagingShardKey(List destinationShardKeyColu /// using (which loads the data via LOAD DATA LOCAL INFILE). /// /// The session-scoped temporary staging table to load into. - /// The source data: a , a array, or an . + /// The source data: a , a sequence of , or an . /// A token to cancel the asynchronous operation. /// The number of rows staged into the temporary table. /// @@ -373,7 +467,7 @@ void OnRowsCopied(object sender, SingleStoreRowsCopiedEventArgs e) var result = source switch { DataTable dataTable => await bulkCopy.WriteToServerAsync(dataTable, cancellationToken).ConfigureAwait(false), - DataRow[] rows => await bulkCopy.WriteToServerAsync(rows, GetDataRowColumnCount(rows), cancellationToken).ConfigureAwait(false), + IEnumerable dataRows => await StageDataRowsAsync(bulkCopy, dataRows, cancellationToken).ConfigureAwait(false), IDataReader dataReader => await bulkCopy.WriteToServerAsync(dataReader, cancellationToken).ConfigureAwait(false), _ => throw new ArgumentException($"Unsupported source type '{source.GetType()}'.", nameof(source)), }; @@ -392,20 +486,25 @@ void OnRowsCopied(object sender, SingleStoreRowsCopiedEventArgs e) } /// - /// Determines the column count to pass to for a array. + /// Stages a sequence of objects, supplying the column count that + /// requires up front. /// /// - /// requires - /// the source column count up front, which is taken from the owning of the first row. - /// Empty input is short-circuited before staging, so the array is expected to be non-empty here; it is still - /// guarded so an unexpected empty array fails clearly rather than dereferencing a missing row. + /// needs the + /// source column count before it enumerates the rows, which is taken from the owning + /// of the first row. The sequence is materialized first so it can be inspected for the first row and then + /// enumerated again by the bulk copy without re-running (and possibly exhausting) a lazy source. Empty input + /// is short-circuited before staging, so the sequence is expected to be non-empty here; it is still guarded so + /// an unexpected empty sequence fails clearly rather than dereferencing a missing row. /// - private static int GetDataRowColumnCount(DataRow[] rows) + private static ValueTask StageDataRowsAsync(SingleStoreBulkCopy bulkCopy, IEnumerable dataRows, CancellationToken cancellationToken) { - if (rows.Length == 0) - throw new ArgumentException("Cannot stage an empty DataRow array.", nameof(rows)); + var rows = dataRows as IReadOnlyList ?? dataRows.ToList(); + if (rows.Count == 0) + throw new ArgumentException("Cannot stage an empty sequence of rows.", nameof(dataRows)); - return rows[0].Table.Columns.Count; + var columnCount = rows[0].Table.Columns.Count; + return bulkCopy.WriteToServerAsync(rows, columnCount, cancellationToken); } /// @@ -482,7 +581,7 @@ private static int GetDataRowColumnCount(DataRow[] rows) /// operation result. /// /// - private async Task ExecuteUpdateAsync(string tempTableName, CancellationToken cancellationToken) + private async Task ExecuteUpdateAsync(string tempTableName, CancellationToken cancellationToken) { // Assign each non-key mapped column from the staging row: t.`c1` = s.`c1`, t.`c2` = s.`c2` ... var setClause = string.Join( @@ -535,7 +634,7 @@ private async Task ExecuteUpdateAsync(string tempTableName, CancellationTok /// is used deliberately so cleanup still runs after a cancelled or timed-out operation. /// /// - private async Task DropStagingTableAsync(string? tempTableName) + private async Task DropStagingTableAsync(string? tempTableName) { if (string.IsNullOrEmpty(tempTableName)) return; @@ -564,7 +663,7 @@ private async Task DropStagingTableAsync(string? tempTableName) /// Builds the key-column equi-join predicate shared by the match-count query and the update, joining the /// destination table (alias t) to the staging table (alias s) on every key column. /// - private string BuildKeyJoinCondition() => + private string BuildKeyJoinCondition() => string.Join( " AND ", KeyColumns.Select(k => $"t.{IdentifierHelper.QuoteIdentifier(k)} = s.{IdentifierHelper.QuoteIdentifier(k)}")); From 66c2ac03b45c69246229e9c87263d12242ccc43b Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Thu, 18 Jun 2026 16:49:35 +0300 Subject: [PATCH 13/28] add general WriteToServer that combines all the methods --- .../SingleStoreBulkUpdate.cs | 171 +++++++++++++----- .../Utilities/SchemaDetector.cs | 27 +-- 2 files changed, 144 insertions(+), 54 deletions(-) diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index 9d1961de4..5cda19599 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -168,13 +168,87 @@ public async ValueTask WriteToServerAsync(IDataRead /// /// The source data: a , a sequence of , or an . /// A token to cancel the asynchronous operation. - private ValueTask WriteToServerAsync(IOBehavior ioBehavior, object source, CancellationToken cancellationToken) + private async ValueTask WriteToServerAsync(IOBehavior ioBehavior, object source, CancellationToken cancellationToken) { - // The full orchestration (validate, open connection, create staging table, stage, count, update, - // drop staging table) is implemented in step 1.12. This method is intentionally a placeholder until then. - throw new NotImplementedException(); + // Validate configuration before touching the connection so misconfiguration fails fast and cheaply. + ValidateColumnMappings(); + + var destinationTableName = DestinationTableName ?? + throw new InvalidOperationException("DestinationTableName must be set before calling WriteToServer."); + + // Reset any warnings from a previous call so the result only reflects this operation. + m_warnings.Clear(); + + // Short-circuit input whose row count is known to be zero: there is nothing to stage or update, so avoid + // opening the connection and creating a staging table. (An IDataReader's count is unknown, so it still + // flows through and stages zero rows naturally.) + if (GetRowCount(source) == 0) + return new SingleStoreBulkUpdateResult(m_warnings.AsReadOnly(), rowsStaged: 0, rowsMatched: ComputeRowsMatched ? 0 : -1, rowsUpdated: 0); + + var stopwatch = Stopwatch.StartNew(); + + // All phases must run on one open session because the staging table is a session-scoped temporary table. + // Open the connection if the caller left it closed, and close it again only if we were the ones to open it. + var closeConnection = false; + if (m_connection.State != ConnectionState.Open) + { + await m_connection.OpenAsync(ioBehavior, cancellationToken).ConfigureAwait(false); + closeConnection = true; + } + + string? tempTableName = null; + try + { + // Reject reference tables and shard-key updates, and confirm every mapped column exists. + await ValidateSchemaAsync(destinationTableName, ioBehavior, cancellationToken).ConfigureAwait(false); + + var updateColumns = GetUpdateColumns(); + Log.StartingBulkUpdate(m_logger, destinationTableName, string.Join(", ", KeyColumns), string.Join(", ", updateColumns), GetRowCount(source)); + + // Phase 1: create the staging table mirroring the destination column types. + tempTableName = await CreateStagingTableAsync(destinationTableName, ioBehavior, cancellationToken).ConfigureAwait(false); + + // Phase 2: stage the source rows into the temporary table via SingleStoreBulkCopy. + var rowsStaged = await StageDataAsync(tempTableName, source, ioBehavior, cancellationToken).ConfigureAwait(false); + + // Phase 3 (optional): count how many staged rows match a destination row. + var rowsMatched = await ComputeMatchedRowsAsync(tempTableName, ioBehavior, cancellationToken).ConfigureAwait(false); + if (rowsMatched is { } matched && rowsStaged > matched) + Log.LargeUnmatchedCountForBulkUpdate(m_logger, rowsStaged, matched, rowsStaged - matched); + + // Phase 4: run the UPDATE ... JOIN that copies the non-key values into the matching rows. + var rowsUpdated = await ExecuteUpdateAsync(tempTableName, ioBehavior, cancellationToken).ConfigureAwait(false); + + stopwatch.Stop(); + Log.CompletedBulkUpdate(m_logger, rowsStaged, rowsMatched ?? -1, rowsUpdated, stopwatch.ElapsedMilliseconds); + + // RowsMatched is reported as -1 when ComputeRowsMatched was false (the count was intentionally skipped). + return new SingleStoreBulkUpdateResult(m_warnings.AsReadOnly(), rowsStaged, rowsMatched ?? -1, rowsUpdated); + } + finally + { + // Drop the staging table before closing a connection we opened (a closed connection's session, and + // therefore the temporary table, is already gone). + await DropStagingTableAsync(tempTableName, ioBehavior).ConfigureAwait(false); + + if (closeConnection) + m_connection.Close(); + } } + /// + /// Returns the number of rows in the source for logging, or -1 when the count is not known in advance + /// (for example an , which is consumed as it is staged). + /// + private static int GetRowCount(object source) => + source switch + { + DataTable dataTable => dataTable.Rows.Count, + ICollection dataRows => dataRows.Count, + IReadOnlyCollection dataRows => dataRows.Count, + _ => -1, + }; + private void ValidateColumnMappings() { // Ensure the caller specified at least one key column. @@ -240,15 +314,15 @@ private void ValidateColumnMappings() throw new InvalidOperationException("ColumnMappings must contain at least one non-key column to update."); } - private async ValueTask ValidateSchemaAsync(string tableName, CancellationToken cancellationToken) + private async ValueTask ValidateSchemaAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken) { var schemaDetector = new SchemaDetector(m_connection); // TODO: make changes to support the solutions described here -- https://docs.singlestore.com/cloud/reference/troubleshooting-reference/query-errors/error-1706-hy-000-feature-multi-table-update-delete-with-a-reference-table-as-target-table-is-not-supported-by-memsql/ - if (await schemaDetector.IsReferenceTableAsync(tableName, cancellationToken).ConfigureAwait(false)) + if (await schemaDetector.IsReferenceTableAsync(tableName, ioBehavior, cancellationToken).ConfigureAwait(false)) throw new NotSupportedException($"Target table '{tableName}' is a reference table. Bulk updates on reference tables are not supported in this version."); - var shardKeyColumns = await schemaDetector.GetShardKeyColumnsAsync(tableName, cancellationToken).ConfigureAwait(false); + var shardKeyColumns = await schemaDetector.GetShardKeyColumnsAsync(tableName, ioBehavior, cancellationToken).ConfigureAwait(false); var updateColumns = GetUpdateColumns(); foreach (var updateColumn in updateColumns) @@ -257,7 +331,7 @@ private async ValueTask ValidateSchemaAsync(string tableName, CancellationToken throw new InvalidOperationException($"Column '{updateColumn}' is a shard key. SingleStore does not support updating shard key columns."); } - var schema = await schemaDetector.GetTableSchemaAsync(tableName, cancellationToken).ConfigureAwait(false); + var schema = await schemaDetector.GetTableSchemaAsync(tableName, ioBehavior, cancellationToken).ConfigureAwait(false); var tableColumns = new HashSet(StringComparer.OrdinalIgnoreCase); @@ -310,7 +384,7 @@ private List GetUpdateColumns() => /// because the temporary table is session-scoped. /// /// - private async Task CreateStagingTableAsync(string destinationTableName, CancellationToken cancellationToken) + private async Task CreateStagingTableAsync(string destinationTableName, IOBehavior ioBehavior, CancellationToken cancellationToken) { // Generate a unique temporary table name. The "g" suffix on the GUID guarantees the identifier // starts with a letter regardless of the GUID's first hex digit. @@ -320,8 +394,8 @@ private async Task CreateStagingTableAsync(string destinationTableName, // Pull the exact, server-rendered type definition for every column so the staging columns are // byte-for-byte type compatible with the destination (see remarks). - var columnTypeDefinitions = await schemaDetector.GetColumnTypeDefinitionsAsync(destinationTableName, cancellationToken).ConfigureAwait(false); - var shardKeyColumns = await schemaDetector.GetShardKeyColumnsAsync(destinationTableName, cancellationToken).ConfigureAwait(false); + var columnTypeDefinitions = await schemaDetector.GetColumnTypeDefinitionsAsync(destinationTableName, ioBehavior, cancellationToken).ConfigureAwait(false); + var shardKeyColumns = await schemaDetector.GetShardKeyColumnsAsync(destinationTableName, ioBehavior, cancellationToken).ConfigureAwait(false); // Emit a column definition for each mapped column, preserving the order in which the columns appear // in the destination table is unnecessary here: the staging table only needs the columns to exist by @@ -378,7 +452,7 @@ private async Task CreateStagingTableAsync(string destinationTableName, cmd.CommandText = createTableSql.ToString(); cmd.Transaction = m_transaction; cmd.CommandTimeout = BulkCopyTimeout; - await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(ioBehavior, cancellationToken).ConfigureAwait(false); } Log.CreatedStagingTableForBulkUpdate(m_logger, tempTableName, columnDefinitions.Count); @@ -435,7 +509,7 @@ private List ComputeStagingShardKey(List destinationShardKeyColu /// destination-name relationship identical between staging and the later UPDATE ... JOIN. /// /// - private async Task StageDataAsync(string tempTableName, object source, CancellationToken cancellationToken) + private async Task StageDataAsync(string tempTableName, object source, IOBehavior ioBehavior, CancellationToken cancellationToken) { var bulkCopy = new SingleStoreBulkCopy(m_connection, m_transaction) { @@ -464,13 +538,7 @@ void OnRowsCopied(object sender, SingleStoreRowsCopiedEventArgs e) try { - var result = source switch - { - DataTable dataTable => await bulkCopy.WriteToServerAsync(dataTable, cancellationToken).ConfigureAwait(false), - IEnumerable dataRows => await StageDataRowsAsync(bulkCopy, dataRows, cancellationToken).ConfigureAwait(false), - IDataReader dataReader => await bulkCopy.WriteToServerAsync(dataReader, cancellationToken).ConfigureAwait(false), - _ => throw new ArgumentException($"Unsupported source type '{source.GetType()}'.", nameof(source)), - }; + var result = await StageWithBulkCopyAsync(bulkCopy, source, ioBehavior, cancellationToken).ConfigureAwait(false); m_warnings.AddRange(result.Warnings); @@ -486,25 +554,46 @@ void OnRowsCopied(object sender, SingleStoreRowsCopiedEventArgs e) } /// - /// Stages a sequence of objects, supplying the column count that - /// requires up front. + /// Dispatches the source data to the appropriate overload, selecting the + /// synchronous or asynchronous method according to . /// /// - /// needs the - /// source column count before it enumerates the rows, which is taken from the owning - /// of the first row. The sequence is materialized first so it can be inspected for the first row and then - /// enumerated again by the bulk copy without re-running (and possibly exhausting) a lazy source. Empty input - /// is short-circuited before staging, so the sequence is expected to be non-empty here; it is still guarded so - /// an unexpected empty sequence fails clearly rather than dereferencing a missing row. + /// exposes separate synchronous (WriteToServer) and asynchronous + /// (WriteToServerAsync) methods rather than an overload, so the behavior is + /// selected here. Calling the synchronous methods on the synchronous path keeps the whole operation inline, + /// preserving the no-sync-over-async guarantee that lets the public synchronous overloads block safely. + /// For a sequence, needs the column count before it + /// enumerates the rows (taken from the owning of the first row); the sequence is + /// materialized first so a lazy source is not consumed by the peek and then re-enumerated (empty) by the bulk + /// copy. Empty input is short-circuited before staging, so the sequence is expected to be non-empty here; it + /// is still guarded so an unexpected empty sequence fails clearly rather than dereferencing a missing row. /// - private static ValueTask StageDataRowsAsync(SingleStoreBulkCopy bulkCopy, IEnumerable dataRows, CancellationToken cancellationToken) + private static ValueTask StageWithBulkCopyAsync(SingleStoreBulkCopy bulkCopy, object source, IOBehavior ioBehavior, CancellationToken cancellationToken) { - var rows = dataRows as IReadOnlyList ?? dataRows.ToList(); - if (rows.Count == 0) - throw new ArgumentException("Cannot stage an empty sequence of rows.", nameof(dataRows)); - - var columnCount = rows[0].Table.Columns.Count; - return bulkCopy.WriteToServerAsync(rows, columnCount, cancellationToken); + switch (source) + { + case DataTable dataTable: + return ioBehavior == IOBehavior.Synchronous + ? new ValueTask(bulkCopy.WriteToServer(dataTable)) + : bulkCopy.WriteToServerAsync(dataTable, cancellationToken); + + case IEnumerable dataRows: + var rows = dataRows as IReadOnlyList ?? dataRows.ToList(); + if (rows.Count == 0) + throw new ArgumentException("Cannot stage an empty sequence of rows.", nameof(source)); + var columnCount = rows[0].Table.Columns.Count; + return ioBehavior == IOBehavior.Synchronous + ? new ValueTask(bulkCopy.WriteToServer(rows, columnCount)) + : bulkCopy.WriteToServerAsync(rows, columnCount, cancellationToken); + + case IDataReader dataReader: + return ioBehavior == IOBehavior.Synchronous + ? new ValueTask(bulkCopy.WriteToServer(dataReader)) + : bulkCopy.WriteToServerAsync(dataReader, cancellationToken); + + default: + throw new ArgumentException($"Unsupported source type '{source.GetType()}'.", nameof(source)); + } } /// @@ -530,7 +619,7 @@ private static ValueTask StageDataRowsAsync(SingleSto /// operation because the staging table is session-scoped. /// /// - private async Task ComputeMatchedRowsAsync(string tempTableName, CancellationToken cancellationToken) + private async Task ComputeMatchedRowsAsync(string tempTableName, IOBehavior ioBehavior, CancellationToken cancellationToken) { if (!ComputeRowsMatched) return null; @@ -544,7 +633,7 @@ private static ValueTask StageDataRowsAsync(SingleSto cmd.Transaction = m_transaction; cmd.CommandTimeout = BulkCopyTimeout; - var scalar = await cmd.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + var scalar = await cmd.ExecuteScalarAsync(ioBehavior, cancellationToken).ConfigureAwait(false); // COUNT(*) comes back as a long; convert rather than cast so the boxed type is handled correctly. var rowsMatched = scalar is null or DBNull ? 0 : Convert.ToInt32(scalar, CultureInfo.InvariantCulture); @@ -581,7 +670,7 @@ private static ValueTask StageDataRowsAsync(SingleSto /// operation result. /// /// - private async Task ExecuteUpdateAsync(string tempTableName, CancellationToken cancellationToken) + private async Task ExecuteUpdateAsync(string tempTableName, IOBehavior ioBehavior, CancellationToken cancellationToken) { // Assign each non-key mapped column from the staging row: t.`c1` = s.`c1`, t.`c2` = s.`c2` ... var setClause = string.Join( @@ -604,7 +693,7 @@ private async Task ExecuteUpdateAsync(string tempTableName, CancellationTok m_connection.InfoMessage += OnInfoMessage; try { - var rowsUpdated = await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + var rowsUpdated = await cmd.ExecuteNonQueryAsync(ioBehavior, cancellationToken).ConfigureAwait(false); Log.ExecutedBulkUpdate(m_logger, rowsUpdated); @@ -634,7 +723,7 @@ private async Task ExecuteUpdateAsync(string tempTableName, CancellationTok /// is used deliberately so cleanup still runs after a cancelled or timed-out operation. /// /// - private async Task DropStagingTableAsync(string? tempTableName) + private async Task DropStagingTableAsync(string? tempTableName, IOBehavior ioBehavior) { if (string.IsNullOrEmpty(tempTableName)) return; @@ -650,7 +739,7 @@ private async Task DropStagingTableAsync(string? tempTableName) cmd.CommandText = $"DROP TEMPORARY TABLE IF EXISTS {IdentifierHelper.QuoteIdentifier(tempTableName!)}"; cmd.Transaction = m_transaction; cmd.CommandTimeout = BulkCopyTimeout; - await cmd.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(ioBehavior, CancellationToken.None).ConfigureAwait(false); } catch (Exception ex) { diff --git a/src/SingleStoreConnector/Utilities/SchemaDetector.cs b/src/SingleStoreConnector/Utilities/SchemaDetector.cs index 15d29b1ae..2bfa7ca14 100644 --- a/src/SingleStoreConnector/Utilities/SchemaDetector.cs +++ b/src/SingleStoreConnector/Utilities/SchemaDetector.cs @@ -1,6 +1,7 @@ using System.Globalization; using System.Text; using System.Text.RegularExpressions; +using SingleStoreConnector.Protocol.Serialization; namespace SingleStoreConnector.Utilities; @@ -20,11 +21,11 @@ internal sealed class SchemaDetector(SingleStoreConnection connection) /// /// Detects if the specified table is a reference table. /// - public async Task IsReferenceTableAsync(string tableName, CancellationToken cancellationToken = default) + public async Task IsReferenceTableAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken = default) { EnsureConnectionIsOpen(); - var createTableSql = await GetCreateTableStatementAsync(tableName, cancellationToken) + var createTableSql = await GetCreateTableStatementAsync(tableName, ioBehavior, cancellationToken) .ConfigureAwait(false); return referenceTableRegex.IsMatch(createTableSql); @@ -34,7 +35,7 @@ public async Task IsReferenceTableAsync(string tableName, CancellationToke /// Gets the shard key columns for the specified table. /// /// List of shard key column names, or empty list if no shard key. - public async Task> GetShardKeyColumnsAsync(string tableName, CancellationToken cancellationToken = default) + public async Task> GetShardKeyColumnsAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken = default) { EnsureConnectionIsOpen(); @@ -46,14 +47,14 @@ public async Task> GetShardKeyColumnsAsync(string tableName, Cancel { cmd.CommandText = $"SHOW INDEXES FROM {IdentifierHelper.QuoteQualifiedIdentifier(tableName)}"; - await using var reader = await cmd.ExecuteReaderAsync(cancellationToken) + await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.Default, ioBehavior, cancellationToken) .ConfigureAwait(false); var keyNameOrdinal = reader.GetOrdinal("Key_name"); var columnNameOrdinal = reader.GetOrdinal("Column_name"); var sequenceOrdinal = reader.GetOrdinal("Seq_in_index"); - while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + while (await reader.ReadAsync(ioBehavior, cancellationToken).ConfigureAwait(false)) { if (reader.IsDBNull(keyNameOrdinal)) continue; @@ -84,7 +85,7 @@ public async Task> GetShardKeyColumnsAsync(string tableName, Cancel // Method 2: Parse SHOW CREATE TABLE for SHARD KEY. // This is a fallback for cases where SHOW INDEXES doesn't expose __SHARDKEY. - var createTableSql = await GetCreateTableStatementAsync(tableName, cancellationToken) + var createTableSql = await GetCreateTableStatementAsync(tableName, ioBehavior, cancellationToken) .ConfigureAwait(false); var match = shardKeyRegex.Match(createTableSql); @@ -101,17 +102,17 @@ public async Task> GetShardKeyColumnsAsync(string tableName, Cancel /// /// Gets the CREATE TABLE statement for the specified table. /// - private async Task GetCreateTableStatementAsync(string tableName, CancellationToken cancellationToken) + private async Task GetCreateTableStatementAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken) { EnsureConnectionIsOpen(); using var cmd = m_connection.CreateCommand(); cmd.CommandText = $"SHOW CREATE TABLE {IdentifierHelper.QuoteQualifiedIdentifier(tableName)}"; - await using var reader = await cmd.ExecuteReaderAsync(cancellationToken) + await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.Default, ioBehavior, cancellationToken) .ConfigureAwait(false); - if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + if (await reader.ReadAsync(ioBehavior, cancellationToken).ConfigureAwait(false)) { // SHOW CREATE TABLE returns the CREATE TABLE statement in the second column. return reader.GetString(1); @@ -142,11 +143,11 @@ private async Task GetCreateTableStatementAsync(string tableName, Cancel /// /// /// A case-insensitive map of column name to its type definition. - public async Task> GetColumnTypeDefinitionsAsync(string tableName, CancellationToken cancellationToken = default) + public async Task> GetColumnTypeDefinitionsAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken = default) { EnsureConnectionIsOpen(); - var createTableSql = await GetCreateTableStatementAsync(tableName, cancellationToken) + var createTableSql = await GetCreateTableStatementAsync(tableName, ioBehavior, cancellationToken) .ConfigureAwait(false); var definitions = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -174,14 +175,14 @@ public async Task> GetColumnTypeDefinitionsAsync(stri /// /// Gets column metadata for the specified table. /// - public async Task GetTableSchemaAsync(string tableName, CancellationToken cancellationToken = default) + public async Task GetTableSchemaAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken = default) { EnsureConnectionIsOpen(); using var cmd = m_connection.CreateCommand(); cmd.CommandText = $"SELECT * FROM {IdentifierHelper.QuoteQualifiedIdentifier(tableName)} LIMIT 0"; - await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly, cancellationToken) + await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly, ioBehavior, cancellationToken) .ConfigureAwait(false); return reader.GetSchemaTable() From 67483a4327c237bc6b35b25261de07ebaf257b43 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Thu, 18 Jun 2026 18:04:14 +0300 Subject: [PATCH 14/28] add missing schemadetector tests --- tests/SideBySide/SchemaDetectorTests.cs | 269 ++++++++++++++++++++++-- 1 file changed, 255 insertions(+), 14 deletions(-) diff --git a/tests/SideBySide/SchemaDetectorTests.cs b/tests/SideBySide/SchemaDetectorTests.cs index a28821b57..fe5f78d1b 100644 --- a/tests/SideBySide/SchemaDetectorTests.cs +++ b/tests/SideBySide/SchemaDetectorTests.cs @@ -1,12 +1,17 @@ +using SingleStoreConnector.Protocol.Serialization; using SingleStoreConnector.Utilities; namespace SideBySide; public class SchemaDetectorTests(DatabaseFixture database) : IClassFixture { - [Fact] - public async Task IsReferenceTable_RegularTable_ReturnsFalse() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task IsReferenceTable_RegularTable_ReturnsFalse(bool isAsync) { + var ioBehavior = ToIOBehavior(isAsync); + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); await connection.OpenAsync(); @@ -26,7 +31,7 @@ id INT PRIMARY KEY } var detector = new SchemaDetector(connection); - var isReference = await detector.IsReferenceTableAsync(tableName); + var isReference = await detector.IsReferenceTableAsync(tableName, ioBehavior); Assert.False(isReference); } @@ -36,9 +41,13 @@ id INT PRIMARY KEY } } - [Fact] - public async Task IsReferenceTable_ReferenceTable_ReturnsTrue() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task IsReferenceTable_ReferenceTable_ReturnsTrue(bool isAsync) { + var ioBehavior = ToIOBehavior(isAsync); + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); await connection.OpenAsync(); @@ -58,7 +67,7 @@ id INT PRIMARY KEY } var detector = new SchemaDetector(connection); - var isReference = await detector.IsReferenceTableAsync(tableName); + var isReference = await detector.IsReferenceTableAsync(tableName, ioBehavior); Assert.True(isReference); } @@ -68,9 +77,13 @@ id INT PRIMARY KEY } } - [Fact] - public async Task GetShardKeyColumns_TableWithShardKey_ReturnsColumnsInOrder() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GetShardKeyColumns_TableWithShardKey_ReturnsColumnsInOrder(bool isAsync) { + var ioBehavior = ToIOBehavior(isAsync); + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); await connection.OpenAsync(); @@ -93,7 +106,7 @@ SHARD KEY (tenant_id, user_id) } var detector = new SchemaDetector(connection); - var shardKeys = await detector.GetShardKeyColumnsAsync(tableName); + var shardKeys = await detector.GetShardKeyColumnsAsync(tableName, ioBehavior); Assert.Equal(new[] { "tenant_id", "user_id" }, shardKeys); } @@ -103,9 +116,13 @@ SHARD KEY (tenant_id, user_id) } } - [Fact] - public async Task GetShardKeyColumns_TableWithoutShardKey_ReturnsEmpty() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GetShardKeyColumns_TableWithoutShardKey_ReturnsEmpty(bool isAsync) { + var ioBehavior = ToIOBehavior(isAsync); + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); await connection.OpenAsync(); @@ -125,7 +142,7 @@ id INT PRIMARY KEY } var detector = new SchemaDetector(connection); - var shardKeys = await detector.GetShardKeyColumnsAsync(tableName); + var shardKeys = await detector.GetShardKeyColumnsAsync(tableName, ioBehavior); Assert.Empty(shardKeys); } @@ -136,8 +153,46 @@ id INT PRIMARY KEY } [Fact] - public async Task GetTableSchema_ExistingTable_ReturnsSchema() + public async Task GetShardKeyColumns_SingleColumnShardKey_ReturnsColumn() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "test_single_shard"; + var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + + try + { + await using (var cmd = new SingleStoreCommand($@" + DROP TABLE IF EXISTS {quotedTableName}; + CREATE TABLE {quotedTableName} + ( + id INT, + name VARCHAR(100), + SHARD KEY (id) + );", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var detector = new SchemaDetector(connection); + var shardKeys = await detector.GetShardKeyColumnsAsync(tableName, IOBehavior.Asynchronous); + + Assert.Equal(new[] { "id" }, shardKeys); + } + finally + { + await DropTableIfExistsAsync(connection, tableName); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GetTableSchema_ExistingTable_ReturnsSchema(bool isAsync) { + var ioBehavior = ToIOBehavior(isAsync); + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); await connection.OpenAsync(); @@ -158,7 +213,7 @@ name VARCHAR(100) } var detector = new SchemaDetector(connection); - var schema = await detector.GetTableSchemaAsync(tableName); + var schema = await detector.GetTableSchemaAsync(tableName, ioBehavior); Assert.NotNull(schema); Assert.Contains(schema.Rows.Cast(), row => row["ColumnName"].ToString() == "id"); @@ -170,6 +225,192 @@ name VARCHAR(100) } } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GetColumnTypeDefinitions_SimpleColumns_ReturnsTypesForAllColumns(bool isAsync) + { + var ioBehavior = ToIOBehavior(isAsync); + + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "test_coldefs_simple"; + var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + + try + { + await using (var cmd = new SingleStoreCommand($@" + DROP TABLE IF EXISTS {quotedTableName}; + CREATE TABLE {quotedTableName} + ( + id INT PRIMARY KEY, + name VARCHAR(100) + );", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, ioBehavior); + + Assert.Equal(2, definitions.Count); + Assert.Contains("int", definitions["id"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("varchar(100)", definitions["name"], StringComparison.OrdinalIgnoreCase); + } + finally + { + await DropTableIfExistsAsync(connection, tableName); + } + } + + [Fact] + public async Task GetColumnTypeDefinitions_IsCaseInsensitiveOnColumnName() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "test_coldefs_case"; + var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + + try + { + await using (var cmd = new SingleStoreCommand($@" + DROP TABLE IF EXISTS {quotedTableName}; + CREATE TABLE {quotedTableName} + ( + MyId INT PRIMARY KEY, + MyValue VARCHAR(50) + );", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + // The map must be case-insensitive so callers can look up by the caller-supplied column name. + Assert.True(definitions.ContainsKey("myid")); + Assert.True(definitions.ContainsKey("MYVALUE")); + } + finally + { + await DropTableIfExistsAsync(connection, tableName); + } + } + + [Fact] + public async Task GetColumnTypeDefinitions_LossyTypes_PreservesExactServerDefinition() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "test_coldefs_lossy"; + var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + + try + { + // These types are exactly the ones GetSchemaTable() reports inaccurately (VARBINARY -> BLOB, + // BIT -> BIGINT, UNSIGNED dropped, DECIMAL precision/scale, ENUM members). The definition must + // come verbatim from SHOW CREATE TABLE so the staging column matches the destination. + await using (var cmd = new SingleStoreCommand($@" + DROP TABLE IF EXISTS {quotedTableName}; + CREATE TABLE {quotedTableName} + ( + id INT PRIMARY KEY, + payload VARBINARY(16), + flag BIT(1), + amount DECIMAL(18,4), + quantity INT UNSIGNED, + status ENUM('active','inactive') + );", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + Assert.Contains("varbinary(16)", definitions["payload"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("bit(1)", definitions["flag"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("decimal(18,4)", definitions["amount"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("unsigned", definitions["quantity"], StringComparison.OrdinalIgnoreCase); + + // The ENUM member list (including the comma between members) must survive top-level comma splitting. + Assert.Contains("'active'", definitions["status"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("'inactive'", definitions["status"], StringComparison.OrdinalIgnoreCase); + } + finally + { + await DropTableIfExistsAsync(connection, tableName); + } + } + + [Fact] + public async Task GetColumnTypeDefinitions_ExcludesColumnOptions() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "test_coldefs_options"; + var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + + try + { + await using (var cmd = new SingleStoreCommand($@" + DROP TABLE IF EXISTS {quotedTableName}; + CREATE TABLE {quotedTableName} + ( + id INT PRIMARY KEY, + note VARCHAR(50) NOT NULL DEFAULT 'none' + );", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + // Only the type portion is returned; column options must be stripped so the caller controls nullability. + var noteDefinition = definitions["note"]; + + Assert.Contains("varchar(50)", noteDefinition, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("NOT NULL", noteDefinition, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("DEFAULT", noteDefinition, StringComparison.OrdinalIgnoreCase); + } + finally + { + await DropTableIfExistsAsync(connection, tableName); + } + } + + [Fact] + public async Task IsReferenceTable_NonexistentTable_Throws() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var detector = new SchemaDetector(connection); + + await Assert.ThrowsAnyAsync( + async () => await detector.IsReferenceTableAsync("table_that_does_not_exist", IOBehavior.Asynchronous)); + } + + [Fact] + public async Task GetTableSchema_ConnectionNotOpen_Throws() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + + // Deliberately do not open the connection: schema detection requires an open session. + var detector = new SchemaDetector(connection); + + await Assert.ThrowsAsync( + async () => await detector.GetTableSchemaAsync("any_table", IOBehavior.Asynchronous)); + } + + private static IOBehavior ToIOBehavior(bool isAsync) => + isAsync ? IOBehavior.Asynchronous : IOBehavior.Synchronous; + private static async Task DropTableIfExistsAsync(SingleStoreConnection connection, string tableName) { var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); From 5a5172307a6ffe414fd2dee3d0b95ba744b2eab6 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Mon, 22 Jun 2026 13:28:04 +0300 Subject: [PATCH 15/28] small changes for better readability --- .../Utilities/IdentifierHelper.cs | 129 +++++++++--------- .../Utilities/SchemaDetector.cs | 7 +- 2 files changed, 66 insertions(+), 70 deletions(-) diff --git a/src/SingleStoreConnector/Utilities/IdentifierHelper.cs b/src/SingleStoreConnector/Utilities/IdentifierHelper.cs index 82fc79874..8866c9f63 100644 --- a/src/SingleStoreConnector/Utilities/IdentifierHelper.cs +++ b/src/SingleStoreConnector/Utilities/IdentifierHelper.cs @@ -8,20 +8,15 @@ namespace SingleStoreConnector.Utilities; internal static class IdentifierHelper { /// - /// Quotes a SQL identifier with backticks, escaping any backticks within the identifier. - /// - /// The identifier to quote. - /// The quoted identifier. - /// If identifier is null, empty, or contains null characters. + /// Quotes a SQL identifier with backticks, escaping any backticks within the identifier. + /// + /// The identifier to quote. + /// The quoted identifier. + /// If identifier is null, empty, or contains null characters. public static string QuoteIdentifier(string identifier) { - if (string.IsNullOrWhiteSpace(identifier)) - throw new ArgumentException("Identifier cannot be null or empty.", nameof(identifier)); - - if (identifier.Contains('\0')) - throw new ArgumentException("Identifier cannot contain null characters.", nameof(identifier)); + ValidateIdentifierInput(identifier, nameof(identifier), "Identifier"); - // Backticks inside the identifier must be doubled. return "`" + identifier.Replace("`", "``") + "`"; } @@ -30,23 +25,21 @@ public static string QuoteIdentifier(string identifier) /// public static string QuoteQualifiedIdentifier(string qualifiedName) { - if (string.IsNullOrWhiteSpace(qualifiedName)) - throw new ArgumentException("Qualified name cannot be null or empty.", nameof(qualifiedName)); - - if (qualifiedName.Contains('\0')) - throw new ArgumentException("Qualified name cannot contain null characters.", nameof(qualifiedName)); - - // Split on dots that are outside backtick-quoted identifier parts. - // This supports names such as: - // database.table - // `database`.`table` - // `database.with.dot`.`table` - // `database`.`table.with.dot` - var parts = SplitQualifiedIdentifier(qualifiedName); + ValidateIdentifierInput(qualifiedName, nameof(qualifiedName), "Qualified name"); + var parts = SplitQualifiedIdentifier(qualifiedName); return string.Join(".", parts.Select(QuoteIdentifier)); } + private static void ValidateIdentifierInput(string value, string paramName, string displayName) + { + if (string.IsNullOrWhiteSpace(value)) + throw new ArgumentException($"{displayName} cannot be null or empty.", paramName); + + if (value.Contains('\0')) + throw new ArgumentException($"{displayName} cannot contain null characters.", paramName); + } + private static List SplitQualifiedIdentifier(string qualifiedName) { var parts = new List(); @@ -57,24 +50,21 @@ private static List SplitQualifiedIdentifier(string qualifiedName) { var ch = qualifiedName[i]; - if (ch == '`') + if (ch == '.' && !inBackticks) { - if (inBackticks) - { - if (i + 1 < qualifiedName.Length && qualifiedName[i + 1] == '`') - { - // Escaped backtick inside a quoted identifier. - current.Append("``"); - i++; - } - else - { - // Closing backtick. - current.Append(ch); - inBackticks = false; - } - } - else if (IsOnlyWhitespace(current)) + AddPart(parts, current, qualifiedName); + continue; + } + + if (ch != '`') + { + current.Append(ch); + continue; + } + + if (!inBackticks) + { + if (IsOnlyWhitespace(current)) { // Opening backtick at the start of an identifier part. current.Append(ch); @@ -82,19 +72,24 @@ private static List SplitQualifiedIdentifier(string qualifiedName) } else { - // Treat backticks in unquoted input as literal identifier characters. - // They will be escaped later by QuoteIdentifier. + // Literal backtick in an unquoted identifier. current.Append(ch); } + + continue; } - else if (ch == '.' && !inBackticks) + + if (i + 1 < qualifiedName.Length && qualifiedName[i + 1] == '`') { - AddPart(parts, current, qualifiedName); - } - else - { - current.Append(ch); + // Escaped backtick inside a quoted identifier. + current.Append("``"); + i++; + continue; } + + // Closing backtick. + current.Append(ch); + inBackticks = false; } if (inBackticks) @@ -126,28 +121,32 @@ private static string UnquoteIdentifierPart(string part, string qualifiedName) { var ch = part[i]; - if (ch == '`') + if (ch != '`') { - if (i + 1 < part.Length && part[i + 1] == '`') - { - identifier.Append('`'); - i++; - } - else - { - if (i != part.Length - 1) - throw new ArgumentException("Qualified name contains unexpected characters after a quoted identifier.", nameof(qualifiedName)); - - return identifier.ToString(); - } + identifier.Append(ch); + continue; } - else + + // Escaped backtick: `` means one literal ` inside the identifier. + if (i + 1 < part.Length && part[i + 1] == '`') { - identifier.Append(ch); + identifier.Append('`'); + i++; + continue; } + + // Single backtick closes the quoted identifier. + if (i == part.Length - 1) + return identifier.ToString(); + + throw new ArgumentException( + "Qualified name contains unexpected characters after a quoted identifier.", + nameof(qualifiedName)); } - throw new ArgumentException("Qualified name contains an unterminated quoted identifier.", nameof(qualifiedName)); + throw new ArgumentException( + "Qualified name contains an unterminated quoted identifier.", + nameof(qualifiedName)); } private static bool IsOnlyWhitespace(StringBuilder builder) diff --git a/src/SingleStoreConnector/Utilities/SchemaDetector.cs b/src/SingleStoreConnector/Utilities/SchemaDetector.cs index 2bfa7ca14..c263c2c6c 100644 --- a/src/SingleStoreConnector/Utilities/SchemaDetector.cs +++ b/src/SingleStoreConnector/Utilities/SchemaDetector.cs @@ -128,12 +128,9 @@ private async Task GetCreateTableStatementAsync(string tableName, IOBeha /// /// /// The returned type definitions are taken verbatim from the server rather than reconstructed from - /// GetSchemaTable(). The schema table is lossy for several SingleStore types: - /// for example VARBINARY is reported as BLOB, BIT(1) as BIGINT, the - /// UNSIGNED flag is dropped, and character set, collation and ENUM/SET member - /// lists are not exposed at all. Copying the definition verbatim guarantees the staging column has the + /// GetSchemaTable(). Copying the definition verbatim guarantees the staging column has the /// exact same type as the destination column, which keeps key-column equality (including collation) - /// well defined in the UPDATE ... JOIN. + /// well-defined in the UPDATE ... JOIN. /// /// /// Only the type portion is returned (data type, any parenthesised arguments, UNSIGNED/ZEROFILL, From 4383f47b19440f2baac86aec0bab72fe9f5259c3 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 23 Jun 2026 17:43:03 +0300 Subject: [PATCH 16/28] updates to SchemaDetectorTests and IdentifierHelperTests --- tests/SideBySide/SchemaDetectorTests.cs | 442 ++++++------------ .../IdentifierHelperTests.cs | 22 +- 2 files changed, 163 insertions(+), 301 deletions(-) diff --git a/tests/SideBySide/SchemaDetectorTests.cs b/tests/SideBySide/SchemaDetectorTests.cs index fe5f78d1b..5ab1e7a9c 100644 --- a/tests/SideBySide/SchemaDetectorTests.cs +++ b/tests/SideBySide/SchemaDetectorTests.cs @@ -8,384 +8,240 @@ public class SchemaDetectorTests(DatabaseFixture database) : IClassFixture(), row => row["ColumnName"].ToString() == "id"); - Assert.Contains(schema.Rows.Cast(), row => row["ColumnName"].ToString() == "name"); - } - finally - { - await DropTableIfExistsAsync(connection, tableName); - } + var tableName = "schema_detector_schema"; + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, name VARCHAR(100)"); + + var detector = new SchemaDetector(connection); + var schema = await detector.GetTableSchemaAsync(tableName, ioBehavior); + + Assert.NotNull(schema); + Assert.Contains(schema.Rows.Cast(), row => row["ColumnName"].ToString() == "id"); + Assert.Contains(schema.Rows.Cast(), row => row["ColumnName"].ToString() == "name"); } [Theory] [InlineData(false)] [InlineData(true)] - public async Task GetColumnTypeDefinitions_SimpleColumns_ReturnsTypesForAllColumns(bool isAsync) + public async Task GetColumnTypeDefinitionsReturnsAllColumns(bool isAsync) { var ioBehavior = ToIOBehavior(isAsync); await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); await connection.OpenAsync(); - var tableName = "test_coldefs_simple"; - var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); - - try - { - await using (var cmd = new SingleStoreCommand($@" - DROP TABLE IF EXISTS {quotedTableName}; - CREATE TABLE {quotedTableName} - ( - id INT PRIMARY KEY, - name VARCHAR(100) - );", connection)) - { - await cmd.ExecuteNonQueryAsync(); - } - - var detector = new SchemaDetector(connection); - var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, ioBehavior); - - Assert.Equal(2, definitions.Count); - Assert.Contains("int", definitions["id"], StringComparison.OrdinalIgnoreCase); - Assert.Contains("varchar(100)", definitions["name"], StringComparison.OrdinalIgnoreCase); - } - finally - { - await DropTableIfExistsAsync(connection, tableName); - } + var tableName = "schema_detector_coldefs_simple"; + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, name VARCHAR(100)"); + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, ioBehavior); + + Assert.Equal(2, definitions.Count); + Assert.Contains("int", definitions["id"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("varchar(100)", definitions["name"], StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task GetColumnTypeDefinitionsIsCaseInsensitive() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "schema_detector_coldefs_case"; + await CreateTableAsync(connection, tableName, "MyId INT PRIMARY KEY, MyValue VARCHAR(50)"); + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + // The map must be case-insensitive so callers can look up by the caller-supplied column name. + Assert.True(definitions.ContainsKey("myid")); + Assert.True(definitions.ContainsKey("MYVALUE")); + } + + [Fact] + public async Task GetColumnTypeDefinitionsPreservesLossyTypes() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "schema_detector_coldefs_lossy"; + + // These types are exactly the ones GetSchemaTable() reports inaccurately (VARBINARY -> BLOB, + // BIT -> BIGINT, UNSIGNED dropped, DECIMAL precision/scale, ENUM members). The definition must + // come verbatim from SHOW CREATE TABLE so the staging column matches the destination. + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, payload VARBINARY(16), flag BIT(1), amount DECIMAL(18,4), quantity INT UNSIGNED, status ENUM('active','inactive')"); + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + Assert.Contains("varbinary(16)", definitions["payload"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("bit(1)", definitions["flag"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("decimal(18,4)", definitions["amount"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("unsigned", definitions["quantity"], StringComparison.OrdinalIgnoreCase); + + // The ENUM member list (including the comma between members) must survive top-level comma splitting. + Assert.Contains("'active'", definitions["status"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("'inactive'", definitions["status"], StringComparison.OrdinalIgnoreCase); } [Fact] - public async Task GetColumnTypeDefinitions_IsCaseInsensitiveOnColumnName() + public async Task GetColumnTypeDefinitionsPreservesCollation() { await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); await connection.OpenAsync(); - var tableName = "test_coldefs_case"; - var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); - - try - { - await using (var cmd = new SingleStoreCommand($@" - DROP TABLE IF EXISTS {quotedTableName}; - CREATE TABLE {quotedTableName} - ( - MyId INT PRIMARY KEY, - MyValue VARCHAR(50) - );", connection)) - { - await cmd.ExecuteNonQueryAsync(); - } - - var detector = new SchemaDetector(connection); - var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); - - // The map must be case-insensitive so callers can look up by the caller-supplied column name. - Assert.True(definitions.ContainsKey("myid")); - Assert.True(definitions.ContainsKey("MYVALUE")); - } - finally - { - await DropTableIfExistsAsync(connection, tableName); - } + var tableName = "schema_detector_coldefs_collation"; + + // Collation is not exposed by GetSchemaTable() but is part of the verbatim definition, and it determines + // how the key-column equality in the UPDATE ... JOIN compares values, so it must be preserved. + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, name VARCHAR(100) COLLATE utf8_bin"); + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + Assert.Contains("varchar(100)", definitions["name"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("collate", definitions["name"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("utf8_bin", definitions["name"], StringComparison.OrdinalIgnoreCase); } [Fact] - public async Task GetColumnTypeDefinitions_LossyTypes_PreservesExactServerDefinition() + public async Task GetColumnTypeDefinitionsUnescapesBacktickInColumnName() { await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); await connection.OpenAsync(); - var tableName = "test_coldefs_lossy"; - var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); - - try - { - // These types are exactly the ones GetSchemaTable() reports inaccurately (VARBINARY -> BLOB, - // BIT -> BIGINT, UNSIGNED dropped, DECIMAL precision/scale, ENUM members). The definition must - // come verbatim from SHOW CREATE TABLE so the staging column matches the destination. - await using (var cmd = new SingleStoreCommand($@" - DROP TABLE IF EXISTS {quotedTableName}; - CREATE TABLE {quotedTableName} - ( - id INT PRIMARY KEY, - payload VARBINARY(16), - flag BIT(1), - amount DECIMAL(18,4), - quantity INT UNSIGNED, - status ENUM('active','inactive') - );", connection)) - { - await cmd.ExecuteNonQueryAsync(); - } - - var detector = new SchemaDetector(connection); - var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); - - Assert.Contains("varbinary(16)", definitions["payload"], StringComparison.OrdinalIgnoreCase); - Assert.Contains("bit(1)", definitions["flag"], StringComparison.OrdinalIgnoreCase); - Assert.Contains("decimal(18,4)", definitions["amount"], StringComparison.OrdinalIgnoreCase); - Assert.Contains("unsigned", definitions["quantity"], StringComparison.OrdinalIgnoreCase); - - // The ENUM member list (including the comma between members) must survive top-level comma splitting. - Assert.Contains("'active'", definitions["status"], StringComparison.OrdinalIgnoreCase); - Assert.Contains("'inactive'", definitions["status"], StringComparison.OrdinalIgnoreCase); - } - finally - { - await DropTableIfExistsAsync(connection, tableName); - } + var tableName = "schema_detector_coldefs_backtick"; + + // A backtick in a column name is doubled in SHOW CREATE TABLE output; the parser must unescape it back + // to a single backtick so the dictionary key matches the real column name. + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, `odd``name` INT"); + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + Assert.True(definitions.ContainsKey("odd`name")); + Assert.Contains("int", definitions["odd`name"], StringComparison.OrdinalIgnoreCase); } [Fact] - public async Task GetColumnTypeDefinitions_ExcludesColumnOptions() + public async Task GetColumnTypeDefinitionsExcludesColumnOptions() { await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); await connection.OpenAsync(); - var tableName = "test_coldefs_options"; - var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); - - try - { - await using (var cmd = new SingleStoreCommand($@" - DROP TABLE IF EXISTS {quotedTableName}; - CREATE TABLE {quotedTableName} - ( - id INT PRIMARY KEY, - note VARCHAR(50) NOT NULL DEFAULT 'none' - );", connection)) - { - await cmd.ExecuteNonQueryAsync(); - } - - var detector = new SchemaDetector(connection); - var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); - - // Only the type portion is returned; column options must be stripped so the caller controls nullability. - var noteDefinition = definitions["note"]; - - Assert.Contains("varchar(50)", noteDefinition, StringComparison.OrdinalIgnoreCase); - Assert.DoesNotContain("NOT NULL", noteDefinition, StringComparison.OrdinalIgnoreCase); - Assert.DoesNotContain("DEFAULT", noteDefinition, StringComparison.OrdinalIgnoreCase); - } - finally - { - await DropTableIfExistsAsync(connection, tableName); - } + var tableName = "schema_detector_coldefs_options"; + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, note VARCHAR(50) NOT NULL DEFAULT 'none'"); + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + // Only the type portion is returned; column options must be stripped so the caller controls nullability. + var noteDefinition = definitions["note"]; + + Assert.Contains("varchar(50)", noteDefinition, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("NOT NULL", noteDefinition, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("DEFAULT", noteDefinition, StringComparison.OrdinalIgnoreCase); } [Fact] - public async Task IsReferenceTable_NonexistentTable_Throws() + public async Task IsReferenceTableThrowsForNonexistentTable() { await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); await connection.OpenAsync(); @@ -397,7 +253,7 @@ await Assert.ThrowsAnyAsync( } [Fact] - public async Task GetTableSchema_ConnectionNotOpen_Throws() + public async Task GetTableSchemaThrowsWhenConnectionNotOpen() { await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); @@ -411,11 +267,17 @@ await Assert.ThrowsAsync( private static IOBehavior ToIOBehavior(bool isAsync) => isAsync ? IOBehavior.Asynchronous : IOBehavior.Synchronous; - private static async Task DropTableIfExistsAsync(SingleStoreConnection connection, string tableName) + private static async Task CreateTableAsync(SingleStoreConnection connection, string tableName, string columnDefinitions) { - var quotedTableName = IdentifierHelper.QuoteIdentifier(tableName); + var quoted = IdentifierHelper.QuoteIdentifier(tableName); + await using var cmd = new SingleStoreCommand($"DROP TABLE IF EXISTS {quoted}; CREATE TABLE {quoted} ({columnDefinitions});", connection); + await cmd.ExecuteNonQueryAsync(); + } - await using var cmd = new SingleStoreCommand($"DROP TABLE IF EXISTS {quotedTableName}", connection); + private static async Task CreateReferenceTableAsync(SingleStoreConnection connection, string tableName, string columnDefinitions) + { + var quoted = IdentifierHelper.QuoteIdentifier(tableName); + await using var cmd = new SingleStoreCommand($"DROP TABLE IF EXISTS {quoted}; CREATE REFERENCE TABLE {quoted} ({columnDefinitions});", connection); await cmd.ExecuteNonQueryAsync(); } } diff --git a/tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs b/tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs index f563d287c..229bf863c 100644 --- a/tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs +++ b/tests/SingleStoreConnector.Tests/IdentifierHelperTests.cs @@ -14,7 +14,7 @@ public class IdentifierHelperTests [InlineData("my`table", "`my``table`")] // Backtick inside [InlineData("用户表", "`用户表`")] // Unicode [InlineData("таблиця", "`таблиця`")] // Unicode - public void QuoteIdentifier_ValidInput_ReturnsQuoted(string input, string expected) + public void QuoteIdentifierQuotesValidInput(string input, string expected) { var result = IdentifierHelper.QuoteIdentifier(input); @@ -25,11 +25,11 @@ public void QuoteIdentifier_ValidInput_ReturnsQuoted(string input, string expect [InlineData(null)] [InlineData("")] [InlineData(" ")] - public void QuoteIdentifier_NullOrEmpty_Throws(string input) + public void QuoteIdentifierThrowsForNullOrEmpty(string input) => Assert.Throws(() => IdentifierHelper.QuoteIdentifier(input)); [Fact] - public void QuoteIdentifier_NullChar_Throws() + public void QuoteIdentifierThrowsForNullChar() => Assert.Throws(() => IdentifierHelper.QuoteIdentifier("table\0name")); [Theory] @@ -40,7 +40,7 @@ public void QuoteIdentifier_NullChar_Throws() [InlineData("db.my table", "`db`.`my table`")] // Space [InlineData("db.用户表", "`db`.`用户表`")] // Unicode [InlineData("db.таблиця", "`db`.`таблиця`")] // Unicode - public void QuoteQualifiedIdentifier_ValidInput_ReturnsQuoted(string input, string expected) + public void QuoteQualifiedIdentifierQuotesValidInput(string input, string expected) { var result = IdentifierHelper.QuoteQualifiedIdentifier(input); @@ -55,7 +55,7 @@ public void QuoteQualifiedIdentifier_ValidInput_ReturnsQuoted(string input, stri [InlineData("`db.with.dot`.`table.with.dot`", "`db.with.dot`.`table.with.dot`")] [InlineData("db.`table.with.dot`", "`db`.`table.with.dot`")] [InlineData("`db.with.dot`.users", "`db.with.dot`.`users`")] - public void QuoteQualifiedIdentifier_AlreadyQuotedInput_ReturnsQuoted(string input, string expected) + public void QuoteQualifiedIdentifierQuotesAlreadyQuotedInput(string input, string expected) { var result = IdentifierHelper.QuoteQualifiedIdentifier(input); @@ -66,7 +66,7 @@ public void QuoteQualifiedIdentifier_AlreadyQuotedInput_ReturnsQuoted(string inp [InlineData("db.my`table", "`db`.`my``table`")] [InlineData("my`db.my`table", "`my``db`.`my``table`")] [InlineData("`my``db`.`my``table`", "`my``db`.`my``table`")] - public void QuoteQualifiedIdentifier_BackticksInsideIdentifiers_ReturnsQuoted(string input, string expected) + public void QuoteQualifiedIdentifierQuotesBackticksInsideIdentifiers(string input, string expected) { var result = IdentifierHelper.QuoteQualifiedIdentifier(input); @@ -77,11 +77,11 @@ public void QuoteQualifiedIdentifier_BackticksInsideIdentifiers_ReturnsQuoted(st [InlineData(null)] [InlineData("")] [InlineData(" ")] - public void QuoteQualifiedIdentifier_NullOrEmpty_Throws(string input) + public void QuoteQualifiedIdentifierThrowsForNullOrEmpty(string input) => Assert.Throws(() => IdentifierHelper.QuoteQualifiedIdentifier(input)); [Fact] - public void QuoteQualifiedIdentifier_NullChar_Throws() + public void QuoteQualifiedIdentifierThrowsForNullChar() => Assert.Throws(() => IdentifierHelper.QuoteQualifiedIdentifier("db.table\0name")); [Theory] @@ -89,19 +89,19 @@ public void QuoteQualifiedIdentifier_NullChar_Throws() [InlineData("db.")] [InlineData("db..users")] [InlineData("db. .users")] - public void QuoteQualifiedIdentifier_EmptyPart_Throws(string input) + public void QuoteQualifiedIdentifierThrowsForEmptyPart(string input) => Assert.Throws(() => IdentifierHelper.QuoteQualifiedIdentifier(input)); [Theory] [InlineData("`db.users")] [InlineData("db.`users")] [InlineData("`db`.`users")] - public void QuoteQualifiedIdentifier_UnterminatedQuotedIdentifier_Throws(string input) + public void QuoteQualifiedIdentifierThrowsForUnterminatedQuotedIdentifier(string input) => Assert.Throws(() => IdentifierHelper.QuoteQualifiedIdentifier(input)); [Theory] [InlineData("`db`extra.users")] [InlineData("db.`users`extra")] - public void QuoteQualifiedIdentifier_UnexpectedCharactersAfterQuotedIdentifier_Throws(string input) + public void QuoteQualifiedIdentifierThrowsForUnexpectedCharactersAfterQuotedIdentifier(string input) => Assert.Throws(() => IdentifierHelper.QuoteQualifiedIdentifier(input)); } From 74eaf40a8ba504d8ccc8485c87b090f1252cbdff Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Thu, 25 Jun 2026 16:39:17 +0300 Subject: [PATCH 17/28] add BulkUpdateTests --- .../SingleStoreBulkUpdate.cs | 4 +- .../Utilities/SchemaDetector.cs | 7 +- tests/SideBySide/BulkUpdateTests.cs | 466 ++++++++++++++++++ 3 files changed, 474 insertions(+), 3 deletions(-) create mode 100644 tests/SideBySide/BulkUpdateTests.cs diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index 5cda19599..c063892dc 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -316,7 +316,7 @@ private void ValidateColumnMappings() private async ValueTask ValidateSchemaAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken) { - var schemaDetector = new SchemaDetector(m_connection); + var schemaDetector = new SchemaDetector(m_connection, m_transaction); // TODO: make changes to support the solutions described here -- https://docs.singlestore.com/cloud/reference/troubleshooting-reference/query-errors/error-1706-hy-000-feature-multi-table-update-delete-with-a-reference-table-as-target-table-is-not-supported-by-memsql/ if (await schemaDetector.IsReferenceTableAsync(tableName, ioBehavior, cancellationToken).ConfigureAwait(false)) @@ -390,7 +390,7 @@ private async Task CreateStagingTableAsync(string destinationTableName, // starts with a letter regardless of the GUID's first hex digit. var tempTableName = $"_bulk_update_staging_g{Guid.NewGuid():N}"; - var schemaDetector = new SchemaDetector(m_connection); + var schemaDetector = new SchemaDetector(m_connection, m_transaction); // Pull the exact, server-rendered type definition for every column so the staging columns are // byte-for-byte type compatible with the destination (see remarks). diff --git a/src/SingleStoreConnector/Utilities/SchemaDetector.cs b/src/SingleStoreConnector/Utilities/SchemaDetector.cs index c263c2c6c..17f627cb3 100644 --- a/src/SingleStoreConnector/Utilities/SchemaDetector.cs +++ b/src/SingleStoreConnector/Utilities/SchemaDetector.cs @@ -5,7 +5,7 @@ namespace SingleStoreConnector.Utilities; -internal sealed class SchemaDetector(SingleStoreConnection connection) +internal sealed class SchemaDetector(SingleStoreConnection connection, SingleStoreTransaction? transaction = null) { private static readonly Regex referenceTableRegex = new(@"CREATE\s+(?:(?:ROWSTORE|COLUMNSTORE)\s+)?REFERENCE\s+TABLE", @@ -18,6 +18,8 @@ internal sealed class SchemaDetector(SingleStoreConnection connection) private readonly SingleStoreConnection m_connection = connection ?? throw new ArgumentNullException(nameof(connection)); + private readonly SingleStoreTransaction? m_transaction = transaction; + /// /// Detects if the specified table is a reference table. /// @@ -45,6 +47,7 @@ public async Task> GetShardKeyColumnsAsync(string tableName, IOBeha // This is preferred because Seq_in_index preserves composite shard key column order. using (var cmd = m_connection.CreateCommand()) { + cmd.Transaction = m_transaction; cmd.CommandText = $"SHOW INDEXES FROM {IdentifierHelper.QuoteQualifiedIdentifier(tableName)}"; await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.Default, ioBehavior, cancellationToken) @@ -107,6 +110,7 @@ private async Task GetCreateTableStatementAsync(string tableName, IOBeha EnsureConnectionIsOpen(); using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_transaction; cmd.CommandText = $"SHOW CREATE TABLE {IdentifierHelper.QuoteQualifiedIdentifier(tableName)}"; await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.Default, ioBehavior, cancellationToken) @@ -177,6 +181,7 @@ public async Task GetTableSchemaAsync(string tableName, IOBehavior io EnsureConnectionIsOpen(); using var cmd = m_connection.CreateCommand(); + cmd.Transaction = m_transaction; cmd.CommandText = $"SELECT * FROM {IdentifierHelper.QuoteQualifiedIdentifier(tableName)} LIMIT 0"; await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly, ioBehavior, cancellationToken) diff --git a/tests/SideBySide/BulkUpdateTests.cs b/tests/SideBySide/BulkUpdateTests.cs new file mode 100644 index 000000000..48b5a56d6 --- /dev/null +++ b/tests/SideBySide/BulkUpdateTests.cs @@ -0,0 +1,466 @@ +namespace SideBySide; + +public class BulkUpdateTests(DatabaseFixture database) : IClassFixture +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UpdatesMatchingRowsWithSingleKey(bool isAsync) + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_basic; +create table bulk_update_basic(id int primary key, name varchar(100), status varchar(50)); +insert into bulk_update_basic values (1, 'Alice', 'active'), (2, 'Bob', 'active'), (3, 'Charlie', 'inactive');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("status", typeof(string)), + }, + Rows = + { + new object[] { 1, "inactive" }, + new object[] { 2, "inactive" }, + }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_basic", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "status"), + }, + }; + + var result = await WriteToServerAsync(bulkUpdate, dataTable, isAsync); + + Assert.Equal(2, result.RowsStaged); + Assert.Equal(2, result.RowsMatched); + Assert.Equal(2, result.RowsUpdated); + + using var selectCommand = new SingleStoreCommand("select status from bulk_update_basic order by id;", connection); + using var reader = await selectCommand.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + Assert.Equal("inactive", reader.GetString(0)); + Assert.True(await reader.ReadAsync()); + Assert.Equal("inactive", reader.GetString(0)); + Assert.True(await reader.ReadAsync()); + Assert.Equal("inactive", reader.GetString(0)); // unchanged (was already inactive) + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UpdatesMatchingRowsWithCompositeKey(bool isAsync) + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_composite; +create table bulk_update_composite(tenant_id int, user_id int, email varchar(100), primary key (tenant_id, user_id)); +insert into bulk_update_composite values (1, 100, 'user100@tenant1.com'), (1, 101, 'user101@tenant1.com'), (2, 100, 'user100@tenant2.com');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("tenant_id", typeof(int)), + new DataColumn("user_id", typeof(int)), + new DataColumn("email", typeof(string)), + }, + Rows = + { + new object[] { 1, 100, "new100@tenant1.com" }, + new object[] { 2, 100, "new100@tenant2.com" }, + }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_composite", + KeyColumns = { "tenant_id", "user_id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "tenant_id"), + new SingleStoreBulkCopyColumnMapping(1, "user_id"), + new SingleStoreBulkCopyColumnMapping(2, "email"), + }, + }; + + var result = await WriteToServerAsync(bulkUpdate, dataTable, isAsync); + + Assert.Equal(2, result.RowsStaged); + Assert.Equal(2, result.RowsMatched); + Assert.Equal(2, result.RowsUpdated); + + using var selectCommand = new SingleStoreCommand("select email from bulk_update_composite where tenant_id = 1 and user_id = 100;", connection); + Assert.Equal("new100@tenant1.com", await selectCommand.ExecuteScalarAsync()); + + // The (1, 101) row shares tenant_id with an updated row but was not in the source: it must be untouched. + using var untouchedCommand = new SingleStoreCommand("select email from bulk_update_composite where tenant_id = 1 and user_id = 101;", connection); + Assert.Equal("user101@tenant1.com", await untouchedCommand.ExecuteScalarAsync()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UpdatesNothingWhenNoKeysMatch(bool isAsync) + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_nomatch; +create table bulk_update_nomatch(id int primary key, value varchar(100)); +insert into bulk_update_nomatch values (1, 'original');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = + { + new object[] { 999, "new" }, + }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_nomatch", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await WriteToServerAsync(bulkUpdate, dataTable, isAsync); + + Assert.Equal(1, result.RowsStaged); + Assert.Equal(0, result.RowsMatched); + Assert.Equal(0, result.RowsUpdated); + + using var selectCommand = new SingleStoreCommand("select value from bulk_update_nomatch where id = 1;", connection); + Assert.Equal("original", await selectCommand.ExecuteScalarAsync()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ThrowsOnDuplicateSourceKeys(bool isAsync) + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_dup; +create table bulk_update_dup(id int primary key, value varchar(100)); +insert into bulk_update_dup values (1, 'original');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = + { + new object[] { 1, "value1" }, + new object[] { 1, "value2" }, // duplicate key violates the staging table's primary key + }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_dup", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await WriteToServerAsync(bulkUpdate, dataTable, isAsync)); + Assert.Equal(SingleStoreErrorCode.DuplicateKeyEntry, exception.ErrorCode); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RollbackLeavesDataUnchanged(bool isAsync) + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_txn; +create table bulk_update_txn(id int primary key, value varchar(100)); +insert into bulk_update_txn values (1, 'original');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = + { + new object[] { 1, "updated" }, + }, + }; + + using (var transaction = await connection.BeginTransactionAsync()) + { + var bulkUpdate = new SingleStoreBulkUpdate(connection, transaction) + { + DestinationTableName = "bulk_update_txn", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await WriteToServerAsync(bulkUpdate, dataTable, isAsync); + Assert.Equal(1, result.RowsUpdated); + + await transaction.RollbackAsync(); + } + + using var selectCommand = new SingleStoreCommand("select value from bulk_update_txn where id = 1;", connection); + Assert.Equal("original", await selectCommand.ExecuteScalarAsync()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task EmptyInputReturnsZeroCounts(bool isAsync) + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_empty; +create table bulk_update_empty(id int primary key, value varchar(100));", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_empty", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await WriteToServerAsync(bulkUpdate, dataTable, isAsync); + + Assert.Equal(0, result.RowsStaged); + Assert.Equal(0, result.RowsMatched); + Assert.Equal(0, result.RowsUpdated); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CommitPersistsChanges(bool isAsync) + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_commit; +create table bulk_update_commit(id int primary key, value varchar(100)); +insert into bulk_update_commit values (1, 'original');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = + { + new object[] { 1, "updated" }, + }, + }; + + using (var transaction = await connection.BeginTransactionAsync()) + { + var bulkUpdate = new SingleStoreBulkUpdate(connection, transaction) + { + DestinationTableName = "bulk_update_commit", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await WriteToServerAsync(bulkUpdate, dataTable, isAsync); + Assert.Equal(1, result.RowsUpdated); + + await transaction.CommitAsync(); + } + + using var selectCommand = new SingleStoreCommand("select value from bulk_update_commit where id = 1;", connection); + Assert.Equal("updated", await selectCommand.ExecuteScalarAsync()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task OpensAndClosesConnectionWhenInitiallyClosed(bool isAsync) + { + using (var setupConnection = new SingleStoreConnection(GetLocalConnectionString(database))) + { + await setupConnection.OpenAsync(); + using var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_autoopen; +create table bulk_update_autoopen(id int primary key, value varchar(100)); +insert into bulk_update_autoopen values (1, 'original');", setupConnection); + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = + { + new object[] { 1, "updated" }, + }, + }; + + // Deliberately leave the connection closed: the bulk update must open it and close it again afterward. + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_autoopen", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + Assert.Equal(ConnectionState.Closed, connection.State); + var result = await WriteToServerAsync(bulkUpdate, dataTable, isAsync); + Assert.Equal(1, result.RowsUpdated); + Assert.Equal(ConnectionState.Closed, connection.State); + + await connection.OpenAsync(); + using var selectCommand = new SingleStoreCommand("select value from bulk_update_autoopen where id = 1;", connection); + Assert.Equal("updated", await selectCommand.ExecuteScalarAsync()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UpdatesFromDataRowSequence(bool isAsync) + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_datarows; +create table bulk_update_datarows(id int primary key, value varchar(100)); +insert into bulk_update_datarows values (1, 'old1'), (2, 'old2'), (3, 'old3');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = + { + new object[] { 1, "new1" }, + new object[] { 2, "new2" }, + new object[] { 3, "new3" }, + }, + }; + + // Select a subset of rows so the source is a DataRow sequence rather than a DataTable. + var dataRows = dataTable.Select("id <> 2"); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_datarows", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = isAsync ? await bulkUpdate.WriteToServerAsync(dataRows) : bulkUpdate.WriteToServer(dataRows); + + Assert.Equal(2, result.RowsStaged); + Assert.Equal(2, result.RowsMatched); + Assert.Equal(2, result.RowsUpdated); + + using var selectCommand = new SingleStoreCommand("select value from bulk_update_datarows order by id;", connection); + using var reader = await selectCommand.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + Assert.Equal("new1", reader.GetString(0)); + Assert.True(await reader.ReadAsync()); + Assert.Equal("old2", reader.GetString(0)); // id = 2 was excluded from the source + Assert.True(await reader.ReadAsync()); + Assert.Equal("new3", reader.GetString(0)); + } + + private static async ValueTask WriteToServerAsync(SingleStoreBulkUpdate bulkUpdate, DataTable dataTable, bool isAsync) => + isAsync ? await bulkUpdate.WriteToServerAsync(dataTable) : bulkUpdate.WriteToServer(dataTable); + + internal static string GetLocalConnectionString(DatabaseFixture database) + { + var csb = new SingleStoreConnectionStringBuilder(database.Connection.ConnectionString) + { + AllowLoadLocalInfile = true, + }; + return csb.ConnectionString; + } +} From 5c06e71a85062637c496ddd5e82a1dd692c78c94 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Thu, 25 Jun 2026 17:09:17 +0300 Subject: [PATCH 18/28] add BulkUpdateAdvanceTests --- tests/SideBySide/BulkUpdateAdvancedTests.cs | 384 ++++++++++++++++++++ 1 file changed, 384 insertions(+) create mode 100644 tests/SideBySide/BulkUpdateAdvancedTests.cs diff --git a/tests/SideBySide/BulkUpdateAdvancedTests.cs b/tests/SideBySide/BulkUpdateAdvancedTests.cs new file mode 100644 index 000000000..5a4396e19 --- /dev/null +++ b/tests/SideBySide/BulkUpdateAdvancedTests.cs @@ -0,0 +1,384 @@ +namespace SideBySide; + +public class BulkUpdateAdvancedTests(DatabaseFixture database) : IClassFixture +{ + [Fact] + public async Task FiresProgressNotifications() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand($@"drop table if exists bulk_update_progress; +create table bulk_update_progress(id int primary key, value int); +insert into bulk_update_progress values {SequentialRows(100)};", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(int)), + }, + }; + for (var i = 1; i <= 100; i++) + dataTable.Rows.Add(i, i * 2); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_progress", + NotifyAfter = 25, + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var eventCount = 0; + bulkUpdate.SingleStoreRowsStaged += (sender, e) => + { + eventCount++; + Assert.True(e.RowsStaged > 0); + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(100, result.RowsStaged); + Assert.True(eventCount > 0, "expected at least one progress event for NotifyAfter=25 over 100 rows"); + } + + [Fact] + public async Task AbortStopsStaging() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand($@"drop table if exists bulk_update_abort; +create table bulk_update_abort(id int primary key, value int); +insert into bulk_update_abort values {SequentialRows(100)};", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(int)), + }, + }; + for (var i = 1; i <= 100; i++) + dataTable.Rows.Add(i, i * 2); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_abort", + NotifyAfter = 25, + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + // Abort on the first progress notification; staging must stop before all rows are sent. + bulkUpdate.SingleStoreRowsStaged += (sender, e) => e.Abort = true; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.True(result.RowsStaged < 100, $"expected staging to stop early after abort, but staged {result.RowsStaged} rows"); + } + + [Fact] + public async Task DoesNotFireProgressNotificationsWhenNotifyAfterIsZero() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand($@"drop table if exists bulk_update_no_notify; +create table bulk_update_no_notify(id int primary key, value int); +insert into bulk_update_no_notify values {SequentialRows(100)};", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(int)), + }, + }; + for (var i = 1; i <= 100; i++) + dataTable.Rows.Add(i, i * 2); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_no_notify", + NotifyAfter = 0, // notifications disabled + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var eventCount = 0; + bulkUpdate.SingleStoreRowsStaged += (sender, e) => eventCount++; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(100, result.RowsStaged); + Assert.Equal(0, eventCount); + } + + [Fact] + public async Task SkipsMatchCountWhenComputeRowsMatchedIsFalse() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_nocount; +create table bulk_update_nocount(id int primary key, value varchar(100)); +insert into bulk_update_nocount values (1, 'original');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = { new object[] { 1, "updated" } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_nocount", + ComputeRowsMatched = false, + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(1, result.RowsStaged); + Assert.Equal(-1, result.RowsMatched); // -1 signals the COUNT was intentionally skipped + Assert.Equal(1, result.RowsUpdated); + + using var selectCommand = new SingleStoreCommand("select value from bulk_update_nocount where id = 1;", connection); + Assert.Equal("updated", await selectCommand.ExecuteScalarAsync()); + } + + [Fact] + public async Task HandlesSpecialCharactersInIdentifiers() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists `my-special-table`; +create table `my-special-table`(`user-id` int primary key, `user name` varchar(100), `select` varchar(50)); +insert into `my-special-table` values (1, 'Alice', 'value1');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("user-id", typeof(int)), + new DataColumn("user name", typeof(string)), + new DataColumn("select", typeof(string)), + }, + Rows = { new object[] { 1, "Alice Updated", "value2" } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "my-special-table", + KeyColumns = { "user-id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "user-id"), + new SingleStoreBulkCopyColumnMapping(1, "user name"), + new SingleStoreBulkCopyColumnMapping(2, "select"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(1, result.RowsStaged); + Assert.Equal(1, result.RowsMatched); + Assert.Equal(1, result.RowsUpdated); + + using var selectCommand = new SingleStoreCommand("select `user name`, `select` from `my-special-table` where `user-id` = 1;", connection); + using var reader = await selectCommand.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + Assert.Equal("Alice Updated", reader.GetString(0)); + Assert.Equal("value2", reader.GetString(1)); + } + + [Fact] + public async Task RoundTripsLossyColumnTypes() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + + // These destination types are exactly the ones GetSchemaTable() reports inaccurately. The staging table + // must mirror them verbatim from SHOW CREATE TABLE, so the values round-trip without conversion errors. + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_lossy; +create table bulk_update_lossy(id int primary key, amount decimal(18,4), quantity int unsigned, status enum('active','inactive')); +insert into bulk_update_lossy values (1, 0.0000, 0, 'active');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("amount", typeof(decimal)), + new DataColumn("quantity", typeof(long)), + new DataColumn("status", typeof(string)), + }, + Rows = { new object[] { 1, 1234.5678m, 4000000000L, "inactive" } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_lossy", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "amount"), + new SingleStoreBulkCopyColumnMapping(2, "quantity"), + new SingleStoreBulkCopyColumnMapping(3, "status"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(1, result.RowsUpdated); + Assert.Empty(result.Warnings); + + using var selectCommand = new SingleStoreCommand("select amount, quantity, status from bulk_update_lossy where id = 1;", connection); + using var reader = await selectCommand.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + Assert.Equal(1234.5678m, reader.GetDecimal(0)); + Assert.Equal(4000000000L, Convert.ToInt64(reader.GetValue(1))); + Assert.Equal("inactive", reader.GetString(2)); + } + + [Fact] + public async Task UpdatesRowsFromDataReader() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_reader_dest; +drop table if exists bulk_update_reader_src; +create table bulk_update_reader_dest(id int primary key, value varchar(100)); +insert into bulk_update_reader_dest values (1, 'old1'), (2, 'old2'); +create table bulk_update_reader_src(id int primary key, value varchar(100)); +insert into bulk_update_reader_src values (1, 'new1'), (2, 'new2');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + // Read source rows on a second connection so the reader is independent of the update connection. + using var readerConnection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await readerConnection.OpenAsync(); + using var selectCommand = new SingleStoreCommand("select id, value from bulk_update_reader_src order by id;", readerConnection); + using var reader = await selectCommand.ExecuteReaderAsync(); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_reader_dest", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(reader); + + Assert.Equal(2, result.RowsStaged); + Assert.Equal(2, result.RowsMatched); + Assert.Equal(2, result.RowsUpdated); + + using var verifyCommand = new SingleStoreCommand("select value from bulk_update_reader_dest where id = 1;", connection); + Assert.Equal("new1", await verifyCommand.ExecuteScalarAsync()); + } + + [Fact] + public async Task UpdatesLargeDataset() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand($@"drop table if exists bulk_update_large; +create table bulk_update_large(id int primary key, value int); +insert into bulk_update_large values {SequentialRows(5000)};", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(int)), + }, + }; + for (var i = 1; i <= 5000; i++) + dataTable.Rows.Add(i, i * 2); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_large", + BulkCopyTimeout = 60, + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(5000, result.RowsStaged); + Assert.Equal(5000, result.RowsMatched); + + using var selectCommand = new SingleStoreCommand("select value from bulk_update_large where id = 2500;", connection); + Assert.Equal(5000, Convert.ToInt32(await selectCommand.ExecuteScalarAsync())); + } + + // Builds a "(1,0),(2,0),...,(count,0)" VALUES list to seed a table with sequential ids. + private static string SequentialRows(int count) + { + var builder = new StringBuilder(); + for (var i = 1; i <= count; i++) + { + if (builder.Length != 0) + builder.Append(','); + builder.Append('(').Append(i).Append(",0)"); + } + + return builder.ToString(); + } +} From bb2df239b5726f4726f877c17f3998088b22c63b Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Fri, 26 Jun 2026 00:19:04 +0300 Subject: [PATCH 19/28] add Validation tests --- tests/SideBySide/BulkUpdateAdvancedTests.cs | 154 +++++++++ tests/SideBySide/BulkUpdateValidationTests.cs | 313 ++++++++++++++++++ 2 files changed, 467 insertions(+) create mode 100644 tests/SideBySide/BulkUpdateValidationTests.cs diff --git a/tests/SideBySide/BulkUpdateAdvancedTests.cs b/tests/SideBySide/BulkUpdateAdvancedTests.cs index 5a4396e19..10b828b1c 100644 --- a/tests/SideBySide/BulkUpdateAdvancedTests.cs +++ b/tests/SideBySide/BulkUpdateAdvancedTests.cs @@ -324,6 +324,160 @@ public async Task UpdatesRowsFromDataReader() Assert.Equal("new1", await verifyCommand.ExecuteScalarAsync()); } + [Fact] + public async Task UpdatesWhenShardKeyAlignsWithKeyColumns() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + + // The destination shard key (id) is a subset of the key columns, so the staging table is sharded the same + // way and the join can run locally. The update must succeed. + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_shard_aligned; +create table bulk_update_shard_aligned(id int, value varchar(100), primary key (id), shard key (id)); +insert into bulk_update_shard_aligned values (1, 'old1'), (2, 'old2');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = + { + new object[] { 1, "new1" }, + new object[] { 2, "new2" }, + }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_shard_aligned", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(2, result.RowsMatched); + Assert.Equal(2, result.RowsUpdated); + + using var selectCommand = new SingleStoreCommand("select value from bulk_update_shard_aligned where id = 1;", connection); + Assert.Equal("new1", await selectCommand.ExecuteScalarAsync()); + } + + [Fact] + public async Task UpdatesWhenCompositeShardKeyOrderDiffersFromKeyColumns() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + + // The destination shard key is (tenant_id, user_id) but the key columns are declared in the opposite order + // (user_id, then tenant_id). ComputeStagingShardKey returns the destination's shard-key columns verbatim, so + // the staging SHARD KEY keeps the destination order (tenant_id, user_id) while the staging PRIMARY KEY uses + // KeyColumns order (user_id, tenant_id). SingleStore only requires the primary key to *contain* every shard + // key column (a set rule, not an ordering/prefix rule), so CREATE TEMPORARY TABLE is valid and the update + // succeeds. Preserving the destination shard-key order also keeps the join co-located. + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_shard_composite; +create table bulk_update_shard_composite(tenant_id int, user_id int, value varchar(100), primary key (tenant_id, user_id), shard key (tenant_id, user_id)); +insert into bulk_update_shard_composite values (1, 100, 'old1'), (2, 200, 'old2');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("user_id", typeof(int)), + new DataColumn("tenant_id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = + { + new object[] { 100, 1, "new1" }, + new object[] { 200, 2, "new2" }, + }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_shard_composite", + KeyColumns = { "user_id", "tenant_id" }, // deliberately the opposite order from the destination shard key + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "user_id"), + new SingleStoreBulkCopyColumnMapping(1, "tenant_id"), + new SingleStoreBulkCopyColumnMapping(2, "value"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(2, result.RowsMatched); + Assert.Equal(2, result.RowsUpdated); + + using var selectCommand = new SingleStoreCommand("select value from bulk_update_shard_composite where tenant_id = 1 and user_id = 100;", connection); + Assert.Equal("new1", await selectCommand.ExecuteScalarAsync()); + } + + [Fact] + public async Task UpdatesWhenShardKeyDoesNotAlignWithKeyColumns() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + + // The destination shard key (region) is not among the key columns, so the staging table cannot be aligned + // and falls back to primary-key distribution (logging a mismatch warning). region is left unmapped so it is + // not treated as a shard-key update. The update must still succeed. + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_shard_mismatch; +create table bulk_update_shard_mismatch(id int, region int, value varchar(100), primary key (id, region), shard key (region)); +insert into bulk_update_shard_mismatch values (1, 10, 'old1'), (2, 20, 'old2');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = + { + new object[] { 1, "new1" }, + new object[] { 2, "new2" }, + }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_shard_mismatch", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(2, result.RowsMatched); + Assert.Equal(2, result.RowsUpdated); + + using var selectCommand = new SingleStoreCommand("select value from bulk_update_shard_mismatch where id = 1;", connection); + Assert.Equal("new1", await selectCommand.ExecuteScalarAsync()); + } + [Fact] public async Task UpdatesLargeDataset() { diff --git a/tests/SideBySide/BulkUpdateValidationTests.cs b/tests/SideBySide/BulkUpdateValidationTests.cs new file mode 100644 index 000000000..81dd31f7f --- /dev/null +++ b/tests/SideBySide/BulkUpdateValidationTests.cs @@ -0,0 +1,313 @@ +namespace SideBySide; + +public class BulkUpdateValidationTests(DatabaseFixture database) : IClassFixture +{ + [Fact] + public async Task ThrowsWhenDestinationTableNameNotSet() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(NewTable("id", "value"))); + Assert.Contains("DestinationTableName", exception.Message); + } + + [Fact] + public async Task ThrowsWhenKeyColumnsEmpty() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "any_table", + ColumnMappings = { new SingleStoreBulkCopyColumnMapping(0, "id") }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(NewTable("id"))); + Assert.Contains("KeyColumns", exception.Message); + } + + [Fact] + public async Task ThrowsWhenColumnMappingsEmpty() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "any_table", + KeyColumns = { "id" }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(NewTable("id"))); + Assert.Contains("ColumnMappings", exception.Message); + } + + [Fact] + public async Task ThrowsWhenKeyColumnNotMapped() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "any_table", + KeyColumns = { "id", "tenant_id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(NewTable("id", "value"))); + Assert.Contains("tenant_id", exception.Message); + } + + [Fact] + public async Task ThrowsWhenNoUpdateColumns() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "any_table", + KeyColumns = { "id" }, + ColumnMappings = { new SingleStoreBulkCopyColumnMapping(0, "id") }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(NewTable("id"))); + Assert.Contains("non-key column", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ThrowsWhenDuplicateKeyColumn() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "any_table", + KeyColumns = { "id", "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(NewTable("id", "value"))); + Assert.Contains("duplicate", exception.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("id", exception.Message); + } + + [Fact] + public async Task ThrowsWhenKeyColumnNameIsEmpty() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "any_table", + KeyColumns = { " " }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(NewTable("id", "value"))); + Assert.Contains("KeyColumns", exception.Message); + } + + [Fact] + public async Task ThrowsWhenDestinationColumnIsEmpty() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "any_table", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, ""), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(NewTable("id", "value"))); + Assert.Contains("DestinationColumn", exception.Message); + } + + [Fact] + public async Task ThrowsWhenMappedColumnDoesNotExistInTargetTable() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_missing_column; +create table bulk_update_missing_column(id int primary key, value varchar(100));", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("missing", typeof(string)), + }, + Rows = { new object[] { 1, "test" } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_missing_column", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "missing"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(dataTable)); + Assert.Contains("missing", exception.Message); + Assert.Contains("does not exist", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ThrowsOnExpressionMapping() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "any_table", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value", "UNHEX(@value)"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(NewTable("id", "value"))); + Assert.Contains("Expression", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ThrowsOnDuplicateDestinationColumn() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "any_table", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + new SingleStoreBulkCopyColumnMapping(2, "value"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(NewTable("id", "value", "value2"))); + Assert.Contains("duplicate", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ThrowsForReferenceTable() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_reference; +create reference table bulk_update_reference(id int primary key, value varchar(100));", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = { new object[] { 1, "test" } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_reference", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(dataTable)); + Assert.Contains("reference table", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task ThrowsWhenUpdatingShardKeyColumn() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + + // Shard on tenant_id, then attempt to update tenant_id (a shard key) as a non-key column. SingleStore does + // not allow updating shard key columns, so this must be rejected. tenant_id is part of the primary key + // because SingleStore requires the primary key to contain all shard key columns. + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_shardkey; +create table bulk_update_shardkey(id int, tenant_id int, value varchar(100), primary key (id, tenant_id), shard key (tenant_id));", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("tenant_id", typeof(int)), + }, + Rows = { new object[] { 1, 2 } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_shardkey", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "tenant_id"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(dataTable)); + Assert.Contains("shard key", exception.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("tenant_id", exception.Message); + } + + private static DataTable NewTable(params string[] columnNames) + { + var dataTable = new DataTable(); + foreach (var columnName in columnNames) + dataTable.Columns.Add(columnName, typeof(string)); + return dataTable; + } +} From 10910e1b24ca5214d7eae23d64d0ebfd58f67c21 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Fri, 26 Jun 2026 14:58:39 +0300 Subject: [PATCH 20/28] add documentation --- .../SingleStoreBulkUpdate.md | 71 ++++++++++ .../SingleStoreBulkUpdate/BulkCopyTimeout.md | 14 ++ .../SingleStoreBulkUpdate/ColumnMappings.md | 15 +++ .../ComputeRowsMatched.md | 14 ++ .../DestinationTableName.md | 18 +++ .../SingleStoreBulkUpdate/KeyColumns.md | 14 ++ .../SingleStoreBulkUpdate/NotifyAfter.md | 14 ++ .../SingleStoreBulkUpdate.md | 22 +++ .../SingleStoreRowsStaged.md | 21 +++ .../SingleStoreBulkUpdate/WriteToServer.md | 71 ++++++++++ .../WriteToServerAsync.md | 77 +++++++++++ .../SingleStoreBulkUpdateResult.md | 22 +++ .../RowsMatched.md | 14 ++ .../SingleStoreBulkUpdateResult/RowsStaged.md | 14 ++ .../RowsUpdated.md | 14 ++ .../SingleStoreBulkUpdateResult/Warnings.md | 15 +++ .../EnableExtendedDataTypes.md | 14 ++ .../SingleStoreDbColumn/VectorDimensions.md | 12 ++ .../VectorElementTypeName.md | 12 ++ .../SingleStoreRowsStagedEventArgs.md | 18 +++ .../SingleStoreRowsStagedEventArgs/Abort.md | 14 ++ .../RowsStaged.md | 14 ++ .../SingleStoreRowsStagedEventHandler.md | 15 +++ docs/content/tutorials/bulk-update.md | 126 ++++++++++++++++++ 24 files changed, 655 insertions(+) create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkCopyTimeout.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ColumnMappings.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ComputeRowsMatched.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/DestinationTableName.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/KeyColumns.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/NotifyAfter.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreBulkUpdate.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreRowsStaged.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/WriteToServer.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/WriteToServerAsync.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsStaged.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsUpdated.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/Warnings.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreConnectionStringBuilder/EnableExtendedDataTypes.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreDbColumn/VectorDimensions.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreDbColumn/VectorElementTypeName.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs/Abort.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs/RowsStaged.md create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventHandler.md create mode 100644 docs/content/tutorials/bulk-update.md diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md new file mode 100644 index 000000000..81118843f --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md @@ -0,0 +1,71 @@ +# SingleStoreBulkUpdate class + +[`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) lets you efficiently update many existing rows in a SingleStore Server table from an in-memory source. It complements [`SingleStoreBulkCopy`](./SingleStoreBulkCopy.md): where bulk copy inserts rows, bulk update modifies rows that already exist, matching them on the columns in [`KeyColumns`](./SingleStoreBulkUpdate/KeyColumns.md). + +The source rows are first staged into a temporary table using [`SingleStoreBulkCopy`](./SingleStoreBulkCopy.md), then a single `UPDATE ... JOIN` copies the non-key column values into the matching rows of the destination table. + +Because staging uses [`SingleStoreBulkCopy`](./SingleStoreBulkCopy.md), which loads data via `LOAD DATA LOCAL INFILE`, the connection string must have `AllowLoadLocalInfile=true` in order to use this class. + +Example code: + +```csharp +// open a connection that is allowed to load local data +await using var connection = new SingleStoreConnection("...;AllowLoadLocalInfile=True"); +await connection.OpenAsync(); + +// the source data; the column ordinals are referenced by the column mappings below +var dataTable = new DataTable +{ + Columns = { new DataColumn("id", typeof(int)), new DataColumn("status", typeof(string)) }, + Rows = { { 1, "active" }, { 2, "disabled" } }, +}; + +// update the "status" column of the rows whose "id" matches +var bulkUpdate = new SingleStoreBulkUpdate(connection) +{ + DestinationTableName = "users", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), // source column 0 -> key column "id" + new SingleStoreBulkCopyColumnMapping(1, "status"), // source column 1 -> updated column "status" + }, +}; +var result = await bulkUpdate.WriteToServerAsync(dataTable); + +// check for problems +if (result.Warnings.Count != 0) { /* handle potential data loss warnings */ } +``` + +```csharp +public sealed class SingleStoreBulkUpdate +``` + +## Public Members + +| name | description | +| --- | --- | +| [SingleStoreBulkUpdate](SingleStoreBulkUpdate/SingleStoreBulkUpdate.md)(…) | Initializes a [`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) object with the specified connection, and optionally the active transaction. | +| [BulkCopyTimeout](SingleStoreBulkUpdate/BulkCopyTimeout.md) { get; set; } | The number of seconds for each phase of the operation to complete before it times out (default `30`). | +| [ColumnMappings](SingleStoreBulkUpdate/ColumnMappings.md) { get; } | A collection of [`SingleStoreBulkCopyColumnMapping`](./SingleStoreBulkCopyColumnMapping.md) objects that map source column ordinals onto destination column names. Every key column and at least one non-key (updated) column must be mapped. | +| [ComputeRowsMatched](SingleStoreBulkUpdate/ComputeRowsMatched.md) { get; set; } | Whether to compute [`RowsMatched`](./SingleStoreBulkUpdateResult/RowsMatched.md) via a `COUNT` query (default `true`). Set this to `false` to skip that query for better performance, in which case [`RowsMatched`](./SingleStoreBulkUpdateResult/RowsMatched.md) is reported as `-1`. | +| [DestinationTableName](SingleStoreBulkUpdate/DestinationTableName.md) { get; set; } | The name of the table whose rows are updated. | +| [KeyColumns](SingleStoreBulkUpdate/KeyColumns.md) { get; } | The columns that identify which rows to update. They form the `JOIN` condition between the destination table and the staging table, so every key column must also appear in [`ColumnMappings`](./SingleStoreBulkUpdate/ColumnMappings.md). | +| [NotifyAfter](SingleStoreBulkUpdate/NotifyAfter.md) { get; set; } | If non-zero, this specifies the number of rows to be staged before raising the [`SingleStoreRowsStaged`](./SingleStoreBulkUpdate/SingleStoreRowsStaged.md) event. This applies only to the staging phase, not to the `UPDATE` execution. | +| event [SingleStoreRowsStaged](SingleStoreBulkUpdate/SingleStoreRowsStaged.md) | This event is raised every time that the number of rows specified by the [`NotifyAfter`](./SingleStoreBulkUpdate/NotifyAfter.md) property have been processed. | +| [WriteToServer](SingleStoreBulkUpdate/WriteToServer.md)(…) | Updates rows in the destination table using the data in the supplied DataTable. (3 methods) | +| [WriteToServerAsync](SingleStoreBulkUpdate/WriteToServerAsync.md)(…) | Asynchronously updates rows in the destination table using the data in the supplied DataTable. (3 methods) | + +## Remarks + +Note: This API is a unique feature of SingleStoreConnector; you must [switch to SingleStoreConnector](https://mysqlconnector.net/overview/installing/) in order to use it. + +The following restrictions apply, and `WriteToServer` throws if they are not met: [`KeyColumns`](./SingleStoreBulkUpdate/KeyColumns.md) is required and every key column must be mapped; at least one non-key column must be mapped; the source must not contain duplicate key values; shard key columns cannot be updated; reference tables are not supported; and expression column mappings are not supported. + +This API is experimental and may change in the future. + +## See Also + +* namespace [SingleStoreConnector](../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkCopyTimeout.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkCopyTimeout.md new file mode 100644 index 000000000..b28769359 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkCopyTimeout.md @@ -0,0 +1,14 @@ +# SingleStoreBulkUpdate.BulkCopyTimeout property + +The number of seconds for each phase of the operation to complete before it times out (default `30`). + +```csharp +public int BulkCopyTimeout { get; set; } +``` + +## See Also + +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ColumnMappings.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ColumnMappings.md new file mode 100644 index 000000000..b0e2bbdbe --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ColumnMappings.md @@ -0,0 +1,15 @@ +# SingleStoreBulkUpdate.ColumnMappings property + +A collection of [`SingleStoreBulkCopyColumnMapping`](../SingleStoreBulkCopyColumnMapping.md) objects that map source column ordinals onto destination column names. Every key column and at least one non-key (updated) column must be mapped. + +```csharp +public List ColumnMappings { get; } +``` + +## See Also + +* class [SingleStoreBulkCopyColumnMapping](../SingleStoreBulkCopyColumnMapping.md) +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ComputeRowsMatched.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ComputeRowsMatched.md new file mode 100644 index 000000000..84fc33de4 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ComputeRowsMatched.md @@ -0,0 +1,14 @@ +# SingleStoreBulkUpdate.ComputeRowsMatched property + +Whether to compute [`RowsMatched`](../SingleStoreBulkUpdateResult/RowsMatched.md) via a `COUNT` query (default `true`). Set this to `false` to skip that query for better performance, in which case [`RowsMatched`](../SingleStoreBulkUpdateResult/RowsMatched.md) is reported as `-1`. + +```csharp +public bool ComputeRowsMatched { get; set; } +``` + +## See Also + +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/DestinationTableName.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/DestinationTableName.md new file mode 100644 index 000000000..7d42b3c7a --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/DestinationTableName.md @@ -0,0 +1,18 @@ +# SingleStoreBulkUpdate.DestinationTableName property + +The name of the table whose rows are updated. + +```csharp +public string? DestinationTableName { get; set; } +``` + +## Remarks + +This name needs to be quoted if it contains special characters. + +## See Also + +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/KeyColumns.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/KeyColumns.md new file mode 100644 index 000000000..9182aad76 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/KeyColumns.md @@ -0,0 +1,14 @@ +# SingleStoreBulkUpdate.KeyColumns property + +The columns that identify which rows to update. They form the `JOIN` condition between the destination table and the staging table, so every key column must also appear in [`ColumnMappings`](./ColumnMappings.md). + +```csharp +public List KeyColumns { get; } +``` + +## See Also + +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/NotifyAfter.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/NotifyAfter.md new file mode 100644 index 000000000..4c7e95a9c --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/NotifyAfter.md @@ -0,0 +1,14 @@ +# SingleStoreBulkUpdate.NotifyAfter property + +If non-zero, this specifies the number of rows to be staged before raising the [`SingleStoreRowsStaged`](./SingleStoreRowsStaged.md) event. This applies only to the staging phase, not to the `UPDATE` execution. + +```csharp +public int NotifyAfter { get; set; } +``` + +## See Also + +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreBulkUpdate.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreBulkUpdate.md new file mode 100644 index 000000000..89b35c0de --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreBulkUpdate.md @@ -0,0 +1,22 @@ +# SingleStoreBulkUpdate constructor + +Initializes a [`SingleStoreBulkUpdate`](../SingleStoreBulkUpdate.md) object with the specified connection, and optionally the active transaction. + +```csharp +public SingleStoreBulkUpdate(SingleStoreConnection connection, + SingleStoreTransaction? transaction = null) +``` + +| parameter | description | +| --- | --- | +| connection | The [`SingleStoreConnection`](../SingleStoreConnection.md) to use. | +| transaction | (Optional) The [`SingleStoreTransaction`](../SingleStoreTransaction.md) to use. | + +## See Also + +* class [SingleStoreConnection](../SingleStoreConnection.md) +* class [SingleStoreTransaction](../SingleStoreTransaction.md) +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreRowsStaged.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreRowsStaged.md new file mode 100644 index 000000000..86b914414 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreRowsStaged.md @@ -0,0 +1,21 @@ +# SingleStoreBulkUpdate.SingleStoreRowsStaged event + +This event is raised every time that the number of rows specified by the [`NotifyAfter`](./NotifyAfter.md) property have been processed. + +```csharp +public event SingleStoreRowsStagedEventHandler? SingleStoreRowsStaged; +``` + +## Remarks + +Receipt of a RowsStaged event does not imply that any rows have been sent to the server or committed. + +The [`Abort`](../SingleStoreRowsStagedEventArgs/Abort.md) property can be set to `true` by the event handler to abort the staging. + +## See Also + +* delegate [SingleStoreRowsStagedEventHandler](../SingleStoreRowsStagedEventHandler.md) +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/WriteToServer.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/WriteToServer.md new file mode 100644 index 000000000..da5835c04 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/WriteToServer.md @@ -0,0 +1,71 @@ +# SingleStoreBulkUpdate.WriteToServer method (1 of 3) + +Updates rows in the destination table using the data in the supplied DataTable. + +```csharp +public SingleStoreBulkUpdateResult WriteToServer(DataTable dataTable) +``` + +| parameter | description | +| --- | --- | +| dataTable | The DataTable containing the key and update column values. | + +## Return Value + +A [`SingleStoreBulkUpdateResult`](../SingleStoreBulkUpdateResult.md) describing the result of the operation. + +## See Also + +* class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + +--- + +# SingleStoreBulkUpdate.WriteToServer method (2 of 3) + +Updates rows in the destination table using the data read from the supplied IDataReader. + +```csharp +public SingleStoreBulkUpdateResult WriteToServer(IDataReader dataReader) +``` + +| parameter | description | +| --- | --- | +| dataReader | The IDataReader to read the key and update column values from. | + +## Return Value + +A [`SingleStoreBulkUpdateResult`](../SingleStoreBulkUpdateResult.md) describing the result of the operation. + +## See Also + +* class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + +--- + +# SingleStoreBulkUpdate.WriteToServer method (3 of 3) + +Updates rows in the destination table using the data in the supplied sequence of DataRow objects. + +```csharp +public SingleStoreBulkUpdateResult WriteToServer(IEnumerable dataRows) +``` + +| parameter | description | +| --- | --- | +| dataRows | The collection of DataRow objects containing the key and update column values. | + +## Return Value + +A [`SingleStoreBulkUpdateResult`](../SingleStoreBulkUpdateResult.md) describing the result of the operation. + +## See Also + +* class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/WriteToServerAsync.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/WriteToServerAsync.md new file mode 100644 index 000000000..20a6dc79f --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/WriteToServerAsync.md @@ -0,0 +1,77 @@ +# SingleStoreBulkUpdate.WriteToServerAsync method (1 of 3) + +Asynchronously updates rows in the destination table using the data in the supplied DataTable. + +```csharp +public ValueTask WriteToServerAsync(DataTable dataTable, + CancellationToken cancellationToken = default) +``` + +| parameter | description | +| --- | --- | +| dataTable | The DataTable containing the key and update column values. | +| cancellationToken | A token to cancel the asynchronous operation. | + +## Return Value + +A [`SingleStoreBulkUpdateResult`](../SingleStoreBulkUpdateResult.md) describing the result of the operation. + +## See Also + +* class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + +--- + +# SingleStoreBulkUpdate.WriteToServerAsync method (2 of 3) + +Asynchronously updates rows in the destination table using the data read from the supplied IDataReader. + +```csharp +public ValueTask WriteToServerAsync(IDataReader dataReader, + CancellationToken cancellationToken = default) +``` + +| parameter | description | +| --- | --- | +| dataReader | The IDataReader to read the key and update column values from. | +| cancellationToken | A token to cancel the asynchronous operation. | + +## Return Value + +A [`SingleStoreBulkUpdateResult`](../SingleStoreBulkUpdateResult.md) describing the result of the operation. + +## See Also + +* class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + +--- + +# SingleStoreBulkUpdate.WriteToServerAsync method (3 of 3) + +Asynchronously updates rows in the destination table using the data in the supplied sequence of DataRow objects. + +```csharp +public ValueTask WriteToServerAsync(IEnumerable dataRows, + CancellationToken cancellationToken = default) +``` + +| parameter | description | +| --- | --- | +| dataRows | The collection of DataRow objects containing the key and update column values. | +| cancellationToken | A token to cancel the asynchronous operation. | + +## Return Value + +A [`SingleStoreBulkUpdateResult`](../SingleStoreBulkUpdateResult.md) describing the result of the operation. + +## See Also + +* class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md new file mode 100644 index 000000000..2293f9837 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md @@ -0,0 +1,22 @@ +# SingleStoreBulkUpdateResult class + +Represents the result of a [`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) operation. + +```csharp +public sealed class SingleStoreBulkUpdateResult +``` + +## Public Members + +| name | description | +| --- | --- | +| [RowsMatched](SingleStoreBulkUpdateResult/RowsMatched.md) { get; } | The number of staged rows that matched rows in the destination table, or `-1` when [`ComputeRowsMatched`](./SingleStoreBulkUpdate/ComputeRowsMatched.md) was set to `false` and the count was not computed. | +| [RowsStaged](SingleStoreBulkUpdateResult/RowsStaged.md) { get; } | The number of rows that were loaded into the staging table during the bulk update operation. | +| [RowsUpdated](SingleStoreBulkUpdateResult/RowsUpdated.md) { get; } | The number of rows that were updated during the bulk update operation. | +| [Warnings](SingleStoreBulkUpdateResult/Warnings.md) { get; } | The warnings, if any. Users of [`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) should check that this collection is empty to avoid potential data loss from failed data type conversions. | + +## See Also + +* namespace [SingleStoreConnector](../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md new file mode 100644 index 000000000..5de88ea79 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md @@ -0,0 +1,14 @@ +# SingleStoreBulkUpdateResult.RowsMatched property + +The number of staged rows that matched rows in the destination table, or `-1` when [`ComputeRowsMatched`](../SingleStoreBulkUpdate/ComputeRowsMatched.md) was set to `false` and the count was not computed. + +```csharp +public int RowsMatched { get; } +``` + +## See Also + +* class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsStaged.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsStaged.md new file mode 100644 index 000000000..66322e538 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsStaged.md @@ -0,0 +1,14 @@ +# SingleStoreBulkUpdateResult.RowsStaged property + +The number of rows that were loaded into the staging table during the bulk update operation. + +```csharp +public int RowsStaged { get; } +``` + +## See Also + +* class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsUpdated.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsUpdated.md new file mode 100644 index 000000000..11329f689 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsUpdated.md @@ -0,0 +1,14 @@ +# SingleStoreBulkUpdateResult.RowsUpdated property + +The number of rows that were updated during the bulk update operation. + +```csharp +public int RowsUpdated { get; } +``` + +## See Also + +* class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/Warnings.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/Warnings.md new file mode 100644 index 000000000..91b0491d3 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/Warnings.md @@ -0,0 +1,15 @@ +# SingleStoreBulkUpdateResult.Warnings property + +The warnings, if any. Users of [`SingleStoreBulkUpdate`](../SingleStoreBulkUpdate.md) should check that this collection is empty to avoid potential data loss from failed data type conversions. + +```csharp +public IReadOnlyList Warnings { get; } +``` + +## See Also + +* class [SingleStoreError](../SingleStoreError.md) +* class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreConnectionStringBuilder/EnableExtendedDataTypes.md b/docs/content/api/SingleStoreConnector/SingleStoreConnectionStringBuilder/EnableExtendedDataTypes.md new file mode 100644 index 000000000..86a37aaf8 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreConnectionStringBuilder/EnableExtendedDataTypes.md @@ -0,0 +1,14 @@ +# SingleStoreConnectionStringBuilder.EnableExtendedDataTypes property + +Enable SingleStore extended type metadata for types such as VECTOR and BSON. + +```csharp +public bool EnableExtendedDataTypes { get; set; } +``` + +## See Also + +* class [SingleStoreConnectionStringBuilder](../SingleStoreConnectionStringBuilder.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreDbColumn/VectorDimensions.md b/docs/content/api/SingleStoreConnector/SingleStoreDbColumn/VectorDimensions.md new file mode 100644 index 000000000..8d78e4707 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreDbColumn/VectorDimensions.md @@ -0,0 +1,12 @@ +# SingleStoreDbColumn.VectorDimensions property + +```csharp +public int? VectorDimensions { get; } +``` + +## See Also + +* class [SingleStoreDbColumn](../SingleStoreDbColumn.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreDbColumn/VectorElementTypeName.md b/docs/content/api/SingleStoreConnector/SingleStoreDbColumn/VectorElementTypeName.md new file mode 100644 index 000000000..4a1e85d2e --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreDbColumn/VectorElementTypeName.md @@ -0,0 +1,12 @@ +# SingleStoreDbColumn.VectorElementTypeName property + +```csharp +public string? VectorElementTypeName { get; } +``` + +## See Also + +* class [SingleStoreDbColumn](../SingleStoreDbColumn.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs.md b/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs.md new file mode 100644 index 000000000..d3cb59d17 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs.md @@ -0,0 +1,18 @@ +# SingleStoreRowsStagedEventArgs class + +```csharp +public sealed class SingleStoreRowsStagedEventArgs : EventArgs +``` + +## Public Members + +| name | description | +| --- | --- | +| [Abort](SingleStoreRowsStagedEventArgs/Abort.md) { get; set; } | Gets or sets a value indicating whether the bulk update operation should be aborted. | +| [RowsStaged](SingleStoreRowsStagedEventArgs/RowsStaged.md) { get; } | Gets a value that returns the number of rows staged during the current bulk update operation. | + +## See Also + +* namespace [SingleStoreConnector](../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs/Abort.md b/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs/Abort.md new file mode 100644 index 000000000..372b63e6c --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs/Abort.md @@ -0,0 +1,14 @@ +# SingleStoreRowsStagedEventArgs.Abort property + +Gets or sets a value indicating whether the bulk update operation should be aborted. + +```csharp +public bool Abort { get; set; } +``` + +## See Also + +* class [SingleStoreRowsStagedEventArgs](../SingleStoreRowsStagedEventArgs.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs/RowsStaged.md b/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs/RowsStaged.md new file mode 100644 index 000000000..b5d724c53 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventArgs/RowsStaged.md @@ -0,0 +1,14 @@ +# SingleStoreRowsStagedEventArgs.RowsStaged property + +Gets a value that returns the number of rows staged during the current bulk update operation. + +```csharp +public long RowsStaged { get; } +``` + +## See Also + +* class [SingleStoreRowsStagedEventArgs](../SingleStoreRowsStagedEventArgs.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventHandler.md b/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventHandler.md new file mode 100644 index 000000000..a14fd4670 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreRowsStagedEventHandler.md @@ -0,0 +1,15 @@ +# SingleStoreRowsStagedEventHandler delegate + +Represents the method that handles the [`SingleStoreRowsStaged`](./SingleStoreBulkUpdate/SingleStoreRowsStaged.md) event of a [`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md). + +```csharp +public delegate void SingleStoreRowsStagedEventHandler(object sender, + SingleStoreRowsStagedEventArgs e); +``` + +## See Also + +* class [SingleStoreRowsStagedEventArgs](./SingleStoreRowsStagedEventArgs.md) +* namespace [SingleStoreConnector](../SingleStoreConnector.md) + + diff --git a/docs/content/tutorials/bulk-update.md b/docs/content/tutorials/bulk-update.md new file mode 100644 index 000000000..df5a56412 --- /dev/null +++ b/docs/content/tutorials/bulk-update.md @@ -0,0 +1,126 @@ +--- +date: 2026-06-26 +menu: + main: + parent: tutorials +title: Bulk Update +customtitle: "Tutorial: Bulk Updating Rows in SingleStore from C#" +weight: 14 +--- + +Bulk Update +=========== + +`SingleStoreBulkUpdate` efficiently updates many existing rows in a SingleStore table from an in-memory source. +It complements [`SingleStoreBulkCopy`](../../api/SingleStoreConnector/SingleStoreBulkCopy/): where bulk copy *inserts* +rows, bulk update *modifies* rows that already exist, matching them on one or more key columns. + +It is much faster than issuing an individual `UPDATE` statement per row, because all of the work is performed in a +single round trip pattern instead of one command per row. + +> **Note:** This API is experimental and may change in the future. + +How it works +------------ + +`SingleStoreBulkUpdate` performs the update in three phases, all on the same connection: + +1. It creates a temporary staging table whose columns mirror the mapped columns of the destination table (their exact + types, lengths, and collations are copied from the destination so values round-trip without conversion). +2. It loads the source rows into that staging table using `SingleStoreBulkCopy`. +3. It runs a single `UPDATE ... JOIN` that copies the non-key column values from the staging table into the matching + rows of the destination table, joining on the key columns. The staging table is then dropped. + +Because staging uses `SingleStoreBulkCopy` (which loads data with `LOAD DATA LOCAL INFILE`), the connection string +*must* have `AllowLoadLocalInfile=true` in order to use this class. + +Basic example +------------- + +```csharp +// open a connection that is allowed to load local data +await using var connection = new SingleStoreConnection("...;AllowLoadLocalInfile=True"); +await connection.OpenAsync(); + +// the source data; the DataTable column names need not match the destination, +// but the source ordinals must match the column mappings below +var dataTable = new DataTable +{ + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("status", typeof(string)), + }, + Rows = + { + { 1, "active" }, + { 2, "disabled" }, + }, +}; + +var bulkUpdate = new SingleStoreBulkUpdate(connection) +{ + DestinationTableName = "users", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), // source column 0 -> key column "id" + new SingleStoreBulkCopyColumnMapping(1, "status"), // source column 1 -> updated column "status" + }, +}; + +var result = await bulkUpdate.WriteToServerAsync(dataTable); + +// check for problems +if (result.Warnings.Count != 0) { /* handle potential data loss warnings */ } +``` + +Column mappings +--------------- + +Each `SingleStoreBulkCopyColumnMapping` maps a **source ordinal** (the zero-based column index in the source data) to a +**destination column name**. The names of the columns in the source `DataTable`/`DataReader` are ignored; only the +ordinal matters. Every key column must be included in the mappings, and at least one non-key column must be mapped so +that there is something to update. + +Source data may be supplied as a `DataTable`, a sequence of `DataRow` objects, or an `IDataReader`. Both synchronous +(`WriteToServer`) and asynchronous (`WriteToServerAsync`) methods are available. + +Interpreting the result +------------------------ + +`WriteToServerAsync` returns a `SingleStoreBulkUpdateResult`: + +* `RowsStaged` — the number of source rows loaded into the staging table. +* `RowsMatched` — the number of staged rows that matched a row in the destination table. This is `-1` when + `ComputeRowsMatched` is set to `false` (see below). +* `RowsUpdated` — the number of rows affected by the `UPDATE`. +* `Warnings` — any warnings raised while staging or updating; check that this is empty to avoid silent data loss from + failed type conversions. + +Performance +----------- + +* Set `ComputeRowsMatched = false` to skip the extra `COUNT(*)` query that populates `RowsMatched`. When disabled, + `RowsMatched` is reported as `-1`. +* Set `BulkCopyTimeout` (in seconds) to control how long each phase may run. +* Set `NotifyAfter` to a non-zero value to receive `SingleStoreRowsStaged` events while rows are being staged; the + event handler can set `Abort = true` to stop staging early. + +Limitations +----------- + +`SingleStoreBulkUpdate` enforces the following restrictions, and will throw if they are not met: + +* `KeyColumns` is required and must contain at least one column. Every key column must also appear in `ColumnMappings`. +* At least one non-key column must be mapped, so there is a column to update. +* Duplicate key values in the source data are rejected; they would collide in the staging table's primary key. +* Shard key columns cannot be updated, because SingleStore does not allow updating a shard key. +* Reference tables are not supported as the destination. +* Expression column mappings (a `SingleStoreBulkCopyColumnMapping` with an `Expression`) are not supported. + +Transactions +------------ + +A `SingleStoreTransaction` may be passed to the constructor. When supplied, all phases participate in that transaction, +so the update can be committed or rolled back atomically with other work on the connection. From 7ae44a1c19279985ed89c5513ebf360106b50a06 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Fri, 26 Jun 2026 15:50:51 +0300 Subject: [PATCH 21/28] rename misleading field + updatethe docs --- docs/content/api/SingleStoreConnector.md | 4 + .../SingleStoreBulkUpdate.md | 6 +- .../SingleStoreBulkUpdate/BulkCopyTimeout.md | 14 ---- .../SingleStoreBulkUpdateResult.md | 2 +- .../RowsUpdated.md | 6 +- .../SingleStoreConnectionStringBuilder.md | 1 + ...ngleStoreConnectorTracingOptionsBuilder.md | 2 +- .../SingleStoreDbColumn.md | 2 + .../SingleStoreConnector/SingleStoreDbType.md | 2 + docs/content/tutorials/bulk-update.md | 7 +- .../SingleStoreBulkUpdate.cs | 80 ++++++++++++++----- .../SingleStoreBulkUpdateResult.cs | 9 ++- tests/SideBySide/BulkUpdateAdvancedTests.cs | 2 +- 13 files changed, 89 insertions(+), 48 deletions(-) delete mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkCopyTimeout.md diff --git a/docs/content/api/SingleStoreConnector.md b/docs/content/api/SingleStoreConnector.md index 183053f5c..a2d58994e 100644 --- a/docs/content/api/SingleStoreConnector.md +++ b/docs/content/api/SingleStoreConnector.md @@ -16,6 +16,8 @@ | class [SingleStoreBulkLoader](./SingleStoreConnector/SingleStoreBulkLoader.md) | [`SingleStoreBulkLoader`](./SingleStoreConnector/SingleStoreBulkLoader.md) lets you efficiently load a SingleStore Server Table with data from a CSV or TSV file or Stream. | | enum [SingleStoreBulkLoaderConflictOption](./SingleStoreConnector/SingleStoreBulkLoaderConflictOption.md) | | | enum [SingleStoreBulkLoaderPriority](./SingleStoreConnector/SingleStoreBulkLoaderPriority.md) | | +| class [SingleStoreBulkUpdate](./SingleStoreConnector/SingleStoreBulkUpdate.md) | [`SingleStoreBulkUpdate`](./SingleStoreConnector/SingleStoreBulkUpdate.md) lets you efficiently update many existing rows in a SingleStore table from an in-memory source. It complements [`SingleStoreBulkCopy`](./SingleStoreConnector/SingleStoreBulkCopy.md): where bulk copy inserts rows, bulk update modifies rows that already exist, matching them on the columns in [`KeyColumns`](./SingleStoreConnector/SingleStoreBulkUpdate/KeyColumns.md). | +| class [SingleStoreBulkUpdateResult](./SingleStoreConnector/SingleStoreBulkUpdateResult.md) | Represents the result of a [`SingleStoreBulkUpdate`](./SingleStoreConnector/SingleStoreBulkUpdate.md) operation. | | enum [SingleStoreCertificateStoreLocation](./SingleStoreConnector/SingleStoreCertificateStoreLocation.md) | | | class [SingleStoreCommand](./SingleStoreConnector/SingleStoreCommand.md) | [`SingleStoreCommand`](./SingleStoreConnector/SingleStoreCommand.md) represents a SQL statement or stored procedure name to execute against a SingleStore database. | | class [SingleStoreCommandBuilder](./SingleStoreConnector/SingleStoreCommandBuilder.md) | | @@ -54,6 +56,8 @@ | class [SingleStoreProvidePasswordContext](./SingleStoreConnector/SingleStoreProvidePasswordContext.md) | Provides context for the [`ProvidePasswordCallback`](./SingleStoreConnector/SingleStoreConnection/ProvidePasswordCallback.md) delegate. | | class [SingleStoreRowsCopiedEventArgs](./SingleStoreConnector/SingleStoreRowsCopiedEventArgs.md) | | | delegate [SingleStoreRowsCopiedEventHandler](./SingleStoreConnector/SingleStoreRowsCopiedEventHandler.md) | Represents the method that handles the [`SingleStoreRowsCopied`](./SingleStoreConnector/SingleStoreBulkCopy/SingleStoreRowsCopied.md) event of a [`SingleStoreBulkCopy`](./SingleStoreConnector/SingleStoreBulkCopy.md). | +| class [SingleStoreRowsStagedEventArgs](./SingleStoreConnector/SingleStoreRowsStagedEventArgs.md) | | +| delegate [SingleStoreRowsStagedEventHandler](./SingleStoreConnector/SingleStoreRowsStagedEventHandler.md) | Represents the method that handles the [`SingleStoreRowsStaged`](./SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreRowsStaged.md) event of a [`SingleStoreBulkUpdate`](./SingleStoreConnector/SingleStoreBulkUpdate.md). | | class [SingleStoreRowUpdatedEventArgs](./SingleStoreConnector/SingleStoreRowUpdatedEventArgs.md) | | | delegate [SingleStoreRowUpdatedEventHandler](./SingleStoreConnector/SingleStoreRowUpdatedEventHandler.md) | | | class [SingleStoreRowUpdatingEventArgs](./SingleStoreConnector/SingleStoreRowUpdatingEventArgs.md) | | diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md index 81118843f..8b203f64f 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md @@ -1,6 +1,6 @@ # SingleStoreBulkUpdate class -[`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) lets you efficiently update many existing rows in a SingleStore Server table from an in-memory source. It complements [`SingleStoreBulkCopy`](./SingleStoreBulkCopy.md): where bulk copy inserts rows, bulk update modifies rows that already exist, matching them on the columns in [`KeyColumns`](./SingleStoreBulkUpdate/KeyColumns.md). +[`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) lets you efficiently update many existing rows in a SingleStore table from an in-memory source. It complements [`SingleStoreBulkCopy`](./SingleStoreBulkCopy.md): where bulk copy inserts rows, bulk update modifies rows that already exist, matching them on the columns in [`KeyColumns`](./SingleStoreBulkUpdate/KeyColumns.md). The source rows are first staged into a temporary table using [`SingleStoreBulkCopy`](./SingleStoreBulkCopy.md), then a single `UPDATE ... JOIN` copies the non-key column values into the matching rows of the destination table. @@ -46,7 +46,7 @@ public sealed class SingleStoreBulkUpdate | name | description | | --- | --- | | [SingleStoreBulkUpdate](SingleStoreBulkUpdate/SingleStoreBulkUpdate.md)(…) | Initializes a [`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) object with the specified connection, and optionally the active transaction. | -| [BulkCopyTimeout](SingleStoreBulkUpdate/BulkCopyTimeout.md) { get; set; } | The number of seconds for each phase of the operation to complete before it times out (default `30`). | +| [BulkUpdateTimeout](SingleStoreBulkUpdate/BulkUpdateTimeout.md) { get; set; } | The number of seconds for each phase of the operation to complete before it times out (default `30`). | | [ColumnMappings](SingleStoreBulkUpdate/ColumnMappings.md) { get; } | A collection of [`SingleStoreBulkCopyColumnMapping`](./SingleStoreBulkCopyColumnMapping.md) objects that map source column ordinals onto destination column names. Every key column and at least one non-key (updated) column must be mapped. | | [ComputeRowsMatched](SingleStoreBulkUpdate/ComputeRowsMatched.md) { get; set; } | Whether to compute [`RowsMatched`](./SingleStoreBulkUpdateResult/RowsMatched.md) via a `COUNT` query (default `true`). Set this to `false` to skip that query for better performance, in which case [`RowsMatched`](./SingleStoreBulkUpdateResult/RowsMatched.md) is reported as `-1`. | | [DestinationTableName](SingleStoreBulkUpdate/DestinationTableName.md) { get; set; } | The name of the table whose rows are updated. | @@ -58,8 +58,6 @@ public sealed class SingleStoreBulkUpdate ## Remarks -Note: This API is a unique feature of SingleStoreConnector; you must [switch to SingleStoreConnector](https://mysqlconnector.net/overview/installing/) in order to use it. - The following restrictions apply, and `WriteToServer` throws if they are not met: [`KeyColumns`](./SingleStoreBulkUpdate/KeyColumns.md) is required and every key column must be mapped; at least one non-key column must be mapped; the source must not contain duplicate key values; shard key columns cannot be updated; reference tables are not supported; and expression column mappings are not supported. This API is experimental and may change in the future. diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkCopyTimeout.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkCopyTimeout.md deleted file mode 100644 index b28769359..000000000 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkCopyTimeout.md +++ /dev/null @@ -1,14 +0,0 @@ -# SingleStoreBulkUpdate.BulkCopyTimeout property - -The number of seconds for each phase of the operation to complete before it times out (default `30`). - -```csharp -public int BulkCopyTimeout { get; set; } -``` - -## See Also - -* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) -* namespace [SingleStoreConnector](../../SingleStoreConnector.md) - - diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md index 2293f9837..6a8ea582f 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md @@ -12,7 +12,7 @@ public sealed class SingleStoreBulkUpdateResult | --- | --- | | [RowsMatched](SingleStoreBulkUpdateResult/RowsMatched.md) { get; } | The number of staged rows that matched rows in the destination table, or `-1` when [`ComputeRowsMatched`](./SingleStoreBulkUpdate/ComputeRowsMatched.md) was set to `false` and the count was not computed. | | [RowsStaged](SingleStoreBulkUpdateResult/RowsStaged.md) { get; } | The number of rows that were loaded into the staging table during the bulk update operation. | -| [RowsUpdated](SingleStoreBulkUpdateResult/RowsUpdated.md) { get; } | The number of rows that were updated during the bulk update operation. | +| [RowsUpdated](SingleStoreBulkUpdateResult/RowsUpdated.md) { get; } | The number of rows affected by the `UPDATE`, as reported by the server. | | [Warnings](SingleStoreBulkUpdateResult/Warnings.md) { get; } | The warnings, if any. Users of [`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) should check that this collection is empty to avoid potential data loss from failed data type conversions. | ## See Also diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsUpdated.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsUpdated.md index 11329f689..e74f620da 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsUpdated.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsUpdated.md @@ -1,11 +1,15 @@ # SingleStoreBulkUpdateResult.RowsUpdated property -The number of rows that were updated during the bulk update operation. +The number of rows affected by the `UPDATE`, as reported by the server. ```csharp public int RowsUpdated { get; } ``` +## Remarks + +The exact meaning depends on the connection's [`UseAffectedRows`](../SingleStoreConnectionStringBuilder/UseAffectedRows.md) setting. With the default (`UseAffectedRows=false`) this is the number of rows matched by the update — including rows that already held the new values — and therefore typically equals [`RowsMatched`](./RowsMatched.md). With `UseAffectedRows=true` it is the number of rows whose values actually changed. + ## See Also * class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) diff --git a/docs/content/api/SingleStoreConnector/SingleStoreConnectionStringBuilder.md b/docs/content/api/SingleStoreConnector/SingleStoreConnectionStringBuilder.md index 85f704fe4..eb0c5e47f 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreConnectionStringBuilder.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreConnectionStringBuilder.md @@ -35,6 +35,7 @@ public sealed class SingleStoreConnectionStringBuilder : DbConnectionStringBuild | [DateTimeKind](SingleStoreConnectionStringBuilder/DateTimeKind.md) { get; set; } | The [`DateTimeKind`](./SingleStoreConnectionStringBuilder/DateTimeKind.md) to use when deserializing `DATETIME` values. | | [DefaultCommandTimeout](SingleStoreConnectionStringBuilder/DefaultCommandTimeout.md) { get; set; } | The length of time (in seconds) each command can execute before the query is cancelled on the server, or zero to disable timeouts. | | [DnsCheckInterval](SingleStoreConnectionStringBuilder/DnsCheckInterval.md) { get; set; } | The number of seconds between checks for DNS changes, or 0 to disable periodic checks. | +| [EnableExtendedDataTypes](SingleStoreConnectionStringBuilder/EnableExtendedDataTypes.md) { get; set; } | Enable SingleStore extended type metadata for types such as VECTOR and BSON. | | [ForceSynchronous](SingleStoreConnectionStringBuilder/ForceSynchronous.md) { get; set; } | Forces all async methods to execute synchronously. This can be useful for debugging. | | [GuidFormat](SingleStoreConnectionStringBuilder/GuidFormat.md) { get; set; } | Determines which column type (if any) should be read as a Guid. | | [IgnoreCommandTransaction](SingleStoreConnectionStringBuilder/IgnoreCommandTransaction.md) { get; set; } | Does not check the [`Transaction`](./SingleStoreCommand/Transaction.md) property for validity when executing a command. | diff --git a/docs/content/api/SingleStoreConnector/SingleStoreConnectorTracingOptionsBuilder.md b/docs/content/api/SingleStoreConnector/SingleStoreConnectorTracingOptionsBuilder.md index bec3dadfa..dfafc2c05 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreConnectorTracingOptionsBuilder.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreConnectorTracingOptionsBuilder.md @@ -11,7 +11,7 @@ public sealed class SingleStoreConnectorTracingOptionsBuilder | name | description | | --- | --- | | [SingleStoreConnectorTracingOptionsBuilder](SingleStoreConnectorTracingOptionsBuilder/SingleStoreConnectorTracingOptionsBuilder.md)() | The default constructor. | -| [EnableResultSetHeaderEvent](SingleStoreConnectorTracingOptionsBuilder/EnableResultSetHeaderEvent.md)(…) | Gets or sets a value indicating whether to enable the "time-to-first-read" event. Default is true to preserve existing behavior. | +| [EnableResultSetHeaderEvent](SingleStoreConnectorTracingOptionsBuilder/EnableResultSetHeaderEvent.md)(…) | Gets or sets a value indicating whether to enable the "read-result-set-header" event. Default is false; set to true to opt in to this event. | ## See Also diff --git a/docs/content/api/SingleStoreConnector/SingleStoreDbColumn.md b/docs/content/api/SingleStoreConnector/SingleStoreDbColumn.md index ad6b96b7f..ff18d96d5 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreDbColumn.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreDbColumn.md @@ -10,6 +10,8 @@ public sealed class SingleStoreDbColumn : DbColumn | --- | --- | | [ProviderType](SingleStoreDbColumn/ProviderType.md) { get; } | | | [TableName](SingleStoreDbColumn/TableName.md) { get; } | Gets the name of the table that the column belongs to. This will be the alias if the table is aliased in the query. | +| [VectorDimensions](SingleStoreDbColumn/VectorDimensions.md) { get; } | | +| [VectorElementTypeName](SingleStoreDbColumn/VectorElementTypeName.md) { get; } | | ## See Also diff --git a/docs/content/api/SingleStoreConnector/SingleStoreDbType.md b/docs/content/api/SingleStoreConnector/SingleStoreDbType.md index f2785adcf..48c23eb6f 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreDbType.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreDbType.md @@ -51,6 +51,8 @@ public enum SingleStoreDbType | LongText | `751` | | | Text | `752` | | | Guid | `800` | | +| Bson | `801` | | +| Vector | `802` | | ## See Also diff --git a/docs/content/tutorials/bulk-update.md b/docs/content/tutorials/bulk-update.md index df5a56412..1bfd2b5f8 100644 --- a/docs/content/tutorials/bulk-update.md +++ b/docs/content/tutorials/bulk-update.md @@ -94,7 +94,10 @@ Interpreting the result * `RowsStaged` — the number of source rows loaded into the staging table. * `RowsMatched` — the number of staged rows that matched a row in the destination table. This is `-1` when `ComputeRowsMatched` is set to `false` (see below). -* `RowsUpdated` — the number of rows affected by the `UPDATE`. +* `RowsUpdated` — the number of rows affected by the `UPDATE`, as reported by the server. Its exact meaning depends on + the connection's `UseAffectedRows` setting: with the default (`UseAffectedRows=false`) it counts the rows *matched* by + the update — including rows that already held the new values — so it typically equals `RowsMatched`; with + `UseAffectedRows=true` it counts only the rows whose values actually *changed*. * `Warnings` — any warnings raised while staging or updating; check that this is empty to avoid silent data loss from failed type conversions. @@ -103,7 +106,7 @@ Performance * Set `ComputeRowsMatched = false` to skip the extra `COUNT(*)` query that populates `RowsMatched`. When disabled, `RowsMatched` is reported as `-1`. -* Set `BulkCopyTimeout` (in seconds) to control how long each phase may run. +* Set `BulkUpdateTimeout` (in seconds) to control how long each phase may run. * Set `NotifyAfter` to a non-zero value to receive `SingleStoreRowsStaged` events while rows are being staged; the event handler can set `Abort = true` to stop staging early. diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index c063892dc..f97488c2c 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -12,13 +12,48 @@ namespace SingleStoreConnector; // TODO: consider upsert support in a future version. /// -/// Provides efficient bulk update operations for SingleStore databases. +/// lets you efficiently update many existing rows in a SingleStore table +/// from an in-memory source. It complements : where bulk copy inserts rows, +/// bulk update modifies rows that already exist, matching them on the columns in . +/// The source rows are first staged into a temporary table using , then a single +/// UPDATE ... JOIN copies the non-key column values into the matching rows of the destination table. +/// Because staging uses , which loads data via LOAD DATA LOCAL INFILE, +/// the connection string must have AllowLoadLocalInfile=true in order to use this class. +/// Example code: +/// +/// // open a connection that is allowed to load local data +/// await using var connection = new SingleStoreConnection("...;AllowLoadLocalInfile=True"); +/// await connection.OpenAsync(); +/// +/// // the source data; the column ordinals are referenced by the column mappings below +/// var dataTable = new DataTable +/// { +/// Columns = { new DataColumn("id", typeof(int)), new DataColumn("status", typeof(string)) }, +/// Rows = { { 1, "active" }, { 2, "disabled" } }, +/// }; +/// +/// // update the "status" column of the rows whose "id" matches +/// var bulkUpdate = new SingleStoreBulkUpdate(connection) +/// { +/// DestinationTableName = "users", +/// KeyColumns = { "id" }, +/// ColumnMappings = +/// { +/// new SingleStoreBulkCopyColumnMapping(0, "id"), // source column 0 -> key column "id" +/// new SingleStoreBulkCopyColumnMapping(1, "status"), // source column 1 -> updated column "status" +/// }, +/// }; +/// var result = await bulkUpdate.WriteToServerAsync(dataTable); +/// +/// // check for problems +/// if (result.Warnings.Count != 0) { /* handle potential data loss warnings */ } +/// /// /// -/// -/// This class stages source rows into a temporary table using , -/// then updates matching rows in the destination table using a single UPDATE ... JOIN statement. -/// +/// The following restrictions apply, and WriteToServer throws if they are not met: +/// is required and every key column must be mapped; at least one non-key column must be mapped; the source must not +/// contain duplicate key values; shard key columns cannot be updated; reference tables are not supported; and +/// expression column mappings are not supported. /// This API is experimental and may change in the future. /// public sealed class SingleStoreBulkUpdate @@ -39,37 +74,38 @@ public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransa } /// - /// Gets or sets the name of the destination table. + /// The name of the table whose rows are updated. /// + /// This name needs to be quoted if it contains special characters. public string? DestinationTableName { get; set; } /// - /// Gets the list of key columns used for the JOIN condition. - /// These columns identify which rows to update. + /// The columns that identify which rows to update. They form the JOIN condition between the destination + /// table and the staging table, so every key column must also appear in . /// public List KeyColumns { get; } /// - /// Gets the collection of column mappings between source data and destination table. + /// A collection of objects that map source column ordinals onto + /// destination column names. Every key column and at least one non-key (updated) column must be mapped. /// public List ColumnMappings { get; } /// - /// Gets or sets the timeout in seconds for bulk operations. + /// The number of seconds for each phase of the operation to complete before it times out (default 30). /// - public int BulkCopyTimeout { get; set; } = 30; + public int BulkUpdateTimeout { get; set; } = 30; /// - /// Gets or sets the number of rows to stage before firing the SingleStoreRowsStaged event. - /// Only applies to the staging phase (LOAD DATA), not the UPDATE execution. - /// Set to 0 to disable progress notifications. + /// If non-zero, this specifies the number of rows to be staged before raising the + /// event. This applies only to the staging phase, not to the UPDATE execution. /// public int NotifyAfter { get; set; } /// - /// Gets or sets whether to compute the RowsMatched count via a COUNT query. - /// Default is true. Set to false to skip the COUNT query for better performance. - /// When false, RowsMatched will be null in the result. + /// Whether to compute via a COUNT query (default true). + /// Set this to false to skip that query for better performance, in which case + /// is reported as -1. /// public bool ComputeRowsMatched { get; set; } = true; @@ -451,7 +487,7 @@ private async Task CreateStagingTableAsync(string destinationTableName, { cmd.CommandText = createTableSql.ToString(); cmd.Transaction = m_transaction; - cmd.CommandTimeout = BulkCopyTimeout; + cmd.CommandTimeout = BulkUpdateTimeout; await cmd.ExecuteNonQueryAsync(ioBehavior, cancellationToken).ConfigureAwait(false); } @@ -514,7 +550,7 @@ private async Task StageDataAsync(string tempTableName, object source, IOBe var bulkCopy = new SingleStoreBulkCopy(m_connection, m_transaction) { DestinationTableName = tempTableName, - BulkCopyTimeout = BulkCopyTimeout, + BulkCopyTimeout = BulkUpdateTimeout, NotifyAfter = NotifyAfter, }; @@ -631,7 +667,7 @@ private static ValueTask StageWithBulkCopyAsync(Singl using var cmd = m_connection.CreateCommand(); cmd.CommandText = countSql; cmd.Transaction = m_transaction; - cmd.CommandTimeout = BulkCopyTimeout; + cmd.CommandTimeout = BulkUpdateTimeout; var scalar = await cmd.ExecuteScalarAsync(ioBehavior, cancellationToken).ConfigureAwait(false); @@ -685,7 +721,7 @@ private async Task ExecuteUpdateAsync(string tempTableName, IOBehavior ioBe using var cmd = m_connection.CreateCommand(); cmd.CommandText = updateSql; cmd.Transaction = m_transaction; - cmd.CommandTimeout = BulkCopyTimeout; + cmd.CommandTimeout = BulkUpdateTimeout; // Collect any warnings raised during the UPDATE. Errors is already IReadOnlyList. void OnInfoMessage(object sender, SingleStoreInfoMessageEventArgs args) => m_warnings.AddRange(args.Errors); @@ -738,7 +774,7 @@ private async Task DropStagingTableAsync(string? tempTableName, IOBehavior ioBeh using var cmd = m_connection.CreateCommand(); cmd.CommandText = $"DROP TEMPORARY TABLE IF EXISTS {IdentifierHelper.QuoteIdentifier(tempTableName!)}"; cmd.Transaction = m_transaction; - cmd.CommandTimeout = BulkCopyTimeout; + cmd.CommandTimeout = BulkUpdateTimeout; await cmd.ExecuteNonQueryAsync(ioBehavior, CancellationToken.None).ConfigureAwait(false); } catch (Exception ex) diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs b/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs index 5e40629ec..92d5e2b9f 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs @@ -17,13 +17,18 @@ public sealed class SingleStoreBulkUpdateResult public int RowsStaged { get; } /// - /// The number of staged rows that matched rows in the destination table. + /// The number of staged rows that matched rows in the destination table, or -1 when + /// was set to false and the count was not computed. /// public int RowsMatched { get; } /// - /// The number of rows that were updated during the bulk update operation. + /// The number of rows affected by the UPDATE, as reported by the server. /// + /// The exact meaning depends on the connection's + /// setting. With the default (UseAffectedRows=false) this is the number of rows matched by the update — + /// including rows that already held the new values — and therefore typically equals . With + /// UseAffectedRows=true it is the number of rows whose values actually changed. public int RowsUpdated { get; } internal SingleStoreBulkUpdateResult( diff --git a/tests/SideBySide/BulkUpdateAdvancedTests.cs b/tests/SideBySide/BulkUpdateAdvancedTests.cs index 10b828b1c..3cb8818ab 100644 --- a/tests/SideBySide/BulkUpdateAdvancedTests.cs +++ b/tests/SideBySide/BulkUpdateAdvancedTests.cs @@ -504,7 +504,7 @@ public async Task UpdatesLargeDataset() var bulkUpdate = new SingleStoreBulkUpdate(connection) { DestinationTableName = "bulk_update_large", - BulkCopyTimeout = 60, + BulkUpdateTimeout = 60, KeyColumns = { "id" }, ColumnMappings = { From ce4b9a8076b16e981cfaea6b620a7d3196032979 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 30 Jun 2026 15:17:34 +0300 Subject: [PATCH 22/28] resolve issues (abort performing partial update instead of aborting the whole op, result warnings possibly being mutated) --- .../BulkUpdateTimeout.md | 14 +++ .../SingleStoreBulkUpdate.cs | 59 ++++++++++--- .../Utilities/SchemaDetector.cs | 42 +++++++++ tests/SideBySide/BulkUpdateAdvancedTests.cs | 11 ++- tests/SideBySide/BulkUpdateTests.cs | 86 +++++++++++++++++++ tests/SideBySide/BulkUpdateValidationTests.cs | 39 +++++++++ 6 files changed, 236 insertions(+), 15 deletions(-) create mode 100644 docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkUpdateTimeout.md diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkUpdateTimeout.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkUpdateTimeout.md new file mode 100644 index 000000000..323b4f455 --- /dev/null +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkUpdateTimeout.md @@ -0,0 +1,14 @@ +# SingleStoreBulkUpdate.BulkUpdateTimeout property + +The number of seconds for each phase of the operation to complete before it times out (default `30`). + +```csharp +public int BulkUpdateTimeout { get; set; } +``` + +## See Also + +* class [SingleStoreBulkUpdate](../SingleStoreBulkUpdate.md) +* namespace [SingleStoreConnector](../../SingleStoreConnector.md) + + diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index f97488c2c..f9aeaedee 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -215,11 +215,17 @@ private async ValueTask WriteToServerAsync(IOBehavi // Reset any warnings from a previous call so the result only reflects this operation. m_warnings.Clear(); + // Materialize a lazy DataRow sequence once, so its row count is known (which makes empty input return a + // consistent zero-count result regardless of source type) and so SingleStoreBulkCopy can enumerate it + // without re-running the original (possibly single-use) source. + if (source is IEnumerable dataRows && source is not ICollection && source is not IReadOnlyCollection) + source = dataRows.ToList(); + // Short-circuit input whose row count is known to be zero: there is nothing to stage or update, so avoid // opening the connection and creating a staging table. (An IDataReader's count is unknown, so it still // flows through and stages zero rows naturally.) if (GetRowCount(source) == 0) - return new SingleStoreBulkUpdateResult(m_warnings.AsReadOnly(), rowsStaged: 0, rowsMatched: ComputeRowsMatched ? 0 : -1, rowsUpdated: 0); + return CreateResult(rowsStaged: 0, rowsMatched: ComputeRowsMatched ? 0 : -1, rowsUpdated: 0); var stopwatch = Stopwatch.StartNew(); @@ -245,7 +251,16 @@ private async ValueTask WriteToServerAsync(IOBehavi tempTableName = await CreateStagingTableAsync(destinationTableName, ioBehavior, cancellationToken).ConfigureAwait(false); // Phase 2: stage the source rows into the temporary table via SingleStoreBulkCopy. - var rowsStaged = await StageDataAsync(tempTableName, source, ioBehavior, cancellationToken).ConfigureAwait(false); + var (rowsStaged, aborted) = await StageDataAsync(tempTableName, source, ioBehavior, cancellationToken).ConfigureAwait(false); + + // If the caller aborted staging via the SingleStoreRowsStaged event, abort the whole operation: do not + // run the UPDATE, so no rows are modified. Only the staging table (dropped below) was touched. + if (aborted) + { + stopwatch.Stop(); + Log.CompletedBulkUpdate(m_logger, rowsStaged, -1, 0, stopwatch.ElapsedMilliseconds); + return CreateResult(rowsStaged, rowsMatched: -1, rowsUpdated: 0); + } // Phase 3 (optional): count how many staged rows match a destination row. var rowsMatched = await ComputeMatchedRowsAsync(tempTableName, ioBehavior, cancellationToken).ConfigureAwait(false); @@ -259,7 +274,7 @@ private async ValueTask WriteToServerAsync(IOBehavi Log.CompletedBulkUpdate(m_logger, rowsStaged, rowsMatched ?? -1, rowsUpdated, stopwatch.ElapsedMilliseconds); // RowsMatched is reported as -1 when ComputeRowsMatched was false (the count was intentionally skipped). - return new SingleStoreBulkUpdateResult(m_warnings.AsReadOnly(), rowsStaged, rowsMatched ?? -1, rowsUpdated); + return CreateResult(rowsStaged, rowsMatched ?? -1, rowsUpdated); } finally { @@ -285,6 +300,13 @@ private static int GetRowCount(object source) => _ => -1, }; + /// + /// Builds the operation result, snapshotting the warnings collected so far into a new list so that a result + /// returned from one call is not mutated when the same instance is reused. + /// + private SingleStoreBulkUpdateResult CreateResult(int rowsStaged, int rowsMatched, int rowsUpdated) => + new(new List(m_warnings), rowsStaged, rowsMatched, rowsUpdated); + private void ValidateColumnMappings() { // Ensure the caller specified at least one key column. @@ -383,6 +405,19 @@ private async ValueTask ValidateSchemaAsync(string tableName, IOBehavior ioBehav if (!tableColumns.Contains(columnMapping.DestinationColumn)) throw new InvalidOperationException($"Column '{columnMapping.DestinationColumn}' does not exist in target table '{tableName}'."); } + + // Reject mapped generated (computed) columns up front with a clear error. Their value is derived from an + // expression, so they cannot be staged or assigned in the UPDATE; without this check the operation would + // fail later with a confusing server error when the UPDATE tries to write to the generated column. + var generatedColumns = await schemaDetector.GetGeneratedColumnsAsync(tableName, ioBehavior, cancellationToken).ConfigureAwait(false); + if (generatedColumns.Count != 0) + { + foreach (var columnMapping in ColumnMappings) + { + if (generatedColumns.Contains(columnMapping.DestinationColumn)) + throw new NotSupportedException($"Column '{columnMapping.DestinationColumn}' is a generated (computed) column, which cannot be updated by SingleStoreBulkUpdate."); + } + } } private List GetUpdateColumns() => @@ -545,7 +580,7 @@ private List ComputeStagingShardKey(List destinationShardKeyColu /// destination-name relationship identical between staging and the later UPDATE ... JOIN. /// /// - private async Task StageDataAsync(string tempTableName, object source, IOBehavior ioBehavior, CancellationToken cancellationToken) + private async Task<(int RowsStaged, bool Aborted)> StageDataAsync(string tempTableName, object source, IOBehavior ioBehavior, CancellationToken cancellationToken) { var bulkCopy = new SingleStoreBulkCopy(m_connection, m_transaction) { @@ -560,12 +595,16 @@ private async Task StageDataAsync(string tempTableName, object source, IOBe // Re-raise SingleStoreBulkCopy's progress event as a bulk-update staging event, and propagate the // caller's request to abort. Only subscribe when progress notifications are actually requested. + var aborted = false; void OnRowsCopied(object sender, SingleStoreRowsCopiedEventArgs e) { var args = new SingleStoreRowsStagedEventArgs { RowsStaged = e.RowsCopied }; SingleStoreRowsStaged?.Invoke(this, args); if (args.Abort) + { + aborted = true; e.Abort = true; + } } var notifyProgress = NotifyAfter > 0 && SingleStoreRowsStaged is not null; @@ -580,7 +619,7 @@ void OnRowsCopied(object sender, SingleStoreRowsCopiedEventArgs e) Log.StagedDataForBulkUpdate(m_logger, result.RowsInserted, result.Warnings.Count); - return result.RowsInserted; + return (result.RowsInserted, aborted); } finally { @@ -598,11 +637,9 @@ void OnRowsCopied(object sender, SingleStoreRowsCopiedEventArgs e) /// (WriteToServerAsync) methods rather than an overload, so the behavior is /// selected here. Calling the synchronous methods on the synchronous path keeps the whole operation inline, /// preserving the no-sync-over-async guarantee that lets the public synchronous overloads block safely. - /// For a sequence, needs the column count before it - /// enumerates the rows (taken from the owning of the first row); the sequence is - /// materialized first so a lazy source is not consumed by the peek and then re-enumerated (empty) by the bulk - /// copy. Empty input is short-circuited before staging, so the sequence is expected to be non-empty here; it - /// is still guarded so an unexpected empty sequence fails clearly rather than dereferencing a missing row. + /// For a sequence, needs the column count up front + /// (taken from the owning of the first row); the caller has already materialized any + /// lazy sequence and short-circuited empty input, so the sequence is a non-empty collection here. /// private static ValueTask StageWithBulkCopyAsync(SingleStoreBulkCopy bulkCopy, object source, IOBehavior ioBehavior, CancellationToken cancellationToken) { @@ -615,8 +652,6 @@ private static ValueTask StageWithBulkCopyAsync(Singl case IEnumerable dataRows: var rows = dataRows as IReadOnlyList ?? dataRows.ToList(); - if (rows.Count == 0) - throw new ArgumentException("Cannot stage an empty sequence of rows.", nameof(source)); var columnCount = rows[0].Table.Columns.Count; return ioBehavior == IOBehavior.Synchronous ? new ValueTask(bulkCopy.WriteToServer(rows, columnCount)) diff --git a/src/SingleStoreConnector/Utilities/SchemaDetector.cs b/src/SingleStoreConnector/Utilities/SchemaDetector.cs index 17f627cb3..dd9b54a25 100644 --- a/src/SingleStoreConnector/Utilities/SchemaDetector.cs +++ b/src/SingleStoreConnector/Utilities/SchemaDetector.cs @@ -173,6 +173,48 @@ public async Task> GetColumnTypeDefinitionsAsync(stri return definitions; } + /// + /// Gets the names of the generated (computed) columns of the specified table, as declared in + /// SHOW CREATE TABLE via an AS (...) / GENERATED ALWAYS AS (...) / COMPUTED clause. + /// + /// A case-insensitive set of generated column names, or an empty set if there are none. + public async Task> GetGeneratedColumnsAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken = default) + { + EnsureConnectionIsOpen(); + + var createTableSql = await GetCreateTableStatementAsync(tableName, ioBehavior, cancellationToken) + .ConfigureAwait(false); + + var generatedColumns = new HashSet(StringComparer.OrdinalIgnoreCase); + + var body = ExtractTableBody(createTableSql, tableName); + foreach (var item in SplitTopLevel(body)) + { + var trimmed = item.TrimStart(); + + // Only column definitions (which begin with a backtick-quoted name) can be generated columns. + if (trimmed.Length == 0 || trimmed[0] != '`') + continue; + + var (columnName, rest) = ParseQuotedNameAndRest(trimmed); + + // A generated column declares its expression with an "AS (...)" clause (optionally preceded by + // GENERATED ALWAYS), or uses the COMPUTED keyword. Look for these tokens at the top level of the + // remaining definition so a matching word inside the type, a string default or a comment is ignored. + foreach (var token in TokenizeTopLevel(rest)) + { + if (string.Equals(token, "AS", StringComparison.OrdinalIgnoreCase) || + string.Equals(token, "COMPUTED", StringComparison.OrdinalIgnoreCase)) + { + generatedColumns.Add(columnName); + break; + } + } + } + + return generatedColumns; + } + /// /// Gets column metadata for the specified table. /// diff --git a/tests/SideBySide/BulkUpdateAdvancedTests.cs b/tests/SideBySide/BulkUpdateAdvancedTests.cs index 3cb8818ab..ed243f921 100644 --- a/tests/SideBySide/BulkUpdateAdvancedTests.cs +++ b/tests/SideBySide/BulkUpdateAdvancedTests.cs @@ -51,7 +51,7 @@ public async Task FiresProgressNotifications() } [Fact] - public async Task AbortStopsStaging() + public async Task AbortLeavesDestinationUnchanged() { using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); await connection.OpenAsync(); @@ -85,12 +85,17 @@ public async Task AbortStopsStaging() }, }; - // Abort on the first progress notification; staging must stop before all rows are sent. + // Abort on the first progress notification. Aborting must cancel the whole operation, not perform a + // partial update: no rows are updated and the destination table is left unchanged. bulkUpdate.SingleStoreRowsStaged += (sender, e) => e.Abort = true; var result = await bulkUpdate.WriteToServerAsync(dataTable); - Assert.True(result.RowsStaged < 100, $"expected staging to stop early after abort, but staged {result.RowsStaged} rows"); + Assert.Equal(0, result.RowsUpdated); + + // Every row must still hold its seeded value of 0 (the update would have set value = id * 2). + using var selectCommand = new SingleStoreCommand("select count(*) from bulk_update_abort where value <> 0;", connection); + Assert.Equal(0L, Convert.ToInt64(await selectCommand.ExecuteScalarAsync())); } [Fact] diff --git a/tests/SideBySide/BulkUpdateTests.cs b/tests/SideBySide/BulkUpdateTests.cs index 48b5a56d6..16d8efb66 100644 --- a/tests/SideBySide/BulkUpdateTests.cs +++ b/tests/SideBySide/BulkUpdateTests.cs @@ -452,6 +452,92 @@ public async Task UpdatesFromDataRowSequence(bool isAsync) Assert.Equal("new3", reader.GetString(0)); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task EmptyDataRowSequenceReturnsZeroCounts(bool isAsync) + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_empty_rows; +create table bulk_update_empty_rows(id int primary key, value varchar(100));", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + }; + + // An empty DataRow sequence must behave like an empty DataTable: return zero counts without error. + var dataRows = dataTable.Select("id > 0"); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_empty_rows", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = isAsync ? await bulkUpdate.WriteToServerAsync(dataRows) : bulkUpdate.WriteToServer(dataRows); + + Assert.Equal(0, result.RowsStaged); + Assert.Equal(0, result.RowsMatched); + Assert.Equal(0, result.RowsUpdated); + } + + [Fact] + public async Task ResultWarningsAreNotMutatedByLaterCall() + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_warnings; +create table bulk_update_warnings(id int primary key, value varchar(100)); +insert into bulk_update_warnings values (1, 'original');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = { new object[] { 1, "updated" } }, + }; + + // Reuse the same instance for two operations; the first result's Warnings collection must be a snapshot + // that is unaffected by the second call (which clears and repopulates the internal warnings list). + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_warnings", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var firstResult = await bulkUpdate.WriteToServerAsync(dataTable); + var secondResult = await bulkUpdate.WriteToServerAsync(dataTable); + + // Each result must own an independent snapshot of the warnings rather than a view over a shared list that + // the second call clears and repopulates. + Assert.NotSame(firstResult.Warnings, secondResult.Warnings); + } + private static async ValueTask WriteToServerAsync(SingleStoreBulkUpdate bulkUpdate, DataTable dataTable, bool isAsync) => isAsync ? await bulkUpdate.WriteToServerAsync(dataTable) : bulkUpdate.WriteToServer(dataTable); diff --git a/tests/SideBySide/BulkUpdateValidationTests.cs b/tests/SideBySide/BulkUpdateValidationTests.cs index 81dd31f7f..410e64994 100644 --- a/tests/SideBySide/BulkUpdateValidationTests.cs +++ b/tests/SideBySide/BulkUpdateValidationTests.cs @@ -303,6 +303,45 @@ public async Task ThrowsWhenUpdatingShardKeyColumn() Assert.Contains("tenant_id", exception.Message); } + [Fact] + public async Task ThrowsWhenUpdatingGeneratedColumn() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + + // total is a generated (computed) column; its value is derived from an expression and cannot be updated. + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_generated; +create table bulk_update_generated(id int primary key, price int, total as (price * 2) persisted int);", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("total", typeof(int)), + }, + Rows = { new object[] { 1, 100 } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_generated", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "total"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(dataTable)); + Assert.Contains("generated", exception.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("total", exception.Message); + } + private static DataTable NewTable(params string[] columnNames) { var dataTable = new DataTable(); From 473129527d8ba1d0b2fc1f681fe3a420063c2fd2 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 30 Jun 2026 16:21:38 +0300 Subject: [PATCH 23/28] docs updates + additional tests --- .../SingleStoreBulkUpdate.md | 6 +- .../ComputeRowsMatched.md | 2 +- .../SingleStoreRowsStaged.md | 2 +- .../SingleStoreBulkUpdateResult.md | 4 +- .../{RowsUpdated.md => RowsAffected.md} | 4 +- .../RowsMatched.md | 4 +- docs/content/tutorials/bulk-update.md | 23 +- .../SingleStoreBulkUpdate.cs | 31 +-- .../SingleStoreBulkUpdateResult.cs | 12 +- .../Utilities/SchemaDetector.cs | 7 +- tests/SideBySide/BulkUpdateAdvancedTests.cs | 217 +++++++++++++++++- tests/SideBySide/BulkUpdateTests.cs | 18 +- tests/SideBySide/SchemaDetectorTests.cs | 58 +++++ 13 files changed, 334 insertions(+), 54 deletions(-) rename docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/{RowsUpdated.md => RowsAffected.md} (89%) diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md index 8b203f64f..605bb8bcb 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md @@ -48,7 +48,7 @@ public sealed class SingleStoreBulkUpdate | [SingleStoreBulkUpdate](SingleStoreBulkUpdate/SingleStoreBulkUpdate.md)(…) | Initializes a [`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) object with the specified connection, and optionally the active transaction. | | [BulkUpdateTimeout](SingleStoreBulkUpdate/BulkUpdateTimeout.md) { get; set; } | The number of seconds for each phase of the operation to complete before it times out (default `30`). | | [ColumnMappings](SingleStoreBulkUpdate/ColumnMappings.md) { get; } | A collection of [`SingleStoreBulkCopyColumnMapping`](./SingleStoreBulkCopyColumnMapping.md) objects that map source column ordinals onto destination column names. Every key column and at least one non-key (updated) column must be mapped. | -| [ComputeRowsMatched](SingleStoreBulkUpdate/ComputeRowsMatched.md) { get; set; } | Whether to compute [`RowsMatched`](./SingleStoreBulkUpdateResult/RowsMatched.md) via a `COUNT` query (default `true`). Set this to `false` to skip that query for better performance, in which case [`RowsMatched`](./SingleStoreBulkUpdateResult/RowsMatched.md) is reported as `-1`. | +| [ComputeRowsMatched](SingleStoreBulkUpdate/ComputeRowsMatched.md) { get; set; } | Whether to compute [`RowsMatched`](./SingleStoreBulkUpdateResult/RowsMatched.md) via a `COUNT` query (default `true`). Set this to `false` to skip that query for better performance, in which case [`RowsMatched`](./SingleStoreBulkUpdateResult/RowsMatched.md) is `null`. | | [DestinationTableName](SingleStoreBulkUpdate/DestinationTableName.md) { get; set; } | The name of the table whose rows are updated. | | [KeyColumns](SingleStoreBulkUpdate/KeyColumns.md) { get; } | The columns that identify which rows to update. They form the `JOIN` condition between the destination table and the staging table, so every key column must also appear in [`ColumnMappings`](./SingleStoreBulkUpdate/ColumnMappings.md). | | [NotifyAfter](SingleStoreBulkUpdate/NotifyAfter.md) { get; set; } | If non-zero, this specifies the number of rows to be staged before raising the [`SingleStoreRowsStaged`](./SingleStoreBulkUpdate/SingleStoreRowsStaged.md) event. This applies only to the staging phase, not to the `UPDATE` execution. | @@ -58,7 +58,9 @@ public sealed class SingleStoreBulkUpdate ## Remarks -The following restrictions apply, and `WriteToServer` throws if they are not met: [`KeyColumns`](./SingleStoreBulkUpdate/KeyColumns.md) is required and every key column must be mapped; at least one non-key column must be mapped; the source must not contain duplicate key values; shard key columns cannot be updated; reference tables are not supported; and expression column mappings are not supported. +The following restrictions apply, and `WriteToServer` throws if they are not met: [`KeyColumns`](./SingleStoreBulkUpdate/KeyColumns.md) is required and every key column must be mapped; at least one non-key column must be mapped; the source must not contain duplicate key values; shard key columns and generated (computed) columns cannot be updated; reference tables are not supported; and expression column mappings are not supported. + +An instance of this class is not thread-safe; do not share an instance across concurrent operations. This API is experimental and may change in the future. diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ComputeRowsMatched.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ComputeRowsMatched.md index 84fc33de4..ff791efee 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ComputeRowsMatched.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/ComputeRowsMatched.md @@ -1,6 +1,6 @@ # SingleStoreBulkUpdate.ComputeRowsMatched property -Whether to compute [`RowsMatched`](../SingleStoreBulkUpdateResult/RowsMatched.md) via a `COUNT` query (default `true`). Set this to `false` to skip that query for better performance, in which case [`RowsMatched`](../SingleStoreBulkUpdateResult/RowsMatched.md) is reported as `-1`. +Whether to compute [`RowsMatched`](../SingleStoreBulkUpdateResult/RowsMatched.md) via a `COUNT` query (default `true`). Set this to `false` to skip that query for better performance, in which case [`RowsMatched`](../SingleStoreBulkUpdateResult/RowsMatched.md) is `null`. ```csharp public bool ComputeRowsMatched { get; set; } diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreRowsStaged.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreRowsStaged.md index 86b914414..be37b6ed1 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreRowsStaged.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/SingleStoreRowsStaged.md @@ -10,7 +10,7 @@ public event SingleStoreRowsStagedEventHandler? SingleStoreRowsStaged; Receipt of a RowsStaged event does not imply that any rows have been sent to the server or committed. -The [`Abort`](../SingleStoreRowsStagedEventArgs/Abort.md) property can be set to `true` by the event handler to abort the staging. +The [`Abort`](../SingleStoreRowsStagedEventArgs/Abort.md) property can be set to `true` by the event handler to cancel the operation. Aborting stops staging and skips the `UPDATE`, so no rows in the destination table are modified. ## See Also diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md index 6a8ea582f..0c2c1c435 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md @@ -10,9 +10,9 @@ public sealed class SingleStoreBulkUpdateResult | name | description | | --- | --- | -| [RowsMatched](SingleStoreBulkUpdateResult/RowsMatched.md) { get; } | The number of staged rows that matched rows in the destination table, or `-1` when [`ComputeRowsMatched`](./SingleStoreBulkUpdate/ComputeRowsMatched.md) was set to `false` and the count was not computed. | +| [RowsAffected](SingleStoreBulkUpdateResult/RowsAffected.md) { get; } | The number of rows affected by the `UPDATE`, as reported by the server. | +| [RowsMatched](SingleStoreBulkUpdateResult/RowsMatched.md) { get; } | The number of staged rows that matched rows in the destination table, or `null` when [`ComputeRowsMatched`](./SingleStoreBulkUpdate/ComputeRowsMatched.md) was set to `false` and the count was not computed. | | [RowsStaged](SingleStoreBulkUpdateResult/RowsStaged.md) { get; } | The number of rows that were loaded into the staging table during the bulk update operation. | -| [RowsUpdated](SingleStoreBulkUpdateResult/RowsUpdated.md) { get; } | The number of rows affected by the `UPDATE`, as reported by the server. | | [Warnings](SingleStoreBulkUpdateResult/Warnings.md) { get; } | The warnings, if any. Users of [`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) should check that this collection is empty to avoid potential data loss from failed data type conversions. | ## See Also diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsUpdated.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsAffected.md similarity index 89% rename from docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsUpdated.md rename to docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsAffected.md index e74f620da..987ae5571 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsUpdated.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsAffected.md @@ -1,9 +1,9 @@ -# SingleStoreBulkUpdateResult.RowsUpdated property +# SingleStoreBulkUpdateResult.RowsAffected property The number of rows affected by the `UPDATE`, as reported by the server. ```csharp -public int RowsUpdated { get; } +public int RowsAffected { get; } ``` ## Remarks diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md index 5de88ea79..1801646fe 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md @@ -1,9 +1,9 @@ # SingleStoreBulkUpdateResult.RowsMatched property -The number of staged rows that matched rows in the destination table, or `-1` when [`ComputeRowsMatched`](../SingleStoreBulkUpdate/ComputeRowsMatched.md) was set to `false` and the count was not computed. +The number of staged rows that matched rows in the destination table, or `null` when [`ComputeRowsMatched`](../SingleStoreBulkUpdate/ComputeRowsMatched.md) was set to `false` and the count was not computed. ```csharp -public int RowsMatched { get; } +public int? RowsMatched { get; } ``` ## See Also diff --git a/docs/content/tutorials/bulk-update.md b/docs/content/tutorials/bulk-update.md index 1bfd2b5f8..7005194d3 100644 --- a/docs/content/tutorials/bulk-update.md +++ b/docs/content/tutorials/bulk-update.md @@ -92,9 +92,9 @@ Interpreting the result `WriteToServerAsync` returns a `SingleStoreBulkUpdateResult`: * `RowsStaged` — the number of source rows loaded into the staging table. -* `RowsMatched` — the number of staged rows that matched a row in the destination table. This is `-1` when +* `RowsMatched` — the number of staged rows that matched a row in the destination table. This is `null` when `ComputeRowsMatched` is set to `false` (see below). -* `RowsUpdated` — the number of rows affected by the `UPDATE`, as reported by the server. Its exact meaning depends on +* `RowsAffected` — the number of rows affected by the `UPDATE`, as reported by the server. Its exact meaning depends on the connection's `UseAffectedRows` setting: with the default (`UseAffectedRows=false`) it counts the rows *matched* by the update — including rows that already held the new values — so it typically equals `RowsMatched`; with `UseAffectedRows=true` it counts only the rows whose values actually *changed*. @@ -105,10 +105,11 @@ Performance ----------- * Set `ComputeRowsMatched = false` to skip the extra `COUNT(*)` query that populates `RowsMatched`. When disabled, - `RowsMatched` is reported as `-1`. + `RowsMatched` is `null`. * Set `BulkUpdateTimeout` (in seconds) to control how long each phase may run. * Set `NotifyAfter` to a non-zero value to receive `SingleStoreRowsStaged` events while rows are being staged; the - event handler can set `Abort = true` to stop staging early. + event handler can set `Abort = true` to cancel the operation. Aborting stops staging and skips the `UPDATE` + entirely, so no rows in the destination table are modified. Limitations ----------- @@ -119,9 +120,23 @@ Limitations * At least one non-key column must be mapped, so there is a column to update. * Duplicate key values in the source data are rejected; they would collide in the staging table's primary key. * Shard key columns cannot be updated, because SingleStore does not allow updating a shard key. +* Generated (computed) columns cannot be updated, and are rejected if mapped. * Reference tables are not supported as the destination. * Expression column mappings (a `SingleStoreBulkCopyColumnMapping` with an `Expression`) are not supported. +Other things to be aware of: + +* **Key column types.** The key columns become the primary key of the staging table, so they must be types that + SingleStore allows in a primary key. Large `TEXT`/`BLOB`/`JSON`/spatial columns are not usable as key columns. +* **Required privileges.** In addition to `UPDATE` on the destination table, the connection needs permission to run + `SHOW CREATE TABLE`, `SHOW INDEXES`, and a schema-only `SELECT` against it, because the operation inspects the + table's schema before updating. +* **`IDataReader` source.** When the source is an `IDataReader`, it must be opened on a *different* connection than + the one used for the bulk update. The update connection runs schema queries, creates the staging table, and loads + data, so it cannot have an open reader on it at the same time. +* **Thread safety.** A `SingleStoreBulkUpdate` instance is not thread-safe. Do not share an instance across concurrent + operations. + Transactions ------------ diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index f9aeaedee..17f14b569 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -52,8 +52,9 @@ namespace SingleStoreConnector; /// /// The following restrictions apply, and WriteToServer throws if they are not met: /// is required and every key column must be mapped; at least one non-key column must be mapped; the source must not -/// contain duplicate key values; shard key columns cannot be updated; reference tables are not supported; and -/// expression column mappings are not supported. +/// contain duplicate key values; shard key columns and generated (computed) columns cannot be updated; reference tables +/// are not supported; and expression column mappings are not supported. +/// An instance of this class is not thread-safe; do not share an instance across concurrent operations. /// This API is experimental and may change in the future. /// public sealed class SingleStoreBulkUpdate @@ -105,7 +106,7 @@ public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransa /// /// Whether to compute via a COUNT query (default true). /// Set this to false to skip that query for better performance, in which case - /// is reported as -1. + /// is null. /// public bool ComputeRowsMatched { get; set; } = true; @@ -114,7 +115,9 @@ public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransa /// /// /// Receipt of a RowsStaged event does not imply that any rows have been sent to the server or committed. - /// The property can be set to true by the event handler to abort the staging. + /// The property can be set to true by the event handler + /// to cancel the operation. Aborting stops staging and skips the UPDATE, so no rows in the destination table + /// are modified. /// public event SingleStoreRowsStagedEventHandler? SingleStoreRowsStaged; @@ -225,7 +228,7 @@ private async ValueTask WriteToServerAsync(IOBehavi // opening the connection and creating a staging table. (An IDataReader's count is unknown, so it still // flows through and stages zero rows naturally.) if (GetRowCount(source) == 0) - return CreateResult(rowsStaged: 0, rowsMatched: ComputeRowsMatched ? 0 : -1, rowsUpdated: 0); + return CreateResult(rowsStaged: 0, rowsMatched: ComputeRowsMatched ? 0 : null, rowsAffected: 0); var stopwatch = Stopwatch.StartNew(); @@ -259,7 +262,7 @@ private async ValueTask WriteToServerAsync(IOBehavi { stopwatch.Stop(); Log.CompletedBulkUpdate(m_logger, rowsStaged, -1, 0, stopwatch.ElapsedMilliseconds); - return CreateResult(rowsStaged, rowsMatched: -1, rowsUpdated: 0); + return CreateResult(rowsStaged, rowsMatched: null, rowsAffected: 0); } // Phase 3 (optional): count how many staged rows match a destination row. @@ -268,13 +271,13 @@ private async ValueTask WriteToServerAsync(IOBehavi Log.LargeUnmatchedCountForBulkUpdate(m_logger, rowsStaged, matched, rowsStaged - matched); // Phase 4: run the UPDATE ... JOIN that copies the non-key values into the matching rows. - var rowsUpdated = await ExecuteUpdateAsync(tempTableName, ioBehavior, cancellationToken).ConfigureAwait(false); + var rowsAffected = await ExecuteUpdateAsync(tempTableName, ioBehavior, cancellationToken).ConfigureAwait(false); stopwatch.Stop(); - Log.CompletedBulkUpdate(m_logger, rowsStaged, rowsMatched ?? -1, rowsUpdated, stopwatch.ElapsedMilliseconds); + Log.CompletedBulkUpdate(m_logger, rowsStaged, rowsMatched ?? -1, rowsAffected, stopwatch.ElapsedMilliseconds); - // RowsMatched is reported as -1 when ComputeRowsMatched was false (the count was intentionally skipped). - return CreateResult(rowsStaged, rowsMatched ?? -1, rowsUpdated); + // RowsMatched is null when ComputeRowsMatched was false (the count was intentionally skipped). + return CreateResult(rowsStaged, rowsMatched, rowsAffected); } finally { @@ -304,8 +307,8 @@ private static int GetRowCount(object source) => /// Builds the operation result, snapshotting the warnings collected so far into a new list so that a result /// returned from one call is not mutated when the same instance is reused. /// - private SingleStoreBulkUpdateResult CreateResult(int rowsStaged, int rowsMatched, int rowsUpdated) => - new(new List(m_warnings), rowsStaged, rowsMatched, rowsUpdated); + private SingleStoreBulkUpdateResult CreateResult(int rowsStaged, int? rowsMatched, int rowsAffected) => + new(new List(m_warnings), rowsStaged, rowsMatched, rowsAffected); private void ValidateColumnMappings() { @@ -374,7 +377,7 @@ private void ValidateColumnMappings() private async ValueTask ValidateSchemaAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken) { - var schemaDetector = new SchemaDetector(m_connection, m_transaction); + var schemaDetector = new SchemaDetector(m_connection, m_transaction, BulkUpdateTimeout); // TODO: make changes to support the solutions described here -- https://docs.singlestore.com/cloud/reference/troubleshooting-reference/query-errors/error-1706-hy-000-feature-multi-table-update-delete-with-a-reference-table-as-target-table-is-not-supported-by-memsql/ if (await schemaDetector.IsReferenceTableAsync(tableName, ioBehavior, cancellationToken).ConfigureAwait(false)) @@ -461,7 +464,7 @@ private async Task CreateStagingTableAsync(string destinationTableName, // starts with a letter regardless of the GUID's first hex digit. var tempTableName = $"_bulk_update_staging_g{Guid.NewGuid():N}"; - var schemaDetector = new SchemaDetector(m_connection, m_transaction); + var schemaDetector = new SchemaDetector(m_connection, m_transaction, BulkUpdateTimeout); // Pull the exact, server-rendered type definition for every column so the staging columns are // byte-for-byte type compatible with the destination (see remarks). diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs b/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs index 92d5e2b9f..98c46c56f 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs @@ -17,10 +17,10 @@ public sealed class SingleStoreBulkUpdateResult public int RowsStaged { get; } /// - /// The number of staged rows that matched rows in the destination table, or -1 when + /// The number of staged rows that matched rows in the destination table, or null when /// was set to false and the count was not computed. /// - public int RowsMatched { get; } + public int? RowsMatched { get; } /// /// The number of rows affected by the UPDATE, as reported by the server. @@ -29,17 +29,17 @@ public sealed class SingleStoreBulkUpdateResult /// setting. With the default (UseAffectedRows=false) this is the number of rows matched by the update — /// including rows that already held the new values — and therefore typically equals . With /// UseAffectedRows=true it is the number of rows whose values actually changed. - public int RowsUpdated { get; } + public int RowsAffected { get; } internal SingleStoreBulkUpdateResult( IReadOnlyList warnings, int rowsStaged, - int rowsMatched, - int rowsUpdated) + int? rowsMatched, + int rowsAffected) { Warnings = warnings; RowsStaged = rowsStaged; RowsMatched = rowsMatched; - RowsUpdated = rowsUpdated; + RowsAffected = rowsAffected; } } diff --git a/src/SingleStoreConnector/Utilities/SchemaDetector.cs b/src/SingleStoreConnector/Utilities/SchemaDetector.cs index dd9b54a25..004e9c43c 100644 --- a/src/SingleStoreConnector/Utilities/SchemaDetector.cs +++ b/src/SingleStoreConnector/Utilities/SchemaDetector.cs @@ -5,7 +5,7 @@ namespace SingleStoreConnector.Utilities; -internal sealed class SchemaDetector(SingleStoreConnection connection, SingleStoreTransaction? transaction = null) +internal sealed class SchemaDetector(SingleStoreConnection connection, SingleStoreTransaction? transaction = null, int commandTimeout = 0) { private static readonly Regex referenceTableRegex = new(@"CREATE\s+(?:(?:ROWSTORE|COLUMNSTORE)\s+)?REFERENCE\s+TABLE", @@ -20,6 +20,8 @@ internal sealed class SchemaDetector(SingleStoreConnection connection, SingleSto private readonly SingleStoreTransaction? m_transaction = transaction; + private readonly int m_commandTimeout = commandTimeout; + /// /// Detects if the specified table is a reference table. /// @@ -48,6 +50,7 @@ public async Task> GetShardKeyColumnsAsync(string tableName, IOBeha using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = m_transaction; + cmd.CommandTimeout = m_commandTimeout; cmd.CommandText = $"SHOW INDEXES FROM {IdentifierHelper.QuoteQualifiedIdentifier(tableName)}"; await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.Default, ioBehavior, cancellationToken) @@ -111,6 +114,7 @@ private async Task GetCreateTableStatementAsync(string tableName, IOBeha using var cmd = m_connection.CreateCommand(); cmd.Transaction = m_transaction; + cmd.CommandTimeout = m_commandTimeout; cmd.CommandText = $"SHOW CREATE TABLE {IdentifierHelper.QuoteQualifiedIdentifier(tableName)}"; await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.Default, ioBehavior, cancellationToken) @@ -224,6 +228,7 @@ public async Task GetTableSchemaAsync(string tableName, IOBehavior io using var cmd = m_connection.CreateCommand(); cmd.Transaction = m_transaction; + cmd.CommandTimeout = m_commandTimeout; cmd.CommandText = $"SELECT * FROM {IdentifierHelper.QuoteQualifiedIdentifier(tableName)} LIMIT 0"; await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SchemaOnly, ioBehavior, cancellationToken) diff --git a/tests/SideBySide/BulkUpdateAdvancedTests.cs b/tests/SideBySide/BulkUpdateAdvancedTests.cs index ed243f921..26a1a0248 100644 --- a/tests/SideBySide/BulkUpdateAdvancedTests.cs +++ b/tests/SideBySide/BulkUpdateAdvancedTests.cs @@ -91,7 +91,7 @@ public async Task AbortLeavesDestinationUnchanged() var result = await bulkUpdate.WriteToServerAsync(dataTable); - Assert.Equal(0, result.RowsUpdated); + Assert.Equal(0, result.RowsAffected); // Every row must still hold its seeded value of 0 (the update would have set value = id * 2). using var selectCommand = new SingleStoreCommand("select count(*) from bulk_update_abort where value <> 0;", connection); @@ -142,6 +142,60 @@ public async Task DoesNotFireProgressNotificationsWhenNotifyAfterIsZero() Assert.Equal(0, eventCount); } + [Theory] + [InlineData(false, 2)] // CLIENT_FOUND_ROWS: RowsAffected counts matched rows, including the unchanged one + [InlineData(true, 1)] // RowsAffected counts only the row whose value actually changed + public async Task RowsAffectedDependsOnUseAffectedRows(bool useAffectedRows, int expectedRowsAffected) + { + var csb = new SingleStoreConnectionStringBuilder(database.Connection.ConnectionString) + { + AllowLoadLocalInfile = true, + UseAffectedRows = useAffectedRows, + }; + + using var connection = new SingleStoreConnection(csb.ConnectionString); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_affected; +create table bulk_update_affected(id int primary key, value varchar(100)); +insert into bulk_update_affected values (1, 'unchanged'), (2, 'before');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + // Row 1 is updated to the value it already holds (no change); row 2 changes. Both match the join, so the + // difference between the two settings is whether RowsAffected counts the unchanged matched row. + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = + { + new object[] { 1, "unchanged" }, + new object[] { 2, "after" }, + }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_affected", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + // RowsMatched always reflects the join (both rows) regardless of the connection setting. + Assert.Equal(2, result.RowsMatched); + Assert.Equal(expectedRowsAffected, result.RowsAffected); + } + [Fact] public async Task SkipsMatchCountWhenComputeRowsMatchedIsFalse() { @@ -179,8 +233,8 @@ public async Task SkipsMatchCountWhenComputeRowsMatchedIsFalse() var result = await bulkUpdate.WriteToServerAsync(dataTable); Assert.Equal(1, result.RowsStaged); - Assert.Equal(-1, result.RowsMatched); // -1 signals the COUNT was intentionally skipped - Assert.Equal(1, result.RowsUpdated); + Assert.Null(result.RowsMatched); // null signals the COUNT was intentionally skipped + Assert.Equal(1, result.RowsAffected); using var selectCommand = new SingleStoreCommand("select value from bulk_update_nocount where id = 1;", connection); Assert.Equal("updated", await selectCommand.ExecuteScalarAsync()); @@ -225,7 +279,7 @@ public async Task HandlesSpecialCharactersInIdentifiers() Assert.Equal(1, result.RowsStaged); Assert.Equal(1, result.RowsMatched); - Assert.Equal(1, result.RowsUpdated); + Assert.Equal(1, result.RowsAffected); using var selectCommand = new SingleStoreCommand("select `user name`, `select` from `my-special-table` where `user-id` = 1;", connection); using var reader = await selectCommand.ExecuteReaderAsync(); @@ -276,7 +330,7 @@ public async Task RoundTripsLossyColumnTypes() var result = await bulkUpdate.WriteToServerAsync(dataTable); - Assert.Equal(1, result.RowsUpdated); + Assert.Equal(1, result.RowsAffected); Assert.Empty(result.Warnings); using var selectCommand = new SingleStoreCommand("select amount, quantity, status from bulk_update_lossy where id = 1;", connection); @@ -287,6 +341,148 @@ public async Task RoundTripsLossyColumnTypes() Assert.Equal("inactive", reader.GetString(2)); } + [Fact] + public async Task RoundTripsBinaryColumn() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + + // VARBINARY is staged directly (not via an UNHEX expression mapping the caller provides); SingleStoreBulkCopy + // applies the hex conversion itself based on the staging column's type, so the bytes must round-trip exactly. + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_binary; +create table bulk_update_binary(id int primary key, payload varbinary(16)); +insert into bulk_update_binary values (1, NULL);", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var payload = new byte[] { 0x00, 0x01, 0xFE, 0xFF, 0x10, 0x20 }; + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("payload", typeof(byte[])), + }, + Rows = { new object[] { 1, payload } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_binary", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "payload"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(1, result.RowsAffected); + Assert.Empty(result.Warnings); + + using var selectCommand = new SingleStoreCommand("select payload from bulk_update_binary where id = 1;", connection); + Assert.Equal(payload, (byte[]) (await selectCommand.ExecuteScalarAsync())!); + } + + [Fact] + public async Task RoundTripsBitColumn() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + + // BIT is staged directly; SingleStoreBulkCopy converts the staged value with CAST(... AS UNSIGNED) based on + // the staging column's type, so the bit value must round-trip. + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_bit; +create table bulk_update_bit(id int primary key, flags bit(8)); +insert into bulk_update_bit values (1, b'00000000');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("flags", typeof(ulong)), + }, + Rows = { new object[] { 1, 0b1010_0101UL } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_bit", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "flags"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(1, result.RowsAffected); + Assert.Empty(result.Warnings); + + using var selectCommand = new SingleStoreCommand("select flags from bulk_update_bit where id = 1;", connection); + using var reader = await selectCommand.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + Assert.Equal(0b1010_0101UL, reader.GetUInt64(0)); + } + + [SkippableFact(ServerFeatures.ExtendedDataTypes)] + public async Task RoundTripsVectorColumn() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + + // VECTOR carries both a dimension count and an element type. The staging column mirrors VECTOR(3, F32) + // verbatim, and SingleStoreBulkCopy reconstructs the value with UNHEX(...):>VECTOR(3, F32) from that + // staging column's metadata, so the vector must round-trip exactly. + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_vector; +create table bulk_update_vector(id int primary key, embedding vector(3, F32)); +insert into bulk_update_vector values (1, '[0,0,0]');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var embedding = new[] { 1.5f, -2.5f, 3.25f }; + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("embedding", typeof(float[])), + }, + Rows = { new object[] { 1, embedding } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_vector", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "embedding"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(1, result.RowsAffected); + Assert.Empty(result.Warnings); + + using var selectCommand = new SingleStoreCommand("select embedding from bulk_update_vector where id = 1;", connection); + using var reader = await selectCommand.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + Assert.Equal(embedding, reader.GetFieldValue>(0).ToArray()); + } + [Fact] public async Task UpdatesRowsFromDataReader() { @@ -302,7 +498,8 @@ public async Task UpdatesRowsFromDataReader() await cmd.ExecuteNonQueryAsync(); } - // Read source rows on a second connection so the reader is independent of the update connection. + // Read source rows on a second connection: an IDataReader source must not be open on the bulk update's own + // connection, which needs to run schema queries, create the staging table, and load data. using var readerConnection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); await readerConnection.OpenAsync(); using var selectCommand = new SingleStoreCommand("select id, value from bulk_update_reader_src order by id;", readerConnection); @@ -323,7 +520,7 @@ public async Task UpdatesRowsFromDataReader() Assert.Equal(2, result.RowsStaged); Assert.Equal(2, result.RowsMatched); - Assert.Equal(2, result.RowsUpdated); + Assert.Equal(2, result.RowsAffected); using var verifyCommand = new SingleStoreCommand("select value from bulk_update_reader_dest where id = 1;", connection); Assert.Equal("new1", await verifyCommand.ExecuteScalarAsync()); @@ -372,7 +569,7 @@ public async Task UpdatesWhenShardKeyAlignsWithKeyColumns() var result = await bulkUpdate.WriteToServerAsync(dataTable); Assert.Equal(2, result.RowsMatched); - Assert.Equal(2, result.RowsUpdated); + Assert.Equal(2, result.RowsAffected); using var selectCommand = new SingleStoreCommand("select value from bulk_update_shard_aligned where id = 1;", connection); Assert.Equal("new1", await selectCommand.ExecuteScalarAsync()); @@ -427,7 +624,7 @@ public async Task UpdatesWhenCompositeShardKeyOrderDiffersFromKeyColumns() var result = await bulkUpdate.WriteToServerAsync(dataTable); Assert.Equal(2, result.RowsMatched); - Assert.Equal(2, result.RowsUpdated); + Assert.Equal(2, result.RowsAffected); using var selectCommand = new SingleStoreCommand("select value from bulk_update_shard_composite where tenant_id = 1 and user_id = 100;", connection); Assert.Equal("new1", await selectCommand.ExecuteScalarAsync()); @@ -477,7 +674,7 @@ public async Task UpdatesWhenShardKeyDoesNotAlignWithKeyColumns() var result = await bulkUpdate.WriteToServerAsync(dataTable); Assert.Equal(2, result.RowsMatched); - Assert.Equal(2, result.RowsUpdated); + Assert.Equal(2, result.RowsAffected); using var selectCommand = new SingleStoreCommand("select value from bulk_update_shard_mismatch where id = 1;", connection); Assert.Equal("new1", await selectCommand.ExecuteScalarAsync()); diff --git a/tests/SideBySide/BulkUpdateTests.cs b/tests/SideBySide/BulkUpdateTests.cs index 16d8efb66..454096495 100644 --- a/tests/SideBySide/BulkUpdateTests.cs +++ b/tests/SideBySide/BulkUpdateTests.cs @@ -45,7 +45,7 @@ public async Task UpdatesMatchingRowsWithSingleKey(bool isAsync) Assert.Equal(2, result.RowsStaged); Assert.Equal(2, result.RowsMatched); - Assert.Equal(2, result.RowsUpdated); + Assert.Equal(2, result.RowsAffected); using var selectCommand = new SingleStoreCommand("select status from bulk_update_basic order by id;", connection); using var reader = await selectCommand.ExecuteReaderAsync(); @@ -102,7 +102,7 @@ public async Task UpdatesMatchingRowsWithCompositeKey(bool isAsync) Assert.Equal(2, result.RowsStaged); Assert.Equal(2, result.RowsMatched); - Assert.Equal(2, result.RowsUpdated); + Assert.Equal(2, result.RowsAffected); using var selectCommand = new SingleStoreCommand("select email from bulk_update_composite where tenant_id = 1 and user_id = 100;", connection); Assert.Equal("new100@tenant1.com", await selectCommand.ExecuteScalarAsync()); @@ -154,7 +154,7 @@ public async Task UpdatesNothingWhenNoKeysMatch(bool isAsync) Assert.Equal(1, result.RowsStaged); Assert.Equal(0, result.RowsMatched); - Assert.Equal(0, result.RowsUpdated); + Assert.Equal(0, result.RowsAffected); using var selectCommand = new SingleStoreCommand("select value from bulk_update_nomatch where id = 1;", connection); Assert.Equal("original", await selectCommand.ExecuteScalarAsync()); @@ -244,7 +244,7 @@ public async Task RollbackLeavesDataUnchanged(bool isAsync) }; var result = await WriteToServerAsync(bulkUpdate, dataTable, isAsync); - Assert.Equal(1, result.RowsUpdated); + Assert.Equal(1, result.RowsAffected); await transaction.RollbackAsync(); } @@ -290,7 +290,7 @@ public async Task EmptyInputReturnsZeroCounts(bool isAsync) Assert.Equal(0, result.RowsStaged); Assert.Equal(0, result.RowsMatched); - Assert.Equal(0, result.RowsUpdated); + Assert.Equal(0, result.RowsAffected); } [Theory] @@ -334,7 +334,7 @@ public async Task CommitPersistsChanges(bool isAsync) }; var result = await WriteToServerAsync(bulkUpdate, dataTable, isAsync); - Assert.Equal(1, result.RowsUpdated); + Assert.Equal(1, result.RowsAffected); await transaction.CommitAsync(); } @@ -385,7 +385,7 @@ public async Task OpensAndClosesConnectionWhenInitiallyClosed(bool isAsync) Assert.Equal(ConnectionState.Closed, connection.State); var result = await WriteToServerAsync(bulkUpdate, dataTable, isAsync); - Assert.Equal(1, result.RowsUpdated); + Assert.Equal(1, result.RowsAffected); Assert.Equal(ConnectionState.Closed, connection.State); await connection.OpenAsync(); @@ -440,7 +440,7 @@ public async Task UpdatesFromDataRowSequence(bool isAsync) Assert.Equal(2, result.RowsStaged); Assert.Equal(2, result.RowsMatched); - Assert.Equal(2, result.RowsUpdated); + Assert.Equal(2, result.RowsAffected); using var selectCommand = new SingleStoreCommand("select value from bulk_update_datarows order by id;", connection); using var reader = await selectCommand.ExecuteReaderAsync(); @@ -492,7 +492,7 @@ public async Task EmptyDataRowSequenceReturnsZeroCounts(bool isAsync) Assert.Equal(0, result.RowsStaged); Assert.Equal(0, result.RowsMatched); - Assert.Equal(0, result.RowsUpdated); + Assert.Equal(0, result.RowsAffected); } [Fact] diff --git a/tests/SideBySide/SchemaDetectorTests.cs b/tests/SideBySide/SchemaDetectorTests.cs index 5ab1e7a9c..2cf0056bc 100644 --- a/tests/SideBySide/SchemaDetectorTests.cs +++ b/tests/SideBySide/SchemaDetectorTests.cs @@ -240,6 +240,64 @@ public async Task GetColumnTypeDefinitionsExcludesColumnOptions() Assert.DoesNotContain("DEFAULT", noteDefinition, StringComparison.OrdinalIgnoreCase); } + [Fact] + public async Task GetColumnTypeDefinitionsHandlesCommentWithSpecialCharacters() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "schema_detector_coldefs_comment"; + + // A column comment can contain the characters the parser keys on: top-level commas (which separate + // columns), parentheses (which the type-args/paren tracking uses), and quotes/backticks. None of these + // must confuse SplitTopLevel, the column-name parsing, or the type extraction. The comment is a column + // option, so it must also be excluded from the returned type definition. + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, value VARCHAR(50) COMMENT 'a, b (c) ''d'' `e`', other INT"); + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + // All three columns must be discovered (the comment must not have swallowed "other" or split it early). + Assert.Equal(3, definitions.Count); + Assert.Contains("varchar(50)", definitions["value"], StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("comment", definitions["value"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("int", definitions["other"], StringComparison.OrdinalIgnoreCase); + } + + [SkippableFact(ServerFeatures.Json)] + public async Task GetColumnTypeDefinitionsHandlesJsonColumn() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "schema_detector_coldefs_json"; + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, data JSON"); + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + Assert.Contains("json", definitions["data"], StringComparison.OrdinalIgnoreCase); + } + + [SkippableFact(ServerFeatures.ExtendedDataTypes)] + public async Task GetColumnTypeDefinitionsHandlesVectorColumn() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "schema_detector_coldefs_vector"; + + // VECTOR has parenthesised arguments (dimension count, and possibly an element type) that must be kept as + // part of the type token rather than split on or truncated. + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, embedding VECTOR(4)"); + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + Assert.Contains("vector", definitions["embedding"], StringComparison.OrdinalIgnoreCase); + Assert.Contains("4", definitions["embedding"], StringComparison.OrdinalIgnoreCase); + } + [Fact] public async Task IsReferenceTableThrowsForNonexistentTable() { From a918d2244c94f4727530d3db6ab0369978b7fa89 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Tue, 30 Jun 2026 18:14:10 +0300 Subject: [PATCH 24/28] revert change to packages.lock.json --- .../packages.lock.json | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json b/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json index 6ab93bf5e..e466b627d 100644 --- a/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json +++ b/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json @@ -4,15 +4,15 @@ "net10.0": { "Microsoft.DotNet.ILCompiler": { "type": "Direct", - "requested": "[10.0.8, )", - "resolved": "10.0.8", - "contentHash": "IG4ZS8fP7whR7OeFjC2mxq9n/yd/QsUliHdQ/Kp0OJ+RObeSJx4HLLGs7/Sxghk/U+S3rmpUOsgLheqBcqlMUw==" + "requested": "[10.0.7, )", + "resolved": "10.0.7", + "contentHash": "S53itVb3/l5wEyvdOukwxGzRtw7JSKuh2GurW3W5R57w4nxbZZ+7qCu+5jVlowLmgGrQ7yxJLZdpJc1Vpv8aig==" }, "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[10.0.8, )", - "resolved": "10.0.8", - "contentHash": "SyfHfPwnylzDkj7u6KiwiIgVP+gc1Ud0ybomjT/sOGJLrGC6KLWR3zUvnTTcwwGK3YqGXsFWEl0opREH11H8pg==" + "requested": "[10.0.7, )", + "resolved": "10.0.7", + "contentHash": "Y0O2XrQiNYsW9UM2GhH/8faornu9pYXdd9YBcb/CrBnCOw5QtFqgh8bFd5zjxbrdoSVxGPUn1hWSrKtD0SEPzg==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", @@ -80,15 +80,15 @@ "net8.0": { "Microsoft.DotNet.ILCompiler": { "type": "Direct", - "requested": "[8.0.27, )", - "resolved": "8.0.27", - "contentHash": "2RDzeGQisWc0e4uJpT8WmMQ8Xccf4bZr4bVVV9iGnfnEZCPo//G01MLLCmGHScwLX4zCegNZRNftgUfgWlBb+g==" + "requested": "[8.0.26, )", + "resolved": "8.0.26", + "contentHash": "TopkZJbGfDfqNRQatno3/UZh8t/mo4tlglGVtcUAZDEplx/DhmBUSoT7UM7M6W0/ah1Wbe6e46FOLDyHL8K4nQ==" }, "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.27, )", - "resolved": "8.0.27", - "contentHash": "rQi9TxifHRnXP7lVRZH05DxD2/XGbJp12q0ozcbrlBlBnyyzssFTH/2vLhtKWUp2CT1qVscTrcYTFiwTyKPKRg==" + "requested": "[8.0.26, )", + "resolved": "8.0.26", + "contentHash": "o7/yVssM2r9Wyln2s9edBd5ANZXqdSdBI+g7JqXkyJmXrhs2WsJp25K5yPnYrTgdKBCjKB8bg+O2oew4sgzFaA==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", From fbecd203f7e2d2b4df3434a9433444913d2048c2 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 1 Jul 2026 14:42:13 +0300 Subject: [PATCH 25/28] added config snapshot + minor changes --- .../SingleStoreBulkUpdate.md | 4 +- .../BulkUpdateTimeout.md | 2 +- .../SingleStoreBulkUpdateResult.md | 2 +- .../RowsMatched.md | 6 +- docs/content/tutorials/bulk-update.md | 36 ++++- src/SingleStoreConnector/Logging/EventIds.cs | 7 +- src/SingleStoreConnector/Logging/Log.cs | 3 - .../SingleStoreBulkUpdate.cs | 146 +++++++++++------- .../SingleStoreBulkUpdateResult.cs | 5 +- .../Utilities/SchemaDetector.cs | 11 +- tests/SideBySide/BulkUpdateTests.cs | 102 ++++++++++++ tests/SideBySide/BulkUpdateValidationTests.cs | 40 +++++ tests/SideBySide/SchemaDetectorTests.cs | 39 +++++ .../packages.lock.json | 24 +-- 14 files changed, 345 insertions(+), 82 deletions(-) diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md index 605bb8bcb..08bde4227 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate.md @@ -46,7 +46,7 @@ public sealed class SingleStoreBulkUpdate | name | description | | --- | --- | | [SingleStoreBulkUpdate](SingleStoreBulkUpdate/SingleStoreBulkUpdate.md)(…) | Initializes a [`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) object with the specified connection, and optionally the active transaction. | -| [BulkUpdateTimeout](SingleStoreBulkUpdate/BulkUpdateTimeout.md) { get; set; } | The number of seconds for each phase of the operation to complete before it times out (default `30`). | +| [BulkUpdateTimeout](SingleStoreBulkUpdate/BulkUpdateTimeout.md) { get; set; } | The number of seconds for each phase of the operation to complete before it times out, or `0` for no timeout (the default). A single bulk update can spend a long time staging, counting, or updating, so a finite timeout should be chosen deliberately. | | [ColumnMappings](SingleStoreBulkUpdate/ColumnMappings.md) { get; } | A collection of [`SingleStoreBulkCopyColumnMapping`](./SingleStoreBulkCopyColumnMapping.md) objects that map source column ordinals onto destination column names. Every key column and at least one non-key (updated) column must be mapped. | | [ComputeRowsMatched](SingleStoreBulkUpdate/ComputeRowsMatched.md) { get; set; } | Whether to compute [`RowsMatched`](./SingleStoreBulkUpdateResult/RowsMatched.md) via a `COUNT` query (default `true`). Set this to `false` to skip that query for better performance, in which case [`RowsMatched`](./SingleStoreBulkUpdateResult/RowsMatched.md) is `null`. | | [DestinationTableName](SingleStoreBulkUpdate/DestinationTableName.md) { get; set; } | The name of the table whose rows are updated. | @@ -58,7 +58,7 @@ public sealed class SingleStoreBulkUpdate ## Remarks -The following restrictions apply, and `WriteToServer` throws if they are not met: [`KeyColumns`](./SingleStoreBulkUpdate/KeyColumns.md) is required and every key column must be mapped; at least one non-key column must be mapped; the source must not contain duplicate key values; shard key columns and generated (computed) columns cannot be updated; reference tables are not supported; and expression column mappings are not supported. +The following restrictions apply, and `WriteToServer` throws if they are not met: [`KeyColumns`](./SingleStoreBulkUpdate/KeyColumns.md) is required and every key column must be mapped; at least one non-key column must be mapped; the source must not contain duplicate key values; shard key columns cannot be updated; generated (computed) columns cannot be mapped; reference tables are not supported; and expression column mappings are not supported. An instance of this class is not thread-safe; do not share an instance across concurrent operations. diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkUpdateTimeout.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkUpdateTimeout.md index 323b4f455..c42497657 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkUpdateTimeout.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdate/BulkUpdateTimeout.md @@ -1,6 +1,6 @@ # SingleStoreBulkUpdate.BulkUpdateTimeout property -The number of seconds for each phase of the operation to complete before it times out (default `30`). +The number of seconds for each phase of the operation to complete before it times out, or `0` for no timeout (the default). A single bulk update can spend a long time staging, counting, or updating, so a finite timeout should be chosen deliberately. ```csharp public int BulkUpdateTimeout { get; set; } diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md index 0c2c1c435..2ba56162c 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult.md @@ -11,7 +11,7 @@ public sealed class SingleStoreBulkUpdateResult | name | description | | --- | --- | | [RowsAffected](SingleStoreBulkUpdateResult/RowsAffected.md) { get; } | The number of rows affected by the `UPDATE`, as reported by the server. | -| [RowsMatched](SingleStoreBulkUpdateResult/RowsMatched.md) { get; } | The number of staged rows that matched rows in the destination table, or `null` when [`ComputeRowsMatched`](./SingleStoreBulkUpdate/ComputeRowsMatched.md) was set to `false` and the count was not computed. | +| [RowsMatched](SingleStoreBulkUpdateResult/RowsMatched.md) { get; } | The number of destination rows matched by the join on the key columns, or `null` when [`ComputeRowsMatched`](./SingleStoreBulkUpdate/ComputeRowsMatched.md) was set to `false` and the count was not computed. | | [RowsStaged](SingleStoreBulkUpdateResult/RowsStaged.md) { get; } | The number of rows that were loaded into the staging table during the bulk update operation. | | [Warnings](SingleStoreBulkUpdateResult/Warnings.md) { get; } | The warnings, if any. Users of [`SingleStoreBulkUpdate`](./SingleStoreBulkUpdate.md) should check that this collection is empty to avoid potential data loss from failed data type conversions. | diff --git a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md index 1801646fe..7c76223a5 100644 --- a/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md +++ b/docs/content/api/SingleStoreConnector/SingleStoreBulkUpdateResult/RowsMatched.md @@ -1,11 +1,15 @@ # SingleStoreBulkUpdateResult.RowsMatched property -The number of staged rows that matched rows in the destination table, or `null` when [`ComputeRowsMatched`](../SingleStoreBulkUpdate/ComputeRowsMatched.md) was set to `false` and the count was not computed. +The number of destination rows matched by the join on the key columns, or `null` when [`ComputeRowsMatched`](../SingleStoreBulkUpdate/ComputeRowsMatched.md) was set to `false` and the count was not computed. ```csharp public int? RowsMatched { get; } ``` +## Remarks + +When the destination key columns are unique this equals the number of staged rows that matched a destination row. If the key columns are not unique, a single staged row can match several destination rows, so this (and [`RowsAffected`](./RowsAffected.md)) can exceed the number of staged rows. + ## See Also * class [SingleStoreBulkUpdateResult](../SingleStoreBulkUpdateResult.md) diff --git a/docs/content/tutorials/bulk-update.md b/docs/content/tutorials/bulk-update.md index 7005194d3..229d9d24e 100644 --- a/docs/content/tutorials/bulk-update.md +++ b/docs/content/tutorials/bulk-update.md @@ -92,8 +92,10 @@ Interpreting the result `WriteToServerAsync` returns a `SingleStoreBulkUpdateResult`: * `RowsStaged` — the number of source rows loaded into the staging table. -* `RowsMatched` — the number of staged rows that matched a row in the destination table. This is `null` when - `ComputeRowsMatched` is set to `false` (see below). +* `RowsMatched` — the number of destination rows matched by the join on the key columns. When the destination key + columns are unique this equals the number of staged rows that matched; if they are not unique, a single staged row + can match several destination rows, so this can exceed `RowsStaged`. This is `null` when `ComputeRowsMatched` is set + to `false` (see below). * `RowsAffected` — the number of rows affected by the `UPDATE`, as reported by the server. Its exact meaning depends on the connection's `UseAffectedRows` setting: with the default (`UseAffectedRows=false`) it counts the rows *matched* by the update — including rows that already held the new values — so it typically equals `RowsMatched`; with @@ -106,7 +108,8 @@ Performance * Set `ComputeRowsMatched = false` to skip the extra `COUNT(*)` query that populates `RowsMatched`. When disabled, `RowsMatched` is `null`. -* Set `BulkUpdateTimeout` (in seconds) to control how long each phase may run. +* Set `BulkUpdateTimeout` (in seconds) to bound how long each phase may run. It defaults to `0` (no timeout), so a + finite value should be set deliberately if a phase must not run unbounded. * Set `NotifyAfter` to a non-zero value to receive `SingleStoreRowsStaged` events while rows are being staged; the event handler can set `Abort = true` to cancel the operation. Aborting stops staging and skips the `UPDATE` entirely, so no rows in the destination table are modified. @@ -120,12 +123,17 @@ Limitations * At least one non-key column must be mapped, so there is a column to update. * Duplicate key values in the source data are rejected; they would collide in the staging table's primary key. * Shard key columns cannot be updated, because SingleStore does not allow updating a shard key. -* Generated (computed) columns cannot be updated, and are rejected if mapped. +* Generated (computed) columns cannot be mapped (neither as key nor update columns), because the staging table cannot + reproduce a generated column's definition. * Reference tables are not supported as the destination. * Expression column mappings (a `SingleStoreBulkCopyColumnMapping` with an `Expression`) are not supported. Other things to be aware of: +* **Key columns need not be unique in the destination.** The key columns identify rows to update via a join; they are + not required to be a unique or primary key on the destination table. If they are not unique, a single source row can + update multiple destination rows, and `RowsMatched`/`RowsAffected` can exceed the number of source rows. Duplicate + keys are only rejected in the *source* data (they would collide in the staging table's primary key). * **Key column types.** The key columns become the primary key of the staging table, so they must be types that SingleStore allows in a primary key. Large `TEXT`/`BLOB`/`JSON`/spatial columns are not usable as key columns. * **Required privileges.** In addition to `UPDATE` on the destination table, the connection needs permission to run @@ -142,3 +150,23 @@ Transactions A `SingleStoreTransaction` may be passed to the constructor. When supplied, all phases participate in that transaction, so the update can be committed or rolled back atomically with other work on the connection. + +Future directions +----------------- + +This API is experimental. Several of the limitations above are deliberate scope decisions for the first version rather +than restrictions imposed by SingleStore, and may be relaxed in a future version: + +* **`KeyColumns` is required.** A future version could auto-detect the destination table's primary key when + `KeyColumns` is not set. +* **Expression column mappings are rejected.** Because the source is staged into a real temporary table, a + `SingleStoreBulkCopyColumnMapping.Expression` (which relies on user variables during `LOAD DATA`) is not supported; + a future version could apply expressions when populating the staging table. +* **Generated (computed) columns cannot be mapped.** This could be supported for key columns if the staging table + learned to reproduce a generated column's underlying type. +* **Row counts are `int`.** `RowsStaged`, `RowsMatched`, and `RowsAffected` are `int` for consistency with + `SingleStoreBulkCopyResult`; they could widen to `long` if very large updates need it. +* **Update only.** There is no upsert (insert-or-update) mode; a future version could add one. + +By contrast, the following are SingleStore behaviors rather than choices made here, and are not expected to change: +updating shard key columns is not allowed, and reference tables are not supported as the destination. diff --git a/src/SingleStoreConnector/Logging/EventIds.cs b/src/SingleStoreConnector/Logging/EventIds.cs index 83ad95277..8759e3f51 100644 --- a/src/SingleStoreConnector/Logging/EventIds.cs +++ b/src/SingleStoreConnector/Logging/EventIds.cs @@ -159,10 +159,9 @@ internal static class EventIds public const int QueriedMatchCountForBulkUpdate = 2803; public const int ExecutedBulkUpdate = 2804; public const int CompletedBulkUpdate = 2805; - public const int KeyColumnsNotIndexedForBulkUpdate = 2806; - public const int ShardKeyMismatchForBulkUpdate = 2807; - public const int LargeUnmatchedCountForBulkUpdate = 2808; - public const int FailedToDropStagingTableForBulkUpdate = 2809; + public const int ShardKeyMismatchForBulkUpdate = 2806; + public const int LargeUnmatchedCountForBulkUpdate = 2807; + public const int FailedToDropStagingTableForBulkUpdate = 2808; // Connection pool events, 3000-3099 public const int WaitingForAvailableSession = 3000; diff --git a/src/SingleStoreConnector/Logging/Log.cs b/src/SingleStoreConnector/Logging/Log.cs index 990bb1945..d01af6c23 100644 --- a/src/SingleStoreConnector/Logging/Log.cs +++ b/src/SingleStoreConnector/Logging/Log.cs @@ -401,9 +401,6 @@ internal static partial class Log [LoggerMessage(EventIds.CompletedBulkUpdate, LogLevel.Information, "Completed bulk update: Staged={Staged}, Matched={Matched}, Updated={Updated}, Duration={Duration}ms")] public static partial void CompletedBulkUpdate(ILogger logger, int staged, int matched, int updated, long duration); - [LoggerMessage(EventIds.KeyColumnsNotIndexedForBulkUpdate, LogLevel.Warning, "Key columns not indexed for bulk update, performance may be slow: {Columns}")] - public static partial void KeyColumnsNotIndexedForBulkUpdate(ILogger logger, string columns); - [LoggerMessage(EventIds.ShardKeyMismatchForBulkUpdate, LogLevel.Warning, "Shard key mismatch for bulk update: Staging=[{StagingKey}], Target=[{TargetKey}]")] public static partial void ShardKeyMismatchForBulkUpdate(ILogger logger, string stagingKey, string targetKey); diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index 17f14b569..a73e7444c 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -52,8 +52,8 @@ namespace SingleStoreConnector; /// /// The following restrictions apply, and WriteToServer throws if they are not met: /// is required and every key column must be mapped; at least one non-key column must be mapped; the source must not -/// contain duplicate key values; shard key columns and generated (computed) columns cannot be updated; reference tables -/// are not supported; and expression column mappings are not supported. +/// contain duplicate key values; shard key columns cannot be updated; generated (computed) columns cannot be mapped; +/// reference tables are not supported; and expression column mappings are not supported. /// An instance of this class is not thread-safe; do not share an instance across concurrent operations. /// This API is experimental and may change in the future. /// @@ -93,9 +93,11 @@ public SingleStoreBulkUpdate(SingleStoreConnection connection, SingleStoreTransa public List ColumnMappings { get; } /// - /// The number of seconds for each phase of the operation to complete before it times out (default 30). + /// The number of seconds for each phase of the operation to complete before it times out, or 0 for no + /// timeout (the default). A single bulk update can spend a long time staging, counting, or updating, so a + /// finite timeout should be chosen deliberately. /// - public int BulkUpdateTimeout { get; set; } = 30; + public int BulkUpdateTimeout { get; set; } /// /// If non-zero, this specifies the number of rows to be staged before raising the @@ -212,8 +214,10 @@ private async ValueTask WriteToServerAsync(IOBehavi // Validate configuration before touching the connection so misconfiguration fails fast and cheaply. ValidateColumnMappings(); - var destinationTableName = DestinationTableName ?? - throw new InvalidOperationException("DestinationTableName must be set before calling WriteToServer."); + // Snapshot all operation inputs now so that a SingleStoreRowsStaged event handler (which fires between + // staging and the UPDATE) cannot change the destination table, key columns or mappings mid-operation and + // produce a staging table and UPDATE built from different configurations. + var plan = CreatePlan(); // Reset any warnings from a previous call so the result only reflects this operation. m_warnings.Clear(); @@ -241,20 +245,23 @@ private async ValueTask WriteToServerAsync(IOBehavi closeConnection = true; } + // A single SchemaDetector per operation so the destination's SHOW CREATE TABLE is fetched once and reused + // across reference-table, shard-key, generated-column and column-type inspection. + var schemaDetector = new SchemaDetector(m_connection, m_transaction, BulkUpdateTimeout); + string? tempTableName = null; try { - // Reject reference tables and shard-key updates, and confirm every mapped column exists. - await ValidateSchemaAsync(destinationTableName, ioBehavior, cancellationToken).ConfigureAwait(false); + // Reject reference tables and shard-key/generated updates, and confirm every mapped column exists. + await ValidateSchemaAsync(schemaDetector, plan, ioBehavior, cancellationToken).ConfigureAwait(false); - var updateColumns = GetUpdateColumns(); - Log.StartingBulkUpdate(m_logger, destinationTableName, string.Join(", ", KeyColumns), string.Join(", ", updateColumns), GetRowCount(source)); + Log.StartingBulkUpdate(m_logger, plan.DestinationTableName, string.Join(", ", plan.KeyColumns), string.Join(", ", plan.UpdateColumns), GetRowCount(source)); // Phase 1: create the staging table mirroring the destination column types. - tempTableName = await CreateStagingTableAsync(destinationTableName, ioBehavior, cancellationToken).ConfigureAwait(false); + tempTableName = await CreateStagingTableAsync(schemaDetector, plan, ioBehavior, cancellationToken).ConfigureAwait(false); // Phase 2: stage the source rows into the temporary table via SingleStoreBulkCopy. - var (rowsStaged, aborted) = await StageDataAsync(tempTableName, source, ioBehavior, cancellationToken).ConfigureAwait(false); + var (rowsStaged, aborted) = await StageDataAsync(plan, tempTableName, source, ioBehavior, cancellationToken).ConfigureAwait(false); // If the caller aborted staging via the SingleStoreRowsStaged event, abort the whole operation: do not // run the UPDATE, so no rows are modified. Only the staging table (dropped below) was touched. @@ -266,12 +273,12 @@ private async ValueTask WriteToServerAsync(IOBehavi } // Phase 3 (optional): count how many staged rows match a destination row. - var rowsMatched = await ComputeMatchedRowsAsync(tempTableName, ioBehavior, cancellationToken).ConfigureAwait(false); + var rowsMatched = await ComputeMatchedRowsAsync(plan, tempTableName, ioBehavior, cancellationToken).ConfigureAwait(false); if (rowsMatched is { } matched && rowsStaged > matched) Log.LargeUnmatchedCountForBulkUpdate(m_logger, rowsStaged, matched, rowsStaged - matched); // Phase 4: run the UPDATE ... JOIN that copies the non-key values into the matching rows. - var rowsAffected = await ExecuteUpdateAsync(tempTableName, ioBehavior, cancellationToken).ConfigureAwait(false); + var rowsAffected = await ExecuteUpdateAsync(plan, tempTableName, ioBehavior, cancellationToken).ConfigureAwait(false); stopwatch.Stop(); Log.CompletedBulkUpdate(m_logger, rowsStaged, rowsMatched ?? -1, rowsAffected, stopwatch.ElapsedMilliseconds); @@ -290,6 +297,38 @@ private async ValueTask WriteToServerAsync(IOBehavi } } + /// + /// Captures an immutable snapshot of the operation inputs (destination table, key columns, column mappings and + /// the derived update columns) so the rest of the operation is unaffected by later mutations of the public + /// properties. + /// + private BulkUpdatePlan CreatePlan() + { + var destinationTableName = DestinationTableName ?? + throw new InvalidOperationException("DestinationTableName must be set before calling WriteToServer."); + + return new BulkUpdatePlan( + destinationTableName, + [.. KeyColumns], + [.. ColumnMappings], + GetUpdateColumns()); + } + + /// + /// An immutable snapshot of the inputs for a single bulk update operation. + /// + private sealed class BulkUpdatePlan( + string destinationTableName, + IReadOnlyList keyColumns, + IReadOnlyList columnMappings, + IReadOnlyList updateColumns) + { + public string DestinationTableName { get; } = destinationTableName; + public IReadOnlyList KeyColumns { get; } = keyColumns; + public IReadOnlyList ColumnMappings { get; } = columnMappings; + public IReadOnlyList UpdateColumns { get; } = updateColumns; + } + /// /// Returns the number of rows in the source for logging, or -1 when the count is not known in advance /// (for example an , which is consumed as it is staged). @@ -307,7 +346,7 @@ private static int GetRowCount(object source) => /// Builds the operation result, snapshotting the warnings collected so far into a new list so that a result /// returned from one call is not mutated when the same instance is reused. /// - private SingleStoreBulkUpdateResult CreateResult(int rowsStaged, int? rowsMatched, int rowsAffected) => + private SingleStoreBulkUpdateResult CreateResult(int rowsStaged, int? rowsMatched, int rowsAffected) => new(new List(m_warnings), rowsStaged, rowsMatched, rowsAffected); private void ValidateColumnMappings() @@ -375,18 +414,17 @@ private void ValidateColumnMappings() throw new InvalidOperationException("ColumnMappings must contain at least one non-key column to update."); } - private async ValueTask ValidateSchemaAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken) + private async ValueTask ValidateSchemaAsync(SchemaDetector schemaDetector, BulkUpdatePlan plan, IOBehavior ioBehavior, CancellationToken cancellationToken) { - var schemaDetector = new SchemaDetector(m_connection, m_transaction, BulkUpdateTimeout); + var tableName = plan.DestinationTableName; // TODO: make changes to support the solutions described here -- https://docs.singlestore.com/cloud/reference/troubleshooting-reference/query-errors/error-1706-hy-000-feature-multi-table-update-delete-with-a-reference-table-as-target-table-is-not-supported-by-memsql/ if (await schemaDetector.IsReferenceTableAsync(tableName, ioBehavior, cancellationToken).ConfigureAwait(false)) throw new NotSupportedException($"Target table '{tableName}' is a reference table. Bulk updates on reference tables are not supported in this version."); var shardKeyColumns = await schemaDetector.GetShardKeyColumnsAsync(tableName, ioBehavior, cancellationToken).ConfigureAwait(false); - var updateColumns = GetUpdateColumns(); - foreach (var updateColumn in updateColumns) + foreach (var updateColumn in plan.UpdateColumns) { if (shardKeyColumns.Contains(updateColumn, StringComparer.OrdinalIgnoreCase)) throw new InvalidOperationException($"Column '{updateColumn}' is a shard key. SingleStore does not support updating shard key columns."); @@ -403,22 +441,24 @@ private async ValueTask ValidateSchemaAsync(string tableName, IOBehavior ioBehav tableColumns.Add(columnName); } - foreach (var columnMapping in ColumnMappings) + foreach (var columnMapping in plan.ColumnMappings) { if (!tableColumns.Contains(columnMapping.DestinationColumn)) throw new InvalidOperationException($"Column '{columnMapping.DestinationColumn}' does not exist in target table '{tableName}'."); } - // Reject mapped generated (computed) columns up front with a clear error. Their value is derived from an - // expression, so they cannot be staged or assigned in the UPDATE; without this check the operation would - // fail later with a confusing server error when the UPDATE tries to write to the generated column. + // Reject mapped generated (computed) columns with a clear error. An update column cannot be assigned (its + // value is derived from an expression), and a key column cannot be staged either: the staging table mirrors + // the destination column's type, but a generated column's definition (AS (expr) [PERSISTED] type) has no + // plain, reproducible column type to copy. Without this check the operation would fail later with a + // confusing server error. var generatedColumns = await schemaDetector.GetGeneratedColumnsAsync(tableName, ioBehavior, cancellationToken).ConfigureAwait(false); if (generatedColumns.Count != 0) { - foreach (var columnMapping in ColumnMappings) + foreach (var columnMapping in plan.ColumnMappings) { if (generatedColumns.Contains(columnMapping.DestinationColumn)) - throw new NotSupportedException($"Column '{columnMapping.DestinationColumn}' is a generated (computed) column, which cannot be updated by SingleStoreBulkUpdate."); + throw new NotSupportedException($"Column '{columnMapping.DestinationColumn}' is a generated (computed) column, which is not supported by SingleStoreBulkUpdate."); } } } @@ -434,7 +474,9 @@ private List GetUpdateColumns() => /// Creates a session-scoped temporary staging table containing only the mapped (key + update) columns /// of the destination table, ready to receive the source data via . /// - /// The destination table whose column types are mirrored. + /// The schema detector used to read the destination table's definition. + /// The snapshot of the operation inputs. + /// Whether to perform database I/O synchronously or asynchronously. /// A token to cancel the asynchronous operation. /// The name of the created temporary table. /// @@ -444,7 +486,7 @@ private List GetUpdateColumns() => /// VARBINARY is reported as BLOB and BIT(1) as BIGINT, and UNSIGNED, /// character set, collation and ENUM/SET member lists are not exposed), so copying the exact /// definition is the only way to guarantee the staging column matches the destination column. Matching the - /// collation in particular keeps the key-column equality used by the UPDATE ... JOIN well defined. + /// collation in particular keeps the key-column equality used by the UPDATE ... JOIN well-defined. /// /// /// The key columns form the staging table's PRIMARY KEY, so they are always declared NOT NULL @@ -458,13 +500,12 @@ private List GetUpdateColumns() => /// because the temporary table is session-scoped. /// /// - private async Task CreateStagingTableAsync(string destinationTableName, IOBehavior ioBehavior, CancellationToken cancellationToken) + private async Task CreateStagingTableAsync(SchemaDetector schemaDetector, BulkUpdatePlan plan, IOBehavior ioBehavior, CancellationToken cancellationToken) { - // Generate a unique temporary table name. The "g" suffix on the GUID guarantees the identifier - // starts with a letter regardless of the GUID's first hex digit. - var tempTableName = $"_bulk_update_staging_g{Guid.NewGuid():N}"; + // Generate a unique temporary table name. + var tempTableName = $"_bulk_update_staging_{Guid.NewGuid():N}"; - var schemaDetector = new SchemaDetector(m_connection, m_transaction, BulkUpdateTimeout); + var destinationTableName = plan.DestinationTableName; // Pull the exact, server-rendered type definition for every column so the staging columns are // byte-for-byte type compatible with the destination (see remarks). @@ -474,11 +515,11 @@ private async Task CreateStagingTableAsync(string destinationTableName, // Emit a column definition for each mapped column, preserving the order in which the columns appear // in the destination table is unnecessary here: the staging table only needs the columns to exist by // name. SingleStoreBulkCopy maps source ordinals to these destination column names when staging. - var keyColumnSet = new HashSet(KeyColumns, StringComparer.OrdinalIgnoreCase); + var keyColumnSet = new HashSet(plan.KeyColumns, StringComparer.OrdinalIgnoreCase); var seenColumns = new HashSet(StringComparer.OrdinalIgnoreCase); - var columnDefinitions = new List(ColumnMappings.Count); + var columnDefinitions = new List(plan.ColumnMappings.Count); - foreach (var mapping in ColumnMappings) + foreach (var mapping in plan.ColumnMappings) { var columnName = mapping.DestinationColumn; @@ -500,12 +541,13 @@ private async Task CreateStagingTableAsync(string destinationTableName, // The key columns identify the rows to update, so they are the natural primary key of the staging // table. This also rejects duplicate keys in the source data with a clear primary-key violation. - var primaryKey = $"PRIMARY KEY ({string.Join(", ", KeyColumns.Select(IdentifierHelper.QuoteIdentifier))})"; + var primaryKey = $"PRIMARY KEY ({string.Join(", ", plan.KeyColumns.Select(IdentifierHelper.QuoteIdentifier))})"; - var stagingShardKey = ComputeStagingShardKey(shardKeyColumns, keyColumnSet); + var stagingShardKey = ComputeStagingShardKey(plan, shardKeyColumns, keyColumnSet); + // Create the staging table as ROWSTORE explicitly: it is small, primary-key indexed and short-lived var createTableSql = new StringBuilder(); - createTableSql.Append("CREATE TEMPORARY TABLE "); + createTableSql.Append("CREATE ROWSTORE TEMPORARY TABLE "); createTableSql.Append(IdentifierHelper.QuoteIdentifier(tempTableName)); createTableSql.Append(" ("); createTableSql.Append(string.Join(", ", columnDefinitions)); @@ -545,7 +587,7 @@ private async Task CreateStagingTableAsync(string destinationTableName, /// sharded on a column that is not a join key — the staging table cannot be aligned, so we fall back to the /// primary-key distribution (by returning an empty list, which omits an explicit shard key) and warn. /// - private List ComputeStagingShardKey(List destinationShardKeyColumns, HashSet keyColumnSet) + private List ComputeStagingShardKey(BulkUpdatePlan plan, List destinationShardKeyColumns, HashSet keyColumnSet) { if (destinationShardKeyColumns.Count == 0) return []; @@ -555,7 +597,7 @@ private List ComputeStagingShardKey(List destinationShardKeyColu Log.ShardKeyMismatchForBulkUpdate( m_logger, - string.Join(", ", KeyColumns), + string.Join(", ", plan.KeyColumns), string.Join(", ", destinationShardKeyColumns)); return []; @@ -583,7 +625,7 @@ private List ComputeStagingShardKey(List destinationShardKeyColu /// destination-name relationship identical between staging and the later UPDATE ... JOIN. /// /// - private async Task<(int RowsStaged, bool Aborted)> StageDataAsync(string tempTableName, object source, IOBehavior ioBehavior, CancellationToken cancellationToken) + private async Task<(int RowsStaged, bool Aborted)> StageDataAsync(BulkUpdatePlan plan, string tempTableName, object source, IOBehavior ioBehavior, CancellationToken cancellationToken) { var bulkCopy = new SingleStoreBulkCopy(m_connection, m_transaction) { @@ -593,7 +635,7 @@ private List ComputeStagingShardKey(List destinationShardKeyColu }; // Forward our column mappings unchanged: source ordinal -> staging column name. - foreach (var mapping in ColumnMappings) + foreach (var mapping in plan.ColumnMappings) bulkCopy.ColumnMappings.Add(mapping); // Re-raise SingleStoreBulkCopy's progress event as a bulk-update staging event, and propagate the @@ -693,14 +735,14 @@ private static ValueTask StageWithBulkCopyAsync(Singl /// operation because the staging table is session-scoped. /// /// - private async Task ComputeMatchedRowsAsync(string tempTableName, IOBehavior ioBehavior, CancellationToken cancellationToken) + private async Task ComputeMatchedRowsAsync(BulkUpdatePlan plan, string tempTableName, IOBehavior ioBehavior, CancellationToken cancellationToken) { if (!ComputeRowsMatched) return null; var countSql = - $"SELECT COUNT(*) FROM {IdentifierHelper.QuoteQualifiedIdentifier(DestinationTableName!)} AS t " + - $"INNER JOIN {IdentifierHelper.QuoteIdentifier(tempTableName)} AS s ON {BuildKeyJoinCondition()}"; + $"SELECT COUNT(*) FROM {IdentifierHelper.QuoteQualifiedIdentifier(plan.DestinationTableName)} AS t " + + $"INNER JOIN {IdentifierHelper.QuoteIdentifier(tempTableName)} AS s ON {BuildKeyJoinCondition(plan)}"; using var cmd = m_connection.CreateCommand(); cmd.CommandText = countSql; @@ -744,16 +786,16 @@ private static ValueTask StageWithBulkCopyAsync(Singl /// operation result. /// /// - private async Task ExecuteUpdateAsync(string tempTableName, IOBehavior ioBehavior, CancellationToken cancellationToken) + private async Task ExecuteUpdateAsync(BulkUpdatePlan plan, string tempTableName, IOBehavior ioBehavior, CancellationToken cancellationToken) { // Assign each non-key mapped column from the staging row: t.`c1` = s.`c1`, t.`c2` = s.`c2` ... var setClause = string.Join( ", ", - GetUpdateColumns().Select(c => $"t.{IdentifierHelper.QuoteIdentifier(c)} = s.{IdentifierHelper.QuoteIdentifier(c)}")); + plan.UpdateColumns.Select(c => $"t.{IdentifierHelper.QuoteIdentifier(c)} = s.{IdentifierHelper.QuoteIdentifier(c)}")); var updateSql = - $"UPDATE {IdentifierHelper.QuoteQualifiedIdentifier(DestinationTableName!)} AS t " + - $"INNER JOIN {IdentifierHelper.QuoteIdentifier(tempTableName)} AS s ON {BuildKeyJoinCondition()} " + + $"UPDATE {IdentifierHelper.QuoteQualifiedIdentifier(plan.DestinationTableName)} AS t " + + $"INNER JOIN {IdentifierHelper.QuoteIdentifier(tempTableName)} AS s ON {BuildKeyJoinCondition(plan)} " + $"SET {setClause}"; using var cmd = m_connection.CreateCommand(); @@ -761,7 +803,7 @@ private async Task ExecuteUpdateAsync(string tempTableName, IOBehavior ioBe cmd.Transaction = m_transaction; cmd.CommandTimeout = BulkUpdateTimeout; - // Collect any warnings raised during the UPDATE. Errors is already IReadOnlyList. + // Collect any warnings raised during the UPDATE. Errors are already IReadOnlyList. void OnInfoMessage(object sender, SingleStoreInfoMessageEventArgs args) => m_warnings.AddRange(args.Errors); m_connection.InfoMessage += OnInfoMessage; @@ -826,10 +868,10 @@ private async Task DropStagingTableAsync(string? tempTableName, IOBehavior ioBeh /// Builds the key-column equi-join predicate shared by the match-count query and the update, joining the /// destination table (alias t) to the staging table (alias s) on every key column. /// - private string BuildKeyJoinCondition() => + private static string BuildKeyJoinCondition(BulkUpdatePlan plan) => string.Join( " AND ", - KeyColumns.Select(k => $"t.{IdentifierHelper.QuoteIdentifier(k)} = s.{IdentifierHelper.QuoteIdentifier(k)}")); + plan.KeyColumns.Select(k => $"t.{IdentifierHelper.QuoteIdentifier(k)} = s.{IdentifierHelper.QuoteIdentifier(k)}")); private readonly SingleStoreConnection m_connection; private readonly SingleStoreTransaction? m_transaction; diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs b/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs index 98c46c56f..b890a5baf 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs @@ -17,9 +17,12 @@ public sealed class SingleStoreBulkUpdateResult public int RowsStaged { get; } /// - /// The number of staged rows that matched rows in the destination table, or null when + /// The number of destination rows matched by the join on the key columns, or null when /// was set to false and the count was not computed. /// + /// When the destination key columns are unique this equals the number of staged rows that matched a + /// destination row. If the key columns are not unique, a single staged row can match several destination rows, + /// so this (and ) can exceed the number of staged rows. public int? RowsMatched { get; } /// diff --git a/src/SingleStoreConnector/Utilities/SchemaDetector.cs b/src/SingleStoreConnector/Utilities/SchemaDetector.cs index 004e9c43c..ba01d9d7a 100644 --- a/src/SingleStoreConnector/Utilities/SchemaDetector.cs +++ b/src/SingleStoreConnector/Utilities/SchemaDetector.cs @@ -22,6 +22,10 @@ internal sealed class SchemaDetector(SingleStoreConnection connection, SingleSto private readonly int m_commandTimeout = commandTimeout; + // Caches the SHOW CREATE TABLE text per table name so a single operation that inspects reference-table status, + // shard keys, column type definitions and generated columns only queries the server once. + private readonly Dictionary m_createTableCache = new(StringComparer.OrdinalIgnoreCase); + /// /// Detects if the specified table is a reference table. /// @@ -112,6 +116,9 @@ private async Task GetCreateTableStatementAsync(string tableName, IOBeha { EnsureConnectionIsOpen(); + if (m_createTableCache.TryGetValue(tableName, out var cached)) + return cached; + using var cmd = m_connection.CreateCommand(); cmd.Transaction = m_transaction; cmd.CommandTimeout = m_commandTimeout; @@ -123,7 +130,9 @@ private async Task GetCreateTableStatementAsync(string tableName, IOBeha if (await reader.ReadAsync(ioBehavior, cancellationToken).ConfigureAwait(false)) { // SHOW CREATE TABLE returns the CREATE TABLE statement in the second column. - return reader.GetString(1); + var createTableSql = reader.GetString(1); + m_createTableCache[tableName] = createTableSql; + return createTableSql; } throw new InvalidOperationException($"Unable to retrieve CREATE TABLE statement for {tableName}."); diff --git a/tests/SideBySide/BulkUpdateTests.cs b/tests/SideBySide/BulkUpdateTests.cs index 454096495..93c4b0b44 100644 --- a/tests/SideBySide/BulkUpdateTests.cs +++ b/tests/SideBySide/BulkUpdateTests.cs @@ -538,6 +538,108 @@ public async Task ResultWarningsAreNotMutatedByLaterCall() Assert.NotSame(firstResult.Warnings, secondResult.Warnings); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UpdatesWithSchemaQualifiedDestinationTableName(bool isAsync) + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_qualified; +create table bulk_update_qualified(id int primary key, value varchar(100)); +insert into bulk_update_qualified values (1, 'original');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = { new object[] { 1, "updated" } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + // Fully-qualified database.table name. + DestinationTableName = $"{connection.Database}.bulk_update_qualified", + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await WriteToServerAsync(bulkUpdate, dataTable, isAsync); + + Assert.Equal(1, result.RowsAffected); + + using var selectCommand = new SingleStoreCommand("select value from bulk_update_qualified where id = 1;", connection); + Assert.Equal("updated", await selectCommand.ExecuteScalarAsync()); + } + + [Fact] + public async Task MutatingConfigInProgressEventDoesNotAffectOperation() + { + using var connection = new SingleStoreConnection(GetLocalConnectionString(database)); + await connection.OpenAsync(); + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_mutate; +drop table if exists bulk_update_mutate_other; +create table bulk_update_mutate(id int primary key, value varchar(100)); +create table bulk_update_mutate_other(id int primary key, value varchar(100)); +insert into bulk_update_mutate values (1, 'original'), (2, 'original');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("id", typeof(int)), + new DataColumn("value", typeof(string)), + }, + }; + for (var i = 1; i <= 2; i++) + dataTable.Rows.Add(i, "updated"); + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_mutate", + NotifyAfter = 1, + KeyColumns = { "id" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "id"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + // The handler fires between staging and the UPDATE and tries to repoint the operation at another table and + // drop a mapping. Because the operation snapshots its configuration up front, these mutations must have no + // effect: the original table is updated and the other table is left untouched. + bulkUpdate.SingleStoreRowsStaged += (sender, e) => + { + bulkUpdate.DestinationTableName = "bulk_update_mutate_other"; + bulkUpdate.KeyColumns.Clear(); + bulkUpdate.ColumnMappings.Clear(); + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(2, result.RowsAffected); + + using var updatedCommand = new SingleStoreCommand("select count(*) from bulk_update_mutate where value = 'updated';", connection); + Assert.Equal(2L, Convert.ToInt64(await updatedCommand.ExecuteScalarAsync())); + + using var otherCommand = new SingleStoreCommand("select count(*) from bulk_update_mutate_other;", connection); + Assert.Equal(0L, Convert.ToInt64(await otherCommand.ExecuteScalarAsync())); + } + private static async ValueTask WriteToServerAsync(SingleStoreBulkUpdate bulkUpdate, DataTable dataTable, bool isAsync) => isAsync ? await bulkUpdate.WriteToServerAsync(dataTable) : bulkUpdate.WriteToServer(dataTable); diff --git a/tests/SideBySide/BulkUpdateValidationTests.cs b/tests/SideBySide/BulkUpdateValidationTests.cs index 410e64994..46eb14e50 100644 --- a/tests/SideBySide/BulkUpdateValidationTests.cs +++ b/tests/SideBySide/BulkUpdateValidationTests.cs @@ -342,6 +342,46 @@ public async Task ThrowsWhenUpdatingGeneratedColumn() Assert.Contains("total", exception.Message); } + [Fact] + public async Task ThrowsWhenGeneratedColumnUsedAsKey() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + + // gen_key is a generated column used as a key column. It is still rejected: a generated column has no plain + // column type that the staging table can reproduce, so it cannot be staged even as a key. + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_generated_key; +create table bulk_update_generated_key(id int, gen_key as (id * 10) persisted int, value varchar(100), primary key (id, gen_key));", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("gen_key", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = { new object[] { 10, "updated" } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_generated_key", + KeyColumns = { "gen_key" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "gen_key"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var exception = await Assert.ThrowsAsync(async () => await bulkUpdate.WriteToServerAsync(dataTable)); + Assert.Contains("generated", exception.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("gen_key", exception.Message); + } + private static DataTable NewTable(params string[] columnNames) { var dataTable = new DataTable(); diff --git a/tests/SideBySide/SchemaDetectorTests.cs b/tests/SideBySide/SchemaDetectorTests.cs index 2cf0056bc..35d0c0b72 100644 --- a/tests/SideBySide/SchemaDetectorTests.cs +++ b/tests/SideBySide/SchemaDetectorTests.cs @@ -298,6 +298,45 @@ public async Task GetColumnTypeDefinitionsHandlesVectorColumn() Assert.Contains("4", definitions["embedding"], StringComparison.OrdinalIgnoreCase); } + [Fact] + public async Task GetColumnTypeDefinitionsHandlesMultiWordType() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "schema_detector_coldefs_multiword"; + + // DOUBLE PRECISION is a multi-token type name; the parser must keep both words rather than stopping after + // the first token (which would drop "PRECISION" or misread it as a column option). + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, measurement DOUBLE PRECISION"); + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + Assert.Contains("double", definitions["measurement"], StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task GetColumnTypeDefinitionsPreservesCharacterSetAndCollation() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "schema_detector_coldefs_charset"; + + // A string column can carry both a CHARACTER SET and a COLLATE modifier; both are part of the type and must + // be preserved so the staging column compares values the same way as the destination. + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, name VARCHAR(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"); + + var detector = new SchemaDetector(connection); + var definitions = await detector.GetColumnTypeDefinitionsAsync(tableName, IOBehavior.Asynchronous); + + var nameDefinition = definitions["name"]; + Assert.Contains("varchar(100)", nameDefinition, StringComparison.OrdinalIgnoreCase); + Assert.Contains("utf8mb4", nameDefinition, StringComparison.OrdinalIgnoreCase); + Assert.Contains("collate", nameDefinition, StringComparison.OrdinalIgnoreCase); + } + [Fact] public async Task IsReferenceTableThrowsForNonexistentTable() { diff --git a/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json b/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json index e466b627d..b5bae9ca2 100644 --- a/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json +++ b/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json @@ -4,15 +4,15 @@ "net10.0": { "Microsoft.DotNet.ILCompiler": { "type": "Direct", - "requested": "[10.0.7, )", - "resolved": "10.0.7", - "contentHash": "S53itVb3/l5wEyvdOukwxGzRtw7JSKuh2GurW3W5R57w4nxbZZ+7qCu+5jVlowLmgGrQ7yxJLZdpJc1Vpv8aig==" + "requested": "[10.0.8, )", + "resolved": "10.0.8", + "contentHash": "IG4ZS8fP7whR7OeFjC2mxq9n/yd/QsUliHdQ/Kp0OJ+RObeSJx4HLLGs7/Sxghk/U+S3rmpUOsgLheqBcqlMUw==" }, "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[10.0.7, )", - "resolved": "10.0.7", - "contentHash": "Y0O2XrQiNYsW9UM2GhH/8faornu9pYXdd9YBcb/CrBnCOw5QtFqgh8bFd5zjxbrdoSVxGPUn1hWSrKtD0SEPzg==" + "requested": "[10.0.9, )", + "resolved": "10.0.9", + "contentHash": "ZuAAT2Qp2l1W6urbmGoA8t8psucpTCpwjDbGYzMSUewt+IEEHz6aVRBJjrDYOu7znIxDS1fprg/VDpAdXSHi7Q==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", @@ -80,15 +80,15 @@ "net8.0": { "Microsoft.DotNet.ILCompiler": { "type": "Direct", - "requested": "[8.0.26, )", - "resolved": "8.0.26", - "contentHash": "TopkZJbGfDfqNRQatno3/UZh8t/mo4tlglGVtcUAZDEplx/DhmBUSoT7UM7M6W0/ah1Wbe6e46FOLDyHL8K4nQ==" + "requested": "[8.0.27, )", + "resolved": "8.0.27", + "contentHash": "2RDzeGQisWc0e4uJpT8WmMQ8Xccf4bZr4bVVV9iGnfnEZCPo//G01MLLCmGHScwLX4zCegNZRNftgUfgWlBb+g==" }, "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.26, )", - "resolved": "8.0.26", - "contentHash": "o7/yVssM2r9Wyln2s9edBd5ANZXqdSdBI+g7JqXkyJmXrhs2WsJp25K5yPnYrTgdKBCjKB8bg+O2oew4sgzFaA==" + "requested": "[8.0.28, )", + "resolved": "8.0.28", + "contentHash": "XMqgVjlLxLqWmEh3c49haXLQwsMNtvo6YscUaqfvEGfg1iA8hnYgkUVq3i9Zu9gKeNKMWiiZKVwZExc/qyEAsQ==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", From 2c6849681d82c0199ca6cf9b8a7aa82de163cb84 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 1 Jul 2026 21:27:54 +0300 Subject: [PATCH 26/28] another block of minor changes and fixes --- docs/content/tutorials/bulk-update.md | 21 +++++--- .../SingleStoreBulkUpdate.cs | 28 +++++----- .../SingleStoreBulkUpdateResult.cs | 11 ++-- .../Utilities/SchemaDetector.cs | 54 ++++++++++++------- tests/SideBySide/BulkUpdateAdvancedTests.cs | 49 ++++++++++++++++- tests/SideBySide/SchemaDetectorTests.cs | 18 +++++++ 6 files changed, 136 insertions(+), 45 deletions(-) diff --git a/docs/content/tutorials/bulk-update.md b/docs/content/tutorials/bulk-update.md index 229d9d24e..637fb2902 100644 --- a/docs/content/tutorials/bulk-update.md +++ b/docs/content/tutorials/bulk-update.md @@ -92,10 +92,10 @@ Interpreting the result `WriteToServerAsync` returns a `SingleStoreBulkUpdateResult`: * `RowsStaged` — the number of source rows loaded into the staging table. -* `RowsMatched` — the number of destination rows matched by the join on the key columns. When the destination key - columns are unique this equals the number of staged rows that matched; if they are not unique, a single staged row - can match several destination rows, so this can exceed `RowsStaged`. This is `null` when `ComputeRowsMatched` is set - to `false` (see below). +* `RowsMatched` — the number of staged rows that matched at least one destination row on the key columns. This never + exceeds `RowsStaged`, so `RowsStaged - RowsMatched` is the number of source rows that matched nothing. (`RowsAffected` + can still exceed this when the destination keys are not unique — see below.) This is `null` when `ComputeRowsMatched` + is set to `false` (see below). * `RowsAffected` — the number of rows affected by the `UPDATE`, as reported by the server. Its exact meaning depends on the connection's `UseAffectedRows` setting: with the default (`UseAffectedRows=false`) it counts the rows *matched* by the update — including rows that already held the new values — so it typically equals `RowsMatched`; with @@ -132,8 +132,9 @@ Other things to be aware of: * **Key columns need not be unique in the destination.** The key columns identify rows to update via a join; they are not required to be a unique or primary key on the destination table. If they are not unique, a single source row can - update multiple destination rows, and `RowsMatched`/`RowsAffected` can exceed the number of source rows. Duplicate - keys are only rejected in the *source* data (they would collide in the staging table's primary key). + update multiple destination rows, so `RowsAffected` can exceed the number of source rows (`RowsMatched`, which counts + matched source rows, cannot). Duplicate keys are only rejected in the *source* data (they would collide in the + staging table's primary key). * **Key column types.** The key columns become the primary key of the staging table, so they must be types that SingleStore allows in a primary key. Large `TEXT`/`BLOB`/`JSON`/spatial columns are not usable as key columns. * **Required privileges.** In addition to `UPDATE` on the destination table, the connection needs permission to run @@ -167,6 +168,10 @@ than restrictions imposed by SingleStore, and may be relaxed in a future version * **Row counts are `int`.** `RowsStaged`, `RowsMatched`, and `RowsAffected` are `int` for consistency with `SingleStoreBulkCopyResult`; they could widen to `long` if very large updates need it. * **Update only.** There is no upsert (insert-or-update) mode; a future version could add one. +* **Reference tables are not supported as the destination.** The `UPDATE ... JOIN` this API generates fails on a + reference-table target joined to a (sharded) staging table (SingleStore error 1706). This is a scope decision, not + a hard limit — a future version could support it by staging into a reference table or by using an upsert, per the + [documented workarounds](https://docs.singlestore.com/cloud/reference/troubleshooting-reference/query-errors/error-1706-hy-000-feature-multi-table-update-delete-with-a-reference-table-as-target-table-is-not-supported-by-memsql/). -By contrast, the following are SingleStore behaviors rather than choices made here, and are not expected to change: -updating shard key columns is not allowed, and reference tables are not supported as the destination. +By contrast, the following is a SingleStore behavior rather than a choice made here, and is not expected to change: +updating shard key columns is not allowed. diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index a73e7444c..f6c04bfba 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -713,26 +713,27 @@ private static ValueTask StageWithBulkCopyAsync(Singl } /// - /// Counts how many staged rows match a row in the destination table, joined on the key columns. + /// Counts how many staged rows match at least one row in the destination table, joined on the key columns. /// /// The session-scoped staging table populated by . /// A token to cancel the asynchronous operation. /// - /// The number of staged rows that match a destination row, or when + /// The number of staged rows that match at least one destination row, or when /// is (the count was intentionally skipped). /// /// /// - /// This runs an extra SELECT COUNT(*) over the same INNER JOIN that the subsequent - /// UPDATE ... JOIN uses, letting the caller distinguish staged rows that updated a destination row - /// from staged rows that matched nothing. Callers that do not need this distinction can set - /// to to skip the query. + /// This counts staged rows (via WHERE EXISTS) rather than joined pairs, so the result never exceeds the + /// number of staged rows even when the destination key columns are not unique and a single staged row matches + /// several destination rows. It lets the caller distinguish staged rows that matched a destination row from + /// staged rows that matched nothing (RowsStaged - RowsMatched). Callers that do not need this distinction + /// can set to to skip the query. /// /// - /// The join uses the key columns, which were created in the staging table with the destination's exact - /// type and collation (see ), so this count is consistent with the - /// rows the UPDATE will match. It must run on the same open connection/transaction as the rest of the - /// operation because the staging table is session-scoped. + /// The match uses the key columns, which were created in the staging table with the destination's exact type and + /// collation (see ), so this count is consistent with the rows the UPDATE + /// will match. It must run on the same open connection/transaction as the rest of the operation because the + /// staging table is session-scoped. /// /// private async Task ComputeMatchedRowsAsync(BulkUpdatePlan plan, string tempTableName, IOBehavior ioBehavior, CancellationToken cancellationToken) @@ -740,9 +741,12 @@ private static ValueTask StageWithBulkCopyAsync(Singl if (!ComputeRowsMatched) return null; + // Count staged rows that have at least one matching destination row. Counting from the staging side with + // EXISTS (rather than COUNT(*) over an INNER JOIN) keeps the result <= RowsStaged when the destination key + // columns are not unique, so RowsStaged - RowsMatched is a correct count of unmatched staged rows. var countSql = - $"SELECT COUNT(*) FROM {IdentifierHelper.QuoteQualifiedIdentifier(plan.DestinationTableName)} AS t " + - $"INNER JOIN {IdentifierHelper.QuoteIdentifier(tempTableName)} AS s ON {BuildKeyJoinCondition(plan)}"; + $"SELECT COUNT(*) FROM {IdentifierHelper.QuoteIdentifier(tempTableName)} AS s " + + $"WHERE EXISTS (SELECT 1 FROM {IdentifierHelper.QuoteQualifiedIdentifier(plan.DestinationTableName)} AS t WHERE {BuildKeyJoinCondition(plan)})"; using var cmd = m_connection.CreateCommand(); cmd.CommandText = countSql; diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs b/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs index b890a5baf..a1b90ce0d 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdateResult.cs @@ -17,12 +17,13 @@ public sealed class SingleStoreBulkUpdateResult public int RowsStaged { get; } /// - /// The number of destination rows matched by the join on the key columns, or null when - /// was set to false and the count was not computed. + /// The number of staged rows that matched at least one row in the destination table on the key columns, or + /// null when was set to false and the count + /// was not computed. /// - /// When the destination key columns are unique this equals the number of staged rows that matched a - /// destination row. If the key columns are not unique, a single staged row can match several destination rows, - /// so this (and ) can exceed the number of staged rows. + /// This never exceeds ; RowsStaged - RowsMatched is the number of staged + /// rows that matched no destination row. Note that can still exceed this when the + /// destination key columns are not unique, because a single staged row can update several destination rows. public int? RowsMatched { get; } /// diff --git a/src/SingleStoreConnector/Utilities/SchemaDetector.cs b/src/SingleStoreConnector/Utilities/SchemaDetector.cs index ba01d9d7a..56ac73a4e 100644 --- a/src/SingleStoreConnector/Utilities/SchemaDetector.cs +++ b/src/SingleStoreConnector/Utilities/SchemaDetector.cs @@ -7,12 +7,17 @@ namespace SingleStoreConnector.Utilities; internal sealed class SchemaDetector(SingleStoreConnection connection, SingleStoreTransaction? transaction = null, int commandTimeout = 0) { + // Anchored to the start of the statement: the CREATE ... TABLE clause is always the first thing in + // SHOW CREATE TABLE output, so anchoring avoids a false positive from the phrase appearing in a column + // comment or default value later in the statement. private static readonly Regex referenceTableRegex = - new(@"CREATE\s+(?:(?:ROWSTORE|COLUMNSTORE)\s+)?REFERENCE\s+TABLE", + new(@"\A\s*CREATE\s+(?:(?:ROWSTORE|COLUMNSTORE)\s+)?REFERENCE\s+TABLE", RegexOptions.IgnoreCase | RegexOptions.Compiled); + // Matches a top-level SHARD KEY clause (optionally with an index name), anchored to the start of a table-body + // item. The optional name allows forms such as SHARD KEY `name` (col, ...). private static readonly Regex shardKeyRegex = - new(@"SHARD\s+KEY(?:\s+(?:`(?:``|[^`])*`|[^\s(]+))?\s*\((?[^)]*)\)", + new(@"\ASHARD\s+KEY(?:\s+(?:`(?:``|[^`])*`|[^\s(]+))?\s*\((?[^)]*)\)", RegexOptions.IgnoreCase | RegexOptions.Compiled); private readonly SingleStoreConnection m_connection = @@ -94,19 +99,24 @@ public async Task> GetShardKeyColumnsAsync(string tableName, IOBeha } // Method 2: Parse SHOW CREATE TABLE for SHARD KEY. - // This is a fallback for cases where SHOW INDEXES doesn't expose __SHARDKEY. + // This is a fallback for cases where SHOW INDEXES doesn't expose __SHARDKEY. Only the top-level table-body + // items are examined (not the raw statement) so that a "SHARD KEY(...)" phrase inside a column comment or + // default value cannot be mistaken for the real shard key clause. var createTableSql = await GetCreateTableStatementAsync(tableName, ioBehavior, cancellationToken) .ConfigureAwait(false); - var match = shardKeyRegex.Match(createTableSql); - if (!match.Success) - return new List(); + var body = ExtractTableBody(createTableSql, tableName); + foreach (var item in SplitTopLevel(body)) + { + var match = shardKeyRegex.Match(item.TrimStart()); + if (!match.Success) + continue; - var shardKeyList = match.Groups["columns"].Value; - if (string.IsNullOrWhiteSpace(shardKeyList)) - return new List(); + var shardKeyList = match.Groups["columns"].Value; + return string.IsNullOrWhiteSpace(shardKeyList) ? new List() : ParseIdentifierList(shardKeyList); + } - return ParseIdentifierList(shardKeyList); + return new List(); } /// @@ -187,10 +197,17 @@ public async Task> GetColumnTypeDefinitionsAsync(stri } /// - /// Gets the names of the generated (computed) columns of the specified table, as declared in - /// SHOW CREATE TABLE via an AS (...) / GENERATED ALWAYS AS (...) / COMPUTED clause. + /// Gets the names of the persistent computed (generated) columns of the specified table, as declared in + /// SHOW CREATE TABLE. /// - /// A case-insensitive set of generated column names, or an empty set if there are none. + /// + /// SingleStore declares a persistent computed column as `col` AS <expression> PERSISTED <type> + /// (see the + /// documentation). The AS keyword is what distinguishes such a column definition from a regular one; + /// the expression that follows is not necessarily parenthesised. (The COMPUTED keyword only appears in + /// DESC output, never in SHOW CREATE TABLE, so it is not matched here.) + /// + /// A case-insensitive set of computed column names, or an empty set if there are none. public async Task> GetGeneratedColumnsAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken = default) { EnsureConnectionIsOpen(); @@ -205,19 +222,18 @@ public async Task> GetGeneratedColumnsAsync(string tableName, IO { var trimmed = item.TrimStart(); - // Only column definitions (which begin with a backtick-quoted name) can be generated columns. + // Only column definitions (which begin with a backtick-quoted name) can be computed columns. if (trimmed.Length == 0 || trimmed[0] != '`') continue; var (columnName, rest) = ParseQuotedNameAndRest(trimmed); - // A generated column declares its expression with an "AS (...)" clause (optionally preceded by - // GENERATED ALWAYS), or uses the COMPUTED keyword. Look for these tokens at the top level of the - // remaining definition so a matching word inside the type, a string default or a comment is ignored. + // A persistent computed column declares its expression with an "AS" clause: `col` AS PERSISTED . + // Look for the AS keyword at the top level so a matching word inside the type, a string default or a + // comment is ignored. foreach (var token in TokenizeTopLevel(rest)) { - if (string.Equals(token, "AS", StringComparison.OrdinalIgnoreCase) || - string.Equals(token, "COMPUTED", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(token, "AS", StringComparison.OrdinalIgnoreCase)) { generatedColumns.Add(columnName); break; diff --git a/tests/SideBySide/BulkUpdateAdvancedTests.cs b/tests/SideBySide/BulkUpdateAdvancedTests.cs index 26a1a0248..499364826 100644 --- a/tests/SideBySide/BulkUpdateAdvancedTests.cs +++ b/tests/SideBySide/BulkUpdateAdvancedTests.cs @@ -191,11 +191,58 @@ public async Task RowsAffectedDependsOnUseAffectedRows(bool useAffectedRows, int var result = await bulkUpdate.WriteToServerAsync(dataTable); - // RowsMatched always reflects the join (both rows) regardless of the connection setting. + // RowsMatched counts the matched source rows (both) regardless of the connection setting. Assert.Equal(2, result.RowsMatched); Assert.Equal(expectedRowsAffected, result.RowsAffected); } + [Fact] + public async Task NonUniqueDestinationKeyMatchesMultipleRows() + { + using var connection = new SingleStoreConnection(BulkUpdateTests.GetLocalConnectionString(database)); + await connection.OpenAsync(); + + // The destination key column (grp) is not unique: one staged row matches two destination rows. RowsMatched + // counts the single matched staged row (never exceeding RowsStaged), while RowsAffected reflects the two + // destination rows the UPDATE touched. + using (var cmd = new SingleStoreCommand(@"drop table if exists bulk_update_nonunique; +create rowstore table bulk_update_nonunique(id int primary key, grp int, value varchar(100)); +insert into bulk_update_nonunique values (1, 100, 'old'), (2, 100, 'old'), (3, 200, 'old');", connection)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var dataTable = new DataTable + { + Columns = + { + new DataColumn("grp", typeof(int)), + new DataColumn("value", typeof(string)), + }, + Rows = { new object[] { 100, "updated" } }, + }; + + var bulkUpdate = new SingleStoreBulkUpdate(connection) + { + DestinationTableName = "bulk_update_nonunique", + KeyColumns = { "grp" }, + ColumnMappings = + { + new SingleStoreBulkCopyColumnMapping(0, "grp"), + new SingleStoreBulkCopyColumnMapping(1, "value"), + }, + }; + + var result = await bulkUpdate.WriteToServerAsync(dataTable); + + Assert.Equal(1, result.RowsStaged); + Assert.Equal(1, result.RowsMatched); // the single staged row matched (not the 2 destination rows it joined) + Assert.Equal(2, result.RowsAffected); // both grp=100 destination rows were updated + + using var selectCommand = new SingleStoreCommand("select count(*) from bulk_update_nonunique where grp = 100 and value = 'updated';", connection); + Assert.Equal(2L, Convert.ToInt64(await selectCommand.ExecuteScalarAsync())); + } + [Fact] public async Task SkipsMatchCountWhenComputeRowsMatchedIsFalse() { diff --git a/tests/SideBySide/SchemaDetectorTests.cs b/tests/SideBySide/SchemaDetectorTests.cs index 35d0c0b72..ae7e216f2 100644 --- a/tests/SideBySide/SchemaDetectorTests.cs +++ b/tests/SideBySide/SchemaDetectorTests.cs @@ -43,6 +43,24 @@ public async Task IsReferenceTableReturnsTrueForReferenceTable(bool isAsync) Assert.True(isReference); } + [Fact] + public async Task IsReferenceTableReturnsFalseWhenPhraseAppearsInComment() + { + await using var connection = new SingleStoreConnection(database.Connection.ConnectionString); + await connection.OpenAsync(); + + var tableName = "schema_detector_reference_comment"; + + // A regular table whose column comment contains the phrase "CREATE REFERENCE TABLE" must not be detected as + // a reference table: reference-table detection is anchored to the start of the CREATE statement. + await CreateTableAsync(connection, tableName, "id INT PRIMARY KEY, note VARCHAR(50) COMMENT 'copied via CREATE REFERENCE TABLE x'"); + + var detector = new SchemaDetector(connection); + var isReference = await detector.IsReferenceTableAsync(tableName, IOBehavior.Asynchronous); + + Assert.False(isReference); + } + [Theory] [InlineData(false)] [InlineData(true)] From d7c01daa5a9db0435714500ff66f1b3ac7dc9eda Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Thu, 2 Jul 2026 13:40:58 +0300 Subject: [PATCH 27/28] small fixes --- .../SingleStoreBulkUpdate.cs | 38 +++++++++---------- .../Utilities/SchemaDetector.cs | 17 ++++++++- .../packages.lock.json | 24 ++++++------ 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs index f6c04bfba..b4b83b84a 100644 --- a/src/SingleStoreConnector/SingleStoreBulkUpdate.cs +++ b/src/SingleStoreConnector/SingleStoreBulkUpdate.cs @@ -690,25 +690,25 @@ private static ValueTask StageWithBulkCopyAsync(Singl { switch (source) { - case DataTable dataTable: - return ioBehavior == IOBehavior.Synchronous - ? new ValueTask(bulkCopy.WriteToServer(dataTable)) - : bulkCopy.WriteToServerAsync(dataTable, cancellationToken); - - case IEnumerable dataRows: - var rows = dataRows as IReadOnlyList ?? dataRows.ToList(); - var columnCount = rows[0].Table.Columns.Count; - return ioBehavior == IOBehavior.Synchronous - ? new ValueTask(bulkCopy.WriteToServer(rows, columnCount)) - : bulkCopy.WriteToServerAsync(rows, columnCount, cancellationToken); - - case IDataReader dataReader: - return ioBehavior == IOBehavior.Synchronous - ? new ValueTask(bulkCopy.WriteToServer(dataReader)) - : bulkCopy.WriteToServerAsync(dataReader, cancellationToken); - - default: - throw new ArgumentException($"Unsupported source type '{source.GetType()}'.", nameof(source)); + case DataTable dataTable: + return ioBehavior == IOBehavior.Synchronous + ? new ValueTask(bulkCopy.WriteToServer(dataTable)) + : bulkCopy.WriteToServerAsync(dataTable, cancellationToken); + + case IEnumerable dataRows: + var rows = dataRows as IReadOnlyList ?? dataRows.ToList(); + var columnCount = rows[0].Table.Columns.Count; + return ioBehavior == IOBehavior.Synchronous + ? new ValueTask(bulkCopy.WriteToServer(rows, columnCount)) + : bulkCopy.WriteToServerAsync(rows, columnCount, cancellationToken); + + case IDataReader dataReader: + return ioBehavior == IOBehavior.Synchronous + ? new ValueTask(bulkCopy.WriteToServer(dataReader)) + : bulkCopy.WriteToServerAsync(dataReader, cancellationToken); + + default: + throw new ArgumentException($"Unsupported source type '{source.GetType()}'.", nameof(source)); } } diff --git a/src/SingleStoreConnector/Utilities/SchemaDetector.cs b/src/SingleStoreConnector/Utilities/SchemaDetector.cs index 56ac73a4e..60a1a28ff 100644 --- a/src/SingleStoreConnector/Utilities/SchemaDetector.cs +++ b/src/SingleStoreConnector/Utilities/SchemaDetector.cs @@ -31,6 +31,10 @@ internal sealed class SchemaDetector(SingleStoreConnection connection, SingleSto // shard keys, column type definitions and generated columns only queries the server once. private readonly Dictionary m_createTableCache = new(StringComparer.OrdinalIgnoreCase); + // Caches the shard key columns per table name so SHOW INDEXES is not issued more than once per operation + // (GetShardKeyColumnsAsync is called during both schema validation and staging-table creation). + private readonly Dictionary> m_shardKeyCache = new(StringComparer.OrdinalIgnoreCase); + /// /// Detects if the specified table is a reference table. /// @@ -45,10 +49,21 @@ public async Task IsReferenceTableAsync(string tableName, IOBehavior ioBeh } /// - /// Gets the shard key columns for the specified table. + /// Gets the shard key columns for the specified table, caching the result so SHOW INDEXES is issued at + /// most once per table for the lifetime of this . /// /// List of shard key column names, or empty list if no shard key. public async Task> GetShardKeyColumnsAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken = default) + { + if (m_shardKeyCache.TryGetValue(tableName, out var cached)) + return cached; + + var shardKeyColumns = await ComputeShardKeyColumnsAsync(tableName, ioBehavior, cancellationToken).ConfigureAwait(false); + m_shardKeyCache[tableName] = shardKeyColumns; + return shardKeyColumns; + } + + private async Task> ComputeShardKeyColumnsAsync(string tableName, IOBehavior ioBehavior, CancellationToken cancellationToken) { EnsureConnectionIsOpen(); diff --git a/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json b/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json index b5bae9ca2..e466b627d 100644 --- a/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json +++ b/tests/SingleStoreConnector.NativeAot.Tests/packages.lock.json @@ -4,15 +4,15 @@ "net10.0": { "Microsoft.DotNet.ILCompiler": { "type": "Direct", - "requested": "[10.0.8, )", - "resolved": "10.0.8", - "contentHash": "IG4ZS8fP7whR7OeFjC2mxq9n/yd/QsUliHdQ/Kp0OJ+RObeSJx4HLLGs7/Sxghk/U+S3rmpUOsgLheqBcqlMUw==" + "requested": "[10.0.7, )", + "resolved": "10.0.7", + "contentHash": "S53itVb3/l5wEyvdOukwxGzRtw7JSKuh2GurW3W5R57w4nxbZZ+7qCu+5jVlowLmgGrQ7yxJLZdpJc1Vpv8aig==" }, "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[10.0.9, )", - "resolved": "10.0.9", - "contentHash": "ZuAAT2Qp2l1W6urbmGoA8t8psucpTCpwjDbGYzMSUewt+IEEHz6aVRBJjrDYOu7znIxDS1fprg/VDpAdXSHi7Q==" + "requested": "[10.0.7, )", + "resolved": "10.0.7", + "contentHash": "Y0O2XrQiNYsW9UM2GhH/8faornu9pYXdd9YBcb/CrBnCOw5QtFqgh8bFd5zjxbrdoSVxGPUn1hWSrKtD0SEPzg==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", @@ -80,15 +80,15 @@ "net8.0": { "Microsoft.DotNet.ILCompiler": { "type": "Direct", - "requested": "[8.0.27, )", - "resolved": "8.0.27", - "contentHash": "2RDzeGQisWc0e4uJpT8WmMQ8Xccf4bZr4bVVV9iGnfnEZCPo//G01MLLCmGHScwLX4zCegNZRNftgUfgWlBb+g==" + "requested": "[8.0.26, )", + "resolved": "8.0.26", + "contentHash": "TopkZJbGfDfqNRQatno3/UZh8t/mo4tlglGVtcUAZDEplx/DhmBUSoT7UM7M6W0/ah1Wbe6e46FOLDyHL8K4nQ==" }, "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.28, )", - "resolved": "8.0.28", - "contentHash": "XMqgVjlLxLqWmEh3c49haXLQwsMNtvo6YscUaqfvEGfg1iA8hnYgkUVq3i9Zu9gKeNKMWiiZKVwZExc/qyEAsQ==" + "requested": "[8.0.26, )", + "resolved": "8.0.26", + "contentHash": "o7/yVssM2r9Wyln2s9edBd5ANZXqdSdBI+g7JqXkyJmXrhs2WsJp25K5yPnYrTgdKBCjKB8bg+O2oew4sgzFaA==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", From 27794cc6725995a55689d668c581c4b292d6aaf9 Mon Sep 17 00:00:00 2001 From: Olha Kramarenko Date: Wed, 8 Jul 2026 13:11:03 +0300 Subject: [PATCH 28/28] resolve comments --- docs/content/tutorials/bulk-update.md | 104 +++++++++----------------- 1 file changed, 34 insertions(+), 70 deletions(-) diff --git a/docs/content/tutorials/bulk-update.md b/docs/content/tutorials/bulk-update.md index 637fb2902..e2669136a 100644 --- a/docs/content/tutorials/bulk-update.md +++ b/docs/content/tutorials/bulk-update.md @@ -12,11 +12,9 @@ Bulk Update =========== `SingleStoreBulkUpdate` efficiently updates many existing rows in a SingleStore table from an in-memory source. -It complements [`SingleStoreBulkCopy`](../../api/SingleStoreConnector/SingleStoreBulkCopy/): where bulk copy *inserts* -rows, bulk update *modifies* rows that already exist, matching them on one or more key columns. +It complements [`SingleStoreBulkCopy`](../../api/SingleStoreConnector/SingleStoreBulkCopy/): where bulk copy *inserts* rows, bulk update *modifies* rows that already exist, matching them on one or more key columns. -It is much faster than issuing an individual `UPDATE` statement per row, because all of the work is performed in a -single round trip pattern instead of one command per row. +It is much faster than issuing an individual `UPDATE` statement per row, because all of the work is performed in a single round-trip pattern instead of one command per row. > **Note:** This API is experimental and may change in the future. @@ -25,14 +23,11 @@ How it works `SingleStoreBulkUpdate` performs the update in three phases, all on the same connection: -1. It creates a temporary staging table whose columns mirror the mapped columns of the destination table (their exact - types, lengths, and collations are copied from the destination so values round-trip without conversion). +1. It creates a temporary staging table whose columns mirror the mapped columns of the destination table (their exact types, lengths, and collations are copied from the destination so values round-trip without conversion). 2. It loads the source rows into that staging table using `SingleStoreBulkCopy`. -3. It runs a single `UPDATE ... JOIN` that copies the non-key column values from the staging table into the matching - rows of the destination table, joining on the key columns. The staging table is then dropped. +3. It runs a single `UPDATE ... JOIN` that copies the non-key column values from the staging table into the matching rows of the destination table, joining on the key columns. The staging table is then dropped. -Because staging uses `SingleStoreBulkCopy` (which loads data with `LOAD DATA LOCAL INFILE`), the connection string -*must* have `AllowLoadLocalInfile=true` in order to use this class. +Because staging uses `SingleStoreBulkCopy` (which loads data with `LOAD DATA LOCAL INFILE`), the connection string *must* have `AllowLoadLocalInfile=true` in order to use this class. Basic example ------------- @@ -78,13 +73,9 @@ if (result.Warnings.Count != 0) { /* handle potential data loss warnings */ } Column mappings --------------- -Each `SingleStoreBulkCopyColumnMapping` maps a **source ordinal** (the zero-based column index in the source data) to a -**destination column name**. The names of the columns in the source `DataTable`/`DataReader` are ignored; only the -ordinal matters. Every key column must be included in the mappings, and at least one non-key column must be mapped so -that there is something to update. +Each `SingleStoreBulkCopyColumnMapping` maps a **source ordinal** (the zero-based column index in the source data) to a **destination column name**. The names of the columns in the source `DataTable`/`DataReader` are ignored; only the ordinal matters. Every key column must be included in the mappings, and at least one non-key column must be mapped so that there is something to update. -Source data may be supplied as a `DataTable`, a sequence of `DataRow` objects, or an `IDataReader`. Both synchronous -(`WriteToServer`) and asynchronous (`WriteToServerAsync`) methods are available. +Source data may be supplied as a `DataTable`, a sequence of `DataRow` objects, or an `IDataReader`. Both synchronous (`WriteToServer`) and asynchronous (`WriteToServerAsync`) methods are available. Interpreting the result ------------------------ @@ -92,27 +83,22 @@ Interpreting the result `WriteToServerAsync` returns a `SingleStoreBulkUpdateResult`: * `RowsStaged` — the number of source rows loaded into the staging table. -* `RowsMatched` — the number of staged rows that matched at least one destination row on the key columns. This never - exceeds `RowsStaged`, so `RowsStaged - RowsMatched` is the number of source rows that matched nothing. (`RowsAffected` - can still exceed this when the destination keys are not unique — see below.) This is `null` when `ComputeRowsMatched` - is set to `false` (see below). -* `RowsAffected` — the number of rows affected by the `UPDATE`, as reported by the server. Its exact meaning depends on - the connection's `UseAffectedRows` setting: with the default (`UseAffectedRows=false`) it counts the rows *matched* by - the update — including rows that already held the new values — so it typically equals `RowsMatched`; with - `UseAffectedRows=true` it counts only the rows whose values actually *changed*. -* `Warnings` — any warnings raised while staging or updating; check that this is empty to avoid silent data loss from - failed type conversions. +* `RowsMatched` — the number of staged rows that matched at least one destination row on the key columns. This never exceeds `RowsStaged`, so `RowsStaged - RowsMatched` is the number of source rows that matched nothing. (`RowsAffected` can still exceed this when the destination keys are not unique — see below.) This is `null` when `ComputeRowsMatched` is set to `false` (see below). +* `RowsAffected` — the number of rows affected by the `UPDATE`, as reported by the server. Its exact meaning depends on the connection's `UseAffectedRows` setting: with the default (`UseAffectedRows=false`) it counts the rows *matched* by the update — including rows that already held the new values — so it typically equals `RowsMatched`; with `UseAffectedRows=true` it counts only the rows whose values actually *changed*. +* `Warnings` — any warnings raised while staging or updating; check that this is empty to avoid silent data loss from failed type conversions. Performance ----------- -* Set `ComputeRowsMatched = false` to skip the extra `COUNT(*)` query that populates `RowsMatched`. When disabled, - `RowsMatched` is `null`. -* Set `BulkUpdateTimeout` (in seconds) to bound how long each phase may run. It defaults to `0` (no timeout), so a - finite value should be set deliberately if a phase must not run unbounded. -* Set `NotifyAfter` to a non-zero value to receive `SingleStoreRowsStaged` events while rows are being staged; the - event handler can set `Abort = true` to cancel the operation. Aborting stops staging and skips the `UPDATE` - entirely, so no rows in the destination table are modified. +`SingleStoreBulkUpdate` pays a fixed cost up front to create and load a staging table, then updates all matching rows with a single `UPDATE ... JOIN`. As a result, its cost usually grows much more slowly with row count than issuing one `UPDATE` per row or using `SingleStoreBatch` for many individual updates. + +In local benchmarks, that fixed setup cost made `SingleStoreBulkUpdate` slower for 100 rows, roughly comparable around 1,000 rows, and significantly faster from 10,000 rows onward. In remote benchmarks, where per-row updates pay repeated network round trips, `SingleStoreBulkUpdate` was already faster at 100 rows and the advantage grew rapidly with larger batches. + +Prefer `SingleStoreBulkUpdate` when updating many rows or when updating over a remote connection. For a small, fixed number of rows on a low-latency connection, a plain looped `UPDATE` or `SingleStoreBatch` may be simpler and comparable. + +* Set `ComputeRowsMatched = false` to skip the extra `COUNT(*)` query that populates `RowsMatched`. When disabled, `RowsMatched` is `null`. +* Set `BulkUpdateTimeout` (in seconds) to bound how long each phase may run. It defaults to `0` (no timeout), so a finite value should be set deliberately if a phase must not run unbounded. +* Set `NotifyAfter` to a non-zero value to receive `SingleStoreRowsStaged` events while rows are being staged; the event handler can set `Abort = true` to cancel the operation. Aborting stops staging and skips the `UPDATE` entirely, so no rows in the destination table are modified. Limitations ----------- @@ -123,55 +109,33 @@ Limitations * At least one non-key column must be mapped, so there is a column to update. * Duplicate key values in the source data are rejected; they would collide in the staging table's primary key. * Shard key columns cannot be updated, because SingleStore does not allow updating a shard key. -* Generated (computed) columns cannot be mapped (neither as key nor update columns), because the staging table cannot - reproduce a generated column's definition. +* Generated (computed) columns cannot be mapped (neither as key nor update columns), because the staging table cannot reproduce a generated column's definition. * Reference tables are not supported as the destination. * Expression column mappings (a `SingleStoreBulkCopyColumnMapping` with an `Expression`) are not supported. Other things to be aware of: -* **Key columns need not be unique in the destination.** The key columns identify rows to update via a join; they are - not required to be a unique or primary key on the destination table. If they are not unique, a single source row can - update multiple destination rows, so `RowsAffected` can exceed the number of source rows (`RowsMatched`, which counts - matched source rows, cannot). Duplicate keys are only rejected in the *source* data (they would collide in the - staging table's primary key). -* **Key column types.** The key columns become the primary key of the staging table, so they must be types that - SingleStore allows in a primary key. Large `TEXT`/`BLOB`/`JSON`/spatial columns are not usable as key columns. -* **Required privileges.** In addition to `UPDATE` on the destination table, the connection needs permission to run - `SHOW CREATE TABLE`, `SHOW INDEXES`, and a schema-only `SELECT` against it, because the operation inspects the - table's schema before updating. -* **`IDataReader` source.** When the source is an `IDataReader`, it must be opened on a *different* connection than - the one used for the bulk update. The update connection runs schema queries, creates the staging table, and loads - data, so it cannot have an open reader on it at the same time. -* **Thread safety.** A `SingleStoreBulkUpdate` instance is not thread-safe. Do not share an instance across concurrent - operations. +* **Key columns need not be unique in the destination.** The key columns identify rows to update via a join; they are not required to be a unique or primary key on the destination table. If they are not unique, a single source row can update multiple destination rows, so `RowsAffected` can exceed the number of source rows (`RowsMatched`, which counts matched source rows, cannot). Duplicate keys are only rejected in the *source* data (they would collide in the staging table's primary key). +* **Key column types.** The key columns become the primary key of the staging table, so they must be types that SingleStore allows in a primary key. Large `TEXT`/`BLOB`/`JSON`/spatial columns are not usable as key columns. +* **Required privileges.** In addition to `UPDATE` on the destination table, the connection needs permission to run `SHOW CREATE TABLE`, `SHOW INDEXES`, and a schema-only `SELECT` against it, because the operation inspects the table's schema before updating. +* **`IDataReader` source.** When the source is an `IDataReader`, it must be opened on a *different* connection than the one used for the bulk update. The update connection runs schema queries, creates the staging table, and loads data, so it cannot have an open reader on it at the same time. +* **Thread safety.** A `SingleStoreBulkUpdate` instance is not thread-safe. Do not share an instance across concurrent operations. Transactions ------------ -A `SingleStoreTransaction` may be passed to the constructor. When supplied, all phases participate in that transaction, -so the update can be committed or rolled back atomically with other work on the connection. +A `SingleStoreTransaction` may be passed to the constructor. When supplied, all phases participate in that transaction, so the update can be committed or rolled back atomically with other work on the connection. Future directions ----------------- -This API is experimental. Several of the limitations above are deliberate scope decisions for the first version rather -than restrictions imposed by SingleStore, and may be relaxed in a future version: - -* **`KeyColumns` is required.** A future version could auto-detect the destination table's primary key when - `KeyColumns` is not set. -* **Expression column mappings are rejected.** Because the source is staged into a real temporary table, a - `SingleStoreBulkCopyColumnMapping.Expression` (which relies on user variables during `LOAD DATA`) is not supported; - a future version could apply expressions when populating the staging table. -* **Generated (computed) columns cannot be mapped.** This could be supported for key columns if the staging table - learned to reproduce a generated column's underlying type. -* **Row counts are `int`.** `RowsStaged`, `RowsMatched`, and `RowsAffected` are `int` for consistency with - `SingleStoreBulkCopyResult`; they could widen to `long` if very large updates need it. +This API is experimental. Several of the limitations above are deliberate scope decisions for the first version rather than restrictions imposed by SingleStore, and may be relaxed in a future version: + +* **`KeyColumns` is required.** A future version could auto-detect the destination table's primary key when `KeyColumns` is not set. +* **Expression column mappings are rejected.** Because the source is staged into a real temporary table, a `SingleStoreBulkCopyColumnMapping.Expression` (which relies on user variables during `LOAD DATA`) is not supported; a future version could apply expressions when populating the staging table. +* **Generated (computed) columns cannot be mapped.** This could be supported for key columns if the staging table learned to reproduce a generated column's underlying type. +* **Row counts are `int`.** `RowsStaged`, `RowsMatched`, and `RowsAffected` are `int` for consistency with `SingleStoreBulkCopyResult`; they could widen to `long` if very large updates need it. * **Update only.** There is no upsert (insert-or-update) mode; a future version could add one. -* **Reference tables are not supported as the destination.** The `UPDATE ... JOIN` this API generates fails on a - reference-table target joined to a (sharded) staging table (SingleStore error 1706). This is a scope decision, not - a hard limit — a future version could support it by staging into a reference table or by using an upsert, per the - [documented workarounds](https://docs.singlestore.com/cloud/reference/troubleshooting-reference/query-errors/error-1706-hy-000-feature-multi-table-update-delete-with-a-reference-table-as-target-table-is-not-supported-by-memsql/). +* **Reference tables are not supported as the destination.** The `UPDATE ... JOIN` this API generates fails on a reference-table target joined to a (sharded) staging table (SingleStore error 1706). This is a scope decision, not a hard limit — a future version could support it by staging into a reference table or by using an upsert, per the [documented workarounds](https://docs.singlestore.com/cloud/reference/troubleshooting-reference/query-errors/error-1706-hy-000-feature-multi-table-update-delete-with-a-reference-table-as-target-table-is-not-supported-by-memsql/). -By contrast, the following is a SingleStore behavior rather than a choice made here, and is not expected to change: -updating shard key columns is not allowed. +By contrast, the following is a SingleStore behavior rather than a choice made here, and is not expected to change: updating shard key columns is not allowed.