diff --git a/src/sql/doc.md b/src/sql/doc.md index 8beb54e1a..eb090ea2c 100644 --- a/src/sql/doc.md +++ b/src/sql/doc.md @@ -203,5 +203,5 @@ For postgres, setup the fantest database and user account via: In addition, both JDBC drivers must be installed, and "etc/sql/config.props" must have a reference to the classpath of both drivers, e.g. - java.drivers=java.drivers=com.mysql.cj.jdbc.Driver,org.postgresql.Driver + java.drivers=com.mysql.cj.jdbc.Driver,org.postgresql.Driver diff --git a/src/sql/fan/BatchExecutor.fan b/src/sql/fan/BatchExecutor.fan index 0b85e1e93..82a19b53a 100644 --- a/src/sql/fan/BatchExecutor.fan +++ b/src/sql/fan/BatchExecutor.fan @@ -28,21 +28,25 @@ class BatchExecutor queued.add(params) if (queued.size > maxChunkSize) { - results.addAll(stmt.executeBatch(queued)) + r := stmt.executeBatch(queued) + result.updateCounts.addAll(r.updateCounts) + result.keys.addAll(r.keys) queued.clear } } - ** Finish execution, and return the concatenated results from each - ** call to Statement.executeBatch(). - Int?[] finish() + ** Finish execution, and return the collated results from each call + ** to Statement.executeBatch(). + BatchResult finish() { if (queued.size > 0) { - results.addAll(stmt.executeBatch(queued)) + r := stmt.executeBatch(queued) + result.updateCounts.addAll(r.updateCounts) + result.keys.addAll(r.keys) queued.clear } - return results + return result } ////////////////////////////////////////////////////////////////////////// @@ -53,6 +57,6 @@ class BatchExecutor private Int maxChunkSize private [Str:Obj][] queued := [Str:Obj][,] - private Int?[] results := [,] + private BatchResult result := BatchResult(Int?[,], Obj?[,]) } diff --git a/src/sql/fan/Statement.fan b/src/sql/fan/Statement.fan index 6d9f5269a..6f2365ba2 100644 --- a/src/sql/fan/Statement.fan +++ b/src/sql/fan/Statement.fan @@ -66,14 +66,23 @@ class Statement ** ** Execute a batch of commands on a prepared Statement. If all commands - ** execute successfully, returns an array of update counts. + ** execute successfully, return a BatchResult containing an array of update + ** counts, and an array of auto-generated keys. ** - ** For each element in the array, if the element is non-null, then it - ** represents an update count giving the number of rows in the database that - ** were affected by the command's execution. + ** For each element in the update counts array, if the element is non-null, + ** then it represents an update count giving the number of rows in the + ** database that were affected by the command's execution. If a given array + ** element is null, it indicates that the command was processed successfully + ** but that the number of rows affected is unknown. ** - ** If a given array element is null, it indicates that the command was - ** processed successfully but that the number of rows affected is unknown. + ** The keys array contains the auto-generated keys, if any, for each command. + ** An element is null if the driver did not return a key for that command. + ** Note that some drivers return the keys even if they were not + ** auto-generated. + ** + ** If the driver returns auto-generated keys for some but not all of the + ** commands, throw SqlErr since the keys cannot be correlated to their + ** commands. ** ** If one of the commands in a batch update fails to execute properly, this ** method throws a SqlErr that wraps a java.sql.BatchUpdateException, and a @@ -81,7 +90,7 @@ class Statement ** the batch, consistent with the underlying DBMS -- either always continuing ** to process commands or never continuing to process commands. ** - native Int?[] executeBatch([Str:Obj]?[] paramsList) + native BatchResult executeBatch([Str:Obj]?[] paramsList) ** ** If the last execute has more results from a multi-result stored @@ -118,3 +127,23 @@ class Statement } +************************************************************************** +** BatchResult +************************************************************************** + +** BatchResult contains the result of a call to Statement.executeBatch(). +class BatchResult +{ + new make(Int?[] updateCounts, Obj?[] keys) + { + this.updateCounts = updateCounts + this.keys = keys + } + + ** The number of rows that were affected by each command's execution. + Int?[] updateCounts + + ** The keys that were auto-generated by each command. + Obj?[] keys +} + diff --git a/src/sql/java/StatementPeer.java b/src/sql/java/StatementPeer.java index c394a94ca..ce5ccc8c1 100644 --- a/src/sql/java/StatementPeer.java +++ b/src/sql/java/StatementPeer.java @@ -305,10 +305,7 @@ private Object executeResult(Statement self, boolean isResultSet) List keys = null; while (rs.next()) { - // get key as Long or String - Object key; - try { key = rs.getLong(1); } - catch (Exception e) { key = rs.getString(1); } + Object key = readGenKey(rs); // lazily create keys list with proper type if (keys == null) @@ -330,12 +327,60 @@ private Object executeResult(Statement self, boolean isResultSet) return Long.valueOf(-1); } - public List executeBatch(Statement self, List paramsList) + /** + * Read the auto-generated key for the current row as a Long, or as a + * String for databases like Oracle which do not allow access as an Int. + */ + private static Object readGenKey(ResultSet rs) + throws SQLException + { + try { return rs.getLong(1); } + catch (Exception e) { return rs.getString(1); } + } + + /** + * Auto-generated keys for the batch just executed, with one entry per + * command. Entries are null if the driver returned no keys at all. If + * the driver returns keys for some but not all of the commands they cannot + * be correlated to their commands, so that case is an error. + */ + private List readBatchGenKeys(PreparedStatement pstmt, int numCommands) + throws SQLException + { + List keys = List.make(Sys.ObjType.toNullable(), numCommands); + + if (isAutoKeys) + { + ResultSet rs = pstmt.getGeneratedKeys(); + try + { + while (rs.next()) keys.add(readGenKey(rs)); + } + finally + { + rs.close(); + } + + if (keys.size() == numCommands) return keys; + + if (keys.size() != 0) + throw SqlErr.make("Driver returned " + keys.size() + + " auto-generated keys for " + numCommands + " batch commands"); + } + + // no keys returned: one null per command + for (int i = 0; i < numCommands; i++) keys.add(null); + return keys; + } + + public BatchResult executeBatch(Statement self, List paramsList) { if (!prepared) throw SqlErr.make("Statement has not been prepared."); PreparedStatement pstmt = (PreparedStatement)stmt; + List genKeys = List.make(Sys.ObjType.toNullable(), 0); + try { // add batch @@ -348,8 +393,11 @@ public List executeBatch(Statement self, List paramsList) // execute batch int[] exec = pstmt.executeBatch(); + // gather auto-generated keys before the statement is reused + genKeys = readBatchGenKeys(pstmt, (int)paramsList.size()); + // process result - List result = List.make(Sys.IntType, exec.length); + List updateCounts = List.make(Sys.IntType, exec.length); for (int i = 0; i < exec.length; i++) { int n = exec[i]; @@ -358,9 +406,10 @@ public List executeBatch(Statement self, List paramsList) // java.sql.Statement.SUCCESS_NO_INFO. We treat that as a null, // indicating that the command was processed successfully but that the // number of rows affected is unknown. - result.add(n < 0 ? null : (long)n); + updateCounts.add(n < 0 ? null : (long)n); } - return result; + + return BatchResult.make(updateCounts, genKeys); } catch (SQLException ex) { diff --git a/src/sql/test/SqlTest.fan b/src/sql/test/SqlTest.fan index b36342320..ff6edc5d9 100644 --- a/src/sql/test/SqlTest.fan +++ b/src/sql/test/SqlTest.fan @@ -48,8 +48,15 @@ class SqlTest : Test transactions preparedStmts executeStmts - batchExecute - batchExecutor + + batchUpdate + batchInsert + batchInsertAuto + + batchExecutorUpdate + batchExecutorInsert + batchExecutorInsertAuto + withPrepare mysqlVariable postgresBuf @@ -530,7 +537,7 @@ class SqlTest : Test // Batch execution ////////////////////////////////////////////////////////////////////////// - Void batchExecute() + Void batchUpdate() { params := [Str:Obj][,] stmt := db.sql("select farmer_id from farmers") @@ -548,9 +555,13 @@ class SqlTest : Test // executeBatch stmt.prepare res := stmt.executeBatch(params) + // The result will be an array filled with '1', // indicating that each row was updated. - verifyEq(res, Int[,].fill(1, params.size)) + verifyEq(res.updateCounts, Int[,].fill(1, params.size)) + + // The keys will all be null, because nothing was autogenerated + verifyEq(res.keys, Obj?[,].fill(null, params.size)) // double check db.commit @@ -562,7 +573,103 @@ class SqlTest : Test } } - Void batchExecutor() + Void batchInsert() + { + // drop and re-create + if (db.meta.tableExists("batch_insert")) + db.sql("drop table batch_insert").execute + + if (dbType == DbType.postgres) + { + db.sql( + "create table batch_insert ( + id int, + name text not null)").execute + } + else + { + db.sql( + "create table batch_insert ( + id int not null, + name varchar(255) not null, + primary key (id))").execute + } + + // batch insert + stmt := db.sql( + "insert into batch_insert (id, name) values (@id, @name)").prepare + res := stmt.executeBatch([ + ["id": 1, "name":"aaa"], + ["id": 2, "name":"bbb"], + ["id": 3, "name":"ccc"], + ["id": 4, "name":"ddd"], + ]) + + // check keys. Note that postgres populates the keys even + // though they weren't auto-generated + if (dbType == DbType.postgres) + verifyEq(res.keys, Obj?[1,2,3,4]) + else + verifyEq(res.keys, Obj?[null,null,null,null]) + + // make sure records were created + stmt = db.sql("select * from batch_insert order by id") + n := 0 + names := ["aaa", "bbb", "ccc", "ddd"] + stmt.queryEach(null) |r| { + verifyEq(r->id, n+1) + verifyEq(r->name, names[n]) + n++ + } + } + + Void batchInsertAuto() + { + // drop and re-create + if (db.meta.tableExists("batch_insert_auto")) + db.sql("drop table batch_insert_auto").execute + + if (dbType == DbType.postgres) + { + db.sql( + "create table batch_insert_auto ( + id serial, + name text not null)").execute + } + else + { + db.sql( + "create table batch_insert_auto ( + id int auto_increment not null, + name varchar(255) not null, + primary key (id))").execute + } + + // batch insert + stmt := db.sql( + "insert into batch_insert_auto (name) values (@name)").prepare + res := stmt.executeBatch([ + ["name":"aaa"], + ["name":"bbb"], + ["name":"ccc"], + ["name":"ddd"], + ]) + + // check auto-generated keys + verifyEq(res.keys, Obj?[1,2,3,4]) + + // make sure records were created + stmt = db.sql("select * from batch_insert_auto order by id") + n := 0 + names := ["aaa", "bbb", "ccc", "ddd"] + stmt.queryEach(null) |r| { + verifyEq(r->id, n+1) + verifyEq(r->name, names[n]) + n++ + } + } + + Void batchExecutorUpdate() { ids := Int[,] stmt := db.sql("select farmer_id from farmers") @@ -577,7 +684,7 @@ class SqlTest : Test res := batch.finish // The result will be an array filled with '1', // indicating that each row was updated. - verifyEq(res, Int?[,].fill(1, ids.size)) + verifyEq(res.updateCounts, Int?[,].fill(1, ids.size)) // double check db.commit @@ -589,6 +696,106 @@ class SqlTest : Test } } + Void batchExecutorInsert() + { + // drop and re-create + if (db.meta.tableExists("batch_insert")) + db.sql("drop table batch_insert").execute + + if (dbType == DbType.postgres) + { + db.sql( + "create table batch_insert ( + id int, + name text not null)").execute + } + else + { + db.sql( + "create table batch_insert ( + id int not null, + name varchar(255) not null, + primary key (id))").execute + } + + // batch insert + stmt := db.sql( + "insert into batch_insert (id, name) values (@id, @name)").prepare + batch := BatchExecutor(stmt, 3) + [ + ["id": 1, "name":"aaa"], + ["id": 2, "name":"bbb"], + ["id": 3, "name":"ccc"], + ["id": 4, "name":"ddd"], + ].each |x| { batch.add(x) } + res := batch.finish + + // check keys. Note that postgres populates the keys even + // though they weren't auto-generated + if (dbType == DbType.postgres) + verifyEq(res.keys, Obj?[1,2,3,4]) + else + verifyEq(res.keys, Obj?[null,null,null,null]) + + // make sure records were created + stmt = db.sql("select * from batch_insert order by id") + n := 0 + names := ["aaa", "bbb", "ccc", "ddd"] + stmt.queryEach(null) |r| { + verifyEq(r->id, n+1) + verifyEq(r->name, names[n]) + n++ + } + } + + Void batchExecutorInsertAuto() + { + // drop and re-create + if (db.meta.tableExists("batch_insert_auto")) + db.sql("drop table batch_insert_auto").execute + + if (dbType == DbType.postgres) + { + db.sql( + "create table batch_insert_auto ( + id serial, + name text not null)").execute + } + else + { + db.sql( + "create table batch_insert_auto ( + id int auto_increment not null, + name varchar(255) not null, + primary key (id))").execute + } + + // batch insert + stmt := db.sql( + "insert into batch_insert_auto (name) values (@name)").prepare + batch := BatchExecutor(stmt, 3) + [ + ["name":"aaa"], + ["name":"bbb"], + ["name":"ccc"], + ["name":"ddd"], + ].each |x| { batch.add(x) } + res := batch.finish + + // check auto-generated keys + verifyEq(res.keys, Obj?[1,2,3,4]) + + // make sure records were created + stmt = db.sql("select * from batch_insert_auto order by id") + n := 0 + names := ["aaa", "bbb", "ccc", "ddd"] + stmt.queryEach(null) |r| { + verifyEq(r->id, n+1) + verifyEq(r->name, names[n]) + n++ + } + } + ////////////////////////////////////////////////////////////////////////// // Pool //////////////////////////////////////////////////////////////////////////