From 81c0c77f27bf3ea4ff1ee2e85d782d4c73ff9161 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 12 May 2026 16:44:33 +0900 Subject: [PATCH 01/76] Add missing include in Cluster.pm The postmaster test 004_negotiate.pl could fail due to IO::Socket::INET gone missing, in environments that cannot use Unix sockets. Oversight in the backport done in 6dffaeb8e54c, so like the other commit this is applied across the v14~17 range. Per buildfarm member drongo. Security: CVE-2026-6479 Backpatch-through: 14 --- src/test/perl/PostgresNode.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index b4424079fa..11c2c6594b 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -98,6 +98,7 @@ use File::Path qw(rmtree mkpath); use File::Spec; use File::stat qw(stat); use File::Temp (); +use IO::Socket::INET; use IPC::Run; use PostgresVersion; use RecursiveCopy; From e3c4e374648e033aa864e9b002090ac84b3d1aab Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 13 May 2026 11:44:31 +0900 Subject: [PATCH 02/76] Fix stale COPY progress during logical replication table sync Previously, pg_stat_progress_copy in the subscriber could continue to show the initial COPY operation for logical replication table synchronization as active even after the data copy had finished. The stale progress entry remained visible until synchronization caught up with the publisher. This happened because the table synchronization code called BeginCopyFrom() and CopyFrom(), but failed to call EndCopyFrom() afterward. This commit fixes the issue by adding the missing EndCopyFrom() call so that the COPY progress state in the subscriber is cleared as soon as the initial data copy completes. Backpatch to all supported branches. Author: Shinya Kato Reviewed-by: Fujii Masao Reviewed-by: ChangAo Chen Reviewed-by: Chao Li Discussion: https://postgr.es/m/CAOzEurQKuy3RiPkd=25PEwEzaqHuGvEOf=X7vaVzhgNjaukYzA@mail.gmail.com Backpatch-through: 14 --- src/backend/replication/logical/tablesync.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index c02d2b7b5f..dcd96da4fa 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -901,6 +901,7 @@ copy_table(Relation rel) /* Do the copy */ (void) CopyFrom(cstate); + EndCopyFrom(cstate); logicalrep_rel_close(relmapentry, NoLock); } From 5c00f4e2e3bcee6931ae93429d53f7c2a4f46156 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 13 May 2026 14:43:52 +0900 Subject: [PATCH 03/76] Add more tests for corrupted data with pglz_decompress() Two cases fixed by 2b5ba2a0a141 were not covered, to emulate the handling of corrupted data, for: - set control bit with a valid 2-byte match tag where offset is 0. - set control bit with a valid 2-byte match tag where offset exceeds output written. Oversight in 67d318e70402. Reviewed-by: Ayush Tiwari Discussion: https://postgr.es/m/agF4xkIdRcrCIprs@paquier.xyz Backpatch-through: 14 --- src/test/regress/input/compression_pglz.source | 10 ++++++++++ src/test/regress/output/compression_pglz.source | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/test/regress/input/compression_pglz.source b/src/test/regress/input/compression_pglz.source index 499ac4cee5..90eee2034c 100644 --- a/src/test/regress/input/compression_pglz.source +++ b/src/test/regress/input/compression_pglz.source @@ -42,6 +42,16 @@ SELECT test_pglz_decompress('\x01ff'::bytea, 1024, true); SELECT test_pglz_decompress('\x010f01'::bytea, 1024, false); SELECT test_pglz_decompress('\x010f01'::bytea, 1024, true); +-- Corrupted compressed data. Set control bit with a valid 2-byte match +-- tag where offset exceeds output written. +SELECT test_pglz_decompress('\x011001'::bytea, 1024, false); +SELECT test_pglz_decompress('\x011001'::bytea, 1024, true); + +-- Corrupted compressed data. Set control bit with a valid 2-byte match +-- tag where offset is 0. +SELECT test_pglz_decompress('\x010300'::bytea, 1024, false); +SELECT test_pglz_decompress('\x010300'::bytea, 1024, true); + -- Clean up DROP FUNCTION test_pglz_compress; DROP FUNCTION test_pglz_decompress; diff --git a/src/test/regress/output/compression_pglz.source b/src/test/regress/output/compression_pglz.source index 910a20acf0..b46632842d 100644 --- a/src/test/regress/output/compression_pglz.source +++ b/src/test/regress/output/compression_pglz.source @@ -56,6 +56,18 @@ SELECT test_pglz_decompress('\x010f01'::bytea, 1024, false); ERROR: pglz_decompress failed SELECT test_pglz_decompress('\x010f01'::bytea, 1024, true); ERROR: pglz_decompress failed +-- Corrupted compressed data. Set control bit with a valid 2-byte match +-- tag where offset exceeds output written. +SELECT test_pglz_decompress('\x011001'::bytea, 1024, false); +ERROR: pglz_decompress failed +SELECT test_pglz_decompress('\x011001'::bytea, 1024, true); +ERROR: pglz_decompress failed +-- Corrupted compressed data. Set control bit with a valid 2-byte match +-- tag where offset is 0. +SELECT test_pglz_decompress('\x010300'::bytea, 1024, false); +ERROR: pglz_decompress failed +SELECT test_pglz_decompress('\x010300'::bytea, 1024, true); +ERROR: pglz_decompress failed -- Clean up DROP FUNCTION test_pglz_compress; DROP FUNCTION test_pglz_decompress; From 1de0a711db9b0656733789df44c7bc4e4ddfc9fd Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Thu, 14 May 2026 13:11:49 -0500 Subject: [PATCH 04/76] refint: Fix segfault in check_foreign_key(). When an UPDATE statement triggers check_foreign_key() with the action set to "cascade", it generates more UPDATE statements to modify the key values in referencing relations. If a new key value is NULL, SPI_getvalue() returns a NULL pointer, which is subsequently passed to quote_literal_cstr(), causing a segfault. To fix, skip quoting when a new key value is NULL and insert an unquoted NULL keyword instead. Oversight in commit 260e97733b. While the refint documentation recommends marking primary key columns NOT NULL, the aforementioned scenario accidentally worked on platforms where snprintf() substitutes "(null)" for NULL pointers. Note that for character-type columns, the old code quoted "(null)" as a string literal, so this didn't always produce correct results. But it still seems better to fix this than to reject cases that previously worked. Reported-by: Nikita Kalinin Author: Ayush Tiwari Reviewed-by: Pierre Forstmann Discussion: https://postgr.es/m/19476-bd04ea6241345303%40postgresql.org Backpatch-through: 14 --- contrib/spi/refint.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/spi/refint.c b/contrib/spi/refint.c index cbef463230..413411c173 100644 --- a/contrib/spi/refint.c +++ b/contrib/spi/refint.c @@ -480,7 +480,8 @@ check_foreign_key(PG_FUNCTION_ARGS) nv = SPI_getvalue(newtuple, tupdesc, fn); appendStringInfo(&sql, " %s = %s ", - args2[k], quote_literal_cstr(nv)); + args2[k], + nv ? quote_literal_cstr(nv) : "NULL"); if (k < nkeys) appendStringInfoString(&sql, ", "); } From 092b570a72dd71544090f62fdda6e7e7702df2b8 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 15 May 2026 18:02:54 +0900 Subject: [PATCH 05/76] Re-add regression tests for ltree and intarray These tests have been removed by 906ea101d0d5, due to some of them being unstable in the buildfarm with low max_stack_depth values. They are now reworked so as they should be more portable. The tests to cover the findoprnd() overflows use a balanced tree to avoid using too much stack, per a suggestion and an investigation by Tom Lane. Note: This is initially applied only on HEAD; a backpatch will follow should the buildfarm be fine with the situation. Discussion: https://postgr.es/m/agZc6XecyE7E7fep@paquier.xyz Backpatch-through: 14 --- contrib/intarray/expected/_int.out | 15 +++++++++++++++ contrib/intarray/sql/_int.sql | 12 ++++++++++++ contrib/ltree/expected/ltree.out | 24 ++++++++++++++++++++++++ contrib/ltree/sql/ltree.sql | 20 ++++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/contrib/intarray/expected/_int.out b/contrib/intarray/expected/_int.out index 64d8878763..f2b2b09403 100644 --- a/contrib/intarray/expected/_int.out +++ b/contrib/intarray/expected/_int.out @@ -398,6 +398,21 @@ SELECT '1&(2&(4&(5|!6)))'::query_int; 1 & 2 & 4 & ( 5 | !6 ) (1 row) +-- Test for overflow of the int16 "left" field in findoprnd(). +-- This query uses a balanced binary tree to avoid using too much stack. +DO $$ +DECLARE + e text := '1'; +BEGIN + FOR i IN 1..15 LOOP + e := '(' || e || '&' || e || ')'; + END LOOP; + PERFORM ('0|' || e)::query_int; +END; +$$; +ERROR: query_int expression is too complex +CONTEXT: SQL statement "SELECT ('0|' || e)::query_int" +PL/pgSQL function inline_code_block line 8 at PERFORM CREATE TABLE test__int( a int[] ); \copy test__int from 'data/test__int.data' ANALYZE test__int; diff --git a/contrib/intarray/sql/_int.sql b/contrib/intarray/sql/_int.sql index ba4c298151..f4871d0a7a 100644 --- a/contrib/intarray/sql/_int.sql +++ b/contrib/intarray/sql/_int.sql @@ -75,6 +75,18 @@ SELECT '1&2&4&5&6'::query_int; SELECT '1&(2&(4&(5|6)))'::query_int; SELECT '1&(2&(4&(5|!6)))'::query_int; +-- Test for overflow of the int16 "left" field in findoprnd(). +-- This query uses a balanced binary tree to avoid using too much stack. +DO $$ +DECLARE + e text := '1'; +BEGIN + FOR i IN 1..15 LOOP + e := '(' || e || '&' || e || ')'; + END LOOP; + PERFORM ('0|' || e)::query_int; +END; +$$; CREATE TABLE test__int( a int[] ); \copy test__int from 'data/test__int.data' diff --git a/contrib/ltree/expected/ltree.out b/contrib/ltree/expected/ltree.out index 28c321a4cf..20f6ad7d0f 100644 --- a/contrib/ltree/expected/ltree.out +++ b/contrib/ltree/expected/ltree.out @@ -1267,6 +1267,21 @@ SELECT 'tree.awdfg_qwerty'::ltree @ 'tree & aw_rw%*'::ltxtquery; f (1 row) +-- Test for overflow of the int16 "left" field in findoprnd(). +-- This query uses a balanced binary tree to avoid using too much stack. +DO $$ +DECLARE + e text := 'a'; +BEGIN + FOR i IN 1..14 LOOP + e := '(' || e || '&' || e || ')'; + END LOOP; + PERFORM ('b|' || e)::ltxtquery; +END; +$$; +ERROR: ltxtquery is too large +CONTEXT: SQL statement "SELECT ('b|' || e)::ltxtquery" +PL/pgSQL function inline_code_block line 8 at PERFORM --arrays SELECT '{1.2.3}'::ltree[] @> '1.2.3.4'; ?column? @@ -8089,3 +8104,12 @@ SELECT count(*) FROM _ltreetest WHERE t ? '{23.*.1,23.*.2}' ; 15 (1 row) +-- Test for overflow of lquery_level.totallen. +SELECT (repeat('x', 255) || repeat('|' || repeat('x', 255), 256))::lquery; +ERROR: lquery level is too large +DETAIL: Total size of level exceeds the maximum allowed (65535 bytes). +--- Test for overflow of lquery_level.numvar, with a set of single-char +--- variants in one level. +SELECT (repeat('a|', 65535) || 'a')::lquery; +ERROR: lquery level has too many variants +DETAIL: Number of variants exceeds the maximum allowed (65535). diff --git a/contrib/ltree/sql/ltree.sql b/contrib/ltree/sql/ltree.sql index 2a612e347d..b187b53c52 100644 --- a/contrib/ltree/sql/ltree.sql +++ b/contrib/ltree/sql/ltree.sql @@ -246,6 +246,19 @@ SELECT 'tree.awdfg'::ltree @ 'tree & aWdfg@'::ltxtquery; SELECT 'tree.awdfg_qwerty'::ltree @ 'tree & aw_qw%*'::ltxtquery; SELECT 'tree.awdfg_qwerty'::ltree @ 'tree & aw_rw%*'::ltxtquery; +-- Test for overflow of the int16 "left" field in findoprnd(). +-- This query uses a balanced binary tree to avoid using too much stack. +DO $$ +DECLARE + e text := 'a'; +BEGIN + FOR i IN 1..14 LOOP + e := '(' || e || '&' || e || ')'; + END LOOP; + PERFORM ('b|' || e)::ltxtquery; +END; +$$; + --arrays SELECT '{1.2.3}'::ltree[] @> '1.2.3.4'; @@ -384,3 +397,10 @@ SELECT count(*) FROM _ltreetest WHERE t ~ '23.*{1}.1' ; SELECT count(*) FROM _ltreetest WHERE t ~ '23.*.1' ; SELECT count(*) FROM _ltreetest WHERE t ~ '23.*.2' ; SELECT count(*) FROM _ltreetest WHERE t ? '{23.*.1,23.*.2}' ; + +-- Test for overflow of lquery_level.totallen. +SELECT (repeat('x', 255) || repeat('|' || repeat('x', 255), 256))::lquery; + +--- Test for overflow of lquery_level.numvar, with a set of single-char +--- variants in one level. +SELECT (repeat('a|', 65535) || 'a')::lquery; From 4c35d93e49ef9eab825996bacc3aa230b4fe11f4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 15 May 2026 18:32:33 -0400 Subject: [PATCH 06/76] Doc: fix release-note typo. This mention of memcpy() should of course have said memcmp(). Reported-by: chris@chrullrich.net Author: Tom Lane Discussion: https://postgr.es/m/177883653690.764749.14038057906859461991@wrigleys.postgresql.org Backpatch-through: 14 --- doc/src/sgml/release-14.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/release-14.sgml b/doc/src/sgml/release-14.sgml index b714f75c90..13393f36e2 100644 --- a/doc/src/sgml/release-14.sgml +++ b/doc/src/sgml/release-14.sgml @@ -299,7 +299,7 @@ Branch: REL_14_STABLE [b282280e9] 2026-05-11 05:13:51 -0700 Use timingsafe_bcmp() instead - of memcpy() or strcmp() + of memcmp() or strcmp() when checking passwords, hashes, etc. It is not known whether the data dependency of those functions is usefully exploitable in any of these places, but in the interests of safety, replace them. From 510a05f07cb5ff86fbbedac848ee5c80b8bbfd83 Mon Sep 17 00:00:00 2001 From: Noah Misch Date: Sat, 16 May 2026 18:01:35 -0700 Subject: [PATCH 07/76] Use ereport(ERROR), not Assert(), for publisher tuples missing columns. Three locations use Assert() to guard against a mismatch between the number of columns advertised in the RELATION message and the number actually received in the subsequent INSERT/UPDATE tuple message. Since these values originate from the publisher, the check must survive into production builds. A malicious or buggy publisher can send a RELATION claiming N columns and an INSERT claiming M < N columns. The subscriber's apply worker indexes into colvalues[]/colstatus[] using column indices from the RELATION message's attribute map, causing a heap out-of-bounds read when the tuple's column array is smaller than expected. We've looked, without success, for a scenario in which the publisher holds sufficient control over these out-of-bounds bytes to exploit this or even to reach a SIGSEGV. Despite not finding one, the code has been fragile. Back-patch to v14 (all supported versions). Reported-by: Varik Matevosyan Author: Varik Matevosyan Discussion: https://postgr.es/m/CA+bBoog3cCogktzfLb9bppUByu-10B3CFp8u=iKXG_OvtAguCw@mail.gmail.com Backpatch-through: 14 --- src/backend/replication/logical/worker.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 93a7714922..5bcefd4523 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -553,9 +553,15 @@ slot_store_data(TupleTableSlot *slot, LogicalRepRelMapEntry *rel, if (!att->attisdropped && remoteattnum >= 0) { - StringInfo colvalue = &tupleData->colvalues[remoteattnum]; + StringInfo colvalue; + + if (remoteattnum >= tupleData->ncols) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("logical replication column %d not found in tuple: only %d column(s) received", + remoteattnum + 1, tupleData->ncols))); - Assert(remoteattnum < tupleData->ncols); + colvalue = &tupleData->colvalues[remoteattnum]; errarg.remote_attnum = remoteattnum; @@ -677,7 +683,11 @@ slot_modify_data(TupleTableSlot *slot, TupleTableSlot *srcslot, if (remoteattnum < 0) continue; - Assert(remoteattnum < tupleData->ncols); + if (remoteattnum >= tupleData->ncols) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("logical replication column %d not found in tuple: only %d column(s) received", + remoteattnum + 1, tupleData->ncols))); if (tupleData->colstatus[remoteattnum] != LOGICALREP_COLUMN_UNCHANGED) { @@ -1421,7 +1431,12 @@ apply_handle_update(StringInfo s) if (!att->attisdropped && remoteattnum >= 0) { - Assert(remoteattnum < newtup.ncols); + if (remoteattnum >= newtup.ncols) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("logical replication column %d not found in tuple: only %d column(s) received", + remoteattnum + 1, newtup.ncols))); + if (newtup.colstatus[remoteattnum] != LOGICALREP_COLUMN_UNCHANGED) target_rte->updatedCols = bms_add_member(target_rte->updatedCols, From 5552a15a3ed2de5b7f0afab9d55f107ebd824d35 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 20 May 2026 15:54:13 +0900 Subject: [PATCH 08/76] pg_recvlogical: Honor source cluster file permissions for output files Commit c37b3d08ca6 attempted to preserve group permissions on pg_recvlogical output files when group access was enabled on the source cluster. However, the output files were still created with a fixed S_IRUSR | S_IWUSR mode, preventing group-read permissions from being applied. This commit fixes the issue by creating output files with pg_file_create_mode instead of a hard-coded mode. This allows pg_recvlogical to correctly preserve group permissions from the source cluster. Backpatch to all supported branches. Author: Fujii Masao Reviewed-by: Srinath Reddy Sadipiralla Discussion: https://postgr.es/m/CAHGQGwHhpizYzMo3nFP4GkNMueSNMY3QfC-gBN1VTXtuiANDvw@mail.gmail.com Backpatch-through: 14 --- doc/src/sgml/ref/pg_recvlogical.sgml | 2 +- src/bin/pg_basebackup/pg_recvlogical.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/ref/pg_recvlogical.sgml b/doc/src/sgml/ref/pg_recvlogical.sgml index e4b3955edc..c395505f20 100644 --- a/doc/src/sgml/ref/pg_recvlogical.sgml +++ b/doc/src/sgml/ref/pg_recvlogical.sgml @@ -423,7 +423,7 @@ PostgreSQL documentation pg_recvlogical will preserve group permissions on - the received WAL files if group permissions are enabled on the source + the output files if group permissions are enabled on the source cluster. diff --git a/src/bin/pg_basebackup/pg_recvlogical.c b/src/bin/pg_basebackup/pg_recvlogical.c index 55139ee31a..4eec0d8eee 100644 --- a/src/bin/pg_basebackup/pg_recvlogical.c +++ b/src/bin/pg_basebackup/pg_recvlogical.c @@ -328,7 +328,7 @@ StreamLogicalLog(void) outfd = fileno(stdout); else outfd = open(outfile, O_CREAT | O_APPEND | O_WRONLY | PG_BINARY, - S_IRUSR | S_IWUSR); + pg_file_create_mode); if (outfd == -1) { pg_log_error("could not open log file \"%s\": %m", outfile); From e18b77153c740122a0eadde39ebfd5899156143e Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sat, 23 May 2026 08:10:18 +0900 Subject: [PATCH 09/76] Avoid exposing WAL receiver raw conninfo during timeline jumps When reusing an existing WAL receiver after it has reached WALRCV_WAITING for new instructions, RequestXLogStreaming() copied PrimaryConnInfo into WalRcv->conninfo before switching the state to WALRCV_RESTARTING. At that point ready_to_display could still be true, so pg_stat_wal_receiver could expose the raw connection string, including sensitive fields, but it should only show the user-displayable version of the connection string. WALRCV_RESTARTING does not establish a new connection. The waiting WAL receiver reuses its existing connection and only needs a new startpoint and timeline, so there is no need to copy the raw connection string into shared memory again. Let's only copy conninfo when launching a new WAL receiver after WALRCV_STOPPED, not while waiting for instructions. This commit adds coverage for the case fixed by this commit to the timeline-switch test by verifying that the WAL receiver conninfo remains consistent across the jump. Backpatch all the way down, as this issue is possible since pg_stat_wal_receiver has been introduced. Author: Chao Li Reviewed-by: Michael Paquier Discussion: https://postgr.es/m/EF91FF76-1E2B-4F3B-9162-290B4DC517FF@gmail.com Backpatch-through: 14 --- src/backend/replication/walreceiverfuncs.c | 14 +++++++++----- src/test/recovery/t/004_timeline_switch.pl | 17 ++++++++++++++--- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/backend/replication/walreceiverfuncs.c b/src/backend/replication/walreceiverfuncs.c index 6f0acbfdef..6ef2d7c0d4 100644 --- a/src/backend/replication/walreceiverfuncs.c +++ b/src/backend/replication/walreceiverfuncs.c @@ -265,11 +265,6 @@ RequestXLogStreaming(TimeLineID tli, XLogRecPtr recptr, const char *conninfo, Assert(walrcv->walRcvState == WALRCV_STOPPED || walrcv->walRcvState == WALRCV_WAITING); - if (conninfo != NULL) - strlcpy((char *) walrcv->conninfo, conninfo, MAXCONNINFO); - else - walrcv->conninfo[0] = '\0'; - /* * Use configured replication slot if present, and ignore the value of * create_temp_slot as the slot name should be persistent. Otherwise, use @@ -287,10 +282,19 @@ RequestXLogStreaming(TimeLineID tli, XLogRecPtr recptr, const char *conninfo, walrcv->is_temp_slot = create_temp_slot; } + /* + * While waiting for instructions, the WAL receiver uses the same + * connection, so do not clobber the user-visible conninfo already saved. + */ if (walrcv->walRcvState == WALRCV_STOPPED) { launch = true; walrcv->walRcvState = WALRCV_STARTING; + + if (conninfo != NULL) + strlcpy((char *) walrcv->conninfo, conninfo, MAXCONNINFO); + else + walrcv->conninfo[0] = '\0'; } else walrcv->walRcvState = WALRCV_RESTARTING; diff --git a/src/test/recovery/t/004_timeline_switch.pl b/src/test/recovery/t/004_timeline_switch.pl index edfb2bef53..bbff6e97cd 100644 --- a/src/test/recovery/t/004_timeline_switch.pl +++ b/src/test/recovery/t/004_timeline_switch.pl @@ -7,7 +7,7 @@ use File::Path qw(rmtree); use PostgresNode; use TestLib; -use Test::More tests => 5; +use Test::More tests => 6; $ENV{PGDATABASE} = 'postgres'; @@ -50,11 +50,15 @@ stdout => \$psql_out); is($psql_out, 't', "promotion of standby with pg_promote"); -# Switch standby 2 to replay from standby 1 +# Switch standby 2 to replay from standby 1. During the timeline switch, +# the WAL receiver process on standby 2 should not be stopped, and the +# new primary connection string should not be visible +# in pg_stat_wal_receiver. +my $secret = 'dont_show_me'; my $connstr_1 = $node_standby_1->connstr; $node_standby_2->append_conf( 'postgresql.conf', qq( -primary_conninfo='$connstr_1' +primary_conninfo='$connstr_1 password=$secret' )); # Rotate logfile before restarting, for the log checks done below. @@ -97,6 +101,13 @@ is($wr_pid_before_switch, $wr_pid_after_switch, 'WAL receiver PID matches across timeline jumps'); +my $raw_conninfo_count = $node_standby_2->safe_psql('postgres', + "SELECT count(*) FROM pg_stat_wal_receiver WHERE conninfo LIKE '%$secret%'" +); + +is($raw_conninfo_count, '0', + 'pg_stat_wal_receiver.conninfo not updated across timeline jumps'); + # Ensure that a standby is able to follow a primary on a newer timeline # when WAL archiving is enabled. From 75710266732e049adc7fac6d885a3cba1474825b Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 25 May 2026 14:39:07 +0900 Subject: [PATCH 10/76] Fix size check in statext_dependencies_deserialize() The check for the minimum expected bytea size of a MVDependencies object was using SizeOfItem() for its calculation. This macro uses the number of attributes in a single dependency. This minimum size calculation should be based on MinSizeOfItems(), that computes the minimum expected size as the header plus the minimally-sized number of dependency items. Oversight in d08c44f7a4ec. Author: Ilia Evdokimov Discussion: https://postgr.es/m/4b8d299d-2505-4c30-bf80-0f697410db35@tantorlabs.com Backpatch-through: 14 --- src/backend/statistics/dependencies.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c index dee234b06f..85e56d1028 100644 --- a/src/backend/statistics/dependencies.c +++ b/src/backend/statistics/dependencies.c @@ -539,7 +539,7 @@ statext_dependencies_deserialize(bytea *data) elog(ERROR, "invalid zero-length item array in MVDependencies"); /* what minimum bytea size do we expect for those parameters */ - min_expected_size = SizeOfItem(dependencies->ndeps); + min_expected_size = MinSizeOfItems(dependencies->ndeps); if (VARSIZE_ANY_EXHDR(data) < min_expected_size) elog(ERROR, "invalid dependencies size %zd (expected at least %zd)", From a96b051a98e346fe03f5fb27f21b539842295745 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 25 May 2026 18:15:49 -0400 Subject: [PATCH 11/76] Fix missed ReleaseVariableStats() in intarray's _int_matchsel(). Given a WHERE clause like "int[] @@ query_int" or "query_int ~~ int[]" where the query_int side is a table column having statistics, _int_matchsel() exited without remembering to free the statistics tuple. This would typically lead to warnings about cache refcount leakage, like WARNING: resource was not closed: cache pg_statistic (73), tuple 42/12 has count 1 It's been wrong since this code was added, in commit c6fbe6d6f. Bug: #19492 Reported-by: Man Zeng Author: Man Zeng Reviewed-by: Tom Lane Discussion: https://postgr.es/m/19492-ddcd0e22399ef85a@postgresql.org Backpatch-through: 14 --- contrib/intarray/_int_selfuncs.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/intarray/_int_selfuncs.c b/contrib/intarray/_int_selfuncs.c index 37b277e7f1..6c61b772fd 100644 --- a/contrib/intarray/_int_selfuncs.c +++ b/contrib/intarray/_int_selfuncs.c @@ -152,7 +152,10 @@ _int_matchsel(PG_FUNCTION_ARGS) * query_int. */ if (vardata.vartype != INT4ARRAYOID) + { + ReleaseVariableStats(vardata); PG_RETURN_FLOAT8(DEFAULT_EQ_SEL); + } /* * Can't do anything useful if the something is not a constant, either. From 8007d118524a5d43568217341fab58b590930322 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 27 May 2026 14:52:31 +0900 Subject: [PATCH 12/76] Fix race conditions in ProcKill()'s lock-group freelist handling This commit fixes two bugs in ProcKill()'s lock-group teardown freelist publication: * a double push of the leader's PGPROC that corrupts the freelist. * a leak of the last follower's PGPROC slot. ProcKill()'s lock-group teardown had two PGPROC freelist updates scattered through the function, done under two separate freeProcsLock acquisitions: * A follower's push of the leader's PGPROC, done when a follower is the last group member exiting. * Every backend's self-push at the bottom of the function. The two freelist updates were coordinated only by inspecting proc->lockGroupLeader, which a follower could clear as a side effect of pushing the leader. This coordination was broken. For example, with two concurrent backends: * The follower clears leader->lockGroupLeader and pushes the leader's PGPROC under leader_lwlock. * The follower does not clear its own proc->lockGroupLeader, being skipped. * When the leader reaches the bottom of ProcKill(), it sees a NULL proc->lockGroupLeader (the follower cleared it) and pushes itself, causing a second dlist_push_tail() of the same node onto the same freelist. * The follower at the bottom sees its own proc->lockGroupLeader being not NULL (never cleared) and skips its own push, causing its own slot to leak. This commit refactors the freelist manipulation to be done in two distinct phases, each step using its own lock acquisition to ensure that each freelist operation happens in an isolated manner for each backend (follower or leader): - First, under a single leader_lwlock acquisition, check the state of the lock-group. Depending on if we are dealing with a follower and/or a leader, and if the leader has exited before a follower, then set some state booleans that define which actions should be taken with the freelist. - Second, under a single freeProcsLock acquisition, perform the cleanup actions, self-push of a backend and/or push of the leader back to the freelist. This is an old issue, dating back to 9.6 where parallel workers and lock grouping has been added. Author: Vlad Lesin Reviewed-by: Andrey Borodin Reviewed-by: Michael Paquier Discussion: https://postgr.es/m/d2983796-2603-41b7-a66e-fc8489ddb954@gmail.com Backpatch-through: 14 --- src/backend/storage/lmgr/proc.c | 79 ++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 26 deletions(-) diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index e36b187947..62b9f635d5 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -822,7 +822,10 @@ static void ProcKill(int code, Datum arg) { PGPROC *proc; + PGPROC *leader; PGPROC *volatile *procgloballist; + bool push_leader; + bool push_self; Assert(MyProc != NULL); @@ -860,36 +863,59 @@ ProcKill(int code, Datum arg) /* Also cleanup all the temporary slots. */ ReplicationSlotCleanup(); + proc = MyProc; + procgloballist = proc->procgloballist; + /* - * Detach from any lock group of which we are a member. If the leader - * exist before all other group members, its PGPROC will remain allocated - * until the last group process exits; that process must return the - * leader's PGPROC to the appropriate list. + * Detach from any lock group of which we are a member, deciding under + * leader_lwlock whether we (via push_self) and/or the leader (via + * push_leader) need to be pushed onto a freelist. The actual pushes + * happen after evaluating if any of these are required, under a single + * ProcGlobal->freeProcsLock. + * + * The decision whether any of the freelists needs to be updated is taken + * under a single leader_lwlock. */ - if (MyProc->lockGroupLeader != NULL) + push_leader = false; + push_self = true; + leader = NULL; + + if (proc->lockGroupLeader != NULL) { - PGPROC *leader = MyProc->lockGroupLeader; - LWLock *leader_lwlock = LockHashPartitionLockByProc(leader); + LWLock *leader_lwlock; + + leader = proc->lockGroupLeader; + leader_lwlock = LockHashPartitionLockByProc(leader); LWLockAcquire(leader_lwlock, LW_EXCLUSIVE); Assert(!dlist_is_empty(&leader->lockGroupMembers)); - dlist_delete(&MyProc->lockGroupLink); + dlist_delete(&proc->lockGroupLink); if (dlist_is_empty(&leader->lockGroupMembers)) { leader->lockGroupLeader = NULL; - if (leader != MyProc) + if (leader != proc) { - procgloballist = leader->procgloballist; - - /* Leader exited first; return its PGPROC. */ - SpinLockAcquire(ProcStructLock); - leader->links.next = (SHM_QUEUE *) *procgloballist; - *procgloballist = leader; - SpinLockRelease(ProcStructLock); + /* + * We are the last follower and the leader exited earlier; its + * PGPROC is still allocated and must be pushed here. + */ + push_leader = true; + proc->lockGroupLeader = NULL; } } - else if (leader != MyProc) - MyProc->lockGroupLeader = NULL; + else if (leader != proc) + { + /* Non-last follower; leader still present in the group. */ + proc->lockGroupLeader = NULL; + } + else + { + /* + * We are the leader and followers remain. Skip our own push; the + * last follower to exit will push us back to the freelist. + */ + push_self = false; + } LWLockRelease(leader_lwlock); } @@ -905,20 +931,21 @@ ProcKill(int code, Datum arg) SwitchBackToLocalLatch(); pgstat_reset_wait_event_storage(); - proc = MyProc; MyProc = NULL; DisownLatch(&proc->procLatch); - procgloballist = proc->procgloballist; SpinLockAcquire(ProcStructLock); + if (push_leader) + { + /* Return leader PGPROC (and semaphore) to appropriate freelist */ + PGPROC *volatile *leadergloballist = leader->procgloballist; - /* - * If we're still a member of a locking group, that means we're a leader - * which has somehow exited before its children. The last remaining child - * will release our PGPROC. Otherwise, release it now. - */ - if (proc->lockGroupLeader == NULL) + leader->links.next = (SHM_QUEUE *) *leadergloballist; + *leadergloballist = leader; + } + if (push_self) { + Assert(proc->lockGroupLeader == NULL); /* Since lockGroupLeader is NULL, lockGroupMembers should be empty. */ Assert(dlist_is_empty(&proc->lockGroupMembers)); From db4d12fc97ab2f4542092782fbcc934d584d0850 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 27 May 2026 17:20:00 +0900 Subject: [PATCH 13/76] Fix procLatch ownership race in ProcKill() DisownLatch() was executed after the PGPROC entry of the process terminated is pushed back into a freelist. A newly-forked backend that recycles the slot could call OwnLatch() and PANIC with a "latch already owned by PID", taking down the server. There were two scenarios related to lock groups where this issue could be reached: * A follower pushes the leader's PGPROC back to the freelist while the leader has not yet called DisownLatch() in its own ProcKill(). * A leader outliving all its followers pushes its own PGPROC onto the freelist before reaching DisownLatch(), which would be the most common scenario. This issue is fixed by calling SwitchBackToLocalLatch() and DisownLatch() at an earlier phase of ProcKill(), before any freelist manipulation happens, so that the slot of the backend terminated is never exposed as owning a latch. Note that pgstat_reset_wait_event_storage() is kept at a later stage. An upcoming commit will take advantage of that by introducing a test able to check the original PANIC scenario. Author: Vlad Lesin Reviewed-by: Andrey Borodin Reviewed-by: Michael Paquier Discussion: https://postgr.es/m/d2983796-2603-41b7-a66e-fc8489ddb954@gmail.com Backpatch-through: 14 --- src/backend/storage/lmgr/proc.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 62b9f635d5..450fb2a71d 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -863,6 +863,24 @@ ProcKill(int code, Datum arg) /* Also cleanup all the temporary slots. */ ReplicationSlotCleanup(); + /* + * Reset MyLatch to the process local one and disown the shared latch, so + * that signal handlers et al can continue using the latch after the + * shared latch isn't ours anymore. + * + * DisownLatch() must happen before our PGPROC can appear on a freelist: a + * newly-forked backend that pops our slot and calls OwnLatch() would + * PANIC on a still-owned latch. + * + * pgstat_reset_wait_event_storage() is intentionally deferred until after + * the lock-group block so that wait_event_info remains visible in our + * PGPROC slot while we may be observed there. It is safe to defer + * because our slot is not yet on any freelist at this point, and useful + * for testing purposes. + */ + SwitchBackToLocalLatch(); + DisownLatch(&MyProc->procLatch); + proc = MyProc; procgloballist = proc->procgloballist; @@ -919,20 +937,10 @@ ProcKill(int code, Datum arg) LWLockRelease(leader_lwlock); } - /* - * Reset MyLatch to the process local one. This is so that signal - * handlers et al can continue using the latch after the shared latch - * isn't ours anymore. - * - * Similarly, stop reporting wait events to MyProc->wait_event_info. - * - * After that clear MyProc and disown the shared latch. - */ - SwitchBackToLocalLatch(); + /* See comment above, close to DisownLatch() */ pgstat_reset_wait_event_storage(); MyProc = NULL; - DisownLatch(&proc->procLatch); SpinLockAcquire(ProcStructLock); if (push_leader) From 2bb60eb4feab76ac5ea2ea6f15111b569ea48b62 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 27 May 2026 11:49:50 +0300 Subject: [PATCH 14/76] Fix self-deadlock when replaying WAL generated by older minor version Commit 77dff5d937 introduced a SimpleLruWriteAll() call when replaying multixact WAL records generated by older minor versions. However, SimpleLruWriteAll() acquires the SLRU lock and on v16 and below, it's called while already holding the lock, leading to self-deadlock. Version 17 and 18 did not have that problem, because in those versions the lock is acquired later in the function. To fix, acquire MultiXactOffsetSLRULock later in RecordNewMultiXact(), at the same place where it's acquired on version 17 and 18. Author: Andrey Borodin Reported-by: Radim Marek Discussion: https://www.postgresql.org/message-id/19490-9c59c6a583513b99@postgresql.org Backpatch-through: 14-16 --- src/backend/access/transam/multixact.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index de79d1d607..0b26ea7be3 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -887,8 +887,6 @@ RecordNewMultiXact(MultiXactId multi, MultiXactOffset offset, MultiXactOffset *next_offptr; MultiXactOffset next_offset; - LWLockAcquire(MultiXactOffsetSLRULock, LW_EXCLUSIVE); - /* position of this multixid in the offsets SLRU area */ pageno = MultiXactIdToOffsetPage(multi); entryno = MultiXactIdToOffsetEntry(multi); @@ -950,6 +948,8 @@ RecordNewMultiXact(MultiXactId multi, MultiXactOffset offset, { elog(DEBUG1, "next offsets page is not initialized, initializing it now"); + LWLockAcquire(MultiXactOffsetSLRULock, LW_EXCLUSIVE); + /* Create and zero the page */ slotno = SimpleLruZeroPage(MultiXactOffsetCtl, next_pageno); @@ -957,6 +957,8 @@ RecordNewMultiXact(MultiXactId multi, MultiXactOffset offset, SimpleLruWritePage(MultiXactOffsetCtl, slotno); Assert(!MultiXactOffsetCtl->shared->page_dirty[slotno]); + LWLockRelease(MultiXactOffsetSLRULock); + /* * Remember that we initialized the page, so that we don't zero it * again at the XLOG_MULTIXACT_ZERO_OFF_PAGE record. @@ -975,6 +977,7 @@ RecordNewMultiXact(MultiXactId multi, MultiXactOffset offset, * concurrently, we might race ahead and get called before the previous * multixid. */ + LWLockAcquire(MultiXactOffsetSLRULock, LW_EXCLUSIVE); /* * Note: we pass the MultiXactId to SimpleLruReadPage as the "transaction" From 36b6ed2606e18066ca1ae95d877aef8e98fad31d Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 27 May 2026 18:35:55 +0300 Subject: [PATCH 15/76] Don't try to record dependency on a dropped column's datatype When creating a relation with a dropped column, we called recordDependencyOn() also on the datatype of the dropped column, which is always InvalidOid. In versions 15 and above, that was harmless because recordDependencyOn() considers InvalidOid as a pinned object, and skips over it. On version 14, isPinnedObject() does not consider InvalidOid as pinned, so we created a bogus pg_depend entry with refobjectid == 0. As far as I can tell, the only case when AddNewAttributeTuples() is called with dropped columns is when performing a table-rewriting ALTER TABLE command. That temporarily creates a new relation with the same columns, including dropped ones, then swaps the relations, and drops the newly created table again. So even on version 14, the bogus pg_depend entry was only on the transient relation that was dropped at the end of the ALTER TABLE command, which was harmless. Even though this is harmless, let's be tidy, similar to commit 713bce9484. The reason I noticed this now and why I backported this, is because the next commit will add code to acquire locks on the referenced objects, and we don't want to acquire a lock on InvalidOid. Discussion: https://postgr.es/m/ZiYjn0eVc7pxVY45@ip-10-97-1-34.eu-west-3.compute.internal Backpatch-through: 14 --- src/backend/catalog/heap.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 5404eee001..a847c76244 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -885,6 +885,9 @@ AddNewAttributeTuples(Oid new_rel_oid, /* add dependencies on their datatypes and collations */ for (int i = 0; i < natts; i++) { + if (tupdesc->attrs[i].attisdropped) + continue; + /* Add dependency info */ ObjectAddressSubSet(myself, RelationRelationId, new_rel_oid, i + 1); ObjectAddressSet(referenced, TypeRelationId, From 5100bdbd3ba20caf00c8074b61326f80d9255a65 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 27 May 2026 18:35:58 +0300 Subject: [PATCH 16/76] Avoid orphaned objects dependencies Concurrent DDL can leave behind objects referencing other objects that no longer exist. This can happen if an object is dropped, while a new object that depends on it is created concurrently. For example: session 1: BEGIN; CREATE FUNCTION myschema.myfunc() ...; session 2: DROP SCHEMA myschema; session 1: COMMIT; DROP SCHEMA does check that there are no objects dependending on the schema being dropped, but it does not see objects being concurrently created by other sessions. Even if it did, this scenario would still fail: session 1: BEGIN: DROP SCHEMA myschema; session 2: CREATE FUNCTION myschema.myfunc() ...; session 1: COMMIT; When the DROP SCHEMA runs, the schema was empty, but the new function is created in it before the dropping transaction completes. The CREATE FUNCTION does not see that the schema is concurrently being dropped. In both of these scenarios, the function is left behind in the schema that no longer exists. To fix, acquire AccessShareLock on all referenced objects when recording dependencies. This conflicts with the AccessExclusiveLock taken by DROP, preventing the race. After acquiring the lock, verify that the object still exists, and if it was dropped concurrently, report an error. We already had such a mechanism for shared dependencies, but for some reason we didn't do it for in-database dependendies. Ideally the locks would be acquired much earlier when creating a new object, but that will require modifying a lot of callers. This check while recording the dependency is a nice wholesale protection, and even if we change all the CREATE commands to acquire locks earlier, it's still good to have this as a backstop to catch any cases where we forgot to do so. The patch adds a few tests for some cases that left behind orphaned objects before this. It also adds a test for roles, which already had such protection, although that test is partially disabled because the error message includes an OID which is not predictable. Author: Bertrand Drouvot Reviewed-by: Heikki Linnakangas Discussion: https://postgr.es/m/ZiYjn0eVc7pxVY45@ip-10-97-1-34.eu-west-3.compute.internal Backpatch-through: 14 --- src/backend/catalog/pg_depend.c | 136 +++++++++++++++++ .../expected/ddl-dependency-locking.out | 137 ++++++++++++++++++ src/test/isolation/isolation_schedule | 1 + .../specs/ddl-dependency-locking.spec | 104 +++++++++++++ src/test/regress/expected/alter_table.out | 11 +- 5 files changed, 384 insertions(+), 5 deletions(-) create mode 100644 src/test/isolation/expected/ddl-dependency-locking.out create mode 100644 src/test/isolation/specs/ddl-dependency-locking.spec diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c index 55a2da35e4..d8b6feed70 100644 --- a/src/backend/catalog/pg_depend.c +++ b/src/backend/catalog/pg_depend.c @@ -17,6 +17,7 @@ #include "access/genam.h" #include "access/htup_details.h" #include "access/table.h" +#include "catalog/catalog.h" #include "catalog/dependency.h" #include "catalog/indexing.h" #include "catalog/pg_constraint.h" @@ -25,13 +26,17 @@ #include "catalog/pg_type.h" #include "commands/extension.h" #include "miscadmin.h" +#include "storage/lmgr.h" +#include "storage/lock.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/rel.h" +#include "utils/snapmgr.h" #include "utils/syscache.h" static bool isObjectPinned(const ObjectAddress *object, Relation rel); +static void dependencyLockAndCheckObject(Oid classId, Oid objectId); /* @@ -104,6 +109,13 @@ recordMultipleDependencies(const ObjectAddress *depender, if (isObjectPinned(referenced, dependDesc)) continue; + /* + * Make sure the new referenced object doesn't go away while we record + * the dependency. DROP routines should lock the object exclusively + * before they check dependencies. + */ + dependencyLockAndCheckObject(referenced->classId, referenced->objectId); + if (slot_init_count < max_slots) { slot[slot_stored_count] = MakeSingleTupleTableSlot(RelationGetDescr(dependDesc), @@ -506,6 +518,13 @@ changeDependencyFor(Oid classId, Oid objectId, return 1; } + /* + * Make sure the new referenced object doesn't go away while we record the + * dependency. + */ + if (!newIsPinned) + dependencyLockAndCheckObject(refClassId, newRefObjectId); + /* There should be existing dependency record(s), so search. */ ScanKeyInit(&key[0], Anum_pg_depend_classid, @@ -747,6 +766,123 @@ isObjectPinned(const ObjectAddress *object, Relation rel) } +/* + * dependencyLockAndCheckObject + * + * Lock the object that we are about to record a dependency on. After it's + * locked, verify that it hasn't been dropped while we weren't looking. If it + * has been dropped, throw an an error. + * + * If the caller already holds a lock that conflicts with DROP + * (AccessShareLock or stronger), this does nothing. Callers should acquire + * locks already when they look up the dependent objects, but many callers + * currently do not. This is a backstop to make sure that we don't record a + * bogus reference permanently in the catalogs in that case. In the future, + * after we have tightened up all the callers to acquire locks earlier, this + * could just verify that the object is already locked and throw an error if + * not. + */ +static void +dependencyLockAndCheckObject(Oid classId, Oid objectId) +{ + /* + * Note: Pinned objects cannot be dropped concurrently, and callers + * checked this already. objectId really should be valid too, but when + * this locking was added there were some corner cases where we created a + * transient dependency on InvalidOid. Probably shouldn't happen anymore, + * but let's tolerate it. + */ + if (!OidIsValid(objectId)) + return; + + if (classId != RelationRelationId) + { + LOCKTAG tag; + int cache; + Relation rel; + SysScanDesc scan; + ScanKeyData skey; + HeapTuple tuple; + + SET_LOCKTAG_OBJECT(tag, + MyDatabaseId, + classId, + objectId, + 0); + + if (LockOrStrongerHeldByMe(&tag, AccessShareLock)) + return; + + /* Assume we should lock the whole object not a sub-object */ + LockDatabaseObject(classId, objectId, 0, AccessShareLock); + + /* + * Check that the object still exists. If the catalog has a suitable + * syscache, check that first. + */ + cache = get_object_catcache_oid(classId); + if (cache != -1) + { + if (SearchSysCacheExists1(cache, ObjectIdGetDatum(objectId))) + return; + } + + /* + * If it's not found in the syscache, or there's no suitable syscache + * we can use, scan the catalog table using SnapshotSelf. This + * handles the case that it's an object we just created (for example, + * if it's a composite type created as part of creating a table). + */ + rel = table_open(classId, AccessShareLock); + + ScanKeyInit(&skey, + get_object_attnum_oid(classId), + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(objectId)); + + scan = systable_beginscan(rel, get_object_oid_index(classId), + true, SnapshotSelf, 1, &skey); + + tuple = systable_getnext(scan); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("dependent %s was concurrently dropped", + get_object_class_descr(classId)))); + + systable_endscan(scan); + table_close(rel, AccessShareLock); + } + else if (IsSharedRelation(objectId)) + { + /* Shared relations are considered pinned, ignore them */ + } + else + { + /* + * Same logic for pg_class entries, but locking relations is handled + * by different functions. + * + * Callers are more careful with locking relations than other objects, + * so we should already have a lock on the relation, or on another + * object that indirectly prevents the relation from being dropped. + * For example, we might have a strong lock on a table while adding + * dependency to its index. However, we cannot detect the indirectly + * protected case here easily. To err on the safe side, acquire a + * lock directly on the relation if we're not holding one already. + */ + if (CheckRelationOidLockedByMe(objectId, AccessShareLock, true)) + return; + LockRelationOid(objectId, AccessShareLock); + + if (SearchSysCacheExists1(RELOID, ObjectIdGetDatum(objectId))) + return; + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("dependent relation was concurrently dropped"))); + } +} + /* * Various special-purpose lookups and manipulations of pg_depend. */ diff --git a/src/test/isolation/expected/ddl-dependency-locking.out b/src/test/isolation/expected/ddl-dependency-locking.out new file mode 100644 index 0000000000..3bf9e43572 --- /dev/null +++ b/src/test/isolation/expected/ddl-dependency-locking.out @@ -0,0 +1,137 @@ +Parsed test spec with 2 sessions + +starting permutation: s1_begin s1_create_function_in_schema s2_drop_schema s1_commit +step s1_begin: BEGIN; +step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; +step s2_drop_schema: DROP SCHEMA testschema; +step s1_commit: COMMIT; +step s2_drop_schema: <... completed> +ERROR: cannot drop schema testschema because other objects depend on it + +starting permutation: s2_begin s2_drop_schema s1_create_function_in_schema s2_commit +step s2_begin: BEGIN; +step s2_drop_schema: DROP SCHEMA testschema; +step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; +step s2_commit: COMMIT; +step s1_create_function_in_schema: <... completed> +ERROR: dependent schema was concurrently dropped + +starting permutation: s1_begin s1_alter_function_schema s2_drop_alterschema s1_commit +step s1_begin: BEGIN; +step s1_alter_function_schema: ALTER FUNCTION public.falter() SET SCHEMA alterschema; +step s2_drop_alterschema: DROP SCHEMA alterschema; +step s1_commit: COMMIT; +step s2_drop_alterschema: <... completed> +ERROR: cannot drop schema alterschema because other objects depend on it + +starting permutation: s2_begin s2_drop_alterschema s1_alter_function_schema s2_commit +step s2_begin: BEGIN; +step s2_drop_alterschema: DROP SCHEMA alterschema; +step s1_alter_function_schema: ALTER FUNCTION public.falter() SET SCHEMA alterschema; +step s2_commit: COMMIT; +step s1_alter_function_schema: <... completed> +ERROR: dependent schema was concurrently dropped + +starting permutation: s1_begin s1_create_function_with_argtype s2_drop_foo_type s1_commit +step s1_begin: BEGIN; +step s1_create_function_with_argtype: CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; +step s2_drop_foo_type: DROP TYPE public.foo; +step s1_commit: COMMIT; +step s2_drop_foo_type: <... completed> +ERROR: cannot drop type foo because other objects depend on it + +starting permutation: s2_begin s2_drop_foo_type s1_create_function_with_argtype s2_commit +step s2_begin: BEGIN; +step s2_drop_foo_type: DROP TYPE public.foo; +step s1_create_function_with_argtype: CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; +step s2_commit: COMMIT; +step s1_create_function_with_argtype: <... completed> +ERROR: dependent type was concurrently dropped + +starting permutation: s1_begin s1_create_function_with_rettype s2_drop_foo_rettype s1_commit +step s1_begin: BEGIN; +step s1_create_function_with_rettype: CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; +step s2_drop_foo_rettype: DROP DOMAIN id; +step s1_commit: COMMIT; +step s2_drop_foo_rettype: <... completed> +ERROR: cannot drop type id because other objects depend on it + +starting permutation: s2_begin s2_drop_foo_rettype s1_create_function_with_rettype s2_commit +step s2_begin: BEGIN; +step s2_drop_foo_rettype: DROP DOMAIN id; +step s1_create_function_with_rettype: CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; +step s2_commit: COMMIT; +step s1_create_function_with_rettype: <... completed> +ERROR: dependent type was concurrently dropped + +starting permutation: s1_begin s1_create_function_with_function s2_drop_function_f s1_commit +step s1_begin: BEGIN; +step s1_create_function_with_function: CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; +step s2_drop_function_f: DROP FUNCTION f(); +step s1_commit: COMMIT; +step s2_drop_function_f: <... completed> +ERROR: cannot drop function f() because other objects depend on it + +starting permutation: s2_begin s2_drop_function_f s1_create_function_with_function s2_commit +step s2_begin: BEGIN; +step s2_drop_function_f: DROP FUNCTION f(); +step s1_create_function_with_function: CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; +step s2_commit: COMMIT; +step s1_create_function_with_function: <... completed> +ERROR: dependent function was concurrently dropped + +starting permutation: s1_begin s1_create_domain_with_domain s2_drop_domain_id s1_commit +step s1_begin: BEGIN; +step s1_create_domain_with_domain: CREATE DOMAIN idid as id; +step s2_drop_domain_id: DROP DOMAIN id; +step s1_commit: COMMIT; +step s2_drop_domain_id: <... completed> +ERROR: cannot drop type id because other objects depend on it + +starting permutation: s2_begin s2_drop_domain_id s1_create_domain_with_domain s2_commit +step s2_begin: BEGIN; +step s2_drop_domain_id: DROP DOMAIN id; +step s1_create_domain_with_domain: CREATE DOMAIN idid as id; +step s2_commit: COMMIT; +step s1_create_domain_with_domain: <... completed> +ERROR: dependent type was concurrently dropped + +starting permutation: s1_begin s1_create_table_with_type s2_drop_footab_type s1_commit +step s1_begin: BEGIN; +step s1_create_table_with_type: CREATE TABLE tabtype(a footab); +step s2_drop_footab_type: DROP TYPE public.footab; +step s1_commit: COMMIT; +step s2_drop_footab_type: <... completed> +ERROR: cannot drop type footab because other objects depend on it + +starting permutation: s2_begin s2_drop_footab_type s1_create_table_with_type s2_commit +step s2_begin: BEGIN; +step s2_drop_footab_type: DROP TYPE public.footab; +step s1_create_table_with_type: CREATE TABLE tabtype(a footab); +step s2_commit: COMMIT; +step s1_create_table_with_type: <... completed> +ERROR: dependent type was concurrently dropped + +starting permutation: s1_begin s1_create_server_with_fdw_wrapper s2_drop_fdw_wrapper s1_commit +step s1_begin: BEGIN; +step s1_create_server_with_fdw_wrapper: CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; +step s2_drop_fdw_wrapper: DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; +step s1_commit: COMMIT; +step s2_drop_fdw_wrapper: <... completed> +ERROR: cannot drop foreign-data wrapper fdw_wrapper because other objects depend on it + +starting permutation: s2_begin s2_drop_fdw_wrapper s1_create_server_with_fdw_wrapper s2_commit +step s2_begin: BEGIN; +step s2_drop_fdw_wrapper: DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; +step s1_create_server_with_fdw_wrapper: CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; +step s2_commit: COMMIT; +step s1_create_server_with_fdw_wrapper: <... completed> +ERROR: dependent foreign-data wrapper was concurrently dropped + +starting permutation: s1_begin s1_alter_function_owner s2_drop_role s1_commit +step s1_begin: BEGIN; +step s1_alter_function_owner: ALTER FUNCTION public.falter() OWNER TO regress_dependency; +step s2_drop_role: DROP ROLE regress_dependency; +step s1_commit: COMMIT; +step s2_drop_role: <... completed> +ERROR: role "regress_dependency" cannot be dropped because some objects depend on it diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 3abc1fa3d6..6ad0e33f3f 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -104,3 +104,4 @@ test: truncate-conflict test: serializable-parallel test: serializable-parallel-2 test: serializable-parallel-3 +test: ddl-dependency-locking diff --git a/src/test/isolation/specs/ddl-dependency-locking.spec b/src/test/isolation/specs/ddl-dependency-locking.spec new file mode 100644 index 0000000000..de5bd88d35 --- /dev/null +++ b/src/test/isolation/specs/ddl-dependency-locking.spec @@ -0,0 +1,104 @@ +# Test that concurrent DROP and CREATE commands do not leave behind +# references to non-existent objects. + +setup +{ + CREATE SCHEMA testschema; + CREATE SCHEMA alterschema; + CREATE TYPE public.foo as enum ('one', 'two'); + CREATE TYPE public.footab as enum ('three', 'four'); + CREATE DOMAIN id AS int; + CREATE FUNCTION f() RETURNS int LANGUAGE SQL RETURN 1; + CREATE FUNCTION public.falter() RETURNS int LANGUAGE SQL RETURN 1; + CREATE FOREIGN DATA WRAPPER fdw_wrapper; + CREATE ROLE regress_dependency; +} + +teardown +{ + DROP FUNCTION IF EXISTS testschema.foo(); + DROP FUNCTION IF EXISTS fooargtype(num foo); + DROP FUNCTION IF EXISTS footrettype(); + DROP FUNCTION IF EXISTS foofunc(); + DROP FUNCTION IF EXISTS public.falter(); + DROP FUNCTION IF EXISTS alterschema.falter(); + DROP DOMAIN IF EXISTS idid; + DROP SERVER IF EXISTS srv_fdw_wrapper; + DROP TABLE IF EXISTS tabtype; + DROP SCHEMA IF EXISTS testschema; + DROP SCHEMA IF EXISTS alterschema; + DROP TYPE IF EXISTS public.foo; + DROP TYPE IF EXISTS public.footab; + DROP DOMAIN IF EXISTS id; + DROP FUNCTION IF EXISTS f(); + DROP FOREIGN DATA WRAPPER IF EXISTS fdw_wrapper; + DROP ROLE regress_dependency; +} + +session "s1" + +step "s1_begin" { BEGIN; } +step "s1_create_function_in_schema" { CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; } +step "s1_create_function_with_argtype" { CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; } +step "s1_create_function_with_rettype" { CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; } +step "s1_create_function_with_function" { CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; } +step "s1_alter_function_owner" { ALTER FUNCTION public.falter() OWNER TO regress_dependency; } +step "s1_alter_function_schema" { ALTER FUNCTION public.falter() SET SCHEMA alterschema; } +step "s1_create_domain_with_domain" { CREATE DOMAIN idid as id; } +step "s1_create_table_with_type" { CREATE TABLE tabtype(a footab); } +step "s1_create_server_with_fdw_wrapper" { CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; } +step "s1_commit" { COMMIT; } + +session "s2" + +step "s2_begin" { BEGIN; } +step "s2_drop_schema" { DROP SCHEMA testschema; } +step "s2_drop_alterschema" { DROP SCHEMA alterschema; } +step "s2_drop_foo_type" { DROP TYPE public.foo; } +step "s2_drop_foo_rettype" { DROP DOMAIN id; } +step "s2_drop_footab_type" { DROP TYPE public.footab; } +step "s2_drop_function_f" { DROP FUNCTION f(); } +step "s2_drop_domain_id" { DROP DOMAIN id; } +step "s2_drop_fdw_wrapper" { DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; } +step "s2_drop_role" { DROP ROLE regress_dependency; } +step "s2_commit" { COMMIT; } + +# create function - drop schema +permutation "s1_begin" "s1_create_function_in_schema" "s2_drop_schema" "s1_commit" +permutation "s2_begin" "s2_drop_schema" "s1_create_function_in_schema" "s2_commit" + +# alter function - drop schema +permutation "s1_begin" "s1_alter_function_schema" "s2_drop_alterschema" "s1_commit" +permutation "s2_begin" "s2_drop_alterschema" "s1_alter_function_schema" "s2_commit" + +# create function - drop argtype +permutation "s1_begin" "s1_create_function_with_argtype" "s2_drop_foo_type" "s1_commit" +permutation "s2_begin" "s2_drop_foo_type" "s1_create_function_with_argtype" "s2_commit" + +# create function - drop rettype +permutation "s1_begin" "s1_create_function_with_rettype" "s2_drop_foo_rettype" "s1_commit" +permutation "s2_begin" "s2_drop_foo_rettype" "s1_create_function_with_rettype" "s2_commit" + +# create function - drop function used in its body +permutation "s1_begin" "s1_create_function_with_function" "s2_drop_function_f" "s1_commit" +permutation "s2_begin" "s2_drop_function_f" "s1_create_function_with_function" "s2_commit" + +# create domain over domain - drop the base domain +permutation "s1_begin" "s1_create_domain_with_domain" "s2_drop_domain_id" "s1_commit" +permutation "s2_begin" "s2_drop_domain_id" "s1_create_domain_with_domain" "s2_commit" + +# create table - drop type used in column +permutation "s1_begin" "s1_create_table_with_type" "s2_drop_footab_type" "s1_commit" +permutation "s2_begin" "s2_drop_footab_type" "s1_create_table_with_type" "s2_commit" + +# create server - drop foreign data wrapper +permutation "s1_begin" "s1_create_server_with_fdw_wrapper" "s2_drop_fdw_wrapper" "s1_commit" +permutation "s2_begin" "s2_drop_fdw_wrapper" "s1_create_server_with_fdw_wrapper" "s2_commit" + +# create function - drop owner role +permutation "s1_begin" "s1_alter_function_owner" "s2_drop_role" "s1_commit" + +# XXX: This permutation is disabled because the error message, "role +# was concurrently dropped", contains an OID that is not stable. +# +# permutation "s2_begin" "s2_drop_role" "s1_alter_function_owner" "s2_commit" diff --git a/src/test/regress/expected/alter_table.out b/src/test/regress/expected/alter_table.out index 7fdcdc1c6c..812e77e893 100644 --- a/src/test/regress/expected/alter_table.out +++ b/src/test/regress/expected/alter_table.out @@ -2843,11 +2843,12 @@ begin; alter table alterlock2 add constraint alterlock2nv foreign key (f1) references alterlock (f1) NOT VALID; select * from my_locks order by 1; - relname | max_lockmode -------------+----------------------- - alterlock | ShareRowExclusiveLock - alterlock2 | ShareRowExclusiveLock -(2 rows) + relname | max_lockmode +----------------+----------------------- + alterlock | ShareRowExclusiveLock + alterlock2 | ShareRowExclusiveLock + alterlock_pkey | AccessShareLock +(3 rows) commit; begin; From b67b2cd702725f79ff16b7e01b3d49e54a360a2a Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Thu, 28 May 2026 11:34:14 -0400 Subject: [PATCH 17/76] Make stack depth check work with asan's use-after-return With address sanitizer's stack-use-after-return check, stack variables are moved to heap allocations, to allow to detect references to the memory at a later time. That broke our stack-depth check, which is why we had to disable detect_stack_use_after_return in CI. Luckily __builtin_frame_address() works correctly, even under asan, so use that. We started using __builtin_frame_address() with de447bb8e6fb, however as of that commit we just used it for the stack base address, not for the value to compare to the base address. Now we use it for both. When building without __builtin_frame_address() support, we continue to use stack variables for the stack depth determination. Reviewed-by: Tom Lane Discussion: https://postgr.es/m/2kk4z4odvuyrg7qlwjd7ft4eron4cle4btb33v4qatgsdkayir@gj6e62rgsel4 Backpatch-through: 14 --- src/backend/tcop/postgres.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 97486f2eb9..0e64b04bd1 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3436,7 +3436,8 @@ set_stack_base(void) /* * Set up reference point for stack depth checking. On recent gcc we use * __builtin_frame_address() to avoid a warning about storing a local - * variable's address in a long-lived variable. + * variable's address in a long-lived variable. This is also important + * with address sanitizer, see comment in stack_is_too_deep(). */ #ifdef HAVE__BUILTIN_FRAME_ADDRESS stack_base_ptr = __builtin_frame_address(0); @@ -3498,13 +3499,28 @@ check_stack_depth(void) bool stack_is_too_deep(void) { +#ifndef HAVE__BUILTIN_FRAME_ADDRESS char stack_top_loc; +#endif long stack_depth; + char *stack_address; + + /* + * With address sanitizer's stack-use-after-return check, stack variables + * are moved to heap allocations, to allow to detect references to the + * memory at a later time. That would break our stack-depth check. Luckily + * __builtin_frame_address() works correctly, even under asan. + */ +#ifndef HAVE__BUILTIN_FRAME_ADDRESS + stack_address = &stack_top_loc; +#else + stack_address = (char *) __builtin_frame_address(0); +#endif /* - * Compute distance from reference point to my local variables + * Compute distance from reference point to my stack frame. */ - stack_depth = (long) (stack_base_ptr - &stack_top_loc); + stack_depth = (long) (stack_base_ptr - stack_address); /* * Take abs value, since stacks grow up on some machines, down on others From d616e741fe61838f15d22bf470bca86b786b3554 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 28 May 2026 21:27:50 +0300 Subject: [PATCH 18/76] Use term "referenced" rather than "dependent" in dependency locking Reported-by: Kyotaro Horiguchi Discussion: https://www.postgresql.org/message-id/20260528.114608.488039299811669368.horikyota.ntt@gmail.com Backpatch-through: 14 --- src/backend/catalog/pg_depend.c | 6 +++--- .../expected/ddl-dependency-locking.out | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/backend/catalog/pg_depend.c b/src/backend/catalog/pg_depend.c index d8b6feed70..8eb86d3112 100644 --- a/src/backend/catalog/pg_depend.c +++ b/src/backend/catalog/pg_depend.c @@ -775,7 +775,7 @@ isObjectPinned(const ObjectAddress *object, Relation rel) * * If the caller already holds a lock that conflicts with DROP * (AccessShareLock or stronger), this does nothing. Callers should acquire - * locks already when they look up the dependent objects, but many callers + * locks already when they look up the referenced objects, but many callers * currently do not. This is a backstop to make sure that we don't record a * bogus reference permanently in the catalogs in that case. In the future, * after we have tightened up all the callers to acquire locks earlier, this @@ -847,7 +847,7 @@ dependencyLockAndCheckObject(Oid classId, Oid objectId) if (!HeapTupleIsValid(tuple)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("dependent %s was concurrently dropped", + errmsg("referenced %s was concurrently dropped", get_object_class_descr(classId)))); systable_endscan(scan); @@ -879,7 +879,7 @@ dependencyLockAndCheckObject(Oid classId, Oid objectId) return; ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("dependent relation was concurrently dropped"))); + errmsg("referenced relation was concurrently dropped"))); } } diff --git a/src/test/isolation/expected/ddl-dependency-locking.out b/src/test/isolation/expected/ddl-dependency-locking.out index 3bf9e43572..636de28102 100644 --- a/src/test/isolation/expected/ddl-dependency-locking.out +++ b/src/test/isolation/expected/ddl-dependency-locking.out @@ -14,7 +14,7 @@ step s2_drop_schema: DROP SCHEMA testschema; step s1_create_function_in_schema: CREATE FUNCTION testschema.foo() RETURNS int AS 'select 1' LANGUAGE sql; step s2_commit: COMMIT; step s1_create_function_in_schema: <... completed> -ERROR: dependent schema was concurrently dropped +ERROR: referenced schema was concurrently dropped starting permutation: s1_begin s1_alter_function_schema s2_drop_alterschema s1_commit step s1_begin: BEGIN; @@ -30,7 +30,7 @@ step s2_drop_alterschema: DROP SCHEMA alterschema; step s1_alter_function_schema: ALTER FUNCTION public.falter() SET SCHEMA alterschema; step s2_commit: COMMIT; step s1_alter_function_schema: <... completed> -ERROR: dependent schema was concurrently dropped +ERROR: referenced schema was concurrently dropped starting permutation: s1_begin s1_create_function_with_argtype s2_drop_foo_type s1_commit step s1_begin: BEGIN; @@ -46,7 +46,7 @@ step s2_drop_foo_type: DROP TYPE public.foo; step s1_create_function_with_argtype: CREATE FUNCTION fooargtype(num foo) RETURNS int AS 'select 1' LANGUAGE sql; step s2_commit: COMMIT; step s1_create_function_with_argtype: <... completed> -ERROR: dependent type was concurrently dropped +ERROR: referenced type was concurrently dropped starting permutation: s1_begin s1_create_function_with_rettype s2_drop_foo_rettype s1_commit step s1_begin: BEGIN; @@ -62,7 +62,7 @@ step s2_drop_foo_rettype: DROP DOMAIN id; step s1_create_function_with_rettype: CREATE FUNCTION footrettype() RETURNS id LANGUAGE sql RETURN 1; step s2_commit: COMMIT; step s1_create_function_with_rettype: <... completed> -ERROR: dependent type was concurrently dropped +ERROR: referenced type was concurrently dropped starting permutation: s1_begin s1_create_function_with_function s2_drop_function_f s1_commit step s1_begin: BEGIN; @@ -78,7 +78,7 @@ step s2_drop_function_f: DROP FUNCTION f(); step s1_create_function_with_function: CREATE FUNCTION foofunc() RETURNS int LANGUAGE SQL RETURN f() + 1; step s2_commit: COMMIT; step s1_create_function_with_function: <... completed> -ERROR: dependent function was concurrently dropped +ERROR: referenced function was concurrently dropped starting permutation: s1_begin s1_create_domain_with_domain s2_drop_domain_id s1_commit step s1_begin: BEGIN; @@ -94,7 +94,7 @@ step s2_drop_domain_id: DROP DOMAIN id; step s1_create_domain_with_domain: CREATE DOMAIN idid as id; step s2_commit: COMMIT; step s1_create_domain_with_domain: <... completed> -ERROR: dependent type was concurrently dropped +ERROR: referenced type was concurrently dropped starting permutation: s1_begin s1_create_table_with_type s2_drop_footab_type s1_commit step s1_begin: BEGIN; @@ -110,7 +110,7 @@ step s2_drop_footab_type: DROP TYPE public.footab; step s1_create_table_with_type: CREATE TABLE tabtype(a footab); step s2_commit: COMMIT; step s1_create_table_with_type: <... completed> -ERROR: dependent type was concurrently dropped +ERROR: referenced type was concurrently dropped starting permutation: s1_begin s1_create_server_with_fdw_wrapper s2_drop_fdw_wrapper s1_commit step s1_begin: BEGIN; @@ -126,7 +126,7 @@ step s2_drop_fdw_wrapper: DROP FOREIGN DATA WRAPPER fdw_wrapper RESTRICT; step s1_create_server_with_fdw_wrapper: CREATE SERVER srv_fdw_wrapper FOREIGN DATA WRAPPER fdw_wrapper; step s2_commit: COMMIT; step s1_create_server_with_fdw_wrapper: <... completed> -ERROR: dependent foreign-data wrapper was concurrently dropped +ERROR: referenced foreign-data wrapper was concurrently dropped starting permutation: s1_begin s1_alter_function_owner s2_drop_role s1_commit step s1_begin: BEGIN; From 74d3482f45dd8924579a9e5d10f8b15756e9db24 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 3 Jun 2026 12:47:34 +0900 Subject: [PATCH 19/76] Fix copy-paste error in hash_record_extended() The code failed to initialize the second isnull argument passed to FunctionCallInvoke(). This is harmless for existing in-core extended hash support functions, since FunctionCallInvoke() does not use the value (note that all the in-core extended hash functions are strict), examining only the argument values. However, extension-provided extended hash functions could be affected if they inspect PG_ARGISNULL(1). Oversight in 01e658fa74cb. Author: Man Zeng Discussion: https://postgr.es/m/tencent_7818173C01E01836109848C3@qq.com Backpatch-through: 14 --- src/backend/utils/adt/rowtypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/utils/adt/rowtypes.c b/src/backend/utils/adt/rowtypes.c index 1a71fdbc33..469ed3df8e 100644 --- a/src/backend/utils/adt/rowtypes.c +++ b/src/backend/utils/adt/rowtypes.c @@ -1995,7 +1995,7 @@ hash_record_extended(PG_FUNCTION_ARGS) locfcinfo->args[0].value = values[i]; locfcinfo->args[0].isnull = false; locfcinfo->args[1].value = Int64GetDatum(seed); - locfcinfo->args[0].isnull = false; + locfcinfo->args[1].isnull = false; element_hash = DatumGetUInt64(FunctionCallInvoke(locfcinfo)); /* We don't expect hash support functions to return null */ From 968c508457b3542387c5ee47426565dd20846ab0 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 3 Jun 2026 18:47:31 +0900 Subject: [PATCH 20/76] Fix race in ReplicationSlotRelease() for ephemeral slots When releasing an ephemeral replication slot, ReplicationSlotRelease() drops the slot via ReplicationSlotDropAcquired(). However, after dropping the slot, ReplicationSlotRelease() continued to use its local "slot" pointer, which still referenced the dropped slot's former shared-memory entry. It could then update fields such as effective_xmin in that entry. Once an ephemeral slot has been dropped (via ReplicationSlotDropAcquired()), its slot array entry can be reused immediately by another backend creating a new slot. As a result, those updates could corrupt the state of an unrelated replication slot. Fix by skipping those shared-memory updates for phemeral slots and performing them only for non-ephemeral slots, whose shared-memory entries remain valid after release. Backpatch to all supported versions. Author: Zhijie Hou Reviewed-by: Masao Fujii Reviewed-by: Srinath Reddy Sadipiralla Reviewed-by: Xuneng Zhou Discussion: https://postgr.es/m/TY4PR01MB177184FF9EE916F577E1F554194082@TY4PR01MB17718.jpnprd01.prod.outlook.com Backpatch-through: 14 --- src/backend/replication/slot.c | 52 ++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c index 78e5566b5d..ef2f8787ee 100644 --- a/src/backend/replication/slot.c +++ b/src/backend/replication/slot.c @@ -509,35 +509,37 @@ ReplicationSlotRelease(void) */ ReplicationSlotDropAcquired(); } - - /* - * If slot needed to temporarily restrain both data and catalog xmin to - * create the catalog snapshot, remove that temporary constraint. - * Snapshots can only be exported while the initial snapshot is still - * acquired. - */ - if (!TransactionIdIsValid(slot->data.xmin) && - TransactionIdIsValid(slot->effective_xmin)) - { - SpinLockAcquire(&slot->mutex); - slot->effective_xmin = InvalidTransactionId; - SpinLockRelease(&slot->mutex); - ReplicationSlotsComputeRequiredXmin(false); - } - - if (slot->data.persistency == RS_PERSISTENT) + else { /* - * Mark persistent slot inactive. We're not freeing it, just - * disconnecting, but wake up others that may be waiting for it. + * If slot needed to temporarily restrain both data and catalog xmin + * to create the catalog snapshot, remove that temporary constraint. + * Snapshots can only be exported while the initial snapshot is still + * acquired. */ - SpinLockAcquire(&slot->mutex); - slot->active_pid = 0; - SpinLockRelease(&slot->mutex); - ConditionVariableBroadcast(&slot->active_cv); - } + if (!TransactionIdIsValid(slot->data.xmin) && + TransactionIdIsValid(slot->effective_xmin)) + { + SpinLockAcquire(&slot->mutex); + slot->effective_xmin = InvalidTransactionId; + SpinLockRelease(&slot->mutex); + ReplicationSlotsComputeRequiredXmin(false); + } - MyReplicationSlot = NULL; + if (slot->data.persistency == RS_PERSISTENT) + { + /* + * Mark persistent slot inactive. We're not freeing it, just + * disconnecting, but wake up others that may be waiting for it. + */ + SpinLockAcquire(&slot->mutex); + slot->active_pid = 0; + SpinLockRelease(&slot->mutex); + ConditionVariableBroadcast(&slot->active_cv); + } + + MyReplicationSlot = NULL; + } /* might not have been set when we've been a plain slot */ LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); From 7bdff3e890e3d1149e9753b61667ccd14f15e6f4 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 4 Jun 2026 11:37:43 -0400 Subject: [PATCH 21/76] Fix another case of indirectly casting away const. Like 8f1791c61, this fixes a case of implicitly casting away const by not treating the result of strrchr() on a const pointer as const. This was missed at the time because the machines reporting those warnings weren't building with --with-llvm. While here, clean up another infelicity: in the probably- impossible case that the input string contains only one dot, this function would call pnstrdup() with a length of -1 and thereby emit a module name equal to the function name. It seems to me we should emit modname = NULL instead. Also remove a useless Assert and two redundant assignments. Back-patch, as 8f1791c61 was, so that users of back branches don't see this warning when building with late-model gcc. Reported-by: hubert depesz lubaczewski Author: Tom Lane Discussion: https://postgr.es/m/aiGNJ89PBqvq2Yyz@depesz.com Backpatch-through: 14 --- src/backend/jit/llvm/llvmjit.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c index 5f208bad81..e4011814cd 100644 --- a/src/backend/jit/llvm/llvmjit.c +++ b/src/backend/jit/llvm/llvmjit.c @@ -1191,9 +1191,6 @@ llvm_create_types(void) void llvm_split_symbol_name(const char *name, char **modname, char **funcname) { - *modname = NULL; - *funcname = NULL; - /* * Module function names are pgextern.$module.$funcname */ @@ -1203,14 +1200,21 @@ llvm_split_symbol_name(const char *name, char **modname, char **funcname) * Symbol names cannot contain a ., therefore we can split based on * first and last occurrence of one. */ - *funcname = rindex(name, '.'); - (*funcname)++; /* jump over . */ - - *modname = pnstrdup(name + strlen("pgextern."), - *funcname - name - strlen("pgextern.") - 1); - Assert(funcname); + const char *lastdot; - *funcname = pstrdup(*funcname); + name += strlen("pgextern."); + lastdot = strrchr(name, '.'); + if (lastdot) + { + *modname = pnstrdup(name, lastdot - name); + *funcname = pstrdup(lastdot + 1); + } + else + { + /* hmm, no second dot? */ + *modname = NULL; + *funcname = pstrdup(name); + } } else { From 262cc4df28976622aed47239c66f0bf56b1ce7bf Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 4 Jun 2026 12:24:51 -0400 Subject: [PATCH 22/76] Improve reporting of invalid weight symbols in setweight() et al. This commit addresses two related issues: tsvector_filter() assumed it could print an incorrect weight value with %c. This could result in an invalidly-encoded error message if the database encoding is multibyte and the char value has its high bit set. Weight values that are ASCII control characters could render illegibly too. Fix by printing such values in octal (\ooo), similarly to how charout() would render them. tsvector_setweight() and tsvector_setweight_by_filter() reported the same unrecognized-weight error condition with elog(), as though it were an internal error. That'd not translate, would produce an unwanted XX000 SQLSTATE code, and also reported the bad value as a decimal integer which seems unhelpful. Fix by refactoring so that all three functions share one copy of the code that interprets a weight argument. The invalid-encoding aspect seems to me (tgl) to justify back-patching. Author: Ewan Young Reviewed-by: Tom Lane Discussion: https://postgr.es/m/CAON2xHNaeLAUzRCXL5AmXLcXaSE_gWAVjWQRmLzc_oZ=1_Vf4Q@mail.gmail.com Backpatch-through: 14 --- src/backend/utils/adt/tsvector_op.c | 87 ++++++++++------------------- 1 file changed, 30 insertions(+), 57 deletions(-) diff --git a/src/backend/utils/adt/tsvector_op.c b/src/backend/utils/adt/tsvector_op.c index d9562e209d..cd8030af40 100644 --- a/src/backend/utils/adt/tsvector_op.c +++ b/src/backend/utils/adt/tsvector_op.c @@ -203,17 +203,10 @@ tsvector_length(PG_FUNCTION_ARGS) PG_RETURN_INT32(ret); } -Datum -tsvector_setweight(PG_FUNCTION_ARGS) +static int +parse_weight(char cw) { - TSVector in = PG_GETARG_TSVECTOR(0); - char cw = PG_GETARG_CHAR(1); - TSVector out; - int i, - j; - WordEntry *entry; - WordEntryPos *p; - int w = 0; + int w; switch (cw) { @@ -234,9 +227,32 @@ tsvector_setweight(PG_FUNCTION_ARGS) w = 0; break; default: - /* internal error */ - elog(ERROR, "unrecognized weight: %d", cw); + /* Avoid printing non-ASCII bytes, else we have encoding issues */ + if (cw >= ' ' && cw < 0x7f) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unrecognized weight: \"%c\"", cw))); + else /* use \ooo format, like charout() */ + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unrecognized weight: \"\\%03o\"", + (unsigned char) cw))); } + return w; +} + + +Datum +tsvector_setweight(PG_FUNCTION_ARGS) +{ + TSVector in = PG_GETARG_TSVECTOR(0); + char cw = PG_GETARG_CHAR(1); + TSVector out; + int i, + j; + WordEntry *entry; + WordEntryPos *p; + int w = parse_weight(cw); out = (TSVector) palloc(VARSIZE(in)); memcpy(out, in, VARSIZE(in)); @@ -281,28 +297,7 @@ tsvector_setweight_by_filter(PG_FUNCTION_ARGS) Datum *dlexemes; bool *nulls; - switch (char_weight) - { - case 'A': - case 'a': - weight = 3; - break; - case 'B': - case 'b': - weight = 2; - break; - case 'C': - case 'c': - weight = 1; - break; - case 'D': - case 'd': - weight = 0; - break; - default: - /* internal error */ - elog(ERROR, "unrecognized weight: %c", char_weight); - } + weight = parse_weight(char_weight); tsout = (TSVector) palloc(VARSIZE(tsin)); memcpy(tsout, tsin, VARSIZE(tsin)); @@ -840,29 +835,7 @@ tsvector_filter(PG_FUNCTION_ARGS) errmsg("weight array may not contain nulls"))); char_weight = DatumGetChar(dweights[i]); - switch (char_weight) - { - case 'A': - case 'a': - mask = mask | 8; - break; - case 'B': - case 'b': - mask = mask | 4; - break; - case 'C': - case 'c': - mask = mask | 2; - break; - case 'D': - case 'd': - mask = mask | 1; - break; - default: - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("unrecognized weight: \"%c\"", char_weight))); - } + mask |= 1 << parse_weight(char_weight); } tsout = (TSVector) palloc0(VARSIZE(tsin)); From 8bb935d619f6397ca91742195965d20b0ee5df6c Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 5 Jun 2026 07:50:18 +0900 Subject: [PATCH 23/76] Fix off-by-one with NFC recomposition for Hangul U+11A7 (TBASE) The NFC recomposition incorrectly included TBASE as a valid T syllable, which is incorrect based on the Unicode specification (TBASE is one below the start of the range, range beginning at U+11A8). This would cause the TBASE to be silently swallowed in the normalization, leading to an incorrect result. A couple of regression tests are added to check more patterns with Hangul recomposition and decomposition, on top of a test to check the problem with TBASE. Diego has submitted the code fix, and I have written the tests. Author: Diego Frias Co-authored-by: Michael Paquier Discussion: https://postgr.es/m/B92ED640-7D4A-4505-B09F-3548F58CBB16@dzfrias.dev Backpatch-through: 14 --- src/common/unicode_norm.c | 2 +- src/test/regress/expected/unicode.out | 78 +++++++++++++++++++++++++++ src/test/regress/sql/unicode.sql | 20 +++++++ 3 files changed, 99 insertions(+), 1 deletion(-) diff --git a/src/common/unicode_norm.c b/src/common/unicode_norm.c index 783a37eb3d..7b9d78ce16 100644 --- a/src/common/unicode_norm.c +++ b/src/common/unicode_norm.c @@ -236,7 +236,7 @@ recompose_code(uint32 start, uint32 code, uint32 *result) /* Check if two current characters are LV and T */ else if (start >= SBASE && start < (SBASE + SCOUNT) && ((start - SBASE) % TCOUNT) == 0 && - code >= TBASE && code < (TBASE + TCOUNT)) + code > TBASE && code < (TBASE + TCOUNT)) { /* make syllable of form LVT */ uint32 tindex = code - TBASE; diff --git a/src/test/regress/expected/unicode.out b/src/test/regress/expected/unicode.out index f2713a2326..ab0081165d 100644 --- a/src/test/regress/expected/unicode.out +++ b/src/test/regress/expected/unicode.out @@ -87,3 +87,81 @@ ORDER BY num; SELECT is_normalized('abc', 'def'); -- run-time error ERROR: invalid normalization form: def +-- Hangul NFC recomposition tests +-- L+V -> LV composition (first and last) +SELECT normalize(U&'\1100\1161', NFC) = U&'\AC00' COLLATE "C" AS hangul_lv_first; + hangul_lv_first +----------------- + t +(1 row) + +SELECT normalize(U&'\1112\1175', NFC) = U&'\D788' COLLATE "C" AS hangul_lv_last; + hangul_lv_last +---------------- + t +(1 row) + +-- LV+T -> LVT composition +SELECT normalize(U&'\AC00\11A8', NFC) = U&'\AC01' COLLATE "C" AS hangul_lvt_first_t; + hangul_lvt_first_t +-------------------- + t +(1 row) + +SELECT normalize(U&'\AC00\11C2', NFC) = U&'\AC1B' COLLATE "C" AS hangul_lvt_last_t; + hangul_lvt_last_t +------------------- + t +(1 row) + +SELECT normalize(U&'\D788\11A8', NFC) = U&'\D789' COLLATE "C" AS hangul_lvt_last_lv; + hangul_lvt_last_lv +-------------------- + t +(1 row) + +-- L+V+T -> LVT composition +SELECT normalize(U&'\1100\1161\11A8', NFC) = U&'\AC01' COLLATE "C" AS hangul_full_lvt; + hangul_full_lvt +----------------- + t +(1 row) + +SELECT normalize(U&'\1112\1175\11C2', NFC) = U&'\D7A3' COLLATE "C" AS hangul_full_lvt; + hangul_full_lvt +----------------- + t +(1 row) + +-- TBASE invalid T syllable +SELECT normalize(U&'\AC00\11A7', NFC) = U&'\AC00\11A7' COLLATE "C" AS hangul_tbase_not_combined; + hangul_tbase_not_combined +--------------------------- + t +(1 row) + +SELECT normalize(U&'\1100\1161\11A7', NFC) = U&'\AC00\11A7' COLLATE "C" AS hangul_lv_tbase_separate; + hangul_lv_tbase_separate +-------------------------- + t +(1 row) + +-- Hangul NFD decomposition tests +SELECT normalize(U&'\AC00', NFD) = U&'\1100\1161' COLLATE "C" AS hangul_nfd_lv; + hangul_nfd_lv +--------------- + t +(1 row) + +SELECT normalize(U&'\AC01', NFD) = U&'\1100\1161\11A8' COLLATE "C" AS hangul_nfd_lvt; + hangul_nfd_lvt +---------------- + t +(1 row) + +SELECT normalize(U&'\D7A3', NFD) = U&'\1112\1175\11C2' COLLATE "C" AS hangul_nfd_last; + hangul_nfd_last +----------------- + t +(1 row) + diff --git a/src/test/regress/sql/unicode.sql b/src/test/regress/sql/unicode.sql index 63cd523f85..95c5a7ac18 100644 --- a/src/test/regress/sql/unicode.sql +++ b/src/test/regress/sql/unicode.sql @@ -32,3 +32,23 @@ FROM ORDER BY num; SELECT is_normalized('abc', 'def'); -- run-time error + +-- Hangul NFC recomposition tests +-- L+V -> LV composition (first and last) +SELECT normalize(U&'\1100\1161', NFC) = U&'\AC00' COLLATE "C" AS hangul_lv_first; +SELECT normalize(U&'\1112\1175', NFC) = U&'\D788' COLLATE "C" AS hangul_lv_last; +-- LV+T -> LVT composition +SELECT normalize(U&'\AC00\11A8', NFC) = U&'\AC01' COLLATE "C" AS hangul_lvt_first_t; +SELECT normalize(U&'\AC00\11C2', NFC) = U&'\AC1B' COLLATE "C" AS hangul_lvt_last_t; +SELECT normalize(U&'\D788\11A8', NFC) = U&'\D789' COLLATE "C" AS hangul_lvt_last_lv; +-- L+V+T -> LVT composition +SELECT normalize(U&'\1100\1161\11A8', NFC) = U&'\AC01' COLLATE "C" AS hangul_full_lvt; +SELECT normalize(U&'\1112\1175\11C2', NFC) = U&'\D7A3' COLLATE "C" AS hangul_full_lvt; +-- TBASE invalid T syllable +SELECT normalize(U&'\AC00\11A7', NFC) = U&'\AC00\11A7' COLLATE "C" AS hangul_tbase_not_combined; +SELECT normalize(U&'\1100\1161\11A7', NFC) = U&'\AC00\11A7' COLLATE "C" AS hangul_lv_tbase_separate; + +-- Hangul NFD decomposition tests +SELECT normalize(U&'\AC00', NFD) = U&'\1100\1161' COLLATE "C" AS hangul_nfd_lv; +SELECT normalize(U&'\AC01', NFD) = U&'\1100\1161\11A8' COLLATE "C" AS hangul_nfd_lvt; +SELECT normalize(U&'\D7A3', NFD) = U&'\1112\1175\11C2' COLLATE "C" AS hangul_nfd_last; From 5b72d0279be1f1e981abddff95bbf1913b2b16fc Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Fri, 5 Jun 2026 12:08:05 -0500 Subject: [PATCH 24/76] refint: Remove plan cache. Presently, refint stores plans in a per-backend cache to avoid re-preparing in each call. This has a few problems. For one, check_foreign_key() embeds the new key values in its cascade-UPDATE queries, so a cached plan reuses the values from preparation. Also, the cache is never invalidated, so it can return stale entries that cause other problems. There may very well be more bugs lurking. We could spend a lot of time trying to address all these problems, but this module is primarily intended as sample code, and by all indications, it sees minimal use. Furthermore, there is a growing consensus for removing refint in v20. However, since we'll need to support it on the back-branches for a while longer, it probably still makes sense to fix some of the more egregious bugs. Therefore, let's just remove refint's plan cache entirely. That means we'll re-prepare on every call, but that seems quite unlikely to bother anyone. On v17 and older versions, the regression test for triggers fails after this change, so I've borrowed pieces of commit 8cfbdf8f4d to fix it. Author: Ayush Tiwari Discussion: https://postgr.es/m/CAJTYsWXU%2BfhuzrEd_bnrxyGH3%2Bny8QRQC2QHf3ws6s9iki3c2Q%40mail.gmail.com Backpatch-through: 14 --- contrib/spi/refint.c | 352 ++++++++----------------- src/test/regress/expected/triggers.out | 37 ++- src/test/regress/sql/triggers.sql | 10 +- 3 files changed, 137 insertions(+), 262 deletions(-) diff --git a/contrib/spi/refint.c b/contrib/spi/refint.c index 413411c173..80b9ef650a 100644 --- a/contrib/spi/refint.c +++ b/contrib/spi/refint.c @@ -12,25 +12,10 @@ #include "commands/trigger.h" #include "executor/spi.h" #include "utils/builtins.h" -#include "utils/memutils.h" #include "utils/rel.h" PG_MODULE_MAGIC; -typedef struct -{ - char *ident; - int nplans; - SPIPlanPtr *splan; -} EPlan; - -static EPlan *FPlans = NULL; -static int nFPlans = 0; -static EPlan *PPlans = NULL; -static int nPPlans = 0; - -static EPlan *find_plan(char *ident, EPlan **eplan, int *nplans); - /* * check_primary_key () -- check that key in tuple being inserted/updated * references existing tuple in "primary" table. @@ -56,12 +41,12 @@ check_primary_key(PG_FUNCTION_ARGS) Relation rel; /* triggered relation */ HeapTuple tuple = NULL; /* tuple to return */ TupleDesc tupdesc; /* tuple description */ - EPlan *plan; /* prepared plan */ + SPIPlanPtr pplan; /* prepared plan */ Oid *argtypes = NULL; /* key types to prepare execution plan */ bool isnull; /* to know is some column NULL or not */ - char ident[2 * NAMEDATALEN]; /* to identify myself */ int ret; int i; + StringInfoData sql; #ifdef DEBUG_QUERY elog(DEBUG4, "check_primary_key: Enter Function"); @@ -118,16 +103,8 @@ check_primary_key(PG_FUNCTION_ARGS) */ kvals = (Datum *) palloc(nkeys * sizeof(Datum)); - /* - * Construct ident string as TriggerName $ TriggeredRelationId and try to - * find prepared execution plan. - */ - snprintf(ident, sizeof(ident), "%s$%u", trigger->tgname, rel->rd_id); - plan = find_plan(ident, &PPlans, &nPPlans); - - /* if there is no plan then allocate argtypes for preparation */ - if (plan->nplans <= 0) - argtypes = (Oid *) palloc(nkeys * sizeof(Oid)); + /* allocate argtypes for preparation */ + argtypes = (Oid *) palloc(nkeys * sizeof(Oid)); /* For each column in key ... */ for (i = 0; i < nkeys; i++) @@ -156,57 +133,36 @@ check_primary_key(PG_FUNCTION_ARGS) return PointerGetDatum(tuple); } - if (plan->nplans <= 0) /* Get typeId of column */ - argtypes[i] = SPI_gettypeid(tupdesc, fnumber); + /* Get typeId of column */ + argtypes[i] = SPI_gettypeid(tupdesc, fnumber); } + initStringInfo(&sql); + /* - * If we have to prepare plan ... + * Construct query: SELECT 1 FROM _referenced_relation_ WHERE Pkey1 = $1 + * [AND Pkey2 = $2 [...]] */ - if (plan->nplans <= 0) + appendStringInfo(&sql, "select 1 from %s where ", relname); + for (i = 1; i <= nkeys; i++) { - SPIPlanPtr pplan; - StringInfoData sql; - - initStringInfo(&sql); - - /* - * Construct query: SELECT 1 FROM _referenced_relation_ WHERE Pkey1 = - * $1 [AND Pkey2 = $2 [...]] - */ - appendStringInfo(&sql, "select 1 from %s where ", relname); - for (i = 1; i <= nkeys; i++) - { - appendStringInfo(&sql, "%s = $%d ", args[i + nkeys], i); - if (i < nkeys) - appendStringInfoString(&sql, "and "); - } - - /* Prepare plan for query */ - pplan = SPI_prepare(sql.data, nkeys, argtypes); - if (pplan == NULL) - /* internal error */ - elog(ERROR, "check_primary_key: SPI_prepare returned %s", SPI_result_code_string(SPI_result)); + appendStringInfo(&sql, "%s = $%d ", args[i + nkeys], i); + if (i < nkeys) + appendStringInfoString(&sql, "and "); + } - /* - * Remember that SPI_prepare places plan in current memory context - - * so, we have to save plan in TopMemoryContext for later use. - */ - if (SPI_keepplan(pplan)) - /* internal error */ - elog(ERROR, "check_primary_key: SPI_keepplan failed"); - plan->splan = (SPIPlanPtr *) MemoryContextAlloc(TopMemoryContext, - sizeof(SPIPlanPtr)); - *(plan->splan) = pplan; - plan->nplans = 1; + /* Prepare plan for query */ + pplan = SPI_prepare(sql.data, nkeys, argtypes); + if (pplan == NULL) + /* internal error */ + elog(ERROR, "check_primary_key: SPI_prepare returned %s", SPI_result_code_string(SPI_result)); - pfree(sql.data); - } + pfree(sql.data); /* * Ok, execute prepared plan. */ - ret = SPI_execp(*(plan->splan), kvals, NULL, 1); + ret = SPI_execp(pplan, kvals, NULL, 1); /* we have no NULLs - so we pass ^^^^ here */ if (ret < 0) @@ -258,15 +214,15 @@ check_foreign_key(PG_FUNCTION_ARGS) HeapTuple trigtuple = NULL; /* tuple to being changed */ HeapTuple newtuple = NULL; /* tuple to return */ TupleDesc tupdesc; /* tuple description */ - EPlan *plan; /* prepared plan(s) */ + SPIPlanPtr *splan; /* prepared plan(s) */ Oid *argtypes = NULL; /* key types to prepare execution plan */ bool isnull; /* to know is some column NULL or not */ bool isequal = true; /* are keys in both tuples equal (in UPDATE) */ - char ident[2 * NAMEDATALEN]; /* to identify myself */ int is_update = 0; int ret; int i, r; + char **args2; #ifdef DEBUG_QUERY elog(DEBUG4, "check_foreign_key: Enter Function"); @@ -343,24 +299,8 @@ check_foreign_key(PG_FUNCTION_ARGS) */ kvals = (Datum *) palloc(nkeys * sizeof(Datum)); - /* - * Construct ident string as TriggerName $ TriggeredRelationId and try to - * find prepared execution plan(s). - */ - snprintf(ident, sizeof(ident), "%s$%u", trigger->tgname, rel->rd_id); - plan = find_plan(ident, &FPlans, &nFPlans); - - /* if there is no plan(s) then allocate argtypes for preparation */ - if (plan->nplans <= 0) - argtypes = (Oid *) palloc(nkeys * sizeof(Oid)); - - /* - * else - check that we have exactly nrefs plan(s) ready - */ - else if (plan->nplans != nrefs) - /* internal error */ - elog(ERROR, "%s: check_foreign_key: # of plans changed in meantime", - trigger->tgname); + /* allocate argtypes for preparation */ + argtypes = (Oid *) palloc(nkeys * sizeof(Oid)); /* For each column in key ... */ for (i = 0; i < nkeys; i++) @@ -408,143 +348,124 @@ check_foreign_key(PG_FUNCTION_ARGS) isequal = false; } - if (plan->nplans <= 0) /* Get typeId of column */ - argtypes[i] = SPI_gettypeid(tupdesc, fnumber); + /* Get typeId of column */ + argtypes[i] = SPI_gettypeid(tupdesc, fnumber); } args_temp = args; nargs -= nkeys; args += nkeys; + args2 = args; - /* - * If we have to prepare plans ... - */ - if (plan->nplans <= 0) + splan = (SPIPlanPtr *) palloc(nrefs * sizeof(SPIPlanPtr)); + + for (r = 0; r < nrefs; r++) { + StringInfoData sql; SPIPlanPtr pplan; - char **args2 = args; - plan->splan = (SPIPlanPtr *) MemoryContextAlloc(TopMemoryContext, - nrefs * sizeof(SPIPlanPtr)); + initStringInfo(&sql); - for (r = 0; r < nrefs; r++) - { - StringInfoData sql; - - initStringInfo(&sql); - - relname = args2[0]; - - /*--------- - * For 'R'estrict action we construct SELECT query: - * - * SELECT 1 - * FROM _referencing_relation_ - * WHERE Fkey1 = $1 [AND Fkey2 = $2 [...]] - * - * to check is tuple referenced or not. - *--------- - */ - if (action == 'r') - appendStringInfo(&sql, "select 1 from %s where ", relname); - - /*--------- - * For 'C'ascade action we construct DELETE query - * - * DELETE - * FROM _referencing_relation_ - * WHERE Fkey1 = $1 [AND Fkey2 = $2 [...]] - * - * to delete all referencing tuples. - *--------- - */ - - /* - * Max : Cascade with UPDATE query i create update query that - * updates new key values in referenced tables - */ - - - else if (action == 'c') - { - if (is_update == 1) - { - int fn; - char *nv; - int k; - - appendStringInfo(&sql, "update %s set ", relname); - for (k = 1; k <= nkeys; k++) - { - fn = SPI_fnumber(tupdesc, args_temp[k - 1]); - Assert(fn > 0); /* already checked above */ - nv = SPI_getvalue(newtuple, tupdesc, fn); - - appendStringInfo(&sql, " %s = %s ", - args2[k], - nv ? quote_literal_cstr(nv) : "NULL"); - if (k < nkeys) - appendStringInfoString(&sql, ", "); - } - appendStringInfoString(&sql, " where "); + relname = args2[0]; + + /*--------- + * For 'R'estrict action we construct SELECT query: + * + * SELECT 1 + * FROM _referencing_relation_ + * WHERE Fkey1 = $1 [AND Fkey2 = $2 [...]] + * + * to check is tuple referenced or not. + *--------- + */ + if (action == 'r') + appendStringInfo(&sql, "select 1 from %s where ", relname); + + /*--------- + * For 'C'ascade action we construct DELETE query + * + * DELETE + * FROM _referencing_relation_ + * WHERE Fkey1 = $1 [AND Fkey2 = $2 [...]] + * + * to delete all referencing tuples. + *--------- + */ - } - else - /* DELETE */ - appendStringInfo(&sql, "delete from %s where ", relname); + /* + * Max : Cascade with UPDATE query i create update query that updates + * new key values in referenced tables + */ - } - /* - * For 'S'etnull action we construct UPDATE query - UPDATE - * _referencing_relation_ SET Fkey1 null [, Fkey2 null [...]] - * WHERE Fkey1 = $1 [AND Fkey2 = $2 [...]] - to set key columns in - * all referencing tuples to NULL. - */ - else if (action == 's') + else if (action == 'c') + { + if (is_update == 1) { + int fn; + char *nv; + int k; + appendStringInfo(&sql, "update %s set ", relname); - for (i = 1; i <= nkeys; i++) + for (k = 1; k <= nkeys; k++) { - appendStringInfo(&sql, "%s = null", args2[i]); - if (i < nkeys) + fn = SPI_fnumber(tupdesc, args_temp[k - 1]); + Assert(fn > 0); /* already checked above */ + nv = SPI_getvalue(newtuple, tupdesc, fn); + + appendStringInfo(&sql, " %s = %s ", + args2[k], + nv ? quote_literal_cstr(nv) : "NULL"); + if (k < nkeys) appendStringInfoString(&sql, ", "); } appendStringInfoString(&sql, " where "); } + else + /* DELETE */ + appendStringInfo(&sql, "delete from %s where ", relname); + } - /* Construct WHERE qual */ + /* + * For 'S'etnull action we construct UPDATE query - UPDATE + * _referencing_relation_ SET Fkey1 null [, Fkey2 null [...]] WHERE + * Fkey1 = $1 [AND Fkey2 = $2 [...]] - to set key columns in all + * referencing tuples to NULL. + */ + else if (action == 's') + { + appendStringInfo(&sql, "update %s set ", relname); for (i = 1; i <= nkeys; i++) { - appendStringInfo(&sql, "%s = $%d ", args2[i], i); + appendStringInfo(&sql, "%s = null", args2[i]); if (i < nkeys) - appendStringInfoString(&sql, "and "); + appendStringInfoString(&sql, ", "); } + appendStringInfoString(&sql, " where "); + } - /* Prepare plan for query */ - pplan = SPI_prepare(sql.data, nkeys, argtypes); - if (pplan == NULL) - /* internal error */ - elog(ERROR, "check_foreign_key: SPI_prepare returned %s", SPI_result_code_string(SPI_result)); + /* Construct WHERE qual */ + for (i = 1; i <= nkeys; i++) + { + appendStringInfo(&sql, "%s = $%d ", args2[i], i); + if (i < nkeys) + appendStringInfoString(&sql, "and "); + } - /* - * Remember that SPI_prepare places plan in current memory context - * - so, we have to save plan in Top memory context for later use. - */ - if (SPI_keepplan(pplan)) - /* internal error */ - elog(ERROR, "check_foreign_key: SPI_keepplan failed"); + /* Prepare plan for query */ + pplan = SPI_prepare(sql.data, nkeys, argtypes); + if (pplan == NULL) + /* internal error */ + elog(ERROR, "check_foreign_key: SPI_prepare returned %s", SPI_result_code_string(SPI_result)); - plan->splan[r] = pplan; + splan[r] = pplan; - args2 += nkeys + 1; /* to the next relation */ + args2 += nkeys + 1; /* to the next relation */ #ifdef DEBUG_QUERY - elog(DEBUG4, "check_foreign_key Debug Query is : %s ", sql.data); + elog(DEBUG4, "check_foreign_key Debug Query is : %s ", sql.data); #endif - pfree(sql.data); - } - plan->nplans = nrefs; + pfree(sql.data); } /* @@ -569,9 +490,7 @@ check_foreign_key(PG_FUNCTION_ARGS) relname = args[0]; - snprintf(ident, sizeof(ident), "%s$%u", trigger->tgname, rel->rd_id); - plan = find_plan(ident, &FPlans, &nFPlans); - ret = SPI_execp(plan->splan[r], kvals, NULL, tcount); + ret = SPI_execp(splan[r], kvals, NULL, tcount); /* we have no NULLs - so we pass ^^^^ here */ if (ret < 0) @@ -604,46 +523,3 @@ check_foreign_key(PG_FUNCTION_ARGS) return PointerGetDatum((newtuple == NULL) ? trigtuple : newtuple); } - -static EPlan * -find_plan(char *ident, EPlan **eplan, int *nplans) -{ - EPlan *newp; - int i; - MemoryContext oldcontext; - - /* - * All allocations done for the plans need to happen in a session-safe - * context. - */ - oldcontext = MemoryContextSwitchTo(TopMemoryContext); - - if (*nplans > 0) - { - for (i = 0; i < *nplans; i++) - { - if (strcmp((*eplan)[i].ident, ident) == 0) - break; - } - if (i != *nplans) - { - MemoryContextSwitchTo(oldcontext); - return (*eplan + i); - } - *eplan = (EPlan *) repalloc(*eplan, (i + 1) * sizeof(EPlan)); - newp = *eplan + i; - } - else - { - newp = *eplan = (EPlan *) palloc(sizeof(EPlan)); - (*nplans) = i = 0; - } - - newp->ident = pstrdup(ident); - newp->nplans = 0; - newp->splan = NULL; - (*nplans)++; - - MemoryContextSwitchTo(oldcontext); - return newp; -} diff --git a/src/test/regress/expected/triggers.out b/src/test/regress/expected/triggers.out index 768f2e6070..901ce8449a 100644 --- a/src/test/regress/expected/triggers.out +++ b/src/test/regress/expected/triggers.out @@ -20,12 +20,12 @@ create unique index pkeys_i on pkeys (pkey1, pkey2); -- (fkey3) --> fkeys2 (pkey23) -- create trigger check_fkeys_pkey_exist - before insert or update on fkeys + after insert or update on fkeys for each row execute function check_primary_key ('fkey1', 'fkey2', 'pkeys', 'pkey1', 'pkey2'); create trigger check_fkeys_pkey2_exist - before insert or update on fkeys + after insert or update on fkeys for each row execute function check_primary_key ('fkey3', 'fkeys2', 'pkey23'); -- @@ -33,7 +33,7 @@ create trigger check_fkeys_pkey2_exist -- (fkey21, fkey22) --> pkeys (pkey1, pkey2) -- create trigger check_fkeys2_pkey_exist - before insert or update on fkeys2 + after insert or update on fkeys2 for each row execute procedure check_primary_key ('fkey21', 'fkey22', 'pkeys', 'pkey1', 'pkey2'); @@ -48,7 +48,7 @@ COMMENT ON TRIGGER check_fkeys2_pkey_exist ON fkeys2 IS NULL; -- fkeys (fkey1, fkey2) and fkeys2 (fkey21, fkey22) -- create trigger check_pkeys_fkey_cascade - before delete or update on pkeys + after delete or update on pkeys for each row execute procedure check_foreign_key (2, 'cascade', 'pkey1', 'pkey2', @@ -59,7 +59,7 @@ create trigger check_pkeys_fkey_cascade -- fkeys (fkey3) -- create trigger check_fkeys2_fkey_restrict - before delete or update on fkeys2 + after delete or update on fkeys2 for each row execute procedure check_foreign_key (1, 'restrict', 'pkey23', 'fkeys', 'fkey3'); insert into fkeys2 values (10, '1', 1); @@ -91,11 +91,10 @@ NOTICE: check_pkeys_fkey_cascade: 1 tuple(s) of fkeys are deleted NOTICE: check_pkeys_fkey_cascade: 1 tuple(s) of fkeys2 are deleted update pkeys set pkey1 = 7, pkey2 = '70' where pkey1 = 50 and pkey2 = '5'; NOTICE: check_pkeys_fkey_cascade: 1 tuple(s) of fkeys are deleted -ERROR: "check_fkeys2_fkey_restrict": tuple is referenced in "fkeys" -CONTEXT: SQL statement "delete from fkeys2 where fkey21 = $1 and fkey22 = $2 " -update pkeys set pkey1 = 7, pkey2 = '70' where pkey1 = 10 and pkey2 = '1'; -NOTICE: check_pkeys_fkey_cascade: 1 tuple(s) of fkeys are deleted NOTICE: check_pkeys_fkey_cascade: 1 tuple(s) of fkeys2 are deleted +update pkeys set pkey1 = 7, pkey2 = '70' where pkey1 = 10 and pkey2 = '1'; +ERROR: duplicate key value violates unique constraint "pkeys_i" +DETAIL: Key (pkey1, pkey2)=(7, 70) already exists. SELECT trigger_name, event_manipulation, event_object_schema, event_object_table, action_order, action_condition, action_orientation, action_timing, action_reference_old_table, action_reference_new_table @@ -104,16 +103,16 @@ SELECT trigger_name, event_manipulation, event_object_schema, event_object_table ORDER BY trigger_name COLLATE "C", 2; trigger_name | event_manipulation | event_object_schema | event_object_table | action_order | action_condition | action_orientation | action_timing | action_reference_old_table | action_reference_new_table ----------------------------+--------------------+---------------------+--------------------+--------------+------------------+--------------------+---------------+----------------------------+---------------------------- - check_fkeys2_fkey_restrict | DELETE | public | fkeys2 | 1 | | ROW | BEFORE | | - check_fkeys2_fkey_restrict | UPDATE | public | fkeys2 | 1 | | ROW | BEFORE | | - check_fkeys2_pkey_exist | INSERT | public | fkeys2 | 1 | | ROW | BEFORE | | - check_fkeys2_pkey_exist | UPDATE | public | fkeys2 | 2 | | ROW | BEFORE | | - check_fkeys_pkey2_exist | INSERT | public | fkeys | 1 | | ROW | BEFORE | | - check_fkeys_pkey2_exist | UPDATE | public | fkeys | 1 | | ROW | BEFORE | | - check_fkeys_pkey_exist | INSERT | public | fkeys | 2 | | ROW | BEFORE | | - check_fkeys_pkey_exist | UPDATE | public | fkeys | 2 | | ROW | BEFORE | | - check_pkeys_fkey_cascade | DELETE | public | pkeys | 1 | | ROW | BEFORE | | - check_pkeys_fkey_cascade | UPDATE | public | pkeys | 1 | | ROW | BEFORE | | + check_fkeys2_fkey_restrict | DELETE | public | fkeys2 | 1 | | ROW | AFTER | | + check_fkeys2_fkey_restrict | UPDATE | public | fkeys2 | 1 | | ROW | AFTER | | + check_fkeys2_pkey_exist | INSERT | public | fkeys2 | 1 | | ROW | AFTER | | + check_fkeys2_pkey_exist | UPDATE | public | fkeys2 | 2 | | ROW | AFTER | | + check_fkeys_pkey2_exist | INSERT | public | fkeys | 1 | | ROW | AFTER | | + check_fkeys_pkey2_exist | UPDATE | public | fkeys | 1 | | ROW | AFTER | | + check_fkeys_pkey_exist | INSERT | public | fkeys | 2 | | ROW | AFTER | | + check_fkeys_pkey_exist | UPDATE | public | fkeys | 2 | | ROW | AFTER | | + check_pkeys_fkey_cascade | DELETE | public | pkeys | 1 | | ROW | AFTER | | + check_pkeys_fkey_cascade | UPDATE | public | pkeys | 1 | | ROW | AFTER | | (10 rows) DROP TABLE pkeys; diff --git a/src/test/regress/sql/triggers.sql b/src/test/regress/sql/triggers.sql index 71f3b6d465..d8e09e4212 100644 --- a/src/test/regress/sql/triggers.sql +++ b/src/test/regress/sql/triggers.sql @@ -24,13 +24,13 @@ create unique index pkeys_i on pkeys (pkey1, pkey2); -- (fkey3) --> fkeys2 (pkey23) -- create trigger check_fkeys_pkey_exist - before insert or update on fkeys + after insert or update on fkeys for each row execute function check_primary_key ('fkey1', 'fkey2', 'pkeys', 'pkey1', 'pkey2'); create trigger check_fkeys_pkey2_exist - before insert or update on fkeys + after insert or update on fkeys for each row execute function check_primary_key ('fkey3', 'fkeys2', 'pkey23'); @@ -39,7 +39,7 @@ create trigger check_fkeys_pkey2_exist -- (fkey21, fkey22) --> pkeys (pkey1, pkey2) -- create trigger check_fkeys2_pkey_exist - before insert or update on fkeys2 + after insert or update on fkeys2 for each row execute procedure check_primary_key ('fkey21', 'fkey22', 'pkeys', 'pkey1', 'pkey2'); @@ -55,7 +55,7 @@ COMMENT ON TRIGGER check_fkeys2_pkey_exist ON fkeys2 IS NULL; -- fkeys (fkey1, fkey2) and fkeys2 (fkey21, fkey22) -- create trigger check_pkeys_fkey_cascade - before delete or update on pkeys + after delete or update on pkeys for each row execute procedure check_foreign_key (2, 'cascade', 'pkey1', 'pkey2', @@ -67,7 +67,7 @@ create trigger check_pkeys_fkey_cascade -- fkeys (fkey3) -- create trigger check_fkeys2_fkey_restrict - before delete or update on fkeys2 + after delete or update on fkeys2 for each row execute procedure check_foreign_key (1, 'restrict', 'pkey23', 'fkeys', 'fkey3'); From 1eda3eb0753ac6c788c11830e9abc0821f7afd48 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Sat, 6 Jun 2026 08:16:46 +0900 Subject: [PATCH 25/76] pg_surgery: Fix off-by-one bug with heap offset heap_force_common() declared a boolean array indexed with an OffsetNumber for a size of MaxHeapTuplesPerPage. OffsetNumbers are 1-based, so an input TID whose offset number equals MaxHeapTuplesPerPage wrote one byte past the end of the stack array, crashing the server. Like heapam_handler.c, this commit changes the array so as it uses a 0-based index, substracting one from the OffsetNumbers. Reported-by: Wang Yuelin Reviewed-by: Ashutosh Sharma Discussion: https://postgr.es/m/20260604002256.40f1fd544@smtp.qiye.163.com Backpatch-through: 14 --- contrib/pg_surgery/heap_surgery.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c index d31e5f31fd..1f514f6fa8 100644 --- a/contrib/pg_surgery/heap_surgery.c +++ b/contrib/pg_surgery/heap_surgery.c @@ -206,8 +206,8 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt) } /* Mark it for processing. */ - Assert(offno < MaxHeapTuplesPerPage); - include_this_tid[offno] = true; + Assert(offno <= MaxHeapTuplesPerPage); + include_this_tid[offno - 1] = true; } /* @@ -225,7 +225,7 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt) { ItemId itemid; - if (!include_this_tid[curoff]) + if (!include_this_tid[curoff - 1]) continue; itemid = PageGetItemId(page, curoff); From a4ca91ea18916c66fa3539c2fd19d8ba571ce7e8 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 8 Jun 2026 14:38:01 +0900 Subject: [PATCH 26/76] psql: Fix expanded aligned output When a table's columns are narrower than the record header line, the expanded aligned format produced misaligned output because the data column width was not adjusted to match the record header width, leading to output like: +-[ RECORD 1 ]-+ | a | 10 | | b | 20 | +---+----+ This commit adjusts the output so as the column width match with the header line, giving: +-[ RECORD 1 ]-+ | a | 10 | | b | 20 | +---+----------+ Author: Pavel Stehule Reviewed-by: Chao Li Discussion: https://postgr.es/m/CAFj8pRCzGpsr9zTHbtTd4mGh2YPJqOEgLgt8JLiopuYA9_1xGw@mail.gmail.com Backpatch-through: 14 --- src/fe_utils/print.c | 7 ++++--- src/test/regress/expected/psql.out | 26 ++++++++++++++++++++++++++ src/test/regress/sql/psql.sql | 11 +++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/fe_utils/print.c b/src/fe_utils/print.c index 2d0f78b8a2..15ea8fbc2d 100644 --- a/src/fe_utils/print.c +++ b/src/fe_utils/print.c @@ -1346,9 +1346,10 @@ print_aligned_vertical(const printTableContent *cont, } /* - * Calculate available width for data in wrapped mode + * Determine data column width: fit output width in wrapped mode, or + * ensure alignment with the record header line in aligned mode. */ - if (cont->opt->format == PRINT_WRAPPED) + if (cont->opt->format == PRINT_WRAPPED || cont->opt->format == PRINT_ALIGNED) { unsigned int swidth, rwidth = 0, @@ -1420,7 +1421,7 @@ print_aligned_vertical(const printTableContent *cont, if (width < rwidth) width = rwidth; - if (output_columns > 0) + if (cont->opt->format == PRINT_WRAPPED && output_columns > 0) { unsigned int min_width; diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out index 1cd4fea70c..d92b671b71 100644 --- a/src/test/regress/expected/psql.out +++ b/src/test/regress/expected/psql.out @@ -2674,6 +2674,32 @@ execute q; +------------------+-------------------+ deallocate q; +-- expanded output with short-width columns +\pset border 2 +\pset expanded on +create table psql_short_tab(a int, b int); +insert into psql_short_tab values(10,20),(30,40); +\pset format aligned +select * from psql_short_tab; ++-[ RECORD 1 ]-+ +| a | 10 | +| b | 20 | ++-[ RECORD 2 ]-+ +| a | 30 | +| b | 40 | ++---+----------+ + +\pset format wrapped +select * from psql_short_tab; ++-[ RECORD 1 ]-+ +| a | 10 | +| b | 20 | ++-[ RECORD 2 ]-+ +| a | 30 | +| b | 40 | ++---+----------+ + +drop table psql_short_tab; \pset linestyle ascii \pset border 1 -- support table for output-format tests (useful to create a footer) diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql index c24438e02d..0cc96aacda 100644 --- a/src/test/regress/sql/psql.sql +++ b/src/test/regress/sql/psql.sql @@ -416,6 +416,17 @@ execute q; deallocate q; +-- expanded output with short-width columns +\pset border 2 +\pset expanded on +create table psql_short_tab(a int, b int); +insert into psql_short_tab values(10,20),(30,40); +\pset format aligned +select * from psql_short_tab; +\pset format wrapped +select * from psql_short_tab; +drop table psql_short_tab; + \pset linestyle ascii \pset border 1 From 9e8fd9f7ab56965dfe012d6be1df152ea8f62aec Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Mon, 8 Jun 2026 17:11:12 +0900 Subject: [PATCH 27/76] ecpg: Reject multiple header items in GET/SET DESCRIPTOR Previously, ecpg accepted multiple descriptor header items in GET DESCRIPTOR and SET DESCRIPTOR, but generated broken C code when they were used. Although the grammar allowed this syntax, the implementation did not actually support it. This commit tightens the ecpg grammar so the header form of GET/SET DESCRIPTOR accepts only a single header item, matching the implementation and preventing generation of broken C code. Also update the documentation synopsis accordingly. Backpatch to all supported versions. Author: Masashi Kamura Reviewed-by: Hayato Kuroda Reviewed-by: Lakshmi G Reviewed-by: Fujii Masao Discussion: https://postgr.es/m/OS9PR01MB13174AD7D1829D0644B6BB90E9447A@OS9PR01MB13174.jpnprd01.prod.outlook.com Backpatch-through: 14 --- doc/src/sgml/ecpg.sgml | 6 +++--- src/interfaces/ecpg/preproc/ecpg.trailer | 12 ++---------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/doc/src/sgml/ecpg.sgml b/doc/src/sgml/ecpg.sgml index 6f9d44cbd7..d84025db6a 100644 --- a/doc/src/sgml/ecpg.sgml +++ b/doc/src/sgml/ecpg.sgml @@ -7276,7 +7276,7 @@ EXEC SQL EXECUTE IMMEDIATE :command; -GET DESCRIPTOR descriptor_name :cvariable = descriptor_header_item [, ... ] +GET DESCRIPTOR descriptor_name :cvariable = descriptor_header_item GET DESCRIPTOR descriptor_name VALUE column_number :cvariable = descriptor_item [, ... ] @@ -7295,7 +7295,7 @@ GET DESCRIPTOR descriptor_name VALU This command has two forms: The first form retrieves - descriptor header items, which apply to the result + descriptor header item, which applies to the result set in its entirety. One example is the row count. The second form, which requires the column number as additional parameter, retrieves information about a particular column. Examples are @@ -7771,7 +7771,7 @@ EXEC SQL SET CONNECTION = con1; -SET DESCRIPTOR descriptor_name descriptor_header_item = value [, ... ] +SET DESCRIPTOR descriptor_name descriptor_header_item = value SET DESCRIPTOR descriptor_name VALUE number descriptor_item = value [, ...] diff --git a/src/interfaces/ecpg/preproc/ecpg.trailer b/src/interfaces/ecpg/preproc/ecpg.trailer index b65e787611..1b6504fbbd 100644 --- a/src/interfaces/ecpg/preproc/ecpg.trailer +++ b/src/interfaces/ecpg/preproc/ecpg.trailer @@ -1178,27 +1178,19 @@ ECPGDeallocateDescr: DEALLOCATE SQL_DESCRIPTOR quoted_ident_stringvar * manipulate a descriptor header */ -ECPGGetDescriptorHeader: SQL_GET SQL_DESCRIPTOR quoted_ident_stringvar ECPGGetDescHeaderItems +ECPGGetDescriptorHeader: SQL_GET SQL_DESCRIPTOR quoted_ident_stringvar ECPGGetDescHeaderItem { $$ = $3; } ; -ECPGGetDescHeaderItems: ECPGGetDescHeaderItem - | ECPGGetDescHeaderItems ',' ECPGGetDescHeaderItem - ; - ECPGGetDescHeaderItem: cvariable '=' desc_header_item { push_assignment($1, $3); } ; -ECPGSetDescriptorHeader: SET SQL_DESCRIPTOR quoted_ident_stringvar ECPGSetDescHeaderItems +ECPGSetDescriptorHeader: SET SQL_DESCRIPTOR quoted_ident_stringvar ECPGSetDescHeaderItem { $$ = $3; } ; -ECPGSetDescHeaderItems: ECPGSetDescHeaderItem - | ECPGSetDescHeaderItems ',' ECPGSetDescHeaderItem - ; - ECPGSetDescHeaderItem: desc_header_item '=' IntConstVar { push_assignment($3, $1); From 673161b63e245674a1bebf5583264ba8ca283946 Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Mon, 8 Jun 2026 10:33:52 -0500 Subject: [PATCH 28/76] doc: Expand on proper use of refint. The security team has received a couple of reports about potential SQL injection via refint's trigger arguments. We discussed this while preparing CVE-2026-6637 and concluded that forcibly quoting these arguments is more likely to break working code than to prevent exploits. Unlike data values, the table/column names come from trigger arguments, and there is little reason for a trigger author to put hostile inputs into those arguments. So, let's document it accordingly. Reported-by: Nikolay Samokhvalov Reported-by: Alex Young Reported-by: Satyanarayana Narlapuram Suggested-by: Noah Misch Reviewed-by: Noah Misch Reviewed-by: Fujii Masao Reviewed-by: Christoph Berg Reviewed-by: Satyanarayana Narlapuram Discussion: https://postgr.es/m/ahXP7z7nsfGPOZ3T%40nathan Backpatch-through: 14 --- doc/src/sgml/contrib-spi.sgml | 58 ++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/contrib-spi.sgml b/doc/src/sgml/contrib-spi.sgml index fed6f24932..cad6d4f289 100644 --- a/doc/src/sgml/contrib-spi.sgml +++ b/doc/src/sgml/contrib-spi.sgml @@ -34,6 +34,14 @@ key mechanism, of course, but the module is still useful as an example.) + + + refint requires a + secure schema usage pattern and + data types where the equality operator is named =. + + + check_primary_key() checks the referencing table. To use, create a BEFORE INSERT OR UPDATE trigger using this @@ -44,6 +52,29 @@ keys, create a trigger for each reference. + + + The referenced table name and column name arguments to + check_primary_key() are copied as-is into internally + generated SQL statements and therefore must be double-quoted by the user as + necessary in the CREATE TRIGGER command. See + for more information about quoting + SQL identifiers. Conversely, the referencing table + column name arguments should not be double quoted. See the following mock + example of proper use of check_primary_key(): + +CREATE TRIGGER mytrigger +BEFORE INSERT OR UPDATE ON referencing_table +FOR EACH ROW EXECUTE PROCEDURE +check_primary_key ( + 'column A', 'column B', -- referencing table columns + 'myschema."referenced table"', -- referenced table + '"column A"', '"column B"' -- referenced table columns +); + + + + check_foreign_key() checks the referenced table. To use, create a BEFORE DELETE OR UPDATE trigger using this @@ -53,13 +84,38 @@ (cascade — to delete the referencing row, restrict — to abort transaction if referencing keys exist, setnull — to set referencing key fields to null), - the triggered table's column names which form the primary/unique key, then + the referenced table's column names which form the primary/unique key, then the referencing table name and column names (repeated for as many referencing tables as were specified by first argument). Note that the primary/unique key columns should be marked NOT NULL and should have a unique index. + + + The referencing table name and column name arguments + to check_foreign_key() are copied as-is into + internally generated SQL statements and therefore must be double-quoted by + the user as necessary in the CREATE TRIGGER command. + See for more information about + quoting SQL identifiers. Conversely, the referenced + table column name arguments should not be double quoted. See the following + mock example of proper use of check_foreign_key(): + +CREATE TRIGGER mytrigger +BEFORE DELETE OR UPDATE ON referenced_table +FOR EACH ROW EXECUTE PROCEDURE +check_foreign_key ( + 1, -- number of referencing tables + 'cascade', -- action + 'column A', 'column B', -- referenced table columns + 'myschema."referencing table"', -- referencing table + '"column A"', '"column B"' -- referencing table columns +); + + + + There are examples in refint.example. From 64778fac724ac4e70473937f2d94d397af5a2785 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Mon, 8 Jun 2026 11:48:07 -0400 Subject: [PATCH 29/76] Fix missed checks for hashability of container-type equality. The operators for array_eq, record_eq, range_eq, and multirange_eq are all marked oprcanhash, but there's a pitfall: their hash functions can fail at runtime if the contained type(s) are not hashable. Therefore, the planner has to check hashability of the contained types before deciding it can use hashing in these cases. Not every place had gotten this memo, and noplace at all had considered the issue for ranges or multiranges. In particular we could attempt to use hashing for a ScalarArrayOpExpr on a container type when it won't actually work, leading to "could not identify a hash function ..." runtime failures. For the most part we should fix this in the lookup functions provided by lsyscache.c, to wit get_op_hash_functions and op_hashjoinable. But there's a problem: get_op_hash_functions is not passed the input data type it would need to check. We mustn't change the API of that exported function in a back-patched fix, and even if we wanted to, its call sites in the executor mostly don't have easy access to the required data type OID. Fortunately, the executor call sites don't actually need fixing, because it's expected that the planner verified hashability before building a plan that requires it. Therefore, leave get_op_hash_functions as-is and invent a wrapper function get_op_hash_functions_ext that does the additional checking needed in the planner's uses. We also need to fix hash_ok_operator (extending the fix in 647889667). While at it, neaten up a couple of places in lookup_type_cache where relevant code for multirange cases was written differently from the code for other container types. Note: while this touches pg_operator.dat, it's only to add oid_symbol macros. So there's no on-disk data change and no need for a catversion bump. Reported-by: Andrei Lepikhov Author: Andrei Lepikhov Co-authored-by: Tom Lane Discussion: https://postgr.es/m/ed221f95-f09b-4a9c-b05b-e1fed621ec87@gmail.com Backpatch-through: 14 --- src/backend/optimizer/plan/subselect.c | 4 +- src/backend/optimizer/util/clauses.c | 6 +- src/backend/utils/cache/lsyscache.c | 85 ++++++++++++++++++++++++-- src/backend/utils/cache/typcache.c | 25 +++----- src/include/catalog/pg_operator.dat | 4 +- src/include/utils/lsyscache.h | 2 + 6 files changed, 100 insertions(+), 26 deletions(-) diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index 0912c2aa15..9b97154743 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -850,7 +850,9 @@ hash_ok_operator(OpExpr *expr) if (list_length(expr->args) != 2) return false; if (opid == ARRAY_EQ_OP || - opid == RECORD_EQ_OP) + opid == RECORD_EQ_OP || + opid == RANGE_EQ_OP || + opid == MULTIRANGE_EQ_OP) { /* these are strict, but must check input type to ensure hashable */ Node *leftarg = linitial(expr->args); diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index 02676ce404..8b48e8830c 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -2204,13 +2204,15 @@ convert_saop_to_hashed_saop_walker(Node *node, void *context) if (IsA(node, ScalarArrayOpExpr)) { ScalarArrayOpExpr *saop = (ScalarArrayOpExpr *) node; - Expr *arrayarg = (Expr *) lsecond(saop->args); + Node *leftarg = (Node *) linitial(saop->args); + Node *arrayarg = (Node *) lsecond(saop->args); Oid lefthashfunc; Oid righthashfunc; if (saop->useOr && arrayarg && IsA(arrayarg, Const) && !((Const *) arrayarg)->constisnull && - get_op_hash_functions(saop->opno, &lefthashfunc, &righthashfunc) && + get_op_hash_functions_ext(saop->opno, exprType(leftarg), + &lefthashfunc, &righthashfunc) && lefthashfunc == righthashfunc) { Datum arrdatum = ((Const *) arrayarg)->constvalue; diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index 83d3850728..3802f31526 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -403,6 +403,12 @@ get_mergejoin_opfamilies(Oid opno) * * Returns true if able to find the requested operator(s), false if not. * (This indicates that the operator should not have been marked oprcanhash.) + * + * Callers must beware that for container types (arrays, records, ranges) + * this function will succeed for array_eq etc, but the hash function could + * fail at runtime if the contained type(s) are not hashable. If it is + * possible that the operator is one of these, precheck with op_hashjoinable + * or get_op_hash_functions_ext. */ bool get_compatible_hash_operators(Oid opno, @@ -503,6 +509,12 @@ get_compatible_hash_operators(Oid opno, * * Returns true if able to find the requested function(s), false if not. * (This indicates that the operator should not have been marked oprcanhash.) + * + * Callers must beware that for container types (arrays, records, ranges) + * this function will succeed for array_eq etc, but the hash function could + * fail at runtime if the contained type(s) are not hashable. If it is + * possible that the operator is one of these, use get_op_hash_functions_ext + * or precheck with op_hashjoinable. */ bool get_op_hash_functions(Oid opno, @@ -585,6 +597,55 @@ get_op_hash_functions(Oid opno, return result; } +/* + * get_op_hash_functions_ext + * As above, but verify hashability in container-type cases. + * + * As with op_hashjoinable, assume the left input type is sufficient + * to disambiguate container-type cases. + */ +bool +get_op_hash_functions_ext(Oid opno, Oid inputtype, + RegProcedure *lhs_procno, RegProcedure *rhs_procno) +{ + TypeCacheEntry *typentry; + + /* Ensure output args are initialized on failure */ + if (lhs_procno) + *lhs_procno = InvalidOid; + if (rhs_procno) + *rhs_procno = InvalidOid; + + /* As in op_hashjoinable, let the typcache handle the hard cases */ + if (opno == ARRAY_EQ_OP) + { + typentry = lookup_type_cache(inputtype, TYPECACHE_HASH_PROC); + if (typentry->hash_proc != F_HASH_ARRAY) + return false; + } + else if (opno == RECORD_EQ_OP) + { + typentry = lookup_type_cache(inputtype, TYPECACHE_HASH_PROC); + if (typentry->hash_proc != F_HASH_RECORD) + return false; + } + else if (opno == RANGE_EQ_OP) + { + typentry = lookup_type_cache(inputtype, TYPECACHE_HASH_PROC); + if (typentry->hash_proc != F_HASH_RANGE) + return false; + } + else if (opno == MULTIRANGE_EQ_OP) + { + typentry = lookup_type_cache(inputtype, TYPECACHE_HASH_PROC); + if (typentry->hash_proc != F_HASH_MULTIRANGE) + return false; + } + + /* OK, do the normal lookup */ + return get_op_hash_functions(opno, lhs_procno, rhs_procno); +} + /* * get_op_btree_interpretation * Given an operator's OID, find out which btree opfamilies it belongs to, @@ -1412,7 +1473,8 @@ op_mergejoinable(Oid opno, Oid inputtype) * For array_eq or record_eq, we can sort if the element or field types * are all sortable. We could implement all the checks for that here, but * the typcache already does that and caches the results too, so let's - * rely on the typcache. + * rely on the typcache. We do not need similar special cases for ranges + * or multiranges, because their subtypes are required to be sortable. */ if (opno == ARRAY_EQ_OP) { @@ -1447,10 +1509,11 @@ op_mergejoinable(Oid opno, Oid inputtype) * Returns true if the operator is hashjoinable. (There must be a suitable * hash opfamily entry for this operator if it is so marked.) * - * In some cases (currently only array_eq), hashjoinability depends on the - * specific input data type the operator is invoked for, so that must be - * passed as well. We currently assume that only one input's type is needed - * to check this --- by convention, pass the left input's data type. + * In some cases (currently array_eq, record_eq, range_eq, multirange_eq), + * hashjoinability depends on the specific input data type the operator is + * invoked for, so that must be passed as well. We currently assume that only + * one input's type is needed to check this --- by convention, pass the left + * input's data type. */ bool op_hashjoinable(Oid opno, Oid inputtype) @@ -1472,6 +1535,18 @@ op_hashjoinable(Oid opno, Oid inputtype) if (typentry->hash_proc == F_HASH_RECORD) result = true; } + else if (opno == RANGE_EQ_OP) + { + typentry = lookup_type_cache(inputtype, TYPECACHE_HASH_PROC); + if (typentry->hash_proc == F_HASH_RANGE) + result = true; + } + else if (opno == MULTIRANGE_EQ_OP) + { + typentry = lookup_type_cache(inputtype, TYPECACHE_HASH_PROC); + if (typentry->hash_proc == F_HASH_MULTIRANGE) + result = true; + } else { /* For all other operators, rely on pg_operator.oprcanhash */ diff --git a/src/backend/utils/cache/typcache.c b/src/backend/utils/cache/typcache.c index 27f92b7283..2b5446594e 100644 --- a/src/backend/utils/cache/typcache.c +++ b/src/backend/utils/cache/typcache.c @@ -691,8 +691,9 @@ lookup_type_cache(Oid type_id, int flags) HASHSTANDARD_PROC); /* - * As above, make sure hash_array, hash_record, or hash_range will - * succeed. + * As above, make sure hash_array, hash_record, hash_range, or + * hash_multirange will succeed. Here we do need to check the range + * cases. */ if (hash_proc == F_HASH_ARRAY && !array_element_has_hashing(typentry)) @@ -703,12 +704,8 @@ lookup_type_cache(Oid type_id, int flags) else if (hash_proc == F_HASH_RANGE && !range_element_has_hashing(typentry)) hash_proc = InvalidOid; - - /* - * Likewise for hash_multirange. - */ - if (hash_proc == F_HASH_MULTIRANGE && - !multirange_element_has_hashing(typentry)) + else if (hash_proc == F_HASH_MULTIRANGE && + !multirange_element_has_hashing(typentry)) hash_proc = InvalidOid; /* Force update of hash_proc_finfo only if we're changing state */ @@ -740,8 +737,8 @@ lookup_type_cache(Oid type_id, int flags) HASHEXTENDED_PROC); /* - * As above, make sure hash_array_extended, hash_record_extended, or - * hash_range_extended will succeed. + * As above, make sure hash_array_extended, hash_record_extended, + * hash_range_extended, or hash_multirange_extended will succeed. */ if (hash_extended_proc == F_HASH_ARRAY_EXTENDED && !array_element_has_extended_hashing(typentry)) @@ -752,12 +749,8 @@ lookup_type_cache(Oid type_id, int flags) else if (hash_extended_proc == F_HASH_RANGE_EXTENDED && !range_element_has_extended_hashing(typentry)) hash_extended_proc = InvalidOid; - - /* - * Likewise for hash_multirange_extended. - */ - if (hash_extended_proc == F_HASH_MULTIRANGE_EXTENDED && - !multirange_element_has_extended_hashing(typentry)) + else if (hash_extended_proc == F_HASH_MULTIRANGE_EXTENDED && + !multirange_element_has_extended_hashing(typentry)) hash_extended_proc = InvalidOid; /* Force update of proc finfo only if we're changing state */ diff --git a/src/include/catalog/pg_operator.dat b/src/include/catalog/pg_operator.dat index 89c73acd68..57e9a6ad71 100644 --- a/src/include/catalog/pg_operator.dat +++ b/src/include/catalog/pg_operator.dat @@ -3074,7 +3074,7 @@ oprrest => 'scalargesel', oprjoin => 'scalargejoinsel' }, # generic range type operators -{ oid => '3882', descr => 'equal', +{ oid => '3882', oid_symbol => 'RANGE_EQ_OP', descr => 'equal', oprname => '=', oprcanmerge => 't', oprcanhash => 't', oprleft => 'anyrange', oprright => 'anyrange', oprresult => 'bool', oprcom => '=(anyrange,anyrange)', oprnegate => '<>(anyrange,anyrange)', oprcode => 'range_eq', @@ -3279,7 +3279,7 @@ oprname => '@@', oprleft => 'jsonb', oprright => 'jsonpath', oprresult => 'bool', oprcode => 'jsonb_path_match_opr(jsonb,jsonpath)', oprrest => 'matchingsel', oprjoin => 'matchingjoinsel' }, -{ oid => '2860', descr => 'equal', +{ oid => '2860', oid_symbol => 'MULTIRANGE_EQ_OP', descr => 'equal', oprname => '=', oprcanmerge => 't', oprcanhash => 't', oprleft => 'anymultirange', oprright => 'anymultirange', oprresult => 'bool', oprcom => '=(anymultirange,anymultirange)', diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index ac1ddb8130..9c3d26aec9 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -83,6 +83,8 @@ extern bool get_compatible_hash_operators(Oid opno, Oid *lhs_opno, Oid *rhs_opno); extern bool get_op_hash_functions(Oid opno, RegProcedure *lhs_procno, RegProcedure *rhs_procno); +extern bool get_op_hash_functions_ext(Oid opno, Oid inputtype, + RegProcedure *lhs_procno, RegProcedure *rhs_procno); extern List *get_op_btree_interpretation(Oid opno); extern bool equality_ops_are_compatible(Oid opno1, Oid opno2); extern bool comparison_ops_are_compatible(Oid opno1, Oid opno2); From 1e04581729421944e18f4a4a10f6d7a8a8e95169 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Mon, 8 Jun 2026 11:47:40 -0700 Subject: [PATCH 30/76] dict_synonym.c: remove incorrect outlen. Previously, outlen was miscalculated if case_sensitive was false and str_tolower() changed the byte length of the string. If outlen was too large, pnstrdup() would stop at the NUL terminator, preventing overrun. But if outlen was too small, it would cause truncation. Fix by just removing outlen. It was only used in a single site, which could just as well use pstrdup(). Discussion: https://postgre.es/m/1101e1a3afbbabb503317069c40374b82e6f4cac.camel@j-davis.com Reviewed-by: Tristan Partin Backpatch-through: 14 --- src/backend/tsearch/dict_synonym.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/backend/tsearch/dict_synonym.c b/src/backend/tsearch/dict_synonym.c index 8c99ecaa0a..853f07bfd9 100644 --- a/src/backend/tsearch/dict_synonym.c +++ b/src/backend/tsearch/dict_synonym.c @@ -22,7 +22,6 @@ typedef struct { char *in; char *out; - int outlen; uint16 flags; } Syn; @@ -187,7 +186,6 @@ dsynonym_init(PG_FUNCTION_ARGS) d->syn[cur].out = lowerstr(starto); } - d->syn[cur].outlen = strlen(starto); d->syn[cur].flags = flags; cur++; @@ -234,7 +232,7 @@ dsynonym_lexize(PG_FUNCTION_ARGS) PG_RETURN_POINTER(NULL); res = palloc0(sizeof(TSLexeme) * 2); - res[0].lexeme = pnstrdup(found->out, found->outlen); + res[0].lexeme = pstrdup(found->out); res[0].flags = found->flags; PG_RETURN_POINTER(res); From f3f901a53a17da5c4f21c1df628d187b6aff1889 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Thu, 11 Jun 2026 14:29:29 +0900 Subject: [PATCH 31/76] xml2: Fix crash with namespace nodes in xpath_nodeset() pgxmlNodeSetToText() passed nodeTab[i]->doc to xmlNodeDump() without checking the node type, which could cause a crash as a XML_NAMESPACE_DECL maps to a xmlNs struct. The passed-in code would then be dereferenced in xmlNodeDump(). This commit switches the code to render XML_NAMESPACE_DECL nodes with xmlXPathCastNodeToString(), like xpath_table(). Some tests are added, written by me. Author: Andrey Chernyy Co-authored-by: Michael Paquier Discussion: https://postgr.es/m/20260611031436.5afde3cb@andrnote Backpatch-through: 14 --- contrib/xml2/expected/xml2.out | 8 ++++++++ contrib/xml2/expected/xml2_1.out | 8 ++++++++ contrib/xml2/sql/xml2.sql | 4 ++++ contrib/xml2/xpath.c | 15 +++++++++++---- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/contrib/xml2/expected/xml2.out b/contrib/xml2/expected/xml2.out index 3027e4df86..b802599dec 100644 --- a/contrib/xml2/expected/xml2.out +++ b/contrib/xml2/expected/xml2.out @@ -207,6 +207,14 @@ SELECT xslt_process('cim30400', + '//namespace::foo'); + xpath_nodeset +---------------------- + http://icl.com/saxon +(1 row) + -- possible security exploit SELECT xslt_process('Hello from XML', $$cim30400 $$::text, 'n1="v1",n2="v2",n3="v3",n4="v4",n5="v5",n6="v6",n7="v7",n8="v8",n9="v9",n10="v10",n11="v11",n12="v12"'::text); ERROR: xslt_process() is not available without libxslt +-- xpath_nodeset() with namespace node +SELECT xpath_nodeset('', + '//namespace::foo'); + xpath_nodeset +---------------------- + http://icl.com/saxon +(1 row) + -- possible security exploit SELECT xslt_process('Hello from XML', $$cim30400 $$::text, 'n1="v1",n2="v2",n3="v3",n4="v4",n5="v5",n6="v6",n7="v7",n8="v8",n9="v9",n10="v10",n11="v11",n12="v12"'::text); +-- xpath_nodeset() with namespace node +SELECT xpath_nodeset('', + '//namespace::foo'); + -- possible security exploit SELECT xslt_process('Hello from XML', $$nodeTab[i]; + if ((septagname != NULL) && (xmlStrlen(septagname) > 0)) { xmlBufferWriteChar(buf, "<"); xmlBufferWriteCHAR(buf, septagname); xmlBufferWriteChar(buf, ">"); } - xmlNodeDump(buf, - nodeset->nodeTab[i]->doc, - nodeset->nodeTab[i], - 1, 0); + + /* + * XML_NAMESPACE_DECL nodes are xmlNs structs, that cannot + * be processed by xmlNodeDump(). + */ + if (node->type == XML_NAMESPACE_DECL) + xmlBufferWriteCHAR(buf, xmlXPathCastNodeToString(node)); + else + xmlNodeDump(buf, node->doc, node, 1, 0); if ((septagname != NULL) && (xmlStrlen(septagname) > 0)) { From 58b91fc73a8679a78e7dc1430169a555ee5c8a81 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 11 Jun 2026 12:33:48 +0300 Subject: [PATCH 32/76] seg: Fix seg_out() to preserve the upper boundary's certainty indicator When printing the upper boundary of a seg interval, seg_out() decided whether to emit the certainty indicator ('<', '>' or '~') by testing the upper indicator (u_ext) for '<' and '>', but mistakenly tested the lower indicator (l_ext) for '~'. This is a copy-and-paste slip from the symmetric code that prints the lower boundary a few lines above. The consequences for valid input were: * A '~' on the upper boundary was dropped on output, e.g. '1.5 .. ~2.5'::seg printed as '1.5 .. 2.5'. * When the lower boundary carried '~' but the upper boundary had no indicator, the wrong test matched and sprintf(p, "%c", seg->u_ext) wrote a NUL byte (u_ext == '\0'), which truncated the result string and silently lost the entire upper boundary, e.g. '~6.5 .. 8.5'::seg printed as '~6.5 .. '. Certainty indicators are documented to be preserved on output (they are ignored by the operators, but kept as comments), so this broke the input/output round-trip for the affected values. The bug has existed since seg was added. It went unnoticed because the existing regression tests only exercised certainty indicators on single-point segs, which are printed by a different branch of seg_out(). Add tests that place indicators on both boundaries of an interval. Author: Ewan Young Discussion: https://www.postgresql.org/message-id/CAON2xHPYeRRCEVAv8XfE18KsEsEHCiYcJ5fOsoxFuMEfpxF1=g@mail.gmail.com Backpatch-through: 14 --- contrib/seg/expected/seg.out | 45 +++++++++++++++++++++++++++++++++++- contrib/seg/seg.c | 2 +- contrib/seg/sql/seg.sql | 11 ++++++++- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/contrib/seg/expected/seg.out b/contrib/seg/expected/seg.out index 2320464dd4..9dbcf35e07 100644 --- a/contrib/seg/expected/seg.out +++ b/contrib/seg/expected/seg.out @@ -263,7 +263,8 @@ SELECT '12.345678901234560000000000000000000000000000000000000000000000000000000 12.3457 (1 row) --- Numbers with certainty indicators +-- Numbers and ranges with certainty indicators. Certainty indicators +-- are stored and preserved on output, but ignored by operators. SELECT '~6.5'::seg AS seg; seg ------ @@ -300,6 +301,48 @@ SELECT '> 6.5'::seg AS seg; >6.5 (1 row) +SELECT '~1.5 .. 2.5'::seg AS seg; + seg +------------- + ~1.5 .. 2.5 +(1 row) + +SELECT '1.5 .. ~2.5'::seg AS seg; + seg +------------- + 1.5 .. ~2.5 +(1 row) + +SELECT '~1.5 .. ~2.5'::seg AS seg; + seg +-------------- + ~1.5 .. ~2.5 +(1 row) + +SELECT '<1.5 .. 2.5'::seg AS seg; + seg +------------- + <1.5 .. 2.5 +(1 row) + +SELECT '1.5 .. <2.5'::seg AS seg; + seg +------------- + 1.5 .. <2.5 +(1 row) + +SELECT '>1.5 .. 2.5'::seg AS seg; + seg +------------- + >1.5 .. 2.5 +(1 row) + +SELECT '1.5 .. >2.5'::seg AS seg; + seg +------------- + 1.5 .. >2.5 +(1 row) + -- Open intervals SELECT '0..'::seg AS seg; seg diff --git a/contrib/seg/seg.c b/contrib/seg/seg.c index 91b8a79600..1f665051ab 100644 --- a/contrib/seg/seg.c +++ b/contrib/seg/seg.c @@ -147,7 +147,7 @@ seg_out(PG_FUNCTION_ARGS) { /* print the upper boundary if exists */ p += sprintf(p, " "); - if (seg->u_ext == '>' || seg->u_ext == '<' || seg->l_ext == '~') + if (seg->u_ext == '>' || seg->u_ext == '<' || seg->u_ext == '~') p += sprintf(p, "%c", seg->u_ext); p += restore(p, seg->upper, seg->u_sigd); } diff --git a/contrib/seg/sql/seg.sql b/contrib/seg/sql/seg.sql index a027d4de97..0081854c01 100644 --- a/contrib/seg/sql/seg.sql +++ b/contrib/seg/sql/seg.sql @@ -63,7 +63,8 @@ SELECT '12.34567890123456'::seg AS seg; -- Same, with a very long input SELECT '12.3456789012345600000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'::seg AS seg; --- Numbers with certainty indicators +-- Numbers and ranges with certainty indicators. Certainty indicators +-- are stored and preserved on output, but ignored by operators. SELECT '~6.5'::seg AS seg; SELECT '<6.5'::seg AS seg; SELECT '>6.5'::seg AS seg; @@ -71,6 +72,14 @@ SELECT '~ 6.5'::seg AS seg; SELECT '< 6.5'::seg AS seg; SELECT '> 6.5'::seg AS seg; +SELECT '~1.5 .. 2.5'::seg AS seg; +SELECT '1.5 .. ~2.5'::seg AS seg; +SELECT '~1.5 .. ~2.5'::seg AS seg; +SELECT '<1.5 .. 2.5'::seg AS seg; +SELECT '1.5 .. <2.5'::seg AS seg; +SELECT '>1.5 .. 2.5'::seg AS seg; +SELECT '1.5 .. >2.5'::seg AS seg; + -- Open intervals SELECT '0..'::seg AS seg; SELECT '0...'::seg AS seg; From 41876c8d77834021d9b1f2b4f62c27c886f47c3b Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 12 Jun 2026 10:25:59 +0900 Subject: [PATCH 33/76] Fix handling of namespace nodes in xpath() (xml) xpath() attempted to call xmlCopyNode() and xmlNodeDump() on a XML_NAMESPACE_DECL, finishing with a confusing error: =# SELECT xpath('//namespace::foo', ''); ERROR: 53200: could not copy node CONTEXT: SQL function "xpath" statement 1 xpath() is changed so as it goes through xmlXPathCastNodeToString() instead, that is able to handle namespace nodes. xml2 uses the same solution. This issue has been discovered while digging into 9d33a5a804db. Author: Michael Paquier Discussion: https://postgr.es/m/aioT7ui_ZJ9RMlfM@paquier.xyz Backpatch-through: 14 --- src/backend/utils/adt/xml.c | 4 +++- src/test/regress/expected/xml.out | 6 ++++++ src/test/regress/expected/xml_1.out | 6 ++++++ src/test/regress/sql/xml.sql | 1 + 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/adt/xml.c b/src/backend/utils/adt/xml.c index 98dcc04122..9b5e2e17b1 100644 --- a/src/backend/utils/adt/xml.c +++ b/src/backend/utils/adt/xml.c @@ -3855,7 +3855,9 @@ xml_xmlnodetoxmltype(xmlNodePtr cur, PgXmlErrorContext *xmlerrcxt) { xmltype *result = NULL; - if (cur->type != XML_ATTRIBUTE_NODE && cur->type != XML_TEXT_NODE) + if (cur->type != XML_ATTRIBUTE_NODE && + cur->type != XML_TEXT_NODE && + cur->type != XML_NAMESPACE_DECL) { void (*volatile nodefree) (xmlNodePtr) = NULL; volatile xmlBufferPtr buf = NULL; diff --git a/src/test/regress/expected/xml.out b/src/test/regress/expected/xml.out index f9b7ec0bab..5924deff66 100644 --- a/src/test/regress/expected/xml.out +++ b/src/test/regress/expected/xml.out @@ -710,6 +710,12 @@ SELECT xpath('root', ''); {} (1 row) +SELECT xpath('//namespace::foo', ''); + xpath +-------------------- + {http://127.0.0.1} +(1 row) + -- Round-trip non-ASCII data through xpath(). DO $$ DECLARE diff --git a/src/test/regress/expected/xml_1.out b/src/test/regress/expected/xml_1.out index fcf5d0f4aa..d22a6f8215 100644 --- a/src/test/regress/expected/xml_1.out +++ b/src/test/regress/expected/xml_1.out @@ -624,6 +624,12 @@ LINE 1: SELECT xpath('root', ''); ^ DETAIL: This functionality requires the server to be built with libxml support. HINT: You need to rebuild PostgreSQL using --with-libxml. +SELECT xpath('//namespace::foo', ''); +ERROR: unsupported XML feature +LINE 1: SELECT xpath('//namespace::foo', ''); -- Round-trip non-ASCII data through xpath(). DO $$ From e3a4e9edd5c0ed1d1ae1ec00fbc27cf8f8598515 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Fri, 12 Jun 2026 11:08:33 +0900 Subject: [PATCH 34/76] doc: fix reference for finding replication slots to drop Commit a70bce43fb added instructions on how to recover if PostgreSQL refuses to issue new transaction IDs because of imminent wraparound, but when describing how to find replication slots that should be dropped, it referred to pg_stat_replication where it should have referenced pg_replication_slots. In passing, decorate references to views with tags. Backpatch to all supported versions. Reported-By: Sanjaya Waruna Author: Laurenz Albe Reviewed-by: Robert Treat Reviewed-by: Fujii Masao Discussion: https://postgr.es/m/176767268098.1084085.10345048667224193115@wrigleys.postgresql.org Backpatch-through: 14 --- doc/src/sgml/maintenance.sgml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/maintenance.sgml b/doc/src/sgml/maintenance.sgml index 3aae2a53a1..1ab5bfd0ae 100644 --- a/doc/src/sgml/maintenance.sgml +++ b/doc/src/sgml/maintenance.sgml @@ -663,20 +663,20 @@ HINT: Stop the postmaster and vacuum that database in single-user mode. Resolve old prepared transactions. You can find these by checking - pg_prepared_xacts for rows where + pg_prepared_xacts for rows where age(transactionid) is large. Such transactions should be committed or rolled back. End long-running open transactions. You can find these by checking - pg_stat_activity for rows where + pg_stat_activity for rows where age(backend_xid) or age(backend_xmin) is large. Such transactions should be committed or rolled back, or the session can be terminated using pg_terminate_backend. Drop any old replication slots. Use - pg_stat_replication to + pg_replication_slots to find slots where age(xmin) or age(catalog_xmin) is large. In many cases, such slots were created for replication to servers that no longer exist, or that have been down for a long time. If you drop a slot for a server From a17f39aa2f5f5867f5e2afc87ec280caa1f65e62 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Fri, 12 Jun 2026 12:37:21 +0900 Subject: [PATCH 35/76] Update expected regression test output for xml_2.out This one has been forgotten in 8bf257aebac1. Per report from buildfarm member massasauga. Backpatch-through: 14 --- src/test/regress/expected/xml_2.out | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/regress/expected/xml_2.out b/src/test/regress/expected/xml_2.out index 044a4917d8..43c5fe9122 100644 --- a/src/test/regress/expected/xml_2.out +++ b/src/test/regress/expected/xml_2.out @@ -696,6 +696,12 @@ SELECT xpath('root', ''); {} (1 row) +SELECT xpath('//namespace::foo', ''); + xpath +-------------------- + {http://127.0.0.1} +(1 row) + -- Round-trip non-ASCII data through xpath(). DO $$ DECLARE From 086652c02f49ca04c3123faf14b2f5cabdf31f7d Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson Date: Fri, 12 Jun 2026 13:57:22 +0200 Subject: [PATCH 36/76] Fix compilation with OpenSSL 4 OpenSSL 4.0.0 changed some parameters and returnvalues to const, so we need to update our declarations and subsequently cast away const- ness from a few callsites to make libpq build without warnings. This is tested with OpenSSL 1.1.1 through 4.0.0 as well as with LibreSSL. No functional change is introduced, this commit only allows postgres to be compiled against OpenSSL 4.0.0 without warnings. There is also an errormessage change in OpenSSL 4.0.0 which needed to be covered by our testharness. This will be backpatched to all supported branches since they are all equally likely to be built against OpenSSL 4.0.0 as it becomes available in distributions. Backpatching will be done once it has been in master for a few days without issues. Author: Daniel Gustafsson Reviewed-by: Michael Paquier Discussion: https://postgr.es/m/066B07BB-85FA-487C-BE8C-40F791CFC3C4@yesql.se Backpatch-through: 14 --- contrib/sslinfo/sslinfo.c | 24 +++++++++++----------- src/backend/libpq/be-secure-openssl.c | 26 ++++++++++++------------ src/interfaces/libpq/fe-secure-openssl.c | 13 ++++++------ src/test/ssl/t/001_ssltests.pl | 4 ++-- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/contrib/sslinfo/sslinfo.c b/contrib/sslinfo/sslinfo.c index 30cae0bb98..4f8a118bc9 100644 --- a/contrib/sslinfo/sslinfo.c +++ b/contrib/sslinfo/sslinfo.c @@ -21,8 +21,8 @@ PG_MODULE_MAGIC; -static Datum X509_NAME_field_to_text(X509_NAME *name, text *fieldName); -static Datum ASN1_STRING_to_text(ASN1_STRING *str); +static Datum X509_NAME_field_to_text(const X509_NAME *name, text *fieldName); +static Datum ASN1_STRING_to_text(const ASN1_STRING *str); /* * Function context for data persisting over repeated calls. @@ -145,7 +145,7 @@ ssl_client_serial(PG_FUNCTION_ARGS) * function. */ static Datum -ASN1_STRING_to_text(ASN1_STRING *str) +ASN1_STRING_to_text(const ASN1_STRING *str) { BIO *membuf; size_t size; @@ -160,7 +160,7 @@ ASN1_STRING_to_text(ASN1_STRING *str) (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("could not create OpenSSL BIO structure"))); (void) BIO_set_close(membuf, BIO_CLOSE); - ASN1_STRING_print_ex(membuf, str, + ASN1_STRING_print_ex(membuf, unconstify(ASN1_STRING *, str), ((ASN1_STRFLGS_RFC2253 & ~ASN1_STRFLGS_ESC_MSB) | ASN1_STRFLGS_UTF8_CONVERT)); /* ensure null termination of the BIO's content */ @@ -191,12 +191,12 @@ ASN1_STRING_to_text(ASN1_STRING *str) * part of name */ static Datum -X509_NAME_field_to_text(X509_NAME *name, text *fieldName) +X509_NAME_field_to_text(const X509_NAME *name, text *fieldName) { char *string_fieldname; int nid, index; - ASN1_STRING *data; + const ASN1_STRING *data; string_fieldname = text_to_cstring(fieldName); nid = OBJ_txt2nid(string_fieldname); @@ -206,10 +206,10 @@ X509_NAME_field_to_text(X509_NAME *name, text *fieldName) errmsg("invalid X.509 field name: \"%s\"", string_fieldname))); pfree(string_fieldname); - index = X509_NAME_get_index_by_NID(name, nid, -1); + index = X509_NAME_get_index_by_NID(unconstify(X509_NAME *, name), nid, -1); if (index < 0) return (Datum) 0; - data = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(name, index)); + data = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(unconstify(X509_NAME *, name), index)); return ASN1_STRING_to_text(data); } @@ -418,8 +418,8 @@ ssl_extension_info(PG_FUNCTION_ARGS) HeapTuple tuple; Datum result; BIO *membuf; - X509_EXTENSION *ext; - ASN1_OBJECT *obj; + const X509_EXTENSION *ext; + const ASN1_OBJECT *obj; int nid; int len; @@ -432,7 +432,7 @@ ssl_extension_info(PG_FUNCTION_ARGS) /* Get the extension from the certificate */ ext = X509_get_ext(cert, call_cntr); - obj = X509_EXTENSION_get_object(ext); + obj = X509_EXTENSION_get_object(unconstify(X509_EXTENSION *, ext)); /* Get the extension name */ nid = OBJ_obj2nid(obj); @@ -445,7 +445,7 @@ ssl_extension_info(PG_FUNCTION_ARGS) nulls[0] = false; /* Get the extension value */ - if (X509V3_EXT_print(membuf, ext, 0, 0) <= 0) + if (X509V3_EXT_print(membuf, unconstify(X509_EXTENSION *, ext), 0, 0) <= 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("could not print extension value in certificate at position %d", diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index 8df8ed3c90..8fa4963ced 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -64,7 +64,7 @@ static bool initialize_dh(SSL_CTX *context, bool isServerStart); static bool initialize_ecdh(SSL_CTX *context, bool isServerStart); static const char *SSLerrmessage(unsigned long ecode); -static char *X509_NAME_to_cstring(X509_NAME *name); +static char *X509_NAME_to_cstring(const X509_NAME *name); static SSL_CTX *SSL_context = NULL; static bool SSL_initialized = false; @@ -580,18 +580,18 @@ be_tls_open_server(Port *port) if (port->peer != NULL) { int len; - X509_NAME *x509name = X509_get_subject_name(port->peer); + const X509_NAME *x509name = X509_get_subject_name(port->peer); char *peer_dn; BIO *bio = NULL; BUF_MEM *bio_buf = NULL; - len = X509_NAME_get_text_by_NID(x509name, NID_commonName, NULL, 0); + len = X509_NAME_get_text_by_NID(unconstify(X509_NAME *, x509name), NID_commonName, NULL, 0); if (len != -1) { char *peer_cn; peer_cn = MemoryContextAlloc(TopMemoryContext, len + 1); - r = X509_NAME_get_text_by_NID(x509name, NID_commonName, peer_cn, + r = X509_NAME_get_text_by_NID(unconstify(X509_NAME *, x509name), NID_commonName, peer_cn, len + 1); peer_cn[len] = '\0'; if (r != len) @@ -632,7 +632,7 @@ be_tls_open_server(Port *port) * which make regular expression matching a bit easier. Also note that * it prints the Subject fields in reverse order. */ - X509_NAME_print_ex(bio, x509name, 0, XN_FLAG_RFC2253); + X509_NAME_print_ex(bio, unconstify(X509_NAME *, x509name), 0, XN_FLAG_RFC2253); if (BIO_get_mem_ptr(bio, &bio_buf) <= 0) { BIO_free(bio); @@ -1406,14 +1406,14 @@ be_tls_get_certificate_hash(Port *port, size_t *len) * */ static char * -X509_NAME_to_cstring(X509_NAME *name) +X509_NAME_to_cstring(const X509_NAME *name) { BIO *membuf = BIO_new(BIO_s_mem()); int i, nid, - count = X509_NAME_entry_count(name); - X509_NAME_ENTRY *e; - ASN1_STRING *v; + count = X509_NAME_entry_count(unconstify(X509_NAME *, name)); + const X509_NAME_ENTRY *e; + const ASN1_STRING *v; const char *field_name; size_t size; char nullterm; @@ -1429,13 +1429,13 @@ X509_NAME_to_cstring(X509_NAME *name) (void) BIO_set_close(membuf, BIO_CLOSE); for (i = 0; i < count; i++) { - e = X509_NAME_get_entry(name, i); - nid = OBJ_obj2nid(X509_NAME_ENTRY_get_object(e)); + e = X509_NAME_get_entry(unconstify(X509_NAME *, name), i); + nid = OBJ_obj2nid(X509_NAME_ENTRY_get_object(unconstify(X509_NAME_ENTRY *, e))); if (nid == NID_undef) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("could not get NID for ASN1_OBJECT object"))); - v = X509_NAME_ENTRY_get_data(e); + v = X509_NAME_ENTRY_get_data(unconstify(X509_NAME_ENTRY *, e)); field_name = OBJ_nid2sn(nid); if (field_name == NULL) field_name = OBJ_nid2ln(nid); @@ -1444,7 +1444,7 @@ X509_NAME_to_cstring(X509_NAME *name) (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("could not convert NID %d to an ASN1_OBJECT structure", nid))); BIO_printf(membuf, "/%s=", field_name); - ASN1_STRING_print_ex(membuf, v, + ASN1_STRING_print_ex(membuf, unconstify(ASN1_STRING *, v), ((ASN1_STRFLGS_RFC2253 & ~ASN1_STRFLGS_ESC_MSB) | ASN1_STRFLGS_UTF8_CONVERT)); } diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index 5f340494b7..915a23cf20 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -64,7 +64,7 @@ static int verify_cb(int ok, X509_STORE_CTX *ctx); static int openssl_verify_peer_name_matches_certificate_name(PGconn *conn, - ASN1_STRING *name, + const ASN1_STRING *name, char **store_name); static void destroy_ssl_system(void); static int initialize_SSL(PGconn *conn); @@ -481,7 +481,8 @@ verify_cb(int ok, X509_STORE_CTX *ctx) * into a plain C string. */ static int -openssl_verify_peer_name_matches_certificate_name(PGconn *conn, ASN1_STRING *name_entry, +openssl_verify_peer_name_matches_certificate_name(PGconn *conn, + const ASN1_STRING *name_entry, char **store_name) { int len; @@ -501,7 +502,7 @@ openssl_verify_peer_name_matches_certificate_name(PGconn *conn, ASN1_STRING *nam #ifdef HAVE_ASN1_STRING_GET0_DATA namedata = ASN1_STRING_get0_data(name_entry); #else - namedata = ASN1_STRING_data(name_entry); + namedata = ASN1_STRING_data(unconstify(ASN1_STRING *, name_entry)); #endif len = ASN1_STRING_length(name_entry); @@ -570,20 +571,20 @@ pgtls_verify_peer_name_matches_certificate_guts(PGconn *conn, */ if (*names_examined == 0) { - X509_NAME *subject_name; + const X509_NAME *subject_name; subject_name = X509_get_subject_name(conn->peer); if (subject_name != NULL) { int cn_index; - cn_index = X509_NAME_get_index_by_NID(subject_name, + cn_index = X509_NAME_get_index_by_NID(unconstify(X509_NAME *, subject_name), NID_commonName, -1); if (cn_index >= 0) { (*names_examined)++; rc = openssl_verify_peer_name_matches_certificate_name(conn, - X509_NAME_ENTRY_get_data(X509_NAME_get_entry(subject_name, cn_index)), + X509_NAME_ENTRY_get_data(X509_NAME_get_entry(unconstify(X509_NAME *, subject_name), cn_index)), first_name); } } diff --git a/src/test/ssl/t/001_ssltests.pl b/src/test/ssl/t/001_ssltests.pl index cc7bd98c83..f6b20186f1 100644 --- a/src/test/ssl/t/001_ssltests.pl +++ b/src/test/ssl/t/001_ssltests.pl @@ -538,7 +538,7 @@ $node->connect_fails( "$common_connstr user=ssltestuser sslcert=ssl/client-revoked.crt sslkey=ssl/client-revoked_tmp.key", "certificate authorization fails with revoked client cert", - expected_stderr => qr|SSL error: ssl[a-z0-9/]* alert certificate revoked|, + expected_stderr => qr!SSL error: (ssl[a-z0-9/]*|tls) alert certificate revoked!, # revoked certificates should not authenticate the user log_unlike => [qr/connection authenticated:/],); @@ -591,7 +591,7 @@ $node->connect_fails( "$common_connstr user=ssltestuser sslcert=ssl/client-revoked.crt sslkey=ssl/client-revoked_tmp.key", "certificate authorization fails with revoked client cert with server-side CRL directory", - expected_stderr => qr|SSL error: ssl[a-z0-9/]* alert certificate revoked|); + expected_stderr => qr!SSL error: (ssl[a-z0-9/]*|tls) alert certificate revoked!); # clean up foreach my $key (@keys) From b31fa1f87f373d8d2fdc43a489ef5611de7a2af7 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Fri, 12 Jun 2026 10:20:34 -0400 Subject: [PATCH 37/76] Don't try to import a non-exported object in vcregress.pl Commit ca9e9b08e453 wrongly tried to import devnull from File::Spec, but it's not exported, you just call the method via the class. This was harmless until modern perls complained, so stop doing that. Per buildfarm failures. Backpatch 14 thru 16 --- src/tools/msvc/vcregress.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl index 57b1a0c221..9a94481868 100644 --- a/src/tools/msvc/vcregress.pl +++ b/src/tools/msvc/vcregress.pl @@ -14,7 +14,7 @@ use File::Copy; use File::Find (); use File::Path qw(rmtree); -use File::Spec qw(devnull); +use File::Spec; use FindBin; use lib $FindBin::RealBin; From 7974f94a02a1b39d7c84737ced41890386aaae67 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Fri, 12 Jun 2026 18:05:25 -0400 Subject: [PATCH 38/76] Adjust cross-version upgrade tests for seg_out() fix Commit 0e1f1ed157e taught seg_out() to print the certainty indicator on an interval's upper boundary, but it was back-patched only as far as v14. When upgrading from an older release, the old server prints the one test_seg row exercising that case ('4.6 .. ~7.0') without the indicator, so the pre- and post-upgrade dumps do not match. Make AdjustUpgrade.pm delete just that row; seg's comparison function does distinguish the certainty indicators, so the otherwise identical row '4.6 .. 7.0' is unaffected. Back-patch to all supported branches. Per buildfarm members crake and fairywren. Discussion: https://postgr.es/m/5ccbdbde-6467-4a10-bf4d-0be73a05ce8d@dunslane.net --- src/test/perl/PostgreSQL/Test/AdjustUpgrade.pm | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test/perl/PostgreSQL/Test/AdjustUpgrade.pm b/src/test/perl/PostgreSQL/Test/AdjustUpgrade.pm index fc9efb795b..c0ac3cc0d7 100644 --- a/src/test/perl/PostgreSQL/Test/AdjustUpgrade.pm +++ b/src/test/perl/PostgreSQL/Test/AdjustUpgrade.pm @@ -123,6 +123,14 @@ sub adjust_database_contents 'drop function if exists public.putenv(text)', 'drop function if exists public.wait_pid(integer)'); } + + # delete seg row that pre-14 was printed incorrectly but would now + # be printed correctly + if ($dbnames{contrib_regression_seg}) + { + _add_st($result, 'contrib_regression_seg', + "delete from test_seg where s = '4.6 .. ~7.0'"); + } } # user table OIDs are gone from release 12 on From af09b18cbadbb91ffe617b9ee5549de420e9b21a Mon Sep 17 00:00:00 2001 From: Alexander Korotkov Date: Sun, 14 Jun 2026 02:49:05 +0300 Subject: [PATCH 39/76] amcheck: Use correct varlena size accessor in bt_normalize_tuple() bt_normalize_tuple() uses VARSIZE() to get the size of varlena, even though it's not yet known, that it has a 4-byte header. Fix this by replacing a accessor with a universal VARSIZE_ANY(). Backpatch to all supported versions. Reported-by: Andres Freund Discussion: https://postgr.es/m/7ckc7oka4bvafkf5bwlqs6ygrhlsbhz25ppozfch7zbuxcx3rf%40e4pr4oqenalc Author: Andrey Borodin Reviewed-by: Alexander Korotkov Backpatch-through: 14 --- contrib/amcheck/verify_nbtree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/amcheck/verify_nbtree.c b/contrib/amcheck/verify_nbtree.c index f75667d806..15794b2641 100644 --- a/contrib/amcheck/verify_nbtree.c +++ b/contrib/amcheck/verify_nbtree.c @@ -2673,7 +2673,7 @@ bt_normalize_tuple(BtreeCheckState *state, IndexTuple itup) ItemPointerGetOffsetNumber(&(itup->t_tid)), RelationGetRelationName(state->rel)))); else if (!VARATT_IS_COMPRESSED(DatumGetPointer(normalized[i])) && - VARSIZE(DatumGetPointer(normalized[i])) > TOAST_INDEX_TARGET && + VARSIZE_ANY(DatumGetPointer(normalized[i])) > TOAST_INDEX_TARGET && (att->attstorage == TYPSTORAGE_EXTENDED || att->attstorage == TYPSTORAGE_MAIN)) { From b94996ddd70bbabb2c5ce2a3078751b4f4552dad Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sun, 14 Jun 2026 11:01:48 -0400 Subject: [PATCH 40/76] Doc: remove stale entry for removed aclitem[] ~ aclitem operator. Commit 2f70fdb06 removed the deprecated containment operator ~(aclitem[],aclitem) from the catalogs, but missed removing its entry from the documentation. (Arguably the blame should fall on c62dd80cd, which added this entry in contravention of the longstanding policy that we don't document deprecated aliases in the first place.) Author: Shinya Kato Reviewed-by: Tom Lane Discussion: https://postgr.es/m/CAOzEurQSyR5psWukyhUz1LtxyO55C2Vfp0Fmt8w2jGKxhszQmQ@mail.gmail.com Backpatch-through: 14 --- doc/src/sgml/func.sgml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 9be4a44162..d1bf8e32fc 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -22695,20 +22695,6 @@ SELECT has_function_privilege('joeuser', 'myfunc(int, text)', 'execute'); t - - - - aclitem[] ~ aclitem - boolean - - - This is a deprecated alias for @>. - - - '{calvin=r*w/hobbes,hobbes=r*w*/postgres}'::aclitem[] ~ 'calvin=r*/hobbes'::aclitem - t - - From 102689827fa7253e4a46ec76603f2148a6bbbfee Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 15 Jun 2026 11:38:02 +0900 Subject: [PATCH 41/76] Trim regression test expected output for xml This commit reduces the number of expected output files for the "xml" test from three to two (well, mostly one, see below for details). xml_2.out existed to handle some differences in output due to libxml2 2.9.3, due to some error context missing (085423e3e326). This file is removed, by tweaking the XML inputs to trigger the same error patterns for the problematic 2.9.3 and other libxml2 versions. This part is authored by Tom Lane. xml_1.out (no libxml2 support) is reduced in size by adding an \if query that exits the test early. This still checks NO_XML_SUPPORT() through xmlin(). The rest of the test is skipped if XML input cannot be handled by the backend. This part has been written by me. Author: Tom Lane Author: Michael Paquier Reviewed-by: Daniel Gustafsson Discussion: https://postgr.es/m/aiu6CXO67q-s70n5@paquier.xyz Backpatch-through: 14 --- src/test/regress/expected/xml.out | 56 +- src/test/regress/expected/xml_1.out | 1412 +----------------------- src/test/regress/expected/xml_2.out | 1557 --------------------------- src/test/regress/sql/xml.sql | 21 +- 4 files changed, 52 insertions(+), 2994 deletions(-) delete mode 100644 src/test/regress/expected/xml_2.out diff --git a/src/test/regress/expected/xml.out b/src/test/regress/expected/xml.out index 5924deff66..c5783ee3f5 100644 --- a/src/test/regress/expected/xml.out +++ b/src/test/regress/expected/xml.out @@ -4,13 +4,19 @@ CREATE TABLE xmltest ( ); INSERT INTO xmltest VALUES (1, 'one'); INSERT INTO xmltest VALUES (2, 'two'); -INSERT INTO xmltest VALUES (3, 'three '); ERROR: invalid XML content -LINE 1: INSERT INTO xmltest VALUES (3, 'three '); ^ -DETAIL: line 1: Couldn't find end of Start Tag wrong line 1 -three + ^ +-- If no XML data could be inserted, skip the tests as the server has been +-- compiled without libxml support. +SELECT count(*) = 0 AS skip_test FROM xmltest \gset +\if :skip_test +\quit +\endif SELECT * FROM xmltest; id | data ----+-------------------- @@ -58,13 +64,13 @@ SELECT xmlconcat(1, 2); ERROR: argument of XMLCONCAT must be type xml, not type integer LINE 1: SELECT xmlconcat(1, 2); ^ -SELECT xmlconcat('bad', ' '); ERROR: invalid XML content -LINE 1: SELECT xmlconcat('bad', ' '); ^ -DETAIL: line 1: Couldn't find end of Start Tag syntax line 1 - + ^ SELECT xmlconcat('', NULL, ''); xmlconcat -------------- @@ -240,13 +246,13 @@ SELECT xmlparse(content ''); (1 row) -SELECT xmlparse(content '&idontexist;'); +SELECT xmlparse(content '&idontexist; '); ERROR: invalid XML content DETAIL: line 1: Entity 'idontexist' not defined -&idontexist; +&idontexist; ^ line 1: Opening and ending tag mismatch: twoerrors line 1 and unbalanced -&idontexist; +&idontexist; ^ SELECT xmlparse(content ''); xmlparse @@ -254,11 +260,11 @@ SELECT xmlparse(content ''); (1 row) -SELECT xmlparse(document ' '); +SELECT xmlparse(document '!'); ERROR: invalid XML document DETAIL: line 1: Start tag expected, '<' not found - - ^ +! +^ SELECT xmlparse(document 'abc'); ERROR: invalid XML document DETAIL: line 1: Start tag expected, '<' not found @@ -270,21 +276,21 @@ SELECT xmlparse(document 'x'); x (1 row) -SELECT xmlparse(document '&'); +SELECT xmlparse(document '& '); ERROR: invalid XML document DETAIL: line 1: xmlParseEntityRef: no name -& +& ^ line 1: Opening and ending tag mismatch: invalidentity line 1 and abc -& +& ^ -SELECT xmlparse(document '&idontexist;'); +SELECT xmlparse(document '&idontexist; '); ERROR: invalid XML document DETAIL: line 1: Entity 'idontexist' not defined -&idontexist; +&idontexist; ^ line 1: Opening and ending tag mismatch: undefinedentity line 1 and abc -&idontexist; +&idontexist; ^ SELECT xmlparse(document ''); xmlparse @@ -298,13 +304,13 @@ SELECT xmlparse(document ''); (1 row) -SELECT xmlparse(document '&idontexist;'); +SELECT xmlparse(document '&idontexist; '); ERROR: invalid XML document DETAIL: line 1: Entity 'idontexist' not defined -&idontexist; +&idontexist; ^ line 1: Opening and ending tag mismatch: twoerrors line 1 and unbalanced -&idontexist; +&idontexist; ^ SELECT xmlparse(document ''); xmlparse diff --git a/src/test/regress/expected/xml_1.out b/src/test/regress/expected/xml_1.out index d22a6f8215..ad3d518490 100644 --- a/src/test/regress/expected/xml_1.out +++ b/src/test/regress/expected/xml_1.out @@ -14,1412 +14,14 @@ LINE 1: INSERT INTO xmltest VALUES (2, 'two'); ^ DETAIL: This functionality requires the server to be built with libxml support. HINT: You need to rebuild PostgreSQL using --with-libxml. -INSERT INTO xmltest VALUES (3, 'three '); ERROR: unsupported XML feature -LINE 1: INSERT INTO xmltest VALUES (3, 'three '); ^ DETAIL: This functionality requires the server to be built with libxml support. HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT * FROM xmltest; - id | data -----+------ -(0 rows) - -SELECT xmlcomment('test'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlcomment('-test'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlcomment('test-'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlcomment('--test'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlcomment('te st'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlconcat(xmlcomment('hello'), - xmlelement(NAME qux, 'foo'), - xmlcomment('world')); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlconcat('hello', 'you'); -ERROR: unsupported XML feature -LINE 1: SELECT xmlconcat('hello', 'you'); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlconcat(1, 2); -ERROR: argument of XMLCONCAT must be type xml, not type integer -LINE 1: SELECT xmlconcat(1, 2); - ^ -SELECT xmlconcat('bad', '', NULL, ''); -ERROR: unsupported XML feature -LINE 1: SELECT xmlconcat('', NULL, '', NULL, ''); -ERROR: unsupported XML feature -LINE 1: SELECT xmlconcat('', NULL, 'r'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlelement(name foo, xml 'br'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlelement(name foo, array[1, 2, 3]); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SET xmlbinary TO base64; -SELECT xmlelement(name foo, bytea 'bar'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SET xmlbinary TO hex; -SELECT xmlelement(name foo, bytea 'bar'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlelement(name foo, xmlattributes(true as bar)); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlelement(name foo, xmlattributes('2009-04-09 00:24:37'::timestamp as bar)); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlelement(name foo, xmlattributes('infinity'::timestamp as bar)); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlelement(name foo, xmlattributes('<>&"''' as funny, xml 'br' as funnier)); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(content ''); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(content ' '); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(content 'abc'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(content 'x'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(content '&'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(content '&idontexist;'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(content ''); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(content ''); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(content '&idontexist;'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(content ''); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(document ' '); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(document 'abc'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(document 'x'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(document '&'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(document '&idontexist;'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(document ''); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(document ''); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(document '&idontexist;'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlparse(document ''); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlpi(name foo); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlpi(name xml); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlpi(name xmlstuff); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlpi(name foo, 'bar'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlpi(name foo, 'in?>valid'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlpi(name foo, null); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlpi(name xml, null); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlpi(name xmlstuff, null); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlpi(name "xml-stylesheet", 'href="mystyle.css" type="text/css"'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlpi(name foo, ' bar'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlroot(xml '', version no value, standalone no value); -ERROR: unsupported XML feature -LINE 1: SELECT xmlroot(xml '', version no value, standalone no... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlroot(xml '', version '2.0'); -ERROR: unsupported XML feature -LINE 1: SELECT xmlroot(xml '', version '2.0'); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlroot(xml '', version no value, standalone yes); -ERROR: unsupported XML feature -LINE 1: SELECT xmlroot(xml '', version no value, standalone ye... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlroot(xml '', version no value, standalone yes); -ERROR: unsupported XML feature -LINE 1: SELECT xmlroot(xml '', version no... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlroot(xmlroot(xml '', version '1.0'), version '1.1', standalone no); -ERROR: unsupported XML feature -LINE 1: SELECT xmlroot(xmlroot(xml '', version '1.0'), version... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlroot('', version no value, standalone no); -ERROR: unsupported XML feature -LINE 1: SELECT xmlroot('... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlroot('', version no value, standalone no value); -ERROR: unsupported XML feature -LINE 1: SELECT xmlroot('... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlroot('', version no value); -ERROR: unsupported XML feature -LINE 1: SELECT xmlroot('... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlroot ( - xmlelement ( - name gazonk, - xmlattributes ( - 'val' AS name, - 1 + 1 AS num - ), - xmlelement ( - NAME qux, - 'foo' - ) - ), - version '1.0', - standalone yes -); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlserialize(content data as character varying(20)) FROM xmltest; - xmlserialize --------------- -(0 rows) - -SELECT xmlserialize(content 'good' as char(10)); -ERROR: unsupported XML feature -LINE 1: SELECT xmlserialize(content 'good' as char(10)); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlserialize(document 'bad' as text); -ERROR: unsupported XML feature -LINE 1: SELECT xmlserialize(document 'bad' as text); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml 'bar' IS DOCUMENT; -ERROR: unsupported XML feature -LINE 1: SELECT xml 'bar' IS DOCUMENT; - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml 'barfoo' IS DOCUMENT; -ERROR: unsupported XML feature -LINE 1: SELECT xml 'barfoo' IS DOCUMENT; - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml '' IS NOT DOCUMENT; -ERROR: unsupported XML feature -LINE 1: SELECT xml '' IS NOT DOCUMENT; - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml 'abc' IS NOT DOCUMENT; -ERROR: unsupported XML feature -LINE 1: SELECT xml 'abc' IS NOT DOCUMENT; - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT '<>' IS NOT DOCUMENT; -ERROR: unsupported XML feature -LINE 1: SELECT '<>' IS NOT DOCUMENT; - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlagg(data) FROM xmltest; - xmlagg --------- - -(1 row) - -SELECT xmlagg(data) FROM xmltest WHERE id > 10; - xmlagg --------- - -(1 row) - -SELECT xmlelement(name employees, xmlagg(xmlelement(name name, name))) FROM emp; -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. --- Check mapping SQL identifier to XML name -SELECT xmlpi(name ":::_xml_abc135.%-&_"); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmlpi(name "123"); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -PREPARE foo (xml) AS SELECT xmlconcat('', $1); -ERROR: unsupported XML feature -LINE 1: PREPARE foo (xml) AS SELECT xmlconcat('', $1); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SET XML OPTION DOCUMENT; -EXECUTE foo (''); -ERROR: prepared statement "foo" does not exist -EXECUTE foo ('bad'); -ERROR: prepared statement "foo" does not exist -SELECT xml ''; -ERROR: unsupported XML feature -LINE 1: SELECT xml ''; - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SET XML OPTION CONTENT; -EXECUTE foo (''); -ERROR: prepared statement "foo" does not exist -EXECUTE foo ('good'); -ERROR: prepared statement "foo" does not exist -SELECT xml ' '; -ERROR: unsupported XML feature -LINE 1: SELECT xml ' '; -ERROR: unsupported XML feature -LINE 1: SELECT xml ' '; -ERROR: unsupported XML feature -LINE 1: SELECT xml ''; - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml ' oops '; -ERROR: unsupported XML feature -LINE 1: SELECT xml ' oops '; - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml ' '; -ERROR: unsupported XML feature -LINE 1: SELECT xml ' '; - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml ''; -ERROR: unsupported XML feature -LINE 1: SELECT xml ''; - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. --- Test backwards parsing -CREATE VIEW xmlview1 AS SELECT xmlcomment('test'); -CREATE VIEW xmlview2 AS SELECT xmlconcat('hello', 'you'); -ERROR: unsupported XML feature -LINE 1: CREATE VIEW xmlview2 AS SELECT xmlconcat('hello', 'you'); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -CREATE VIEW xmlview3 AS SELECT xmlelement(name element, xmlattributes (1 as ":one:", 'deuce' as two), 'content&'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -CREATE VIEW xmlview4 AS SELECT xmlelement(name employee, xmlforest(name, age, salary as pay)) FROM emp; -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -CREATE VIEW xmlview5 AS SELECT xmlparse(content 'x'); -CREATE VIEW xmlview6 AS SELECT xmlpi(name foo, 'bar'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -CREATE VIEW xmlview7 AS SELECT xmlroot(xml '', version no value, standalone yes); -ERROR: unsupported XML feature -LINE 1: CREATE VIEW xmlview7 AS SELECT xmlroot(xml '', version... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -CREATE VIEW xmlview8 AS SELECT xmlserialize(content 'good' as char(10)); -ERROR: unsupported XML feature -LINE 1: ...EATE VIEW xmlview8 AS SELECT xmlserialize(content 'good' as ... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -CREATE VIEW xmlview9 AS SELECT xmlserialize(content 'good' as text); -ERROR: unsupported XML feature -LINE 1: ...EATE VIEW xmlview9 AS SELECT xmlserialize(content 'good' as ... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT table_name, view_definition FROM information_schema.views - WHERE table_name LIKE 'xmlview%' ORDER BY 1; - table_name | view_definition -------------+-------------------------------------------------------------------------------- - xmlview1 | SELECT xmlcomment('test'::text) AS xmlcomment; - xmlview5 | SELECT XMLPARSE(CONTENT 'x'::text STRIP WHITESPACE) AS "xmlparse"; -(2 rows) - --- Text XPath expressions evaluation -SELECT xpath('/value', data) FROM xmltest; - xpath -------- -(0 rows) - -SELECT xpath(NULL, NULL) IS NULL FROM xmltest; - ?column? ----------- -(0 rows) - -SELECT xpath('', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('', ''); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xpath('//text()', 'number one'); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('//text()', 'number one', ARRAY[ARRAY['loc', 'http://127.0.0.1']]); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('//loc:piece/@id', 'number one', ARRAY[ARRAY['loc', 'http://127.0.0.1']]); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('//loc:piece', 'number one', ARRAY[ARRAY['loc', 'http://127.0.0.1']]); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('//loc:piece', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('//@value', ''); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xpath('''<>''', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('''<>''', ''); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xpath('count(//*)', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('count(//*)', ''); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xpath('count(//*)=0', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('count(//*)=0', ''); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xpath('count(//*)=3', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('count(//*)=3', ''); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xpath('name(/*)', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('name(/*)', ''); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xpath('/nosuchtag', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('/nosuchtag', ''); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xpath('root', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('root', ''); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xpath('//namespace::foo', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('//namespace::foo', ''; - degree_symbol text; - res xml[]; -BEGIN - -- Per the documentation, except when the server encoding is UTF8, xpath() - -- may not work on non-ASCII data. The untranslatable_character and - -- undefined_function traps below, currently dead code, will become relevant - -- if we remove this limitation. - IF current_setting('server_encoding') <> 'UTF8' THEN - RAISE LOG 'skip: encoding % unsupported for xpath', - current_setting('server_encoding'); - RETURN; - END IF; - - degree_symbol := convert_from('\xc2b0', 'UTF8'); - res := xpath('text()', (xml_declaration || - '' || degree_symbol || '')::xml); - IF degree_symbol <> res[1]::text THEN - RAISE 'expected % (%), got % (%)', - degree_symbol, convert_to(degree_symbol, 'UTF8'), - res[1], convert_to(res[1]::text, 'UTF8'); - END IF; -EXCEPTION - -- character with byte sequence 0xc2 0xb0 in encoding "UTF8" has no equivalent in encoding "LATIN8" - WHEN untranslatable_character - -- default conversion function for encoding "UTF8" to "MULE_INTERNAL" does not exist - OR undefined_function - -- unsupported XML feature - OR feature_not_supported THEN - RAISE LOG 'skip: %', SQLERRM; -END -$$; --- Test xmlexists and xpath_exists -SELECT xmlexists('//town[text() = ''Toronto'']' PASSING BY REF 'Bidford-on-AvonCwmbranBristol'); -ERROR: unsupported XML feature -LINE 1: ...sts('//town[text() = ''Toronto'']' PASSING BY REF 'Bidford-on-AvonCwmbranBristol'); -ERROR: unsupported XML feature -LINE 1: ...sts('//town[text() = ''Cwmbran'']' PASSING BY REF ''); -ERROR: unsupported XML feature -LINE 1: ...LECT xmlexists('count(/nosuchtag)' PASSING BY REF '')... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xpath_exists('//town[text() = ''Toronto'']','Bidford-on-AvonCwmbranBristol'::xml); -ERROR: unsupported XML feature -LINE 1: ...ELECT xpath_exists('//town[text() = ''Toronto'']','Bidford-on-AvonCwmbranBristol'::xml); -ERROR: unsupported XML feature -LINE 1: ...ELECT xpath_exists('//town[text() = ''Cwmbran'']',''::xml); -ERROR: unsupported XML feature -LINE 1: SELECT xpath_exists('count(/nosuchtag)', ''::xml); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -INSERT INTO xmltest VALUES (4, 'BudvarfreeCarlinglots'::xml); -ERROR: unsupported XML feature -LINE 1: INSERT INTO xmltest VALUES (4, 'BudvarMolsonfreeCarlinglots'::xml); -ERROR: unsupported XML feature -LINE 1: INSERT INTO xmltest VALUES (5, 'MolsonBudvarfreeCarlinglots'::xml); -ERROR: unsupported XML feature -LINE 1: INSERT INTO xmltest VALUES (6, 'MolsonfreeCarlinglots'::xml); -ERROR: unsupported XML feature -LINE 1: INSERT INTO xmltest VALUES (7, 'number one'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml_is_well_formed('bar'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml_is_well_formed('bar'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml_is_well_formed('&'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml_is_well_formed('&idontexist;'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml_is_well_formed(''); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml_is_well_formed(''); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xml_is_well_formed('&idontexist;'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SET xmloption TO CONTENT; -SELECT xml_is_well_formed('abc'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. --- Since xpath() deals with namespaces, it's a bit stricter about --- what's well-formed and what's not. If we don't obey these rules --- (i.e. ignore namespace-related errors from libxml), xpath() --- fails in subtle ways. The following would for example produce --- the xml value --- --- which is invalid because '<' may not appear un-escaped in --- attribute values. --- Since different libxml versions emit slightly different --- error messages, we suppress the DETAIL in this test. -\set VERBOSITY terse -SELECT xpath('/*', ''); -ERROR: unsupported XML feature at character 20 -\set VERBOSITY default --- Again, the XML isn't well-formed for namespace purposes -SELECT xpath('/*', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('/*', ''); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. --- XPath deprecates relative namespaces, but they're not supposed to --- throw an error, only a warning. -SELECT xpath('/*', ''); -ERROR: unsupported XML feature -LINE 1: SELECT xpath('/*', ''); - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. --- External entity references should not leak filesystem information. -SELECT XMLPARSE(DOCUMENT ']>&c;'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT XMLPARSE(DOCUMENT ']>&c;'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. --- This might or might not load the requested DTD, but it mustn't throw error. -SELECT XMLPARSE(DOCUMENT ' '); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. --- XMLPATH tests -CREATE TABLE xmldata(data xml); -INSERT INTO xmldata VALUES(' - - AU - Australia - 3 - - - CN - China - 3 - - - HK - HongKong - 3 - - - IN - India - 3 - - - JP - Japan - 3Sinzo Abe - - - SG - Singapore - 3791 - -'); -ERROR: unsupported XML feature -LINE 1: INSERT INTO xmldata VALUES(' - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. --- XMLTABLE with columns -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME/text()' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); - id | _id | country_name | country_id | region_id | size | unit | premier_name -----+-----+--------------+------------+-----------+------+------+-------------- -(0 rows) - -CREATE VIEW xmltableview1 AS SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME/text()' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); -SELECT * FROM xmltableview1; - id | _id | country_name | country_id | region_id | size | unit | premier_name -----+-----+--------------+------------+-----------+------+------+-------------- -(0 rows) - -\sv xmltableview1 -CREATE OR REPLACE VIEW public.xmltableview1 AS - SELECT "xmltable".id, - "xmltable"._id, - "xmltable".country_name, - "xmltable".country_id, - "xmltable".region_id, - "xmltable".size, - "xmltable".unit, - "xmltable".premier_name - FROM ( SELECT xmldata.data - FROM xmldata) x, - LATERAL XMLTABLE(('/ROWS/ROW'::text) PASSING (x.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME/text()'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) -EXPLAIN (COSTS OFF) SELECT * FROM xmltableview1; - QUERY PLAN ------------------------------------------ - Nested Loop - -> Seq Scan on xmldata - -> Table Function Scan on "xmltable" -(3 rows) - -EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM xmltableview1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Nested Loop - Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name - -> Seq Scan on public.xmldata - Output: xmldata.data - -> Table Function Scan on "xmltable" - Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name - Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME/text()'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) -(7 rows) - --- errors -SELECT * FROM XMLTABLE (ROW () PASSING null COLUMNS v1 timestamp) AS f (v1, v2); -ERROR: XMLTABLE function has 1 columns available but 2 columns specified --- XMLNAMESPACES tests -SELECT * FROM XMLTABLE(XMLNAMESPACES('http://x.y' AS zz), - '/zz:rows/zz:row' - PASSING '10' - COLUMNS a int PATH 'zz:a'); -ERROR: unsupported XML feature -LINE 3: PASSING '10' - COLUMNS a int PATH 'Zz:a'); -ERROR: unsupported XML feature -LINE 3: PASSING '10' - COLUMNS a int PATH 'a'); -ERROR: unsupported XML feature -LINE 3: PASSING '' - COLUMNS a text PATH 'foo/namespace::node()'); -ERROR: unsupported XML feature -LINE 2: PASSING '' - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. --- used in prepare statements -PREPARE pp AS -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); -EXECUTE pp; - id | _id | country_name | country_id | region_id | size | unit | premier_name -----+-----+--------------+------------+-----------+------+------+-------------- -(0 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int); - COUNTRY_NAME | REGION_ID ---------------+----------- -(0 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id FOR ORDINALITY, "COUNTRY_NAME" text, "REGION_ID" int); - id | COUNTRY_NAME | REGION_ID -----+--------------+----------- -(0 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int); - id | COUNTRY_NAME | REGION_ID -----+--------------+----------- -(0 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id'); - id ----- -(0 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id FOR ORDINALITY); - id ----- -(0 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int, rawdata xml PATH '.'); - id | COUNTRY_NAME | REGION_ID | rawdata -----+--------------+-----------+--------- -(0 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int, rawdata xml PATH './*'); - id | COUNTRY_NAME | REGION_ID | rawdata -----+--------------+-----------+--------- -(0 rows) - -SELECT * FROM xmltable('/root' passing 'a1aa2a bbbbxxxcccc' COLUMNS element text); -ERROR: unsupported XML feature -LINE 1: SELECT * FROM xmltable('/root' passing 'a1aa1aa2a bbbbxxxcccc' COLUMNS element text PATH 'element/text()'); -- should fail -ERROR: unsupported XML feature -LINE 1: SELECT * FROM xmltable('/root' passing 'a1a &"<>!foo]]>2' columns c text); -ERROR: unsupported XML feature -LINE 1: select * from xmltable('d/r' passing ''"&<>' COLUMNS ent text); -ERROR: unsupported XML feature -LINE 1: SELECT * FROM xmltable('/x/a' PASSING '''"&<>' COLUMNS ent xml); -ERROR: unsupported XML feature -LINE 1: SELECT * FROM xmltable('/x/a' PASSING '' Seq Scan on public.xmldata - Output: xmldata.data - -> Table Function Scan on "xmltable" - Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name - Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) -(7 rows) - --- test qual -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; - COUNTRY_NAME | REGION_ID ---------------+----------- -(0 rows) - -EXPLAIN (VERBOSE, COSTS OFF) -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Nested Loop - Output: "xmltable"."COUNTRY_NAME", "xmltable"."REGION_ID" - -> Seq Scan on public.xmldata - Output: xmldata.data - -> Table Function Scan on "xmltable" - Output: "xmltable"."COUNTRY_NAME", "xmltable"."REGION_ID" - Table Function Call: XMLTABLE(('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]'::text) PASSING (xmldata.data) COLUMNS "COUNTRY_NAME" text, "REGION_ID" integer) - Filter: ("xmltable"."COUNTRY_NAME" = 'Japan'::text) -(8 rows) - --- should to work with more data -INSERT INTO xmldata VALUES(' - - CZ - Czech Republic - 2Milos Zeman - - - DE - Germany - 2 - - - FR - France - 2 - -'); -ERROR: unsupported XML feature -LINE 1: INSERT INTO xmldata VALUES(' - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -INSERT INTO xmldata VALUES(' - - EG - Egypt - 1 - - - SD - Sudan - 1 - -'); -ERROR: unsupported XML feature -LINE 1: INSERT INTO xmldata VALUES(' - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); - id | _id | country_name | country_id | region_id | size | unit | premier_name -----+-----+--------------+------------+-----------+------+------+-------------- -(0 rows) - -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') - WHERE region_id = 2; - id | _id | country_name | country_id | region_id | size | unit | premier_name -----+-----+--------------+------------+-----------+------+------+-------------- -(0 rows) - -EXPLAIN (VERBOSE, COSTS OFF) -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') - WHERE region_id = 2; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - Nested Loop - Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name - -> Seq Scan on public.xmldata - Output: xmldata.data - -> Table Function Scan on "xmltable" - Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name - Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) - Filter: ("xmltable".region_id = 2) -(8 rows) - --- should fail, NULL value -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE' NOT NULL, - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); - id | _id | country_name | country_id | region_id | size | unit | premier_name -----+-----+--------------+------------+-----------+------+------+-------------- -(0 rows) - --- if all is ok, then result is empty --- one line xml test -WITH - x AS (SELECT proname, proowner, procost::numeric, pronargs, - array_to_string(proargnames,',') as proargnames, - case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes - FROM pg_proc WHERE proname = 'f_leak'), - y AS (SELECT xmlelement(name proc, - xmlforest(proname, proowner, - procost, pronargs, - proargnames, proargtypes)) as proc - FROM x), - z AS (SELECT xmltable.* - FROM y, - LATERAL xmltable('/proc' PASSING proc - COLUMNS proname name, - proowner oid, - procost float, - pronargs int, - proargnames text, - proargtypes text)) - SELECT * FROM z - EXCEPT SELECT * FROM x; -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. --- multi line xml test, result should be empty too -WITH - x AS (SELECT proname, proowner, procost::numeric, pronargs, - array_to_string(proargnames,',') as proargnames, - case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes - FROM pg_proc), - y AS (SELECT xmlelement(name data, - xmlagg(xmlelement(name proc, - xmlforest(proname, proowner, procost, - pronargs, proargnames, proargtypes)))) as doc - FROM x), - z AS (SELECT xmltable.* - FROM y, - LATERAL xmltable('/data/proc' PASSING doc - COLUMNS proname name, - proowner oid, - procost float, - pronargs int, - proargnames text, - proargtypes text)) - SELECT * FROM z - EXCEPT SELECT * FROM x; -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -CREATE TABLE xmltest2(x xml, _path text); -INSERT INTO xmltest2 VALUES('1', 'A'); -ERROR: unsupported XML feature -LINE 1: INSERT INTO xmltest2 VALUES('1', 'A')... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -INSERT INTO xmltest2 VALUES('2', 'B'); -ERROR: unsupported XML feature -LINE 1: INSERT INTO xmltest2 VALUES('2', 'B')... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -INSERT INTO xmltest2 VALUES('3', 'C'); -ERROR: unsupported XML feature -LINE 1: INSERT INTO xmltest2 VALUES('3', 'C')... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -INSERT INTO xmltest2 VALUES('2', 'D'); -ERROR: unsupported XML feature -LINE 1: INSERT INTO xmltest2 VALUES('2', 'D')... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -SELECT xmltable.* FROM xmltest2, LATERAL xmltable('/d/r' PASSING x COLUMNS a int PATH '' || lower(_path) || 'c'); - a ---- -(0 rows) - -SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH '.'); - a ---- -(0 rows) - -SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH 'x' DEFAULT ascii(_path) - 54); - a ---- -(0 rows) - --- XPath result can be boolean or number too -SELECT * FROM XMLTABLE('*' PASSING 'a' COLUMNS a xml PATH '.', b text PATH '.', c text PATH '"hi"', d boolean PATH '. = "a"', e integer PATH 'string-length(.)'); -ERROR: unsupported XML feature -LINE 1: SELECT * FROM XMLTABLE('*' PASSING 'a' COLUMNS a xml ... - ^ -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. -\x -SELECT * FROM XMLTABLE('*' PASSING 'pre&deeppost' COLUMNS x xml PATH 'node()', y xml PATH '/'); -ERROR: unsupported XML feature -LINE 1: SELECT * FROM XMLTABLE('*' PASSING 'pre"', b xml PATH '""'); -ERROR: unsupported XML feature -DETAIL: This functionality requires the server to be built with libxml support. -HINT: You need to rebuild PostgreSQL using --with-libxml. +-- If no XML data could be inserted, skip the tests as the server has been +-- compiled without libxml support. +SELECT count(*) = 0 AS skip_test FROM xmltest \gset +\if :skip_test +\quit diff --git a/src/test/regress/expected/xml_2.out b/src/test/regress/expected/xml_2.out deleted file mode 100644 index 43c5fe9122..0000000000 --- a/src/test/regress/expected/xml_2.out +++ /dev/null @@ -1,1557 +0,0 @@ -CREATE TABLE xmltest ( - id int, - data xml -); -INSERT INTO xmltest VALUES (1, 'one'); -INSERT INTO xmltest VALUES (2, 'two'); -INSERT INTO xmltest VALUES (3, 'one - 2 | two -(2 rows) - -SELECT xmlcomment('test'); - xmlcomment -------------- - -(1 row) - -SELECT xmlcomment('-test'); - xmlcomment --------------- - -(1 row) - -SELECT xmlcomment('test-'); -ERROR: invalid XML comment -SELECT xmlcomment('--test'); -ERROR: invalid XML comment -SELECT xmlcomment('te st'); - xmlcomment --------------- - -(1 row) - -SELECT xmlconcat(xmlcomment('hello'), - xmlelement(NAME qux, 'foo'), - xmlcomment('world')); - xmlconcat ----------------------------------------- - foo -(1 row) - -SELECT xmlconcat('hello', 'you'); - xmlconcat ------------ - helloyou -(1 row) - -SELECT xmlconcat(1, 2); -ERROR: argument of XMLCONCAT must be type xml, not type integer -LINE 1: SELECT xmlconcat(1, 2); - ^ -SELECT xmlconcat('bad', '', NULL, ''); - xmlconcat --------------- - -(1 row) - -SELECT xmlconcat('', NULL, ''); - xmlconcat ------------------------------------ - -(1 row) - -SELECT xmlconcat(NULL); - xmlconcat ------------ - -(1 row) - -SELECT xmlconcat(NULL, NULL); - xmlconcat ------------ - -(1 row) - -SELECT xmlelement(name element, - xmlattributes (1 as one, 'deuce' as two), - 'content'); - xmlelement ------------------------------------------------- - content -(1 row) - -SELECT xmlelement(name element, - xmlattributes ('unnamed and wrong')); -ERROR: unnamed XML attribute value must be a column reference -LINE 2: xmlattributes ('unnamed and wrong')); - ^ -SELECT xmlelement(name element, xmlelement(name nested, 'stuff')); - xmlelement -------------------------------------------- - stuff -(1 row) - -SELECT xmlelement(name employee, xmlforest(name, age, salary as pay)) FROM emp; - xmlelement ----------------------------------------------------------------------- - sharon251000 - sam302000 - bill201000 - jeff23600 - cim30400 - linda19100 -(6 rows) - -SELECT xmlelement(name duplicate, xmlattributes(1 as a, 2 as b, 3 as a)); -ERROR: XML attribute name "a" appears more than once -LINE 1: ...ment(name duplicate, xmlattributes(1 as a, 2 as b, 3 as a)); - ^ -SELECT xmlelement(name num, 37); - xmlelement ---------------- - 37 -(1 row) - -SELECT xmlelement(name foo, text 'bar'); - xmlelement ----------------- - bar -(1 row) - -SELECT xmlelement(name foo, xml 'bar'); - xmlelement ----------------- - bar -(1 row) - -SELECT xmlelement(name foo, text 'br'); - xmlelement -------------------------- - b<a/>r -(1 row) - -SELECT xmlelement(name foo, xml 'br'); - xmlelement -------------------- - br -(1 row) - -SELECT xmlelement(name foo, array[1, 2, 3]); - xmlelement -------------------------------------------------------------------------- - 123 -(1 row) - -SET xmlbinary TO base64; -SELECT xmlelement(name foo, bytea 'bar'); - xmlelement ------------------ - YmFy -(1 row) - -SET xmlbinary TO hex; -SELECT xmlelement(name foo, bytea 'bar'); - xmlelement -------------------- - 626172 -(1 row) - -SELECT xmlelement(name foo, xmlattributes(true as bar)); - xmlelement -------------------- - -(1 row) - -SELECT xmlelement(name foo, xmlattributes('2009-04-09 00:24:37'::timestamp as bar)); - xmlelement ----------------------------------- - -(1 row) - -SELECT xmlelement(name foo, xmlattributes('infinity'::timestamp as bar)); -ERROR: timestamp out of range -DETAIL: XML does not support infinite timestamp values. -SELECT xmlelement(name foo, xmlattributes('<>&"''' as funny, xml 'br' as funnier)); - xmlelement ------------------------------------------------------------- - -(1 row) - -SELECT xmlparse(content ''); - xmlparse ----------- - -(1 row) - -SELECT xmlparse(content ' '); - xmlparse ----------- - -(1 row) - -SELECT xmlparse(content 'abc'); - xmlparse ----------- - abc -(1 row) - -SELECT xmlparse(content 'x'); - xmlparse --------------- - x -(1 row) - -SELECT xmlparse(content '&'); -ERROR: invalid XML content -DETAIL: line 1: xmlParseEntityRef: no name -& - ^ -SELECT xmlparse(content '&idontexist;'); -ERROR: invalid XML content -DETAIL: line 1: Entity 'idontexist' not defined -&idontexist; - ^ -SELECT xmlparse(content ''); - xmlparse ---------------------------- - -(1 row) - -SELECT xmlparse(content ''); - xmlparse --------------------------------- - -(1 row) - -SELECT xmlparse(content '&idontexist;'); -ERROR: invalid XML content -DETAIL: line 1: Entity 'idontexist' not defined -&idontexist; - ^ -line 1: Opening and ending tag mismatch: twoerrors line 1 and unbalanced -SELECT xmlparse(content ''); - xmlparse ---------------------- - -(1 row) - -SELECT xmlparse(document ' '); -ERROR: invalid XML document -DETAIL: line 1: Start tag expected, '<' not found -SELECT xmlparse(document 'abc'); -ERROR: invalid XML document -DETAIL: line 1: Start tag expected, '<' not found -abc -^ -SELECT xmlparse(document 'x'); - xmlparse --------------- - x -(1 row) - -SELECT xmlparse(document '&'); -ERROR: invalid XML document -DETAIL: line 1: xmlParseEntityRef: no name -& - ^ -line 1: Opening and ending tag mismatch: invalidentity line 1 and abc -SELECT xmlparse(document '&idontexist;'); -ERROR: invalid XML document -DETAIL: line 1: Entity 'idontexist' not defined -&idontexist; - ^ -line 1: Opening and ending tag mismatch: undefinedentity line 1 and abc -SELECT xmlparse(document ''); - xmlparse ---------------------------- - -(1 row) - -SELECT xmlparse(document ''); - xmlparse --------------------------------- - -(1 row) - -SELECT xmlparse(document '&idontexist;'); -ERROR: invalid XML document -DETAIL: line 1: Entity 'idontexist' not defined -&idontexist; - ^ -line 1: Opening and ending tag mismatch: twoerrors line 1 and unbalanced -SELECT xmlparse(document ''); - xmlparse ---------------------- - -(1 row) - -SELECT xmlpi(name foo); - xmlpi ---------- - -(1 row) - -SELECT xmlpi(name xml); -ERROR: invalid XML processing instruction -DETAIL: XML processing instruction target name cannot be "xml". -SELECT xmlpi(name xmlstuff); - xmlpi --------------- - -(1 row) - -SELECT xmlpi(name foo, 'bar'); - xmlpi -------------- - -(1 row) - -SELECT xmlpi(name foo, 'in?>valid'); -ERROR: invalid XML processing instruction -DETAIL: XML processing instruction cannot contain "?>". -SELECT xmlpi(name foo, null); - xmlpi -------- - -(1 row) - -SELECT xmlpi(name xml, null); -ERROR: invalid XML processing instruction -DETAIL: XML processing instruction target name cannot be "xml". -SELECT xmlpi(name xmlstuff, null); - xmlpi -------- - -(1 row) - -SELECT xmlpi(name "xml-stylesheet", 'href="mystyle.css" type="text/css"'); - xmlpi -------------------------------------------------------- - -(1 row) - -SELECT xmlpi(name foo, ' bar'); - xmlpi -------------- - -(1 row) - -SELECT xmlroot(xml '', version no value, standalone no value); - xmlroot ---------- - -(1 row) - -SELECT xmlroot(xml '', version '2.0'); - xmlroot ------------------------------ - -(1 row) - -SELECT xmlroot(xml '', version no value, standalone yes); - xmlroot ----------------------------------------------- - -(1 row) - -SELECT xmlroot(xml '', version no value, standalone yes); - xmlroot ----------------------------------------------- - -(1 row) - -SELECT xmlroot(xmlroot(xml '', version '1.0'), version '1.1', standalone no); - xmlroot ---------------------------------------------- - -(1 row) - -SELECT xmlroot('', version no value, standalone no); - xmlroot ---------------------------------------------- - -(1 row) - -SELECT xmlroot('', version no value, standalone no value); - xmlroot ---------- - -(1 row) - -SELECT xmlroot('', version no value); - xmlroot ----------------------------------------------- - -(1 row) - -SELECT xmlroot ( - xmlelement ( - name gazonk, - xmlattributes ( - 'val' AS name, - 1 + 1 AS num - ), - xmlelement ( - NAME qux, - 'foo' - ) - ), - version '1.0', - standalone yes -); - xmlroot ------------------------------------------------------------------------------------------- - foo -(1 row) - -SELECT xmlserialize(content data as character varying(20)) FROM xmltest; - xmlserialize --------------------- - one - two -(2 rows) - -SELECT xmlserialize(content 'good' as char(10)); - xmlserialize --------------- - good -(1 row) - -SELECT xmlserialize(document 'bad' as text); -ERROR: not an XML document -SELECT xml 'bar' IS DOCUMENT; - ?column? ----------- - t -(1 row) - -SELECT xml 'barfoo' IS DOCUMENT; - ?column? ----------- - f -(1 row) - -SELECT xml '' IS NOT DOCUMENT; - ?column? ----------- - f -(1 row) - -SELECT xml 'abc' IS NOT DOCUMENT; - ?column? ----------- - t -(1 row) - -SELECT '<>' IS NOT DOCUMENT; -ERROR: invalid XML content -LINE 1: SELECT '<>' IS NOT DOCUMENT; - ^ -DETAIL: line 1: StartTag: invalid element name -<> - ^ -SELECT xmlagg(data) FROM xmltest; - xmlagg --------------------------------------- - onetwo -(1 row) - -SELECT xmlagg(data) FROM xmltest WHERE id > 10; - xmlagg --------- - -(1 row) - -SELECT xmlelement(name employees, xmlagg(xmlelement(name name, name))) FROM emp; - xmlelement --------------------------------------------------------------------------------------------------------------------------------- - sharonsambilljeffcimlinda -(1 row) - --- Check mapping SQL identifier to XML name -SELECT xmlpi(name ":::_xml_abc135.%-&_"); - xmlpi -------------------------------------------------- - -(1 row) - -SELECT xmlpi(name "123"); - xmlpi ---------------- - -(1 row) - -PREPARE foo (xml) AS SELECT xmlconcat('', $1); -SET XML OPTION DOCUMENT; -EXECUTE foo (''); - xmlconcat --------------- - -(1 row) - -EXECUTE foo ('bad'); -ERROR: invalid XML document -LINE 1: EXECUTE foo ('bad'); - ^ -DETAIL: line 1: Start tag expected, '<' not found -bad -^ -SELECT xml ''; -ERROR: invalid XML document -LINE 1: SELECT xml ''; - ^ -DETAIL: line 1: Extra content at the end of the document - - ^ -SET XML OPTION CONTENT; -EXECUTE foo (''); - xmlconcat --------------- - -(1 row) - -EXECUTE foo ('good'); - xmlconcat ------------- - good -(1 row) - -SELECT xml ' '; - xml --------------------------------------------------------------------- - -(1 row) - -SELECT xml ' '; - xml ------------------------------- - -(1 row) - -SELECT xml ''; - xml ------------------- - -(1 row) - -SELECT xml ' oops '; -ERROR: invalid XML content -LINE 1: SELECT xml ' oops '; - ^ -DETAIL: line 1: StartTag: invalid element name - oops - ^ -SELECT xml ' '; -ERROR: invalid XML content -LINE 1: SELECT xml ' '; - ^ -DETAIL: line 1: StartTag: invalid element name - - ^ -SELECT xml ''; -ERROR: invalid XML content -LINE 1: SELECT xml ''; - ^ -DETAIL: line 1: Extra content at the end of the document - - ^ --- Test backwards parsing -CREATE VIEW xmlview1 AS SELECT xmlcomment('test'); -CREATE VIEW xmlview2 AS SELECT xmlconcat('hello', 'you'); -CREATE VIEW xmlview3 AS SELECT xmlelement(name element, xmlattributes (1 as ":one:", 'deuce' as two), 'content&'); -CREATE VIEW xmlview4 AS SELECT xmlelement(name employee, xmlforest(name, age, salary as pay)) FROM emp; -CREATE VIEW xmlview5 AS SELECT xmlparse(content 'x'); -CREATE VIEW xmlview6 AS SELECT xmlpi(name foo, 'bar'); -CREATE VIEW xmlview7 AS SELECT xmlroot(xml '', version no value, standalone yes); -CREATE VIEW xmlview8 AS SELECT xmlserialize(content 'good' as char(10)); -CREATE VIEW xmlview9 AS SELECT xmlserialize(content 'good' as text); -SELECT table_name, view_definition FROM information_schema.views - WHERE table_name LIKE 'xmlview%' ORDER BY 1; - table_name | view_definition -------------+------------------------------------------------------------------------------------------------------------------- - xmlview1 | SELECT xmlcomment('test'::text) AS xmlcomment; - xmlview2 | SELECT XMLCONCAT('hello'::xml, 'you'::xml) AS "xmlconcat"; - xmlview3 | SELECT XMLELEMENT(NAME element, XMLATTRIBUTES(1 AS ":one:", 'deuce' AS two), 'content&') AS "xmlelement"; - xmlview4 | SELECT XMLELEMENT(NAME employee, XMLFOREST(emp.name AS name, emp.age AS age, emp.salary AS pay)) AS "xmlelement"+ - | FROM emp; - xmlview5 | SELECT XMLPARSE(CONTENT 'x'::text STRIP WHITESPACE) AS "xmlparse"; - xmlview6 | SELECT XMLPI(NAME foo, 'bar'::text) AS "xmlpi"; - xmlview7 | SELECT XMLROOT(''::xml, VERSION NO VALUE, STANDALONE YES) AS "xmlroot"; - xmlview8 | SELECT (XMLSERIALIZE(CONTENT 'good'::xml AS character(10)))::character(10) AS "xmlserialize"; - xmlview9 | SELECT XMLSERIALIZE(CONTENT 'good'::xml AS text) AS "xmlserialize"; -(9 rows) - --- Text XPath expressions evaluation -SELECT xpath('/value', data) FROM xmltest; - xpath ----------------------- - {one} - {two} -(2 rows) - -SELECT xpath(NULL, NULL) IS NULL FROM xmltest; - ?column? ----------- - t - t -(2 rows) - -SELECT xpath('', ''); -ERROR: empty XPath expression -CONTEXT: SQL function "xpath" statement 1 -SELECT xpath('//text()', 'number one'); - xpath ----------------- - {"number one"} -(1 row) - -SELECT xpath('//loc:piece/@id', 'number one', ARRAY[ARRAY['loc', 'http://127.0.0.1']]); - xpath -------- - {1,2} -(1 row) - -SELECT xpath('//loc:piece', 'number one', ARRAY[ARRAY['loc', 'http://127.0.0.1']]); - xpath ------------------------------------------------------------------------------------------------------------------------------------------------- - {"number one",""} -(1 row) - -SELECT xpath('//loc:piece', 'number one', ARRAY[ARRAY['loc', 'http://127.0.0.1']]); - xpath ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {"number one",""} -(1 row) - -SELECT xpath('//b', 'one two three etc'); - xpath -------------------------- - {two,etc} -(1 row) - -SELECT xpath('//text()', '<'); - xpath --------- - {<} -(1 row) - -SELECT xpath('//@value', ''); - xpath --------- - {<} -(1 row) - -SELECT xpath('''<>''', ''); - xpath ---------------------------- - {<<invalid>>} -(1 row) - -SELECT xpath('count(//*)', ''); - xpath -------- - {3} -(1 row) - -SELECT xpath('count(//*)=0', ''); - xpath ---------- - {false} -(1 row) - -SELECT xpath('count(//*)=3', ''); - xpath --------- - {true} -(1 row) - -SELECT xpath('name(/*)', ''); - xpath --------- - {root} -(1 row) - -SELECT xpath('/nosuchtag', ''); - xpath -------- - {} -(1 row) - -SELECT xpath('root', ''); - xpath ------------ - {} -(1 row) - -SELECT xpath('//namespace::foo', ''); - xpath --------------------- - {http://127.0.0.1} -(1 row) - --- Round-trip non-ASCII data through xpath(). -DO $$ -DECLARE - xml_declaration text := ''; - degree_symbol text; - res xml[]; -BEGIN - -- Per the documentation, except when the server encoding is UTF8, xpath() - -- may not work on non-ASCII data. The untranslatable_character and - -- undefined_function traps below, currently dead code, will become relevant - -- if we remove this limitation. - IF current_setting('server_encoding') <> 'UTF8' THEN - RAISE LOG 'skip: encoding % unsupported for xpath', - current_setting('server_encoding'); - RETURN; - END IF; - - degree_symbol := convert_from('\xc2b0', 'UTF8'); - res := xpath('text()', (xml_declaration || - '' || degree_symbol || '')::xml); - IF degree_symbol <> res[1]::text THEN - RAISE 'expected % (%), got % (%)', - degree_symbol, convert_to(degree_symbol, 'UTF8'), - res[1], convert_to(res[1]::text, 'UTF8'); - END IF; -EXCEPTION - -- character with byte sequence 0xc2 0xb0 in encoding "UTF8" has no equivalent in encoding "LATIN8" - WHEN untranslatable_character - -- default conversion function for encoding "UTF8" to "MULE_INTERNAL" does not exist - OR undefined_function - -- unsupported XML feature - OR feature_not_supported THEN - RAISE LOG 'skip: %', SQLERRM; -END -$$; --- Test xmlexists and xpath_exists -SELECT xmlexists('//town[text() = ''Toronto'']' PASSING BY REF 'Bidford-on-AvonCwmbranBristol'); - xmlexists ------------ - f -(1 row) - -SELECT xmlexists('//town[text() = ''Cwmbran'']' PASSING BY REF 'Bidford-on-AvonCwmbranBristol'); - xmlexists ------------ - t -(1 row) - -SELECT xmlexists('count(/nosuchtag)' PASSING BY REF ''); - xmlexists ------------ - t -(1 row) - -SELECT xpath_exists('//town[text() = ''Toronto'']','Bidford-on-AvonCwmbranBristol'::xml); - xpath_exists --------------- - f -(1 row) - -SELECT xpath_exists('//town[text() = ''Cwmbran'']','Bidford-on-AvonCwmbranBristol'::xml); - xpath_exists --------------- - t -(1 row) - -SELECT xpath_exists('count(/nosuchtag)', ''::xml); - xpath_exists --------------- - t -(1 row) - -INSERT INTO xmltest VALUES (4, 'BudvarfreeCarlinglots'::xml); -INSERT INTO xmltest VALUES (5, 'MolsonfreeCarlinglots'::xml); -INSERT INTO xmltest VALUES (6, 'BudvarfreeCarlinglots'::xml); -INSERT INTO xmltest VALUES (7, 'MolsonfreeCarlinglots'::xml); -SELECT COUNT(id) FROM xmltest WHERE xmlexists('/menu/beer' PASSING data); - count -------- - 0 -(1 row) - -SELECT COUNT(id) FROM xmltest WHERE xmlexists('/menu/beer' PASSING BY REF data BY REF); - count -------- - 0 -(1 row) - -SELECT COUNT(id) FROM xmltest WHERE xmlexists('/menu/beers' PASSING BY REF data); - count -------- - 2 -(1 row) - -SELECT COUNT(id) FROM xmltest WHERE xmlexists('/menu/beers/name[text() = ''Molson'']' PASSING BY REF data); - count -------- - 1 -(1 row) - -SELECT COUNT(id) FROM xmltest WHERE xpath_exists('/menu/beer',data); - count -------- - 0 -(1 row) - -SELECT COUNT(id) FROM xmltest WHERE xpath_exists('/menu/beers',data); - count -------- - 2 -(1 row) - -SELECT COUNT(id) FROM xmltest WHERE xpath_exists('/menu/beers/name[text() = ''Molson'']',data); - count -------- - 1 -(1 row) - -SELECT COUNT(id) FROM xmltest WHERE xpath_exists('/myns:menu/myns:beer',data,ARRAY[ARRAY['myns','http://myns.com']]); - count -------- - 0 -(1 row) - -SELECT COUNT(id) FROM xmltest WHERE xpath_exists('/myns:menu/myns:beers',data,ARRAY[ARRAY['myns','http://myns.com']]); - count -------- - 2 -(1 row) - -SELECT COUNT(id) FROM xmltest WHERE xpath_exists('/myns:menu/myns:beers/myns:name[text() = ''Molson'']',data,ARRAY[ARRAY['myns','http://myns.com']]); - count -------- - 1 -(1 row) - -CREATE TABLE query ( expr TEXT ); -INSERT INTO query VALUES ('/menu/beers/cost[text() = ''lots'']'); -SELECT COUNT(id) FROM xmltest, query WHERE xmlexists(expr PASSING BY REF data); - count -------- - 2 -(1 row) - --- Test xml_is_well_formed and variants -SELECT xml_is_well_formed_document('bar'); - xml_is_well_formed_document ------------------------------ - t -(1 row) - -SELECT xml_is_well_formed_document('abc'); - xml_is_well_formed_document ------------------------------ - f -(1 row) - -SELECT xml_is_well_formed_content('bar'); - xml_is_well_formed_content ----------------------------- - t -(1 row) - -SELECT xml_is_well_formed_content('abc'); - xml_is_well_formed_content ----------------------------- - t -(1 row) - -SET xmloption TO DOCUMENT; -SELECT xml_is_well_formed('abc'); - xml_is_well_formed --------------------- - f -(1 row) - -SELECT xml_is_well_formed('<>'); - xml_is_well_formed --------------------- - f -(1 row) - -SELECT xml_is_well_formed(''); - xml_is_well_formed --------------------- - t -(1 row) - -SELECT xml_is_well_formed('bar'); - xml_is_well_formed --------------------- - t -(1 row) - -SELECT xml_is_well_formed('barbaz'); - xml_is_well_formed --------------------- - f -(1 row) - -SELECT xml_is_well_formed('number one'); - xml_is_well_formed --------------------- - t -(1 row) - -SELECT xml_is_well_formed('bar'); - xml_is_well_formed --------------------- - f -(1 row) - -SELECT xml_is_well_formed('bar'); - xml_is_well_formed --------------------- - t -(1 row) - -SELECT xml_is_well_formed('&'); - xml_is_well_formed --------------------- - f -(1 row) - -SELECT xml_is_well_formed('&idontexist;'); - xml_is_well_formed --------------------- - f -(1 row) - -SELECT xml_is_well_formed(''); - xml_is_well_formed --------------------- - t -(1 row) - -SELECT xml_is_well_formed(''); - xml_is_well_formed --------------------- - t -(1 row) - -SELECT xml_is_well_formed('&idontexist;'); - xml_is_well_formed --------------------- - f -(1 row) - -SET xmloption TO CONTENT; -SELECT xml_is_well_formed('abc'); - xml_is_well_formed --------------------- - t -(1 row) - --- Since xpath() deals with namespaces, it's a bit stricter about --- what's well-formed and what's not. If we don't obey these rules --- (i.e. ignore namespace-related errors from libxml), xpath() --- fails in subtle ways. The following would for example produce --- the xml value --- --- which is invalid because '<' may not appear un-escaped in --- attribute values. --- Since different libxml versions emit slightly different --- error messages, we suppress the DETAIL in this test. -\set VERBOSITY terse -SELECT xpath('/*', ''); -ERROR: could not parse XML document -\set VERBOSITY default --- Again, the XML isn't well-formed for namespace purposes -SELECT xpath('/*', ''); -ERROR: could not parse XML document -DETAIL: line 1: Namespace prefix nosuchprefix on tag is not defined - - ^ -CONTEXT: SQL function "xpath" statement 1 --- XPath deprecates relative namespaces, but they're not supposed to --- throw an error, only a warning. -SELECT xpath('/*', ''); -WARNING: line 1: xmlns: URI relative is not absolute - - ^ - xpath --------------------------------------- - {""} -(1 row) - --- External entity references should not leak filesystem information. -SELECT XMLPARSE(DOCUMENT ']>&c;'); - xmlparse ------------------------------------------------------------------ - ]>&c; -(1 row) - -SELECT XMLPARSE(DOCUMENT ']>&c;'); - xmlparse ------------------------------------------------------------------------ - ]>&c; -(1 row) - --- This might or might not load the requested DTD, but it mustn't throw error. -SELECT XMLPARSE(DOCUMENT ' '); - xmlparse ------------------------------------------------------------------------------------------------------------------------------------------------------- -   -(1 row) - --- XMLPATH tests -CREATE TABLE xmldata(data xml); -INSERT INTO xmldata VALUES(' - - AU - Australia - 3 - - - CN - China - 3 - - - HK - HongKong - 3 - - - IN - India - 3 - - - JP - Japan - 3Sinzo Abe - - - SG - Singapore - 3791 - -'); --- XMLTABLE with columns -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME/text()' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); - id | _id | country_name | country_id | region_id | size | unit | premier_name -----+-----+--------------+------------+-----------+------+------+--------------- - 1 | 1 | Australia | AU | 3 | | | not specified - 2 | 2 | China | CN | 3 | | | not specified - 3 | 3 | HongKong | HK | 3 | | | not specified - 4 | 4 | India | IN | 3 | | | not specified - 5 | 5 | Japan | JP | 3 | | | Sinzo Abe - 6 | 6 | Singapore | SG | 3 | 791 | km | not specified -(6 rows) - -CREATE VIEW xmltableview1 AS SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME/text()' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); -SELECT * FROM xmltableview1; - id | _id | country_name | country_id | region_id | size | unit | premier_name -----+-----+--------------+------------+-----------+------+------+--------------- - 1 | 1 | Australia | AU | 3 | | | not specified - 2 | 2 | China | CN | 3 | | | not specified - 3 | 3 | HongKong | HK | 3 | | | not specified - 4 | 4 | India | IN | 3 | | | not specified - 5 | 5 | Japan | JP | 3 | | | Sinzo Abe - 6 | 6 | Singapore | SG | 3 | 791 | km | not specified -(6 rows) - -\sv xmltableview1 -CREATE OR REPLACE VIEW public.xmltableview1 AS - SELECT "xmltable".id, - "xmltable"._id, - "xmltable".country_name, - "xmltable".country_id, - "xmltable".region_id, - "xmltable".size, - "xmltable".unit, - "xmltable".premier_name - FROM ( SELECT xmldata.data - FROM xmldata) x, - LATERAL XMLTABLE(('/ROWS/ROW'::text) PASSING (x.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME/text()'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) -EXPLAIN (COSTS OFF) SELECT * FROM xmltableview1; - QUERY PLAN ------------------------------------------ - Nested Loop - -> Seq Scan on xmldata - -> Table Function Scan on "xmltable" -(3 rows) - -EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM xmltableview1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Nested Loop - Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name - -> Seq Scan on public.xmldata - Output: xmldata.data - -> Table Function Scan on "xmltable" - Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name - Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME/text()'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) -(7 rows) - --- errors -SELECT * FROM XMLTABLE (ROW () PASSING null COLUMNS v1 timestamp) AS f (v1, v2); -ERROR: XMLTABLE function has 1 columns available but 2 columns specified --- XMLNAMESPACES tests -SELECT * FROM XMLTABLE(XMLNAMESPACES('http://x.y' AS zz), - '/zz:rows/zz:row' - PASSING '10' - COLUMNS a int PATH 'zz:a'); - a ----- - 10 -(1 row) - -CREATE VIEW xmltableview2 AS SELECT * FROM XMLTABLE(XMLNAMESPACES('http://x.y' AS "Zz"), - '/Zz:rows/Zz:row' - PASSING '10' - COLUMNS a int PATH 'Zz:a'); -SELECT * FROM xmltableview2; - a ----- - 10 -(1 row) - -\sv xmltableview2 -CREATE OR REPLACE VIEW public.xmltableview2 AS - SELECT "xmltable".a - FROM XMLTABLE(XMLNAMESPACES ('http://x.y'::text AS "Zz"), ('/Zz:rows/Zz:row'::text) PASSING ('10'::xml) COLUMNS a integer PATH ('Zz:a'::text)) -SELECT * FROM XMLTABLE(XMLNAMESPACES(DEFAULT 'http://x.y'), - '/rows/row' - PASSING '10' - COLUMNS a int PATH 'a'); -ERROR: DEFAULT namespace is not supported -SELECT * FROM XMLTABLE('.' - PASSING '' - COLUMNS a text PATH 'foo/namespace::node()'); - a --------------------------------------- - http://www.w3.org/XML/1998/namespace -(1 row) - --- used in prepare statements -PREPARE pp AS -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); -EXECUTE pp; - id | _id | country_name | country_id | region_id | size | unit | premier_name -----+-----+--------------+------------+-----------+------+------+--------------- - 1 | 1 | Australia | AU | 3 | | | not specified - 2 | 2 | China | CN | 3 | | | not specified - 3 | 3 | HongKong | HK | 3 | | | not specified - 4 | 4 | India | IN | 3 | | | not specified - 5 | 5 | Japan | JP | 3 | | | Sinzo Abe - 6 | 6 | Singapore | SG | 3 | 791 | km | not specified -(6 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int); - COUNTRY_NAME | REGION_ID ---------------+----------- - India | 3 - Japan | 3 -(2 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id FOR ORDINALITY, "COUNTRY_NAME" text, "REGION_ID" int); - id | COUNTRY_NAME | REGION_ID -----+--------------+----------- - 1 | India | 3 - 2 | Japan | 3 -(2 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int); - id | COUNTRY_NAME | REGION_ID -----+--------------+----------- - 4 | India | 3 - 5 | Japan | 3 -(2 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id'); - id ----- - 4 - 5 -(2 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id FOR ORDINALITY); - id ----- - 1 - 2 -(2 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int, rawdata xml PATH '.'); - id | COUNTRY_NAME | REGION_ID | rawdata -----+--------------+-----------+------------------------------------------------------------------ - 4 | India | 3 | + - | | | IN + - | | | India + - | | | 3 + - | | | - 5 | Japan | 3 | + - | | | JP + - | | | Japan + - | | | 3Sinzo Abe+ - | | | -(2 rows) - -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS id int PATH '@id', "COUNTRY_NAME" text, "REGION_ID" int, rawdata xml PATH './*'); - id | COUNTRY_NAME | REGION_ID | rawdata -----+--------------+-----------+----------------------------------------------------------------------------------------------------------------------------- - 4 | India | 3 | INIndia3 - 5 | Japan | 3 | JPJapan3Sinzo Abe -(2 rows) - -SELECT * FROM xmltable('/root' passing 'a1aa2a bbbbxxxcccc' COLUMNS element text); - element ----------------------- - a1aa2a bbbbxxxcccc -(1 row) - -SELECT * FROM xmltable('/root' passing 'a1aa2a bbbbxxxcccc' COLUMNS element text PATH 'element/text()'); -- should fail -ERROR: more than one value returned by column XPath expression --- CDATA test -select * from xmltable('d/r' passing ' &"<>!foo]]>2' columns c text); - c -------------------------- - &"<>!foo - 2 -(2 rows) - --- XML builtin entities -SELECT * FROM xmltable('/x/a' PASSING ''"&<>' COLUMNS ent text); - ent ------ - ' - " - & - < - > -(5 rows) - -SELECT * FROM xmltable('/x/a' PASSING ''"&<>' COLUMNS ent xml); - ent ------------------- - ' - " - & - < - > -(5 rows) - -EXPLAIN (VERBOSE, COSTS OFF) -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - Nested Loop - Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name - -> Seq Scan on public.xmldata - Output: xmldata.data - -> Table Function Scan on "xmltable" - Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name - Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) -(7 rows) - --- test qual -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; - COUNTRY_NAME | REGION_ID ---------------+----------- - Japan | 3 -(1 row) - -EXPLAIN (VERBOSE, COSTS OFF) -SELECT xmltable.* FROM xmldata, LATERAL xmltable('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]' PASSING data COLUMNS "COUNTRY_NAME" text, "REGION_ID" int) WHERE "COUNTRY_NAME" = 'Japan'; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Nested Loop - Output: "xmltable"."COUNTRY_NAME", "xmltable"."REGION_ID" - -> Seq Scan on public.xmldata - Output: xmldata.data - -> Table Function Scan on "xmltable" - Output: "xmltable"."COUNTRY_NAME", "xmltable"."REGION_ID" - Table Function Call: XMLTABLE(('/ROWS/ROW[COUNTRY_NAME="Japan" or COUNTRY_NAME="India"]'::text) PASSING (xmldata.data) COLUMNS "COUNTRY_NAME" text, "REGION_ID" integer) - Filter: ("xmltable"."COUNTRY_NAME" = 'Japan'::text) -(8 rows) - --- should to work with more data -INSERT INTO xmldata VALUES(' - - CZ - Czech Republic - 2Milos Zeman - - - DE - Germany - 2 - - - FR - France - 2 - -'); -INSERT INTO xmldata VALUES(' - - EG - Egypt - 1 - - - SD - Sudan - 1 - -'); -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); - id | _id | country_name | country_id | region_id | size | unit | premier_name -----+-----+----------------+------------+-----------+------+------+--------------- - 1 | 1 | Australia | AU | 3 | | | not specified - 2 | 2 | China | CN | 3 | | | not specified - 3 | 3 | HongKong | HK | 3 | | | not specified - 4 | 4 | India | IN | 3 | | | not specified - 5 | 5 | Japan | JP | 3 | | | Sinzo Abe - 6 | 6 | Singapore | SG | 3 | 791 | km | not specified - 10 | 1 | Czech Republic | CZ | 2 | | | Milos Zeman - 11 | 2 | Germany | DE | 2 | | | not specified - 12 | 3 | France | FR | 2 | | | not specified - 20 | 1 | Egypt | EG | 1 | | | not specified - 21 | 2 | Sudan | SD | 1 | | | not specified -(11 rows) - -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') - WHERE region_id = 2; - id | _id | country_name | country_id | region_id | size | unit | premier_name -----+-----+----------------+------------+-----------+------+------+--------------- - 10 | 1 | Czech Republic | CZ | 2 | | | Milos Zeman - 11 | 2 | Germany | DE | 2 | | | not specified - 12 | 3 | France | FR | 2 | | | not specified -(3 rows) - -EXPLAIN (VERBOSE, COSTS OFF) -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE', - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified') - WHERE region_id = 2; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - Nested Loop - Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name - -> Seq Scan on public.xmldata - Output: xmldata.data - -> Table Function Scan on "xmltable" - Output: "xmltable".id, "xmltable"._id, "xmltable".country_name, "xmltable".country_id, "xmltable".region_id, "xmltable".size, "xmltable".unit, "xmltable".premier_name - Table Function Call: XMLTABLE(('/ROWS/ROW'::text) PASSING (xmldata.data) COLUMNS id integer PATH ('@id'::text), _id FOR ORDINALITY, country_name text PATH ('COUNTRY_NAME'::text) NOT NULL, country_id text PATH ('COUNTRY_ID'::text), region_id integer PATH ('REGION_ID'::text), size double precision PATH ('SIZE'::text), unit text PATH ('SIZE/@unit'::text), premier_name text DEFAULT ('not specified'::text) PATH ('PREMIER_NAME'::text)) - Filter: ("xmltable".region_id = 2) -(8 rows) - --- should fail, NULL value -SELECT xmltable.* - FROM (SELECT data FROM xmldata) x, - LATERAL XMLTABLE('/ROWS/ROW' - PASSING data - COLUMNS id int PATH '@id', - _id FOR ORDINALITY, - country_name text PATH 'COUNTRY_NAME' NOT NULL, - country_id text PATH 'COUNTRY_ID', - region_id int PATH 'REGION_ID', - size float PATH 'SIZE' NOT NULL, - unit text PATH 'SIZE/@unit', - premier_name text PATH 'PREMIER_NAME' DEFAULT 'not specified'); -ERROR: null is not allowed in column "size" --- if all is ok, then result is empty --- one line xml test -WITH - x AS (SELECT proname, proowner, procost::numeric, pronargs, - array_to_string(proargnames,',') as proargnames, - case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes - FROM pg_proc WHERE proname = 'f_leak'), - y AS (SELECT xmlelement(name proc, - xmlforest(proname, proowner, - procost, pronargs, - proargnames, proargtypes)) as proc - FROM x), - z AS (SELECT xmltable.* - FROM y, - LATERAL xmltable('/proc' PASSING proc - COLUMNS proname name, - proowner oid, - procost float, - pronargs int, - proargnames text, - proargtypes text)) - SELECT * FROM z - EXCEPT SELECT * FROM x; - proname | proowner | procost | pronargs | proargnames | proargtypes ----------+----------+---------+----------+-------------+------------- -(0 rows) - --- multi line xml test, result should be empty too -WITH - x AS (SELECT proname, proowner, procost::numeric, pronargs, - array_to_string(proargnames,',') as proargnames, - case when proargtypes <> '' then array_to_string(proargtypes::oid[],',') end as proargtypes - FROM pg_proc), - y AS (SELECT xmlelement(name data, - xmlagg(xmlelement(name proc, - xmlforest(proname, proowner, procost, - pronargs, proargnames, proargtypes)))) as doc - FROM x), - z AS (SELECT xmltable.* - FROM y, - LATERAL xmltable('/data/proc' PASSING doc - COLUMNS proname name, - proowner oid, - procost float, - pronargs int, - proargnames text, - proargtypes text)) - SELECT * FROM z - EXCEPT SELECT * FROM x; - proname | proowner | procost | pronargs | proargnames | proargtypes ----------+----------+---------+----------+-------------+------------- -(0 rows) - -CREATE TABLE xmltest2(x xml, _path text); -INSERT INTO xmltest2 VALUES('1', 'A'); -INSERT INTO xmltest2 VALUES('2', 'B'); -INSERT INTO xmltest2 VALUES('3', 'C'); -INSERT INTO xmltest2 VALUES('2', 'D'); -SELECT xmltable.* FROM xmltest2, LATERAL xmltable('/d/r' PASSING x COLUMNS a int PATH '' || lower(_path) || 'c'); - a ---- - 1 - 2 - 3 - 2 -(4 rows) - -SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH '.'); - a ---- - 1 - 2 - 3 - 2 -(4 rows) - -SELECT xmltable.* FROM xmltest2, LATERAL xmltable(('/d/r/' || lower(_path) || 'c') PASSING x COLUMNS a int PATH 'x' DEFAULT ascii(_path) - 54); - a ----- - 11 - 12 - 13 - 14 -(4 rows) - --- XPath result can be boolean or number too -SELECT * FROM XMLTABLE('*' PASSING 'a' COLUMNS a xml PATH '.', b text PATH '.', c text PATH '"hi"', d boolean PATH '. = "a"', e integer PATH 'string-length(.)'); - a | b | c | d | e -----------+---+----+---+--- - a | a | hi | t | 1 -(1 row) - -\x -SELECT * FROM XMLTABLE('*' PASSING 'pre&deeppost' COLUMNS x xml PATH 'node()', y xml PATH '/'); --[ RECORD 1 ]----------------------------------------------------------- -x | pre&deeppost -y | pre&deeppost+ - | - -\x -SELECT * FROM XMLTABLE('.' PASSING XMLELEMENT(NAME a) columns a varchar(20) PATH '""', b xml PATH '""'); - a | b ---------+-------------- - | <foo/> -(1 row) - diff --git a/src/test/regress/sql/xml.sql b/src/test/regress/sql/xml.sql index ea2de44deb..43a9e414bf 100644 --- a/src/test/regress/sql/xml.sql +++ b/src/test/regress/sql/xml.sql @@ -5,7 +5,14 @@ CREATE TABLE xmltest ( INSERT INTO xmltest VALUES (1, 'one'); INSERT INTO xmltest VALUES (2, 'two'); -INSERT INTO xmltest VALUES (3, 'three '); + +-- If no XML data could be inserted, skip the tests as the server has been +-- compiled without libxml support. +SELECT count(*) = 0 AS skip_test FROM xmltest \gset +\if :skip_test +\quit +\endif SELECT * FROM xmltest; @@ -23,7 +30,7 @@ SELECT xmlconcat(xmlcomment('hello'), SELECT xmlconcat('hello', 'you'); SELECT xmlconcat(1, 2); -SELECT xmlconcat('bad', ' '); SELECT xmlconcat('', NULL, ''); SELECT xmlconcat('', NULL, ''); SELECT xmlconcat(NULL); @@ -68,17 +75,17 @@ SELECT xmlparse(content '&'); SELECT xmlparse(content '&idontexist;'); SELECT xmlparse(content ''); SELECT xmlparse(content ''); -SELECT xmlparse(content '&idontexist;'); +SELECT xmlparse(content '&idontexist; '); SELECT xmlparse(content ''); -SELECT xmlparse(document ' '); +SELECT xmlparse(document '!'); SELECT xmlparse(document 'abc'); SELECT xmlparse(document 'x'); -SELECT xmlparse(document '&'); -SELECT xmlparse(document '&idontexist;'); +SELECT xmlparse(document '& '); +SELECT xmlparse(document '&idontexist; '); SELECT xmlparse(document ''); SELECT xmlparse(document ''); -SELECT xmlparse(document '&idontexist;'); +SELECT xmlparse(document '&idontexist; '); SELECT xmlparse(document ''); From 1b79c8d1a58195c1c6bedd336ce509f0b01c14d7 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 15 Jun 2026 11:28:45 +0300 Subject: [PATCH 42/76] Fix PQdescribePrepared with more than 7498 params If a query has more than 7498 params, the ParameterDescription message exceeds the 30000 byte limit on messages that are not specifically marked as possibly being longer than that (VALID_LONG_MESSAGE_TYPE). To fix, add ParameterDescription to the list. Author: Ning Sun Discussion: https://www.postgresql.org/message-id/dbfb4b65-0aa8-470a-8b87-b6496160b28a@gmail.com Backpatch-through: 14 --- src/interfaces/libpq/fe-protocol3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c index 1d01991329..3c6f927eac 100644 --- a/src/interfaces/libpq/fe-protocol3.c +++ b/src/interfaces/libpq/fe-protocol3.c @@ -39,7 +39,7 @@ */ #define VALID_LONG_MESSAGE_TYPE(id) \ ((id) == 'T' || (id) == 'D' || (id) == 'd' || (id) == 'V' || \ - (id) == 'E' || (id) == 'N' || (id) == 'A') + (id) == 'E' || (id) == 'N' || (id) == 'A' || (id) == 't') static void handleSyncLoss(PGconn *conn, char id, int msgLength); From 924172c56549c39362d64cf01e312a8be229509f Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Mon, 15 Jun 2026 12:16:38 -0500 Subject: [PATCH 43/76] doc: Fix "Prev" link. Presently, the "Prev" link on the page for background workers sends you to the middle of the previous chapter instead of the actual previous page. This appears to be caused by a libxml2 bug, but regardless, a minimal fix is to change the link generation code to use [position()=last()] instead of [last()] in the predicate on the union of reverse axes. Reviewed-by: Ayush Tiwari Discussion: https://postgr.es/m/aim4AZorFKaC7Wrf%40nathan Backpatch-through: 14 --- doc/src/sgml/stylesheet-speedup-xhtml.xsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/stylesheet-speedup-xhtml.xsl b/doc/src/sgml/stylesheet-speedup-xhtml.xsl index da0f2b5a97..a3b3692ba0 100644 --- a/doc/src/sgml/stylesheet-speedup-xhtml.xsl +++ b/doc/src/sgml/stylesheet-speedup-xhtml.xsl @@ -208,7 +208,7 @@ |ancestor::article[1] |ancestor::topic[1] |preceding::sect1[1] - |ancestor::sect1[1])[last()]"/> + |ancestor::sect1[1])[position()=last()]"/> slotname); This is fine as long as options->slotname doesn't contain a double quote mark, but what if it does? In principle this'd allow injection of harmful options into replication commands, in the probably-unlikely case that a slot name comes from untrustworthy input. We ought to clean that up. Moreover, even the places that were trying to be more careful generally got it wrong, because they used quoting subroutines intended for SQL commands rather than something that will work with the replication-command scanner repl_scanner.l. For example, several places naively use PQescapeLiteral() to quote option values for replication commands. If the string contains a backslash, PQescapeLiteral() will produce E'...' literal syntax, which repl_scanner.l doesn't recognize. Another near miss was to use quote_identifier() to quote identifiers. That function won't quote valid lowercase identifiers unless they match SQL keywords ... but in this context, replication keywords are what matter. Neither of these errors seem to risk string injection, but they definitely can cause syntax errors in replication commands that ought to be valid. We can clean all this up by using simple quoting logic that just doubles single or double quotes respectively. Or at least, we could if repl_scanner.l handled doubled double quotes in identifiers, but for some reason it doesn't! So the first step in this fix has to be to fix that. (The fact that we'll later reject slot names containing double quotes is very far short of justifying this omission.) Having done that, this patch runs around and applies correct quoting in all places that generate replication commands containing strings coming from outside the immediate context. Probably some of these places are safe because of restrictions elsewhere, but it seems best to just quote all the time. This was originally reported as a security bug, which it could be if replication slot names or parameters were to originate from untrustworthy sources. But the security team concluded that that was a very improbable situation, so we're just going to fix this as a regular bug. Reported-by: Team Dhiutsa Author: Tom Lane Reviewed-by: Ayush Tiwari Discussion: https://postgr.es/m/1648659.1781287310@sss.pgh.pa.us Backpatch-through: 14 --- src/backend/commands/subscriptioncmds.c | 30 +++++++- .../libpqwalreceiver/libpqwalreceiver.c | 71 ++++++++++--------- src/backend/replication/repl_scanner.l | 4 ++ src/bin/pg_basebackup/pg_recvlogical.c | 14 ++-- src/bin/pg_basebackup/receivelog.c | 28 +++++--- src/bin/pg_basebackup/streamutil.c | 33 +++++++-- src/bin/pg_basebackup/streamutil.h | 6 ++ 7 files changed, 133 insertions(+), 53 deletions(-) diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index bbfd860c40..a0d310c935 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -320,6 +320,32 @@ publicationListToArray(List *publist) return PointerGetDatum(arr); } +/* + * Append a suitably-quoted identifier or string literal to buf. + * "quote" should be either a double-quote or single-quote character. + * + * Caution: this quoting logic is sufficient for identifiers and literals + * in the replication grammar, but not always in regular SQL. Specifically, + * it'd fail for a string literal if standard_conforming_strings is off. + */ +static void +appendQuotedString(StringInfo buf, const char *str, char quote) +{ + appendStringInfoChar(buf, quote); + while (*str) + { + char c = *str++; + + if (c == quote) + appendStringInfoChar(buf, c); + appendStringInfoChar(buf, c); + } + appendStringInfoChar(buf, quote); +} + +#define appendQuotedIdentifier(b, s) appendQuotedString(b, s, '"') +#define appendQuotedLiteral(b, s) appendQuotedString(b, s, '\'') + /* * Create new subscription. */ @@ -1331,7 +1357,9 @@ ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missi load_file("libpqwalreceiver", false); initStringInfo(&cmd); - appendStringInfo(&cmd, "DROP_REPLICATION_SLOT %s WAIT", quote_identifier(slotname)); + appendStringInfoString(&cmd, "DROP_REPLICATION_SLOT "); + appendQuotedIdentifier(&cmd, slotname); + appendStringInfoString(&cmd, " WAIT"); PG_TRY(); { diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 99c5f9f878..98a786348b 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -103,7 +103,7 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = { /* Prototypes for private functions */ static PGresult *libpqrcv_PQexec(PGconn *streamConn, const char *query); static PGresult *libpqrcv_PQgetResult(PGconn *streamConn); -static char *stringlist_to_identifierstr(PGconn *conn, List *strings); +static char *stringlist_to_identifierstr(List *strings); /* * Module initialization function @@ -392,6 +392,32 @@ libpqrcv_server_version(WalReceiverConn *conn) return PQserverVersion(conn->streamConn); } +/* + * Append a suitably-quoted identifier or string literal to buf. + * "quote" should be either a double-quote or single-quote character. + * + * Caution: this quoting logic is sufficient for identifiers and literals + * in the replication grammar, but not always in regular SQL. Specifically, + * it'd fail for a string literal if standard_conforming_strings is off. + */ +static void +appendQuotedString(StringInfo buf, const char *str, char quote) +{ + appendStringInfoChar(buf, quote); + while (*str) + { + char c = *str++; + + if (c == quote) + appendStringInfoChar(buf, c); + appendStringInfoChar(buf, c); + } + appendStringInfoChar(buf, quote); +} + +#define appendQuotedIdentifier(b, s) appendQuotedString(b, s, '"') +#define appendQuotedLiteral(b, s) appendQuotedString(b, s, '\'') + /* * Start streaming WAL data from given streaming options. * @@ -417,8 +443,10 @@ libpqrcv_startstreaming(WalReceiverConn *conn, /* Build the command. */ appendStringInfoString(&cmd, "START_REPLICATION"); if (options->slotname != NULL) - appendStringInfo(&cmd, " SLOT \"%s\"", - options->slotname); + { + appendStringInfoString(&cmd, " SLOT "); + appendQuotedIdentifier(&cmd, options->slotname); + } if (options->logical) appendStringInfoString(&cmd, " LOGICAL"); @@ -433,7 +461,6 @@ libpqrcv_startstreaming(WalReceiverConn *conn, { char *pubnames_str; List *pubnames; - char *pubnames_literal; appendStringInfoString(&cmd, " ("); @@ -445,21 +472,9 @@ libpqrcv_startstreaming(WalReceiverConn *conn, appendStringInfoString(&cmd, ", streaming 'on'"); pubnames = options->proto.logical.publication_names; - pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames); - if (!pubnames_str) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), /* likely guess */ - errmsg("could not start WAL streaming: %s", - pchomp(PQerrorMessage(conn->streamConn))))); - pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str, - strlen(pubnames_str)); - if (!pubnames_literal) - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), /* likely guess */ - errmsg("could not start WAL streaming: %s", - pchomp(PQerrorMessage(conn->streamConn))))); - appendStringInfo(&cmd, ", publication_names %s", pubnames_literal); - PQfreemem(pubnames_literal); + pubnames_str = stringlist_to_identifierstr(pubnames); + appendStringInfoString(&cmd, ", publication_names "); + appendQuotedLiteral(&cmd, pubnames_str); pfree(pubnames_str); if (options->proto.logical.binary && @@ -865,7 +880,8 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, initStringInfo(&cmd); - appendStringInfo(&cmd, "CREATE_REPLICATION_SLOT \"%s\"", slotname); + appendStringInfoString(&cmd, "CREATE_REPLICATION_SLOT "); + appendQuotedIdentifier(&cmd, slotname); if (temporary) appendStringInfoString(&cmd, " TEMPORARY"); @@ -1082,10 +1098,10 @@ libpqrcv_exec(WalReceiverConn *conn, const char *query, * * This is essentially the reverse of SplitIdentifierString. * - * The caller should free the result. + * The caller should pfree the result. */ static char * -stringlist_to_identifierstr(PGconn *conn, List *strings) +stringlist_to_identifierstr(List *strings) { ListCell *lc; StringInfoData res; @@ -1096,21 +1112,12 @@ stringlist_to_identifierstr(PGconn *conn, List *strings) foreach(lc, strings) { char *val = strVal(lfirst(lc)); - char *val_escaped; if (first) first = false; else appendStringInfoChar(&res, ','); - - val_escaped = PQescapeIdentifier(conn, val, strlen(val)); - if (!val_escaped) - { - free(res.data); - return NULL; - } - appendStringInfoString(&res, val_escaped); - PQfreemem(val_escaped); + appendQuotedIdentifier(&res, val); } return res.data; diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 8a075e2d92..ee6cb7b237 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -186,6 +186,10 @@ MANIFEST_CHECKSUMS { return K_MANIFEST_CHECKSUMS; } return IDENT; } +{xddouble} { + addlitchar('"'); + } + {xdinside} { addlit(yytext, yyleng); } diff --git a/src/bin/pg_basebackup/pg_recvlogical.c b/src/bin/pg_basebackup/pg_recvlogical.c index 4eec0d8eee..24529ae36a 100644 --- a/src/bin/pg_basebackup/pg_recvlogical.c +++ b/src/bin/pg_basebackup/pg_recvlogical.c @@ -231,8 +231,9 @@ StreamLogicalLog(void) /* Initiate the replication stream at specified location */ query = createPQExpBuffer(); - appendPQExpBuffer(query, "START_REPLICATION SLOT \"%s\" LOGICAL %X/%X", - replication_slot, LSN_FORMAT_ARGS(startpos)); + appendPQExpBufferStr(query, "START_REPLICATION SLOT "); + AppendQuotedIdentifier(query, replication_slot); + appendPQExpBuffer(query, " LOGICAL %X/%X", LSN_FORMAT_ARGS(startpos)); /* print options if there are any */ if (noptions) @@ -245,11 +246,14 @@ StreamLogicalLog(void) appendPQExpBufferStr(query, ", "); /* write option name */ - appendPQExpBuffer(query, "\"%s\"", options[(i * 2)]); + AppendQuotedIdentifier(query, options[i * 2]); /* write option value if specified */ - if (options[(i * 2) + 1] != NULL) - appendPQExpBuffer(query, " '%s'", options[(i * 2) + 1]); + if (options[i * 2 + 1] != NULL) + { + appendPQExpBufferChar(query, ' '); + AppendQuotedLiteral(query, options[i * 2 + 1]); + } } if (noptions) diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c index 73620e0daf..e89903c790 100644 --- a/src/bin/pg_basebackup/receivelog.c +++ b/src/bin/pg_basebackup/receivelog.c @@ -448,8 +448,7 @@ CheckServerVersionForStreaming(PGconn *conn) bool ReceiveXlogStream(PGconn *conn, StreamCtl *stream) { - char query[128]; - char slotcmd[128]; + PQExpBuffer query; PGresult *res; XLogRecPtr stoppos; @@ -474,7 +473,6 @@ ReceiveXlogStream(PGconn *conn, StreamCtl *stream) if (stream->replication_slot != NULL) { reportFlushPosition = true; - sprintf(slotcmd, "SLOT \"%s\" ", stream->replication_slot); } else { @@ -482,7 +480,6 @@ ReceiveXlogStream(PGconn *conn, StreamCtl *stream) reportFlushPosition = true; else reportFlushPosition = false; - slotcmd[0] = 0; } if (stream->sysidentifier != NULL) @@ -535,8 +532,10 @@ ReceiveXlogStream(PGconn *conn, StreamCtl *stream) */ if (!existsTimeLineHistoryFile(stream)) { - snprintf(query, sizeof(query), "TIMELINE_HISTORY %u", stream->timeline); - res = PQexec(conn, query); + query = createPQExpBuffer(); + appendPQExpBuffer(query, "TIMELINE_HISTORY %u", stream->timeline); + res = PQexec(conn, query->data); + destroyPQExpBuffer(query); if (PQresultStatus(res) != PGRES_TUPLES_OK) { /* FIXME: we might send it ok, but get an error */ @@ -572,11 +571,18 @@ ReceiveXlogStream(PGconn *conn, StreamCtl *stream) return true; /* Initiate the replication stream at specified location */ - snprintf(query, sizeof(query), "START_REPLICATION %s%X/%X TIMELINE %u", - slotcmd, - LSN_FORMAT_ARGS(stream->startpos), - stream->timeline); - res = PQexec(conn, query); + query = createPQExpBuffer(); + appendPQExpBufferStr(query, "START_REPLICATION"); + if (stream->replication_slot != NULL) + { + appendPQExpBufferStr(query, " SLOT "); + AppendQuotedIdentifier(query, stream->replication_slot); + } + appendPQExpBuffer(query, " %X/%X TIMELINE %u", + LSN_FORMAT_ARGS(stream->startpos), + stream->timeline); + res = PQexec(conn, query->data); + destroyPQExpBuffer(query); if (PQresultStatus(res) != PGRES_COPY_BOTH) { pg_log_error("could not send replication command \"%s\": %s", diff --git a/src/bin/pg_basebackup/streamutil.c b/src/bin/pg_basebackup/streamutil.c index f8764a853b..4f7d89704f 100644 --- a/src/bin/pg_basebackup/streamutil.c +++ b/src/bin/pg_basebackup/streamutil.c @@ -498,7 +498,8 @@ CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, Assert(slot_name != NULL); /* Build query */ - appendPQExpBuffer(query, "CREATE_REPLICATION_SLOT \"%s\"", slot_name); + appendPQExpBufferStr(query, "CREATE_REPLICATION_SLOT "); + AppendQuotedIdentifier(query, slot_name); if (is_temporary) appendPQExpBufferStr(query, " TEMPORARY"); if (is_physical) @@ -509,7 +510,8 @@ CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, } else { - appendPQExpBuffer(query, " LOGICAL \"%s\"", plugin); + appendPQExpBufferStr(query, " LOGICAL "); + AppendQuotedIdentifier(query, plugin); if (PQserverVersion(conn) >= 100000) /* pg_recvlogical doesn't use an exported snapshot, so suppress */ appendPQExpBufferStr(query, " NOEXPORT_SNAPSHOT"); @@ -570,8 +572,8 @@ DropReplicationSlot(PGconn *conn, const char *slot_name) query = createPQExpBuffer(); /* Build query */ - appendPQExpBuffer(query, "DROP_REPLICATION_SLOT \"%s\"", - slot_name); + appendPQExpBufferStr(query, "DROP_REPLICATION_SLOT "); + AppendQuotedIdentifier(query, slot_name); res = PQexec(conn, query->data); if (PQresultStatus(res) != PGRES_COMMAND_OK) { @@ -600,6 +602,29 @@ DropReplicationSlot(PGconn *conn, const char *slot_name) } +/* + * Append a suitably-quoted identifier or string literal to buf. + * "quote" should be either a double-quote or single-quote character. + * + * Caution: this quoting logic is sufficient for identifiers and literals + * in the replication grammar, but not always in regular SQL. Specifically, + * it'd fail for a string literal if standard_conforming_strings is off. + */ +void +AppendQuotedString(PQExpBuffer buf, const char *str, char quote) +{ + appendPQExpBufferChar(buf, quote); + while (*str) + { + char c = *str++; + + if (c == quote) + appendPQExpBufferChar(buf, c); + appendPQExpBufferChar(buf, c); + } + appendPQExpBufferChar(buf, quote); +} + /* * Frontend version of GetCurrentTimestamp(), since we are not linked with * backend code. diff --git a/src/bin/pg_basebackup/streamutil.h b/src/bin/pg_basebackup/streamutil.h index 10f87ad0c1..a5ae1b91d6 100644 --- a/src/bin/pg_basebackup/streamutil.h +++ b/src/bin/pg_basebackup/streamutil.h @@ -15,6 +15,7 @@ #include "access/xlogdefs.h" #include "datatype/timestamp.h" #include "libpq-fe.h" +#include "pqexpbuffer.h" extern const char *progname; extern char *connection_string; @@ -40,6 +41,11 @@ extern bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name); + +extern void AppendQuotedString(PQExpBuffer buf, const char *str, char quote); +#define AppendQuotedIdentifier(b, s) AppendQuotedString(b, s, '"') +#define AppendQuotedLiteral(b, s) AppendQuotedString(b, s, '\'') + extern bool RetrieveWalSegSize(PGconn *conn); extern TimestampTz feGetCurrentTimestamp(void); extern void feTimestampDifference(TimestampTz start_time, TimestampTz stop_time, From f528a5606a836289c8cf3faa09874439b6f79c8f Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 16 Jun 2026 09:27:00 +0300 Subject: [PATCH 45/76] Fix int32 overflow in ltree_compare() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The expression (len_diff * 10 * (an + 1)) used as the return value of ltree_compare() is computed at int32 width. With LTREE_MAX_LEVELS = 65535, the product can exceed INT32_MAX once an ltree has more than ~14,653 levels, which causes the result to wrap and invert its sign. That corrupts btree ordering as well as the "magnitude" consumed by ltree_penalty() for GiST page splits. To fix, split ltree_compare() into two functions. The new ltree_compare_distance() function returns a float, which won't overflow. It's used by the ltree_penalty() caller. All the other callers only care about the sign of the return value, i.e. which of the arguments is greater, so change ltree_compare() to not multiply the result with (10 * (an + 1)), which avoids the overflow for those callers. Existing btree or GiST indexes on ltree columns containing values with more than ~14,653 levels may be corrupt and should be REINDEXed. Add a regression test based on the reporter's PoC. Author: Ayush Tiwari Reported-by: 王跃林 Discussion: https://www.postgresql.org/message-id/AI6AnABgKW93Qbx1jVzi84r9.8.1781322625756.Hmail.3020001251%40tju.edu.cn Backpatch-through: 14 --- contrib/ltree/expected/ltree.out | 10 +++++++ contrib/ltree/ltree.h | 1 + contrib/ltree/ltree_gist.c | 6 ++-- contrib/ltree/ltree_op.c | 49 ++++++++++++++++++++++++++++---- contrib/ltree/sql/ltree.sql | 6 ++++ 5 files changed, 63 insertions(+), 9 deletions(-) diff --git a/contrib/ltree/expected/ltree.out b/contrib/ltree/expected/ltree.out index 20f6ad7d0f..4572638265 100644 --- a/contrib/ltree/expected/ltree.out +++ b/contrib/ltree/expected/ltree.out @@ -8113,3 +8113,13 @@ DETAIL: Total size of level exceeds the maximum allowed (65535 bytes). SELECT (repeat('a|', 65535) || 'a')::lquery; ERROR: lquery level has too many variants DETAIL: Number of variants exceeds the maximum allowed (65535). +-- Test that ltree_compare() does not overflow with very deep paths. +WITH s AS (SELECT 'a'::ltree AS v), + l AS (SELECT (repeat('a.', 14999) || 'a')::ltree AS v) +SELECT (l.v > s.v) AS gt_ok, (l.v < s.v) AS lt_ok, (l.v = s.v) AS eq_ok + FROM s, l; + gt_ok | lt_ok | eq_ok +-------+-------+------- + t | f | f +(1 row) + diff --git a/contrib/ltree/ltree.h b/contrib/ltree/ltree.h index c3b38fcde4..31a2d71199 100644 --- a/contrib/ltree/ltree.h +++ b/contrib/ltree/ltree.h @@ -193,6 +193,7 @@ bool ltree_execute(ITEM *curitem, void *checkval, bool calcnot, bool (*chkcond) (void *checkval, ITEM *val)); int ltree_compare(const ltree *a, const ltree *b); +float ltree_compare_distance(const ltree *a, const ltree *b); bool inner_isparent(const ltree *c, const ltree *p); bool compare_subnode(ltree_level *t, char *qn, int len, bool prefix, bool ci); ltree *lca_inner(ltree **a, int len); diff --git a/contrib/ltree/ltree_gist.c b/contrib/ltree/ltree_gist.c index f5b4155594..f14bc3c0c1 100644 --- a/contrib/ltree/ltree_gist.c +++ b/contrib/ltree/ltree_gist.c @@ -261,11 +261,11 @@ ltree_penalty(PG_FUNCTION_ARGS) ltree_gist *newval = (ltree_gist *) DatumGetPointer(((GISTENTRY *) PG_GETARG_POINTER(1))->key); float *penalty = (float *) PG_GETARG_POINTER(2); int siglen = LTREE_GET_SIGLEN(); - int32 cmpr, + float cmpr, cmpl; - cmpl = ltree_compare(LTG_GETLNODE(origval, siglen), LTG_GETLNODE(newval, siglen)); - cmpr = ltree_compare(LTG_GETRNODE(newval, siglen), LTG_GETRNODE(origval, siglen)); + cmpl = ltree_compare_distance(LTG_GETLNODE(origval, siglen), LTG_GETLNODE(newval, siglen)); + cmpr = ltree_compare_distance(LTG_GETRNODE(newval, siglen), LTG_GETRNODE(origval, siglen)); *penalty = Max(cmpl, 0) + Max(cmpr, 0); diff --git a/contrib/ltree/ltree_op.c b/contrib/ltree/ltree_op.c index 778dbf1e98..9367c04d01 100644 --- a/contrib/ltree/ltree_op.c +++ b/contrib/ltree/ltree_op.c @@ -38,6 +38,9 @@ PG_FUNCTION_INFO_V1(ltree2text); PG_FUNCTION_INFO_V1(text2ltree); PG_FUNCTION_INFO_V1(ltreeparentsel); +/* + * btree-comparison function. + */ int ltree_compare(const ltree *a, const ltree *b) { @@ -50,18 +53,52 @@ ltree_compare(const ltree *a, const ltree *b) { int res; - if ((res = memcmp(al->name, bl->name, Min(al->len, bl->len))) == 0) + res = memcmp(al->name, bl->name, Min(al->len, bl->len)); + if (res == 0) + { + if (al->len != bl->len) + return (int) al->len - (int) bl->len; + } + else + return res; + + an--; + bn--; + al = LEVEL_NEXT(al); + bl = LEVEL_NEXT(bl); + } + + return a->numlevel - b->numlevel; +} + +/* + * Returns a "distance" between a and b. If a < b, the distance is negative, + * consistent with the ltree_compare() ordering. + */ +float +ltree_compare_distance(const ltree *a, const ltree *b) +{ + ltree_level *al = LTREE_FIRST(a); + ltree_level *bl = LTREE_FIRST(b); + int an = a->numlevel; + int bn = b->numlevel; + + while (an > 0 && bn > 0) + { + int res; + + res = memcmp(al->name, bl->name, Min(al->len, bl->len)); + if (res == 0) { if (al->len != bl->len) - return (al->len - bl->len) * 10 * (an + 1); + return (float) (al->len - bl->len) * 10.0 * (an + 1); } else { if (res < 0) - res = -1; + return -1.0 * 10.0 * (an + 1); else - res = 1; - return res * 10 * (an + 1); + return 1.0 * 10.0 * (an + 1); } an--; @@ -70,7 +107,7 @@ ltree_compare(const ltree *a, const ltree *b) bl = LEVEL_NEXT(bl); } - return (a->numlevel - b->numlevel) * 10 * (an + 1); + return ((float) (a->numlevel - b->numlevel)) * 10.0 * (an + 1); } #define RUNCMP \ diff --git a/contrib/ltree/sql/ltree.sql b/contrib/ltree/sql/ltree.sql index b187b53c52..f47f81238d 100644 --- a/contrib/ltree/sql/ltree.sql +++ b/contrib/ltree/sql/ltree.sql @@ -404,3 +404,9 @@ SELECT (repeat('x', 255) || repeat('|' || repeat('x', 255), 256))::lquery; --- Test for overflow of lquery_level.numvar, with a set of single-char --- variants in one level. SELECT (repeat('a|', 65535) || 'a')::lquery; + +-- Test that ltree_compare() does not overflow with very deep paths. +WITH s AS (SELECT 'a'::ltree AS v), + l AS (SELECT (repeat('a.', 14999) || 'a')::ltree AS v) +SELECT (l.v > s.v) AS gt_ok, (l.v < s.v) AS lt_ok, (l.v = s.v) AS eq_ok + FROM s, l; From 0939aad564029ff78e1fb9b79bf8b1dd1c0b38d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Herrera?= Date: Tue, 16 Jun 2026 18:13:15 +0200 Subject: [PATCH 46/76] logical decoding: Correctly free speculative insertion The error path in ReorderBufferProcessTXN was not freeing (reorderbuffer.c's representation of) a speculative insertion record correctly. In assert-enabled builds, this leads to an assertion failure. In production builds, I see no effect; there may be a small transient leak, but in an improbable code path such as this, such a leak is not of any significance. For users running with assertions enabled, the crash is annoying. Fix by having ReorderBufferProcessTXN() free the speculative insert ahead of freeing the rest of the transaction, and no longer try to handle that insert as a separate argument to ReorderBufferResetTXN(). This code came in with commit 7259736a6e5b (14-era). Backpatch all the way back. In branches 14-16, also backpatch the assertion that originally fails in the problem scenario, which was added by dbed2e36625d (originally backpatched to 17), that at the end of ReorderBufferReturnTXN() the in-memory size of the transaction is zero. Author: Vishal Prasanna Reviewed-by: Hayato Kuroda Backpatch-through: 14 Discussion: https://postgr.es/m/19c7623e882.4080fd5426212.311756747309556767@zohocorp.com --- .../replication/logical/reorderbuffer.c | 25 +++++------ src/test/subscription/t/100_bugs.pl | 43 ++++++++++++++++++- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index b708877d96..bdcd428894 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -474,6 +474,9 @@ ReorderBufferReturnTXN(ReorderBuffer *rb, ReorderBufferTXN *txn) /* Reset the toast hash */ ReorderBufferToastReset(rb, txn); + /* All changes must be deallocated */ + Assert(txn->size == 0); + pfree(txn); } @@ -2029,8 +2032,7 @@ static void ReorderBufferResetTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, Snapshot snapshot_now, CommandId command_id, - XLogRecPtr last_lsn, - ReorderBufferChange *specinsert) + XLogRecPtr last_lsn) { /* Discard the changes that we just streamed */ ReorderBufferTruncateTXN(rb, txn, rbtxn_prepared(txn)); @@ -2038,13 +2040,6 @@ ReorderBufferResetTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, /* Free all resources allocated for toast reconstruction */ ReorderBufferToastReset(rb, txn); - /* Return the spec insert change if it is not NULL */ - if (specinsert != NULL) - { - ReorderBufferReturnChange(rb, specinsert, true); - specinsert = NULL; - } - /* * For the streaming case, stop the stream and remember the command ID and * snapshot for the streaming run. @@ -2303,7 +2298,7 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, * CheckTableNotInUse() and locking. */ - /* clear out a pending (and thus failed) speculation */ + /* clear out a pending (= failed) speculative insertion */ if (specinsert != NULL) { ReorderBufferReturnChange(rb, specinsert, true); @@ -2589,6 +2584,13 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, if (using_subtxn) RollbackAndReleaseCurrentSubTransaction(); + /* Free the specinsert change before freeing the ReorderBufferTXN */ + if (specinsert != NULL) + { + ReorderBufferReturnChange(rb, specinsert, true); + specinsert = NULL; + } + /* * The error code ERRCODE_TRANSACTION_ROLLBACK indicates a concurrent * abort of the (sub)transaction we are streaming or preparing. We @@ -2615,8 +2617,7 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn, /* Reset the TXN so that it is allowed to stream remaining data. */ ReorderBufferResetTXN(rb, txn, snapshot_now, - command_id, prev_lsn, - specinsert); + command_id, prev_lsn); } else { diff --git a/src/test/subscription/t/100_bugs.pl b/src/test/subscription/t/100_bugs.pl index 235227c772..81ff79fce4 100644 --- a/src/test/subscription/t/100_bugs.pl +++ b/src/test/subscription/t/100_bugs.pl @@ -6,7 +6,7 @@ use warnings; use PostgresNode; use TestLib; -use Test::More tests => 11; +use Test::More tests => 12; # Bug #15114 @@ -392,3 +392,44 @@ $node_publisher->safe_psql('postgres', "DROP DATABASE regress_db"); $node_publisher->stop('fast'); + +# https://postgr.es/m/19c7623e882.4080fd5426212.311756747309556767%40zohocorp.com + +# The bug was that when an ERROR was raised while processing an INSERT ... ON +# CONFLICT statement, the decoded change misses to be free'd. This can cause an +# assertion failure if enabled. + +$node_publisher->rotate_logfile(); +$node_publisher->start(); + +$node_publisher->safe_psql( + 'postgres', qq( + CREATE TABLE tab_upsert (a INT PRIMARY KEY, b INT); + SELECT * FROM pg_create_logical_replication_slot('upsert_slot', 'pgoutput'); + INSERT INTO tab_upsert (a, b) VALUES (1, 1) + ON CONFLICT(a) DO UPDATE SET b = excluded.b; +)); + +# Decode the changes without a publication and +# verify that the logical decoder doesn't crash. +($ret, $stdout, $stderr) = $node_publisher->psql( + 'postgres', qq( + SELECT * + FROM pg_logical_slot_peek_binary_changes( + 'upsert_slot', + NULL, + NULL, + 'proto_version', '1', + 'publication_names', 'pub_that_does_not_exist' + ); +)); + +ok( $stderr =~ qr/publication "pub_that_does_not_exist" does not exist/, + 'peek logical changes with non-existent publication throws error' +); + +# Clean up +$node_publisher->safe_psql('postgres', "SELECT pg_drop_replication_slot('upsert_slot')"); +$node_publisher->safe_psql('postgres', "DROP TABLE tab_upsert"); + +$node_publisher->stop('fast'); From d75146456fa5ebcb50a084cd4e2db61ab108c233 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 17 Jun 2026 08:42:13 +0900 Subject: [PATCH 47/76] Fix another instability in recovery TAP test 004_timeline_switch The test did not wait for the standby to be connected to the primary. This breaks one assumption at the beginning of the test, where the primary is stopped to ensure that all its records are flushed to both standbys before moving on with its next steps. If standby_1 finishes ahead of standby_2, the test would be able work fine as the former waits for the latter. The opposite is not true, standby_2 getting ahead of standby_1 would cause the test to fail on timeout when standby_1 attempts to connect to standby_2. This commit adds an additional polling query after the two standbys are started, checking that both standbys are connected to the primary before processing with the initial steps of the test. Like 7185eddf0522, backpatch down to v14. Author: Sergey Tatarintsev Reviewed-by: Ewan Young Discussion: https://postgr.es/m/fea4190e-f8b5-4432-a52d-bcbee5f34366@postgrespro.ru Backpatch-through: 14 --- src/test/recovery/t/004_timeline_switch.pl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/recovery/t/004_timeline_switch.pl b/src/test/recovery/t/004_timeline_switch.pl index bbff6e97cd..ef99485473 100644 --- a/src/test/recovery/t/004_timeline_switch.pl +++ b/src/test/recovery/t/004_timeline_switch.pl @@ -33,6 +33,10 @@ has_streaming => 1); $node_standby_2->start; +# Wait for standby_1 and standby_2 connection to the primary. +$node_primary->poll_query_until('postgres', + "SELECT count(1) = 2 FROM pg_stat_replication"); + # Create some content on primary $node_primary->safe_psql('postgres', "CREATE TABLE tab_int AS SELECT generate_series(1,1000) AS a"); From 639fff5118a846fdb604824a87f4d77cc286d89d Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 17 Jun 2026 11:04:41 -0400 Subject: [PATCH 48/76] jsonb_plperl, jsonb_plpython: Fix unguarded recursion and loops. Add check_stack_depth() to Jsonb_to_SV, SV_to_JsonbValue, PLyObject_FromJsonbContainer, and PLyObject_ToJsonbValue. Without this, deeply nested JSONB values can crash the backend with SIGSEGV instead of raising a proper error. Also add CHECK_FOR_INTERRUPTS() to the while loop in SV_to_JsonbValue that dereferences chains of Perl references, so that a circular reference (e.g. $x = \$x) can be cancelled by the user instead of spinning indefinitely. (We looked at detecting such circular references, but it seems more trouble than it's worth.) Author: Aleksander Alekseev Reviewed-by: Tom Lane Discussion: https://postgr.es/m/CAJ7c6TPbjkzUk4qJ5dHvDNEz0hBuFue3A-XWz_=897z+BC+z8A@mail.gmail.com Backpatch-through: 14 --- contrib/jsonb_plperl/jsonb_plperl.c | 15 +++++++++++++++ contrib/jsonb_plpython/jsonb_plpython.c | 7 +++++++ 2 files changed, 22 insertions(+) diff --git a/contrib/jsonb_plperl/jsonb_plperl.c b/contrib/jsonb_plperl/jsonb_plperl.c index 22e90afe1b..2f585f083a 100644 --- a/contrib/jsonb_plperl/jsonb_plperl.c +++ b/contrib/jsonb_plperl/jsonb_plperl.c @@ -3,6 +3,7 @@ #include #include "fmgr.h" +#include "miscadmin.h" #include "plperl.h" #include "plperl_helpers.h" #include "utils/fmgrprotos.h" @@ -64,6 +65,9 @@ Jsonb_to_SV(JsonbContainer *jsonb) JsonbIterator *it; JsonbIteratorToken r; + /* this can recurse via JsonbValue_to_SV() */ + check_stack_depth(); + it = JsonbIteratorInit(jsonb); r = JsonbIteratorNext(&it, &v, true); @@ -177,9 +181,20 @@ SV_to_JsonbValue(SV *in, JsonbParseState **jsonb_state, bool is_elem) dTHX; JsonbValue out; /* result */ + /* this can recurse via AV_to_JsonbValue() or HV_to_JsonbValue() */ + check_stack_depth(); + /* Dereference references recursively. */ while (SvROK(in)) + { + /* + * It's possible for circular references to make this an infinite + * loop. Checking for such a situation seems like much more trouble + * than it's worth, but let's provide a way to break out of the loop. + */ + CHECK_FOR_INTERRUPTS(); in = SvRV(in); + } switch (SvTYPE(in)) { diff --git a/contrib/jsonb_plpython/jsonb_plpython.c b/contrib/jsonb_plpython/jsonb_plpython.c index 836c178770..8d95f09f23 100644 --- a/contrib/jsonb_plpython/jsonb_plpython.c +++ b/contrib/jsonb_plpython/jsonb_plpython.c @@ -1,5 +1,6 @@ #include "postgres.h" +#include "miscadmin.h" #include "plpy_elog.h" #include "plpy_typeio.h" #include "plpython.h" @@ -145,6 +146,9 @@ PLyObject_FromJsonbContainer(JsonbContainer *jsonb) JsonbIterator *it; PyObject *result; + /* this can recurse via PLyObject_FromJsonbValue() */ + check_stack_depth(); + it = JsonbIteratorInit(jsonb); r = JsonbIteratorNext(&it, &v, true); @@ -415,6 +419,9 @@ PLyObject_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state, bool is_ele { JsonbValue *out; + /* this can recurse via PLyMapping_ToJsonbValue() */ + check_stack_depth(); + if (!(PyString_Check(obj) || PyUnicode_Check(obj))) { if (PySequence_Check(obj)) From 3640143270a9f1f115cf03a16bee3fd469ba1116 Mon Sep 17 00:00:00 2001 From: Amit Langote Date: Thu, 18 Jun 2026 15:51:38 +0900 Subject: [PATCH 49/76] Report undefined jsonpath variable when no variables are supplied The two-argument jsonb @? and @@ operators invoke the jsonpath executor with no variable set. In that case getJsonPathVariable() treated any "$name" reference as JSON null and continued evaluating, instead of reporting the variable as undefined. This produced incorrect results -- for example '42'::jsonb @? '$"x"' returned true -- and, for some malformed or hostile jsonpath expressions with deeply nested predicates, allowed essentially unbounded memory consumption that could get the backend killed by the OOM killer. Report the undefined variable as an error in this case as well, reusing the message already emitted when a variable is not found among supplied variables. This matches the behavior of v17 and later, where the jsonpath executor was reorganized. Stopping at the first undefined variable reference also resolves the reported memory-growth case. Note this is a user-visible change in the back branches: a jsonpath expression that references a variable while no variables are supplied now raises an error rather than silently evaluating it as NULL. The previous behavior was incorrect, so the change is judged worthwhile. Bug: #19458 Reported-by: Andrey Rachitskiy Author: Andrey Rachitskiy Reviewed-by: Andrey Borodin Reviewed-by: Nikita Malakhov Reviewed-by: Amit Langote Discussion: https://postgr.es/m/19458-a69c98bc498333ba@postgresql.org Backpatch-through: 14-16 --- src/backend/utils/adt/jsonpath_exec.c | 13 +++++++------ src/test/regress/expected/jsonb_jsonpath.out | 7 +++++++ src/test/regress/sql/jsonb_jsonpath.sql | 5 +++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c index 10ec66c629..34420849ea 100644 --- a/src/backend/utils/adt/jsonpath_exec.c +++ b/src/backend/utils/adt/jsonpath_exec.c @@ -2128,14 +2128,15 @@ getJsonPathVariable(JsonPathExecContext *cxt, JsonPathItem *variable, JsonbValue tmp; JsonbValue *v; - if (!vars) - { - value->type = jbvNull; - return; - } - Assert(variable->type == jpiVariable); varName = jspGetString(variable, &varNameLength); + + if (!vars) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("could not find jsonpath variable \"%s\"", + pnstrdup(varName, varNameLength)))); + tmp.type = jbvString; tmp.val.string.val = varName; tmp.val.string.len = varNameLength; diff --git a/src/test/regress/expected/jsonb_jsonpath.out b/src/test/regress/expected/jsonb_jsonpath.out index 6659bc9091..eafb421c7a 100644 --- a/src/test/regress/expected/jsonb_jsonpath.out +++ b/src/test/regress/expected/jsonb_jsonpath.out @@ -487,6 +487,13 @@ select * from jsonb_path_query('{"a": 10}', '$'); select * from jsonb_path_query('{"a": 10}', '$ ? (@.a < $value)'); ERROR: could not find jsonpath variable "value" +-- the @? and @@ operators supply no variables, so a variable reference +-- must be reported as undefined rather than silently treated as NULL +-- (the latter gave wrong results and could drive unbounded memory use) +select jsonb '42' @? '$"no_such_var"'; +ERROR: could not find jsonpath variable "no_such_var" +select jsonb '42' @@ '$"no_such_var" == 1'; +ERROR: could not find jsonpath variable "no_such_var" select * from jsonb_path_query('{"a": 10}', '$ ? (@.a < $value)', '1'); ERROR: "vars" argument is not an object DETAIL: Jsonpath parameters should be encoded as key-value pairs of "vars" object. diff --git a/src/test/regress/sql/jsonb_jsonpath.sql b/src/test/regress/sql/jsonb_jsonpath.sql index e0ce509264..8163fc6713 100644 --- a/src/test/regress/sql/jsonb_jsonpath.sql +++ b/src/test/regress/sql/jsonb_jsonpath.sql @@ -98,6 +98,11 @@ select jsonb_path_query('[1,2,3]', '$[last ? (@.type() == "string")]', silent => select * from jsonb_path_query('{"a": 10}', '$'); select * from jsonb_path_query('{"a": 10}', '$ ? (@.a < $value)'); +-- the @? and @@ operators supply no variables, so a variable reference +-- must be reported as undefined rather than silently treated as NULL +-- (the latter gave wrong results and could drive unbounded memory use) +select jsonb '42' @? '$"no_such_var"'; +select jsonb '42' @@ '$"no_such_var" == 1'; select * from jsonb_path_query('{"a": 10}', '$ ? (@.a < $value)', '1'); select * from jsonb_path_query('{"a": 10}', '$ ? (@.a < $value)', '[{"value" : 13}]'); select * from jsonb_path_query('{"a": 10}', '$ ? (@.a < $value)', '{"value" : 13}'); From 1d216f1e772369c0275b46029821de6802fe40b9 Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Thu, 18 Jun 2026 09:31:27 -0500 Subject: [PATCH 50/76] doc: Fix "Prev" link, take 2. Commit 6678b58d78 fixed a wrong "Prev" link by changing the link generation code to use [position()=last()] instead of [last()] in the predicate on the union of reverse axes. Unfortunately, that caused documentation builds to take much longer. To fix, combine the "preceding" and "ancestor" steps into one "preceding" step and one "ancestor" step, and revert the predicate back to [last()]. The smaller union evades the libxml2 bug while avoiding the build time regression. Reported-by: Tom Lane Tested-by: Tom Lane Discussion: https://postgr.es/m/1132496.1781718007%40sss.pgh.pa.us Backpatch-through: 14 --- doc/src/sgml/stylesheet-speedup-xhtml.xsl | 52 +++++++++++------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/doc/src/sgml/stylesheet-speedup-xhtml.xsl b/doc/src/sgml/stylesheet-speedup-xhtml.xsl index a3b3692ba0..0e8e97c7e5 100644 --- a/doc/src/sgml/stylesheet-speedup-xhtml.xsl +++ b/doc/src/sgml/stylesheet-speedup-xhtml.xsl @@ -183,32 +183,32 @@ + select="(preceding::*[self::book + or self::preface + or self::chapter + or self::appendix + or self::part + or self::reference + or self::refentry + or self::colophon + or self::article + or self::topic + or self::sect1 + or self::bibliography[parent::article or parent::book or parent::part] + or self ::glossary[parent::article or parent::book or parent::part] + or self::index[$generate.index != 0] + [parent::article or parent::book or parent::part] + or self::setindex[$generate.index != 0]][1] + |ancestor::*[self::set + or self::book + or self::preface + or self::chapter + or self::appendix + or self::part + or self::reference + or self::article + or self::topic + or self::sect1][1])[last()]"/> Date: Thu, 18 Jun 2026 11:29:49 -0500 Subject: [PATCH 52/76] Silence "may be used uninitialized" compiler warning. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Newer gcc warns that this "actual_arg_types" variable may be used uninitialized, but visual inspection indicates there's no bug. To silence the warning, initialize the variable to zeros. Bug: #19485 Reported-by: Hans Buschmann Tested-by: Erik Rijkers Tested-by: Hans Buschmann Reviewed-by: Tristan Partin Reviewed-by: Álvaro Herrera Discussion: https://postgr.es/m/19485-2b03231a775756f1%40postgresql.org Discussion: https://postgr.es/m/6c52a1a6612948519468d46cb224a8c4%40nidsa.net --- src/backend/optimizer/util/clauses.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c index 8b48e8830c..af63bf66da 100644 --- a/src/backend/optimizer/util/clauses.c +++ b/src/backend/optimizer/util/clauses.c @@ -4216,7 +4216,7 @@ recheck_cast_function_args(List *args, Oid result_type, { Form_pg_proc funcform = (Form_pg_proc) GETSTRUCT(func_tuple); int nargs; - Oid actual_arg_types[FUNC_MAX_ARGS]; + Oid actual_arg_types[FUNC_MAX_ARGS] = {0}; Oid declared_arg_types[FUNC_MAX_ARGS]; Oid rettype; ListCell *lc; From 4b3bc6b714531438cee299648f12e70c5cc38ebb Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 19 Jun 2026 12:52:00 -0400 Subject: [PATCH 53/76] Make pg_mkdir_p() tolerant of a concurrent directory creation. pg_mkdir_p creates each missing path component with a stat() followed by mkdir(). If the stat() reports the component as absent but another process creates it in the window before this process's mkdir(), mkdir() fails with EEXIST and pg_mkdir_p treated that as a hard error -- unlike "mkdir -p", which is meant to be idempotent and race-tolerant. This shows up when several processes concurrently create paths that share an ancestor directory: for example, parallel initdb runs whose data directories live under a common temporary directory. One process wins the race to create the shared ancestor and the others fail with could not create directory "...": File exists Fix this race condition by first trying mkdir() and only attempting stat() if it fails with EEXIST. On Windows, there's an additional problem: stat() opens a file handle and participates in share-mode locking, which means it can transiently fail on a directory another process is concurrently creating. Use GetFileAttributes() instead: it requests only FILE_READ_ATTRIBUTES and is exempt from share-mode denial, so it reliably sees a concurrently-created directory. I (tgl) also chose to back-patch 039f7ee0f's effects on this function, so that pgmkdirp.c remains identical in all live branches. Author: Andrew Dunstan Co-authored-by: Tom Lane Discussion: https://postgr.es/m/3ca004de-e49b-4471-b8aa-fd656e70f68c@dunslane.net Backpatch-through: 14 --- src/port/pgmkdirp.c | 49 ++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/src/port/pgmkdirp.c b/src/port/pgmkdirp.c index d943559760..3e6b06fce7 100644 --- a/src/port/pgmkdirp.c +++ b/src/port/pgmkdirp.c @@ -56,7 +56,6 @@ int pg_mkdir_p(char *path, int omode) { - struct stat sb; mode_t numask, oumask; int last, @@ -73,7 +72,7 @@ pg_mkdir_p(char *path, int omode) if (p[0] == '/' && p[1] == '/') { /* network drive */ - p = strstr(p + 2, "/"); + p = strchr(p + 2, '/'); if (p == NULL) { errno = EINVAL; @@ -119,24 +118,46 @@ pg_mkdir_p(char *path, int omode) if (last) (void) umask(oumask); - /* check for pre-existing directory */ - if (stat(path, &sb) == 0) + if (mkdir(path, last ? omode : S_IRWXU | S_IRWXG | S_IRWXO) < 0) { - if (!S_ISDIR(sb.st_mode)) + /* + * If we got EEXIST because there's already a directory there, + * don't complain. + */ +#ifndef WIN32 + int save_errno = errno; + struct stat sb; + + if (save_errno != EEXIST || + stat(path, &sb) != 0 || + !S_ISDIR(sb.st_mode)) { - if (last) - errno = EEXIST; - else - errno = ENOTDIR; + /* Don't let stat replace mkdir's errno */ + errno = save_errno; retval = -1; break; } +#else /* WIN32 */ + /* + * On Windows, stat() opens a handle and can transiently fail on a + * directory another process is concurrently creating. Probe with + * a path-based attribute query instead: it requests only + * FILE_READ_ATTRIBUTES and is exempt from share-mode denial, so + * it reliably sees a concurrently-created directory. We assume + * GetFileAttributes() won't change errno. + */ + DWORD attr = GetFileAttributes(path); + + if (errno != EEXIST || + attr == INVALID_FILE_ATTRIBUTES || + !(attr & FILE_ATTRIBUTE_DIRECTORY)) + { + retval = -1; + break; + } +#endif /* WIN32 */ } - else if (mkdir(path, last ? omode : S_IRWXU | S_IRWXG | S_IRWXO) < 0) - { - retval = -1; - break; - } + if (!last) *p = '/'; } From e8fbe5d838f50d0641d40f98b44f3698ad0846c1 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 23 Jun 2026 16:52:21 +0900 Subject: [PATCH 54/76] doc: Describe better handling of indexes in ALTER TABLE ATTACH PARTITION When ALTER TABLE ... ATTACH PARTITION matches partition indexes to the parent table's indexes, invalid indexes are skipped. This commit improves the documentation to describe what e90e9275f56 has changed: invalid indexes are skipped, and only valid indexes are considered for a match. Author: Mohamed Ali Reviewed-by: Sami Imseih Discussion: https://postgr.es/m/CAGnOmWpAMaE-BOkpwM6mJnHcpS2QZ8yLSSaqmz+vryEsbCWWWA@mail.gmail.com Backpatch-through: 14 --- doc/src/sgml/ref/alter_table.sgml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 4a2f46d239..63421cae16 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -932,10 +932,11 @@ WITH ( MODULUS numeric_literal, REM as a partition of the target table. The table can be attached as a partition for specific values using FOR VALUES or as a default partition by using DEFAULT. - For each index in the target table, a corresponding - one will be created in the attached table; or, if an equivalent - index already exists, it will be attached to the target table's index, - as if ALTER INDEX ATTACH PARTITION had been executed. + For each index in the target table, if a valid equivalent index + already exists in the partition, it will be attached to the target + table's index, as if ALTER INDEX ATTACH PARTITION had been executed; + otherwise, a new corresponding index will be created. Invalid indexes + on the partition are skipped. Note that if the existing table is a foreign table, it is currently not allowed to attach the table as a partition of the target table if there are UNIQUE indexes on the target table. (See also From e520ad34b482cc3a441f10ded0320d42db2625ed Mon Sep 17 00:00:00 2001 From: Richard Guo Date: Wed, 24 Jun 2026 09:09:48 +0900 Subject: [PATCH 55/76] plperl: Fix NULL pointer dereference for forged array object In get_perl_array_ref(), for a PostgreSQL::InServer::ARRAY object, we look up its "array" key with hv_fetch_string() and then inspect the returned SV. However, hv_fetch_string() returns a NULL pointer when the key is absent, and the code dereferenced that result without first checking whether the pointer itself was NULL. As a result, a plperl function returning a forged PostgreSQL::InServer::ARRAY object that lacks the "array" key would crash the backend with a segmentation fault. Fix this by checking the pointer returned by hv_fetch_string() before dereferencing it, matching how other callers in this file already guard the result. With the check in place, such an object falls through to the existing error report instead of crashing. Author: Xing Guo Reviewed-by: Richard Guo Discussion: https://postgr.es/m/CACpMh+DYgcnqZwQLXXuxQcehJTd7T8UmKWSLsK4mFBEp9G2ajA@mail.gmail.com Backpatch-through: 14 --- src/pl/plperl/expected/plperl_array.out | 7 +++++++ src/pl/plperl/plperl.c | 2 +- src/pl/plperl/sql/plperl_array.sql | 7 +++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/pl/plperl/expected/plperl_array.out b/src/pl/plperl/expected/plperl_array.out index bd04a062fb..03e5629d69 100644 --- a/src/pl/plperl/expected/plperl_array.out +++ b/src/pl/plperl/expected/plperl_array.out @@ -274,3 +274,10 @@ select perl_setof_array('{{1}, {2}, {3}}'); {3} (3 rows) +-- Test a forged PostgreSQL::InServer::ARRAY object lacking the 'array' key +CREATE OR REPLACE FUNCTION perl_forged_array() RETURNS integer[] AS $$ + return bless {}, "PostgreSQL::InServer::ARRAY"; +$$ LANGUAGE plperl; +SELECT perl_forged_array(); +ERROR: could not get array reference from PostgreSQL::InServer::ARRAY object +CONTEXT: PL/Perl function "perl_forged_array" diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c index fe6efdb374..b3834f7142 100644 --- a/src/pl/plperl/plperl.c +++ b/src/pl/plperl/plperl.c @@ -1151,7 +1151,7 @@ get_perl_array_ref(SV *sv) HV *hv = (HV *) SvRV(sv); SV **sav = hv_fetch_string(hv, "array"); - if (*sav && SvOK(*sav) && SvROK(*sav) && + if (sav && *sav && SvOK(*sav) && SvROK(*sav) && SvTYPE(SvRV(*sav)) == SVt_PVAV) return *sav; diff --git a/src/pl/plperl/sql/plperl_array.sql b/src/pl/plperl/sql/plperl_array.sql index ca63b5db62..cd1d7e34c5 100644 --- a/src/pl/plperl/sql/plperl_array.sql +++ b/src/pl/plperl/sql/plperl_array.sql @@ -206,3 +206,10 @@ create or replace function perl_setof_array(integer[]) returns setof integer[] l $$; select perl_setof_array('{{1}, {2}, {3}}'); + +-- Test a forged PostgreSQL::InServer::ARRAY object lacking the 'array' key +CREATE OR REPLACE FUNCTION perl_forged_array() RETURNS integer[] AS $$ + return bless {}, "PostgreSQL::InServer::ARRAY"; +$$ LANGUAGE plperl; + +SELECT perl_forged_array(); From 0b7719f744e694a2a1946f7ddf230bf4fdfad20c Mon Sep 17 00:00:00 2001 From: Richard Guo Date: Mon, 29 Jun 2026 11:44:35 +0900 Subject: [PATCH 56/76] plpython: Fix NULL pointer dereferences for broken sequence and mapping objects PL/Python and its hstore and jsonb transforms build SQL values from Python containers by calling Python C API functions that can return NULL, and in several places the result was used without first checking it. On the sequence side, PySequence_GetItem() is used when converting a returned sequence into a SQL array or composite value, when reading the argument list passed to plpy.execute() or plpy.cursor(), and when reading the list of type names given to plpy.prepare(). On the mapping side, the hstore and jsonb transforms call PyMapping_Size() and PyMapping_Items() and then index the result with PyList_GetItem() and PyTuple_GetItem(). All of these return NULL (or -1), with a Python exception set, for a broken object: for example one whose __getitem__() or items() raises, or which reports a length that disagrees with what it actually yields. The unchecked result was then dereferenced, crashing the backend. Fix this by checking the result of each call and reporting a regular error if it failed, so that the underlying Python exception is surfaced instead of taking down the session. Author: Richard Guo Reviewed-by: Ayush Tiwari Discussion: https://postgr.es/m/CAMbWs49BKM9wP6m8bCXEpHwQKp7usvOGV6Jf=J7FYr_BCpxLqg@mail.gmail.com Backpatch-through: 14 --- .../expected/hstore_plpython.out | 65 ++++++++++++++ contrib/hstore_plpython/hstore_plpython.c | 16 ++++ .../hstore_plpython/sql/hstore_plpython.sql | 65 ++++++++++++++ .../expected/jsonb_plpython.out | 89 +++++++++++++++++++ contrib/jsonb_plpython/jsonb_plpython.c | 21 ++++- contrib/jsonb_plpython/sql/jsonb_plpython.sql | 77 ++++++++++++++++ .../plpython/expected/plpython_composite.out | 16 ++++ src/pl/plpython/expected/plpython_spi.out | 51 +++++++++++ src/pl/plpython/expected/plpython_types.out | 16 ++++ src/pl/plpython/expected/plpython_types_3.out | 16 ++++ src/pl/plpython/plpy_cursorobject.c | 5 ++ src/pl/plpython/plpy_spi.c | 10 +++ src/pl/plpython/plpy_typeio.c | 9 +- src/pl/plpython/sql/plpython_composite.sql | 12 +++ src/pl/plpython/sql/plpython_spi.sql | 39 ++++++++ src/pl/plpython/sql/plpython_types.sql | 13 +++ 16 files changed, 516 insertions(+), 4 deletions(-) diff --git a/contrib/hstore_plpython/expected/hstore_plpython.out b/contrib/hstore_plpython/expected/hstore_plpython.out index 57d83fa2db..b1837715ce 100644 --- a/contrib/hstore_plpython/expected/hstore_plpython.out +++ b/contrib/hstore_plpython/expected/hstore_plpython.out @@ -43,6 +43,71 @@ SELECT test1bad(); ERROR: not a Python mapping CONTEXT: while creating return value PL/Python function "test1bad" +-- A mapping whose items() raises should be reported as an error, not crash +-- the backend +CREATE FUNCTION test1broken() RETURNS hstore +LANGUAGE plpython3u +TRANSFORM FOR TYPE hstore +AS $$ +class C(dict): + def items(self): + raise ValueError('items failed') +d = C() +d['x'] = 1 +return d +$$; +SELECT test1broken(); +ERROR: could not get items from Python mapping +CONTEXT: while creating return value +PL/Python function "test1broken" +-- Likewise for a mapping whose items() does not return key/value pairs +CREATE FUNCTION test1malformed() RETURNS hstore +LANGUAGE plpython3u +TRANSFORM FOR TYPE hstore +AS $$ +class C(dict): + def items(self): + return [42] +d = C() +d['x'] = 1 +return d +$$; +SELECT test1malformed(); +ERROR: items() of a Python mapping must return key/value pairs +CONTEXT: while creating return value +PL/Python function "test1malformed" +-- Likewise for a mapping whose items() yields fewer pairs than its length +CREATE FUNCTION test1short() RETURNS hstore +LANGUAGE plpython3u +TRANSFORM FOR TYPE hstore +AS $$ +class C(dict): + def items(self): + return [] +d = C() +d['x'] = 1 +return d +$$; +SELECT test1short(); +ERROR: items() of a Python mapping must return key/value pairs +CONTEXT: while creating return value +PL/Python function "test1short" +-- Likewise for a mapping whose __len__() raises +CREATE FUNCTION test1brokenlen() RETURNS hstore +LANGUAGE plpython3u +TRANSFORM FOR TYPE hstore +AS $$ +class C(dict): + def __len__(self): + raise ValueError('len failed') +d = C() +d['x'] = 1 +return d +$$; +SELECT test1brokenlen(); +ERROR: could not get size of Python mapping +CONTEXT: while creating return value +PL/Python function "test1brokenlen" -- test hstore[] -> python CREATE FUNCTION test1arr(val hstore[]) RETURNS int LANGUAGE plpythonu diff --git a/contrib/hstore_plpython/hstore_plpython.c b/contrib/hstore_plpython/hstore_plpython.c index 2d144043ab..db67bb3685 100644 --- a/contrib/hstore_plpython/hstore_plpython.c +++ b/contrib/hstore_plpython/hstore_plpython.c @@ -145,7 +145,16 @@ plpython_to_hstore(PG_FUNCTION_ARGS) errmsg("not a Python mapping"))); pcount = PyMapping_Size(dict); + if (pcount < 0) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("could not get size of Python mapping"))); + items = PyMapping_Items(dict); + if (items == NULL) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("could not get items from Python mapping"))); PG_TRY(); { @@ -162,6 +171,13 @@ plpython_to_hstore(PG_FUNCTION_ARGS) PyObject *value; tuple = PyList_GetItem(items, i); + + /* The mapping's items() must yield key/value pairs */ + if (tuple == NULL || !PyTuple_Check(tuple) || PyTuple_Size(tuple) < 2) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("items() of a Python mapping must return key/value pairs"))); + key = PyTuple_GetItem(tuple, 0); value = PyTuple_GetItem(tuple, 1); diff --git a/contrib/hstore_plpython/sql/hstore_plpython.sql b/contrib/hstore_plpython/sql/hstore_plpython.sql index 1aa4416512..8548b7dbe0 100644 --- a/contrib/hstore_plpython/sql/hstore_plpython.sql +++ b/contrib/hstore_plpython/sql/hstore_plpython.sql @@ -38,6 +38,71 @@ $$; SELECT test1bad(); +-- A mapping whose items() raises should be reported as an error, not crash +-- the backend +CREATE FUNCTION test1broken() RETURNS hstore +LANGUAGE plpython3u +TRANSFORM FOR TYPE hstore +AS $$ +class C(dict): + def items(self): + raise ValueError('items failed') +d = C() +d['x'] = 1 +return d +$$; + +SELECT test1broken(); + + +-- Likewise for a mapping whose items() does not return key/value pairs +CREATE FUNCTION test1malformed() RETURNS hstore +LANGUAGE plpython3u +TRANSFORM FOR TYPE hstore +AS $$ +class C(dict): + def items(self): + return [42] +d = C() +d['x'] = 1 +return d +$$; + +SELECT test1malformed(); + + +-- Likewise for a mapping whose items() yields fewer pairs than its length +CREATE FUNCTION test1short() RETURNS hstore +LANGUAGE plpython3u +TRANSFORM FOR TYPE hstore +AS $$ +class C(dict): + def items(self): + return [] +d = C() +d['x'] = 1 +return d +$$; + +SELECT test1short(); + + +-- Likewise for a mapping whose __len__() raises +CREATE FUNCTION test1brokenlen() RETURNS hstore +LANGUAGE plpython3u +TRANSFORM FOR TYPE hstore +AS $$ +class C(dict): + def __len__(self): + raise ValueError('len failed') +d = C() +d['x'] = 1 +return d +$$; + +SELECT test1brokenlen(); + + -- test hstore[] -> python CREATE FUNCTION test1arr(val hstore[]) RETURNS int LANGUAGE plpythonu diff --git a/contrib/jsonb_plpython/expected/jsonb_plpython.out b/contrib/jsonb_plpython/expected/jsonb_plpython.out index b491fe9cc6..f9bde49209 100644 --- a/contrib/jsonb_plpython/expected/jsonb_plpython.out +++ b/contrib/jsonb_plpython/expected/jsonb_plpython.out @@ -304,3 +304,92 @@ SELECT test_dict1(); {"": 2, "a": 1, "33": 3} (1 row) +-- A custom sequence whose __getitem__ raises should be reported as an error, +-- not crash the backend +CREATE FUNCTION test_broken_sequence() RETURNS jsonb +LANGUAGE plpython3u +TRANSFORM FOR TYPE jsonb +AS $$ +class C: + def __len__(self): + return 2 + def __getitem__(self, i): + raise ValueError('getitem failed') +return C() +$$; +SELECT test_broken_sequence(); +ERROR: could not get element 0 from sequence +DETAIL: ValueError: getitem failed +CONTEXT: Traceback (most recent call last): +while creating return value +PL/Python function "test_broken_sequence" +-- A mapping whose items() raises should be reported as an error, not crash +-- the backend +CREATE FUNCTION test_broken_mapping() RETURNS jsonb +LANGUAGE plpython3u +TRANSFORM FOR TYPE jsonb +AS $$ +class C(dict): + def items(self): + raise ValueError('items failed') +d = C() +d['x'] = 1 +return d +$$; +SELECT test_broken_mapping(); +ERROR: could not get items from Python mapping +DETAIL: ValueError: items failed +CONTEXT: Traceback (most recent call last): +while creating return value +PL/Python function "test_broken_mapping" +-- Likewise for a mapping whose items() does not return key/value pairs +CREATE FUNCTION test_malformed_mapping() RETURNS jsonb +LANGUAGE plpython3u +TRANSFORM FOR TYPE jsonb +AS $$ +class C(dict): + def items(self): + return [42] +d = C() +d['x'] = 1 +return d +$$; +SELECT test_malformed_mapping(); +ERROR: items() of a Python mapping must return key/value pairs +CONTEXT: while creating return value +PL/Python function "test_malformed_mapping" +-- Likewise for a mapping whose items() yields fewer pairs than its length +CREATE FUNCTION test_short_mapping() RETURNS jsonb +LANGUAGE plpython3u +TRANSFORM FOR TYPE jsonb +AS $$ +class C(dict): + def items(self): + return [] +d = C() +d['x'] = 1 +return d +$$; +SELECT test_short_mapping(); +ERROR: items() of a Python mapping must return key/value pairs +DETAIL: IndexError: list index out of range +CONTEXT: while creating return value +PL/Python function "test_short_mapping" +-- Likewise for a mapping whose __len__() raises +CREATE FUNCTION test_broken_len_mapping() RETURNS jsonb +LANGUAGE plpython3u +TRANSFORM FOR TYPE jsonb +AS $$ +class C(dict): + def __len__(self): + raise ValueError('len failed') +d = C() +d['x'] = 1 +return d +$$; +SELECT test_broken_len_mapping(); +ERROR: could not get size of Python mapping +DETAIL: ValueError: len failed +CONTEXT: Traceback (most recent call last): +while creating return value +PL/Python function "test_broken_len_mapping" diff --git a/contrib/jsonb_plpython/jsonb_plpython.c b/contrib/jsonb_plpython/jsonb_plpython.c index 8d95f09f23..1c7097a407 100644 --- a/contrib/jsonb_plpython/jsonb_plpython.c +++ b/contrib/jsonb_plpython/jsonb_plpython.c @@ -277,7 +277,12 @@ PLyMapping_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state) JsonbValue *volatile out; pcount = PyMapping_Size(obj); + if (pcount < 0) + PLy_elog(ERROR, "could not get size of Python mapping"); + items = PyMapping_Items(obj); + if (items == NULL) + PLy_elog(ERROR, "could not get items from Python mapping"); PG_TRY(); { @@ -289,8 +294,15 @@ PLyMapping_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state) { JsonbValue jbvKey; PyObject *item = PyList_GetItem(items, i); - PyObject *key = PyTuple_GetItem(item, 0); - PyObject *value = PyTuple_GetItem(item, 1); + PyObject *key; + PyObject *value; + + /* The mapping's items() must yield key/value pairs */ + if (item == NULL || !PyTuple_Check(item) || PyTuple_Size(item) < 2) + PLy_elog(ERROR, "items() of a Python mapping must return key/value pairs"); + + key = PyTuple_GetItem(item, 0); + value = PyTuple_GetItem(item, 1); /* Python dictionary can have None as key */ if (key == Py_None) @@ -342,7 +354,10 @@ PLySequence_ToJsonbValue(PyObject *obj, JsonbParseState **jsonb_state) for (i = 0; i < pcount; i++) { value = PySequence_GetItem(obj, i); - Assert(value); + + /* PySequence_GetItem() can return NULL, with an exception set */ + if (value == NULL) + PLy_elog(ERROR, "could not get element %d from sequence", (int) i); (void) PLyObject_ToJsonbValue(value, jsonb_state, true); Py_XDECREF(value); diff --git a/contrib/jsonb_plpython/sql/jsonb_plpython.sql b/contrib/jsonb_plpython/sql/jsonb_plpython.sql index 2ee1bca0a9..78578fa3d8 100644 --- a/contrib/jsonb_plpython/sql/jsonb_plpython.sql +++ b/contrib/jsonb_plpython/sql/jsonb_plpython.sql @@ -181,3 +181,80 @@ return x $$; SELECT test_dict1(); + +-- A custom sequence whose __getitem__ raises should be reported as an error, +-- not crash the backend +CREATE FUNCTION test_broken_sequence() RETURNS jsonb +LANGUAGE plpython3u +TRANSFORM FOR TYPE jsonb +AS $$ +class C: + def __len__(self): + return 2 + def __getitem__(self, i): + raise ValueError('getitem failed') +return C() +$$; + +SELECT test_broken_sequence(); + +-- A mapping whose items() raises should be reported as an error, not crash +-- the backend +CREATE FUNCTION test_broken_mapping() RETURNS jsonb +LANGUAGE plpython3u +TRANSFORM FOR TYPE jsonb +AS $$ +class C(dict): + def items(self): + raise ValueError('items failed') +d = C() +d['x'] = 1 +return d +$$; + +SELECT test_broken_mapping(); + +-- Likewise for a mapping whose items() does not return key/value pairs +CREATE FUNCTION test_malformed_mapping() RETURNS jsonb +LANGUAGE plpython3u +TRANSFORM FOR TYPE jsonb +AS $$ +class C(dict): + def items(self): + return [42] +d = C() +d['x'] = 1 +return d +$$; + +SELECT test_malformed_mapping(); + +-- Likewise for a mapping whose items() yields fewer pairs than its length +CREATE FUNCTION test_short_mapping() RETURNS jsonb +LANGUAGE plpython3u +TRANSFORM FOR TYPE jsonb +AS $$ +class C(dict): + def items(self): + return [] +d = C() +d['x'] = 1 +return d +$$; + +SELECT test_short_mapping(); + +-- Likewise for a mapping whose __len__() raises +CREATE FUNCTION test_broken_len_mapping() RETURNS jsonb +LANGUAGE plpython3u +TRANSFORM FOR TYPE jsonb +AS $$ +class C(dict): + def __len__(self): + raise ValueError('len failed') +d = C() +d['x'] = 1 +return d +$$; + +SELECT test_broken_len_mapping(); diff --git a/src/pl/plpython/expected/plpython_composite.out b/src/pl/plpython/expected/plpython_composite.out index b9111210d5..a860dd24a5 100644 --- a/src/pl/plpython/expected/plpython_composite.out +++ b/src/pl/plpython/expected/plpython_composite.out @@ -606,3 +606,19 @@ DETAIL: Missing left parenthesis. HINT: To return a composite type in an array, return the composite type as a Python tuple, e.g., "[('foo',)]". CONTEXT: while creating return value PL/Python function "composite_type_as_list_broken" +-- A custom sequence whose length matches the tuple but whose __getitem__ +-- raises should be reported as an error, not crash the backend. +CREATE FUNCTION composite_type_as_broken_sequence() RETURNS type_record AS $$ +class C: + def __len__(self): + return 2 + def __getitem__(self, i): + raise ValueError('getitem failed') +return C() +$$ LANGUAGE plpython3u; +SELECT * FROM composite_type_as_broken_sequence(); +ERROR: could not get element 0 from sequence +DETAIL: ValueError: getitem failed +CONTEXT: Traceback (most recent call last): +while creating return value +PL/Python function "composite_type_as_broken_sequence" diff --git a/src/pl/plpython/expected/plpython_spi.out b/src/pl/plpython/expected/plpython_spi.out index a09df68c7d..eeed3d25a7 100644 --- a/src/pl/plpython/expected/plpython_spi.out +++ b/src/pl/plpython/expected/plpython_spi.out @@ -464,3 +464,54 @@ SELECT plan_composite_args(); (3,label) (1 row) +-- A custom argument sequence whose length matches the plan but whose +-- __getitem__ raises should be reported as an error, not crash the backend. +CREATE FUNCTION plan_broken_arg_sequence() RETURNS void AS $$ +plan = plpy.prepare("select $1", ["int4"]) +class C: + def __len__(self): + return 1 + def __getitem__(self, i): + raise ValueError('getitem failed') +plpy.execute(plan, C()) +$$ LANGUAGE plpython3u; +SELECT plan_broken_arg_sequence(); +ERROR: spiexceptions.ExternalRoutineException: could not get element 0 from sequence +DETAIL: ValueError: getitem failed +CONTEXT: Traceback (most recent call last): + PL/Python function "plan_broken_arg_sequence", line 8, in + plpy.execute(plan, C()) +PL/Python function "plan_broken_arg_sequence" +-- Likewise for the type-name list passed to plpy.prepare(). +CREATE FUNCTION prepare_broken_type_sequence() RETURNS void AS $$ +class C: + def __len__(self): + return 1 + def __getitem__(self, i): + raise ValueError('getitem failed') +plpy.prepare("select $1", C()) +$$ LANGUAGE plpython3u; +SELECT prepare_broken_type_sequence(); +ERROR: spiexceptions.ExternalRoutineException: could not get element 0 from sequence +DETAIL: ValueError: getitem failed +CONTEXT: Traceback (most recent call last): + PL/Python function "prepare_broken_type_sequence", line 7, in + plpy.prepare("select $1", C()) +PL/Python function "prepare_broken_type_sequence" +-- Likewise for the argument sequence passed to plpy.cursor(). +CREATE FUNCTION cursor_broken_arg_sequence() RETURNS void AS $$ +plan = plpy.prepare("select $1", ["int4"]) +class C: + def __len__(self): + return 1 + def __getitem__(self, i): + raise ValueError('getitem failed') +plpy.cursor(plan, C()) +$$ LANGUAGE plpython3u; +SELECT cursor_broken_arg_sequence(); +ERROR: spiexceptions.ExternalRoutineException: could not get element 0 from sequence +DETAIL: ValueError: getitem failed +CONTEXT: Traceback (most recent call last): + PL/Python function "cursor_broken_arg_sequence", line 8, in + plpy.cursor(plan, C()) +PL/Python function "cursor_broken_arg_sequence" diff --git a/src/pl/plpython/expected/plpython_types.out b/src/pl/plpython/expected/plpython_types.out index d1776365c3..6ef55a4efc 100644 --- a/src/pl/plpython/expected/plpython_types.out +++ b/src/pl/plpython/expected/plpython_types.out @@ -796,6 +796,22 @@ SELECT * FROM test_type_conversion_array_error(); ERROR: return value of function with array return type is not a Python sequence CONTEXT: while creating return value PL/Python function "test_type_conversion_array_error" +-- A custom sequence whose __getitem__ raises should be reported as an error, +-- not crash the backend. +CREATE FUNCTION test_type_conversion_array_getitem_fail() RETURNS int[] AS $$ +class C: + def __len__(self): + return 2 + def __getitem__(self, i): + raise ValueError('getitem failed') +return C() +$$ LANGUAGE plpython3u; +SELECT * FROM test_type_conversion_array_getitem_fail(); +ERROR: could not get element 0 from sequence +DETAIL: ValueError: getitem failed +CONTEXT: Traceback (most recent call last): +while creating return value +PL/Python function "test_type_conversion_array_getitem_fail" -- -- Domains over arrays -- diff --git a/src/pl/plpython/expected/plpython_types_3.out b/src/pl/plpython/expected/plpython_types_3.out index 6fadd5462d..ce362cbbc3 100644 --- a/src/pl/plpython/expected/plpython_types_3.out +++ b/src/pl/plpython/expected/plpython_types_3.out @@ -796,6 +796,22 @@ SELECT * FROM test_type_conversion_array_error(); ERROR: return value of function with array return type is not a Python sequence CONTEXT: while creating return value PL/Python function "test_type_conversion_array_error" +-- A custom sequence whose __getitem__ raises should be reported as an error, +-- not crash the backend. +CREATE FUNCTION test_type_conversion_array_getitem_fail() RETURNS int[] AS $$ +class C: + def __len__(self): + return 2 + def __getitem__(self, i): + raise ValueError('getitem failed') +return C() +$$ LANGUAGE plpython3u; +SELECT * FROM test_type_conversion_array_getitem_fail(); +ERROR: could not get element 0 from sequence +DETAIL: ValueError: getitem failed +CONTEXT: Traceback (most recent call last): +while creating return value +PL/Python function "test_type_conversion_array_getitem_fail" -- -- Domains over arrays -- diff --git a/src/pl/plpython/plpy_cursorobject.c b/src/pl/plpython/plpy_cursorobject.c index d783819e82..06229fcd76 100644 --- a/src/pl/plpython/plpy_cursorobject.c +++ b/src/pl/plpython/plpy_cursorobject.c @@ -231,6 +231,11 @@ PLy_cursor_plan(PyObject *ob, PyObject *args) PyObject *elem; elem = PySequence_GetItem(args, j); + + /* PySequence_GetItem() can return NULL, with an exception set */ + if (elem == NULL) + PLy_elog(ERROR, "could not get element %d from sequence", j); + PG_TRY(); { bool isnull; diff --git a/src/pl/plpython/plpy_spi.c b/src/pl/plpython/plpy_spi.c index 53af9c05ef..594f0a67ab 100644 --- a/src/pl/plpython/plpy_spi.c +++ b/src/pl/plpython/plpy_spi.c @@ -90,6 +90,11 @@ PLy_spi_prepare(PyObject *self, PyObject *args) int32 typmod; optr = PySequence_GetItem(list, i); + + /* PySequence_GetItem() can return NULL, with an exception set */ + if (optr == NULL) + PLy_elog(ERROR, "could not get element %d from sequence", i); + if (PyString_Check(optr)) sptr = PyString_AsString(optr); else if (PyUnicode_Check(optr)) @@ -254,6 +259,11 @@ PLy_spi_execute_plan(PyObject *ob, PyObject *list, long limit) PyObject *elem; elem = PySequence_GetItem(list, j); + + /* PySequence_GetItem() can return NULL, with an exception set */ + if (elem == NULL) + PLy_elog(ERROR, "could not get element %d from sequence", j); + PG_TRY(); { bool isnull; diff --git a/src/pl/plpython/plpy_typeio.c b/src/pl/plpython/plpy_typeio.c index 707fe416b2..35be8bfd8a 100644 --- a/src/pl/plpython/plpy_typeio.c +++ b/src/pl/plpython/plpy_typeio.c @@ -1214,6 +1214,10 @@ PLySequence_ToArray_recurse(PyObject *obj, ArrayBuildState **astatep, /* fetch the array element */ PyObject *subobj = PySequence_GetItem(obj, i); + /* PySequence_GetItem() can return NULL, with an exception set */ + if (subobj == NULL) + PLy_elog(ERROR, "could not get element %d from sequence", i); + /* need PG_TRY to ensure we release the subobj's refcount */ PG_TRY(); { @@ -1460,7 +1464,10 @@ PLySequence_ToComposite(PLyObToDatum *arg, TupleDesc desc, PyObject *sequence) PG_TRY(); { value = PySequence_GetItem(sequence, idx); - Assert(value); + + /* PySequence_GetItem() can return NULL, with an exception set */ + if (value == NULL) + PLy_elog(ERROR, "could not get element %d from sequence", idx); values[i] = att->func(att, value, &nulls[i], false); diff --git a/src/pl/plpython/sql/plpython_composite.sql b/src/pl/plpython/sql/plpython_composite.sql index 844b761bf1..d7ac6463c1 100644 --- a/src/pl/plpython/sql/plpython_composite.sql +++ b/src/pl/plpython/sql/plpython_composite.sql @@ -233,3 +233,15 @@ CREATE FUNCTION composite_type_as_list_broken() RETURNS type_record[] AS $$ return [['first', 1]]; $$ LANGUAGE plpythonu; SELECT * FROM composite_type_as_list_broken(); + +-- A custom sequence whose length matches the tuple but whose __getitem__ +-- raises should be reported as an error, not crash the backend. +CREATE FUNCTION composite_type_as_broken_sequence() RETURNS type_record AS $$ +class C: + def __len__(self): + return 2 + def __getitem__(self, i): + raise ValueError('getitem failed') +return C() +$$ LANGUAGE plpython3u; +SELECT * FROM composite_type_as_broken_sequence(); diff --git a/src/pl/plpython/sql/plpython_spi.sql b/src/pl/plpython/sql/plpython_spi.sql index dd77833ed5..93d2b0d90c 100644 --- a/src/pl/plpython/sql/plpython_spi.sql +++ b/src/pl/plpython/sql/plpython_spi.sql @@ -320,3 +320,42 @@ SELECT cursor_fetch_next_empty(); SELECT cursor_plan(); SELECT cursor_plan_wrong_args(); SELECT plan_composite_args(); + +-- A custom argument sequence whose length matches the plan but whose +-- __getitem__ raises should be reported as an error, not crash the backend. +CREATE FUNCTION plan_broken_arg_sequence() RETURNS void AS $$ +plan = plpy.prepare("select $1", ["int4"]) +class C: + def __len__(self): + return 1 + def __getitem__(self, i): + raise ValueError('getitem failed') +plpy.execute(plan, C()) +$$ LANGUAGE plpython3u; + +SELECT plan_broken_arg_sequence(); + +-- Likewise for the type-name list passed to plpy.prepare(). +CREATE FUNCTION prepare_broken_type_sequence() RETURNS void AS $$ +class C: + def __len__(self): + return 1 + def __getitem__(self, i): + raise ValueError('getitem failed') +plpy.prepare("select $1", C()) +$$ LANGUAGE plpython3u; + +SELECT prepare_broken_type_sequence(); + +-- Likewise for the argument sequence passed to plpy.cursor(). +CREATE FUNCTION cursor_broken_arg_sequence() RETURNS void AS $$ +plan = plpy.prepare("select $1", ["int4"]) +class C: + def __len__(self): + return 1 + def __getitem__(self, i): + raise ValueError('getitem failed') +plpy.cursor(plan, C()) +$$ LANGUAGE plpython3u; + +SELECT cursor_broken_arg_sequence(); diff --git a/src/pl/plpython/sql/plpython_types.sql b/src/pl/plpython/sql/plpython_types.sql index a8ceb66111..b64fac4d17 100644 --- a/src/pl/plpython/sql/plpython_types.sql +++ b/src/pl/plpython/sql/plpython_types.sql @@ -417,6 +417,19 @@ $$ LANGUAGE plpythonu; SELECT * FROM test_type_conversion_array_error(); +-- A custom sequence whose __getitem__ raises should be reported as an error, +-- not crash the backend. +CREATE FUNCTION test_type_conversion_array_getitem_fail() RETURNS int[] AS $$ +class C: + def __len__(self): + return 2 + def __getitem__(self, i): + raise ValueError('getitem failed') +return C() +$$ LANGUAGE plpython3u; + +SELECT * FROM test_type_conversion_array_getitem_fail(); + -- -- Domains over arrays From 309dc4526de5ca4cb871fae80a1470feb0fad231 Mon Sep 17 00:00:00 2001 From: Richard Guo Date: Mon, 29 Jun 2026 14:21:15 +0900 Subject: [PATCH 57/76] Use plpythonu in plpython tests added by commit 0b7719f74 Commit 0b7719f74 added regression tests that spell the language name as plpython3u. That works on the master branch, but on v14 the plpython tests must use the unversioned name plpythonu: the plpython3u extension is built only in Python 3 builds, and for those builds regress-python3-mangle.mk rewrites plpythonu to plpython3u in the test files. In a Python 2 build the new tests instead failed with "language \"plpython3u\" does not exist". Spell the language as plpythonu in the new tests, matching every other plpython test in this branch, so they work in both Python 2 and Python 3 builds. This is needed only in v14; later branches no longer support Python 2 and have dropped the mangling step. Per buildfarm member hippopotamus. --- contrib/hstore_plpython/expected/hstore_plpython.out | 8 ++++---- contrib/hstore_plpython/sql/hstore_plpython.sql | 8 ++++---- contrib/jsonb_plpython/expected/jsonb_plpython.out | 10 +++++----- contrib/jsonb_plpython/sql/jsonb_plpython.sql | 10 +++++----- src/pl/plpython/expected/plpython_composite.out | 2 +- src/pl/plpython/expected/plpython_spi.out | 6 +++--- src/pl/plpython/expected/plpython_types.out | 2 +- src/pl/plpython/sql/plpython_composite.sql | 2 +- src/pl/plpython/sql/plpython_spi.sql | 6 +++--- src/pl/plpython/sql/plpython_types.sql | 2 +- 10 files changed, 28 insertions(+), 28 deletions(-) diff --git a/contrib/hstore_plpython/expected/hstore_plpython.out b/contrib/hstore_plpython/expected/hstore_plpython.out index b1837715ce..121d0248d4 100644 --- a/contrib/hstore_plpython/expected/hstore_plpython.out +++ b/contrib/hstore_plpython/expected/hstore_plpython.out @@ -46,7 +46,7 @@ PL/Python function "test1bad" -- A mapping whose items() raises should be reported as an error, not crash -- the backend CREATE FUNCTION test1broken() RETURNS hstore -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE hstore AS $$ class C(dict): @@ -62,7 +62,7 @@ CONTEXT: while creating return value PL/Python function "test1broken" -- Likewise for a mapping whose items() does not return key/value pairs CREATE FUNCTION test1malformed() RETURNS hstore -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE hstore AS $$ class C(dict): @@ -78,7 +78,7 @@ CONTEXT: while creating return value PL/Python function "test1malformed" -- Likewise for a mapping whose items() yields fewer pairs than its length CREATE FUNCTION test1short() RETURNS hstore -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE hstore AS $$ class C(dict): @@ -94,7 +94,7 @@ CONTEXT: while creating return value PL/Python function "test1short" -- Likewise for a mapping whose __len__() raises CREATE FUNCTION test1brokenlen() RETURNS hstore -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE hstore AS $$ class C(dict): diff --git a/contrib/hstore_plpython/sql/hstore_plpython.sql b/contrib/hstore_plpython/sql/hstore_plpython.sql index 8548b7dbe0..9923349dc0 100644 --- a/contrib/hstore_plpython/sql/hstore_plpython.sql +++ b/contrib/hstore_plpython/sql/hstore_plpython.sql @@ -41,7 +41,7 @@ SELECT test1bad(); -- A mapping whose items() raises should be reported as an error, not crash -- the backend CREATE FUNCTION test1broken() RETURNS hstore -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE hstore AS $$ class C(dict): @@ -57,7 +57,7 @@ SELECT test1broken(); -- Likewise for a mapping whose items() does not return key/value pairs CREATE FUNCTION test1malformed() RETURNS hstore -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE hstore AS $$ class C(dict): @@ -73,7 +73,7 @@ SELECT test1malformed(); -- Likewise for a mapping whose items() yields fewer pairs than its length CREATE FUNCTION test1short() RETURNS hstore -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE hstore AS $$ class C(dict): @@ -89,7 +89,7 @@ SELECT test1short(); -- Likewise for a mapping whose __len__() raises CREATE FUNCTION test1brokenlen() RETURNS hstore -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE hstore AS $$ class C(dict): diff --git a/contrib/jsonb_plpython/expected/jsonb_plpython.out b/contrib/jsonb_plpython/expected/jsonb_plpython.out index f9bde49209..36712f4559 100644 --- a/contrib/jsonb_plpython/expected/jsonb_plpython.out +++ b/contrib/jsonb_plpython/expected/jsonb_plpython.out @@ -307,7 +307,7 @@ SELECT test_dict1(); -- A custom sequence whose __getitem__ raises should be reported as an error, -- not crash the backend CREATE FUNCTION test_broken_sequence() RETURNS jsonb -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE jsonb AS $$ class C: @@ -326,7 +326,7 @@ PL/Python function "test_broken_sequence" -- A mapping whose items() raises should be reported as an error, not crash -- the backend CREATE FUNCTION test_broken_mapping() RETURNS jsonb -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE jsonb AS $$ class C(dict): @@ -344,7 +344,7 @@ while creating return value PL/Python function "test_broken_mapping" -- Likewise for a mapping whose items() does not return key/value pairs CREATE FUNCTION test_malformed_mapping() RETURNS jsonb -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE jsonb AS $$ class C(dict): @@ -360,7 +360,7 @@ CONTEXT: while creating return value PL/Python function "test_malformed_mapping" -- Likewise for a mapping whose items() yields fewer pairs than its length CREATE FUNCTION test_short_mapping() RETURNS jsonb -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE jsonb AS $$ class C(dict): @@ -377,7 +377,7 @@ CONTEXT: while creating return value PL/Python function "test_short_mapping" -- Likewise for a mapping whose __len__() raises CREATE FUNCTION test_broken_len_mapping() RETURNS jsonb -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE jsonb AS $$ class C(dict): diff --git a/contrib/jsonb_plpython/sql/jsonb_plpython.sql b/contrib/jsonb_plpython/sql/jsonb_plpython.sql index 78578fa3d8..40832a49b3 100644 --- a/contrib/jsonb_plpython/sql/jsonb_plpython.sql +++ b/contrib/jsonb_plpython/sql/jsonb_plpython.sql @@ -185,7 +185,7 @@ SELECT test_dict1(); -- A custom sequence whose __getitem__ raises should be reported as an error, -- not crash the backend CREATE FUNCTION test_broken_sequence() RETURNS jsonb -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE jsonb AS $$ class C: @@ -201,7 +201,7 @@ SELECT test_broken_sequence(); -- A mapping whose items() raises should be reported as an error, not crash -- the backend CREATE FUNCTION test_broken_mapping() RETURNS jsonb -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE jsonb AS $$ class C(dict): @@ -216,7 +216,7 @@ SELECT test_broken_mapping(); -- Likewise for a mapping whose items() does not return key/value pairs CREATE FUNCTION test_malformed_mapping() RETURNS jsonb -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE jsonb AS $$ class C(dict): @@ -231,7 +231,7 @@ SELECT test_malformed_mapping(); -- Likewise for a mapping whose items() yields fewer pairs than its length CREATE FUNCTION test_short_mapping() RETURNS jsonb -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE jsonb AS $$ class C(dict): @@ -246,7 +246,7 @@ SELECT test_short_mapping(); -- Likewise for a mapping whose __len__() raises CREATE FUNCTION test_broken_len_mapping() RETURNS jsonb -LANGUAGE plpython3u +LANGUAGE plpythonu TRANSFORM FOR TYPE jsonb AS $$ class C(dict): diff --git a/src/pl/plpython/expected/plpython_composite.out b/src/pl/plpython/expected/plpython_composite.out index a860dd24a5..e503de54b6 100644 --- a/src/pl/plpython/expected/plpython_composite.out +++ b/src/pl/plpython/expected/plpython_composite.out @@ -615,7 +615,7 @@ class C: def __getitem__(self, i): raise ValueError('getitem failed') return C() -$$ LANGUAGE plpython3u; +$$ LANGUAGE plpythonu; SELECT * FROM composite_type_as_broken_sequence(); ERROR: could not get element 0 from sequence DETAIL: ValueError: getitem failed diff --git a/src/pl/plpython/expected/plpython_spi.out b/src/pl/plpython/expected/plpython_spi.out index eeed3d25a7..6cd9275b76 100644 --- a/src/pl/plpython/expected/plpython_spi.out +++ b/src/pl/plpython/expected/plpython_spi.out @@ -474,7 +474,7 @@ class C: def __getitem__(self, i): raise ValueError('getitem failed') plpy.execute(plan, C()) -$$ LANGUAGE plpython3u; +$$ LANGUAGE plpythonu; SELECT plan_broken_arg_sequence(); ERROR: spiexceptions.ExternalRoutineException: could not get element 0 from sequence DETAIL: ValueError: getitem failed @@ -490,7 +490,7 @@ class C: def __getitem__(self, i): raise ValueError('getitem failed') plpy.prepare("select $1", C()) -$$ LANGUAGE plpython3u; +$$ LANGUAGE plpythonu; SELECT prepare_broken_type_sequence(); ERROR: spiexceptions.ExternalRoutineException: could not get element 0 from sequence DETAIL: ValueError: getitem failed @@ -507,7 +507,7 @@ class C: def __getitem__(self, i): raise ValueError('getitem failed') plpy.cursor(plan, C()) -$$ LANGUAGE plpython3u; +$$ LANGUAGE plpythonu; SELECT cursor_broken_arg_sequence(); ERROR: spiexceptions.ExternalRoutineException: could not get element 0 from sequence DETAIL: ValueError: getitem failed diff --git a/src/pl/plpython/expected/plpython_types.out b/src/pl/plpython/expected/plpython_types.out index 6ef55a4efc..98c7428eeb 100644 --- a/src/pl/plpython/expected/plpython_types.out +++ b/src/pl/plpython/expected/plpython_types.out @@ -805,7 +805,7 @@ class C: def __getitem__(self, i): raise ValueError('getitem failed') return C() -$$ LANGUAGE plpython3u; +$$ LANGUAGE plpythonu; SELECT * FROM test_type_conversion_array_getitem_fail(); ERROR: could not get element 0 from sequence DETAIL: ValueError: getitem failed diff --git a/src/pl/plpython/sql/plpython_composite.sql b/src/pl/plpython/sql/plpython_composite.sql index d7ac6463c1..5537d75472 100644 --- a/src/pl/plpython/sql/plpython_composite.sql +++ b/src/pl/plpython/sql/plpython_composite.sql @@ -243,5 +243,5 @@ class C: def __getitem__(self, i): raise ValueError('getitem failed') return C() -$$ LANGUAGE plpython3u; +$$ LANGUAGE plpythonu; SELECT * FROM composite_type_as_broken_sequence(); diff --git a/src/pl/plpython/sql/plpython_spi.sql b/src/pl/plpython/sql/plpython_spi.sql index 93d2b0d90c..95481580c7 100644 --- a/src/pl/plpython/sql/plpython_spi.sql +++ b/src/pl/plpython/sql/plpython_spi.sql @@ -331,7 +331,7 @@ class C: def __getitem__(self, i): raise ValueError('getitem failed') plpy.execute(plan, C()) -$$ LANGUAGE plpython3u; +$$ LANGUAGE plpythonu; SELECT plan_broken_arg_sequence(); @@ -343,7 +343,7 @@ class C: def __getitem__(self, i): raise ValueError('getitem failed') plpy.prepare("select $1", C()) -$$ LANGUAGE plpython3u; +$$ LANGUAGE plpythonu; SELECT prepare_broken_type_sequence(); @@ -356,6 +356,6 @@ class C: def __getitem__(self, i): raise ValueError('getitem failed') plpy.cursor(plan, C()) -$$ LANGUAGE plpython3u; +$$ LANGUAGE plpythonu; SELECT cursor_broken_arg_sequence(); diff --git a/src/pl/plpython/sql/plpython_types.sql b/src/pl/plpython/sql/plpython_types.sql index b64fac4d17..5a6fdb26b1 100644 --- a/src/pl/plpython/sql/plpython_types.sql +++ b/src/pl/plpython/sql/plpython_types.sql @@ -426,7 +426,7 @@ class C: def __getitem__(self, i): raise ValueError('getitem failed') return C() -$$ LANGUAGE plpython3u; +$$ LANGUAGE plpythonu; SELECT * FROM test_type_conversion_array_getitem_fail(); From 255bce44884ec5e12414760e5865a95dedb39925 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Wed, 1 Jul 2026 13:27:22 -0400 Subject: [PATCH 58/76] btree_gist: fix NaN handling in float4/float8 opclasses. The float4 and float8 btree_gist opclasses compared keys with raw C operators (==, <, >). IEEE 754 makes every comparison involving NaN false, so GiST disagreed with the regular float comparison operators and with the btree opclass, which uses float[4|8]_cmp_internal() (so that all NaNs are equal and NaN sorts after every non-NaN value). In addition, the penalty and distance functions were not careful about NaNs, and the penalty functions could also misbehave for IEEE infinities. Wrong answers from the penalty functions would probably do no more than make the index non-optimal, but the distance mistakes were visible from SQL. To fix, make the comparison functions rely on the same NaN-aware comparison functions the core code uses, and rewrite the penalty and distance functions to follow the rules that NaNs are equal but maximally far away from non-NaNs. The penalty_num() code was formerly shared between integral and float cases, but I chose to make two copies so that the integral cases are not saddled with the extra logic for NaNs and infinities/overflows. I also rewrote it as static inline functions instead of an unreadable and uncommented macro. The float penalty functions were previously unreached by the regression tests, so add new test cases to exercise them. There's no on-disk format change, but users who have NaN entries in a btree_gist index would be well advised to reindex it. Bug: #19501 Bug: #19524 Reported-by: Man Zeng Reported-by: Yuelin Wang <3020001251@tju.edu.cn> Author: Bill Kim Co-authored-by: Tom Lane Discussion: https://postgr.es/m/19501-3bff3bbc97f1e7c9@postgresql.org Discussion: https://postgr.es/m/19524-9559d302c8455664@postgresql.org Discussion: https://postgr.es/m/CAMQXxcgbtD2LXfX0tpgvOizxP-XxrCHV2ZDy4By_TZnJMsxXWQ@mail.gmail.com Backpatch-through: 14 --- contrib/btree_gist/btree_float4.c | 59 ++++++++--- contrib/btree_gist/btree_float8.c | 52 +++++++--- contrib/btree_gist/btree_utils_num.h | 131 +++++++++++++++++++++--- contrib/btree_gist/data/float4.data | 3 + contrib/btree_gist/data/float8.data | 3 + contrib/btree_gist/expected/float4.out | 51 +++++++-- contrib/btree_gist/expected/float8.out | 51 +++++++-- contrib/btree_gist/expected/numeric.out | 48 ++++----- contrib/btree_gist/sql/float4.sql | 17 +++ contrib/btree_gist/sql/float8.sql | 17 +++ 10 files changed, 345 insertions(+), 87 deletions(-) diff --git a/contrib/btree_gist/btree_float4.c b/contrib/btree_gist/btree_float4.c index 3604c73313..c8fd19a24c 100644 --- a/contrib/btree_gist/btree_float4.c +++ b/contrib/btree_gist/btree_float4.c @@ -24,30 +24,36 @@ PG_FUNCTION_INFO_V1(gbt_float4_distance); PG_FUNCTION_INFO_V1(gbt_float4_penalty); PG_FUNCTION_INFO_V1(gbt_float4_same); +/* + * Use the NaN-aware comparators from utils/float.h, so that our results + * will agree with standard btree indexes. Note that penalty and distance + * functions below must also cope with NaNs, in particular with the policy + * that all NaNs are equal. + */ static bool gbt_float4gt(const void *a, const void *b, FmgrInfo *flinfo) { - return (*((const float4 *) a) > *((const float4 *) b)); + return float4_gt(*((const float4 *) a), *((const float4 *) b)); } static bool gbt_float4ge(const void *a, const void *b, FmgrInfo *flinfo) { - return (*((const float4 *) a) >= *((const float4 *) b)); + return float4_ge(*((const float4 *) a), *((const float4 *) b)); } static bool gbt_float4eq(const void *a, const void *b, FmgrInfo *flinfo) { - return (*((const float4 *) a) == *((const float4 *) b)); + return float4_eq(*((const float4 *) a), *((const float4 *) b)); } static bool gbt_float4le(const void *a, const void *b, FmgrInfo *flinfo) { - return (*((const float4 *) a) <= *((const float4 *) b)); + return float4_le(*((const float4 *) a), *((const float4 *) b)); } static bool gbt_float4lt(const void *a, const void *b, FmgrInfo *flinfo) { - return (*((const float4 *) a) < *((const float4 *) b)); + return float4_lt(*((const float4 *) a), *((const float4 *) b)); } static int @@ -55,22 +61,33 @@ gbt_float4key_cmp(const void *a, const void *b, FmgrInfo *flinfo) { float4KEY *ia = (float4KEY *) (((const Nsrt *) a)->t); float4KEY *ib = (float4KEY *) (((const Nsrt *) b)->t); + int res; - if (ia->lower == ib->lower) - { - if (ia->upper == ib->upper) - return 0; - - return (ia->upper > ib->upper) ? 1 : -1; - } - - return (ia->lower > ib->lower) ? 1 : -1; + res = float4_cmp_internal(ia->lower, ib->lower); + if (res != 0) + return res; + return float4_cmp_internal(ia->upper, ib->upper); } static float8 gbt_float4_dist(const void *a, const void *b, FmgrInfo *flinfo) { - return GET_FLOAT_DISTANCE(float4, a, b); + float8 arg1 = *(const float4 *) a; + float8 arg2 = *(const float4 *) b; + float8 r; + + r = arg1 - arg2; + /* needn't consider isinf case here, must be due to input infinity */ + if (unlikely(isnan(r))) + { + if (isnan(arg1) && isnan(arg2)) + r = 0.0; /* treat NaNs as equal */ + else if (isnan(arg1) || isnan(arg2)) + r = get_float8_infinity(); /* max dist for NaN vs non-NaN */ + else + r = 0.0; /* must be Inf - Inf case */ + } + return fabs(r); } @@ -99,7 +116,15 @@ float4_dist(PG_FUNCTION_ARGS) r = a - b; CHECKFLOATVAL(r, isinf(a) || isinf(b), true); - + if (unlikely(isnan(r))) + { + if (isnan(a) && isnan(b)) + r = 0.0; /* treat NaNs as equal */ + else if (isnan(a) || isnan(b)) + r = get_float4_infinity(); /* max dist for NaN vs non-NaN */ + else + r = 0.0; /* must be Inf - Inf case */ + } PG_RETURN_FLOAT4(Abs(r)); } @@ -185,7 +210,7 @@ gbt_float4_penalty(PG_FUNCTION_ARGS) float4KEY *newentry = (float4KEY *) DatumGetPointer(((GISTENTRY *) PG_GETARG_POINTER(1))->key); float *result = (float *) PG_GETARG_POINTER(2); - penalty_num(result, origentry->lower, origentry->upper, newentry->lower, newentry->upper); + float_penalty_num(result, origentry->lower, origentry->upper, newentry->lower, newentry->upper); PG_RETURN_POINTER(result); diff --git a/contrib/btree_gist/btree_float8.c b/contrib/btree_gist/btree_float8.c index 10a5262aaa..369307c7a1 100644 --- a/contrib/btree_gist/btree_float8.c +++ b/contrib/btree_gist/btree_float8.c @@ -25,30 +25,36 @@ PG_FUNCTION_INFO_V1(gbt_float8_penalty); PG_FUNCTION_INFO_V1(gbt_float8_same); +/* + * Use the NaN-aware comparators from utils/float.h, so that our results + * will agree with standard btree indexes. Note that penalty and distance + * functions below must also cope with NaNs, in particular with the policy + * that all NaNs are equal. + */ static bool gbt_float8gt(const void *a, const void *b, FmgrInfo *flinfo) { - return (*((const float8 *) a) > *((const float8 *) b)); + return float8_gt(*((const float8 *) a), *((const float8 *) b)); } static bool gbt_float8ge(const void *a, const void *b, FmgrInfo *flinfo) { - return (*((const float8 *) a) >= *((const float8 *) b)); + return float8_ge(*((const float8 *) a), *((const float8 *) b)); } static bool gbt_float8eq(const void *a, const void *b, FmgrInfo *flinfo) { - return (*((const float8 *) a) == *((const float8 *) b)); + return float8_eq(*((const float8 *) a), *((const float8 *) b)); } static bool gbt_float8le(const void *a, const void *b, FmgrInfo *flinfo) { - return (*((const float8 *) a) <= *((const float8 *) b)); + return float8_le(*((const float8 *) a), *((const float8 *) b)); } static bool gbt_float8lt(const void *a, const void *b, FmgrInfo *flinfo) { - return (*((const float8 *) a) < *((const float8 *) b)); + return float8_lt(*((const float8 *) a), *((const float8 *) b)); } static int @@ -56,16 +62,12 @@ gbt_float8key_cmp(const void *a, const void *b, FmgrInfo *flinfo) { float8KEY *ia = (float8KEY *) (((const Nsrt *) a)->t); float8KEY *ib = (float8KEY *) (((const Nsrt *) b)->t); + int res; - if (ia->lower == ib->lower) - { - if (ia->upper == ib->upper) - return 0; - - return (ia->upper > ib->upper) ? 1 : -1; - } - - return (ia->lower > ib->lower) ? 1 : -1; + res = float8_cmp_internal(ia->lower, ib->lower); + if (res != 0) + return res; + return float8_cmp_internal(ia->upper, ib->upper); } static float8 @@ -77,7 +79,15 @@ gbt_float8_dist(const void *a, const void *b, FmgrInfo *flinfo) r = arg1 - arg2; CHECKFLOATVAL(r, isinf(arg1) || isinf(arg2), true); - + if (unlikely(isnan(r))) + { + if (isnan(arg1) && isnan(arg2)) + r = 0.0; /* treat NaNs as equal */ + else if (isnan(arg1) || isnan(arg2)) + r = get_float8_infinity(); /* max dist for NaN vs non-NaN */ + else + r = 0.0; /* must be Inf - Inf case */ + } return Abs(r); } @@ -107,7 +117,15 @@ float8_dist(PG_FUNCTION_ARGS) r = a - b; CHECKFLOATVAL(r, isinf(a) || isinf(b), true); - + if (unlikely(isnan(r))) + { + if (isnan(a) && isnan(b)) + r = 0.0; /* treat NaNs as equal */ + else if (isnan(a) || isnan(b)) + r = get_float8_infinity(); /* max dist for NaN vs non-NaN */ + else + r = 0.0; /* must be Inf - Inf case */ + } PG_RETURN_FLOAT8(Abs(r)); } @@ -192,7 +210,7 @@ gbt_float8_penalty(PG_FUNCTION_ARGS) float8KEY *newentry = (float8KEY *) DatumGetPointer(((GISTENTRY *) PG_GETARG_POINTER(1))->key); float *result = (float *) PG_GETARG_POINTER(2); - penalty_num(result, origentry->lower, origentry->upper, newentry->lower, newentry->upper); + float_penalty_num(result, origentry->lower, origentry->upper, newentry->lower, newentry->upper); PG_RETURN_POINTER(result); diff --git a/contrib/btree_gist/btree_utils_num.h b/contrib/btree_gist/btree_utils_num.h index cec6986172..3b09358653 100644 --- a/contrib/btree_gist/btree_utils_num.h +++ b/contrib/btree_gist/btree_utils_num.h @@ -9,6 +9,7 @@ #include "access/gist.h" #include "btree_gist.h" +#include "utils/float.h" #include "utils/rel.h" typedef char GBT_NUMKEY; @@ -59,21 +60,124 @@ typedef struct /* - * Note: The factor 0.49 in following macro avoids floating point overflows + * Compute penalty for expanding a range olower..oupper to nlower..nupper. + * + * Although the arguments are declared double, they must not be NaN nor + * large enough to risk overflows in the calculations herein. We only + * actually use this for integral data types, so there's no hazard. + */ +static inline float +penalty_num_impl(double olower, double oupper, + double nlower, double nupper, + int natts) +{ + float result = 0.0F; + double tmp = 0.0; + + /* Add penalty for expanding upper bound */ + if (nupper > oupper) + tmp += nupper - oupper; + /* Add penalty for expanding lower bound */ + if (olower > nlower) + tmp += olower - nlower; + if (tmp > 0.0) + { + /* Ensure result is non-zero, even if next step underflows to zero */ + result += FLT_MIN; + /* Scale penalty to 0 .. 1 */ + result += (float) (tmp / (tmp + (oupper - olower))); + /* Scale to 0 .. FLT_MAX / (natts + 1) */ + result *= FLT_MAX / (natts + 1); + } + return result; +} + +/* + * As above, but the input values are float4 or float8, so we must cope + * with NaNs, infinities, and overflows. + */ +static inline float +float_penalty_num_impl(double olower, double oupper, + double nlower, double nupper, + int natts) +{ + float result = 0.0F; + double tmp = 0.0; + + /* Add penalty for expanding upper bound */ + if (float8_gt(nupper, oupper)) + { + double delta = nupper - oupper; + + if (unlikely(isnan(delta))) + { + /* oupper couldn't be NaN here, see float8_gt */ + if (isnan(nupper)) + delta = FLT_MAX; /* max penalty for NaN vs non-NaN */ + else + delta = 0.0; /* must be Inf - Inf case */ + } + else if (delta > FLT_MAX) + delta = FLT_MAX; /* clamp to FLT_MAX, esp for infinity */ + tmp += delta; + } + /* Add penalty for expanding lower bound */ + if (float8_gt(olower, nlower)) + { + double delta = olower - nlower; + + if (unlikely(isnan(delta))) + { + /* nlower couldn't be NaN here, see float8_gt */ + if (isnan(olower)) + delta = FLT_MAX; /* max penalty for NaN vs non-NaN */ + else + delta = 0.0; /* must be Inf - Inf case */ + } + else if (delta > FLT_MAX) + delta = FLT_MAX; /* clamp to FLT_MAX, esp for infinity */ + tmp += delta; + } + if (tmp > 0.0) + { + double delta = oupper - olower; + + /* Clamp delta (the original range size) to 0 .. FLT_MAX */ + if (unlikely(isnan(delta))) + { + /* here, we must deal with olower possibly being NaN */ + if (isnan(oupper) && isnan(olower)) + delta = 0.0; /* treat NaNs as equal */ + else if (isnan(oupper) || isnan(olower)) + delta = FLT_MAX; /* max penalty for NaN vs non-NaN */ + else + delta = 0.0; /* must be Inf - Inf case */ + } + else if (delta > FLT_MAX) + delta = FLT_MAX; /* clamp to FLT_MAX, esp for infinity */ + /* Ensure result is non-zero, even if next step underflows to zero */ + result += FLT_MIN; + /* Scale penalty to 0 .. 1 */ + result += (float) (tmp / (tmp + delta)); + /* Scale to 0 .. FLT_MAX / (natts + 1) */ + result *= FLT_MAX / (natts + 1); + } + return result; +} + +/* + * These macros provide backwards-compatible notation for callers. */ #define penalty_num(result,olower,oupper,nlower,nupper) do { \ - double tmp = 0.0F; \ - (*(result)) = 0.0F; \ - if ( (nupper) > (oupper) ) \ - tmp += ( ((double)nupper)*0.49F - ((double)oupper)*0.49F ); \ - if ( (olower) > (nlower) ) \ - tmp += ( ((double)olower)*0.49F - ((double)nlower)*0.49F ); \ - if (tmp > 0.0F) \ - { \ - (*(result)) += FLT_MIN; \ - (*(result)) += (float) ( ((double)(tmp)) / ( (double)(tmp) + ( ((double)(oupper))*0.49F - ((double)(olower))*0.49F ) ) ); \ - (*(result)) *= (FLT_MAX / (((GISTENTRY *) PG_GETARG_POINTER(0))->rel->rd_att->natts + 1)); \ - } \ + GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); \ + *(result) = penalty_num_impl(olower, oupper, nlower, nupper, \ + entry->rel->rd_att->natts); \ +} while (0) + +#define float_penalty_num(result,olower,oupper,nlower,nupper) do { \ + GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); \ + *(result) = float_penalty_num_impl(olower, oupper, nlower, nupper, \ + entry->rel->rd_att->natts); \ } while (0) @@ -87,6 +191,7 @@ typedef struct (ivp)->day * (24.0 * SECS_PER_HOUR) + \ (ivp)->month * (30.0 * SECS_PER_DAY)) +/* This macro is not safe to use with actual float inputs, only integers */ #define GET_FLOAT_DISTANCE(t, arg1, arg2) Abs( ((float8) *((const t *) (arg1))) - ((float8) *((const t *) (arg2))) ) /* diff --git a/contrib/btree_gist/data/float4.data b/contrib/btree_gist/data/float4.data index 947955e468..af7d09f00d 100644 --- a/contrib/btree_gist/data/float4.data +++ b/contrib/btree_gist/data/float4.data @@ -298,6 +298,9 @@ \N 2972.381398 220.199877 +Infinity +-Infinity +NaN 3542.561032 -2168.024176 -3305.714558 diff --git a/contrib/btree_gist/data/float8.data b/contrib/btree_gist/data/float8.data index ff21226e06..b60e22f957 100644 --- a/contrib/btree_gist/data/float8.data +++ b/contrib/btree_gist/data/float8.data @@ -298,6 +298,9 @@ 27770.539968 13275.355549 -4267.695804 +Infinity +-Infinity +NaN \N \N 38915.525185 diff --git a/contrib/btree_gist/expected/float4.out b/contrib/btree_gist/expected/float4.out index dfe732049e..a917b79a63 100644 --- a/contrib/btree_gist/expected/float4.out +++ b/contrib/btree_gist/expected/float4.out @@ -5,13 +5,13 @@ SET enable_seqscan=on; SELECT count(*) FROM float4tmp WHERE a < -179.0; count ------- - 244 + 245 (1 row) SELECT count(*) FROM float4tmp WHERE a <= -179.0; count ------- - 245 + 246 (1 row) SELECT count(*) FROM float4tmp WHERE a = -179.0; @@ -23,13 +23,13 @@ SELECT count(*) FROM float4tmp WHERE a = -179.0; SELECT count(*) FROM float4tmp WHERE a >= -179.0; count ------- - 303 + 305 (1 row) SELECT count(*) FROM float4tmp WHERE a > -179.0; count ------- - 302 + 304 (1 row) SELECT a, a <-> '-179.0' FROM float4tmp ORDER BY a <-> '-179.0' LIMIT 3; @@ -45,13 +45,13 @@ SET enable_seqscan=off; SELECT count(*) FROM float4tmp WHERE a < -179.0::float4; count ------- - 244 + 245 (1 row) SELECT count(*) FROM float4tmp WHERE a <= -179.0::float4; count ------- - 245 + 246 (1 row) SELECT count(*) FROM float4tmp WHERE a = -179.0::float4; @@ -63,13 +63,13 @@ SELECT count(*) FROM float4tmp WHERE a = -179.0::float4; SELECT count(*) FROM float4tmp WHERE a >= -179.0::float4; count ------- - 303 + 305 (1 row) SELECT count(*) FROM float4tmp WHERE a > -179.0::float4; count ------- - 302 + 304 (1 row) EXPLAIN (COSTS OFF) @@ -89,3 +89,38 @@ SELECT a, a <-> '-179.0' FROM float4tmp ORDER BY a <-> '-179.0' LIMIT 3; -158.17741 | 20.822586 (3 rows) +-- EXCLUDE constraint must block a duplicate NaN, same as it does for finite +-- values. +CREATE TABLE float4excl (a float4, EXCLUDE USING gist (a WITH =)); +INSERT INTO float4excl VALUES ('NaN'::float4); +INSERT INTO float4excl VALUES ('NaN'::float4); -- expect: violates EXCLUDE +ERROR: conflicting key value violates exclusion constraint "float4excl_a_excl" +DETAIL: Key (a)=(NaN) conflicts with existing key (a)=(NaN). +SELECT count(*) FROM float4excl; + count +------- + 1 +(1 row) + +-- Test double-column index +CREATE INDEX float4idx2 ON float4tmp USING gist ( a, abs(a) ); +EXPLAIN (COSTS OFF) +SELECT count(*) FROM float4tmp WHERE abs(a) = 179.0::float4; + QUERY PLAN +-------------------------------------------------- + Aggregate + -> Bitmap Heap Scan on float4tmp + Recheck Cond: (abs(a) = '179'::real) + -> Bitmap Index Scan on float4idx2 + Index Cond: (abs(a) = '179'::real) +(5 rows) + +SELECT count(*) FROM float4tmp WHERE abs(a) = 179.0::float4; + count +------- + 1 +(1 row) + +RESET enable_seqscan; +RESET enable_indexscan; +RESET enable_bitmapscan; diff --git a/contrib/btree_gist/expected/float8.out b/contrib/btree_gist/expected/float8.out index ebd0ef3d68..194bd210ac 100644 --- a/contrib/btree_gist/expected/float8.out +++ b/contrib/btree_gist/expected/float8.out @@ -5,13 +5,13 @@ SET enable_seqscan=on; SELECT count(*) FROM float8tmp WHERE a < -1890.0; count ------- - 237 + 238 (1 row) SELECT count(*) FROM float8tmp WHERE a <= -1890.0; count ------- - 238 + 239 (1 row) SELECT count(*) FROM float8tmp WHERE a = -1890.0; @@ -23,13 +23,13 @@ SELECT count(*) FROM float8tmp WHERE a = -1890.0; SELECT count(*) FROM float8tmp WHERE a >= -1890.0; count ------- - 307 + 309 (1 row) SELECT count(*) FROM float8tmp WHERE a > -1890.0; count ------- - 306 + 308 (1 row) SELECT a, a <-> '-1890.0' FROM float8tmp ORDER BY a <-> '-1890.0' LIMIT 3; @@ -45,13 +45,13 @@ SET enable_seqscan=off; SELECT count(*) FROM float8tmp WHERE a < -1890.0::float8; count ------- - 237 + 238 (1 row) SELECT count(*) FROM float8tmp WHERE a <= -1890.0::float8; count ------- - 238 + 239 (1 row) SELECT count(*) FROM float8tmp WHERE a = -1890.0::float8; @@ -63,13 +63,13 @@ SELECT count(*) FROM float8tmp WHERE a = -1890.0::float8; SELECT count(*) FROM float8tmp WHERE a >= -1890.0::float8; count ------- - 307 + 309 (1 row) SELECT count(*) FROM float8tmp WHERE a > -1890.0::float8; count ------- - 306 + 308 (1 row) EXPLAIN (COSTS OFF) @@ -89,3 +89,38 @@ SELECT a, a <-> '-1890.0' FROM float8tmp ORDER BY a <-> '-1890.0' LIMIT 3; -1769.73634 | 120.26366000000007 (3 rows) +-- EXCLUDE constraint must block a duplicate NaN, same as it does for finite +-- values. +CREATE TABLE float8excl (a float8, EXCLUDE USING gist (a WITH =)); +INSERT INTO float8excl VALUES ('NaN'::float8); +INSERT INTO float8excl VALUES ('NaN'::float8); -- expect: violates EXCLUDE +ERROR: conflicting key value violates exclusion constraint "float8excl_a_excl" +DETAIL: Key (a)=(NaN) conflicts with existing key (a)=(NaN). +SELECT count(*) FROM float8excl; + count +------- + 1 +(1 row) + +-- Test double-column index +CREATE INDEX float8idx2 ON float8tmp USING gist ( a, abs(a) ); +EXPLAIN (COSTS OFF) +SELECT count(*) FROM float8tmp WHERE abs(a) = 1890.0::float8; + QUERY PLAN +--------------------------------------------------------------- + Aggregate + -> Bitmap Heap Scan on float8tmp + Recheck Cond: (abs(a) = '1890'::double precision) + -> Bitmap Index Scan on float8idx2 + Index Cond: (abs(a) = '1890'::double precision) +(5 rows) + +SELECT count(*) FROM float8tmp WHERE abs(a) = 1890.0::float8; + count +------- + 1 +(1 row) + +RESET enable_seqscan; +RESET enable_indexscan; +RESET enable_bitmapscan; diff --git a/contrib/btree_gist/expected/numeric.out b/contrib/btree_gist/expected/numeric.out index ae839b8ec8..34c1e56806 100644 --- a/contrib/btree_gist/expected/numeric.out +++ b/contrib/btree_gist/expected/numeric.out @@ -7,13 +7,13 @@ SET enable_seqscan=on; SELECT count(*) FROM numerictmp WHERE a < -1890.0; count ------- - 505 + 506 (1 row) SELECT count(*) FROM numerictmp WHERE a <= -1890.0; count ------- - 506 + 507 (1 row) SELECT count(*) FROM numerictmp WHERE a = -1890.0; @@ -25,37 +25,37 @@ SELECT count(*) FROM numerictmp WHERE a = -1890.0; SELECT count(*) FROM numerictmp WHERE a >= -1890.0; count ------- - 597 + 599 (1 row) SELECT count(*) FROM numerictmp WHERE a > -1890.0; count ------- - 596 + 598 (1 row) SELECT count(*) FROM numerictmp WHERE a < 'NaN' ; count ------- - 1100 + 1102 (1 row) SELECT count(*) FROM numerictmp WHERE a <= 'NaN' ; count ------- - 1102 + 1105 (1 row) SELECT count(*) FROM numerictmp WHERE a = 'NaN' ; count ------- - 2 + 3 (1 row) SELECT count(*) FROM numerictmp WHERE a >= 'NaN' ; count ------- - 2 + 3 (1 row) SELECT count(*) FROM numerictmp WHERE a > 'NaN' ; @@ -67,13 +67,13 @@ SELECT count(*) FROM numerictmp WHERE a > 'NaN' ; SELECT count(*) FROM numerictmp WHERE a < 0 ; count ------- - 523 + 524 (1 row) SELECT count(*) FROM numerictmp WHERE a <= 0 ; count ------- - 526 + 527 (1 row) SELECT count(*) FROM numerictmp WHERE a = 0 ; @@ -85,13 +85,13 @@ SELECT count(*) FROM numerictmp WHERE a = 0 ; SELECT count(*) FROM numerictmp WHERE a >= 0 ; count ------- - 579 + 581 (1 row) SELECT count(*) FROM numerictmp WHERE a > 0 ; count ------- - 576 + 578 (1 row) CREATE INDEX numericidx ON numerictmp USING gist ( a ); @@ -99,13 +99,13 @@ SET enable_seqscan=off; SELECT count(*) FROM numerictmp WHERE a < -1890.0; count ------- - 505 + 506 (1 row) SELECT count(*) FROM numerictmp WHERE a <= -1890.0; count ------- - 506 + 507 (1 row) SELECT count(*) FROM numerictmp WHERE a = -1890.0; @@ -117,37 +117,37 @@ SELECT count(*) FROM numerictmp WHERE a = -1890.0; SELECT count(*) FROM numerictmp WHERE a >= -1890.0; count ------- - 597 + 599 (1 row) SELECT count(*) FROM numerictmp WHERE a > -1890.0; count ------- - 596 + 598 (1 row) SELECT count(*) FROM numerictmp WHERE a < 'NaN' ; count ------- - 1100 + 1102 (1 row) SELECT count(*) FROM numerictmp WHERE a <= 'NaN' ; count ------- - 1102 + 1105 (1 row) SELECT count(*) FROM numerictmp WHERE a = 'NaN' ; count ------- - 2 + 3 (1 row) SELECT count(*) FROM numerictmp WHERE a >= 'NaN' ; count ------- - 2 + 3 (1 row) SELECT count(*) FROM numerictmp WHERE a > 'NaN' ; @@ -159,13 +159,13 @@ SELECT count(*) FROM numerictmp WHERE a > 'NaN' ; SELECT count(*) FROM numerictmp WHERE a < 0 ; count ------- - 523 + 524 (1 row) SELECT count(*) FROM numerictmp WHERE a <= 0 ; count ------- - 526 + 527 (1 row) SELECT count(*) FROM numerictmp WHERE a = 0 ; @@ -177,13 +177,13 @@ SELECT count(*) FROM numerictmp WHERE a = 0 ; SELECT count(*) FROM numerictmp WHERE a >= 0 ; count ------- - 579 + 581 (1 row) SELECT count(*) FROM numerictmp WHERE a > 0 ; count ------- - 576 + 578 (1 row) -- Test index-only scans diff --git a/contrib/btree_gist/sql/float4.sql b/contrib/btree_gist/sql/float4.sql index 3da1ce953c..71de5d5cf4 100644 --- a/contrib/btree_gist/sql/float4.sql +++ b/contrib/btree_gist/sql/float4.sql @@ -35,3 +35,20 @@ SELECT count(*) FROM float4tmp WHERE a > -179.0::float4; EXPLAIN (COSTS OFF) SELECT a, a <-> '-179.0' FROM float4tmp ORDER BY a <-> '-179.0' LIMIT 3; SELECT a, a <-> '-179.0' FROM float4tmp ORDER BY a <-> '-179.0' LIMIT 3; + +-- EXCLUDE constraint must block a duplicate NaN, same as it does for finite +-- values. +CREATE TABLE float4excl (a float4, EXCLUDE USING gist (a WITH =)); +INSERT INTO float4excl VALUES ('NaN'::float4); +INSERT INTO float4excl VALUES ('NaN'::float4); -- expect: violates EXCLUDE +SELECT count(*) FROM float4excl; + +-- Test double-column index +CREATE INDEX float4idx2 ON float4tmp USING gist ( a, abs(a) ); +EXPLAIN (COSTS OFF) +SELECT count(*) FROM float4tmp WHERE abs(a) = 179.0::float4; +SELECT count(*) FROM float4tmp WHERE abs(a) = 179.0::float4; + +RESET enable_seqscan; +RESET enable_indexscan; +RESET enable_bitmapscan; diff --git a/contrib/btree_gist/sql/float8.sql b/contrib/btree_gist/sql/float8.sql index e1e819b37f..a0fc84f94b 100644 --- a/contrib/btree_gist/sql/float8.sql +++ b/contrib/btree_gist/sql/float8.sql @@ -35,3 +35,20 @@ SELECT count(*) FROM float8tmp WHERE a > -1890.0::float8; EXPLAIN (COSTS OFF) SELECT a, a <-> '-1890.0' FROM float8tmp ORDER BY a <-> '-1890.0' LIMIT 3; SELECT a, a <-> '-1890.0' FROM float8tmp ORDER BY a <-> '-1890.0' LIMIT 3; + +-- EXCLUDE constraint must block a duplicate NaN, same as it does for finite +-- values. +CREATE TABLE float8excl (a float8, EXCLUDE USING gist (a WITH =)); +INSERT INTO float8excl VALUES ('NaN'::float8); +INSERT INTO float8excl VALUES ('NaN'::float8); -- expect: violates EXCLUDE +SELECT count(*) FROM float8excl; + +-- Test double-column index +CREATE INDEX float8idx2 ON float8tmp USING gist ( a, abs(a) ); +EXPLAIN (COSTS OFF) +SELECT count(*) FROM float8tmp WHERE abs(a) = 1890.0::float8; +SELECT count(*) FROM float8tmp WHERE abs(a) = 1890.0::float8; + +RESET enable_seqscan; +RESET enable_indexscan; +RESET enable_bitmapscan; From 802dc79df63b6af33429e99c497a7ecc8dca378d Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Fri, 3 Jul 2026 11:22:15 +0900 Subject: [PATCH 59/76] Remove replication slot advice from MultiXact wraparound hints Previously, MultiXactId wraparound hints suggested dropping stale replication slots. While that advice is appropriate for transaction ID wraparound, where replication slots can hold back XID horizons, it was misleading for MultiXactId wraparound. Following it could lead users to drop replication slots unnecessarily without helping resolve the MultiXactId wraparound condition. MultiXact cleanup is not directly delayed by replication slots. Instead, it depends on whether old MultiXactIds can still be seen as live by running transactions. This commit removes the replication slot advice from MultiXactId wraparound hints, and documents that stale replication slots are normally not relevant to resolving MultiXactId wraparound problems. Backpatch to all supported branches. BUG #18876 Reported-by: Haruka Takatsuka Author: Fujii Masao Discussion: https://postgr.es/m/18876-0d0b53bad5a1f4c1@postgresql.org Backpatch-through: 14 --- doc/src/sgml/maintenance.sgml | 6 ++++++ src/backend/access/transam/multixact.c | 12 ++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/maintenance.sgml b/doc/src/sgml/maintenance.sgml index 1ab5bfd0ae..f494c69cdc 100644 --- a/doc/src/sgml/maintenance.sgml +++ b/doc/src/sgml/maintenance.sgml @@ -800,6 +800,12 @@ HINT: Stop the postmaster and vacuum that database in single-user mode. Running transactions and prepared transactions can be ignored if there is no chance that they might appear in a multixact. + + Unlike transaction ID wraparound, replication slots do not + directly hold back multixact cleanup. Dropping stale replication + slots is therefore not usually relevant to resolving multixact ID + wraparound problems. + MXID information is not directly visible in system views such as pg_stat_activity; however, looking for old XIDs is still a good diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index 0b26ea7be3..50c8d9220e 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -1158,14 +1158,14 @@ GetNewMultiXactId(int nmembers, MultiXactOffset *offset) errmsg("database is not accepting commands that generate new MultiXactIds to avoid wraparound data loss in database \"%s\"", oldest_datname), errhint("Execute a database-wide VACUUM in that database.\n" - "You might also need to commit or roll back old prepared transactions, or drop stale replication slots."))); + "You might also need to commit or roll back old prepared transactions."))); else ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("database is not accepting commands that generate new MultiXactIds to avoid wraparound data loss in database with OID %u", oldest_datoid), errhint("Execute a database-wide VACUUM in that database.\n" - "You might also need to commit or roll back old prepared transactions, or drop stale replication slots."))); + "You might also need to commit or roll back old prepared transactions."))); } /* @@ -1189,7 +1189,7 @@ GetNewMultiXactId(int nmembers, MultiXactOffset *offset) oldest_datname, multiWrapLimit - result), errhint("Execute a database-wide VACUUM in that database.\n" - "You might also need to commit or roll back old prepared transactions, or drop stale replication slots."))); + "You might also need to commit or roll back old prepared transactions."))); else ereport(WARNING, (errmsg_plural("database with OID %u must be vacuumed before %u more MultiXactId is used", @@ -1198,7 +1198,7 @@ GetNewMultiXactId(int nmembers, MultiXactOffset *offset) oldest_datoid, multiWrapLimit - result), errhint("Execute a database-wide VACUUM in that database.\n" - "You might also need to commit or roll back old prepared transactions, or drop stale replication slots."))); + "You might also need to commit or roll back old prepared transactions."))); } /* Re-acquire lock and start over */ @@ -2476,7 +2476,7 @@ SetMultiXactIdLimit(MultiXactId oldest_datminmxid, Oid oldest_datoid, oldest_datname, multiWrapLimit - curMulti), errhint("To avoid a database shutdown, execute a database-wide VACUUM in that database.\n" - "You might also need to commit or roll back old prepared transactions, or drop stale replication slots."))); + "You might also need to commit or roll back old prepared transactions."))); else ereport(WARNING, (errmsg_plural("database with OID %u must be vacuumed before %u more MultiXactId is used", @@ -2485,7 +2485,7 @@ SetMultiXactIdLimit(MultiXactId oldest_datminmxid, Oid oldest_datoid, oldest_datoid, multiWrapLimit - curMulti), errhint("To avoid a database shutdown, execute a database-wide VACUUM in that database.\n" - "You might also need to commit or roll back old prepared transactions, or drop stale replication slots."))); + "You might also need to commit or roll back old prepared transactions."))); } } From 2d44bb9009cdc0454d863a03a78812b4259b3381 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Fri, 3 Jul 2026 13:50:51 +0900 Subject: [PATCH 60/76] psql: Fix \df tab completion for procedures Commit fb421231daa extended \df to include procedures, but its tab completion continued not to show procedures. Update \df tab completion to include procedures as well. Backpatch to all supported versions. Author: Erik Wienhold Reviewed-by: Surya Poondla Reviewed-by: Fujii Masao Discussion: https://postgr.es/m/10fbfdfe-80f6-4ef9-b8b3-f7be0eb53a50@ewie.name Backpatch-through: 14 --- src/bin/psql/tab-complete.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 4f0416d6b3..2749841323 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -4036,7 +4036,7 @@ psql_completion(const char *text, int start, int end) else if (TailMatchesCS("\\dew*")) COMPLETE_WITH_QUERY(Query_for_list_of_fdws); else if (TailMatchesCS("\\df*")) - COMPLETE_WITH_VERSIONED_SCHEMA_QUERY(Query_for_list_of_functions, NULL); + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_routines, NULL); else if (HeadMatchesCS("\\df*")) COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_datatypes, NULL); From 286f9a3cec4b9c4b1d1c49d0b35b6ef074ed2629 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 3 Jul 2026 13:50:14 -0400 Subject: [PATCH 61/76] Fix btree_gist's NotEqual strategy on internal index pages. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gbt_var_consistent() handled the <> (BtreeGistNotEqual) strategy without distinguishing leaf from internal pages, unlike every other strategy. In particular, it tried to apply the datatype-specific f_eq method, which is completely wrong since internal keys might not have the same representation as leaf keys. This led to OOB reads and potentially crashes, and most likely to wrong query results as well. On leaf pages we can apply the inverse of what the Equal strategy does. On internal pages, use a correct implementation of what the previous code intended: we can descend if the query value equals both bounds, *so long as the bounds aren't truncated*. With truncated bounds we don't quite know the range of what's below, so we must always descend. Adjust the code in gbt_num_consistent() to look similar, too. This fixes a performance buglet in that there's no need to do two comparisons on a leaf entry, but the main point is just to keep code consistency. Reported-by: 王跃林 Author: Ayush Tiwari Reviewed-by: Tom Lane Discussion: https://postgr.es/m/AH*AvQCYKhQGVvPWi1GiU4oY.8.1781609375063.Hmail.3020001251@tju.edu.cn Backpatch-through: 14 --- contrib/btree_gist/btree_utils_num.c | 15 +++++++++++++-- contrib/btree_gist/btree_utils_var.c | 23 +++++++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/contrib/btree_gist/btree_utils_num.c b/contrib/btree_gist/btree_utils_num.c index 7564a403c7..a71f41066f 100644 --- a/contrib/btree_gist/btree_utils_num.c +++ b/contrib/btree_gist/btree_utils_num.c @@ -289,8 +289,19 @@ gbt_num_consistent(const GBT_NUMKEY_R *key, retval = tinfo->f_le(query, key->upper, flinfo); break; case BtreeGistNotEqualStrategyNumber: - retval = (!(tinfo->f_eq(query, key->lower, flinfo) && - tinfo->f_eq(query, key->upper, flinfo))); + if (is_leaf) + retval = !(tinfo->f_eq(query, key->lower, flinfo)); + else + { + /* + * If the upper/lower bounds are equal, then all entries below + * this node must have exactly that value. So we can avoid + * descending if the query equals both bounds. In all other + * cases, we must descend. + */ + retval = !(tinfo->f_eq(query, key->lower, flinfo) && + tinfo->f_eq(query, key->upper, flinfo)); + } break; default: retval = false; diff --git a/contrib/btree_gist/btree_utils_var.c b/contrib/btree_gist/btree_utils_var.c index 9d93b3c775..27358d3501 100644 --- a/contrib/btree_gist/btree_utils_var.c +++ b/contrib/btree_gist/btree_utils_var.c @@ -572,6 +572,13 @@ gbt_var_consistent(GBT_VARKEY_R *key, { bool retval = false; + /* + * Remember that f_cmp is for internal pages, f_eq etc for leaf pages, and + * on internal pages we need to check gbt_var_node_pf_match too. + * + * The leaf-page tests use swapped operands (e.g., f_gt(query, lower) + * means "lower < query"), which is why they look reversed. + */ switch (strategy) { case BTLessEqualStrategyNumber: @@ -612,8 +619,20 @@ gbt_var_consistent(GBT_VARKEY_R *key, || gbt_var_node_pf_match(key, query, tinfo); break; case BtreeGistNotEqualStrategyNumber: - retval = !(tinfo->f_eq(query, key->lower, collation, flinfo) && - tinfo->f_eq(query, key->upper, collation, flinfo)); + if (is_leaf) + retval = !(tinfo->f_eq(query, key->lower, collation, flinfo)); + else + { + /* + * If the upper/lower bounds are equal and not truncated, then + * all entries below this node must have exactly that value. + * So we can avoid descending if the query equals both bounds. + * In all other cases, we must descend. + */ + retval = tinfo->trnc || + !(tinfo->f_cmp(query, key->lower, collation, flinfo) == 0 && + tinfo->f_cmp(query, key->upper, collation, flinfo) == 0); + } break; default: retval = false; From 1b17a6e3cd89f76ea5db6260c79db620d5a7793b Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Sat, 4 Jul 2026 11:34:26 -0400 Subject: [PATCH 62/76] Disallow renaming a rule to "_RETURN". ON SELECT rules must be named "_RETURN", while other kinds of rules must not be; this ancient restriction is depended on by various client code. We successfully enforced this convention in most places, but ALTER RULE allowed renaming a non-SELECT rule to "_RETURN". Notably, that would break dump/restore, since the eventual CREATE RULE command would reject the name. While at it, remove DefineQueryRewrite's hack to substitute "_RETURN" for the convention that was used before 7.3. We dropped other server-side code that supported restoring pre-7.3 dumps some time ago (notably in e58a59975 and nearby commits), but this bit was missed. Bug: #19543 Reported-by: Adam Pickering Author: Tom Lane Discussion: https://postgr.es/m/19543-461228e77f3b32fc@postgresql.org Backpatch-through: 14 --- src/backend/rewrite/rewriteDefine.c | 36 +++++++++++++---------------- src/test/regress/expected/rules.out | 2 ++ src/test/regress/sql/rules.sql | 1 + 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/backend/rewrite/rewriteDefine.c b/src/backend/rewrite/rewriteDefine.c index 08ed31f9e4..fedf425436 100644 --- a/src/backend/rewrite/rewriteDefine.c +++ b/src/backend/rewrite/rewriteDefine.c @@ -389,26 +389,11 @@ DefineQueryRewrite(const char *rulename, * ... and finally the rule must be named _RETURN. */ if (strcmp(rulename, ViewSelectRuleName) != 0) - { - /* - * In versions before 7.3, the expected name was _RETviewname. For - * backwards compatibility with old pg_dump output, accept that - * and silently change it to _RETURN. Since this is just a quick - * backwards-compatibility hack, limit the number of characters - * checked to a few less than NAMEDATALEN; this saves having to - * worry about where a multibyte character might have gotten - * truncated. - */ - if (strncmp(rulename, "_RET", 4) != 0 || - strncmp(rulename + 4, RelationGetRelationName(event_relation), - NAMEDATALEN - 4 - 4) != 0) - ereport(ERROR, - (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), - errmsg("view rule for \"%s\" must be named \"%s\"", - RelationGetRelationName(event_relation), - ViewSelectRuleName))); - rulename = pstrdup(ViewSelectRuleName); - } + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("view rule for \"%s\" must be named \"%s\"", + RelationGetRelationName(event_relation), + ViewSelectRuleName))); /* * Are we converting a relation to a view? @@ -1026,6 +1011,17 @@ RenameRewriteRule(RangeVar *relation, const char *oldName, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), errmsg("renaming an ON SELECT rule is not allowed"))); + /* + * Conversely, if it's not an ON SELECT rule then it must *not* be named + * _RETURN. + */ + if (strcmp(newName, ViewSelectRuleName) == 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("non-view rule for \"%s\" must not be named \"%s\"", + RelationGetRelationName(targetrel), + ViewSelectRuleName))); + /* OK, do the update */ namestrcpy(&(ruleform->rulename), newName); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 942caf8921..4216aebc9e 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -3321,6 +3321,8 @@ ALTER RULE NewInsertRule ON rule_v1 RENAME TO "_RETURN"; -- already exists ERROR: rule "_RETURN" for relation "rule_v1" already exists ALTER RULE "_RETURN" ON rule_v1 RENAME TO abc; -- ON SELECT rule cannot be renamed ERROR: renaming an ON SELECT rule is not allowed +ALTER RULE rtest_t4_ins1 ON rtest_t4 RENAME TO "_RETURN"; -- also disallowed +ERROR: non-view rule for "rtest_t4" must not be named "_RETURN" DROP VIEW rule_v1; DROP TABLE rule_t1; -- diff --git a/src/test/regress/sql/rules.sql b/src/test/regress/sql/rules.sql index 0c7f3df45c..05101017d7 100644 --- a/src/test/regress/sql/rules.sql +++ b/src/test/regress/sql/rules.sql @@ -1106,6 +1106,7 @@ SELECT * FROM rule_v1; ALTER RULE InsertRule ON rule_v1 RENAME TO NewInsertRule; -- doesn't exist ALTER RULE NewInsertRule ON rule_v1 RENAME TO "_RETURN"; -- already exists ALTER RULE "_RETURN" ON rule_v1 RENAME TO abc; -- ON SELECT rule cannot be renamed +ALTER RULE rtest_t4_ins1 ON rtest_t4 RENAME TO "_RETURN"; -- also disallowed DROP VIEW rule_v1; DROP TABLE rule_t1; From 0115650de9374888f2e64d8dae5fbfd06ec45c7e Mon Sep 17 00:00:00 2001 From: Robert Haas Date: Mon, 6 Jul 2026 12:12:41 -0400 Subject: [PATCH 63/76] Prevent satisfies_hash_partition from crashing with VARIADIC NULL. Commit f3b0897a1213f46b4d3a99a7f8ef3a4b32e03572 fixed some related problems, but overlooked this one. That commit first appeared in PostgreSQL 11, so back-patch to all supported branches. Backpatch-through: 14 Discussion: http://postgr.es/m/CA+TgmobsvQw3F+KRYT83=N3teh8D2t-oPR=U06QDZJE3viCJRg@mail.gmail.com Reviewed-by: Tender Wang Reviewed-by: Ewan Young --- src/backend/partitioning/partbounds.c | 14 +++++++++++++- src/test/regress/expected/hash_part.out | 7 +++++++ src/test/regress/sql/hash_part.sql | 3 +++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/backend/partitioning/partbounds.c b/src/backend/partitioning/partbounds.c index 4c3b7df313..abb7d410a9 100644 --- a/src/backend/partitioning/partbounds.c +++ b/src/backend/partitioning/partbounds.c @@ -4777,6 +4777,12 @@ satisfies_hash_partition(PG_FUNCTION_ARGS) fcinfo->flinfo->fn_mcxt); } } + else if (PG_ARGISNULL(3)) + { + /* Special case for VARIADIC NULL::sometype[] */ + relation_close(parent, NoLock); + PG_RETURN_BOOL(false); + } else { ArrayType *variadic_array = PG_GETARG_ARRAYTYPE_P(3); @@ -4847,12 +4853,18 @@ satisfies_hash_partition(PG_FUNCTION_ARGS) } else { - ArrayType *variadic_array = PG_GETARG_ARRAYTYPE_P(3); + ArrayType *variadic_array; int i; int nelems; Datum *datum; bool *isnull; + /* Special case for VARIADIC NULL::sometype[] */ + if (PG_ARGISNULL(3)) + PG_RETURN_BOOL(false); + + variadic_array = PG_GETARG_ARRAYTYPE_P(3); + deconstruct_array(variadic_array, my_extra->variadic_type, my_extra->variadic_typlen, diff --git a/src/test/regress/expected/hash_part.out b/src/test/regress/expected/hash_part.out index ac3aabee02..c54b74abf6 100644 --- a/src/test/regress/expected/hash_part.out +++ b/src/test/regress/expected/hash_part.out @@ -40,6 +40,13 @@ SELECT satisfies_hash_partition('mchash'::regclass, 4, NULL, NULL); f (1 row) +-- variadic null +SELECT satisfies_hash_partition('mchash'::regclass, 4, 0, VARIADIC NULL::int[]); + satisfies_hash_partition +-------------------------- + f +(1 row) + -- too many arguments SELECT satisfies_hash_partition('mchash'::regclass, 4, 0, NULL::int, NULL::text, NULL::json); ERROR: number of partitioning columns (2) does not match number of partition keys provided (3) diff --git a/src/test/regress/sql/hash_part.sql b/src/test/regress/sql/hash_part.sql index e7eb36542c..0eca58469a 100644 --- a/src/test/regress/sql/hash_part.sql +++ b/src/test/regress/sql/hash_part.sql @@ -35,6 +35,9 @@ SELECT satisfies_hash_partition('mchash'::regclass, NULL, 0, NULL); -- remainder is null SELECT satisfies_hash_partition('mchash'::regclass, 4, NULL, NULL); +-- variadic null +SELECT satisfies_hash_partition('mchash'::regclass, 4, 0, VARIADIC NULL::int[]); + -- too many arguments SELECT satisfies_hash_partition('mchash'::regclass, 4, 0, NULL::int, NULL::text, NULL::json); From f59a592f9b2fe64affdbc63187d2da763df0caf0 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 7 Jul 2026 18:11:28 +0300 Subject: [PATCH 64/76] pg_dump: check for _beginthreadex() failure in parallel dump ParallelBackupStart() stored _beginthreadex()'s return value as the worker's thread handle without checking it. On failure that value is 0, which would later reach WaitForMultipleObjects() as a null handle, caught only by an Assert. The fork() path already calls pg_fatal() when it fails; do the same for _beginthreadex(), as pgbench does. Author: Bryan Green Discussion: https://www.postgresql.org/message-id/8c712d76-ecf7-4749-a6d8-dddc01f298ec@gmail.com Backpatch-through: 14 --- src/bin/pg_dump/parallel.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bin/pg_dump/parallel.c b/src/bin/pg_dump/parallel.c index f1577e785f..323a97c69b 100644 --- a/src/bin/pg_dump/parallel.c +++ b/src/bin/pg_dump/parallel.c @@ -979,6 +979,8 @@ ParallelBackupStart(ArchiveHandle *AH) handle = _beginthreadex(NULL, 0, (void *) &init_spawned_worker_win32, wi, 0, &(slot->threadId)); + if (handle == 0) + fatal("could not create worker thread: %m"); slot->hThread = handle; slot->workerStatus = WRKR_IDLE; #else /* !WIN32 */ From 1b018b5d10561a49fcd987a0a659ec06fbd6ee8f Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 7 Jul 2026 18:45:34 +0300 Subject: [PATCH 65/76] libpq: Extend "read pending" check from SSL to GSS An extra check for pending bytes in the SSL layer has been part of pqReadReady() for a very long time (79ff2e96d). But when GSS transport encryption was added, it didn't receive the same treatment. (As 79ff2e96d notes, "The bug that I fixed in this patch is exceptionally hard to reproduce reliably.") Without that check, it's possible to hit a hang in gssencmode, if the server splits a large libpq message such that the final message in a streamed response is part of the same wrapped token as the split message: DataRowDataRowDataRowDataRowDataRowData -- token boundary -- RowDataRowCommandCompleteReadyForQuery If the split message takes up enough memory to nearly fill libpq's receive buffer, libpq may return from pqReadData() before the later messages are pulled out of the PqGSSRecvBuffer. Without additional socket activity from the server, pqReadReady() (via pqSocketCheck()) will never again return true, hanging the connection. Pull the pending-bytes check into the pqsecure API layer, where both SSL and GSS now implement it. Note that this does not fix the root problem! Third party clients of libpq have no way to call pqsecure_read_is_pending() in their own polling. This just brings the GSS implementation up to par with the existing SSL workaround; a broader fix is left to a subsequent commit. In preparation for the broader fix, this patch already changes the *_read_pending() functions to return the number of bytes in the buffer rather than just a boolean. The current callers don't need that, but the subsequent fix will. Author: Jacob Champion Discussion: https://postgr.es/m/CAOYmi%2BmpymrgZ76Jre2dx_PwRniS9YZojwH0rZnTuiGHCsj0rA%40mail.gmail.com Backpatch-through: 14 --- src/interfaces/libpq/fe-misc.c | 6 ++-- src/interfaces/libpq/fe-secure-gssapi.c | 7 +++++ src/interfaces/libpq/fe-secure-openssl.c | 36 ++++++++++++++++++++++-- src/interfaces/libpq/fe-secure.c | 22 +++++++++++++++ src/interfaces/libpq/libpq-int.h | 6 ++-- 5 files changed, 68 insertions(+), 9 deletions(-) diff --git a/src/interfaces/libpq/fe-misc.c b/src/interfaces/libpq/fe-misc.c index aa079e82ea..e153937746 100644 --- a/src/interfaces/libpq/fe-misc.c +++ b/src/interfaces/libpq/fe-misc.c @@ -1087,14 +1087,12 @@ pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time) return -1; } -#ifdef USE_SSL - /* Check for SSL library buffering read bytes */ - if (forRead && conn->ssl_in_use && pgtls_read_pending(conn)) + /* Check for SSL/GSS library buffering read bytes */ + if (forRead && pqsecure_bytes_pending(conn) != 0) { /* short-circuit the select */ return 1; } -#endif /* We will retry as long as we get EINTR */ do diff --git a/src/interfaces/libpq/fe-secure-gssapi.c b/src/interfaces/libpq/fe-secure-gssapi.c index 9508a1a8b2..bc151b2904 100644 --- a/src/interfaces/libpq/fe-secure-gssapi.c +++ b/src/interfaces/libpq/fe-secure-gssapi.c @@ -475,6 +475,13 @@ gss_read(PGconn *conn, void *recv_buffer, size_t length, ssize_t *ret) return PGRES_POLLING_OK; } +ssize_t +pg_GSS_bytes_pending(PGconn *conn) +{ + Assert(PqGSSResultLength >= PqGSSResultNext); + return (ssize_t) (PqGSSResultLength - PqGSSResultNext); +} + /* * Negotiate GSSAPI transport for a connection. When complete, returns * PGRES_POLLING_OK. Will return PGRES_POLLING_READING or diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index 915a23cf20..b20c4d2981 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -259,10 +259,40 @@ pgtls_read(PGconn *conn, void *ptr, size_t len) return n; } -bool -pgtls_read_pending(PGconn *conn) +ssize_t +pgtls_bytes_pending(PGconn *conn) { - return SSL_pending(conn->ssl) > 0; + int pending; + + /* + * OpenSSL readahead is documented to break SSL_pending(). + */ + Assert(!SSL_get_read_ahead(conn->ssl)); + + pending = SSL_pending(conn->ssl); + if (pending < 0) + { + /* shouldn't be possible */ + Assert(false); + appendPQExpBufferStr(&conn->errorMessage, + "OpenSSL reports negative bytes pending\n"); + return -1; + } + else if (pending == INT_MAX) + { + /* + * If we ever found a legitimate way to hit this, we'd need to loop + * around in the caller to call pgtls_bytes_pending() again. Throw an + * error rather than complicate the code in that way, because + * SSL_read() should be bounded to the size of a single TLS record, + * and conn->inBuffer can't currently go past INT_MAX in size anyway. + */ + appendPQExpBufferStr(&conn->errorMessage, + "OpenSSL reports INT_MAX bytes pending"); + return -1; + } + + return (ssize_t) pending; } ssize_t diff --git a/src/interfaces/libpq/fe-secure.c b/src/interfaces/libpq/fe-secure.c index 63bb13353d..e9462864b5 100644 --- a/src/interfaces/libpq/fe-secure.c +++ b/src/interfaces/libpq/fe-secure.c @@ -284,6 +284,28 @@ pqsecure_raw_read(PGconn *conn, void *ptr, size_t len) return n; } +/* + * Return the number of bytes available in the transport buffer. + * + * If pqsecure_read() is called for this number of bytes, it's guaranteed to + * return successfully without reading from the underlying socket. + */ +ssize_t +pqsecure_bytes_pending(PGconn *conn) +{ +#ifdef USE_SSL + if (conn->ssl_in_use) + return pgtls_bytes_pending(conn); +#endif +#ifdef ENABLE_GSS + if (conn->gssenc) + return pg_GSS_bytes_pending(conn); +#endif + + /* Plaintext connections have no transport buffer. */ + return 0; +} + /* * Write data to a secure connection. * diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 6f34adb9f7..9ac8f32e27 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -726,6 +726,7 @@ extern int pqsecure_initialize(PGconn *, bool, bool); extern PostgresPollingStatusType pqsecure_open_client(PGconn *); extern void pqsecure_close(PGconn *); extern ssize_t pqsecure_read(PGconn *, void *ptr, size_t len); +extern ssize_t pqsecure_bytes_pending(PGconn *); extern ssize_t pqsecure_write(PGconn *, const void *ptr, size_t len); extern ssize_t pqsecure_raw_read(PGconn *, void *ptr, size_t len); extern ssize_t pqsecure_raw_write(PGconn *, const void *ptr, size_t len); @@ -779,9 +780,9 @@ extern void pgtls_close(PGconn *conn); extern ssize_t pgtls_read(PGconn *conn, void *ptr, size_t len); /* - * Is there unread data waiting in the SSL read buffer? + * Return the number of bytes available in the transport buffer. */ -extern bool pgtls_read_pending(PGconn *conn); +extern ssize_t pgtls_bytes_pending(PGconn *conn); /* * Write data to a secure connection. @@ -835,6 +836,7 @@ extern PostgresPollingStatusType pqsecure_open_gss(PGconn *conn); */ extern ssize_t pg_GSS_write(PGconn *conn, const void *ptr, size_t len); extern ssize_t pg_GSS_read(PGconn *conn, void *ptr, size_t len); +extern ssize_t pg_GSS_bytes_pending(PGconn *conn); #endif /* === in libpq-trace.c === */ From d3888c90a07f1a2134a162eace20d6e32eb38038 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 7 Jul 2026 18:45:37 +0300 Subject: [PATCH 66/76] libpq: Drain all pending bytes from SSL/GSS during pqReadData() The previous commit strengthened a workaround for a hang when large messages are split across TLS records/GSS tokens. Because that workaround is implemented in libpq internals, it can only help us when libpq itself is polling on the socket. In nonblocking situations, where the client above libpq is expected to poll, the same bugs can show up. As a contrived example, consider a large protocol-2.0 error coming back from a server during PQconnectPoll(), split in an odd way across two records: -- TLS record (8192-byte payload) -- EEEE[...repeated a total of 8192 times] -- TLS record (8193-byte payload) -- EEEE[...repeated a total of 8192 times]\0 The first record will fill the first half of the libpq receive buffer, which is 16k long by default. The second record completely fills the last half with its first 8192 bytes, leaving the terminating NULL in the OpenSSL buffer. Since we still haven't seen the terminator at our level, PQconnectPoll() will return PGRES_POLLING_READING, expecting to come back when the server has sent "the rest" of the data. But there is nothing left to read from the socket; OpenSSL had to pull all of the data in the 8193-byte record off of the wire to decrypt it. A real server would probably not split up the records this way, nor keep the connection open after sending a fatal connection error. But servers that regularly use larger TLS records can get the libpq receive buffer into the same state if DataRows are big enough, as reported on the list. While the PostgreSQL server doesn't use larger TLS records like that, other non-PostgreSQL servers that implement the wire protocol are known to do that, as well as proxies that sit between the server and the client This is a layering violation. libpq makes decisions based on data in the application buffer, above the transport buffer (whether SSL or GSS), but clients are polling the socket below the transport buffer. One way to fix this in a backportable way, without changing APIs too much, is to ensure data never stays in the transport buffer. Then pqReadData's postconditions will look similar for both raw sockets and SSL/GSS: any available data is either in the application buffer, or still on the socket. Building on the prior commit, make pqReadData() to drain all pending data from the transport layer into conn->inBuffer, expanding the buffer as necessary. This is not particularly efficient from an architectural perspective (the pqsecure_read() implementations take care to fit their packets into the current buffer, and that effort is now completely discarded), but it's hopefully easier to reason about than a full rewrite would be for the back branches. Author: Jacob Champion Reviewed-by: Mark Dilger Reviewed-by: solai v Reported-by: Lars Kanis Discussion: https://postgr.es/m/2039ac58-d3e0-434b-ac1a-2a987f3b4cb1%40greiz-reinsdorf.de Backpatch-through: 14 --- src/interfaces/libpq/fe-misc.c | 145 ++++++++++++++++++++++- src/interfaces/libpq/fe-secure-openssl.c | 4 +- src/interfaces/libpq/fe-secure.c | 3 +- 3 files changed, 148 insertions(+), 4 deletions(-) diff --git a/src/interfaces/libpq/fe-misc.c b/src/interfaces/libpq/fe-misc.c index e153937746..3633b125ba 100644 --- a/src/interfaces/libpq/fe-misc.c +++ b/src/interfaces/libpq/fe-misc.c @@ -58,6 +58,8 @@ static int pqSendSome(PGconn *conn, int len); static int pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time); static int pqSocketPoll(int sock, int forRead, int forWrite, time_t end_time); +static int pqReadData_internal(PGconn *conn); +static int pqDrainPending(PGconn *conn); /* * PQlibVersion: return the libpq version number @@ -581,6 +583,13 @@ pqPutMsgEnd(PGconn *conn) /* ---------- * pqReadData: read more data, if any is available + * + * Upon a successful return, callers may assume that either 1) all available + * bytes have been consumed from the socket, or 2) the socket is still marked + * readable by the OS. (In other words: after a successful pqReadData, it's + * safe to tell a client to poll for readable bytes on the socket without any + * further draining of the SSL/GSS transport buffers.) + * * Possible return values: * 1: successfully loaded at least one more byte * 0: no data is presently available, but no error detected @@ -593,8 +602,7 @@ pqPutMsgEnd(PGconn *conn) int pqReadData(PGconn *conn) { - int someread = 0; - int nread; + int available; if (conn->sock == PGINVALID_SOCKET) { @@ -603,6 +611,40 @@ pqReadData(PGconn *conn) return -1; } + available = pqReadData_internal(conn); + if (available < 0) + return -1; + else if (available > 0) + { + /* + * Make sure there are no bytes stuck in layers between conn->inBuffer + * and the socket, to make it safe for clients to poll on PQsocket(). + */ + if (pqDrainPending(conn)) + return -1; + } + else + { + /* + * If we're not returning any bytes from the underlying transport, + * that must imply there aren't any in the transport buffer... + */ + Assert(pqsecure_bytes_pending(conn) == 0); + } + + return available; +} + +/* + * Workhorse for pqReadData(). It's kept separate from the pqDrainPending() + * logic to avoid adding to this function's goto complexity. + */ +static int +pqReadData_internal(PGconn *conn) +{ + int someread = 0; + int nread; + /* Left-justify any data in the buffer to make room */ if (conn->inStart < conn->inEnd) { @@ -790,6 +832,105 @@ pqReadData(PGconn *conn) return -1; } +/*--- + * Drain any transport data that is already buffered in userspace and add it + * to conn->inBuffer, enlarging inBuffer if necessary. The drain fails if + * inBuffer cannot be made to hold all available transport data. + * + * We assume that the underlying secure transport implementation does not + * attempt to read any more data from the socket while draining the transport + * buffer. After a successful return, pqsecure_bytes_pending() must be zero. + * + * This operation is necessary to prevent deadlock, due to a layering + * violation designed into our asynchronous client API: pqReadData() and all + * the parsing routines above it receive data from the SSL/GSS transport + * buffer, but clients poll on the raw PQsocket() handle. So data can be + * "lost" in the intermediate layer if we don't take it out here. + * + * To illustrate what we're trying to prevent, say that the server is sending + * two messages at once in response to a query (Aaaa and Bb), the libpq buffer + * is five characters in size, and TLS records max out at three-character + * payloads. Here's what would happen if pqReadData() didn't call + * pqDrainPending(): + * + * Client libpq SSL Socket + * | | | | + * | [ ] [ ] [ ] [1] Buffers are empty, client is + * x --------------------------> | polling on socket + * | | | | + * | [ ] [ ] [xxx] [2] First record is received; poll + * | <-------------------------- | signals read-ready + * | | | | + * x ---> [ ] [ ] [xxx] [3] Client calls PQconsumeInput() + * | | | | + * | [ ] -> [ ] [xxx] [4] libpq calls pqReadData() to fill + * | | | | the receive buffer + * | [ ] [Aaa] <-- [ ] [5] SSL pulls payload off the wire + * | | | | and decrypts it + * | [Aaa ] <- [ ] [ ] [6] pqsecure_read() takes all data + * | | | | + * | <--- [Aaa ] [ ] [ ] [7] PQconsumeInput() returns with a + * x --------------------------> | partial message, PQisBusy() is + * | | | | still true, client polls again + * | [Aaa ] [ ] [xxx] [8] Second record is received; poll + * | <-------------------------- | signals read-ready + * | | | | + * x ---> [Aaa ] [ ] [xxx] [9] Client calls PQconsumeInput() + * | | | | + * | [Aaa ] -> [ ] [xxx] [10] libpq calls pqReadData() to fill + * | | | | the receive buffer + * | [Aaa ] [aBb] <-- [ ] [11] SSL decrypts + * | | | | + * | [AaaaB] <- [b ] [ ] [12] pqsecure_read() fills its + * | | | | buffer, taking only two bytes + * | <--- [AaaaB] [b ] [ ] [13] PQconsumeInput() returns with a + * | | | | complete message buffered; + * | | | | PQisBusy() is false + * x ---> [AaaaB] [b ] [ ] [14] Client calls PQgetResult() + * | | | | + * | <--- [B ] [b ] [ ] [15] Aaaa is returned; PQisBusy() is + * x --------------------------> | true and client polls again + * . | | . + * . [B ] [b ] . [16] No packets, and client hangs. + * . | | . + * + * The pqDrainPending() call fixes the above scenario at step [13]. Before + * returning to the Client, it first expands the libpq buffer and moves the + * remaining data from the SSL buffer to the libpq buffer. + * + * The function returns 0 on success and -1 on error. Success means that + * there was no data pending or it was successfully drained to conn->inBuffer. + * On error, conn->errorMessage is set. + */ +static int +pqDrainPending(PGconn *conn) +{ + ssize_t bytes_pending; + ssize_t nread; + + bytes_pending = pqsecure_bytes_pending(conn); + if (bytes_pending <= 0) + return bytes_pending; + + /* Expand the input buffer if necessary. */ + if (pqCheckInBufferSpace(conn->inEnd + (size_t) bytes_pending, conn)) + return -1; /* errorMessage already set */ + + nread = pqsecure_read(conn, conn->inBuffer + conn->inEnd, + bytes_pending); + conn->inEnd += nread; + + /* When there are bytes pending, the read function is not supposed to fail */ + if (nread != bytes_pending) + { + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("drained only %zu of %zd pending bytes in transport buffer\n"), + nread, bytes_pending); + return -1; + } + return 0; +} + /* * pqSendSome: send data waiting in the output buffer. * diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index b20c4d2981..3c73071518 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -265,7 +265,9 @@ pgtls_bytes_pending(PGconn *conn) int pending; /* - * OpenSSL readahead is documented to break SSL_pending(). + * OpenSSL readahead is documented to break SSL_pending(). Plus, we can't + * afford to have OpenSSL take bytes off the socket without processing + * them; that breaks the postconditions for pqsecure_drain_pending(). */ Assert(!SSL_get_read_ahead(conn->ssl)); diff --git a/src/interfaces/libpq/fe-secure.c b/src/interfaces/libpq/fe-secure.c index e9462864b5..feb50f7e22 100644 --- a/src/interfaces/libpq/fe-secure.c +++ b/src/interfaces/libpq/fe-secure.c @@ -288,7 +288,8 @@ pqsecure_raw_read(PGconn *conn, void *ptr, size_t len) * Return the number of bytes available in the transport buffer. * * If pqsecure_read() is called for this number of bytes, it's guaranteed to - * return successfully without reading from the underlying socket. + * return successfully without reading from the underlying socket. See + * pqDrainPending() for a more complete discussion of the concepts involved. */ ssize_t pqsecure_bytes_pending(PGconn *conn) From 1e8f393f2d037d3821e79349c8fac22ce697e8cd Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 8 Jul 2026 09:04:31 +0900 Subject: [PATCH 67/76] doc: Fix typo in rule-system view example Commit dcb00495236 accidentally changed the final expanded query's condition to > 2 while rewriting the example into SQL operator notation. The original query and the preceding rewritten forms all use >= 2, and view expansion should preserve that qualification. This commit changes the final condition from > 2 to >= 2. Backpatch to all supported versions. Reported-by: Yaroslav Saburov Author: Fujii Masao Reviewed-by: Daniel Gustafsson Discussion: https://postgr.es/m/178248467618.108999.9966122434342474006@wrigleys.postgresql.org Backpatch-through: 14 --- doc/src/sgml/rules.sgml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sgml/rules.sgml b/doc/src/sgml/rules.sgml index 4aa4e00e01..ce9c93dacf 100644 --- a/doc/src/sgml/rules.sgml +++ b/doc/src/sgml/rules.sgml @@ -624,7 +624,7 @@ SELECT shoe_ready.shoename, shoe_ready.sh_avail, WHERE rsl.sl_color = rsh.slcolor AND rsl.sl_len_cm >= rsh.slminlen_cm AND rsl.sl_len_cm <= rsh.slmaxlen_cm) shoe_ready - WHERE shoe_ready.total_avail > 2; + WHERE shoe_ready.total_avail >= 2; From fd224fe90823de3df4f209c09a5794606ff70dc7 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Wed, 8 Jul 2026 12:45:20 +0900 Subject: [PATCH 68/76] doc: Clarify COPY FROM WHERE expression restrictions Commit aa606b9316a disallowed generated columns in COPY FROM WHERE expressions, and commit 21c69dc73f9 disallowed system columns. However, the COPY reference page still mentions only the restriction on subqueries. Update the documentation to also list generated columns and system columns as unsupported in COPY FROM WHERE expressions. Backpatch the generated-column documentation change to all supported versions. Backpatch the system-column documentation change to v19, where that restriction was introduced. Author: Fujii Masao Reviewed-by: Ayush Tiwari Discussion: https://postgr.es/m/CAHGQGwEgxErc54yVOAVWCsr1O=8pgw4oKRPuEQ9mfhkoYGR_XA@mail.gmail.com Backpatch-through: 14 --- doc/src/sgml/ref/copy.sgml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 9055d4e675..47e50afd22 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -379,10 +379,11 @@ WHERE condition - Currently, subqueries are not allowed in WHERE - expressions, and the evaluation does not see any changes made by the - COPY itself (this matters when the expression - contains calls to VOLATILE functions). + Currently, subqueries and generated columns are not allowed in + WHERE expressions, and the evaluation does not see + any changes made by the COPY itself (this matters + when the expression contains calls to VOLATILE + functions). From f5f569713b871ca6771e6fbaf5d4de16d4df1728 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 9 Jul 2026 18:34:24 +0300 Subject: [PATCH 69/76] ssl: Include limits.h to get INT_MAX when using LibreSSL When compiling against OpenSSL, the header is indirectly included via openssl/ossl_typ.h from openssl/conf.h, but the LibreSSL version of ossl_typ.h does not include which cause compiler failure due to missing symbol (since ffd080d94fe). Fix by explicitly including . Author: Daniel Gustafsson Discussion: https://www.postgresql.org/message-id/6A9E7815-BD5A-4C31-A515-48159823406B@yesql.se Backpatch-through: 14 --- src/interfaces/libpq/fe-secure-openssl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index 3c73071518..48cec07cef 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "libpq-fe.h" #include "fe-auth.h" From 7532f2117c4e422ca08896473689f589309219c3 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 9 Jul 2026 18:34:27 +0300 Subject: [PATCH 70/76] libpq: Make error checks in the new buffer draining code more robust Check explicitly for pqsecure_read() returning an error. It shouldn't fail, and we would've caught it in the check for a short read, but better to be explicit so that the error message is more informative. We also shouldn't update 'inEnd' when the read fails, although that too is just pro forma as we will bail out and close the connection on error. Reported-by: Peter Eisentraut Discussion: https://www.postgresql.org/message-id/34844e8c-267c-4daf-b1e0-f26059a4a7d3@eisentraut.org Backpatch-through: 14 --- src/interfaces/libpq/fe-misc.c | 11 ++++++++--- src/interfaces/libpq/fe-secure.c | 5 +++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/interfaces/libpq/fe-misc.c b/src/interfaces/libpq/fe-misc.c index 3633b125ba..01f55732b0 100644 --- a/src/interfaces/libpq/fe-misc.c +++ b/src/interfaces/libpq/fe-misc.c @@ -918,13 +918,18 @@ pqDrainPending(PGconn *conn) nread = pqsecure_read(conn, conn->inBuffer + conn->inEnd, bytes_pending); - conn->inEnd += nread; - /* When there are bytes pending, the read function is not supposed to fail */ + /* + * When there are bytes pending, pqsecure_read() is not supposed to fail + * or do a short read, but let's check anyway to be safe. + */ + if (nread < 0) + return -1; + conn->inEnd += nread; if (nread != bytes_pending) { appendPQExpBuffer(&conn->errorMessage, - libpq_gettext("drained only %zu of %zd pending bytes in transport buffer\n"), + libpq_gettext("drained only %zd of %zd pending bytes in transport buffer\n"), nread, bytes_pending); return -1; } diff --git a/src/interfaces/libpq/fe-secure.c b/src/interfaces/libpq/fe-secure.c index feb50f7e22..b9ce02a665 100644 --- a/src/interfaces/libpq/fe-secure.c +++ b/src/interfaces/libpq/fe-secure.c @@ -288,8 +288,9 @@ pqsecure_raw_read(PGconn *conn, void *ptr, size_t len) * Return the number of bytes available in the transport buffer. * * If pqsecure_read() is called for this number of bytes, it's guaranteed to - * return successfully without reading from the underlying socket. See - * pqDrainPending() for a more complete discussion of the concepts involved. + * return successfully with the same number of bytes, without reading from the + * underlying socket. See pqDrainPending() for a more complete discussion of + * the concepts involved. */ ssize_t pqsecure_bytes_pending(PGconn *conn) From 5c1004b0d33426f642808133c282c09cdd0a4f86 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Sat, 11 Jul 2026 15:14:50 +0200 Subject: [PATCH 71/76] Shorten pg_attribute_always_inline to pg_always_inline The pg_attribute_always_inline macro name is so long it forces pgindent to format the code in strange ways. Which may incentivize patch authors to either structure the code in strange ways (e.g. reorder prototypes), use shorter names, etc. Neither is very desirable for code readability. This shortens the name by removing the _attribute_ part. It also makes it more consistent with pg_noinline, which does not have the _attribute_ part either. Backpatched to all supported branches, to prevent conflicts when backpatching other fixes. The backbranches however keep both the old and new macro name, so that existing code keeps working. Author: Andres Freund Reviewed-by: Peter Geoghegan Reviewed-by: Tomas Vondra Discussion: https://postgr.es/m/bqqdehahpoa36igpictuqyn2s2mexk3t3ehidh2ffd2slb35e5@rzgksuiszgbg Backpatch-through: 14 --- src/backend/executor/execExprInterp.c | 50 +++++++++++++-------------- src/backend/executor/execTuples.c | 6 ++-- src/backend/executor/nodeHashjoin.c | 2 +- src/backend/utils/cache/catcache.c | 2 +- src/include/c.h | 9 ++++- 5 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c index 27b0f8a0a7..24eb53a0b2 100644 --- a/src/backend/executor/execExprInterp.c +++ b/src/backend/executor/execExprInterp.c @@ -167,24 +167,24 @@ static Datum ExecJustAssignOuterVarVirt(ExprState *state, ExprContext *econtext, static Datum ExecJustAssignScanVarVirt(ExprState *state, ExprContext *econtext, bool *isnull); /* execution helper functions */ -static pg_attribute_always_inline void ExecEvalArrayCompareInternal(FunctionCallInfo fcinfo, - ArrayType *arr, - int16 typlen, - bool typbyval, - char typalign, - bool useOr, - Datum *result, - bool *resultnull); -static pg_attribute_always_inline void ExecAggPlainTransByVal(AggState *aggstate, - AggStatePerTrans pertrans, - AggStatePerGroup pergroup, - ExprContext *aggcontext, - int setno); -static pg_attribute_always_inline void ExecAggPlainTransByRef(AggState *aggstate, - AggStatePerTrans pertrans, - AggStatePerGroup pergroup, - ExprContext *aggcontext, - int setno); +static pg_always_inline void ExecEvalArrayCompareInternal(FunctionCallInfo fcinfo, + ArrayType *arr, + int16 typlen, + bool typbyval, + char typalign, + bool useOr, + Datum *result, + bool *resultnull); +static pg_always_inline void ExecAggPlainTransByVal(AggState *aggstate, + AggStatePerTrans pertrans, + AggStatePerGroup pergroup, + ExprContext *aggcontext, + int setno); +static pg_always_inline void ExecAggPlainTransByRef(AggState *aggstate, + AggStatePerTrans pertrans, + AggStatePerGroup pergroup, + ExprContext *aggcontext, + int setno); /* * ScalarArrayOpExprHashEntry @@ -2063,7 +2063,7 @@ get_cached_rowtype(Oid type_id, int32 typmod, */ /* implementation of ExecJust(Inner|Outer|Scan)Var */ -static pg_attribute_always_inline Datum +static pg_always_inline Datum ExecJustVarImpl(ExprState *state, TupleTableSlot *slot, bool *isnull) { ExprEvalStep *op = &state->steps[1]; @@ -2101,7 +2101,7 @@ ExecJustScanVar(ExprState *state, ExprContext *econtext, bool *isnull) } /* implementation of ExecJustAssign(Inner|Outer|Scan)Var */ -static pg_attribute_always_inline Datum +static pg_always_inline Datum ExecJustAssignVarImpl(ExprState *state, TupleTableSlot *inslot, bool *isnull) { ExprEvalStep *op = &state->steps[1]; @@ -2196,7 +2196,7 @@ ExecJustConst(ExprState *state, ExprContext *econtext, bool *isnull) } /* implementation of ExecJust(Inner|Outer|Scan)VarVirt */ -static pg_attribute_always_inline Datum +static pg_always_inline Datum ExecJustVarVirtImpl(ExprState *state, TupleTableSlot *slot, bool *isnull) { ExprEvalStep *op = &state->steps[0]; @@ -2239,7 +2239,7 @@ ExecJustScanVarVirt(ExprState *state, ExprContext *econtext, bool *isnull) } /* implementation of ExecJustAssign(Inner|Outer|Scan)VarVirt */ -static pg_attribute_always_inline Datum +static pg_always_inline Datum ExecJustAssignVarVirtImpl(ExprState *state, TupleTableSlot *inslot, bool *isnull) { ExprEvalStep *op = &state->steps[0]; @@ -3393,7 +3393,7 @@ ExecEvalScalarArrayOp(ExprState *state, ExprEvalStep *op) * Callers must handle the strict LHS-is-NULL; return NULL fast path prior to * calling this. */ -static pg_attribute_always_inline void +static pg_always_inline void ExecEvalArrayCompareInternal(FunctionCallInfo fcinfo, ArrayType *arr, int16 typlen, bool typbyval, char typalign, bool useOr, Datum *result, bool *resultnull) @@ -4376,7 +4376,7 @@ ExecEvalAggOrderedTransTuple(ExprState *state, ExprEvalStep *op, } /* implementation of transition function invocation for byval types */ -static pg_attribute_always_inline void +static pg_always_inline void ExecAggPlainTransByVal(AggState *aggstate, AggStatePerTrans pertrans, AggStatePerGroup pergroup, ExprContext *aggcontext, int setno) @@ -4408,7 +4408,7 @@ ExecAggPlainTransByVal(AggState *aggstate, AggStatePerTrans pertrans, } /* implementation of transition function invocation for byref types */ -static pg_attribute_always_inline void +static pg_always_inline void ExecAggPlainTransByRef(AggState *aggstate, AggStatePerTrans pertrans, AggStatePerGroup pergroup, ExprContext *aggcontext, int setno) diff --git a/src/backend/executor/execTuples.c b/src/backend/executor/execTuples.c index 5004b3b165..5f6d92e242 100644 --- a/src/backend/executor/execTuples.c +++ b/src/backend/executor/execTuples.c @@ -71,8 +71,8 @@ static TupleDesc ExecTypeFromTLInternal(List *targetList, bool skipjunk); -static pg_attribute_always_inline void slot_deform_heap_tuple(TupleTableSlot *slot, HeapTuple tuple, uint32 *offp, - int natts); +static pg_always_inline void slot_deform_heap_tuple(TupleTableSlot *slot, HeapTuple tuple, uint32 *offp, + int natts); static inline void tts_buffer_heap_store_tuple(TupleTableSlot *slot, HeapTuple tuple, Buffer buffer, @@ -921,7 +921,7 @@ tts_buffer_heap_store_tuple(TupleTableSlot *slot, HeapTuple tuple, * This is marked as always inline, so the different offp for different types * of slots gets optimized away. */ -static pg_attribute_always_inline void +static pg_always_inline void slot_deform_heap_tuple(TupleTableSlot *slot, HeapTuple tuple, uint32 *offp, int natts) { diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index 43569fc58b..f2ebea5f4b 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -167,7 +167,7 @@ static void ExecParallelHashJoinPartitionOuter(HashJoinState *node); * the other one is "outer". * ---------------------------------------------------------------- */ -static pg_attribute_always_inline TupleTableSlot * +static pg_always_inline TupleTableSlot * ExecHashJoinImpl(PlanState *pstate, bool parallel) { HashJoinState *node = castNode(HashJoinState, pstate); diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c index 655920e019..a65eb03b58 100644 --- a/src/backend/utils/cache/catcache.c +++ b/src/backend/utils/cache/catcache.c @@ -947,7 +947,7 @@ RehashCatCache(CatCache *cp) * * Call CatalogCacheInitializeCache() if not yet done. */ -pg_attribute_always_inline +pg_always_inline static void ConditionalCatalogCacheInitializeCache(CatCache *cache) { diff --git a/src/include/c.h b/src/include/c.h index 61af588de0..596c2425b8 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -190,19 +190,26 @@ #endif /* - * Use "pg_attribute_always_inline" in place of "inline" for functions that + * Use "pg_always_inline" in place of "inline" for functions that * we wish to force inlining of, even when the compiler's heuristics would * choose not to. But, if possible, don't force inlining in unoptimized * debug builds. + * + * XXX The "pg_attribute_always_inline" variant is kept for backwards + * compatibility with existing code. All new code should use the shorter + * variant "pg_always_inline." */ #if (defined(__GNUC__) && __GNUC__ > 3 && defined(__OPTIMIZE__)) || defined(__SUNPRO_C) || defined(__IBMC__) /* GCC > 3, Sunpro and XLC support always_inline via __attribute__ */ +#define pg_always_inline __attribute__((always_inline)) inline #define pg_attribute_always_inline __attribute__((always_inline)) inline #elif defined(_MSC_VER) /* MSVC has a special keyword for this */ +#define pg_always_inline __forceinline #define pg_attribute_always_inline __forceinline #else /* Otherwise, the best we can do is to say "inline" */ +#define pg_always_inline inline #define pg_attribute_always_inline inline #endif From 99eb806ab48f77d06b9e902742d0b91f289b1ffa Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Wed, 15 Jul 2026 10:03:46 +0900 Subject: [PATCH 72/76] Include check on polpermissive relcache for policies equalPolicy() is used in the relation cache to check if two policy definitions are equivalent, but missed to check for polpermissive. ALTER POLICY cannot switch a policy to be PERMISSIVE or RESTRICTIVE, so this would need a dropped and then re-created policy, which would trigger a relcache invalidation. Anyway, there is no harm in being consistent in the check, and if one decides to add an ALTER POLICY to switch PERMISSIVE or RESTRICTIVE, we would be silently in trouble. Author: Andreas Lind Reviewed-by: Laurenz Albe Discussion: https://postgr.es/m/CAMxA3rv1CS6R7JR5ojz-3CmCEnZEFrqu+XXTnGbLRWrjJRH7sA@mail.gmail.com Backpatch-through: 14 --- src/backend/utils/cache/relcache.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 70d197292d..d47a4ac223 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -942,6 +942,8 @@ equalPolicy(RowSecurityPolicy *policy1, RowSecurityPolicy *policy2) if (policy1->polcmd != policy2->polcmd) return false; + if (policy1->permissive != policy2->permissive) + return false; if (policy1->hassublinks != policy2->hassublinks) return false; if (strcmp(policy1->policy_name, policy2->policy_name) != 0) From e6e8277f420f3b348ad5e414761196c5d02acb82 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Thu, 16 Jul 2026 13:35:36 +0900 Subject: [PATCH 73/76] doc: Fix log_parameter_max_length docs to reference log_min_duration_statement The documentation for log_parameter_max_length said it affects messages generated by log_duration. However, log_duration alone does not log bind parameter values, so this is misleading. This commit updates the documentation to reference log_min_duration_statement, which can log bind parameters, to better reflect actual behavior. Backpatch to all supported versions. Author: Fujii Masao Reviewed-by: Surya Poondla Discussion: https://postgr.es/m/CAHGQGwGnCVMVz8-LU9F8Sh57bkQX3jMZzx7age7M0LFEz5=Fog@mail.gmail.com Backpatch-through: 14 --- doc/src/sgml/config.sgml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 2308c87ca2..cd625271ea 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -7199,9 +7199,10 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' This setting only affects log messages printed as a result of , - , and related settings. Non-zero - values of this setting add some overhead, particularly if parameters - are sent in binary form, since then conversion to text is required. + , and related settings. + Non-zero values of this setting add some overhead, particularly + if parameters are sent in binary form, since then conversion to + text is required. From 2f4f193fdbe057029b87ae45822e22e7a8b85693 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Thu, 16 Jul 2026 13:38:34 +0900 Subject: [PATCH 74/76] Check CREATE_REPLICATION_SLOT response shape in libpqwalreceiver Previously, libpqrcv_create_slot() checked only that CREATE_REPLICATION_SLOT returned PGRES_TUPLES_OK before reading values from the first row. If the server unexpectedly returned an invalid result, such as zero rows, PQgetvalue() could return NULL, leading to a crash while parsing the LSN. Other replication commands, such as IDENTIFY_SYSTEM, already validate the response shape before accessing result values, but CREATE_REPLICATION_SLOT did not. Fix this by verifying that CREATE_REPLICATION_SLOT response contains exactly one row with four fields, and report a protocol violation otherwise. Backpatch to all supported versions. Bug: #19547 Reported-by: Yuelin Wang <1217816127@qq.com> Author: Kenny Chen Reviewed-by: Hayato Kuroda Reviewed-by: Fujii Masao Discussion: https://postgr.es/m/19547-f7986f668f71e788@postgresql.org Discussion: https://postgr.es/m/CAPXstDtW2iqe+DJAOTQTX+rRziJp2UhZSo1+HRj1COAtbu+nKw@mail.gmail.com Backpatch-through: 14 --- .../replication/libpqwalreceiver/libpqwalreceiver.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 98a786348b..1228188fce 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -919,6 +919,14 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname, slotname, pchomp(PQerrorMessage(conn->streamConn))))); } + /* CREATE_REPLICATION_SLOT returns a single row with four columns */ + if (PQnfields(res) != 4 || PQntuples(res) != 1) + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("invalid response from primary server"), + errdetail("Could not create replication slot \"%s\": got %d rows and %d fields, expected %d rows and %d fields.", + slotname, PQntuples(res), PQnfields(res), 1, 4))); + if (lsn) *lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid, CStringGetDatum(PQgetvalue(res, 0, 1)))); From 9d950d83e7fc894478fa4291f593f2ce4067b5d5 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 6 Jun 2018 15:58:49 +0300 Subject: [PATCH 75/76] Push down join quals into lateral subqueries. We don't normally push down join quals into subqueries, because that would require creating parameterized plans. However, if the plan is already parameterized because it's LATERAL, we might as well push down any additional join quals, that refer the same relations that are already referenced within the subquery. This changes the behavior of the sublevels_up parameters to ReplaceVarsFromTargetList(). The targetlist entries used to replace vars are no longer offset by that amount. I'm not sure what the original thinking on it was, but all the existing callers passed sublevels_up = 0, so I hope this is OK.. XXX: fixes by reshke --- src/backend/optimizer/path/allpaths.c | 141 ++++++++++++++---- src/backend/optimizer/plan/createplan.c | 30 ++++ src/backend/optimizer/prep/prepunion.c | 4 +- src/backend/optimizer/util/pathnode.c | 6 +- src/backend/rewrite/rewriteManip.c | 6 +- src/include/nodes/pathnodes.h | 1 + src/include/optimizer/pathnode.h | 5 +- .../regress/expected/subselect_pushdown.out | 91 +++++++++++ src/test/regress/parallel_schedule | 4 +- src/test/regress/sql/subselect_pushdown.sql | 54 +++++++ 10 files changed, 304 insertions(+), 38 deletions(-) create mode 100644 src/test/regress/expected/subselect_pushdown.out create mode 100644 src/test/regress/sql/subselect_pushdown.sql diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index f535ef0ed3..433ee363ef 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -138,12 +138,11 @@ static bool qual_is_pushdown_safe(Query *subquery, Index rti, RestrictInfo *rinfo, pushdown_safety_info *safetyInfo); static void subquery_push_qual(Query *subquery, - RangeTblEntry *rte, Index rti, Node *qual); + RangeTblEntry *rte, Index rti, Node *qual, int sublevels_up); static void recurse_push_qual(Node *setOp, Query *topquery, - RangeTblEntry *rte, Index rti, Node *qual); + RangeTblEntry *rte, Index rti, Node *qual, int sublevels_up); static void remove_unused_subquery_outputs(Query *subquery, RelOptInfo *rel); - /* * make_one_rel * Finds all possible access paths for executing a query, returning a @@ -2101,6 +2100,11 @@ has_multiple_baserels(PlannerInfo *root) * So the paths made here will be parameterized if the subquery contains * LATERAL references, otherwise not. As long as that's true, there's no need * for a separate set_subquery_size phase: just make the paths right away. + * + * (If a subquery is LATERAL, though, we do push down join clauses that refer + * to relations that the subquery already references laterally. Pushing down + * such quals won't make the subquery any more lateral, so there's no reason + * not to.) */ static void set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, @@ -2113,6 +2117,7 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, double tuple_fraction; RelOptInfo *sub_final_rel; ListCell *lc; + List *pushed_down_ec_joins = NIL; /* * Must copy the Query so that planning doesn't mess up the RTE contents @@ -2123,8 +2128,7 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, /* * If it's a LATERAL subquery, it might contain some Vars of the current - * query level, requiring it to be treated as parameterized, even though - * we don't support pushing down join quals into subqueries. + * query level, requiring it to be treated as parameterized. */ required_outer = rel->lateral_relids; @@ -2162,39 +2166,112 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, * pseudoconstant clauses; better to have the gating node above the * subquery. * + * Join clauses are only pushed down, if the subquery is LATERAL, and + * the join clause only refers to relations that the subquery already + * depends on. It might be useful to push down other join clauses, too, + * but then we would need to plan the subquery multiple times, to create + * parameterized paths, which seems too expensive. + * * Non-pushed-down clauses will get evaluated as qpquals of the * SubqueryScan node. * * XXX Are there any cases where we want to make a policy decision not to * push down a pushable qual, because it'd result in a worse plan? */ - if (rel->baserestrictinfo != NIL && + if ((rel->baserestrictinfo != NIL || + (!bms_is_empty(required_outer) && (rel->joininfo || rel->has_eclass_joins))) && subquery_is_pushdown_safe(subquery, subquery, &safetyInfo)) { /* OK to consider pushing down individual quals */ - List *upperrestrictlist = NIL; ListCell *l; + Bitmapset *available_relids; - foreach(l, rel->baserestrictinfo) + if (rel->baserestrictinfo) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(l); + List *upperrestrictlist = NIL; - if (!rinfo->pseudoconstant && - qual_is_pushdown_safe(subquery, rti, rinfo, &safetyInfo)) + foreach(l, rel->baserestrictinfo) { - Node *clause = (Node *) rinfo->clause; + RestrictInfo *rinfo = (RestrictInfo *) lfirst(l); - /* Push it down */ - subquery_push_qual(subquery, rte, rti, clause); + if (!rinfo->pseudoconstant && + qual_is_pushdown_safe(subquery, rti, rinfo, &safetyInfo)) + { + Node *clause = (Node *) rinfo->clause; + /* Push it down */ + subquery_push_qual(subquery, rte, rti, clause, 0); + } + else + { + /* Keep it in the upper query */ + upperrestrictlist = lappend(upperrestrictlist, rinfo); + } } - else + rel->baserestrictinfo = upperrestrictlist; + /* We don't bother recomputing baserestrict_min_security */ + } + + /* + * Push down join quals, as well. But only for LATERAL, and only for those + * relations that are "required" anyway. + */ + if (!bms_is_empty(required_outer)) + { + available_relids = bms_copy(required_outer); + available_relids = bms_add_member(available_relids, rti); + + if (rel->joininfo) { - /* Keep it in the upper query */ - upperrestrictlist = lappend(upperrestrictlist, rinfo); + ListCell *lc; + List *upperjoinlist = NIL; + + foreach(lc, rel->joininfo) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + Node *clause = (Node *) rinfo->clause; + + if (!rinfo->pseudoconstant && + bms_is_subset(rinfo->required_relids, available_relids) && + qual_is_pushdown_safe(subquery, rti, rinfo, &safetyInfo)) + { + /* Push it down */ + subquery_push_qual(subquery, rte, rti, clause, 0); + } + else + { + /* Keep it in the upper query */ + upperjoinlist = lappend(upperjoinlist, rinfo); + } + } + rel->joininfo = upperjoinlist; + } + + if (rel->has_eclass_joins) + { + List *clauses; + + clauses = generate_join_implied_equalities(root, + available_relids, + required_outer, + rel); + + foreach(lc, clauses) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + Node *clause = (Node *) rinfo->clause; + + if (!rinfo->pseudoconstant && + qual_is_pushdown_safe(subquery, rti, rinfo, &safetyInfo)) + { + /* Push it down */ + Assert(bms_is_subset(rinfo->required_relids, available_relids)); + subquery_push_qual(subquery, rte, rti, clause, 0); + + pushed_down_ec_joins = lappend(pushed_down_ec_joins, clause); + } + } } } - rel->baserestrictinfo = upperrestrictlist; - /* We don't bother recomputing baserestrict_min_security */ } pfree(safetyInfo.unsafeColumns); @@ -2272,7 +2349,7 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, /* Generate outer path using this subpath */ add_path(rel, (Path *) create_subqueryscan_path(root, rel, subpath, - pathkeys, required_outer)); + pathkeys, required_outer, pushed_down_ec_joins)); } /* If outer rel allows parallelism, do same for partial paths. */ @@ -2298,7 +2375,7 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, add_partial_path(rel, (Path *) create_subqueryscan_path(root, rel, subpath, pathkeys, - required_outer)); + required_outer, pushed_down_ec_joins)); } } } @@ -3512,13 +3589,13 @@ qual_is_pushdown_safe(Query *subquery, Index rti, RestrictInfo *rinfo, * subquery_push_qual - push down a qual that we have determined is safe */ static void -subquery_push_qual(Query *subquery, RangeTblEntry *rte, Index rti, Node *qual) +subquery_push_qual(Query *subquery, RangeTblEntry *rte, Index rti, Node *qual, int sublevels_up) { if (subquery->setOperations != NULL) { /* Recurse to push it separately to each component query */ recurse_push_qual(subquery->setOperations, subquery, - rte, rti, qual); + rte, rti, qual, sublevels_up); } else { @@ -3526,12 +3603,18 @@ subquery_push_qual(Query *subquery, RangeTblEntry *rte, Index rti, Node *qual) * We need to replace Vars in the qual (which must refer to outputs of * the subquery) with copies of the subquery's targetlist expressions. * Note that at this point, any uplevel Vars in the qual should have - * been replaced with Params, so they need no work. + * been replaced with Params, so they need no work. But in a join qual, + * there can be Vars referring to other relations at the same level. + * We need to increment varlevelsup of those, so that when the qual is + * pushed down, they refer to the parent query. * * This step also ensures that when we are pushing into a setop tree, * each component query gets its own copy of the qual. */ - qual = ReplaceVarsFromTargetList(qual, rti, 0, rte, + qual = copyObject(qual); + IncrementVarSublevelsUp(qual, sublevels_up + 1, 0); + + qual = ReplaceVarsFromTargetList(qual, rti, sublevels_up + 1, rte, subquery->targetList, REPLACEVARS_REPORT_ERROR, 0, &subquery->hasSubLinks); @@ -3560,7 +3643,7 @@ subquery_push_qual(Query *subquery, RangeTblEntry *rte, Index rti, Node *qual) */ static void recurse_push_qual(Node *setOp, Query *topquery, - RangeTblEntry *rte, Index rti, Node *qual) + RangeTblEntry *rte, Index rti, Node *qual, int sublevels_up) { if (IsA(setOp, RangeTblRef)) { @@ -3569,14 +3652,14 @@ recurse_push_qual(Node *setOp, Query *topquery, Query *subquery = subrte->subquery; Assert(subquery != NULL); - subquery_push_qual(subquery, rte, rti, qual); + subquery_push_qual(subquery, rte, rti, qual, sublevels_up + 1); } else if (IsA(setOp, SetOperationStmt)) { SetOperationStmt *op = (SetOperationStmt *) setOp; - recurse_push_qual(op->larg, topquery, rte, rti, qual); - recurse_push_qual(op->rarg, topquery, rte, rti, qual); + recurse_push_qual(op->larg, topquery, rte, rti, qual, sublevels_up); + recurse_push_qual(op->rarg, topquery, rte, rti, qual, sublevels_up); } else { diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index adfe9e686d..52df0d4663 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -3647,6 +3647,9 @@ create_subqueryscan_plan(PlannerInfo *root, SubqueryScanPath *best_path, RelOptInfo *rel = best_path->path.parent; Index scan_relid = rel->relid; Plan *subplan; + ListCell *l; + List *qpqual; + List *sq_quals = best_path->pushed_down_ec_joins; /* it should be a subquery base rel... */ Assert(scan_relid > 0); @@ -3659,6 +3662,33 @@ create_subqueryscan_plan(PlannerInfo *root, SubqueryScanPath *best_path, */ subplan = create_plan(rel->subroot, best_path->subpath); + /* + * If we had pushed down any join clauses to the subquery, we don't need + * to re-check them in the SubqueryScan node. + * + * This only applies to join clauses derived from equivalence classes. + * Non-join quals, and non-EC-derived join clauses are immediately removed + * from 'baserestrictinfo' and 'joininfo' when they're pushed down, so we + * won't need to worry about them here. + */ + qpqual = NIL; + foreach (l, scan_clauses) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, l); + + if (rinfo->pseudoconstant) + continue; /* we may drop pseudoconstants here */ + if (list_member_ptr(sq_quals, rinfo)) + continue; /* simple duplicate */ + if (is_redundant_derived_clause(rinfo, sq_quals)) + continue; /* derived from same EquivalenceClass */ + if (!contain_mutable_functions((Node *) rinfo->clause) && + predicate_implied_by(list_make1(rinfo->clause), sq_quals, false)) + continue; /* provably implied by indexquals */ + qpqual = lappend(qpqual, rinfo); + } + scan_clauses = qpqual; + /* Sort clauses into best execution order */ scan_clauses = order_qual_clauses(root, scan_clauses); diff --git a/src/backend/optimizer/prep/prepunion.c b/src/backend/optimizer/prep/prepunion.c index 78f582ba2e..16160c73e8 100644 --- a/src/backend/optimizer/prep/prepunion.c +++ b/src/backend/optimizer/prep/prepunion.c @@ -291,7 +291,7 @@ recurse_set_operations(Node *setOp, PlannerInfo *root, * soon too, likely.) */ path = (Path *) create_subqueryscan_path(root, rel, subpath, - NIL, NULL); + NIL, NULL, NIL); add_path(rel, path); @@ -309,7 +309,7 @@ recurse_set_operations(Node *setOp, PlannerInfo *root, partial_subpath = linitial(final_rel->partial_pathlist); partial_path = (Path *) create_subqueryscan_path(root, rel, partial_subpath, - NIL, NULL); + NIL, NULL, NIL); add_partial_path(rel, partial_path); } diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 33c40f375a..bb591520eb 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1999,7 +1999,7 @@ create_gather_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, */ SubqueryScanPath * create_subqueryscan_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, - List *pathkeys, Relids required_outer) + List *pathkeys, Relids required_outer, List *pushed_down_ec_joins) { SubqueryScanPath *pathnode = makeNode(SubqueryScanPath); @@ -2014,6 +2014,7 @@ create_subqueryscan_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, pathnode->path.parallel_workers = subpath->parallel_workers; pathnode->path.pathkeys = pathkeys; pathnode->subpath = subpath; + pathnode->pushed_down_ec_joins = pushed_down_ec_joins; cost_subqueryscan(pathnode, root, rel, pathnode->path.param_info); @@ -3904,7 +3905,8 @@ reparameterize_path(PlannerInfo *root, Path *path, rel, spath->subpath, spath->path.pathkeys, - required_outer); + required_outer, + spath->pushed_down_ec_joins); } case T_Result: /* Supported only for RTE_RESULT scan paths */ diff --git a/src/backend/rewrite/rewriteManip.c b/src/backend/rewrite/rewriteManip.c index 8cf58291ec..a7145df760 100644 --- a/src/backend/rewrite/rewriteManip.c +++ b/src/backend/rewrite/rewriteManip.c @@ -1408,6 +1408,7 @@ typedef struct List *targetlist; ReplaceVarsNoMatchOption nomatch_option; int nomatch_varno; + int min_sublevels_up; } ReplaceVarsFromTargetList_context; static Node * @@ -1493,8 +1494,8 @@ ReplaceVarsFromTargetList_callback(Var *var, Expr *newnode = copyObject(tle->expr); /* Must adjust varlevelsup if tlist item is from higher query */ - if (var->varlevelsup > 0) - IncrementVarSublevelsUp((Node *) newnode, var->varlevelsup, 0); + if (var->varlevelsup + rcon->min_sublevels_up > 0) + IncrementVarSublevelsUp((Node *) newnode, var->varlevelsup - rcon->min_sublevels_up, 0); /* * Check to see if the tlist item contains a PARAM_MULTIEXPR Param, @@ -1530,6 +1531,7 @@ ReplaceVarsFromTargetList(Node *node, context.targetlist = targetlist; context.nomatch_option = nomatch_option; context.nomatch_varno = nomatch_varno; + context.min_sublevels_up = sublevels_up; return replace_rte_variables(node, target_varno, sublevels_up, ReplaceVarsFromTargetList_callback, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 2a8977c972..d38d60a4ca 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -1379,6 +1379,7 @@ typedef struct SubqueryScanPath { Path path; Path *subpath; /* path representing subquery execution */ + List *pushed_down_ec_joins; /* pushed-down quals derived from ECs */ } SubqueryScanPath; /* diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 2922c0cdc1..5bae1c4d29 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -103,8 +103,9 @@ extern GatherMergePath *create_gather_merge_path(PlannerInfo *root, Relids required_outer, double *rows); extern SubqueryScanPath *create_subqueryscan_path(PlannerInfo *root, - RelOptInfo *rel, Path *subpath, - List *pathkeys, Relids required_outer); + RelOptInfo *rel, Path *subpath, + List *pathkeys, Relids required_outer, + List *pushed_down_ec_joins); extern Path *create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, List *pathkeys, Relids required_outer); extern Path *create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel, diff --git a/src/test/regress/expected/subselect_pushdown.out b/src/test/regress/expected/subselect_pushdown.out new file mode 100644 index 0000000000..f160bc84c8 --- /dev/null +++ b/src/test/regress/expected/subselect_pushdown.out @@ -0,0 +1,91 @@ +-- Test pushdown of quals into subqueries. +create table smalltab (i int4, j int4); +create table bigtab (i int4, j int4); +insert into smalltab values (1, 1), (100000, 100000); +insert into bigtab select g,g from generate_series(1, 100000) g; +analyze smalltab, bigtab; +create index bigtab_i on bigtab (i); +-- Push down restriction quals. +explain (costs off) +select * from smalltab, +( + select bigtab.i, avg(bigtab.j) + from bigtab + group by bigtab.i +) as subq(i, avg) +where smalltab.i = subq.i and smalltab.i = 123; + QUERY PLAN +------------------------------------------------- + Nested Loop + -> Seq Scan on smalltab + Filter: (i = 123) + -> GroupAggregate + Group Key: bigtab.i + -> Index Scan using bigtab_i on bigtab + Index Cond: (i = 123) +(7 rows) + +-- Join quals are not currently pushed down +explain (costs off) +select * from smalltab, +( + select bigtab.i, avg(bigtab.j) + from bigtab + group by bigtab.i +) as subq(i, avg) +where smalltab.i = subq.i; + QUERY PLAN +------------------------------------------------- + Merge Join + Merge Cond: (smalltab.i = bigtab.i) + -> Sort + Sort Key: smalltab.i + -> Seq Scan on smalltab + -> GroupAggregate + Group Key: bigtab.i + -> Index Scan using bigtab_i on bigtab +(8 rows) + +-- Except when the subquery is LATERAL, and already references the other relation. +-- Such join clauses can be pushed down. +explain (costs off) +select * from smalltab, +lateral ( + select bigtab.i, avg(bigtab.j) + from bigtab + where bigtab.j = smalltab.j + group by bigtab.i +) as subq(i, avg) +where smalltab.i < subq.i; + QUERY PLAN +------------------------------------------------- + Nested Loop + -> Seq Scan on smalltab + -> GroupAggregate + Group Key: bigtab.i + -> Index Scan using bigtab_i on bigtab + Index Cond: (smalltab.i < i) + Filter: (j = smalltab.j) +(7 rows) + +-- Multiple join clauses constructed from equivalence classes +explain (costs off) +select * from smalltab, +lateral ( + select bigtab.i, bigtab.j, avg(bigtab.j) + from bigtab + where bigtab.j/2 = smalltab.j / 2 + group by bigtab.i, bigtab.j +) as subq(i, j, avg) +where smalltab.i = subq.i and smalltab.j = subq.j; + QUERY PLAN +--------------------------------------------------------------------------- + Nested Loop + -> Seq Scan on smalltab + -> GroupAggregate + Group Key: bigtab.i, bigtab.j + -> Index Scan using bigtab_i on bigtab + Index Cond: (smalltab.i = i) + Filter: ((smalltab.j = j) AND ((j / 2) = (smalltab.j / 2))) +(7 rows) + diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 52ef93828f..21421332ea 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -71,7 +71,9 @@ test: sanity_check # Note: the ignore: line does not run random, just mark it as ignorable # ---------- ignore: random -test: select_into select_distinct select_distinct_on select_implicit select_having subselect union case join aggregates transactions random portals arrays btree_index hash_index update delete namespace prepared_xacts +test: select_into select_distinct select_distinct_on select_implicit select_having union case join aggregates transactions random portals arrays btree_index hash_index update delete namespace prepared_xacts + +test: subselect subselect_pushdown # ---------- # Another group of parallel tests diff --git a/src/test/regress/sql/subselect_pushdown.sql b/src/test/regress/sql/subselect_pushdown.sql new file mode 100644 index 0000000000..c9e7a43794 --- /dev/null +++ b/src/test/regress/sql/subselect_pushdown.sql @@ -0,0 +1,54 @@ +-- Test pushdown of quals into subqueries. + +create table smalltab (i int4, j int4); +create table bigtab (i int4, j int4); + +insert into smalltab values (1, 1), (100000, 100000); +insert into bigtab select g,g from generate_series(1, 100000) g; + +analyze smalltab, bigtab; + +create index bigtab_i on bigtab (i); + +-- Push down restriction quals. +explain (costs off) +select * from smalltab, +( + select bigtab.i, avg(bigtab.j) + from bigtab + group by bigtab.i +) as subq(i, avg) +where smalltab.i = subq.i and smalltab.i = 123; + +-- Join quals are not currently pushed down +explain (costs off) +select * from smalltab, +( + select bigtab.i, avg(bigtab.j) + from bigtab + group by bigtab.i +) as subq(i, avg) +where smalltab.i = subq.i; + +-- Except when the subquery is LATERAL, and already references the other relation. +-- Such join clauses can be pushed down. +explain (costs off) +select * from smalltab, +lateral ( + select bigtab.i, avg(bigtab.j) + from bigtab + where bigtab.j = smalltab.j + group by bigtab.i +) as subq(i, avg) +where smalltab.i < subq.i; + +-- Multiple join clauses constructed from equivalence classes +explain (costs off) +select * from smalltab, +lateral ( + select bigtab.i, bigtab.j, avg(bigtab.j) + from bigtab + where bigtab.j/2 = smalltab.j / 2 + group by bigtab.i, bigtab.j +) as subq(i, j, avg) +where smalltab.i = subq.i and smalltab.j = subq.j; From 6752c38a9c0d5fe1c8f440ed453a662301456f7e Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 6 Jun 2018 20:07:31 +0300 Subject: [PATCH 76/76] WIP: Allow pushing join quals down into subqueries. Plan subqueries a second time, to create parameterized plans with join quals. XXX: fixes by reshke --- src/backend/nodes/outfuncs.c | 2 - src/backend/optimizer/path/allpaths.c | 151 +++++++++++++++--- src/backend/optimizer/path/costsize.c | 3 +- src/backend/optimizer/plan/createplan.c | 30 +++- src/backend/optimizer/plan/setrefs.c | 8 +- src/backend/optimizer/plan/subselect.c | 4 +- src/backend/optimizer/prep/prepunion.c | 22 +-- src/backend/optimizer/util/pathnode.c | 16 +- src/backend/optimizer/util/relnode.c | 9 +- src/backend/utils/adt/selfuncs.c | 25 ++- src/include/nodes/pathnodes.h | 8 +- src/include/optimizer/cost.h | 2 +- src/include/optimizer/pathnode.h | 8 +- .../regress/expected/subselect_pushdown.out | 17 +- src/test/regress/sql/subselect_pushdown.sql | 7 +- 15 files changed, 233 insertions(+), 79 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 58c2590698..623c17f716 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2373,8 +2373,6 @@ _outRelOptInfo(StringInfo str, const RelOptInfo *node) WRITE_FLOAT_FIELD(tuples, "%.0f"); WRITE_FLOAT_FIELD(allvisfrac, "%.6f"); WRITE_BITMAPSET_FIELD(eclass_indexes); - WRITE_NODE_FIELD(subroot); - WRITE_NODE_FIELD(subplan_params); WRITE_INT_FIELD(rel_parallel_workers); WRITE_UINT_FIELD(amflags); WRITE_OID_FIELD(serverid); diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 433ee363ef..c52c5666d3 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -91,6 +91,13 @@ static void set_tablesample_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); static void set_foreign_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); + +static void add_subqueryscan_variant(PlannerInfo *root, RelOptInfo *rel, + Index rti, RangeTblEntry *rte, + Bitmapset *required_outer, + Query *subquery, List *pushed_down_clauses, double tuple_fraction, + bool update_estimates); + static void set_foreign_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); static void set_append_rel_size(PlannerInfo *root, RelOptInfo *rel, @@ -2111,13 +2118,13 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) { Query *parse = root->parse; + Query *unparameterized_subquery; Query *subquery = rte->subquery; Relids required_outer; pushdown_safety_info safetyInfo; double tuple_fraction; - RelOptInfo *sub_final_rel; - ListCell *lc; List *pushed_down_ec_joins = NIL; + bool sq_is_pushdown_safe; /* * Must copy the Query so that planning doesn't mess up the RTE contents @@ -2178,9 +2185,10 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, * XXX Are there any cases where we want to make a policy decision not to * push down a pushable qual, because it'd result in a worse plan? */ - if ((rel->baserestrictinfo != NIL || - (!bms_is_empty(required_outer) && (rel->joininfo || rel->has_eclass_joins))) && - subquery_is_pushdown_safe(subquery, subquery, &safetyInfo)) + sq_is_pushdown_safe = subquery_is_pushdown_safe(subquery, subquery, &safetyInfo); + if (sq_is_pushdown_safe && + (rel->baserestrictinfo != NIL || + (!bms_is_empty(required_outer) && (rel->joininfo || rel->has_eclass_joins)))) { /* OK to consider pushing down individual quals */ ListCell *l; @@ -2249,6 +2257,7 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, if (rel->has_eclass_joins) { List *clauses; + ListCell *lc; clauses = generate_join_implied_equalities(root, available_relids, @@ -2274,8 +2283,6 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, } } - pfree(safetyInfo.unsafeColumns); - /* * The upper query might not use all the subquery's output columns; if * not, we can simplify. @@ -2299,16 +2306,112 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, else tuple_fraction = root->tuple_fraction; + unparameterized_subquery = copyObject(subquery); + + add_subqueryscan_variant(root, rel, rti, rte, + required_outer, subquery, pushed_down_ec_joins, tuple_fraction, true); + + /* + * Also create parameterized join paths, where we push the join condition + * down to the subquery. + * + * To keep the planning time reasonable, this is all-or-nothing. We try to + * push all join conditions down to the subquery, and create paths for that. + * We don't create paths for every combination of join conditions that we + * could push down. + */ + if ((rel->has_eclass_joins || rel->joininfo) && + sq_is_pushdown_safe) + { + List *clauses; + ListCell *lc; + List *pushed_down_clauses = list_copy(pushed_down_ec_joins); + Bitmapset *available_relids; + Bitmapset *other_relids; + + subquery = copyObject(unparameterized_subquery); + + required_outer = bms_copy(required_outer); + + foreach(lc, rel->joininfo) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + Node *clause = (Node *) rinfo->clause; + + if (!rinfo->pseudoconstant && + qual_is_pushdown_safe(subquery, rti, rinfo, &safetyInfo)) + { + /* Push it down */ + required_outer = bms_union(required_outer, + pull_varnos(root, clause)); + required_outer = bms_del_member(required_outer, rti); + + subquery_push_qual(subquery, rte, rti, clause, 0); + + pushed_down_clauses = lappend(pushed_down_clauses, rinfo); + } + } + + /* + * We already pushed down any join quals with LATERAL referenced rels, don't add + * them again. + */ + available_relids = bms_difference(root->all_baserels, rel->lateral_referencers); + other_relids = bms_del_member(bms_copy(available_relids), rti); + + clauses = generate_join_implied_equalities(root, + available_relids, + other_relids, + rel); + foreach(lc, clauses) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + Node *clause = (Node *) rinfo->clause; + + if (!rinfo->pseudoconstant && + qual_is_pushdown_safe(subquery, rti, rinfo, &safetyInfo)) + { + /* Push it down */ + required_outer = bms_union(required_outer, + pull_varnos(root, clause)); + required_outer = bms_del_member(required_outer, rti); + + subquery_push_qual(subquery, rte, rti, clause, 0); + + pushed_down_clauses = lappend(pushed_down_clauses, rinfo); + } + } + if (pushed_down_clauses) + add_subqueryscan_variant(root, rel, rti, rte, + required_outer, + subquery, pushed_down_clauses, tuple_fraction, false); + } + + pfree(safetyInfo.unsafeColumns); +} + +static void +add_subqueryscan_variant(PlannerInfo *root, RelOptInfo *rel, + Index rti, RangeTblEntry *rte, + Bitmapset *required_outer, + Query *subquery, List *pushed_down_clauses, double tuple_fraction, + bool update_estimates) +{ + RelOptInfo *sub_final_rel; + ListCell *lc; + PlannerInfo *subroot; + List *subplan_params; + /* plan_params should not be in use in current query level */ Assert(root->plan_params == NIL); /* Generate a subroot and Paths for the subquery */ - rel->subroot = subquery_planner(root->glob, subquery, - root, - false, tuple_fraction); + subroot = subquery_planner(root->glob, subquery, + root, + false, tuple_fraction); /* Isolate the params needed by this specific subplan */ - rel->subplan_params = root->plan_params; + subplan_params = root->plan_params; root->plan_params = NIL; /* @@ -2316,7 +2419,7 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, * so, it's desirable to produce an unadorned dummy path so that we will * recognize appropriate optimizations at this query level. */ - sub_final_rel = fetch_upper_rel(rel->subroot, UPPERREL_FINAL, NULL); + sub_final_rel = fetch_upper_rel(subroot, UPPERREL_FINAL, NULL); if (IS_DUMMY_REL(sub_final_rel)) { @@ -2328,8 +2431,13 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, * Mark rel with estimated output rows, width, etc. Note that we have to * do this before generating outer-query paths, else cost_subqueryscan is * not happy. + * + * Don't overwrite the estimates when we're creating parameterized paths + * for joins. The estimate for a parameterized path includes the effects + * of the join clauses. */ - set_subquery_size_estimates(root, rel); + if (update_estimates) + set_subquery_size_estimates(root, rel, subroot); /* * For each Path that subquery_planner produced, make a SubqueryScanPath @@ -2348,8 +2456,8 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, /* Generate outer path using this subpath */ add_path(rel, (Path *) - create_subqueryscan_path(root, rel, subpath, - pathkeys, required_outer, pushed_down_ec_joins)); + create_subqueryscan_path(root, rel, subroot, subplan_params, subpath, + pathkeys, required_outer, pushed_down_clauses)); } /* If outer rel allows parallelism, do same for partial paths. */ @@ -2373,9 +2481,9 @@ set_subquery_pathlist(PlannerInfo *root, RelOptInfo *rel, /* Generate outer path using this subpath */ add_partial_path(rel, (Path *) - create_subqueryscan_path(root, rel, subpath, + create_subqueryscan_path(root, rel, subroot, subplan_params, subpath, pathkeys, - required_outer, pushed_down_ec_joins)); + required_outer, pushed_down_clauses)); } } } @@ -3531,6 +3639,7 @@ qual_is_pushdown_safe(Query *subquery, Index rti, RestrictInfo *rinfo, * such Vars must refer to subselect output columns ... unless this is * part of a LATERAL subquery, in which case there could be lateral * references. + * Examine all Vars used in clause. */ vars = pull_var_clause(qual, PVC_INCLUDE_PLACEHOLDERS); foreach(vl, vars) @@ -3551,15 +3660,11 @@ qual_is_pushdown_safe(Query *subquery, Index rti, RestrictInfo *rinfo, } /* - * Punt if we find any lateral references. It would be safe to push - * these down, but we'd have to convert them into outer references, - * which subquery_push_qual lacks the infrastructure to do. The case - * arises so seldom that it doesn't seem worth working hard on. + * XXX: diff with upstream */ if (var->varno != rti) { - safe = false; - break; + continue; } /* Subqueries have no system columns */ diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 7967f22911..f1dd800a0a 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -5494,9 +5494,8 @@ get_foreign_key_join_selectivity(PlannerInfo *root, * We set the same fields as set_baserel_size_estimates. */ void -set_subquery_size_estimates(PlannerInfo *root, RelOptInfo *rel) +set_subquery_size_estimates(PlannerInfo *root, RelOptInfo *rel, PlannerInfo *subroot) { - PlannerInfo *subroot = rel->subroot; RelOptInfo *sub_final_rel; ListCell *lc; diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 52df0d4663..230534ca91 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -3649,18 +3649,40 @@ create_subqueryscan_plan(PlannerInfo *root, SubqueryScanPath *best_path, Plan *subplan; ListCell *l; List *qpqual; - List *sq_quals = best_path->pushed_down_ec_joins; + List *sq_quals = best_path->pushed_down_clauses; /* it should be a subquery base rel... */ Assert(scan_relid > 0); Assert(rel->rtekind == RTE_SUBQUERY); + Assert(rel->chosen_plan == NULL); /* * Recursively create Plan from Path for subquery. Since we are entering * a different planner context (subroot), recurse to create_plan not * create_plan_recurse. */ - subplan = create_plan(rel->subroot, best_path->subpath); + subplan = create_plan(best_path->subroot, best_path->subpath); + + /* + * If this path used join quals that were pushed down to the subquery, + * we don't need to re-check those quals on the SubqueryScan node itself. + */ + if (best_path->pushed_down_clauses) + { + List *new_clauses = NIL; + ListCell *l; + + foreach(l, scan_clauses) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, l); + + if (list_member_ptr(best_path->pushed_down_clauses, rinfo)) + continue; + + new_clauses = lappend(new_clauses, rinfo); + } + scan_clauses = new_clauses; + } /* * If we had pushed down any join clauses to the subquery, we don't need @@ -3701,7 +3723,7 @@ create_subqueryscan_plan(PlannerInfo *root, SubqueryScanPath *best_path, scan_clauses = (List *) replace_nestloop_params(root, (Node *) scan_clauses); process_subquery_nestloop_params(root, - rel->subplan_params); + best_path->subplan_params); } scan_plan = make_subqueryscan(tlist, @@ -3711,6 +3733,8 @@ create_subqueryscan_plan(PlannerInfo *root, SubqueryScanPath *best_path, copy_generic_path_info(&scan_plan->scan.plan, &best_path->path); + rel->chosen_plan = best_path->subroot; + return scan_plan; } diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index e361ae841b..82dc97d1dd 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -418,12 +418,12 @@ add_rtes_to_flat_rtable(PlannerInfo *root, bool recursing) * that some upper query level is treating this one as dummy, * and so we won't scan this level's plan tree at all. */ - if (rel->subroot == NULL) + if (rel->chosen_plan == NULL) flatten_unplanned_rtes(glob, rte); else if (recursing || - IS_DUMMY_REL(fetch_upper_rel(rel->subroot, + IS_DUMMY_REL(fetch_upper_rel(rel->chosen_plan, UPPERREL_FINAL, NULL))) - add_rtes_to_flat_rtable(rel->subroot, true); + add_rtes_to_flat_rtable(rel->chosen_plan, true); } } rti++; @@ -1238,7 +1238,7 @@ set_subqueryscan_references(PlannerInfo *root, rel = find_base_rel(root, plan->scan.scanrelid); /* Recursively process the subplan */ - plan->subplan = set_plan_references(rel->subroot, plan->subplan); + plan->subplan = set_plan_references(rel->chosen_plan, plan->subplan); if (trivial_subqueryscan(plan)) { diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index 9b97154743..f151415e41 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -2387,11 +2387,11 @@ finalize_plan(PlannerInfo *root, Plan *plan, /* We must run finalize_plan on the subquery */ rel = find_base_rel(root, sscan->scan.scanrelid); - subquery_params = rel->subroot->outer_params; + subquery_params = rel->chosen_plan->outer_params; if (gather_param >= 0) subquery_params = bms_add_member(bms_copy(subquery_params), gather_param); - finalize_plan(rel->subroot, sscan->subplan, gather_param, + finalize_plan(rel->chosen_plan, sscan->subplan, gather_param, subquery_params, NULL); /* Now we can add its extParams to the parent's params */ diff --git a/src/backend/optimizer/prep/prepunion.c b/src/backend/optimizer/prep/prepunion.c index 16160c73e8..a8a03e2e79 100644 --- a/src/backend/optimizer/prep/prepunion.c +++ b/src/backend/optimizer/prep/prepunion.c @@ -236,10 +236,10 @@ recurse_set_operations(Node *setOp, PlannerInfo *root, Assert(root->plan_params == NIL); /* Generate a subroot and Paths for the subquery */ - subroot = rel->subroot = subquery_planner(root->glob, subquery, - root, - false, - root->tuple_fraction); + subroot = subquery_planner(root->glob, subquery, + root, + false, + root->tuple_fraction); /* * It should not be possible for the primitive query to contain any @@ -265,7 +265,7 @@ recurse_set_operations(Node *setOp, PlannerInfo *root, * to do this before generating outer-query paths, else * cost_subqueryscan is not happy. */ - set_subquery_size_estimates(root, rel); + set_subquery_size_estimates(root, rel, subroot); /* * Since we may want to add a partial path to this relation, we must @@ -290,8 +290,12 @@ recurse_set_operations(Node *setOp, PlannerInfo *root, * the SubqueryScanPath with nil pathkeys. (XXX that should change * soon too, likely.) */ - path = (Path *) create_subqueryscan_path(root, rel, subpath, - NIL, NULL, NIL); + path = (Path *) create_subqueryscan_path(root, rel, + subroot, + NIL, + subpath, + NIL, + NULL, NIL); add_path(rel, path); @@ -308,8 +312,8 @@ recurse_set_operations(Node *setOp, PlannerInfo *root, partial_subpath = linitial(final_rel->partial_pathlist); partial_path = (Path *) - create_subqueryscan_path(root, rel, partial_subpath, - NIL, NULL, NIL); + create_subqueryscan_path(root, rel, subroot, NIL, partial_subpath, NIL, + NULL, NIL); add_partial_path(rel, partial_path); } diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index bb591520eb..54a2e497cf 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1998,8 +1998,12 @@ create_gather_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, * returning the pathnode. */ SubqueryScanPath * -create_subqueryscan_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, - List *pathkeys, Relids required_outer, List *pushed_down_ec_joins) +create_subqueryscan_path(PlannerInfo *root, RelOptInfo *rel, + PlannerInfo *subroot, + List *subplan_params, + Path *subpath, + List *pathkeys, Relids required_outer, + List *pushed_down_clauses) { SubqueryScanPath *pathnode = makeNode(SubqueryScanPath); @@ -2014,7 +2018,9 @@ create_subqueryscan_path(PlannerInfo *root, RelOptInfo *rel, Path *subpath, pathnode->path.parallel_workers = subpath->parallel_workers; pathnode->path.pathkeys = pathkeys; pathnode->subpath = subpath; - pathnode->pushed_down_ec_joins = pushed_down_ec_joins; + pathnode->subplan_params = subplan_params; + pathnode->subroot = subroot; + pathnode->pushed_down_clauses = pushed_down_clauses; cost_subqueryscan(pathnode, root, rel, pathnode->path.param_info); @@ -3903,10 +3909,12 @@ reparameterize_path(PlannerInfo *root, Path *path, return (Path *) create_subqueryscan_path(root, rel, + spath->subroot, + spath->subplan_params, spath->subpath, spath->path.pathkeys, required_outer, - spath->pushed_down_ec_joins); + spath->pushed_down_clauses); } case T_Result: /* Supported only for RTE_RESULT scan paths */ diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index a496f56b4d..c0de894579 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -231,8 +231,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->tuples = 0; rel->allvisfrac = 0; rel->eclass_indexes = NULL; - rel->subroot = NULL; - rel->subplan_params = NIL; + rel->rel_parallel_workers = -1; /* set up in get_relation_info */ rel->amflags = 0; rel->serverid = InvalidOid; @@ -644,8 +643,7 @@ build_join_rel(PlannerInfo *root, joinrel->tuples = 0; joinrel->allvisfrac = 0; joinrel->eclass_indexes = NULL; - joinrel->subroot = NULL; - joinrel->subplan_params = NIL; + joinrel->rel_parallel_workers = -1; joinrel->amflags = 0; joinrel->serverid = InvalidOid; @@ -826,9 +824,8 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->tuples = 0; joinrel->allvisfrac = 0; joinrel->eclass_indexes = NULL; - joinrel->subroot = NULL; - joinrel->subplan_params = NIL; joinrel->amflags = 0; + joinrel->chosen_plan = NULL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c index 64e5398b6a..3a2f6677ee 100644 --- a/src/backend/utils/adt/selfuncs.c +++ b/src/backend/utils/adt/selfuncs.c @@ -5346,6 +5346,7 @@ examine_simple_variable(PlannerInfo *root, Var *var, Query *subquery = rte->subquery; RelOptInfo *rel; TargetEntry *ste; + PlannerInfo *subroot; /* * Punt if it's a whole-row var rather than a plain column reference. @@ -5375,10 +5376,24 @@ examine_simple_variable(PlannerInfo *root, Var *var, */ rel = find_base_rel(root, var->varno); - /* If the subquery hasn't been planned yet, we have to punt */ - if (rel->subroot == NULL) + if (rel->chosen_plan) + subroot = rel->chosen_plan; + else if (rel->pathlist && IsA(linitial(rel->pathlist), SubqueryScanPath)) + { + /* + * Use the estimates from the first path. XXX: what if it's a parameterized + * path? + */ + SubqueryScanPath *sqpath = (SubqueryScanPath *) linitial(rel->pathlist); + + subroot = sqpath->subroot; + } + else + { + /* If the subquery hasn't been planned yet, we have to punt */ return; - Assert(IsA(rel->subroot, PlannerInfo)); + } + Assert(IsA(subroot, PlannerInfo)); /* * Switch our attention to the subquery as mangled by the planner. It @@ -5388,7 +5403,7 @@ examine_simple_variable(PlannerInfo *root, Var *var, * planning, Vars in the targetlist might have gotten replaced, and we * need to see the replacement expressions. */ - subquery = rel->subroot->parse; + subquery = subroot->parse; Assert(IsA(subquery, Query)); /* Get the subquery output expression referenced by the upper Var */ @@ -5440,7 +5455,7 @@ examine_simple_variable(PlannerInfo *root, Var *var, * if the underlying column is unique, the subquery may have * joined to other tables in a way that creates duplicates. */ - examine_simple_variable(rel->subroot, var, vardata); + examine_simple_variable(subroot, var, vardata); } } else diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index d38d60a4ca..fa78a9cfa8 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -722,8 +722,7 @@ typedef struct RelOptInfo double allvisfrac; Bitmapset *eclass_indexes; /* Indexes in PlannerInfo's eq_classes list of * ECs that mention this rel */ - PlannerInfo *subroot; /* if subquery */ - List *subplan_params; /* if subquery */ + PlannerInfo *chosen_plan; int rel_parallel_workers; /* wanted number of parallel workers */ uint32 amflags; /* Bitmask of optional features supported by * the table AM */ @@ -1379,7 +1378,10 @@ typedef struct SubqueryScanPath { Path path; Path *subpath; /* path representing subquery execution */ - List *pushed_down_ec_joins; /* pushed-down quals derived from ECs */ + PlannerInfo *subroot; /* */ + List *subplan_params; /* */ + + List *pushed_down_clauses; /* pushed-down join quals */ } SubqueryScanPath; /* diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index 2113bc82de..947c5a28c0 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -195,7 +195,7 @@ extern void set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, RelOptInfo *inner_rel, SpecialJoinInfo *sjinfo, List *restrictlist); -extern void set_subquery_size_estimates(PlannerInfo *root, RelOptInfo *rel); +extern void set_subquery_size_estimates(PlannerInfo *root, RelOptInfo *rel, PlannerInfo *subroot); extern void set_function_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern void set_values_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern void set_cte_size_estimates(PlannerInfo *root, RelOptInfo *rel, diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 5bae1c4d29..f48cd77aac 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -103,9 +103,11 @@ extern GatherMergePath *create_gather_merge_path(PlannerInfo *root, Relids required_outer, double *rows); extern SubqueryScanPath *create_subqueryscan_path(PlannerInfo *root, - RelOptInfo *rel, Path *subpath, - List *pathkeys, Relids required_outer, - List *pushed_down_ec_joins); + RelOptInfo *rel, + PlannerInfo *subroot, + List *subplan_params, + Path *subpath, + List *pathkeys, Relids required_outer, List *pushed_down_clauses); extern Path *create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, List *pathkeys, Relids required_outer); extern Path *create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel, diff --git a/src/test/regress/expected/subselect_pushdown.out b/src/test/regress/expected/subselect_pushdown.out index f160bc84c8..792f66e7fd 100644 --- a/src/test/regress/expected/subselect_pushdown.out +++ b/src/test/regress/expected/subselect_pushdown.out @@ -25,7 +25,7 @@ where smalltab.i = subq.i and smalltab.i = 123; Index Cond: (i = 123) (7 rows) --- Join quals are not currently pushed down +-- Push down join quals. explain (costs off) select * from smalltab, ( @@ -36,18 +36,17 @@ select * from smalltab, where smalltab.i = subq.i; QUERY PLAN ------------------------------------------------- - Merge Join - Merge Cond: (smalltab.i = bigtab.i) - -> Sort - Sort Key: smalltab.i - -> Seq Scan on smalltab + Nested Loop + -> Seq Scan on smalltab -> GroupAggregate Group Key: bigtab.i -> Index Scan using bigtab_i on bigtab -(8 rows) + Index Cond: (smalltab.i = i) +(6 rows) --- Except when the subquery is LATERAL, and already references the other relation. --- Such join clauses can be pushed down. +-- Subquery is LATERAL, and already references the other relation. The join +-- qual is always pushed down in that case, as the plan is "parameterized" +-- in respect to the other relation even if it was not pushed down. explain (costs off) select * from smalltab, lateral ( diff --git a/src/test/regress/sql/subselect_pushdown.sql b/src/test/regress/sql/subselect_pushdown.sql index c9e7a43794..d6431d314c 100644 --- a/src/test/regress/sql/subselect_pushdown.sql +++ b/src/test/regress/sql/subselect_pushdown.sql @@ -20,7 +20,7 @@ select * from smalltab, ) as subq(i, avg) where smalltab.i = subq.i and smalltab.i = 123; --- Join quals are not currently pushed down +-- Push down join quals. explain (costs off) select * from smalltab, ( @@ -30,8 +30,9 @@ select * from smalltab, ) as subq(i, avg) where smalltab.i = subq.i; --- Except when the subquery is LATERAL, and already references the other relation. --- Such join clauses can be pushed down. +-- Subquery is LATERAL, and already references the other relation. The join +-- qual is always pushed down in that case, as the plan is "parameterized" +-- in respect to the other relation even if it was not pushed down. explain (costs off) select * from smalltab, lateral (